1/* drivers/atm/eni.c - Efficient Networks ENI155P device driver */
   2 
   3/* Written 1995-2000 by Werner Almesberger, EPFL LRC/ICA */
   4 
   5
   6#include <linux/module.h>
   7#include <linux/kernel.h>
   8#include <linux/mm.h>
   9#include <linux/pci.h>
  10#include <linux/errno.h>
  11#include <linux/atm.h>
  12#include <linux/atmdev.h>
  13#include <linux/sonet.h>
  14#include <linux/skbuff.h>
  15#include <linux/time.h>
  16#include <linux/delay.h>
  17#include <linux/uio.h>
  18#include <linux/init.h>
  19#include <linux/atm_eni.h>
  20#include <linux/bitops.h>
  21#include <linux/slab.h>
  22#include <asm/io.h>
  23#include <linux/atomic.h>
  24#include <asm/uaccess.h>
  25#include <asm/string.h>
  26#include <asm/byteorder.h>
  27
  28#include "tonga.h"
  29#include "midway.h"
  30#include "suni.h"
  31#include "eni.h"
  32
  33#if !defined(__i386__) && !defined(__x86_64__)
  34#ifndef ioremap_nocache
  35#define ioremap_nocache(X,Y) ioremap(X,Y)
  36#endif 
  37#endif
  38
  39/*
  40 * TODO:
  41 *
  42 * Show stoppers
  43 *  none
  44 *
  45 * Minor
  46 *  - OAM support
  47 *  - fix bugs listed below
  48 */
  49
  50/*
  51 * KNOWN BUGS:
  52 *
  53 * - may run into JK-JK bug and deadlock
  54 * - should allocate UBR channel first
  55 * - buffer space allocation algorithm is stupid
  56 *   (RX: should be maxSDU+maxdelay*rate
  57 *    TX: should be maxSDU+min(maxSDU,maxdelay*rate) )
  58 * - doesn't support OAM cells
  59 * - eni_put_free may hang if not putting memory fragments that _complete_
  60 *   2^n block (never happens in real life, though)
  61 */
  62
  63
  64#if 0
  65#define DPRINTK(format,args...) printk(KERN_DEBUG format,##args)
  66#else
  67#define DPRINTK(format,args...)
  68#endif
  69
  70
  71#ifndef CONFIG_ATM_ENI_TUNE_BURST
  72#define CONFIG_ATM_ENI_BURST_TX_8W
  73#define CONFIG_ATM_ENI_BURST_RX_4W
  74#endif
  75
  76
  77#ifndef CONFIG_ATM_ENI_DEBUG
  78
  79
  80#define NULLCHECK(x)
  81
  82#define EVENT(s,a,b)
  83
  84
  85static void event_dump(void)
  86{
  87}
  88
  89
  90#else
  91
  92
  93/* 
  94 * NULL pointer checking
  95 */
  96
  97#define NULLCHECK(x) \
  98	if ((unsigned long) (x) < 0x30) \
  99		printk(KERN_CRIT #x "==0x%lx\n",(unsigned long) (x))
 100
 101/*
 102 * Very extensive activity logging. Greatly improves bug detection speed but
 103 * costs a few Mbps if enabled.
 104 */
 105
 106#define EV 64
 107
 108static const char *ev[EV];
 109static unsigned long ev_a[EV],ev_b[EV];
 110static int ec = 0;
 111
 112
 113static void EVENT(const char *s,unsigned long a,unsigned long b)
 114{
 115	ev[ec] = s; 
 116	ev_a[ec] = a;
 117	ev_b[ec] = b;
 118	ec = (ec+1) % EV;
 119}
 120
 121
 122static void event_dump(void)
 123{
 124	int n,i;
 125
 126	for (n = 0; n < EV; n++) {
 127		i = (ec+n) % EV;
 128		printk(KERN_NOTICE);
 129		printk(ev[i] ? ev[i] : "(null)",ev_a[i],ev_b[i]);
 130	}
 131}
 132
 133
 134#endif /* CONFIG_ATM_ENI_DEBUG */
 135
 136
 137/*
 138 * NExx   must not be equal at end
 139 * EExx   may be equal at end
 140 * xxPJOK verify validity of pointer jumps
 141 * xxPMOK operating on a circular buffer of "c" words
 142 */
 143
 144#define NEPJOK(a0,a1,b) \
 145    ((a0) < (a1) ? (b) <= (a0) || (b) > (a1) : (b) <= (a0) && (b) > (a1))
 146#define EEPJOK(a0,a1,b) \
 147    ((a0) < (a1) ? (b) < (a0) || (b) >= (a1) : (b) < (a0) && (b) >= (a1))
 148#define NEPMOK(a0,d,b,c) NEPJOK(a0,(a0+d) & (c-1),b)
 149#define EEPMOK(a0,d,b,c) EEPJOK(a0,(a0+d) & (c-1),b)
 150
 151
 152static int tx_complete = 0,dma_complete = 0,queued = 0,requeued = 0,
 153  backlogged = 0,rx_enqueued = 0,rx_dequeued = 0,pushed = 0,submitted = 0,
 154  putting = 0;
 155
 156static struct atm_dev *eni_boards = NULL;
 157
 158/* Read/write registers on card */
 159#define eni_in(r)	readl(eni_dev->reg+(r)*4)
 160#define eni_out(v,r)	writel((v),eni_dev->reg+(r)*4)
 161
 162
 163/*-------------------------------- utilities --------------------------------*/
 164
 165
 166static void dump_mem(struct eni_dev *eni_dev)
 167{
 168	int i;
 169
 170	for (i = 0; i < eni_dev->free_len; i++)
 171		printk(KERN_DEBUG "  %d: %p %d\n",i,
 172		    eni_dev->free_list[i].start,
 173		    1 << eni_dev->free_list[i].order);
 174}
 175
 176
 177static void dump(struct atm_dev *dev)
 178{
 179	struct eni_dev *eni_dev;
 180
 181	int i;
 182
 183	eni_dev = ENI_DEV(dev);
 184	printk(KERN_NOTICE "Free memory\n");
 185	dump_mem(eni_dev);
 186	printk(KERN_NOTICE "TX buffers\n");
 187	for (i = 0; i < NR_CHAN; i++)
 188		if (eni_dev->tx[i].send)
 189			printk(KERN_NOTICE "  TX %d @ %p: %ld\n",i,
 190			    eni_dev->tx[i].send,eni_dev->tx[i].words*4);
 191	printk(KERN_NOTICE "RX buffers\n");
 192	for (i = 0; i < 1024; i++)
 193		if (eni_dev->rx_map[i] && ENI_VCC(eni_dev->rx_map[i])->rx)
 194			printk(KERN_NOTICE "  RX %d @ %p: %ld\n",i,
 195			    ENI_VCC(eni_dev->rx_map[i])->recv,
 196			    ENI_VCC(eni_dev->rx_map[i])->words*4);
 197	printk(KERN_NOTICE "----\n");
 198}
 199
 200
 201static void eni_put_free(struct eni_dev *eni_dev, void __iomem *start,
 202    unsigned long size)
 203{
 204	struct eni_free *list;
 205	int len,order;
 206
 207	DPRINTK("init 0x%lx+%ld(0x%lx)\n",start,size,size);
 208	start += eni_dev->base_diff;
 209	list = eni_dev->free_list;
 210	len = eni_dev->free_len;
 211	while (size) {
 212		if (len >= eni_dev->free_list_size) {
 213			printk(KERN_CRIT "eni_put_free overflow (%p,%ld)\n",
 214			    start,size);
 215			break;
 216		}
 217		for (order = 0; !(((unsigned long)start | size) & (1 << order)); order++);
 218		if (MID_MIN_BUF_SIZE > (1 << order)) {
 219			printk(KERN_CRIT "eni_put_free: order %d too small\n",
 220			    order);
 221			break;
 222		}
 223		list[len].start = (void __iomem *) start;
 224		list[len].order = order;
 225		len++;
 226		start += 1 << order;
 227		size -= 1 << order;
 228	}
 229	eni_dev->free_len = len;
 230	/*dump_mem(eni_dev);*/
 231}
 232
 233
 234static void __iomem *eni_alloc_mem(struct eni_dev *eni_dev, unsigned long *size)
 235{
 236	struct eni_free *list;
 237	void __iomem *start;
 238	int len,i,order,best_order,index;
 239
 240	list = eni_dev->free_list;
 241	len = eni_dev->free_len;
 242	if (*size < MID_MIN_BUF_SIZE) *size = MID_MIN_BUF_SIZE;
 243	if (*size > MID_MAX_BUF_SIZE) return NULL;
 244	for (order = 0; (1 << order) < *size; order++);
 
 245	DPRINTK("trying: %ld->%d\n",*size,order);
 246	best_order = 65; /* we don't have more than 2^64 of anything ... */
 247	index = 0; /* silence GCC */
 248	for (i = 0; i < len; i++)
 249		if (list[i].order == order) {
 250			best_order = order;
 251			index = i;
 252			break;
 253		}
 254		else if (best_order > list[i].order && list[i].order > order) {
 255				best_order = list[i].order;
 256				index = i;
 257			}
 258	if (best_order == 65) return NULL;
 259	start = list[index].start-eni_dev->base_diff;
 260	list[index] = list[--len];
 261	eni_dev->free_len = len;
 262	*size = 1 << order;
 263	eni_put_free(eni_dev,start+*size,(1 << best_order)-*size);
 264	DPRINTK("%ld bytes (order %d) at 0x%lx\n",*size,order,start);
 265	memset_io(start,0,*size);       /* never leak data */
 266	/*dump_mem(eni_dev);*/
 267	return start;
 268}
 269
 270
 271static void eni_free_mem(struct eni_dev *eni_dev, void __iomem *start,
 272    unsigned long size)
 273{
 274	struct eni_free *list;
 275	int len,i,order;
 276
 277	start += eni_dev->base_diff;
 278	list = eni_dev->free_list;
 279	len = eni_dev->free_len;
 280	for (order = -1; size; order++) size >>= 1;
 281	DPRINTK("eni_free_mem: %p+0x%lx (order %d)\n",start,size,order);
 282	for (i = 0; i < len; i++)
 283		if (((unsigned long) list[i].start) == ((unsigned long)start^(1 << order)) &&
 284		    list[i].order == order) {
 285			DPRINTK("match[%d]: 0x%lx/0x%lx(0x%x), %d/%d\n",i,
 286			    list[i].start,start,1 << order,list[i].order,order);
 287			list[i] = list[--len];
 288			start = (void __iomem *) ((unsigned long) start & ~(unsigned long) (1 << order));
 289			order++;
 290			i = -1;
 291			continue;
 292		}
 293	if (len >= eni_dev->free_list_size) {
 294		printk(KERN_ALERT "eni_free_mem overflow (%p,%d)\n",start,
 295		    order);
 296		return;
 297	}
 298	list[len].start = start;
 299	list[len].order = order;
 300	eni_dev->free_len = len+1;
 301	/*dump_mem(eni_dev);*/
 302}
 303
 304
 305/*----------------------------------- RX ------------------------------------*/
 306
 307
 308#define ENI_VCC_NOS ((struct atm_vcc *) 1)
 309
 310
 311static void rx_ident_err(struct atm_vcc *vcc)
 312{
 313	struct atm_dev *dev;
 314	struct eni_dev *eni_dev;
 315	struct eni_vcc *eni_vcc;
 316
 317	dev = vcc->dev;
 318	eni_dev = ENI_DEV(dev);
 319	/* immediately halt adapter */
 320	eni_out(eni_in(MID_MC_S) &
 321	    ~(MID_DMA_ENABLE | MID_TX_ENABLE | MID_RX_ENABLE),MID_MC_S);
 322	/* dump useful information */
 323	eni_vcc = ENI_VCC(vcc);
 324	printk(KERN_ALERT DEV_LABEL "(itf %d): driver error - RX ident "
 325	    "mismatch\n",dev->number);
 326	printk(KERN_ALERT "  VCI %d, rxing %d, words %ld\n",vcc->vci,
 327	    eni_vcc->rxing,eni_vcc->words);
 328	printk(KERN_ALERT "  host descr 0x%lx, rx pos 0x%lx, descr value "
 329	    "0x%x\n",eni_vcc->descr,eni_vcc->rx_pos,
 330	    (unsigned) readl(eni_vcc->recv+eni_vcc->descr*4));
 331	printk(KERN_ALERT "  last %p, servicing %d\n",eni_vcc->last,
 332	    eni_vcc->servicing);
 333	EVENT("---dump ends here---\n",0,0);
 334	printk(KERN_NOTICE "---recent events---\n");
 335	event_dump();
 336	ENI_DEV(dev)->fast = NULL; /* really stop it */
 337	ENI_DEV(dev)->slow = NULL;
 338	skb_queue_head_init(&ENI_DEV(dev)->rx_queue);
 339}
 340
 341
 342static int do_rx_dma(struct atm_vcc *vcc,struct sk_buff *skb,
 343    unsigned long skip,unsigned long size,unsigned long eff)
 344{
 345	struct eni_dev *eni_dev;
 346	struct eni_vcc *eni_vcc;
 347	u32 dma_rd,dma_wr;
 348	u32 dma[RX_DMA_BUF*2];
 349	dma_addr_t paddr;
 350	unsigned long here;
 351	int i,j;
 352
 353	eni_dev = ENI_DEV(vcc->dev);
 354	eni_vcc = ENI_VCC(vcc);
 355	paddr = 0; /* GCC, shut up */
 356	if (skb) {
 357		paddr = dma_map_single(&eni_dev->pci_dev->dev,skb->data,skb->len,
 358				       DMA_FROM_DEVICE);
 359		if (dma_mapping_error(&eni_dev->pci_dev->dev, paddr))
 360			goto dma_map_error;
 361		ENI_PRV_PADDR(skb) = paddr;
 362		if (paddr & 3)
 363			printk(KERN_CRIT DEV_LABEL "(itf %d): VCI %d has "
 364			    "mis-aligned RX data (0x%lx)\n",vcc->dev->number,
 365			    vcc->vci,(unsigned long) paddr);
 366		ENI_PRV_SIZE(skb) = size+skip;
 367		    /* PDU plus descriptor */
 368		ATM_SKB(skb)->vcc = vcc;
 369	}
 370	j = 0;
 371	if ((eff && skip) || 1) { /* @@@ actually, skip is always == 1 ... */
 372		here = (eni_vcc->descr+skip) & (eni_vcc->words-1);
 373		dma[j++] = (here << MID_DMA_COUNT_SHIFT) | (vcc->vci
 374		    << MID_DMA_VCI_SHIFT) | MID_DT_JK;
 375		j++;
 376	}
 377	here = (eni_vcc->descr+size+skip) & (eni_vcc->words-1);
 378	if (!eff) size += skip;
 379	else {
 380		unsigned long words;
 381
 382		if (!size) {
 383			DPRINTK("strange things happen ...\n");
 384			EVENT("strange things happen ... (skip=%ld,eff=%ld)\n",
 385			    size,eff);
 386		}
 387		words = eff;
 388		if (paddr & 15) {
 389			unsigned long init;
 390
 391			init = 4-((paddr & 15) >> 2);
 392			if (init > words) init = words;
 393			dma[j++] = MID_DT_WORD | (init << MID_DMA_COUNT_SHIFT) |
 394			    (vcc->vci << MID_DMA_VCI_SHIFT);
 395			dma[j++] = paddr;
 396			paddr += init << 2;
 397			words -= init;
 398		}
 399#ifdef CONFIG_ATM_ENI_BURST_RX_16W /* may work with some PCI chipsets ... */
 400		if (words & ~15) {
 401			dma[j++] = MID_DT_16W | ((words >> 4) <<
 402			    MID_DMA_COUNT_SHIFT) | (vcc->vci <<
 403			    MID_DMA_VCI_SHIFT);
 404			dma[j++] = paddr;
 405			paddr += (words & ~15) << 2;
 406			words &= 15;
 407		}
 408#endif
 409#ifdef CONFIG_ATM_ENI_BURST_RX_8W  /* works only with *some* PCI chipsets ... */
 410		if (words & ~7) {
 411			dma[j++] = MID_DT_8W | ((words >> 3) <<
 412			    MID_DMA_COUNT_SHIFT) | (vcc->vci <<
 413			    MID_DMA_VCI_SHIFT);
 414			dma[j++] = paddr;
 415			paddr += (words & ~7) << 2;
 416			words &= 7;
 417		}
 418#endif
 419#ifdef CONFIG_ATM_ENI_BURST_RX_4W /* recommended */
 420		if (words & ~3) {
 421			dma[j++] = MID_DT_4W | ((words >> 2) <<
 422			    MID_DMA_COUNT_SHIFT) | (vcc->vci <<
 423			    MID_DMA_VCI_SHIFT);
 424			dma[j++] = paddr;
 425			paddr += (words & ~3) << 2;
 426			words &= 3;
 427		}
 428#endif
 429#ifdef CONFIG_ATM_ENI_BURST_RX_2W /* probably useless if RX_4W, RX_8W, ... */
 430		if (words & ~1) {
 431			dma[j++] = MID_DT_2W | ((words >> 1) <<
 432			    MID_DMA_COUNT_SHIFT) | (vcc->vci <<
 433			    MID_DMA_VCI_SHIFT);
 434			dma[j++] = paddr;
 435			paddr += (words & ~1) << 2;
 436			words &= 1;
 437		}
 438#endif
 439		if (words) {
 440			dma[j++] = MID_DT_WORD | (words << MID_DMA_COUNT_SHIFT)
 441			    | (vcc->vci << MID_DMA_VCI_SHIFT);
 442			dma[j++] = paddr;
 443		}
 444	}
 445	if (size != eff) {
 446		dma[j++] = (here << MID_DMA_COUNT_SHIFT) |
 447		    (vcc->vci << MID_DMA_VCI_SHIFT) | MID_DT_JK;
 448		j++;
 449	}
 450	if (!j || j > 2*RX_DMA_BUF) {
 451		printk(KERN_CRIT DEV_LABEL "!j or j too big!!!\n");
 452		goto trouble;
 453	}
 454	dma[j-2] |= MID_DMA_END;
 455	j = j >> 1;
 456	dma_wr = eni_in(MID_DMA_WR_RX);
 457	dma_rd = eni_in(MID_DMA_RD_RX);
 458	/*
 459	 * Can I move the dma_wr pointer by 2j+1 positions without overwriting
 460	 * data that hasn't been read (position of dma_rd) yet ?
 461	 */
 462	if (!NEPMOK(dma_wr,j+j+1,dma_rd,NR_DMA_RX)) { /* @@@ +1 is ugly */
 463		printk(KERN_WARNING DEV_LABEL "(itf %d): RX DMA full\n",
 464		    vcc->dev->number);
 465		goto trouble;
 466	}
 467        for (i = 0; i < j; i++) {
 468		writel(dma[i*2],eni_dev->rx_dma+dma_wr*8);
 469		writel(dma[i*2+1],eni_dev->rx_dma+dma_wr*8+4);
 470		dma_wr = (dma_wr+1) & (NR_DMA_RX-1);
 471        }
 472	if (skb) {
 473		ENI_PRV_POS(skb) = eni_vcc->descr+size+1;
 474		skb_queue_tail(&eni_dev->rx_queue,skb);
 475		eni_vcc->last = skb;
 476rx_enqueued++;
 477	}
 478	eni_vcc->descr = here;
 479	eni_out(dma_wr,MID_DMA_WR_RX);
 480	return 0;
 481
 482trouble:
 483	if (paddr)
 484		dma_unmap_single(&eni_dev->pci_dev->dev,paddr,skb->len,
 485				 DMA_FROM_DEVICE);
 486dma_map_error:
 487	if (skb) dev_kfree_skb_irq(skb);
 488	return -1;
 489}
 490
 491
 492static void discard(struct atm_vcc *vcc,unsigned long size)
 493{
 494	struct eni_vcc *eni_vcc;
 495
 496	eni_vcc = ENI_VCC(vcc);
 497	EVENT("discard (size=%ld)\n",size,0);
 498	while (do_rx_dma(vcc,NULL,1,size,0)) EVENT("BUSY LOOP",0,0);
 499	    /* could do a full fallback, but that might be more expensive */
 500	if (eni_vcc->rxing) ENI_PRV_POS(eni_vcc->last) += size+1;
 501	else eni_vcc->rx_pos = (eni_vcc->rx_pos+size+1) & (eni_vcc->words-1);
 502}
 503
 504
 505/*
 506 * TODO: should check whether direct copies (without DMA setup, dequeuing on
 507 * interrupt, etc.) aren't much faster for AAL0
 508 */
 509
 510static int rx_aal0(struct atm_vcc *vcc)
 511{
 512	struct eni_vcc *eni_vcc;
 513	unsigned long descr;
 514	unsigned long length;
 515	struct sk_buff *skb;
 516
 517	DPRINTK(">rx_aal0\n");
 518	eni_vcc = ENI_VCC(vcc);
 519	descr = readl(eni_vcc->recv+eni_vcc->descr*4);
 520	if ((descr & MID_RED_IDEN) != (MID_RED_RX_ID << MID_RED_SHIFT)) {
 521		rx_ident_err(vcc);
 522		return 1;
 523	}
 524	if (descr & MID_RED_T) {
 525		DPRINTK(DEV_LABEL "(itf %d): trashing empty cell\n",
 526		    vcc->dev->number);
 527		length = 0;
 528		atomic_inc(&vcc->stats->rx_err);
 529	}
 530	else {
 531		length = ATM_CELL_SIZE-1; /* no HEC */
 532	}
 533	skb = length ? atm_alloc_charge(vcc,length,GFP_ATOMIC) : NULL;
 534	if (!skb) {
 535		discard(vcc,length >> 2);
 536		return 0;
 537	}
 538	skb_put(skb,length);
 539	skb->tstamp = eni_vcc->timestamp;
 540	DPRINTK("got len %ld\n",length);
 541	if (do_rx_dma(vcc,skb,1,length >> 2,length >> 2)) return 1;
 542	eni_vcc->rxing++;
 543	return 0;
 544}
 545
 546
 547static int rx_aal5(struct atm_vcc *vcc)
 548{
 549	struct eni_vcc *eni_vcc;
 550	unsigned long descr;
 551	unsigned long size,eff,length;
 552	struct sk_buff *skb;
 553
 554	EVENT("rx_aal5\n",0,0);
 555	DPRINTK(">rx_aal5\n");
 556	eni_vcc = ENI_VCC(vcc);
 557	descr = readl(eni_vcc->recv+eni_vcc->descr*4);
 558	if ((descr & MID_RED_IDEN) != (MID_RED_RX_ID << MID_RED_SHIFT)) {
 559		rx_ident_err(vcc);
 560		return 1;
 561	}
 562	if (descr & (MID_RED_T | MID_RED_CRC_ERR)) {
 563		if (descr & MID_RED_T) {
 564			EVENT("empty cell (descr=0x%lx)\n",descr,0);
 565			DPRINTK(DEV_LABEL "(itf %d): trashing empty cell\n",
 566			    vcc->dev->number);
 567			size = 0;
 568		}
 569		else {
 570			static unsigned long silence = 0;
 571
 572			if (time_after(jiffies, silence) || silence == 0) {
 573				printk(KERN_WARNING DEV_LABEL "(itf %d): "
 574				    "discarding PDU(s) with CRC error\n",
 575				    vcc->dev->number);
 576				silence = (jiffies+2*HZ)|1;
 577			}
 578			size = (descr & MID_RED_COUNT)*(ATM_CELL_PAYLOAD >> 2);
 579			EVENT("CRC error (descr=0x%lx,size=%ld)\n",descr,
 580			    size);
 581		}
 582		eff = length = 0;
 583		atomic_inc(&vcc->stats->rx_err);
 584	}
 585	else {
 586		size = (descr & MID_RED_COUNT)*(ATM_CELL_PAYLOAD >> 2);
 587		DPRINTK("size=%ld\n",size);
 588		length = readl(eni_vcc->recv+(((eni_vcc->descr+size-1) &
 589		    (eni_vcc->words-1)))*4) & 0xffff;
 590				/* -trailer(2)+header(1) */
 591		if (length && length <= (size << 2)-8 && length <=
 592		  ATM_MAX_AAL5_PDU) eff = (length+3) >> 2;
 593		else {				 /* ^ trailer length (8) */
 594			EVENT("bad PDU (descr=0x08%lx,length=%ld)\n",descr,
 595			    length);
 596			printk(KERN_ERR DEV_LABEL "(itf %d): bad AAL5 PDU "
 597			    "(VCI=%d,length=%ld,size=%ld (descr 0x%lx))\n",
 598			    vcc->dev->number,vcc->vci,length,size << 2,descr);
 599			length = eff = 0;
 600			atomic_inc(&vcc->stats->rx_err);
 601		}
 602	}
 603	skb = eff ? atm_alloc_charge(vcc,eff << 2,GFP_ATOMIC) : NULL;
 604	if (!skb) {
 605		discard(vcc,size);
 606		return 0;
 607	}
 608	skb_put(skb,length);
 609	DPRINTK("got len %ld\n",length);
 610	if (do_rx_dma(vcc,skb,1,size,eff)) return 1;
 611	eni_vcc->rxing++;
 612	return 0;
 613}
 614
 615
 616static inline int rx_vcc(struct atm_vcc *vcc)
 617{
 618	void __iomem *vci_dsc;
 619	unsigned long tmp;
 620	struct eni_vcc *eni_vcc;
 621
 622	eni_vcc = ENI_VCC(vcc);
 623	vci_dsc = ENI_DEV(vcc->dev)->vci+vcc->vci*16;
 624	EVENT("rx_vcc(1)\n",0,0);
 625	while (eni_vcc->descr != (tmp = (readl(vci_dsc+4) & MID_VCI_DESCR) >>
 626	    MID_VCI_DESCR_SHIFT)) {
 627		EVENT("rx_vcc(2: host dsc=0x%lx, nic dsc=0x%lx)\n",
 628		    eni_vcc->descr,tmp);
 629		DPRINTK("CB_DESCR %ld REG_DESCR %d\n",ENI_VCC(vcc)->descr,
 630		    (((unsigned) readl(vci_dsc+4) & MID_VCI_DESCR) >>
 631		    MID_VCI_DESCR_SHIFT));
 632		if (ENI_VCC(vcc)->rx(vcc)) return 1;
 633	}
 634	/* clear IN_SERVICE flag */
 635	writel(readl(vci_dsc) & ~MID_VCI_IN_SERVICE,vci_dsc);
 636	/*
 637	 * If new data has arrived between evaluating the while condition and
 638	 * clearing IN_SERVICE, we wouldn't be notified until additional data
 639	 * follows. So we have to loop again to be sure.
 640	 */
 641	EVENT("rx_vcc(3)\n",0,0);
 642	while (ENI_VCC(vcc)->descr != (tmp = (readl(vci_dsc+4) & MID_VCI_DESCR)
 643	    >> MID_VCI_DESCR_SHIFT)) {
 644		EVENT("rx_vcc(4: host dsc=0x%lx, nic dsc=0x%lx)\n",
 645		    eni_vcc->descr,tmp);
 646		DPRINTK("CB_DESCR %ld REG_DESCR %d\n",ENI_VCC(vcc)->descr,
 647		    (((unsigned) readl(vci_dsc+4) & MID_VCI_DESCR) >>
 648		    MID_VCI_DESCR_SHIFT));
 649		if (ENI_VCC(vcc)->rx(vcc)) return 1;
 650	}
 651	return 0;
 652}
 653
 654
 655static void poll_rx(struct atm_dev *dev)
 656{
 657	struct eni_dev *eni_dev;
 658	struct atm_vcc *curr;
 659
 660	eni_dev = ENI_DEV(dev);
 661	while ((curr = eni_dev->fast)) {
 662		EVENT("poll_rx.fast\n",0,0);
 663		if (rx_vcc(curr)) return;
 664		eni_dev->fast = ENI_VCC(curr)->next;
 665		ENI_VCC(curr)->next = ENI_VCC_NOS;
 666		barrier();
 667		ENI_VCC(curr)->servicing--;
 668	}
 669	while ((curr = eni_dev->slow)) {
 670		EVENT("poll_rx.slow\n",0,0);
 671		if (rx_vcc(curr)) return;
 672		eni_dev->slow = ENI_VCC(curr)->next;
 673		ENI_VCC(curr)->next = ENI_VCC_NOS;
 674		barrier();
 675		ENI_VCC(curr)->servicing--;
 676	}
 677}
 678
 679
 680static void get_service(struct atm_dev *dev)
 681{
 682	struct eni_dev *eni_dev;
 683	struct atm_vcc *vcc;
 684	unsigned long vci;
 685
 686	DPRINTK(">get_service\n");
 687	eni_dev = ENI_DEV(dev);
 688	while (eni_in(MID_SERV_WRITE) != eni_dev->serv_read) {
 689		vci = readl(eni_dev->service+eni_dev->serv_read*4);
 690		eni_dev->serv_read = (eni_dev->serv_read+1) & (NR_SERVICE-1);
 691		vcc = eni_dev->rx_map[vci & 1023];
 692		if (!vcc) {
 693			printk(KERN_CRIT DEV_LABEL "(itf %d): VCI %ld not "
 694			    "found\n",dev->number,vci);
 695			continue; /* nasty but we try to go on anyway */
 696			/* @@@ nope, doesn't work */
 697		}
 698		EVENT("getting from service\n",0,0);
 699		if (ENI_VCC(vcc)->next != ENI_VCC_NOS) {
 700			EVENT("double service\n",0,0);
 701			DPRINTK("Grr, servicing VCC %ld twice\n",vci);
 702			continue;
 703		}
 704		ENI_VCC(vcc)->timestamp = ktime_get_real();
 705		ENI_VCC(vcc)->next = NULL;
 706		if (vcc->qos.rxtp.traffic_class == ATM_CBR) {
 707			if (eni_dev->fast)
 708				ENI_VCC(eni_dev->last_fast)->next = vcc;
 709			else eni_dev->fast = vcc;
 710			eni_dev->last_fast = vcc;
 711		}
 712		else {
 713			if (eni_dev->slow)
 714				ENI_VCC(eni_dev->last_slow)->next = vcc;
 715			else eni_dev->slow = vcc;
 716			eni_dev->last_slow = vcc;
 717		}
 718putting++;
 719		ENI_VCC(vcc)->servicing++;
 720	}
 721}
 722
 723
 724static void dequeue_rx(struct atm_dev *dev)
 725{
 726	struct eni_dev *eni_dev;
 727	struct eni_vcc *eni_vcc;
 728	struct atm_vcc *vcc;
 729	struct sk_buff *skb;
 730	void __iomem *vci_dsc;
 731	int first;
 732
 733	eni_dev = ENI_DEV(dev);
 734	first = 1;
 735	while (1) {
 736		skb = skb_dequeue(&eni_dev->rx_queue);
 737		if (!skb) {
 738			if (first) {
 739				DPRINTK(DEV_LABEL "(itf %d): RX but not "
 740				    "rxing\n",dev->number);
 741				EVENT("nothing to dequeue\n",0,0);
 742			}
 743			break;
 744		}
 745		EVENT("dequeued (size=%ld,pos=0x%lx)\n",ENI_PRV_SIZE(skb),
 746		    ENI_PRV_POS(skb));
 747rx_dequeued++;
 748		vcc = ATM_SKB(skb)->vcc;
 749		eni_vcc = ENI_VCC(vcc);
 750		first = 0;
 751		vci_dsc = eni_dev->vci+vcc->vci*16;
 752		if (!EEPMOK(eni_vcc->rx_pos,ENI_PRV_SIZE(skb),
 753		    (readl(vci_dsc+4) & MID_VCI_READ) >> MID_VCI_READ_SHIFT,
 754		    eni_vcc->words)) {
 755			EVENT("requeuing\n",0,0);
 756			skb_queue_head(&eni_dev->rx_queue,skb);
 757			break;
 758		}
 759		eni_vcc->rxing--;
 760		eni_vcc->rx_pos = ENI_PRV_POS(skb) & (eni_vcc->words-1);
 761		dma_unmap_single(&eni_dev->pci_dev->dev,ENI_PRV_PADDR(skb),skb->len,
 762			         DMA_TO_DEVICE);
 763		if (!skb->len) dev_kfree_skb_irq(skb);
 764		else {
 765			EVENT("pushing (len=%ld)\n",skb->len,0);
 766			if (vcc->qos.aal == ATM_AAL0)
 767				*(unsigned long *) skb->data =
 768				    ntohl(*(unsigned long *) skb->data);
 769			memset(skb->cb,0,sizeof(struct eni_skb_prv));
 770			vcc->push(vcc,skb);
 771			pushed++;
 772		}
 773		atomic_inc(&vcc->stats->rx);
 774	}
 775	wake_up(&eni_dev->rx_wait);
 776}
 777
 778
 779static int open_rx_first(struct atm_vcc *vcc)
 780{
 781	struct eni_dev *eni_dev;
 782	struct eni_vcc *eni_vcc;
 783	unsigned long size;
 784
 785	DPRINTK("open_rx_first\n");
 786	eni_dev = ENI_DEV(vcc->dev);
 787	eni_vcc = ENI_VCC(vcc);
 788	eni_vcc->rx = NULL;
 789	if (vcc->qos.rxtp.traffic_class == ATM_NONE) return 0;
 790	size = vcc->qos.rxtp.max_sdu*eni_dev->rx_mult/100;
 791	if (size > MID_MAX_BUF_SIZE && vcc->qos.rxtp.max_sdu <=
 792	    MID_MAX_BUF_SIZE)
 793		size = MID_MAX_BUF_SIZE;
 794	eni_vcc->recv = eni_alloc_mem(eni_dev,&size);
 795	DPRINTK("rx at 0x%lx\n",eni_vcc->recv);
 796	eni_vcc->words = size >> 2;
 797	if (!eni_vcc->recv) return -ENOBUFS;
 798	eni_vcc->rx = vcc->qos.aal == ATM_AAL5 ? rx_aal5 : rx_aal0;
 799	eni_vcc->descr = 0;
 800	eni_vcc->rx_pos = 0;
 801	eni_vcc->rxing = 0;
 802	eni_vcc->servicing = 0;
 803	eni_vcc->next = ENI_VCC_NOS;
 804	return 0;
 805}
 806
 807
 808static int open_rx_second(struct atm_vcc *vcc)
 809{
 810	void __iomem *here;
 811	struct eni_dev *eni_dev;
 812	struct eni_vcc *eni_vcc;
 813	unsigned long size;
 814	int order;
 815
 816	DPRINTK("open_rx_second\n");
 817	eni_dev = ENI_DEV(vcc->dev);
 818	eni_vcc = ENI_VCC(vcc);
 819	if (!eni_vcc->rx) return 0;
 820	/* set up VCI descriptor */
 821	here = eni_dev->vci+vcc->vci*16;
 822	DPRINTK("loc 0x%x\n",(unsigned) (eni_vcc->recv-eni_dev->ram)/4);
 823	size = eni_vcc->words >> 8;
 824	for (order = -1; size; order++) size >>= 1;
 825	writel(0,here+4); /* descr, read = 0 */
 826	writel(0,here+8); /* write, state, count = 0 */
 827	if (eni_dev->rx_map[vcc->vci])
 828		printk(KERN_CRIT DEV_LABEL "(itf %d): BUG - VCI %d already "
 829		    "in use\n",vcc->dev->number,vcc->vci);
 830	eni_dev->rx_map[vcc->vci] = vcc; /* now it counts */
 831	writel(((vcc->qos.aal != ATM_AAL5 ? MID_MODE_RAW : MID_MODE_AAL5) <<
 832	    MID_VCI_MODE_SHIFT) | MID_VCI_PTI_MODE |
 833	    (((eni_vcc->recv-eni_dev->ram) >> (MID_LOC_SKIP+2)) <<
 834	    MID_VCI_LOCATION_SHIFT) | (order << MID_VCI_SIZE_SHIFT),here);
 835	return 0;
 836}
 837
 838
 839static void close_rx(struct atm_vcc *vcc)
 840{
 841	DECLARE_WAITQUEUE(wait,current);
 842	void __iomem *here;
 843	struct eni_dev *eni_dev;
 844	struct eni_vcc *eni_vcc;
 845
 846	eni_vcc = ENI_VCC(vcc);
 847	if (!eni_vcc->rx) return;
 848	eni_dev = ENI_DEV(vcc->dev);
 849	if (vcc->vpi != ATM_VPI_UNSPEC && vcc->vci != ATM_VCI_UNSPEC) {
 850		here = eni_dev->vci+vcc->vci*16;
 851		/* block receiver */
 852		writel((readl(here) & ~MID_VCI_MODE) | (MID_MODE_TRASH <<
 853		    MID_VCI_MODE_SHIFT),here);
 854		/* wait for receiver to become idle */
 855		udelay(27);
 856		/* discard pending cell */
 857		writel(readl(here) & ~MID_VCI_IN_SERVICE,here);
 858		/* don't accept any new ones */
 859		eni_dev->rx_map[vcc->vci] = NULL;
 860		/* wait for RX queue to drain */
 861		DPRINTK("eni_close: waiting for RX ...\n");
 862		EVENT("RX closing\n",0,0);
 863		add_wait_queue(&eni_dev->rx_wait,&wait);
 864		set_current_state(TASK_UNINTERRUPTIBLE);
 865		barrier();
 866		for (;;) {
 867			/* transition service->rx: rxing++, servicing-- */
 868			if (!eni_vcc->servicing) {
 869				barrier();
 870				if (!eni_vcc->rxing) break;
 871			}
 872			EVENT("drain PDUs (rx %ld, serv %ld)\n",eni_vcc->rxing,
 873			    eni_vcc->servicing);
 874			printk(KERN_INFO "%d+%d RX left\n",eni_vcc->servicing,
 875			    eni_vcc->rxing);
 876			schedule();
 877			set_current_state(TASK_UNINTERRUPTIBLE);
 878		}
 879		for (;;) {
 880			int at_end;
 881			u32 tmp;
 882
 883			tasklet_disable(&eni_dev->task);
 884			tmp = readl(eni_dev->vci+vcc->vci*16+4) & MID_VCI_READ;
 885			at_end = eni_vcc->rx_pos == tmp >> MID_VCI_READ_SHIFT;
 886			tasklet_enable(&eni_dev->task);
 887			if (at_end) break;
 888			EVENT("drain discard (host 0x%lx, nic 0x%lx)\n",
 889			    eni_vcc->rx_pos,tmp);
 890			printk(KERN_INFO "draining RX: host 0x%lx, nic 0x%x\n",
 891			    eni_vcc->rx_pos,tmp);
 892			schedule();
 893			set_current_state(TASK_UNINTERRUPTIBLE);
 894		}
 895		set_current_state(TASK_RUNNING);
 896		remove_wait_queue(&eni_dev->rx_wait,&wait);
 897	}
 898	eni_free_mem(eni_dev,eni_vcc->recv,eni_vcc->words << 2);
 899	eni_vcc->rx = NULL;
 900}
 901
 902
 903static int start_rx(struct atm_dev *dev)
 904{
 905	struct eni_dev *eni_dev;
 906
 907	eni_dev = ENI_DEV(dev);
 908	eni_dev->rx_map = (struct atm_vcc **) get_zeroed_page(GFP_KERNEL);
 909	if (!eni_dev->rx_map) {
 910		printk(KERN_ERR DEV_LABEL "(itf %d): couldn't get free page\n",
 911		    dev->number);
 912		free_page((unsigned long) eni_dev->free_list);
 913		return -ENOMEM;
 914	}
 915	eni_dev->rx_mult = DEFAULT_RX_MULT;
 916	eni_dev->fast = eni_dev->last_fast = NULL;
 917	eni_dev->slow = eni_dev->last_slow = NULL;
 918	init_waitqueue_head(&eni_dev->rx_wait);
 919	skb_queue_head_init(&eni_dev->rx_queue);
 920	eni_dev->serv_read = eni_in(MID_SERV_WRITE);
 921	eni_out(0,MID_DMA_WR_RX);
 922	return 0;
 923}
 924
 925
 926/*----------------------------------- TX ------------------------------------*/
 927
 928
 929enum enq_res { enq_ok,enq_next,enq_jam };
 930
 931
 932static inline void put_dma(int chan,u32 *dma,int *j,dma_addr_t paddr,
 933    u32 size)
 934{
 935	u32 init,words;
 936
 937	DPRINTK("put_dma: 0x%lx+0x%x\n",(unsigned long) paddr,size);
 938	EVENT("put_dma: 0x%lx+0x%lx\n",(unsigned long) paddr,size);
 939#if 0 /* don't complain anymore */
 940	if (paddr & 3)
 941		printk(KERN_ERR "put_dma: unaligned addr (0x%lx)\n",paddr);
 942	if (size & 3)
 943		printk(KERN_ERR "put_dma: unaligned size (0x%lx)\n",size);
 944#endif
 945	if (paddr & 3) {
 946		init = 4-(paddr & 3);
 947		if (init > size || size < 7) init = size;
 948		DPRINTK("put_dma: %lx DMA: %d/%d bytes\n",
 949		    (unsigned long) paddr,init,size);
 950		dma[(*j)++] = MID_DT_BYTE | (init << MID_DMA_COUNT_SHIFT) |
 951		    (chan << MID_DMA_CHAN_SHIFT);
 952		dma[(*j)++] = paddr;
 953		paddr += init;
 954		size -= init;
 955	}
 956	words = size >> 2;
 957	size &= 3;
 958	if (words && (paddr & 31)) {
 959		init = 8-((paddr & 31) >> 2);
 960		if (init > words) init = words;
 961		DPRINTK("put_dma: %lx DMA: %d/%d words\n",
 962		    (unsigned long) paddr,init,words);
 963		dma[(*j)++] = MID_DT_WORD | (init << MID_DMA_COUNT_SHIFT) |
 964		    (chan << MID_DMA_CHAN_SHIFT);
 965		dma[(*j)++] = paddr;
 966		paddr += init << 2;
 967		words -= init;
 968	}
 969#ifdef CONFIG_ATM_ENI_BURST_TX_16W /* may work with some PCI chipsets ... */
 970	if (words & ~15) {
 971		DPRINTK("put_dma: %lx DMA: %d*16/%d words\n",
 972		    (unsigned long) paddr,words >> 4,words);
 973		dma[(*j)++] = MID_DT_16W | ((words >> 4) << MID_DMA_COUNT_SHIFT)
 974		    | (chan << MID_DMA_CHAN_SHIFT);
 975		dma[(*j)++] = paddr;
 976		paddr += (words & ~15) << 2;
 977		words &= 15;
 978	}
 979#endif
 980#ifdef CONFIG_ATM_ENI_BURST_TX_8W /* recommended */
 981	if (words & ~7) {
 982		DPRINTK("put_dma: %lx DMA: %d*8/%d words\n",
 983		    (unsigned long) paddr,words >> 3,words);
 984		dma[(*j)++] = MID_DT_8W | ((words >> 3) << MID_DMA_COUNT_SHIFT)
 985		    | (chan << MID_DMA_CHAN_SHIFT);
 986		dma[(*j)++] = paddr;
 987		paddr += (words & ~7) << 2;
 988		words &= 7;
 989	}
 990#endif
 991#ifdef CONFIG_ATM_ENI_BURST_TX_4W /* probably useless if TX_8W or TX_16W */
 992	if (words & ~3) {
 993		DPRINTK("put_dma: %lx DMA: %d*4/%d words\n",
 994		    (unsigned long) paddr,words >> 2,words);
 995		dma[(*j)++] = MID_DT_4W | ((words >> 2) << MID_DMA_COUNT_SHIFT)
 996		    | (chan << MID_DMA_CHAN_SHIFT);
 997		dma[(*j)++] = paddr;
 998		paddr += (words & ~3) << 2;
 999		words &= 3;
1000	}
1001#endif
1002#ifdef CONFIG_ATM_ENI_BURST_TX_2W /* probably useless if TX_4W, TX_8W, ... */
1003	if (words & ~1) {
1004		DPRINTK("put_dma: %lx DMA: %d*2/%d words\n",
1005		    (unsigned long) paddr,words >> 1,words);
1006		dma[(*j)++] = MID_DT_2W | ((words >> 1) << MID_DMA_COUNT_SHIFT)
1007		    | (chan << MID_DMA_CHAN_SHIFT);
1008		dma[(*j)++] = paddr;
1009		paddr += (words & ~1) << 2;
1010		words &= 1;
1011	}
1012#endif
1013	if (words) {
1014		DPRINTK("put_dma: %lx DMA: %d words\n",(unsigned long) paddr,
1015		    words);
1016		dma[(*j)++] = MID_DT_WORD | (words << MID_DMA_COUNT_SHIFT) |
1017		    (chan << MID_DMA_CHAN_SHIFT);
1018		dma[(*j)++] = paddr;
1019		paddr += words << 2;
1020	}
1021	if (size) {
1022		DPRINTK("put_dma: %lx DMA: %d bytes\n",(unsigned long) paddr,
1023		    size);
1024		dma[(*j)++] = MID_DT_BYTE | (size << MID_DMA_COUNT_SHIFT) |
1025		    (chan << MID_DMA_CHAN_SHIFT);
1026		dma[(*j)++] = paddr;
1027	}
1028}
1029
1030
1031static enum enq_res do_tx(struct sk_buff *skb)
1032{
1033	struct atm_vcc *vcc;
1034	struct eni_dev *eni_dev;
1035	struct eni_vcc *eni_vcc;
1036	struct eni_tx *tx;
1037	dma_addr_t paddr;
1038	u32 dma_rd,dma_wr;
1039	u32 size; /* in words */
1040	int aal5,dma_size,i,j;
1041
1042	DPRINTK(">do_tx\n");
1043	NULLCHECK(skb);
1044	EVENT("do_tx: skb=0x%lx, %ld bytes\n",(unsigned long) skb,skb->len);
1045	vcc = ATM_SKB(skb)->vcc;
1046	NULLCHECK(vcc);
1047	eni_dev = ENI_DEV(vcc->dev);
1048	NULLCHECK(eni_dev);
1049	eni_vcc = ENI_VCC(vcc);
1050	tx = eni_vcc->tx;
1051	NULLCHECK(tx);
1052#if 0 /* Enable this for testing with the "align" program */
1053	{
1054		unsigned int hack = *((char *) skb->data)-'0';
1055
1056		if (hack < 8) {
1057			skb->data += hack;
1058			skb->len -= hack;
1059		}
1060	}
1061#endif
1062#if 0 /* should work now */
1063	if ((unsigned long) skb->data & 3)
1064		printk(KERN_ERR DEV_LABEL "(itf %d): VCI %d has mis-aligned "
1065		    "TX data\n",vcc->dev->number,vcc->vci);
1066#endif
1067	/*
1068	 * Potential future IP speedup: make hard_header big enough to put
1069	 * segmentation descriptor directly into PDU. Saves: 4 slave writes,
1070	 * 1 DMA xfer & 2 DMA'ed bytes (protocol layering is for wimps :-)
1071	 */
1072
1073	aal5 = vcc->qos.aal == ATM_AAL5;
1074	/* check space in buffer */
1075	if (!aal5)
1076		size = (ATM_CELL_PAYLOAD >> 2)+TX_DESCR_SIZE;
1077			/* cell without HEC plus segmentation header (includes
1078			   four-byte cell header) */
1079	else {
1080		size = skb->len+4*AAL5_TRAILER+ATM_CELL_PAYLOAD-1;
1081			/* add AAL5 trailer */
1082		size = ((size-(size % ATM_CELL_PAYLOAD)) >> 2)+TX_DESCR_SIZE;
1083						/* add segmentation header */
1084	}
1085	/*
1086	 * Can I move tx_pos by size bytes without getting closer than TX_GAP
1087	 * to the read pointer ? TX_GAP means to leave some space for what
1088	 * the manual calls "too close".
1089	 */
1090	if (!NEPMOK(tx->tx_pos,size+TX_GAP,
1091	    eni_in(MID_TX_RDPTR(tx->index)),tx->words)) {
1092		DPRINTK(DEV_LABEL "(itf %d): TX full (size %d)\n",
1093		    vcc->dev->number,size);
1094		return enq_next;
1095	}
1096	/* check DMA */
1097	dma_wr = eni_in(MID_DMA_WR_TX);
1098	dma_rd = eni_in(MID_DMA_RD_TX);
1099	dma_size = 3; /* JK for descriptor and final fill, plus final size
1100			 mis-alignment fix */
1101DPRINTK("iovcnt = %d\n",skb_shinfo(skb)->nr_frags);
1102	if (!skb_shinfo(skb)->nr_frags) dma_size += 5;
1103	else dma_size += 5*(skb_shinfo(skb)->nr_frags+1);
1104	if (dma_size > TX_DMA_BUF) {
1105		printk(KERN_CRIT DEV_LABEL "(itf %d): needs %d DMA entries "
1106		    "(got only %d)\n",vcc->dev->number,dma_size,TX_DMA_BUF);
1107	}
1108	DPRINTK("dma_wr is %d, tx_pos is %ld\n",dma_wr,tx->tx_pos);
1109	if (dma_wr != dma_rd && ((dma_rd+NR_DMA_TX-dma_wr) & (NR_DMA_TX-1)) <
1110	     dma_size) {
1111		printk(KERN_WARNING DEV_LABEL "(itf %d): TX DMA full\n",
1112		    vcc->dev->number);
1113		return enq_jam;
1114	}
1115	paddr = dma_map_single(&eni_dev->pci_dev->dev,skb->data,skb->len,
1116			       DMA_TO_DEVICE);
1117	ENI_PRV_PADDR(skb) = paddr;
1118	/* prepare DMA queue entries */
1119	j = 0;
1120	eni_dev->dma[j++] = (((tx->tx_pos+TX_DESCR_SIZE) & (tx->words-1)) <<
1121	     MID_DMA_COUNT_SHIFT) | (tx->index << MID_DMA_CHAN_SHIFT) |
1122	     MID_DT_JK;
1123	j++;
1124	if (!skb_shinfo(skb)->nr_frags)
1125		if (aal5) put_dma(tx->index,eni_dev->dma,&j,paddr,skb->len);
1126		else put_dma(tx->index,eni_dev->dma,&j,paddr+4,skb->len-4);
1127	else {
1128DPRINTK("doing direct send\n"); /* @@@ well, this doesn't work anyway */
1129		for (i = -1; i < skb_shinfo(skb)->nr_frags; i++)
1130			if (i == -1)
1131				put_dma(tx->index,eni_dev->dma,&j,(unsigned long)
1132				    skb->data,
1133				    skb_headlen(skb));
1134			else
1135				put_dma(tx->index,eni_dev->dma,&j,(unsigned long)
1136				    skb_frag_page(&skb_shinfo(skb)->frags[i]) +
1137					skb_shinfo(skb)->frags[i].page_offset,
1138				    skb_frag_size(&skb_shinfo(skb)->frags[i]));
1139	}
1140	if (skb->len & 3) {
1141		put_dma(tx->index, eni_dev->dma, &j, eni_dev->zero.dma,
1142			4 - (skb->len & 3));
1143	}
1144	/* JK for AAL5 trailer - AAL0 doesn't need it, but who cares ... */
1145	eni_dev->dma[j++] = (((tx->tx_pos+size) & (tx->words-1)) <<
1146	     MID_DMA_COUNT_SHIFT) | (tx->index << MID_DMA_CHAN_SHIFT) |
1147	     MID_DMA_END | MID_DT_JK;
1148	j++;
1149	DPRINTK("DMA at end: %d\n",j);
1150	/* store frame */
1151	writel((MID_SEG_TX_ID << MID_SEG_ID_SHIFT) |
1152	    (aal5 ? MID_SEG_AAL5 : 0) | (tx->prescaler << MID_SEG_PR_SHIFT) |
1153	    (tx->resolution << MID_SEG_RATE_SHIFT) |
1154	    (size/(ATM_CELL_PAYLOAD/4)),tx->send+tx->tx_pos*4);
1155/*printk("dsc = 0x%08lx\n",(unsigned long) readl(tx->send+tx->tx_pos*4));*/
1156	writel((vcc->vci << MID_SEG_VCI_SHIFT) |
1157            (aal5 ? 0 : (skb->data[3] & 0xf)) |
1158	    (ATM_SKB(skb)->atm_options & ATM_ATMOPT_CLP ? MID_SEG_CLP : 0),
1159	    tx->send+((tx->tx_pos+1) & (tx->words-1))*4);
1160	DPRINTK("size: %d, len:%d\n",size,skb->len);
1161	if (aal5)
1162		writel(skb->len,tx->send+
1163                    ((tx->tx_pos+size-AAL5_TRAILER) & (tx->words-1))*4);
1164	j = j >> 1;
1165	for (i = 0; i < j; i++) {
1166		writel(eni_dev->dma[i*2],eni_dev->tx_dma+dma_wr*8);
1167		writel(eni_dev->dma[i*2+1],eni_dev->tx_dma+dma_wr*8+4);
1168		dma_wr = (dma_wr+1) & (NR_DMA_TX-1);
1169	}
1170	ENI_PRV_POS(skb) = tx->tx_pos;
1171	ENI_PRV_SIZE(skb) = size;
1172	ENI_VCC(vcc)->txing += size;
1173	tx->tx_pos = (tx->tx_pos+size) & (tx->words-1);
1174	DPRINTK("dma_wr set to %d, tx_pos is now %ld\n",dma_wr,tx->tx_pos);
1175	eni_out(dma_wr,MID_DMA_WR_TX);
1176	skb_queue_tail(&eni_dev->tx_queue,skb);
1177queued++;
1178	return enq_ok;
1179}
1180
1181
1182static void poll_tx(struct atm_dev *dev)
1183{
1184	struct eni_tx *tx;
1185	struct sk_buff *skb;
1186	enum enq_res res;
1187	int i;
1188
1189	DPRINTK(">poll_tx\n");
1190	for (i = NR_CHAN-1; i >= 0; i--) {
1191		tx = &ENI_DEV(dev)->tx[i];
1192		if (tx->send)
1193			while ((skb = skb_dequeue(&tx->backlog))) {
1194				res = do_tx(skb);
1195				if (res == enq_ok) continue;
1196				DPRINTK("re-queuing TX PDU\n");
1197				skb_queue_head(&tx->backlog,skb);
1198requeued++;
1199				if (res == enq_jam) return;
1200				break;
1201			}
1202	}
1203}
1204
1205
1206static void dequeue_tx(struct atm_dev *dev)
1207{
1208	struct eni_dev *eni_dev;
1209	struct atm_vcc *vcc;
1210	struct sk_buff *skb;
1211	struct eni_tx *tx;
1212
1213	NULLCHECK(dev);
1214	eni_dev = ENI_DEV(dev);
1215	NULLCHECK(eni_dev);
1216	while ((skb = skb_dequeue(&eni_dev->tx_queue))) {
1217		vcc = ATM_SKB(skb)->vcc;
1218		NULLCHECK(vcc);
1219		tx = ENI_VCC(vcc)->tx;
1220		NULLCHECK(ENI_VCC(vcc)->tx);
1221		DPRINTK("dequeue_tx: next 0x%lx curr 0x%x\n",ENI_PRV_POS(skb),
1222		    (unsigned) eni_in(MID_TX_DESCRSTART(tx->index)));
1223		if (ENI_VCC(vcc)->txing < tx->words && ENI_PRV_POS(skb) ==
1224		    eni_in(MID_TX_DESCRSTART(tx->index))) {
1225			skb_queue_head(&eni_dev->tx_queue,skb);
1226			break;
1227		}
1228		ENI_VCC(vcc)->txing -= ENI_PRV_SIZE(skb);
1229		dma_unmap_single(&eni_dev->pci_dev->dev,ENI_PRV_PADDR(skb),skb->len,
1230				 DMA_TO_DEVICE);
1231		if (vcc->pop) vcc->pop(vcc,skb);
1232		else dev_kfree_skb_irq(skb);
1233		atomic_inc(&vcc->stats->tx);
1234		wake_up(&eni_dev->tx_wait);
1235dma_complete++;
1236	}
1237}
1238
1239
1240static struct eni_tx *alloc_tx(struct eni_dev *eni_dev,int ubr)
1241{
1242	int i;
1243
1244	for (i = !ubr; i < NR_CHAN; i++)
1245		if (!eni_dev->tx[i].send) return eni_dev->tx+i;
1246	return NULL;
1247}
1248
1249
1250static int comp_tx(struct eni_dev *eni_dev,int *pcr,int reserved,int *pre,
1251    int *res,int unlimited)
1252{
1253	static const int pre_div[] = { 4,16,128,2048 };
1254	    /* 2^(((x+2)^2-(x+2))/2+1) */
1255
1256	if (unlimited) *pre = *res = 0;
1257	else {
1258		if (*pcr > 0) {
1259			int div;
1260
1261			for (*pre = 0; *pre < 3; (*pre)++)
1262				if (TS_CLOCK/pre_div[*pre]/64 <= *pcr) break;
1263			div = pre_div[*pre]**pcr;
1264			DPRINTK("min div %d\n",div);
1265			*res = TS_CLOCK/div-1;
1266		}
1267		else {
1268			int div;
1269
1270			if (!*pcr) *pcr = eni_dev->tx_bw+reserved;
1271			for (*pre = 3; *pre >= 0; (*pre)--)
1272				if (TS_CLOCK/pre_div[*pre]/64 > -*pcr) break;
1273			if (*pre < 3) (*pre)++; /* else fail later */
1274			div = pre_div[*pre]*-*pcr;
1275			DPRINTK("max div %d\n",div);
1276			*res = DIV_ROUND_UP(TS_CLOCK, div)-1;
1277		}
1278		if (*res < 0) *res = 0;
1279		if (*res > MID_SEG_MAX_RATE) *res = MID_SEG_MAX_RATE;
1280	}
1281	*pcr = TS_CLOCK/pre_div[*pre]/(*res+1);
1282	DPRINTK("out pcr: %d (%d:%d)\n",*pcr,*pre,*res);
1283	return 0;
1284}
1285
1286
1287static int reserve_or_set_tx(struct atm_vcc *vcc,struct atm_trafprm *txtp,
1288    int set_rsv,int set_shp)
1289{
1290	struct eni_dev *eni_dev = ENI_DEV(vcc->dev);
1291	struct eni_vcc *eni_vcc = ENI_VCC(vcc);
1292	struct eni_tx *tx;
1293	unsigned long size;
1294	void __iomem *mem;
1295	int rate,ubr,unlimited,new_tx;
1296	int pre,res,order;
1297	int error;
1298
1299	rate = atm_pcr_goal(txtp);
1300	ubr = txtp->traffic_class == ATM_UBR;
1301	unlimited = ubr && (!rate || rate <= -ATM_OC3_PCR ||
1302	    rate >= ATM_OC3_PCR);
1303	if (!unlimited) {
1304		size = txtp->max_sdu*eni_dev->tx_mult/100;
1305		if (size > MID_MAX_BUF_SIZE && txtp->max_sdu <=
1306		    MID_MAX_BUF_SIZE)
1307			size = MID_MAX_BUF_SIZE;
1308	}
1309	else {
1310		if (eni_dev->ubr) {
1311			eni_vcc->tx = eni_dev->ubr;
1312			txtp->pcr = ATM_OC3_PCR;
1313			return 0;
1314		}
1315		size = UBR_BUFFER;
1316	}
1317	new_tx = !eni_vcc->tx;
1318	mem = NULL; /* for gcc */
1319	if (!new_tx) tx = eni_vcc->tx;
1320	else {
1321		mem = eni_alloc_mem(eni_dev,&size);
1322		if (!mem) return -ENOBUFS;
1323		tx = alloc_tx(eni_dev,unlimited);
1324		if (!tx) {
1325			eni_free_mem(eni_dev,mem,size);
1326			return -EBUSY;
1327		}
1328		DPRINTK("got chan %d\n",tx->index);
1329		tx->reserved = tx->shaping = 0;
1330		tx->send = mem;
1331		tx->words = size >> 2;
1332		skb_queue_head_init(&tx->backlog);
1333		for (order = 0; size > (1 << (order+10)); order++);
1334		eni_out((order << MID_SIZE_SHIFT) |
1335		    ((tx->send-eni_dev->ram) >> (MID_LOC_SKIP+2)),
1336		    MID_TX_PLACE(tx->index));
1337		tx->tx_pos = eni_in(MID_TX_DESCRSTART(tx->index)) &
1338		    MID_DESCR_START;
1339	}
1340	error = comp_tx(eni_dev,&rate,tx->reserved,&pre,&res,unlimited);
1341	if (!error  && txtp->min_pcr > rate) error = -EINVAL;
1342	if (!error && txtp->max_pcr && txtp->max_pcr != ATM_MAX_PCR &&
1343	    txtp->max_pcr < rate) error = -EINVAL;
1344	if (!error && !ubr && rate > eni_dev->tx_bw+tx->reserved)
1345		error = -EINVAL;
1346	if (!error && set_rsv && !set_shp && rate < tx->shaping)
1347		error = -EINVAL;
1348	if (!error && !set_rsv && rate > tx->reserved && !ubr)
1349		error = -EINVAL;
1350	if (error) {
1351		if (new_tx) {
1352			tx->send = NULL;
1353			eni_free_mem(eni_dev,mem,size);
1354		}
1355		return error;
1356	}
1357	txtp->pcr = rate;
1358	if (set_rsv && !ubr) {
1359		eni_dev->tx_bw += tx->reserved;
1360		tx->reserved = rate;
1361		eni_dev->tx_bw -= rate;
1362	}
1363	if (set_shp || (unlimited && new_tx)) {
1364		if (unlimited && new_tx) eni_dev->ubr = tx;
1365		tx->prescaler = pre;
1366		tx->resolution = res;
1367		tx->shaping = rate;
1368	}
1369	if (set_shp) eni_vcc->tx = tx;
1370	DPRINTK("rsv %d shp %d\n",tx->reserved,tx->shaping);
1371	return 0;
1372}
1373
1374
1375static int open_tx_first(struct atm_vcc *vcc)
1376{
1377	ENI_VCC(vcc)->tx = NULL;
1378	if (vcc->qos.txtp.traffic_class == ATM_NONE) return 0;
1379	ENI_VCC(vcc)->txing = 0;
1380	return reserve_or_set_tx(vcc,&vcc->qos.txtp,1,1);
1381}
1382
1383
1384static int open_tx_second(struct atm_vcc *vcc)
1385{
1386	return 0; /* nothing to do */
1387}
1388
1389
1390static void close_tx(struct atm_vcc *vcc)
1391{
1392	DECLARE_WAITQUEUE(wait,current);
1393	struct eni_dev *eni_dev;
1394	struct eni_vcc *eni_vcc;
1395
1396	eni_vcc = ENI_VCC(vcc);
1397	if (!eni_vcc->tx) return;
1398	eni_dev = ENI_DEV(vcc->dev);
1399	/* wait for TX queue to drain */
1400	DPRINTK("eni_close: waiting for TX ...\n");
1401	add_wait_queue(&eni_dev->tx_wait,&wait);
1402	set_current_state(TASK_UNINTERRUPTIBLE);
1403	for (;;) {
1404		int txing;
1405
1406		tasklet_disable(&eni_dev->task);
1407		txing = skb_peek(&eni_vcc->tx->backlog) || eni_vcc->txing;
1408		tasklet_enable(&eni_dev->task);
1409		if (!txing) break;
1410		DPRINTK("%d TX left\n",eni_vcc->txing);
1411		schedule();
1412		set_current_state(TASK_UNINTERRUPTIBLE);
1413	}
1414	set_current_state(TASK_RUNNING);
1415	remove_wait_queue(&eni_dev->tx_wait,&wait);
1416	if (eni_vcc->tx != eni_dev->ubr) {
1417		/*
1418		 * Looping a few times in here is probably far cheaper than
1419		 * keeping track of TX completions all the time, so let's poll
1420		 * a bit ...
1421		 */
1422		while (eni_in(MID_TX_RDPTR(eni_vcc->tx->index)) !=
1423		    eni_in(MID_TX_DESCRSTART(eni_vcc->tx->index)))
1424			schedule();
1425		eni_free_mem(eni_dev,eni_vcc->tx->send,eni_vcc->tx->words << 2);
1426		eni_vcc->tx->send = NULL;
1427		eni_dev->tx_bw += eni_vcc->tx->reserved;
1428	}
1429	eni_vcc->tx = NULL;
1430}
1431
1432
1433static int start_tx(struct atm_dev *dev)
1434{
1435	struct eni_dev *eni_dev;
1436	int i;
1437
1438	eni_dev = ENI_DEV(dev);
1439	eni_dev->lost = 0;
1440	eni_dev->tx_bw = ATM_OC3_PCR;
1441	eni_dev->tx_mult = DEFAULT_TX_MULT;
1442	init_waitqueue_head(&eni_dev->tx_wait);
1443	eni_dev->ubr = NULL;
1444	skb_queue_head_init(&eni_dev->tx_queue);
1445	eni_out(0,MID_DMA_WR_TX);
1446	for (i = 0; i < NR_CHAN; i++) {
1447		eni_dev->tx[i].send = NULL;
1448		eni_dev->tx[i].index = i;
1449	}
1450	return 0;
1451}
1452
1453
1454/*--------------------------------- common ----------------------------------*/
1455
1456
1457#if 0 /* may become useful again when tuning things */
1458
1459static void foo(void)
1460{
1461printk(KERN_INFO
1462  "tx_complete=%d,dma_complete=%d,queued=%d,requeued=%d,sub=%d,\n"
1463  "backlogged=%d,rx_enqueued=%d,rx_dequeued=%d,putting=%d,pushed=%d\n",
1464  tx_complete,dma_complete,queued,requeued,submitted,backlogged,
1465  rx_enqueued,rx_dequeued,putting,pushed);
1466if (eni_boards) printk(KERN_INFO "loss: %ld\n",ENI_DEV(eni_boards)->lost);
1467}
1468
1469#endif
1470
1471
1472static void bug_int(struct atm_dev *dev,unsigned long reason)
1473{
1474	DPRINTK(">bug_int\n");
1475	if (reason & MID_DMA_ERR_ACK)
1476		printk(KERN_CRIT DEV_LABEL "(itf %d): driver error - DMA "
1477		    "error\n",dev->number);
1478	if (reason & MID_TX_IDENT_MISM)
1479		printk(KERN_CRIT DEV_LABEL "(itf %d): driver error - ident "
1480		    "mismatch\n",dev->number);
1481	if (reason & MID_TX_DMA_OVFL)
1482		printk(KERN_CRIT DEV_LABEL "(itf %d): driver error - DMA "
1483		    "overflow\n",dev->number);
1484	EVENT("---dump ends here---\n",0,0);
1485	printk(KERN_NOTICE "---recent events---\n");
1486	event_dump();
1487}
1488
1489
1490static irqreturn_t eni_int(int irq,void *dev_id)
1491{
1492	struct atm_dev *dev;
1493	struct eni_dev *eni_dev;
1494	u32 reason;
1495
1496	DPRINTK(">eni_int\n");
1497	dev = dev_id;
1498	eni_dev = ENI_DEV(dev);
1499	reason = eni_in(MID_ISA);
1500	DPRINTK(DEV_LABEL ": int 0x%lx\n",(unsigned long) reason);
1501	/*
1502	 * Must handle these two right now, because reading ISA doesn't clear
1503	 * them, so they re-occur and we never make it to the tasklet. Since
1504	 * they're rare, we don't mind the occasional invocation of eni_tasklet
1505	 * with eni_dev->events == 0.
1506	 */
1507	if (reason & MID_STAT_OVFL) {
1508		EVENT("stat overflow\n",0,0);
1509		eni_dev->lost += eni_in(MID_STAT) & MID_OVFL_TRASH;
1510	}
1511	if (reason & MID_SUNI_INT) {
1512		EVENT("SUNI int\n",0,0);
1513		dev->phy->interrupt(dev);
1514#if 0
1515		foo();
1516#endif
1517	}
1518	spin_lock(&eni_dev->lock);
1519	eni_dev->events |= reason;
1520	spin_unlock(&eni_dev->lock);
1521	tasklet_schedule(&eni_dev->task);
1522	return IRQ_HANDLED;
1523}
1524
1525
1526static void eni_tasklet(unsigned long data)
1527{
1528	struct atm_dev *dev = (struct atm_dev *) data;
1529	struct eni_dev *eni_dev = ENI_DEV(dev);
1530	unsigned long flags;
1531	u32 events;
1532
1533	DPRINTK("eni_tasklet (dev %p)\n",dev);
1534	spin_lock_irqsave(&eni_dev->lock,flags);
1535	events = xchg(&eni_dev->events,0);
1536	spin_unlock_irqrestore(&eni_dev->lock,flags);
1537	if (events & MID_RX_DMA_COMPLETE) {
1538		EVENT("INT: RX DMA complete, starting dequeue_rx\n",0,0);
1539		dequeue_rx(dev);
1540		EVENT("dequeue_rx done, starting poll_rx\n",0,0);
1541		poll_rx(dev);
1542		EVENT("poll_rx done\n",0,0);
1543		/* poll_tx ? */
1544	}
1545	if (events & MID_SERVICE) {
1546		EVENT("INT: service, starting get_service\n",0,0);
1547		get_service(dev);
1548		EVENT("get_service done, starting poll_rx\n",0,0);
1549		poll_rx(dev);
1550		EVENT("poll_rx done\n",0,0);
1551	}
1552 	if (events & MID_TX_DMA_COMPLETE) {
1553		EVENT("INT: TX DMA COMPLETE\n",0,0);
1554		dequeue_tx(dev);
1555	}
1556	if (events & MID_TX_COMPLETE) {
1557		EVENT("INT: TX COMPLETE\n",0,0);
1558tx_complete++;
1559		wake_up(&eni_dev->tx_wait);
1560		/* poll_rx ? */
1561	}
1562	if (events & (MID_DMA_ERR_ACK | MID_TX_IDENT_MISM | MID_TX_DMA_OVFL)) {
1563		EVENT("bug interrupt\n",0,0);
1564		bug_int(dev,events);
1565	}
1566	poll_tx(dev);
1567}
1568
1569
1570/*--------------------------------- entries ---------------------------------*/
1571
1572
1573static char * const media_name[] = {
1574    "MMF", "SMF", "MMF", "03?", /*  0- 3 */
1575    "UTP", "05?", "06?", "07?", /*  4- 7 */
1576    "TAXI","09?", "10?", "11?", /*  8-11 */
1577    "12?", "13?", "14?", "15?", /* 12-15 */
1578    "MMF", "SMF", "18?", "19?", /* 16-19 */
1579    "UTP", "21?", "22?", "23?", /* 20-23 */
1580    "24?", "25?", "26?", "27?", /* 24-27 */
1581    "28?", "29?", "30?", "31?"  /* 28-31 */
1582};
1583
1584
1585#define SET_SEPROM \
1586  ({ if (!error && !pci_error) { \
1587    pci_error = pci_write_config_byte(eni_dev->pci_dev,PCI_TONGA_CTRL,tonga); \
1588    udelay(10); /* 10 usecs */ \
1589  } })
1590#define GET_SEPROM \
1591  ({ if (!error && !pci_error) { \
1592    pci_error = pci_read_config_byte(eni_dev->pci_dev,PCI_TONGA_CTRL,&tonga); \
1593    udelay(10); /* 10 usecs */ \
1594  } })
1595
1596
1597static int get_esi_asic(struct atm_dev *dev)
1598{
1599	struct eni_dev *eni_dev;
1600	unsigned char tonga;
1601	int error,failed,pci_error;
1602	int address,i,j;
1603
1604	eni_dev = ENI_DEV(dev);
1605	error = pci_error = 0;
1606	tonga = SEPROM_MAGIC | SEPROM_DATA | SEPROM_CLK;
1607	SET_SEPROM;
1608	for (i = 0; i < ESI_LEN && !error && !pci_error; i++) {
1609		/* start operation */
1610		tonga |= SEPROM_DATA;
1611		SET_SEPROM;
1612		tonga |= SEPROM_CLK;
1613		SET_SEPROM;
1614		tonga &= ~SEPROM_DATA;
1615		SET_SEPROM;
1616		tonga &= ~SEPROM_CLK;
1617		SET_SEPROM;
1618		/* send address */
1619		address = ((i+SEPROM_ESI_BASE) << 1)+1;
1620		for (j = 7; j >= 0; j--) {
1621			tonga = (address >> j) & 1 ? tonga | SEPROM_DATA :
1622			    tonga & ~SEPROM_DATA;
1623			SET_SEPROM;
1624			tonga |= SEPROM_CLK;
1625			SET_SEPROM;
1626			tonga &= ~SEPROM_CLK;
1627			SET_SEPROM;
1628		}
1629		/* get ack */
1630		tonga |= SEPROM_DATA;
1631		SET_SEPROM;
1632		tonga |= SEPROM_CLK;
1633		SET_SEPROM;
1634		GET_SEPROM;
1635		failed = tonga & SEPROM_DATA;
1636		tonga &= ~SEPROM_CLK;
1637		SET_SEPROM;
1638		tonga |= SEPROM_DATA;
1639		SET_SEPROM;
1640		if (failed) error = -EIO;
1641		else {
1642			dev->esi[i] = 0;
1643			for (j = 7; j >= 0; j--) {
1644				dev->esi[i] <<= 1;
1645				tonga |= SEPROM_DATA;
1646				SET_SEPROM;
1647				tonga |= SEPROM_CLK;
1648				SET_SEPROM;
1649				GET_SEPROM;
1650				if (tonga & SEPROM_DATA) dev->esi[i] |= 1;
1651				tonga &= ~SEPROM_CLK;
1652				SET_SEPROM;
1653				tonga |= SEPROM_DATA;
1654				SET_SEPROM;
1655			}
1656			/* get ack */
1657			tonga |= SEPROM_DATA;
1658			SET_SEPROM;
1659			tonga |= SEPROM_CLK;
1660			SET_SEPROM;
1661			GET_SEPROM;
1662			if (!(tonga & SEPROM_DATA)) error = -EIO;
1663			tonga &= ~SEPROM_CLK;
1664			SET_SEPROM;
1665			tonga |= SEPROM_DATA;
1666			SET_SEPROM;
1667		}
1668		/* stop operation */
1669		tonga &= ~SEPROM_DATA;
1670		SET_SEPROM;
1671		tonga |= SEPROM_CLK;
1672		SET_SEPROM;
1673		tonga |= SEPROM_DATA;
1674		SET_SEPROM;
1675	}
1676	if (pci_error) {
1677		printk(KERN_ERR DEV_LABEL "(itf %d): error reading ESI "
1678		    "(0x%02x)\n",dev->number,pci_error);
1679		error = -EIO;
1680	}
1681	return error;
1682}
1683
1684
1685#undef SET_SEPROM
1686#undef GET_SEPROM
1687
1688
1689static int get_esi_fpga(struct atm_dev *dev, void __iomem *base)
1690{
1691	void __iomem *mac_base;
1692	int i;
1693
1694	mac_base = base+EPROM_SIZE-sizeof(struct midway_eprom);
1695	for (i = 0; i < ESI_LEN; i++) dev->esi[i] = readb(mac_base+(i^3));
1696	return 0;
1697}
1698
1699
1700static int eni_do_init(struct atm_dev *dev)
1701{
1702	struct midway_eprom __iomem *eprom;
1703	struct eni_dev *eni_dev;
1704	struct pci_dev *pci_dev;
1705	unsigned long real_base;
1706	void __iomem *base;
1707	int error,i,last;
1708
1709	DPRINTK(">eni_init\n");
1710	dev->ci_range.vpi_bits = 0;
1711	dev->ci_range.vci_bits = NR_VCI_LD;
1712	dev->link_rate = ATM_OC3_PCR;
1713	eni_dev = ENI_DEV(dev);
1714	pci_dev = eni_dev->pci_dev;
1715	real_base = pci_resource_start(pci_dev, 0);
1716	eni_dev->irq = pci_dev->irq;
1717	if ((error = pci_write_config_word(pci_dev,PCI_COMMAND,
1718	    PCI_COMMAND_MEMORY |
1719	    (eni_dev->asic ? PCI_COMMAND_PARITY | PCI_COMMAND_SERR : 0)))) {
1720		printk(KERN_ERR DEV_LABEL "(itf %d): can't enable memory "
1721		    "(0x%02x)\n",dev->number,error);
1722		return -EIO;
1723	}
1724	printk(KERN_NOTICE DEV_LABEL "(itf %d): rev.%d,base=0x%lx,irq=%d,",
1725	    dev->number,pci_dev->revision,real_base,eni_dev->irq);
1726	if (!(base = ioremap_nocache(real_base,MAP_MAX_SIZE))) {
1727		printk("\n");
1728		printk(KERN_ERR DEV_LABEL "(itf %d): can't set up page "
1729		    "mapping\n",dev->number);
1730		return error;
1731	}
1732	eni_dev->ioaddr = base;
1733	eni_dev->base_diff = real_base - (unsigned long) base;
1734	/* id may not be present in ASIC Tonga boards - check this @@@ */
1735	if (!eni_dev->asic) {
1736		eprom = (base+EPROM_SIZE-sizeof(struct midway_eprom));
1737		if (readl(&eprom->magic) != ENI155_MAGIC) {
1738			printk("\n");
1739			printk(KERN_ERR DEV_LABEL
1740			       "(itf %d): bad magic - expected 0x%x, got 0x%x\n",
1741			       dev->number, ENI155_MAGIC,
1742			       (unsigned)readl(&eprom->magic));
1743			error = -EINVAL;
1744			goto unmap;
1745		}
1746	}
1747	eni_dev->phy = base+PHY_BASE;
1748	eni_dev->reg = base+REG_BASE;
1749	eni_dev->ram = base+RAM_BASE;
1750	last = MAP_MAX_SIZE-RAM_BASE;
1751	for (i = last-RAM_INCREMENT; i >= 0; i -= RAM_INCREMENT) {
1752		writel(0x55555555,eni_dev->ram+i);
1753		if (readl(eni_dev->ram+i) != 0x55555555) last = i;
1754		else {
1755			writel(0xAAAAAAAA,eni_dev->ram+i);
1756			if (readl(eni_dev->ram+i) != 0xAAAAAAAA) last = i;
1757			else writel(i,eni_dev->ram+i);
1758		}
1759	}
1760	for (i = 0; i < last; i += RAM_INCREMENT)
1761		if (readl(eni_dev->ram+i) != i) break;
1762	eni_dev->mem = i;
1763	memset_io(eni_dev->ram,0,eni_dev->mem);
1764	/* TODO: should shrink allocation now */
1765	printk("mem=%dkB (",eni_dev->mem >> 10);
1766	/* TODO: check for non-SUNI, check for TAXI ? */
1767	if (!(eni_in(MID_RES_ID_MCON) & 0x200) != !eni_dev->asic) {
1768		printk(")\n");
1769		printk(KERN_ERR DEV_LABEL "(itf %d): ERROR - wrong id 0x%x\n",
1770		    dev->number,(unsigned) eni_in(MID_RES_ID_MCON));
1771		error = -EINVAL;
1772		goto unmap;
1773	}
1774	error = eni_dev->asic ? get_esi_asic(dev) : get_esi_fpga(dev,base);
1775	if (error)
1776		goto unmap;
1777	for (i = 0; i < ESI_LEN; i++)
1778		printk("%s%02X",i ? "-" : "",dev->esi[i]);
1779	printk(")\n");
1780	printk(KERN_NOTICE DEV_LABEL "(itf %d): %s,%s\n",dev->number,
1781	    eni_in(MID_RES_ID_MCON) & 0x200 ? "ASIC" : "FPGA",
1782	    media_name[eni_in(MID_RES_ID_MCON) & DAUGTHER_ID]);
1783
1784	error = suni_init(dev);
1785	if (error)
1786		goto unmap;
1787out:
1788	return error;
1789unmap:
1790	iounmap(base);
1791	goto out;
1792}
1793
1794static void eni_do_release(struct atm_dev *dev)
1795{
1796	struct eni_dev *ed = ENI_DEV(dev);
1797
1798	dev->phy->stop(dev);
1799	dev->phy = NULL;
1800	iounmap(ed->ioaddr);
1801}
1802
1803static int eni_start(struct atm_dev *dev)
1804{
1805	struct eni_dev *eni_dev;
1806	
1807	void __iomem *buf;
1808	unsigned long buffer_mem;
1809	int error;
1810
1811	DPRINTK(">eni_start\n");
1812	eni_dev = ENI_DEV(dev);
1813	if (request_irq(eni_dev->irq,&eni_int,IRQF_SHARED,DEV_LABEL,dev)) {
1814		printk(KERN_ERR DEV_LABEL "(itf %d): IRQ%d is already in use\n",
1815		    dev->number,eni_dev->irq);
1816		error = -EAGAIN;
1817		goto out;
1818	}
1819	pci_set_master(eni_dev->pci_dev);
1820	if ((error = pci_write_config_word(eni_dev->pci_dev,PCI_COMMAND,
1821	    PCI_COMMAND_MEMORY | PCI_COMMAND_MASTER |
1822	    (eni_dev->asic ? PCI_COMMAND_PARITY | PCI_COMMAND_SERR : 0)))) {
1823		printk(KERN_ERR DEV_LABEL "(itf %d): can't enable memory+"
1824		    "master (0x%02x)\n",dev->number,error);
1825		goto free_irq;
1826	}
1827	if ((error = pci_write_config_byte(eni_dev->pci_dev,PCI_TONGA_CTRL,
1828	    END_SWAP_DMA))) {
1829		printk(KERN_ERR DEV_LABEL "(itf %d): can't set endian swap "
1830		    "(0x%02x)\n",dev->number,error);
1831		goto free_irq;
1832	}
1833	/* determine addresses of internal tables */
1834	eni_dev->vci = eni_dev->ram;
1835	eni_dev->rx_dma = eni_dev->ram+NR_VCI*16;
1836	eni_dev->tx_dma = eni_dev->rx_dma+NR_DMA_RX*8;
1837	eni_dev->service = eni_dev->tx_dma+NR_DMA_TX*8;
1838	buf = eni_dev->service+NR_SERVICE*4;
1839	DPRINTK("vci 0x%lx,rx 0x%lx, tx 0x%lx,srv 0x%lx,buf 0x%lx\n",
1840	     eni_dev->vci,eni_dev->rx_dma,eni_dev->tx_dma,
1841	     eni_dev->service,buf);
1842	spin_lock_init(&eni_dev->lock);
1843	tasklet_init(&eni_dev->task,eni_tasklet,(unsigned long) dev);
1844	eni_dev->events = 0;
1845	/* initialize memory management */
1846	buffer_mem = eni_dev->mem - (buf - eni_dev->ram);
1847	eni_dev->free_list_size = buffer_mem/MID_MIN_BUF_SIZE/2;
1848	eni_dev->free_list = kmalloc(
1849	    sizeof(struct eni_free)*(eni_dev->free_list_size+1),GFP_KERNEL);
 
1850	if (!eni_dev->free_list) {
1851		printk(KERN_ERR DEV_LABEL "(itf %d): couldn't get free page\n",
1852		    dev->number);
1853		error = -ENOMEM;
1854		goto free_irq;
1855	}
1856	eni_dev->free_len = 0;
1857	eni_put_free(eni_dev,buf,buffer_mem);
1858	memset_io(eni_dev->vci,0,16*NR_VCI); /* clear VCI table */
1859	/*
1860	 * byte_addr  free (k)
1861	 * 0x00000000     512  VCI table
1862	 * 0x00004000	  496  RX DMA
1863	 * 0x00005000	  492  TX DMA
1864	 * 0x00006000	  488  service list
1865	 * 0x00007000	  484  buffers
1866	 * 0x00080000	    0  end (512kB)
1867	 */
1868	eni_out(0xffffffff,MID_IE);
1869	error = start_tx(dev);
1870	if (error) goto free_list;
1871	error = start_rx(dev);
1872	if (error) goto free_list;
1873	error = dev->phy->start(dev);
1874	if (error) goto free_list;
1875	eni_out(eni_in(MID_MC_S) | (1 << MID_INT_SEL_SHIFT) |
1876	    MID_TX_LOCK_MODE | MID_DMA_ENABLE | MID_TX_ENABLE | MID_RX_ENABLE,
1877	    MID_MC_S);
1878	    /* Tonga uses SBus INTReq1 */
1879	(void) eni_in(MID_ISA); /* clear Midway interrupts */
1880	return 0;
1881
1882free_list:
1883	kfree(eni_dev->free_list);
1884
1885free_irq:
1886	free_irq(eni_dev->irq, dev);
1887
1888out:
1889	return error;
1890}
1891
1892
1893static void eni_close(struct atm_vcc *vcc)
1894{
1895	DPRINTK(">eni_close\n");
1896	if (!ENI_VCC(vcc)) return;
1897	clear_bit(ATM_VF_READY,&vcc->flags);
1898	close_rx(vcc);
1899	close_tx(vcc);
1900	DPRINTK("eni_close: done waiting\n");
1901	/* deallocate memory */
1902	kfree(ENI_VCC(vcc));
1903	vcc->dev_data = NULL;
1904	clear_bit(ATM_VF_ADDR,&vcc->flags);
1905	/*foo();*/
1906}
1907
1908
1909static int eni_open(struct atm_vcc *vcc)
1910{
1911	struct eni_vcc *eni_vcc;
1912	int error;
1913	short vpi = vcc->vpi;
1914	int vci = vcc->vci;
1915
1916	DPRINTK(">eni_open\n");
1917	EVENT("eni_open\n",0,0);
1918	if (!test_bit(ATM_VF_PARTIAL,&vcc->flags))
1919		vcc->dev_data = NULL;
1920	if (vci != ATM_VPI_UNSPEC && vpi != ATM_VCI_UNSPEC)
1921		set_bit(ATM_VF_ADDR,&vcc->flags);
1922	if (vcc->qos.aal != ATM_AAL0 && vcc->qos.aal != ATM_AAL5)
1923		return -EINVAL;
1924	DPRINTK(DEV_LABEL "(itf %d): open %d.%d\n",vcc->dev->number,vcc->vpi,
1925	    vcc->vci);
1926	if (!test_bit(ATM_VF_PARTIAL,&vcc->flags)) {
1927		eni_vcc = kmalloc(sizeof(struct eni_vcc),GFP_KERNEL);
1928		if (!eni_vcc) return -ENOMEM;
1929		vcc->dev_data = eni_vcc;
1930		eni_vcc->tx = NULL; /* for eni_close after open_rx */
1931		if ((error = open_rx_first(vcc))) {
1932			eni_close(vcc);
1933			return error;
1934		}
1935		if ((error = open_tx_first(vcc))) {
1936			eni_close(vcc);
1937			return error;
1938		}
1939	}
1940	if (vci == ATM_VPI_UNSPEC || vpi == ATM_VCI_UNSPEC) return 0;
1941	if ((error = open_rx_second(vcc))) {
1942		eni_close(vcc);
1943		return error;
1944	}
1945	if ((error = open_tx_second(vcc))) {
1946		eni_close(vcc);
1947		return error;
1948	}
1949	set_bit(ATM_VF_READY,&vcc->flags);
1950	/* should power down SUNI while !ref_count @@@ */
1951	return 0;
1952}
1953
1954
1955static int eni_change_qos(struct atm_vcc *vcc,struct atm_qos *qos,int flgs)
1956{
1957	struct eni_dev *eni_dev = ENI_DEV(vcc->dev);
1958	struct eni_tx *tx = ENI_VCC(vcc)->tx;
1959	struct sk_buff *skb;
1960	int error,rate,rsv,shp;
1961
1962	if (qos->txtp.traffic_class == ATM_NONE) return 0;
1963	if (tx == eni_dev->ubr) return -EBADFD;
1964	rate = atm_pcr_goal(&qos->txtp);
1965	if (rate < 0) rate = -rate;
1966	rsv = shp = 0;
1967	if ((flgs & ATM_MF_DEC_RSV) && rate && rate < tx->reserved) rsv = 1;
1968	if ((flgs & ATM_MF_INC_RSV) && (!rate || rate > tx->reserved)) rsv = 1;
1969	if ((flgs & ATM_MF_DEC_SHP) && rate && rate < tx->shaping) shp = 1;
1970	if ((flgs & ATM_MF_INC_SHP) && (!rate || rate > tx->shaping)) shp = 1;
1971	if (!rsv && !shp) return 0;
1972	error = reserve_or_set_tx(vcc,&qos->txtp,rsv,shp);
1973	if (error) return error;
1974	if (shp && !(flgs & ATM_MF_IMMED)) return 0;
1975	/*
1976	 * Walk through the send buffer and patch the rate information in all
1977	 * segmentation buffer descriptors of this VCC.
1978	 */
1979	tasklet_disable(&eni_dev->task);
1980	skb_queue_walk(&eni_dev->tx_queue, skb) {
1981		void __iomem *dsc;
1982
1983		if (ATM_SKB(skb)->vcc != vcc) continue;
1984		dsc = tx->send+ENI_PRV_POS(skb)*4;
1985		writel((readl(dsc) & ~(MID_SEG_RATE | MID_SEG_PR)) |
1986		    (tx->prescaler << MID_SEG_PR_SHIFT) |
1987		    (tx->resolution << MID_SEG_RATE_SHIFT), dsc);
1988	}
1989	tasklet_enable(&eni_dev->task);
1990	return 0;
1991}
1992
1993
1994static int eni_ioctl(struct atm_dev *dev,unsigned int cmd,void __user *arg)
1995{
1996	struct eni_dev *eni_dev = ENI_DEV(dev);
1997
1998	if (cmd == ENI_MEMDUMP) {
1999		if (!capable(CAP_NET_ADMIN)) return -EPERM;
2000		printk(KERN_WARNING "Please use /proc/atm/" DEV_LABEL ":%d "
2001		    "instead of obsolete ioctl ENI_MEMDUMP\n",dev->number);
2002		dump(dev);
2003		return 0;
2004	}
2005	if (cmd == ENI_SETMULT) {
2006		struct eni_multipliers mult;
2007
2008		if (!capable(CAP_NET_ADMIN)) return -EPERM;
2009		if (copy_from_user(&mult, arg,
2010		    sizeof(struct eni_multipliers)))
2011			return -EFAULT;
2012		if ((mult.tx && mult.tx <= 100) || (mult.rx &&mult.rx <= 100) ||
2013		    mult.tx > 65536 || mult.rx > 65536)
2014			return -EINVAL;
2015		if (mult.tx) eni_dev->tx_mult = mult.tx;
2016		if (mult.rx) eni_dev->rx_mult = mult.rx;
2017		return 0;
2018	}
2019	if (cmd == ATM_SETCIRANGE) {
2020		struct atm_cirange ci;
2021
2022		if (copy_from_user(&ci, arg,sizeof(struct atm_cirange)))
2023			return -EFAULT;
2024		if ((ci.vpi_bits == 0 || ci.vpi_bits == ATM_CI_MAX) &&
2025		    (ci.vci_bits == NR_VCI_LD || ci.vpi_bits == ATM_CI_MAX))
2026		    return 0;
2027		return -EINVAL;
2028	}
2029	if (!dev->phy->ioctl) return -ENOIOCTLCMD;
2030	return dev->phy->ioctl(dev,cmd,arg);
2031}
2032
2033
2034static int eni_getsockopt(struct atm_vcc *vcc,int level,int optname,
2035    void __user *optval,int optlen)
2036{
2037	return -EINVAL;
2038}
2039
2040
2041static int eni_setsockopt(struct atm_vcc *vcc,int level,int optname,
2042    void __user *optval,unsigned int optlen)
2043{
2044	return -EINVAL;
2045}
2046
2047
2048static int eni_send(struct atm_vcc *vcc,struct sk_buff *skb)
2049{
2050	enum enq_res res;
2051
2052	DPRINTK(">eni_send\n");
2053	if (!ENI_VCC(vcc)->tx) {
2054		if (vcc->pop) vcc->pop(vcc,skb);
2055		else dev_kfree_skb(skb);
2056		return -EINVAL;
2057	}
2058	if (!skb) {
2059		printk(KERN_CRIT "!skb in eni_send ?\n");
2060		if (vcc->pop) vcc->pop(vcc,skb);
2061		return -EINVAL;
2062	}
2063	if (vcc->qos.aal == ATM_AAL0) {
2064		if (skb->len != ATM_CELL_SIZE-1) {
2065			if (vcc->pop) vcc->pop(vcc,skb);
2066			else dev_kfree_skb(skb);
2067			return -EINVAL;
2068		}
2069		*(u32 *) skb->data = htonl(*(u32 *) skb->data);
2070	}
2071submitted++;
2072	ATM_SKB(skb)->vcc = vcc;
2073	tasklet_disable(&ENI_DEV(vcc->dev)->task);
2074	res = do_tx(skb);
2075	tasklet_enable(&ENI_DEV(vcc->dev)->task);
2076	if (res == enq_ok) return 0;
2077	skb_queue_tail(&ENI_VCC(vcc)->tx->backlog,skb);
2078backlogged++;
2079	tasklet_schedule(&ENI_DEV(vcc->dev)->task);
2080	return 0;
2081}
2082
2083static void eni_phy_put(struct atm_dev *dev,unsigned char value,
2084    unsigned long addr)
2085{
2086	writel(value,ENI_DEV(dev)->phy+addr*4);
2087}
2088
2089
2090
2091static unsigned char eni_phy_get(struct atm_dev *dev,unsigned long addr)
2092{
2093	return readl(ENI_DEV(dev)->phy+addr*4);
2094}
2095
2096
2097static int eni_proc_read(struct atm_dev *dev,loff_t *pos,char *page)
2098{
2099	struct sock *s;
2100	static const char *signal[] = { "LOST","unknown","okay" };
2101	struct eni_dev *eni_dev = ENI_DEV(dev);
2102	struct atm_vcc *vcc;
2103	int left,i;
2104
2105	left = *pos;
2106	if (!left)
2107		return sprintf(page,DEV_LABEL "(itf %d) signal %s, %dkB, "
2108		    "%d cps remaining\n",dev->number,signal[(int) dev->signal],
2109		    eni_dev->mem >> 10,eni_dev->tx_bw);
2110	if (!--left)
2111		return sprintf(page,"%4sBursts: TX"
2112#if !defined(CONFIG_ATM_ENI_BURST_TX_16W) && \
2113    !defined(CONFIG_ATM_ENI_BURST_TX_8W) && \
2114    !defined(CONFIG_ATM_ENI_BURST_TX_4W) && \
2115    !defined(CONFIG_ATM_ENI_BURST_TX_2W)
2116		    " none"
2117#endif
2118#ifdef CONFIG_ATM_ENI_BURST_TX_16W
2119		    " 16W"
2120#endif
2121#ifdef CONFIG_ATM_ENI_BURST_TX_8W
2122		    " 8W"
2123#endif
2124#ifdef CONFIG_ATM_ENI_BURST_TX_4W
2125		    " 4W"
2126#endif
2127#ifdef CONFIG_ATM_ENI_BURST_TX_2W
2128		    " 2W"
2129#endif
2130		    ", RX"
2131#if !defined(CONFIG_ATM_ENI_BURST_RX_16W) && \
2132    !defined(CONFIG_ATM_ENI_BURST_RX_8W) && \
2133    !defined(CONFIG_ATM_ENI_BURST_RX_4W) && \
2134    !defined(CONFIG_ATM_ENI_BURST_RX_2W)
2135		    " none"
2136#endif
2137#ifdef CONFIG_ATM_ENI_BURST_RX_16W
2138		    " 16W"
2139#endif
2140#ifdef CONFIG_ATM_ENI_BURST_RX_8W
2141		    " 8W"
2142#endif
2143#ifdef CONFIG_ATM_ENI_BURST_RX_4W
2144		    " 4W"
2145#endif
2146#ifdef CONFIG_ATM_ENI_BURST_RX_2W
2147		    " 2W"
2148#endif
2149#ifndef CONFIG_ATM_ENI_TUNE_BURST
2150		    " (default)"
2151#endif
2152		    "\n","");
2153	if (!--left) 
2154		return sprintf(page,"%4sBuffer multipliers: tx %d%%, rx %d%%\n",
2155		    "",eni_dev->tx_mult,eni_dev->rx_mult);
2156	for (i = 0; i < NR_CHAN; i++) {
2157		struct eni_tx *tx = eni_dev->tx+i;
2158
2159		if (!tx->send) continue;
2160		if (!--left) {
2161			return sprintf(page, "tx[%d]:    0x%lx-0x%lx "
2162			    "(%6ld bytes), rsv %d cps, shp %d cps%s\n",i,
2163			    (unsigned long) (tx->send - eni_dev->ram),
2164			    tx->send-eni_dev->ram+tx->words*4-1,tx->words*4,
2165			    tx->reserved,tx->shaping,
2166			    tx == eni_dev->ubr ? " (UBR)" : "");
2167		}
2168		if (--left) continue;
2169		return sprintf(page,"%10sbacklog %u packets\n","",
2170		    skb_queue_len(&tx->backlog));
2171	}
2172	read_lock(&vcc_sklist_lock);
2173	for(i = 0; i < VCC_HTABLE_SIZE; ++i) {
2174		struct hlist_head *head = &vcc_hash[i];
2175
2176		sk_for_each(s, head) {
2177			struct eni_vcc *eni_vcc;
2178			int length;
2179
2180			vcc = atm_sk(s);
2181			if (vcc->dev != dev)
2182				continue;
2183			eni_vcc = ENI_VCC(vcc);
2184			if (--left) continue;
2185			length = sprintf(page,"vcc %4d: ",vcc->vci);
2186			if (eni_vcc->rx) {
2187				length += sprintf(page+length, "0x%lx-0x%lx "
2188				    "(%6ld bytes)",
2189				    (unsigned long) (eni_vcc->recv - eni_dev->ram),
2190				    eni_vcc->recv-eni_dev->ram+eni_vcc->words*4-1,
2191				    eni_vcc->words*4);
2192				if (eni_vcc->tx) length += sprintf(page+length,", ");
2193			}
2194			if (eni_vcc->tx)
2195				length += sprintf(page+length,"tx[%d], txing %d bytes",
2196				    eni_vcc->tx->index,eni_vcc->txing);
2197			page[length] = '\n';
2198			read_unlock(&vcc_sklist_lock);
2199			return length+1;
2200		}
2201	}
2202	read_unlock(&vcc_sklist_lock);
2203	for (i = 0; i < eni_dev->free_len; i++) {
2204		struct eni_free *fe = eni_dev->free_list+i;
2205		unsigned long offset;
2206
2207		if (--left) continue;
2208		offset = (unsigned long) eni_dev->ram+eni_dev->base_diff;
2209		return sprintf(page,"free      %p-%p (%6d bytes)\n",
2210		    fe->start-offset,fe->start-offset+(1 << fe->order)-1,
2211		    1 << fe->order);
2212	}
2213	return 0;
2214}
2215
2216
2217static const struct atmdev_ops ops = {
2218	.open		= eni_open,
2219	.close		= eni_close,
2220	.ioctl		= eni_ioctl,
2221	.getsockopt	= eni_getsockopt,
2222	.setsockopt	= eni_setsockopt,
2223	.send		= eni_send,
2224	.phy_put	= eni_phy_put,
2225	.phy_get	= eni_phy_get,
2226	.change_qos	= eni_change_qos,
2227	.proc_read	= eni_proc_read
2228};
2229
2230
2231static int eni_init_one(struct pci_dev *pci_dev,
2232			const struct pci_device_id *ent)
2233{
2234	struct atm_dev *dev;
2235	struct eni_dev *eni_dev;
2236	struct eni_zero *zero;
2237	int rc;
2238
2239	rc = pci_enable_device(pci_dev);
2240	if (rc < 0)
2241		goto out;
2242
2243	rc = dma_set_mask_and_coherent(&pci_dev->dev, DMA_BIT_MASK(32));
2244	if (rc < 0)
2245		goto out;
2246
2247	rc = -ENOMEM;
2248	eni_dev = kmalloc(sizeof(struct eni_dev), GFP_KERNEL);
2249	if (!eni_dev)
2250		goto err_disable;
2251
2252	zero = &eni_dev->zero;
2253	zero->addr = dma_alloc_coherent(&pci_dev->dev,
2254					ENI_ZEROES_SIZE, &zero->dma, GFP_KERNEL);
2255	if (!zero->addr)
2256		goto err_kfree;
2257
2258	dev = atm_dev_register(DEV_LABEL, &pci_dev->dev, &ops, -1, NULL);
2259	if (!dev)
2260		goto err_free_consistent;
2261
2262	dev->dev_data = eni_dev;
2263	pci_set_drvdata(pci_dev, dev);
2264	eni_dev->pci_dev = pci_dev;
2265	eni_dev->asic = ent->driver_data;
2266
2267	rc = eni_do_init(dev);
2268	if (rc < 0)
2269		goto err_unregister;
2270
2271	rc = eni_start(dev);
2272	if (rc < 0)
2273		goto err_eni_release;
2274
2275	eni_dev->more = eni_boards;
2276	eni_boards = dev;
2277out:
2278	return rc;
2279
2280err_eni_release:
2281	eni_do_release(dev);
2282err_unregister:
2283	atm_dev_deregister(dev);
2284err_free_consistent:
2285	dma_free_coherent(&pci_dev->dev, ENI_ZEROES_SIZE, zero->addr, zero->dma);
2286err_kfree:
2287	kfree(eni_dev);
2288err_disable:
2289	pci_disable_device(pci_dev);
2290	goto out;
2291}
2292
2293
2294static struct pci_device_id eni_pci_tbl[] = {
2295	{ PCI_VDEVICE(EF, PCI_DEVICE_ID_EF_ATM_FPGA), 0 /* FPGA */ },
2296	{ PCI_VDEVICE(EF, PCI_DEVICE_ID_EF_ATM_ASIC), 1 /* ASIC */ },
2297	{ 0, }
2298};
2299MODULE_DEVICE_TABLE(pci,eni_pci_tbl);
2300
2301
2302static void eni_remove_one(struct pci_dev *pdev)
2303{
2304	struct atm_dev *dev = pci_get_drvdata(pdev);
2305	struct eni_dev *ed = ENI_DEV(dev);
2306	struct eni_zero *zero = &ed->zero;
2307
2308	eni_do_release(dev);
2309	atm_dev_deregister(dev);
2310	dma_free_coherent(&pdev->dev, ENI_ZEROES_SIZE, zero->addr, zero->dma);
2311	kfree(ed);
2312	pci_disable_device(pdev);
2313}
2314
2315
2316static struct pci_driver eni_driver = {
2317	.name		= DEV_LABEL,
2318	.id_table	= eni_pci_tbl,
2319	.probe		= eni_init_one,
2320	.remove		= eni_remove_one,
2321};
2322
2323
2324static int __init eni_init(void)
2325{
2326	struct sk_buff *skb; /* dummy for sizeof */
2327
2328	if (sizeof(skb->cb) < sizeof(struct eni_skb_prv)) {
2329		printk(KERN_ERR "eni_detect: skb->cb is too small (%Zd < %Zd)\n",
2330		    sizeof(skb->cb),sizeof(struct eni_skb_prv));
2331		return -EIO;
2332	}
2333	return pci_register_driver(&eni_driver);
2334}
2335
2336
2337module_init(eni_init);
2338/* @@@ since exit routine not defined, this module can not be unloaded */
2339
2340MODULE_LICENSE("GPL");