Loading...
Note: File does not exist in v6.8.
1/*
2 * Copyright 2012 Tilera Corporation. All Rights Reserved.
3 *
4 * This program is free software; you can redistribute it and/or
5 * modify it under the terms of the GNU General Public License
6 * as published by the Free Software Foundation, version 2.
7 *
8 * This program is distributed in the hope that it will be useful, but
9 * WITHOUT ANY WARRANTY; without even the implied warranty of
10 * MERCHANTABILITY OR FITNESS FOR A PARTICULAR PURPOSE, GOOD TITLE or
11 * NON INFRINGEMENT. See the GNU General Public License for
12 * more details.
13 */
14
15#include <linux/module.h>
16#include <linux/init.h>
17#include <linux/moduleparam.h>
18#include <linux/sched.h>
19#include <linux/kernel.h> /* printk() */
20#include <linux/slab.h> /* kmalloc() */
21#include <linux/errno.h> /* error codes */
22#include <linux/types.h> /* size_t */
23#include <linux/interrupt.h>
24#include <linux/in.h>
25#include <linux/irq.h>
26#include <linux/netdevice.h> /* struct device, and other headers */
27#include <linux/etherdevice.h> /* eth_type_trans */
28#include <linux/skbuff.h>
29#include <linux/ioctl.h>
30#include <linux/cdev.h>
31#include <linux/hugetlb.h>
32#include <linux/in6.h>
33#include <linux/timer.h>
34#include <linux/hrtimer.h>
35#include <linux/ktime.h>
36#include <linux/io.h>
37#include <linux/ctype.h>
38#include <linux/ip.h>
39#include <linux/ipv6.h>
40#include <linux/tcp.h>
41#include <linux/net_tstamp.h>
42#include <linux/ptp_clock_kernel.h>
43#include <linux/tick.h>
44
45#include <asm/checksum.h>
46#include <asm/homecache.h>
47#include <gxio/mpipe.h>
48#include <arch/sim.h>
49
50/* Default transmit lockup timeout period, in jiffies. */
51#define TILE_NET_TIMEOUT (5 * HZ)
52
53/* The maximum number of distinct channels (idesc.channel is 5 bits). */
54#define TILE_NET_CHANNELS 32
55
56/* Maximum number of idescs to handle per "poll". */
57#define TILE_NET_BATCH 128
58
59/* Maximum number of packets to handle per "poll". */
60#define TILE_NET_WEIGHT 64
61
62/* Maximum Jumbo Packet MTU */
63#define TILE_JUMBO_MAX_MTU 9000
64
65/* Number of entries in each iqueue. */
66#define IQUEUE_ENTRIES 512
67
68/* Number of entries in each equeue. */
69#define EQUEUE_ENTRIES 2048
70
71/* Total header bytes per equeue slot. Must be big enough for 2 bytes
72 * of NET_IP_ALIGN alignment, plus 14 bytes (?) of L2 header, plus up to
73 * 60 bytes of actual TCP header. We round up to align to cache lines.
74 */
75#define HEADER_BYTES 128
76
77/* Maximum completions per cpu per device (must be a power of two).
78 * ISSUE: What is the right number here? If this is too small, then
79 * egress might block waiting for free space in a completions array.
80 * ISSUE: At the least, allocate these only for initialized echannels.
81 */
82#define TILE_NET_MAX_COMPS 64
83
84#define MAX_FRAGS (MAX_SKB_FRAGS + 1)
85
86/* The "kinds" of buffer stacks (small/large/jumbo). */
87#define MAX_KINDS 3
88
89/* Size of completions data to allocate.
90 * ISSUE: Probably more than needed since we don't use all the channels.
91 */
92#define COMPS_SIZE (TILE_NET_CHANNELS * sizeof(struct tile_net_comps))
93
94/* Size of NotifRing data to allocate. */
95#define NOTIF_RING_SIZE (IQUEUE_ENTRIES * sizeof(gxio_mpipe_idesc_t))
96
97/* Timeout to wake the per-device TX timer after we stop the queue.
98 * We don't want the timeout too short (adds overhead, and might end
99 * up causing stop/wake/stop/wake cycles) or too long (affects performance).
100 * For the 10 Gb NIC, 30 usec means roughly 30+ 1500-byte packets.
101 */
102#define TX_TIMER_DELAY_USEC 30
103
104/* Timeout to wake the per-cpu egress timer to free completions. */
105#define EGRESS_TIMER_DELAY_USEC 1000
106
107MODULE_AUTHOR("Tilera Corporation");
108MODULE_LICENSE("GPL");
109
110/* A "packet fragment" (a chunk of memory). */
111struct frag {
112 void *buf;
113 size_t length;
114};
115
116/* A single completion. */
117struct tile_net_comp {
118 /* The "complete_count" when the completion will be complete. */
119 s64 when;
120 /* The buffer to be freed when the completion is complete. */
121 struct sk_buff *skb;
122};
123
124/* The completions for a given cpu and echannel. */
125struct tile_net_comps {
126 /* The completions. */
127 struct tile_net_comp comp_queue[TILE_NET_MAX_COMPS];
128 /* The number of completions used. */
129 unsigned long comp_next;
130 /* The number of completions freed. */
131 unsigned long comp_last;
132};
133
134/* The transmit wake timer for a given cpu and echannel. */
135struct tile_net_tx_wake {
136 int tx_queue_idx;
137 struct hrtimer timer;
138 struct net_device *dev;
139};
140
141/* Info for a specific cpu. */
142struct tile_net_info {
143 /* Our cpu. */
144 int my_cpu;
145 /* A timer for handling egress completions. */
146 struct hrtimer egress_timer;
147 /* True if "egress_timer" is scheduled. */
148 bool egress_timer_scheduled;
149 struct info_mpipe {
150 /* Packet queue. */
151 gxio_mpipe_iqueue_t iqueue;
152 /* The NAPI struct. */
153 struct napi_struct napi;
154 /* Number of buffers (by kind) which must still be provided. */
155 unsigned int num_needed_buffers[MAX_KINDS];
156 /* instance id. */
157 int instance;
158 /* True if iqueue is valid. */
159 bool has_iqueue;
160 /* NAPI flags. */
161 bool napi_added;
162 bool napi_enabled;
163 /* Comps for each egress channel. */
164 struct tile_net_comps *comps_for_echannel[TILE_NET_CHANNELS];
165 /* Transmit wake timer for each egress channel. */
166 struct tile_net_tx_wake tx_wake[TILE_NET_CHANNELS];
167 } mpipe[NR_MPIPE_MAX];
168};
169
170/* Info for egress on a particular egress channel. */
171struct tile_net_egress {
172 /* The "equeue". */
173 gxio_mpipe_equeue_t *equeue;
174 /* The headers for TSO. */
175 unsigned char *headers;
176};
177
178/* Info for a specific device. */
179struct tile_net_priv {
180 /* Our network device. */
181 struct net_device *dev;
182 /* The primary link. */
183 gxio_mpipe_link_t link;
184 /* The primary channel, if open, else -1. */
185 int channel;
186 /* The "loopify" egress link, if needed. */
187 gxio_mpipe_link_t loopify_link;
188 /* The "loopify" egress channel, if open, else -1. */
189 int loopify_channel;
190 /* The egress channel (channel or loopify_channel). */
191 int echannel;
192 /* mPIPE instance, 0 or 1. */
193 int instance;
194 /* The timestamp config. */
195 struct hwtstamp_config stamp_cfg;
196};
197
198static struct mpipe_data {
199 /* The ingress irq. */
200 int ingress_irq;
201
202 /* The "context" for all devices. */
203 gxio_mpipe_context_t context;
204
205 /* Egress info, indexed by "priv->echannel"
206 * (lazily created as needed).
207 */
208 struct tile_net_egress
209 egress_for_echannel[TILE_NET_CHANNELS];
210
211 /* Devices currently associated with each channel.
212 * NOTE: The array entry can become NULL after ifconfig down, but
213 * we do not free the underlying net_device structures, so it is
214 * safe to use a pointer after reading it from this array.
215 */
216 struct net_device
217 *tile_net_devs_for_channel[TILE_NET_CHANNELS];
218
219 /* The actual memory allocated for the buffer stacks. */
220 void *buffer_stack_vas[MAX_KINDS];
221
222 /* The amount of memory allocated for each buffer stack. */
223 size_t buffer_stack_bytes[MAX_KINDS];
224
225 /* The first buffer stack index
226 * (small = +0, large = +1, jumbo = +2).
227 */
228 int first_buffer_stack;
229
230 /* The buckets. */
231 int first_bucket;
232 int num_buckets;
233
234 /* PTP-specific data. */
235 struct ptp_clock *ptp_clock;
236 struct ptp_clock_info caps;
237
238 /* Lock for ptp accessors. */
239 struct mutex ptp_lock;
240
241} mpipe_data[NR_MPIPE_MAX] = {
242 [0 ... (NR_MPIPE_MAX - 1)] {
243 .ingress_irq = -1,
244 .first_buffer_stack = -1,
245 .first_bucket = -1,
246 .num_buckets = 1
247 }
248};
249
250/* A mutex for "tile_net_devs_for_channel". */
251static DEFINE_MUTEX(tile_net_devs_for_channel_mutex);
252
253/* The per-cpu info. */
254static DEFINE_PER_CPU(struct tile_net_info, per_cpu_info);
255
256
257/* The buffer size enums for each buffer stack.
258 * See arch/tile/include/gxio/mpipe.h for the set of possible values.
259 * We avoid the "10384" size because it can induce "false chaining"
260 * on "cut-through" jumbo packets.
261 */
262static gxio_mpipe_buffer_size_enum_t buffer_size_enums[MAX_KINDS] = {
263 GXIO_MPIPE_BUFFER_SIZE_128,
264 GXIO_MPIPE_BUFFER_SIZE_1664,
265 GXIO_MPIPE_BUFFER_SIZE_16384
266};
267
268/* Text value of tile_net.cpus if passed as a module parameter. */
269static char *network_cpus_string;
270
271/* The actual cpus in "network_cpus". */
272static struct cpumask network_cpus_map;
273
274/* If "tile_net.loopify=LINK" was specified, this is "LINK". */
275static char *loopify_link_name;
276
277/* If "tile_net.custom" was specified, this is true. */
278static bool custom_flag;
279
280/* If "tile_net.jumbo=NUM" was specified, this is "NUM". */
281static uint jumbo_num;
282
283/* Obtain mpipe instance from struct tile_net_priv given struct net_device. */
284static inline int mpipe_instance(struct net_device *dev)
285{
286 struct tile_net_priv *priv = netdev_priv(dev);
287 return priv->instance;
288}
289
290/* The "tile_net.cpus" argument specifies the cpus that are dedicated
291 * to handle ingress packets.
292 *
293 * The parameter should be in the form "tile_net.cpus=m-n[,x-y]", where
294 * m, n, x, y are integer numbers that represent the cpus that can be
295 * neither a dedicated cpu nor a dataplane cpu.
296 */
297static bool network_cpus_init(void)
298{
299 int rc;
300
301 if (network_cpus_string == NULL)
302 return false;
303
304 rc = cpulist_parse_crop(network_cpus_string, &network_cpus_map);
305 if (rc != 0) {
306 pr_warn("tile_net.cpus=%s: malformed cpu list\n",
307 network_cpus_string);
308 return false;
309 }
310
311 /* Remove dedicated cpus. */
312 cpumask_and(&network_cpus_map, &network_cpus_map, cpu_possible_mask);
313
314 if (cpumask_empty(&network_cpus_map)) {
315 pr_warn("Ignoring empty tile_net.cpus='%s'.\n",
316 network_cpus_string);
317 return false;
318 }
319
320 pr_info("Linux network CPUs: %*pbl\n",
321 cpumask_pr_args(&network_cpus_map));
322 return true;
323}
324
325module_param_named(cpus, network_cpus_string, charp, 0444);
326MODULE_PARM_DESC(cpus, "cpulist of cores that handle network interrupts");
327
328/* The "tile_net.loopify=LINK" argument causes the named device to
329 * actually use "loop0" for ingress, and "loop1" for egress. This
330 * allows an app to sit between the actual link and linux, passing
331 * (some) packets along to linux, and forwarding (some) packets sent
332 * out by linux.
333 */
334module_param_named(loopify, loopify_link_name, charp, 0444);
335MODULE_PARM_DESC(loopify, "name the device to use loop0/1 for ingress/egress");
336
337/* The "tile_net.custom" argument causes us to ignore the "conventional"
338 * classifier metadata, in particular, the "l2_offset".
339 */
340module_param_named(custom, custom_flag, bool, 0444);
341MODULE_PARM_DESC(custom, "indicates a (heavily) customized classifier");
342
343/* The "tile_net.jumbo" argument causes us to support "jumbo" packets,
344 * and to allocate the given number of "jumbo" buffers.
345 */
346module_param_named(jumbo, jumbo_num, uint, 0444);
347MODULE_PARM_DESC(jumbo, "the number of buffers to support jumbo packets");
348
349/* Atomically update a statistics field.
350 * Note that on TILE-Gx, this operation is fire-and-forget on the
351 * issuing core (single-cycle dispatch) and takes only a few cycles
352 * longer than a regular store when the request reaches the home cache.
353 * No expensive bus management overhead is required.
354 */
355static void tile_net_stats_add(unsigned long value, unsigned long *field)
356{
357 BUILD_BUG_ON(sizeof(atomic_long_t) != sizeof(unsigned long));
358 atomic_long_add(value, (atomic_long_t *)field);
359}
360
361/* Allocate and push a buffer. */
362static bool tile_net_provide_buffer(int instance, int kind)
363{
364 struct mpipe_data *md = &mpipe_data[instance];
365 gxio_mpipe_buffer_size_enum_t bse = buffer_size_enums[kind];
366 size_t bs = gxio_mpipe_buffer_size_enum_to_buffer_size(bse);
367 const unsigned long buffer_alignment = 128;
368 struct sk_buff *skb;
369 int len;
370
371 len = sizeof(struct sk_buff **) + buffer_alignment + bs;
372 skb = dev_alloc_skb(len);
373 if (skb == NULL)
374 return false;
375
376 /* Make room for a back-pointer to 'skb' and guarantee alignment. */
377 skb_reserve(skb, sizeof(struct sk_buff **));
378 skb_reserve(skb, -(long)skb->data & (buffer_alignment - 1));
379
380 /* Save a back-pointer to 'skb'. */
381 *(struct sk_buff **)(skb->data - sizeof(struct sk_buff **)) = skb;
382
383 /* Make sure "skb" and the back-pointer have been flushed. */
384 wmb();
385
386 gxio_mpipe_push_buffer(&md->context, md->first_buffer_stack + kind,
387 (void *)va_to_tile_io_addr(skb->data));
388
389 return true;
390}
391
392/* Convert a raw mpipe buffer to its matching skb pointer. */
393static struct sk_buff *mpipe_buf_to_skb(void *va)
394{
395 /* Acquire the associated "skb". */
396 struct sk_buff **skb_ptr = va - sizeof(*skb_ptr);
397 struct sk_buff *skb = *skb_ptr;
398
399 /* Paranoia. */
400 if (skb->data != va) {
401 /* Panic here since there's a reasonable chance
402 * that corrupt buffers means generic memory
403 * corruption, with unpredictable system effects.
404 */
405 panic("Corrupt linux buffer! va=%p, skb=%p, skb->data=%p",
406 va, skb, skb->data);
407 }
408
409 return skb;
410}
411
412static void tile_net_pop_all_buffers(int instance, int stack)
413{
414 struct mpipe_data *md = &mpipe_data[instance];
415
416 for (;;) {
417 tile_io_addr_t addr =
418 (tile_io_addr_t)gxio_mpipe_pop_buffer(&md->context,
419 stack);
420 if (addr == 0)
421 break;
422 dev_kfree_skb_irq(mpipe_buf_to_skb(tile_io_addr_to_va(addr)));
423 }
424}
425
426/* Provide linux buffers to mPIPE. */
427static void tile_net_provide_needed_buffers(void)
428{
429 struct tile_net_info *info = this_cpu_ptr(&per_cpu_info);
430 int instance, kind;
431 for (instance = 0; instance < NR_MPIPE_MAX &&
432 info->mpipe[instance].has_iqueue; instance++) {
433 for (kind = 0; kind < MAX_KINDS; kind++) {
434 while (info->mpipe[instance].num_needed_buffers[kind]
435 != 0) {
436 if (!tile_net_provide_buffer(instance, kind)) {
437 pr_notice("Tile %d still needs"
438 " some buffers\n",
439 info->my_cpu);
440 return;
441 }
442 info->mpipe[instance].
443 num_needed_buffers[kind]--;
444 }
445 }
446 }
447}
448
449/* Get RX timestamp, and store it in the skb. */
450static void tile_rx_timestamp(struct tile_net_priv *priv, struct sk_buff *skb,
451 gxio_mpipe_idesc_t *idesc)
452{
453 if (unlikely(priv->stamp_cfg.rx_filter != HWTSTAMP_FILTER_NONE)) {
454 struct skb_shared_hwtstamps *shhwtstamps = skb_hwtstamps(skb);
455 memset(shhwtstamps, 0, sizeof(*shhwtstamps));
456 shhwtstamps->hwtstamp = ktime_set(idesc->time_stamp_sec,
457 idesc->time_stamp_ns);
458 }
459}
460
461/* Get TX timestamp, and store it in the skb. */
462static void tile_tx_timestamp(struct sk_buff *skb, int instance)
463{
464 struct skb_shared_info *shtx = skb_shinfo(skb);
465 if (unlikely((shtx->tx_flags & SKBTX_HW_TSTAMP) != 0)) {
466 struct mpipe_data *md = &mpipe_data[instance];
467 struct skb_shared_hwtstamps shhwtstamps;
468 struct timespec64 ts;
469
470 shtx->tx_flags |= SKBTX_IN_PROGRESS;
471 gxio_mpipe_get_timestamp(&md->context, &ts);
472 memset(&shhwtstamps, 0, sizeof(shhwtstamps));
473 shhwtstamps.hwtstamp = ktime_set(ts.tv_sec, ts.tv_nsec);
474 skb_tstamp_tx(skb, &shhwtstamps);
475 }
476}
477
478/* Use ioctl() to enable or disable TX or RX timestamping. */
479static int tile_hwtstamp_set(struct net_device *dev, struct ifreq *rq)
480{
481 struct hwtstamp_config config;
482 struct tile_net_priv *priv = netdev_priv(dev);
483
484 if (copy_from_user(&config, rq->ifr_data, sizeof(config)))
485 return -EFAULT;
486
487 if (config.flags) /* reserved for future extensions */
488 return -EINVAL;
489
490 switch (config.tx_type) {
491 case HWTSTAMP_TX_OFF:
492 case HWTSTAMP_TX_ON:
493 break;
494 default:
495 return -ERANGE;
496 }
497
498 switch (config.rx_filter) {
499 case HWTSTAMP_FILTER_NONE:
500 break;
501 case HWTSTAMP_FILTER_ALL:
502 case HWTSTAMP_FILTER_SOME:
503 case HWTSTAMP_FILTER_PTP_V1_L4_EVENT:
504 case HWTSTAMP_FILTER_PTP_V1_L4_SYNC:
505 case HWTSTAMP_FILTER_PTP_V1_L4_DELAY_REQ:
506 case HWTSTAMP_FILTER_PTP_V2_L4_EVENT:
507 case HWTSTAMP_FILTER_PTP_V2_L4_SYNC:
508 case HWTSTAMP_FILTER_PTP_V2_L4_DELAY_REQ:
509 case HWTSTAMP_FILTER_PTP_V2_L2_EVENT:
510 case HWTSTAMP_FILTER_PTP_V2_L2_SYNC:
511 case HWTSTAMP_FILTER_PTP_V2_L2_DELAY_REQ:
512 case HWTSTAMP_FILTER_PTP_V2_EVENT:
513 case HWTSTAMP_FILTER_PTP_V2_SYNC:
514 case HWTSTAMP_FILTER_PTP_V2_DELAY_REQ:
515 config.rx_filter = HWTSTAMP_FILTER_ALL;
516 break;
517 default:
518 return -ERANGE;
519 }
520
521 if (copy_to_user(rq->ifr_data, &config, sizeof(config)))
522 return -EFAULT;
523
524 priv->stamp_cfg = config;
525 return 0;
526}
527
528static int tile_hwtstamp_get(struct net_device *dev, struct ifreq *rq)
529{
530 struct tile_net_priv *priv = netdev_priv(dev);
531
532 if (copy_to_user(rq->ifr_data, &priv->stamp_cfg,
533 sizeof(priv->stamp_cfg)))
534 return -EFAULT;
535
536 return 0;
537}
538
539static inline bool filter_packet(struct net_device *dev, void *buf)
540{
541 /* Filter packets received before we're up. */
542 if (dev == NULL || !(dev->flags & IFF_UP))
543 return true;
544
545 /* Filter out packets that aren't for us. */
546 if (!(dev->flags & IFF_PROMISC) &&
547 !is_multicast_ether_addr(buf) &&
548 !ether_addr_equal(dev->dev_addr, buf))
549 return true;
550
551 return false;
552}
553
554static void tile_net_receive_skb(struct net_device *dev, struct sk_buff *skb,
555 gxio_mpipe_idesc_t *idesc, unsigned long len)
556{
557 struct tile_net_info *info = this_cpu_ptr(&per_cpu_info);
558 struct tile_net_priv *priv = netdev_priv(dev);
559 int instance = priv->instance;
560
561 /* Encode the actual packet length. */
562 skb_put(skb, len);
563
564 skb->protocol = eth_type_trans(skb, dev);
565
566 /* Acknowledge "good" hardware checksums. */
567 if (idesc->cs && idesc->csum_seed_val == 0xFFFF)
568 skb->ip_summed = CHECKSUM_UNNECESSARY;
569
570 /* Get RX timestamp from idesc. */
571 tile_rx_timestamp(priv, skb, idesc);
572
573 napi_gro_receive(&info->mpipe[instance].napi, skb);
574
575 /* Update stats. */
576 tile_net_stats_add(1, &dev->stats.rx_packets);
577 tile_net_stats_add(len, &dev->stats.rx_bytes);
578
579 /* Need a new buffer. */
580 if (idesc->size == buffer_size_enums[0])
581 info->mpipe[instance].num_needed_buffers[0]++;
582 else if (idesc->size == buffer_size_enums[1])
583 info->mpipe[instance].num_needed_buffers[1]++;
584 else
585 info->mpipe[instance].num_needed_buffers[2]++;
586}
587
588/* Handle a packet. Return true if "processed", false if "filtered". */
589static bool tile_net_handle_packet(int instance, gxio_mpipe_idesc_t *idesc)
590{
591 struct tile_net_info *info = this_cpu_ptr(&per_cpu_info);
592 struct mpipe_data *md = &mpipe_data[instance];
593 struct net_device *dev = md->tile_net_devs_for_channel[idesc->channel];
594 uint8_t l2_offset;
595 void *va;
596 void *buf;
597 unsigned long len;
598 bool filter;
599
600 /* Drop packets for which no buffer was available (which can
601 * happen under heavy load), or for which the me/tr/ce flags
602 * are set (which can happen for jumbo cut-through packets,
603 * or with a customized classifier).
604 */
605 if (idesc->be || idesc->me || idesc->tr || idesc->ce) {
606 if (dev)
607 tile_net_stats_add(1, &dev->stats.rx_errors);
608 goto drop;
609 }
610
611 /* Get the "l2_offset", if allowed. */
612 l2_offset = custom_flag ? 0 : gxio_mpipe_idesc_get_l2_offset(idesc);
613
614 /* Get the VA (including NET_IP_ALIGN bytes of "headroom"). */
615 va = tile_io_addr_to_va((unsigned long)idesc->va);
616
617 /* Get the actual packet start/length. */
618 buf = va + l2_offset;
619 len = idesc->l2_size - l2_offset;
620
621 /* Point "va" at the raw buffer. */
622 va -= NET_IP_ALIGN;
623
624 filter = filter_packet(dev, buf);
625 if (filter) {
626 if (dev)
627 tile_net_stats_add(1, &dev->stats.rx_dropped);
628drop:
629 gxio_mpipe_iqueue_drop(&info->mpipe[instance].iqueue, idesc);
630 } else {
631 struct sk_buff *skb = mpipe_buf_to_skb(va);
632
633 /* Skip headroom, and any custom header. */
634 skb_reserve(skb, NET_IP_ALIGN + l2_offset);
635
636 tile_net_receive_skb(dev, skb, idesc, len);
637 }
638
639 gxio_mpipe_iqueue_consume(&info->mpipe[instance].iqueue, idesc);
640 return !filter;
641}
642
643/* Handle some packets for the current CPU.
644 *
645 * This function handles up to TILE_NET_BATCH idescs per call.
646 *
647 * ISSUE: Since we do not provide new buffers until this function is
648 * complete, we must initially provide enough buffers for each network
649 * cpu to fill its iqueue and also its batched idescs.
650 *
651 * ISSUE: The "rotting packet" race condition occurs if a packet
652 * arrives after the queue appears to be empty, and before the
653 * hypervisor interrupt is re-enabled.
654 */
655static int tile_net_poll(struct napi_struct *napi, int budget)
656{
657 struct tile_net_info *info = this_cpu_ptr(&per_cpu_info);
658 unsigned int work = 0;
659 gxio_mpipe_idesc_t *idesc;
660 int instance, i, n;
661 struct mpipe_data *md;
662 struct info_mpipe *info_mpipe =
663 container_of(napi, struct info_mpipe, napi);
664
665 if (budget <= 0)
666 goto done;
667
668 instance = info_mpipe->instance;
669 while ((n = gxio_mpipe_iqueue_try_peek(
670 &info_mpipe->iqueue,
671 &idesc)) > 0) {
672 for (i = 0; i < n; i++) {
673 if (i == TILE_NET_BATCH)
674 goto done;
675 if (tile_net_handle_packet(instance,
676 idesc + i)) {
677 if (++work >= budget)
678 goto done;
679 }
680 }
681 }
682
683 /* There are no packets left. */
684 napi_complete(&info_mpipe->napi);
685
686 md = &mpipe_data[instance];
687 /* Re-enable hypervisor interrupts. */
688 gxio_mpipe_enable_notif_ring_interrupt(
689 &md->context, info->mpipe[instance].iqueue.ring);
690
691 /* HACK: Avoid the "rotting packet" problem. */
692 if (gxio_mpipe_iqueue_try_peek(&info_mpipe->iqueue, &idesc) > 0)
693 napi_schedule(&info_mpipe->napi);
694
695 /* ISSUE: Handle completions? */
696
697done:
698 tile_net_provide_needed_buffers();
699
700 return work;
701}
702
703/* Handle an ingress interrupt from an instance on the current cpu. */
704static irqreturn_t tile_net_handle_ingress_irq(int irq, void *id)
705{
706 struct tile_net_info *info = this_cpu_ptr(&per_cpu_info);
707 napi_schedule(&info->mpipe[(uint64_t)id].napi);
708 return IRQ_HANDLED;
709}
710
711/* Free some completions. This must be called with interrupts blocked. */
712static int tile_net_free_comps(gxio_mpipe_equeue_t *equeue,
713 struct tile_net_comps *comps,
714 int limit, bool force_update)
715{
716 int n = 0;
717 while (comps->comp_last < comps->comp_next) {
718 unsigned int cid = comps->comp_last % TILE_NET_MAX_COMPS;
719 struct tile_net_comp *comp = &comps->comp_queue[cid];
720 if (!gxio_mpipe_equeue_is_complete(equeue, comp->when,
721 force_update || n == 0))
722 break;
723 dev_kfree_skb_irq(comp->skb);
724 comps->comp_last++;
725 if (++n == limit)
726 break;
727 }
728 return n;
729}
730
731/* Add a completion. This must be called with interrupts blocked.
732 * tile_net_equeue_try_reserve() will have ensured a free completion entry.
733 */
734static void add_comp(gxio_mpipe_equeue_t *equeue,
735 struct tile_net_comps *comps,
736 uint64_t when, struct sk_buff *skb)
737{
738 int cid = comps->comp_next % TILE_NET_MAX_COMPS;
739 comps->comp_queue[cid].when = when;
740 comps->comp_queue[cid].skb = skb;
741 comps->comp_next++;
742}
743
744static void tile_net_schedule_tx_wake_timer(struct net_device *dev,
745 int tx_queue_idx)
746{
747 struct tile_net_info *info = &per_cpu(per_cpu_info, tx_queue_idx);
748 struct tile_net_priv *priv = netdev_priv(dev);
749 int instance = priv->instance;
750 struct tile_net_tx_wake *tx_wake =
751 &info->mpipe[instance].tx_wake[priv->echannel];
752
753 hrtimer_start(&tx_wake->timer,
754 TX_TIMER_DELAY_USEC * 1000UL,
755 HRTIMER_MODE_REL_PINNED);
756}
757
758static enum hrtimer_restart tile_net_handle_tx_wake_timer(struct hrtimer *t)
759{
760 struct tile_net_tx_wake *tx_wake =
761 container_of(t, struct tile_net_tx_wake, timer);
762 netif_wake_subqueue(tx_wake->dev, tx_wake->tx_queue_idx);
763 return HRTIMER_NORESTART;
764}
765
766/* Make sure the egress timer is scheduled. */
767static void tile_net_schedule_egress_timer(void)
768{
769 struct tile_net_info *info = this_cpu_ptr(&per_cpu_info);
770
771 if (!info->egress_timer_scheduled) {
772 hrtimer_start(&info->egress_timer,
773 EGRESS_TIMER_DELAY_USEC * 1000UL,
774 HRTIMER_MODE_REL_PINNED);
775 info->egress_timer_scheduled = true;
776 }
777}
778
779/* The "function" for "info->egress_timer".
780 *
781 * This timer will reschedule itself as long as there are any pending
782 * completions expected for this tile.
783 */
784static enum hrtimer_restart tile_net_handle_egress_timer(struct hrtimer *t)
785{
786 struct tile_net_info *info = this_cpu_ptr(&per_cpu_info);
787 unsigned long irqflags;
788 bool pending = false;
789 int i, instance;
790
791 local_irq_save(irqflags);
792
793 /* The timer is no longer scheduled. */
794 info->egress_timer_scheduled = false;
795
796 /* Free all possible comps for this tile. */
797 for (instance = 0; instance < NR_MPIPE_MAX &&
798 info->mpipe[instance].has_iqueue; instance++) {
799 for (i = 0; i < TILE_NET_CHANNELS; i++) {
800 struct tile_net_egress *egress =
801 &mpipe_data[instance].egress_for_echannel[i];
802 struct tile_net_comps *comps =
803 info->mpipe[instance].comps_for_echannel[i];
804 if (!egress || comps->comp_last >= comps->comp_next)
805 continue;
806 tile_net_free_comps(egress->equeue, comps, -1, true);
807 pending = pending ||
808 (comps->comp_last < comps->comp_next);
809 }
810 }
811
812 /* Reschedule timer if needed. */
813 if (pending)
814 tile_net_schedule_egress_timer();
815
816 local_irq_restore(irqflags);
817
818 return HRTIMER_NORESTART;
819}
820
821/* PTP clock operations. */
822
823static int ptp_mpipe_adjfreq(struct ptp_clock_info *ptp, s32 ppb)
824{
825 int ret = 0;
826 struct mpipe_data *md = container_of(ptp, struct mpipe_data, caps);
827 mutex_lock(&md->ptp_lock);
828 if (gxio_mpipe_adjust_timestamp_freq(&md->context, ppb))
829 ret = -EINVAL;
830 mutex_unlock(&md->ptp_lock);
831 return ret;
832}
833
834static int ptp_mpipe_adjtime(struct ptp_clock_info *ptp, s64 delta)
835{
836 int ret = 0;
837 struct mpipe_data *md = container_of(ptp, struct mpipe_data, caps);
838 mutex_lock(&md->ptp_lock);
839 if (gxio_mpipe_adjust_timestamp(&md->context, delta))
840 ret = -EBUSY;
841 mutex_unlock(&md->ptp_lock);
842 return ret;
843}
844
845static int ptp_mpipe_gettime(struct ptp_clock_info *ptp,
846 struct timespec64 *ts)
847{
848 int ret = 0;
849 struct mpipe_data *md = container_of(ptp, struct mpipe_data, caps);
850 mutex_lock(&md->ptp_lock);
851 if (gxio_mpipe_get_timestamp(&md->context, ts))
852 ret = -EBUSY;
853 mutex_unlock(&md->ptp_lock);
854 return ret;
855}
856
857static int ptp_mpipe_settime(struct ptp_clock_info *ptp,
858 const struct timespec64 *ts)
859{
860 int ret = 0;
861 struct mpipe_data *md = container_of(ptp, struct mpipe_data, caps);
862 mutex_lock(&md->ptp_lock);
863 if (gxio_mpipe_set_timestamp(&md->context, ts))
864 ret = -EBUSY;
865 mutex_unlock(&md->ptp_lock);
866 return ret;
867}
868
869static int ptp_mpipe_enable(struct ptp_clock_info *ptp,
870 struct ptp_clock_request *request, int on)
871{
872 return -EOPNOTSUPP;
873}
874
875static struct ptp_clock_info ptp_mpipe_caps = {
876 .owner = THIS_MODULE,
877 .name = "mPIPE clock",
878 .max_adj = 999999999,
879 .n_ext_ts = 0,
880 .n_pins = 0,
881 .pps = 0,
882 .adjfreq = ptp_mpipe_adjfreq,
883 .adjtime = ptp_mpipe_adjtime,
884 .gettime64 = ptp_mpipe_gettime,
885 .settime64 = ptp_mpipe_settime,
886 .enable = ptp_mpipe_enable,
887};
888
889/* Sync mPIPE's timestamp up with Linux system time and register PTP clock. */
890static void register_ptp_clock(struct net_device *dev, struct mpipe_data *md)
891{
892 struct timespec64 ts;
893
894 ktime_get_ts64(&ts);
895 gxio_mpipe_set_timestamp(&md->context, &ts);
896
897 mutex_init(&md->ptp_lock);
898 md->caps = ptp_mpipe_caps;
899 md->ptp_clock = ptp_clock_register(&md->caps, NULL);
900 if (IS_ERR(md->ptp_clock))
901 netdev_err(dev, "ptp_clock_register failed %ld\n",
902 PTR_ERR(md->ptp_clock));
903}
904
905/* Initialize PTP fields in a new device. */
906static void init_ptp_dev(struct tile_net_priv *priv)
907{
908 priv->stamp_cfg.rx_filter = HWTSTAMP_FILTER_NONE;
909 priv->stamp_cfg.tx_type = HWTSTAMP_TX_OFF;
910}
911
912/* Helper functions for "tile_net_update()". */
913static void enable_ingress_irq(void *irq)
914{
915 enable_percpu_irq((long)irq, 0);
916}
917
918static void disable_ingress_irq(void *irq)
919{
920 disable_percpu_irq((long)irq);
921}
922
923/* Helper function for tile_net_open() and tile_net_stop().
924 * Always called under tile_net_devs_for_channel_mutex.
925 */
926static int tile_net_update(struct net_device *dev)
927{
928 static gxio_mpipe_rules_t rules; /* too big to fit on the stack */
929 bool saw_channel = false;
930 int instance = mpipe_instance(dev);
931 struct mpipe_data *md = &mpipe_data[instance];
932 int channel;
933 int rc;
934 int cpu;
935
936 saw_channel = false;
937 gxio_mpipe_rules_init(&rules, &md->context);
938
939 for (channel = 0; channel < TILE_NET_CHANNELS; channel++) {
940 if (md->tile_net_devs_for_channel[channel] == NULL)
941 continue;
942 if (!saw_channel) {
943 saw_channel = true;
944 gxio_mpipe_rules_begin(&rules, md->first_bucket,
945 md->num_buckets, NULL);
946 gxio_mpipe_rules_set_headroom(&rules, NET_IP_ALIGN);
947 }
948 gxio_mpipe_rules_add_channel(&rules, channel);
949 }
950
951 /* NOTE: This can fail if there is no classifier.
952 * ISSUE: Can anything else cause it to fail?
953 */
954 rc = gxio_mpipe_rules_commit(&rules);
955 if (rc != 0) {
956 netdev_warn(dev, "gxio_mpipe_rules_commit: mpipe[%d] %d\n",
957 instance, rc);
958 return -EIO;
959 }
960
961 /* Update all cpus, sequentially (to protect "netif_napi_add()").
962 * We use on_each_cpu to handle the IPI mask or unmask.
963 */
964 if (!saw_channel)
965 on_each_cpu(disable_ingress_irq,
966 (void *)(long)(md->ingress_irq), 1);
967 for_each_online_cpu(cpu) {
968 struct tile_net_info *info = &per_cpu(per_cpu_info, cpu);
969
970 if (!info->mpipe[instance].has_iqueue)
971 continue;
972 if (saw_channel) {
973 if (!info->mpipe[instance].napi_added) {
974 netif_napi_add(dev, &info->mpipe[instance].napi,
975 tile_net_poll, TILE_NET_WEIGHT);
976 info->mpipe[instance].napi_added = true;
977 }
978 if (!info->mpipe[instance].napi_enabled) {
979 napi_enable(&info->mpipe[instance].napi);
980 info->mpipe[instance].napi_enabled = true;
981 }
982 } else {
983 if (info->mpipe[instance].napi_enabled) {
984 napi_disable(&info->mpipe[instance].napi);
985 info->mpipe[instance].napi_enabled = false;
986 }
987 /* FIXME: Drain the iqueue. */
988 }
989 }
990 if (saw_channel)
991 on_each_cpu(enable_ingress_irq,
992 (void *)(long)(md->ingress_irq), 1);
993
994 /* HACK: Allow packets to flow in the simulator. */
995 if (saw_channel)
996 sim_enable_mpipe_links(instance, -1);
997
998 return 0;
999}
1000
1001/* Initialize a buffer stack. */
1002static int create_buffer_stack(struct net_device *dev,
1003 int kind, size_t num_buffers)
1004{
1005 pte_t hash_pte = pte_set_home((pte_t) { 0 }, PAGE_HOME_HASH);
1006 int instance = mpipe_instance(dev);
1007 struct mpipe_data *md = &mpipe_data[instance];
1008 size_t needed = gxio_mpipe_calc_buffer_stack_bytes(num_buffers);
1009 int stack_idx = md->first_buffer_stack + kind;
1010 void *va;
1011 int i, rc;
1012
1013 /* Round up to 64KB and then use alloc_pages() so we get the
1014 * required 64KB alignment.
1015 */
1016 md->buffer_stack_bytes[kind] =
1017 ALIGN(needed, 64 * 1024);
1018
1019 va = alloc_pages_exact(md->buffer_stack_bytes[kind], GFP_KERNEL);
1020 if (va == NULL) {
1021 netdev_err(dev,
1022 "Could not alloc %zd bytes for buffer stack %d\n",
1023 md->buffer_stack_bytes[kind], kind);
1024 return -ENOMEM;
1025 }
1026
1027 /* Initialize the buffer stack. */
1028 rc = gxio_mpipe_init_buffer_stack(&md->context, stack_idx,
1029 buffer_size_enums[kind], va,
1030 md->buffer_stack_bytes[kind], 0);
1031 if (rc != 0) {
1032 netdev_err(dev, "gxio_mpipe_init_buffer_stack: mpipe[%d] %d\n",
1033 instance, rc);
1034 free_pages_exact(va, md->buffer_stack_bytes[kind]);
1035 return rc;
1036 }
1037
1038 md->buffer_stack_vas[kind] = va;
1039
1040 rc = gxio_mpipe_register_client_memory(&md->context, stack_idx,
1041 hash_pte, 0);
1042 if (rc != 0) {
1043 netdev_err(dev,
1044 "gxio_mpipe_register_client_memory: mpipe[%d] %d\n",
1045 instance, rc);
1046 return rc;
1047 }
1048
1049 /* Provide initial buffers. */
1050 for (i = 0; i < num_buffers; i++) {
1051 if (!tile_net_provide_buffer(instance, kind)) {
1052 netdev_err(dev, "Cannot allocate initial sk_bufs!\n");
1053 return -ENOMEM;
1054 }
1055 }
1056
1057 return 0;
1058}
1059
1060/* Allocate and initialize mpipe buffer stacks, and register them in
1061 * the mPIPE TLBs, for small, large, and (possibly) jumbo packet sizes.
1062 * This routine supports tile_net_init_mpipe(), below.
1063 */
1064static int init_buffer_stacks(struct net_device *dev,
1065 int network_cpus_count)
1066{
1067 int num_kinds = MAX_KINDS - (jumbo_num == 0);
1068 size_t num_buffers;
1069 int rc;
1070 int instance = mpipe_instance(dev);
1071 struct mpipe_data *md = &mpipe_data[instance];
1072
1073 /* Allocate the buffer stacks. */
1074 rc = gxio_mpipe_alloc_buffer_stacks(&md->context, num_kinds, 0, 0);
1075 if (rc < 0) {
1076 netdev_err(dev,
1077 "gxio_mpipe_alloc_buffer_stacks: mpipe[%d] %d\n",
1078 instance, rc);
1079 return rc;
1080 }
1081 md->first_buffer_stack = rc;
1082
1083 /* Enough small/large buffers to (normally) avoid buffer errors. */
1084 num_buffers =
1085 network_cpus_count * (IQUEUE_ENTRIES + TILE_NET_BATCH);
1086
1087 /* Allocate the small memory stack. */
1088 if (rc >= 0)
1089 rc = create_buffer_stack(dev, 0, num_buffers);
1090
1091 /* Allocate the large buffer stack. */
1092 if (rc >= 0)
1093 rc = create_buffer_stack(dev, 1, num_buffers);
1094
1095 /* Allocate the jumbo buffer stack if needed. */
1096 if (rc >= 0 && jumbo_num != 0)
1097 rc = create_buffer_stack(dev, 2, jumbo_num);
1098
1099 return rc;
1100}
1101
1102/* Allocate per-cpu resources (memory for completions and idescs).
1103 * This routine supports tile_net_init_mpipe(), below.
1104 */
1105static int alloc_percpu_mpipe_resources(struct net_device *dev,
1106 int cpu, int ring)
1107{
1108 struct tile_net_info *info = &per_cpu(per_cpu_info, cpu);
1109 int order, i, rc;
1110 int instance = mpipe_instance(dev);
1111 struct mpipe_data *md = &mpipe_data[instance];
1112 struct page *page;
1113 void *addr;
1114
1115 /* Allocate the "comps". */
1116 order = get_order(COMPS_SIZE);
1117 page = homecache_alloc_pages(GFP_KERNEL, order, cpu);
1118 if (page == NULL) {
1119 netdev_err(dev, "Failed to alloc %zd bytes comps memory\n",
1120 COMPS_SIZE);
1121 return -ENOMEM;
1122 }
1123 addr = pfn_to_kaddr(page_to_pfn(page));
1124 memset(addr, 0, COMPS_SIZE);
1125 for (i = 0; i < TILE_NET_CHANNELS; i++)
1126 info->mpipe[instance].comps_for_echannel[i] =
1127 addr + i * sizeof(struct tile_net_comps);
1128
1129 /* If this is a network cpu, create an iqueue. */
1130 if (cpumask_test_cpu(cpu, &network_cpus_map)) {
1131 order = get_order(NOTIF_RING_SIZE);
1132 page = homecache_alloc_pages(GFP_KERNEL, order, cpu);
1133 if (page == NULL) {
1134 netdev_err(dev,
1135 "Failed to alloc %zd bytes iqueue memory\n",
1136 NOTIF_RING_SIZE);
1137 return -ENOMEM;
1138 }
1139 addr = pfn_to_kaddr(page_to_pfn(page));
1140 rc = gxio_mpipe_iqueue_init(&info->mpipe[instance].iqueue,
1141 &md->context, ring++, addr,
1142 NOTIF_RING_SIZE, 0);
1143 if (rc < 0) {
1144 netdev_err(dev,
1145 "gxio_mpipe_iqueue_init failed: %d\n", rc);
1146 return rc;
1147 }
1148 info->mpipe[instance].has_iqueue = true;
1149 }
1150
1151 return ring;
1152}
1153
1154/* Initialize NotifGroup and buckets.
1155 * This routine supports tile_net_init_mpipe(), below.
1156 */
1157static int init_notif_group_and_buckets(struct net_device *dev,
1158 int ring, int network_cpus_count)
1159{
1160 int group, rc;
1161 int instance = mpipe_instance(dev);
1162 struct mpipe_data *md = &mpipe_data[instance];
1163
1164 /* Allocate one NotifGroup. */
1165 rc = gxio_mpipe_alloc_notif_groups(&md->context, 1, 0, 0);
1166 if (rc < 0) {
1167 netdev_err(dev, "gxio_mpipe_alloc_notif_groups: mpipe[%d] %d\n",
1168 instance, rc);
1169 return rc;
1170 }
1171 group = rc;
1172
1173 /* Initialize global num_buckets value. */
1174 if (network_cpus_count > 4)
1175 md->num_buckets = 256;
1176 else if (network_cpus_count > 1)
1177 md->num_buckets = 16;
1178
1179 /* Allocate some buckets, and set global first_bucket value. */
1180 rc = gxio_mpipe_alloc_buckets(&md->context, md->num_buckets, 0, 0);
1181 if (rc < 0) {
1182 netdev_err(dev, "gxio_mpipe_alloc_buckets: mpipe[%d] %d\n",
1183 instance, rc);
1184 return rc;
1185 }
1186 md->first_bucket = rc;
1187
1188 /* Init group and buckets. */
1189 rc = gxio_mpipe_init_notif_group_and_buckets(
1190 &md->context, group, ring, network_cpus_count,
1191 md->first_bucket, md->num_buckets,
1192 GXIO_MPIPE_BUCKET_STICKY_FLOW_LOCALITY);
1193 if (rc != 0) {
1194 netdev_err(dev, "gxio_mpipe_init_notif_group_and_buckets: "
1195 "mpipe[%d] %d\n", instance, rc);
1196 return rc;
1197 }
1198
1199 return 0;
1200}
1201
1202/* Create an irq and register it, then activate the irq and request
1203 * interrupts on all cores. Note that "ingress_irq" being initialized
1204 * is how we know not to call tile_net_init_mpipe() again.
1205 * This routine supports tile_net_init_mpipe(), below.
1206 */
1207static int tile_net_setup_interrupts(struct net_device *dev)
1208{
1209 int cpu, rc, irq;
1210 int instance = mpipe_instance(dev);
1211 struct mpipe_data *md = &mpipe_data[instance];
1212
1213 irq = md->ingress_irq;
1214 if (irq < 0) {
1215 irq = irq_alloc_hwirq(-1);
1216 if (!irq) {
1217 netdev_err(dev,
1218 "create_irq failed: mpipe[%d] %d\n",
1219 instance, irq);
1220 return irq;
1221 }
1222 tile_irq_activate(irq, TILE_IRQ_PERCPU);
1223
1224 rc = request_irq(irq, tile_net_handle_ingress_irq,
1225 0, "tile_net", (void *)((uint64_t)instance));
1226
1227 if (rc != 0) {
1228 netdev_err(dev, "request_irq failed: mpipe[%d] %d\n",
1229 instance, rc);
1230 irq_free_hwirq(irq);
1231 return rc;
1232 }
1233 md->ingress_irq = irq;
1234 }
1235
1236 for_each_online_cpu(cpu) {
1237 struct tile_net_info *info = &per_cpu(per_cpu_info, cpu);
1238 if (info->mpipe[instance].has_iqueue) {
1239 gxio_mpipe_request_notif_ring_interrupt(&md->context,
1240 cpu_x(cpu), cpu_y(cpu), KERNEL_PL, irq,
1241 info->mpipe[instance].iqueue.ring);
1242 }
1243 }
1244
1245 return 0;
1246}
1247
1248/* Undo any state set up partially by a failed call to tile_net_init_mpipe. */
1249static void tile_net_init_mpipe_fail(int instance)
1250{
1251 int kind, cpu;
1252 struct mpipe_data *md = &mpipe_data[instance];
1253
1254 /* Do cleanups that require the mpipe context first. */
1255 for (kind = 0; kind < MAX_KINDS; kind++) {
1256 if (md->buffer_stack_vas[kind] != NULL) {
1257 tile_net_pop_all_buffers(instance,
1258 md->first_buffer_stack +
1259 kind);
1260 }
1261 }
1262
1263 /* Destroy mpipe context so the hardware no longer owns any memory. */
1264 gxio_mpipe_destroy(&md->context);
1265
1266 for_each_online_cpu(cpu) {
1267 struct tile_net_info *info = &per_cpu(per_cpu_info, cpu);
1268 free_pages(
1269 (unsigned long)(
1270 info->mpipe[instance].comps_for_echannel[0]),
1271 get_order(COMPS_SIZE));
1272 info->mpipe[instance].comps_for_echannel[0] = NULL;
1273 free_pages((unsigned long)(info->mpipe[instance].iqueue.idescs),
1274 get_order(NOTIF_RING_SIZE));
1275 info->mpipe[instance].iqueue.idescs = NULL;
1276 }
1277
1278 for (kind = 0; kind < MAX_KINDS; kind++) {
1279 if (md->buffer_stack_vas[kind] != NULL) {
1280 free_pages_exact(md->buffer_stack_vas[kind],
1281 md->buffer_stack_bytes[kind]);
1282 md->buffer_stack_vas[kind] = NULL;
1283 }
1284 }
1285
1286 md->first_buffer_stack = -1;
1287 md->first_bucket = -1;
1288}
1289
1290/* The first time any tilegx network device is opened, we initialize
1291 * the global mpipe state. If this step fails, we fail to open the
1292 * device, but if it succeeds, we never need to do it again, and since
1293 * tile_net can't be unloaded, we never undo it.
1294 *
1295 * Note that some resources in this path (buffer stack indices,
1296 * bindings from init_buffer_stack, etc.) are hypervisor resources
1297 * that are freed implicitly by gxio_mpipe_destroy().
1298 */
1299static int tile_net_init_mpipe(struct net_device *dev)
1300{
1301 int rc;
1302 int cpu;
1303 int first_ring, ring;
1304 int instance = mpipe_instance(dev);
1305 struct mpipe_data *md = &mpipe_data[instance];
1306 int network_cpus_count = cpumask_weight(&network_cpus_map);
1307
1308 if (!hash_default) {
1309 netdev_err(dev, "Networking requires hash_default!\n");
1310 return -EIO;
1311 }
1312
1313 rc = gxio_mpipe_init(&md->context, instance);
1314 if (rc != 0) {
1315 netdev_err(dev, "gxio_mpipe_init: mpipe[%d] %d\n",
1316 instance, rc);
1317 return -EIO;
1318 }
1319
1320 /* Set up the buffer stacks. */
1321 rc = init_buffer_stacks(dev, network_cpus_count);
1322 if (rc != 0)
1323 goto fail;
1324
1325 /* Allocate one NotifRing for each network cpu. */
1326 rc = gxio_mpipe_alloc_notif_rings(&md->context,
1327 network_cpus_count, 0, 0);
1328 if (rc < 0) {
1329 netdev_err(dev, "gxio_mpipe_alloc_notif_rings failed %d\n",
1330 rc);
1331 goto fail;
1332 }
1333
1334 /* Init NotifRings per-cpu. */
1335 first_ring = rc;
1336 ring = first_ring;
1337 for_each_online_cpu(cpu) {
1338 rc = alloc_percpu_mpipe_resources(dev, cpu, ring);
1339 if (rc < 0)
1340 goto fail;
1341 ring = rc;
1342 }
1343
1344 /* Initialize NotifGroup and buckets. */
1345 rc = init_notif_group_and_buckets(dev, first_ring, network_cpus_count);
1346 if (rc != 0)
1347 goto fail;
1348
1349 /* Create and enable interrupts. */
1350 rc = tile_net_setup_interrupts(dev);
1351 if (rc != 0)
1352 goto fail;
1353
1354 /* Register PTP clock and set mPIPE timestamp, if configured. */
1355 register_ptp_clock(dev, md);
1356
1357 return 0;
1358
1359fail:
1360 tile_net_init_mpipe_fail(instance);
1361 return rc;
1362}
1363
1364/* Create persistent egress info for a given egress channel.
1365 * Note that this may be shared between, say, "gbe0" and "xgbe0".
1366 * ISSUE: Defer header allocation until TSO is actually needed?
1367 */
1368static int tile_net_init_egress(struct net_device *dev, int echannel)
1369{
1370 static int ering = -1;
1371 struct page *headers_page, *edescs_page, *equeue_page;
1372 gxio_mpipe_edesc_t *edescs;
1373 gxio_mpipe_equeue_t *equeue;
1374 unsigned char *headers;
1375 int headers_order, edescs_order, equeue_order;
1376 size_t edescs_size;
1377 int rc = -ENOMEM;
1378 int instance = mpipe_instance(dev);
1379 struct mpipe_data *md = &mpipe_data[instance];
1380
1381 /* Only initialize once. */
1382 if (md->egress_for_echannel[echannel].equeue != NULL)
1383 return 0;
1384
1385 /* Allocate memory for the "headers". */
1386 headers_order = get_order(EQUEUE_ENTRIES * HEADER_BYTES);
1387 headers_page = alloc_pages(GFP_KERNEL, headers_order);
1388 if (headers_page == NULL) {
1389 netdev_warn(dev,
1390 "Could not alloc %zd bytes for TSO headers.\n",
1391 PAGE_SIZE << headers_order);
1392 goto fail;
1393 }
1394 headers = pfn_to_kaddr(page_to_pfn(headers_page));
1395
1396 /* Allocate memory for the "edescs". */
1397 edescs_size = EQUEUE_ENTRIES * sizeof(*edescs);
1398 edescs_order = get_order(edescs_size);
1399 edescs_page = alloc_pages(GFP_KERNEL, edescs_order);
1400 if (edescs_page == NULL) {
1401 netdev_warn(dev,
1402 "Could not alloc %zd bytes for eDMA ring.\n",
1403 edescs_size);
1404 goto fail_headers;
1405 }
1406 edescs = pfn_to_kaddr(page_to_pfn(edescs_page));
1407
1408 /* Allocate memory for the "equeue". */
1409 equeue_order = get_order(sizeof(*equeue));
1410 equeue_page = alloc_pages(GFP_KERNEL, equeue_order);
1411 if (equeue_page == NULL) {
1412 netdev_warn(dev,
1413 "Could not alloc %zd bytes for equeue info.\n",
1414 PAGE_SIZE << equeue_order);
1415 goto fail_edescs;
1416 }
1417 equeue = pfn_to_kaddr(page_to_pfn(equeue_page));
1418
1419 /* Allocate an edma ring (using a one entry "free list"). */
1420 if (ering < 0) {
1421 rc = gxio_mpipe_alloc_edma_rings(&md->context, 1, 0, 0);
1422 if (rc < 0) {
1423 netdev_warn(dev, "gxio_mpipe_alloc_edma_rings: "
1424 "mpipe[%d] %d\n", instance, rc);
1425 goto fail_equeue;
1426 }
1427 ering = rc;
1428 }
1429
1430 /* Initialize the equeue. */
1431 rc = gxio_mpipe_equeue_init(equeue, &md->context, ering, echannel,
1432 edescs, edescs_size, 0);
1433 if (rc != 0) {
1434 netdev_err(dev, "gxio_mpipe_equeue_init: mpipe[%d] %d\n",
1435 instance, rc);
1436 goto fail_equeue;
1437 }
1438
1439 /* Don't reuse the ering later. */
1440 ering = -1;
1441
1442 if (jumbo_num != 0) {
1443 /* Make sure "jumbo" packets can be egressed safely. */
1444 if (gxio_mpipe_equeue_set_snf_size(equeue, 10368) < 0) {
1445 /* ISSUE: There is no "gxio_mpipe_equeue_destroy()". */
1446 netdev_warn(dev, "Jumbo packets may not be egressed"
1447 " properly on channel %d\n", echannel);
1448 }
1449 }
1450
1451 /* Done. */
1452 md->egress_for_echannel[echannel].equeue = equeue;
1453 md->egress_for_echannel[echannel].headers = headers;
1454 return 0;
1455
1456fail_equeue:
1457 __free_pages(equeue_page, equeue_order);
1458
1459fail_edescs:
1460 __free_pages(edescs_page, edescs_order);
1461
1462fail_headers:
1463 __free_pages(headers_page, headers_order);
1464
1465fail:
1466 return rc;
1467}
1468
1469/* Return channel number for a newly-opened link. */
1470static int tile_net_link_open(struct net_device *dev, gxio_mpipe_link_t *link,
1471 const char *link_name)
1472{
1473 int instance = mpipe_instance(dev);
1474 struct mpipe_data *md = &mpipe_data[instance];
1475 int rc = gxio_mpipe_link_open(link, &md->context, link_name, 0);
1476 if (rc < 0) {
1477 netdev_err(dev, "Failed to open '%s', mpipe[%d], %d\n",
1478 link_name, instance, rc);
1479 return rc;
1480 }
1481 if (jumbo_num != 0) {
1482 u32 attr = GXIO_MPIPE_LINK_RECEIVE_JUMBO;
1483 rc = gxio_mpipe_link_set_attr(link, attr, 1);
1484 if (rc != 0) {
1485 netdev_err(dev,
1486 "Cannot receive jumbo packets on '%s'\n",
1487 link_name);
1488 gxio_mpipe_link_close(link);
1489 return rc;
1490 }
1491 }
1492 rc = gxio_mpipe_link_channel(link);
1493 if (rc < 0 || rc >= TILE_NET_CHANNELS) {
1494 netdev_err(dev, "gxio_mpipe_link_channel bad value: %d\n", rc);
1495 gxio_mpipe_link_close(link);
1496 return -EINVAL;
1497 }
1498 return rc;
1499}
1500
1501/* Help the kernel activate the given network interface. */
1502static int tile_net_open(struct net_device *dev)
1503{
1504 struct tile_net_priv *priv = netdev_priv(dev);
1505 int cpu, rc, instance;
1506
1507 mutex_lock(&tile_net_devs_for_channel_mutex);
1508
1509 /* Get the instance info. */
1510 rc = gxio_mpipe_link_instance(dev->name);
1511 if (rc < 0 || rc >= NR_MPIPE_MAX) {
1512 mutex_unlock(&tile_net_devs_for_channel_mutex);
1513 return -EIO;
1514 }
1515
1516 priv->instance = rc;
1517 instance = rc;
1518 if (!mpipe_data[rc].context.mmio_fast_base) {
1519 /* Do one-time initialization per instance the first time
1520 * any device is opened.
1521 */
1522 rc = tile_net_init_mpipe(dev);
1523 if (rc != 0)
1524 goto fail;
1525 }
1526
1527 /* Determine if this is the "loopify" device. */
1528 if (unlikely((loopify_link_name != NULL) &&
1529 !strcmp(dev->name, loopify_link_name))) {
1530 rc = tile_net_link_open(dev, &priv->link, "loop0");
1531 if (rc < 0)
1532 goto fail;
1533 priv->channel = rc;
1534 rc = tile_net_link_open(dev, &priv->loopify_link, "loop1");
1535 if (rc < 0)
1536 goto fail;
1537 priv->loopify_channel = rc;
1538 priv->echannel = rc;
1539 } else {
1540 rc = tile_net_link_open(dev, &priv->link, dev->name);
1541 if (rc < 0)
1542 goto fail;
1543 priv->channel = rc;
1544 priv->echannel = rc;
1545 }
1546
1547 /* Initialize egress info (if needed). Once ever, per echannel. */
1548 rc = tile_net_init_egress(dev, priv->echannel);
1549 if (rc != 0)
1550 goto fail;
1551
1552 mpipe_data[instance].tile_net_devs_for_channel[priv->channel] = dev;
1553
1554 rc = tile_net_update(dev);
1555 if (rc != 0)
1556 goto fail;
1557
1558 mutex_unlock(&tile_net_devs_for_channel_mutex);
1559
1560 /* Initialize the transmit wake timer for this device for each cpu. */
1561 for_each_online_cpu(cpu) {
1562 struct tile_net_info *info = &per_cpu(per_cpu_info, cpu);
1563 struct tile_net_tx_wake *tx_wake =
1564 &info->mpipe[instance].tx_wake[priv->echannel];
1565
1566 hrtimer_init(&tx_wake->timer, CLOCK_MONOTONIC,
1567 HRTIMER_MODE_REL);
1568 tx_wake->tx_queue_idx = cpu;
1569 tx_wake->timer.function = tile_net_handle_tx_wake_timer;
1570 tx_wake->dev = dev;
1571 }
1572
1573 for_each_online_cpu(cpu)
1574 netif_start_subqueue(dev, cpu);
1575 netif_carrier_on(dev);
1576 return 0;
1577
1578fail:
1579 if (priv->loopify_channel >= 0) {
1580 if (gxio_mpipe_link_close(&priv->loopify_link) != 0)
1581 netdev_warn(dev, "Failed to close loopify link!\n");
1582 priv->loopify_channel = -1;
1583 }
1584 if (priv->channel >= 0) {
1585 if (gxio_mpipe_link_close(&priv->link) != 0)
1586 netdev_warn(dev, "Failed to close link!\n");
1587 priv->channel = -1;
1588 }
1589 priv->echannel = -1;
1590 mpipe_data[instance].tile_net_devs_for_channel[priv->channel] = NULL;
1591 mutex_unlock(&tile_net_devs_for_channel_mutex);
1592
1593 /* Don't return raw gxio error codes to generic Linux. */
1594 return (rc > -512) ? rc : -EIO;
1595}
1596
1597/* Help the kernel deactivate the given network interface. */
1598static int tile_net_stop(struct net_device *dev)
1599{
1600 struct tile_net_priv *priv = netdev_priv(dev);
1601 int cpu;
1602 int instance = priv->instance;
1603 struct mpipe_data *md = &mpipe_data[instance];
1604
1605 for_each_online_cpu(cpu) {
1606 struct tile_net_info *info = &per_cpu(per_cpu_info, cpu);
1607 struct tile_net_tx_wake *tx_wake =
1608 &info->mpipe[instance].tx_wake[priv->echannel];
1609
1610 hrtimer_cancel(&tx_wake->timer);
1611 netif_stop_subqueue(dev, cpu);
1612 }
1613
1614 mutex_lock(&tile_net_devs_for_channel_mutex);
1615 md->tile_net_devs_for_channel[priv->channel] = NULL;
1616 (void)tile_net_update(dev);
1617 if (priv->loopify_channel >= 0) {
1618 if (gxio_mpipe_link_close(&priv->loopify_link) != 0)
1619 netdev_warn(dev, "Failed to close loopify link!\n");
1620 priv->loopify_channel = -1;
1621 }
1622 if (priv->channel >= 0) {
1623 if (gxio_mpipe_link_close(&priv->link) != 0)
1624 netdev_warn(dev, "Failed to close link!\n");
1625 priv->channel = -1;
1626 }
1627 priv->echannel = -1;
1628 mutex_unlock(&tile_net_devs_for_channel_mutex);
1629
1630 return 0;
1631}
1632
1633/* Determine the VA for a fragment. */
1634static inline void *tile_net_frag_buf(skb_frag_t *f)
1635{
1636 unsigned long pfn = page_to_pfn(skb_frag_page(f));
1637 return pfn_to_kaddr(pfn) + f->page_offset;
1638}
1639
1640/* Acquire a completion entry and an egress slot, or if we can't,
1641 * stop the queue and schedule the tx_wake timer.
1642 */
1643static s64 tile_net_equeue_try_reserve(struct net_device *dev,
1644 int tx_queue_idx,
1645 struct tile_net_comps *comps,
1646 gxio_mpipe_equeue_t *equeue,
1647 int num_edescs)
1648{
1649 /* Try to acquire a completion entry. */
1650 if (comps->comp_next - comps->comp_last < TILE_NET_MAX_COMPS - 1 ||
1651 tile_net_free_comps(equeue, comps, 32, false) != 0) {
1652
1653 /* Try to acquire an egress slot. */
1654 s64 slot = gxio_mpipe_equeue_try_reserve(equeue, num_edescs);
1655 if (slot >= 0)
1656 return slot;
1657
1658 /* Freeing some completions gives the equeue time to drain. */
1659 tile_net_free_comps(equeue, comps, TILE_NET_MAX_COMPS, false);
1660
1661 slot = gxio_mpipe_equeue_try_reserve(equeue, num_edescs);
1662 if (slot >= 0)
1663 return slot;
1664 }
1665
1666 /* Still nothing; give up and stop the queue for a short while. */
1667 netif_stop_subqueue(dev, tx_queue_idx);
1668 tile_net_schedule_tx_wake_timer(dev, tx_queue_idx);
1669 return -1;
1670}
1671
1672/* Determine how many edesc's are needed for TSO.
1673 *
1674 * Sometimes, if "sendfile()" requires copying, we will be called with
1675 * "data" containing the header and payload, with "frags" being empty.
1676 * Sometimes, for example when using NFS over TCP, a single segment can
1677 * span 3 fragments. This requires special care.
1678 */
1679static int tso_count_edescs(struct sk_buff *skb)
1680{
1681 struct skb_shared_info *sh = skb_shinfo(skb);
1682 unsigned int sh_len = skb_transport_offset(skb) + tcp_hdrlen(skb);
1683 unsigned int data_len = skb->len - sh_len;
1684 unsigned int p_len = sh->gso_size;
1685 long f_id = -1; /* id of the current fragment */
1686 long f_size = skb_headlen(skb) - sh_len; /* current fragment size */
1687 long f_used = 0; /* bytes used from the current fragment */
1688 long n; /* size of the current piece of payload */
1689 int num_edescs = 0;
1690 int segment;
1691
1692 for (segment = 0; segment < sh->gso_segs; segment++) {
1693
1694 unsigned int p_used = 0;
1695
1696 /* One edesc for header and for each piece of the payload. */
1697 for (num_edescs++; p_used < p_len; num_edescs++) {
1698
1699 /* Advance as needed. */
1700 while (f_used >= f_size) {
1701 f_id++;
1702 f_size = skb_frag_size(&sh->frags[f_id]);
1703 f_used = 0;
1704 }
1705
1706 /* Use bytes from the current fragment. */
1707 n = p_len - p_used;
1708 if (n > f_size - f_used)
1709 n = f_size - f_used;
1710 f_used += n;
1711 p_used += n;
1712 }
1713
1714 /* The last segment may be less than gso_size. */
1715 data_len -= p_len;
1716 if (data_len < p_len)
1717 p_len = data_len;
1718 }
1719
1720 return num_edescs;
1721}
1722
1723/* Prepare modified copies of the skbuff headers. */
1724static void tso_headers_prepare(struct sk_buff *skb, unsigned char *headers,
1725 s64 slot)
1726{
1727 struct skb_shared_info *sh = skb_shinfo(skb);
1728 struct iphdr *ih;
1729 struct ipv6hdr *ih6;
1730 struct tcphdr *th;
1731 unsigned int sh_len = skb_transport_offset(skb) + tcp_hdrlen(skb);
1732 unsigned int data_len = skb->len - sh_len;
1733 unsigned char *data = skb->data;
1734 unsigned int ih_off, th_off, p_len;
1735 unsigned int isum_seed, tsum_seed, seq;
1736 unsigned int uninitialized_var(id);
1737 int is_ipv6;
1738 long f_id = -1; /* id of the current fragment */
1739 long f_size = skb_headlen(skb) - sh_len; /* current fragment size */
1740 long f_used = 0; /* bytes used from the current fragment */
1741 long n; /* size of the current piece of payload */
1742 int segment;
1743
1744 /* Locate original headers and compute various lengths. */
1745 is_ipv6 = skb_is_gso_v6(skb);
1746 if (is_ipv6) {
1747 ih6 = ipv6_hdr(skb);
1748 ih_off = skb_network_offset(skb);
1749 } else {
1750 ih = ip_hdr(skb);
1751 ih_off = skb_network_offset(skb);
1752 isum_seed = ((0xFFFF - ih->check) +
1753 (0xFFFF - ih->tot_len) +
1754 (0xFFFF - ih->id));
1755 id = ntohs(ih->id);
1756 }
1757
1758 th = tcp_hdr(skb);
1759 th_off = skb_transport_offset(skb);
1760 p_len = sh->gso_size;
1761
1762 tsum_seed = th->check + (0xFFFF ^ htons(skb->len));
1763 seq = ntohl(th->seq);
1764
1765 /* Prepare all the headers. */
1766 for (segment = 0; segment < sh->gso_segs; segment++) {
1767 unsigned char *buf;
1768 unsigned int p_used = 0;
1769
1770 /* Copy to the header memory for this segment. */
1771 buf = headers + (slot % EQUEUE_ENTRIES) * HEADER_BYTES +
1772 NET_IP_ALIGN;
1773 memcpy(buf, data, sh_len);
1774
1775 /* Update copied ip header. */
1776 if (is_ipv6) {
1777 ih6 = (struct ipv6hdr *)(buf + ih_off);
1778 ih6->payload_len = htons(sh_len + p_len - ih_off -
1779 sizeof(*ih6));
1780 } else {
1781 ih = (struct iphdr *)(buf + ih_off);
1782 ih->tot_len = htons(sh_len + p_len - ih_off);
1783 ih->id = htons(id++);
1784 ih->check = csum_long(isum_seed + ih->tot_len +
1785 ih->id) ^ 0xffff;
1786 }
1787
1788 /* Update copied tcp header. */
1789 th = (struct tcphdr *)(buf + th_off);
1790 th->seq = htonl(seq);
1791 th->check = csum_long(tsum_seed + htons(sh_len + p_len));
1792 if (segment != sh->gso_segs - 1) {
1793 th->fin = 0;
1794 th->psh = 0;
1795 }
1796
1797 /* Skip past the header. */
1798 slot++;
1799
1800 /* Skip past the payload. */
1801 while (p_used < p_len) {
1802
1803 /* Advance as needed. */
1804 while (f_used >= f_size) {
1805 f_id++;
1806 f_size = skb_frag_size(&sh->frags[f_id]);
1807 f_used = 0;
1808 }
1809
1810 /* Use bytes from the current fragment. */
1811 n = p_len - p_used;
1812 if (n > f_size - f_used)
1813 n = f_size - f_used;
1814 f_used += n;
1815 p_used += n;
1816
1817 slot++;
1818 }
1819
1820 seq += p_len;
1821
1822 /* The last segment may be less than gso_size. */
1823 data_len -= p_len;
1824 if (data_len < p_len)
1825 p_len = data_len;
1826 }
1827
1828 /* Flush the headers so they are ready for hardware DMA. */
1829 wmb();
1830}
1831
1832/* Pass all the data to mpipe for egress. */
1833static void tso_egress(struct net_device *dev, gxio_mpipe_equeue_t *equeue,
1834 struct sk_buff *skb, unsigned char *headers, s64 slot)
1835{
1836 struct skb_shared_info *sh = skb_shinfo(skb);
1837 int instance = mpipe_instance(dev);
1838 struct mpipe_data *md = &mpipe_data[instance];
1839 unsigned int sh_len = skb_transport_offset(skb) + tcp_hdrlen(skb);
1840 unsigned int data_len = skb->len - sh_len;
1841 unsigned int p_len = sh->gso_size;
1842 gxio_mpipe_edesc_t edesc_head = { { 0 } };
1843 gxio_mpipe_edesc_t edesc_body = { { 0 } };
1844 long f_id = -1; /* id of the current fragment */
1845 long f_size = skb_headlen(skb) - sh_len; /* current fragment size */
1846 long f_used = 0; /* bytes used from the current fragment */
1847 void *f_data = skb->data + sh_len;
1848 long n; /* size of the current piece of payload */
1849 unsigned long tx_packets = 0, tx_bytes = 0;
1850 unsigned int csum_start;
1851 int segment;
1852
1853 /* Prepare to egress the headers: set up header edesc. */
1854 csum_start = skb_checksum_start_offset(skb);
1855 edesc_head.csum = 1;
1856 edesc_head.csum_start = csum_start;
1857 edesc_head.csum_dest = csum_start + skb->csum_offset;
1858 edesc_head.xfer_size = sh_len;
1859
1860 /* This is only used to specify the TLB. */
1861 edesc_head.stack_idx = md->first_buffer_stack;
1862 edesc_body.stack_idx = md->first_buffer_stack;
1863
1864 /* Egress all the edescs. */
1865 for (segment = 0; segment < sh->gso_segs; segment++) {
1866 unsigned char *buf;
1867 unsigned int p_used = 0;
1868
1869 /* Egress the header. */
1870 buf = headers + (slot % EQUEUE_ENTRIES) * HEADER_BYTES +
1871 NET_IP_ALIGN;
1872 edesc_head.va = va_to_tile_io_addr(buf);
1873 gxio_mpipe_equeue_put_at(equeue, edesc_head, slot);
1874 slot++;
1875
1876 /* Egress the payload. */
1877 while (p_used < p_len) {
1878 void *va;
1879
1880 /* Advance as needed. */
1881 while (f_used >= f_size) {
1882 f_id++;
1883 f_size = skb_frag_size(&sh->frags[f_id]);
1884 f_data = tile_net_frag_buf(&sh->frags[f_id]);
1885 f_used = 0;
1886 }
1887
1888 va = f_data + f_used;
1889
1890 /* Use bytes from the current fragment. */
1891 n = p_len - p_used;
1892 if (n > f_size - f_used)
1893 n = f_size - f_used;
1894 f_used += n;
1895 p_used += n;
1896
1897 /* Egress a piece of the payload. */
1898 edesc_body.va = va_to_tile_io_addr(va);
1899 edesc_body.xfer_size = n;
1900 edesc_body.bound = !(p_used < p_len);
1901 gxio_mpipe_equeue_put_at(equeue, edesc_body, slot);
1902 slot++;
1903 }
1904
1905 tx_packets++;
1906 tx_bytes += sh_len + p_len;
1907
1908 /* The last segment may be less than gso_size. */
1909 data_len -= p_len;
1910 if (data_len < p_len)
1911 p_len = data_len;
1912 }
1913
1914 /* Update stats. */
1915 tile_net_stats_add(tx_packets, &dev->stats.tx_packets);
1916 tile_net_stats_add(tx_bytes, &dev->stats.tx_bytes);
1917}
1918
1919/* Do "TSO" handling for egress.
1920 *
1921 * Normally drivers set NETIF_F_TSO only to support hardware TSO;
1922 * otherwise the stack uses scatter-gather to implement GSO in software.
1923 * On our testing, enabling GSO support (via NETIF_F_SG) drops network
1924 * performance down to around 7.5 Gbps on the 10G interfaces, although
1925 * also dropping cpu utilization way down, to under 8%. But
1926 * implementing "TSO" in the driver brings performance back up to line
1927 * rate, while dropping cpu usage even further, to less than 4%. In
1928 * practice, profiling of GSO shows that skb_segment() is what causes
1929 * the performance overheads; we benefit in the driver from using
1930 * preallocated memory to duplicate the TCP/IP headers.
1931 */
1932static int tile_net_tx_tso(struct sk_buff *skb, struct net_device *dev)
1933{
1934 struct tile_net_info *info = this_cpu_ptr(&per_cpu_info);
1935 struct tile_net_priv *priv = netdev_priv(dev);
1936 int channel = priv->echannel;
1937 int instance = priv->instance;
1938 struct mpipe_data *md = &mpipe_data[instance];
1939 struct tile_net_egress *egress = &md->egress_for_echannel[channel];
1940 struct tile_net_comps *comps =
1941 info->mpipe[instance].comps_for_echannel[channel];
1942 gxio_mpipe_equeue_t *equeue = egress->equeue;
1943 unsigned long irqflags;
1944 int num_edescs;
1945 s64 slot;
1946
1947 /* Determine how many mpipe edesc's are needed. */
1948 num_edescs = tso_count_edescs(skb);
1949
1950 local_irq_save(irqflags);
1951
1952 /* Try to acquire a completion entry and an egress slot. */
1953 slot = tile_net_equeue_try_reserve(dev, skb->queue_mapping, comps,
1954 equeue, num_edescs);
1955 if (slot < 0) {
1956 local_irq_restore(irqflags);
1957 return NETDEV_TX_BUSY;
1958 }
1959
1960 /* Set up copies of header data properly. */
1961 tso_headers_prepare(skb, egress->headers, slot);
1962
1963 /* Actually pass the data to the network hardware. */
1964 tso_egress(dev, equeue, skb, egress->headers, slot);
1965
1966 /* Add a completion record. */
1967 add_comp(equeue, comps, slot + num_edescs - 1, skb);
1968
1969 local_irq_restore(irqflags);
1970
1971 /* Make sure the egress timer is scheduled. */
1972 tile_net_schedule_egress_timer();
1973
1974 return NETDEV_TX_OK;
1975}
1976
1977/* Analyze the body and frags for a transmit request. */
1978static unsigned int tile_net_tx_frags(struct frag *frags,
1979 struct sk_buff *skb,
1980 void *b_data, unsigned int b_len)
1981{
1982 unsigned int i, n = 0;
1983
1984 struct skb_shared_info *sh = skb_shinfo(skb);
1985
1986 if (b_len != 0) {
1987 frags[n].buf = b_data;
1988 frags[n++].length = b_len;
1989 }
1990
1991 for (i = 0; i < sh->nr_frags; i++) {
1992 skb_frag_t *f = &sh->frags[i];
1993 frags[n].buf = tile_net_frag_buf(f);
1994 frags[n++].length = skb_frag_size(f);
1995 }
1996
1997 return n;
1998}
1999
2000/* Help the kernel transmit a packet. */
2001static int tile_net_tx(struct sk_buff *skb, struct net_device *dev)
2002{
2003 struct tile_net_info *info = this_cpu_ptr(&per_cpu_info);
2004 struct tile_net_priv *priv = netdev_priv(dev);
2005 int instance = priv->instance;
2006 struct mpipe_data *md = &mpipe_data[instance];
2007 struct tile_net_egress *egress =
2008 &md->egress_for_echannel[priv->echannel];
2009 gxio_mpipe_equeue_t *equeue = egress->equeue;
2010 struct tile_net_comps *comps =
2011 info->mpipe[instance].comps_for_echannel[priv->echannel];
2012 unsigned int len = skb->len;
2013 unsigned char *data = skb->data;
2014 unsigned int num_edescs;
2015 struct frag frags[MAX_FRAGS];
2016 gxio_mpipe_edesc_t edescs[MAX_FRAGS];
2017 unsigned long irqflags;
2018 gxio_mpipe_edesc_t edesc = { { 0 } };
2019 unsigned int i;
2020 s64 slot;
2021
2022 if (skb_is_gso(skb))
2023 return tile_net_tx_tso(skb, dev);
2024
2025 num_edescs = tile_net_tx_frags(frags, skb, data, skb_headlen(skb));
2026
2027 /* This is only used to specify the TLB. */
2028 edesc.stack_idx = md->first_buffer_stack;
2029
2030 /* Prepare the edescs. */
2031 for (i = 0; i < num_edescs; i++) {
2032 edesc.xfer_size = frags[i].length;
2033 edesc.va = va_to_tile_io_addr(frags[i].buf);
2034 edescs[i] = edesc;
2035 }
2036
2037 /* Mark the final edesc. */
2038 edescs[num_edescs - 1].bound = 1;
2039
2040 /* Add checksum info to the initial edesc, if needed. */
2041 if (skb->ip_summed == CHECKSUM_PARTIAL) {
2042 unsigned int csum_start = skb_checksum_start_offset(skb);
2043 edescs[0].csum = 1;
2044 edescs[0].csum_start = csum_start;
2045 edescs[0].csum_dest = csum_start + skb->csum_offset;
2046 }
2047
2048 local_irq_save(irqflags);
2049
2050 /* Try to acquire a completion entry and an egress slot. */
2051 slot = tile_net_equeue_try_reserve(dev, skb->queue_mapping, comps,
2052 equeue, num_edescs);
2053 if (slot < 0) {
2054 local_irq_restore(irqflags);
2055 return NETDEV_TX_BUSY;
2056 }
2057
2058 for (i = 0; i < num_edescs; i++)
2059 gxio_mpipe_equeue_put_at(equeue, edescs[i], slot++);
2060
2061 /* Store TX timestamp if needed. */
2062 tile_tx_timestamp(skb, instance);
2063
2064 /* Add a completion record. */
2065 add_comp(equeue, comps, slot - 1, skb);
2066
2067 /* NOTE: Use ETH_ZLEN for short packets (e.g. 42 < 60). */
2068 tile_net_stats_add(1, &dev->stats.tx_packets);
2069 tile_net_stats_add(max_t(unsigned int, len, ETH_ZLEN),
2070 &dev->stats.tx_bytes);
2071
2072 local_irq_restore(irqflags);
2073
2074 /* Make sure the egress timer is scheduled. */
2075 tile_net_schedule_egress_timer();
2076
2077 return NETDEV_TX_OK;
2078}
2079
2080/* Return subqueue id on this core (one per core). */
2081static u16 tile_net_select_queue(struct net_device *dev, struct sk_buff *skb,
2082 void *accel_priv, select_queue_fallback_t fallback)
2083{
2084 return smp_processor_id();
2085}
2086
2087/* Deal with a transmit timeout. */
2088static void tile_net_tx_timeout(struct net_device *dev)
2089{
2090 int cpu;
2091
2092 for_each_online_cpu(cpu)
2093 netif_wake_subqueue(dev, cpu);
2094}
2095
2096/* Ioctl commands. */
2097static int tile_net_ioctl(struct net_device *dev, struct ifreq *rq, int cmd)
2098{
2099 if (cmd == SIOCSHWTSTAMP)
2100 return tile_hwtstamp_set(dev, rq);
2101 if (cmd == SIOCGHWTSTAMP)
2102 return tile_hwtstamp_get(dev, rq);
2103
2104 return -EOPNOTSUPP;
2105}
2106
2107/* Change the Ethernet address of the NIC.
2108 *
2109 * The hypervisor driver does not support changing MAC address. However,
2110 * the hardware does not do anything with the MAC address, so the address
2111 * which gets used on outgoing packets, and which is accepted on incoming
2112 * packets, is completely up to us.
2113 *
2114 * Returns 0 on success, negative on failure.
2115 */
2116static int tile_net_set_mac_address(struct net_device *dev, void *p)
2117{
2118 struct sockaddr *addr = p;
2119
2120 if (!is_valid_ether_addr(addr->sa_data))
2121 return -EINVAL;
2122 memcpy(dev->dev_addr, addr->sa_data, dev->addr_len);
2123 return 0;
2124}
2125
2126#ifdef CONFIG_NET_POLL_CONTROLLER
2127/* Polling 'interrupt' - used by things like netconsole to send skbs
2128 * without having to re-enable interrupts. It's not called while
2129 * the interrupt routine is executing.
2130 */
2131static void tile_net_netpoll(struct net_device *dev)
2132{
2133 int instance = mpipe_instance(dev);
2134 struct tile_net_info *info = this_cpu_ptr(&per_cpu_info);
2135 struct mpipe_data *md = &mpipe_data[instance];
2136
2137 disable_percpu_irq(md->ingress_irq);
2138 napi_schedule(&info->mpipe[instance].napi);
2139 enable_percpu_irq(md->ingress_irq, 0);
2140}
2141#endif
2142
2143static const struct net_device_ops tile_net_ops = {
2144 .ndo_open = tile_net_open,
2145 .ndo_stop = tile_net_stop,
2146 .ndo_start_xmit = tile_net_tx,
2147 .ndo_select_queue = tile_net_select_queue,
2148 .ndo_do_ioctl = tile_net_ioctl,
2149 .ndo_tx_timeout = tile_net_tx_timeout,
2150 .ndo_set_mac_address = tile_net_set_mac_address,
2151#ifdef CONFIG_NET_POLL_CONTROLLER
2152 .ndo_poll_controller = tile_net_netpoll,
2153#endif
2154};
2155
2156/* The setup function.
2157 *
2158 * This uses ether_setup() to assign various fields in dev, including
2159 * setting IFF_BROADCAST and IFF_MULTICAST, then sets some extra fields.
2160 */
2161static void tile_net_setup(struct net_device *dev)
2162{
2163 netdev_features_t features = 0;
2164
2165 ether_setup(dev);
2166 dev->netdev_ops = &tile_net_ops;
2167 dev->watchdog_timeo = TILE_NET_TIMEOUT;
2168
2169 /* MTU range: 68 - 1500 or 9000 */
2170 dev->mtu = ETH_DATA_LEN;
2171 dev->min_mtu = ETH_MIN_MTU;
2172 dev->max_mtu = jumbo_num ? TILE_JUMBO_MAX_MTU : ETH_DATA_LEN;
2173
2174 features |= NETIF_F_HW_CSUM;
2175 features |= NETIF_F_SG;
2176 features |= NETIF_F_TSO;
2177 features |= NETIF_F_TSO6;
2178
2179 dev->hw_features |= features;
2180 dev->vlan_features |= features;
2181 dev->features |= features;
2182}
2183
2184/* Allocate the device structure, register the device, and obtain the
2185 * MAC address from the hypervisor.
2186 */
2187static void tile_net_dev_init(const char *name, const uint8_t *mac)
2188{
2189 int ret;
2190 struct net_device *dev;
2191 struct tile_net_priv *priv;
2192
2193 /* HACK: Ignore "loop" links. */
2194 if (strncmp(name, "loop", 4) == 0)
2195 return;
2196
2197 /* Allocate the device structure. Normally, "name" is a
2198 * template, instantiated by register_netdev(), but not for us.
2199 */
2200 dev = alloc_netdev_mqs(sizeof(*priv), name, NET_NAME_UNKNOWN,
2201 tile_net_setup, NR_CPUS, 1);
2202 if (!dev) {
2203 pr_err("alloc_netdev_mqs(%s) failed\n", name);
2204 return;
2205 }
2206
2207 /* Initialize "priv". */
2208 priv = netdev_priv(dev);
2209 priv->dev = dev;
2210 priv->channel = -1;
2211 priv->loopify_channel = -1;
2212 priv->echannel = -1;
2213 init_ptp_dev(priv);
2214
2215 /* Get the MAC address and set it in the device struct; this must
2216 * be done before the device is opened. If the MAC is all zeroes,
2217 * we use a random address, since we're probably on the simulator.
2218 */
2219 if (!is_zero_ether_addr(mac))
2220 ether_addr_copy(dev->dev_addr, mac);
2221 else
2222 eth_hw_addr_random(dev);
2223
2224 /* Register the network device. */
2225 ret = register_netdev(dev);
2226 if (ret) {
2227 netdev_err(dev, "register_netdev failed %d\n", ret);
2228 free_netdev(dev);
2229 return;
2230 }
2231}
2232
2233/* Per-cpu module initialization. */
2234static void tile_net_init_module_percpu(void *unused)
2235{
2236 struct tile_net_info *info = this_cpu_ptr(&per_cpu_info);
2237 int my_cpu = smp_processor_id();
2238 int instance;
2239
2240 for (instance = 0; instance < NR_MPIPE_MAX; instance++) {
2241 info->mpipe[instance].has_iqueue = false;
2242 info->mpipe[instance].instance = instance;
2243 }
2244 info->my_cpu = my_cpu;
2245
2246 /* Initialize the egress timer. */
2247 hrtimer_init(&info->egress_timer, CLOCK_MONOTONIC, HRTIMER_MODE_REL);
2248 info->egress_timer.function = tile_net_handle_egress_timer;
2249}
2250
2251/* Module initialization. */
2252static int __init tile_net_init_module(void)
2253{
2254 int i;
2255 char name[GXIO_MPIPE_LINK_NAME_LEN];
2256 uint8_t mac[6];
2257
2258 pr_info("Tilera Network Driver\n");
2259
2260 BUILD_BUG_ON(NR_MPIPE_MAX != 2);
2261
2262 mutex_init(&tile_net_devs_for_channel_mutex);
2263
2264 /* Initialize each CPU. */
2265 on_each_cpu(tile_net_init_module_percpu, NULL, 1);
2266
2267 /* Find out what devices we have, and initialize them. */
2268 for (i = 0; gxio_mpipe_link_enumerate_mac(i, name, mac) >= 0; i++)
2269 tile_net_dev_init(name, mac);
2270
2271 if (!network_cpus_init())
2272 cpumask_and(&network_cpus_map, housekeeping_cpumask(),
2273 cpu_online_mask);
2274
2275 return 0;
2276}
2277
2278module_init(tile_net_init_module);