1#!/bin/bash
  2#
  3# Multiqueue: Using pktgen threads for sending on multiple CPUs
  4#  * adding devices to kernel threads which are in the same NUMA node
  5#  * bound devices queue's irq affinity to the threads, 1:1 mapping
  6#  * notice the naming scheme for keeping device names unique
  7#  * nameing scheme: dev@thread_number
  8#  * flow variation via random UDP source port
  9#
 10basedir=`dirname $0`
 11source ${basedir}/functions.sh
 12root_check_run_with_sudo "$@"
 13#
 14# Required param: -i dev in $DEV
 15source ${basedir}/parameters.sh
 16
 17# Trap EXIT first
 18trap_exit
 19
 20# Base Config
 21[ -z "$COUNT" ]     && COUNT="20000000"   # Zero means indefinitely
 22[ -z "$CLONE_SKB" ] && CLONE_SKB="0"
 23
 24# Flow variation random source port between min and max
 25UDP_SRC_MIN=9
 26UDP_SRC_MAX=109
 27
 28node=`get_iface_node $DEV`
 29irq_array=(`get_iface_irqs $DEV`)
 30cpu_array=(`get_node_cpus $node`)
 31
 32[ $THREADS -gt ${#irq_array[*]} -o $THREADS -gt ${#cpu_array[*]}  ] && \
 33	err 1 "Thread number $THREADS exceeds: min (${#irq_array[*]},${#cpu_array[*]})"
 34
 35# (example of setting default params in your script)
 36if [ -z "$DEST_IP" ]; then
 37    [ -z "$IP6" ] && DEST_IP="198.18.0.42" || DEST_IP="FD00::1"
 38fi
 39[ -z "$DST_MAC" ] && DST_MAC="90:e2:ba:ff:ff:ff"
 40if [ -n "$DEST_IP" ]; then
 41    validate_addr${IP6} $DEST_IP
 42    read -r DST_MIN DST_MAX <<< $(parse_addr${IP6} $DEST_IP)
 43fi
 44if [ -n "$DST_PORT" ]; then
 45    read -r UDP_DST_MIN UDP_DST_MAX <<< $(parse_ports $DST_PORT)
 46    validate_ports $UDP_DST_MIN $UDP_DST_MAX
 47fi
 48
 49# General cleanup everything since last run
 50[ -z "$APPEND" ] && pg_ctrl "reset"
 51
 52# Threads are specified with parameter -t value in $THREADS
 53for ((i = 0; i < $THREADS; i++)); do
 54    # The device name is extended with @name, using thread number to
 55    # make then unique, but any name will do.
 56    # Set the queue's irq affinity to this $thread (processor)
 57    # if '-f' is designated, offset cpu id
 58    thread=${cpu_array[$((i+F_THREAD))]}
 59    dev=${DEV}@${thread}
 60    echo $thread > /proc/irq/${irq_array[$i]}/smp_affinity_list
 61    info "irq ${irq_array[$i]} is set affinity to `cat /proc/irq/${irq_array[$i]}/smp_affinity_list`"
 62
 63    # Add remove all other devices and add_device $dev to thread
 64    [ -z "$APPEND" ] && pg_thread $thread "rem_device_all"
 65    pg_thread $thread "add_device" $dev
 66
 67    # select queue and bind the queue and $dev in 1:1 relationship
 68    queue_num=$i
 69    info "queue number is $queue_num"
 70    pg_set $dev "queue_map_min $queue_num"
 71    pg_set $dev "queue_map_max $queue_num"
 72
 73    # Notice config queue to map to cpu (mirrors smp_processor_id())
 74    # It is beneficial to map IRQ /proc/irq/*/smp_affinity 1:1 to CPU number
 75    pg_set $dev "flag QUEUE_MAP_CPU"
 76
 77    # Base config of dev
 78    pg_set $dev "count $COUNT"
 79    pg_set $dev "clone_skb $CLONE_SKB"
 80    pg_set $dev "pkt_size $PKT_SIZE"
 81    pg_set $dev "delay $DELAY"
 82
 83    # Flag example disabling timestamping
 84    pg_set $dev "flag NO_TIMESTAMP"
 85
 86    # Destination
 87    pg_set $dev "dst_mac $DST_MAC"
 88    pg_set $dev "dst${IP6}_min $DST_MIN"
 89    pg_set $dev "dst${IP6}_max $DST_MAX"
 90
 91    if [ -n "$DST_PORT" ]; then
 92	# Single destination port or random port range
 93	pg_set $dev "flag UDPDST_RND"
 94	pg_set $dev "udp_dst_min $UDP_DST_MIN"
 95	pg_set $dev "udp_dst_max $UDP_DST_MAX"
 96    fi
 97
 98    [ ! -z "$UDP_CSUM" ] && pg_set $dev "flag UDPCSUM"
 99
100    # Setup random UDP port src range
101    pg_set $dev "flag UDPSRC_RND"
102    pg_set $dev "udp_src_min $UDP_SRC_MIN"
103    pg_set $dev "udp_src_max $UDP_SRC_MAX"
104done
105
106# Run if user hits control-c
107function print_result() {
108    # Print results
109    for ((i = 0; i < $THREADS; i++)); do
110        thread=${cpu_array[$((i+F_THREAD))]}
111        dev=${DEV}@${thread}
112        echo "Device: $dev"
113        cat /proc/net/pktgen/$dev | grep -A2 "Result:"
114    done
115}
116# trap keyboard interrupt (Ctrl-C)
117trap true SIGINT
118
119# start_run
120if [ -z "$APPEND" ]; then
121    echo "Running... ctrl^C to stop" >&2
122    pg_ctrl "start"
123    echo "Done" >&2
124
125    print_result
126else
127    echo "Append mode: config done. Do more or use 'pg_ctrl start' to run"
128fi