1 /* 2 * Authors: 3 * Copyright 2001, 2002 by Robert Olsson <robert.olsson@its.uu.se> 4 * Uppsala University and 5 * Swedish University of Agricultural Sciences 6 * 7 * Alexey Kuznetsov <kuznet@ms2.inr.ac.ru> 8 * Ben Greear <greearb@candelatech.com> 9 * Jens Låås <jens.laas@data.slu.se> 10 * 11 * This program is free software; you can redistribute it and/or 12 * modify it under the terms of the GNU General Public License 13 * as published by the Free Software Foundation; either version 14 * 2 of the License, or (at your option) any later version. 15 * 16 * 17 * A tool for loading the network with preconfigurated packets. 18 * The tool is implemented as a linux module. Parameters are output 19 * device, delay (to hard_xmit), number of packets, and whether 20 * to use multiple SKBs or just the same one. 21 * pktgen uses the installed interface's output routine. 22 * 23 * Additional hacking by: 24 * 25 * Jens.Laas@data.slu.se 26 * Improved by ANK. 010120. 27 * Improved by ANK even more. 010212. 28 * MAC address typo fixed. 010417 --ro 29 * Integrated. 020301 --DaveM 30 * Added multiskb option 020301 --DaveM 31 * Scaling of results. 020417--sigurdur@linpro.no 32 * Significant re-work of the module: 33 * * Convert to threaded model to more efficiently be able to transmit 34 * and receive on multiple interfaces at once. 35 * * Converted many counters to __u64 to allow longer runs. 36 * * Allow configuration of ranges, like min/max IP address, MACs, 37 * and UDP-ports, for both source and destination, and can 38 * set to use a random distribution or sequentially walk the range. 39 * * Can now change most values after starting. 40 * * Place 12-byte packet in UDP payload with magic number, 41 * sequence number, and timestamp. 42 * * Add receiver code that detects dropped pkts, re-ordered pkts, and 43 * latencies (with micro-second) precision. 44 * * Add IOCTL interface to easily get counters & configuration. 45 * --Ben Greear <greearb@candelatech.com> 46 * 47 * Renamed multiskb to clone_skb and cleaned up sending core for two distinct 48 * skb modes. A clone_skb=0 mode for Ben "ranges" work and a clone_skb != 0 49 * as a "fastpath" with a configurable number of clones after alloc's. 50 * clone_skb=0 means all packets are allocated this also means ranges time 51 * stamps etc can be used. clone_skb=100 means 1 malloc is followed by 100 52 * clones. 53 * 54 * Also moved to /proc/net/pktgen/ 55 * --ro 56 * 57 * Sept 10: Fixed threading/locking. Lots of bone-headed and more clever 58 * mistakes. Also merged in DaveM's patch in the -pre6 patch. 59 * --Ben Greear <greearb@candelatech.com> 60 * 61 * Integrated to 2.5.x 021029 --Lucio Maciel (luciomaciel@zipmail.com.br) 62 * 63 * 64 * 021124 Finished major redesign and rewrite for new functionality. 65 * See Documentation/networking/pktgen.txt for how to use this. 66 * 67 * The new operation: 68 * For each CPU one thread/process is created at start. This process checks 69 * for running devices in the if_list and sends packets until count is 0 it 70 * also the thread checks the thread->control which is used for inter-process 71 * communication. controlling process "posts" operations to the threads this 72 * way. 73 * The if_list is RCU protected, and the if_lock remains to protect updating 74 * of if_list, from "add_device" as it invoked from userspace (via proc write). 75 * 76 * By design there should only be *one* "controlling" process. In practice 77 * multiple write accesses gives unpredictable result. Understood by "write" 78 * to /proc gives result code thats should be read be the "writer". 79 * For practical use this should be no problem. 80 * 81 * Note when adding devices to a specific CPU there good idea to also assign 82 * /proc/irq/XX/smp_affinity so TX-interrupts gets bound to the same CPU. 83 * --ro 84 * 85 * Fix refcount off by one if first packet fails, potential null deref, 86 * memleak 030710- KJP 87 * 88 * First "ranges" functionality for ipv6 030726 --ro 89 * 90 * Included flow support. 030802 ANK. 91 * 92 * Fixed unaligned access on IA-64 Grant Grundler <grundler@parisc-linux.org> 93 * 94 * Remove if fix from added Harald Welte <laforge@netfilter.org> 040419 95 * ia64 compilation fix from Aron Griffis <aron@hp.com> 040604 96 * 97 * New xmit() return, do_div and misc clean up by Stephen Hemminger 98 * <shemminger@osdl.org> 040923 99 * 100 * Randy Dunlap fixed u64 printk compiler warning 101 * 102 * Remove FCS from BW calculation. Lennert Buytenhek <buytenh@wantstofly.org> 103 * New time handling. Lennert Buytenhek <buytenh@wantstofly.org> 041213 104 * 105 * Corrections from Nikolai Malykh (nmalykh@bilim.com) 106 * Removed unused flags F_SET_SRCMAC & F_SET_SRCIP 041230 107 * 108 * interruptible_sleep_on_timeout() replaced Nishanth Aravamudan <nacc@us.ibm.com> 109 * 050103 110 * 111 * MPLS support by Steven Whitehouse <steve@chygwyn.com> 112 * 113 * 802.1Q/Q-in-Q support by Francesco Fondelli (FF) <francesco.fondelli@gmail.com> 114 * 115 * Fixed src_mac command to set source mac of packet to value specified in 116 * command by Adit Ranadive <adit.262@gmail.com> 117 * 118 */ 119 120 #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt 121 122 #include <linux/sys.h> 123 #include <linux/types.h> 124 #include <linux/module.h> 125 #include <linux/moduleparam.h> 126 #include <linux/kernel.h> 127 #include <linux/mutex.h> 128 #include <linux/sched.h> 129 #include <linux/slab.h> 130 #include <linux/vmalloc.h> 131 #include <linux/unistd.h> 132 #include <linux/string.h> 133 #include <linux/ptrace.h> 134 #include <linux/errno.h> 135 #include <linux/ioport.h> 136 #include <linux/interrupt.h> 137 #include <linux/capability.h> 138 #include <linux/hrtimer.h> 139 #include <linux/freezer.h> 140 #include <linux/delay.h> 141 #include <linux/timer.h> 142 #include <linux/list.h> 143 #include <linux/init.h> 144 #include <linux/skbuff.h> 145 #include <linux/netdevice.h> 146 #include <linux/inet.h> 147 #include <linux/inetdevice.h> 148 #include <linux/rtnetlink.h> 149 #include <linux/if_arp.h> 150 #include <linux/if_vlan.h> 151 #include <linux/in.h> 152 #include <linux/ip.h> 153 #include <linux/ipv6.h> 154 #include <linux/udp.h> 155 #include <linux/proc_fs.h> 156 #include <linux/seq_file.h> 157 #include <linux/wait.h> 158 #include <linux/etherdevice.h> 159 #include <linux/kthread.h> 160 #include <linux/prefetch.h> 161 #include <net/net_namespace.h> 162 #include <net/checksum.h> 163 #include <net/ipv6.h> 164 #include <net/udp.h> 165 #include <net/ip6_checksum.h> 166 #include <net/addrconf.h> 167 #ifdef CONFIG_XFRM 168 #include <net/xfrm.h> 169 #endif 170 #include <net/netns/generic.h> 171 #include <asm/byteorder.h> 172 #include <linux/rcupdate.h> 173 #include <linux/bitops.h> 174 #include <linux/io.h> 175 #include <linux/timex.h> 176 #include <linux/uaccess.h> 177 #include <asm/dma.h> 178 #include <asm/div64.h> /* do_div */ 179 180 #define VERSION "2.75" 181 #define IP_NAME_SZ 32 182 #define MAX_MPLS_LABELS 16 /* This is the max label stack depth */ 183 #define MPLS_STACK_BOTTOM htonl(0x00000100) 184 185 #define func_enter() pr_debug("entering %s\n", __func__); 186 187 /* Device flag bits */ 188 #define F_IPSRC_RND (1<<0) /* IP-Src Random */ 189 #define F_IPDST_RND (1<<1) /* IP-Dst Random */ 190 #define F_UDPSRC_RND (1<<2) /* UDP-Src Random */ 191 #define F_UDPDST_RND (1<<3) /* UDP-Dst Random */ 192 #define F_MACSRC_RND (1<<4) /* MAC-Src Random */ 193 #define F_MACDST_RND (1<<5) /* MAC-Dst Random */ 194 #define F_TXSIZE_RND (1<<6) /* Transmit size is random */ 195 #define F_IPV6 (1<<7) /* Interface in IPV6 Mode */ 196 #define F_MPLS_RND (1<<8) /* Random MPLS labels */ 197 #define F_VID_RND (1<<9) /* Random VLAN ID */ 198 #define F_SVID_RND (1<<10) /* Random SVLAN ID */ 199 #define F_FLOW_SEQ (1<<11) /* Sequential flows */ 200 #define F_IPSEC_ON (1<<12) /* ipsec on for flows */ 201 #define F_QUEUE_MAP_RND (1<<13) /* queue map Random */ 202 #define F_QUEUE_MAP_CPU (1<<14) /* queue map mirrors smp_processor_id() */ 203 #define F_NODE (1<<15) /* Node memory alloc*/ 204 #define F_UDPCSUM (1<<16) /* Include UDP checksum */ 205 #define F_NO_TIMESTAMP (1<<17) /* Don't timestamp packets (default TS) */ 206 207 /* Thread control flag bits */ 208 #define T_STOP (1<<0) /* Stop run */ 209 #define T_RUN (1<<1) /* Start run */ 210 #define T_REMDEVALL (1<<2) /* Remove all devs */ 211 #define T_REMDEV (1<<3) /* Remove one dev */ 212 213 /* Xmit modes */ 214 #define M_START_XMIT 0 /* Default normal TX */ 215 #define M_NETIF_RECEIVE 1 /* Inject packets into stack */ 216 217 /* If lock -- protects updating of if_list */ 218 #define if_lock(t) spin_lock(&(t->if_lock)); 219 #define if_unlock(t) spin_unlock(&(t->if_lock)); 220 221 /* Used to help with determining the pkts on receive */ 222 #define PKTGEN_MAGIC 0xbe9be955 223 #define PG_PROC_DIR "pktgen" 224 #define PGCTRL "pgctrl" 225 226 #define MAX_CFLOWS 65536 227 228 #define VLAN_TAG_SIZE(x) ((x)->vlan_id == 0xffff ? 0 : 4) 229 #define SVLAN_TAG_SIZE(x) ((x)->svlan_id == 0xffff ? 0 : 4) 230 231 struct flow_state { 232 __be32 cur_daddr; 233 int count; 234 #ifdef CONFIG_XFRM 235 struct xfrm_state *x; 236 #endif 237 __u32 flags; 238 }; 239 240 /* flow flag bits */ 241 #define F_INIT (1<<0) /* flow has been initialized */ 242 243 struct pktgen_dev { 244 /* 245 * Try to keep frequent/infrequent used vars. separated. 246 */ 247 struct proc_dir_entry *entry; /* proc file */ 248 struct pktgen_thread *pg_thread;/* the owner */ 249 struct list_head list; /* chaining in the thread's run-queue */ 250 struct rcu_head rcu; /* freed by RCU */ 251 252 int running; /* if false, the test will stop */ 253 254 /* If min != max, then we will either do a linear iteration, or 255 * we will do a random selection from within the range. 256 */ 257 __u32 flags; 258 int xmit_mode; 259 int min_pkt_size; 260 int max_pkt_size; 261 int pkt_overhead; /* overhead for MPLS, VLANs, IPSEC etc */ 262 int nfrags; 263 int removal_mark; /* non-zero => the device is marked for 264 * removal by worker thread */ 265 266 struct page *page; 267 u64 delay; /* nano-seconds */ 268 269 __u64 count; /* Default No packets to send */ 270 __u64 sofar; /* How many pkts we've sent so far */ 271 __u64 tx_bytes; /* How many bytes we've transmitted */ 272 __u64 errors; /* Errors when trying to transmit, */ 273 274 /* runtime counters relating to clone_skb */ 275 276 __u32 clone_count; 277 int last_ok; /* Was last skb sent? 278 * Or a failed transmit of some sort? 279 * This will keep sequence numbers in order 280 */ 281 ktime_t next_tx; 282 ktime_t started_at; 283 ktime_t stopped_at; 284 u64 idle_acc; /* nano-seconds */ 285 286 __u32 seq_num; 287 288 int clone_skb; /* 289 * Use multiple SKBs during packet gen. 290 * If this number is greater than 1, then 291 * that many copies of the same packet will be 292 * sent before a new packet is allocated. 293 * If you want to send 1024 identical packets 294 * before creating a new packet, 295 * set clone_skb to 1024. 296 */ 297 298 char dst_min[IP_NAME_SZ]; /* IP, ie 1.2.3.4 */ 299 char dst_max[IP_NAME_SZ]; /* IP, ie 1.2.3.4 */ 300 char src_min[IP_NAME_SZ]; /* IP, ie 1.2.3.4 */ 301 char src_max[IP_NAME_SZ]; /* IP, ie 1.2.3.4 */ 302 303 struct in6_addr in6_saddr; 304 struct in6_addr in6_daddr; 305 struct in6_addr cur_in6_daddr; 306 struct in6_addr cur_in6_saddr; 307 /* For ranges */ 308 struct in6_addr min_in6_daddr; 309 struct in6_addr max_in6_daddr; 310 struct in6_addr min_in6_saddr; 311 struct in6_addr max_in6_saddr; 312 313 /* If we're doing ranges, random or incremental, then this 314 * defines the min/max for those ranges. 315 */ 316 __be32 saddr_min; /* inclusive, source IP address */ 317 __be32 saddr_max; /* exclusive, source IP address */ 318 __be32 daddr_min; /* inclusive, dest IP address */ 319 __be32 daddr_max; /* exclusive, dest IP address */ 320 321 __u16 udp_src_min; /* inclusive, source UDP port */ 322 __u16 udp_src_max; /* exclusive, source UDP port */ 323 __u16 udp_dst_min; /* inclusive, dest UDP port */ 324 __u16 udp_dst_max; /* exclusive, dest UDP port */ 325 326 /* DSCP + ECN */ 327 __u8 tos; /* six MSB of (former) IPv4 TOS 328 are for dscp codepoint */ 329 __u8 traffic_class; /* ditto for the (former) Traffic Class in IPv6 330 (see RFC 3260, sec. 4) */ 331 332 /* MPLS */ 333 unsigned int nr_labels; /* Depth of stack, 0 = no MPLS */ 334 __be32 labels[MAX_MPLS_LABELS]; 335 336 /* VLAN/SVLAN (802.1Q/Q-in-Q) */ 337 __u8 vlan_p; 338 __u8 vlan_cfi; 339 __u16 vlan_id; /* 0xffff means no vlan tag */ 340 341 __u8 svlan_p; 342 __u8 svlan_cfi; 343 __u16 svlan_id; /* 0xffff means no svlan tag */ 344 345 __u32 src_mac_count; /* How many MACs to iterate through */ 346 __u32 dst_mac_count; /* How many MACs to iterate through */ 347 348 unsigned char dst_mac[ETH_ALEN]; 349 unsigned char src_mac[ETH_ALEN]; 350 351 __u32 cur_dst_mac_offset; 352 __u32 cur_src_mac_offset; 353 __be32 cur_saddr; 354 __be32 cur_daddr; 355 __u16 ip_id; 356 __u16 cur_udp_dst; 357 __u16 cur_udp_src; 358 __u16 cur_queue_map; 359 __u32 cur_pkt_size; 360 __u32 last_pkt_size; 361 362 __u8 hh[14]; 363 /* = { 364 0x00, 0x80, 0xC8, 0x79, 0xB3, 0xCB, 365 366 We fill in SRC address later 367 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 368 0x08, 0x00 369 }; 370 */ 371 __u16 pad; /* pad out the hh struct to an even 16 bytes */ 372 373 struct sk_buff *skb; /* skb we are to transmit next, used for when we 374 * are transmitting the same one multiple times 375 */ 376 struct net_device *odev; /* The out-going device. 377 * Note that the device should have it's 378 * pg_info pointer pointing back to this 379 * device. 380 * Set when the user specifies the out-going 381 * device name (not when the inject is 382 * started as it used to do.) 383 */ 384 char odevname[32]; 385 struct flow_state *flows; 386 unsigned int cflows; /* Concurrent flows (config) */ 387 unsigned int lflow; /* Flow length (config) */ 388 unsigned int nflows; /* accumulated flows (stats) */ 389 unsigned int curfl; /* current sequenced flow (state)*/ 390 391 u16 queue_map_min; 392 u16 queue_map_max; 393 __u32 skb_priority; /* skb priority field */ 394 unsigned int burst; /* number of duplicated packets to burst */ 395 int node; /* Memory node */ 396 397 #ifdef CONFIG_XFRM 398 __u8 ipsmode; /* IPSEC mode (config) */ 399 __u8 ipsproto; /* IPSEC type (config) */ 400 __u32 spi; 401 struct dst_entry dst; 402 struct dst_ops dstops; 403 #endif 404 char result[512]; 405 }; 406 407 struct pktgen_hdr { 408 __be32 pgh_magic; 409 __be32 seq_num; 410 __be32 tv_sec; 411 __be32 tv_usec; 412 }; 413 414 415 static int pg_net_id __read_mostly; 416 417 struct pktgen_net { 418 struct net *net; 419 struct proc_dir_entry *proc_dir; 420 struct list_head pktgen_threads; 421 bool pktgen_exiting; 422 }; 423 424 struct pktgen_thread { 425 spinlock_t if_lock; /* for list of devices */ 426 struct list_head if_list; /* All device here */ 427 struct list_head th_list; 428 struct task_struct *tsk; 429 char result[512]; 430 431 /* Field for thread to receive "posted" events terminate, 432 stop ifs etc. */ 433 434 u32 control; 435 int cpu; 436 437 wait_queue_head_t queue; 438 struct completion start_done; 439 struct pktgen_net *net; 440 }; 441 442 #define REMOVE 1 443 #define FIND 0 444 445 static const char version[] = 446 "Packet Generator for packet performance testing. " 447 "Version: " VERSION "\n"; 448 449 static int pktgen_remove_device(struct pktgen_thread *t, struct pktgen_dev *i); 450 static int pktgen_add_device(struct pktgen_thread *t, const char *ifname); 451 static struct pktgen_dev *pktgen_find_dev(struct pktgen_thread *t, 452 const char *ifname, bool exact); 453 static int pktgen_device_event(struct notifier_block *, unsigned long, void *); 454 static void pktgen_run_all_threads(struct pktgen_net *pn); 455 static void pktgen_reset_all_threads(struct pktgen_net *pn); 456 static void pktgen_stop_all_threads_ifs(struct pktgen_net *pn); 457 458 static void pktgen_stop(struct pktgen_thread *t); 459 static void pktgen_clear_counters(struct pktgen_dev *pkt_dev); 460 461 /* Module parameters, defaults. */ 462 static int pg_count_d __read_mostly = 1000; 463 static int pg_delay_d __read_mostly; 464 static int pg_clone_skb_d __read_mostly; 465 static int debug __read_mostly; 466 467 static DEFINE_MUTEX(pktgen_thread_lock); 468 469 static struct notifier_block pktgen_notifier_block = { 470 .notifier_call = pktgen_device_event, 471 }; 472 473 /* 474 * /proc handling functions 475 * 476 */ 477 478 static int pgctrl_show(struct seq_file *seq, void *v) 479 { 480 seq_puts(seq, version); 481 return 0; 482 } 483 484 static ssize_t pgctrl_write(struct file *file, const char __user *buf, 485 size_t count, loff_t *ppos) 486 { 487 char data[128]; 488 struct pktgen_net *pn = net_generic(current->nsproxy->net_ns, pg_net_id); 489 490 if (!capable(CAP_NET_ADMIN)) 491 return -EPERM; 492 493 if (count == 0) 494 return -EINVAL; 495 496 if (count > sizeof(data)) 497 count = sizeof(data); 498 499 if (copy_from_user(data, buf, count)) 500 return -EFAULT; 501 502 data[count - 1] = 0; /* Strip trailing '\n' and terminate string */ 503 504 if (!strcmp(data, "stop")) 505 pktgen_stop_all_threads_ifs(pn); 506 507 else if (!strcmp(data, "start")) 508 pktgen_run_all_threads(pn); 509 510 else if (!strcmp(data, "reset")) 511 pktgen_reset_all_threads(pn); 512 513 else 514 return -EINVAL; 515 516 return count; 517 } 518 519 static int pgctrl_open(struct inode *inode, struct file *file) 520 { 521 return single_open(file, pgctrl_show, PDE_DATA(inode)); 522 } 523 524 static const struct file_operations pktgen_fops = { 525 .owner = THIS_MODULE, 526 .open = pgctrl_open, 527 .read = seq_read, 528 .llseek = seq_lseek, 529 .write = pgctrl_write, 530 .release = single_release, 531 }; 532 533 static int pktgen_if_show(struct seq_file *seq, void *v) 534 { 535 const struct pktgen_dev *pkt_dev = seq->private; 536 ktime_t stopped; 537 u64 idle; 538 539 seq_printf(seq, 540 "Params: count %llu min_pkt_size: %u max_pkt_size: %u\n", 541 (unsigned long long)pkt_dev->count, pkt_dev->min_pkt_size, 542 pkt_dev->max_pkt_size); 543 544 seq_printf(seq, 545 " frags: %d delay: %llu clone_skb: %d ifname: %s\n", 546 pkt_dev->nfrags, (unsigned long long) pkt_dev->delay, 547 pkt_dev->clone_skb, pkt_dev->odevname); 548 549 seq_printf(seq, " flows: %u flowlen: %u\n", pkt_dev->cflows, 550 pkt_dev->lflow); 551 552 seq_printf(seq, 553 " queue_map_min: %u queue_map_max: %u\n", 554 pkt_dev->queue_map_min, 555 pkt_dev->queue_map_max); 556 557 if (pkt_dev->skb_priority) 558 seq_printf(seq, " skb_priority: %u\n", 559 pkt_dev->skb_priority); 560 561 if (pkt_dev->flags & F_IPV6) { 562 seq_printf(seq, 563 " saddr: %pI6c min_saddr: %pI6c max_saddr: %pI6c\n" 564 " daddr: %pI6c min_daddr: %pI6c max_daddr: %pI6c\n", 565 &pkt_dev->in6_saddr, 566 &pkt_dev->min_in6_saddr, &pkt_dev->max_in6_saddr, 567 &pkt_dev->in6_daddr, 568 &pkt_dev->min_in6_daddr, &pkt_dev->max_in6_daddr); 569 } else { 570 seq_printf(seq, 571 " dst_min: %s dst_max: %s\n", 572 pkt_dev->dst_min, pkt_dev->dst_max); 573 seq_printf(seq, 574 " src_min: %s src_max: %s\n", 575 pkt_dev->src_min, pkt_dev->src_max); 576 } 577 578 seq_puts(seq, " src_mac: "); 579 580 seq_printf(seq, "%pM ", 581 is_zero_ether_addr(pkt_dev->src_mac) ? 582 pkt_dev->odev->dev_addr : pkt_dev->src_mac); 583 584 seq_puts(seq, "dst_mac: "); 585 seq_printf(seq, "%pM\n", pkt_dev->dst_mac); 586 587 seq_printf(seq, 588 " udp_src_min: %d udp_src_max: %d" 589 " udp_dst_min: %d udp_dst_max: %d\n", 590 pkt_dev->udp_src_min, pkt_dev->udp_src_max, 591 pkt_dev->udp_dst_min, pkt_dev->udp_dst_max); 592 593 seq_printf(seq, 594 " src_mac_count: %d dst_mac_count: %d\n", 595 pkt_dev->src_mac_count, pkt_dev->dst_mac_count); 596 597 if (pkt_dev->nr_labels) { 598 unsigned int i; 599 seq_puts(seq, " mpls: "); 600 for (i = 0; i < pkt_dev->nr_labels; i++) 601 seq_printf(seq, "%08x%s", ntohl(pkt_dev->labels[i]), 602 i == pkt_dev->nr_labels-1 ? "\n" : ", "); 603 } 604 605 if (pkt_dev->vlan_id != 0xffff) 606 seq_printf(seq, " vlan_id: %u vlan_p: %u vlan_cfi: %u\n", 607 pkt_dev->vlan_id, pkt_dev->vlan_p, 608 pkt_dev->vlan_cfi); 609 610 if (pkt_dev->svlan_id != 0xffff) 611 seq_printf(seq, " svlan_id: %u vlan_p: %u vlan_cfi: %u\n", 612 pkt_dev->svlan_id, pkt_dev->svlan_p, 613 pkt_dev->svlan_cfi); 614 615 if (pkt_dev->tos) 616 seq_printf(seq, " tos: 0x%02x\n", pkt_dev->tos); 617 618 if (pkt_dev->traffic_class) 619 seq_printf(seq, " traffic_class: 0x%02x\n", pkt_dev->traffic_class); 620 621 if (pkt_dev->burst > 1) 622 seq_printf(seq, " burst: %d\n", pkt_dev->burst); 623 624 if (pkt_dev->node >= 0) 625 seq_printf(seq, " node: %d\n", pkt_dev->node); 626 627 if (pkt_dev->xmit_mode == M_NETIF_RECEIVE) 628 seq_puts(seq, " xmit_mode: netif_receive\n"); 629 630 seq_puts(seq, " Flags: "); 631 632 if (pkt_dev->flags & F_IPV6) 633 seq_puts(seq, "IPV6 "); 634 635 if (pkt_dev->flags & F_IPSRC_RND) 636 seq_puts(seq, "IPSRC_RND "); 637 638 if (pkt_dev->flags & F_IPDST_RND) 639 seq_puts(seq, "IPDST_RND "); 640 641 if (pkt_dev->flags & F_TXSIZE_RND) 642 seq_puts(seq, "TXSIZE_RND "); 643 644 if (pkt_dev->flags & F_UDPSRC_RND) 645 seq_puts(seq, "UDPSRC_RND "); 646 647 if (pkt_dev->flags & F_UDPDST_RND) 648 seq_puts(seq, "UDPDST_RND "); 649 650 if (pkt_dev->flags & F_UDPCSUM) 651 seq_puts(seq, "UDPCSUM "); 652 653 if (pkt_dev->flags & F_NO_TIMESTAMP) 654 seq_puts(seq, "NO_TIMESTAMP "); 655 656 if (pkt_dev->flags & F_MPLS_RND) 657 seq_puts(seq, "MPLS_RND "); 658 659 if (pkt_dev->flags & F_QUEUE_MAP_RND) 660 seq_puts(seq, "QUEUE_MAP_RND "); 661 662 if (pkt_dev->flags & F_QUEUE_MAP_CPU) 663 seq_puts(seq, "QUEUE_MAP_CPU "); 664 665 if (pkt_dev->cflows) { 666 if (pkt_dev->flags & F_FLOW_SEQ) 667 seq_puts(seq, "FLOW_SEQ "); /*in sequence flows*/ 668 else 669 seq_puts(seq, "FLOW_RND "); 670 } 671 672 #ifdef CONFIG_XFRM 673 if (pkt_dev->flags & F_IPSEC_ON) { 674 seq_puts(seq, "IPSEC "); 675 if (pkt_dev->spi) 676 seq_printf(seq, "spi:%u", pkt_dev->spi); 677 } 678 #endif 679 680 if (pkt_dev->flags & F_MACSRC_RND) 681 seq_puts(seq, "MACSRC_RND "); 682 683 if (pkt_dev->flags & F_MACDST_RND) 684 seq_puts(seq, "MACDST_RND "); 685 686 if (pkt_dev->flags & F_VID_RND) 687 seq_puts(seq, "VID_RND "); 688 689 if (pkt_dev->flags & F_SVID_RND) 690 seq_puts(seq, "SVID_RND "); 691 692 if (pkt_dev->flags & F_NODE) 693 seq_puts(seq, "NODE_ALLOC "); 694 695 seq_puts(seq, "\n"); 696 697 /* not really stopped, more like last-running-at */ 698 stopped = pkt_dev->running ? ktime_get() : pkt_dev->stopped_at; 699 idle = pkt_dev->idle_acc; 700 do_div(idle, NSEC_PER_USEC); 701 702 seq_printf(seq, 703 "Current:\n pkts-sofar: %llu errors: %llu\n", 704 (unsigned long long)pkt_dev->sofar, 705 (unsigned long long)pkt_dev->errors); 706 707 seq_printf(seq, 708 " started: %lluus stopped: %lluus idle: %lluus\n", 709 (unsigned long long) ktime_to_us(pkt_dev->started_at), 710 (unsigned long long) ktime_to_us(stopped), 711 (unsigned long long) idle); 712 713 seq_printf(seq, 714 " seq_num: %d cur_dst_mac_offset: %d cur_src_mac_offset: %d\n", 715 pkt_dev->seq_num, pkt_dev->cur_dst_mac_offset, 716 pkt_dev->cur_src_mac_offset); 717 718 if (pkt_dev->flags & F_IPV6) { 719 seq_printf(seq, " cur_saddr: %pI6c cur_daddr: %pI6c\n", 720 &pkt_dev->cur_in6_saddr, 721 &pkt_dev->cur_in6_daddr); 722 } else 723 seq_printf(seq, " cur_saddr: %pI4 cur_daddr: %pI4\n", 724 &pkt_dev->cur_saddr, &pkt_dev->cur_daddr); 725 726 seq_printf(seq, " cur_udp_dst: %d cur_udp_src: %d\n", 727 pkt_dev->cur_udp_dst, pkt_dev->cur_udp_src); 728 729 seq_printf(seq, " cur_queue_map: %u\n", pkt_dev->cur_queue_map); 730 731 seq_printf(seq, " flows: %u\n", pkt_dev->nflows); 732 733 if (pkt_dev->result[0]) 734 seq_printf(seq, "Result: %s\n", pkt_dev->result); 735 else 736 seq_puts(seq, "Result: Idle\n"); 737 738 return 0; 739 } 740 741 742 static int hex32_arg(const char __user *user_buffer, unsigned long maxlen, 743 __u32 *num) 744 { 745 int i = 0; 746 *num = 0; 747 748 for (; i < maxlen; i++) { 749 int value; 750 char c; 751 *num <<= 4; 752 if (get_user(c, &user_buffer[i])) 753 return -EFAULT; 754 value = hex_to_bin(c); 755 if (value >= 0) 756 *num |= value; 757 else 758 break; 759 } 760 return i; 761 } 762 763 static int count_trail_chars(const char __user * user_buffer, 764 unsigned int maxlen) 765 { 766 int i; 767 768 for (i = 0; i < maxlen; i++) { 769 char c; 770 if (get_user(c, &user_buffer[i])) 771 return -EFAULT; 772 switch (c) { 773 case '\"': 774 case '\n': 775 case '\r': 776 case '\t': 777 case ' ': 778 case '=': 779 break; 780 default: 781 goto done; 782 } 783 } 784 done: 785 return i; 786 } 787 788 static long num_arg(const char __user *user_buffer, unsigned long maxlen, 789 unsigned long *num) 790 { 791 int i; 792 *num = 0; 793 794 for (i = 0; i < maxlen; i++) { 795 char c; 796 if (get_user(c, &user_buffer[i])) 797 return -EFAULT; 798 if ((c >= '0') && (c <= '9')) { 799 *num *= 10; 800 *num += c - '0'; 801 } else 802 break; 803 } 804 return i; 805 } 806 807 static int strn_len(const char __user * user_buffer, unsigned int maxlen) 808 { 809 int i; 810 811 for (i = 0; i < maxlen; i++) { 812 char c; 813 if (get_user(c, &user_buffer[i])) 814 return -EFAULT; 815 switch (c) { 816 case '\"': 817 case '\n': 818 case '\r': 819 case '\t': 820 case ' ': 821 goto done_str; 822 default: 823 break; 824 } 825 } 826 done_str: 827 return i; 828 } 829 830 static ssize_t get_labels(const char __user *buffer, struct pktgen_dev *pkt_dev) 831 { 832 unsigned int n = 0; 833 char c; 834 ssize_t i = 0; 835 int len; 836 837 pkt_dev->nr_labels = 0; 838 do { 839 __u32 tmp; 840 len = hex32_arg(&buffer[i], 8, &tmp); 841 if (len <= 0) 842 return len; 843 pkt_dev->labels[n] = htonl(tmp); 844 if (pkt_dev->labels[n] & MPLS_STACK_BOTTOM) 845 pkt_dev->flags |= F_MPLS_RND; 846 i += len; 847 if (get_user(c, &buffer[i])) 848 return -EFAULT; 849 i++; 850 n++; 851 if (n >= MAX_MPLS_LABELS) 852 return -E2BIG; 853 } while (c == ','); 854 855 pkt_dev->nr_labels = n; 856 return i; 857 } 858 859 static ssize_t pktgen_if_write(struct file *file, 860 const char __user * user_buffer, size_t count, 861 loff_t * offset) 862 { 863 struct seq_file *seq = file->private_data; 864 struct pktgen_dev *pkt_dev = seq->private; 865 int i, max, len; 866 char name[16], valstr[32]; 867 unsigned long value = 0; 868 char *pg_result = NULL; 869 int tmp = 0; 870 char buf[128]; 871 872 pg_result = &(pkt_dev->result[0]); 873 874 if (count < 1) { 875 pr_warn("wrong command format\n"); 876 return -EINVAL; 877 } 878 879 max = count; 880 tmp = count_trail_chars(user_buffer, max); 881 if (tmp < 0) { 882 pr_warn("illegal format\n"); 883 return tmp; 884 } 885 i = tmp; 886 887 /* Read variable name */ 888 889 len = strn_len(&user_buffer[i], sizeof(name) - 1); 890 if (len < 0) 891 return len; 892 893 memset(name, 0, sizeof(name)); 894 if (copy_from_user(name, &user_buffer[i], len)) 895 return -EFAULT; 896 i += len; 897 898 max = count - i; 899 len = count_trail_chars(&user_buffer[i], max); 900 if (len < 0) 901 return len; 902 903 i += len; 904 905 if (debug) { 906 size_t copy = min_t(size_t, count, 1023); 907 char tb[copy + 1]; 908 if (copy_from_user(tb, user_buffer, copy)) 909 return -EFAULT; 910 tb[copy] = 0; 911 pr_debug("%s,%lu buffer -:%s:-\n", 912 name, (unsigned long)count, tb); 913 } 914 915 if (!strcmp(name, "min_pkt_size")) { 916 len = num_arg(&user_buffer[i], 10, &value); 917 if (len < 0) 918 return len; 919 920 i += len; 921 if (value < 14 + 20 + 8) 922 value = 14 + 20 + 8; 923 if (value != pkt_dev->min_pkt_size) { 924 pkt_dev->min_pkt_size = value; 925 pkt_dev->cur_pkt_size = value; 926 } 927 sprintf(pg_result, "OK: min_pkt_size=%u", 928 pkt_dev->min_pkt_size); 929 return count; 930 } 931 932 if (!strcmp(name, "max_pkt_size")) { 933 len = num_arg(&user_buffer[i], 10, &value); 934 if (len < 0) 935 return len; 936 937 i += len; 938 if (value < 14 + 20 + 8) 939 value = 14 + 20 + 8; 940 if (value != pkt_dev->max_pkt_size) { 941 pkt_dev->max_pkt_size = value; 942 pkt_dev->cur_pkt_size = value; 943 } 944 sprintf(pg_result, "OK: max_pkt_size=%u", 945 pkt_dev->max_pkt_size); 946 return count; 947 } 948 949 /* Shortcut for min = max */ 950 951 if (!strcmp(name, "pkt_size")) { 952 len = num_arg(&user_buffer[i], 10, &value); 953 if (len < 0) 954 return len; 955 956 i += len; 957 if (value < 14 + 20 + 8) 958 value = 14 + 20 + 8; 959 if (value != pkt_dev->min_pkt_size) { 960 pkt_dev->min_pkt_size = value; 961 pkt_dev->max_pkt_size = value; 962 pkt_dev->cur_pkt_size = value; 963 } 964 sprintf(pg_result, "OK: pkt_size=%u", pkt_dev->min_pkt_size); 965 return count; 966 } 967 968 if (!strcmp(name, "debug")) { 969 len = num_arg(&user_buffer[i], 10, &value); 970 if (len < 0) 971 return len; 972 973 i += len; 974 debug = value; 975 sprintf(pg_result, "OK: debug=%u", debug); 976 return count; 977 } 978 979 if (!strcmp(name, "frags")) { 980 len = num_arg(&user_buffer[i], 10, &value); 981 if (len < 0) 982 return len; 983 984 i += len; 985 pkt_dev->nfrags = value; 986 sprintf(pg_result, "OK: frags=%u", pkt_dev->nfrags); 987 return count; 988 } 989 if (!strcmp(name, "delay")) { 990 len = num_arg(&user_buffer[i], 10, &value); 991 if (len < 0) 992 return len; 993 994 i += len; 995 if (value == 0x7FFFFFFF) 996 pkt_dev->delay = ULLONG_MAX; 997 else 998 pkt_dev->delay = (u64)value; 999 1000 sprintf(pg_result, "OK: delay=%llu", 1001 (unsigned long long) pkt_dev->delay); 1002 return count; 1003 } 1004 if (!strcmp(name, "rate")) { 1005 len = num_arg(&user_buffer[i], 10, &value); 1006 if (len < 0) 1007 return len; 1008 1009 i += len; 1010 if (!value) 1011 return len; 1012 pkt_dev->delay = pkt_dev->min_pkt_size*8*NSEC_PER_USEC/value; 1013 if (debug) 1014 pr_info("Delay set at: %llu ns\n", pkt_dev->delay); 1015 1016 sprintf(pg_result, "OK: rate=%lu", value); 1017 return count; 1018 } 1019 if (!strcmp(name, "ratep")) { 1020 len = num_arg(&user_buffer[i], 10, &value); 1021 if (len < 0) 1022 return len; 1023 1024 i += len; 1025 if (!value) 1026 return len; 1027 pkt_dev->delay = NSEC_PER_SEC/value; 1028 if (debug) 1029 pr_info("Delay set at: %llu ns\n", pkt_dev->delay); 1030 1031 sprintf(pg_result, "OK: rate=%lu", value); 1032 return count; 1033 } 1034 if (!strcmp(name, "udp_src_min")) { 1035 len = num_arg(&user_buffer[i], 10, &value); 1036 if (len < 0) 1037 return len; 1038 1039 i += len; 1040 if (value != pkt_dev->udp_src_min) { 1041 pkt_dev->udp_src_min = value; 1042 pkt_dev->cur_udp_src = value; 1043 } 1044 sprintf(pg_result, "OK: udp_src_min=%u", pkt_dev->udp_src_min); 1045 return count; 1046 } 1047 if (!strcmp(name, "udp_dst_min")) { 1048 len = num_arg(&user_buffer[i], 10, &value); 1049 if (len < 0) 1050 return len; 1051 1052 i += len; 1053 if (value != pkt_dev->udp_dst_min) { 1054 pkt_dev->udp_dst_min = value; 1055 pkt_dev->cur_udp_dst = value; 1056 } 1057 sprintf(pg_result, "OK: udp_dst_min=%u", pkt_dev->udp_dst_min); 1058 return count; 1059 } 1060 if (!strcmp(name, "udp_src_max")) { 1061 len = num_arg(&user_buffer[i], 10, &value); 1062 if (len < 0) 1063 return len; 1064 1065 i += len; 1066 if (value != pkt_dev->udp_src_max) { 1067 pkt_dev->udp_src_max = value; 1068 pkt_dev->cur_udp_src = value; 1069 } 1070 sprintf(pg_result, "OK: udp_src_max=%u", pkt_dev->udp_src_max); 1071 return count; 1072 } 1073 if (!strcmp(name, "udp_dst_max")) { 1074 len = num_arg(&user_buffer[i], 10, &value); 1075 if (len < 0) 1076 return len; 1077 1078 i += len; 1079 if (value != pkt_dev->udp_dst_max) { 1080 pkt_dev->udp_dst_max = value; 1081 pkt_dev->cur_udp_dst = value; 1082 } 1083 sprintf(pg_result, "OK: udp_dst_max=%u", pkt_dev->udp_dst_max); 1084 return count; 1085 } 1086 if (!strcmp(name, "clone_skb")) { 1087 len = num_arg(&user_buffer[i], 10, &value); 1088 if (len < 0) 1089 return len; 1090 if ((value > 0) && 1091 ((pkt_dev->xmit_mode == M_NETIF_RECEIVE) || 1092 !(pkt_dev->odev->priv_flags & IFF_TX_SKB_SHARING))) 1093 return -ENOTSUPP; 1094 i += len; 1095 pkt_dev->clone_skb = value; 1096 1097 sprintf(pg_result, "OK: clone_skb=%d", pkt_dev->clone_skb); 1098 return count; 1099 } 1100 if (!strcmp(name, "count")) { 1101 len = num_arg(&user_buffer[i], 10, &value); 1102 if (len < 0) 1103 return len; 1104 1105 i += len; 1106 pkt_dev->count = value; 1107 sprintf(pg_result, "OK: count=%llu", 1108 (unsigned long long)pkt_dev->count); 1109 return count; 1110 } 1111 if (!strcmp(name, "src_mac_count")) { 1112 len = num_arg(&user_buffer[i], 10, &value); 1113 if (len < 0) 1114 return len; 1115 1116 i += len; 1117 if (pkt_dev->src_mac_count != value) { 1118 pkt_dev->src_mac_count = value; 1119 pkt_dev->cur_src_mac_offset = 0; 1120 } 1121 sprintf(pg_result, "OK: src_mac_count=%d", 1122 pkt_dev->src_mac_count); 1123 return count; 1124 } 1125 if (!strcmp(name, "dst_mac_count")) { 1126 len = num_arg(&user_buffer[i], 10, &value); 1127 if (len < 0) 1128 return len; 1129 1130 i += len; 1131 if (pkt_dev->dst_mac_count != value) { 1132 pkt_dev->dst_mac_count = value; 1133 pkt_dev->cur_dst_mac_offset = 0; 1134 } 1135 sprintf(pg_result, "OK: dst_mac_count=%d", 1136 pkt_dev->dst_mac_count); 1137 return count; 1138 } 1139 if (!strcmp(name, "burst")) { 1140 len = num_arg(&user_buffer[i], 10, &value); 1141 if (len < 0) 1142 return len; 1143 1144 i += len; 1145 if ((value > 1) && (pkt_dev->xmit_mode == M_START_XMIT) && 1146 (!(pkt_dev->odev->priv_flags & IFF_TX_SKB_SHARING))) 1147 return -ENOTSUPP; 1148 pkt_dev->burst = value < 1 ? 1 : value; 1149 sprintf(pg_result, "OK: burst=%d", pkt_dev->burst); 1150 return count; 1151 } 1152 if (!strcmp(name, "node")) { 1153 len = num_arg(&user_buffer[i], 10, &value); 1154 if (len < 0) 1155 return len; 1156 1157 i += len; 1158 1159 if (node_possible(value)) { 1160 pkt_dev->node = value; 1161 sprintf(pg_result, "OK: node=%d", pkt_dev->node); 1162 if (pkt_dev->page) { 1163 put_page(pkt_dev->page); 1164 pkt_dev->page = NULL; 1165 } 1166 } 1167 else 1168 sprintf(pg_result, "ERROR: node not possible"); 1169 return count; 1170 } 1171 if (!strcmp(name, "xmit_mode")) { 1172 char f[32]; 1173 1174 memset(f, 0, 32); 1175 len = strn_len(&user_buffer[i], sizeof(f) - 1); 1176 if (len < 0) 1177 return len; 1178 1179 if (copy_from_user(f, &user_buffer[i], len)) 1180 return -EFAULT; 1181 i += len; 1182 1183 if (strcmp(f, "start_xmit") == 0) { 1184 pkt_dev->xmit_mode = M_START_XMIT; 1185 } else if (strcmp(f, "netif_receive") == 0) { 1186 /* clone_skb set earlier, not supported in this mode */ 1187 if (pkt_dev->clone_skb > 0) 1188 return -ENOTSUPP; 1189 1190 pkt_dev->xmit_mode = M_NETIF_RECEIVE; 1191 1192 /* make sure new packet is allocated every time 1193 * pktgen_xmit() is called 1194 */ 1195 pkt_dev->last_ok = 1; 1196 1197 /* override clone_skb if user passed default value 1198 * at module loading time 1199 */ 1200 pkt_dev->clone_skb = 0; 1201 } else { 1202 sprintf(pg_result, 1203 "xmit_mode -:%s:- unknown\nAvailable modes: %s", 1204 f, "start_xmit, netif_receive\n"); 1205 return count; 1206 } 1207 sprintf(pg_result, "OK: xmit_mode=%s", f); 1208 return count; 1209 } 1210 if (!strcmp(name, "flag")) { 1211 char f[32]; 1212 memset(f, 0, 32); 1213 len = strn_len(&user_buffer[i], sizeof(f) - 1); 1214 if (len < 0) 1215 return len; 1216 1217 if (copy_from_user(f, &user_buffer[i], len)) 1218 return -EFAULT; 1219 i += len; 1220 if (strcmp(f, "IPSRC_RND") == 0) 1221 pkt_dev->flags |= F_IPSRC_RND; 1222 1223 else if (strcmp(f, "!IPSRC_RND") == 0) 1224 pkt_dev->flags &= ~F_IPSRC_RND; 1225 1226 else if (strcmp(f, "TXSIZE_RND") == 0) 1227 pkt_dev->flags |= F_TXSIZE_RND; 1228 1229 else if (strcmp(f, "!TXSIZE_RND") == 0) 1230 pkt_dev->flags &= ~F_TXSIZE_RND; 1231 1232 else if (strcmp(f, "IPDST_RND") == 0) 1233 pkt_dev->flags |= F_IPDST_RND; 1234 1235 else if (strcmp(f, "!IPDST_RND") == 0) 1236 pkt_dev->flags &= ~F_IPDST_RND; 1237 1238 else if (strcmp(f, "UDPSRC_RND") == 0) 1239 pkt_dev->flags |= F_UDPSRC_RND; 1240 1241 else if (strcmp(f, "!UDPSRC_RND") == 0) 1242 pkt_dev->flags &= ~F_UDPSRC_RND; 1243 1244 else if (strcmp(f, "UDPDST_RND") == 0) 1245 pkt_dev->flags |= F_UDPDST_RND; 1246 1247 else if (strcmp(f, "!UDPDST_RND") == 0) 1248 pkt_dev->flags &= ~F_UDPDST_RND; 1249 1250 else if (strcmp(f, "MACSRC_RND") == 0) 1251 pkt_dev->flags |= F_MACSRC_RND; 1252 1253 else if (strcmp(f, "!MACSRC_RND") == 0) 1254 pkt_dev->flags &= ~F_MACSRC_RND; 1255 1256 else if (strcmp(f, "MACDST_RND") == 0) 1257 pkt_dev->flags |= F_MACDST_RND; 1258 1259 else if (strcmp(f, "!MACDST_RND") == 0) 1260 pkt_dev->flags &= ~F_MACDST_RND; 1261 1262 else if (strcmp(f, "MPLS_RND") == 0) 1263 pkt_dev->flags |= F_MPLS_RND; 1264 1265 else if (strcmp(f, "!MPLS_RND") == 0) 1266 pkt_dev->flags &= ~F_MPLS_RND; 1267 1268 else if (strcmp(f, "VID_RND") == 0) 1269 pkt_dev->flags |= F_VID_RND; 1270 1271 else if (strcmp(f, "!VID_RND") == 0) 1272 pkt_dev->flags &= ~F_VID_RND; 1273 1274 else if (strcmp(f, "SVID_RND") == 0) 1275 pkt_dev->flags |= F_SVID_RND; 1276 1277 else if (strcmp(f, "!SVID_RND") == 0) 1278 pkt_dev->flags &= ~F_SVID_RND; 1279 1280 else if (strcmp(f, "FLOW_SEQ") == 0) 1281 pkt_dev->flags |= F_FLOW_SEQ; 1282 1283 else if (strcmp(f, "QUEUE_MAP_RND") == 0) 1284 pkt_dev->flags |= F_QUEUE_MAP_RND; 1285 1286 else if (strcmp(f, "!QUEUE_MAP_RND") == 0) 1287 pkt_dev->flags &= ~F_QUEUE_MAP_RND; 1288 1289 else if (strcmp(f, "QUEUE_MAP_CPU") == 0) 1290 pkt_dev->flags |= F_QUEUE_MAP_CPU; 1291 1292 else if (strcmp(f, "!QUEUE_MAP_CPU") == 0) 1293 pkt_dev->flags &= ~F_QUEUE_MAP_CPU; 1294 #ifdef CONFIG_XFRM 1295 else if (strcmp(f, "IPSEC") == 0) 1296 pkt_dev->flags |= F_IPSEC_ON; 1297 #endif 1298 1299 else if (strcmp(f, "!IPV6") == 0) 1300 pkt_dev->flags &= ~F_IPV6; 1301 1302 else if (strcmp(f, "NODE_ALLOC") == 0) 1303 pkt_dev->flags |= F_NODE; 1304 1305 else if (strcmp(f, "!NODE_ALLOC") == 0) 1306 pkt_dev->flags &= ~F_NODE; 1307 1308 else if (strcmp(f, "UDPCSUM") == 0) 1309 pkt_dev->flags |= F_UDPCSUM; 1310 1311 else if (strcmp(f, "!UDPCSUM") == 0) 1312 pkt_dev->flags &= ~F_UDPCSUM; 1313 1314 else if (strcmp(f, "NO_TIMESTAMP") == 0) 1315 pkt_dev->flags |= F_NO_TIMESTAMP; 1316 1317 else if (strcmp(f, "!NO_TIMESTAMP") == 0) 1318 pkt_dev->flags &= ~F_NO_TIMESTAMP; 1319 1320 else { 1321 sprintf(pg_result, 1322 "Flag -:%s:- unknown\nAvailable flags, (prepend ! to un-set flag):\n%s", 1323 f, 1324 "IPSRC_RND, IPDST_RND, UDPSRC_RND, UDPDST_RND, " 1325 "MACSRC_RND, MACDST_RND, TXSIZE_RND, IPV6, " 1326 "MPLS_RND, VID_RND, SVID_RND, FLOW_SEQ, " 1327 "QUEUE_MAP_RND, QUEUE_MAP_CPU, UDPCSUM, " 1328 "NO_TIMESTAMP, " 1329 #ifdef CONFIG_XFRM 1330 "IPSEC, " 1331 #endif 1332 "NODE_ALLOC\n"); 1333 return count; 1334 } 1335 sprintf(pg_result, "OK: flags=0x%x", pkt_dev->flags); 1336 return count; 1337 } 1338 if (!strcmp(name, "dst_min") || !strcmp(name, "dst")) { 1339 len = strn_len(&user_buffer[i], sizeof(pkt_dev->dst_min) - 1); 1340 if (len < 0) 1341 return len; 1342 1343 if (copy_from_user(buf, &user_buffer[i], len)) 1344 return -EFAULT; 1345 buf[len] = 0; 1346 if (strcmp(buf, pkt_dev->dst_min) != 0) { 1347 memset(pkt_dev->dst_min, 0, sizeof(pkt_dev->dst_min)); 1348 strncpy(pkt_dev->dst_min, buf, len); 1349 pkt_dev->daddr_min = in_aton(pkt_dev->dst_min); 1350 pkt_dev->cur_daddr = pkt_dev->daddr_min; 1351 } 1352 if (debug) 1353 pr_debug("dst_min set to: %s\n", pkt_dev->dst_min); 1354 i += len; 1355 sprintf(pg_result, "OK: dst_min=%s", pkt_dev->dst_min); 1356 return count; 1357 } 1358 if (!strcmp(name, "dst_max")) { 1359 len = strn_len(&user_buffer[i], sizeof(pkt_dev->dst_max) - 1); 1360 if (len < 0) 1361 return len; 1362 1363 1364 if (copy_from_user(buf, &user_buffer[i], len)) 1365 return -EFAULT; 1366 1367 buf[len] = 0; 1368 if (strcmp(buf, pkt_dev->dst_max) != 0) { 1369 memset(pkt_dev->dst_max, 0, sizeof(pkt_dev->dst_max)); 1370 strncpy(pkt_dev->dst_max, buf, len); 1371 pkt_dev->daddr_max = in_aton(pkt_dev->dst_max); 1372 pkt_dev->cur_daddr = pkt_dev->daddr_max; 1373 } 1374 if (debug) 1375 pr_debug("dst_max set to: %s\n", pkt_dev->dst_max); 1376 i += len; 1377 sprintf(pg_result, "OK: dst_max=%s", pkt_dev->dst_max); 1378 return count; 1379 } 1380 if (!strcmp(name, "dst6")) { 1381 len = strn_len(&user_buffer[i], sizeof(buf) - 1); 1382 if (len < 0) 1383 return len; 1384 1385 pkt_dev->flags |= F_IPV6; 1386 1387 if (copy_from_user(buf, &user_buffer[i], len)) 1388 return -EFAULT; 1389 buf[len] = 0; 1390 1391 in6_pton(buf, -1, pkt_dev->in6_daddr.s6_addr, -1, NULL); 1392 snprintf(buf, sizeof(buf), "%pI6c", &pkt_dev->in6_daddr); 1393 1394 pkt_dev->cur_in6_daddr = pkt_dev->in6_daddr; 1395 1396 if (debug) 1397 pr_debug("dst6 set to: %s\n", buf); 1398 1399 i += len; 1400 sprintf(pg_result, "OK: dst6=%s", buf); 1401 return count; 1402 } 1403 if (!strcmp(name, "dst6_min")) { 1404 len = strn_len(&user_buffer[i], sizeof(buf) - 1); 1405 if (len < 0) 1406 return len; 1407 1408 pkt_dev->flags |= F_IPV6; 1409 1410 if (copy_from_user(buf, &user_buffer[i], len)) 1411 return -EFAULT; 1412 buf[len] = 0; 1413 1414 in6_pton(buf, -1, pkt_dev->min_in6_daddr.s6_addr, -1, NULL); 1415 snprintf(buf, sizeof(buf), "%pI6c", &pkt_dev->min_in6_daddr); 1416 1417 pkt_dev->cur_in6_daddr = pkt_dev->min_in6_daddr; 1418 if (debug) 1419 pr_debug("dst6_min set to: %s\n", buf); 1420 1421 i += len; 1422 sprintf(pg_result, "OK: dst6_min=%s", buf); 1423 return count; 1424 } 1425 if (!strcmp(name, "dst6_max")) { 1426 len = strn_len(&user_buffer[i], sizeof(buf) - 1); 1427 if (len < 0) 1428 return len; 1429 1430 pkt_dev->flags |= F_IPV6; 1431 1432 if (copy_from_user(buf, &user_buffer[i], len)) 1433 return -EFAULT; 1434 buf[len] = 0; 1435 1436 in6_pton(buf, -1, pkt_dev->max_in6_daddr.s6_addr, -1, NULL); 1437 snprintf(buf, sizeof(buf), "%pI6c", &pkt_dev->max_in6_daddr); 1438 1439 if (debug) 1440 pr_debug("dst6_max set to: %s\n", buf); 1441 1442 i += len; 1443 sprintf(pg_result, "OK: dst6_max=%s", buf); 1444 return count; 1445 } 1446 if (!strcmp(name, "src6")) { 1447 len = strn_len(&user_buffer[i], sizeof(buf) - 1); 1448 if (len < 0) 1449 return len; 1450 1451 pkt_dev->flags |= F_IPV6; 1452 1453 if (copy_from_user(buf, &user_buffer[i], len)) 1454 return -EFAULT; 1455 buf[len] = 0; 1456 1457 in6_pton(buf, -1, pkt_dev->in6_saddr.s6_addr, -1, NULL); 1458 snprintf(buf, sizeof(buf), "%pI6c", &pkt_dev->in6_saddr); 1459 1460 pkt_dev->cur_in6_saddr = pkt_dev->in6_saddr; 1461 1462 if (debug) 1463 pr_debug("src6 set to: %s\n", buf); 1464 1465 i += len; 1466 sprintf(pg_result, "OK: src6=%s", buf); 1467 return count; 1468 } 1469 if (!strcmp(name, "src_min")) { 1470 len = strn_len(&user_buffer[i], sizeof(pkt_dev->src_min) - 1); 1471 if (len < 0) 1472 return len; 1473 1474 if (copy_from_user(buf, &user_buffer[i], len)) 1475 return -EFAULT; 1476 buf[len] = 0; 1477 if (strcmp(buf, pkt_dev->src_min) != 0) { 1478 memset(pkt_dev->src_min, 0, sizeof(pkt_dev->src_min)); 1479 strncpy(pkt_dev->src_min, buf, len); 1480 pkt_dev->saddr_min = in_aton(pkt_dev->src_min); 1481 pkt_dev->cur_saddr = pkt_dev->saddr_min; 1482 } 1483 if (debug) 1484 pr_debug("src_min set to: %s\n", pkt_dev->src_min); 1485 i += len; 1486 sprintf(pg_result, "OK: src_min=%s", pkt_dev->src_min); 1487 return count; 1488 } 1489 if (!strcmp(name, "src_max")) { 1490 len = strn_len(&user_buffer[i], sizeof(pkt_dev->src_max) - 1); 1491 if (len < 0) 1492 return len; 1493 1494 if (copy_from_user(buf, &user_buffer[i], len)) 1495 return -EFAULT; 1496 buf[len] = 0; 1497 if (strcmp(buf, pkt_dev->src_max) != 0) { 1498 memset(pkt_dev->src_max, 0, sizeof(pkt_dev->src_max)); 1499 strncpy(pkt_dev->src_max, buf, len); 1500 pkt_dev->saddr_max = in_aton(pkt_dev->src_max); 1501 pkt_dev->cur_saddr = pkt_dev->saddr_max; 1502 } 1503 if (debug) 1504 pr_debug("src_max set to: %s\n", pkt_dev->src_max); 1505 i += len; 1506 sprintf(pg_result, "OK: src_max=%s", pkt_dev->src_max); 1507 return count; 1508 } 1509 if (!strcmp(name, "dst_mac")) { 1510 len = strn_len(&user_buffer[i], sizeof(valstr) - 1); 1511 if (len < 0) 1512 return len; 1513 1514 memset(valstr, 0, sizeof(valstr)); 1515 if (copy_from_user(valstr, &user_buffer[i], len)) 1516 return -EFAULT; 1517 1518 if (!mac_pton(valstr, pkt_dev->dst_mac)) 1519 return -EINVAL; 1520 /* Set up Dest MAC */ 1521 ether_addr_copy(&pkt_dev->hh[0], pkt_dev->dst_mac); 1522 1523 sprintf(pg_result, "OK: dstmac %pM", pkt_dev->dst_mac); 1524 return count; 1525 } 1526 if (!strcmp(name, "src_mac")) { 1527 len = strn_len(&user_buffer[i], sizeof(valstr) - 1); 1528 if (len < 0) 1529 return len; 1530 1531 memset(valstr, 0, sizeof(valstr)); 1532 if (copy_from_user(valstr, &user_buffer[i], len)) 1533 return -EFAULT; 1534 1535 if (!mac_pton(valstr, pkt_dev->src_mac)) 1536 return -EINVAL; 1537 /* Set up Src MAC */ 1538 ether_addr_copy(&pkt_dev->hh[6], pkt_dev->src_mac); 1539 1540 sprintf(pg_result, "OK: srcmac %pM", pkt_dev->src_mac); 1541 return count; 1542 } 1543 1544 if (!strcmp(name, "clear_counters")) { 1545 pktgen_clear_counters(pkt_dev); 1546 sprintf(pg_result, "OK: Clearing counters.\n"); 1547 return count; 1548 } 1549 1550 if (!strcmp(name, "flows")) { 1551 len = num_arg(&user_buffer[i], 10, &value); 1552 if (len < 0) 1553 return len; 1554 1555 i += len; 1556 if (value > MAX_CFLOWS) 1557 value = MAX_CFLOWS; 1558 1559 pkt_dev->cflows = value; 1560 sprintf(pg_result, "OK: flows=%u", pkt_dev->cflows); 1561 return count; 1562 } 1563 #ifdef CONFIG_XFRM 1564 if (!strcmp(name, "spi")) { 1565 len = num_arg(&user_buffer[i], 10, &value); 1566 if (len < 0) 1567 return len; 1568 1569 i += len; 1570 pkt_dev->spi = value; 1571 sprintf(pg_result, "OK: spi=%u", pkt_dev->spi); 1572 return count; 1573 } 1574 #endif 1575 if (!strcmp(name, "flowlen")) { 1576 len = num_arg(&user_buffer[i], 10, &value); 1577 if (len < 0) 1578 return len; 1579 1580 i += len; 1581 pkt_dev->lflow = value; 1582 sprintf(pg_result, "OK: flowlen=%u", pkt_dev->lflow); 1583 return count; 1584 } 1585 1586 if (!strcmp(name, "queue_map_min")) { 1587 len = num_arg(&user_buffer[i], 5, &value); 1588 if (len < 0) 1589 return len; 1590 1591 i += len; 1592 pkt_dev->queue_map_min = value; 1593 sprintf(pg_result, "OK: queue_map_min=%u", pkt_dev->queue_map_min); 1594 return count; 1595 } 1596 1597 if (!strcmp(name, "queue_map_max")) { 1598 len = num_arg(&user_buffer[i], 5, &value); 1599 if (len < 0) 1600 return len; 1601 1602 i += len; 1603 pkt_dev->queue_map_max = value; 1604 sprintf(pg_result, "OK: queue_map_max=%u", pkt_dev->queue_map_max); 1605 return count; 1606 } 1607 1608 if (!strcmp(name, "mpls")) { 1609 unsigned int n, cnt; 1610 1611 len = get_labels(&user_buffer[i], pkt_dev); 1612 if (len < 0) 1613 return len; 1614 i += len; 1615 cnt = sprintf(pg_result, "OK: mpls="); 1616 for (n = 0; n < pkt_dev->nr_labels; n++) 1617 cnt += sprintf(pg_result + cnt, 1618 "%08x%s", ntohl(pkt_dev->labels[n]), 1619 n == pkt_dev->nr_labels-1 ? "" : ","); 1620 1621 if (pkt_dev->nr_labels && pkt_dev->vlan_id != 0xffff) { 1622 pkt_dev->vlan_id = 0xffff; /* turn off VLAN/SVLAN */ 1623 pkt_dev->svlan_id = 0xffff; 1624 1625 if (debug) 1626 pr_debug("VLAN/SVLAN auto turned off\n"); 1627 } 1628 return count; 1629 } 1630 1631 if (!strcmp(name, "vlan_id")) { 1632 len = num_arg(&user_buffer[i], 4, &value); 1633 if (len < 0) 1634 return len; 1635 1636 i += len; 1637 if (value <= 4095) { 1638 pkt_dev->vlan_id = value; /* turn on VLAN */ 1639 1640 if (debug) 1641 pr_debug("VLAN turned on\n"); 1642 1643 if (debug && pkt_dev->nr_labels) 1644 pr_debug("MPLS auto turned off\n"); 1645 1646 pkt_dev->nr_labels = 0; /* turn off MPLS */ 1647 sprintf(pg_result, "OK: vlan_id=%u", pkt_dev->vlan_id); 1648 } else { 1649 pkt_dev->vlan_id = 0xffff; /* turn off VLAN/SVLAN */ 1650 pkt_dev->svlan_id = 0xffff; 1651 1652 if (debug) 1653 pr_debug("VLAN/SVLAN turned off\n"); 1654 } 1655 return count; 1656 } 1657 1658 if (!strcmp(name, "vlan_p")) { 1659 len = num_arg(&user_buffer[i], 1, &value); 1660 if (len < 0) 1661 return len; 1662 1663 i += len; 1664 if ((value <= 7) && (pkt_dev->vlan_id != 0xffff)) { 1665 pkt_dev->vlan_p = value; 1666 sprintf(pg_result, "OK: vlan_p=%u", pkt_dev->vlan_p); 1667 } else { 1668 sprintf(pg_result, "ERROR: vlan_p must be 0-7"); 1669 } 1670 return count; 1671 } 1672 1673 if (!strcmp(name, "vlan_cfi")) { 1674 len = num_arg(&user_buffer[i], 1, &value); 1675 if (len < 0) 1676 return len; 1677 1678 i += len; 1679 if ((value <= 1) && (pkt_dev->vlan_id != 0xffff)) { 1680 pkt_dev->vlan_cfi = value; 1681 sprintf(pg_result, "OK: vlan_cfi=%u", pkt_dev->vlan_cfi); 1682 } else { 1683 sprintf(pg_result, "ERROR: vlan_cfi must be 0-1"); 1684 } 1685 return count; 1686 } 1687 1688 if (!strcmp(name, "svlan_id")) { 1689 len = num_arg(&user_buffer[i], 4, &value); 1690 if (len < 0) 1691 return len; 1692 1693 i += len; 1694 if ((value <= 4095) && ((pkt_dev->vlan_id != 0xffff))) { 1695 pkt_dev->svlan_id = value; /* turn on SVLAN */ 1696 1697 if (debug) 1698 pr_debug("SVLAN turned on\n"); 1699 1700 if (debug && pkt_dev->nr_labels) 1701 pr_debug("MPLS auto turned off\n"); 1702 1703 pkt_dev->nr_labels = 0; /* turn off MPLS */ 1704 sprintf(pg_result, "OK: svlan_id=%u", pkt_dev->svlan_id); 1705 } else { 1706 pkt_dev->vlan_id = 0xffff; /* turn off VLAN/SVLAN */ 1707 pkt_dev->svlan_id = 0xffff; 1708 1709 if (debug) 1710 pr_debug("VLAN/SVLAN turned off\n"); 1711 } 1712 return count; 1713 } 1714 1715 if (!strcmp(name, "svlan_p")) { 1716 len = num_arg(&user_buffer[i], 1, &value); 1717 if (len < 0) 1718 return len; 1719 1720 i += len; 1721 if ((value <= 7) && (pkt_dev->svlan_id != 0xffff)) { 1722 pkt_dev->svlan_p = value; 1723 sprintf(pg_result, "OK: svlan_p=%u", pkt_dev->svlan_p); 1724 } else { 1725 sprintf(pg_result, "ERROR: svlan_p must be 0-7"); 1726 } 1727 return count; 1728 } 1729 1730 if (!strcmp(name, "svlan_cfi")) { 1731 len = num_arg(&user_buffer[i], 1, &value); 1732 if (len < 0) 1733 return len; 1734 1735 i += len; 1736 if ((value <= 1) && (pkt_dev->svlan_id != 0xffff)) { 1737 pkt_dev->svlan_cfi = value; 1738 sprintf(pg_result, "OK: svlan_cfi=%u", pkt_dev->svlan_cfi); 1739 } else { 1740 sprintf(pg_result, "ERROR: svlan_cfi must be 0-1"); 1741 } 1742 return count; 1743 } 1744 1745 if (!strcmp(name, "tos")) { 1746 __u32 tmp_value = 0; 1747 len = hex32_arg(&user_buffer[i], 2, &tmp_value); 1748 if (len < 0) 1749 return len; 1750 1751 i += len; 1752 if (len == 2) { 1753 pkt_dev->tos = tmp_value; 1754 sprintf(pg_result, "OK: tos=0x%02x", pkt_dev->tos); 1755 } else { 1756 sprintf(pg_result, "ERROR: tos must be 00-ff"); 1757 } 1758 return count; 1759 } 1760 1761 if (!strcmp(name, "traffic_class")) { 1762 __u32 tmp_value = 0; 1763 len = hex32_arg(&user_buffer[i], 2, &tmp_value); 1764 if (len < 0) 1765 return len; 1766 1767 i += len; 1768 if (len == 2) { 1769 pkt_dev->traffic_class = tmp_value; 1770 sprintf(pg_result, "OK: traffic_class=0x%02x", pkt_dev->traffic_class); 1771 } else { 1772 sprintf(pg_result, "ERROR: traffic_class must be 00-ff"); 1773 } 1774 return count; 1775 } 1776 1777 if (!strcmp(name, "skb_priority")) { 1778 len = num_arg(&user_buffer[i], 9, &value); 1779 if (len < 0) 1780 return len; 1781 1782 i += len; 1783 pkt_dev->skb_priority = value; 1784 sprintf(pg_result, "OK: skb_priority=%i", 1785 pkt_dev->skb_priority); 1786 return count; 1787 } 1788 1789 sprintf(pkt_dev->result, "No such parameter \"%s\"", name); 1790 return -EINVAL; 1791 } 1792 1793 static int pktgen_if_open(struct inode *inode, struct file *file) 1794 { 1795 return single_open(file, pktgen_if_show, PDE_DATA(inode)); 1796 } 1797 1798 static const struct file_operations pktgen_if_fops = { 1799 .owner = THIS_MODULE, 1800 .open = pktgen_if_open, 1801 .read = seq_read, 1802 .llseek = seq_lseek, 1803 .write = pktgen_if_write, 1804 .release = single_release, 1805 }; 1806 1807 static int pktgen_thread_show(struct seq_file *seq, void *v) 1808 { 1809 struct pktgen_thread *t = seq->private; 1810 const struct pktgen_dev *pkt_dev; 1811 1812 BUG_ON(!t); 1813 1814 seq_puts(seq, "Running: "); 1815 1816 rcu_read_lock(); 1817 list_for_each_entry_rcu(pkt_dev, &t->if_list, list) 1818 if (pkt_dev->running) 1819 seq_printf(seq, "%s ", pkt_dev->odevname); 1820 1821 seq_puts(seq, "\nStopped: "); 1822 1823 list_for_each_entry_rcu(pkt_dev, &t->if_list, list) 1824 if (!pkt_dev->running) 1825 seq_printf(seq, "%s ", pkt_dev->odevname); 1826 1827 if (t->result[0]) 1828 seq_printf(seq, "\nResult: %s\n", t->result); 1829 else 1830 seq_puts(seq, "\nResult: NA\n"); 1831 1832 rcu_read_unlock(); 1833 1834 return 0; 1835 } 1836 1837 static ssize_t pktgen_thread_write(struct file *file, 1838 const char __user * user_buffer, 1839 size_t count, loff_t * offset) 1840 { 1841 struct seq_file *seq = file->private_data; 1842 struct pktgen_thread *t = seq->private; 1843 int i, max, len, ret; 1844 char name[40]; 1845 char *pg_result; 1846 1847 if (count < 1) { 1848 // sprintf(pg_result, "Wrong command format"); 1849 return -EINVAL; 1850 } 1851 1852 max = count; 1853 len = count_trail_chars(user_buffer, max); 1854 if (len < 0) 1855 return len; 1856 1857 i = len; 1858 1859 /* Read variable name */ 1860 1861 len = strn_len(&user_buffer[i], sizeof(name) - 1); 1862 if (len < 0) 1863 return len; 1864 1865 memset(name, 0, sizeof(name)); 1866 if (copy_from_user(name, &user_buffer[i], len)) 1867 return -EFAULT; 1868 i += len; 1869 1870 max = count - i; 1871 len = count_trail_chars(&user_buffer[i], max); 1872 if (len < 0) 1873 return len; 1874 1875 i += len; 1876 1877 if (debug) 1878 pr_debug("t=%s, count=%lu\n", name, (unsigned long)count); 1879 1880 if (!t) { 1881 pr_err("ERROR: No thread\n"); 1882 ret = -EINVAL; 1883 goto out; 1884 } 1885 1886 pg_result = &(t->result[0]); 1887 1888 if (!strcmp(name, "add_device")) { 1889 char f[32]; 1890 memset(f, 0, 32); 1891 len = strn_len(&user_buffer[i], sizeof(f) - 1); 1892 if (len < 0) { 1893 ret = len; 1894 goto out; 1895 } 1896 if (copy_from_user(f, &user_buffer[i], len)) 1897 return -EFAULT; 1898 i += len; 1899 mutex_lock(&pktgen_thread_lock); 1900 ret = pktgen_add_device(t, f); 1901 mutex_unlock(&pktgen_thread_lock); 1902 if (!ret) { 1903 ret = count; 1904 sprintf(pg_result, "OK: add_device=%s", f); 1905 } else 1906 sprintf(pg_result, "ERROR: can not add device %s", f); 1907 goto out; 1908 } 1909 1910 if (!strcmp(name, "rem_device_all")) { 1911 mutex_lock(&pktgen_thread_lock); 1912 t->control |= T_REMDEVALL; 1913 mutex_unlock(&pktgen_thread_lock); 1914 schedule_timeout_interruptible(msecs_to_jiffies(125)); /* Propagate thread->control */ 1915 ret = count; 1916 sprintf(pg_result, "OK: rem_device_all"); 1917 goto out; 1918 } 1919 1920 if (!strcmp(name, "max_before_softirq")) { 1921 sprintf(pg_result, "OK: Note! max_before_softirq is obsoleted -- Do not use"); 1922 ret = count; 1923 goto out; 1924 } 1925 1926 ret = -EINVAL; 1927 out: 1928 return ret; 1929 } 1930 1931 static int pktgen_thread_open(struct inode *inode, struct file *file) 1932 { 1933 return single_open(file, pktgen_thread_show, PDE_DATA(inode)); 1934 } 1935 1936 static const struct file_operations pktgen_thread_fops = { 1937 .owner = THIS_MODULE, 1938 .open = pktgen_thread_open, 1939 .read = seq_read, 1940 .llseek = seq_lseek, 1941 .write = pktgen_thread_write, 1942 .release = single_release, 1943 }; 1944 1945 /* Think find or remove for NN */ 1946 static struct pktgen_dev *__pktgen_NN_threads(const struct pktgen_net *pn, 1947 const char *ifname, int remove) 1948 { 1949 struct pktgen_thread *t; 1950 struct pktgen_dev *pkt_dev = NULL; 1951 bool exact = (remove == FIND); 1952 1953 list_for_each_entry(t, &pn->pktgen_threads, th_list) { 1954 pkt_dev = pktgen_find_dev(t, ifname, exact); 1955 if (pkt_dev) { 1956 if (remove) { 1957 pkt_dev->removal_mark = 1; 1958 t->control |= T_REMDEV; 1959 } 1960 break; 1961 } 1962 } 1963 return pkt_dev; 1964 } 1965 1966 /* 1967 * mark a device for removal 1968 */ 1969 static void pktgen_mark_device(const struct pktgen_net *pn, const char *ifname) 1970 { 1971 struct pktgen_dev *pkt_dev = NULL; 1972 const int max_tries = 10, msec_per_try = 125; 1973 int i = 0; 1974 1975 mutex_lock(&pktgen_thread_lock); 1976 pr_debug("%s: marking %s for removal\n", __func__, ifname); 1977 1978 while (1) { 1979 1980 pkt_dev = __pktgen_NN_threads(pn, ifname, REMOVE); 1981 if (pkt_dev == NULL) 1982 break; /* success */ 1983 1984 mutex_unlock(&pktgen_thread_lock); 1985 pr_debug("%s: waiting for %s to disappear....\n", 1986 __func__, ifname); 1987 schedule_timeout_interruptible(msecs_to_jiffies(msec_per_try)); 1988 mutex_lock(&pktgen_thread_lock); 1989 1990 if (++i >= max_tries) { 1991 pr_err("%s: timed out after waiting %d msec for device %s to be removed\n", 1992 __func__, msec_per_try * i, ifname); 1993 break; 1994 } 1995 1996 } 1997 1998 mutex_unlock(&pktgen_thread_lock); 1999 } 2000 2001 static void pktgen_change_name(const struct pktgen_net *pn, struct net_device *dev) 2002 { 2003 struct pktgen_thread *t; 2004 2005 list_for_each_entry(t, &pn->pktgen_threads, th_list) { 2006 struct pktgen_dev *pkt_dev; 2007 2008 rcu_read_lock(); 2009 list_for_each_entry_rcu(pkt_dev, &t->if_list, list) { 2010 if (pkt_dev->odev != dev) 2011 continue; 2012 2013 proc_remove(pkt_dev->entry); 2014 2015 pkt_dev->entry = proc_create_data(dev->name, 0600, 2016 pn->proc_dir, 2017 &pktgen_if_fops, 2018 pkt_dev); 2019 if (!pkt_dev->entry) 2020 pr_err("can't move proc entry for '%s'\n", 2021 dev->name); 2022 break; 2023 } 2024 rcu_read_unlock(); 2025 } 2026 } 2027 2028 static int pktgen_device_event(struct notifier_block *unused, 2029 unsigned long event, void *ptr) 2030 { 2031 struct net_device *dev = netdev_notifier_info_to_dev(ptr); 2032 struct pktgen_net *pn = net_generic(dev_net(dev), pg_net_id); 2033 2034 if (pn->pktgen_exiting) 2035 return NOTIFY_DONE; 2036 2037 /* It is OK that we do not hold the group lock right now, 2038 * as we run under the RTNL lock. 2039 */ 2040 2041 switch (event) { 2042 case NETDEV_CHANGENAME: 2043 pktgen_change_name(pn, dev); 2044 break; 2045 2046 case NETDEV_UNREGISTER: 2047 pktgen_mark_device(pn, dev->name); 2048 break; 2049 } 2050 2051 return NOTIFY_DONE; 2052 } 2053 2054 static struct net_device *pktgen_dev_get_by_name(const struct pktgen_net *pn, 2055 struct pktgen_dev *pkt_dev, 2056 const char *ifname) 2057 { 2058 char b[IFNAMSIZ+5]; 2059 int i; 2060 2061 for (i = 0; ifname[i] != '@'; i++) { 2062 if (i == IFNAMSIZ) 2063 break; 2064 2065 b[i] = ifname[i]; 2066 } 2067 b[i] = 0; 2068 2069 return dev_get_by_name(pn->net, b); 2070 } 2071 2072 2073 /* Associate pktgen_dev with a device. */ 2074 2075 static int pktgen_setup_dev(const struct pktgen_net *pn, 2076 struct pktgen_dev *pkt_dev, const char *ifname) 2077 { 2078 struct net_device *odev; 2079 int err; 2080 2081 /* Clean old setups */ 2082 if (pkt_dev->odev) { 2083 dev_put(pkt_dev->odev); 2084 pkt_dev->odev = NULL; 2085 } 2086 2087 odev = pktgen_dev_get_by_name(pn, pkt_dev, ifname); 2088 if (!odev) { 2089 pr_err("no such netdevice: \"%s\"\n", ifname); 2090 return -ENODEV; 2091 } 2092 2093 if (odev->type != ARPHRD_ETHER) { 2094 pr_err("not an ethernet device: \"%s\"\n", ifname); 2095 err = -EINVAL; 2096 } else if (!netif_running(odev)) { 2097 pr_err("device is down: \"%s\"\n", ifname); 2098 err = -ENETDOWN; 2099 } else { 2100 pkt_dev->odev = odev; 2101 return 0; 2102 } 2103 2104 dev_put(odev); 2105 return err; 2106 } 2107 2108 /* Read pkt_dev from the interface and set up internal pktgen_dev 2109 * structure to have the right information to create/send packets 2110 */ 2111 static void pktgen_setup_inject(struct pktgen_dev *pkt_dev) 2112 { 2113 int ntxq; 2114 2115 if (!pkt_dev->odev) { 2116 pr_err("ERROR: pkt_dev->odev == NULL in setup_inject\n"); 2117 sprintf(pkt_dev->result, 2118 "ERROR: pkt_dev->odev == NULL in setup_inject.\n"); 2119 return; 2120 } 2121 2122 /* make sure that we don't pick a non-existing transmit queue */ 2123 ntxq = pkt_dev->odev->real_num_tx_queues; 2124 2125 if (ntxq <= pkt_dev->queue_map_min) { 2126 pr_warn("WARNING: Requested queue_map_min (zero-based) (%d) exceeds valid range [0 - %d] for (%d) queues on %s, resetting\n", 2127 pkt_dev->queue_map_min, (ntxq ?: 1) - 1, ntxq, 2128 pkt_dev->odevname); 2129 pkt_dev->queue_map_min = (ntxq ?: 1) - 1; 2130 } 2131 if (pkt_dev->queue_map_max >= ntxq) { 2132 pr_warn("WARNING: Requested queue_map_max (zero-based) (%d) exceeds valid range [0 - %d] for (%d) queues on %s, resetting\n", 2133 pkt_dev->queue_map_max, (ntxq ?: 1) - 1, ntxq, 2134 pkt_dev->odevname); 2135 pkt_dev->queue_map_max = (ntxq ?: 1) - 1; 2136 } 2137 2138 /* Default to the interface's mac if not explicitly set. */ 2139 2140 if (is_zero_ether_addr(pkt_dev->src_mac)) 2141 ether_addr_copy(&(pkt_dev->hh[6]), pkt_dev->odev->dev_addr); 2142 2143 /* Set up Dest MAC */ 2144 ether_addr_copy(&(pkt_dev->hh[0]), pkt_dev->dst_mac); 2145 2146 if (pkt_dev->flags & F_IPV6) { 2147 int i, set = 0, err = 1; 2148 struct inet6_dev *idev; 2149 2150 if (pkt_dev->min_pkt_size == 0) { 2151 pkt_dev->min_pkt_size = 14 + sizeof(struct ipv6hdr) 2152 + sizeof(struct udphdr) 2153 + sizeof(struct pktgen_hdr) 2154 + pkt_dev->pkt_overhead; 2155 } 2156 2157 for (i = 0; i < IN6_ADDR_HSIZE; i++) 2158 if (pkt_dev->cur_in6_saddr.s6_addr[i]) { 2159 set = 1; 2160 break; 2161 } 2162 2163 if (!set) { 2164 2165 /* 2166 * Use linklevel address if unconfigured. 2167 * 2168 * use ipv6_get_lladdr if/when it's get exported 2169 */ 2170 2171 rcu_read_lock(); 2172 idev = __in6_dev_get(pkt_dev->odev); 2173 if (idev) { 2174 struct inet6_ifaddr *ifp; 2175 2176 read_lock_bh(&idev->lock); 2177 list_for_each_entry(ifp, &idev->addr_list, if_list) { 2178 if ((ifp->scope & IFA_LINK) && 2179 !(ifp->flags & IFA_F_TENTATIVE)) { 2180 pkt_dev->cur_in6_saddr = ifp->addr; 2181 err = 0; 2182 break; 2183 } 2184 } 2185 read_unlock_bh(&idev->lock); 2186 } 2187 rcu_read_unlock(); 2188 if (err) 2189 pr_err("ERROR: IPv6 link address not available\n"); 2190 } 2191 } else { 2192 if (pkt_dev->min_pkt_size == 0) { 2193 pkt_dev->min_pkt_size = 14 + sizeof(struct iphdr) 2194 + sizeof(struct udphdr) 2195 + sizeof(struct pktgen_hdr) 2196 + pkt_dev->pkt_overhead; 2197 } 2198 2199 pkt_dev->saddr_min = 0; 2200 pkt_dev->saddr_max = 0; 2201 if (strlen(pkt_dev->src_min) == 0) { 2202 2203 struct in_device *in_dev; 2204 2205 rcu_read_lock(); 2206 in_dev = __in_dev_get_rcu(pkt_dev->odev); 2207 if (in_dev) { 2208 if (in_dev->ifa_list) { 2209 pkt_dev->saddr_min = 2210 in_dev->ifa_list->ifa_address; 2211 pkt_dev->saddr_max = pkt_dev->saddr_min; 2212 } 2213 } 2214 rcu_read_unlock(); 2215 } else { 2216 pkt_dev->saddr_min = in_aton(pkt_dev->src_min); 2217 pkt_dev->saddr_max = in_aton(pkt_dev->src_max); 2218 } 2219 2220 pkt_dev->daddr_min = in_aton(pkt_dev->dst_min); 2221 pkt_dev->daddr_max = in_aton(pkt_dev->dst_max); 2222 } 2223 /* Initialize current values. */ 2224 pkt_dev->cur_pkt_size = pkt_dev->min_pkt_size; 2225 if (pkt_dev->min_pkt_size > pkt_dev->max_pkt_size) 2226 pkt_dev->max_pkt_size = pkt_dev->min_pkt_size; 2227 2228 pkt_dev->cur_dst_mac_offset = 0; 2229 pkt_dev->cur_src_mac_offset = 0; 2230 pkt_dev->cur_saddr = pkt_dev->saddr_min; 2231 pkt_dev->cur_daddr = pkt_dev->daddr_min; 2232 pkt_dev->cur_udp_dst = pkt_dev->udp_dst_min; 2233 pkt_dev->cur_udp_src = pkt_dev->udp_src_min; 2234 pkt_dev->nflows = 0; 2235 } 2236 2237 2238 static void spin(struct pktgen_dev *pkt_dev, ktime_t spin_until) 2239 { 2240 ktime_t start_time, end_time; 2241 s64 remaining; 2242 struct hrtimer_sleeper t; 2243 2244 hrtimer_init_on_stack(&t.timer, CLOCK_MONOTONIC, HRTIMER_MODE_ABS); 2245 hrtimer_set_expires(&t.timer, spin_until); 2246 2247 remaining = ktime_to_ns(hrtimer_expires_remaining(&t.timer)); 2248 if (remaining <= 0) { 2249 pkt_dev->next_tx = ktime_add_ns(spin_until, pkt_dev->delay); 2250 return; 2251 } 2252 2253 start_time = ktime_get(); 2254 if (remaining < 100000) { 2255 /* for small delays (<100us), just loop until limit is reached */ 2256 do { 2257 end_time = ktime_get(); 2258 } while (ktime_compare(end_time, spin_until) < 0); 2259 } else { 2260 /* see do_nanosleep */ 2261 hrtimer_init_sleeper(&t, current); 2262 do { 2263 set_current_state(TASK_INTERRUPTIBLE); 2264 hrtimer_start_expires(&t.timer, HRTIMER_MODE_ABS); 2265 2266 if (likely(t.task)) 2267 schedule(); 2268 2269 hrtimer_cancel(&t.timer); 2270 } while (t.task && pkt_dev->running && !signal_pending(current)); 2271 __set_current_state(TASK_RUNNING); 2272 end_time = ktime_get(); 2273 } 2274 2275 pkt_dev->idle_acc += ktime_to_ns(ktime_sub(end_time, start_time)); 2276 pkt_dev->next_tx = ktime_add_ns(spin_until, pkt_dev->delay); 2277 } 2278 2279 static inline void set_pkt_overhead(struct pktgen_dev *pkt_dev) 2280 { 2281 pkt_dev->pkt_overhead = LL_RESERVED_SPACE(pkt_dev->odev); 2282 pkt_dev->pkt_overhead += pkt_dev->nr_labels*sizeof(u32); 2283 pkt_dev->pkt_overhead += VLAN_TAG_SIZE(pkt_dev); 2284 pkt_dev->pkt_overhead += SVLAN_TAG_SIZE(pkt_dev); 2285 } 2286 2287 static inline int f_seen(const struct pktgen_dev *pkt_dev, int flow) 2288 { 2289 return !!(pkt_dev->flows[flow].flags & F_INIT); 2290 } 2291 2292 static inline int f_pick(struct pktgen_dev *pkt_dev) 2293 { 2294 int flow = pkt_dev->curfl; 2295 2296 if (pkt_dev->flags & F_FLOW_SEQ) { 2297 if (pkt_dev->flows[flow].count >= pkt_dev->lflow) { 2298 /* reset time */ 2299 pkt_dev->flows[flow].count = 0; 2300 pkt_dev->flows[flow].flags = 0; 2301 pkt_dev->curfl += 1; 2302 if (pkt_dev->curfl >= pkt_dev->cflows) 2303 pkt_dev->curfl = 0; /*reset */ 2304 } 2305 } else { 2306 flow = prandom_u32() % pkt_dev->cflows; 2307 pkt_dev->curfl = flow; 2308 2309 if (pkt_dev->flows[flow].count > pkt_dev->lflow) { 2310 pkt_dev->flows[flow].count = 0; 2311 pkt_dev->flows[flow].flags = 0; 2312 } 2313 } 2314 2315 return pkt_dev->curfl; 2316 } 2317 2318 2319 #ifdef CONFIG_XFRM 2320 /* If there was already an IPSEC SA, we keep it as is, else 2321 * we go look for it ... 2322 */ 2323 #define DUMMY_MARK 0 2324 static void get_ipsec_sa(struct pktgen_dev *pkt_dev, int flow) 2325 { 2326 struct xfrm_state *x = pkt_dev->flows[flow].x; 2327 struct pktgen_net *pn = net_generic(dev_net(pkt_dev->odev), pg_net_id); 2328 if (!x) { 2329 2330 if (pkt_dev->spi) { 2331 /* We need as quick as possible to find the right SA 2332 * Searching with minimum criteria to archieve this. 2333 */ 2334 x = xfrm_state_lookup_byspi(pn->net, htonl(pkt_dev->spi), AF_INET); 2335 } else { 2336 /* slow path: we dont already have xfrm_state */ 2337 x = xfrm_stateonly_find(pn->net, DUMMY_MARK, 2338 (xfrm_address_t *)&pkt_dev->cur_daddr, 2339 (xfrm_address_t *)&pkt_dev->cur_saddr, 2340 AF_INET, 2341 pkt_dev->ipsmode, 2342 pkt_dev->ipsproto, 0); 2343 } 2344 if (x) { 2345 pkt_dev->flows[flow].x = x; 2346 set_pkt_overhead(pkt_dev); 2347 pkt_dev->pkt_overhead += x->props.header_len; 2348 } 2349 2350 } 2351 } 2352 #endif 2353 static void set_cur_queue_map(struct pktgen_dev *pkt_dev) 2354 { 2355 2356 if (pkt_dev->flags & F_QUEUE_MAP_CPU) 2357 pkt_dev->cur_queue_map = smp_processor_id(); 2358 2359 else if (pkt_dev->queue_map_min <= pkt_dev->queue_map_max) { 2360 __u16 t; 2361 if (pkt_dev->flags & F_QUEUE_MAP_RND) { 2362 t = prandom_u32() % 2363 (pkt_dev->queue_map_max - 2364 pkt_dev->queue_map_min + 1) 2365 + pkt_dev->queue_map_min; 2366 } else { 2367 t = pkt_dev->cur_queue_map + 1; 2368 if (t > pkt_dev->queue_map_max) 2369 t = pkt_dev->queue_map_min; 2370 } 2371 pkt_dev->cur_queue_map = t; 2372 } 2373 pkt_dev->cur_queue_map = pkt_dev->cur_queue_map % pkt_dev->odev->real_num_tx_queues; 2374 } 2375 2376 /* Increment/randomize headers according to flags and current values 2377 * for IP src/dest, UDP src/dst port, MAC-Addr src/dst 2378 */ 2379 static void mod_cur_headers(struct pktgen_dev *pkt_dev) 2380 { 2381 __u32 imn; 2382 __u32 imx; 2383 int flow = 0; 2384 2385 if (pkt_dev->cflows) 2386 flow = f_pick(pkt_dev); 2387 2388 /* Deal with source MAC */ 2389 if (pkt_dev->src_mac_count > 1) { 2390 __u32 mc; 2391 __u32 tmp; 2392 2393 if (pkt_dev->flags & F_MACSRC_RND) 2394 mc = prandom_u32() % pkt_dev->src_mac_count; 2395 else { 2396 mc = pkt_dev->cur_src_mac_offset++; 2397 if (pkt_dev->cur_src_mac_offset >= 2398 pkt_dev->src_mac_count) 2399 pkt_dev->cur_src_mac_offset = 0; 2400 } 2401 2402 tmp = pkt_dev->src_mac[5] + (mc & 0xFF); 2403 pkt_dev->hh[11] = tmp; 2404 tmp = (pkt_dev->src_mac[4] + ((mc >> 8) & 0xFF) + (tmp >> 8)); 2405 pkt_dev->hh[10] = tmp; 2406 tmp = (pkt_dev->src_mac[3] + ((mc >> 16) & 0xFF) + (tmp >> 8)); 2407 pkt_dev->hh[9] = tmp; 2408 tmp = (pkt_dev->src_mac[2] + ((mc >> 24) & 0xFF) + (tmp >> 8)); 2409 pkt_dev->hh[8] = tmp; 2410 tmp = (pkt_dev->src_mac[1] + (tmp >> 8)); 2411 pkt_dev->hh[7] = tmp; 2412 } 2413 2414 /* Deal with Destination MAC */ 2415 if (pkt_dev->dst_mac_count > 1) { 2416 __u32 mc; 2417 __u32 tmp; 2418 2419 if (pkt_dev->flags & F_MACDST_RND) 2420 mc = prandom_u32() % pkt_dev->dst_mac_count; 2421 2422 else { 2423 mc = pkt_dev->cur_dst_mac_offset++; 2424 if (pkt_dev->cur_dst_mac_offset >= 2425 pkt_dev->dst_mac_count) { 2426 pkt_dev->cur_dst_mac_offset = 0; 2427 } 2428 } 2429 2430 tmp = pkt_dev->dst_mac[5] + (mc & 0xFF); 2431 pkt_dev->hh[5] = tmp; 2432 tmp = (pkt_dev->dst_mac[4] + ((mc >> 8) & 0xFF) + (tmp >> 8)); 2433 pkt_dev->hh[4] = tmp; 2434 tmp = (pkt_dev->dst_mac[3] + ((mc >> 16) & 0xFF) + (tmp >> 8)); 2435 pkt_dev->hh[3] = tmp; 2436 tmp = (pkt_dev->dst_mac[2] + ((mc >> 24) & 0xFF) + (tmp >> 8)); 2437 pkt_dev->hh[2] = tmp; 2438 tmp = (pkt_dev->dst_mac[1] + (tmp >> 8)); 2439 pkt_dev->hh[1] = tmp; 2440 } 2441 2442 if (pkt_dev->flags & F_MPLS_RND) { 2443 unsigned int i; 2444 for (i = 0; i < pkt_dev->nr_labels; i++) 2445 if (pkt_dev->labels[i] & MPLS_STACK_BOTTOM) 2446 pkt_dev->labels[i] = MPLS_STACK_BOTTOM | 2447 ((__force __be32)prandom_u32() & 2448 htonl(0x000fffff)); 2449 } 2450 2451 if ((pkt_dev->flags & F_VID_RND) && (pkt_dev->vlan_id != 0xffff)) { 2452 pkt_dev->vlan_id = prandom_u32() & (4096 - 1); 2453 } 2454 2455 if ((pkt_dev->flags & F_SVID_RND) && (pkt_dev->svlan_id != 0xffff)) { 2456 pkt_dev->svlan_id = prandom_u32() & (4096 - 1); 2457 } 2458 2459 if (pkt_dev->udp_src_min < pkt_dev->udp_src_max) { 2460 if (pkt_dev->flags & F_UDPSRC_RND) 2461 pkt_dev->cur_udp_src = prandom_u32() % 2462 (pkt_dev->udp_src_max - pkt_dev->udp_src_min) 2463 + pkt_dev->udp_src_min; 2464 2465 else { 2466 pkt_dev->cur_udp_src++; 2467 if (pkt_dev->cur_udp_src >= pkt_dev->udp_src_max) 2468 pkt_dev->cur_udp_src = pkt_dev->udp_src_min; 2469 } 2470 } 2471 2472 if (pkt_dev->udp_dst_min < pkt_dev->udp_dst_max) { 2473 if (pkt_dev->flags & F_UDPDST_RND) { 2474 pkt_dev->cur_udp_dst = prandom_u32() % 2475 (pkt_dev->udp_dst_max - pkt_dev->udp_dst_min) 2476 + pkt_dev->udp_dst_min; 2477 } else { 2478 pkt_dev->cur_udp_dst++; 2479 if (pkt_dev->cur_udp_dst >= pkt_dev->udp_dst_max) 2480 pkt_dev->cur_udp_dst = pkt_dev->udp_dst_min; 2481 } 2482 } 2483 2484 if (!(pkt_dev->flags & F_IPV6)) { 2485 2486 imn = ntohl(pkt_dev->saddr_min); 2487 imx = ntohl(pkt_dev->saddr_max); 2488 if (imn < imx) { 2489 __u32 t; 2490 if (pkt_dev->flags & F_IPSRC_RND) 2491 t = prandom_u32() % (imx - imn) + imn; 2492 else { 2493 t = ntohl(pkt_dev->cur_saddr); 2494 t++; 2495 if (t > imx) 2496 t = imn; 2497 2498 } 2499 pkt_dev->cur_saddr = htonl(t); 2500 } 2501 2502 if (pkt_dev->cflows && f_seen(pkt_dev, flow)) { 2503 pkt_dev->cur_daddr = pkt_dev->flows[flow].cur_daddr; 2504 } else { 2505 imn = ntohl(pkt_dev->daddr_min); 2506 imx = ntohl(pkt_dev->daddr_max); 2507 if (imn < imx) { 2508 __u32 t; 2509 __be32 s; 2510 if (pkt_dev->flags & F_IPDST_RND) { 2511 2512 do { 2513 t = prandom_u32() % 2514 (imx - imn) + imn; 2515 s = htonl(t); 2516 } while (ipv4_is_loopback(s) || 2517 ipv4_is_multicast(s) || 2518 ipv4_is_lbcast(s) || 2519 ipv4_is_zeronet(s) || 2520 ipv4_is_local_multicast(s)); 2521 pkt_dev->cur_daddr = s; 2522 } else { 2523 t = ntohl(pkt_dev->cur_daddr); 2524 t++; 2525 if (t > imx) { 2526 t = imn; 2527 } 2528 pkt_dev->cur_daddr = htonl(t); 2529 } 2530 } 2531 if (pkt_dev->cflows) { 2532 pkt_dev->flows[flow].flags |= F_INIT; 2533 pkt_dev->flows[flow].cur_daddr = 2534 pkt_dev->cur_daddr; 2535 #ifdef CONFIG_XFRM 2536 if (pkt_dev->flags & F_IPSEC_ON) 2537 get_ipsec_sa(pkt_dev, flow); 2538 #endif 2539 pkt_dev->nflows++; 2540 } 2541 } 2542 } else { /* IPV6 * */ 2543 2544 if (!ipv6_addr_any(&pkt_dev->min_in6_daddr)) { 2545 int i; 2546 2547 /* Only random destinations yet */ 2548 2549 for (i = 0; i < 4; i++) { 2550 pkt_dev->cur_in6_daddr.s6_addr32[i] = 2551 (((__force __be32)prandom_u32() | 2552 pkt_dev->min_in6_daddr.s6_addr32[i]) & 2553 pkt_dev->max_in6_daddr.s6_addr32[i]); 2554 } 2555 } 2556 } 2557 2558 if (pkt_dev->min_pkt_size < pkt_dev->max_pkt_size) { 2559 __u32 t; 2560 if (pkt_dev->flags & F_TXSIZE_RND) { 2561 t = prandom_u32() % 2562 (pkt_dev->max_pkt_size - pkt_dev->min_pkt_size) 2563 + pkt_dev->min_pkt_size; 2564 } else { 2565 t = pkt_dev->cur_pkt_size + 1; 2566 if (t > pkt_dev->max_pkt_size) 2567 t = pkt_dev->min_pkt_size; 2568 } 2569 pkt_dev->cur_pkt_size = t; 2570 } 2571 2572 set_cur_queue_map(pkt_dev); 2573 2574 pkt_dev->flows[flow].count++; 2575 } 2576 2577 2578 #ifdef CONFIG_XFRM 2579 static u32 pktgen_dst_metrics[RTAX_MAX + 1] = { 2580 2581 [RTAX_HOPLIMIT] = 0x5, /* Set a static hoplimit */ 2582 }; 2583 2584 static int pktgen_output_ipsec(struct sk_buff *skb, struct pktgen_dev *pkt_dev) 2585 { 2586 struct xfrm_state *x = pkt_dev->flows[pkt_dev->curfl].x; 2587 int err = 0; 2588 struct net *net = dev_net(pkt_dev->odev); 2589 2590 if (!x) 2591 return 0; 2592 /* XXX: we dont support tunnel mode for now until 2593 * we resolve the dst issue */ 2594 if ((x->props.mode != XFRM_MODE_TRANSPORT) && (pkt_dev->spi == 0)) 2595 return 0; 2596 2597 /* But when user specify an valid SPI, transformation 2598 * supports both transport/tunnel mode + ESP/AH type. 2599 */ 2600 if ((x->props.mode == XFRM_MODE_TUNNEL) && (pkt_dev->spi != 0)) 2601 skb->_skb_refdst = (unsigned long)&pkt_dev->dst | SKB_DST_NOREF; 2602 2603 rcu_read_lock_bh(); 2604 err = x->outer_mode->output(x, skb); 2605 rcu_read_unlock_bh(); 2606 if (err) { 2607 XFRM_INC_STATS(net, LINUX_MIB_XFRMOUTSTATEMODEERROR); 2608 goto error; 2609 } 2610 err = x->type->output(x, skb); 2611 if (err) { 2612 XFRM_INC_STATS(net, LINUX_MIB_XFRMOUTSTATEPROTOERROR); 2613 goto error; 2614 } 2615 spin_lock_bh(&x->lock); 2616 x->curlft.bytes += skb->len; 2617 x->curlft.packets++; 2618 spin_unlock_bh(&x->lock); 2619 error: 2620 return err; 2621 } 2622 2623 static void free_SAs(struct pktgen_dev *pkt_dev) 2624 { 2625 if (pkt_dev->cflows) { 2626 /* let go of the SAs if we have them */ 2627 int i; 2628 for (i = 0; i < pkt_dev->cflows; i++) { 2629 struct xfrm_state *x = pkt_dev->flows[i].x; 2630 if (x) { 2631 xfrm_state_put(x); 2632 pkt_dev->flows[i].x = NULL; 2633 } 2634 } 2635 } 2636 } 2637 2638 static int process_ipsec(struct pktgen_dev *pkt_dev, 2639 struct sk_buff *skb, __be16 protocol) 2640 { 2641 if (pkt_dev->flags & F_IPSEC_ON) { 2642 struct xfrm_state *x = pkt_dev->flows[pkt_dev->curfl].x; 2643 int nhead = 0; 2644 if (x) { 2645 struct ethhdr *eth; 2646 struct iphdr *iph; 2647 int ret; 2648 2649 nhead = x->props.header_len - skb_headroom(skb); 2650 if (nhead > 0) { 2651 ret = pskb_expand_head(skb, nhead, 0, GFP_ATOMIC); 2652 if (ret < 0) { 2653 pr_err("Error expanding ipsec packet %d\n", 2654 ret); 2655 goto err; 2656 } 2657 } 2658 2659 /* ipsec is not expecting ll header */ 2660 skb_pull(skb, ETH_HLEN); 2661 ret = pktgen_output_ipsec(skb, pkt_dev); 2662 if (ret) { 2663 pr_err("Error creating ipsec packet %d\n", ret); 2664 goto err; 2665 } 2666 /* restore ll */ 2667 eth = (struct ethhdr *)skb_push(skb, ETH_HLEN); 2668 memcpy(eth, pkt_dev->hh, 2 * ETH_ALEN); 2669 eth->h_proto = protocol; 2670 2671 /* Update IPv4 header len as well as checksum value */ 2672 iph = ip_hdr(skb); 2673 iph->tot_len = htons(skb->len - ETH_HLEN); 2674 ip_send_check(iph); 2675 } 2676 } 2677 return 1; 2678 err: 2679 kfree_skb(skb); 2680 return 0; 2681 } 2682 #endif 2683 2684 static void mpls_push(__be32 *mpls, struct pktgen_dev *pkt_dev) 2685 { 2686 unsigned int i; 2687 for (i = 0; i < pkt_dev->nr_labels; i++) 2688 *mpls++ = pkt_dev->labels[i] & ~MPLS_STACK_BOTTOM; 2689 2690 mpls--; 2691 *mpls |= MPLS_STACK_BOTTOM; 2692 } 2693 2694 static inline __be16 build_tci(unsigned int id, unsigned int cfi, 2695 unsigned int prio) 2696 { 2697 return htons(id | (cfi << 12) | (prio << 13)); 2698 } 2699 2700 static void pktgen_finalize_skb(struct pktgen_dev *pkt_dev, struct sk_buff *skb, 2701 int datalen) 2702 { 2703 struct timeval timestamp; 2704 struct pktgen_hdr *pgh; 2705 2706 pgh = (struct pktgen_hdr *)skb_put(skb, sizeof(*pgh)); 2707 datalen -= sizeof(*pgh); 2708 2709 if (pkt_dev->nfrags <= 0) { 2710 memset(skb_put(skb, datalen), 0, datalen); 2711 } else { 2712 int frags = pkt_dev->nfrags; 2713 int i, len; 2714 int frag_len; 2715 2716 2717 if (frags > MAX_SKB_FRAGS) 2718 frags = MAX_SKB_FRAGS; 2719 len = datalen - frags * PAGE_SIZE; 2720 if (len > 0) { 2721 memset(skb_put(skb, len), 0, len); 2722 datalen = frags * PAGE_SIZE; 2723 } 2724 2725 i = 0; 2726 frag_len = (datalen/frags) < PAGE_SIZE ? 2727 (datalen/frags) : PAGE_SIZE; 2728 while (datalen > 0) { 2729 if (unlikely(!pkt_dev->page)) { 2730 int node = numa_node_id(); 2731 2732 if (pkt_dev->node >= 0 && (pkt_dev->flags & F_NODE)) 2733 node = pkt_dev->node; 2734 pkt_dev->page = alloc_pages_node(node, GFP_KERNEL | __GFP_ZERO, 0); 2735 if (!pkt_dev->page) 2736 break; 2737 } 2738 get_page(pkt_dev->page); 2739 skb_frag_set_page(skb, i, pkt_dev->page); 2740 skb_shinfo(skb)->frags[i].page_offset = 0; 2741 /*last fragment, fill rest of data*/ 2742 if (i == (frags - 1)) 2743 skb_frag_size_set(&skb_shinfo(skb)->frags[i], 2744 (datalen < PAGE_SIZE ? datalen : PAGE_SIZE)); 2745 else 2746 skb_frag_size_set(&skb_shinfo(skb)->frags[i], frag_len); 2747 datalen -= skb_frag_size(&skb_shinfo(skb)->frags[i]); 2748 skb->len += skb_frag_size(&skb_shinfo(skb)->frags[i]); 2749 skb->data_len += skb_frag_size(&skb_shinfo(skb)->frags[i]); 2750 i++; 2751 skb_shinfo(skb)->nr_frags = i; 2752 } 2753 } 2754 2755 /* Stamp the time, and sequence number, 2756 * convert them to network byte order 2757 */ 2758 pgh->pgh_magic = htonl(PKTGEN_MAGIC); 2759 pgh->seq_num = htonl(pkt_dev->seq_num); 2760 2761 if (pkt_dev->flags & F_NO_TIMESTAMP) { 2762 pgh->tv_sec = 0; 2763 pgh->tv_usec = 0; 2764 } else { 2765 do_gettimeofday(×tamp); 2766 pgh->tv_sec = htonl(timestamp.tv_sec); 2767 pgh->tv_usec = htonl(timestamp.tv_usec); 2768 } 2769 } 2770 2771 static struct sk_buff *pktgen_alloc_skb(struct net_device *dev, 2772 struct pktgen_dev *pkt_dev, 2773 unsigned int extralen) 2774 { 2775 struct sk_buff *skb = NULL; 2776 unsigned int size = pkt_dev->cur_pkt_size + 64 + extralen + 2777 pkt_dev->pkt_overhead; 2778 2779 if (pkt_dev->flags & F_NODE) { 2780 int node = pkt_dev->node >= 0 ? pkt_dev->node : numa_node_id(); 2781 2782 skb = __alloc_skb(NET_SKB_PAD + size, GFP_NOWAIT, 0, node); 2783 if (likely(skb)) { 2784 skb_reserve(skb, NET_SKB_PAD); 2785 skb->dev = dev; 2786 } 2787 } else { 2788 skb = __netdev_alloc_skb(dev, size, GFP_NOWAIT); 2789 } 2790 skb_reserve(skb, LL_RESERVED_SPACE(dev)); 2791 2792 return skb; 2793 } 2794 2795 static struct sk_buff *fill_packet_ipv4(struct net_device *odev, 2796 struct pktgen_dev *pkt_dev) 2797 { 2798 struct sk_buff *skb = NULL; 2799 __u8 *eth; 2800 struct udphdr *udph; 2801 int datalen, iplen; 2802 struct iphdr *iph; 2803 __be16 protocol = htons(ETH_P_IP); 2804 __be32 *mpls; 2805 __be16 *vlan_tci = NULL; /* Encapsulates priority and VLAN ID */ 2806 __be16 *vlan_encapsulated_proto = NULL; /* packet type ID field (or len) for VLAN tag */ 2807 __be16 *svlan_tci = NULL; /* Encapsulates priority and SVLAN ID */ 2808 __be16 *svlan_encapsulated_proto = NULL; /* packet type ID field (or len) for SVLAN tag */ 2809 u16 queue_map; 2810 2811 if (pkt_dev->nr_labels) 2812 protocol = htons(ETH_P_MPLS_UC); 2813 2814 if (pkt_dev->vlan_id != 0xffff) 2815 protocol = htons(ETH_P_8021Q); 2816 2817 /* Update any of the values, used when we're incrementing various 2818 * fields. 2819 */ 2820 mod_cur_headers(pkt_dev); 2821 queue_map = pkt_dev->cur_queue_map; 2822 2823 datalen = (odev->hard_header_len + 16) & ~0xf; 2824 2825 skb = pktgen_alloc_skb(odev, pkt_dev, datalen); 2826 if (!skb) { 2827 sprintf(pkt_dev->result, "No memory"); 2828 return NULL; 2829 } 2830 2831 prefetchw(skb->data); 2832 skb_reserve(skb, datalen); 2833 2834 /* Reserve for ethernet and IP header */ 2835 eth = (__u8 *) skb_push(skb, 14); 2836 mpls = (__be32 *)skb_put(skb, pkt_dev->nr_labels*sizeof(__u32)); 2837 if (pkt_dev->nr_labels) 2838 mpls_push(mpls, pkt_dev); 2839 2840 if (pkt_dev->vlan_id != 0xffff) { 2841 if (pkt_dev->svlan_id != 0xffff) { 2842 svlan_tci = (__be16 *)skb_put(skb, sizeof(__be16)); 2843 *svlan_tci = build_tci(pkt_dev->svlan_id, 2844 pkt_dev->svlan_cfi, 2845 pkt_dev->svlan_p); 2846 svlan_encapsulated_proto = (__be16 *)skb_put(skb, sizeof(__be16)); 2847 *svlan_encapsulated_proto = htons(ETH_P_8021Q); 2848 } 2849 vlan_tci = (__be16 *)skb_put(skb, sizeof(__be16)); 2850 *vlan_tci = build_tci(pkt_dev->vlan_id, 2851 pkt_dev->vlan_cfi, 2852 pkt_dev->vlan_p); 2853 vlan_encapsulated_proto = (__be16 *)skb_put(skb, sizeof(__be16)); 2854 *vlan_encapsulated_proto = htons(ETH_P_IP); 2855 } 2856 2857 skb_set_mac_header(skb, 0); 2858 skb_set_network_header(skb, skb->len); 2859 iph = (struct iphdr *) skb_put(skb, sizeof(struct iphdr)); 2860 2861 skb_set_transport_header(skb, skb->len); 2862 udph = (struct udphdr *) skb_put(skb, sizeof(struct udphdr)); 2863 skb_set_queue_mapping(skb, queue_map); 2864 skb->priority = pkt_dev->skb_priority; 2865 2866 memcpy(eth, pkt_dev->hh, 12); 2867 *(__be16 *) & eth[12] = protocol; 2868 2869 /* Eth + IPh + UDPh + mpls */ 2870 datalen = pkt_dev->cur_pkt_size - 14 - 20 - 8 - 2871 pkt_dev->pkt_overhead; 2872 if (datalen < 0 || datalen < sizeof(struct pktgen_hdr)) 2873 datalen = sizeof(struct pktgen_hdr); 2874 2875 udph->source = htons(pkt_dev->cur_udp_src); 2876 udph->dest = htons(pkt_dev->cur_udp_dst); 2877 udph->len = htons(datalen + 8); /* DATA + udphdr */ 2878 udph->check = 0; 2879 2880 iph->ihl = 5; 2881 iph->version = 4; 2882 iph->ttl = 32; 2883 iph->tos = pkt_dev->tos; 2884 iph->protocol = IPPROTO_UDP; /* UDP */ 2885 iph->saddr = pkt_dev->cur_saddr; 2886 iph->daddr = pkt_dev->cur_daddr; 2887 iph->id = htons(pkt_dev->ip_id); 2888 pkt_dev->ip_id++; 2889 iph->frag_off = 0; 2890 iplen = 20 + 8 + datalen; 2891 iph->tot_len = htons(iplen); 2892 ip_send_check(iph); 2893 skb->protocol = protocol; 2894 skb->dev = odev; 2895 skb->pkt_type = PACKET_HOST; 2896 2897 pktgen_finalize_skb(pkt_dev, skb, datalen); 2898 2899 if (!(pkt_dev->flags & F_UDPCSUM)) { 2900 skb->ip_summed = CHECKSUM_NONE; 2901 } else if (odev->features & NETIF_F_V4_CSUM) { 2902 skb->ip_summed = CHECKSUM_PARTIAL; 2903 skb->csum = 0; 2904 udp4_hwcsum(skb, iph->saddr, iph->daddr); 2905 } else { 2906 __wsum csum = skb_checksum(skb, skb_transport_offset(skb), datalen + 8, 0); 2907 2908 /* add protocol-dependent pseudo-header */ 2909 udph->check = csum_tcpudp_magic(iph->saddr, iph->daddr, 2910 datalen + 8, IPPROTO_UDP, csum); 2911 2912 if (udph->check == 0) 2913 udph->check = CSUM_MANGLED_0; 2914 } 2915 2916 #ifdef CONFIG_XFRM 2917 if (!process_ipsec(pkt_dev, skb, protocol)) 2918 return NULL; 2919 #endif 2920 2921 return skb; 2922 } 2923 2924 static struct sk_buff *fill_packet_ipv6(struct net_device *odev, 2925 struct pktgen_dev *pkt_dev) 2926 { 2927 struct sk_buff *skb = NULL; 2928 __u8 *eth; 2929 struct udphdr *udph; 2930 int datalen, udplen; 2931 struct ipv6hdr *iph; 2932 __be16 protocol = htons(ETH_P_IPV6); 2933 __be32 *mpls; 2934 __be16 *vlan_tci = NULL; /* Encapsulates priority and VLAN ID */ 2935 __be16 *vlan_encapsulated_proto = NULL; /* packet type ID field (or len) for VLAN tag */ 2936 __be16 *svlan_tci = NULL; /* Encapsulates priority and SVLAN ID */ 2937 __be16 *svlan_encapsulated_proto = NULL; /* packet type ID field (or len) for SVLAN tag */ 2938 u16 queue_map; 2939 2940 if (pkt_dev->nr_labels) 2941 protocol = htons(ETH_P_MPLS_UC); 2942 2943 if (pkt_dev->vlan_id != 0xffff) 2944 protocol = htons(ETH_P_8021Q); 2945 2946 /* Update any of the values, used when we're incrementing various 2947 * fields. 2948 */ 2949 mod_cur_headers(pkt_dev); 2950 queue_map = pkt_dev->cur_queue_map; 2951 2952 skb = pktgen_alloc_skb(odev, pkt_dev, 16); 2953 if (!skb) { 2954 sprintf(pkt_dev->result, "No memory"); 2955 return NULL; 2956 } 2957 2958 prefetchw(skb->data); 2959 skb_reserve(skb, 16); 2960 2961 /* Reserve for ethernet and IP header */ 2962 eth = (__u8 *) skb_push(skb, 14); 2963 mpls = (__be32 *)skb_put(skb, pkt_dev->nr_labels*sizeof(__u32)); 2964 if (pkt_dev->nr_labels) 2965 mpls_push(mpls, pkt_dev); 2966 2967 if (pkt_dev->vlan_id != 0xffff) { 2968 if (pkt_dev->svlan_id != 0xffff) { 2969 svlan_tci = (__be16 *)skb_put(skb, sizeof(__be16)); 2970 *svlan_tci = build_tci(pkt_dev->svlan_id, 2971 pkt_dev->svlan_cfi, 2972 pkt_dev->svlan_p); 2973 svlan_encapsulated_proto = (__be16 *)skb_put(skb, sizeof(__be16)); 2974 *svlan_encapsulated_proto = htons(ETH_P_8021Q); 2975 } 2976 vlan_tci = (__be16 *)skb_put(skb, sizeof(__be16)); 2977 *vlan_tci = build_tci(pkt_dev->vlan_id, 2978 pkt_dev->vlan_cfi, 2979 pkt_dev->vlan_p); 2980 vlan_encapsulated_proto = (__be16 *)skb_put(skb, sizeof(__be16)); 2981 *vlan_encapsulated_proto = htons(ETH_P_IPV6); 2982 } 2983 2984 skb_set_mac_header(skb, 0); 2985 skb_set_network_header(skb, skb->len); 2986 iph = (struct ipv6hdr *) skb_put(skb, sizeof(struct ipv6hdr)); 2987 2988 skb_set_transport_header(skb, skb->len); 2989 udph = (struct udphdr *) skb_put(skb, sizeof(struct udphdr)); 2990 skb_set_queue_mapping(skb, queue_map); 2991 skb->priority = pkt_dev->skb_priority; 2992 2993 memcpy(eth, pkt_dev->hh, 12); 2994 *(__be16 *) ð[12] = protocol; 2995 2996 /* Eth + IPh + UDPh + mpls */ 2997 datalen = pkt_dev->cur_pkt_size - 14 - 2998 sizeof(struct ipv6hdr) - sizeof(struct udphdr) - 2999 pkt_dev->pkt_overhead; 3000 3001 if (datalen < 0 || datalen < sizeof(struct pktgen_hdr)) { 3002 datalen = sizeof(struct pktgen_hdr); 3003 net_info_ratelimited("increased datalen to %d\n", datalen); 3004 } 3005 3006 udplen = datalen + sizeof(struct udphdr); 3007 udph->source = htons(pkt_dev->cur_udp_src); 3008 udph->dest = htons(pkt_dev->cur_udp_dst); 3009 udph->len = htons(udplen); 3010 udph->check = 0; 3011 3012 *(__be32 *) iph = htonl(0x60000000); /* Version + flow */ 3013 3014 if (pkt_dev->traffic_class) { 3015 /* Version + traffic class + flow (0) */ 3016 *(__be32 *)iph |= htonl(0x60000000 | (pkt_dev->traffic_class << 20)); 3017 } 3018 3019 iph->hop_limit = 32; 3020 3021 iph->payload_len = htons(udplen); 3022 iph->nexthdr = IPPROTO_UDP; 3023 3024 iph->daddr = pkt_dev->cur_in6_daddr; 3025 iph->saddr = pkt_dev->cur_in6_saddr; 3026 3027 skb->protocol = protocol; 3028 skb->dev = odev; 3029 skb->pkt_type = PACKET_HOST; 3030 3031 pktgen_finalize_skb(pkt_dev, skb, datalen); 3032 3033 if (!(pkt_dev->flags & F_UDPCSUM)) { 3034 skb->ip_summed = CHECKSUM_NONE; 3035 } else if (odev->features & NETIF_F_V6_CSUM) { 3036 skb->ip_summed = CHECKSUM_PARTIAL; 3037 skb->csum_start = skb_transport_header(skb) - skb->head; 3038 skb->csum_offset = offsetof(struct udphdr, check); 3039 udph->check = ~csum_ipv6_magic(&iph->saddr, &iph->daddr, udplen, IPPROTO_UDP, 0); 3040 } else { 3041 __wsum csum = skb_checksum(skb, skb_transport_offset(skb), udplen, 0); 3042 3043 /* add protocol-dependent pseudo-header */ 3044 udph->check = csum_ipv6_magic(&iph->saddr, &iph->daddr, udplen, IPPROTO_UDP, csum); 3045 3046 if (udph->check == 0) 3047 udph->check = CSUM_MANGLED_0; 3048 } 3049 3050 return skb; 3051 } 3052 3053 static struct sk_buff *fill_packet(struct net_device *odev, 3054 struct pktgen_dev *pkt_dev) 3055 { 3056 if (pkt_dev->flags & F_IPV6) 3057 return fill_packet_ipv6(odev, pkt_dev); 3058 else 3059 return fill_packet_ipv4(odev, pkt_dev); 3060 } 3061 3062 static void pktgen_clear_counters(struct pktgen_dev *pkt_dev) 3063 { 3064 pkt_dev->seq_num = 1; 3065 pkt_dev->idle_acc = 0; 3066 pkt_dev->sofar = 0; 3067 pkt_dev->tx_bytes = 0; 3068 pkt_dev->errors = 0; 3069 } 3070 3071 /* Set up structure for sending pkts, clear counters */ 3072 3073 static void pktgen_run(struct pktgen_thread *t) 3074 { 3075 struct pktgen_dev *pkt_dev; 3076 int started = 0; 3077 3078 func_enter(); 3079 3080 rcu_read_lock(); 3081 list_for_each_entry_rcu(pkt_dev, &t->if_list, list) { 3082 3083 /* 3084 * setup odev and create initial packet. 3085 */ 3086 pktgen_setup_inject(pkt_dev); 3087 3088 if (pkt_dev->odev) { 3089 pktgen_clear_counters(pkt_dev); 3090 pkt_dev->skb = NULL; 3091 pkt_dev->started_at = pkt_dev->next_tx = ktime_get(); 3092 3093 set_pkt_overhead(pkt_dev); 3094 3095 strcpy(pkt_dev->result, "Starting"); 3096 pkt_dev->running = 1; /* Cranke yeself! */ 3097 started++; 3098 } else 3099 strcpy(pkt_dev->result, "Error starting"); 3100 } 3101 rcu_read_unlock(); 3102 if (started) 3103 t->control &= ~(T_STOP); 3104 } 3105 3106 static void pktgen_stop_all_threads_ifs(struct pktgen_net *pn) 3107 { 3108 struct pktgen_thread *t; 3109 3110 func_enter(); 3111 3112 mutex_lock(&pktgen_thread_lock); 3113 3114 list_for_each_entry(t, &pn->pktgen_threads, th_list) 3115 t->control |= T_STOP; 3116 3117 mutex_unlock(&pktgen_thread_lock); 3118 } 3119 3120 static int thread_is_running(const struct pktgen_thread *t) 3121 { 3122 const struct pktgen_dev *pkt_dev; 3123 3124 rcu_read_lock(); 3125 list_for_each_entry_rcu(pkt_dev, &t->if_list, list) 3126 if (pkt_dev->running) { 3127 rcu_read_unlock(); 3128 return 1; 3129 } 3130 rcu_read_unlock(); 3131 return 0; 3132 } 3133 3134 static int pktgen_wait_thread_run(struct pktgen_thread *t) 3135 { 3136 while (thread_is_running(t)) { 3137 3138 msleep_interruptible(100); 3139 3140 if (signal_pending(current)) 3141 goto signal; 3142 } 3143 return 1; 3144 signal: 3145 return 0; 3146 } 3147 3148 static int pktgen_wait_all_threads_run(struct pktgen_net *pn) 3149 { 3150 struct pktgen_thread *t; 3151 int sig = 1; 3152 3153 mutex_lock(&pktgen_thread_lock); 3154 3155 list_for_each_entry(t, &pn->pktgen_threads, th_list) { 3156 sig = pktgen_wait_thread_run(t); 3157 if (sig == 0) 3158 break; 3159 } 3160 3161 if (sig == 0) 3162 list_for_each_entry(t, &pn->pktgen_threads, th_list) 3163 t->control |= (T_STOP); 3164 3165 mutex_unlock(&pktgen_thread_lock); 3166 return sig; 3167 } 3168 3169 static void pktgen_run_all_threads(struct pktgen_net *pn) 3170 { 3171 struct pktgen_thread *t; 3172 3173 func_enter(); 3174 3175 mutex_lock(&pktgen_thread_lock); 3176 3177 list_for_each_entry(t, &pn->pktgen_threads, th_list) 3178 t->control |= (T_RUN); 3179 3180 mutex_unlock(&pktgen_thread_lock); 3181 3182 /* Propagate thread->control */ 3183 schedule_timeout_interruptible(msecs_to_jiffies(125)); 3184 3185 pktgen_wait_all_threads_run(pn); 3186 } 3187 3188 static void pktgen_reset_all_threads(struct pktgen_net *pn) 3189 { 3190 struct pktgen_thread *t; 3191 3192 func_enter(); 3193 3194 mutex_lock(&pktgen_thread_lock); 3195 3196 list_for_each_entry(t, &pn->pktgen_threads, th_list) 3197 t->control |= (T_REMDEVALL); 3198 3199 mutex_unlock(&pktgen_thread_lock); 3200 3201 /* Propagate thread->control */ 3202 schedule_timeout_interruptible(msecs_to_jiffies(125)); 3203 3204 pktgen_wait_all_threads_run(pn); 3205 } 3206 3207 static void show_results(struct pktgen_dev *pkt_dev, int nr_frags) 3208 { 3209 __u64 bps, mbps, pps; 3210 char *p = pkt_dev->result; 3211 ktime_t elapsed = ktime_sub(pkt_dev->stopped_at, 3212 pkt_dev->started_at); 3213 ktime_t idle = ns_to_ktime(pkt_dev->idle_acc); 3214 3215 p += sprintf(p, "OK: %llu(c%llu+d%llu) usec, %llu (%dbyte,%dfrags)\n", 3216 (unsigned long long)ktime_to_us(elapsed), 3217 (unsigned long long)ktime_to_us(ktime_sub(elapsed, idle)), 3218 (unsigned long long)ktime_to_us(idle), 3219 (unsigned long long)pkt_dev->sofar, 3220 pkt_dev->cur_pkt_size, nr_frags); 3221 3222 pps = div64_u64(pkt_dev->sofar * NSEC_PER_SEC, 3223 ktime_to_ns(elapsed)); 3224 3225 bps = pps * 8 * pkt_dev->cur_pkt_size; 3226 3227 mbps = bps; 3228 do_div(mbps, 1000000); 3229 p += sprintf(p, " %llupps %lluMb/sec (%llubps) errors: %llu", 3230 (unsigned long long)pps, 3231 (unsigned long long)mbps, 3232 (unsigned long long)bps, 3233 (unsigned long long)pkt_dev->errors); 3234 } 3235 3236 /* Set stopped-at timer, remove from running list, do counters & statistics */ 3237 static int pktgen_stop_device(struct pktgen_dev *pkt_dev) 3238 { 3239 int nr_frags = pkt_dev->skb ? skb_shinfo(pkt_dev->skb)->nr_frags : -1; 3240 3241 if (!pkt_dev->running) { 3242 pr_warn("interface: %s is already stopped\n", 3243 pkt_dev->odevname); 3244 return -EINVAL; 3245 } 3246 3247 pkt_dev->running = 0; 3248 kfree_skb(pkt_dev->skb); 3249 pkt_dev->skb = NULL; 3250 pkt_dev->stopped_at = ktime_get(); 3251 3252 show_results(pkt_dev, nr_frags); 3253 3254 return 0; 3255 } 3256 3257 static struct pktgen_dev *next_to_run(struct pktgen_thread *t) 3258 { 3259 struct pktgen_dev *pkt_dev, *best = NULL; 3260 3261 rcu_read_lock(); 3262 list_for_each_entry_rcu(pkt_dev, &t->if_list, list) { 3263 if (!pkt_dev->running) 3264 continue; 3265 if (best == NULL) 3266 best = pkt_dev; 3267 else if (ktime_compare(pkt_dev->next_tx, best->next_tx) < 0) 3268 best = pkt_dev; 3269 } 3270 rcu_read_unlock(); 3271 3272 return best; 3273 } 3274 3275 static void pktgen_stop(struct pktgen_thread *t) 3276 { 3277 struct pktgen_dev *pkt_dev; 3278 3279 func_enter(); 3280 3281 rcu_read_lock(); 3282 3283 list_for_each_entry_rcu(pkt_dev, &t->if_list, list) { 3284 pktgen_stop_device(pkt_dev); 3285 } 3286 3287 rcu_read_unlock(); 3288 } 3289 3290 /* 3291 * one of our devices needs to be removed - find it 3292 * and remove it 3293 */ 3294 static void pktgen_rem_one_if(struct pktgen_thread *t) 3295 { 3296 struct list_head *q, *n; 3297 struct pktgen_dev *cur; 3298 3299 func_enter(); 3300 3301 list_for_each_safe(q, n, &t->if_list) { 3302 cur = list_entry(q, struct pktgen_dev, list); 3303 3304 if (!cur->removal_mark) 3305 continue; 3306 3307 kfree_skb(cur->skb); 3308 cur->skb = NULL; 3309 3310 pktgen_remove_device(t, cur); 3311 3312 break; 3313 } 3314 } 3315 3316 static void pktgen_rem_all_ifs(struct pktgen_thread *t) 3317 { 3318 struct list_head *q, *n; 3319 struct pktgen_dev *cur; 3320 3321 func_enter(); 3322 3323 /* Remove all devices, free mem */ 3324 3325 list_for_each_safe(q, n, &t->if_list) { 3326 cur = list_entry(q, struct pktgen_dev, list); 3327 3328 kfree_skb(cur->skb); 3329 cur->skb = NULL; 3330 3331 pktgen_remove_device(t, cur); 3332 } 3333 } 3334 3335 static void pktgen_rem_thread(struct pktgen_thread *t) 3336 { 3337 /* Remove from the thread list */ 3338 remove_proc_entry(t->tsk->comm, t->net->proc_dir); 3339 } 3340 3341 static void pktgen_resched(struct pktgen_dev *pkt_dev) 3342 { 3343 ktime_t idle_start = ktime_get(); 3344 schedule(); 3345 pkt_dev->idle_acc += ktime_to_ns(ktime_sub(ktime_get(), idle_start)); 3346 } 3347 3348 static void pktgen_wait_for_skb(struct pktgen_dev *pkt_dev) 3349 { 3350 ktime_t idle_start = ktime_get(); 3351 3352 while (atomic_read(&(pkt_dev->skb->users)) != 1) { 3353 if (signal_pending(current)) 3354 break; 3355 3356 if (need_resched()) 3357 pktgen_resched(pkt_dev); 3358 else 3359 cpu_relax(); 3360 } 3361 pkt_dev->idle_acc += ktime_to_ns(ktime_sub(ktime_get(), idle_start)); 3362 } 3363 3364 static void pktgen_xmit(struct pktgen_dev *pkt_dev) 3365 { 3366 unsigned int burst = ACCESS_ONCE(pkt_dev->burst); 3367 struct net_device *odev = pkt_dev->odev; 3368 struct netdev_queue *txq; 3369 struct sk_buff *skb; 3370 int ret; 3371 3372 /* If device is offline, then don't send */ 3373 if (unlikely(!netif_running(odev) || !netif_carrier_ok(odev))) { 3374 pktgen_stop_device(pkt_dev); 3375 return; 3376 } 3377 3378 /* This is max DELAY, this has special meaning of 3379 * "never transmit" 3380 */ 3381 if (unlikely(pkt_dev->delay == ULLONG_MAX)) { 3382 pkt_dev->next_tx = ktime_add_ns(ktime_get(), ULONG_MAX); 3383 return; 3384 } 3385 3386 /* If no skb or clone count exhausted then get new one */ 3387 if (!pkt_dev->skb || (pkt_dev->last_ok && 3388 ++pkt_dev->clone_count >= pkt_dev->clone_skb)) { 3389 /* build a new pkt */ 3390 kfree_skb(pkt_dev->skb); 3391 3392 pkt_dev->skb = fill_packet(odev, pkt_dev); 3393 if (pkt_dev->skb == NULL) { 3394 pr_err("ERROR: couldn't allocate skb in fill_packet\n"); 3395 schedule(); 3396 pkt_dev->clone_count--; /* back out increment, OOM */ 3397 return; 3398 } 3399 pkt_dev->last_pkt_size = pkt_dev->skb->len; 3400 pkt_dev->clone_count = 0; /* reset counter */ 3401 } 3402 3403 if (pkt_dev->delay && pkt_dev->last_ok) 3404 spin(pkt_dev, pkt_dev->next_tx); 3405 3406 if (pkt_dev->xmit_mode == M_NETIF_RECEIVE) { 3407 skb = pkt_dev->skb; 3408 skb->protocol = eth_type_trans(skb, skb->dev); 3409 atomic_add(burst, &skb->users); 3410 local_bh_disable(); 3411 do { 3412 ret = netif_receive_skb(skb); 3413 if (ret == NET_RX_DROP) 3414 pkt_dev->errors++; 3415 pkt_dev->sofar++; 3416 pkt_dev->seq_num++; 3417 if (atomic_read(&skb->users) != burst) { 3418 /* skb was queued by rps/rfs or taps, 3419 * so cannot reuse this skb 3420 */ 3421 atomic_sub(burst - 1, &skb->users); 3422 /* get out of the loop and wait 3423 * until skb is consumed 3424 */ 3425 break; 3426 } 3427 /* skb was 'freed' by stack, so clean few 3428 * bits and reuse it 3429 */ 3430 #ifdef CONFIG_NET_CLS_ACT 3431 skb->tc_verd = 0; /* reset reclass/redir ttl */ 3432 #endif 3433 } while (--burst > 0); 3434 goto out; /* Skips xmit_mode M_START_XMIT */ 3435 } 3436 3437 txq = skb_get_tx_queue(odev, pkt_dev->skb); 3438 3439 local_bh_disable(); 3440 3441 HARD_TX_LOCK(odev, txq, smp_processor_id()); 3442 3443 if (unlikely(netif_xmit_frozen_or_drv_stopped(txq))) { 3444 ret = NETDEV_TX_BUSY; 3445 pkt_dev->last_ok = 0; 3446 goto unlock; 3447 } 3448 atomic_add(burst, &pkt_dev->skb->users); 3449 3450 xmit_more: 3451 ret = netdev_start_xmit(pkt_dev->skb, odev, txq, --burst > 0); 3452 3453 switch (ret) { 3454 case NETDEV_TX_OK: 3455 pkt_dev->last_ok = 1; 3456 pkt_dev->sofar++; 3457 pkt_dev->seq_num++; 3458 pkt_dev->tx_bytes += pkt_dev->last_pkt_size; 3459 if (burst > 0 && !netif_xmit_frozen_or_drv_stopped(txq)) 3460 goto xmit_more; 3461 break; 3462 case NET_XMIT_DROP: 3463 case NET_XMIT_CN: 3464 case NET_XMIT_POLICED: 3465 /* skb has been consumed */ 3466 pkt_dev->errors++; 3467 break; 3468 default: /* Drivers are not supposed to return other values! */ 3469 net_info_ratelimited("%s xmit error: %d\n", 3470 pkt_dev->odevname, ret); 3471 pkt_dev->errors++; 3472 /* fallthru */ 3473 case NETDEV_TX_LOCKED: 3474 case NETDEV_TX_BUSY: 3475 /* Retry it next time */ 3476 atomic_dec(&(pkt_dev->skb->users)); 3477 pkt_dev->last_ok = 0; 3478 } 3479 if (unlikely(burst)) 3480 atomic_sub(burst, &pkt_dev->skb->users); 3481 unlock: 3482 HARD_TX_UNLOCK(odev, txq); 3483 3484 out: 3485 local_bh_enable(); 3486 3487 /* If pkt_dev->count is zero, then run forever */ 3488 if ((pkt_dev->count != 0) && (pkt_dev->sofar >= pkt_dev->count)) { 3489 pktgen_wait_for_skb(pkt_dev); 3490 3491 /* Done with this */ 3492 pktgen_stop_device(pkt_dev); 3493 } 3494 } 3495 3496 /* 3497 * Main loop of the thread goes here 3498 */ 3499 3500 static int pktgen_thread_worker(void *arg) 3501 { 3502 DEFINE_WAIT(wait); 3503 struct pktgen_thread *t = arg; 3504 struct pktgen_dev *pkt_dev = NULL; 3505 int cpu = t->cpu; 3506 3507 BUG_ON(smp_processor_id() != cpu); 3508 3509 init_waitqueue_head(&t->queue); 3510 complete(&t->start_done); 3511 3512 pr_debug("starting pktgen/%d: pid=%d\n", cpu, task_pid_nr(current)); 3513 3514 set_freezable(); 3515 3516 while (!kthread_should_stop()) { 3517 pkt_dev = next_to_run(t); 3518 3519 if (unlikely(!pkt_dev && t->control == 0)) { 3520 if (t->net->pktgen_exiting) 3521 break; 3522 wait_event_interruptible_timeout(t->queue, 3523 t->control != 0, 3524 HZ/10); 3525 try_to_freeze(); 3526 continue; 3527 } 3528 3529 if (likely(pkt_dev)) { 3530 pktgen_xmit(pkt_dev); 3531 3532 if (need_resched()) 3533 pktgen_resched(pkt_dev); 3534 else 3535 cpu_relax(); 3536 } 3537 3538 if (t->control & T_STOP) { 3539 pktgen_stop(t); 3540 t->control &= ~(T_STOP); 3541 } 3542 3543 if (t->control & T_RUN) { 3544 pktgen_run(t); 3545 t->control &= ~(T_RUN); 3546 } 3547 3548 if (t->control & T_REMDEVALL) { 3549 pktgen_rem_all_ifs(t); 3550 t->control &= ~(T_REMDEVALL); 3551 } 3552 3553 if (t->control & T_REMDEV) { 3554 pktgen_rem_one_if(t); 3555 t->control &= ~(T_REMDEV); 3556 } 3557 3558 try_to_freeze(); 3559 } 3560 3561 pr_debug("%s stopping all device\n", t->tsk->comm); 3562 pktgen_stop(t); 3563 3564 pr_debug("%s removing all device\n", t->tsk->comm); 3565 pktgen_rem_all_ifs(t); 3566 3567 pr_debug("%s removing thread\n", t->tsk->comm); 3568 pktgen_rem_thread(t); 3569 3570 return 0; 3571 } 3572 3573 static struct pktgen_dev *pktgen_find_dev(struct pktgen_thread *t, 3574 const char *ifname, bool exact) 3575 { 3576 struct pktgen_dev *p, *pkt_dev = NULL; 3577 size_t len = strlen(ifname); 3578 3579 rcu_read_lock(); 3580 list_for_each_entry_rcu(p, &t->if_list, list) 3581 if (strncmp(p->odevname, ifname, len) == 0) { 3582 if (p->odevname[len]) { 3583 if (exact || p->odevname[len] != '@') 3584 continue; 3585 } 3586 pkt_dev = p; 3587 break; 3588 } 3589 3590 rcu_read_unlock(); 3591 pr_debug("find_dev(%s) returning %p\n", ifname, pkt_dev); 3592 return pkt_dev; 3593 } 3594 3595 /* 3596 * Adds a dev at front of if_list. 3597 */ 3598 3599 static int add_dev_to_thread(struct pktgen_thread *t, 3600 struct pktgen_dev *pkt_dev) 3601 { 3602 int rv = 0; 3603 3604 /* This function cannot be called concurrently, as its called 3605 * under pktgen_thread_lock mutex, but it can run from 3606 * userspace on another CPU than the kthread. The if_lock() 3607 * is used here to sync with concurrent instances of 3608 * _rem_dev_from_if_list() invoked via kthread, which is also 3609 * updating the if_list */ 3610 if_lock(t); 3611 3612 if (pkt_dev->pg_thread) { 3613 pr_err("ERROR: already assigned to a thread\n"); 3614 rv = -EBUSY; 3615 goto out; 3616 } 3617 3618 pkt_dev->running = 0; 3619 pkt_dev->pg_thread = t; 3620 list_add_rcu(&pkt_dev->list, &t->if_list); 3621 3622 out: 3623 if_unlock(t); 3624 return rv; 3625 } 3626 3627 /* Called under thread lock */ 3628 3629 static int pktgen_add_device(struct pktgen_thread *t, const char *ifname) 3630 { 3631 struct pktgen_dev *pkt_dev; 3632 int err; 3633 int node = cpu_to_node(t->cpu); 3634 3635 /* We don't allow a device to be on several threads */ 3636 3637 pkt_dev = __pktgen_NN_threads(t->net, ifname, FIND); 3638 if (pkt_dev) { 3639 pr_err("ERROR: interface already used\n"); 3640 return -EBUSY; 3641 } 3642 3643 pkt_dev = kzalloc_node(sizeof(struct pktgen_dev), GFP_KERNEL, node); 3644 if (!pkt_dev) 3645 return -ENOMEM; 3646 3647 strcpy(pkt_dev->odevname, ifname); 3648 pkt_dev->flows = vzalloc_node(MAX_CFLOWS * sizeof(struct flow_state), 3649 node); 3650 if (pkt_dev->flows == NULL) { 3651 kfree(pkt_dev); 3652 return -ENOMEM; 3653 } 3654 3655 pkt_dev->removal_mark = 0; 3656 pkt_dev->nfrags = 0; 3657 pkt_dev->delay = pg_delay_d; 3658 pkt_dev->count = pg_count_d; 3659 pkt_dev->sofar = 0; 3660 pkt_dev->udp_src_min = 9; /* sink port */ 3661 pkt_dev->udp_src_max = 9; 3662 pkt_dev->udp_dst_min = 9; 3663 pkt_dev->udp_dst_max = 9; 3664 pkt_dev->vlan_p = 0; 3665 pkt_dev->vlan_cfi = 0; 3666 pkt_dev->vlan_id = 0xffff; 3667 pkt_dev->svlan_p = 0; 3668 pkt_dev->svlan_cfi = 0; 3669 pkt_dev->svlan_id = 0xffff; 3670 pkt_dev->burst = 1; 3671 pkt_dev->node = -1; 3672 3673 err = pktgen_setup_dev(t->net, pkt_dev, ifname); 3674 if (err) 3675 goto out1; 3676 if (pkt_dev->odev->priv_flags & IFF_TX_SKB_SHARING) 3677 pkt_dev->clone_skb = pg_clone_skb_d; 3678 3679 pkt_dev->entry = proc_create_data(ifname, 0600, t->net->proc_dir, 3680 &pktgen_if_fops, pkt_dev); 3681 if (!pkt_dev->entry) { 3682 pr_err("cannot create %s/%s procfs entry\n", 3683 PG_PROC_DIR, ifname); 3684 err = -EINVAL; 3685 goto out2; 3686 } 3687 #ifdef CONFIG_XFRM 3688 pkt_dev->ipsmode = XFRM_MODE_TRANSPORT; 3689 pkt_dev->ipsproto = IPPROTO_ESP; 3690 3691 /* xfrm tunnel mode needs additional dst to extract outter 3692 * ip header protocol/ttl/id field, here creat a phony one. 3693 * instead of looking for a valid rt, which definitely hurting 3694 * performance under such circumstance. 3695 */ 3696 pkt_dev->dstops.family = AF_INET; 3697 pkt_dev->dst.dev = pkt_dev->odev; 3698 dst_init_metrics(&pkt_dev->dst, pktgen_dst_metrics, false); 3699 pkt_dev->dst.child = &pkt_dev->dst; 3700 pkt_dev->dst.ops = &pkt_dev->dstops; 3701 #endif 3702 3703 return add_dev_to_thread(t, pkt_dev); 3704 out2: 3705 dev_put(pkt_dev->odev); 3706 out1: 3707 #ifdef CONFIG_XFRM 3708 free_SAs(pkt_dev); 3709 #endif 3710 vfree(pkt_dev->flows); 3711 kfree(pkt_dev); 3712 return err; 3713 } 3714 3715 static int __net_init pktgen_create_thread(int cpu, struct pktgen_net *pn) 3716 { 3717 struct pktgen_thread *t; 3718 struct proc_dir_entry *pe; 3719 struct task_struct *p; 3720 3721 t = kzalloc_node(sizeof(struct pktgen_thread), GFP_KERNEL, 3722 cpu_to_node(cpu)); 3723 if (!t) { 3724 pr_err("ERROR: out of memory, can't create new thread\n"); 3725 return -ENOMEM; 3726 } 3727 3728 spin_lock_init(&t->if_lock); 3729 t->cpu = cpu; 3730 3731 INIT_LIST_HEAD(&t->if_list); 3732 3733 list_add_tail(&t->th_list, &pn->pktgen_threads); 3734 init_completion(&t->start_done); 3735 3736 p = kthread_create_on_node(pktgen_thread_worker, 3737 t, 3738 cpu_to_node(cpu), 3739 "kpktgend_%d", cpu); 3740 if (IS_ERR(p)) { 3741 pr_err("kernel_thread() failed for cpu %d\n", t->cpu); 3742 list_del(&t->th_list); 3743 kfree(t); 3744 return PTR_ERR(p); 3745 } 3746 kthread_bind(p, cpu); 3747 t->tsk = p; 3748 3749 pe = proc_create_data(t->tsk->comm, 0600, pn->proc_dir, 3750 &pktgen_thread_fops, t); 3751 if (!pe) { 3752 pr_err("cannot create %s/%s procfs entry\n", 3753 PG_PROC_DIR, t->tsk->comm); 3754 kthread_stop(p); 3755 list_del(&t->th_list); 3756 kfree(t); 3757 return -EINVAL; 3758 } 3759 3760 t->net = pn; 3761 get_task_struct(p); 3762 wake_up_process(p); 3763 wait_for_completion(&t->start_done); 3764 3765 return 0; 3766 } 3767 3768 /* 3769 * Removes a device from the thread if_list. 3770 */ 3771 static void _rem_dev_from_if_list(struct pktgen_thread *t, 3772 struct pktgen_dev *pkt_dev) 3773 { 3774 struct list_head *q, *n; 3775 struct pktgen_dev *p; 3776 3777 if_lock(t); 3778 list_for_each_safe(q, n, &t->if_list) { 3779 p = list_entry(q, struct pktgen_dev, list); 3780 if (p == pkt_dev) 3781 list_del_rcu(&p->list); 3782 } 3783 if_unlock(t); 3784 } 3785 3786 static int pktgen_remove_device(struct pktgen_thread *t, 3787 struct pktgen_dev *pkt_dev) 3788 { 3789 pr_debug("remove_device pkt_dev=%p\n", pkt_dev); 3790 3791 if (pkt_dev->running) { 3792 pr_warn("WARNING: trying to remove a running interface, stopping it now\n"); 3793 pktgen_stop_device(pkt_dev); 3794 } 3795 3796 /* Dis-associate from the interface */ 3797 3798 if (pkt_dev->odev) { 3799 dev_put(pkt_dev->odev); 3800 pkt_dev->odev = NULL; 3801 } 3802 3803 /* Remove proc before if_list entry, because add_device uses 3804 * list to determine if interface already exist, avoid race 3805 * with proc_create_data() */ 3806 proc_remove(pkt_dev->entry); 3807 3808 /* And update the thread if_list */ 3809 _rem_dev_from_if_list(t, pkt_dev); 3810 3811 #ifdef CONFIG_XFRM 3812 free_SAs(pkt_dev); 3813 #endif 3814 vfree(pkt_dev->flows); 3815 if (pkt_dev->page) 3816 put_page(pkt_dev->page); 3817 kfree_rcu(pkt_dev, rcu); 3818 return 0; 3819 } 3820 3821 static int __net_init pg_net_init(struct net *net) 3822 { 3823 struct pktgen_net *pn = net_generic(net, pg_net_id); 3824 struct proc_dir_entry *pe; 3825 int cpu, ret = 0; 3826 3827 pn->net = net; 3828 INIT_LIST_HEAD(&pn->pktgen_threads); 3829 pn->pktgen_exiting = false; 3830 pn->proc_dir = proc_mkdir(PG_PROC_DIR, pn->net->proc_net); 3831 if (!pn->proc_dir) { 3832 pr_warn("cannot create /proc/net/%s\n", PG_PROC_DIR); 3833 return -ENODEV; 3834 } 3835 pe = proc_create(PGCTRL, 0600, pn->proc_dir, &pktgen_fops); 3836 if (pe == NULL) { 3837 pr_err("cannot create %s procfs entry\n", PGCTRL); 3838 ret = -EINVAL; 3839 goto remove; 3840 } 3841 3842 for_each_online_cpu(cpu) { 3843 int err; 3844 3845 err = pktgen_create_thread(cpu, pn); 3846 if (err) 3847 pr_warn("Cannot create thread for cpu %d (%d)\n", 3848 cpu, err); 3849 } 3850 3851 if (list_empty(&pn->pktgen_threads)) { 3852 pr_err("Initialization failed for all threads\n"); 3853 ret = -ENODEV; 3854 goto remove_entry; 3855 } 3856 3857 return 0; 3858 3859 remove_entry: 3860 remove_proc_entry(PGCTRL, pn->proc_dir); 3861 remove: 3862 remove_proc_entry(PG_PROC_DIR, pn->net->proc_net); 3863 return ret; 3864 } 3865 3866 static void __net_exit pg_net_exit(struct net *net) 3867 { 3868 struct pktgen_net *pn = net_generic(net, pg_net_id); 3869 struct pktgen_thread *t; 3870 struct list_head *q, *n; 3871 LIST_HEAD(list); 3872 3873 /* Stop all interfaces & threads */ 3874 pn->pktgen_exiting = true; 3875 3876 mutex_lock(&pktgen_thread_lock); 3877 list_splice_init(&pn->pktgen_threads, &list); 3878 mutex_unlock(&pktgen_thread_lock); 3879 3880 list_for_each_safe(q, n, &list) { 3881 t = list_entry(q, struct pktgen_thread, th_list); 3882 list_del(&t->th_list); 3883 kthread_stop(t->tsk); 3884 put_task_struct(t->tsk); 3885 kfree(t); 3886 } 3887 3888 remove_proc_entry(PGCTRL, pn->proc_dir); 3889 remove_proc_entry(PG_PROC_DIR, pn->net->proc_net); 3890 } 3891 3892 static struct pernet_operations pg_net_ops = { 3893 .init = pg_net_init, 3894 .exit = pg_net_exit, 3895 .id = &pg_net_id, 3896 .size = sizeof(struct pktgen_net), 3897 }; 3898 3899 static int __init pg_init(void) 3900 { 3901 int ret = 0; 3902 3903 pr_info("%s", version); 3904 ret = register_pernet_subsys(&pg_net_ops); 3905 if (ret) 3906 return ret; 3907 ret = register_netdevice_notifier(&pktgen_notifier_block); 3908 if (ret) 3909 unregister_pernet_subsys(&pg_net_ops); 3910 3911 return ret; 3912 } 3913 3914 static void __exit pg_cleanup(void) 3915 { 3916 unregister_netdevice_notifier(&pktgen_notifier_block); 3917 unregister_pernet_subsys(&pg_net_ops); 3918 /* Don't need rcu_barrier() due to use of kfree_rcu() */ 3919 } 3920 3921 module_init(pg_init); 3922 module_exit(pg_cleanup); 3923 3924 MODULE_AUTHOR("Robert Olsson <robert.olsson@its.uu.se>"); 3925 MODULE_DESCRIPTION("Packet Generator tool"); 3926 MODULE_LICENSE("GPL"); 3927 MODULE_VERSION(VERSION); 3928 module_param(pg_count_d, int, 0); 3929 MODULE_PARM_DESC(pg_count_d, "Default number of packets to inject"); 3930 module_param(pg_delay_d, int, 0); 3931 MODULE_PARM_DESC(pg_delay_d, "Default delay between packets (nanoseconds)"); 3932 module_param(pg_clone_skb_d, int, 0); 3933 MODULE_PARM_DESC(pg_clone_skb_d, "Default number of copies of the same packet"); 3934 module_param(debug, int, 0); 3935 MODULE_PARM_DESC(debug, "Enable debugging of pktgen module"); 3936