184a14ae8SThomas Gleixner // SPDX-License-Identifier: GPL-2.0-only 21da177e4SLinus Torvalds /* 31da177e4SLinus Torvalds * net/sched/sch_netem.c Network emulator 41da177e4SLinus Torvalds * 51da177e4SLinus Torvalds * Many of the algorithms and ideas for this came from 61da177e4SLinus Torvalds * NIST Net which is not copyrighted. 71da177e4SLinus Torvalds * 81da177e4SLinus Torvalds * Authors: Stephen Hemminger <shemminger@osdl.org> 91da177e4SLinus Torvalds * Catalin(ux aka Dino) BOIE <catab at umbrella dot ro> 101da177e4SLinus Torvalds */ 111da177e4SLinus Torvalds 12b7f080cfSAlexey Dobriyan #include <linux/mm.h> 131da177e4SLinus Torvalds #include <linux/module.h> 145a0e3ad6STejun Heo #include <linux/slab.h> 151da177e4SLinus Torvalds #include <linux/types.h> 161da177e4SLinus Torvalds #include <linux/kernel.h> 171da177e4SLinus Torvalds #include <linux/errno.h> 181da177e4SLinus Torvalds #include <linux/skbuff.h> 1978776d3fSDavid S. Miller #include <linux/vmalloc.h> 201da177e4SLinus Torvalds #include <linux/rtnetlink.h> 2190b41a1cSHagen Paul Pfeifer #include <linux/reciprocal_div.h> 22aec0a40aSEric Dumazet #include <linux/rbtree.h> 231da177e4SLinus Torvalds 24dc5fc579SArnaldo Carvalho de Melo #include <net/netlink.h> 251da177e4SLinus Torvalds #include <net/pkt_sched.h> 26e4ae004bSEric Dumazet #include <net/inet_ecn.h> 271da177e4SLinus Torvalds 28250a65f7Sstephen hemminger #define VERSION "1.3" 29eb229c4cSStephen Hemminger 301da177e4SLinus Torvalds /* Network Emulation Queuing algorithm. 311da177e4SLinus Torvalds ==================================== 321da177e4SLinus Torvalds 331da177e4SLinus Torvalds Sources: [1] Mark Carson, Darrin Santay, "NIST Net - A Linux-based 341da177e4SLinus Torvalds Network Emulation Tool 351da177e4SLinus Torvalds [2] Luigi Rizzo, DummyNet for FreeBSD 361da177e4SLinus Torvalds 371da177e4SLinus Torvalds ---------------------------------------------------------------- 381da177e4SLinus Torvalds 391da177e4SLinus Torvalds This started out as a simple way to delay outgoing packets to 401da177e4SLinus Torvalds test TCP but has grown to include most of the functionality 411da177e4SLinus Torvalds of a full blown network emulator like NISTnet. It can delay 421da177e4SLinus Torvalds packets and add random jitter (and correlation). The random 431da177e4SLinus Torvalds distribution can be loaded from a table as well to provide 441da177e4SLinus Torvalds normal, Pareto, or experimental curves. Packet loss, 451da177e4SLinus Torvalds duplication, and reordering can also be emulated. 461da177e4SLinus Torvalds 471da177e4SLinus Torvalds This qdisc does not do classification that can be handled in 481da177e4SLinus Torvalds layering other disciplines. It does not need to do bandwidth 491da177e4SLinus Torvalds control either since that can be handled by using token 501da177e4SLinus Torvalds bucket or other rate control. 51661b7972Sstephen hemminger 52661b7972Sstephen hemminger Correlated Loss Generator models 53661b7972Sstephen hemminger 54661b7972Sstephen hemminger Added generation of correlated loss according to the 55661b7972Sstephen hemminger "Gilbert-Elliot" model, a 4-state markov model. 56661b7972Sstephen hemminger 57661b7972Sstephen hemminger References: 58661b7972Sstephen hemminger [1] NetemCLG Home http://netgroup.uniroma2.it/NetemCLG 59661b7972Sstephen hemminger [2] S. Salsano, F. Ludovici, A. Ordine, "Definition of a general 60661b7972Sstephen hemminger and intuitive loss model for packet networks and its implementation 61661b7972Sstephen hemminger in the Netem module in the Linux kernel", available in [1] 62661b7972Sstephen hemminger 63661b7972Sstephen hemminger Authors: Stefano Salsano <stefano.salsano at uniroma2.it 64661b7972Sstephen hemminger Fabio Ludovici <fabio.ludovici at yahoo.it> 651da177e4SLinus Torvalds */ 661da177e4SLinus Torvalds 670a9fe5c3SYousuk Seung struct disttable { 680a9fe5c3SYousuk Seung u32 size; 690a9fe5c3SYousuk Seung s16 table[0]; 700a9fe5c3SYousuk Seung }; 710a9fe5c3SYousuk Seung 721da177e4SLinus Torvalds struct netem_sched_data { 73aec0a40aSEric Dumazet /* internal t(ime)fifo qdisc uses t_root and sch->limit */ 74aec0a40aSEric Dumazet struct rb_root t_root; 7550612537SEric Dumazet 76d66280b1SPeter Oskolkov /* a linear queue; reduces rbtree rebalancing when jitter is low */ 77d66280b1SPeter Oskolkov struct sk_buff *t_head; 78d66280b1SPeter Oskolkov struct sk_buff *t_tail; 79d66280b1SPeter Oskolkov 8050612537SEric Dumazet /* optional qdisc for classful handling (NULL at netem init) */ 811da177e4SLinus Torvalds struct Qdisc *qdisc; 8250612537SEric Dumazet 8359cb5c67SPatrick McHardy struct qdisc_watchdog watchdog; 841da177e4SLinus Torvalds 85112f9cb6SDave Taht s64 latency; 86112f9cb6SDave Taht s64 jitter; 87b407621cSStephen Hemminger 881da177e4SLinus Torvalds u32 loss; 89e4ae004bSEric Dumazet u32 ecn; 901da177e4SLinus Torvalds u32 limit; 911da177e4SLinus Torvalds u32 counter; 921da177e4SLinus Torvalds u32 gap; 931da177e4SLinus Torvalds u32 duplicate; 940dca51d3SStephen Hemminger u32 reorder; 95c865e5d9SStephen Hemminger u32 corrupt; 966a031f67SYang Yingliang u64 rate; 9790b41a1cSHagen Paul Pfeifer s32 packet_overhead; 9890b41a1cSHagen Paul Pfeifer u32 cell_size; 99809fa972SHannes Frederic Sowa struct reciprocal_value cell_size_reciprocal; 10090b41a1cSHagen Paul Pfeifer s32 cell_overhead; 1011da177e4SLinus Torvalds 1021da177e4SLinus Torvalds struct crndstate { 103b407621cSStephen Hemminger u32 last; 104b407621cSStephen Hemminger u32 rho; 105c865e5d9SStephen Hemminger } delay_cor, loss_cor, dup_cor, reorder_cor, corrupt_cor; 1061da177e4SLinus Torvalds 1070a9fe5c3SYousuk Seung struct disttable *delay_dist; 108661b7972Sstephen hemminger 109661b7972Sstephen hemminger enum { 110661b7972Sstephen hemminger CLG_RANDOM, 111661b7972Sstephen hemminger CLG_4_STATES, 112661b7972Sstephen hemminger CLG_GILB_ELL, 113661b7972Sstephen hemminger } loss_model; 114661b7972Sstephen hemminger 115a6e2fe17SYang Yingliang enum { 116a6e2fe17SYang Yingliang TX_IN_GAP_PERIOD = 1, 117a6e2fe17SYang Yingliang TX_IN_BURST_PERIOD, 118a6e2fe17SYang Yingliang LOST_IN_GAP_PERIOD, 119a6e2fe17SYang Yingliang LOST_IN_BURST_PERIOD, 120a6e2fe17SYang Yingliang } _4_state_model; 121a6e2fe17SYang Yingliang 122c045a734SYang Yingliang enum { 123c045a734SYang Yingliang GOOD_STATE = 1, 124c045a734SYang Yingliang BAD_STATE, 125c045a734SYang Yingliang } GE_state_model; 126c045a734SYang Yingliang 127661b7972Sstephen hemminger /* Correlated Loss Generation models */ 128661b7972Sstephen hemminger struct clgstate { 129661b7972Sstephen hemminger /* state of the Markov chain */ 130661b7972Sstephen hemminger u8 state; 131661b7972Sstephen hemminger 132661b7972Sstephen hemminger /* 4-states and Gilbert-Elliot models */ 133661b7972Sstephen hemminger u32 a1; /* p13 for 4-states or p for GE */ 134661b7972Sstephen hemminger u32 a2; /* p31 for 4-states or r for GE */ 135661b7972Sstephen hemminger u32 a3; /* p32 for 4-states or h for GE */ 136661b7972Sstephen hemminger u32 a4; /* p14 for 4-states or 1-k for GE */ 137661b7972Sstephen hemminger u32 a5; /* p23 used only in 4-states */ 138661b7972Sstephen hemminger } clg; 139661b7972Sstephen hemminger 140836af83bSDave Taht struct tc_netem_slot slot_config; 141836af83bSDave Taht struct slotstate { 142836af83bSDave Taht u64 slot_next; 143836af83bSDave Taht s32 packets_left; 144836af83bSDave Taht s32 bytes_left; 145836af83bSDave Taht } slot; 146836af83bSDave Taht 1470a9fe5c3SYousuk Seung struct disttable *slot_dist; 1481da177e4SLinus Torvalds }; 1491da177e4SLinus Torvalds 15050612537SEric Dumazet /* Time stamp put into socket buffer control block 15150612537SEric Dumazet * Only valid when skbs are in our internal t(ime)fifo queue. 15256b17425SEric Dumazet * 15356b17425SEric Dumazet * As skb->rbnode uses same storage than skb->next, skb->prev and skb->tstamp, 15456b17425SEric Dumazet * and skb->next & skb->prev are scratch space for a qdisc, 15556b17425SEric Dumazet * we save skb->tstamp value in skb->cb[] before destroying it. 15650612537SEric Dumazet */ 1571da177e4SLinus Torvalds struct netem_skb_cb { 158112f9cb6SDave Taht u64 time_to_send; 1591da177e4SLinus Torvalds }; 1601da177e4SLinus Torvalds 1615f86173bSJussi Kivilinna static inline struct netem_skb_cb *netem_skb_cb(struct sk_buff *skb) 1625f86173bSJussi Kivilinna { 163aec0a40aSEric Dumazet /* we assume we can use skb next/prev/tstamp as storage for rb_node */ 16416bda13dSDavid S. Miller qdisc_cb_private_validate(skb, sizeof(struct netem_skb_cb)); 165175f9c1bSJussi Kivilinna return (struct netem_skb_cb *)qdisc_skb_cb(skb)->data; 1665f86173bSJussi Kivilinna } 1675f86173bSJussi Kivilinna 1681da177e4SLinus Torvalds /* init_crandom - initialize correlated random number generator 1691da177e4SLinus Torvalds * Use entropy source for initial seed. 1701da177e4SLinus Torvalds */ 1711da177e4SLinus Torvalds static void init_crandom(struct crndstate *state, unsigned long rho) 1721da177e4SLinus Torvalds { 1731da177e4SLinus Torvalds state->rho = rho; 17463862b5bSAruna-Hewapathirane state->last = prandom_u32(); 1751da177e4SLinus Torvalds } 1761da177e4SLinus Torvalds 1771da177e4SLinus Torvalds /* get_crandom - correlated random number generator 1781da177e4SLinus Torvalds * Next number depends on last value. 1791da177e4SLinus Torvalds * rho is scaled to avoid floating point. 1801da177e4SLinus Torvalds */ 181b407621cSStephen Hemminger static u32 get_crandom(struct crndstate *state) 1821da177e4SLinus Torvalds { 1831da177e4SLinus Torvalds u64 value, rho; 1841da177e4SLinus Torvalds unsigned long answer; 1851da177e4SLinus Torvalds 1860a9fe5c3SYousuk Seung if (!state || state->rho == 0) /* no correlation */ 18763862b5bSAruna-Hewapathirane return prandom_u32(); 1881da177e4SLinus Torvalds 18963862b5bSAruna-Hewapathirane value = prandom_u32(); 1901da177e4SLinus Torvalds rho = (u64)state->rho + 1; 1911da177e4SLinus Torvalds answer = (value * ((1ull<<32) - rho) + state->last * rho) >> 32; 1921da177e4SLinus Torvalds state->last = answer; 1931da177e4SLinus Torvalds return answer; 1941da177e4SLinus Torvalds } 1951da177e4SLinus Torvalds 196661b7972Sstephen hemminger /* loss_4state - 4-state model loss generator 197661b7972Sstephen hemminger * Generates losses according to the 4-state Markov chain adopted in 198661b7972Sstephen hemminger * the GI (General and Intuitive) loss model. 199661b7972Sstephen hemminger */ 200661b7972Sstephen hemminger static bool loss_4state(struct netem_sched_data *q) 201661b7972Sstephen hemminger { 202661b7972Sstephen hemminger struct clgstate *clg = &q->clg; 20363862b5bSAruna-Hewapathirane u32 rnd = prandom_u32(); 204661b7972Sstephen hemminger 205661b7972Sstephen hemminger /* 20625985edcSLucas De Marchi * Makes a comparison between rnd and the transition 207661b7972Sstephen hemminger * probabilities outgoing from the current state, then decides the 208661b7972Sstephen hemminger * next state and if the next packet has to be transmitted or lost. 209661b7972Sstephen hemminger * The four states correspond to: 210a6e2fe17SYang Yingliang * TX_IN_GAP_PERIOD => successfully transmitted packets within a gap period 211a6e2fe17SYang Yingliang * LOST_IN_BURST_PERIOD => isolated losses within a gap period 212a6e2fe17SYang Yingliang * LOST_IN_GAP_PERIOD => lost packets within a burst period 213a6e2fe17SYang Yingliang * TX_IN_GAP_PERIOD => successfully transmitted packets within a burst period 214661b7972Sstephen hemminger */ 215661b7972Sstephen hemminger switch (clg->state) { 216a6e2fe17SYang Yingliang case TX_IN_GAP_PERIOD: 217661b7972Sstephen hemminger if (rnd < clg->a4) { 218a6e2fe17SYang Yingliang clg->state = LOST_IN_BURST_PERIOD; 219661b7972Sstephen hemminger return true; 220ab6c27beSstephen hemminger } else if (clg->a4 < rnd && rnd < clg->a1 + clg->a4) { 221a6e2fe17SYang Yingliang clg->state = LOST_IN_GAP_PERIOD; 222661b7972Sstephen hemminger return true; 223a6e2fe17SYang Yingliang } else if (clg->a1 + clg->a4 < rnd) { 224a6e2fe17SYang Yingliang clg->state = TX_IN_GAP_PERIOD; 225a6e2fe17SYang Yingliang } 226661b7972Sstephen hemminger 227661b7972Sstephen hemminger break; 228a6e2fe17SYang Yingliang case TX_IN_BURST_PERIOD: 229661b7972Sstephen hemminger if (rnd < clg->a5) { 230a6e2fe17SYang Yingliang clg->state = LOST_IN_GAP_PERIOD; 231661b7972Sstephen hemminger return true; 232a6e2fe17SYang Yingliang } else { 233a6e2fe17SYang Yingliang clg->state = TX_IN_BURST_PERIOD; 234a6e2fe17SYang Yingliang } 235661b7972Sstephen hemminger 236661b7972Sstephen hemminger break; 237a6e2fe17SYang Yingliang case LOST_IN_GAP_PERIOD: 238661b7972Sstephen hemminger if (rnd < clg->a3) 239a6e2fe17SYang Yingliang clg->state = TX_IN_BURST_PERIOD; 240661b7972Sstephen hemminger else if (clg->a3 < rnd && rnd < clg->a2 + clg->a3) { 241a6e2fe17SYang Yingliang clg->state = TX_IN_GAP_PERIOD; 242661b7972Sstephen hemminger } else if (clg->a2 + clg->a3 < rnd) { 243a6e2fe17SYang Yingliang clg->state = LOST_IN_GAP_PERIOD; 244661b7972Sstephen hemminger return true; 245661b7972Sstephen hemminger } 246661b7972Sstephen hemminger break; 247a6e2fe17SYang Yingliang case LOST_IN_BURST_PERIOD: 248a6e2fe17SYang Yingliang clg->state = TX_IN_GAP_PERIOD; 249661b7972Sstephen hemminger break; 250661b7972Sstephen hemminger } 251661b7972Sstephen hemminger 252661b7972Sstephen hemminger return false; 253661b7972Sstephen hemminger } 254661b7972Sstephen hemminger 255661b7972Sstephen hemminger /* loss_gilb_ell - Gilbert-Elliot model loss generator 256661b7972Sstephen hemminger * Generates losses according to the Gilbert-Elliot loss model or 257661b7972Sstephen hemminger * its special cases (Gilbert or Simple Gilbert) 258661b7972Sstephen hemminger * 25925985edcSLucas De Marchi * Makes a comparison between random number and the transition 260661b7972Sstephen hemminger * probabilities outgoing from the current state, then decides the 26125985edcSLucas De Marchi * next state. A second random number is extracted and the comparison 262661b7972Sstephen hemminger * with the loss probability of the current state decides if the next 263661b7972Sstephen hemminger * packet will be transmitted or lost. 264661b7972Sstephen hemminger */ 265661b7972Sstephen hemminger static bool loss_gilb_ell(struct netem_sched_data *q) 266661b7972Sstephen hemminger { 267661b7972Sstephen hemminger struct clgstate *clg = &q->clg; 268661b7972Sstephen hemminger 269661b7972Sstephen hemminger switch (clg->state) { 270c045a734SYang Yingliang case GOOD_STATE: 27163862b5bSAruna-Hewapathirane if (prandom_u32() < clg->a1) 272c045a734SYang Yingliang clg->state = BAD_STATE; 27363862b5bSAruna-Hewapathirane if (prandom_u32() < clg->a4) 274661b7972Sstephen hemminger return true; 2757c2781faSstephen hemminger break; 276c045a734SYang Yingliang case BAD_STATE: 27763862b5bSAruna-Hewapathirane if (prandom_u32() < clg->a2) 278c045a734SYang Yingliang clg->state = GOOD_STATE; 27963862b5bSAruna-Hewapathirane if (prandom_u32() > clg->a3) 280661b7972Sstephen hemminger return true; 281661b7972Sstephen hemminger } 282661b7972Sstephen hemminger 283661b7972Sstephen hemminger return false; 284661b7972Sstephen hemminger } 285661b7972Sstephen hemminger 286661b7972Sstephen hemminger static bool loss_event(struct netem_sched_data *q) 287661b7972Sstephen hemminger { 288661b7972Sstephen hemminger switch (q->loss_model) { 289661b7972Sstephen hemminger case CLG_RANDOM: 290661b7972Sstephen hemminger /* Random packet drop 0 => none, ~0 => all */ 291661b7972Sstephen hemminger return q->loss && q->loss >= get_crandom(&q->loss_cor); 292661b7972Sstephen hemminger 293661b7972Sstephen hemminger case CLG_4_STATES: 294661b7972Sstephen hemminger /* 4state loss model algorithm (used also for GI model) 295661b7972Sstephen hemminger * Extracts a value from the markov 4 state loss generator, 296661b7972Sstephen hemminger * if it is 1 drops a packet and if needed writes the event in 297661b7972Sstephen hemminger * the kernel logs 298661b7972Sstephen hemminger */ 299661b7972Sstephen hemminger return loss_4state(q); 300661b7972Sstephen hemminger 301661b7972Sstephen hemminger case CLG_GILB_ELL: 302661b7972Sstephen hemminger /* Gilbert-Elliot loss model algorithm 303661b7972Sstephen hemminger * Extracts a value from the Gilbert-Elliot loss generator, 304661b7972Sstephen hemminger * if it is 1 drops a packet and if needed writes the event in 305661b7972Sstephen hemminger * the kernel logs 306661b7972Sstephen hemminger */ 307661b7972Sstephen hemminger return loss_gilb_ell(q); 308661b7972Sstephen hemminger } 309661b7972Sstephen hemminger 310661b7972Sstephen hemminger return false; /* not reached */ 311661b7972Sstephen hemminger } 312661b7972Sstephen hemminger 313661b7972Sstephen hemminger 3141da177e4SLinus Torvalds /* tabledist - return a pseudo-randomly distributed value with mean mu and 3151da177e4SLinus Torvalds * std deviation sigma. Uses table lookup to approximate the desired 3161da177e4SLinus Torvalds * distribution, and a uniformly-distributed pseudo-random source. 3171da177e4SLinus Torvalds */ 3189b0ed891SStephen Hemminger static s64 tabledist(s64 mu, s32 sigma, 319b407621cSStephen Hemminger struct crndstate *state, 320b407621cSStephen Hemminger const struct disttable *dist) 3211da177e4SLinus Torvalds { 322112f9cb6SDave Taht s64 x; 323b407621cSStephen Hemminger long t; 324b407621cSStephen Hemminger u32 rnd; 3251da177e4SLinus Torvalds 3261da177e4SLinus Torvalds if (sigma == 0) 3271da177e4SLinus Torvalds return mu; 3281da177e4SLinus Torvalds 3291da177e4SLinus Torvalds rnd = get_crandom(state); 3301da177e4SLinus Torvalds 3311da177e4SLinus Torvalds /* default uniform distribution */ 3321da177e4SLinus Torvalds if (dist == NULL) 333043e337fSMd. Islam return ((rnd % (2 * sigma)) + mu) - sigma; 3341da177e4SLinus Torvalds 3351da177e4SLinus Torvalds t = dist->table[rnd % dist->size]; 3361da177e4SLinus Torvalds x = (sigma % NETEM_DIST_SCALE) * t; 3371da177e4SLinus Torvalds if (x >= 0) 3381da177e4SLinus Torvalds x += NETEM_DIST_SCALE/2; 3391da177e4SLinus Torvalds else 3401da177e4SLinus Torvalds x -= NETEM_DIST_SCALE/2; 3411da177e4SLinus Torvalds 3421da177e4SLinus Torvalds return x / NETEM_DIST_SCALE + (sigma / NETEM_DIST_SCALE) * t + mu; 3431da177e4SLinus Torvalds } 3441da177e4SLinus Torvalds 345bce552fdSStephen Hemminger static u64 packet_time_ns(u64 len, const struct netem_sched_data *q) 3467bc0f28cSHagen Paul Pfeifer { 34790b41a1cSHagen Paul Pfeifer len += q->packet_overhead; 34890b41a1cSHagen Paul Pfeifer 34990b41a1cSHagen Paul Pfeifer if (q->cell_size) { 35090b41a1cSHagen Paul Pfeifer u32 cells = reciprocal_divide(len, q->cell_size_reciprocal); 35190b41a1cSHagen Paul Pfeifer 35290b41a1cSHagen Paul Pfeifer if (len > cells * q->cell_size) /* extra cell needed for remainder */ 35390b41a1cSHagen Paul Pfeifer cells++; 35490b41a1cSHagen Paul Pfeifer len = cells * (q->cell_size + q->cell_overhead); 35590b41a1cSHagen Paul Pfeifer } 356bce552fdSStephen Hemminger 357bce552fdSStephen Hemminger return div64_u64(len * NSEC_PER_SEC, q->rate); 3587bc0f28cSHagen Paul Pfeifer } 3597bc0f28cSHagen Paul Pfeifer 360ff704050Sstephen hemminger static void tfifo_reset(struct Qdisc *sch) 361ff704050Sstephen hemminger { 362ff704050Sstephen hemminger struct netem_sched_data *q = qdisc_priv(sch); 3633aa605f2SEric Dumazet struct rb_node *p = rb_first(&q->t_root); 364ff704050Sstephen hemminger 3653aa605f2SEric Dumazet while (p) { 36618a4c0eaSEric Dumazet struct sk_buff *skb = rb_to_skb(p); 367ff704050Sstephen hemminger 3683aa605f2SEric Dumazet p = rb_next(p); 3693aa605f2SEric Dumazet rb_erase(&skb->rbnode, &q->t_root); 3702f08a9a1SEric Dumazet rtnl_kfree_skbs(skb, skb); 371ff704050Sstephen hemminger } 372d66280b1SPeter Oskolkov 373d66280b1SPeter Oskolkov rtnl_kfree_skbs(q->t_head, q->t_tail); 374d66280b1SPeter Oskolkov q->t_head = NULL; 375d66280b1SPeter Oskolkov q->t_tail = NULL; 376ff704050Sstephen hemminger } 377ff704050Sstephen hemminger 378960fb66eSEric Dumazet static void tfifo_enqueue(struct sk_buff *nskb, struct Qdisc *sch) 37950612537SEric Dumazet { 380aec0a40aSEric Dumazet struct netem_sched_data *q = qdisc_priv(sch); 381112f9cb6SDave Taht u64 tnext = netem_skb_cb(nskb)->time_to_send; 382d66280b1SPeter Oskolkov 383d66280b1SPeter Oskolkov if (!q->t_tail || tnext >= netem_skb_cb(q->t_tail)->time_to_send) { 384d66280b1SPeter Oskolkov if (q->t_tail) 385d66280b1SPeter Oskolkov q->t_tail->next = nskb; 386d66280b1SPeter Oskolkov else 387d66280b1SPeter Oskolkov q->t_head = nskb; 388d66280b1SPeter Oskolkov q->t_tail = nskb; 389d66280b1SPeter Oskolkov } else { 390aec0a40aSEric Dumazet struct rb_node **p = &q->t_root.rb_node, *parent = NULL; 39150612537SEric Dumazet 392aec0a40aSEric Dumazet while (*p) { 393aec0a40aSEric Dumazet struct sk_buff *skb; 39450612537SEric Dumazet 395aec0a40aSEric Dumazet parent = *p; 39618a4c0eaSEric Dumazet skb = rb_to_skb(parent); 39750612537SEric Dumazet if (tnext >= netem_skb_cb(skb)->time_to_send) 398aec0a40aSEric Dumazet p = &parent->rb_right; 399aec0a40aSEric Dumazet else 400aec0a40aSEric Dumazet p = &parent->rb_left; 40150612537SEric Dumazet } 40256b17425SEric Dumazet rb_link_node(&nskb->rbnode, parent, p); 40356b17425SEric Dumazet rb_insert_color(&nskb->rbnode, &q->t_root); 404d66280b1SPeter Oskolkov } 405aec0a40aSEric Dumazet sch->q.qlen++; 40650612537SEric Dumazet } 40750612537SEric Dumazet 4086071bd1aSNeil Horman /* netem can't properly corrupt a megapacket (like we get from GSO), so instead 4096071bd1aSNeil Horman * when we statistically choose to corrupt one, we instead segment it, returning 4106071bd1aSNeil Horman * the first packet to be corrupted, and re-enqueue the remaining frames 4116071bd1aSNeil Horman */ 412520ac30fSEric Dumazet static struct sk_buff *netem_segment(struct sk_buff *skb, struct Qdisc *sch, 413520ac30fSEric Dumazet struct sk_buff **to_free) 4146071bd1aSNeil Horman { 4156071bd1aSNeil Horman struct sk_buff *segs; 4166071bd1aSNeil Horman netdev_features_t features = netif_skb_features(skb); 4176071bd1aSNeil Horman 4186071bd1aSNeil Horman segs = skb_gso_segment(skb, features & ~NETIF_F_GSO_MASK); 4196071bd1aSNeil Horman 4206071bd1aSNeil Horman if (IS_ERR_OR_NULL(segs)) { 421520ac30fSEric Dumazet qdisc_drop(skb, sch, to_free); 4226071bd1aSNeil Horman return NULL; 4236071bd1aSNeil Horman } 4246071bd1aSNeil Horman consume_skb(skb); 4256071bd1aSNeil Horman return segs; 4266071bd1aSNeil Horman } 4276071bd1aSNeil Horman 4280afb51e7SStephen Hemminger /* 4290afb51e7SStephen Hemminger * Insert one skb into qdisc. 4300afb51e7SStephen Hemminger * Note: parent depends on return value to account for queue length. 4310afb51e7SStephen Hemminger * NET_XMIT_DROP: queue length didn't change. 4320afb51e7SStephen Hemminger * NET_XMIT_SUCCESS: one skb was queued. 4330afb51e7SStephen Hemminger */ 434520ac30fSEric Dumazet static int netem_enqueue(struct sk_buff *skb, struct Qdisc *sch, 435520ac30fSEric Dumazet struct sk_buff **to_free) 4361da177e4SLinus Torvalds { 4371da177e4SLinus Torvalds struct netem_sched_data *q = qdisc_priv(sch); 43889e1df74SGuillaume Chazarain /* We don't fill cb now as skb_unshare() may invalidate it */ 43989e1df74SGuillaume Chazarain struct netem_skb_cb *cb; 4400afb51e7SStephen Hemminger struct sk_buff *skb2; 4416071bd1aSNeil Horman struct sk_buff *segs = NULL; 442177b8007SJakub Kicinski unsigned int prev_len = qdisc_pkt_len(skb); 4430afb51e7SStephen Hemminger int count = 1; 4446071bd1aSNeil Horman int rc = NET_XMIT_SUCCESS; 4455845f706SSheng Lan int rc_drop = NET_XMIT_DROP; 4461da177e4SLinus Torvalds 4479410d386SChristoph Paasch /* Do not fool qdisc_drop_all() */ 4489410d386SChristoph Paasch skb->prev = NULL; 4499410d386SChristoph Paasch 4500afb51e7SStephen Hemminger /* Random duplication */ 4510afb51e7SStephen Hemminger if (q->duplicate && q->duplicate >= get_crandom(&q->dup_cor)) 4520afb51e7SStephen Hemminger ++count; 4530afb51e7SStephen Hemminger 454661b7972Sstephen hemminger /* Drop packet? */ 455e4ae004bSEric Dumazet if (loss_event(q)) { 456e4ae004bSEric Dumazet if (q->ecn && INET_ECN_set_ce(skb)) 45725331d6cSJohn Fastabend qdisc_qstats_drop(sch); /* mark packet */ 458e4ae004bSEric Dumazet else 4590afb51e7SStephen Hemminger --count; 460e4ae004bSEric Dumazet } 4610afb51e7SStephen Hemminger if (count == 0) { 46225331d6cSJohn Fastabend qdisc_qstats_drop(sch); 463520ac30fSEric Dumazet __qdisc_drop(skb, to_free); 464c27f339aSJarek Poplawski return NET_XMIT_SUCCESS | __NET_XMIT_BYPASS; 4651da177e4SLinus Torvalds } 4661da177e4SLinus Torvalds 4675a308f40SEric Dumazet /* If a delay is expected, orphan the skb. (orphaning usually takes 4685a308f40SEric Dumazet * place at TX completion time, so _before_ the link transit delay) 4695a308f40SEric Dumazet */ 4705080f39eSNik Unger if (q->latency || q->jitter || q->rate) 471f2f872f9SEric Dumazet skb_orphan_partial(skb); 4724e8a5201SDavid S. Miller 4730afb51e7SStephen Hemminger /* 4740afb51e7SStephen Hemminger * If we need to duplicate packet, then re-insert at top of the 4750afb51e7SStephen Hemminger * qdisc tree, since parent queuer expects that only one 4760afb51e7SStephen Hemminger * skb will be queued. 477d5d75cd6SStephen Hemminger */ 4780afb51e7SStephen Hemminger if (count > 1 && (skb2 = skb_clone(skb, GFP_ATOMIC)) != NULL) { 479159d2c7dSEric Dumazet struct Qdisc *rootq = qdisc_root_bh(sch); 4800afb51e7SStephen Hemminger u32 dupsave = q->duplicate; /* prevent duplicating a dup... */ 481d5d75cd6SStephen Hemminger 482b396cca6SEric Dumazet q->duplicate = 0; 483520ac30fSEric Dumazet rootq->enqueue(skb2, rootq, to_free); 4840afb51e7SStephen Hemminger q->duplicate = dupsave; 4855845f706SSheng Lan rc_drop = NET_XMIT_SUCCESS; 4861da177e4SLinus Torvalds } 4871da177e4SLinus Torvalds 488c865e5d9SStephen Hemminger /* 489c865e5d9SStephen Hemminger * Randomized packet corruption. 490c865e5d9SStephen Hemminger * Make copy if needed since we are modifying 491c865e5d9SStephen Hemminger * If packet is going to be hardware checksummed, then 492c865e5d9SStephen Hemminger * do it now in software before we mangle it. 493c865e5d9SStephen Hemminger */ 494c865e5d9SStephen Hemminger if (q->corrupt && q->corrupt >= get_crandom(&q->corrupt_cor)) { 4956071bd1aSNeil Horman if (skb_is_gso(skb)) { 4963e14c383SJakub Kicinski skb = netem_segment(skb, sch, to_free); 4973e14c383SJakub Kicinski if (!skb) 4985845f706SSheng Lan return rc_drop; 4993e14c383SJakub Kicinski segs = skb->next; 5003e14c383SJakub Kicinski skb_mark_not_on_list(skb); 5013e14c383SJakub Kicinski qdisc_skb_cb(skb)->pkt_len = skb->len; 5026071bd1aSNeil Horman } 5036071bd1aSNeil Horman 5048a6e9c67SEric Dumazet skb = skb_unshare(skb, GFP_ATOMIC); 5058a6e9c67SEric Dumazet if (unlikely(!skb)) { 5068a6e9c67SEric Dumazet qdisc_qstats_drop(sch); 5078a6e9c67SEric Dumazet goto finish_segs; 5088a6e9c67SEric Dumazet } 5098a6e9c67SEric Dumazet if (skb->ip_summed == CHECKSUM_PARTIAL && 5108a6e9c67SEric Dumazet skb_checksum_help(skb)) { 5118a6e9c67SEric Dumazet qdisc_drop(skb, sch, to_free); 5126071bd1aSNeil Horman goto finish_segs; 5136071bd1aSNeil Horman } 514c865e5d9SStephen Hemminger 51563862b5bSAruna-Hewapathirane skb->data[prandom_u32() % skb_headlen(skb)] ^= 51663862b5bSAruna-Hewapathirane 1<<(prandom_u32() % 8); 517c865e5d9SStephen Hemminger } 518c865e5d9SStephen Hemminger 5195845f706SSheng Lan if (unlikely(sch->q.qlen >= sch->limit)) { 5203e14c383SJakub Kicinski /* re-link segs, so that qdisc_drop_all() frees them all */ 5213e14c383SJakub Kicinski skb->next = segs; 5225845f706SSheng Lan qdisc_drop_all(skb, sch, to_free); 5235845f706SSheng Lan return rc_drop; 5245845f706SSheng Lan } 525960fb66eSEric Dumazet 52625331d6cSJohn Fastabend qdisc_qstats_backlog_inc(sch, skb); 527960fb66eSEric Dumazet 5285f86173bSJussi Kivilinna cb = netem_skb_cb(skb); 529f64f9e71SJoe Perches if (q->gap == 0 || /* not doing reordering */ 530a42b4799SVijay Subramanian q->counter < q->gap - 1 || /* inside last reordering gap */ 531f64f9e71SJoe Perches q->reorder < get_crandom(&q->reorder_cor)) { 532112f9cb6SDave Taht u64 now; 533112f9cb6SDave Taht s64 delay; 53407aaa115SStephen Hemminger 53507aaa115SStephen Hemminger delay = tabledist(q->latency, q->jitter, 53607aaa115SStephen Hemminger &q->delay_cor, q->delay_dist); 53707aaa115SStephen Hemminger 538112f9cb6SDave Taht now = ktime_get_ns(); 5397bc0f28cSHagen Paul Pfeifer 5407bc0f28cSHagen Paul Pfeifer if (q->rate) { 5415080f39eSNik Unger struct netem_skb_cb *last = NULL; 5427bc0f28cSHagen Paul Pfeifer 5435080f39eSNik Unger if (sch->q.tail) 5445080f39eSNik Unger last = netem_skb_cb(sch->q.tail); 5455080f39eSNik Unger if (q->t_root.rb_node) { 5465080f39eSNik Unger struct sk_buff *t_skb; 5475080f39eSNik Unger struct netem_skb_cb *t_last; 5485080f39eSNik Unger 54918a4c0eaSEric Dumazet t_skb = skb_rb_last(&q->t_root); 5505080f39eSNik Unger t_last = netem_skb_cb(t_skb); 5515080f39eSNik Unger if (!last || 552d66280b1SPeter Oskolkov t_last->time_to_send > last->time_to_send) 5535080f39eSNik Unger last = t_last; 5545080f39eSNik Unger } 555d66280b1SPeter Oskolkov if (q->t_tail) { 556d66280b1SPeter Oskolkov struct netem_skb_cb *t_last = 557d66280b1SPeter Oskolkov netem_skb_cb(q->t_tail); 558d66280b1SPeter Oskolkov 559d66280b1SPeter Oskolkov if (!last || 560d66280b1SPeter Oskolkov t_last->time_to_send > last->time_to_send) 561d66280b1SPeter Oskolkov last = t_last; 5625080f39eSNik Unger } 5635080f39eSNik Unger 564aec0a40aSEric Dumazet if (last) { 5657bc0f28cSHagen Paul Pfeifer /* 566a13d3104SJohannes Naab * Last packet in queue is reference point (now), 567a13d3104SJohannes Naab * calculate this time bonus and subtract 5687bc0f28cSHagen Paul Pfeifer * from delay. 5697bc0f28cSHagen Paul Pfeifer */ 5705080f39eSNik Unger delay -= last->time_to_send - now; 571112f9cb6SDave Taht delay = max_t(s64, 0, delay); 5725080f39eSNik Unger now = last->time_to_send; 5737bc0f28cSHagen Paul Pfeifer } 574a13d3104SJohannes Naab 575bce552fdSStephen Hemminger delay += packet_time_ns(qdisc_pkt_len(skb), q); 5767bc0f28cSHagen Paul Pfeifer } 5777bc0f28cSHagen Paul Pfeifer 5787c59e25fSPatrick McHardy cb->time_to_send = now + delay; 5791da177e4SLinus Torvalds ++q->counter; 580960fb66eSEric Dumazet tfifo_enqueue(skb, sch); 5811da177e4SLinus Torvalds } else { 5820dca51d3SStephen Hemminger /* 5830dca51d3SStephen Hemminger * Do re-ordering by putting one out of N packets at the front 5840dca51d3SStephen Hemminger * of the queue. 5850dca51d3SStephen Hemminger */ 586112f9cb6SDave Taht cb->time_to_send = ktime_get_ns(); 5870dca51d3SStephen Hemminger q->counter = 0; 5888ba25dadSJarek Poplawski 58959697730SDavid S. Miller __qdisc_enqueue_head(skb, &sch->q); 590eb101924SHagen Paul Pfeifer sch->qstats.requeues++; 591378a2f09SJarek Poplawski } 5921da177e4SLinus Torvalds 5936071bd1aSNeil Horman finish_segs: 5946071bd1aSNeil Horman if (segs) { 595177b8007SJakub Kicinski unsigned int len, last_len; 596177b8007SJakub Kicinski int nb = 0; 597177b8007SJakub Kicinski 598177b8007SJakub Kicinski len = skb->len; 599177b8007SJakub Kicinski 6006071bd1aSNeil Horman while (segs) { 6016071bd1aSNeil Horman skb2 = segs->next; 602a8305bffSDavid S. Miller skb_mark_not_on_list(segs); 6036071bd1aSNeil Horman qdisc_skb_cb(segs)->pkt_len = segs->len; 6046071bd1aSNeil Horman last_len = segs->len; 605520ac30fSEric Dumazet rc = qdisc_enqueue(segs, sch, to_free); 6066071bd1aSNeil Horman if (rc != NET_XMIT_SUCCESS) { 6076071bd1aSNeil Horman if (net_xmit_drop_count(rc)) 6086071bd1aSNeil Horman qdisc_qstats_drop(sch); 6096071bd1aSNeil Horman } else { 6106071bd1aSNeil Horman nb++; 6116071bd1aSNeil Horman len += last_len; 6126071bd1aSNeil Horman } 6136071bd1aSNeil Horman segs = skb2; 6146071bd1aSNeil Horman } 615177b8007SJakub Kicinski qdisc_tree_reduce_backlog(sch, -nb, prev_len - len); 6166071bd1aSNeil Horman } 61710f6dfcfSstephen hemminger return NET_XMIT_SUCCESS; 6181da177e4SLinus Torvalds } 6191da177e4SLinus Torvalds 620836af83bSDave Taht /* Delay the next round with a new future slot with a 621836af83bSDave Taht * correct number of bytes and packets. 622836af83bSDave Taht */ 623836af83bSDave Taht 624836af83bSDave Taht static void get_slot_next(struct netem_sched_data *q, u64 now) 625836af83bSDave Taht { 6260a9fe5c3SYousuk Seung s64 next_delay; 6270a9fe5c3SYousuk Seung 6280a9fe5c3SYousuk Seung if (!q->slot_dist) 6290a9fe5c3SYousuk Seung next_delay = q->slot_config.min_delay + 630836af83bSDave Taht (prandom_u32() * 631836af83bSDave Taht (q->slot_config.max_delay - 632836af83bSDave Taht q->slot_config.min_delay) >> 32); 6330a9fe5c3SYousuk Seung else 6340a9fe5c3SYousuk Seung next_delay = tabledist(q->slot_config.dist_delay, 6350a9fe5c3SYousuk Seung (s32)(q->slot_config.dist_jitter), 6360a9fe5c3SYousuk Seung NULL, q->slot_dist); 6370a9fe5c3SYousuk Seung 6380a9fe5c3SYousuk Seung q->slot.slot_next = now + next_delay; 639836af83bSDave Taht q->slot.packets_left = q->slot_config.max_packets; 640836af83bSDave Taht q->slot.bytes_left = q->slot_config.max_bytes; 641836af83bSDave Taht } 642836af83bSDave Taht 643d66280b1SPeter Oskolkov static struct sk_buff *netem_peek(struct netem_sched_data *q) 644d66280b1SPeter Oskolkov { 645d66280b1SPeter Oskolkov struct sk_buff *skb = skb_rb_first(&q->t_root); 646d66280b1SPeter Oskolkov u64 t1, t2; 647d66280b1SPeter Oskolkov 648d66280b1SPeter Oskolkov if (!skb) 649d66280b1SPeter Oskolkov return q->t_head; 650d66280b1SPeter Oskolkov if (!q->t_head) 651d66280b1SPeter Oskolkov return skb; 652d66280b1SPeter Oskolkov 653d66280b1SPeter Oskolkov t1 = netem_skb_cb(skb)->time_to_send; 654d66280b1SPeter Oskolkov t2 = netem_skb_cb(q->t_head)->time_to_send; 655d66280b1SPeter Oskolkov if (t1 < t2) 656d66280b1SPeter Oskolkov return skb; 657d66280b1SPeter Oskolkov return q->t_head; 658d66280b1SPeter Oskolkov } 659d66280b1SPeter Oskolkov 660d66280b1SPeter Oskolkov static void netem_erase_head(struct netem_sched_data *q, struct sk_buff *skb) 661d66280b1SPeter Oskolkov { 662d66280b1SPeter Oskolkov if (skb == q->t_head) { 663d66280b1SPeter Oskolkov q->t_head = skb->next; 664d66280b1SPeter Oskolkov if (!q->t_head) 665d66280b1SPeter Oskolkov q->t_tail = NULL; 666d66280b1SPeter Oskolkov } else { 667d66280b1SPeter Oskolkov rb_erase(&skb->rbnode, &q->t_root); 668d66280b1SPeter Oskolkov } 669d66280b1SPeter Oskolkov } 670d66280b1SPeter Oskolkov 6711da177e4SLinus Torvalds static struct sk_buff *netem_dequeue(struct Qdisc *sch) 6721da177e4SLinus Torvalds { 6731da177e4SLinus Torvalds struct netem_sched_data *q = qdisc_priv(sch); 6741da177e4SLinus Torvalds struct sk_buff *skb; 6751da177e4SLinus Torvalds 67650612537SEric Dumazet tfifo_dequeue: 677ed760cb8SFlorian Westphal skb = __qdisc_dequeue_head(&sch->q); 678771018e7SStephen Hemminger if (skb) { 67925331d6cSJohn Fastabend qdisc_qstats_backlog_dec(sch, skb); 6800ad2a836SBeshay, Joseph deliver: 681aec0a40aSEric Dumazet qdisc_bstats_update(sch, skb); 682aec0a40aSEric Dumazet return skb; 683aec0a40aSEric Dumazet } 684d66280b1SPeter Oskolkov skb = netem_peek(q); 685d66280b1SPeter Oskolkov if (skb) { 686112f9cb6SDave Taht u64 time_to_send; 687836af83bSDave Taht u64 now = ktime_get_ns(); 68836b7bfe0SEric Dumazet 6890f9f32acSStephen Hemminger /* if more time remaining? */ 69036b7bfe0SEric Dumazet time_to_send = netem_skb_cb(skb)->time_to_send; 691836af83bSDave Taht if (q->slot.slot_next && q->slot.slot_next < time_to_send) 692836af83bSDave Taht get_slot_next(q, now); 693aec0a40aSEric Dumazet 694836af83bSDave Taht if (time_to_send <= now && q->slot.slot_next <= now) { 695d66280b1SPeter Oskolkov netem_erase_head(q, skb); 696aec0a40aSEric Dumazet sch->q.qlen--; 6970ad2a836SBeshay, Joseph qdisc_qstats_backlog_dec(sch, skb); 698aec0a40aSEric Dumazet skb->next = NULL; 699aec0a40aSEric Dumazet skb->prev = NULL; 700bffa72cfSEric Dumazet /* skb->dev shares skb->rbnode area, 701bffa72cfSEric Dumazet * we need to restore its value. 702bffa72cfSEric Dumazet */ 703bffa72cfSEric Dumazet skb->dev = qdisc_dev(sch); 70403c05f0dSJarek Poplawski 705836af83bSDave Taht if (q->slot.slot_next) { 706836af83bSDave Taht q->slot.packets_left--; 707836af83bSDave Taht q->slot.bytes_left -= qdisc_pkt_len(skb); 708836af83bSDave Taht if (q->slot.packets_left <= 0 || 709836af83bSDave Taht q->slot.bytes_left <= 0) 710836af83bSDave Taht get_slot_next(q, now); 711836af83bSDave Taht } 712836af83bSDave Taht 71350612537SEric Dumazet if (q->qdisc) { 71421de12eeSEric Dumazet unsigned int pkt_len = qdisc_pkt_len(skb); 715520ac30fSEric Dumazet struct sk_buff *to_free = NULL; 716520ac30fSEric Dumazet int err; 71750612537SEric Dumazet 718520ac30fSEric Dumazet err = qdisc_enqueue(skb, q->qdisc, &to_free); 719520ac30fSEric Dumazet kfree_skb_list(to_free); 72021de12eeSEric Dumazet if (err != NET_XMIT_SUCCESS && 72121de12eeSEric Dumazet net_xmit_drop_count(err)) { 72225331d6cSJohn Fastabend qdisc_qstats_drop(sch); 7232ccccf5fSWANG Cong qdisc_tree_reduce_backlog(sch, 1, 72421de12eeSEric Dumazet pkt_len); 72550612537SEric Dumazet } 72650612537SEric Dumazet goto tfifo_dequeue; 72750612537SEric Dumazet } 728aec0a40aSEric Dumazet goto deliver; 72911274e5aSStephen Hemminger } 73007aaa115SStephen Hemminger 73150612537SEric Dumazet if (q->qdisc) { 73250612537SEric Dumazet skb = q->qdisc->ops->dequeue(q->qdisc); 73350612537SEric Dumazet if (skb) 73450612537SEric Dumazet goto deliver; 73550612537SEric Dumazet } 736836af83bSDave Taht 737836af83bSDave Taht qdisc_watchdog_schedule_ns(&q->watchdog, 738836af83bSDave Taht max(time_to_send, 739836af83bSDave Taht q->slot.slot_next)); 7400f9f32acSStephen Hemminger } 7410f9f32acSStephen Hemminger 74250612537SEric Dumazet if (q->qdisc) { 74350612537SEric Dumazet skb = q->qdisc->ops->dequeue(q->qdisc); 74450612537SEric Dumazet if (skb) 74550612537SEric Dumazet goto deliver; 74650612537SEric Dumazet } 7470f9f32acSStephen Hemminger return NULL; 7481da177e4SLinus Torvalds } 7491da177e4SLinus Torvalds 7501da177e4SLinus Torvalds static void netem_reset(struct Qdisc *sch) 7511da177e4SLinus Torvalds { 7521da177e4SLinus Torvalds struct netem_sched_data *q = qdisc_priv(sch); 7531da177e4SLinus Torvalds 75450612537SEric Dumazet qdisc_reset_queue(sch); 755ff704050Sstephen hemminger tfifo_reset(sch); 75650612537SEric Dumazet if (q->qdisc) 7571da177e4SLinus Torvalds qdisc_reset(q->qdisc); 75859cb5c67SPatrick McHardy qdisc_watchdog_cancel(&q->watchdog); 7591da177e4SLinus Torvalds } 7601da177e4SLinus Torvalds 7616373a9a2Sstephen hemminger static void dist_free(struct disttable *d) 7626373a9a2Sstephen hemminger { 7634cb28970SWANG Cong kvfree(d); 7646373a9a2Sstephen hemminger } 7656373a9a2Sstephen hemminger 7661da177e4SLinus Torvalds /* 7671da177e4SLinus Torvalds * Distribution data is a variable size payload containing 7681da177e4SLinus Torvalds * signed 16 bit values. 7691da177e4SLinus Torvalds */ 770836af83bSDave Taht 7710a9fe5c3SYousuk Seung static int get_dist_table(struct Qdisc *sch, struct disttable **tbl, 7720a9fe5c3SYousuk Seung const struct nlattr *attr) 7731da177e4SLinus Torvalds { 7746373a9a2Sstephen hemminger size_t n = nla_len(attr)/sizeof(__s16); 7751e90474cSPatrick McHardy const __s16 *data = nla_data(attr); 7767698b4fcSDavid S. Miller spinlock_t *root_lock; 7771da177e4SLinus Torvalds struct disttable *d; 7781da177e4SLinus Torvalds int i; 7791da177e4SLinus Torvalds 780b41d936bSEric Dumazet if (!n || n > NETEM_DIST_MAX) 7811da177e4SLinus Torvalds return -EINVAL; 7821da177e4SLinus Torvalds 783752ade68SMichal Hocko d = kvmalloc(sizeof(struct disttable) + n * sizeof(s16), GFP_KERNEL); 7841da177e4SLinus Torvalds if (!d) 7851da177e4SLinus Torvalds return -ENOMEM; 7861da177e4SLinus Torvalds 7871da177e4SLinus Torvalds d->size = n; 7881da177e4SLinus Torvalds for (i = 0; i < n; i++) 7891da177e4SLinus Torvalds d->table[i] = data[i]; 7901da177e4SLinus Torvalds 791102396aeSJarek Poplawski root_lock = qdisc_root_sleeping_lock(sch); 7927698b4fcSDavid S. Miller 7937698b4fcSDavid S. Miller spin_lock_bh(root_lock); 7940a9fe5c3SYousuk Seung swap(*tbl, d); 7957698b4fcSDavid S. Miller spin_unlock_bh(root_lock); 796bb52c7acSEric Dumazet 797bb52c7acSEric Dumazet dist_free(d); 7981da177e4SLinus Torvalds return 0; 7991da177e4SLinus Torvalds } 8001da177e4SLinus Torvalds 801836af83bSDave Taht static void get_slot(struct netem_sched_data *q, const struct nlattr *attr) 802836af83bSDave Taht { 803836af83bSDave Taht const struct tc_netem_slot *c = nla_data(attr); 804836af83bSDave Taht 805836af83bSDave Taht q->slot_config = *c; 806836af83bSDave Taht if (q->slot_config.max_packets == 0) 807836af83bSDave Taht q->slot_config.max_packets = INT_MAX; 808836af83bSDave Taht if (q->slot_config.max_bytes == 0) 809836af83bSDave Taht q->slot_config.max_bytes = INT_MAX; 810836af83bSDave Taht q->slot.packets_left = q->slot_config.max_packets; 811836af83bSDave Taht q->slot.bytes_left = q->slot_config.max_bytes; 8120a9fe5c3SYousuk Seung if (q->slot_config.min_delay | q->slot_config.max_delay | 8130a9fe5c3SYousuk Seung q->slot_config.dist_jitter) 814836af83bSDave Taht q->slot.slot_next = ktime_get_ns(); 815836af83bSDave Taht else 816836af83bSDave Taht q->slot.slot_next = 0; 817836af83bSDave Taht } 818836af83bSDave Taht 81949545a77SYang Yingliang static void get_correlation(struct netem_sched_data *q, const struct nlattr *attr) 8201da177e4SLinus Torvalds { 8211e90474cSPatrick McHardy const struct tc_netem_corr *c = nla_data(attr); 8221da177e4SLinus Torvalds 8231da177e4SLinus Torvalds init_crandom(&q->delay_cor, c->delay_corr); 8241da177e4SLinus Torvalds init_crandom(&q->loss_cor, c->loss_corr); 8251da177e4SLinus Torvalds init_crandom(&q->dup_cor, c->dup_corr); 8261da177e4SLinus Torvalds } 8271da177e4SLinus Torvalds 82849545a77SYang Yingliang static void get_reorder(struct netem_sched_data *q, const struct nlattr *attr) 8290dca51d3SStephen Hemminger { 8301e90474cSPatrick McHardy const struct tc_netem_reorder *r = nla_data(attr); 8310dca51d3SStephen Hemminger 8320dca51d3SStephen Hemminger q->reorder = r->probability; 8330dca51d3SStephen Hemminger init_crandom(&q->reorder_cor, r->correlation); 8340dca51d3SStephen Hemminger } 8350dca51d3SStephen Hemminger 83649545a77SYang Yingliang static void get_corrupt(struct netem_sched_data *q, const struct nlattr *attr) 837c865e5d9SStephen Hemminger { 8381e90474cSPatrick McHardy const struct tc_netem_corrupt *r = nla_data(attr); 839c865e5d9SStephen Hemminger 840c865e5d9SStephen Hemminger q->corrupt = r->probability; 841c865e5d9SStephen Hemminger init_crandom(&q->corrupt_cor, r->correlation); 842c865e5d9SStephen Hemminger } 843c865e5d9SStephen Hemminger 84449545a77SYang Yingliang static void get_rate(struct netem_sched_data *q, const struct nlattr *attr) 8457bc0f28cSHagen Paul Pfeifer { 8467bc0f28cSHagen Paul Pfeifer const struct tc_netem_rate *r = nla_data(attr); 8477bc0f28cSHagen Paul Pfeifer 8487bc0f28cSHagen Paul Pfeifer q->rate = r->rate; 84990b41a1cSHagen Paul Pfeifer q->packet_overhead = r->packet_overhead; 85090b41a1cSHagen Paul Pfeifer q->cell_size = r->cell_size; 851809fa972SHannes Frederic Sowa q->cell_overhead = r->cell_overhead; 85290b41a1cSHagen Paul Pfeifer if (q->cell_size) 85390b41a1cSHagen Paul Pfeifer q->cell_size_reciprocal = reciprocal_value(q->cell_size); 854809fa972SHannes Frederic Sowa else 855809fa972SHannes Frederic Sowa q->cell_size_reciprocal = (struct reciprocal_value) { 0 }; 8567bc0f28cSHagen Paul Pfeifer } 8577bc0f28cSHagen Paul Pfeifer 85849545a77SYang Yingliang static int get_loss_clg(struct netem_sched_data *q, const struct nlattr *attr) 859661b7972Sstephen hemminger { 860661b7972Sstephen hemminger const struct nlattr *la; 861661b7972Sstephen hemminger int rem; 862661b7972Sstephen hemminger 863661b7972Sstephen hemminger nla_for_each_nested(la, attr, rem) { 864661b7972Sstephen hemminger u16 type = nla_type(la); 865661b7972Sstephen hemminger 866661b7972Sstephen hemminger switch (type) { 867661b7972Sstephen hemminger case NETEM_LOSS_GI: { 868661b7972Sstephen hemminger const struct tc_netem_gimodel *gi = nla_data(la); 869661b7972Sstephen hemminger 8702494654dSstephen hemminger if (nla_len(la) < sizeof(struct tc_netem_gimodel)) { 871661b7972Sstephen hemminger pr_info("netem: incorrect gi model size\n"); 872661b7972Sstephen hemminger return -EINVAL; 873661b7972Sstephen hemminger } 874661b7972Sstephen hemminger 875661b7972Sstephen hemminger q->loss_model = CLG_4_STATES; 876661b7972Sstephen hemminger 8773fbac2a8SYang Yingliang q->clg.state = TX_IN_GAP_PERIOD; 878661b7972Sstephen hemminger q->clg.a1 = gi->p13; 879661b7972Sstephen hemminger q->clg.a2 = gi->p31; 880661b7972Sstephen hemminger q->clg.a3 = gi->p32; 881661b7972Sstephen hemminger q->clg.a4 = gi->p14; 882661b7972Sstephen hemminger q->clg.a5 = gi->p23; 883661b7972Sstephen hemminger break; 884661b7972Sstephen hemminger } 885661b7972Sstephen hemminger 886661b7972Sstephen hemminger case NETEM_LOSS_GE: { 887661b7972Sstephen hemminger const struct tc_netem_gemodel *ge = nla_data(la); 888661b7972Sstephen hemminger 8892494654dSstephen hemminger if (nla_len(la) < sizeof(struct tc_netem_gemodel)) { 8902494654dSstephen hemminger pr_info("netem: incorrect ge model size\n"); 891661b7972Sstephen hemminger return -EINVAL; 892661b7972Sstephen hemminger } 893661b7972Sstephen hemminger 894661b7972Sstephen hemminger q->loss_model = CLG_GILB_ELL; 8953fbac2a8SYang Yingliang q->clg.state = GOOD_STATE; 896661b7972Sstephen hemminger q->clg.a1 = ge->p; 897661b7972Sstephen hemminger q->clg.a2 = ge->r; 898661b7972Sstephen hemminger q->clg.a3 = ge->h; 899661b7972Sstephen hemminger q->clg.a4 = ge->k1; 900661b7972Sstephen hemminger break; 901661b7972Sstephen hemminger } 902661b7972Sstephen hemminger 903661b7972Sstephen hemminger default: 904661b7972Sstephen hemminger pr_info("netem: unknown loss type %u\n", type); 905661b7972Sstephen hemminger return -EINVAL; 906661b7972Sstephen hemminger } 907661b7972Sstephen hemminger } 908661b7972Sstephen hemminger 909661b7972Sstephen hemminger return 0; 910661b7972Sstephen hemminger } 911661b7972Sstephen hemminger 91227a3421eSPatrick McHardy static const struct nla_policy netem_policy[TCA_NETEM_MAX + 1] = { 91327a3421eSPatrick McHardy [TCA_NETEM_CORR] = { .len = sizeof(struct tc_netem_corr) }, 91427a3421eSPatrick McHardy [TCA_NETEM_REORDER] = { .len = sizeof(struct tc_netem_reorder) }, 91527a3421eSPatrick McHardy [TCA_NETEM_CORRUPT] = { .len = sizeof(struct tc_netem_corrupt) }, 9167bc0f28cSHagen Paul Pfeifer [TCA_NETEM_RATE] = { .len = sizeof(struct tc_netem_rate) }, 917661b7972Sstephen hemminger [TCA_NETEM_LOSS] = { .type = NLA_NESTED }, 918e4ae004bSEric Dumazet [TCA_NETEM_ECN] = { .type = NLA_U32 }, 9196a031f67SYang Yingliang [TCA_NETEM_RATE64] = { .type = NLA_U64 }, 92099803171SDave Taht [TCA_NETEM_LATENCY64] = { .type = NLA_S64 }, 92199803171SDave Taht [TCA_NETEM_JITTER64] = { .type = NLA_S64 }, 922836af83bSDave Taht [TCA_NETEM_SLOT] = { .len = sizeof(struct tc_netem_slot) }, 92327a3421eSPatrick McHardy }; 92427a3421eSPatrick McHardy 9252c10b32bSThomas Graf static int parse_attr(struct nlattr *tb[], int maxtype, struct nlattr *nla, 9262c10b32bSThomas Graf const struct nla_policy *policy, int len) 9272c10b32bSThomas Graf { 9282c10b32bSThomas Graf int nested_len = nla_len(nla) - NLA_ALIGN(len); 9292c10b32bSThomas Graf 930661b7972Sstephen hemminger if (nested_len < 0) { 931661b7972Sstephen hemminger pr_info("netem: invalid attributes len %d\n", nested_len); 9322c10b32bSThomas Graf return -EINVAL; 933661b7972Sstephen hemminger } 934661b7972Sstephen hemminger 9352c10b32bSThomas Graf if (nested_len >= nla_attr_size(0)) 9368cb08174SJohannes Berg return nla_parse_deprecated(tb, maxtype, 9378cb08174SJohannes Berg nla_data(nla) + NLA_ALIGN(len), 938fceb6435SJohannes Berg nested_len, policy, NULL); 939661b7972Sstephen hemminger 9402c10b32bSThomas Graf memset(tb, 0, sizeof(struct nlattr *) * (maxtype + 1)); 9412c10b32bSThomas Graf return 0; 9422c10b32bSThomas Graf } 9432c10b32bSThomas Graf 944c865e5d9SStephen Hemminger /* Parse netlink message to set options */ 9452030721cSAlexander Aring static int netem_change(struct Qdisc *sch, struct nlattr *opt, 9462030721cSAlexander Aring struct netlink_ext_ack *extack) 9471da177e4SLinus Torvalds { 9481da177e4SLinus Torvalds struct netem_sched_data *q = qdisc_priv(sch); 949b03f4672SPatrick McHardy struct nlattr *tb[TCA_NETEM_MAX + 1]; 9501da177e4SLinus Torvalds struct tc_netem_qopt *qopt; 95154a4b05cSYang Yingliang struct clgstate old_clg; 95254a4b05cSYang Yingliang int old_loss_model = CLG_RANDOM; 9531da177e4SLinus Torvalds int ret; 9541da177e4SLinus Torvalds 955b03f4672SPatrick McHardy if (opt == NULL) 9561da177e4SLinus Torvalds return -EINVAL; 9571da177e4SLinus Torvalds 9582c10b32bSThomas Graf qopt = nla_data(opt); 9592c10b32bSThomas Graf ret = parse_attr(tb, TCA_NETEM_MAX, opt, netem_policy, sizeof(*qopt)); 960b03f4672SPatrick McHardy if (ret < 0) 961b03f4672SPatrick McHardy return ret; 962b03f4672SPatrick McHardy 96354a4b05cSYang Yingliang /* backup q->clg and q->loss_model */ 96454a4b05cSYang Yingliang old_clg = q->clg; 96554a4b05cSYang Yingliang old_loss_model = q->loss_model; 96654a4b05cSYang Yingliang 96754a4b05cSYang Yingliang if (tb[TCA_NETEM_LOSS]) { 96849545a77SYang Yingliang ret = get_loss_clg(q, tb[TCA_NETEM_LOSS]); 96954a4b05cSYang Yingliang if (ret) { 97054a4b05cSYang Yingliang q->loss_model = old_loss_model; 97154a4b05cSYang Yingliang return ret; 97254a4b05cSYang Yingliang } 97354a4b05cSYang Yingliang } else { 97454a4b05cSYang Yingliang q->loss_model = CLG_RANDOM; 97554a4b05cSYang Yingliang } 97654a4b05cSYang Yingliang 97754a4b05cSYang Yingliang if (tb[TCA_NETEM_DELAY_DIST]) { 9780a9fe5c3SYousuk Seung ret = get_dist_table(sch, &q->delay_dist, 9790a9fe5c3SYousuk Seung tb[TCA_NETEM_DELAY_DIST]); 9800a9fe5c3SYousuk Seung if (ret) 9810a9fe5c3SYousuk Seung goto get_table_failure; 98254a4b05cSYang Yingliang } 9830a9fe5c3SYousuk Seung 9840a9fe5c3SYousuk Seung if (tb[TCA_NETEM_SLOT_DIST]) { 9850a9fe5c3SYousuk Seung ret = get_dist_table(sch, &q->slot_dist, 9860a9fe5c3SYousuk Seung tb[TCA_NETEM_SLOT_DIST]); 9870a9fe5c3SYousuk Seung if (ret) 9880a9fe5c3SYousuk Seung goto get_table_failure; 98954a4b05cSYang Yingliang } 99054a4b05cSYang Yingliang 99150612537SEric Dumazet sch->limit = qopt->limit; 9921da177e4SLinus Torvalds 993112f9cb6SDave Taht q->latency = PSCHED_TICKS2NS(qopt->latency); 994112f9cb6SDave Taht q->jitter = PSCHED_TICKS2NS(qopt->jitter); 9951da177e4SLinus Torvalds q->limit = qopt->limit; 9961da177e4SLinus Torvalds q->gap = qopt->gap; 9970dca51d3SStephen Hemminger q->counter = 0; 9981da177e4SLinus Torvalds q->loss = qopt->loss; 9991da177e4SLinus Torvalds q->duplicate = qopt->duplicate; 10001da177e4SLinus Torvalds 1001bb2f8cc0SStephen Hemminger /* for compatibility with earlier versions. 1002bb2f8cc0SStephen Hemminger * if gap is set, need to assume 100% probability 10030dca51d3SStephen Hemminger */ 1004a362e0a7SStephen Hemminger if (q->gap) 10050dca51d3SStephen Hemminger q->reorder = ~0; 10060dca51d3SStephen Hemminger 1007265eb67fSStephen Hemminger if (tb[TCA_NETEM_CORR]) 100849545a77SYang Yingliang get_correlation(q, tb[TCA_NETEM_CORR]); 10091da177e4SLinus Torvalds 1010265eb67fSStephen Hemminger if (tb[TCA_NETEM_REORDER]) 101149545a77SYang Yingliang get_reorder(q, tb[TCA_NETEM_REORDER]); 10121da177e4SLinus Torvalds 1013265eb67fSStephen Hemminger if (tb[TCA_NETEM_CORRUPT]) 101449545a77SYang Yingliang get_corrupt(q, tb[TCA_NETEM_CORRUPT]); 10151da177e4SLinus Torvalds 10167bc0f28cSHagen Paul Pfeifer if (tb[TCA_NETEM_RATE]) 101749545a77SYang Yingliang get_rate(q, tb[TCA_NETEM_RATE]); 10187bc0f28cSHagen Paul Pfeifer 10196a031f67SYang Yingliang if (tb[TCA_NETEM_RATE64]) 10206a031f67SYang Yingliang q->rate = max_t(u64, q->rate, 10216a031f67SYang Yingliang nla_get_u64(tb[TCA_NETEM_RATE64])); 10226a031f67SYang Yingliang 102399803171SDave Taht if (tb[TCA_NETEM_LATENCY64]) 102499803171SDave Taht q->latency = nla_get_s64(tb[TCA_NETEM_LATENCY64]); 102599803171SDave Taht 102699803171SDave Taht if (tb[TCA_NETEM_JITTER64]) 102799803171SDave Taht q->jitter = nla_get_s64(tb[TCA_NETEM_JITTER64]); 102899803171SDave Taht 1029e4ae004bSEric Dumazet if (tb[TCA_NETEM_ECN]) 1030e4ae004bSEric Dumazet q->ecn = nla_get_u32(tb[TCA_NETEM_ECN]); 1031e4ae004bSEric Dumazet 1032836af83bSDave Taht if (tb[TCA_NETEM_SLOT]) 1033836af83bSDave Taht get_slot(q, tb[TCA_NETEM_SLOT]); 1034836af83bSDave Taht 1035661b7972Sstephen hemminger return ret; 10360a9fe5c3SYousuk Seung 10370a9fe5c3SYousuk Seung get_table_failure: 10380a9fe5c3SYousuk Seung /* recover clg and loss_model, in case of 10390a9fe5c3SYousuk Seung * q->clg and q->loss_model were modified 10400a9fe5c3SYousuk Seung * in get_loss_clg() 10410a9fe5c3SYousuk Seung */ 10420a9fe5c3SYousuk Seung q->clg = old_clg; 10430a9fe5c3SYousuk Seung q->loss_model = old_loss_model; 10440a9fe5c3SYousuk Seung return ret; 10451da177e4SLinus Torvalds } 10461da177e4SLinus Torvalds 1047e63d7dfdSAlexander Aring static int netem_init(struct Qdisc *sch, struct nlattr *opt, 1048e63d7dfdSAlexander Aring struct netlink_ext_ack *extack) 10491da177e4SLinus Torvalds { 10501da177e4SLinus Torvalds struct netem_sched_data *q = qdisc_priv(sch); 10511da177e4SLinus Torvalds int ret; 10521da177e4SLinus Torvalds 1053634576a1SNikolay Aleksandrov qdisc_watchdog_init(&q->watchdog, sch); 1054634576a1SNikolay Aleksandrov 10551da177e4SLinus Torvalds if (!opt) 10561da177e4SLinus Torvalds return -EINVAL; 10571da177e4SLinus Torvalds 1058661b7972Sstephen hemminger q->loss_model = CLG_RANDOM; 10592030721cSAlexander Aring ret = netem_change(sch, opt, extack); 106050612537SEric Dumazet if (ret) 1061250a65f7Sstephen hemminger pr_info("netem: change failed\n"); 10621da177e4SLinus Torvalds return ret; 10631da177e4SLinus Torvalds } 10641da177e4SLinus Torvalds 10651da177e4SLinus Torvalds static void netem_destroy(struct Qdisc *sch) 10661da177e4SLinus Torvalds { 10671da177e4SLinus Torvalds struct netem_sched_data *q = qdisc_priv(sch); 10681da177e4SLinus Torvalds 106959cb5c67SPatrick McHardy qdisc_watchdog_cancel(&q->watchdog); 107050612537SEric Dumazet if (q->qdisc) 107186bd446bSVlad Buslov qdisc_put(q->qdisc); 10726373a9a2Sstephen hemminger dist_free(q->delay_dist); 10730a9fe5c3SYousuk Seung dist_free(q->slot_dist); 10741da177e4SLinus Torvalds } 10751da177e4SLinus Torvalds 1076661b7972Sstephen hemminger static int dump_loss_model(const struct netem_sched_data *q, 1077661b7972Sstephen hemminger struct sk_buff *skb) 1078661b7972Sstephen hemminger { 1079661b7972Sstephen hemminger struct nlattr *nest; 1080661b7972Sstephen hemminger 1081ae0be8deSMichal Kubecek nest = nla_nest_start_noflag(skb, TCA_NETEM_LOSS); 1082661b7972Sstephen hemminger if (nest == NULL) 1083661b7972Sstephen hemminger goto nla_put_failure; 1084661b7972Sstephen hemminger 1085661b7972Sstephen hemminger switch (q->loss_model) { 1086661b7972Sstephen hemminger case CLG_RANDOM: 1087661b7972Sstephen hemminger /* legacy loss model */ 1088661b7972Sstephen hemminger nla_nest_cancel(skb, nest); 1089661b7972Sstephen hemminger return 0; /* no data */ 1090661b7972Sstephen hemminger 1091661b7972Sstephen hemminger case CLG_4_STATES: { 1092661b7972Sstephen hemminger struct tc_netem_gimodel gi = { 1093661b7972Sstephen hemminger .p13 = q->clg.a1, 1094661b7972Sstephen hemminger .p31 = q->clg.a2, 1095661b7972Sstephen hemminger .p32 = q->clg.a3, 1096661b7972Sstephen hemminger .p14 = q->clg.a4, 1097661b7972Sstephen hemminger .p23 = q->clg.a5, 1098661b7972Sstephen hemminger }; 1099661b7972Sstephen hemminger 11001b34ec43SDavid S. Miller if (nla_put(skb, NETEM_LOSS_GI, sizeof(gi), &gi)) 11011b34ec43SDavid S. Miller goto nla_put_failure; 1102661b7972Sstephen hemminger break; 1103661b7972Sstephen hemminger } 1104661b7972Sstephen hemminger case CLG_GILB_ELL: { 1105661b7972Sstephen hemminger struct tc_netem_gemodel ge = { 1106661b7972Sstephen hemminger .p = q->clg.a1, 1107661b7972Sstephen hemminger .r = q->clg.a2, 1108661b7972Sstephen hemminger .h = q->clg.a3, 1109661b7972Sstephen hemminger .k1 = q->clg.a4, 1110661b7972Sstephen hemminger }; 1111661b7972Sstephen hemminger 11121b34ec43SDavid S. Miller if (nla_put(skb, NETEM_LOSS_GE, sizeof(ge), &ge)) 11131b34ec43SDavid S. Miller goto nla_put_failure; 1114661b7972Sstephen hemminger break; 1115661b7972Sstephen hemminger } 1116661b7972Sstephen hemminger } 1117661b7972Sstephen hemminger 1118661b7972Sstephen hemminger nla_nest_end(skb, nest); 1119661b7972Sstephen hemminger return 0; 1120661b7972Sstephen hemminger 1121661b7972Sstephen hemminger nla_put_failure: 1122661b7972Sstephen hemminger nla_nest_cancel(skb, nest); 1123661b7972Sstephen hemminger return -1; 1124661b7972Sstephen hemminger } 1125661b7972Sstephen hemminger 11261da177e4SLinus Torvalds static int netem_dump(struct Qdisc *sch, struct sk_buff *skb) 11271da177e4SLinus Torvalds { 11281da177e4SLinus Torvalds const struct netem_sched_data *q = qdisc_priv(sch); 1129861d7f74Sstephen hemminger struct nlattr *nla = (struct nlattr *) skb_tail_pointer(skb); 11301da177e4SLinus Torvalds struct tc_netem_qopt qopt; 11311da177e4SLinus Torvalds struct tc_netem_corr cor; 11320dca51d3SStephen Hemminger struct tc_netem_reorder reorder; 1133c865e5d9SStephen Hemminger struct tc_netem_corrupt corrupt; 11347bc0f28cSHagen Paul Pfeifer struct tc_netem_rate rate; 1135836af83bSDave Taht struct tc_netem_slot slot; 11361da177e4SLinus Torvalds 1137112f9cb6SDave Taht qopt.latency = min_t(psched_tdiff_t, PSCHED_NS2TICKS(q->latency), 1138112f9cb6SDave Taht UINT_MAX); 1139112f9cb6SDave Taht qopt.jitter = min_t(psched_tdiff_t, PSCHED_NS2TICKS(q->jitter), 1140112f9cb6SDave Taht UINT_MAX); 11411da177e4SLinus Torvalds qopt.limit = q->limit; 11421da177e4SLinus Torvalds qopt.loss = q->loss; 11431da177e4SLinus Torvalds qopt.gap = q->gap; 11441da177e4SLinus Torvalds qopt.duplicate = q->duplicate; 11451b34ec43SDavid S. Miller if (nla_put(skb, TCA_OPTIONS, sizeof(qopt), &qopt)) 11461b34ec43SDavid S. Miller goto nla_put_failure; 11471da177e4SLinus Torvalds 114899803171SDave Taht if (nla_put(skb, TCA_NETEM_LATENCY64, sizeof(q->latency), &q->latency)) 114999803171SDave Taht goto nla_put_failure; 115099803171SDave Taht 115199803171SDave Taht if (nla_put(skb, TCA_NETEM_JITTER64, sizeof(q->jitter), &q->jitter)) 115299803171SDave Taht goto nla_put_failure; 115399803171SDave Taht 11541da177e4SLinus Torvalds cor.delay_corr = q->delay_cor.rho; 11551da177e4SLinus Torvalds cor.loss_corr = q->loss_cor.rho; 11561da177e4SLinus Torvalds cor.dup_corr = q->dup_cor.rho; 11571b34ec43SDavid S. Miller if (nla_put(skb, TCA_NETEM_CORR, sizeof(cor), &cor)) 11581b34ec43SDavid S. Miller goto nla_put_failure; 11590dca51d3SStephen Hemminger 11600dca51d3SStephen Hemminger reorder.probability = q->reorder; 11610dca51d3SStephen Hemminger reorder.correlation = q->reorder_cor.rho; 11621b34ec43SDavid S. Miller if (nla_put(skb, TCA_NETEM_REORDER, sizeof(reorder), &reorder)) 11631b34ec43SDavid S. Miller goto nla_put_failure; 11640dca51d3SStephen Hemminger 1165c865e5d9SStephen Hemminger corrupt.probability = q->corrupt; 1166c865e5d9SStephen Hemminger corrupt.correlation = q->corrupt_cor.rho; 11671b34ec43SDavid S. Miller if (nla_put(skb, TCA_NETEM_CORRUPT, sizeof(corrupt), &corrupt)) 11681b34ec43SDavid S. Miller goto nla_put_failure; 1169c865e5d9SStephen Hemminger 11706a031f67SYang Yingliang if (q->rate >= (1ULL << 32)) { 11712a51c1e8SNicolas Dichtel if (nla_put_u64_64bit(skb, TCA_NETEM_RATE64, q->rate, 11722a51c1e8SNicolas Dichtel TCA_NETEM_PAD)) 11736a031f67SYang Yingliang goto nla_put_failure; 11746a031f67SYang Yingliang rate.rate = ~0U; 11756a031f67SYang Yingliang } else { 11767bc0f28cSHagen Paul Pfeifer rate.rate = q->rate; 11776a031f67SYang Yingliang } 117890b41a1cSHagen Paul Pfeifer rate.packet_overhead = q->packet_overhead; 117990b41a1cSHagen Paul Pfeifer rate.cell_size = q->cell_size; 118090b41a1cSHagen Paul Pfeifer rate.cell_overhead = q->cell_overhead; 11811b34ec43SDavid S. Miller if (nla_put(skb, TCA_NETEM_RATE, sizeof(rate), &rate)) 11821b34ec43SDavid S. Miller goto nla_put_failure; 11837bc0f28cSHagen Paul Pfeifer 1184e4ae004bSEric Dumazet if (q->ecn && nla_put_u32(skb, TCA_NETEM_ECN, q->ecn)) 1185e4ae004bSEric Dumazet goto nla_put_failure; 1186e4ae004bSEric Dumazet 1187661b7972Sstephen hemminger if (dump_loss_model(q, skb) != 0) 1188661b7972Sstephen hemminger goto nla_put_failure; 1189661b7972Sstephen hemminger 11900a9fe5c3SYousuk Seung if (q->slot_config.min_delay | q->slot_config.max_delay | 11910a9fe5c3SYousuk Seung q->slot_config.dist_jitter) { 1192836af83bSDave Taht slot = q->slot_config; 1193836af83bSDave Taht if (slot.max_packets == INT_MAX) 1194836af83bSDave Taht slot.max_packets = 0; 1195836af83bSDave Taht if (slot.max_bytes == INT_MAX) 1196836af83bSDave Taht slot.max_bytes = 0; 1197836af83bSDave Taht if (nla_put(skb, TCA_NETEM_SLOT, sizeof(slot), &slot)) 1198836af83bSDave Taht goto nla_put_failure; 1199836af83bSDave Taht } 1200836af83bSDave Taht 1201861d7f74Sstephen hemminger return nla_nest_end(skb, nla); 12021da177e4SLinus Torvalds 12031e90474cSPatrick McHardy nla_put_failure: 1204861d7f74Sstephen hemminger nlmsg_trim(skb, nla); 12051da177e4SLinus Torvalds return -1; 12061da177e4SLinus Torvalds } 12071da177e4SLinus Torvalds 120810f6dfcfSstephen hemminger static int netem_dump_class(struct Qdisc *sch, unsigned long cl, 120910f6dfcfSstephen hemminger struct sk_buff *skb, struct tcmsg *tcm) 121010f6dfcfSstephen hemminger { 121110f6dfcfSstephen hemminger struct netem_sched_data *q = qdisc_priv(sch); 121210f6dfcfSstephen hemminger 121350612537SEric Dumazet if (cl != 1 || !q->qdisc) /* only one class */ 121410f6dfcfSstephen hemminger return -ENOENT; 121510f6dfcfSstephen hemminger 121610f6dfcfSstephen hemminger tcm->tcm_handle |= TC_H_MIN(1); 121710f6dfcfSstephen hemminger tcm->tcm_info = q->qdisc->handle; 121810f6dfcfSstephen hemminger 121910f6dfcfSstephen hemminger return 0; 122010f6dfcfSstephen hemminger } 122110f6dfcfSstephen hemminger 122210f6dfcfSstephen hemminger static int netem_graft(struct Qdisc *sch, unsigned long arg, struct Qdisc *new, 1223653d6fd6SAlexander Aring struct Qdisc **old, struct netlink_ext_ack *extack) 122410f6dfcfSstephen hemminger { 122510f6dfcfSstephen hemminger struct netem_sched_data *q = qdisc_priv(sch); 122610f6dfcfSstephen hemminger 122786a7996cSWANG Cong *old = qdisc_replace(sch, new, &q->qdisc); 122810f6dfcfSstephen hemminger return 0; 122910f6dfcfSstephen hemminger } 123010f6dfcfSstephen hemminger 123110f6dfcfSstephen hemminger static struct Qdisc *netem_leaf(struct Qdisc *sch, unsigned long arg) 123210f6dfcfSstephen hemminger { 123310f6dfcfSstephen hemminger struct netem_sched_data *q = qdisc_priv(sch); 123410f6dfcfSstephen hemminger return q->qdisc; 123510f6dfcfSstephen hemminger } 123610f6dfcfSstephen hemminger 1237143976ceSWANG Cong static unsigned long netem_find(struct Qdisc *sch, u32 classid) 123810f6dfcfSstephen hemminger { 123910f6dfcfSstephen hemminger return 1; 124010f6dfcfSstephen hemminger } 124110f6dfcfSstephen hemminger 124210f6dfcfSstephen hemminger static void netem_walk(struct Qdisc *sch, struct qdisc_walker *walker) 124310f6dfcfSstephen hemminger { 124410f6dfcfSstephen hemminger if (!walker->stop) { 124510f6dfcfSstephen hemminger if (walker->count >= walker->skip) 124610f6dfcfSstephen hemminger if (walker->fn(sch, 1, walker) < 0) { 124710f6dfcfSstephen hemminger walker->stop = 1; 124810f6dfcfSstephen hemminger return; 124910f6dfcfSstephen hemminger } 125010f6dfcfSstephen hemminger walker->count++; 125110f6dfcfSstephen hemminger } 125210f6dfcfSstephen hemminger } 125310f6dfcfSstephen hemminger 125410f6dfcfSstephen hemminger static const struct Qdisc_class_ops netem_class_ops = { 125510f6dfcfSstephen hemminger .graft = netem_graft, 125610f6dfcfSstephen hemminger .leaf = netem_leaf, 1257143976ceSWANG Cong .find = netem_find, 125810f6dfcfSstephen hemminger .walk = netem_walk, 125910f6dfcfSstephen hemminger .dump = netem_dump_class, 126010f6dfcfSstephen hemminger }; 126110f6dfcfSstephen hemminger 126220fea08bSEric Dumazet static struct Qdisc_ops netem_qdisc_ops __read_mostly = { 12631da177e4SLinus Torvalds .id = "netem", 126410f6dfcfSstephen hemminger .cl_ops = &netem_class_ops, 12651da177e4SLinus Torvalds .priv_size = sizeof(struct netem_sched_data), 12661da177e4SLinus Torvalds .enqueue = netem_enqueue, 12671da177e4SLinus Torvalds .dequeue = netem_dequeue, 126877be155cSJarek Poplawski .peek = qdisc_peek_dequeued, 12691da177e4SLinus Torvalds .init = netem_init, 12701da177e4SLinus Torvalds .reset = netem_reset, 12711da177e4SLinus Torvalds .destroy = netem_destroy, 12721da177e4SLinus Torvalds .change = netem_change, 12731da177e4SLinus Torvalds .dump = netem_dump, 12741da177e4SLinus Torvalds .owner = THIS_MODULE, 12751da177e4SLinus Torvalds }; 12761da177e4SLinus Torvalds 12771da177e4SLinus Torvalds 12781da177e4SLinus Torvalds static int __init netem_module_init(void) 12791da177e4SLinus Torvalds { 1280eb229c4cSStephen Hemminger pr_info("netem: version " VERSION "\n"); 12811da177e4SLinus Torvalds return register_qdisc(&netem_qdisc_ops); 12821da177e4SLinus Torvalds } 12831da177e4SLinus Torvalds static void __exit netem_module_exit(void) 12841da177e4SLinus Torvalds { 12851da177e4SLinus Torvalds unregister_qdisc(&netem_qdisc_ops); 12861da177e4SLinus Torvalds } 12871da177e4SLinus Torvalds module_init(netem_module_init) 12881da177e4SLinus Torvalds module_exit(netem_module_exit) 12891da177e4SLinus Torvalds MODULE_LICENSE("GPL"); 1290