sch_generic.c (3a175cdf439275c3da347b6b42c4e9b652a12904) sch_generic.c (b2d3bcfa26a7a8de41f358a6cae8b848673b3c6e)
1/*
2 * net/sched/sch_generic.c Generic packet scheduler routines.
3 *
4 * This program is free software; you can redistribute it and/or
5 * modify it under the terms of the GNU General Public License
6 * as published by the Free Software Foundation; either version
7 * 2 of the License, or (at your option) any later version.
8 *

--- 12 unchanged lines hidden (view full) ---

21#include <linux/netdevice.h>
22#include <linux/skbuff.h>
23#include <linux/rtnetlink.h>
24#include <linux/init.h>
25#include <linux/rcupdate.h>
26#include <linux/list.h>
27#include <linux/slab.h>
28#include <linux/if_vlan.h>
1/*
2 * net/sched/sch_generic.c Generic packet scheduler routines.
3 *
4 * This program is free software; you can redistribute it and/or
5 * modify it under the terms of the GNU General Public License
6 * as published by the Free Software Foundation; either version
7 * 2 of the License, or (at your option) any later version.
8 *

--- 12 unchanged lines hidden (view full) ---

21#include <linux/netdevice.h>
22#include <linux/skbuff.h>
23#include <linux/rtnetlink.h>
24#include <linux/init.h>
25#include <linux/rcupdate.h>
26#include <linux/list.h>
27#include <linux/slab.h>
28#include <linux/if_vlan.h>
29#include <linux/skb_array.h>
29#include <linux/if_macvlan.h>
30#include <net/sch_generic.h>
31#include <net/pkt_sched.h>
32#include <net/dst.h>
33#include <trace/events/qdisc.h>
30#include <linux/if_macvlan.h>
31#include <net/sch_generic.h>
32#include <net/pkt_sched.h>
33#include <net/dst.h>
34#include <trace/events/qdisc.h>
35#include <net/xfrm.h>
34
35/* Qdisc to use by default */
36const struct Qdisc_ops *default_qdisc_ops = &pfifo_fast_ops;
37EXPORT_SYMBOL(default_qdisc_ops);
38
39/* Main transmission queue. */
40
41/* Modifications to data participating in scheduling must be protected with
42 * qdisc_lock(qdisc) spinlock.
43 *
44 * The idea is the following:
45 * - enqueue, dequeue are serialized via qdisc root lock
46 * - ingress filtering is also serialized via qdisc root lock
47 * - updates to tree and tree walking are only done under the rtnl mutex.
48 */
49
36
37/* Qdisc to use by default */
38const struct Qdisc_ops *default_qdisc_ops = &pfifo_fast_ops;
39EXPORT_SYMBOL(default_qdisc_ops);
40
41/* Main transmission queue. */
42
43/* Modifications to data participating in scheduling must be protected with
44 * qdisc_lock(qdisc) spinlock.
45 *
46 * The idea is the following:
47 * - enqueue, dequeue are serialized via qdisc root lock
48 * - ingress filtering is also serialized via qdisc root lock
49 * - updates to tree and tree walking are only done under the rtnl mutex.
50 */
51
50static inline int dev_requeue_skb(struct sk_buff *skb, struct Qdisc *q)
52static inline struct sk_buff *__skb_dequeue_bad_txq(struct Qdisc *q)
51{
53{
52 q->gso_skb = skb;
53 q->qstats.requeues++;
54 qdisc_qstats_backlog_inc(q, skb);
55 q->q.qlen++; /* it's still part of the queue */
54 const struct netdev_queue *txq = q->dev_queue;
55 spinlock_t *lock = NULL;
56 struct sk_buff *skb;
57
58 if (q->flags & TCQ_F_NOLOCK) {
59 lock = qdisc_lock(q);
60 spin_lock(lock);
61 }
62
63 skb = skb_peek(&q->skb_bad_txq);
64 if (skb) {
65 /* check the reason of requeuing without tx lock first */
66 txq = skb_get_tx_queue(txq->dev, skb);
67 if (!netif_xmit_frozen_or_stopped(txq)) {
68 skb = __skb_dequeue(&q->skb_bad_txq);
69 if (qdisc_is_percpu_stats(q)) {
70 qdisc_qstats_cpu_backlog_dec(q, skb);
71 qdisc_qstats_cpu_qlen_dec(q);
72 } else {
73 qdisc_qstats_backlog_dec(q, skb);
74 q->q.qlen--;
75 }
76 } else {
77 skb = NULL;
78 }
79 }
80
81 if (lock)
82 spin_unlock(lock);
83
84 return skb;
85}
86
87static inline struct sk_buff *qdisc_dequeue_skb_bad_txq(struct Qdisc *q)
88{
89 struct sk_buff *skb = skb_peek(&q->skb_bad_txq);
90
91 if (unlikely(skb))
92 skb = __skb_dequeue_bad_txq(q);
93
94 return skb;
95}
96
97static inline void qdisc_enqueue_skb_bad_txq(struct Qdisc *q,
98 struct sk_buff *skb)
99{
100 spinlock_t *lock = NULL;
101
102 if (q->flags & TCQ_F_NOLOCK) {
103 lock = qdisc_lock(q);
104 spin_lock(lock);
105 }
106
107 __skb_queue_tail(&q->skb_bad_txq, skb);
108
109 if (lock)
110 spin_unlock(lock);
111}
112
113static inline int __dev_requeue_skb(struct sk_buff *skb, struct Qdisc *q)
114{
115 while (skb) {
116 struct sk_buff *next = skb->next;
117
118 __skb_queue_tail(&q->gso_skb, skb);
119 q->qstats.requeues++;
120 qdisc_qstats_backlog_inc(q, skb);
121 q->q.qlen++; /* it's still part of the queue */
122
123 skb = next;
124 }
56 __netif_schedule(q);
57
58 return 0;
59}
60
125 __netif_schedule(q);
126
127 return 0;
128}
129
130static inline int dev_requeue_skb_locked(struct sk_buff *skb, struct Qdisc *q)
131{
132 spinlock_t *lock = qdisc_lock(q);
133
134 spin_lock(lock);
135 while (skb) {
136 struct sk_buff *next = skb->next;
137
138 __skb_queue_tail(&q->gso_skb, skb);
139
140 qdisc_qstats_cpu_requeues_inc(q);
141 qdisc_qstats_cpu_backlog_inc(q, skb);
142 qdisc_qstats_cpu_qlen_inc(q);
143
144 skb = next;
145 }
146 spin_unlock(lock);
147
148 __netif_schedule(q);
149
150 return 0;
151}
152
153static inline int dev_requeue_skb(struct sk_buff *skb, struct Qdisc *q)
154{
155 if (q->flags & TCQ_F_NOLOCK)
156 return dev_requeue_skb_locked(skb, q);
157 else
158 return __dev_requeue_skb(skb, q);
159}
160
61static void try_bulk_dequeue_skb(struct Qdisc *q,
62 struct sk_buff *skb,
63 const struct netdev_queue *txq,
64 int *packets)
65{
66 int bytelimit = qdisc_avail_bulklimit(txq) - skb->len;
67
68 while (bytelimit > 0) {

--- 21 unchanged lines hidden (view full) ---

90 struct sk_buff *nskb;
91 int cnt = 0;
92
93 do {
94 nskb = q->dequeue(q);
95 if (!nskb)
96 break;
97 if (unlikely(skb_get_queue_mapping(nskb) != mapping)) {
161static void try_bulk_dequeue_skb(struct Qdisc *q,
162 struct sk_buff *skb,
163 const struct netdev_queue *txq,
164 int *packets)
165{
166 int bytelimit = qdisc_avail_bulklimit(txq) - skb->len;
167
168 while (bytelimit > 0) {

--- 21 unchanged lines hidden (view full) ---

190 struct sk_buff *nskb;
191 int cnt = 0;
192
193 do {
194 nskb = q->dequeue(q);
195 if (!nskb)
196 break;
197 if (unlikely(skb_get_queue_mapping(nskb) != mapping)) {
98 q->skb_bad_txq = nskb;
99 qdisc_qstats_backlog_inc(q, nskb);
100 q->q.qlen++;
198 qdisc_enqueue_skb_bad_txq(q, nskb);
199
200 if (qdisc_is_percpu_stats(q)) {
201 qdisc_qstats_cpu_backlog_inc(q, nskb);
202 qdisc_qstats_cpu_qlen_inc(q);
203 } else {
204 qdisc_qstats_backlog_inc(q, nskb);
205 q->q.qlen++;
206 }
101 break;
102 }
103 skb->next = nskb;
104 skb = nskb;
105 } while (++cnt < 8);
106 (*packets) += cnt;
107 skb->next = NULL;
108}
109
110/* Note that dequeue_skb can possibly return a SKB list (via skb->next).
111 * A requeued skb (via q->gso_skb) can also be a SKB list.
112 */
113static struct sk_buff *dequeue_skb(struct Qdisc *q, bool *validate,
114 int *packets)
115{
207 break;
208 }
209 skb->next = nskb;
210 skb = nskb;
211 } while (++cnt < 8);
212 (*packets) += cnt;
213 skb->next = NULL;
214}
215
216/* Note that dequeue_skb can possibly return a SKB list (via skb->next).
217 * A requeued skb (via q->gso_skb) can also be a SKB list.
218 */
219static struct sk_buff *dequeue_skb(struct Qdisc *q, bool *validate,
220 int *packets)
221{
116 struct sk_buff *skb = q->gso_skb;
117 const struct netdev_queue *txq = q->dev_queue;
222 const struct netdev_queue *txq = q->dev_queue;
223 struct sk_buff *skb = NULL;
118
119 *packets = 1;
224
225 *packets = 1;
120 if (unlikely(skb)) {
226 if (unlikely(!skb_queue_empty(&q->gso_skb))) {
227 spinlock_t *lock = NULL;
228
229 if (q->flags & TCQ_F_NOLOCK) {
230 lock = qdisc_lock(q);
231 spin_lock(lock);
232 }
233
234 skb = skb_peek(&q->gso_skb);
235
236 /* skb may be null if another cpu pulls gso_skb off in between
237 * empty check and lock.
238 */
239 if (!skb) {
240 if (lock)
241 spin_unlock(lock);
242 goto validate;
243 }
244
121 /* skb in gso_skb were already validated */
122 *validate = false;
245 /* skb in gso_skb were already validated */
246 *validate = false;
247 if (xfrm_offload(skb))
248 *validate = true;
123 /* check the reason of requeuing without tx lock first */
124 txq = skb_get_tx_queue(txq->dev, skb);
125 if (!netif_xmit_frozen_or_stopped(txq)) {
249 /* check the reason of requeuing without tx lock first */
250 txq = skb_get_tx_queue(txq->dev, skb);
251 if (!netif_xmit_frozen_or_stopped(txq)) {
126 q->gso_skb = NULL;
127 qdisc_qstats_backlog_dec(q, skb);
128 q->q.qlen--;
129 } else
252 skb = __skb_dequeue(&q->gso_skb);
253 if (qdisc_is_percpu_stats(q)) {
254 qdisc_qstats_cpu_backlog_dec(q, skb);
255 qdisc_qstats_cpu_qlen_dec(q);
256 } else {
257 qdisc_qstats_backlog_dec(q, skb);
258 q->q.qlen--;
259 }
260 } else {
130 skb = NULL;
261 skb = NULL;
131 goto trace;
132 }
133 *validate = true;
134 skb = q->skb_bad_txq;
135 if (unlikely(skb)) {
136 /* check the reason of requeuing without tx lock first */
137 txq = skb_get_tx_queue(txq->dev, skb);
138 if (!netif_xmit_frozen_or_stopped(txq)) {
139 q->skb_bad_txq = NULL;
140 qdisc_qstats_backlog_dec(q, skb);
141 q->q.qlen--;
142 goto bulk;
143 }
262 }
144 skb = NULL;
263 if (lock)
264 spin_unlock(lock);
145 goto trace;
146 }
265 goto trace;
266 }
147 if (!(q->flags & TCQ_F_ONETXQUEUE) ||
148 !netif_xmit_frozen_or_stopped(txq))
149 skb = q->dequeue(q);
267validate:
268 *validate = true;
269
270 if ((q->flags & TCQ_F_ONETXQUEUE) &&
271 netif_xmit_frozen_or_stopped(txq))
272 return skb;
273
274 skb = qdisc_dequeue_skb_bad_txq(q);
275 if (unlikely(skb))
276 goto bulk;
277 skb = q->dequeue(q);
150 if (skb) {
151bulk:
152 if (qdisc_may_bulk(q))
153 try_bulk_dequeue_skb(q, skb, txq, packets);
154 else
155 try_bulk_dequeue_skb_slow(q, skb, packets);
156 }
157trace:
158 trace_qdisc_dequeue(q, txq, *packets, skb);
159 return skb;
160}
161
162/*
163 * Transmit possibly several skbs, and handle the return status as
164 * required. Owning running seqcount bit guarantees that
165 * only one CPU can execute this function.
166 *
167 * Returns to the caller:
278 if (skb) {
279bulk:
280 if (qdisc_may_bulk(q))
281 try_bulk_dequeue_skb(q, skb, txq, packets);
282 else
283 try_bulk_dequeue_skb_slow(q, skb, packets);
284 }
285trace:
286 trace_qdisc_dequeue(q, txq, *packets, skb);
287 return skb;
288}
289
290/*
291 * Transmit possibly several skbs, and handle the return status as
292 * required. Owning running seqcount bit guarantees that
293 * only one CPU can execute this function.
294 *
295 * Returns to the caller:
168 * 0 - queue is empty or throttled.
169 * >0 - queue is not empty.
296 * false - hardware queue frozen backoff
297 * true - feel free to send more pkts
170 */
298 */
171int sch_direct_xmit(struct sk_buff *skb, struct Qdisc *q,
172 struct net_device *dev, struct netdev_queue *txq,
173 spinlock_t *root_lock, bool validate)
299bool sch_direct_xmit(struct sk_buff *skb, struct Qdisc *q,
300 struct net_device *dev, struct netdev_queue *txq,
301 spinlock_t *root_lock, bool validate)
174{
175 int ret = NETDEV_TX_BUSY;
302{
303 int ret = NETDEV_TX_BUSY;
304 bool again = false;
176
177 /* And release qdisc */
305
306 /* And release qdisc */
178 spin_unlock(root_lock);
307 if (root_lock)
308 spin_unlock(root_lock);
179
180 /* Note that we validate skb (GSO, checksum, ...) outside of locks */
181 if (validate)
309
310 /* Note that we validate skb (GSO, checksum, ...) outside of locks */
311 if (validate)
182 skb = validate_xmit_skb_list(skb, dev);
312 skb = validate_xmit_skb_list(skb, dev, &again);
183
313
314#ifdef CONFIG_XFRM_OFFLOAD
315 if (unlikely(again)) {
316 if (root_lock)
317 spin_lock(root_lock);
318
319 dev_requeue_skb(skb, q);
320 return false;
321 }
322#endif
323
184 if (likely(skb)) {
185 HARD_TX_LOCK(dev, txq, smp_processor_id());
186 if (!netif_xmit_frozen_or_stopped(txq))
187 skb = dev_hard_start_xmit(skb, dev, txq, &ret);
188
189 HARD_TX_UNLOCK(dev, txq);
190 } else {
324 if (likely(skb)) {
325 HARD_TX_LOCK(dev, txq, smp_processor_id());
326 if (!netif_xmit_frozen_or_stopped(txq))
327 skb = dev_hard_start_xmit(skb, dev, txq, &ret);
328
329 HARD_TX_UNLOCK(dev, txq);
330 } else {
191 spin_lock(root_lock);
192 return qdisc_qlen(q);
331 if (root_lock)
332 spin_lock(root_lock);
333 return true;
193 }
334 }
194 spin_lock(root_lock);
195
335
196 if (dev_xmit_complete(ret)) {
197 /* Driver sent out skb successfully or skb was consumed */
198 ret = qdisc_qlen(q);
199 } else {
336 if (root_lock)
337 spin_lock(root_lock);
338
339 if (!dev_xmit_complete(ret)) {
200 /* Driver returned NETDEV_TX_BUSY - requeue skb */
201 if (unlikely(ret != NETDEV_TX_BUSY))
202 net_warn_ratelimited("BUG %s code %d qlen %d\n",
203 dev->name, ret, q->q.qlen);
204
340 /* Driver returned NETDEV_TX_BUSY - requeue skb */
341 if (unlikely(ret != NETDEV_TX_BUSY))
342 net_warn_ratelimited("BUG %s code %d qlen %d\n",
343 dev->name, ret, q->q.qlen);
344
205 ret = dev_requeue_skb(skb, q);
345 dev_requeue_skb(skb, q);
346 return false;
206 }
207
208 if (ret && netif_xmit_frozen_or_stopped(txq))
347 }
348
349 if (ret && netif_xmit_frozen_or_stopped(txq))
209 ret = 0;
350 return false;
210
351
211 return ret;
352 return true;
212}
213
214/*
215 * NOTE: Called under qdisc_lock(q) with locally disabled BH.
216 *
217 * running seqcount guarantees only one CPU can process
218 * this qdisc at a time. qdisc_lock(q) serializes queue accesses for
219 * this queue.

--- 5 unchanged lines hidden (view full) ---

225 *
226 * Note, that this procedure can be called by a watchdog timer
227 *
228 * Returns to the caller:
229 * 0 - queue is empty or throttled.
230 * >0 - queue is not empty.
231 *
232 */
353}
354
355/*
356 * NOTE: Called under qdisc_lock(q) with locally disabled BH.
357 *
358 * running seqcount guarantees only one CPU can process
359 * this qdisc at a time. qdisc_lock(q) serializes queue accesses for
360 * this queue.

--- 5 unchanged lines hidden (view full) ---

366 *
367 * Note, that this procedure can be called by a watchdog timer
368 *
369 * Returns to the caller:
370 * 0 - queue is empty or throttled.
371 * >0 - queue is not empty.
372 *
373 */
233static inline int qdisc_restart(struct Qdisc *q, int *packets)
374static inline bool qdisc_restart(struct Qdisc *q, int *packets)
234{
375{
376 spinlock_t *root_lock = NULL;
235 struct netdev_queue *txq;
236 struct net_device *dev;
377 struct netdev_queue *txq;
378 struct net_device *dev;
237 spinlock_t *root_lock;
238 struct sk_buff *skb;
239 bool validate;
240
241 /* Dequeue packet */
242 skb = dequeue_skb(q, &validate, packets);
243 if (unlikely(!skb))
379 struct sk_buff *skb;
380 bool validate;
381
382 /* Dequeue packet */
383 skb = dequeue_skb(q, &validate, packets);
384 if (unlikely(!skb))
244 return 0;
385 return false;
245
386
246 root_lock = qdisc_lock(q);
387 if (!(q->flags & TCQ_F_NOLOCK))
388 root_lock = qdisc_lock(q);
389
247 dev = qdisc_dev(q);
248 txq = skb_get_tx_queue(dev, skb);
249
250 return sch_direct_xmit(skb, q, dev, txq, root_lock, validate);
251}
252
253void __qdisc_run(struct Qdisc *q)
254{

--- 7 unchanged lines hidden (view full) ---

262 * 2. another process needs the CPU;
263 */
264 quota -= packets;
265 if (quota <= 0 || need_resched()) {
266 __netif_schedule(q);
267 break;
268 }
269 }
390 dev = qdisc_dev(q);
391 txq = skb_get_tx_queue(dev, skb);
392
393 return sch_direct_xmit(skb, q, dev, txq, root_lock, validate);
394}
395
396void __qdisc_run(struct Qdisc *q)
397{

--- 7 unchanged lines hidden (view full) ---

405 * 2. another process needs the CPU;
406 */
407 quota -= packets;
408 if (quota <= 0 || need_resched()) {
409 __netif_schedule(q);
410 break;
411 }
412 }
270
271 qdisc_run_end(q);
272}
273
274unsigned long dev_trans_start(struct net_device *dev)
275{
276 unsigned long val, res;
277 unsigned int i;
278
279 if (is_vlan_dev(dev))

--- 84 unchanged lines hidden (view full) ---

364 *
365 * Device has detected that carrier.
366 */
367void netif_carrier_on(struct net_device *dev)
368{
369 if (test_and_clear_bit(__LINK_STATE_NOCARRIER, &dev->state)) {
370 if (dev->reg_state == NETREG_UNINITIALIZED)
371 return;
413}
414
415unsigned long dev_trans_start(struct net_device *dev)
416{
417 unsigned long val, res;
418 unsigned int i;
419
420 if (is_vlan_dev(dev))

--- 84 unchanged lines hidden (view full) ---

505 *
506 * Device has detected that carrier.
507 */
508void netif_carrier_on(struct net_device *dev)
509{
510 if (test_and_clear_bit(__LINK_STATE_NOCARRIER, &dev->state)) {
511 if (dev->reg_state == NETREG_UNINITIALIZED)
512 return;
372 atomic_inc(&dev->carrier_changes);
513 atomic_inc(&dev->carrier_up_count);
373 linkwatch_fire_event(dev);
374 if (netif_running(dev))
375 __netdev_watchdog_up(dev);
376 }
377}
378EXPORT_SYMBOL(netif_carrier_on);
379
380/**
381 * netif_carrier_off - clear carrier
382 * @dev: network device
383 *
384 * Device has detected loss of carrier.
385 */
386void netif_carrier_off(struct net_device *dev)
387{
388 if (!test_and_set_bit(__LINK_STATE_NOCARRIER, &dev->state)) {
389 if (dev->reg_state == NETREG_UNINITIALIZED)
390 return;
514 linkwatch_fire_event(dev);
515 if (netif_running(dev))
516 __netdev_watchdog_up(dev);
517 }
518}
519EXPORT_SYMBOL(netif_carrier_on);
520
521/**
522 * netif_carrier_off - clear carrier
523 * @dev: network device
524 *
525 * Device has detected loss of carrier.
526 */
527void netif_carrier_off(struct net_device *dev)
528{
529 if (!test_and_set_bit(__LINK_STATE_NOCARRIER, &dev->state)) {
530 if (dev->reg_state == NETREG_UNINITIALIZED)
531 return;
391 atomic_inc(&dev->carrier_changes);
532 atomic_inc(&dev->carrier_down_count);
392 linkwatch_fire_event(dev);
393 }
394}
395EXPORT_SYMBOL(netif_carrier_off);
396
397/* "NOOP" scheduler: the best scheduler, recommended for all interfaces
398 under all circumstances. It is difficult to invent anything faster or
399 cheaper.

--- 32 unchanged lines hidden (view full) ---

432 .ops = &noop_qdisc_ops,
433 .q.lock = __SPIN_LOCK_UNLOCKED(noop_qdisc.q.lock),
434 .dev_queue = &noop_netdev_queue,
435 .running = SEQCNT_ZERO(noop_qdisc.running),
436 .busylock = __SPIN_LOCK_UNLOCKED(noop_qdisc.busylock),
437};
438EXPORT_SYMBOL(noop_qdisc);
439
533 linkwatch_fire_event(dev);
534 }
535}
536EXPORT_SYMBOL(netif_carrier_off);
537
538/* "NOOP" scheduler: the best scheduler, recommended for all interfaces
539 under all circumstances. It is difficult to invent anything faster or
540 cheaper.

--- 32 unchanged lines hidden (view full) ---

573 .ops = &noop_qdisc_ops,
574 .q.lock = __SPIN_LOCK_UNLOCKED(noop_qdisc.q.lock),
575 .dev_queue = &noop_netdev_queue,
576 .running = SEQCNT_ZERO(noop_qdisc.running),
577 .busylock = __SPIN_LOCK_UNLOCKED(noop_qdisc.busylock),
578};
579EXPORT_SYMBOL(noop_qdisc);
580
440static int noqueue_init(struct Qdisc *qdisc, struct nlattr *opt)
581static int noqueue_init(struct Qdisc *qdisc, struct nlattr *opt,
582 struct netlink_ext_ack *extack)
441{
442 /* register_qdisc() assigns a default of noop_enqueue if unset,
443 * but __dev_queue_xmit() treats noqueue only as such
444 * if this is NULL - so clear it here. */
445 qdisc->enqueue = NULL;
446 return 0;
447}
448

--- 14 unchanged lines hidden (view full) ---

463/* 3-band FIFO queue: old style, but should be a bit faster than
464 generic prio+fifo combination.
465 */
466
467#define PFIFO_FAST_BANDS 3
468
469/*
470 * Private data for a pfifo_fast scheduler containing:
583{
584 /* register_qdisc() assigns a default of noop_enqueue if unset,
585 * but __dev_queue_xmit() treats noqueue only as such
586 * if this is NULL - so clear it here. */
587 qdisc->enqueue = NULL;
588 return 0;
589}
590

--- 14 unchanged lines hidden (view full) ---

605/* 3-band FIFO queue: old style, but should be a bit faster than
606 generic prio+fifo combination.
607 */
608
609#define PFIFO_FAST_BANDS 3
610
611/*
612 * Private data for a pfifo_fast scheduler containing:
471 * - queues for the three band
472 * - bitmap indicating which of the bands contain skbs
613 * - rings for priority bands
473 */
474struct pfifo_fast_priv {
614 */
615struct pfifo_fast_priv {
475 u32 bitmap;
476 struct qdisc_skb_head q[PFIFO_FAST_BANDS];
616 struct skb_array q[PFIFO_FAST_BANDS];
477};
478
617};
618
479/*
480 * Convert a bitmap to the first band number where an skb is queued, where:
481 * bitmap=0 means there are no skbs on any band.
482 * bitmap=1 means there is an skb on band 0.
483 * bitmap=7 means there are skbs on all 3 bands, etc.
484 */
485static const int bitmap2band[] = {-1, 0, 1, 0, 2, 0, 1, 0};
486
487static inline struct qdisc_skb_head *band2list(struct pfifo_fast_priv *priv,
488 int band)
619static inline struct skb_array *band2list(struct pfifo_fast_priv *priv,
620 int band)
489{
621{
490 return priv->q + band;
622 return &priv->q[band];
491}
492
493static int pfifo_fast_enqueue(struct sk_buff *skb, struct Qdisc *qdisc,
494 struct sk_buff **to_free)
495{
623}
624
625static int pfifo_fast_enqueue(struct sk_buff *skb, struct Qdisc *qdisc,
626 struct sk_buff **to_free)
627{
496 if (qdisc->q.qlen < qdisc_dev(qdisc)->tx_queue_len) {
497 int band = prio2band[skb->priority & TC_PRIO_MAX];
498 struct pfifo_fast_priv *priv = qdisc_priv(qdisc);
499 struct qdisc_skb_head *list = band2list(priv, band);
628 int band = prio2band[skb->priority & TC_PRIO_MAX];
629 struct pfifo_fast_priv *priv = qdisc_priv(qdisc);
630 struct skb_array *q = band2list(priv, band);
631 int err;
500
632
501 priv->bitmap |= (1 << band);
502 qdisc->q.qlen++;
503 return __qdisc_enqueue_tail(skb, qdisc, list);
504 }
633 err = skb_array_produce(q, skb);
505
634
506 return qdisc_drop(skb, qdisc, to_free);
635 if (unlikely(err))
636 return qdisc_drop_cpu(skb, qdisc, to_free);
637
638 qdisc_qstats_cpu_qlen_inc(qdisc);
639 qdisc_qstats_cpu_backlog_inc(qdisc, skb);
640 return NET_XMIT_SUCCESS;
507}
508
509static struct sk_buff *pfifo_fast_dequeue(struct Qdisc *qdisc)
510{
511 struct pfifo_fast_priv *priv = qdisc_priv(qdisc);
641}
642
643static struct sk_buff *pfifo_fast_dequeue(struct Qdisc *qdisc)
644{
645 struct pfifo_fast_priv *priv = qdisc_priv(qdisc);
512 int band = bitmap2band[priv->bitmap];
646 struct sk_buff *skb = NULL;
647 int band;
513
648
514 if (likely(band >= 0)) {
515 struct qdisc_skb_head *qh = band2list(priv, band);
516 struct sk_buff *skb = __qdisc_dequeue_head(qh);
649 for (band = 0; band < PFIFO_FAST_BANDS && !skb; band++) {
650 struct skb_array *q = band2list(priv, band);
517
651
518 if (likely(skb != NULL)) {
519 qdisc_qstats_backlog_dec(qdisc, skb);
520 qdisc_bstats_update(qdisc, skb);
521 }
652 if (__skb_array_empty(q))
653 continue;
522
654
523 qdisc->q.qlen--;
524 if (qh->qlen == 0)
525 priv->bitmap &= ~(1 << band);
526
527 return skb;
655 skb = skb_array_consume_bh(q);
528 }
656 }
657 if (likely(skb)) {
658 qdisc_qstats_cpu_backlog_dec(qdisc, skb);
659 qdisc_bstats_cpu_update(qdisc, skb);
660 qdisc_qstats_cpu_qlen_dec(qdisc);
661 }
529
662
530 return NULL;
663 return skb;
531}
532
533static struct sk_buff *pfifo_fast_peek(struct Qdisc *qdisc)
534{
535 struct pfifo_fast_priv *priv = qdisc_priv(qdisc);
664}
665
666static struct sk_buff *pfifo_fast_peek(struct Qdisc *qdisc)
667{
668 struct pfifo_fast_priv *priv = qdisc_priv(qdisc);
536 int band = bitmap2band[priv->bitmap];
669 struct sk_buff *skb = NULL;
670 int band;
537
671
538 if (band >= 0) {
539 struct qdisc_skb_head *qh = band2list(priv, band);
672 for (band = 0; band < PFIFO_FAST_BANDS && !skb; band++) {
673 struct skb_array *q = band2list(priv, band);
540
674
541 return qh->head;
675 skb = __skb_array_peek(q);
542 }
543
676 }
677
544 return NULL;
678 return skb;
545}
546
547static void pfifo_fast_reset(struct Qdisc *qdisc)
548{
679}
680
681static void pfifo_fast_reset(struct Qdisc *qdisc)
682{
549 int prio;
683 int i, band;
550 struct pfifo_fast_priv *priv = qdisc_priv(qdisc);
551
684 struct pfifo_fast_priv *priv = qdisc_priv(qdisc);
685
552 for (prio = 0; prio < PFIFO_FAST_BANDS; prio++)
553 __qdisc_reset_queue(band2list(priv, prio));
686 for (band = 0; band < PFIFO_FAST_BANDS; band++) {
687 struct skb_array *q = band2list(priv, band);
688 struct sk_buff *skb;
554
689
555 priv->bitmap = 0;
556 qdisc->qstats.backlog = 0;
557 qdisc->q.qlen = 0;
690 /* NULL ring is possible if destroy path is due to a failed
691 * skb_array_init() in pfifo_fast_init() case.
692 */
693 if (!q->ring.queue)
694 continue;
695
696 while ((skb = skb_array_consume_bh(q)) != NULL)
697 kfree_skb(skb);
698 }
699
700 for_each_possible_cpu(i) {
701 struct gnet_stats_queue *q = per_cpu_ptr(qdisc->cpu_qstats, i);
702
703 q->backlog = 0;
704 q->qlen = 0;
705 }
558}
559
560static int pfifo_fast_dump(struct Qdisc *qdisc, struct sk_buff *skb)
561{
562 struct tc_prio_qopt opt = { .bands = PFIFO_FAST_BANDS };
563
564 memcpy(&opt.priomap, prio2band, TC_PRIO_MAX + 1);
565 if (nla_put(skb, TCA_OPTIONS, sizeof(opt), &opt))
566 goto nla_put_failure;
567 return skb->len;
568
569nla_put_failure:
570 return -1;
571}
572
706}
707
708static int pfifo_fast_dump(struct Qdisc *qdisc, struct sk_buff *skb)
709{
710 struct tc_prio_qopt opt = { .bands = PFIFO_FAST_BANDS };
711
712 memcpy(&opt.priomap, prio2band, TC_PRIO_MAX + 1);
713 if (nla_put(skb, TCA_OPTIONS, sizeof(opt), &opt))
714 goto nla_put_failure;
715 return skb->len;
716
717nla_put_failure:
718 return -1;
719}
720
573static int pfifo_fast_init(struct Qdisc *qdisc, struct nlattr *opt)
721static int pfifo_fast_init(struct Qdisc *qdisc, struct nlattr *opt,
722 struct netlink_ext_ack *extack)
574{
723{
575 int prio;
724 unsigned int qlen = qdisc_dev(qdisc)->tx_queue_len;
576 struct pfifo_fast_priv *priv = qdisc_priv(qdisc);
725 struct pfifo_fast_priv *priv = qdisc_priv(qdisc);
726 int prio;
577
727
578 for (prio = 0; prio < PFIFO_FAST_BANDS; prio++)
579 qdisc_skb_head_init(band2list(priv, prio));
728 /* guard against zero length rings */
729 if (!qlen)
730 return -EINVAL;
580
731
732 for (prio = 0; prio < PFIFO_FAST_BANDS; prio++) {
733 struct skb_array *q = band2list(priv, prio);
734 int err;
735
736 err = skb_array_init(q, qlen, GFP_KERNEL);
737 if (err)
738 return -ENOMEM;
739 }
740
581 /* Can by-pass the queue discipline */
582 qdisc->flags |= TCQ_F_CAN_BYPASS;
583 return 0;
584}
585
741 /* Can by-pass the queue discipline */
742 qdisc->flags |= TCQ_F_CAN_BYPASS;
743 return 0;
744}
745
746static void pfifo_fast_destroy(struct Qdisc *sch)
747{
748 struct pfifo_fast_priv *priv = qdisc_priv(sch);
749 int prio;
750
751 for (prio = 0; prio < PFIFO_FAST_BANDS; prio++) {
752 struct skb_array *q = band2list(priv, prio);
753
754 /* NULL ring is possible if destroy path is due to a failed
755 * skb_array_init() in pfifo_fast_init() case.
756 */
757 if (!q->ring.queue)
758 continue;
759 /* Destroy ring but no need to kfree_skb because a call to
760 * pfifo_fast_reset() has already done that work.
761 */
762 ptr_ring_cleanup(&q->ring, NULL);
763 }
764}
765
586struct Qdisc_ops pfifo_fast_ops __read_mostly = {
587 .id = "pfifo_fast",
588 .priv_size = sizeof(struct pfifo_fast_priv),
589 .enqueue = pfifo_fast_enqueue,
590 .dequeue = pfifo_fast_dequeue,
591 .peek = pfifo_fast_peek,
592 .init = pfifo_fast_init,
766struct Qdisc_ops pfifo_fast_ops __read_mostly = {
767 .id = "pfifo_fast",
768 .priv_size = sizeof(struct pfifo_fast_priv),
769 .enqueue = pfifo_fast_enqueue,
770 .dequeue = pfifo_fast_dequeue,
771 .peek = pfifo_fast_peek,
772 .init = pfifo_fast_init,
773 .destroy = pfifo_fast_destroy,
593 .reset = pfifo_fast_reset,
594 .dump = pfifo_fast_dump,
595 .owner = THIS_MODULE,
774 .reset = pfifo_fast_reset,
775 .dump = pfifo_fast_dump,
776 .owner = THIS_MODULE,
777 .static_flags = TCQ_F_NOLOCK | TCQ_F_CPUSTATS,
596};
597EXPORT_SYMBOL(pfifo_fast_ops);
598
599static struct lock_class_key qdisc_tx_busylock;
600static struct lock_class_key qdisc_running_key;
601
602struct Qdisc *qdisc_alloc(struct netdev_queue *dev_queue,
778};
779EXPORT_SYMBOL(pfifo_fast_ops);
780
781static struct lock_class_key qdisc_tx_busylock;
782static struct lock_class_key qdisc_running_key;
783
784struct Qdisc *qdisc_alloc(struct netdev_queue *dev_queue,
603 const struct Qdisc_ops *ops)
785 const struct Qdisc_ops *ops,
786 struct netlink_ext_ack *extack)
604{
605 void *p;
606 struct Qdisc *sch;
607 unsigned int size = QDISC_ALIGN(sizeof(*sch)) + ops->priv_size;
608 int err = -ENOBUFS;
609 struct net_device *dev;
610
611 if (!dev_queue) {
787{
788 void *p;
789 struct Qdisc *sch;
790 unsigned int size = QDISC_ALIGN(sizeof(*sch)) + ops->priv_size;
791 int err = -ENOBUFS;
792 struct net_device *dev;
793
794 if (!dev_queue) {
795 NL_SET_ERR_MSG(extack, "No device queue given");
612 err = -EINVAL;
613 goto errout;
614 }
615
616 dev = dev_queue->dev;
617 p = kzalloc_node(size, GFP_KERNEL,
618 netdev_queue_numa_node_read(dev_queue));
619

--- 5 unchanged lines hidden (view full) ---

625 kfree(p);
626 p = kzalloc_node(size + QDISC_ALIGNTO - 1, GFP_KERNEL,
627 netdev_queue_numa_node_read(dev_queue));
628 if (!p)
629 goto errout;
630 sch = (struct Qdisc *) QDISC_ALIGN((unsigned long) p);
631 sch->padded = (char *) sch - (char *) p;
632 }
796 err = -EINVAL;
797 goto errout;
798 }
799
800 dev = dev_queue->dev;
801 p = kzalloc_node(size, GFP_KERNEL,
802 netdev_queue_numa_node_read(dev_queue));
803

--- 5 unchanged lines hidden (view full) ---

809 kfree(p);
810 p = kzalloc_node(size + QDISC_ALIGNTO - 1, GFP_KERNEL,
811 netdev_queue_numa_node_read(dev_queue));
812 if (!p)
813 goto errout;
814 sch = (struct Qdisc *) QDISC_ALIGN((unsigned long) p);
815 sch->padded = (char *) sch - (char *) p;
816 }
817 __skb_queue_head_init(&sch->gso_skb);
818 __skb_queue_head_init(&sch->skb_bad_txq);
633 qdisc_skb_head_init(&sch->q);
634 spin_lock_init(&sch->q.lock);
635
819 qdisc_skb_head_init(&sch->q);
820 spin_lock_init(&sch->q.lock);
821
822 if (ops->static_flags & TCQ_F_CPUSTATS) {
823 sch->cpu_bstats =
824 netdev_alloc_pcpu_stats(struct gnet_stats_basic_cpu);
825 if (!sch->cpu_bstats)
826 goto errout1;
827
828 sch->cpu_qstats = alloc_percpu(struct gnet_stats_queue);
829 if (!sch->cpu_qstats) {
830 free_percpu(sch->cpu_bstats);
831 goto errout1;
832 }
833 }
834
636 spin_lock_init(&sch->busylock);
637 lockdep_set_class(&sch->busylock,
638 dev->qdisc_tx_busylock ?: &qdisc_tx_busylock);
639
640 seqcount_init(&sch->running);
641 lockdep_set_class(&sch->running,
642 dev->qdisc_running_key ?: &qdisc_running_key);
643
644 sch->ops = ops;
835 spin_lock_init(&sch->busylock);
836 lockdep_set_class(&sch->busylock,
837 dev->qdisc_tx_busylock ?: &qdisc_tx_busylock);
838
839 seqcount_init(&sch->running);
840 lockdep_set_class(&sch->running,
841 dev->qdisc_running_key ?: &qdisc_running_key);
842
843 sch->ops = ops;
844 sch->flags = ops->static_flags;
645 sch->enqueue = ops->enqueue;
646 sch->dequeue = ops->dequeue;
647 sch->dev_queue = dev_queue;
648 dev_hold(dev);
649 refcount_set(&sch->refcnt, 1);
650
651 return sch;
845 sch->enqueue = ops->enqueue;
846 sch->dequeue = ops->dequeue;
847 sch->dev_queue = dev_queue;
848 dev_hold(dev);
849 refcount_set(&sch->refcnt, 1);
850
851 return sch;
852errout1:
853 kfree(p);
652errout:
653 return ERR_PTR(err);
654}
655
656struct Qdisc *qdisc_create_dflt(struct netdev_queue *dev_queue,
657 const struct Qdisc_ops *ops,
854errout:
855 return ERR_PTR(err);
856}
857
858struct Qdisc *qdisc_create_dflt(struct netdev_queue *dev_queue,
859 const struct Qdisc_ops *ops,
658 unsigned int parentid)
860 unsigned int parentid,
861 struct netlink_ext_ack *extack)
659{
660 struct Qdisc *sch;
661
862{
863 struct Qdisc *sch;
864
662 if (!try_module_get(ops->owner))
865 if (!try_module_get(ops->owner)) {
866 NL_SET_ERR_MSG(extack, "Failed to increase module reference counter");
663 return NULL;
867 return NULL;
868 }
664
869
665 sch = qdisc_alloc(dev_queue, ops);
870 sch = qdisc_alloc(dev_queue, ops, extack);
666 if (IS_ERR(sch)) {
667 module_put(ops->owner);
668 return NULL;
669 }
670 sch->parent = parentid;
671
871 if (IS_ERR(sch)) {
872 module_put(ops->owner);
873 return NULL;
874 }
875 sch->parent = parentid;
876
672 if (!ops->init || ops->init(sch, NULL) == 0)
877 if (!ops->init || ops->init(sch, NULL, extack) == 0)
673 return sch;
674
675 qdisc_destroy(sch);
676 return NULL;
677}
678EXPORT_SYMBOL(qdisc_create_dflt);
679
680/* Under qdisc_lock(qdisc) and BH! */
681
682void qdisc_reset(struct Qdisc *qdisc)
683{
684 const struct Qdisc_ops *ops = qdisc->ops;
878 return sch;
879
880 qdisc_destroy(sch);
881 return NULL;
882}
883EXPORT_SYMBOL(qdisc_create_dflt);
884
885/* Under qdisc_lock(qdisc) and BH! */
886
887void qdisc_reset(struct Qdisc *qdisc)
888{
889 const struct Qdisc_ops *ops = qdisc->ops;
890 struct sk_buff *skb, *tmp;
685
686 if (ops->reset)
687 ops->reset(qdisc);
688
891
892 if (ops->reset)
893 ops->reset(qdisc);
894
689 kfree_skb(qdisc->skb_bad_txq);
690 qdisc->skb_bad_txq = NULL;
895 skb_queue_walk_safe(&qdisc->gso_skb, skb, tmp) {
896 __skb_unlink(skb, &qdisc->gso_skb);
897 kfree_skb_list(skb);
898 }
691
899
692 if (qdisc->gso_skb) {
693 kfree_skb_list(qdisc->gso_skb);
694 qdisc->gso_skb = NULL;
900 skb_queue_walk_safe(&qdisc->skb_bad_txq, skb, tmp) {
901 __skb_unlink(skb, &qdisc->skb_bad_txq);
902 kfree_skb_list(skb);
695 }
903 }
904
696 qdisc->q.qlen = 0;
697 qdisc->qstats.backlog = 0;
698}
699EXPORT_SYMBOL(qdisc_reset);
700
905 qdisc->q.qlen = 0;
906 qdisc->qstats.backlog = 0;
907}
908EXPORT_SYMBOL(qdisc_reset);
909
701static void qdisc_free(struct Qdisc *qdisc)
910void qdisc_free(struct Qdisc *qdisc)
702{
703 if (qdisc_is_percpu_stats(qdisc)) {
704 free_percpu(qdisc->cpu_bstats);
705 free_percpu(qdisc->cpu_qstats);
706 }
707
708 kfree((char *) qdisc - qdisc->padded);
709}
710
711void qdisc_destroy(struct Qdisc *qdisc)
712{
713 const struct Qdisc_ops *ops = qdisc->ops;
911{
912 if (qdisc_is_percpu_stats(qdisc)) {
913 free_percpu(qdisc->cpu_bstats);
914 free_percpu(qdisc->cpu_qstats);
915 }
916
917 kfree((char *) qdisc - qdisc->padded);
918}
919
920void qdisc_destroy(struct Qdisc *qdisc)
921{
922 const struct Qdisc_ops *ops = qdisc->ops;
923 struct sk_buff *skb, *tmp;
714
715 if (qdisc->flags & TCQ_F_BUILTIN ||
716 !refcount_dec_and_test(&qdisc->refcnt))
717 return;
718
719#ifdef CONFIG_NET_SCHED
720 qdisc_hash_del(qdisc);
721
722 qdisc_put_stab(rtnl_dereference(qdisc->stab));
723#endif
724 gen_kill_estimator(&qdisc->rate_est);
725 if (ops->reset)
726 ops->reset(qdisc);
727 if (ops->destroy)
728 ops->destroy(qdisc);
729
730 module_put(ops->owner);
731 dev_put(qdisc_dev(qdisc));
732
924
925 if (qdisc->flags & TCQ_F_BUILTIN ||
926 !refcount_dec_and_test(&qdisc->refcnt))
927 return;
928
929#ifdef CONFIG_NET_SCHED
930 qdisc_hash_del(qdisc);
931
932 qdisc_put_stab(rtnl_dereference(qdisc->stab));
933#endif
934 gen_kill_estimator(&qdisc->rate_est);
935 if (ops->reset)
936 ops->reset(qdisc);
937 if (ops->destroy)
938 ops->destroy(qdisc);
939
940 module_put(ops->owner);
941 dev_put(qdisc_dev(qdisc));
942
733 kfree_skb_list(qdisc->gso_skb);
734 kfree_skb(qdisc->skb_bad_txq);
943 skb_queue_walk_safe(&qdisc->gso_skb, skb, tmp) {
944 __skb_unlink(skb, &qdisc->gso_skb);
945 kfree_skb_list(skb);
946 }
947
948 skb_queue_walk_safe(&qdisc->skb_bad_txq, skb, tmp) {
949 __skb_unlink(skb, &qdisc->skb_bad_txq);
950 kfree_skb_list(skb);
951 }
952
735 qdisc_free(qdisc);
736}
737EXPORT_SYMBOL(qdisc_destroy);
738
739/* Attach toplevel qdisc to device queue. */
740struct Qdisc *dev_graft_qdisc(struct netdev_queue *dev_queue,
741 struct Qdisc *qdisc)
742{
743 struct Qdisc *oqdisc = dev_queue->qdisc_sleeping;
744 spinlock_t *root_lock;
745
746 root_lock = qdisc_lock(oqdisc);
747 spin_lock_bh(root_lock);
748
953 qdisc_free(qdisc);
954}
955EXPORT_SYMBOL(qdisc_destroy);
956
957/* Attach toplevel qdisc to device queue. */
958struct Qdisc *dev_graft_qdisc(struct netdev_queue *dev_queue,
959 struct Qdisc *qdisc)
960{
961 struct Qdisc *oqdisc = dev_queue->qdisc_sleeping;
962 spinlock_t *root_lock;
963
964 root_lock = qdisc_lock(oqdisc);
965 spin_lock_bh(root_lock);
966
749 /* Prune old scheduler */
750 if (oqdisc && refcount_read(&oqdisc->refcnt) <= 1)
751 qdisc_reset(oqdisc);
752
753 /* ... and graft new one */
754 if (qdisc == NULL)
755 qdisc = &noop_qdisc;
756 dev_queue->qdisc_sleeping = qdisc;
757 rcu_assign_pointer(dev_queue->qdisc, &noop_qdisc);
758
759 spin_unlock_bh(root_lock);
760

--- 6 unchanged lines hidden (view full) ---

767 void *_unused)
768{
769 struct Qdisc *qdisc;
770 const struct Qdisc_ops *ops = default_qdisc_ops;
771
772 if (dev->priv_flags & IFF_NO_QUEUE)
773 ops = &noqueue_qdisc_ops;
774
967 /* ... and graft new one */
968 if (qdisc == NULL)
969 qdisc = &noop_qdisc;
970 dev_queue->qdisc_sleeping = qdisc;
971 rcu_assign_pointer(dev_queue->qdisc, &noop_qdisc);
972
973 spin_unlock_bh(root_lock);
974

--- 6 unchanged lines hidden (view full) ---

981 void *_unused)
982{
983 struct Qdisc *qdisc;
984 const struct Qdisc_ops *ops = default_qdisc_ops;
985
986 if (dev->priv_flags & IFF_NO_QUEUE)
987 ops = &noqueue_qdisc_ops;
988
775 qdisc = qdisc_create_dflt(dev_queue, ops, TC_H_ROOT);
989 qdisc = qdisc_create_dflt(dev_queue, ops, TC_H_ROOT, NULL);
776 if (!qdisc) {
777 netdev_info(dev, "activation failed\n");
778 return;
779 }
780 if (!netif_is_multiqueue(dev))
781 qdisc->flags |= TCQ_F_ONETXQUEUE | TCQ_F_NOPARENT;
782 dev_queue->qdisc_sleeping = qdisc;
783}

--- 6 unchanged lines hidden (view full) ---

790 txq = netdev_get_tx_queue(dev, 0);
791
792 if (!netif_is_multiqueue(dev) ||
793 dev->priv_flags & IFF_NO_QUEUE) {
794 netdev_for_each_tx_queue(dev, attach_one_default_qdisc, NULL);
795 dev->qdisc = txq->qdisc_sleeping;
796 qdisc_refcount_inc(dev->qdisc);
797 } else {
990 if (!qdisc) {
991 netdev_info(dev, "activation failed\n");
992 return;
993 }
994 if (!netif_is_multiqueue(dev))
995 qdisc->flags |= TCQ_F_ONETXQUEUE | TCQ_F_NOPARENT;
996 dev_queue->qdisc_sleeping = qdisc;
997}

--- 6 unchanged lines hidden (view full) ---

1004 txq = netdev_get_tx_queue(dev, 0);
1005
1006 if (!netif_is_multiqueue(dev) ||
1007 dev->priv_flags & IFF_NO_QUEUE) {
1008 netdev_for_each_tx_queue(dev, attach_one_default_qdisc, NULL);
1009 dev->qdisc = txq->qdisc_sleeping;
1010 qdisc_refcount_inc(dev->qdisc);
1011 } else {
798 qdisc = qdisc_create_dflt(txq, &mq_qdisc_ops, TC_H_ROOT);
1012 qdisc = qdisc_create_dflt(txq, &mq_qdisc_ops, TC_H_ROOT, NULL);
799 if (qdisc) {
800 dev->qdisc = qdisc;
801 qdisc->ops->attach(qdisc);
802 }
803 }
804#ifdef CONFIG_NET_SCHED
805 if (dev->qdisc != &noop_qdisc)
806 qdisc_hash_add(dev->qdisc, false);

--- 73 unchanged lines hidden (view full) ---

880 for (i = 0; i < dev->num_tx_queues; i++) {
881 struct netdev_queue *dev_queue;
882 spinlock_t *root_lock;
883 struct Qdisc *q;
884 int val;
885
886 dev_queue = netdev_get_tx_queue(dev, i);
887 q = dev_queue->qdisc_sleeping;
1013 if (qdisc) {
1014 dev->qdisc = qdisc;
1015 qdisc->ops->attach(qdisc);
1016 }
1017 }
1018#ifdef CONFIG_NET_SCHED
1019 if (dev->qdisc != &noop_qdisc)
1020 qdisc_hash_add(dev->qdisc, false);

--- 73 unchanged lines hidden (view full) ---

1094 for (i = 0; i < dev->num_tx_queues; i++) {
1095 struct netdev_queue *dev_queue;
1096 spinlock_t *root_lock;
1097 struct Qdisc *q;
1098 int val;
1099
1100 dev_queue = netdev_get_tx_queue(dev, i);
1101 q = dev_queue->qdisc_sleeping;
888 root_lock = qdisc_lock(q);
889
1102
890 spin_lock_bh(root_lock);
1103 if (q->flags & TCQ_F_NOLOCK) {
1104 val = test_bit(__QDISC_STATE_SCHED, &q->state);
1105 } else {
1106 root_lock = qdisc_lock(q);
1107 spin_lock_bh(root_lock);
891
1108
892 val = (qdisc_is_running(q) ||
893 test_bit(__QDISC_STATE_SCHED, &q->state));
1109 val = (qdisc_is_running(q) ||
1110 test_bit(__QDISC_STATE_SCHED, &q->state));
894
1111
895 spin_unlock_bh(root_lock);
1112 spin_unlock_bh(root_lock);
1113 }
896
897 if (val)
898 return true;
899 }
900 return false;
901}
902
1114
1115 if (val)
1116 return true;
1117 }
1118 return false;
1119}
1120
1121static void dev_qdisc_reset(struct net_device *dev,
1122 struct netdev_queue *dev_queue,
1123 void *none)
1124{
1125 struct Qdisc *qdisc = dev_queue->qdisc_sleeping;
1126
1127 if (qdisc)
1128 qdisc_reset(qdisc);
1129}
1130
903/**
904 * dev_deactivate_many - deactivate transmissions on several devices
905 * @head: list of devices to deactivate
906 *
907 * This function returns only when all outstanding transmissions
908 * have completed, unless all devices are in dismantle phase.
909 */
910void dev_deactivate_many(struct list_head *head)
911{
912 struct net_device *dev;
1131/**
1132 * dev_deactivate_many - deactivate transmissions on several devices
1133 * @head: list of devices to deactivate
1134 *
1135 * This function returns only when all outstanding transmissions
1136 * have completed, unless all devices are in dismantle phase.
1137 */
1138void dev_deactivate_many(struct list_head *head)
1139{
1140 struct net_device *dev;
913 bool sync_needed = false;
914
915 list_for_each_entry(dev, head, close_list) {
916 netdev_for_each_tx_queue(dev, dev_deactivate_queue,
917 &noop_qdisc);
918 if (dev_ingress_queue(dev))
919 dev_deactivate_queue(dev, dev_ingress_queue(dev),
920 &noop_qdisc);
921
922 dev_watchdog_down(dev);
1141
1142 list_for_each_entry(dev, head, close_list) {
1143 netdev_for_each_tx_queue(dev, dev_deactivate_queue,
1144 &noop_qdisc);
1145 if (dev_ingress_queue(dev))
1146 dev_deactivate_queue(dev, dev_ingress_queue(dev),
1147 &noop_qdisc);
1148
1149 dev_watchdog_down(dev);
923 sync_needed |= !dev->dismantle;
924 }
925
926 /* Wait for outstanding qdisc-less dev_queue_xmit calls.
927 * This is avoided if all devices are in dismantle phase :
928 * Caller will call synchronize_net() for us
929 */
1150 }
1151
1152 /* Wait for outstanding qdisc-less dev_queue_xmit calls.
1153 * This is avoided if all devices are in dismantle phase :
1154 * Caller will call synchronize_net() for us
1155 */
930 if (sync_needed)
931 synchronize_net();
1156 synchronize_net();
932
933 /* Wait for outstanding qdisc_run calls. */
1157
1158 /* Wait for outstanding qdisc_run calls. */
934 list_for_each_entry(dev, head, close_list)
1159 list_for_each_entry(dev, head, close_list) {
935 while (some_qdisc_is_busy(dev))
936 yield();
1160 while (some_qdisc_is_busy(dev))
1161 yield();
1162 /* The new qdisc is assigned at this point so we can safely
1163 * unwind stale skb lists and qdisc statistics
1164 */
1165 netdev_for_each_tx_queue(dev, dev_qdisc_reset, NULL);
1166 if (dev_ingress_queue(dev))
1167 dev_qdisc_reset(dev, dev_ingress_queue(dev), NULL);
1168 }
937}
938
939void dev_deactivate(struct net_device *dev)
940{
941 LIST_HEAD(single);
942
943 list_add(&dev->close_list, &single);
944 dev_deactivate_many(&single);

--- 4 unchanged lines hidden (view full) ---

949static void dev_init_scheduler_queue(struct net_device *dev,
950 struct netdev_queue *dev_queue,
951 void *_qdisc)
952{
953 struct Qdisc *qdisc = _qdisc;
954
955 rcu_assign_pointer(dev_queue->qdisc, qdisc);
956 dev_queue->qdisc_sleeping = qdisc;
1169}
1170
1171void dev_deactivate(struct net_device *dev)
1172{
1173 LIST_HEAD(single);
1174
1175 list_add(&dev->close_list, &single);
1176 dev_deactivate_many(&single);

--- 4 unchanged lines hidden (view full) ---

1181static void dev_init_scheduler_queue(struct net_device *dev,
1182 struct netdev_queue *dev_queue,
1183 void *_qdisc)
1184{
1185 struct Qdisc *qdisc = _qdisc;
1186
1187 rcu_assign_pointer(dev_queue->qdisc, qdisc);
1188 dev_queue->qdisc_sleeping = qdisc;
1189 __skb_queue_head_init(&qdisc->gso_skb);
1190 __skb_queue_head_init(&qdisc->skb_bad_txq);
957}
958
959void dev_init_scheduler(struct net_device *dev)
960{
961 dev->qdisc = &noop_qdisc;
962 netdev_for_each_tx_queue(dev, dev_init_scheduler_queue, &noop_qdisc);
963 if (dev_ingress_queue(dev))
964 dev_init_scheduler_queue(dev, dev_ingress_queue(dev), &noop_qdisc);

--- 113 unchanged lines hidden --- 		1191}
1192
1193void dev_init_scheduler(struct net_device *dev)
1194{
1195 dev->qdisc = &noop_qdisc;
1196 netdev_for_each_tx_queue(dev, dev_init_scheduler_queue, &noop_qdisc);
1197 if (dev_ingress_queue(dev))
1198 dev_init_scheduler_queue(dev, dev_ingress_queue(dev), &noop_qdisc);

--- 113 unchanged lines hidden ---