2 * net/sched/sch_api.c Packet scheduler API.
4 * This program is free software; you can redistribute it and/or
5 * modify it under the terms of the GNU General Public License
6 * as published by the Free Software Foundation; either version
7 * 2 of the License, or (at your option) any later version.
9 * Authors: Alexey Kuznetsov, <kuznet@ms2.inr.ac.ru>
13 * Rani Assaf <rani@magic.metawire.com> :980802: JIFFIES and CPU clock sources are repaired.
14 * Eduardo J. Blanco <ejbs@netlabs.com.uy> :990222: kmod support
15 * Jamal Hadi Salim <hadi@nortelnetworks.com>: 990601: ingress support
18 #include <linux/module.h>
19 #include <linux/types.h>
20 #include <linux/kernel.h>
21 #include <linux/string.h>
22 #include <linux/errno.h>
23 #include <linux/skbuff.h>
24 #include <linux/init.h>
25 #include <linux/proc_fs.h>
26 #include <linux/seq_file.h>
27 #include <linux/kmod.h>
28 #include <linux/list.h>
29 #include <linux/hrtimer.h>
31 #include <net/net_namespace.h>
33 #include <net/netlink.h>
34 #include <net/pkt_sched.h>
36 static int qdisc_notify(struct sk_buff *oskb, struct nlmsghdr *n, u32 clid,
37 struct Qdisc *old, struct Qdisc *new);
38 static int tclass_notify(struct sk_buff *oskb, struct nlmsghdr *n,
39 struct Qdisc *q, unsigned long cl, int event);
46 This file consists of two interrelated parts:
48 1. queueing disciplines manager frontend.
49 2. traffic classes manager frontend.
51 Generally, queueing discipline ("qdisc") is a black box,
52 which is able to enqueue packets and to dequeue them (when
53 device is ready to send something) in order and at times
54 determined by algorithm hidden in it.
56 qdisc's are divided to two categories:
57 - "queues", which have no internal structure visible from outside.
58 - "schedulers", which split all the packets to "traffic classes",
59 using "packet classifiers" (look at cls_api.c)
61 In turn, classes may have child qdiscs (as rule, queues)
62 attached to them etc. etc. etc.
64 The goal of the routines in this file is to translate
65 information supplied by user in the form of handles
66 to more intelligible for kernel form, to make some sanity
67 checks and part of work, which is common to all qdiscs
68 and to provide rtnetlink notifications.
70 All real intelligent work is done inside qdisc modules.
74 Every discipline has two major routines: enqueue and dequeue.
78 dequeue usually returns a skb to send. It is allowed to return NULL,
79 but it does not mean that queue is empty, it just means that
80 discipline does not want to send anything this time.
81 Queue is really empty if q->q.qlen == 0.
82 For complicated disciplines with multiple queues q->q is not
83 real packet queue, but however q->q.qlen must be valid.
87 enqueue returns 0, if packet was enqueued successfully.
88 If packet (this one or another one) was dropped, it returns
90 NET_XMIT_DROP - this packet dropped
91 Expected action: do not backoff, but wait until queue will clear.
92 NET_XMIT_CN - probably this packet enqueued, but another one dropped.
93 Expected action: backoff or ignore
94 NET_XMIT_POLICED - dropped by police.
95 Expected action: backoff or error to real-time apps.
101 requeues once dequeued packet. It is used for non-standard or
102 just buggy devices, which can defer output even if netif_queue_stopped()=0.
106 returns qdisc to initial state: purge all buffers, clear all
107 timers, counters (except for statistics) etc.
111 initializes newly created qdisc.
115 destroys resources allocated by init and during lifetime of qdisc.
119 changes qdisc parameters.
122 /* Protects list of registered TC modules. It is pure SMP lock. */
123 static DEFINE_RWLOCK(qdisc_mod_lock);
126 /************************************************
127 * Queueing disciplines manipulation. *
128 ************************************************/
131 /* The list of all installed queueing disciplines. */
133 static struct Qdisc_ops *qdisc_base;
135 /* Register/uregister queueing discipline */
137 int register_qdisc(struct Qdisc_ops *qops)
139 struct Qdisc_ops *q, **qp;
142 write_lock(&qdisc_mod_lock);
143 for (qp = &qdisc_base; (q = *qp) != NULL; qp = &q->next)
144 if (!strcmp(qops->id, q->id))
147 if (qops->enqueue == NULL)
148 qops->enqueue = noop_qdisc_ops.enqueue;
149 if (qops->requeue == NULL)
150 qops->requeue = noop_qdisc_ops.requeue;
151 if (qops->dequeue == NULL)
152 qops->dequeue = noop_qdisc_ops.dequeue;
158 write_unlock(&qdisc_mod_lock);
161 EXPORT_SYMBOL(register_qdisc);
163 int unregister_qdisc(struct Qdisc_ops *qops)
165 struct Qdisc_ops *q, **qp;
168 write_lock(&qdisc_mod_lock);
169 for (qp = &qdisc_base; (q=*qp)!=NULL; qp = &q->next)
177 write_unlock(&qdisc_mod_lock);
180 EXPORT_SYMBOL(unregister_qdisc);
182 /* We know handle. Find qdisc among all qdisc's attached to device
183 (root qdisc, all its children, children of children etc.)
186 struct Qdisc *qdisc_lookup(struct net_device *dev, u32 handle)
190 list_for_each_entry(q, &dev->qdisc_list, list) {
191 if (q->handle == handle)
197 static struct Qdisc *qdisc_leaf(struct Qdisc *p, u32 classid)
201 const struct Qdisc_class_ops *cops = p->ops->cl_ops;
205 cl = cops->get(p, classid);
209 leaf = cops->leaf(p, cl);
214 /* Find queueing discipline by name */
216 static struct Qdisc_ops *qdisc_lookup_ops(struct nlattr *kind)
218 struct Qdisc_ops *q = NULL;
221 read_lock(&qdisc_mod_lock);
222 for (q = qdisc_base; q; q = q->next) {
223 if (nla_strcmp(kind, q->id) == 0) {
224 if (!try_module_get(q->owner))
229 read_unlock(&qdisc_mod_lock);
234 static struct qdisc_rate_table *qdisc_rtab_list;
236 struct qdisc_rate_table *qdisc_get_rtab(struct tc_ratespec *r, struct nlattr *tab)
238 struct qdisc_rate_table *rtab;
240 for (rtab = qdisc_rtab_list; rtab; rtab = rtab->next) {
241 if (memcmp(&rtab->rate, r, sizeof(struct tc_ratespec)) == 0) {
247 if (tab == NULL || r->rate == 0 || r->cell_log == 0 ||
248 nla_len(tab) != TC_RTAB_SIZE)
251 rtab = kmalloc(sizeof(*rtab), GFP_KERNEL);
255 memcpy(rtab->data, nla_data(tab), 1024);
256 rtab->next = qdisc_rtab_list;
257 qdisc_rtab_list = rtab;
261 EXPORT_SYMBOL(qdisc_get_rtab);
263 void qdisc_put_rtab(struct qdisc_rate_table *tab)
265 struct qdisc_rate_table *rtab, **rtabp;
267 if (!tab || --tab->refcnt)
270 for (rtabp = &qdisc_rtab_list; (rtab=*rtabp) != NULL; rtabp = &rtab->next) {
278 EXPORT_SYMBOL(qdisc_put_rtab);
280 static enum hrtimer_restart qdisc_watchdog(struct hrtimer *timer)
282 struct qdisc_watchdog *wd = container_of(timer, struct qdisc_watchdog,
285 wd->qdisc->flags &= ~TCQ_F_THROTTLED;
287 __netif_schedule(wd->qdisc);
289 return HRTIMER_NORESTART;
292 void qdisc_watchdog_init(struct qdisc_watchdog *wd, struct Qdisc *qdisc)
294 hrtimer_init(&wd->timer, CLOCK_MONOTONIC, HRTIMER_MODE_ABS);
295 wd->timer.function = qdisc_watchdog;
298 EXPORT_SYMBOL(qdisc_watchdog_init);
300 void qdisc_watchdog_schedule(struct qdisc_watchdog *wd, psched_time_t expires)
304 wd->qdisc->flags |= TCQ_F_THROTTLED;
305 time = ktime_set(0, 0);
306 time = ktime_add_ns(time, PSCHED_US2NS(expires));
307 hrtimer_start(&wd->timer, time, HRTIMER_MODE_ABS);
309 EXPORT_SYMBOL(qdisc_watchdog_schedule);
311 void qdisc_watchdog_cancel(struct qdisc_watchdog *wd)
313 hrtimer_cancel(&wd->timer);
314 wd->qdisc->flags &= ~TCQ_F_THROTTLED;
316 EXPORT_SYMBOL(qdisc_watchdog_cancel);
318 struct hlist_head *qdisc_class_hash_alloc(unsigned int n)
320 unsigned int size = n * sizeof(struct hlist_head), i;
321 struct hlist_head *h;
323 if (size <= PAGE_SIZE)
324 h = kmalloc(size, GFP_KERNEL);
326 h = (struct hlist_head *)
327 __get_free_pages(GFP_KERNEL, get_order(size));
330 for (i = 0; i < n; i++)
331 INIT_HLIST_HEAD(&h[i]);
336 static void qdisc_class_hash_free(struct hlist_head *h, unsigned int n)
338 unsigned int size = n * sizeof(struct hlist_head);
340 if (size <= PAGE_SIZE)
343 free_pages((unsigned long)h, get_order(size));
346 void qdisc_class_hash_grow(struct Qdisc *sch, struct Qdisc_class_hash *clhash)
348 struct Qdisc_class_common *cl;
349 struct hlist_node *n, *next;
350 struct hlist_head *nhash, *ohash;
351 unsigned int nsize, nmask, osize;
354 /* Rehash when load factor exceeds 0.75 */
355 if (clhash->hashelems * 4 <= clhash->hashsize * 3)
357 nsize = clhash->hashsize * 2;
359 nhash = qdisc_class_hash_alloc(nsize);
363 ohash = clhash->hash;
364 osize = clhash->hashsize;
367 for (i = 0; i < osize; i++) {
368 hlist_for_each_entry_safe(cl, n, next, &ohash[i], hnode) {
369 h = qdisc_class_hash(cl->classid, nmask);
370 hlist_add_head(&cl->hnode, &nhash[h]);
373 clhash->hash = nhash;
374 clhash->hashsize = nsize;
375 clhash->hashmask = nmask;
376 sch_tree_unlock(sch);
378 qdisc_class_hash_free(ohash, osize);
380 EXPORT_SYMBOL(qdisc_class_hash_grow);
382 int qdisc_class_hash_init(struct Qdisc_class_hash *clhash)
384 unsigned int size = 4;
386 clhash->hash = qdisc_class_hash_alloc(size);
387 if (clhash->hash == NULL)
389 clhash->hashsize = size;
390 clhash->hashmask = size - 1;
391 clhash->hashelems = 0;
394 EXPORT_SYMBOL(qdisc_class_hash_init);
396 void qdisc_class_hash_destroy(struct Qdisc_class_hash *clhash)
398 qdisc_class_hash_free(clhash->hash, clhash->hashsize);
400 EXPORT_SYMBOL(qdisc_class_hash_destroy);
402 void qdisc_class_hash_insert(struct Qdisc_class_hash *clhash,
403 struct Qdisc_class_common *cl)
407 INIT_HLIST_NODE(&cl->hnode);
408 h = qdisc_class_hash(cl->classid, clhash->hashmask);
409 hlist_add_head(&cl->hnode, &clhash->hash[h]);
412 EXPORT_SYMBOL(qdisc_class_hash_insert);
414 void qdisc_class_hash_remove(struct Qdisc_class_hash *clhash,
415 struct Qdisc_class_common *cl)
417 hlist_del(&cl->hnode);
420 EXPORT_SYMBOL(qdisc_class_hash_remove);
422 /* Allocate an unique handle from space managed by kernel */
424 static u32 qdisc_alloc_handle(struct net_device *dev)
427 static u32 autohandle = TC_H_MAKE(0x80000000U, 0);
430 autohandle += TC_H_MAKE(0x10000U, 0);
431 if (autohandle == TC_H_MAKE(TC_H_ROOT, 0))
432 autohandle = TC_H_MAKE(0x80000000U, 0);
433 } while (qdisc_lookup(dev, autohandle) && --i > 0);
435 return i>0 ? autohandle : 0;
438 /* Attach toplevel qdisc to device queue. */
440 static struct Qdisc *dev_graft_qdisc(struct netdev_queue *dev_queue,
443 spinlock_t *root_lock;
444 struct Qdisc *oqdisc;
448 if (qdisc && qdisc->flags&TCQ_F_INGRESS)
452 oqdisc = dev_queue->qdisc;
454 oqdisc = dev_queue->qdisc_sleeping;
457 root_lock = qdisc_root_lock(oqdisc);
458 spin_lock_bh(root_lock);
461 /* Prune old scheduler */
462 if (oqdisc && atomic_read(&oqdisc->refcnt) <= 1) {
465 dev_queue->qdisc = NULL;
467 dev_queue->qdisc = qdisc;
471 /* Prune old scheduler */
472 if (oqdisc && atomic_read(&oqdisc->refcnt) <= 1)
475 /* ... and graft new one */
478 dev_queue->qdisc_sleeping = qdisc;
479 dev_queue->qdisc = &noop_qdisc;
482 spin_unlock_bh(root_lock);
487 void qdisc_tree_decrease_qlen(struct Qdisc *sch, unsigned int n)
489 const struct Qdisc_class_ops *cops;
495 while ((parentid = sch->parent)) {
496 if (TC_H_MAJ(parentid) == TC_H_MAJ(TC_H_INGRESS))
499 sch = qdisc_lookup(qdisc_dev(sch), TC_H_MAJ(parentid));
501 WARN_ON(parentid != TC_H_ROOT);
504 cops = sch->ops->cl_ops;
505 if (cops->qlen_notify) {
506 cl = cops->get(sch, parentid);
507 cops->qlen_notify(sch, cl);
513 EXPORT_SYMBOL(qdisc_tree_decrease_qlen);
515 static void notify_and_destroy(struct sk_buff *skb, struct nlmsghdr *n, u32 clid,
516 struct Qdisc *old, struct Qdisc *new)
519 qdisc_notify(skb, n, clid, old, new);
522 spin_lock_bh(&old->q.lock);
524 spin_unlock_bh(&old->q.lock);
528 /* Graft qdisc "new" to class "classid" of qdisc "parent" or
531 * When appropriate send a netlink notification using 'skb'
534 * On success, destroy old qdisc.
537 static int qdisc_graft(struct net_device *dev, struct Qdisc *parent,
538 struct sk_buff *skb, struct nlmsghdr *n, u32 classid,
539 struct Qdisc *new, struct Qdisc *old)
541 struct Qdisc *q = old;
544 if (parent == NULL) {
545 unsigned int i, num_q, ingress;
548 num_q = dev->num_tx_queues;
549 if (q && q->flags & TCQ_F_INGRESS) {
554 if (dev->flags & IFF_UP)
557 for (i = 0; i < num_q; i++) {
558 struct netdev_queue *dev_queue = &dev->rx_queue;
561 dev_queue = netdev_get_tx_queue(dev, i);
564 old = dev_graft_qdisc(dev_queue, q);
566 old = dev_graft_qdisc(dev_queue, new);
568 atomic_inc(&new->refcnt);
570 notify_and_destroy(skb, n, classid, old, new);
573 if (dev->flags & IFF_UP)
576 const struct Qdisc_class_ops *cops = parent->ops->cl_ops;
581 unsigned long cl = cops->get(parent, classid);
583 err = cops->graft(parent, cl, new, &old);
584 cops->put(parent, cl);
588 notify_and_destroy(skb, n, classid, old, new);
594 Allocate and initialize new qdisc.
596 Parameters are passed via opt.
599 static struct Qdisc *
600 qdisc_create(struct net_device *dev, struct netdev_queue *dev_queue,
601 u32 parent, u32 handle, struct nlattr **tca, int *errp)
604 struct nlattr *kind = tca[TCA_KIND];
606 struct Qdisc_ops *ops;
608 ops = qdisc_lookup_ops(kind);
610 if (ops == NULL && kind != NULL) {
612 if (nla_strlcpy(name, kind, IFNAMSIZ) < IFNAMSIZ) {
613 /* We dropped the RTNL semaphore in order to
614 * perform the module load. So, even if we
615 * succeeded in loading the module we have to
616 * tell the caller to replay the request. We
617 * indicate this using -EAGAIN.
618 * We replay the request because the device may
619 * go away in the mean time.
622 request_module("sch_%s", name);
624 ops = qdisc_lookup_ops(kind);
626 /* We will try again qdisc_lookup_ops,
627 * so don't keep a reference.
629 module_put(ops->owner);
641 sch = qdisc_alloc(dev_queue, ops);
647 sch->parent = parent;
649 if (handle == TC_H_INGRESS) {
650 sch->flags |= TCQ_F_INGRESS;
651 handle = TC_H_MAKE(TC_H_INGRESS, 0);
654 handle = qdisc_alloc_handle(dev);
661 sch->handle = handle;
663 if (!ops->init || (err = ops->init(sch, tca[TCA_OPTIONS])) == 0) {
665 err = gen_new_estimator(&sch->bstats, &sch->rate_est,
666 qdisc_root_lock(sch),
670 * Any broken qdiscs that would require
671 * a ops->reset() here? The qdisc was never
672 * in action so it shouldn't be necessary.
679 spin_lock_bh(&dev->qdisc_list_lock);
680 list_add_tail(&sch->list, &dev->qdisc_list);
681 spin_unlock_bh(&dev->qdisc_list_lock);
687 kfree((char *) sch - sch->padded);
689 module_put(ops->owner);
695 static int qdisc_change(struct Qdisc *sch, struct nlattr **tca)
697 if (tca[TCA_OPTIONS]) {
700 if (sch->ops->change == NULL)
702 err = sch->ops->change(sch, tca[TCA_OPTIONS]);
707 gen_replace_estimator(&sch->bstats, &sch->rate_est,
708 qdisc_root_lock(sch), tca[TCA_RATE]);
712 struct check_loop_arg
714 struct qdisc_walker w;
719 static int check_loop_fn(struct Qdisc *q, unsigned long cl, struct qdisc_walker *w);
721 static int check_loop(struct Qdisc *q, struct Qdisc *p, int depth)
723 struct check_loop_arg arg;
725 if (q->ops->cl_ops == NULL)
728 arg.w.stop = arg.w.skip = arg.w.count = 0;
729 arg.w.fn = check_loop_fn;
732 q->ops->cl_ops->walk(q, &arg.w);
733 return arg.w.stop ? -ELOOP : 0;
737 check_loop_fn(struct Qdisc *q, unsigned long cl, struct qdisc_walker *w)
740 const struct Qdisc_class_ops *cops = q->ops->cl_ops;
741 struct check_loop_arg *arg = (struct check_loop_arg *)w;
743 leaf = cops->leaf(q, cl);
745 if (leaf == arg->p || arg->depth > 7)
747 return check_loop(leaf, arg->p, arg->depth + 1);
756 static int tc_get_qdisc(struct sk_buff *skb, struct nlmsghdr *n, void *arg)
758 struct net *net = sock_net(skb->sk);
759 struct tcmsg *tcm = NLMSG_DATA(n);
760 struct nlattr *tca[TCA_MAX + 1];
761 struct net_device *dev;
762 u32 clid = tcm->tcm_parent;
763 struct Qdisc *q = NULL;
764 struct Qdisc *p = NULL;
767 if (net != &init_net)
770 if ((dev = __dev_get_by_index(&init_net, tcm->tcm_ifindex)) == NULL)
773 err = nlmsg_parse(n, sizeof(*tcm), tca, TCA_MAX, NULL);
778 if (clid != TC_H_ROOT) {
779 if (TC_H_MAJ(clid) != TC_H_MAJ(TC_H_INGRESS)) {
780 if ((p = qdisc_lookup(dev, TC_H_MAJ(clid))) == NULL)
782 q = qdisc_leaf(p, clid);
783 } else { /* ingress */
784 q = dev->rx_queue.qdisc;
787 struct netdev_queue *dev_queue;
788 dev_queue = netdev_get_tx_queue(dev, 0);
789 q = dev_queue->qdisc_sleeping;
794 if (tcm->tcm_handle && q->handle != tcm->tcm_handle)
797 if ((q = qdisc_lookup(dev, tcm->tcm_handle)) == NULL)
801 if (tca[TCA_KIND] && nla_strcmp(tca[TCA_KIND], q->ops->id))
804 if (n->nlmsg_type == RTM_DELQDISC) {
809 if ((err = qdisc_graft(dev, p, skb, n, clid, NULL, q)) != 0)
812 qdisc_notify(skb, n, clid, NULL, q);
821 static int tc_modify_qdisc(struct sk_buff *skb, struct nlmsghdr *n, void *arg)
823 struct net *net = sock_net(skb->sk);
825 struct nlattr *tca[TCA_MAX + 1];
826 struct net_device *dev;
831 if (net != &init_net)
835 /* Reinit, just in case something touches this. */
837 clid = tcm->tcm_parent;
840 if ((dev = __dev_get_by_index(&init_net, tcm->tcm_ifindex)) == NULL)
843 err = nlmsg_parse(n, sizeof(*tcm), tca, TCA_MAX, NULL);
848 if (clid != TC_H_ROOT) {
849 if (clid != TC_H_INGRESS) {
850 if ((p = qdisc_lookup(dev, TC_H_MAJ(clid))) == NULL)
852 q = qdisc_leaf(p, clid);
853 } else { /*ingress */
854 q = dev->rx_queue.qdisc;
857 struct netdev_queue *dev_queue;
858 dev_queue = netdev_get_tx_queue(dev, 0);
859 q = dev_queue->qdisc_sleeping;
862 /* It may be default qdisc, ignore it */
863 if (q && q->handle == 0)
866 if (!q || !tcm->tcm_handle || q->handle != tcm->tcm_handle) {
867 if (tcm->tcm_handle) {
868 if (q && !(n->nlmsg_flags&NLM_F_REPLACE))
870 if (TC_H_MIN(tcm->tcm_handle))
872 if ((q = qdisc_lookup(dev, tcm->tcm_handle)) == NULL)
874 if (n->nlmsg_flags&NLM_F_EXCL)
876 if (tca[TCA_KIND] && nla_strcmp(tca[TCA_KIND], q->ops->id))
879 (p && check_loop(q, p, 0)))
881 atomic_inc(&q->refcnt);
887 /* This magic test requires explanation.
889 * We know, that some child q is already
890 * attached to this parent and have choice:
891 * either to change it or to create/graft new one.
893 * 1. We are allowed to create/graft only
894 * if CREATE and REPLACE flags are set.
896 * 2. If EXCL is set, requestor wanted to say,
897 * that qdisc tcm_handle is not expected
898 * to exist, so that we choose create/graft too.
900 * 3. The last case is when no flags are set.
901 * Alas, it is sort of hole in API, we
902 * cannot decide what to do unambiguously.
903 * For now we select create/graft, if
904 * user gave KIND, which does not match existing.
906 if ((n->nlmsg_flags&NLM_F_CREATE) &&
907 (n->nlmsg_flags&NLM_F_REPLACE) &&
908 ((n->nlmsg_flags&NLM_F_EXCL) ||
910 nla_strcmp(tca[TCA_KIND], q->ops->id))))
915 if (!tcm->tcm_handle)
917 q = qdisc_lookup(dev, tcm->tcm_handle);
920 /* Change qdisc parameters */
923 if (n->nlmsg_flags&NLM_F_EXCL)
925 if (tca[TCA_KIND] && nla_strcmp(tca[TCA_KIND], q->ops->id))
927 err = qdisc_change(q, tca);
929 qdisc_notify(skb, n, clid, NULL, q);
933 if (!(n->nlmsg_flags&NLM_F_CREATE))
935 if (clid == TC_H_INGRESS)
936 q = qdisc_create(dev, &dev->rx_queue,
937 tcm->tcm_parent, tcm->tcm_parent,
940 q = qdisc_create(dev, netdev_get_tx_queue(dev, 0),
941 tcm->tcm_parent, tcm->tcm_handle,
951 spinlock_t *root_lock;
953 err = qdisc_graft(dev, p, skb, n, clid, q, NULL);
956 root_lock = qdisc_root_lock(q);
957 spin_lock_bh(root_lock);
959 spin_unlock_bh(root_lock);
967 static int tc_fill_qdisc(struct sk_buff *skb, struct Qdisc *q, u32 clid,
968 u32 pid, u32 seq, u16 flags, int event)
971 struct nlmsghdr *nlh;
972 unsigned char *b = skb_tail_pointer(skb);
975 nlh = NLMSG_NEW(skb, pid, seq, event, sizeof(*tcm), flags);
976 tcm = NLMSG_DATA(nlh);
977 tcm->tcm_family = AF_UNSPEC;
980 tcm->tcm_ifindex = qdisc_dev(q)->ifindex;
981 tcm->tcm_parent = clid;
982 tcm->tcm_handle = q->handle;
983 tcm->tcm_info = atomic_read(&q->refcnt);
984 NLA_PUT_STRING(skb, TCA_KIND, q->ops->id);
985 if (q->ops->dump && q->ops->dump(q, skb) < 0)
986 goto nla_put_failure;
987 q->qstats.qlen = q->q.qlen;
989 if (gnet_stats_start_copy_compat(skb, TCA_STATS2, TCA_STATS,
990 TCA_XSTATS, qdisc_root_lock(q), &d) < 0)
991 goto nla_put_failure;
993 if (q->ops->dump_stats && q->ops->dump_stats(q, &d) < 0)
994 goto nla_put_failure;
996 if (gnet_stats_copy_basic(&d, &q->bstats) < 0 ||
997 gnet_stats_copy_rate_est(&d, &q->rate_est) < 0 ||
998 gnet_stats_copy_queue(&d, &q->qstats) < 0)
999 goto nla_put_failure;
1001 if (gnet_stats_finish_copy(&d) < 0)
1002 goto nla_put_failure;
1004 nlh->nlmsg_len = skb_tail_pointer(skb) - b;
1013 static int qdisc_notify(struct sk_buff *oskb, struct nlmsghdr *n,
1014 u32 clid, struct Qdisc *old, struct Qdisc *new)
1016 struct sk_buff *skb;
1017 u32 pid = oskb ? NETLINK_CB(oskb).pid : 0;
1019 skb = alloc_skb(NLMSG_GOODSIZE, GFP_KERNEL);
1023 if (old && old->handle) {
1024 if (tc_fill_qdisc(skb, old, clid, pid, n->nlmsg_seq, 0, RTM_DELQDISC) < 0)
1028 if (tc_fill_qdisc(skb, new, clid, pid, n->nlmsg_seq, old ? NLM_F_REPLACE : 0, RTM_NEWQDISC) < 0)
1033 return rtnetlink_send(skb, &init_net, pid, RTNLGRP_TC, n->nlmsg_flags&NLM_F_ECHO);
1040 static int tc_dump_qdisc(struct sk_buff *skb, struct netlink_callback *cb)
1042 struct net *net = sock_net(skb->sk);
1045 struct net_device *dev;
1048 if (net != &init_net)
1051 s_idx = cb->args[0];
1052 s_q_idx = q_idx = cb->args[1];
1053 read_lock(&dev_base_lock);
1055 for_each_netdev(&init_net, dev) {
1061 list_for_each_entry(q, &dev->qdisc_list, list) {
1062 if (q_idx < s_q_idx) {
1066 if (tc_fill_qdisc(skb, q, q->parent, NETLINK_CB(cb->skb).pid,
1067 cb->nlh->nlmsg_seq, NLM_F_MULTI, RTM_NEWQDISC) <= 0)
1076 read_unlock(&dev_base_lock);
1079 cb->args[1] = q_idx;
1086 /************************************************
1087 * Traffic classes manipulation. *
1088 ************************************************/
1092 static int tc_ctl_tclass(struct sk_buff *skb, struct nlmsghdr *n, void *arg)
1094 struct net *net = sock_net(skb->sk);
1095 struct netdev_queue *dev_queue;
1096 struct tcmsg *tcm = NLMSG_DATA(n);
1097 struct nlattr *tca[TCA_MAX + 1];
1098 struct net_device *dev;
1099 struct Qdisc *q = NULL;
1100 const struct Qdisc_class_ops *cops;
1101 unsigned long cl = 0;
1102 unsigned long new_cl;
1103 u32 pid = tcm->tcm_parent;
1104 u32 clid = tcm->tcm_handle;
1105 u32 qid = TC_H_MAJ(clid);
1108 if (net != &init_net)
1111 if ((dev = __dev_get_by_index(&init_net, tcm->tcm_ifindex)) == NULL)
1114 err = nlmsg_parse(n, sizeof(*tcm), tca, TCA_MAX, NULL);
1119 parent == TC_H_UNSPEC - unspecified parent.
1120 parent == TC_H_ROOT - class is root, which has no parent.
1121 parent == X:0 - parent is root class.
1122 parent == X:Y - parent is a node in hierarchy.
1123 parent == 0:Y - parent is X:Y, where X:0 is qdisc.
1125 handle == 0:0 - generate handle from kernel pool.
1126 handle == 0:Y - class is X:Y, where X:0 is qdisc.
1127 handle == X:Y - clear.
1128 handle == X:0 - root class.
1131 /* Step 1. Determine qdisc handle X:0 */
1133 dev_queue = netdev_get_tx_queue(dev, 0);
1134 if (pid != TC_H_ROOT) {
1135 u32 qid1 = TC_H_MAJ(pid);
1138 /* If both majors are known, they must be identical. */
1143 } else if (qid == 0)
1144 qid = dev_queue->qdisc_sleeping->handle;
1146 /* Now qid is genuine qdisc handle consistent
1147 both with parent and child.
1149 TC_H_MAJ(pid) still may be unspecified, complete it now.
1152 pid = TC_H_MAKE(qid, pid);
1155 qid = dev_queue->qdisc_sleeping->handle;
1158 /* OK. Locate qdisc */
1159 if ((q = qdisc_lookup(dev, qid)) == NULL)
1162 /* An check that it supports classes */
1163 cops = q->ops->cl_ops;
1167 /* Now try to get class */
1169 if (pid == TC_H_ROOT)
1172 clid = TC_H_MAKE(qid, clid);
1175 cl = cops->get(q, clid);
1179 if (n->nlmsg_type != RTM_NEWTCLASS || !(n->nlmsg_flags&NLM_F_CREATE))
1182 switch (n->nlmsg_type) {
1185 if (n->nlmsg_flags&NLM_F_EXCL)
1189 err = cops->delete(q, cl);
1191 tclass_notify(skb, n, q, cl, RTM_DELTCLASS);
1194 err = tclass_notify(skb, n, q, cl, RTM_NEWTCLASS);
1203 err = cops->change(q, clid, pid, tca, &new_cl);
1205 tclass_notify(skb, n, q, new_cl, RTM_NEWTCLASS);
1215 static int tc_fill_tclass(struct sk_buff *skb, struct Qdisc *q,
1217 u32 pid, u32 seq, u16 flags, int event)
1220 struct nlmsghdr *nlh;
1221 unsigned char *b = skb_tail_pointer(skb);
1223 const struct Qdisc_class_ops *cl_ops = q->ops->cl_ops;
1225 nlh = NLMSG_NEW(skb, pid, seq, event, sizeof(*tcm), flags);
1226 tcm = NLMSG_DATA(nlh);
1227 tcm->tcm_family = AF_UNSPEC;
1228 tcm->tcm_ifindex = qdisc_dev(q)->ifindex;
1229 tcm->tcm_parent = q->handle;
1230 tcm->tcm_handle = q->handle;
1232 NLA_PUT_STRING(skb, TCA_KIND, q->ops->id);
1233 if (cl_ops->dump && cl_ops->dump(q, cl, skb, tcm) < 0)
1234 goto nla_put_failure;
1236 if (gnet_stats_start_copy_compat(skb, TCA_STATS2, TCA_STATS,
1237 TCA_XSTATS, qdisc_root_lock(q), &d) < 0)
1238 goto nla_put_failure;
1240 if (cl_ops->dump_stats && cl_ops->dump_stats(q, cl, &d) < 0)
1241 goto nla_put_failure;
1243 if (gnet_stats_finish_copy(&d) < 0)
1244 goto nla_put_failure;
1246 nlh->nlmsg_len = skb_tail_pointer(skb) - b;
1255 static int tclass_notify(struct sk_buff *oskb, struct nlmsghdr *n,
1256 struct Qdisc *q, unsigned long cl, int event)
1258 struct sk_buff *skb;
1259 u32 pid = oskb ? NETLINK_CB(oskb).pid : 0;
1261 skb = alloc_skb(NLMSG_GOODSIZE, GFP_KERNEL);
1265 if (tc_fill_tclass(skb, q, cl, pid, n->nlmsg_seq, 0, event) < 0) {
1270 return rtnetlink_send(skb, &init_net, pid, RTNLGRP_TC, n->nlmsg_flags&NLM_F_ECHO);
1273 struct qdisc_dump_args
1275 struct qdisc_walker w;
1276 struct sk_buff *skb;
1277 struct netlink_callback *cb;
1280 static int qdisc_class_dump(struct Qdisc *q, unsigned long cl, struct qdisc_walker *arg)
1282 struct qdisc_dump_args *a = (struct qdisc_dump_args *)arg;
1284 return tc_fill_tclass(a->skb, q, cl, NETLINK_CB(a->cb->skb).pid,
1285 a->cb->nlh->nlmsg_seq, NLM_F_MULTI, RTM_NEWTCLASS);
1288 static int tc_dump_tclass(struct sk_buff *skb, struct netlink_callback *cb)
1290 struct net *net = sock_net(skb->sk);
1293 struct net_device *dev;
1295 struct tcmsg *tcm = (struct tcmsg*)NLMSG_DATA(cb->nlh);
1296 struct qdisc_dump_args arg;
1298 if (net != &init_net)
1301 if (cb->nlh->nlmsg_len < NLMSG_LENGTH(sizeof(*tcm)))
1303 if ((dev = dev_get_by_index(&init_net, tcm->tcm_ifindex)) == NULL)
1309 list_for_each_entry(q, &dev->qdisc_list, list) {
1310 if (t < s_t || !q->ops->cl_ops ||
1312 TC_H_MAJ(tcm->tcm_parent) != q->handle)) {
1317 memset(&cb->args[1], 0, sizeof(cb->args)-sizeof(cb->args[0]));
1318 arg.w.fn = qdisc_class_dump;
1322 arg.w.skip = cb->args[1];
1324 q->ops->cl_ops->walk(q, &arg.w);
1325 cb->args[1] = arg.w.count;
1337 /* Main classifier routine: scans classifier chain attached
1338 to this qdisc, (optionally) tests for protocol and asks
1339 specific classifiers.
1341 int tc_classify_compat(struct sk_buff *skb, struct tcf_proto *tp,
1342 struct tcf_result *res)
1344 __be16 protocol = skb->protocol;
1347 for (; tp; tp = tp->next) {
1348 if ((tp->protocol == protocol ||
1349 tp->protocol == htons(ETH_P_ALL)) &&
1350 (err = tp->classify(skb, tp, res)) >= 0) {
1351 #ifdef CONFIG_NET_CLS_ACT
1352 if (err != TC_ACT_RECLASSIFY && skb->tc_verd)
1353 skb->tc_verd = SET_TC_VERD(skb->tc_verd, 0);
1360 EXPORT_SYMBOL(tc_classify_compat);
1362 int tc_classify(struct sk_buff *skb, struct tcf_proto *tp,
1363 struct tcf_result *res)
1367 #ifdef CONFIG_NET_CLS_ACT
1368 struct tcf_proto *otp = tp;
1371 protocol = skb->protocol;
1373 err = tc_classify_compat(skb, tp, res);
1374 #ifdef CONFIG_NET_CLS_ACT
1375 if (err == TC_ACT_RECLASSIFY) {
1376 u32 verd = G_TC_VERD(skb->tc_verd);
1379 if (verd++ >= MAX_REC_LOOP) {
1380 printk("rule prio %u protocol %02x reclassify loop, "
1382 tp->prio&0xffff, ntohs(tp->protocol));
1385 skb->tc_verd = SET_TC_VERD(skb->tc_verd, verd);
1391 EXPORT_SYMBOL(tc_classify);
1393 void tcf_destroy(struct tcf_proto *tp)
1395 tp->ops->destroy(tp);
1396 module_put(tp->ops->owner);
1400 void tcf_destroy_chain(struct tcf_proto **fl)
1402 struct tcf_proto *tp;
1404 while ((tp = *fl) != NULL) {
1409 EXPORT_SYMBOL(tcf_destroy_chain);
1411 #ifdef CONFIG_PROC_FS
1412 static int psched_show(struct seq_file *seq, void *v)
1416 hrtimer_get_res(CLOCK_MONOTONIC, &ts);
1417 seq_printf(seq, "%08x %08x %08x %08x\n",
1418 (u32)NSEC_PER_USEC, (u32)PSCHED_US2NS(1),
1420 (u32)NSEC_PER_SEC/(u32)ktime_to_ns(timespec_to_ktime(ts)));
1425 static int psched_open(struct inode *inode, struct file *file)
1427 return single_open(file, psched_show, PDE(inode)->data);
1430 static const struct file_operations psched_fops = {
1431 .owner = THIS_MODULE,
1432 .open = psched_open,
1434 .llseek = seq_lseek,
1435 .release = single_release,
1439 static int __init pktsched_init(void)
1441 register_qdisc(&pfifo_qdisc_ops);
1442 register_qdisc(&bfifo_qdisc_ops);
1443 proc_net_fops_create(&init_net, "psched", 0, &psched_fops);
1445 rtnl_register(PF_UNSPEC, RTM_NEWQDISC, tc_modify_qdisc, NULL);
1446 rtnl_register(PF_UNSPEC, RTM_DELQDISC, tc_get_qdisc, NULL);
1447 rtnl_register(PF_UNSPEC, RTM_GETQDISC, tc_get_qdisc, tc_dump_qdisc);
1448 rtnl_register(PF_UNSPEC, RTM_NEWTCLASS, tc_ctl_tclass, NULL);
1449 rtnl_register(PF_UNSPEC, RTM_DELTCLASS, tc_ctl_tclass, NULL);
1450 rtnl_register(PF_UNSPEC, RTM_GETTCLASS, tc_ctl_tclass, tc_dump_tclass);
1455 subsys_initcall(pktsched_init);