X-Git-Url: https://err.no/cgi-bin/gitweb.cgi?a=blobdiff_plain;f=net%2Fsched%2Fsch_generic.c;h=9c9cd4d94890d1b1c5fb7d582af730f37248cfc0;hb=e9e80ea5f2129e135e3a6fa260314b1c6d99b19a;hp=739a8711ab30c42df56b7f4efd02f902d8e5003a;hpb=7698b4fcabcd790efc4f226bada1e7b5870653af;p=linux-2.6 diff --git a/net/sched/sch_generic.c b/net/sched/sch_generic.c index 739a8711ab..9c9cd4d948 100644 --- a/net/sched/sch_generic.c +++ b/net/sched/sch_generic.c @@ -29,59 +29,27 @@ /* Main transmission queue. */ /* Modifications to data participating in scheduling must be protected with - * queue->lock spinlock. + * qdisc_root_lock(qdisc) spinlock. * * The idea is the following: - * - enqueue, dequeue are serialized via top level device - * spinlock queue->lock. - * - ingress filtering is serialized via top level device - * spinlock dev->rx_queue.lock. + * - enqueue, dequeue are serialized via qdisc root lock + * - ingress filtering is also serialized via qdisc root lock * - updates to tree and tree walking are only done under the rtnl mutex. */ -void qdisc_lock_tree(struct net_device *dev) - __acquires(dev->rx_queue.lock) -{ - unsigned int i; - - local_bh_disable(); - for (i = 0; i < dev->num_tx_queues; i++) { - struct netdev_queue *txq = netdev_get_tx_queue(dev, i); - spin_lock(&txq->lock); - } - spin_lock(&dev->rx_queue.lock); -} -EXPORT_SYMBOL(qdisc_lock_tree); - -void qdisc_unlock_tree(struct net_device *dev) - __releases(dev->rx_queue.lock) -{ - unsigned int i; - - spin_unlock(&dev->rx_queue.lock); - for (i = 0; i < dev->num_tx_queues; i++) { - struct netdev_queue *txq = netdev_get_tx_queue(dev, i); - spin_unlock(&txq->lock); - } - local_bh_enable(); -} -EXPORT_SYMBOL(qdisc_unlock_tree); - static inline int qdisc_qlen(struct Qdisc *q) { return q->q.qlen; } -static inline int dev_requeue_skb(struct sk_buff *skb, - struct netdev_queue *dev_queue, - struct Qdisc *q) +static inline int dev_requeue_skb(struct sk_buff *skb, struct Qdisc *q) { if (unlikely(skb->next)) q->gso_skb = skb; else q->ops->requeue(skb, q); - netif_schedule_queue(dev_queue); + __netif_schedule(q); return 0; } @@ -121,22 +89,22 @@ static inline int handle_dev_cpu_collision(struct sk_buff *skb, * some time. */ __get_cpu_var(netdev_rx_stat).cpu_collision++; - ret = dev_requeue_skb(skb, dev_queue, q); + ret = dev_requeue_skb(skb, q); } return ret; } /* - * NOTE: Called under queue->lock with locally disabled BH. + * NOTE: Called under qdisc_lock(q) with locally disabled BH. * * __QDISC_STATE_RUNNING guarantees only one CPU can process - * this qdisc at a time. queue->lock serializes queue accesses for - * this queue AND txq->qdisc pointer itself. + * this qdisc at a time. qdisc_lock(q) serializes queue accesses for + * this queue. * * netif_tx_lock serializes accesses to device driver. * - * queue->lock and netif_tx_lock are mutually exclusive, + * qdisc_lock(q) and netif_tx_lock are mutually exclusive, * if one is grabbed, another must be free. * * Note, that this procedure can be called by a watchdog timer @@ -146,9 +114,9 @@ static inline int handle_dev_cpu_collision(struct sk_buff *skb, * >0 - queue is not empty. * */ -static inline int qdisc_restart(struct netdev_queue *txq, - struct Qdisc *q) +static inline int qdisc_restart(struct Qdisc *q) { + struct netdev_queue *txq; int ret = NETDEV_TX_BUSY; struct net_device *dev; spinlock_t *root_lock; @@ -163,10 +131,12 @@ static inline int qdisc_restart(struct netdev_queue *txq, /* And release qdisc */ spin_unlock(root_lock); - dev = txq->dev; + dev = qdisc_dev(q); + txq = netdev_get_tx_queue(dev, skb_get_queue_mapping(skb)); HARD_TX_LOCK(dev, txq, smp_processor_id()); - if (!netif_subqueue_stopped(dev, skb)) + if (!netif_tx_queue_stopped(txq) && + !netif_tx_queue_frozen(txq)) ret = dev_hard_start_xmit(skb, dev, txq); HARD_TX_UNLOCK(dev, txq); @@ -189,29 +159,29 @@ static inline int qdisc_restart(struct netdev_queue *txq, printk(KERN_WARNING "BUG %s code %d qlen %d\n", dev->name, ret, q->q.qlen); - ret = dev_requeue_skb(skb, txq, q); + ret = dev_requeue_skb(skb, q); break; } + if (ret && (netif_tx_queue_stopped(txq) || + netif_tx_queue_frozen(txq))) + ret = 0; + return ret; } -void __qdisc_run(struct netdev_queue *txq) +void __qdisc_run(struct Qdisc *q) { unsigned long start_time = jiffies; - struct Qdisc *q = txq->qdisc; - - while (qdisc_restart(txq, q)) { - if (netif_tx_queue_stopped(txq)) - break; + while (qdisc_restart(q)) { /* * Postpone processing if * 1. another process needs the CPU; * 2. we've been doing it for too long. */ if (need_resched() || jiffies != start_time) { - netif_schedule_queue(txq); + __netif_schedule(q); break; } } @@ -244,9 +214,9 @@ static void dev_watchdog(unsigned long arg) if (some_queue_stopped && time_after(jiffies, (dev->trans_start + dev->watchdog_timeo))) { - printk(KERN_INFO "NETDEV WATCHDOG: %s: " - "transmit timed out\n", - dev->name); + char drivername[64]; + printk(KERN_INFO "NETDEV WATCHDOG: %s (%s): transmit timed out\n", + dev->name, netdev_drivername(dev, drivername, 64)); dev->tx_timeout(dev); WARN_ON_ONCE(1); } @@ -349,7 +319,6 @@ struct Qdisc_ops noop_qdisc_ops __read_mostly = { }; static struct netdev_queue noop_netdev_queue = { - .lock = __SPIN_LOCK_UNLOCKED(noop_netdev_queue.lock), .qdisc = &noop_qdisc, }; @@ -359,6 +328,7 @@ struct Qdisc noop_qdisc = { .flags = TCQ_F_BUILTIN, .ops = &noop_qdisc_ops, .list = LIST_HEAD_INIT(noop_qdisc.list), + .q.lock = __SPIN_LOCK_UNLOCKED(noop_qdisc.q.lock), .dev_queue = &noop_netdev_queue, }; EXPORT_SYMBOL(noop_qdisc); @@ -372,12 +342,19 @@ static struct Qdisc_ops noqueue_qdisc_ops __read_mostly = { .owner = THIS_MODULE, }; +static struct Qdisc noqueue_qdisc; +static struct netdev_queue noqueue_netdev_queue = { + .qdisc = &noqueue_qdisc, +}; + static struct Qdisc noqueue_qdisc = { .enqueue = NULL, .dequeue = noop_dequeue, .flags = TCQ_F_BUILTIN, .ops = &noqueue_qdisc_ops, .list = LIST_HEAD_INIT(noqueue_qdisc.list), + .q.lock = __SPIN_LOCK_UNLOCKED(noqueue_qdisc.q.lock), + .dev_queue = &noqueue_netdev_queue, }; @@ -530,7 +507,7 @@ errout: } EXPORT_SYMBOL(qdisc_create_dflt); -/* Under queue->lock and BH! */ +/* Under qdisc_root_lock(qdisc) and BH! */ void qdisc_reset(struct Qdisc *qdisc) { @@ -547,28 +524,36 @@ EXPORT_SYMBOL(qdisc_reset); static void __qdisc_destroy(struct rcu_head *head) { struct Qdisc *qdisc = container_of(head, struct Qdisc, q_rcu); + const struct Qdisc_ops *ops = qdisc->ops; + +#ifdef CONFIG_NET_SCHED + qdisc_put_stab(qdisc->stab); +#endif + gen_kill_estimator(&qdisc->bstats, &qdisc->rate_est); + if (ops->reset) + ops->reset(qdisc); + if (ops->destroy) + ops->destroy(qdisc); + + module_put(ops->owner); + dev_put(qdisc_dev(qdisc)); + + kfree_skb(qdisc->gso_skb); + kfree((char *) qdisc - qdisc->padded); } -/* Under queue->lock and BH! */ +/* Under qdisc_root_lock(qdisc) and BH! */ void qdisc_destroy(struct Qdisc *qdisc) { - const struct Qdisc_ops *ops = qdisc->ops; - if (qdisc->flags & TCQ_F_BUILTIN || !atomic_dec_and_test(&qdisc->refcnt)) return; - list_del(&qdisc->list); - gen_kill_estimator(&qdisc->bstats, &qdisc->rate_est); - if (ops->reset) - ops->reset(qdisc); - if (ops->destroy) - ops->destroy(qdisc); + if (qdisc->parent) + list_del(&qdisc->list); - module_put(ops->owner); - dev_put(qdisc_dev(qdisc)); call_rcu(&qdisc->q_rcu, __qdisc_destroy); } EXPORT_SYMBOL(qdisc_destroy); @@ -599,7 +584,6 @@ static void attach_one_default_qdisc(struct net_device *dev, printk(KERN_INFO "%s: activation failed\n", dev->name); return; } - list_add_tail(&qdisc->list, &dev_queue->qdisc_list); } else { qdisc = &noqueue_qdisc; } @@ -610,13 +594,12 @@ static void transition_one_qdisc(struct net_device *dev, struct netdev_queue *dev_queue, void *_need_watchdog) { + struct Qdisc *new_qdisc = dev_queue->qdisc_sleeping; int *need_watchdog_p = _need_watchdog; - spin_lock_bh(&dev_queue->lock); - rcu_assign_pointer(dev_queue->qdisc, dev_queue->qdisc_sleeping); - if (dev_queue->qdisc != &noqueue_qdisc) + rcu_assign_pointer(dev_queue->qdisc, new_qdisc); + if (need_watchdog_p && new_qdisc != &noqueue_qdisc) *need_watchdog_p = 1; - spin_unlock_bh(&dev_queue->lock); } void dev_activate(struct net_device *dev) @@ -638,6 +621,7 @@ void dev_activate(struct net_device *dev) need_watchdog = 0; netdev_for_each_tx_queue(dev, transition_one_qdisc, &need_watchdog); + transition_one_qdisc(dev, &dev->rx_queue, NULL); if (need_watchdog) { dev->trans_start = jiffies; @@ -650,23 +634,17 @@ static void dev_deactivate_queue(struct net_device *dev, void *_qdisc_default) { struct Qdisc *qdisc_default = _qdisc_default; - struct sk_buff *skb = NULL; struct Qdisc *qdisc; - spin_lock_bh(&dev_queue->lock); - qdisc = dev_queue->qdisc; if (qdisc) { + spin_lock_bh(qdisc_lock(qdisc)); + dev_queue->qdisc = qdisc_default; qdisc_reset(qdisc); - skb = qdisc->gso_skb; - qdisc->gso_skb = NULL; + spin_unlock_bh(qdisc_lock(qdisc)); } - - spin_unlock_bh(&dev_queue->lock); - - kfree_skb(skb); } static bool some_qdisc_is_running(struct net_device *dev, int lock) @@ -702,6 +680,7 @@ void dev_deactivate(struct net_device *dev) bool running; netdev_for_each_tx_queue(dev, dev_deactivate_queue, &noop_qdisc); + dev_deactivate_queue(dev, &dev->rx_queue, &noop_qdisc); dev_watchdog_down(dev); @@ -738,15 +717,12 @@ static void dev_init_scheduler_queue(struct net_device *dev, dev_queue->qdisc = qdisc; dev_queue->qdisc_sleeping = qdisc; - INIT_LIST_HEAD(&dev_queue->qdisc_list); } void dev_init_scheduler(struct net_device *dev) { - qdisc_lock_tree(dev); netdev_for_each_tx_queue(dev, dev_init_scheduler_queue, &noop_qdisc); - dev_init_scheduler_queue(dev, &dev->rx_queue, NULL); - qdisc_unlock_tree(dev); + dev_init_scheduler_queue(dev, &dev->rx_queue, &noop_qdisc); setup_timer(&dev->watchdog_timer, dev_watchdog, (unsigned long)dev); } @@ -759,18 +735,20 @@ static void shutdown_scheduler_queue(struct net_device *dev, struct Qdisc *qdisc_default = _qdisc_default; if (qdisc) { + spinlock_t *root_lock = qdisc_root_lock(qdisc); + dev_queue->qdisc = qdisc_default; dev_queue->qdisc_sleeping = qdisc_default; + spin_lock_bh(root_lock); qdisc_destroy(qdisc); + spin_unlock_bh(root_lock); } } void dev_shutdown(struct net_device *dev) { - qdisc_lock_tree(dev); netdev_for_each_tx_queue(dev, shutdown_scheduler_queue, &noop_qdisc); - shutdown_scheduler_queue(dev, &dev->rx_queue, NULL); - BUG_TRAP(!timer_pending(&dev->watchdog_timer)); - qdisc_unlock_tree(dev); + shutdown_scheduler_queue(dev, &dev->rx_queue, &noop_qdisc); + WARN_ON(timer_pending(&dev->watchdog_timer)); }