From f47c11eeccc8820010992eb32dbe7370a08f8bd3 Mon Sep 17 00:00:00 2001 From: "David S. Miller" Date: Fri, 24 Jun 2005 20:18:35 -0700 Subject: [PATCH] [TG3]: Eliminate all hw IRQ handler spinlocks. Move all driver spinlocks to be taken at sw IRQ context only. This fixes the skb_copy() we were doing with hw IRQs disabled (which is illegal and triggers a BUG() with HIGHMEM enabled). It also simplifies the locking all over the driver tremendously. We accomplish this feat by creating a special sequence to synchronize with the hw IRQ handler using a binary state and synchronize_irq(). This idea is from Herbert Xu. Thanks to Michael Chan for helping to track down all of the race conditions in initial versions of this code. Signed-off-by: David S. Miller --- drivers/net/tg3.c | 304 ++++++++++++++++++++-------------------------- drivers/net/tg3.h | 24 +++- 2 files changed, 149 insertions(+), 179 deletions(-) diff --git a/drivers/net/tg3.c b/drivers/net/tg3.c index fef1d08710..8b8aa2ad57 100644 --- a/drivers/net/tg3.c +++ b/drivers/net/tg3.c @@ -337,12 +337,10 @@ static struct { static void tg3_write_indirect_reg32(struct tg3 *tp, u32 off, u32 val) { if ((tp->tg3_flags & TG3_FLAG_PCIX_TARGET_HWBUG) != 0) { - unsigned long flags; - - spin_lock_irqsave(&tp->indirect_lock, flags); + spin_lock_bh(&tp->indirect_lock); pci_write_config_dword(tp->pdev, TG3PCI_REG_BASE_ADDR, off); pci_write_config_dword(tp->pdev, TG3PCI_REG_DATA, val); - spin_unlock_irqrestore(&tp->indirect_lock, flags); + spin_unlock_bh(&tp->indirect_lock); } else { writel(val, tp->regs + off); if ((tp->tg3_flags & TG3_FLAG_5701_REG_WRITE_BUG) != 0) @@ -353,12 +351,10 @@ static void tg3_write_indirect_reg32(struct tg3 *tp, u32 off, u32 val) static void _tw32_flush(struct tg3 *tp, u32 off, u32 val) { if ((tp->tg3_flags & TG3_FLAG_PCIX_TARGET_HWBUG) != 0) { - unsigned long flags; - - spin_lock_irqsave(&tp->indirect_lock, flags); + spin_lock_bh(&tp->indirect_lock); pci_write_config_dword(tp->pdev, TG3PCI_REG_BASE_ADDR, off); pci_write_config_dword(tp->pdev, TG3PCI_REG_DATA, val); - spin_unlock_irqrestore(&tp->indirect_lock, flags); + spin_unlock_bh(&tp->indirect_lock); } else { void __iomem *dest = tp->regs + off; writel(val, dest); @@ -398,28 +394,24 @@ static inline void _tw32_tx_mbox(struct tg3 *tp, u32 off, u32 val) static void tg3_write_mem(struct tg3 *tp, u32 off, u32 val) { - unsigned long flags; - - spin_lock_irqsave(&tp->indirect_lock, flags); + spin_lock_bh(&tp->indirect_lock); pci_write_config_dword(tp->pdev, TG3PCI_MEM_WIN_BASE_ADDR, off); pci_write_config_dword(tp->pdev, TG3PCI_MEM_WIN_DATA, val); /* Always leave this as zero. */ pci_write_config_dword(tp->pdev, TG3PCI_MEM_WIN_BASE_ADDR, 0); - spin_unlock_irqrestore(&tp->indirect_lock, flags); + spin_unlock_bh(&tp->indirect_lock); } static void tg3_read_mem(struct tg3 *tp, u32 off, u32 *val) { - unsigned long flags; - - spin_lock_irqsave(&tp->indirect_lock, flags); + spin_lock_bh(&tp->indirect_lock); pci_write_config_dword(tp->pdev, TG3PCI_MEM_WIN_BASE_ADDR, off); pci_read_config_dword(tp->pdev, TG3PCI_MEM_WIN_DATA, val); /* Always leave this as zero. */ pci_write_config_dword(tp->pdev, TG3PCI_MEM_WIN_BASE_ADDR, 0); - spin_unlock_irqrestore(&tp->indirect_lock, flags); + spin_unlock_bh(&tp->indirect_lock); } static void tg3_disable_ints(struct tg3 *tp) @@ -443,7 +435,7 @@ static void tg3_enable_ints(struct tg3 *tp) tw32_mailbox(MAILBOX_INTERRUPT_0 + TG3_64BIT_REG_LOW, (tp->last_tag << 24)); tr32(MAILBOX_INTERRUPT_0 + TG3_64BIT_REG_LOW); - + tp->irq_sync = 0; tg3_cond_int(tp); } @@ -504,7 +496,8 @@ static inline void tg3_netif_start(struct tg3 *tp) * (such as after tg3_init_hw) */ netif_poll_enable(tp->dev); - tg3_cond_int(tp); + tp->hw_status->status |= SD_STATUS_UPDATED; + tg3_enable_ints(tp); } static void tg3_switch_clocks(struct tg3 *tp) @@ -2578,7 +2571,7 @@ static void tg3_tx(struct tg3 *tp) sw_idx = NEXT_TX(sw_idx); } - dev_kfree_skb_irq(skb); + dev_kfree_skb(skb); } tp->tx_cons = sw_idx; @@ -2884,11 +2877,8 @@ static int tg3_poll(struct net_device *netdev, int *budget) { struct tg3 *tp = netdev_priv(netdev); struct tg3_hw_status *sblk = tp->hw_status; - unsigned long flags; int done; - spin_lock_irqsave(&tp->lock, flags); - /* handle link change and other phy events */ if (!(tp->tg3_flags & (TG3_FLAG_USE_LINKCHG_REG | @@ -2896,7 +2886,9 @@ static int tg3_poll(struct net_device *netdev, int *budget) if (sblk->status & SD_STATUS_LINK_CHG) { sblk->status = SD_STATUS_UPDATED | (sblk->status & ~SD_STATUS_LINK_CHG); + spin_lock(&tp->lock); tg3_setup_phy(tp, 0); + spin_unlock(&tp->lock); } } @@ -2907,8 +2899,6 @@ static int tg3_poll(struct net_device *netdev, int *budget) spin_unlock(&tp->tx_lock); } - spin_unlock_irqrestore(&tp->lock, flags); - /* run RX thread, within the bounds set by NAPI. * All RX "locking" is done by ensuring outside * code synchronizes with dev->poll() @@ -2934,15 +2924,49 @@ static int tg3_poll(struct net_device *netdev, int *budget) /* if no more work, tell net stack and NIC we're done */ done = !tg3_has_work(tp); if (done) { - spin_lock_irqsave(&tp->lock, flags); - __netif_rx_complete(netdev); + spin_lock(&tp->lock); + netif_rx_complete(netdev); tg3_restart_ints(tp); - spin_unlock_irqrestore(&tp->lock, flags); + spin_unlock(&tp->lock); } return (done ? 0 : 1); } +static void tg3_irq_quiesce(struct tg3 *tp) +{ + BUG_ON(tp->irq_sync); + + tp->irq_sync = 1; + smp_mb(); + + synchronize_irq(tp->pdev->irq); +} + +static inline int tg3_irq_sync(struct tg3 *tp) +{ + return tp->irq_sync; +} + +/* Fully shutdown all tg3 driver activity elsewhere in the system. + * If irq_sync is non-zero, then the IRQ handler must be synchronized + * with as well. Most of the time, this is not necessary except when + * shutting down the device. + */ +static inline void tg3_full_lock(struct tg3 *tp, int irq_sync) +{ + if (irq_sync) + tg3_irq_quiesce(tp); + spin_lock_bh(&tp->lock); + spin_lock(&tp->tx_lock); +} + +static inline void tg3_full_unlock(struct tg3 *tp) +{ + spin_unlock(&tp->tx_lock); + spin_unlock_bh(&tp->lock); +} + /* MSI ISR - No need to check for interrupt sharing and no need to * flush status block and interrupt mailbox. PCI ordering rules * guarantee that MSI will arrive after the status block. @@ -2952,9 +2976,6 @@ static irqreturn_t tg3_msi(int irq, void *dev_id, struct pt_regs *regs) struct net_device *dev = dev_id; struct tg3 *tp = netdev_priv(dev); struct tg3_hw_status *sblk = tp->hw_status; - unsigned long flags; - - spin_lock_irqsave(&tp->lock, flags); /* * Writing any value to intr-mbox-0 clears PCI INTA# and @@ -2966,6 +2987,8 @@ static irqreturn_t tg3_msi(int irq, void *dev_id, struct pt_regs *regs) tw32_mailbox(MAILBOX_INTERRUPT_0 + TG3_64BIT_REG_LOW, 0x00000001); tp->last_tag = sblk->status_tag; rmb(); + if (tg3_irq_sync(tp)) + goto out; sblk->status &= ~SD_STATUS_UPDATED; if (likely(tg3_has_work(tp))) netif_rx_schedule(dev); /* schedule NAPI poll */ @@ -2974,9 +2997,7 @@ static irqreturn_t tg3_msi(int irq, void *dev_id, struct pt_regs *regs) tw32_mailbox(MAILBOX_INTERRUPT_0 + TG3_64BIT_REG_LOW, tp->last_tag << 24); } - - spin_unlock_irqrestore(&tp->lock, flags); - +out: return IRQ_RETVAL(1); } @@ -2985,11 +3006,8 @@ static irqreturn_t tg3_interrupt(int irq, void *dev_id, struct pt_regs *regs) struct net_device *dev = dev_id; struct tg3 *tp = netdev_priv(dev); struct tg3_hw_status *sblk = tp->hw_status; - unsigned long flags; unsigned int handled = 1; - spin_lock_irqsave(&tp->lock, flags); - /* In INTx mode, it is possible for the interrupt to arrive at * the CPU before the status block posted prior to the interrupt. * Reading the PCI State register will confirm whether the @@ -3006,6 +3024,8 @@ static irqreturn_t tg3_interrupt(int irq, void *dev_id, struct pt_regs *regs) */ tw32_mailbox(MAILBOX_INTERRUPT_0 + TG3_64BIT_REG_LOW, 0x00000001); + if (tg3_irq_sync(tp)) + goto out; sblk->status &= ~SD_STATUS_UPDATED; if (likely(tg3_has_work(tp))) netif_rx_schedule(dev); /* schedule NAPI poll */ @@ -3020,9 +3040,7 @@ static irqreturn_t tg3_interrupt(int irq, void *dev_id, struct pt_regs *regs) } else { /* shared interrupt */ handled = 0; } - - spin_unlock_irqrestore(&tp->lock, flags); - +out: return IRQ_RETVAL(handled); } @@ -3031,11 +3049,8 @@ static irqreturn_t tg3_interrupt_tagged(int irq, void *dev_id, struct pt_regs *r struct net_device *dev = dev_id; struct tg3 *tp = netdev_priv(dev); struct tg3_hw_status *sblk = tp->hw_status; - unsigned long flags; unsigned int handled = 1; - spin_lock_irqsave(&tp->lock, flags); - /* In INTx mode, it is possible for the interrupt to arrive at * the CPU before the status block posted prior to the interrupt. * Reading the PCI State register will confirm whether the @@ -3054,6 +3069,8 @@ static irqreturn_t tg3_interrupt_tagged(int irq, void *dev_id, struct pt_regs *r 0x00000001); tp->last_tag = sblk->status_tag; rmb(); + if (tg3_irq_sync(tp)) + goto out; sblk->status &= ~SD_STATUS_UPDATED; if (likely(tg3_has_work(tp))) netif_rx_schedule(dev); /* schedule NAPI poll */ @@ -3068,9 +3085,7 @@ static irqreturn_t tg3_interrupt_tagged(int irq, void *dev_id, struct pt_regs *r } else { /* shared interrupt */ handled = 0; } - - spin_unlock_irqrestore(&tp->lock, flags); - +out: return IRQ_RETVAL(handled); } @@ -3109,8 +3124,7 @@ static void tg3_reset_task(void *_data) tg3_netif_stop(tp); - spin_lock_irq(&tp->lock); - spin_lock(&tp->tx_lock); + tg3_full_lock(tp, 1); restart_timer = tp->tg3_flags2 & TG3_FLG2_RESTART_TIMER; tp->tg3_flags2 &= ~TG3_FLG2_RESTART_TIMER; @@ -3120,8 +3134,7 @@ static void tg3_reset_task(void *_data) tg3_netif_start(tp); - spin_unlock(&tp->tx_lock); - spin_unlock_irq(&tp->lock); + tg3_full_unlock(tp); if (restart_timer) mod_timer(&tp->timer, jiffies + 1); @@ -3227,39 +3240,21 @@ static int tg3_start_xmit(struct sk_buff *skb, struct net_device *dev) unsigned int i; u32 len, entry, base_flags, mss; int would_hit_hwbug; - unsigned long flags; len = skb_headlen(skb); /* No BH disabling for tx_lock here. We are running in BH disabled * context and TX reclaim runs via tp->poll inside of a software - * interrupt. Rejoice! - * - * Actually, things are not so simple. If we are to take a hw - * IRQ here, we can deadlock, consider: - * - * CPU1 CPU2 - * tg3_start_xmit - * take tp->tx_lock - * tg3_timer - * take tp->lock - * tg3_interrupt - * spin on tp->lock - * spin on tp->tx_lock - * - * So we really do need to disable interrupts when taking - * tx_lock here. + * interrupt. Furthermore, IRQ processing runs lockless so we have + * no IRQ context deadlocks to worry about either. Rejoice! */ - local_irq_save(flags); - if (!spin_trylock(&tp->tx_lock)) { - local_irq_restore(flags); + if (!spin_trylock(&tp->tx_lock)) return NETDEV_TX_LOCKED; - } /* This is a hard error, log it. */ if (unlikely(TX_BUFFS_AVAIL(tp) <= (skb_shinfo(skb)->nr_frags + 1))) { netif_stop_queue(dev); - spin_unlock_irqrestore(&tp->tx_lock, flags); + spin_unlock(&tp->tx_lock); printk(KERN_ERR PFX "%s: BUG! Tx Ring full when queue awake!\n", dev->name); return NETDEV_TX_BUSY; @@ -3424,7 +3419,7 @@ static int tg3_start_xmit(struct sk_buff *skb, struct net_device *dev) out_unlock: mmiowb(); - spin_unlock_irqrestore(&tp->tx_lock, flags); + spin_unlock(&tp->tx_lock); dev->trans_start = jiffies; @@ -3458,8 +3453,8 @@ static int tg3_change_mtu(struct net_device *dev, int new_mtu) } tg3_netif_stop(tp); - spin_lock_irq(&tp->lock); - spin_lock(&tp->tx_lock); + + tg3_full_lock(tp, 1); tg3_halt(tp, RESET_KIND_SHUTDOWN, 1); @@ -3469,8 +3464,7 @@ static int tg3_change_mtu(struct net_device *dev, int new_mtu) tg3_netif_start(tp); - spin_unlock(&tp->tx_lock); - spin_unlock_irq(&tp->lock); + tg3_full_unlock(tp); return 0; } @@ -5091,9 +5085,9 @@ static int tg3_set_mac_addr(struct net_device *dev, void *p) memcpy(dev->dev_addr, addr->sa_data, dev->addr_len); - spin_lock_irq(&tp->lock); + spin_lock_bh(&tp->lock); __tg3_set_mac_addr(tp); - spin_unlock_irq(&tp->lock); + spin_unlock_bh(&tp->lock); return 0; } @@ -5805,10 +5799,8 @@ static void tg3_periodic_fetch_stats(struct tg3 *tp) static void tg3_timer(unsigned long __opaque) { struct tg3 *tp = (struct tg3 *) __opaque; - unsigned long flags; - spin_lock_irqsave(&tp->lock, flags); - spin_lock(&tp->tx_lock); + spin_lock(&tp->lock); if (!(tp->tg3_flags & TG3_FLAG_TAGGED_STATUS)) { /* All of this garbage is because when using non-tagged @@ -5825,8 +5817,7 @@ static void tg3_timer(unsigned long __opaque) if (!(tr32(WDMAC_MODE) & WDMAC_MODE_ENABLE)) { tp->tg3_flags2 |= TG3_FLG2_RESTART_TIMER; - spin_unlock(&tp->tx_lock); - spin_unlock_irqrestore(&tp->lock, flags); + spin_unlock(&tp->lock); schedule_work(&tp->reset_task); return; } @@ -5894,8 +5885,7 @@ static void tg3_timer(unsigned long __opaque) tp->asf_counter = tp->asf_multiplier; } - spin_unlock(&tp->tx_lock); - spin_unlock_irqrestore(&tp->lock, flags); + spin_unlock(&tp->lock); tp->timer.expires = jiffies + tp->timer_offset; add_timer(&tp->timer); @@ -6010,14 +6000,12 @@ static int tg3_test_msi(struct tg3 *tp) /* Need to reset the chip because the MSI cycle may have terminated * with Master Abort. */ - spin_lock_irq(&tp->lock); - spin_lock(&tp->tx_lock); + tg3_full_lock(tp, 1); tg3_halt(tp, RESET_KIND_SHUTDOWN, 1); err = tg3_init_hw(tp); - spin_unlock(&tp->tx_lock); - spin_unlock_irq(&tp->lock); + tg3_full_unlock(tp); if (err) free_irq(tp->pdev->irq, dev); @@ -6030,14 +6018,12 @@ static int tg3_open(struct net_device *dev) struct tg3 *tp = netdev_priv(dev); int err; - spin_lock_irq(&tp->lock); - spin_lock(&tp->tx_lock); + tg3_full_lock(tp, 0); tg3_disable_ints(tp); tp->tg3_flags &= ~TG3_FLAG_INIT_COMPLETE; - spin_unlock(&tp->tx_lock); - spin_unlock_irq(&tp->lock); + tg3_full_unlock(tp); /* The placement of this call is tied * to the setup and use of Host TX descriptors. @@ -6084,8 +6070,7 @@ static int tg3_open(struct net_device *dev) return err; } - spin_lock_irq(&tp->lock); - spin_lock(&tp->tx_lock); + tg3_full_lock(tp, 0); err = tg3_init_hw(tp); if (err) { @@ -6109,8 +6094,7 @@ static int tg3_open(struct net_device *dev) tp->timer.function = tg3_timer; } - spin_unlock(&tp->tx_lock); - spin_unlock_irq(&tp->lock); + tg3_full_unlock(tp); if (err) { free_irq(tp->pdev->irq, dev); @@ -6126,8 +6110,7 @@ static int tg3_open(struct net_device *dev) err = tg3_test_msi(tp); if (err) { - spin_lock_irq(&tp->lock); - spin_lock(&tp->tx_lock); + tg3_full_lock(tp, 0); if (tp->tg3_flags2 & TG3_FLG2_USING_MSI) { pci_disable_msi(tp->pdev); @@ -6137,22 +6120,19 @@ static int tg3_open(struct net_device *dev) tg3_free_rings(tp); tg3_free_consistent(tp); - spin_unlock(&tp->tx_lock); - spin_unlock_irq(&tp->lock); + tg3_full_unlock(tp); return err; } } - spin_lock_irq(&tp->lock); - spin_lock(&tp->tx_lock); + tg3_full_lock(tp, 0); add_timer(&tp->timer); tp->tg3_flags |= TG3_FLAG_INIT_COMPLETE; tg3_enable_ints(tp); - spin_unlock(&tp->tx_lock); - spin_unlock_irq(&tp->lock); + tg3_full_unlock(tp); netif_start_queue(dev); @@ -6398,8 +6378,7 @@ static int tg3_close(struct net_device *dev) del_timer_sync(&tp->timer); - spin_lock_irq(&tp->lock); - spin_lock(&tp->tx_lock); + tg3_full_lock(tp, 1); #if 0 tg3_dump_state(tp); #endif @@ -6413,8 +6392,7 @@ static int tg3_close(struct net_device *dev) TG3_FLAG_GOT_SERDES_FLOWCTL); netif_carrier_off(tp->dev); - spin_unlock(&tp->tx_lock); - spin_unlock_irq(&tp->lock); + tg3_full_unlock(tp); free_irq(tp->pdev->irq, dev); if (tp->tg3_flags2 & TG3_FLG2_USING_MSI) { @@ -6451,16 +6429,15 @@ static unsigned long calc_crc_errors(struct tg3 *tp) if (!(tp->tg3_flags2 & TG3_FLG2_PHY_SERDES) && (GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5700 || GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5701)) { - unsigned long flags; u32 val; - spin_lock_irqsave(&tp->lock, flags); + spin_lock_bh(&tp->lock); if (!tg3_readphy(tp, 0x1e, &val)) { tg3_writephy(tp, 0x1e, val | 0x8000); tg3_readphy(tp, 0x14, &val); } else val = 0; - spin_unlock_irqrestore(&tp->lock, flags); + spin_unlock_bh(&tp->lock); tp->phy_crc_errors += val; @@ -6722,11 +6699,9 @@ static void tg3_set_rx_mode(struct net_device *dev) { struct tg3 *tp = netdev_priv(dev); - spin_lock_irq(&tp->lock); - spin_lock(&tp->tx_lock); + tg3_full_lock(tp, 0); __tg3_set_rx_mode(dev); - spin_unlock(&tp->tx_lock); - spin_unlock_irq(&tp->lock); + tg3_full_unlock(tp); } #define TG3_REGDUMP_LEN (32 * 1024) @@ -6748,8 +6723,7 @@ static void tg3_get_regs(struct net_device *dev, memset(p, 0, TG3_REGDUMP_LEN); - spin_lock_irq(&tp->lock); - spin_lock(&tp->tx_lock); + tg3_full_lock(tp, 0); #define __GET_REG32(reg) (*(p)++ = tr32(reg)) #define GET_REG32_LOOP(base,len) \ @@ -6799,8 +6773,7 @@ do { p = (u32 *)(orig_p + (reg)); \ #undef GET_REG32_LOOP #undef GET_REG32_1 - spin_unlock(&tp->tx_lock); - spin_unlock_irq(&tp->lock); + tg3_full_unlock(tp); } static int tg3_get_eeprom_len(struct net_device *dev) @@ -6976,8 +6949,7 @@ static int tg3_set_settings(struct net_device *dev, struct ethtool_cmd *cmd) return -EINVAL; } - spin_lock_irq(&tp->lock); - spin_lock(&tp->tx_lock); + tg3_full_lock(tp, 0); tp->link_config.autoneg = cmd->autoneg; if (cmd->autoneg == AUTONEG_ENABLE) { @@ -6993,8 +6965,7 @@ static int tg3_set_settings(struct net_device *dev, struct ethtool_cmd *cmd) if (netif_running(dev)) tg3_setup_phy(tp, 1); - spin_unlock(&tp->tx_lock); - spin_unlock_irq(&tp->lock); + tg3_full_unlock(tp); return 0; } @@ -7030,12 +7001,12 @@ static int tg3_set_wol(struct net_device *dev, struct ethtool_wolinfo *wol) !(tp->tg3_flags & TG3_FLAG_SERDES_WOL_CAP)) return -EINVAL; - spin_lock_irq(&tp->lock); + spin_lock_bh(&tp->lock); if (wol->wolopts & WAKE_MAGIC) tp->tg3_flags |= TG3_FLAG_WOL_ENABLE; else tp->tg3_flags &= ~TG3_FLAG_WOL_ENABLE; - spin_unlock_irq(&tp->lock); + spin_unlock_bh(&tp->lock); return 0; } @@ -7075,7 +7046,7 @@ static int tg3_nway_reset(struct net_device *dev) if (!netif_running(dev)) return -EAGAIN; - spin_lock_irq(&tp->lock); + spin_lock_bh(&tp->lock); r = -EINVAL; tg3_readphy(tp, MII_BMCR, &bmcr); if (!tg3_readphy(tp, MII_BMCR, &bmcr) && @@ -7083,7 +7054,7 @@ static int tg3_nway_reset(struct net_device *dev) tg3_writephy(tp, MII_BMCR, bmcr | BMCR_ANRESTART); r = 0; } - spin_unlock_irq(&tp->lock); + spin_unlock_bh(&tp->lock); return r; } @@ -7114,8 +7085,7 @@ static int tg3_set_ringparam(struct net_device *dev, struct ethtool_ringparam *e if (netif_running(dev)) tg3_netif_stop(tp); - spin_lock_irq(&tp->lock); - spin_lock(&tp->tx_lock); + tg3_full_lock(tp, 0); tp->rx_pending = ering->rx_pending; @@ -7131,8 +7101,7 @@ static int tg3_set_ringparam(struct net_device *dev, struct ethtool_ringparam *e tg3_netif_start(tp); } - spin_unlock(&tp->tx_lock); - spin_unlock_irq(&tp->lock); + tg3_full_unlock(tp); return 0; } @@ -7153,8 +7122,8 @@ static int tg3_set_pauseparam(struct net_device *dev, struct ethtool_pauseparam if (netif_running(dev)) tg3_netif_stop(tp); - spin_lock_irq(&tp->lock); - spin_lock(&tp->tx_lock); + tg3_full_lock(tp, 1); + if (epause->autoneg) tp->tg3_flags |= TG3_FLAG_PAUSE_AUTONEG; else @@ -7173,8 +7142,8 @@ static int tg3_set_pauseparam(struct net_device *dev, struct ethtool_pauseparam tg3_init_hw(tp); tg3_netif_start(tp); } - spin_unlock(&tp->tx_lock); - spin_unlock_irq(&tp->lock); + + tg3_full_unlock(tp); return 0; } @@ -7195,12 +7164,12 @@ static int tg3_set_rx_csum(struct net_device *dev, u32 data) return 0; } - spin_lock_irq(&tp->lock); + spin_lock_bh(&tp->lock); if (data) tp->tg3_flags |= TG3_FLAG_RX_CHECKSUMS; else tp->tg3_flags &= ~TG3_FLAG_RX_CHECKSUMS; - spin_unlock_irq(&tp->lock); + spin_unlock_bh(&tp->lock); return 0; } @@ -7722,8 +7691,7 @@ static void tg3_self_test(struct net_device *dev, struct ethtool_test *etest, if (netif_running(dev)) tg3_netif_stop(tp); - spin_lock_irq(&tp->lock); - spin_lock(&tp->tx_lock); + tg3_full_lock(tp, 1); tg3_halt(tp, RESET_KIND_SUSPEND, 1); tg3_nvram_lock(tp); @@ -7745,14 +7713,14 @@ static void tg3_self_test(struct net_device *dev, struct ethtool_test *etest, data[4] = 1; } - spin_unlock(&tp->tx_lock); - spin_unlock_irq(&tp->lock); + tg3_full_unlock(tp); + if (tg3_test_interrupt(tp) != 0) { etest->flags |= ETH_TEST_FL_FAILED; data[5] = 1; } - spin_lock_irq(&tp->lock); - spin_lock(&tp->tx_lock); + + tg3_full_lock(tp, 0); tg3_halt(tp, RESET_KIND_SHUTDOWN, 1); if (netif_running(dev)) { @@ -7760,8 +7728,8 @@ static void tg3_self_test(struct net_device *dev, struct ethtool_test *etest, tg3_init_hw(tp); tg3_netif_start(tp); } - spin_unlock(&tp->tx_lock); - spin_unlock_irq(&tp->lock); + + tg3_full_unlock(tp); } } @@ -7782,9 +7750,9 @@ static int tg3_ioctl(struct net_device *dev, struct ifreq *ifr, int cmd) if (tp->tg3_flags2 & TG3_FLG2_PHY_SERDES) break; /* We have no PHY */ - spin_lock_irq(&tp->lock); + spin_lock_bh(&tp->lock); err = tg3_readphy(tp, data->reg_num & 0x1f, &mii_regval); - spin_unlock_irq(&tp->lock); + spin_unlock_bh(&tp->lock); data->val_out = mii_regval; @@ -7798,9 +7766,9 @@ static int tg3_ioctl(struct net_device *dev, struct ifreq *ifr, int cmd) if (!capable(CAP_NET_ADMIN)) return -EPERM; - spin_lock_irq(&tp->lock); + spin_lock_bh(&tp->lock); err = tg3_writephy(tp, data->reg_num & 0x1f, data->val_in); - spin_unlock_irq(&tp->lock); + spin_unlock_bh(&tp->lock); return err; @@ -7816,28 +7784,24 @@ static void tg3_vlan_rx_register(struct net_device *dev, struct vlan_group *grp) { struct tg3 *tp = netdev_priv(dev); - spin_lock_irq(&tp->lock); - spin_lock(&tp->tx_lock); + tg3_full_lock(tp, 0); tp->vlgrp = grp; /* Update RX_MODE_KEEP_VLAN_TAG bit in RX_MODE register. */ __tg3_set_rx_mode(dev); - spin_unlock(&tp->tx_lock); - spin_unlock_irq(&tp->lock); + tg3_full_unlock(tp); } static void tg3_vlan_rx_kill_vid(struct net_device *dev, unsigned short vid) { struct tg3 *tp = netdev_priv(dev); - spin_lock_irq(&tp->lock); - spin_lock(&tp->tx_lock); + tg3_full_lock(tp, 0); if (tp->vlgrp) tp->vlgrp->vlan_devices[vid] = NULL; - spin_unlock(&tp->tx_lock); - spin_unlock_irq(&tp->lock); + tg3_full_unlock(tp); } #endif @@ -10168,24 +10132,19 @@ static int tg3_suspend(struct pci_dev *pdev, pm_message_t state) del_timer_sync(&tp->timer); - spin_lock_irq(&tp->lock); - spin_lock(&tp->tx_lock); + tg3_full_lock(tp, 1); tg3_disable_ints(tp); - spin_unlock(&tp->tx_lock); - spin_unlock_irq(&tp->lock); + tg3_full_unlock(tp); netif_device_detach(dev); - spin_lock_irq(&tp->lock); - spin_lock(&tp->tx_lock); + tg3_full_lock(tp, 0); tg3_halt(tp, RESET_KIND_SHUTDOWN, 1); - spin_unlock(&tp->tx_lock); - spin_unlock_irq(&tp->lock); + tg3_full_unlock(tp); err = tg3_set_power_state(tp, pci_choose_state(pdev, state)); if (err) { - spin_lock_irq(&tp->lock); - spin_lock(&tp->tx_lock); + tg3_full_lock(tp, 0); tg3_init_hw(tp); @@ -10195,8 +10154,7 @@ static int tg3_suspend(struct pci_dev *pdev, pm_message_t state) netif_device_attach(dev); tg3_netif_start(tp); - spin_unlock(&tp->tx_lock); - spin_unlock_irq(&tp->lock); + tg3_full_unlock(tp); } return err; @@ -10219,8 +10177,7 @@ static int tg3_resume(struct pci_dev *pdev) netif_device_attach(dev); - spin_lock_irq(&tp->lock); - spin_lock(&tp->tx_lock); + tg3_full_lock(tp, 0); tg3_init_hw(tp); @@ -10231,8 +10188,7 @@ static int tg3_resume(struct pci_dev *pdev) tg3_netif_start(tp); - spin_unlock(&tp->tx_lock); - spin_unlock_irq(&tp->lock); + tg3_full_unlock(tp); return 0; } diff --git a/drivers/net/tg3.h b/drivers/net/tg3.h index 993f84c93d..99c5f9675a 100644 --- a/drivers/net/tg3.h +++ b/drivers/net/tg3.h @@ -2006,17 +2006,31 @@ struct tg3_ethtool_stats { struct tg3 { /* begin "general, frequently-used members" cacheline section */ + /* If the IRQ handler (which runs lockless) needs to be + * quiesced, the following bitmask state is used. The + * SYNC flag is set by non-IRQ context code to initiate + * the quiescence. + * + * When the IRQ handler notices that SYNC is set, it + * disables interrupts and returns. + * + * When all outstanding IRQ handlers have returned after + * the SYNC flag has been set, the setter can be assured + * that interrupts will no longer get run. + * + * In this way all SMP driver locks are never acquired + * in hw IRQ context, only sw IRQ context or lower. + */ + unsigned int irq_sync; + /* SMP locking strategy: * * lock: Held during all operations except TX packet * processing. * - * tx_lock: Held during tg3_start_xmit{,_4gbug} and tg3_tx + * tx_lock: Held during tg3_start_xmit and tg3_tx * - * If you want to shut up all asynchronous processing you must - * acquire both locks, 'lock' taken before 'tx_lock'. IRQs must - * be disabled to take 'lock' but only softirq disabling is - * necessary for acquisition of 'tx_lock'. + * Both of these locks are to be held with BH safety. */ spinlock_t lock; spinlock_t indirect_lock; -- 2.39.5