From: Divy Le Ray Date: Thu, 22 May 2008 01:56:21 +0000 (-0700) Subject: cxgb3 - Add page support to jumbo frame Rx queue X-Git-Tag: v2.6.27-rc1~969^2~394^2~21 X-Git-Url: https://err.no/cgi-bin/gitweb.cgi?a=commitdiff_plain;h=7385ecf339c504933a98581c2056d83b69b2a82b;p=linux-2.6 cxgb3 - Add page support to jumbo frame Rx queue Add page support to Jumbo frame Rx queues. Signed-off-by: Divy Le Ray Signed-off-by: Jeff Garzik --- diff --git a/drivers/net/cxgb3/adapter.h b/drivers/net/cxgb3/adapter.h index acebe431d0..263e4faf45 100644 --- a/drivers/net/cxgb3/adapter.h +++ b/drivers/net/cxgb3/adapter.h @@ -92,6 +92,7 @@ struct sge_fl { /* SGE per free-buffer list state */ unsigned int gen; /* free list generation */ struct fl_pg_chunk pg_chunk;/* page chunk cache */ unsigned int use_pages; /* whether FL uses pages or sk_buffs */ + unsigned int order; /* order of page allocations */ struct rx_desc *desc; /* address of HW Rx descriptor ring */ struct rx_sw_desc *sdesc; /* address of SW Rx descriptor ring */ dma_addr_t phys_addr; /* physical address of HW ring start */ @@ -116,12 +117,15 @@ struct sge_rspq { /* state for an SGE response queue */ unsigned int polling; /* is the queue serviced through NAPI? */ unsigned int holdoff_tmr; /* interrupt holdoff timer in 100ns */ unsigned int next_holdoff; /* holdoff time for next interrupt */ + unsigned int rx_recycle_buf; /* whether recycling occurred + within current sop-eop */ struct rsp_desc *desc; /* address of HW response ring */ dma_addr_t phys_addr; /* physical address of the ring */ unsigned int cntxt_id; /* SGE context id for the response q */ spinlock_t lock; /* guards response processing */ struct sk_buff *rx_head; /* offload packet receive queue head */ struct sk_buff *rx_tail; /* offload packet receive queue tail */ + struct sk_buff *pg_skb; /* used to build frag list in napi handler */ unsigned long offload_pkts; unsigned long offload_bundles; diff --git a/drivers/net/cxgb3/sge.c b/drivers/net/cxgb3/sge.c index 0741deb86c..3e91be55e1 100644 --- a/drivers/net/cxgb3/sge.c +++ b/drivers/net/cxgb3/sge.c @@ -55,6 +55,9 @@ * directly. */ #define FL0_PG_CHUNK_SIZE 2048 +#define FL0_PG_ORDER 0 +#define FL1_PG_CHUNK_SIZE (PAGE_SIZE > 8192 ? 16384 : 8192) +#define FL1_PG_ORDER (PAGE_SIZE > 8192 ? 0 : 1) #define SGE_RX_DROP_THRES 16 @@ -359,7 +362,7 @@ static void free_rx_bufs(struct pci_dev *pdev, struct sge_fl *q) } if (q->pg_chunk.page) { - __free_page(q->pg_chunk.page); + __free_pages(q->pg_chunk.page, q->order); q->pg_chunk.page = NULL; } } @@ -396,10 +399,11 @@ static inline int add_one_rx_buf(void *va, unsigned int len, return 0; } -static int alloc_pg_chunk(struct sge_fl *q, struct rx_sw_desc *sd, gfp_t gfp) +static int alloc_pg_chunk(struct sge_fl *q, struct rx_sw_desc *sd, gfp_t gfp, + unsigned int order) { if (!q->pg_chunk.page) { - q->pg_chunk.page = alloc_page(gfp); + q->pg_chunk.page = alloc_pages(gfp, order); if (unlikely(!q->pg_chunk.page)) return -ENOMEM; q->pg_chunk.va = page_address(q->pg_chunk.page); @@ -408,7 +412,7 @@ static int alloc_pg_chunk(struct sge_fl *q, struct rx_sw_desc *sd, gfp_t gfp) sd->pg_chunk = q->pg_chunk; q->pg_chunk.offset += q->buf_size; - if (q->pg_chunk.offset == PAGE_SIZE) + if (q->pg_chunk.offset == (PAGE_SIZE << order)) q->pg_chunk.page = NULL; else { q->pg_chunk.va += q->buf_size; @@ -439,7 +443,7 @@ static int refill_fl(struct adapter *adap, struct sge_fl *q, int n, gfp_t gfp) int err; if (q->use_pages) { - if (unlikely(alloc_pg_chunk(q, sd, gfp))) { + if (unlikely(alloc_pg_chunk(q, sd, gfp, q->order))) { nomem: q->alloc_failed++; break; } @@ -484,7 +488,8 @@ nomem: q->alloc_failed++; static inline void __refill_fl(struct adapter *adap, struct sge_fl *fl) { - refill_fl(adap, fl, min(16U, fl->size - fl->credits), GFP_ATOMIC); + refill_fl(adap, fl, min(16U, fl->size - fl->credits), + GFP_ATOMIC | __GFP_COMP); } /** @@ -759,19 +764,22 @@ use_orig_buf: * that are page chunks rather than sk_buffs. */ static struct sk_buff *get_packet_pg(struct adapter *adap, struct sge_fl *fl, - unsigned int len, unsigned int drop_thres) + struct sge_rspq *q, unsigned int len, + unsigned int drop_thres) { - struct sk_buff *skb = NULL; + struct sk_buff *newskb, *skb; struct rx_sw_desc *sd = &fl->sdesc[fl->cidx]; - if (len <= SGE_RX_COPY_THRES) { - skb = alloc_skb(len, GFP_ATOMIC); - if (likely(skb != NULL)) { - __skb_put(skb, len); + newskb = skb = q->pg_skb; + + if (!skb && (len <= SGE_RX_COPY_THRES)) { + newskb = alloc_skb(len, GFP_ATOMIC); + if (likely(newskb != NULL)) { + __skb_put(newskb, len); pci_dma_sync_single_for_cpu(adap->pdev, pci_unmap_addr(sd, dma_addr), len, PCI_DMA_FROMDEVICE); - memcpy(skb->data, sd->pg_chunk.va, len); + memcpy(newskb->data, sd->pg_chunk.va, len); pci_dma_sync_single_for_device(adap->pdev, pci_unmap_addr(sd, dma_addr), len, PCI_DMA_FROMDEVICE); @@ -780,14 +788,16 @@ static struct sk_buff *get_packet_pg(struct adapter *adap, struct sge_fl *fl, recycle: fl->credits--; recycle_rx_buf(adap, fl, fl->cidx); - return skb; + q->rx_recycle_buf++; + return newskb; } - if (unlikely(fl->credits <= drop_thres)) + if (unlikely(q->rx_recycle_buf || (!skb && fl->credits <= drop_thres))) goto recycle; - skb = alloc_skb(SGE_RX_PULL_LEN, GFP_ATOMIC); - if (unlikely(!skb)) { + if (!skb) + newskb = alloc_skb(SGE_RX_PULL_LEN, GFP_ATOMIC); + if (unlikely(!newskb)) { if (!drop_thres) return NULL; goto recycle; @@ -795,21 +805,29 @@ recycle: pci_unmap_single(adap->pdev, pci_unmap_addr(sd, dma_addr), fl->buf_size, PCI_DMA_FROMDEVICE); - __skb_put(skb, SGE_RX_PULL_LEN); - memcpy(skb->data, sd->pg_chunk.va, SGE_RX_PULL_LEN); - skb_fill_page_desc(skb, 0, sd->pg_chunk.page, - sd->pg_chunk.offset + SGE_RX_PULL_LEN, - len - SGE_RX_PULL_LEN); - skb->len = len; - skb->data_len = len - SGE_RX_PULL_LEN; - skb->truesize += skb->data_len; + if (!skb) { + __skb_put(newskb, SGE_RX_PULL_LEN); + memcpy(newskb->data, sd->pg_chunk.va, SGE_RX_PULL_LEN); + skb_fill_page_desc(newskb, 0, sd->pg_chunk.page, + sd->pg_chunk.offset + SGE_RX_PULL_LEN, + len - SGE_RX_PULL_LEN); + newskb->len = len; + newskb->data_len = len - SGE_RX_PULL_LEN; + } else { + skb_fill_page_desc(newskb, skb_shinfo(newskb)->nr_frags, + sd->pg_chunk.page, + sd->pg_chunk.offset, len); + newskb->len += len; + newskb->data_len += len; + } + newskb->truesize += newskb->data_len; fl->credits--; /* * We do not refill FLs here, we let the caller do it to overlap a * prefetch. */ - return skb; + return newskb; } /** @@ -1966,6 +1984,12 @@ static inline int is_new_response(const struct rsp_desc *r, return (r->intr_gen & F_RSPD_GEN2) == q->gen; } +static inline void clear_rspq_bufstate(struct sge_rspq * const q) +{ + q->pg_skb = NULL; + q->rx_recycle_buf = 0; +} + #define RSPD_GTS_MASK (F_RSPD_TXQ0_GTS | F_RSPD_TXQ1_GTS) #define RSPD_CTRL_MASK (RSPD_GTS_MASK | \ V_RSPD_TXQ0_CR(M_RSPD_TXQ0_CR) | \ @@ -2003,10 +2027,11 @@ static int process_responses(struct adapter *adap, struct sge_qset *qs, q->next_holdoff = q->holdoff_tmr; while (likely(budget_left && is_new_response(r, q))) { - int eth, ethpad = 2; + int packet_complete, eth, ethpad = 2; struct sk_buff *skb = NULL; u32 len, flags = ntohl(r->flags); - __be32 rss_hi = *(const __be32 *)r, rss_lo = r->rss_hdr.rss_hash_val; + __be32 rss_hi = *(const __be32 *)r, + rss_lo = r->rss_hdr.rss_hash_val; eth = r->rss_hdr.opcode == CPL_RX_PKT; @@ -2044,8 +2069,11 @@ no_mem: #endif __refill_fl(adap, fl); - skb = get_packet_pg(adap, fl, G_RSPD_LEN(len), - eth ? SGE_RX_DROP_THRES : 0); + skb = get_packet_pg(adap, fl, q, + G_RSPD_LEN(len), + eth ? + SGE_RX_DROP_THRES : 0); + q->pg_skb = skb; } else skb = get_packet(adap, fl, G_RSPD_LEN(len), eth ? SGE_RX_DROP_THRES : 0); @@ -2079,7 +2107,11 @@ no_mem: q->credits = 0; } - if (likely(skb != NULL)) { + packet_complete = flags & + (F_RSPD_EOP | F_RSPD_IMM_DATA_VALID | + F_RSPD_ASYNC_NOTIF); + + if (skb != NULL && packet_complete) { if (eth) rx_eth(adap, q, skb, ethpad); else { @@ -2091,6 +2123,9 @@ no_mem: offload_skbs, ngathered); } + + if (flags & F_RSPD_EOP) + clear_rspq_bufstate(q); } --budget_left; } @@ -2706,10 +2741,18 @@ int t3_sge_alloc_qset(struct adapter *adapter, unsigned int id, int nports, #else q->fl[0].buf_size = SGE_RX_SM_BUF_SIZE + sizeof(struct cpl_rx_data); #endif - q->fl[0].use_pages = FL0_PG_CHUNK_SIZE > 0; +#if FL1_PG_CHUNK_SIZE > 0 + q->fl[1].buf_size = FL1_PG_CHUNK_SIZE; +#else q->fl[1].buf_size = is_offload(adapter) ? (16 * 1024) - SKB_DATA_ALIGN(sizeof(struct skb_shared_info)) : MAX_FRAME_SIZE + 2 + sizeof(struct cpl_rx_pkt); +#endif + + q->fl[0].use_pages = FL0_PG_CHUNK_SIZE > 0; + q->fl[1].use_pages = FL1_PG_CHUNK_SIZE > 0; + q->fl[0].order = FL0_PG_ORDER; + q->fl[1].order = FL1_PG_ORDER; spin_lock_irq(&adapter->sge.reg_lock); @@ -2760,7 +2803,8 @@ int t3_sge_alloc_qset(struct adapter *adapter, unsigned int id, int nports, q->adap = adapter; q->netdev = dev; t3_update_qset_coalesce(q, p); - avail = refill_fl(adapter, &q->fl[0], q->fl[0].size, GFP_KERNEL); + avail = refill_fl(adapter, &q->fl[0], q->fl[0].size, + GFP_KERNEL | __GFP_COMP); if (!avail) { CH_ALERT(adapter, "free list queue 0 initialization failed\n"); goto err; @@ -2769,7 +2813,8 @@ int t3_sge_alloc_qset(struct adapter *adapter, unsigned int id, int nports, CH_WARN(adapter, "free list queue 0 enabled with %d credits\n", avail); - avail = refill_fl(adapter, &q->fl[1], q->fl[1].size, GFP_KERNEL); + avail = refill_fl(adapter, &q->fl[1], q->fl[1].size, + GFP_KERNEL | __GFP_COMP); if (avail < q->fl[1].size) CH_WARN(adapter, "free list queue 1 enabled with %d credits\n", avail); @@ -2905,7 +2950,7 @@ void t3_sge_prep(struct adapter *adap, struct sge_params *p) q->coalesce_usecs = 5; q->rspq_size = 1024; q->fl_size = 1024; - q->jumbo_size = 512; + q->jumbo_size = 512; q->txq_size[TXQ_ETH] = 1024; q->txq_size[TXQ_OFLD] = 1024; q->txq_size[TXQ_CTRL] = 256;