2 * Copyright (c) 2005-2008 Chelsio, Inc. All rights reserved.
4 * This software is available to you under a choice of one of two
5 * licenses. You may choose to be licensed under the terms of the GNU
6 * General Public License (GPL) Version 2, available from the file
7 * COPYING in the main directory of this source tree, or the
8 * OpenIB.org BSD license below:
10 * Redistribution and use in source and binary forms, with or
11 * without modification, are permitted provided that the following
14 * - Redistributions of source code must retain the above
15 * copyright notice, this list of conditions and the following
18 * - Redistributions in binary form must reproduce the above
19 * copyright notice, this list of conditions and the following
20 * disclaimer in the documentation and/or other materials
21 * provided with the distribution.
23 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
24 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
25 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
26 * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
27 * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
28 * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
29 * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
32 #include <linux/skbuff.h>
33 #include <linux/netdevice.h>
34 #include <linux/etherdevice.h>
35 #include <linux/if_vlan.h>
37 #include <linux/tcp.h>
38 #include <linux/dma-mapping.h>
44 #include "firmware_exports.h"
48 #define SGE_RX_SM_BUF_SIZE 1536
50 #define SGE_RX_COPY_THRES 256
51 #define SGE_RX_PULL_LEN 128
53 #define SGE_PG_RSVD SMP_CACHE_BYTES
55 * Page chunk size for FL0 buffers if FL0 is to be populated with page chunks.
56 * It must be a divisor of PAGE_SIZE. If set to 0 FL0 will use sk_buffs
59 #define FL0_PG_CHUNK_SIZE 2048
60 #define FL0_PG_ORDER 0
61 #define FL0_PG_ALLOC_SIZE (PAGE_SIZE << FL0_PG_ORDER)
62 #define FL1_PG_CHUNK_SIZE (PAGE_SIZE > 8192 ? 16384 : 8192)
63 #define FL1_PG_ORDER (PAGE_SIZE > 8192 ? 0 : 1)
64 #define FL1_PG_ALLOC_SIZE (PAGE_SIZE << FL1_PG_ORDER)
66 #define SGE_RX_DROP_THRES 16
67 #define RX_RECLAIM_PERIOD (HZ/4)
70 * Max number of Rx buffers we replenish at a time.
72 #define MAX_RX_REFILL 16U
74 * Period of the Tx buffer reclaim timer. This timer does not need to run
75 * frequently as Tx buffers are usually reclaimed by new Tx packets.
77 #define TX_RECLAIM_PERIOD (HZ / 4)
78 #define TX_RECLAIM_TIMER_CHUNK 64U
79 #define TX_RECLAIM_CHUNK 16U
81 /* WR size in bytes */
82 #define WR_LEN (WR_FLITS * 8)
85 * Types of Tx queues in each queue set. Order here matters, do not change.
87 enum { TXQ_ETH, TXQ_OFLD, TXQ_CTRL };
89 /* Values for sge_txq.flags */
91 TXQ_RUNNING = 1 << 0, /* fetch engine is running */
92 TXQ_LAST_PKT_DB = 1 << 1, /* last packet rang the doorbell */
96 __be64 flit[TX_DESC_FLITS];
106 struct tx_sw_desc { /* SW state per Tx descriptor */
108 u8 eop; /* set if last descriptor for packet */
109 u8 addr_idx; /* buffer index of first SGL entry in descriptor */
110 u8 fragidx; /* first page fragment associated with descriptor */
111 s8 sflit; /* start flit of first SGL entry in descriptor */
114 struct rx_sw_desc { /* SW state per Rx descriptor */
117 struct fl_pg_chunk pg_chunk;
119 DECLARE_PCI_UNMAP_ADDR(dma_addr);
122 struct rsp_desc { /* response queue descriptor */
123 struct rss_header rss_hdr;
131 * Holds unmapping information for Tx packets that need deferred unmapping.
132 * This structure lives at skb->head and must be allocated by callers.
134 struct deferred_unmap_info {
135 struct pci_dev *pdev;
136 dma_addr_t addr[MAX_SKB_FRAGS + 1];
140 * Maps a number of flits to the number of Tx descriptors that can hold them.
143 * desc = 1 + (flits - 2) / (WR_FLITS - 1).
145 * HW allows up to 4 descriptors to be combined into a WR.
147 static u8 flit_desc_map[] = {
149 #if SGE_NUM_GENBITS == 1
150 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,
151 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2,
152 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3,
153 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4
154 #elif SGE_NUM_GENBITS == 2
155 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,
156 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2,
157 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3,
158 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4,
160 # error "SGE_NUM_GENBITS must be 1 or 2"
164 static inline struct sge_qset *fl_to_qset(const struct sge_fl *q, int qidx)
166 return container_of(q, struct sge_qset, fl[qidx]);
169 static inline struct sge_qset *rspq_to_qset(const struct sge_rspq *q)
171 return container_of(q, struct sge_qset, rspq);
174 static inline struct sge_qset *txq_to_qset(const struct sge_txq *q, int qidx)
176 return container_of(q, struct sge_qset, txq[qidx]);
180 * refill_rspq - replenish an SGE response queue
181 * @adapter: the adapter
182 * @q: the response queue to replenish
183 * @credits: how many new responses to make available
185 * Replenishes a response queue by making the supplied number of responses
188 static inline void refill_rspq(struct adapter *adapter,
189 const struct sge_rspq *q, unsigned int credits)
192 t3_write_reg(adapter, A_SG_RSPQ_CREDIT_RETURN,
193 V_RSPQ(q->cntxt_id) | V_CREDITS(credits));
197 * need_skb_unmap - does the platform need unmapping of sk_buffs?
199 * Returns true if the platfrom needs sk_buff unmapping. The compiler
200 * optimizes away unecessary code if this returns true.
202 static inline int need_skb_unmap(void)
205 * This structure is used to tell if the platfrom needs buffer
206 * unmapping by checking if DECLARE_PCI_UNMAP_ADDR defines anything.
209 DECLARE_PCI_UNMAP_ADDR(addr);
212 return sizeof(struct dummy) != 0;
216 * unmap_skb - unmap a packet main body and its page fragments
218 * @q: the Tx queue containing Tx descriptors for the packet
219 * @cidx: index of Tx descriptor
220 * @pdev: the PCI device
222 * Unmap the main body of an sk_buff and its page fragments, if any.
223 * Because of the fairly complicated structure of our SGLs and the desire
224 * to conserve space for metadata, the information necessary to unmap an
225 * sk_buff is spread across the sk_buff itself (buffer lengths), the HW Tx
226 * descriptors (the physical addresses of the various data buffers), and
227 * the SW descriptor state (assorted indices). The send functions
228 * initialize the indices for the first packet descriptor so we can unmap
229 * the buffers held in the first Tx descriptor here, and we have enough
230 * information at this point to set the state for the next Tx descriptor.
232 * Note that it is possible to clean up the first descriptor of a packet
233 * before the send routines have written the next descriptors, but this
234 * race does not cause any problem. We just end up writing the unmapping
235 * info for the descriptor first.
237 static inline void unmap_skb(struct sk_buff *skb, struct sge_txq *q,
238 unsigned int cidx, struct pci_dev *pdev)
240 const struct sg_ent *sgp;
241 struct tx_sw_desc *d = &q->sdesc[cidx];
242 int nfrags, frag_idx, curflit, j = d->addr_idx;
244 sgp = (struct sg_ent *)&q->desc[cidx].flit[d->sflit];
245 frag_idx = d->fragidx;
247 if (frag_idx == 0 && skb_headlen(skb)) {
248 pci_unmap_single(pdev, be64_to_cpu(sgp->addr[0]),
249 skb_headlen(skb), PCI_DMA_TODEVICE);
253 curflit = d->sflit + 1 + j;
254 nfrags = skb_shinfo(skb)->nr_frags;
256 while (frag_idx < nfrags && curflit < WR_FLITS) {
257 pci_unmap_page(pdev, be64_to_cpu(sgp->addr[j]),
258 skb_shinfo(skb)->frags[frag_idx].size,
269 if (frag_idx < nfrags) { /* SGL continues into next Tx descriptor */
270 d = cidx + 1 == q->size ? q->sdesc : d + 1;
271 d->fragidx = frag_idx;
273 d->sflit = curflit - WR_FLITS - j; /* sflit can be -1 */
278 * free_tx_desc - reclaims Tx descriptors and their buffers
279 * @adapter: the adapter
280 * @q: the Tx queue to reclaim descriptors from
281 * @n: the number of descriptors to reclaim
283 * Reclaims Tx descriptors from an SGE Tx queue and frees the associated
284 * Tx buffers. Called with the Tx queue lock held.
286 static void free_tx_desc(struct adapter *adapter, struct sge_txq *q,
289 struct tx_sw_desc *d;
290 struct pci_dev *pdev = adapter->pdev;
291 unsigned int cidx = q->cidx;
293 const int need_unmap = need_skb_unmap() &&
294 q->cntxt_id >= FW_TUNNEL_SGEEC_START;
298 if (d->skb) { /* an SGL is present */
300 unmap_skb(d->skb, q, cidx, pdev);
305 if (++cidx == q->size) {
314 * reclaim_completed_tx - reclaims completed Tx descriptors
315 * @adapter: the adapter
316 * @q: the Tx queue to reclaim completed descriptors from
317 * @chunk: maximum number of descriptors to reclaim
319 * Reclaims Tx descriptors that the SGE has indicated it has processed,
320 * and frees the associated buffers if possible. Called with the Tx
323 static inline unsigned int reclaim_completed_tx(struct adapter *adapter,
327 unsigned int reclaim = q->processed - q->cleaned;
329 reclaim = min(chunk, reclaim);
331 free_tx_desc(adapter, q, reclaim);
332 q->cleaned += reclaim;
333 q->in_use -= reclaim;
335 return q->processed - q->cleaned;
339 * should_restart_tx - are there enough resources to restart a Tx queue?
342 * Checks if there are enough descriptors to restart a suspended Tx queue.
344 static inline int should_restart_tx(const struct sge_txq *q)
346 unsigned int r = q->processed - q->cleaned;
348 return q->in_use - r < (q->size >> 1);
351 static void clear_rx_desc(struct pci_dev *pdev, const struct sge_fl *q,
352 struct rx_sw_desc *d)
354 if (q->use_pages && d->pg_chunk.page) {
355 (*d->pg_chunk.p_cnt)--;
356 if (!*d->pg_chunk.p_cnt)
359 q->alloc_size, PCI_DMA_FROMDEVICE);
361 put_page(d->pg_chunk.page);
362 d->pg_chunk.page = NULL;
364 pci_unmap_single(pdev, pci_unmap_addr(d, dma_addr),
365 q->buf_size, PCI_DMA_FROMDEVICE);
372 * free_rx_bufs - free the Rx buffers on an SGE free list
373 * @pdev: the PCI device associated with the adapter
374 * @rxq: the SGE free list to clean up
376 * Release the buffers on an SGE free-buffer Rx queue. HW fetching from
377 * this queue should be stopped before calling this function.
379 static void free_rx_bufs(struct pci_dev *pdev, struct sge_fl *q)
381 unsigned int cidx = q->cidx;
383 while (q->credits--) {
384 struct rx_sw_desc *d = &q->sdesc[cidx];
387 clear_rx_desc(pdev, q, d);
388 if (++cidx == q->size)
392 if (q->pg_chunk.page) {
393 __free_pages(q->pg_chunk.page, q->order);
394 q->pg_chunk.page = NULL;
399 * add_one_rx_buf - add a packet buffer to a free-buffer list
400 * @va: buffer start VA
401 * @len: the buffer length
402 * @d: the HW Rx descriptor to write
403 * @sd: the SW Rx descriptor to write
404 * @gen: the generation bit value
405 * @pdev: the PCI device associated with the adapter
407 * Add a buffer of the given length to the supplied HW and SW Rx
410 static inline int add_one_rx_buf(void *va, unsigned int len,
411 struct rx_desc *d, struct rx_sw_desc *sd,
412 unsigned int gen, struct pci_dev *pdev)
416 mapping = pci_map_single(pdev, va, len, PCI_DMA_FROMDEVICE);
417 if (unlikely(pci_dma_mapping_error(pdev, mapping)))
420 pci_unmap_addr_set(sd, dma_addr, mapping);
422 d->addr_lo = cpu_to_be32(mapping);
423 d->addr_hi = cpu_to_be32((u64) mapping >> 32);
425 d->len_gen = cpu_to_be32(V_FLD_GEN1(gen));
426 d->gen2 = cpu_to_be32(V_FLD_GEN2(gen));
430 static inline int add_one_rx_chunk(dma_addr_t mapping, struct rx_desc *d,
433 d->addr_lo = cpu_to_be32(mapping);
434 d->addr_hi = cpu_to_be32((u64) mapping >> 32);
436 d->len_gen = cpu_to_be32(V_FLD_GEN1(gen));
437 d->gen2 = cpu_to_be32(V_FLD_GEN2(gen));
441 static int alloc_pg_chunk(struct adapter *adapter, struct sge_fl *q,
442 struct rx_sw_desc *sd, gfp_t gfp,
445 if (!q->pg_chunk.page) {
448 q->pg_chunk.page = alloc_pages(gfp, order);
449 if (unlikely(!q->pg_chunk.page))
451 q->pg_chunk.va = page_address(q->pg_chunk.page);
452 q->pg_chunk.p_cnt = q->pg_chunk.va + (PAGE_SIZE << order) -
454 q->pg_chunk.offset = 0;
455 mapping = pci_map_page(adapter->pdev, q->pg_chunk.page,
456 0, q->alloc_size, PCI_DMA_FROMDEVICE);
457 q->pg_chunk.mapping = mapping;
459 sd->pg_chunk = q->pg_chunk;
461 prefetch(sd->pg_chunk.p_cnt);
463 q->pg_chunk.offset += q->buf_size;
464 if (q->pg_chunk.offset == (PAGE_SIZE << order))
465 q->pg_chunk.page = NULL;
467 q->pg_chunk.va += q->buf_size;
468 get_page(q->pg_chunk.page);
471 if (sd->pg_chunk.offset == 0)
472 *sd->pg_chunk.p_cnt = 1;
474 *sd->pg_chunk.p_cnt += 1;
479 static inline void ring_fl_db(struct adapter *adap, struct sge_fl *q)
481 if (q->pend_cred >= q->credits / 4) {
483 t3_write_reg(adap, A_SG_KDOORBELL, V_EGRCNTX(q->cntxt_id));
488 * refill_fl - refill an SGE free-buffer list
489 * @adapter: the adapter
490 * @q: the free-list to refill
491 * @n: the number of new buffers to allocate
492 * @gfp: the gfp flags for allocating new buffers
494 * (Re)populate an SGE free-buffer list with up to @n new packet buffers,
495 * allocated with the supplied gfp flags. The caller must assure that
496 * @n does not exceed the queue's capacity.
498 static int refill_fl(struct adapter *adap, struct sge_fl *q, int n, gfp_t gfp)
500 struct rx_sw_desc *sd = &q->sdesc[q->pidx];
501 struct rx_desc *d = &q->desc[q->pidx];
502 unsigned int count = 0;
509 if (unlikely(alloc_pg_chunk(adap, q, sd, gfp,
511 nomem: q->alloc_failed++;
514 mapping = sd->pg_chunk.mapping + sd->pg_chunk.offset;
515 pci_unmap_addr_set(sd, dma_addr, mapping);
517 add_one_rx_chunk(mapping, d, q->gen);
518 pci_dma_sync_single_for_device(adap->pdev, mapping,
519 q->buf_size - SGE_PG_RSVD,
524 struct sk_buff *skb = alloc_skb(q->buf_size, gfp);
529 buf_start = skb->data;
530 err = add_one_rx_buf(buf_start, q->buf_size, d, sd,
533 clear_rx_desc(adap->pdev, q, sd);
540 if (++q->pidx == q->size) {
550 q->pend_cred += count;
556 static inline void __refill_fl(struct adapter *adap, struct sge_fl *fl)
558 refill_fl(adap, fl, min(MAX_RX_REFILL, fl->size - fl->credits),
559 GFP_ATOMIC | __GFP_COMP);
563 * recycle_rx_buf - recycle a receive buffer
564 * @adapter: the adapter
565 * @q: the SGE free list
566 * @idx: index of buffer to recycle
568 * Recycles the specified buffer on the given free list by adding it at
569 * the next available slot on the list.
571 static void recycle_rx_buf(struct adapter *adap, struct sge_fl *q,
574 struct rx_desc *from = &q->desc[idx];
575 struct rx_desc *to = &q->desc[q->pidx];
577 q->sdesc[q->pidx] = q->sdesc[idx];
578 to->addr_lo = from->addr_lo; /* already big endian */
579 to->addr_hi = from->addr_hi; /* likewise */
581 to->len_gen = cpu_to_be32(V_FLD_GEN1(q->gen));
582 to->gen2 = cpu_to_be32(V_FLD_GEN2(q->gen));
584 if (++q->pidx == q->size) {
595 * alloc_ring - allocate resources for an SGE descriptor ring
596 * @pdev: the PCI device
597 * @nelem: the number of descriptors
598 * @elem_size: the size of each descriptor
599 * @sw_size: the size of the SW state associated with each ring element
600 * @phys: the physical address of the allocated ring
601 * @metadata: address of the array holding the SW state for the ring
603 * Allocates resources for an SGE descriptor ring, such as Tx queues,
604 * free buffer lists, or response queues. Each SGE ring requires
605 * space for its HW descriptors plus, optionally, space for the SW state
606 * associated with each HW entry (the metadata). The function returns
607 * three values: the virtual address for the HW ring (the return value
608 * of the function), the physical address of the HW ring, and the address
611 static void *alloc_ring(struct pci_dev *pdev, size_t nelem, size_t elem_size,
612 size_t sw_size, dma_addr_t * phys, void *metadata)
614 size_t len = nelem * elem_size;
616 void *p = dma_alloc_coherent(&pdev->dev, len, phys, GFP_KERNEL);
620 if (sw_size && metadata) {
621 s = kcalloc(nelem, sw_size, GFP_KERNEL);
624 dma_free_coherent(&pdev->dev, len, p, *phys);
627 *(void **)metadata = s;
634 * t3_reset_qset - reset a sge qset
637 * Reset the qset structure.
638 * the NAPI structure is preserved in the event of
639 * the qset's reincarnation, for example during EEH recovery.
641 static void t3_reset_qset(struct sge_qset *q)
644 !(q->adap->flags & NAPI_INIT)) {
645 memset(q, 0, sizeof(*q));
650 memset(&q->rspq, 0, sizeof(q->rspq));
651 memset(q->fl, 0, sizeof(struct sge_fl) * SGE_RXQ_PER_SET);
652 memset(q->txq, 0, sizeof(struct sge_txq) * SGE_TXQ_PER_SET);
654 q->tx_reclaim_timer.function = NULL; /* for t3_stop_sge_timers() */
655 q->rx_reclaim_timer.function = NULL;
657 napi_free_frags(&q->napi);
662 * free_qset - free the resources of an SGE queue set
663 * @adapter: the adapter owning the queue set
666 * Release the HW and SW resources associated with an SGE queue set, such
667 * as HW contexts, packet buffers, and descriptor rings. Traffic to the
668 * queue set must be quiesced prior to calling this.
670 static void t3_free_qset(struct adapter *adapter, struct sge_qset *q)
673 struct pci_dev *pdev = adapter->pdev;
675 for (i = 0; i < SGE_RXQ_PER_SET; ++i)
677 spin_lock_irq(&adapter->sge.reg_lock);
678 t3_sge_disable_fl(adapter, q->fl[i].cntxt_id);
679 spin_unlock_irq(&adapter->sge.reg_lock);
680 free_rx_bufs(pdev, &q->fl[i]);
681 kfree(q->fl[i].sdesc);
682 dma_free_coherent(&pdev->dev,
684 sizeof(struct rx_desc), q->fl[i].desc,
688 for (i = 0; i < SGE_TXQ_PER_SET; ++i)
689 if (q->txq[i].desc) {
690 spin_lock_irq(&adapter->sge.reg_lock);
691 t3_sge_enable_ecntxt(adapter, q->txq[i].cntxt_id, 0);
692 spin_unlock_irq(&adapter->sge.reg_lock);
693 if (q->txq[i].sdesc) {
694 free_tx_desc(adapter, &q->txq[i],
696 kfree(q->txq[i].sdesc);
698 dma_free_coherent(&pdev->dev,
700 sizeof(struct tx_desc),
701 q->txq[i].desc, q->txq[i].phys_addr);
702 __skb_queue_purge(&q->txq[i].sendq);
706 spin_lock_irq(&adapter->sge.reg_lock);
707 t3_sge_disable_rspcntxt(adapter, q->rspq.cntxt_id);
708 spin_unlock_irq(&adapter->sge.reg_lock);
709 dma_free_coherent(&pdev->dev,
710 q->rspq.size * sizeof(struct rsp_desc),
711 q->rspq.desc, q->rspq.phys_addr);
718 * init_qset_cntxt - initialize an SGE queue set context info
720 * @id: the queue set id
722 * Initializes the TIDs and context ids for the queues of a queue set.
724 static void init_qset_cntxt(struct sge_qset *qs, unsigned int id)
726 qs->rspq.cntxt_id = id;
727 qs->fl[0].cntxt_id = 2 * id;
728 qs->fl[1].cntxt_id = 2 * id + 1;
729 qs->txq[TXQ_ETH].cntxt_id = FW_TUNNEL_SGEEC_START + id;
730 qs->txq[TXQ_ETH].token = FW_TUNNEL_TID_START + id;
731 qs->txq[TXQ_OFLD].cntxt_id = FW_OFLD_SGEEC_START + id;
732 qs->txq[TXQ_CTRL].cntxt_id = FW_CTRL_SGEEC_START + id;
733 qs->txq[TXQ_CTRL].token = FW_CTRL_TID_START + id;
737 * sgl_len - calculates the size of an SGL of the given capacity
738 * @n: the number of SGL entries
740 * Calculates the number of flits needed for a scatter/gather list that
741 * can hold the given number of entries.
743 static inline unsigned int sgl_len(unsigned int n)
745 /* alternatively: 3 * (n / 2) + 2 * (n & 1) */
746 return (3 * n) / 2 + (n & 1);
750 * flits_to_desc - returns the num of Tx descriptors for the given flits
751 * @n: the number of flits
753 * Calculates the number of Tx descriptors needed for the supplied number
756 static inline unsigned int flits_to_desc(unsigned int n)
758 BUG_ON(n >= ARRAY_SIZE(flit_desc_map));
759 return flit_desc_map[n];
763 * get_packet - return the next ingress packet buffer from a free list
764 * @adap: the adapter that received the packet
765 * @fl: the SGE free list holding the packet
766 * @len: the packet length including any SGE padding
767 * @drop_thres: # of remaining buffers before we start dropping packets
769 * Get the next packet from a free list and complete setup of the
770 * sk_buff. If the packet is small we make a copy and recycle the
771 * original buffer, otherwise we use the original buffer itself. If a
772 * positive drop threshold is supplied packets are dropped and their
773 * buffers recycled if (a) the number of remaining buffers is under the
774 * threshold and the packet is too big to copy, or (b) the packet should
775 * be copied but there is no memory for the copy.
777 static struct sk_buff *get_packet(struct adapter *adap, struct sge_fl *fl,
778 unsigned int len, unsigned int drop_thres)
780 struct sk_buff *skb = NULL;
781 struct rx_sw_desc *sd = &fl->sdesc[fl->cidx];
783 prefetch(sd->skb->data);
786 if (len <= SGE_RX_COPY_THRES) {
787 skb = alloc_skb(len, GFP_ATOMIC);
788 if (likely(skb != NULL)) {
790 pci_dma_sync_single_for_cpu(adap->pdev,
791 pci_unmap_addr(sd, dma_addr), len,
793 memcpy(skb->data, sd->skb->data, len);
794 pci_dma_sync_single_for_device(adap->pdev,
795 pci_unmap_addr(sd, dma_addr), len,
797 } else if (!drop_thres)
800 recycle_rx_buf(adap, fl, fl->cidx);
804 if (unlikely(fl->credits < drop_thres) &&
805 refill_fl(adap, fl, min(MAX_RX_REFILL, fl->size - fl->credits - 1),
806 GFP_ATOMIC | __GFP_COMP) == 0)
810 pci_unmap_single(adap->pdev, pci_unmap_addr(sd, dma_addr),
811 fl->buf_size, PCI_DMA_FROMDEVICE);
814 __refill_fl(adap, fl);
819 * get_packet_pg - return the next ingress packet buffer from a free list
820 * @adap: the adapter that received the packet
821 * @fl: the SGE free list holding the packet
822 * @len: the packet length including any SGE padding
823 * @drop_thres: # of remaining buffers before we start dropping packets
825 * Get the next packet from a free list populated with page chunks.
826 * If the packet is small we make a copy and recycle the original buffer,
827 * otherwise we attach the original buffer as a page fragment to a fresh
828 * sk_buff. If a positive drop threshold is supplied packets are dropped
829 * and their buffers recycled if (a) the number of remaining buffers is
830 * under the threshold and the packet is too big to copy, or (b) there's
833 * Note: this function is similar to @get_packet but deals with Rx buffers
834 * that are page chunks rather than sk_buffs.
836 static struct sk_buff *get_packet_pg(struct adapter *adap, struct sge_fl *fl,
837 struct sge_rspq *q, unsigned int len,
838 unsigned int drop_thres)
840 struct sk_buff *newskb, *skb;
841 struct rx_sw_desc *sd = &fl->sdesc[fl->cidx];
843 dma_addr_t dma_addr = pci_unmap_addr(sd, dma_addr);
845 newskb = skb = q->pg_skb;
846 if (!skb && (len <= SGE_RX_COPY_THRES)) {
847 newskb = alloc_skb(len, GFP_ATOMIC);
848 if (likely(newskb != NULL)) {
849 __skb_put(newskb, len);
850 pci_dma_sync_single_for_cpu(adap->pdev, dma_addr, len,
852 memcpy(newskb->data, sd->pg_chunk.va, len);
853 pci_dma_sync_single_for_device(adap->pdev, dma_addr,
856 } else if (!drop_thres)
860 recycle_rx_buf(adap, fl, fl->cidx);
865 if (unlikely(q->rx_recycle_buf || (!skb && fl->credits <= drop_thres)))
868 prefetch(sd->pg_chunk.p_cnt);
871 newskb = alloc_skb(SGE_RX_PULL_LEN, GFP_ATOMIC);
873 if (unlikely(!newskb)) {
879 pci_dma_sync_single_for_cpu(adap->pdev, dma_addr, len,
881 (*sd->pg_chunk.p_cnt)--;
882 if (!*sd->pg_chunk.p_cnt)
883 pci_unmap_page(adap->pdev,
884 sd->pg_chunk.mapping,
888 __skb_put(newskb, SGE_RX_PULL_LEN);
889 memcpy(newskb->data, sd->pg_chunk.va, SGE_RX_PULL_LEN);
890 skb_fill_page_desc(newskb, 0, sd->pg_chunk.page,
891 sd->pg_chunk.offset + SGE_RX_PULL_LEN,
892 len - SGE_RX_PULL_LEN);
894 newskb->data_len = len - SGE_RX_PULL_LEN;
895 newskb->truesize += newskb->data_len;
897 skb_fill_page_desc(newskb, skb_shinfo(newskb)->nr_frags,
899 sd->pg_chunk.offset, len);
901 newskb->data_len += len;
902 newskb->truesize += len;
907 * We do not refill FLs here, we let the caller do it to overlap a
914 * get_imm_packet - return the next ingress packet buffer from a response
915 * @resp: the response descriptor containing the packet data
917 * Return a packet containing the immediate data of the given response.
919 static inline struct sk_buff *get_imm_packet(const struct rsp_desc *resp)
921 struct sk_buff *skb = alloc_skb(IMMED_PKT_SIZE, GFP_ATOMIC);
924 __skb_put(skb, IMMED_PKT_SIZE);
925 skb_copy_to_linear_data(skb, resp->imm_data, IMMED_PKT_SIZE);
931 * calc_tx_descs - calculate the number of Tx descriptors for a packet
934 * Returns the number of Tx descriptors needed for the given Ethernet
935 * packet. Ethernet packets require addition of WR and CPL headers.
937 static inline unsigned int calc_tx_descs(const struct sk_buff *skb)
941 if (skb->len <= WR_LEN - sizeof(struct cpl_tx_pkt))
944 flits = sgl_len(skb_shinfo(skb)->nr_frags + 1) + 2;
945 if (skb_shinfo(skb)->gso_size)
947 return flits_to_desc(flits);
951 * make_sgl - populate a scatter/gather list for a packet
953 * @sgp: the SGL to populate
954 * @start: start address of skb main body data to include in the SGL
955 * @len: length of skb main body data to include in the SGL
956 * @pdev: the PCI device
958 * Generates a scatter/gather list for the buffers that make up a packet
959 * and returns the SGL size in 8-byte words. The caller must size the SGL
962 static inline unsigned int make_sgl(const struct sk_buff *skb,
963 struct sg_ent *sgp, unsigned char *start,
964 unsigned int len, struct pci_dev *pdev)
967 unsigned int i, j = 0, nfrags;
970 mapping = pci_map_single(pdev, start, len, PCI_DMA_TODEVICE);
971 sgp->len[0] = cpu_to_be32(len);
972 sgp->addr[0] = cpu_to_be64(mapping);
976 nfrags = skb_shinfo(skb)->nr_frags;
977 for (i = 0; i < nfrags; i++) {
978 skb_frag_t *frag = &skb_shinfo(skb)->frags[i];
980 mapping = pci_map_page(pdev, frag->page, frag->page_offset,
981 frag->size, PCI_DMA_TODEVICE);
982 sgp->len[j] = cpu_to_be32(frag->size);
983 sgp->addr[j] = cpu_to_be64(mapping);
990 return ((nfrags + (len != 0)) * 3) / 2 + j;
994 * check_ring_tx_db - check and potentially ring a Tx queue's doorbell
998 * Ring the doorbel if a Tx queue is asleep. There is a natural race,
999 * where the HW is going to sleep just after we checked, however,
1000 * then the interrupt handler will detect the outstanding TX packet
1001 * and ring the doorbell for us.
1003 * When GTS is disabled we unconditionally ring the doorbell.
1005 static inline void check_ring_tx_db(struct adapter *adap, struct sge_txq *q)
1008 clear_bit(TXQ_LAST_PKT_DB, &q->flags);
1009 if (test_and_set_bit(TXQ_RUNNING, &q->flags) == 0) {
1010 set_bit(TXQ_LAST_PKT_DB, &q->flags);
1011 t3_write_reg(adap, A_SG_KDOORBELL,
1012 F_SELEGRCNTX | V_EGRCNTX(q->cntxt_id));
1015 wmb(); /* write descriptors before telling HW */
1016 t3_write_reg(adap, A_SG_KDOORBELL,
1017 F_SELEGRCNTX | V_EGRCNTX(q->cntxt_id));
1021 static inline void wr_gen2(struct tx_desc *d, unsigned int gen)
1023 #if SGE_NUM_GENBITS == 2
1024 d->flit[TX_DESC_FLITS - 1] = cpu_to_be64(gen);
1029 * write_wr_hdr_sgl - write a WR header and, optionally, SGL
1030 * @ndesc: number of Tx descriptors spanned by the SGL
1031 * @skb: the packet corresponding to the WR
1032 * @d: first Tx descriptor to be written
1033 * @pidx: index of above descriptors
1034 * @q: the SGE Tx queue
1036 * @flits: number of flits to the start of the SGL in the first descriptor
1037 * @sgl_flits: the SGL size in flits
1038 * @gen: the Tx descriptor generation
1039 * @wr_hi: top 32 bits of WR header based on WR type (big endian)
1040 * @wr_lo: low 32 bits of WR header based on WR type (big endian)
1042 * Write a work request header and an associated SGL. If the SGL is
1043 * small enough to fit into one Tx descriptor it has already been written
1044 * and we just need to write the WR header. Otherwise we distribute the
1045 * SGL across the number of descriptors it spans.
1047 static void write_wr_hdr_sgl(unsigned int ndesc, struct sk_buff *skb,
1048 struct tx_desc *d, unsigned int pidx,
1049 const struct sge_txq *q,
1050 const struct sg_ent *sgl,
1051 unsigned int flits, unsigned int sgl_flits,
1052 unsigned int gen, __be32 wr_hi,
1055 struct work_request_hdr *wrp = (struct work_request_hdr *)d;
1056 struct tx_sw_desc *sd = &q->sdesc[pidx];
1059 if (need_skb_unmap()) {
1065 if (likely(ndesc == 1)) {
1067 wrp->wr_hi = htonl(F_WR_SOP | F_WR_EOP | V_WR_DATATYPE(1) |
1068 V_WR_SGLSFLT(flits)) | wr_hi;
1070 wrp->wr_lo = htonl(V_WR_LEN(flits + sgl_flits) |
1071 V_WR_GEN(gen)) | wr_lo;
1074 unsigned int ogen = gen;
1075 const u64 *fp = (const u64 *)sgl;
1076 struct work_request_hdr *wp = wrp;
1078 wrp->wr_hi = htonl(F_WR_SOP | V_WR_DATATYPE(1) |
1079 V_WR_SGLSFLT(flits)) | wr_hi;
1082 unsigned int avail = WR_FLITS - flits;
1084 if (avail > sgl_flits)
1086 memcpy(&d->flit[flits], fp, avail * sizeof(*fp));
1096 if (++pidx == q->size) {
1104 wrp = (struct work_request_hdr *)d;
1105 wrp->wr_hi = htonl(V_WR_DATATYPE(1) |
1106 V_WR_SGLSFLT(1)) | wr_hi;
1107 wrp->wr_lo = htonl(V_WR_LEN(min(WR_FLITS,
1109 V_WR_GEN(gen)) | wr_lo;
1114 wrp->wr_hi |= htonl(F_WR_EOP);
1116 wp->wr_lo = htonl(V_WR_LEN(WR_FLITS) | V_WR_GEN(ogen)) | wr_lo;
1117 wr_gen2((struct tx_desc *)wp, ogen);
1118 WARN_ON(ndesc != 0);
1123 * write_tx_pkt_wr - write a TX_PKT work request
1124 * @adap: the adapter
1125 * @skb: the packet to send
1126 * @pi: the egress interface
1127 * @pidx: index of the first Tx descriptor to write
1128 * @gen: the generation value to use
1130 * @ndesc: number of descriptors the packet will occupy
1131 * @compl: the value of the COMPL bit to use
1133 * Generate a TX_PKT work request to send the supplied packet.
1135 static void write_tx_pkt_wr(struct adapter *adap, struct sk_buff *skb,
1136 const struct port_info *pi,
1137 unsigned int pidx, unsigned int gen,
1138 struct sge_txq *q, unsigned int ndesc,
1141 unsigned int flits, sgl_flits, cntrl, tso_info;
1142 struct sg_ent *sgp, sgl[MAX_SKB_FRAGS / 2 + 1];
1143 struct tx_desc *d = &q->desc[pidx];
1144 struct cpl_tx_pkt *cpl = (struct cpl_tx_pkt *)d;
1146 cpl->len = htonl(skb->len);
1147 cntrl = V_TXPKT_INTF(pi->port_id);
1149 if (vlan_tx_tag_present(skb) && pi->vlan_grp)
1150 cntrl |= F_TXPKT_VLAN_VLD | V_TXPKT_VLAN(vlan_tx_tag_get(skb));
1152 tso_info = V_LSO_MSS(skb_shinfo(skb)->gso_size);
1155 struct cpl_tx_pkt_lso *hdr = (struct cpl_tx_pkt_lso *)cpl;
1158 cntrl |= V_TXPKT_OPCODE(CPL_TX_PKT_LSO);
1159 hdr->cntrl = htonl(cntrl);
1160 eth_type = skb_network_offset(skb) == ETH_HLEN ?
1161 CPL_ETH_II : CPL_ETH_II_VLAN;
1162 tso_info |= V_LSO_ETH_TYPE(eth_type) |
1163 V_LSO_IPHDR_WORDS(ip_hdr(skb)->ihl) |
1164 V_LSO_TCPHDR_WORDS(tcp_hdr(skb)->doff);
1165 hdr->lso_info = htonl(tso_info);
1168 cntrl |= V_TXPKT_OPCODE(CPL_TX_PKT);
1169 cntrl |= F_TXPKT_IPCSUM_DIS; /* SW calculates IP csum */
1170 cntrl |= V_TXPKT_L4CSUM_DIS(skb->ip_summed != CHECKSUM_PARTIAL);
1171 cpl->cntrl = htonl(cntrl);
1173 if (skb->len <= WR_LEN - sizeof(*cpl)) {
1174 q->sdesc[pidx].skb = NULL;
1176 skb_copy_from_linear_data(skb, &d->flit[2],
1179 skb_copy_bits(skb, 0, &d->flit[2], skb->len);
1181 flits = (skb->len + 7) / 8 + 2;
1182 cpl->wr.wr_hi = htonl(V_WR_BCNTLFLT(skb->len & 7) |
1183 V_WR_OP(FW_WROPCODE_TUNNEL_TX_PKT)
1184 | F_WR_SOP | F_WR_EOP | compl);
1186 cpl->wr.wr_lo = htonl(V_WR_LEN(flits) | V_WR_GEN(gen) |
1187 V_WR_TID(q->token));
1196 sgp = ndesc == 1 ? (struct sg_ent *)&d->flit[flits] : sgl;
1197 sgl_flits = make_sgl(skb, sgp, skb->data, skb_headlen(skb), adap->pdev);
1199 write_wr_hdr_sgl(ndesc, skb, d, pidx, q, sgl, flits, sgl_flits, gen,
1200 htonl(V_WR_OP(FW_WROPCODE_TUNNEL_TX_PKT) | compl),
1201 htonl(V_WR_TID(q->token)));
1204 static inline void t3_stop_tx_queue(struct netdev_queue *txq,
1205 struct sge_qset *qs, struct sge_txq *q)
1207 netif_tx_stop_queue(txq);
1208 set_bit(TXQ_ETH, &qs->txq_stopped);
1213 * eth_xmit - add a packet to the Ethernet Tx queue
1215 * @dev: the egress net device
1217 * Add a packet to an SGE Tx queue. Runs with softirqs disabled.
1219 int t3_eth_xmit(struct sk_buff *skb, struct net_device *dev)
1222 unsigned int ndesc, pidx, credits, gen, compl;
1223 const struct port_info *pi = netdev_priv(dev);
1224 struct adapter *adap = pi->adapter;
1225 struct netdev_queue *txq;
1226 struct sge_qset *qs;
1230 * The chip min packet length is 9 octets but play safe and reject
1231 * anything shorter than an Ethernet header.
1233 if (unlikely(skb->len < ETH_HLEN)) {
1235 return NETDEV_TX_OK;
1238 qidx = skb_get_queue_mapping(skb);
1240 q = &qs->txq[TXQ_ETH];
1241 txq = netdev_get_tx_queue(dev, qidx);
1243 reclaim_completed_tx(adap, q, TX_RECLAIM_CHUNK);
1245 credits = q->size - q->in_use;
1246 ndesc = calc_tx_descs(skb);
1248 if (unlikely(credits < ndesc)) {
1249 t3_stop_tx_queue(txq, qs, q);
1250 dev_err(&adap->pdev->dev,
1251 "%s: Tx ring %u full while queue awake!\n",
1252 dev->name, q->cntxt_id & 7);
1253 return NETDEV_TX_BUSY;
1257 if (unlikely(credits - ndesc < q->stop_thres)) {
1258 t3_stop_tx_queue(txq, qs, q);
1260 if (should_restart_tx(q) &&
1261 test_and_clear_bit(TXQ_ETH, &qs->txq_stopped)) {
1263 netif_tx_wake_queue(txq);
1268 q->unacked += ndesc;
1269 compl = (q->unacked & 8) << (S_WR_COMPL - 3);
1273 if (q->pidx >= q->size) {
1278 /* update port statistics */
1279 if (skb->ip_summed == CHECKSUM_COMPLETE)
1280 qs->port_stats[SGE_PSTAT_TX_CSUM]++;
1281 if (skb_shinfo(skb)->gso_size)
1282 qs->port_stats[SGE_PSTAT_TSO]++;
1283 if (vlan_tx_tag_present(skb) && pi->vlan_grp)
1284 qs->port_stats[SGE_PSTAT_VLANINS]++;
1287 * We do not use Tx completion interrupts to free DMAd Tx packets.
1288 * This is good for performamce but means that we rely on new Tx
1289 * packets arriving to run the destructors of completed packets,
1290 * which open up space in their sockets' send queues. Sometimes
1291 * we do not get such new packets causing Tx to stall. A single
1292 * UDP transmitter is a good example of this situation. We have
1293 * a clean up timer that periodically reclaims completed packets
1294 * but it doesn't run often enough (nor do we want it to) to prevent
1295 * lengthy stalls. A solution to this problem is to run the
1296 * destructor early, after the packet is queued but before it's DMAd.
1297 * A cons is that we lie to socket memory accounting, but the amount
1298 * of extra memory is reasonable (limited by the number of Tx
1299 * descriptors), the packets do actually get freed quickly by new
1300 * packets almost always, and for protocols like TCP that wait for
1301 * acks to really free up the data the extra memory is even less.
1302 * On the positive side we run the destructors on the sending CPU
1303 * rather than on a potentially different completing CPU, usually a
1304 * good thing. We also run them without holding our Tx queue lock,
1305 * unlike what reclaim_completed_tx() would otherwise do.
1307 * Run the destructor before telling the DMA engine about the packet
1308 * to make sure it doesn't complete and get freed prematurely.
1310 if (likely(!skb_shared(skb)))
1313 write_tx_pkt_wr(adap, skb, pi, pidx, gen, q, ndesc, compl);
1314 check_ring_tx_db(adap, q);
1315 return NETDEV_TX_OK;
1319 * write_imm - write a packet into a Tx descriptor as immediate data
1320 * @d: the Tx descriptor to write
1322 * @len: the length of packet data to write as immediate data
1323 * @gen: the generation bit value to write
1325 * Writes a packet as immediate data into a Tx descriptor. The packet
1326 * contains a work request at its beginning. We must write the packet
1327 * carefully so the SGE doesn't read it accidentally before it's written
1330 static inline void write_imm(struct tx_desc *d, struct sk_buff *skb,
1331 unsigned int len, unsigned int gen)
1333 struct work_request_hdr *from = (struct work_request_hdr *)skb->data;
1334 struct work_request_hdr *to = (struct work_request_hdr *)d;
1336 if (likely(!skb->data_len))
1337 memcpy(&to[1], &from[1], len - sizeof(*from));
1339 skb_copy_bits(skb, sizeof(*from), &to[1], len - sizeof(*from));
1341 to->wr_hi = from->wr_hi | htonl(F_WR_SOP | F_WR_EOP |
1342 V_WR_BCNTLFLT(len & 7));
1344 to->wr_lo = from->wr_lo | htonl(V_WR_GEN(gen) |
1345 V_WR_LEN((len + 7) / 8));
1351 * check_desc_avail - check descriptor availability on a send queue
1352 * @adap: the adapter
1353 * @q: the send queue
1354 * @skb: the packet needing the descriptors
1355 * @ndesc: the number of Tx descriptors needed
1356 * @qid: the Tx queue number in its queue set (TXQ_OFLD or TXQ_CTRL)
1358 * Checks if the requested number of Tx descriptors is available on an
1359 * SGE send queue. If the queue is already suspended or not enough
1360 * descriptors are available the packet is queued for later transmission.
1361 * Must be called with the Tx queue locked.
1363 * Returns 0 if enough descriptors are available, 1 if there aren't
1364 * enough descriptors and the packet has been queued, and 2 if the caller
1365 * needs to retry because there weren't enough descriptors at the
1366 * beginning of the call but some freed up in the mean time.
1368 static inline int check_desc_avail(struct adapter *adap, struct sge_txq *q,
1369 struct sk_buff *skb, unsigned int ndesc,
1372 if (unlikely(!skb_queue_empty(&q->sendq))) {
1373 addq_exit:__skb_queue_tail(&q->sendq, skb);
1376 if (unlikely(q->size - q->in_use < ndesc)) {
1377 struct sge_qset *qs = txq_to_qset(q, qid);
1379 set_bit(qid, &qs->txq_stopped);
1380 smp_mb__after_clear_bit();
1382 if (should_restart_tx(q) &&
1383 test_and_clear_bit(qid, &qs->txq_stopped))
1393 * reclaim_completed_tx_imm - reclaim completed control-queue Tx descs
1394 * @q: the SGE control Tx queue
1396 * This is a variant of reclaim_completed_tx() that is used for Tx queues
1397 * that send only immediate data (presently just the control queues) and
1398 * thus do not have any sk_buffs to release.
1400 static inline void reclaim_completed_tx_imm(struct sge_txq *q)
1402 unsigned int reclaim = q->processed - q->cleaned;
1404 q->in_use -= reclaim;
1405 q->cleaned += reclaim;
1408 static inline int immediate(const struct sk_buff *skb)
1410 return skb->len <= WR_LEN;
1414 * ctrl_xmit - send a packet through an SGE control Tx queue
1415 * @adap: the adapter
1416 * @q: the control queue
1419 * Send a packet through an SGE control Tx queue. Packets sent through
1420 * a control queue must fit entirely as immediate data in a single Tx
1421 * descriptor and have no page fragments.
1423 static int ctrl_xmit(struct adapter *adap, struct sge_txq *q,
1424 struct sk_buff *skb)
1427 struct work_request_hdr *wrp = (struct work_request_hdr *)skb->data;
1429 if (unlikely(!immediate(skb))) {
1432 return NET_XMIT_SUCCESS;
1435 wrp->wr_hi |= htonl(F_WR_SOP | F_WR_EOP);
1436 wrp->wr_lo = htonl(V_WR_TID(q->token));
1438 spin_lock(&q->lock);
1439 again:reclaim_completed_tx_imm(q);
1441 ret = check_desc_avail(adap, q, skb, 1, TXQ_CTRL);
1442 if (unlikely(ret)) {
1444 spin_unlock(&q->lock);
1450 write_imm(&q->desc[q->pidx], skb, skb->len, q->gen);
1453 if (++q->pidx >= q->size) {
1457 spin_unlock(&q->lock);
1459 t3_write_reg(adap, A_SG_KDOORBELL,
1460 F_SELEGRCNTX | V_EGRCNTX(q->cntxt_id));
1461 return NET_XMIT_SUCCESS;
1465 * restart_ctrlq - restart a suspended control queue
1466 * @qs: the queue set cotaining the control queue
1468 * Resumes transmission on a suspended Tx control queue.
1470 static void restart_ctrlq(unsigned long data)
1472 struct sk_buff *skb;
1473 struct sge_qset *qs = (struct sge_qset *)data;
1474 struct sge_txq *q = &qs->txq[TXQ_CTRL];
1476 spin_lock(&q->lock);
1477 again:reclaim_completed_tx_imm(q);
1479 while (q->in_use < q->size &&
1480 (skb = __skb_dequeue(&q->sendq)) != NULL) {
1482 write_imm(&q->desc[q->pidx], skb, skb->len, q->gen);
1484 if (++q->pidx >= q->size) {
1491 if (!skb_queue_empty(&q->sendq)) {
1492 set_bit(TXQ_CTRL, &qs->txq_stopped);
1493 smp_mb__after_clear_bit();
1495 if (should_restart_tx(q) &&
1496 test_and_clear_bit(TXQ_CTRL, &qs->txq_stopped))
1501 spin_unlock(&q->lock);
1503 t3_write_reg(qs->adap, A_SG_KDOORBELL,
1504 F_SELEGRCNTX | V_EGRCNTX(q->cntxt_id));
1508 * Send a management message through control queue 0
1510 int t3_mgmt_tx(struct adapter *adap, struct sk_buff *skb)
1514 ret = ctrl_xmit(adap, &adap->sge.qs[0].txq[TXQ_CTRL], skb);
1521 * deferred_unmap_destructor - unmap a packet when it is freed
1524 * This is the packet destructor used for Tx packets that need to remain
1525 * mapped until they are freed rather than until their Tx descriptors are
1528 static void deferred_unmap_destructor(struct sk_buff *skb)
1531 const dma_addr_t *p;
1532 const struct skb_shared_info *si;
1533 const struct deferred_unmap_info *dui;
1535 dui = (struct deferred_unmap_info *)skb->head;
1538 if (skb->tail - skb->transport_header)
1539 pci_unmap_single(dui->pdev, *p++,
1540 skb->tail - skb->transport_header,
1543 si = skb_shinfo(skb);
1544 for (i = 0; i < si->nr_frags; i++)
1545 pci_unmap_page(dui->pdev, *p++, si->frags[i].size,
1549 static void setup_deferred_unmapping(struct sk_buff *skb, struct pci_dev *pdev,
1550 const struct sg_ent *sgl, int sgl_flits)
1553 struct deferred_unmap_info *dui;
1555 dui = (struct deferred_unmap_info *)skb->head;
1557 for (p = dui->addr; sgl_flits >= 3; sgl++, sgl_flits -= 3) {
1558 *p++ = be64_to_cpu(sgl->addr[0]);
1559 *p++ = be64_to_cpu(sgl->addr[1]);
1562 *p = be64_to_cpu(sgl->addr[0]);
1566 * write_ofld_wr - write an offload work request
1567 * @adap: the adapter
1568 * @skb: the packet to send
1570 * @pidx: index of the first Tx descriptor to write
1571 * @gen: the generation value to use
1572 * @ndesc: number of descriptors the packet will occupy
1574 * Write an offload work request to send the supplied packet. The packet
1575 * data already carry the work request with most fields populated.
1577 static void write_ofld_wr(struct adapter *adap, struct sk_buff *skb,
1578 struct sge_txq *q, unsigned int pidx,
1579 unsigned int gen, unsigned int ndesc)
1581 unsigned int sgl_flits, flits;
1582 struct work_request_hdr *from;
1583 struct sg_ent *sgp, sgl[MAX_SKB_FRAGS / 2 + 1];
1584 struct tx_desc *d = &q->desc[pidx];
1586 if (immediate(skb)) {
1587 q->sdesc[pidx].skb = NULL;
1588 write_imm(d, skb, skb->len, gen);
1592 /* Only TX_DATA builds SGLs */
1594 from = (struct work_request_hdr *)skb->data;
1595 memcpy(&d->flit[1], &from[1],
1596 skb_transport_offset(skb) - sizeof(*from));
1598 flits = skb_transport_offset(skb) / 8;
1599 sgp = ndesc == 1 ? (struct sg_ent *)&d->flit[flits] : sgl;
1600 sgl_flits = make_sgl(skb, sgp, skb_transport_header(skb),
1601 skb->tail - skb->transport_header,
1603 if (need_skb_unmap()) {
1604 setup_deferred_unmapping(skb, adap->pdev, sgp, sgl_flits);
1605 skb->destructor = deferred_unmap_destructor;
1608 write_wr_hdr_sgl(ndesc, skb, d, pidx, q, sgl, flits, sgl_flits,
1609 gen, from->wr_hi, from->wr_lo);
1613 * calc_tx_descs_ofld - calculate # of Tx descriptors for an offload packet
1616 * Returns the number of Tx descriptors needed for the given offload
1617 * packet. These packets are already fully constructed.
1619 static inline unsigned int calc_tx_descs_ofld(const struct sk_buff *skb)
1621 unsigned int flits, cnt;
1623 if (skb->len <= WR_LEN)
1624 return 1; /* packet fits as immediate data */
1626 flits = skb_transport_offset(skb) / 8; /* headers */
1627 cnt = skb_shinfo(skb)->nr_frags;
1628 if (skb->tail != skb->transport_header)
1630 return flits_to_desc(flits + sgl_len(cnt));
1634 * ofld_xmit - send a packet through an offload queue
1635 * @adap: the adapter
1636 * @q: the Tx offload queue
1639 * Send an offload packet through an SGE offload queue.
1641 static int ofld_xmit(struct adapter *adap, struct sge_txq *q,
1642 struct sk_buff *skb)
1645 unsigned int ndesc = calc_tx_descs_ofld(skb), pidx, gen;
1647 spin_lock(&q->lock);
1648 again: reclaim_completed_tx(adap, q, TX_RECLAIM_CHUNK);
1650 ret = check_desc_avail(adap, q, skb, ndesc, TXQ_OFLD);
1651 if (unlikely(ret)) {
1653 skb->priority = ndesc; /* save for restart */
1654 spin_unlock(&q->lock);
1664 if (q->pidx >= q->size) {
1668 spin_unlock(&q->lock);
1670 write_ofld_wr(adap, skb, q, pidx, gen, ndesc);
1671 check_ring_tx_db(adap, q);
1672 return NET_XMIT_SUCCESS;
1676 * restart_offloadq - restart a suspended offload queue
1677 * @qs: the queue set cotaining the offload queue
1679 * Resumes transmission on a suspended Tx offload queue.
1681 static void restart_offloadq(unsigned long data)
1683 struct sk_buff *skb;
1684 struct sge_qset *qs = (struct sge_qset *)data;
1685 struct sge_txq *q = &qs->txq[TXQ_OFLD];
1686 const struct port_info *pi = netdev_priv(qs->netdev);
1687 struct adapter *adap = pi->adapter;
1689 spin_lock(&q->lock);
1690 again: reclaim_completed_tx(adap, q, TX_RECLAIM_CHUNK);
1692 while ((skb = skb_peek(&q->sendq)) != NULL) {
1693 unsigned int gen, pidx;
1694 unsigned int ndesc = skb->priority;
1696 if (unlikely(q->size - q->in_use < ndesc)) {
1697 set_bit(TXQ_OFLD, &qs->txq_stopped);
1698 smp_mb__after_clear_bit();
1700 if (should_restart_tx(q) &&
1701 test_and_clear_bit(TXQ_OFLD, &qs->txq_stopped))
1711 if (q->pidx >= q->size) {
1715 __skb_unlink(skb, &q->sendq);
1716 spin_unlock(&q->lock);
1718 write_ofld_wr(adap, skb, q, pidx, gen, ndesc);
1719 spin_lock(&q->lock);
1721 spin_unlock(&q->lock);
1724 set_bit(TXQ_RUNNING, &q->flags);
1725 set_bit(TXQ_LAST_PKT_DB, &q->flags);
1728 t3_write_reg(adap, A_SG_KDOORBELL,
1729 F_SELEGRCNTX | V_EGRCNTX(q->cntxt_id));
1733 * queue_set - return the queue set a packet should use
1736 * Maps a packet to the SGE queue set it should use. The desired queue
1737 * set is carried in bits 1-3 in the packet's priority.
1739 static inline int queue_set(const struct sk_buff *skb)
1741 return skb->priority >> 1;
1745 * is_ctrl_pkt - return whether an offload packet is a control packet
1748 * Determines whether an offload packet should use an OFLD or a CTRL
1749 * Tx queue. This is indicated by bit 0 in the packet's priority.
1751 static inline int is_ctrl_pkt(const struct sk_buff *skb)
1753 return skb->priority & 1;
1757 * t3_offload_tx - send an offload packet
1758 * @tdev: the offload device to send to
1761 * Sends an offload packet. We use the packet priority to select the
1762 * appropriate Tx queue as follows: bit 0 indicates whether the packet
1763 * should be sent as regular or control, bits 1-3 select the queue set.
1765 int t3_offload_tx(struct t3cdev *tdev, struct sk_buff *skb)
1767 struct adapter *adap = tdev2adap(tdev);
1768 struct sge_qset *qs = &adap->sge.qs[queue_set(skb)];
1770 if (unlikely(is_ctrl_pkt(skb)))
1771 return ctrl_xmit(adap, &qs->txq[TXQ_CTRL], skb);
1773 return ofld_xmit(adap, &qs->txq[TXQ_OFLD], skb);
1777 * offload_enqueue - add an offload packet to an SGE offload receive queue
1778 * @q: the SGE response queue
1781 * Add a new offload packet to an SGE response queue's offload packet
1782 * queue. If the packet is the first on the queue it schedules the RX
1783 * softirq to process the queue.
1785 static inline void offload_enqueue(struct sge_rspq *q, struct sk_buff *skb)
1787 int was_empty = skb_queue_empty(&q->rx_queue);
1789 __skb_queue_tail(&q->rx_queue, skb);
1792 struct sge_qset *qs = rspq_to_qset(q);
1794 napi_schedule(&qs->napi);
1799 * deliver_partial_bundle - deliver a (partial) bundle of Rx offload pkts
1800 * @tdev: the offload device that will be receiving the packets
1801 * @q: the SGE response queue that assembled the bundle
1802 * @skbs: the partial bundle
1803 * @n: the number of packets in the bundle
1805 * Delivers a (partial) bundle of Rx offload packets to an offload device.
1807 static inline void deliver_partial_bundle(struct t3cdev *tdev,
1809 struct sk_buff *skbs[], int n)
1812 q->offload_bundles++;
1813 tdev->recv(tdev, skbs, n);
1818 * ofld_poll - NAPI handler for offload packets in interrupt mode
1819 * @dev: the network device doing the polling
1820 * @budget: polling budget
1822 * The NAPI handler for offload packets when a response queue is serviced
1823 * by the hard interrupt handler, i.e., when it's operating in non-polling
1824 * mode. Creates small packet batches and sends them through the offload
1825 * receive handler. Batches need to be of modest size as we do prefetches
1826 * on the packets in each.
1828 static int ofld_poll(struct napi_struct *napi, int budget)
1830 struct sge_qset *qs = container_of(napi, struct sge_qset, napi);
1831 struct sge_rspq *q = &qs->rspq;
1832 struct adapter *adapter = qs->adap;
1835 while (work_done < budget) {
1836 struct sk_buff *skb, *tmp, *skbs[RX_BUNDLE_SIZE];
1837 struct sk_buff_head queue;
1840 spin_lock_irq(&q->lock);
1841 __skb_queue_head_init(&queue);
1842 skb_queue_splice_init(&q->rx_queue, &queue);
1843 if (skb_queue_empty(&queue)) {
1844 napi_complete(napi);
1845 spin_unlock_irq(&q->lock);
1848 spin_unlock_irq(&q->lock);
1851 skb_queue_walk_safe(&queue, skb, tmp) {
1852 if (work_done >= budget)
1856 __skb_unlink(skb, &queue);
1857 prefetch(skb->data);
1858 skbs[ngathered] = skb;
1859 if (++ngathered == RX_BUNDLE_SIZE) {
1860 q->offload_bundles++;
1861 adapter->tdev.recv(&adapter->tdev, skbs,
1866 if (!skb_queue_empty(&queue)) {
1867 /* splice remaining packets back onto Rx queue */
1868 spin_lock_irq(&q->lock);
1869 skb_queue_splice(&queue, &q->rx_queue);
1870 spin_unlock_irq(&q->lock);
1872 deliver_partial_bundle(&adapter->tdev, q, skbs, ngathered);
1879 * rx_offload - process a received offload packet
1880 * @tdev: the offload device receiving the packet
1881 * @rq: the response queue that received the packet
1883 * @rx_gather: a gather list of packets if we are building a bundle
1884 * @gather_idx: index of the next available slot in the bundle
1886 * Process an ingress offload pakcet and add it to the offload ingress
1887 * queue. Returns the index of the next available slot in the bundle.
1889 static inline int rx_offload(struct t3cdev *tdev, struct sge_rspq *rq,
1890 struct sk_buff *skb, struct sk_buff *rx_gather[],
1891 unsigned int gather_idx)
1893 skb_reset_mac_header(skb);
1894 skb_reset_network_header(skb);
1895 skb_reset_transport_header(skb);
1898 rx_gather[gather_idx++] = skb;
1899 if (gather_idx == RX_BUNDLE_SIZE) {
1900 tdev->recv(tdev, rx_gather, RX_BUNDLE_SIZE);
1902 rq->offload_bundles++;
1905 offload_enqueue(rq, skb);
1911 * restart_tx - check whether to restart suspended Tx queues
1912 * @qs: the queue set to resume
1914 * Restarts suspended Tx queues of an SGE queue set if they have enough
1915 * free resources to resume operation.
1917 static void restart_tx(struct sge_qset *qs)
1919 if (test_bit(TXQ_ETH, &qs->txq_stopped) &&
1920 should_restart_tx(&qs->txq[TXQ_ETH]) &&
1921 test_and_clear_bit(TXQ_ETH, &qs->txq_stopped)) {
1922 qs->txq[TXQ_ETH].restarts++;
1923 if (netif_running(qs->netdev))
1924 netif_tx_wake_queue(qs->tx_q);
1927 if (test_bit(TXQ_OFLD, &qs->txq_stopped) &&
1928 should_restart_tx(&qs->txq[TXQ_OFLD]) &&
1929 test_and_clear_bit(TXQ_OFLD, &qs->txq_stopped)) {
1930 qs->txq[TXQ_OFLD].restarts++;
1931 tasklet_schedule(&qs->txq[TXQ_OFLD].qresume_tsk);
1933 if (test_bit(TXQ_CTRL, &qs->txq_stopped) &&
1934 should_restart_tx(&qs->txq[TXQ_CTRL]) &&
1935 test_and_clear_bit(TXQ_CTRL, &qs->txq_stopped)) {
1936 qs->txq[TXQ_CTRL].restarts++;
1937 tasklet_schedule(&qs->txq[TXQ_CTRL].qresume_tsk);
1942 * cxgb3_arp_process - process an ARP request probing a private IP address
1943 * @adapter: the adapter
1944 * @skb: the skbuff containing the ARP request
1946 * Check if the ARP request is probing the private IP address
1947 * dedicated to iSCSI, generate an ARP reply if so.
1949 static void cxgb3_arp_process(struct adapter *adapter, struct sk_buff *skb)
1951 struct net_device *dev = skb->dev;
1952 struct port_info *pi;
1954 unsigned char *arp_ptr;
1961 skb_reset_network_header(skb);
1964 if (arp->ar_op != htons(ARPOP_REQUEST))
1967 arp_ptr = (unsigned char *)(arp + 1);
1969 arp_ptr += dev->addr_len;
1970 memcpy(&sip, arp_ptr, sizeof(sip));
1971 arp_ptr += sizeof(sip);
1972 arp_ptr += dev->addr_len;
1973 memcpy(&tip, arp_ptr, sizeof(tip));
1975 pi = netdev_priv(dev);
1976 if (tip != pi->iscsi_ipv4addr)
1979 arp_send(ARPOP_REPLY, ETH_P_ARP, sip, dev, tip, sha,
1980 dev->dev_addr, sha);
1984 static inline int is_arp(struct sk_buff *skb)
1986 return skb->protocol == htons(ETH_P_ARP);
1990 * rx_eth - process an ingress ethernet packet
1991 * @adap: the adapter
1992 * @rq: the response queue that received the packet
1994 * @pad: amount of padding at the start of the buffer
1996 * Process an ingress ethernet pakcet and deliver it to the stack.
1997 * The padding is 2 if the packet was delivered in an Rx buffer and 0
1998 * if it was immediate data in a response.
2000 static void rx_eth(struct adapter *adap, struct sge_rspq *rq,
2001 struct sk_buff *skb, int pad, int lro)
2003 struct cpl_rx_pkt *p = (struct cpl_rx_pkt *)(skb->data + pad);
2004 struct sge_qset *qs = rspq_to_qset(rq);
2005 struct port_info *pi;
2007 skb_pull(skb, sizeof(*p) + pad);
2008 skb->protocol = eth_type_trans(skb, adap->port[p->iff]);
2009 pi = netdev_priv(skb->dev);
2010 if ((pi->rx_offload & T3_RX_CSUM) && p->csum_valid &&
2011 p->csum == htons(0xffff) && !p->fragment) {
2012 qs->port_stats[SGE_PSTAT_RX_CSUM_GOOD]++;
2013 skb->ip_summed = CHECKSUM_UNNECESSARY;
2015 skb->ip_summed = CHECKSUM_NONE;
2016 skb_record_rx_queue(skb, qs - &adap->sge.qs[0]);
2018 if (unlikely(p->vlan_valid)) {
2019 struct vlan_group *grp = pi->vlan_grp;
2021 qs->port_stats[SGE_PSTAT_VLANEX]++;
2024 vlan_gro_receive(&qs->napi, grp,
2025 ntohs(p->vlan), skb);
2027 if (unlikely(pi->iscsi_ipv4addr &&
2029 unsigned short vtag = ntohs(p->vlan) &
2031 skb->dev = vlan_group_get_device(grp,
2033 cxgb3_arp_process(adap, skb);
2035 __vlan_hwaccel_rx(skb, grp, ntohs(p->vlan),
2039 dev_kfree_skb_any(skb);
2040 } else if (rq->polling) {
2042 napi_gro_receive(&qs->napi, skb);
2044 if (unlikely(pi->iscsi_ipv4addr && is_arp(skb)))
2045 cxgb3_arp_process(adap, skb);
2046 netif_receive_skb(skb);
2052 static inline int is_eth_tcp(u32 rss)
2054 return G_HASHTYPE(ntohl(rss)) == RSS_HASH_4_TUPLE;
2058 * lro_add_page - add a page chunk to an LRO session
2059 * @adap: the adapter
2060 * @qs: the associated queue set
2061 * @fl: the free list containing the page chunk to add
2062 * @len: packet length
2063 * @complete: Indicates the last fragment of a frame
2065 * Add a received packet contained in a page chunk to an existing LRO
2068 static void lro_add_page(struct adapter *adap, struct sge_qset *qs,
2069 struct sge_fl *fl, int len, int complete)
2071 struct rx_sw_desc *sd = &fl->sdesc[fl->cidx];
2072 struct sk_buff *skb = NULL;
2073 struct cpl_rx_pkt *cpl;
2074 struct skb_frag_struct *rx_frag;
2079 skb = napi_get_frags(&qs->napi);
2085 pci_dma_sync_single_for_cpu(adap->pdev,
2086 pci_unmap_addr(sd, dma_addr),
2087 fl->buf_size - SGE_PG_RSVD,
2088 PCI_DMA_FROMDEVICE);
2090 (*sd->pg_chunk.p_cnt)--;
2091 if (!*sd->pg_chunk.p_cnt)
2092 pci_unmap_page(adap->pdev,
2093 sd->pg_chunk.mapping,
2095 PCI_DMA_FROMDEVICE);
2098 put_page(sd->pg_chunk.page);
2104 rx_frag = skb_shinfo(skb)->frags;
2105 nr_frags = skb_shinfo(skb)->nr_frags;
2108 offset = 2 + sizeof(struct cpl_rx_pkt);
2109 qs->lro_va = sd->pg_chunk.va + 2;
2113 prefetch(qs->lro_va);
2115 rx_frag += nr_frags;
2116 rx_frag->page = sd->pg_chunk.page;
2117 rx_frag->page_offset = sd->pg_chunk.offset + offset;
2118 rx_frag->size = len;
2121 skb->data_len += len;
2122 skb->truesize += len;
2123 skb_shinfo(skb)->nr_frags++;
2128 skb->ip_summed = CHECKSUM_UNNECESSARY;
2131 if (unlikely(cpl->vlan_valid)) {
2132 struct net_device *dev = qs->netdev;
2133 struct port_info *pi = netdev_priv(dev);
2134 struct vlan_group *grp = pi->vlan_grp;
2136 if (likely(grp != NULL)) {
2137 vlan_gro_frags(&qs->napi, grp, ntohs(cpl->vlan));
2141 napi_gro_frags(&qs->napi);
2145 * handle_rsp_cntrl_info - handles control information in a response
2146 * @qs: the queue set corresponding to the response
2147 * @flags: the response control flags
2149 * Handles the control information of an SGE response, such as GTS
2150 * indications and completion credits for the queue set's Tx queues.
2151 * HW coalesces credits, we don't do any extra SW coalescing.
2153 static inline void handle_rsp_cntrl_info(struct sge_qset *qs, u32 flags)
2155 unsigned int credits;
2158 if (flags & F_RSPD_TXQ0_GTS)
2159 clear_bit(TXQ_RUNNING, &qs->txq[TXQ_ETH].flags);
2162 credits = G_RSPD_TXQ0_CR(flags);
2164 qs->txq[TXQ_ETH].processed += credits;
2166 credits = G_RSPD_TXQ2_CR(flags);
2168 qs->txq[TXQ_CTRL].processed += credits;
2171 if (flags & F_RSPD_TXQ1_GTS)
2172 clear_bit(TXQ_RUNNING, &qs->txq[TXQ_OFLD].flags);
2174 credits = G_RSPD_TXQ1_CR(flags);
2176 qs->txq[TXQ_OFLD].processed += credits;
2180 * check_ring_db - check if we need to ring any doorbells
2181 * @adapter: the adapter
2182 * @qs: the queue set whose Tx queues are to be examined
2183 * @sleeping: indicates which Tx queue sent GTS
2185 * Checks if some of a queue set's Tx queues need to ring their doorbells
2186 * to resume transmission after idling while they still have unprocessed
2189 static void check_ring_db(struct adapter *adap, struct sge_qset *qs,
2190 unsigned int sleeping)
2192 if (sleeping & F_RSPD_TXQ0_GTS) {
2193 struct sge_txq *txq = &qs->txq[TXQ_ETH];
2195 if (txq->cleaned + txq->in_use != txq->processed &&
2196 !test_and_set_bit(TXQ_LAST_PKT_DB, &txq->flags)) {
2197 set_bit(TXQ_RUNNING, &txq->flags);
2198 t3_write_reg(adap, A_SG_KDOORBELL, F_SELEGRCNTX |
2199 V_EGRCNTX(txq->cntxt_id));
2203 if (sleeping & F_RSPD_TXQ1_GTS) {
2204 struct sge_txq *txq = &qs->txq[TXQ_OFLD];
2206 if (txq->cleaned + txq->in_use != txq->processed &&
2207 !test_and_set_bit(TXQ_LAST_PKT_DB, &txq->flags)) {
2208 set_bit(TXQ_RUNNING, &txq->flags);
2209 t3_write_reg(adap, A_SG_KDOORBELL, F_SELEGRCNTX |
2210 V_EGRCNTX(txq->cntxt_id));
2216 * is_new_response - check if a response is newly written
2217 * @r: the response descriptor
2218 * @q: the response queue
2220 * Returns true if a response descriptor contains a yet unprocessed
2223 static inline int is_new_response(const struct rsp_desc *r,
2224 const struct sge_rspq *q)
2226 return (r->intr_gen & F_RSPD_GEN2) == q->gen;
2229 static inline void clear_rspq_bufstate(struct sge_rspq * const q)
2232 q->rx_recycle_buf = 0;
2235 #define RSPD_GTS_MASK (F_RSPD_TXQ0_GTS | F_RSPD_TXQ1_GTS)
2236 #define RSPD_CTRL_MASK (RSPD_GTS_MASK | \
2237 V_RSPD_TXQ0_CR(M_RSPD_TXQ0_CR) | \
2238 V_RSPD_TXQ1_CR(M_RSPD_TXQ1_CR) | \
2239 V_RSPD_TXQ2_CR(M_RSPD_TXQ2_CR))
2241 /* How long to delay the next interrupt in case of memory shortage, in 0.1us. */
2242 #define NOMEM_INTR_DELAY 2500
2245 * process_responses - process responses from an SGE response queue
2246 * @adap: the adapter
2247 * @qs: the queue set to which the response queue belongs
2248 * @budget: how many responses can be processed in this round
2250 * Process responses from an SGE response queue up to the supplied budget.
2251 * Responses include received packets as well as credits and other events
2252 * for the queues that belong to the response queue's queue set.
2253 * A negative budget is effectively unlimited.
2255 * Additionally choose the interrupt holdoff time for the next interrupt
2256 * on this queue. If the system is under memory shortage use a fairly
2257 * long delay to help recovery.
2259 static int process_responses(struct adapter *adap, struct sge_qset *qs,
2262 struct sge_rspq *q = &qs->rspq;
2263 struct rsp_desc *r = &q->desc[q->cidx];
2264 int budget_left = budget;
2265 unsigned int sleeping = 0;
2266 struct sk_buff *offload_skbs[RX_BUNDLE_SIZE];
2269 q->next_holdoff = q->holdoff_tmr;
2271 while (likely(budget_left && is_new_response(r, q))) {
2272 int packet_complete, eth, ethpad = 2, lro = qs->lro_enabled;
2273 struct sk_buff *skb = NULL;
2274 u32 len, flags = ntohl(r->flags);
2275 __be32 rss_hi = *(const __be32 *)r,
2276 rss_lo = r->rss_hdr.rss_hash_val;
2278 eth = r->rss_hdr.opcode == CPL_RX_PKT;
2280 if (unlikely(flags & F_RSPD_ASYNC_NOTIF)) {
2281 skb = alloc_skb(AN_PKT_SIZE, GFP_ATOMIC);
2285 memcpy(__skb_put(skb, AN_PKT_SIZE), r, AN_PKT_SIZE);
2286 skb->data[0] = CPL_ASYNC_NOTIF;
2287 rss_hi = htonl(CPL_ASYNC_NOTIF << 24);
2289 } else if (flags & F_RSPD_IMM_DATA_VALID) {
2290 skb = get_imm_packet(r);
2291 if (unlikely(!skb)) {
2293 q->next_holdoff = NOMEM_INTR_DELAY;
2295 /* consume one credit since we tried */
2301 } else if ((len = ntohl(r->len_cq)) != 0) {
2304 lro &= eth && is_eth_tcp(rss_hi);
2306 fl = (len & F_RSPD_FLQ) ? &qs->fl[1] : &qs->fl[0];
2307 if (fl->use_pages) {
2308 void *addr = fl->sdesc[fl->cidx].pg_chunk.va;
2311 #if L1_CACHE_BYTES < 128
2312 prefetch(addr + L1_CACHE_BYTES);
2314 __refill_fl(adap, fl);
2316 lro_add_page(adap, qs, fl,
2318 flags & F_RSPD_EOP);
2322 skb = get_packet_pg(adap, fl, q,
2325 SGE_RX_DROP_THRES : 0);
2328 skb = get_packet(adap, fl, G_RSPD_LEN(len),
2329 eth ? SGE_RX_DROP_THRES : 0);
2330 if (unlikely(!skb)) {
2334 } else if (unlikely(r->rss_hdr.opcode == CPL_TRACE_PKT))
2337 if (++fl->cidx == fl->size)
2342 if (flags & RSPD_CTRL_MASK) {
2343 sleeping |= flags & RSPD_GTS_MASK;
2344 handle_rsp_cntrl_info(qs, flags);
2348 if (unlikely(++q->cidx == q->size)) {
2355 if (++q->credits >= (q->size / 4)) {
2356 refill_rspq(adap, q, q->credits);
2360 packet_complete = flags &
2361 (F_RSPD_EOP | F_RSPD_IMM_DATA_VALID |
2362 F_RSPD_ASYNC_NOTIF);
2364 if (skb != NULL && packet_complete) {
2366 rx_eth(adap, q, skb, ethpad, lro);
2369 /* Preserve the RSS info in csum & priority */
2371 skb->priority = rss_lo;
2372 ngathered = rx_offload(&adap->tdev, q, skb,
2377 if (flags & F_RSPD_EOP)
2378 clear_rspq_bufstate(q);
2383 deliver_partial_bundle(&adap->tdev, q, offload_skbs, ngathered);
2386 check_ring_db(adap, qs, sleeping);
2388 smp_mb(); /* commit Tx queue .processed updates */
2389 if (unlikely(qs->txq_stopped != 0))
2392 budget -= budget_left;
2396 static inline int is_pure_response(const struct rsp_desc *r)
2398 __be32 n = r->flags & htonl(F_RSPD_ASYNC_NOTIF | F_RSPD_IMM_DATA_VALID);
2400 return (n | r->len_cq) == 0;
2404 * napi_rx_handler - the NAPI handler for Rx processing
2405 * @napi: the napi instance
2406 * @budget: how many packets we can process in this round
2408 * Handler for new data events when using NAPI.
2410 static int napi_rx_handler(struct napi_struct *napi, int budget)
2412 struct sge_qset *qs = container_of(napi, struct sge_qset, napi);
2413 struct adapter *adap = qs->adap;
2414 int work_done = process_responses(adap, qs, budget);
2416 if (likely(work_done < budget)) {
2417 napi_complete(napi);
2420 * Because we don't atomically flush the following
2421 * write it is possible that in very rare cases it can
2422 * reach the device in a way that races with a new
2423 * response being written plus an error interrupt
2424 * causing the NAPI interrupt handler below to return
2425 * unhandled status to the OS. To protect against
2426 * this would require flushing the write and doing
2427 * both the write and the flush with interrupts off.
2428 * Way too expensive and unjustifiable given the
2429 * rarity of the race.
2431 * The race cannot happen at all with MSI-X.
2433 t3_write_reg(adap, A_SG_GTS, V_RSPQ(qs->rspq.cntxt_id) |
2434 V_NEWTIMER(qs->rspq.next_holdoff) |
2435 V_NEWINDEX(qs->rspq.cidx));
2441 * Returns true if the device is already scheduled for polling.
2443 static inline int napi_is_scheduled(struct napi_struct *napi)
2445 return test_bit(NAPI_STATE_SCHED, &napi->state);
2449 * process_pure_responses - process pure responses from a response queue
2450 * @adap: the adapter
2451 * @qs: the queue set owning the response queue
2452 * @r: the first pure response to process
2454 * A simpler version of process_responses() that handles only pure (i.e.,
2455 * non data-carrying) responses. Such respones are too light-weight to
2456 * justify calling a softirq under NAPI, so we handle them specially in
2457 * the interrupt handler. The function is called with a pointer to a
2458 * response, which the caller must ensure is a valid pure response.
2460 * Returns 1 if it encounters a valid data-carrying response, 0 otherwise.
2462 static int process_pure_responses(struct adapter *adap, struct sge_qset *qs,
2465 struct sge_rspq *q = &qs->rspq;
2466 unsigned int sleeping = 0;
2469 u32 flags = ntohl(r->flags);
2472 if (unlikely(++q->cidx == q->size)) {
2479 if (flags & RSPD_CTRL_MASK) {
2480 sleeping |= flags & RSPD_GTS_MASK;
2481 handle_rsp_cntrl_info(qs, flags);
2485 if (++q->credits >= (q->size / 4)) {
2486 refill_rspq(adap, q, q->credits);
2489 } while (is_new_response(r, q) && is_pure_response(r));
2492 check_ring_db(adap, qs, sleeping);
2494 smp_mb(); /* commit Tx queue .processed updates */
2495 if (unlikely(qs->txq_stopped != 0))
2498 return is_new_response(r, q);
2502 * handle_responses - decide what to do with new responses in NAPI mode
2503 * @adap: the adapter
2504 * @q: the response queue
2506 * This is used by the NAPI interrupt handlers to decide what to do with
2507 * new SGE responses. If there are no new responses it returns -1. If
2508 * there are new responses and they are pure (i.e., non-data carrying)
2509 * it handles them straight in hard interrupt context as they are very
2510 * cheap and don't deliver any packets. Finally, if there are any data
2511 * signaling responses it schedules the NAPI handler. Returns 1 if it
2512 * schedules NAPI, 0 if all new responses were pure.
2514 * The caller must ascertain NAPI is not already running.
2516 static inline int handle_responses(struct adapter *adap, struct sge_rspq *q)
2518 struct sge_qset *qs = rspq_to_qset(q);
2519 struct rsp_desc *r = &q->desc[q->cidx];
2521 if (!is_new_response(r, q))
2523 if (is_pure_response(r) && process_pure_responses(adap, qs, r) == 0) {
2524 t3_write_reg(adap, A_SG_GTS, V_RSPQ(q->cntxt_id) |
2525 V_NEWTIMER(q->holdoff_tmr) | V_NEWINDEX(q->cidx));
2528 napi_schedule(&qs->napi);
2533 * The MSI-X interrupt handler for an SGE response queue for the non-NAPI case
2534 * (i.e., response queue serviced in hard interrupt).
2536 irqreturn_t t3_sge_intr_msix(int irq, void *cookie)
2538 struct sge_qset *qs = cookie;
2539 struct adapter *adap = qs->adap;
2540 struct sge_rspq *q = &qs->rspq;
2542 spin_lock(&q->lock);
2543 if (process_responses(adap, qs, -1) == 0)
2544 q->unhandled_irqs++;
2545 t3_write_reg(adap, A_SG_GTS, V_RSPQ(q->cntxt_id) |
2546 V_NEWTIMER(q->next_holdoff) | V_NEWINDEX(q->cidx));
2547 spin_unlock(&q->lock);
2552 * The MSI-X interrupt handler for an SGE response queue for the NAPI case
2553 * (i.e., response queue serviced by NAPI polling).
2555 static irqreturn_t t3_sge_intr_msix_napi(int irq, void *cookie)
2557 struct sge_qset *qs = cookie;
2558 struct sge_rspq *q = &qs->rspq;
2560 spin_lock(&q->lock);
2562 if (handle_responses(qs->adap, q) < 0)
2563 q->unhandled_irqs++;
2564 spin_unlock(&q->lock);
2569 * The non-NAPI MSI interrupt handler. This needs to handle data events from
2570 * SGE response queues as well as error and other async events as they all use
2571 * the same MSI vector. We use one SGE response queue per port in this mode
2572 * and protect all response queues with queue 0's lock.
2574 static irqreturn_t t3_intr_msi(int irq, void *cookie)
2576 int new_packets = 0;
2577 struct adapter *adap = cookie;
2578 struct sge_rspq *q = &adap->sge.qs[0].rspq;
2580 spin_lock(&q->lock);
2582 if (process_responses(adap, &adap->sge.qs[0], -1)) {
2583 t3_write_reg(adap, A_SG_GTS, V_RSPQ(q->cntxt_id) |
2584 V_NEWTIMER(q->next_holdoff) | V_NEWINDEX(q->cidx));
2588 if (adap->params.nports == 2 &&
2589 process_responses(adap, &adap->sge.qs[1], -1)) {
2590 struct sge_rspq *q1 = &adap->sge.qs[1].rspq;
2592 t3_write_reg(adap, A_SG_GTS, V_RSPQ(q1->cntxt_id) |
2593 V_NEWTIMER(q1->next_holdoff) |
2594 V_NEWINDEX(q1->cidx));
2598 if (!new_packets && t3_slow_intr_handler(adap) == 0)
2599 q->unhandled_irqs++;
2601 spin_unlock(&q->lock);
2605 static int rspq_check_napi(struct sge_qset *qs)
2607 struct sge_rspq *q = &qs->rspq;
2609 if (!napi_is_scheduled(&qs->napi) &&
2610 is_new_response(&q->desc[q->cidx], q)) {
2611 napi_schedule(&qs->napi);
2618 * The MSI interrupt handler for the NAPI case (i.e., response queues serviced
2619 * by NAPI polling). Handles data events from SGE response queues as well as
2620 * error and other async events as they all use the same MSI vector. We use
2621 * one SGE response queue per port in this mode and protect all response
2622 * queues with queue 0's lock.
2624 static irqreturn_t t3_intr_msi_napi(int irq, void *cookie)
2627 struct adapter *adap = cookie;
2628 struct sge_rspq *q = &adap->sge.qs[0].rspq;
2630 spin_lock(&q->lock);
2632 new_packets = rspq_check_napi(&adap->sge.qs[0]);
2633 if (adap->params.nports == 2)
2634 new_packets += rspq_check_napi(&adap->sge.qs[1]);
2635 if (!new_packets && t3_slow_intr_handler(adap) == 0)
2636 q->unhandled_irqs++;
2638 spin_unlock(&q->lock);
2643 * A helper function that processes responses and issues GTS.
2645 static inline int process_responses_gts(struct adapter *adap,
2646 struct sge_rspq *rq)
2650 work = process_responses(adap, rspq_to_qset(rq), -1);
2651 t3_write_reg(adap, A_SG_GTS, V_RSPQ(rq->cntxt_id) |
2652 V_NEWTIMER(rq->next_holdoff) | V_NEWINDEX(rq->cidx));
2657 * The legacy INTx interrupt handler. This needs to handle data events from
2658 * SGE response queues as well as error and other async events as they all use
2659 * the same interrupt pin. We use one SGE response queue per port in this mode
2660 * and protect all response queues with queue 0's lock.
2662 static irqreturn_t t3_intr(int irq, void *cookie)
2664 int work_done, w0, w1;
2665 struct adapter *adap = cookie;
2666 struct sge_rspq *q0 = &adap->sge.qs[0].rspq;
2667 struct sge_rspq *q1 = &adap->sge.qs[1].rspq;
2669 spin_lock(&q0->lock);
2671 w0 = is_new_response(&q0->desc[q0->cidx], q0);
2672 w1 = adap->params.nports == 2 &&
2673 is_new_response(&q1->desc[q1->cidx], q1);
2675 if (likely(w0 | w1)) {
2676 t3_write_reg(adap, A_PL_CLI, 0);
2677 t3_read_reg(adap, A_PL_CLI); /* flush */
2680 process_responses_gts(adap, q0);
2683 process_responses_gts(adap, q1);
2685 work_done = w0 | w1;
2687 work_done = t3_slow_intr_handler(adap);
2689 spin_unlock(&q0->lock);
2690 return IRQ_RETVAL(work_done != 0);
2694 * Interrupt handler for legacy INTx interrupts for T3B-based cards.
2695 * Handles data events from SGE response queues as well as error and other
2696 * async events as they all use the same interrupt pin. We use one SGE
2697 * response queue per port in this mode and protect all response queues with
2700 static irqreturn_t t3b_intr(int irq, void *cookie)
2703 struct adapter *adap = cookie;
2704 struct sge_rspq *q0 = &adap->sge.qs[0].rspq;
2706 t3_write_reg(adap, A_PL_CLI, 0);
2707 map = t3_read_reg(adap, A_SG_DATA_INTR);
2709 if (unlikely(!map)) /* shared interrupt, most likely */
2712 spin_lock(&q0->lock);
2714 if (unlikely(map & F_ERRINTR))
2715 t3_slow_intr_handler(adap);
2717 if (likely(map & 1))
2718 process_responses_gts(adap, q0);
2721 process_responses_gts(adap, &adap->sge.qs[1].rspq);
2723 spin_unlock(&q0->lock);
2728 * NAPI interrupt handler for legacy INTx interrupts for T3B-based cards.
2729 * Handles data events from SGE response queues as well as error and other
2730 * async events as they all use the same interrupt pin. We use one SGE
2731 * response queue per port in this mode and protect all response queues with
2734 static irqreturn_t t3b_intr_napi(int irq, void *cookie)
2737 struct adapter *adap = cookie;
2738 struct sge_qset *qs0 = &adap->sge.qs[0];
2739 struct sge_rspq *q0 = &qs0->rspq;
2741 t3_write_reg(adap, A_PL_CLI, 0);
2742 map = t3_read_reg(adap, A_SG_DATA_INTR);
2744 if (unlikely(!map)) /* shared interrupt, most likely */
2747 spin_lock(&q0->lock);
2749 if (unlikely(map & F_ERRINTR))
2750 t3_slow_intr_handler(adap);
2752 if (likely(map & 1))
2753 napi_schedule(&qs0->napi);
2756 napi_schedule(&adap->sge.qs[1].napi);
2758 spin_unlock(&q0->lock);
2763 * t3_intr_handler - select the top-level interrupt handler
2764 * @adap: the adapter
2765 * @polling: whether using NAPI to service response queues
2767 * Selects the top-level interrupt handler based on the type of interrupts
2768 * (MSI-X, MSI, or legacy) and whether NAPI will be used to service the
2771 irq_handler_t t3_intr_handler(struct adapter *adap, int polling)
2773 if (adap->flags & USING_MSIX)
2774 return polling ? t3_sge_intr_msix_napi : t3_sge_intr_msix;
2775 if (adap->flags & USING_MSI)
2776 return polling ? t3_intr_msi_napi : t3_intr_msi;
2777 if (adap->params.rev > 0)
2778 return polling ? t3b_intr_napi : t3b_intr;
2782 #define SGE_PARERR (F_CPPARITYERROR | F_OCPARITYERROR | F_RCPARITYERROR | \
2783 F_IRPARITYERROR | V_ITPARITYERROR(M_ITPARITYERROR) | \
2784 V_FLPARITYERROR(M_FLPARITYERROR) | F_LODRBPARITYERROR | \
2785 F_HIDRBPARITYERROR | F_LORCQPARITYERROR | \
2787 #define SGE_FRAMINGERR (F_UC_REQ_FRAMINGERROR | F_R_REQ_FRAMINGERROR)
2788 #define SGE_FATALERR (SGE_PARERR | SGE_FRAMINGERR | F_RSPQCREDITOVERFOW | \
2792 * t3_sge_err_intr_handler - SGE async event interrupt handler
2793 * @adapter: the adapter
2795 * Interrupt handler for SGE asynchronous (non-data) events.
2797 void t3_sge_err_intr_handler(struct adapter *adapter)
2799 unsigned int v, status = t3_read_reg(adapter, A_SG_INT_CAUSE) &
2802 if (status & SGE_PARERR)
2803 CH_ALERT(adapter, "SGE parity error (0x%x)\n",
2804 status & SGE_PARERR);
2805 if (status & SGE_FRAMINGERR)
2806 CH_ALERT(adapter, "SGE framing error (0x%x)\n",
2807 status & SGE_FRAMINGERR);
2809 if (status & F_RSPQCREDITOVERFOW)
2810 CH_ALERT(adapter, "SGE response queue credit overflow\n");
2812 if (status & F_RSPQDISABLED) {
2813 v = t3_read_reg(adapter, A_SG_RSPQ_FL_STATUS);
2816 "packet delivered to disabled response queue "
2817 "(0x%x)\n", (v >> S_RSPQ0DISABLED) & 0xff);
2820 if (status & (F_HIPIODRBDROPERR | F_LOPIODRBDROPERR))
2821 CH_ALERT(adapter, "SGE dropped %s priority doorbell\n",
2822 status & F_HIPIODRBDROPERR ? "high" : "lo");
2824 t3_write_reg(adapter, A_SG_INT_CAUSE, status);
2825 if (status & SGE_FATALERR)
2826 t3_fatal_err(adapter);
2830 * sge_timer_tx - perform periodic maintenance of an SGE qset
2831 * @data: the SGE queue set to maintain
2833 * Runs periodically from a timer to perform maintenance of an SGE queue
2834 * set. It performs two tasks:
2836 * Cleans up any completed Tx descriptors that may still be pending.
2837 * Normal descriptor cleanup happens when new packets are added to a Tx
2838 * queue so this timer is relatively infrequent and does any cleanup only
2839 * if the Tx queue has not seen any new packets in a while. We make a
2840 * best effort attempt to reclaim descriptors, in that we don't wait
2841 * around if we cannot get a queue's lock (which most likely is because
2842 * someone else is queueing new packets and so will also handle the clean
2843 * up). Since control queues use immediate data exclusively we don't
2844 * bother cleaning them up here.
2847 static void sge_timer_tx(unsigned long data)
2849 struct sge_qset *qs = (struct sge_qset *)data;
2850 struct port_info *pi = netdev_priv(qs->netdev);
2851 struct adapter *adap = pi->adapter;
2852 unsigned int tbd[SGE_TXQ_PER_SET] = {0, 0};
2853 unsigned long next_period;
2855 if (__netif_tx_trylock(qs->tx_q)) {
2856 tbd[TXQ_ETH] = reclaim_completed_tx(adap, &qs->txq[TXQ_ETH],
2857 TX_RECLAIM_TIMER_CHUNK);
2858 __netif_tx_unlock(qs->tx_q);
2861 if (spin_trylock(&qs->txq[TXQ_OFLD].lock)) {
2862 tbd[TXQ_OFLD] = reclaim_completed_tx(adap, &qs->txq[TXQ_OFLD],
2863 TX_RECLAIM_TIMER_CHUNK);
2864 spin_unlock(&qs->txq[TXQ_OFLD].lock);
2867 next_period = TX_RECLAIM_PERIOD >>
2868 (max(tbd[TXQ_ETH], tbd[TXQ_OFLD]) /
2869 TX_RECLAIM_TIMER_CHUNK);
2870 mod_timer(&qs->tx_reclaim_timer, jiffies + next_period);
2874 * sge_timer_rx - perform periodic maintenance of an SGE qset
2875 * @data: the SGE queue set to maintain
2877 * a) Replenishes Rx queues that have run out due to memory shortage.
2878 * Normally new Rx buffers are added when existing ones are consumed but
2879 * when out of memory a queue can become empty. We try to add only a few
2880 * buffers here, the queue will be replenished fully as these new buffers
2881 * are used up if memory shortage has subsided.
2883 * b) Return coalesced response queue credits in case a response queue is
2887 static void sge_timer_rx(unsigned long data)
2890 struct sge_qset *qs = (struct sge_qset *)data;
2891 struct port_info *pi = netdev_priv(qs->netdev);
2892 struct adapter *adap = pi->adapter;
2895 lock = adap->params.rev > 0 ?
2896 &qs->rspq.lock : &adap->sge.qs[0].rspq.lock;
2898 if (!spin_trylock_irq(lock))
2901 if (napi_is_scheduled(&qs->napi))
2904 if (adap->params.rev < 4) {
2905 status = t3_read_reg(adap, A_SG_RSPQ_FL_STATUS);
2907 if (status & (1 << qs->rspq.cntxt_id)) {
2909 if (qs->rspq.credits) {
2911 refill_rspq(adap, &qs->rspq, 1);
2912 qs->rspq.restarted++;
2913 t3_write_reg(adap, A_SG_RSPQ_FL_STATUS,
2914 1 << qs->rspq.cntxt_id);
2919 if (qs->fl[0].credits < qs->fl[0].size)
2920 __refill_fl(adap, &qs->fl[0]);
2921 if (qs->fl[1].credits < qs->fl[1].size)
2922 __refill_fl(adap, &qs->fl[1]);
2925 spin_unlock_irq(lock);
2927 mod_timer(&qs->rx_reclaim_timer, jiffies + RX_RECLAIM_PERIOD);
2931 * t3_update_qset_coalesce - update coalescing settings for a queue set
2932 * @qs: the SGE queue set
2933 * @p: new queue set parameters
2935 * Update the coalescing settings for an SGE queue set. Nothing is done
2936 * if the queue set is not initialized yet.
2938 void t3_update_qset_coalesce(struct sge_qset *qs, const struct qset_params *p)
2940 qs->rspq.holdoff_tmr = max(p->coalesce_usecs * 10, 1U);/* can't be 0 */
2941 qs->rspq.polling = p->polling;
2942 qs->napi.poll = p->polling ? napi_rx_handler : ofld_poll;
2946 * t3_sge_alloc_qset - initialize an SGE queue set
2947 * @adapter: the adapter
2948 * @id: the queue set id
2949 * @nports: how many Ethernet ports will be using this queue set
2950 * @irq_vec_idx: the IRQ vector index for response queue interrupts
2951 * @p: configuration parameters for this queue set
2952 * @ntxq: number of Tx queues for the queue set
2953 * @netdev: net device associated with this queue set
2954 * @netdevq: net device TX queue associated with this queue set
2956 * Allocate resources and initialize an SGE queue set. A queue set
2957 * comprises a response queue, two Rx free-buffer queues, and up to 3
2958 * Tx queues. The Tx queues are assigned roles in the order Ethernet
2959 * queue, offload queue, and control queue.
2961 int t3_sge_alloc_qset(struct adapter *adapter, unsigned int id, int nports,
2962 int irq_vec_idx, const struct qset_params *p,
2963 int ntxq, struct net_device *dev,
2964 struct netdev_queue *netdevq)
2966 int i, avail, ret = -ENOMEM;
2967 struct sge_qset *q = &adapter->sge.qs[id];
2969 init_qset_cntxt(q, id);
2970 setup_timer(&q->tx_reclaim_timer, sge_timer_tx, (unsigned long)q);
2971 setup_timer(&q->rx_reclaim_timer, sge_timer_rx, (unsigned long)q);
2973 q->fl[0].desc = alloc_ring(adapter->pdev, p->fl_size,
2974 sizeof(struct rx_desc),
2975 sizeof(struct rx_sw_desc),
2976 &q->fl[0].phys_addr, &q->fl[0].sdesc);
2980 q->fl[1].desc = alloc_ring(adapter->pdev, p->jumbo_size,
2981 sizeof(struct rx_desc),
2982 sizeof(struct rx_sw_desc),
2983 &q->fl[1].phys_addr, &q->fl[1].sdesc);
2987 q->rspq.desc = alloc_ring(adapter->pdev, p->rspq_size,
2988 sizeof(struct rsp_desc), 0,
2989 &q->rspq.phys_addr, NULL);
2993 for (i = 0; i < ntxq; ++i) {
2995 * The control queue always uses immediate data so does not
2996 * need to keep track of any sk_buffs.
2998 size_t sz = i == TXQ_CTRL ? 0 : sizeof(struct tx_sw_desc);
3000 q->txq[i].desc = alloc_ring(adapter->pdev, p->txq_size[i],
3001 sizeof(struct tx_desc), sz,
3002 &q->txq[i].phys_addr,
3004 if (!q->txq[i].desc)
3008 q->txq[i].size = p->txq_size[i];
3009 spin_lock_init(&q->txq[i].lock);
3010 skb_queue_head_init(&q->txq[i].sendq);
3013 tasklet_init(&q->txq[TXQ_OFLD].qresume_tsk, restart_offloadq,
3015 tasklet_init(&q->txq[TXQ_CTRL].qresume_tsk, restart_ctrlq,
3018 q->fl[0].gen = q->fl[1].gen = 1;
3019 q->fl[0].size = p->fl_size;
3020 q->fl[1].size = p->jumbo_size;
3023 q->rspq.size = p->rspq_size;
3024 spin_lock_init(&q->rspq.lock);
3025 skb_queue_head_init(&q->rspq.rx_queue);
3027 q->txq[TXQ_ETH].stop_thres = nports *
3028 flits_to_desc(sgl_len(MAX_SKB_FRAGS + 1) + 3);
3030 #if FL0_PG_CHUNK_SIZE > 0
3031 q->fl[0].buf_size = FL0_PG_CHUNK_SIZE;
3033 q->fl[0].buf_size = SGE_RX_SM_BUF_SIZE + sizeof(struct cpl_rx_data);
3035 #if FL1_PG_CHUNK_SIZE > 0
3036 q->fl[1].buf_size = FL1_PG_CHUNK_SIZE;
3038 q->fl[1].buf_size = is_offload(adapter) ?
3039 (16 * 1024) - SKB_DATA_ALIGN(sizeof(struct skb_shared_info)) :
3040 MAX_FRAME_SIZE + 2 + sizeof(struct cpl_rx_pkt);
3043 q->fl[0].use_pages = FL0_PG_CHUNK_SIZE > 0;
3044 q->fl[1].use_pages = FL1_PG_CHUNK_SIZE > 0;
3045 q->fl[0].order = FL0_PG_ORDER;
3046 q->fl[1].order = FL1_PG_ORDER;
3047 q->fl[0].alloc_size = FL0_PG_ALLOC_SIZE;
3048 q->fl[1].alloc_size = FL1_PG_ALLOC_SIZE;
3050 spin_lock_irq(&adapter->sge.reg_lock);
3052 /* FL threshold comparison uses < */
3053 ret = t3_sge_init_rspcntxt(adapter, q->rspq.cntxt_id, irq_vec_idx,
3054 q->rspq.phys_addr, q->rspq.size,
3055 q->fl[0].buf_size - SGE_PG_RSVD, 1, 0);
3059 for (i = 0; i < SGE_RXQ_PER_SET; ++i) {
3060 ret = t3_sge_init_flcntxt(adapter, q->fl[i].cntxt_id, 0,
3061 q->fl[i].phys_addr, q->fl[i].size,
3062 q->fl[i].buf_size - SGE_PG_RSVD,
3063 p->cong_thres, 1, 0);
3068 ret = t3_sge_init_ecntxt(adapter, q->txq[TXQ_ETH].cntxt_id, USE_GTS,
3069 SGE_CNTXT_ETH, id, q->txq[TXQ_ETH].phys_addr,
3070 q->txq[TXQ_ETH].size, q->txq[TXQ_ETH].token,
3076 ret = t3_sge_init_ecntxt(adapter, q->txq[TXQ_OFLD].cntxt_id,
3077 USE_GTS, SGE_CNTXT_OFLD, id,
3078 q->txq[TXQ_OFLD].phys_addr,
3079 q->txq[TXQ_OFLD].size, 0, 1, 0);
3085 ret = t3_sge_init_ecntxt(adapter, q->txq[TXQ_CTRL].cntxt_id, 0,
3087 q->txq[TXQ_CTRL].phys_addr,
3088 q->txq[TXQ_CTRL].size,
3089 q->txq[TXQ_CTRL].token, 1, 0);
3094 spin_unlock_irq(&adapter->sge.reg_lock);
3099 t3_update_qset_coalesce(q, p);
3101 avail = refill_fl(adapter, &q->fl[0], q->fl[0].size,
3102 GFP_KERNEL | __GFP_COMP);
3104 CH_ALERT(adapter, "free list queue 0 initialization failed\n");
3107 if (avail < q->fl[0].size)
3108 CH_WARN(adapter, "free list queue 0 enabled with %d credits\n",
3111 avail = refill_fl(adapter, &q->fl[1], q->fl[1].size,
3112 GFP_KERNEL | __GFP_COMP);
3113 if (avail < q->fl[1].size)
3114 CH_WARN(adapter, "free list queue 1 enabled with %d credits\n",
3116 refill_rspq(adapter, &q->rspq, q->rspq.size - 1);
3118 t3_write_reg(adapter, A_SG_GTS, V_RSPQ(q->rspq.cntxt_id) |
3119 V_NEWTIMER(q->rspq.holdoff_tmr));
3124 spin_unlock_irq(&adapter->sge.reg_lock);
3126 t3_free_qset(adapter, q);
3131 * t3_start_sge_timers - start SGE timer call backs
3132 * @adap: the adapter
3134 * Starts each SGE queue set's timer call back
3136 void t3_start_sge_timers(struct adapter *adap)
3140 for (i = 0; i < SGE_QSETS; ++i) {
3141 struct sge_qset *q = &adap->sge.qs[i];
3143 if (q->tx_reclaim_timer.function)
3144 mod_timer(&q->tx_reclaim_timer, jiffies + TX_RECLAIM_PERIOD);
3146 if (q->rx_reclaim_timer.function)
3147 mod_timer(&q->rx_reclaim_timer, jiffies + RX_RECLAIM_PERIOD);
3152 * t3_stop_sge_timers - stop SGE timer call backs
3153 * @adap: the adapter
3155 * Stops each SGE queue set's timer call back
3157 void t3_stop_sge_timers(struct adapter *adap)
3161 for (i = 0; i < SGE_QSETS; ++i) {
3162 struct sge_qset *q = &adap->sge.qs[i];
3164 if (q->tx_reclaim_timer.function)
3165 del_timer_sync(&q->tx_reclaim_timer);
3166 if (q->rx_reclaim_timer.function)
3167 del_timer_sync(&q->rx_reclaim_timer);
3172 * t3_free_sge_resources - free SGE resources
3173 * @adap: the adapter
3175 * Frees resources used by the SGE queue sets.
3177 void t3_free_sge_resources(struct adapter *adap)
3181 for (i = 0; i < SGE_QSETS; ++i)
3182 t3_free_qset(adap, &adap->sge.qs[i]);
3186 * t3_sge_start - enable SGE
3187 * @adap: the adapter
3189 * Enables the SGE for DMAs. This is the last step in starting packet
3192 void t3_sge_start(struct adapter *adap)
3194 t3_set_reg_field(adap, A_SG_CONTROL, F_GLOBALENABLE, F_GLOBALENABLE);
3198 * t3_sge_stop - disable SGE operation
3199 * @adap: the adapter
3201 * Disables the DMA engine. This can be called in emeregencies (e.g.,
3202 * from error interrupts) or from normal process context. In the latter
3203 * case it also disables any pending queue restart tasklets. Note that
3204 * if it is called in interrupt context it cannot disable the restart
3205 * tasklets as it cannot wait, however the tasklets will have no effect
3206 * since the doorbells are disabled and the driver will call this again
3207 * later from process context, at which time the tasklets will be stopped
3208 * if they are still running.
3210 void t3_sge_stop(struct adapter *adap)
3212 t3_set_reg_field(adap, A_SG_CONTROL, F_GLOBALENABLE, 0);
3213 if (!in_interrupt()) {
3216 for (i = 0; i < SGE_QSETS; ++i) {
3217 struct sge_qset *qs = &adap->sge.qs[i];
3219 tasklet_kill(&qs->txq[TXQ_OFLD].qresume_tsk);
3220 tasklet_kill(&qs->txq[TXQ_CTRL].qresume_tsk);
3226 * t3_sge_init - initialize SGE
3227 * @adap: the adapter
3228 * @p: the SGE parameters
3230 * Performs SGE initialization needed every time after a chip reset.
3231 * We do not initialize any of the queue sets here, instead the driver
3232 * top-level must request those individually. We also do not enable DMA
3233 * here, that should be done after the queues have been set up.
3235 void t3_sge_init(struct adapter *adap, struct sge_params *p)
3237 unsigned int ctrl, ups = ffs(pci_resource_len(adap->pdev, 2) >> 12);
3239 ctrl = F_DROPPKT | V_PKTSHIFT(2) | F_FLMODE | F_AVOIDCQOVFL |
3240 F_CQCRDTCTRL | F_CONGMODE | F_TNLFLMODE | F_FATLPERREN |
3241 V_HOSTPAGESIZE(PAGE_SHIFT - 11) | F_BIGENDIANINGRESS |
3242 V_USERSPACESIZE(ups ? ups - 1 : 0) | F_ISCSICOALESCING;
3243 #if SGE_NUM_GENBITS == 1
3244 ctrl |= F_EGRGENCTRL;
3246 if (adap->params.rev > 0) {
3247 if (!(adap->flags & (USING_MSIX | USING_MSI)))
3248 ctrl |= F_ONEINTMULTQ | F_OPTONEINTMULTQ;
3250 t3_write_reg(adap, A_SG_CONTROL, ctrl);
3251 t3_write_reg(adap, A_SG_EGR_RCQ_DRB_THRSH, V_HIRCQDRBTHRSH(512) |
3252 V_LORCQDRBTHRSH(512));
3253 t3_write_reg(adap, A_SG_TIMER_TICK, core_ticks_per_usec(adap) / 10);
3254 t3_write_reg(adap, A_SG_CMDQ_CREDIT_TH, V_THRESHOLD(32) |
3255 V_TIMEOUT(200 * core_ticks_per_usec(adap)));
3256 t3_write_reg(adap, A_SG_HI_DRB_HI_THRSH,
3257 adap->params.rev < T3_REV_C ? 1000 : 500);
3258 t3_write_reg(adap, A_SG_HI_DRB_LO_THRSH, 256);
3259 t3_write_reg(adap, A_SG_LO_DRB_HI_THRSH, 1000);
3260 t3_write_reg(adap, A_SG_LO_DRB_LO_THRSH, 256);
3261 t3_write_reg(adap, A_SG_OCO_BASE, V_BASE1(0xfff));
3262 t3_write_reg(adap, A_SG_DRB_PRI_THRESH, 63 * 1024);
3266 * t3_sge_prep - one-time SGE initialization
3267 * @adap: the associated adapter
3268 * @p: SGE parameters
3270 * Performs one-time initialization of SGE SW state. Includes determining
3271 * defaults for the assorted SGE parameters, which admins can change until
3272 * they are used to initialize the SGE.
3274 void t3_sge_prep(struct adapter *adap, struct sge_params *p)
3278 p->max_pkt_size = (16 * 1024) - sizeof(struct cpl_rx_data) -
3279 SKB_DATA_ALIGN(sizeof(struct skb_shared_info));
3281 for (i = 0; i < SGE_QSETS; ++i) {
3282 struct qset_params *q = p->qset + i;
3284 q->polling = adap->params.rev > 0;
3285 q->coalesce_usecs = 5;
3286 q->rspq_size = 1024;
3288 q->jumbo_size = 512;
3289 q->txq_size[TXQ_ETH] = 1024;
3290 q->txq_size[TXQ_OFLD] = 1024;
3291 q->txq_size[TXQ_CTRL] = 256;
3295 spin_lock_init(&adap->sge.reg_lock);
3299 * t3_get_desc - dump an SGE descriptor for debugging purposes
3300 * @qs: the queue set
3301 * @qnum: identifies the specific queue (0..2: Tx, 3:response, 4..5: Rx)
3302 * @idx: the descriptor index in the queue
3303 * @data: where to dump the descriptor contents
3305 * Dumps the contents of a HW descriptor of an SGE queue. Returns the
3306 * size of the descriptor.
3308 int t3_get_desc(const struct sge_qset *qs, unsigned int qnum, unsigned int idx,
3309 unsigned char *data)
3315 if (!qs->txq[qnum].desc || idx >= qs->txq[qnum].size)
3317 memcpy(data, &qs->txq[qnum].desc[idx], sizeof(struct tx_desc));
3318 return sizeof(struct tx_desc);
3322 if (!qs->rspq.desc || idx >= qs->rspq.size)
3324 memcpy(data, &qs->rspq.desc[idx], sizeof(struct rsp_desc));
3325 return sizeof(struct rsp_desc);
3329 if (!qs->fl[qnum].desc || idx >= qs->fl[qnum].size)
3331 memcpy(data, &qs->fl[qnum].desc[idx], sizeof(struct rx_desc));
3332 return sizeof(struct rx_desc);