1c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom/* 2c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom * This file is part of the Chelsio T4 PCI-E SR-IOV Virtual Function Ethernet 3c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom * driver for Linux. 4c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom * 5c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom * Copyright (c) 2009-2010 Chelsio Communications, Inc. All rights reserved. 6c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom * 7c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom * This software is available to you under a choice of one of two 8c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom * licenses. You may choose to be licensed under the terms of the GNU 9c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom * General Public License (GPL) Version 2, available from the file 10c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom * COPYING in the main directory of this source tree, or the 11c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom * OpenIB.org BSD license below: 12c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom * 13c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom * Redistribution and use in source and binary forms, with or 14c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom * without modification, are permitted provided that the following 15c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom * conditions are met: 16c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom * 17c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom * - Redistributions of source code must retain the above 18c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom * copyright notice, this list of conditions and the following 19c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom * disclaimer. 20c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom * 21c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom * - Redistributions in binary form must reproduce the above 22c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom * copyright notice, this list of conditions and the following 23c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom * disclaimer in the documentation and/or other materials 24c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom * provided with the distribution. 25c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom * 26c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, 27c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF 28c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND 29c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS 30c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN 31c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN 32c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 33c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom * SOFTWARE. 34c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom */ 35c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom 36c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom#include <linux/skbuff.h> 37c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom#include <linux/netdevice.h> 38c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom#include <linux/etherdevice.h> 39c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom#include <linux/if_vlan.h> 40c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom#include <linux/ip.h> 41c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom#include <net/ipv6.h> 42c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom#include <net/tcp.h> 43c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom#include <linux/dma-mapping.h> 4470c71606190e9115e5f8363bfcd164c582eb314aPaul Gortmaker#include <linux/prefetch.h> 45c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom 46c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom#include "t4vf_common.h" 47c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom#include "t4vf_defs.h" 48c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom 49c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom#include "../cxgb4/t4_regs.h" 50c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom#include "../cxgb4/t4fw_api.h" 51c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom#include "../cxgb4/t4_msg.h" 52c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom 53c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom/* 54c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom * Decoded Adapter Parameters. 55c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom */ 56c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedomstatic u32 FL_PG_ORDER; /* large page allocation size */ 57c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedomstatic u32 STAT_LEN; /* length of status page at ring end */ 58c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedomstatic u32 PKTSHIFT; /* padding between CPL and packet data */ 59c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedomstatic u32 FL_ALIGN; /* response queue message alignment */ 60c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom 61c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom/* 62c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom * Constants ... 63c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom */ 64c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedomenum { 65c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom /* 66c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom * Egress Queue sizes, producer and consumer indices are all in units 67c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom * of Egress Context Units bytes. Note that as far as the hardware is 68c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom * concerned, the free list is an Egress Queue (the host produces free 69c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom * buffers which the hardware consumes) and free list entries are 70c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom * 64-bit PCI DMA addresses. 71c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom */ 72c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom EQ_UNIT = SGE_EQ_IDXSIZE, 73c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom FL_PER_EQ_UNIT = EQ_UNIT / sizeof(__be64), 74c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom TXD_PER_EQ_UNIT = EQ_UNIT / sizeof(__be64), 75c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom 76c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom /* 77c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom * Max number of TX descriptors we clean up at a time. Should be 78c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom * modest as freeing skbs isn't cheap and it happens while holding 79c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom * locks. We just need to free packets faster than they arrive, we 80c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom * eventually catch up and keep the amortized cost reasonable. 81c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom */ 82c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom MAX_TX_RECLAIM = 16, 83c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom 84c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom /* 85c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom * Max number of Rx buffers we replenish at a time. Again keep this 86c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom * modest, allocating buffers isn't cheap either. 87c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom */ 88c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom MAX_RX_REFILL = 16, 89c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom 90c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom /* 91c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom * Period of the Rx queue check timer. This timer is infrequent as it 92c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom * has something to do only when the system experiences severe memory 93c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom * shortage. 94c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom */ 95c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom RX_QCHECK_PERIOD = (HZ / 2), 96c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom 97c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom /* 98c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom * Period of the TX queue check timer and the maximum number of TX 99c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom * descriptors to be reclaimed by the TX timer. 100c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom */ 101c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom TX_QCHECK_PERIOD = (HZ / 2), 102c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom MAX_TIMER_TX_RECLAIM = 100, 103c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom 104c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom /* 105c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom * An FL with <= FL_STARVE_THRES buffers is starving and a periodic 106c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom * timer will attempt to refill it. 107c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom */ 108c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom FL_STARVE_THRES = 4, 109c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom 110c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom /* 111c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom * Suspend an Ethernet TX queue with fewer available descriptors than 112c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom * this. We always want to have room for a maximum sized packet: 113c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom * inline immediate data + MAX_SKB_FRAGS. This is the same as 114c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom * calc_tx_flits() for a TSO packet with nr_frags == MAX_SKB_FRAGS 115c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom * (see that function and its helpers for a description of the 116c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom * calculation). 117c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom */ 118c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom ETHTXQ_MAX_FRAGS = MAX_SKB_FRAGS + 1, 119c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom ETHTXQ_MAX_SGL_LEN = ((3 * (ETHTXQ_MAX_FRAGS-1))/2 + 120c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom ((ETHTXQ_MAX_FRAGS-1) & 1) + 121c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom 2), 122c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom ETHTXQ_MAX_HDR = (sizeof(struct fw_eth_tx_pkt_vm_wr) + 123c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom sizeof(struct cpl_tx_pkt_lso_core) + 124c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom sizeof(struct cpl_tx_pkt_core)) / sizeof(__be64), 125c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom ETHTXQ_MAX_FLITS = ETHTXQ_MAX_SGL_LEN + ETHTXQ_MAX_HDR, 126c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom 127c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom ETHTXQ_STOP_THRES = 1 + DIV_ROUND_UP(ETHTXQ_MAX_FLITS, TXD_PER_EQ_UNIT), 128c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom 129c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom /* 130c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom * Max TX descriptor space we allow for an Ethernet packet to be 131c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom * inlined into a WR. This is limited by the maximum value which 132c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom * we can specify for immediate data in the firmware Ethernet TX 133c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom * Work Request. 134c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom */ 135c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom MAX_IMM_TX_PKT_LEN = FW_WR_IMMDLEN_MASK, 136c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom 137c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom /* 138c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom * Max size of a WR sent through a control TX queue. 139c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom */ 140c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom MAX_CTRL_WR_LEN = 256, 141c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom 142c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom /* 143c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom * Maximum amount of data which we'll ever need to inline into a 144c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom * TX ring: max(MAX_IMM_TX_PKT_LEN, MAX_CTRL_WR_LEN). 145c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom */ 146c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom MAX_IMM_TX_LEN = (MAX_IMM_TX_PKT_LEN > MAX_CTRL_WR_LEN 147c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom ? MAX_IMM_TX_PKT_LEN 148c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom : MAX_CTRL_WR_LEN), 149c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom 150c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom /* 151c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom * For incoming packets less than RX_COPY_THRES, we copy the data into 152c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom * an skb rather than referencing the data. We allocate enough 153c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom * in-line room in skb's to accommodate pulling in RX_PULL_LEN bytes 154c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom * of the data (header). 155c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom */ 156c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom RX_COPY_THRES = 256, 157c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom RX_PULL_LEN = 128, 158c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom 159eb6c503db3619c43b5aad0ced82e9d91c034ced5Casey Leedom /* 160eb6c503db3619c43b5aad0ced82e9d91c034ced5Casey Leedom * Main body length for sk_buffs used for RX Ethernet packets with 161eb6c503db3619c43b5aad0ced82e9d91c034ced5Casey Leedom * fragments. Should be >= RX_PULL_LEN but possibly bigger to give 162eb6c503db3619c43b5aad0ced82e9d91c034ced5Casey Leedom * pskb_may_pull() some room. 163eb6c503db3619c43b5aad0ced82e9d91c034ced5Casey Leedom */ 164eb6c503db3619c43b5aad0ced82e9d91c034ced5Casey Leedom RX_SKB_LEN = 512, 165eb6c503db3619c43b5aad0ced82e9d91c034ced5Casey Leedom}; 166c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom 167c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom/* 168c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom * Software state per TX descriptor. 169c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom */ 170c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedomstruct tx_sw_desc { 171c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom struct sk_buff *skb; /* socket buffer of TX data source */ 172c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom struct ulptx_sgl *sgl; /* scatter/gather list in TX Queue */ 173c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom}; 174c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom 175c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom/* 176c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom * Software state per RX Free List descriptor. We keep track of the allocated 177c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom * FL page, its size, and its PCI DMA address (if the page is mapped). The FL 178c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom * page size and its PCI DMA mapped state are stored in the low bits of the 179c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom * PCI DMA address as per below. 180c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom */ 181c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedomstruct rx_sw_desc { 182c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom struct page *page; /* Free List page buffer */ 183c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom dma_addr_t dma_addr; /* PCI DMA address (if mapped) */ 184c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom /* and flags (see below) */ 185c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom}; 186c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom 187c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom/* 188c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom * The low bits of rx_sw_desc.dma_addr have special meaning. Note that the 189c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom * SGE also uses the low 4 bits to determine the size of the buffer. It uses 190c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom * those bits to index into the SGE_FL_BUFFER_SIZE[index] register array. 191c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom * Since we only use SGE_FL_BUFFER_SIZE0 and SGE_FL_BUFFER_SIZE1, these low 4 192c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom * bits can only contain a 0 or a 1 to indicate which size buffer we're giving 193c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom * to the SGE. Thus, our software state of "is the buffer mapped for DMA" is 194c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom * maintained in an inverse sense so the hardware never sees that bit high. 195c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom */ 196c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedomenum { 197c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom RX_LARGE_BUF = 1 << 0, /* buffer is SGE_FL_BUFFER_SIZE[1] */ 198c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom RX_UNMAPPED_BUF = 1 << 1, /* buffer is not mapped */ 199c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom}; 200c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom 201c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom/** 202c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom * get_buf_addr - return DMA buffer address of software descriptor 203c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom * @sdesc: pointer to the software buffer descriptor 204c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom * 205c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom * Return the DMA buffer address of a software descriptor (stripping out 206c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom * our low-order flag bits). 207c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom */ 208c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedomstatic inline dma_addr_t get_buf_addr(const struct rx_sw_desc *sdesc) 209c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom{ 210c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom return sdesc->dma_addr & ~(dma_addr_t)(RX_LARGE_BUF | RX_UNMAPPED_BUF); 211c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom} 212c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom 213c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom/** 214c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom * is_buf_mapped - is buffer mapped for DMA? 215c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom * @sdesc: pointer to the software buffer descriptor 216c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom * 217c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom * Determine whether the buffer associated with a software descriptor in 218c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom * mapped for DMA or not. 219c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom */ 220c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedomstatic inline bool is_buf_mapped(const struct rx_sw_desc *sdesc) 221c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom{ 222c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom return !(sdesc->dma_addr & RX_UNMAPPED_BUF); 223c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom} 224c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom 225c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom/** 226c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom * need_skb_unmap - does the platform need unmapping of sk_buffs? 227c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom * 22825985edcedea6396277003854657b5f3cb31a628Lucas De Marchi * Returns true if the platform needs sk_buff unmapping. The compiler 22925985edcedea6396277003854657b5f3cb31a628Lucas De Marchi * optimizes away unnecessary code if this returns true. 230c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom */ 231c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedomstatic inline int need_skb_unmap(void) 232c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom{ 23357b2eaf7ddeae307fac202d82a6fabf5976e575bFUJITA Tomonori#ifdef CONFIG_NEED_DMA_MAP_STATE 23457b2eaf7ddeae307fac202d82a6fabf5976e575bFUJITA Tomonori return 1; 23557b2eaf7ddeae307fac202d82a6fabf5976e575bFUJITA Tomonori#else 23657b2eaf7ddeae307fac202d82a6fabf5976e575bFUJITA Tomonori return 0; 23757b2eaf7ddeae307fac202d82a6fabf5976e575bFUJITA Tomonori#endif 238c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom} 239c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom 240c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom/** 241c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom * txq_avail - return the number of available slots in a TX queue 242c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom * @tq: the TX queue 243c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom * 244c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom * Returns the number of available descriptors in a TX queue. 245c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom */ 246c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedomstatic inline unsigned int txq_avail(const struct sge_txq *tq) 247c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom{ 248c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom return tq->size - 1 - tq->in_use; 249c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom} 250c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom 251c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom/** 252c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom * fl_cap - return the capacity of a Free List 253c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom * @fl: the Free List 254c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom * 255c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom * Returns the capacity of a Free List. The capacity is less than the 256c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom * size because an Egress Queue Index Unit worth of descriptors needs to 257c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom * be left unpopulated, otherwise the Producer and Consumer indices PIDX 258c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom * and CIDX will match and the hardware will think the FL is empty. 259c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom */ 260c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedomstatic inline unsigned int fl_cap(const struct sge_fl *fl) 261c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom{ 262c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom return fl->size - FL_PER_EQ_UNIT; 263c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom} 264c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom 265c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom/** 266c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom * fl_starving - return whether a Free List is starving. 267c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom * @fl: the Free List 268c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom * 269c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom * Tests specified Free List to see whether the number of buffers 270c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom * available to the hardware has falled below our "starvation" 27125985edcedea6396277003854657b5f3cb31a628Lucas De Marchi * threshold. 272c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom */ 273c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedomstatic inline bool fl_starving(const struct sge_fl *fl) 274c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom{ 275c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom return fl->avail - fl->pend_cred <= FL_STARVE_THRES; 276c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom} 277c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom 278c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom/** 279c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom * map_skb - map an skb for DMA to the device 280c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom * @dev: the egress net device 281c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom * @skb: the packet to map 282c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom * @addr: a pointer to the base of the DMA mapping array 283c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom * 284c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom * Map an skb for DMA to the device and return an array of DMA addresses. 285c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom */ 286c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedomstatic int map_skb(struct device *dev, const struct sk_buff *skb, 287c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom dma_addr_t *addr) 288c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom{ 289c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom const skb_frag_t *fp, *end; 290c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom const struct skb_shared_info *si; 291c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom 292c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom *addr = dma_map_single(dev, skb->data, skb_headlen(skb), DMA_TO_DEVICE); 293c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom if (dma_mapping_error(dev, *addr)) 294c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom goto out_err; 295c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom 296c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom si = skb_shinfo(skb); 297c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom end = &si->frags[si->nr_frags]; 298c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom for (fp = si->frags; fp < end; fp++) { 299a0006a86cb19543f126bb2ee3d37baef82080763Ian Campbell *++addr = skb_frag_dma_map(dev, fp, 0, skb_frag_size(fp), 300a0006a86cb19543f126bb2ee3d37baef82080763Ian Campbell DMA_TO_DEVICE); 301c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom if (dma_mapping_error(dev, *addr)) 302c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom goto unwind; 303c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom } 304c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom return 0; 305c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom 306c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedomunwind: 307c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom while (fp-- > si->frags) 3089e903e085262ffbf1fc44a17ac06058aca03524aEric Dumazet dma_unmap_page(dev, *--addr, skb_frag_size(fp), DMA_TO_DEVICE); 309c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom dma_unmap_single(dev, addr[-1], skb_headlen(skb), DMA_TO_DEVICE); 310c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom 311c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedomout_err: 312c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom return -ENOMEM; 313c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom} 314c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom 315c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedomstatic void unmap_sgl(struct device *dev, const struct sk_buff *skb, 316c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom const struct ulptx_sgl *sgl, const struct sge_txq *tq) 317c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom{ 318c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom const struct ulptx_sge_pair *p; 319c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom unsigned int nfrags = skb_shinfo(skb)->nr_frags; 320c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom 321c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom if (likely(skb_headlen(skb))) 322c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom dma_unmap_single(dev, be64_to_cpu(sgl->addr0), 323c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom be32_to_cpu(sgl->len0), DMA_TO_DEVICE); 324c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom else { 325c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom dma_unmap_page(dev, be64_to_cpu(sgl->addr0), 326c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom be32_to_cpu(sgl->len0), DMA_TO_DEVICE); 327c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom nfrags--; 328c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom } 329c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom 330c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom /* 331c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom * the complexity below is because of the possibility of a wrap-around 332c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom * in the middle of an SGL 333c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom */ 334c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom for (p = sgl->sge; nfrags >= 2; nfrags -= 2) { 335c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom if (likely((u8 *)(p + 1) <= (u8 *)tq->stat)) { 336c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedomunmap: 337c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom dma_unmap_page(dev, be64_to_cpu(p->addr[0]), 338c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom be32_to_cpu(p->len[0]), DMA_TO_DEVICE); 339c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom dma_unmap_page(dev, be64_to_cpu(p->addr[1]), 340c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom be32_to_cpu(p->len[1]), DMA_TO_DEVICE); 341c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom p++; 342c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom } else if ((u8 *)p == (u8 *)tq->stat) { 343c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom p = (const struct ulptx_sge_pair *)tq->desc; 344c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom goto unmap; 345c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom } else if ((u8 *)p + 8 == (u8 *)tq->stat) { 346c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom const __be64 *addr = (const __be64 *)tq->desc; 347c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom 348c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom dma_unmap_page(dev, be64_to_cpu(addr[0]), 349c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom be32_to_cpu(p->len[0]), DMA_TO_DEVICE); 350c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom dma_unmap_page(dev, be64_to_cpu(addr[1]), 351c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom be32_to_cpu(p->len[1]), DMA_TO_DEVICE); 352c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom p = (const struct ulptx_sge_pair *)&addr[2]; 353c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom } else { 354c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom const __be64 *addr = (const __be64 *)tq->desc; 355c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom 356c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom dma_unmap_page(dev, be64_to_cpu(p->addr[0]), 357c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom be32_to_cpu(p->len[0]), DMA_TO_DEVICE); 358c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom dma_unmap_page(dev, be64_to_cpu(addr[0]), 359c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom be32_to_cpu(p->len[1]), DMA_TO_DEVICE); 360c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom p = (const struct ulptx_sge_pair *)&addr[1]; 361c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom } 362c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom } 363c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom if (nfrags) { 364c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom __be64 addr; 365c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom 366c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom if ((u8 *)p == (u8 *)tq->stat) 367c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom p = (const struct ulptx_sge_pair *)tq->desc; 368c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom addr = ((u8 *)p + 16 <= (u8 *)tq->stat 369c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom ? p->addr[0] 370c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom : *(const __be64 *)tq->desc); 371c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom dma_unmap_page(dev, be64_to_cpu(addr), be32_to_cpu(p->len[0]), 372c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom DMA_TO_DEVICE); 373c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom } 374c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom} 375c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom 376c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom/** 377c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom * free_tx_desc - reclaims TX descriptors and their buffers 378c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom * @adapter: the adapter 379c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom * @tq: the TX queue to reclaim descriptors from 380c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom * @n: the number of descriptors to reclaim 381c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom * @unmap: whether the buffers should be unmapped for DMA 382c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom * 383c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom * Reclaims TX descriptors from an SGE TX queue and frees the associated 384c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom * TX buffers. Called with the TX queue lock held. 385c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom */ 386c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedomstatic void free_tx_desc(struct adapter *adapter, struct sge_txq *tq, 387c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom unsigned int n, bool unmap) 388c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom{ 389c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom struct tx_sw_desc *sdesc; 390c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom unsigned int cidx = tq->cidx; 391c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom struct device *dev = adapter->pdev_dev; 392c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom 393c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom const int need_unmap = need_skb_unmap() && unmap; 394c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom 395c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom sdesc = &tq->sdesc[cidx]; 396c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom while (n--) { 397c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom /* 398c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom * If we kept a reference to the original TX skb, we need to 399c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom * unmap it from PCI DMA space (if required) and free it. 400c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom */ 401c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom if (sdesc->skb) { 402c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom if (need_unmap) 403c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom unmap_sgl(dev, sdesc->skb, sdesc->sgl, tq); 404c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom kfree_skb(sdesc->skb); 405c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom sdesc->skb = NULL; 406c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom } 407c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom 408c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom sdesc++; 409c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom if (++cidx == tq->size) { 410c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom cidx = 0; 411c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom sdesc = tq->sdesc; 412c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom } 413c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom } 414c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom tq->cidx = cidx; 415c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom} 416c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom 417c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom/* 418c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom * Return the number of reclaimable descriptors in a TX queue. 419c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom */ 420c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedomstatic inline int reclaimable(const struct sge_txq *tq) 421c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom{ 422c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom int hw_cidx = be16_to_cpu(tq->stat->cidx); 423c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom int reclaimable = hw_cidx - tq->cidx; 424c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom if (reclaimable < 0) 425c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom reclaimable += tq->size; 426c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom return reclaimable; 427c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom} 428c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom 429c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom/** 430c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom * reclaim_completed_tx - reclaims completed TX descriptors 431c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom * @adapter: the adapter 432c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom * @tq: the TX queue to reclaim completed descriptors from 433c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom * @unmap: whether the buffers should be unmapped for DMA 434c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom * 435c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom * Reclaims TX descriptors that the SGE has indicated it has processed, 436c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom * and frees the associated buffers if possible. Called with the TX 437c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom * queue locked. 438c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom */ 439c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedomstatic inline void reclaim_completed_tx(struct adapter *adapter, 440c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom struct sge_txq *tq, 441c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom bool unmap) 442c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom{ 443c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom int avail = reclaimable(tq); 444c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom 445c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom if (avail) { 446c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom /* 447c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom * Limit the amount of clean up work we do at a time to keep 448c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom * the TX lock hold time O(1). 449c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom */ 450c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom if (avail > MAX_TX_RECLAIM) 451c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom avail = MAX_TX_RECLAIM; 452c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom 453c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom free_tx_desc(adapter, tq, avail, unmap); 454c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom tq->in_use -= avail; 455c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom } 456c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom} 457c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom 458c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom/** 459c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom * get_buf_size - return the size of an RX Free List buffer. 460c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom * @sdesc: pointer to the software buffer descriptor 461c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom */ 462c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedomstatic inline int get_buf_size(const struct rx_sw_desc *sdesc) 463c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom{ 464c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom return FL_PG_ORDER > 0 && (sdesc->dma_addr & RX_LARGE_BUF) 465c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom ? (PAGE_SIZE << FL_PG_ORDER) 466c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom : PAGE_SIZE; 467c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom} 468c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom 469c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom/** 470c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom * free_rx_bufs - free RX buffers on an SGE Free List 471c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom * @adapter: the adapter 472c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom * @fl: the SGE Free List to free buffers from 473c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom * @n: how many buffers to free 474c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom * 475c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom * Release the next @n buffers on an SGE Free List RX queue. The 476c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom * buffers must be made inaccessible to hardware before calling this 477c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom * function. 478c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom */ 479c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedomstatic void free_rx_bufs(struct adapter *adapter, struct sge_fl *fl, int n) 480c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom{ 481c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom while (n--) { 482c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom struct rx_sw_desc *sdesc = &fl->sdesc[fl->cidx]; 483c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom 484c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom if (is_buf_mapped(sdesc)) 485c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom dma_unmap_page(adapter->pdev_dev, get_buf_addr(sdesc), 486c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom get_buf_size(sdesc), PCI_DMA_FROMDEVICE); 487c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom put_page(sdesc->page); 488c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom sdesc->page = NULL; 489c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom if (++fl->cidx == fl->size) 490c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom fl->cidx = 0; 491c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom fl->avail--; 492c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom } 493c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom} 494c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom 495c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom/** 496c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom * unmap_rx_buf - unmap the current RX buffer on an SGE Free List 497c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom * @adapter: the adapter 498c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom * @fl: the SGE Free List 499c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom * 500c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom * Unmap the current buffer on an SGE Free List RX queue. The 501c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom * buffer must be made inaccessible to HW before calling this function. 502c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom * 503c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom * This is similar to @free_rx_bufs above but does not free the buffer. 504c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom * Do note that the FL still loses any further access to the buffer. 505c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom * This is used predominantly to "transfer ownership" of an FL buffer 506c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom * to another entity (typically an skb's fragment list). 507c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom */ 508c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedomstatic void unmap_rx_buf(struct adapter *adapter, struct sge_fl *fl) 509c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom{ 510c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom struct rx_sw_desc *sdesc = &fl->sdesc[fl->cidx]; 511c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom 512c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom if (is_buf_mapped(sdesc)) 513c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom dma_unmap_page(adapter->pdev_dev, get_buf_addr(sdesc), 514c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom get_buf_size(sdesc), PCI_DMA_FROMDEVICE); 515c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom sdesc->page = NULL; 516c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom if (++fl->cidx == fl->size) 517c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom fl->cidx = 0; 518c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom fl->avail--; 519c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom} 520c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom 521c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom/** 522c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom * ring_fl_db - righ doorbell on free list 523c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom * @adapter: the adapter 524c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom * @fl: the Free List whose doorbell should be rung ... 525c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom * 526c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom * Tell the Scatter Gather Engine that there are new free list entries 527c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom * available. 528c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom */ 529c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedomstatic inline void ring_fl_db(struct adapter *adapter, struct sge_fl *fl) 530c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom{ 531c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom /* 532c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom * The SGE keeps track of its Producer and Consumer Indices in terms 533c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom * of Egress Queue Units so we can only tell it about integral numbers 534c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom * of multiples of Free List Entries per Egress Queue Units ... 535c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom */ 536c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom if (fl->pend_cred >= FL_PER_EQ_UNIT) { 537c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom wmb(); 538c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom t4_write_reg(adapter, T4VF_SGE_BASE_ADDR + SGE_VF_KDOORBELL, 539c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom DBPRIO | 540c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom QID(fl->cntxt_id) | 541c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom PIDX(fl->pend_cred / FL_PER_EQ_UNIT)); 542c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom fl->pend_cred %= FL_PER_EQ_UNIT; 543c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom } 544c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom} 545c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom 546c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom/** 547c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom * set_rx_sw_desc - initialize software RX buffer descriptor 548c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom * @sdesc: pointer to the softwore RX buffer descriptor 549c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom * @page: pointer to the page data structure backing the RX buffer 550c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom * @dma_addr: PCI DMA address (possibly with low-bit flags) 551c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom */ 552c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedomstatic inline void set_rx_sw_desc(struct rx_sw_desc *sdesc, struct page *page, 553c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom dma_addr_t dma_addr) 554c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom{ 555c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom sdesc->page = page; 556c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom sdesc->dma_addr = dma_addr; 557c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom} 558c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom 559c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom/* 560c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom * Support for poisoning RX buffers ... 561c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom */ 562c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom#define POISON_BUF_VAL -1 563c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom 564c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedomstatic inline void poison_buf(struct page *page, size_t sz) 565c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom{ 566c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom#if POISON_BUF_VAL >= 0 567c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom memset(page_address(page), POISON_BUF_VAL, sz); 568c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom#endif 569c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom} 570c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom 571c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom/** 572c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom * refill_fl - refill an SGE RX buffer ring 573c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom * @adapter: the adapter 574c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom * @fl: the Free List ring to refill 575c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom * @n: the number of new buffers to allocate 576c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom * @gfp: the gfp flags for the allocations 577c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom * 578c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom * (Re)populate an SGE free-buffer queue with up to @n new packet buffers, 579c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom * allocated with the supplied gfp flags. The caller must assure that 580c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom * @n does not exceed the queue's capacity -- i.e. (cidx == pidx) _IN 581c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom * EGRESS QUEUE UNITS_ indicates an empty Free List! Returns the number 582c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom * of buffers allocated. If afterwards the queue is found critically low, 583c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom * mark it as starving in the bitmap of starving FLs. 584c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom */ 585c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedomstatic unsigned int refill_fl(struct adapter *adapter, struct sge_fl *fl, 586c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom int n, gfp_t gfp) 587c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom{ 588c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom struct page *page; 589c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom dma_addr_t dma_addr; 590c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom unsigned int cred = fl->avail; 591c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom __be64 *d = &fl->desc[fl->pidx]; 592c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom struct rx_sw_desc *sdesc = &fl->sdesc[fl->pidx]; 593c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom 594c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom /* 595c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom * Sanity: ensure that the result of adding n Free List buffers 596c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom * won't result in wrapping the SGE's Producer Index around to 597c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom * it's Consumer Index thereby indicating an empty Free List ... 598c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom */ 599c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom BUG_ON(fl->avail + n > fl->size - FL_PER_EQ_UNIT); 600c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom 601c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom /* 602c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom * If we support large pages, prefer large buffers and fail over to 603c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom * small pages if we can't allocate large pages to satisfy the refill. 604c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom * If we don't support large pages, drop directly into the small page 605c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom * allocation code. 606c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom */ 607c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom if (FL_PG_ORDER == 0) 608c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom goto alloc_small_pages; 609c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom 610c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom while (n) { 611c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom page = alloc_pages(gfp | __GFP_COMP | __GFP_NOWARN, 612c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom FL_PG_ORDER); 613c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom if (unlikely(!page)) { 614c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom /* 615c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom * We've failed inour attempt to allocate a "large 616c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom * page". Fail over to the "small page" allocation 617c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom * below. 618c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom */ 619c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom fl->large_alloc_failed++; 620c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom break; 621c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom } 622c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom poison_buf(page, PAGE_SIZE << FL_PG_ORDER); 623c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom 624c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom dma_addr = dma_map_page(adapter->pdev_dev, page, 0, 625c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom PAGE_SIZE << FL_PG_ORDER, 626c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom PCI_DMA_FROMDEVICE); 627c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom if (unlikely(dma_mapping_error(adapter->pdev_dev, dma_addr))) { 628c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom /* 629c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom * We've run out of DMA mapping space. Free up the 630c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom * buffer and return with what we've managed to put 631c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom * into the free list. We don't want to fail over to 632c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom * the small page allocation below in this case 633c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom * because DMA mapping resources are typically 634c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom * critical resources once they become scarse. 635c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom */ 636c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom __free_pages(page, FL_PG_ORDER); 637c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom goto out; 638c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom } 639c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom dma_addr |= RX_LARGE_BUF; 640c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom *d++ = cpu_to_be64(dma_addr); 641c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom 642c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom set_rx_sw_desc(sdesc, page, dma_addr); 643c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom sdesc++; 644c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom 645c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom fl->avail++; 646c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom if (++fl->pidx == fl->size) { 647c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom fl->pidx = 0; 648c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom sdesc = fl->sdesc; 649c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom d = fl->desc; 650c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom } 651c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom n--; 652c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom } 653c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom 654c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedomalloc_small_pages: 655c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom while (n--) { 6561f2149c1df50c8c712950872675f46e6e44629f0Eric Dumazet page = alloc_page(gfp | __GFP_NOWARN | __GFP_COLD); 657c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom if (unlikely(!page)) { 658c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom fl->alloc_failed++; 659c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom break; 660c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom } 661c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom poison_buf(page, PAGE_SIZE); 662c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom 663c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom dma_addr = dma_map_page(adapter->pdev_dev, page, 0, PAGE_SIZE, 664c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom PCI_DMA_FROMDEVICE); 665c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom if (unlikely(dma_mapping_error(adapter->pdev_dev, dma_addr))) { 6661f2149c1df50c8c712950872675f46e6e44629f0Eric Dumazet put_page(page); 667c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom break; 668c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom } 669c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom *d++ = cpu_to_be64(dma_addr); 670c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom 671c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom set_rx_sw_desc(sdesc, page, dma_addr); 672c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom sdesc++; 673c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom 674c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom fl->avail++; 675c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom if (++fl->pidx == fl->size) { 676c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom fl->pidx = 0; 677c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom sdesc = fl->sdesc; 678c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom d = fl->desc; 679c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom } 680c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom } 681c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom 682c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedomout: 683c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom /* 684c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom * Update our accounting state to incorporate the new Free List 685c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom * buffers, tell the hardware about them and return the number of 68690802ed9c3dbab2e067bd9fc67a30e66e6774e8fPaul Bolle * buffers which we were able to allocate. 687c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom */ 688c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom cred = fl->avail - cred; 689c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom fl->pend_cred += cred; 690c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom ring_fl_db(adapter, fl); 691c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom 692c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom if (unlikely(fl_starving(fl))) { 693c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom smp_wmb(); 694c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom set_bit(fl->cntxt_id, adapter->sge.starving_fl); 695c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom } 696c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom 697c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom return cred; 698c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom} 699c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom 700c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom/* 701c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom * Refill a Free List to its capacity or the Maximum Refill Increment, 702c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom * whichever is smaller ... 703c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom */ 704c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedomstatic inline void __refill_fl(struct adapter *adapter, struct sge_fl *fl) 705c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom{ 706c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom refill_fl(adapter, fl, 707c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom min((unsigned int)MAX_RX_REFILL, fl_cap(fl) - fl->avail), 708c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom GFP_ATOMIC); 709c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom} 710c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom 711c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom/** 712c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom * alloc_ring - allocate resources for an SGE descriptor ring 713c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom * @dev: the PCI device's core device 714c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom * @nelem: the number of descriptors 715c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom * @hwsize: the size of each hardware descriptor 716c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom * @swsize: the size of each software descriptor 717c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom * @busaddrp: the physical PCI bus address of the allocated ring 718c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom * @swringp: return address pointer for software ring 719c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom * @stat_size: extra space in hardware ring for status information 720c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom * 721c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom * Allocates resources for an SGE descriptor ring, such as TX queues, 722c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom * free buffer lists, response queues, etc. Each SGE ring requires 723c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom * space for its hardware descriptors plus, optionally, space for software 724c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom * state associated with each hardware entry (the metadata). The function 725c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom * returns three values: the virtual address for the hardware ring (the 726c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom * return value of the function), the PCI bus address of the hardware 727c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom * ring (in *busaddrp), and the address of the software ring (in swringp). 728c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom * Both the hardware and software rings are returned zeroed out. 729c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom */ 730c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedomstatic void *alloc_ring(struct device *dev, size_t nelem, size_t hwsize, 731c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom size_t swsize, dma_addr_t *busaddrp, void *swringp, 732c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom size_t stat_size) 733c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom{ 734c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom /* 735c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom * Allocate the hardware ring and PCI DMA bus address space for said. 736c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom */ 737c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom size_t hwlen = nelem * hwsize + stat_size; 738c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom void *hwring = dma_alloc_coherent(dev, hwlen, busaddrp, GFP_KERNEL); 739c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom 740c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom if (!hwring) 741c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom return NULL; 742c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom 743c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom /* 744c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom * If the caller wants a software ring, allocate it and return a 745c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom * pointer to it in *swringp. 746c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom */ 747c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom BUG_ON((swsize != 0) != (swringp != NULL)); 748c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom if (swsize) { 749c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom void *swring = kcalloc(nelem, swsize, GFP_KERNEL); 750c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom 751c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom if (!swring) { 752c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom dma_free_coherent(dev, hwlen, hwring, *busaddrp); 753c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom return NULL; 754c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom } 755c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom *(void **)swringp = swring; 756c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom } 757c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom 758c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom /* 759c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom * Zero out the hardware ring and return its address as our function 760c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom * value. 761c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom */ 762c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom memset(hwring, 0, hwlen); 763c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom return hwring; 764c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom} 765c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom 766c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom/** 767c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom * sgl_len - calculates the size of an SGL of the given capacity 768c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom * @n: the number of SGL entries 769c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom * 770c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom * Calculates the number of flits (8-byte units) needed for a Direct 771c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom * Scatter/Gather List that can hold the given number of entries. 772c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom */ 773c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedomstatic inline unsigned int sgl_len(unsigned int n) 774c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom{ 775c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom /* 776c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom * A Direct Scatter Gather List uses 32-bit lengths and 64-bit PCI DMA 777c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom * addresses. The DSGL Work Request starts off with a 32-bit DSGL 778c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom * ULPTX header, then Length0, then Address0, then, for 1 <= i <= N, 779c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom * repeated sequences of { Length[i], Length[i+1], Address[i], 780c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom * Address[i+1] } (this ensures that all addresses are on 64-bit 781c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom * boundaries). If N is even, then Length[N+1] should be set to 0 and 782c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom * Address[N+1] is omitted. 783c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom * 784c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom * The following calculation incorporates all of the above. It's 785c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom * somewhat hard to follow but, briefly: the "+2" accounts for the 786c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom * first two flits which include the DSGL header, Length0 and 787c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom * Address0; the "(3*(n-1))/2" covers the main body of list entries (3 788c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom * flits for every pair of the remaining N) +1 if (n-1) is odd; and 789c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom * finally the "+((n-1)&1)" adds the one remaining flit needed if 790c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom * (n-1) is odd ... 791c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom */ 792c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom n--; 793c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom return (3 * n) / 2 + (n & 1) + 2; 794c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom} 795c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom 796c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom/** 797c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom * flits_to_desc - returns the num of TX descriptors for the given flits 798c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom * @flits: the number of flits 799c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom * 800c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom * Returns the number of TX descriptors needed for the supplied number 801c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom * of flits. 802c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom */ 803c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedomstatic inline unsigned int flits_to_desc(unsigned int flits) 804c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom{ 805c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom BUG_ON(flits > SGE_MAX_WR_LEN / sizeof(__be64)); 806c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom return DIV_ROUND_UP(flits, TXD_PER_EQ_UNIT); 807c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom} 808c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom 809c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom/** 810c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom * is_eth_imm - can an Ethernet packet be sent as immediate data? 811c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom * @skb: the packet 812c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom * 813c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom * Returns whether an Ethernet packet is small enough to fit completely as 814c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom * immediate data. 815c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom */ 816c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedomstatic inline int is_eth_imm(const struct sk_buff *skb) 817c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom{ 818c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom /* 819c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom * The VF Driver uses the FW_ETH_TX_PKT_VM_WR firmware Work Request 820c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom * which does not accommodate immediate data. We could dike out all 821c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom * of the support code for immediate data but that would tie our hands 822c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom * too much if we ever want to enhace the firmware. It would also 823c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom * create more differences between the PF and VF Drivers. 824c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom */ 825c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom return false; 826c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom} 827c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom 828c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom/** 829c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom * calc_tx_flits - calculate the number of flits for a packet TX WR 830c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom * @skb: the packet 831c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom * 832c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom * Returns the number of flits needed for a TX Work Request for the 833c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom * given Ethernet packet, including the needed WR and CPL headers. 834c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom */ 835c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedomstatic inline unsigned int calc_tx_flits(const struct sk_buff *skb) 836c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom{ 837c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom unsigned int flits; 838c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom 839c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom /* 840c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom * If the skb is small enough, we can pump it out as a work request 841c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom * with only immediate data. In that case we just have to have the 842c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom * TX Packet header plus the skb data in the Work Request. 843c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom */ 844c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom if (is_eth_imm(skb)) 845c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom return DIV_ROUND_UP(skb->len + sizeof(struct cpl_tx_pkt), 846c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom sizeof(__be64)); 847c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom 848c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom /* 849c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom * Otherwise, we're going to have to construct a Scatter gather list 850c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom * of the skb body and fragments. We also include the flits necessary 851c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom * for the TX Packet Work Request and CPL. We always have a firmware 852c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom * Write Header (incorporated as part of the cpl_tx_pkt_lso and 853c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom * cpl_tx_pkt structures), followed by either a TX Packet Write CPL 854c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom * message or, if we're doing a Large Send Offload, an LSO CPL message 855c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom * with an embeded TX Packet Write CPL message. 856c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom */ 857c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom flits = sgl_len(skb_shinfo(skb)->nr_frags + 1); 858c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom if (skb_shinfo(skb)->gso_size) 859c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom flits += (sizeof(struct fw_eth_tx_pkt_vm_wr) + 860c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom sizeof(struct cpl_tx_pkt_lso_core) + 861c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom sizeof(struct cpl_tx_pkt_core)) / sizeof(__be64); 862c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom else 863c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom flits += (sizeof(struct fw_eth_tx_pkt_vm_wr) + 864c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom sizeof(struct cpl_tx_pkt_core)) / sizeof(__be64); 865c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom return flits; 866c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom} 867c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom 868c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom/** 869c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom * write_sgl - populate a Scatter/Gather List for a packet 870c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom * @skb: the packet 871c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom * @tq: the TX queue we are writing into 872c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom * @sgl: starting location for writing the SGL 873c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom * @end: points right after the end of the SGL 874c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom * @start: start offset into skb main-body data to include in the SGL 875c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom * @addr: the list of DMA bus addresses for the SGL elements 876c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom * 877c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom * Generates a Scatter/Gather List for the buffers that make up a packet. 878c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom * The caller must provide adequate space for the SGL that will be written. 879c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom * The SGL includes all of the packet's page fragments and the data in its 880c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom * main body except for the first @start bytes. @pos must be 16-byte 881c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom * aligned and within a TX descriptor with available space. @end points 882c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom * write after the end of the SGL but does not account for any potential 883c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom * wrap around, i.e., @end > @tq->stat. 884c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom */ 885c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedomstatic void write_sgl(const struct sk_buff *skb, struct sge_txq *tq, 886c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom struct ulptx_sgl *sgl, u64 *end, unsigned int start, 887c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom const dma_addr_t *addr) 888c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom{ 889c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom unsigned int i, len; 890c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom struct ulptx_sge_pair *to; 891c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom const struct skb_shared_info *si = skb_shinfo(skb); 892c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom unsigned int nfrags = si->nr_frags; 893c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom struct ulptx_sge_pair buf[MAX_SKB_FRAGS / 2 + 1]; 894c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom 895c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom len = skb_headlen(skb) - start; 896c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom if (likely(len)) { 897c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom sgl->len0 = htonl(len); 898c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom sgl->addr0 = cpu_to_be64(addr[0] + start); 899c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom nfrags++; 900c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom } else { 9019e903e085262ffbf1fc44a17ac06058aca03524aEric Dumazet sgl->len0 = htonl(skb_frag_size(&si->frags[0])); 902c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom sgl->addr0 = cpu_to_be64(addr[1]); 903c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom } 904c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom 905c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom sgl->cmd_nsge = htonl(ULPTX_CMD(ULP_TX_SC_DSGL) | 906c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom ULPTX_NSGE(nfrags)); 907c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom if (likely(--nfrags == 0)) 908c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom return; 909c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom /* 910c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom * Most of the complexity below deals with the possibility we hit the 911c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom * end of the queue in the middle of writing the SGL. For this case 912c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom * only we create the SGL in a temporary buffer and then copy it. 913c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom */ 914c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom to = (u8 *)end > (u8 *)tq->stat ? buf : sgl->sge; 915c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom 916c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom for (i = (nfrags != si->nr_frags); nfrags >= 2; nfrags -= 2, to++) { 9179e903e085262ffbf1fc44a17ac06058aca03524aEric Dumazet to->len[0] = cpu_to_be32(skb_frag_size(&si->frags[i])); 9189e903e085262ffbf1fc44a17ac06058aca03524aEric Dumazet to->len[1] = cpu_to_be32(skb_frag_size(&si->frags[++i])); 919c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom to->addr[0] = cpu_to_be64(addr[i]); 920c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom to->addr[1] = cpu_to_be64(addr[++i]); 921c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom } 922c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom if (nfrags) { 9239e903e085262ffbf1fc44a17ac06058aca03524aEric Dumazet to->len[0] = cpu_to_be32(skb_frag_size(&si->frags[i])); 924c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom to->len[1] = cpu_to_be32(0); 925c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom to->addr[0] = cpu_to_be64(addr[i + 1]); 926c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom } 927c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom if (unlikely((u8 *)end > (u8 *)tq->stat)) { 928c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom unsigned int part0 = (u8 *)tq->stat - (u8 *)sgl->sge, part1; 929c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom 930c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom if (likely(part0)) 931c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom memcpy(sgl->sge, buf, part0); 932c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom part1 = (u8 *)end - (u8 *)tq->stat; 933c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom memcpy(tq->desc, (u8 *)buf + part0, part1); 934c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom end = (void *)tq->desc + part1; 935c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom } 936c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom if ((uintptr_t)end & 8) /* 0-pad to multiple of 16 */ 937c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom *(u64 *)end = 0; 938c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom} 939c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom 940c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom/** 941c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom * check_ring_tx_db - check and potentially ring a TX queue's doorbell 942c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom * @adapter: the adapter 943c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom * @tq: the TX queue 944c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom * @n: number of new descriptors to give to HW 945c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom * 946c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom * Ring the doorbel for a TX queue. 947c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom */ 948c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedomstatic inline void ring_tx_db(struct adapter *adapter, struct sge_txq *tq, 949c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom int n) 950c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom{ 951c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom /* 952c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom * Warn if we write doorbells with the wrong priority and write 953c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom * descriptors before telling HW. 954c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom */ 955c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom WARN_ON((QID(tq->cntxt_id) | PIDX(n)) & DBPRIO); 956c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom wmb(); 957c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom t4_write_reg(adapter, T4VF_SGE_BASE_ADDR + SGE_VF_KDOORBELL, 958c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom QID(tq->cntxt_id) | PIDX(n)); 959c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom} 960c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom 961c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom/** 962c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom * inline_tx_skb - inline a packet's data into TX descriptors 963c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom * @skb: the packet 964c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom * @tq: the TX queue where the packet will be inlined 965c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom * @pos: starting position in the TX queue to inline the packet 966c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom * 967c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom * Inline a packet's contents directly into TX descriptors, starting at 968c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom * the given position within the TX DMA ring. 969c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom * Most of the complexity of this operation is dealing with wrap arounds 970c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom * in the middle of the packet we want to inline. 971c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom */ 972c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedomstatic void inline_tx_skb(const struct sk_buff *skb, const struct sge_txq *tq, 973c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom void *pos) 974c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom{ 975c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom u64 *p; 976c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom int left = (void *)tq->stat - pos; 977c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom 978c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom if (likely(skb->len <= left)) { 979c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom if (likely(!skb->data_len)) 980c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom skb_copy_from_linear_data(skb, pos, skb->len); 981c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom else 982c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom skb_copy_bits(skb, 0, pos, skb->len); 983c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom pos += skb->len; 984c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom } else { 985c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom skb_copy_bits(skb, 0, pos, left); 986c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom skb_copy_bits(skb, left, tq->desc, skb->len - left); 987c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom pos = (void *)tq->desc + (skb->len - left); 988c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom } 989c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom 990c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom /* 0-pad to multiple of 16 */ 991c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom p = PTR_ALIGN(pos, 8); 992c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom if ((uintptr_t)p & 8) 993c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom *p = 0; 994c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom} 995c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom 996c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom/* 997c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom * Figure out what HW csum a packet wants and return the appropriate control 998c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom * bits. 999c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom */ 1000c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedomstatic u64 hwcsum(const struct sk_buff *skb) 1001c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom{ 1002c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom int csum_type; 1003c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom const struct iphdr *iph = ip_hdr(skb); 1004c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom 1005c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom if (iph->version == 4) { 1006c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom if (iph->protocol == IPPROTO_TCP) 1007c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom csum_type = TX_CSUM_TCPIP; 1008c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom else if (iph->protocol == IPPROTO_UDP) 1009c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom csum_type = TX_CSUM_UDPIP; 1010c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom else { 1011c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedomnocsum: 1012c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom /* 1013c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom * unknown protocol, disable HW csum 1014c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom * and hope a bad packet is detected 1015c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom */ 1016c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom return TXPKT_L4CSUM_DIS; 1017c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom } 1018c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom } else { 1019c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom /* 1020c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom * this doesn't work with extension headers 1021c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom */ 1022c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom const struct ipv6hdr *ip6h = (const struct ipv6hdr *)iph; 1023c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom 1024c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom if (ip6h->nexthdr == IPPROTO_TCP) 1025c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom csum_type = TX_CSUM_TCPIP6; 1026c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom else if (ip6h->nexthdr == IPPROTO_UDP) 1027c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom csum_type = TX_CSUM_UDPIP6; 1028c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom else 1029c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom goto nocsum; 1030c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom } 1031c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom 1032c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom if (likely(csum_type >= TX_CSUM_TCPIP)) 1033c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom return TXPKT_CSUM_TYPE(csum_type) | 1034c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom TXPKT_IPHDR_LEN(skb_network_header_len(skb)) | 1035c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom TXPKT_ETHHDR_LEN(skb_network_offset(skb) - ETH_HLEN); 1036c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom else { 1037c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom int start = skb_transport_offset(skb); 1038c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom 1039c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom return TXPKT_CSUM_TYPE(csum_type) | 1040c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom TXPKT_CSUM_START(start) | 1041c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom TXPKT_CSUM_LOC(start + skb->csum_offset); 1042c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom } 1043c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom} 1044c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom 1045c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom/* 1046c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom * Stop an Ethernet TX queue and record that state change. 1047c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom */ 1048c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedomstatic void txq_stop(struct sge_eth_txq *txq) 1049c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom{ 1050c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom netif_tx_stop_queue(txq->txq); 1051c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom txq->q.stops++; 1052c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom} 1053c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom 1054c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom/* 1055c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom * Advance our software state for a TX queue by adding n in use descriptors. 1056c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom */ 1057c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedomstatic inline void txq_advance(struct sge_txq *tq, unsigned int n) 1058c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom{ 1059c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom tq->in_use += n; 1060c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom tq->pidx += n; 1061c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom if (tq->pidx >= tq->size) 1062c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom tq->pidx -= tq->size; 1063c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom} 1064c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom 1065c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom/** 1066c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom * t4vf_eth_xmit - add a packet to an Ethernet TX queue 1067c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom * @skb: the packet 1068c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom * @dev: the egress net device 1069c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom * 1070c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom * Add a packet to an SGE Ethernet TX queue. Runs with softirqs disabled. 1071c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom */ 1072c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedomint t4vf_eth_xmit(struct sk_buff *skb, struct net_device *dev) 1073c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom{ 10747f9dd2fa4ac74d35f7e5200b76bd09533afe4e4cCasey Leedom u32 wr_mid; 1075c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom u64 cntrl, *end; 1076c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom int qidx, credits; 1077c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom unsigned int flits, ndesc; 1078c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom struct adapter *adapter; 1079c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom struct sge_eth_txq *txq; 1080c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom const struct port_info *pi; 1081c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom struct fw_eth_tx_pkt_vm_wr *wr; 1082c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom struct cpl_tx_pkt_core *cpl; 1083c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom const struct skb_shared_info *ssi; 1084c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom dma_addr_t addr[MAX_SKB_FRAGS + 1]; 1085c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom const size_t fw_hdr_copy_len = (sizeof(wr->ethmacdst) + 1086c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom sizeof(wr->ethmacsrc) + 1087c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom sizeof(wr->ethtype) + 1088c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom sizeof(wr->vlantci)); 1089c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom 1090c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom /* 1091c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom * The chip minimum packet length is 10 octets but the firmware 1092c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom * command that we are using requires that we copy the Ethernet header 1093c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom * (including the VLAN tag) into the header so we reject anything 1094c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom * smaller than that ... 1095c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom */ 1096c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom if (unlikely(skb->len < fw_hdr_copy_len)) 1097c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom goto out_free; 1098c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom 1099c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom /* 1100c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom * Figure out which TX Queue we're going to use. 1101c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom */ 1102c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom pi = netdev_priv(dev); 1103c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom adapter = pi->adapter; 1104c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom qidx = skb_get_queue_mapping(skb); 1105c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom BUG_ON(qidx >= pi->nqsets); 1106c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom txq = &adapter->sge.ethtxq[pi->first_qset + qidx]; 1107c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom 1108c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom /* 1109c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom * Take this opportunity to reclaim any TX Descriptors whose DMA 1110c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom * transfers have completed. 1111c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom */ 1112c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom reclaim_completed_tx(adapter, &txq->q, true); 1113c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom 1114c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom /* 1115c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom * Calculate the number of flits and TX Descriptors we're going to 1116c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom * need along with how many TX Descriptors will be left over after 1117c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom * we inject our Work Request. 1118c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom */ 1119c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom flits = calc_tx_flits(skb); 1120c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom ndesc = flits_to_desc(flits); 1121c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom credits = txq_avail(&txq->q) - ndesc; 1122c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom 1123c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom if (unlikely(credits < 0)) { 1124c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom /* 1125c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom * Not enough room for this packet's Work Request. Stop the 1126c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom * TX Queue and return a "busy" condition. The queue will get 1127c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom * started later on when the firmware informs us that space 1128c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom * has opened up. 1129c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom */ 1130c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom txq_stop(txq); 1131c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom dev_err(adapter->pdev_dev, 1132c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom "%s: TX ring %u full while queue awake!\n", 1133c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom dev->name, qidx); 1134c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom return NETDEV_TX_BUSY; 1135c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom } 1136c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom 1137c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom if (!is_eth_imm(skb) && 1138c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom unlikely(map_skb(adapter->pdev_dev, skb, addr) < 0)) { 1139c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom /* 1140c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom * We need to map the skb into PCI DMA space (because it can't 1141c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom * be in-lined directly into the Work Request) and the mapping 1142c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom * operation failed. Record the error and drop the packet. 1143c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom */ 1144c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom txq->mapping_err++; 1145c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom goto out_free; 1146c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom } 1147c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom 11487f9dd2fa4ac74d35f7e5200b76bd09533afe4e4cCasey Leedom wr_mid = FW_WR_LEN16(DIV_ROUND_UP(flits, 2)); 1149c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom if (unlikely(credits < ETHTXQ_STOP_THRES)) { 1150c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom /* 1151c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom * After we're done injecting the Work Request for this 115225985edcedea6396277003854657b5f3cb31a628Lucas De Marchi * packet, we'll be below our "stop threshold" so stop the TX 11537f9dd2fa4ac74d35f7e5200b76bd09533afe4e4cCasey Leedom * Queue now and schedule a request for an SGE Egress Queue 11547f9dd2fa4ac74d35f7e5200b76bd09533afe4e4cCasey Leedom * Update message. The queue will get started later on when 11557f9dd2fa4ac74d35f7e5200b76bd09533afe4e4cCasey Leedom * the firmware processes this Work Request and sends us an 11567f9dd2fa4ac74d35f7e5200b76bd09533afe4e4cCasey Leedom * Egress Queue Status Update message indicating that space 11577f9dd2fa4ac74d35f7e5200b76bd09533afe4e4cCasey Leedom * has opened up. 1158c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom */ 1159c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom txq_stop(txq); 11607f9dd2fa4ac74d35f7e5200b76bd09533afe4e4cCasey Leedom wr_mid |= FW_WR_EQUEQ | FW_WR_EQUIQ; 1161c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom } 1162c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom 1163c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom /* 1164c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom * Start filling in our Work Request. Note that we do _not_ handle 1165c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom * the WR Header wrapping around the TX Descriptor Ring. If our 1166c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom * maximum header size ever exceeds one TX Descriptor, we'll need to 1167c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom * do something else here. 1168c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom */ 1169c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom BUG_ON(DIV_ROUND_UP(ETHTXQ_MAX_HDR, TXD_PER_EQ_UNIT) > 1); 1170c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom wr = (void *)&txq->q.desc[txq->q.pidx]; 11717f9dd2fa4ac74d35f7e5200b76bd09533afe4e4cCasey Leedom wr->equiq_to_len16 = cpu_to_be32(wr_mid); 1172c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom wr->r3[0] = cpu_to_be64(0); 1173c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom wr->r3[1] = cpu_to_be64(0); 1174c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom skb_copy_from_linear_data(skb, (void *)wr->ethmacdst, fw_hdr_copy_len); 1175c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom end = (u64 *)wr + flits; 1176c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom 1177c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom /* 1178c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom * If this is a Large Send Offload packet we'll put in an LSO CPL 1179c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom * message with an encapsulated TX Packet CPL message. Otherwise we 1180c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom * just use a TX Packet CPL message. 1181c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom */ 1182c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom ssi = skb_shinfo(skb); 1183c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom if (ssi->gso_size) { 1184c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom struct cpl_tx_pkt_lso_core *lso = (void *)(wr + 1); 1185c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom bool v6 = (ssi->gso_type & SKB_GSO_TCPV6) != 0; 1186c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom int l3hdr_len = skb_network_header_len(skb); 1187c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom int eth_xtra_len = skb_network_offset(skb) - ETH_HLEN; 1188c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom 1189c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom wr->op_immdlen = 1190c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom cpu_to_be32(FW_WR_OP(FW_ETH_TX_PKT_VM_WR) | 1191c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom FW_WR_IMMDLEN(sizeof(*lso) + 1192c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom sizeof(*cpl))); 1193c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom /* 1194c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom * Fill in the LSO CPL message. 1195c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom */ 1196c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom lso->lso_ctrl = 1197c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom cpu_to_be32(LSO_OPCODE(CPL_TX_PKT_LSO) | 1198c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom LSO_FIRST_SLICE | 1199c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom LSO_LAST_SLICE | 1200c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom LSO_IPV6(v6) | 1201c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom LSO_ETHHDR_LEN(eth_xtra_len/4) | 1202c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom LSO_IPHDR_LEN(l3hdr_len/4) | 1203c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom LSO_TCPHDR_LEN(tcp_hdr(skb)->doff)); 1204c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom lso->ipid_ofst = cpu_to_be16(0); 1205c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom lso->mss = cpu_to_be16(ssi->gso_size); 1206c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom lso->seqno_offset = cpu_to_be32(0); 1207c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom lso->len = cpu_to_be32(skb->len); 1208c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom 1209c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom /* 1210c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom * Set up TX Packet CPL pointer, control word and perform 1211c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom * accounting. 1212c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom */ 1213c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom cpl = (void *)(lso + 1); 1214c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom cntrl = (TXPKT_CSUM_TYPE(v6 ? TX_CSUM_TCPIP6 : TX_CSUM_TCPIP) | 1215c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom TXPKT_IPHDR_LEN(l3hdr_len) | 1216c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom TXPKT_ETHHDR_LEN(eth_xtra_len)); 1217c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom txq->tso++; 1218c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom txq->tx_cso += ssi->gso_segs; 1219c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom } else { 1220c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom int len; 1221c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom 1222c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom len = is_eth_imm(skb) ? skb->len + sizeof(*cpl) : sizeof(*cpl); 1223c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom wr->op_immdlen = 1224c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom cpu_to_be32(FW_WR_OP(FW_ETH_TX_PKT_VM_WR) | 1225c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom FW_WR_IMMDLEN(len)); 1226c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom 1227c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom /* 1228c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom * Set up TX Packet CPL pointer, control word and perform 1229c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom * accounting. 1230c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom */ 1231c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom cpl = (void *)(wr + 1); 1232c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom if (skb->ip_summed == CHECKSUM_PARTIAL) { 1233c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom cntrl = hwcsum(skb) | TXPKT_IPCSUM_DIS; 1234c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom txq->tx_cso++; 1235c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom } else 1236c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom cntrl = TXPKT_L4CSUM_DIS | TXPKT_IPCSUM_DIS; 1237c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom } 1238c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom 1239c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom /* 1240c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom * If there's a VLAN tag present, add that to the list of things to 1241c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom * do in this Work Request. 1242c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom */ 1243c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom if (vlan_tx_tag_present(skb)) { 1244c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom txq->vlan_ins++; 1245c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom cntrl |= TXPKT_VLAN_VLD | TXPKT_VLAN(vlan_tx_tag_get(skb)); 1246c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom } 1247c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom 1248c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom /* 1249c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom * Fill in the TX Packet CPL message header. 1250c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom */ 1251c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom cpl->ctrl0 = cpu_to_be32(TXPKT_OPCODE(CPL_TX_PKT_XT) | 1252c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom TXPKT_INTF(pi->port_id) | 1253c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom TXPKT_PF(0)); 1254c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom cpl->pack = cpu_to_be16(0); 1255c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom cpl->len = cpu_to_be16(skb->len); 1256c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom cpl->ctrl1 = cpu_to_be64(cntrl); 1257c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom 1258c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom#ifdef T4_TRACE 1259c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom T4_TRACE5(adapter->tb[txq->q.cntxt_id & 7], 1260c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom "eth_xmit: ndesc %u, credits %u, pidx %u, len %u, frags %u", 1261c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom ndesc, credits, txq->q.pidx, skb->len, ssi->nr_frags); 1262c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom#endif 1263c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom 1264c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom /* 1265c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom * Fill in the body of the TX Packet CPL message with either in-lined 1266c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom * data or a Scatter/Gather List. 1267c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom */ 1268c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom if (is_eth_imm(skb)) { 1269c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom /* 1270c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom * In-line the packet's data and free the skb since we don't 1271c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom * need it any longer. 1272c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom */ 1273c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom inline_tx_skb(skb, &txq->q, cpl + 1); 1274c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom dev_kfree_skb(skb); 1275c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom } else { 1276c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom /* 1277c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom * Write the skb's Scatter/Gather list into the TX Packet CPL 1278c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom * message and retain a pointer to the skb so we can free it 1279c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom * later when its DMA completes. (We store the skb pointer 1280c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom * in the Software Descriptor corresponding to the last TX 1281c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom * Descriptor used by the Work Request.) 1282c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom * 1283c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom * The retained skb will be freed when the corresponding TX 1284c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom * Descriptors are reclaimed after their DMAs complete. 1285c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom * However, this could take quite a while since, in general, 1286c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom * the hardware is set up to be lazy about sending DMA 1287c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom * completion notifications to us and we mostly perform TX 1288c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom * reclaims in the transmit routine. 1289c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom * 1290c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom * This is good for performamce but means that we rely on new 1291c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom * TX packets arriving to run the destructors of completed 1292c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom * packets, which open up space in their sockets' send queues. 1293c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom * Sometimes we do not get such new packets causing TX to 1294c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom * stall. A single UDP transmitter is a good example of this 1295c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom * situation. We have a clean up timer that periodically 1296c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom * reclaims completed packets but it doesn't run often enough 1297c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom * (nor do we want it to) to prevent lengthy stalls. A 1298c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom * solution to this problem is to run the destructor early, 1299c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom * after the packet is queued but before it's DMAd. A con is 1300c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom * that we lie to socket memory accounting, but the amount of 1301c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom * extra memory is reasonable (limited by the number of TX 1302c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom * descriptors), the packets do actually get freed quickly by 1303c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom * new packets almost always, and for protocols like TCP that 1304c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom * wait for acks to really free up the data the extra memory 1305c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom * is even less. On the positive side we run the destructors 1306c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom * on the sending CPU rather than on a potentially different 130764bb336c8f4de8b281d0d44f2ec2c900b9b28466Casey Leedom * completing CPU, usually a good thing. 1308c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom * 1309c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom * Run the destructor before telling the DMA engine about the 1310c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom * packet to make sure it doesn't complete and get freed 1311c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom * prematurely. 1312c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom */ 1313c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom struct ulptx_sgl *sgl = (struct ulptx_sgl *)(cpl + 1); 1314c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom struct sge_txq *tq = &txq->q; 1315c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom int last_desc; 1316c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom 1317c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom /* 1318c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom * If the Work Request header was an exact multiple of our TX 1319c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom * Descriptor length, then it's possible that the starting SGL 1320c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom * pointer lines up exactly with the end of our TX Descriptor 1321c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom * ring. If that's the case, wrap around to the beginning 1322c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom * here ... 1323c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom */ 1324c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom if (unlikely((void *)sgl == (void *)tq->stat)) { 1325c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom sgl = (void *)tq->desc; 1326c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom end = (void *)((void *)tq->desc + 1327c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom ((void *)end - (void *)tq->stat)); 1328c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom } 1329c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom 1330c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom write_sgl(skb, tq, sgl, end, 0, addr); 1331c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom skb_orphan(skb); 1332c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom 1333c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom last_desc = tq->pidx + ndesc - 1; 1334c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom if (last_desc >= tq->size) 1335c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom last_desc -= tq->size; 1336c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom tq->sdesc[last_desc].skb = skb; 1337c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom tq->sdesc[last_desc].sgl = sgl; 1338c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom } 1339c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom 1340c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom /* 1341c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom * Advance our internal TX Queue state, tell the hardware about 1342c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom * the new TX descriptors and return success. 1343c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom */ 1344c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom txq_advance(&txq->q, ndesc); 1345c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom dev->trans_start = jiffies; 1346c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom ring_tx_db(adapter, &txq->q, ndesc); 1347c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom return NETDEV_TX_OK; 1348c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom 1349c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedomout_free: 1350c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom /* 1351c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom * An error of some sort happened. Free the TX skb and tell the 1352c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom * OS that we've "dealt" with the packet ... 1353c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom */ 1354c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom dev_kfree_skb(skb); 1355c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom return NETDEV_TX_OK; 1356c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom} 1357c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom 1358c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom/** 1359a0006a86cb19543f126bb2ee3d37baef82080763Ian Campbell * copy_frags - copy fragments from gather list into skb_shared_info 1360a0006a86cb19543f126bb2ee3d37baef82080763Ian Campbell * @skb: destination skb 1361a0006a86cb19543f126bb2ee3d37baef82080763Ian Campbell * @gl: source internal packet gather list 1362a0006a86cb19543f126bb2ee3d37baef82080763Ian Campbell * @offset: packet start offset in first page 1363a0006a86cb19543f126bb2ee3d37baef82080763Ian Campbell * 1364a0006a86cb19543f126bb2ee3d37baef82080763Ian Campbell * Copy an internal packet gather list into a Linux skb_shared_info 1365a0006a86cb19543f126bb2ee3d37baef82080763Ian Campbell * structure. 1366a0006a86cb19543f126bb2ee3d37baef82080763Ian Campbell */ 1367a0006a86cb19543f126bb2ee3d37baef82080763Ian Campbellstatic inline void copy_frags(struct sk_buff *skb, 1368a0006a86cb19543f126bb2ee3d37baef82080763Ian Campbell const struct pkt_gl *gl, 1369a0006a86cb19543f126bb2ee3d37baef82080763Ian Campbell unsigned int offset) 1370a0006a86cb19543f126bb2ee3d37baef82080763Ian Campbell{ 1371a0006a86cb19543f126bb2ee3d37baef82080763Ian Campbell int i; 1372a0006a86cb19543f126bb2ee3d37baef82080763Ian Campbell 1373a0006a86cb19543f126bb2ee3d37baef82080763Ian Campbell /* usually there's just one frag */ 1374a0006a86cb19543f126bb2ee3d37baef82080763Ian Campbell __skb_fill_page_desc(skb, 0, gl->frags[0].page, 1375a0006a86cb19543f126bb2ee3d37baef82080763Ian Campbell gl->frags[0].offset + offset, 1376a0006a86cb19543f126bb2ee3d37baef82080763Ian Campbell gl->frags[0].size - offset); 1377a0006a86cb19543f126bb2ee3d37baef82080763Ian Campbell skb_shinfo(skb)->nr_frags = gl->nfrags; 1378a0006a86cb19543f126bb2ee3d37baef82080763Ian Campbell for (i = 1; i < gl->nfrags; i++) 1379a0006a86cb19543f126bb2ee3d37baef82080763Ian Campbell __skb_fill_page_desc(skb, i, gl->frags[i].page, 1380a0006a86cb19543f126bb2ee3d37baef82080763Ian Campbell gl->frags[i].offset, 1381a0006a86cb19543f126bb2ee3d37baef82080763Ian Campbell gl->frags[i].size); 1382a0006a86cb19543f126bb2ee3d37baef82080763Ian Campbell 1383a0006a86cb19543f126bb2ee3d37baef82080763Ian Campbell /* get a reference to the last page, we don't own it */ 1384a0006a86cb19543f126bb2ee3d37baef82080763Ian Campbell get_page(gl->frags[gl->nfrags - 1].page); 1385a0006a86cb19543f126bb2ee3d37baef82080763Ian Campbell} 1386a0006a86cb19543f126bb2ee3d37baef82080763Ian Campbell 1387a0006a86cb19543f126bb2ee3d37baef82080763Ian Campbell/** 1388eb6c503db3619c43b5aad0ced82e9d91c034ced5Casey Leedom * t4vf_pktgl_to_skb - build an sk_buff from a packet gather list 1389eb6c503db3619c43b5aad0ced82e9d91c034ced5Casey Leedom * @gl: the gather list 1390eb6c503db3619c43b5aad0ced82e9d91c034ced5Casey Leedom * @skb_len: size of sk_buff main body if it carries fragments 1391eb6c503db3619c43b5aad0ced82e9d91c034ced5Casey Leedom * @pull_len: amount of data to move to the sk_buff's main body 1392eb6c503db3619c43b5aad0ced82e9d91c034ced5Casey Leedom * 1393eb6c503db3619c43b5aad0ced82e9d91c034ced5Casey Leedom * Builds an sk_buff from the given packet gather list. Returns the 1394eb6c503db3619c43b5aad0ced82e9d91c034ced5Casey Leedom * sk_buff or %NULL if sk_buff allocation failed. 1395eb6c503db3619c43b5aad0ced82e9d91c034ced5Casey Leedom */ 1396eb6c503db3619c43b5aad0ced82e9d91c034ced5Casey Leedomstruct sk_buff *t4vf_pktgl_to_skb(const struct pkt_gl *gl, 1397eb6c503db3619c43b5aad0ced82e9d91c034ced5Casey Leedom unsigned int skb_len, unsigned int pull_len) 1398eb6c503db3619c43b5aad0ced82e9d91c034ced5Casey Leedom{ 1399eb6c503db3619c43b5aad0ced82e9d91c034ced5Casey Leedom struct sk_buff *skb; 1400eb6c503db3619c43b5aad0ced82e9d91c034ced5Casey Leedom 1401eb6c503db3619c43b5aad0ced82e9d91c034ced5Casey Leedom /* 1402eb6c503db3619c43b5aad0ced82e9d91c034ced5Casey Leedom * If the ingress packet is small enough, allocate an skb large enough 1403eb6c503db3619c43b5aad0ced82e9d91c034ced5Casey Leedom * for all of the data and copy it inline. Otherwise, allocate an skb 1404eb6c503db3619c43b5aad0ced82e9d91c034ced5Casey Leedom * with enough room to pull in the header and reference the rest of 1405eb6c503db3619c43b5aad0ced82e9d91c034ced5Casey Leedom * the data via the skb fragment list. 1406eb6c503db3619c43b5aad0ced82e9d91c034ced5Casey Leedom * 1407eb6c503db3619c43b5aad0ced82e9d91c034ced5Casey Leedom * Below we rely on RX_COPY_THRES being less than the smallest Rx 1408eb6c503db3619c43b5aad0ced82e9d91c034ced5Casey Leedom * buff! size, which is expected since buffers are at least 1409eb6c503db3619c43b5aad0ced82e9d91c034ced5Casey Leedom * PAGE_SIZEd. In this case packets up to RX_COPY_THRES have only one 1410eb6c503db3619c43b5aad0ced82e9d91c034ced5Casey Leedom * fragment. 1411eb6c503db3619c43b5aad0ced82e9d91c034ced5Casey Leedom */ 1412eb6c503db3619c43b5aad0ced82e9d91c034ced5Casey Leedom if (gl->tot_len <= RX_COPY_THRES) { 1413eb6c503db3619c43b5aad0ced82e9d91c034ced5Casey Leedom /* small packets have only one fragment */ 1414eb6c503db3619c43b5aad0ced82e9d91c034ced5Casey Leedom skb = alloc_skb(gl->tot_len, GFP_ATOMIC); 1415eb6c503db3619c43b5aad0ced82e9d91c034ced5Casey Leedom if (unlikely(!skb)) 1416eb6c503db3619c43b5aad0ced82e9d91c034ced5Casey Leedom goto out; 1417eb6c503db3619c43b5aad0ced82e9d91c034ced5Casey Leedom __skb_put(skb, gl->tot_len); 1418eb6c503db3619c43b5aad0ced82e9d91c034ced5Casey Leedom skb_copy_to_linear_data(skb, gl->va, gl->tot_len); 1419eb6c503db3619c43b5aad0ced82e9d91c034ced5Casey Leedom } else { 1420eb6c503db3619c43b5aad0ced82e9d91c034ced5Casey Leedom skb = alloc_skb(skb_len, GFP_ATOMIC); 1421eb6c503db3619c43b5aad0ced82e9d91c034ced5Casey Leedom if (unlikely(!skb)) 1422eb6c503db3619c43b5aad0ced82e9d91c034ced5Casey Leedom goto out; 1423eb6c503db3619c43b5aad0ced82e9d91c034ced5Casey Leedom __skb_put(skb, pull_len); 1424eb6c503db3619c43b5aad0ced82e9d91c034ced5Casey Leedom skb_copy_to_linear_data(skb, gl->va, pull_len); 1425eb6c503db3619c43b5aad0ced82e9d91c034ced5Casey Leedom 1426a0006a86cb19543f126bb2ee3d37baef82080763Ian Campbell copy_frags(skb, gl, pull_len); 1427eb6c503db3619c43b5aad0ced82e9d91c034ced5Casey Leedom skb->len = gl->tot_len; 1428eb6c503db3619c43b5aad0ced82e9d91c034ced5Casey Leedom skb->data_len = skb->len - pull_len; 1429eb6c503db3619c43b5aad0ced82e9d91c034ced5Casey Leedom skb->truesize += skb->data_len; 1430eb6c503db3619c43b5aad0ced82e9d91c034ced5Casey Leedom } 1431eb6c503db3619c43b5aad0ced82e9d91c034ced5Casey Leedom 1432eb6c503db3619c43b5aad0ced82e9d91c034ced5Casey Leedomout: 1433eb6c503db3619c43b5aad0ced82e9d91c034ced5Casey Leedom return skb; 1434eb6c503db3619c43b5aad0ced82e9d91c034ced5Casey Leedom} 1435eb6c503db3619c43b5aad0ced82e9d91c034ced5Casey Leedom 1436eb6c503db3619c43b5aad0ced82e9d91c034ced5Casey Leedom/** 1437c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom * t4vf_pktgl_free - free a packet gather list 1438c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom * @gl: the gather list 1439c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom * 1440c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom * Releases the pages of a packet gather list. We do not own the last 1441c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom * page on the list and do not free it. 1442c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom */ 1443c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedomvoid t4vf_pktgl_free(const struct pkt_gl *gl) 1444c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom{ 1445c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom int frag; 1446c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom 1447c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom frag = gl->nfrags - 1; 1448c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom while (frag--) 1449c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom put_page(gl->frags[frag].page); 1450c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom} 1451c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom 1452c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom/** 1453c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom * do_gro - perform Generic Receive Offload ingress packet processing 1454c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom * @rxq: ingress RX Ethernet Queue 1455c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom * @gl: gather list for ingress packet 1456c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom * @pkt: CPL header for last packet fragment 1457c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom * 1458c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom * Perform Generic Receive Offload (GRO) ingress packet processing. 1459c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom * We use the standard Linux GRO interfaces for this. 1460c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom */ 1461c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedomstatic void do_gro(struct sge_eth_rxq *rxq, const struct pkt_gl *gl, 1462c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom const struct cpl_rx_pkt *pkt) 1463c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom{ 1464c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom int ret; 1465c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom struct sk_buff *skb; 1466c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom 1467c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom skb = napi_get_frags(&rxq->rspq.napi); 1468c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom if (unlikely(!skb)) { 1469c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom t4vf_pktgl_free(gl); 1470c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom rxq->stats.rx_drops++; 1471c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom return; 1472c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom } 1473c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom 1474a0006a86cb19543f126bb2ee3d37baef82080763Ian Campbell copy_frags(skb, gl, PKTSHIFT); 1475c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom skb->len = gl->tot_len - PKTSHIFT; 1476c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom skb->data_len = skb->len; 1477c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom skb->truesize += skb->data_len; 1478c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom skb->ip_summed = CHECKSUM_UNNECESSARY; 1479c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom skb_record_rx_queue(skb, rxq->rspq.idx); 1480c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom 148187737663d5d7668c8553c572a5a7c26e86c5842bJiri Pirko if (pkt->vlan_ex) 148287737663d5d7668c8553c572a5a7c26e86c5842bJiri Pirko __vlan_hwaccel_put_tag(skb, be16_to_cpu(pkt->vlan)); 1483c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom ret = napi_gro_frags(&rxq->rspq.napi); 1484c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom 1485c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom if (ret == GRO_HELD) 1486c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom rxq->stats.lro_pkts++; 1487c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom else if (ret == GRO_MERGED || ret == GRO_MERGED_FREE) 1488c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom rxq->stats.lro_merged++; 1489c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom rxq->stats.pkts++; 1490c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom rxq->stats.rx_cso++; 1491c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom} 1492c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom 1493c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom/** 1494c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom * t4vf_ethrx_handler - process an ingress ethernet packet 1495c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom * @rspq: the response queue that received the packet 1496c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom * @rsp: the response queue descriptor holding the RX_PKT message 1497c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom * @gl: the gather list of packet fragments 1498c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom * 1499c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom * Process an ingress ethernet packet and deliver it to the stack. 1500c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom */ 1501c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedomint t4vf_ethrx_handler(struct sge_rspq *rspq, const __be64 *rsp, 1502c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom const struct pkt_gl *gl) 1503c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom{ 1504c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom struct sk_buff *skb; 1505c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom const struct cpl_rx_pkt *pkt = (void *)&rsp[1]; 1506c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom bool csum_ok = pkt->csum_calc && !pkt->err_vec; 1507c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom struct sge_eth_rxq *rxq = container_of(rspq, struct sge_eth_rxq, rspq); 1508c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom 1509c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom /* 1510c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom * If this is a good TCP packet and we have Generic Receive Offload 1511c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom * enabled, handle the packet in the GRO path. 1512c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom */ 1513c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom if ((pkt->l2info & cpu_to_be32(RXF_TCP)) && 1514c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom (rspq->netdev->features & NETIF_F_GRO) && csum_ok && 1515c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom !pkt->ip_frag) { 1516c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom do_gro(rxq, gl, pkt); 1517c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom return 0; 1518c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom } 1519c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom 1520c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom /* 1521eb6c503db3619c43b5aad0ced82e9d91c034ced5Casey Leedom * Convert the Packet Gather List into an skb. 1522c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom */ 1523eb6c503db3619c43b5aad0ced82e9d91c034ced5Casey Leedom skb = t4vf_pktgl_to_skb(gl, RX_SKB_LEN, RX_PULL_LEN); 1524eb6c503db3619c43b5aad0ced82e9d91c034ced5Casey Leedom if (unlikely(!skb)) { 1525eb6c503db3619c43b5aad0ced82e9d91c034ced5Casey Leedom t4vf_pktgl_free(gl); 1526eb6c503db3619c43b5aad0ced82e9d91c034ced5Casey Leedom rxq->stats.rx_drops++; 1527eb6c503db3619c43b5aad0ced82e9d91c034ced5Casey Leedom return 0; 1528c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom } 1529c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom __skb_pull(skb, PKTSHIFT); 1530c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom skb->protocol = eth_type_trans(skb, rspq->netdev); 1531c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom skb_record_rx_queue(skb, rspq->idx); 1532c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom rxq->stats.pkts++; 1533c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom 15342ed28baa7076083b56c1e70ccd927b7870117c59Michał Mirosław if (csum_ok && (rspq->netdev->features & NETIF_F_RXCSUM) && 15352ed28baa7076083b56c1e70ccd927b7870117c59Michał Mirosław !pkt->err_vec && (be32_to_cpu(pkt->l2info) & (RXF_UDP|RXF_TCP))) { 1536c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom if (!pkt->ip_frag) 1537c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom skb->ip_summed = CHECKSUM_UNNECESSARY; 1538c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom else { 1539c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom __sum16 c = (__force __sum16)pkt->csum; 1540c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom skb->csum = csum_unfold(c); 1541c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom skb->ip_summed = CHECKSUM_COMPLETE; 1542c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom } 1543c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom rxq->stats.rx_cso++; 1544c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom } else 1545bc8acf2c8c3e43fcc192762a9f964b3e9a17748bEric Dumazet skb_checksum_none_assert(skb); 1546c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom 154787737663d5d7668c8553c572a5a7c26e86c5842bJiri Pirko if (pkt->vlan_ex) { 1548c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom rxq->stats.vlan_ex++; 154987737663d5d7668c8553c572a5a7c26e86c5842bJiri Pirko __vlan_hwaccel_put_tag(skb, be16_to_cpu(pkt->vlan)); 155087737663d5d7668c8553c572a5a7c26e86c5842bJiri Pirko } 155187737663d5d7668c8553c572a5a7c26e86c5842bJiri Pirko 155287737663d5d7668c8553c572a5a7c26e86c5842bJiri Pirko netif_receive_skb(skb); 1553c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom 1554c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom return 0; 1555c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom} 1556c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom 1557c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom/** 1558c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom * is_new_response - check if a response is newly written 1559c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom * @rc: the response control descriptor 1560c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom * @rspq: the response queue 1561c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom * 1562c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom * Returns true if a response descriptor contains a yet unprocessed 1563c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom * response. 1564c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom */ 1565c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedomstatic inline bool is_new_response(const struct rsp_ctrl *rc, 1566c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom const struct sge_rspq *rspq) 1567c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom{ 1568c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom return RSPD_GEN(rc->type_gen) == rspq->gen; 1569c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom} 1570c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom 1571c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom/** 1572c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom * restore_rx_bufs - put back a packet's RX buffers 1573c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom * @gl: the packet gather list 1574c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom * @fl: the SGE Free List 1575c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom * @nfrags: how many fragments in @si 1576c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom * 1577c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom * Called when we find out that the current packet, @si, can't be 1578c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom * processed right away for some reason. This is a very rare event and 1579c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom * there's no effort to make this suspension/resumption process 1580c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom * particularly efficient. 1581c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom * 1582c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom * We implement the suspension by putting all of the RX buffers associated 1583c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom * with the current packet back on the original Free List. The buffers 1584c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom * have already been unmapped and are left unmapped, we mark them as 1585c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom * unmapped in order to prevent further unmapping attempts. (Effectively 1586c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom * this function undoes the series of @unmap_rx_buf calls which were done 1587c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom * to create the current packet's gather list.) This leaves us ready to 1588c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom * restart processing of the packet the next time we start processing the 1589c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom * RX Queue ... 1590c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom */ 1591c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedomstatic void restore_rx_bufs(const struct pkt_gl *gl, struct sge_fl *fl, 1592c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom int frags) 1593c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom{ 1594c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom struct rx_sw_desc *sdesc; 1595c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom 1596c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom while (frags--) { 1597c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom if (fl->cidx == 0) 1598c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom fl->cidx = fl->size - 1; 1599c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom else 1600c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom fl->cidx--; 1601c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom sdesc = &fl->sdesc[fl->cidx]; 1602c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom sdesc->page = gl->frags[frags].page; 1603c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom sdesc->dma_addr |= RX_UNMAPPED_BUF; 1604c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom fl->avail++; 1605c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom } 1606c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom} 1607c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom 1608c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom/** 1609c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom * rspq_next - advance to the next entry in a response queue 1610c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom * @rspq: the queue 1611c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom * 1612c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom * Updates the state of a response queue to advance it to the next entry. 1613c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom */ 1614c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedomstatic inline void rspq_next(struct sge_rspq *rspq) 1615c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom{ 1616c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom rspq->cur_desc = (void *)rspq->cur_desc + rspq->iqe_len; 1617c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom if (unlikely(++rspq->cidx == rspq->size)) { 1618c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom rspq->cidx = 0; 1619c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom rspq->gen ^= 1; 1620c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom rspq->cur_desc = rspq->desc; 1621c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom } 1622c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom} 1623c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom 1624c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom/** 1625c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom * process_responses - process responses from an SGE response queue 1626c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom * @rspq: the ingress response queue to process 1627c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom * @budget: how many responses can be processed in this round 1628c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom * 1629c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom * Process responses from a Scatter Gather Engine response queue up to 1630c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom * the supplied budget. Responses include received packets as well as 1631c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom * control messages from firmware or hardware. 1632c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom * 1633c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom * Additionally choose the interrupt holdoff time for the next interrupt 1634c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom * on this queue. If the system is under memory shortage use a fairly 1635c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom * long delay to help recovery. 1636c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom */ 1637c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedomint process_responses(struct sge_rspq *rspq, int budget) 1638c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom{ 1639c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom struct sge_eth_rxq *rxq = container_of(rspq, struct sge_eth_rxq, rspq); 1640c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom int budget_left = budget; 1641c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom 1642c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom while (likely(budget_left)) { 1643c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom int ret, rsp_type; 1644c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom const struct rsp_ctrl *rc; 1645c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom 1646c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom rc = (void *)rspq->cur_desc + (rspq->iqe_len - sizeof(*rc)); 1647c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom if (!is_new_response(rc, rspq)) 1648c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom break; 1649c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom 1650c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom /* 1651c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom * Figure out what kind of response we've received from the 1652c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom * SGE. 1653c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom */ 1654c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom rmb(); 1655c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom rsp_type = RSPD_TYPE(rc->type_gen); 1656c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom if (likely(rsp_type == RSP_TYPE_FLBUF)) { 1657a0006a86cb19543f126bb2ee3d37baef82080763Ian Campbell struct page_frag *fp; 1658c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom struct pkt_gl gl; 1659c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom const struct rx_sw_desc *sdesc; 1660c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom u32 bufsz, frag; 1661c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom u32 len = be32_to_cpu(rc->pldbuflen_qid); 1662c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom 1663c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom /* 1664c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom * If we get a "new buffer" message from the SGE we 1665c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom * need to move on to the next Free List buffer. 1666c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom */ 1667c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom if (len & RSPD_NEWBUF) { 1668c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom /* 1669c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom * We get one "new buffer" message when we 1670c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom * first start up a queue so we need to ignore 1671c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom * it when our offset into the buffer is 0. 1672c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom */ 1673c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom if (likely(rspq->offset > 0)) { 1674c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom free_rx_bufs(rspq->adapter, &rxq->fl, 1675c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom 1); 1676c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom rspq->offset = 0; 1677c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom } 1678c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom len = RSPD_LEN(len); 1679c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom } 1680b94e72e23e3ededa23cf18973ffc9a03c7ea8d04Casey Leedom gl.tot_len = len; 1681c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom 1682c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom /* 1683c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom * Gather packet fragments. 1684c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom */ 1685c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom for (frag = 0, fp = gl.frags; /**/; frag++, fp++) { 1686c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom BUG_ON(frag >= MAX_SKB_FRAGS); 1687c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom BUG_ON(rxq->fl.avail == 0); 1688c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom sdesc = &rxq->fl.sdesc[rxq->fl.cidx]; 1689c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom bufsz = get_buf_size(sdesc); 1690c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom fp->page = sdesc->page; 1691a0006a86cb19543f126bb2ee3d37baef82080763Ian Campbell fp->offset = rspq->offset; 1692a0006a86cb19543f126bb2ee3d37baef82080763Ian Campbell fp->size = min(bufsz, len); 1693a0006a86cb19543f126bb2ee3d37baef82080763Ian Campbell len -= fp->size; 1694c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom if (!len) 1695c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom break; 1696c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom unmap_rx_buf(rspq->adapter, &rxq->fl); 1697c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom } 1698c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom gl.nfrags = frag+1; 1699c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom 1700c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom /* 1701c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom * Last buffer remains mapped so explicitly make it 1702c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom * coherent for CPU access and start preloading first 1703c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom * cache line ... 1704c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom */ 1705c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom dma_sync_single_for_cpu(rspq->adapter->pdev_dev, 1706c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom get_buf_addr(sdesc), 1707a0006a86cb19543f126bb2ee3d37baef82080763Ian Campbell fp->size, DMA_FROM_DEVICE); 1708c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom gl.va = (page_address(gl.frags[0].page) + 1709a0006a86cb19543f126bb2ee3d37baef82080763Ian Campbell gl.frags[0].offset); 1710c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom prefetch(gl.va); 1711c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom 1712c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom /* 1713c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom * Hand the new ingress packet to the handler for 1714c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom * this Response Queue. 1715c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom */ 1716c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom ret = rspq->handler(rspq, rspq->cur_desc, &gl); 1717c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom if (likely(ret == 0)) 1718a0006a86cb19543f126bb2ee3d37baef82080763Ian Campbell rspq->offset += ALIGN(fp->size, FL_ALIGN); 1719c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom else 1720c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom restore_rx_bufs(&gl, &rxq->fl, frag); 1721c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom } else if (likely(rsp_type == RSP_TYPE_CPL)) { 1722c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom ret = rspq->handler(rspq, rspq->cur_desc, NULL); 1723c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom } else { 1724c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom WARN_ON(rsp_type > RSP_TYPE_CPL); 1725c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom ret = 0; 1726c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom } 1727c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom 1728c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom if (unlikely(ret)) { 1729c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom /* 1730c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom * Couldn't process descriptor, back off for recovery. 1731c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom * We use the SGE's last timer which has the longest 1732c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom * interrupt coalescing value ... 1733c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom */ 1734c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom const int NOMEM_TIMER_IDX = SGE_NTIMERS-1; 1735c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom rspq->next_intr_params = 1736c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom QINTR_TIMER_IDX(NOMEM_TIMER_IDX); 1737c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom break; 1738c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom } 1739c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom 1740c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom rspq_next(rspq); 1741c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom budget_left--; 1742c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom } 1743c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom 1744c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom /* 1745c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom * If this is a Response Queue with an associated Free List and 1746c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom * at least two Egress Queue units available in the Free List 1747c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom * for new buffer pointers, refill the Free List. 1748c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom */ 1749c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom if (rspq->offset >= 0 && 1750c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom rxq->fl.size - rxq->fl.avail >= 2*FL_PER_EQ_UNIT) 1751c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom __refill_fl(rspq->adapter, &rxq->fl); 1752c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom return budget - budget_left; 1753c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom} 1754c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom 1755c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom/** 1756c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom * napi_rx_handler - the NAPI handler for RX processing 1757c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom * @napi: the napi instance 1758c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom * @budget: how many packets we can process in this round 1759c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom * 1760c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom * Handler for new data events when using NAPI. This does not need any 1761c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom * locking or protection from interrupts as data interrupts are off at 1762c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom * this point and other adapter interrupts do not interfere (the latter 1763c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom * in not a concern at all with MSI-X as non-data interrupts then have 1764c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom * a separate handler). 1765c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom */ 1766c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedomstatic int napi_rx_handler(struct napi_struct *napi, int budget) 1767c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom{ 1768c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom unsigned int intr_params; 1769c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom struct sge_rspq *rspq = container_of(napi, struct sge_rspq, napi); 1770c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom int work_done = process_responses(rspq, budget); 1771c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom 1772c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom if (likely(work_done < budget)) { 1773c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom napi_complete(napi); 1774c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom intr_params = rspq->next_intr_params; 1775c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom rspq->next_intr_params = rspq->intr_params; 1776c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom } else 1777c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom intr_params = QINTR_TIMER_IDX(SGE_TIMER_UPD_CIDX); 1778c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom 177968dc9d36c19aa1fd1633427b419d5e1f44753e8aCasey Leedom if (unlikely(work_done == 0)) 178068dc9d36c19aa1fd1633427b419d5e1f44753e8aCasey Leedom rspq->unhandled_irqs++; 178168dc9d36c19aa1fd1633427b419d5e1f44753e8aCasey Leedom 1782c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom t4_write_reg(rspq->adapter, 1783c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom T4VF_SGE_BASE_ADDR + SGE_VF_GTS, 1784c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom CIDXINC(work_done) | 1785c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom INGRESSQID((u32)rspq->cntxt_id) | 1786c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom SEINTARM(intr_params)); 1787c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom return work_done; 1788c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom} 1789c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom 1790c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom/* 1791c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom * The MSI-X interrupt handler for an SGE response queue for the NAPI case 1792c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom * (i.e., response queue serviced by NAPI polling). 1793c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom */ 1794c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedomirqreturn_t t4vf_sge_intr_msix(int irq, void *cookie) 1795c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom{ 1796c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom struct sge_rspq *rspq = cookie; 1797c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom 1798c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom napi_schedule(&rspq->napi); 1799c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom return IRQ_HANDLED; 1800c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom} 1801c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom 1802c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom/* 1803c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom * Process the indirect interrupt entries in the interrupt queue and kick off 1804c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom * NAPI for each queue that has generated an entry. 1805c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom */ 1806c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedomstatic unsigned int process_intrq(struct adapter *adapter) 1807c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom{ 1808c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom struct sge *s = &adapter->sge; 1809c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom struct sge_rspq *intrq = &s->intrq; 1810c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom unsigned int work_done; 1811c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom 1812c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom spin_lock(&adapter->sge.intrq_lock); 1813c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom for (work_done = 0; ; work_done++) { 1814c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom const struct rsp_ctrl *rc; 1815c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom unsigned int qid, iq_idx; 1816c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom struct sge_rspq *rspq; 1817c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom 1818c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom /* 1819c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom * Grab the next response from the interrupt queue and bail 1820c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom * out if it's not a new response. 1821c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom */ 1822c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom rc = (void *)intrq->cur_desc + (intrq->iqe_len - sizeof(*rc)); 1823c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom if (!is_new_response(rc, intrq)) 1824c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom break; 1825c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom 1826c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom /* 1827c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom * If the response isn't a forwarded interrupt message issue a 1828c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom * error and go on to the next response message. This should 1829c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom * never happen ... 1830c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom */ 1831c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom rmb(); 1832c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom if (unlikely(RSPD_TYPE(rc->type_gen) != RSP_TYPE_INTR)) { 1833c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom dev_err(adapter->pdev_dev, 1834c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom "Unexpected INTRQ response type %d\n", 1835c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom RSPD_TYPE(rc->type_gen)); 1836c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom continue; 1837c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom } 1838c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom 1839c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom /* 1840c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom * Extract the Queue ID from the interrupt message and perform 1841c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom * sanity checking to make sure it really refers to one of our 1842c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom * Ingress Queues which is active and matches the queue's ID. 1843c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom * None of these error conditions should ever happen so we may 1844c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom * want to either make them fatal and/or conditionalized under 1845c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom * DEBUG. 1846c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom */ 1847c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom qid = RSPD_QID(be32_to_cpu(rc->pldbuflen_qid)); 1848c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom iq_idx = IQ_IDX(s, qid); 1849c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom if (unlikely(iq_idx >= MAX_INGQ)) { 1850c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom dev_err(adapter->pdev_dev, 1851c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom "Ingress QID %d out of range\n", qid); 1852c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom continue; 1853c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom } 1854c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom rspq = s->ingr_map[iq_idx]; 1855c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom if (unlikely(rspq == NULL)) { 1856c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom dev_err(adapter->pdev_dev, 1857c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom "Ingress QID %d RSPQ=NULL\n", qid); 1858c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom continue; 1859c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom } 1860c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom if (unlikely(rspq->abs_id != qid)) { 1861c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom dev_err(adapter->pdev_dev, 1862c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom "Ingress QID %d refers to RSPQ %d\n", 1863c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom qid, rspq->abs_id); 1864c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom continue; 1865c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom } 1866c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom 1867c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom /* 1868c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom * Schedule NAPI processing on the indicated Response Queue 1869c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom * and move on to the next entry in the Forwarded Interrupt 1870c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom * Queue. 1871c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom */ 1872c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom napi_schedule(&rspq->napi); 1873c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom rspq_next(intrq); 1874c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom } 1875c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom 1876c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom t4_write_reg(adapter, T4VF_SGE_BASE_ADDR + SGE_VF_GTS, 1877c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom CIDXINC(work_done) | 1878c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom INGRESSQID(intrq->cntxt_id) | 1879c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom SEINTARM(intrq->intr_params)); 1880c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom 1881c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom spin_unlock(&adapter->sge.intrq_lock); 1882c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom 1883c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom return work_done; 1884c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom} 1885c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom 1886c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom/* 1887c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom * The MSI interrupt handler handles data events from SGE response queues as 1888c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom * well as error and other async events as they all use the same MSI vector. 1889c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom */ 1890c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedomirqreturn_t t4vf_intr_msi(int irq, void *cookie) 1891c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom{ 1892c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom struct adapter *adapter = cookie; 1893c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom 1894c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom process_intrq(adapter); 1895c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom return IRQ_HANDLED; 1896c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom} 1897c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom 1898c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom/** 1899c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom * t4vf_intr_handler - select the top-level interrupt handler 1900c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom * @adapter: the adapter 1901c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom * 1902c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom * Selects the top-level interrupt handler based on the type of interrupts 1903c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom * (MSI-X or MSI). 1904c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom */ 1905c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedomirq_handler_t t4vf_intr_handler(struct adapter *adapter) 1906c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom{ 1907c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom BUG_ON((adapter->flags & (USING_MSIX|USING_MSI)) == 0); 1908c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom if (adapter->flags & USING_MSIX) 1909c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom return t4vf_sge_intr_msix; 1910c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom else 1911c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom return t4vf_intr_msi; 1912c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom} 1913c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom 1914c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom/** 1915c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom * sge_rx_timer_cb - perform periodic maintenance of SGE RX queues 1916c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom * @data: the adapter 1917c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom * 1918c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom * Runs periodically from a timer to perform maintenance of SGE RX queues. 1919c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom * 1920c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom * a) Replenishes RX queues that have run out due to memory shortage. 1921c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom * Normally new RX buffers are added when existing ones are consumed but 1922c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom * when out of memory a queue can become empty. We schedule NAPI to do 1923c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom * the actual refill. 1924c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom */ 1925c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedomstatic void sge_rx_timer_cb(unsigned long data) 1926c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom{ 1927c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom struct adapter *adapter = (struct adapter *)data; 1928c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom struct sge *s = &adapter->sge; 1929c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom unsigned int i; 1930c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom 1931c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom /* 1932c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom * Scan the "Starving Free Lists" flag array looking for any Free 1933c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom * Lists in need of more free buffers. If we find one and it's not 1934c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom * being actively polled, then bump its "starving" counter and attempt 1935c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom * to refill it. If we're successful in adding enough buffers to push 1936c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom * the Free List over the starving threshold, then we can clear its 1937c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom * "starving" status. 1938c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom */ 1939c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom for (i = 0; i < ARRAY_SIZE(s->starving_fl); i++) { 1940c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom unsigned long m; 1941c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom 1942c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom for (m = s->starving_fl[i]; m; m &= m - 1) { 1943c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom unsigned int id = __ffs(m) + i * BITS_PER_LONG; 1944c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom struct sge_fl *fl = s->egr_map[id]; 1945c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom 1946c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom clear_bit(id, s->starving_fl); 1947c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom smp_mb__after_clear_bit(); 1948c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom 1949c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom /* 1950c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom * Since we are accessing fl without a lock there's a 1951c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom * small probability of a false positive where we 1952c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom * schedule napi but the FL is no longer starving. 1953c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom * No biggie. 1954c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom */ 1955c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom if (fl_starving(fl)) { 1956c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom struct sge_eth_rxq *rxq; 1957c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom 1958c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom rxq = container_of(fl, struct sge_eth_rxq, fl); 1959c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom if (napi_reschedule(&rxq->rspq.napi)) 1960c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom fl->starving++; 1961c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom else 1962c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom set_bit(id, s->starving_fl); 1963c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom } 1964c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom } 1965c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom } 1966c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom 1967c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom /* 1968c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom * Reschedule the next scan for starving Free Lists ... 1969c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom */ 1970c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom mod_timer(&s->rx_timer, jiffies + RX_QCHECK_PERIOD); 1971c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom} 1972c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom 1973c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom/** 1974c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom * sge_tx_timer_cb - perform periodic maintenance of SGE Tx queues 1975c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom * @data: the adapter 1976c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom * 1977c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom * Runs periodically from a timer to perform maintenance of SGE TX queues. 1978c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom * 1979c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom * b) Reclaims completed Tx packets for the Ethernet queues. Normally 1980c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom * packets are cleaned up by new Tx packets, this timer cleans up packets 1981c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom * when no new packets are being submitted. This is essential for pktgen, 1982c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom * at least. 1983c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom */ 1984c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedomstatic void sge_tx_timer_cb(unsigned long data) 1985c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom{ 1986c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom struct adapter *adapter = (struct adapter *)data; 1987c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom struct sge *s = &adapter->sge; 1988c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom unsigned int i, budget; 1989c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom 1990c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom budget = MAX_TIMER_TX_RECLAIM; 1991c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom i = s->ethtxq_rover; 1992c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom do { 1993c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom struct sge_eth_txq *txq = &s->ethtxq[i]; 1994c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom 1995c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom if (reclaimable(&txq->q) && __netif_tx_trylock(txq->txq)) { 1996c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom int avail = reclaimable(&txq->q); 1997c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom 1998c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom if (avail > budget) 1999c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom avail = budget; 2000c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom 2001c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom free_tx_desc(adapter, &txq->q, avail, true); 2002c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom txq->q.in_use -= avail; 2003c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom __netif_tx_unlock(txq->txq); 2004c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom 2005c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom budget -= avail; 2006c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom if (!budget) 2007c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom break; 2008c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom } 2009c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom 2010c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom i++; 2011c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom if (i >= s->ethqsets) 2012c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom i = 0; 2013c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom } while (i != s->ethtxq_rover); 2014c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom s->ethtxq_rover = i; 2015c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom 2016c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom /* 2017c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom * If we found too many reclaimable packets schedule a timer in the 2018c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom * near future to continue where we left off. Otherwise the next timer 2019c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom * will be at its normal interval. 2020c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom */ 2021c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom mod_timer(&s->tx_timer, jiffies + (budget ? TX_QCHECK_PERIOD : 2)); 2022c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom} 2023c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom 2024c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom/** 2025c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom * t4vf_sge_alloc_rxq - allocate an SGE RX Queue 2026c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom * @adapter: the adapter 2027c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom * @rspq: pointer to to the new rxq's Response Queue to be filled in 2028c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom * @iqasynch: if 0, a normal rspq; if 1, an asynchronous event queue 2029c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom * @dev: the network device associated with the new rspq 2030c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom * @intr_dest: MSI-X vector index (overriden in MSI mode) 2031c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom * @fl: pointer to the new rxq's Free List to be filled in 2032c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom * @hnd: the interrupt handler to invoke for the rspq 2033c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom */ 2034c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedomint t4vf_sge_alloc_rxq(struct adapter *adapter, struct sge_rspq *rspq, 2035c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom bool iqasynch, struct net_device *dev, 2036c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom int intr_dest, 2037c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom struct sge_fl *fl, rspq_handler_t hnd) 2038c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom{ 2039c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom struct port_info *pi = netdev_priv(dev); 2040c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom struct fw_iq_cmd cmd, rpl; 2041c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom int ret, iqandst, flsz = 0; 2042c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom 2043c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom /* 2044c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom * If we're using MSI interrupts and we're not initializing the 2045c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom * Forwarded Interrupt Queue itself, then set up this queue for 2046c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom * indirect interrupts to the Forwarded Interrupt Queue. Obviously 2047c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom * the Forwarded Interrupt Queue must be set up before any other 2048c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom * ingress queue ... 2049c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom */ 2050c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom if ((adapter->flags & USING_MSI) && rspq != &adapter->sge.intrq) { 2051c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom iqandst = SGE_INTRDST_IQ; 2052c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom intr_dest = adapter->sge.intrq.abs_id; 2053c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom } else 2054c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom iqandst = SGE_INTRDST_PCI; 2055c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom 2056c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom /* 2057c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom * Allocate the hardware ring for the Response Queue. The size needs 2058c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom * to be a multiple of 16 which includes the mandatory status entry 2059c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom * (regardless of whether the Status Page capabilities are enabled or 2060c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom * not). 2061c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom */ 2062c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom rspq->size = roundup(rspq->size, 16); 2063c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom rspq->desc = alloc_ring(adapter->pdev_dev, rspq->size, rspq->iqe_len, 2064c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom 0, &rspq->phys_addr, NULL, 0); 2065c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom if (!rspq->desc) 2066c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom return -ENOMEM; 2067c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom 2068c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom /* 2069c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom * Fill in the Ingress Queue Command. Note: Ideally this code would 2070c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom * be in t4vf_hw.c but there are so many parameters and dependencies 2071c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom * on our Linux SGE state that we would end up having to pass tons of 2072c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom * parameters. We'll have to think about how this might be migrated 2073c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom * into OS-independent common code ... 2074c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom */ 2075c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom memset(&cmd, 0, sizeof(cmd)); 2076c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom cmd.op_to_vfn = cpu_to_be32(FW_CMD_OP(FW_IQ_CMD) | 2077c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom FW_CMD_REQUEST | 2078c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom FW_CMD_WRITE | 2079c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom FW_CMD_EXEC); 2080c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom cmd.alloc_to_len16 = cpu_to_be32(FW_IQ_CMD_ALLOC | 2081c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom FW_IQ_CMD_IQSTART(1) | 2082c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom FW_LEN16(cmd)); 2083c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom cmd.type_to_iqandstindex = 2084c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom cpu_to_be32(FW_IQ_CMD_TYPE(FW_IQ_TYPE_FL_INT_CAP) | 2085c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom FW_IQ_CMD_IQASYNCH(iqasynch) | 2086c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom FW_IQ_CMD_VIID(pi->viid) | 2087c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom FW_IQ_CMD_IQANDST(iqandst) | 2088c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom FW_IQ_CMD_IQANUS(1) | 2089c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom FW_IQ_CMD_IQANUD(SGE_UPDATEDEL_INTR) | 2090c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom FW_IQ_CMD_IQANDSTINDEX(intr_dest)); 2091c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom cmd.iqdroprss_to_iqesize = 2092c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom cpu_to_be16(FW_IQ_CMD_IQPCIECH(pi->port_id) | 2093c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom FW_IQ_CMD_IQGTSMODE | 2094c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom FW_IQ_CMD_IQINTCNTTHRESH(rspq->pktcnt_idx) | 2095c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom FW_IQ_CMD_IQESIZE(ilog2(rspq->iqe_len) - 4)); 2096c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom cmd.iqsize = cpu_to_be16(rspq->size); 2097c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom cmd.iqaddr = cpu_to_be64(rspq->phys_addr); 2098c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom 2099c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom if (fl) { 2100c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom /* 2101c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom * Allocate the ring for the hardware free list (with space 2102c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom * for its status page) along with the associated software 2103c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom * descriptor ring. The free list size needs to be a multiple 2104c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom * of the Egress Queue Unit. 2105c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom */ 2106c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom fl->size = roundup(fl->size, FL_PER_EQ_UNIT); 2107c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom fl->desc = alloc_ring(adapter->pdev_dev, fl->size, 2108c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom sizeof(__be64), sizeof(struct rx_sw_desc), 2109c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom &fl->addr, &fl->sdesc, STAT_LEN); 2110c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom if (!fl->desc) { 2111c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom ret = -ENOMEM; 2112c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom goto err; 2113c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom } 2114c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom 2115c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom /* 2116c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom * Calculate the size of the hardware free list ring plus 2117caedda35c6dba5a283e5d87e77a8d19ee4be3183Casey Leedom * Status Page (which the SGE will place after the end of the 2118c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom * free list ring) in Egress Queue Units. 2119c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom */ 2120c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom flsz = (fl->size / FL_PER_EQ_UNIT + 2121c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom STAT_LEN / EQ_UNIT); 2122c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom 2123c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom /* 2124c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom * Fill in all the relevant firmware Ingress Queue Command 2125c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom * fields for the free list. 2126c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom */ 2127c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom cmd.iqns_to_fl0congen = 2128c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom cpu_to_be32( 2129c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom FW_IQ_CMD_FL0HOSTFCMODE(SGE_HOSTFCMODE_NONE) | 2130c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom FW_IQ_CMD_FL0PACKEN | 2131c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom FW_IQ_CMD_FL0PADEN); 2132c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom cmd.fl0dcaen_to_fl0cidxfthresh = 2133c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom cpu_to_be16( 2134c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom FW_IQ_CMD_FL0FBMIN(SGE_FETCHBURSTMIN_64B) | 2135c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom FW_IQ_CMD_FL0FBMAX(SGE_FETCHBURSTMAX_512B)); 2136c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom cmd.fl0size = cpu_to_be16(flsz); 2137c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom cmd.fl0addr = cpu_to_be64(fl->addr); 2138c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom } 2139c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom 2140c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom /* 2141c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom * Issue the firmware Ingress Queue Command and extract the results if 2142c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom * it completes successfully. 2143c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom */ 2144c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom ret = t4vf_wr_mbox(adapter, &cmd, sizeof(cmd), &rpl); 2145c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom if (ret) 2146c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom goto err; 2147c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom 2148c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom netif_napi_add(dev, &rspq->napi, napi_rx_handler, 64); 2149c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom rspq->cur_desc = rspq->desc; 2150c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom rspq->cidx = 0; 2151c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom rspq->gen = 1; 2152c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom rspq->next_intr_params = rspq->intr_params; 2153c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom rspq->cntxt_id = be16_to_cpu(rpl.iqid); 2154c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom rspq->abs_id = be16_to_cpu(rpl.physiqid); 2155c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom rspq->size--; /* subtract status entry */ 2156c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom rspq->adapter = adapter; 2157c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom rspq->netdev = dev; 2158c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom rspq->handler = hnd; 2159c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom 2160c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom /* set offset to -1 to distinguish ingress queues without FL */ 2161c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom rspq->offset = fl ? 0 : -1; 2162c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom 2163c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom if (fl) { 2164c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom fl->cntxt_id = be16_to_cpu(rpl.fl0id); 2165c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom fl->avail = 0; 2166c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom fl->pend_cred = 0; 2167c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom fl->pidx = 0; 2168c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom fl->cidx = 0; 2169c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom fl->alloc_failed = 0; 2170c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom fl->large_alloc_failed = 0; 2171c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom fl->starving = 0; 2172c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom refill_fl(adapter, fl, fl_cap(fl), GFP_KERNEL); 2173c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom } 2174c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom 2175c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom return 0; 2176c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom 2177c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedomerr: 2178c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom /* 2179c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom * An error occurred. Clean up our partial allocation state and 2180c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom * return the error. 2181c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom */ 2182c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom if (rspq->desc) { 2183c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom dma_free_coherent(adapter->pdev_dev, rspq->size * rspq->iqe_len, 2184c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom rspq->desc, rspq->phys_addr); 2185c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom rspq->desc = NULL; 2186c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom } 2187c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom if (fl && fl->desc) { 2188c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom kfree(fl->sdesc); 2189c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom fl->sdesc = NULL; 2190c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom dma_free_coherent(adapter->pdev_dev, flsz * EQ_UNIT, 2191c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom fl->desc, fl->addr); 2192c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom fl->desc = NULL; 2193c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom } 2194c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom return ret; 2195c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom} 2196c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom 2197c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom/** 2198c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom * t4vf_sge_alloc_eth_txq - allocate an SGE Ethernet TX Queue 2199c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom * @adapter: the adapter 2200c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom * @txq: pointer to the new txq to be filled in 2201c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom * @devq: the network TX queue associated with the new txq 2202c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom * @iqid: the relative ingress queue ID to which events relating to 2203c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom * the new txq should be directed 2204c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom */ 2205c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedomint t4vf_sge_alloc_eth_txq(struct adapter *adapter, struct sge_eth_txq *txq, 2206c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom struct net_device *dev, struct netdev_queue *devq, 2207c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom unsigned int iqid) 2208c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom{ 2209c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom int ret, nentries; 2210c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom struct fw_eq_eth_cmd cmd, rpl; 2211c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom struct port_info *pi = netdev_priv(dev); 2212c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom 2213c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom /* 2214caedda35c6dba5a283e5d87e77a8d19ee4be3183Casey Leedom * Calculate the size of the hardware TX Queue (including the Status 2215caedda35c6dba5a283e5d87e77a8d19ee4be3183Casey Leedom * Page on the end of the TX Queue) in units of TX Descriptors. 2216c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom */ 2217c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom nentries = txq->q.size + STAT_LEN / sizeof(struct tx_desc); 2218c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom 2219c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom /* 2220c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom * Allocate the hardware ring for the TX ring (with space for its 2221c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom * status page) along with the associated software descriptor ring. 2222c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom */ 2223c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom txq->q.desc = alloc_ring(adapter->pdev_dev, txq->q.size, 2224c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom sizeof(struct tx_desc), 2225c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom sizeof(struct tx_sw_desc), 2226c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom &txq->q.phys_addr, &txq->q.sdesc, STAT_LEN); 2227c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom if (!txq->q.desc) 2228c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom return -ENOMEM; 2229c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom 2230c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom /* 2231c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom * Fill in the Egress Queue Command. Note: As with the direct use of 2232c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom * the firmware Ingress Queue COmmand above in our RXQ allocation 2233c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom * routine, ideally, this code would be in t4vf_hw.c. Again, we'll 2234c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom * have to see if there's some reasonable way to parameterize it 2235c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom * into the common code ... 2236c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom */ 2237c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom memset(&cmd, 0, sizeof(cmd)); 2238c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom cmd.op_to_vfn = cpu_to_be32(FW_CMD_OP(FW_EQ_ETH_CMD) | 2239c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom FW_CMD_REQUEST | 2240c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom FW_CMD_WRITE | 2241c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom FW_CMD_EXEC); 2242c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom cmd.alloc_to_len16 = cpu_to_be32(FW_EQ_ETH_CMD_ALLOC | 2243c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom FW_EQ_ETH_CMD_EQSTART | 2244c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom FW_LEN16(cmd)); 2245c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom cmd.viid_pkd = cpu_to_be32(FW_EQ_ETH_CMD_VIID(pi->viid)); 2246c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom cmd.fetchszm_to_iqid = 2247c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom cpu_to_be32(FW_EQ_ETH_CMD_HOSTFCMODE(SGE_HOSTFCMODE_STPG) | 2248c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom FW_EQ_ETH_CMD_PCIECHN(pi->port_id) | 2249c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom FW_EQ_ETH_CMD_IQID(iqid)); 2250c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom cmd.dcaen_to_eqsize = 2251c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom cpu_to_be32(FW_EQ_ETH_CMD_FBMIN(SGE_FETCHBURSTMIN_64B) | 2252c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom FW_EQ_ETH_CMD_FBMAX(SGE_FETCHBURSTMAX_512B) | 2253c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom FW_EQ_ETH_CMD_CIDXFTHRESH(SGE_CIDXFLUSHTHRESH_32) | 2254c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom FW_EQ_ETH_CMD_EQSIZE(nentries)); 2255c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom cmd.eqaddr = cpu_to_be64(txq->q.phys_addr); 2256c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom 2257c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom /* 2258c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom * Issue the firmware Egress Queue Command and extract the results if 2259c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom * it completes successfully. 2260c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom */ 2261c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom ret = t4vf_wr_mbox(adapter, &cmd, sizeof(cmd), &rpl); 2262c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom if (ret) { 2263c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom /* 2264c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom * The girmware Ingress Queue Command failed for some reason. 2265c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom * Free up our partial allocation state and return the error. 2266c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom */ 2267c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom kfree(txq->q.sdesc); 2268c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom txq->q.sdesc = NULL; 2269c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom dma_free_coherent(adapter->pdev_dev, 2270c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom nentries * sizeof(struct tx_desc), 2271c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom txq->q.desc, txq->q.phys_addr); 2272c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom txq->q.desc = NULL; 2273c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom return ret; 2274c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom } 2275c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom 2276c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom txq->q.in_use = 0; 2277c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom txq->q.cidx = 0; 2278c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom txq->q.pidx = 0; 2279c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom txq->q.stat = (void *)&txq->q.desc[txq->q.size]; 2280c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom txq->q.cntxt_id = FW_EQ_ETH_CMD_EQID_GET(be32_to_cpu(rpl.eqid_pkd)); 2281c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom txq->q.abs_id = 2282c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom FW_EQ_ETH_CMD_PHYSEQID_GET(be32_to_cpu(rpl.physeqid_pkd)); 2283c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom txq->txq = devq; 2284c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom txq->tso = 0; 2285c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom txq->tx_cso = 0; 2286c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom txq->vlan_ins = 0; 2287c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom txq->q.stops = 0; 2288c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom txq->q.restarts = 0; 2289c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom txq->mapping_err = 0; 2290c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom return 0; 2291c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom} 2292c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom 2293c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom/* 2294c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom * Free the DMA map resources associated with a TX queue. 2295c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom */ 2296c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedomstatic void free_txq(struct adapter *adapter, struct sge_txq *tq) 2297c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom{ 2298c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom dma_free_coherent(adapter->pdev_dev, 2299c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom tq->size * sizeof(*tq->desc) + STAT_LEN, 2300c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom tq->desc, tq->phys_addr); 2301c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom tq->cntxt_id = 0; 2302c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom tq->sdesc = NULL; 2303c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom tq->desc = NULL; 2304c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom} 2305c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom 2306c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom/* 2307c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom * Free the resources associated with a response queue (possibly including a 2308c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom * free list). 2309c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom */ 2310c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedomstatic void free_rspq_fl(struct adapter *adapter, struct sge_rspq *rspq, 2311c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom struct sge_fl *fl) 2312c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom{ 2313c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom unsigned int flid = fl ? fl->cntxt_id : 0xffff; 2314c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom 2315c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom t4vf_iq_free(adapter, FW_IQ_TYPE_FL_INT_CAP, 2316c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom rspq->cntxt_id, flid, 0xffff); 2317c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom dma_free_coherent(adapter->pdev_dev, (rspq->size + 1) * rspq->iqe_len, 2318c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom rspq->desc, rspq->phys_addr); 2319c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom netif_napi_del(&rspq->napi); 2320c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom rspq->netdev = NULL; 2321c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom rspq->cntxt_id = 0; 2322c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom rspq->abs_id = 0; 2323c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom rspq->desc = NULL; 2324c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom 2325c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom if (fl) { 2326c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom free_rx_bufs(adapter, fl, fl->avail); 2327c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom dma_free_coherent(adapter->pdev_dev, 2328c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom fl->size * sizeof(*fl->desc) + STAT_LEN, 2329c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom fl->desc, fl->addr); 2330c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom kfree(fl->sdesc); 2331c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom fl->sdesc = NULL; 2332c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom fl->cntxt_id = 0; 2333c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom fl->desc = NULL; 2334c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom } 2335c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom} 2336c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom 2337c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom/** 2338c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom * t4vf_free_sge_resources - free SGE resources 2339c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom * @adapter: the adapter 2340c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom * 2341c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom * Frees resources used by the SGE queue sets. 2342c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom */ 2343c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedomvoid t4vf_free_sge_resources(struct adapter *adapter) 2344c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom{ 2345c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom struct sge *s = &adapter->sge; 2346c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom struct sge_eth_rxq *rxq = s->ethrxq; 2347c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom struct sge_eth_txq *txq = s->ethtxq; 2348c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom struct sge_rspq *evtq = &s->fw_evtq; 2349c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom struct sge_rspq *intrq = &s->intrq; 2350c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom int qs; 2351c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom 2352b97d13a53d63c7db1d05d54298c7a12f86c4fbadCasey Leedom for (qs = 0; qs < adapter->sge.ethqsets; qs++, rxq++, txq++) { 2353c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom if (rxq->rspq.desc) 2354c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom free_rspq_fl(adapter, &rxq->rspq, &rxq->fl); 2355c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom if (txq->q.desc) { 2356c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom t4vf_eth_eq_free(adapter, txq->q.cntxt_id); 2357c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom free_tx_desc(adapter, &txq->q, txq->q.in_use, true); 2358c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom kfree(txq->q.sdesc); 2359c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom free_txq(adapter, &txq->q); 2360c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom } 2361c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom } 2362c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom if (evtq->desc) 2363c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom free_rspq_fl(adapter, evtq, NULL); 2364c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom if (intrq->desc) 2365c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom free_rspq_fl(adapter, intrq, NULL); 2366c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom} 2367c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom 2368c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom/** 2369c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom * t4vf_sge_start - enable SGE operation 2370c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom * @adapter: the adapter 2371c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom * 2372c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom * Start tasklets and timers associated with the DMA engine. 2373c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom */ 2374c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedomvoid t4vf_sge_start(struct adapter *adapter) 2375c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom{ 2376c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom adapter->sge.ethtxq_rover = 0; 2377c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom mod_timer(&adapter->sge.rx_timer, jiffies + RX_QCHECK_PERIOD); 2378c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom mod_timer(&adapter->sge.tx_timer, jiffies + TX_QCHECK_PERIOD); 2379c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom} 2380c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom 2381c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom/** 2382c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom * t4vf_sge_stop - disable SGE operation 2383c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom * @adapter: the adapter 2384c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom * 2385c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom * Stop tasklets and timers associated with the DMA engine. Note that 2386c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom * this is effective only if measures have been taken to disable any HW 2387c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom * events that may restart them. 2388c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom */ 2389c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedomvoid t4vf_sge_stop(struct adapter *adapter) 2390c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom{ 2391c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom struct sge *s = &adapter->sge; 2392c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom 2393c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom if (s->rx_timer.function) 2394c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom del_timer_sync(&s->rx_timer); 2395c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom if (s->tx_timer.function) 2396c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom del_timer_sync(&s->tx_timer); 2397c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom} 2398c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom 2399c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom/** 2400c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom * t4vf_sge_init - initialize SGE 2401c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom * @adapter: the adapter 2402c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom * 2403c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom * Performs SGE initialization needed every time after a chip reset. 2404c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom * We do not initialize any of the queue sets here, instead the driver 2405c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom * top-level must request those individually. We also do not enable DMA 2406c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom * here, that should be done after the queues have been set up. 2407c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom */ 2408c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedomint t4vf_sge_init(struct adapter *adapter) 2409c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom{ 2410c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom struct sge_params *sge_params = &adapter->params.sge; 2411c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom u32 fl0 = sge_params->sge_fl_buffer_size[0]; 2412c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom u32 fl1 = sge_params->sge_fl_buffer_size[1]; 2413c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom struct sge *s = &adapter->sge; 2414c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom 2415c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom /* 2416c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom * Start by vetting the basic SGE parameters which have been set up by 2417c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom * the Physical Function Driver. Ideally we should be able to deal 2418c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom * with _any_ configuration. Practice is different ... 2419c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom */ 2420c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom if (fl0 != PAGE_SIZE || (fl1 != 0 && fl1 <= fl0)) { 2421c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom dev_err(adapter->pdev_dev, "bad SGE FL buffer sizes [%d, %d]\n", 2422c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom fl0, fl1); 2423c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom return -EINVAL; 2424c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom } 2425c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom if ((sge_params->sge_control & RXPKTCPLMODE) == 0) { 2426c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom dev_err(adapter->pdev_dev, "bad SGE CPL MODE\n"); 2427c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom return -EINVAL; 2428c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom } 2429c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom 2430c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom /* 2431c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom * Now translate the adapter parameters into our internal forms. 2432c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom */ 2433c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom if (fl1) 2434c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom FL_PG_ORDER = ilog2(fl1) - PAGE_SHIFT; 2435c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom STAT_LEN = ((sge_params->sge_control & EGRSTATUSPAGESIZE) ? 128 : 64); 2436c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom PKTSHIFT = PKTSHIFT_GET(sge_params->sge_control); 2437c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom FL_ALIGN = 1 << (INGPADBOUNDARY_GET(sge_params->sge_control) + 2438b3003be36a3c9215cd17182349981581de269048Casey Leedom SGE_INGPADBOUNDARY_SHIFT); 2439c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom 2440c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom /* 2441c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom * Set up tasklet timers. 2442c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom */ 2443c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom setup_timer(&s->rx_timer, sge_rx_timer_cb, (unsigned long)adapter); 2444c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom setup_timer(&s->tx_timer, sge_tx_timer_cb, (unsigned long)adapter); 2445c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom 2446c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom /* 2447c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom * Initialize Forwarded Interrupt Queue lock. 2448c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom */ 2449c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom spin_lock_init(&s->intrq_lock); 2450c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom 2451c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom return 0; 2452c6e0d91464da214081af546496dd3a4b6d19db70Casey Leedom} 2453