18199d3a79c224bbe5943fa08684e1f93a17881b0Christoph Lameter/***************************************************************************** 28199d3a79c224bbe5943fa08684e1f93a17881b0Christoph Lameter * * 38199d3a79c224bbe5943fa08684e1f93a17881b0Christoph Lameter * File: sge.c * 4559fb51ba7e66fe298b8355fabde1275b7def35fScott Bardone * $Revision: 1.26 $ * 5559fb51ba7e66fe298b8355fabde1275b7def35fScott Bardone * $Date: 2005/06/21 18:29:48 $ * 68199d3a79c224bbe5943fa08684e1f93a17881b0Christoph Lameter * Description: * 78199d3a79c224bbe5943fa08684e1f93a17881b0Christoph Lameter * DMA engine. * 88199d3a79c224bbe5943fa08684e1f93a17881b0Christoph Lameter * part of the Chelsio 10Gb Ethernet Driver. * 98199d3a79c224bbe5943fa08684e1f93a17881b0Christoph Lameter * * 108199d3a79c224bbe5943fa08684e1f93a17881b0Christoph Lameter * This program is free software; you can redistribute it and/or modify * 118199d3a79c224bbe5943fa08684e1f93a17881b0Christoph Lameter * it under the terms of the GNU General Public License, version 2, as * 128199d3a79c224bbe5943fa08684e1f93a17881b0Christoph Lameter * published by the Free Software Foundation. * 138199d3a79c224bbe5943fa08684e1f93a17881b0Christoph Lameter * * 148199d3a79c224bbe5943fa08684e1f93a17881b0Christoph Lameter * You should have received a copy of the GNU General Public License along * 158199d3a79c224bbe5943fa08684e1f93a17881b0Christoph Lameter * with this program; if not, write to the Free Software Foundation, Inc., * 168199d3a79c224bbe5943fa08684e1f93a17881b0Christoph Lameter * 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA. * 178199d3a79c224bbe5943fa08684e1f93a17881b0Christoph Lameter * * 188199d3a79c224bbe5943fa08684e1f93a17881b0Christoph Lameter * THIS SOFTWARE IS PROVIDED ``AS IS'' AND WITHOUT ANY EXPRESS OR IMPLIED * 198199d3a79c224bbe5943fa08684e1f93a17881b0Christoph Lameter * WARRANTIES, INCLUDING, WITHOUT LIMITATION, THE IMPLIED WARRANTIES OF * 208199d3a79c224bbe5943fa08684e1f93a17881b0Christoph Lameter * MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE. * 218199d3a79c224bbe5943fa08684e1f93a17881b0Christoph Lameter * * 228199d3a79c224bbe5943fa08684e1f93a17881b0Christoph Lameter * http://www.chelsio.com * 238199d3a79c224bbe5943fa08684e1f93a17881b0Christoph Lameter * * 248199d3a79c224bbe5943fa08684e1f93a17881b0Christoph Lameter * Copyright (c) 2003 - 2005 Chelsio Communications, Inc. * 258199d3a79c224bbe5943fa08684e1f93a17881b0Christoph Lameter * All rights reserved. * 268199d3a79c224bbe5943fa08684e1f93a17881b0Christoph Lameter * * 278199d3a79c224bbe5943fa08684e1f93a17881b0Christoph Lameter * Maintainers: maintainers@chelsio.com * 288199d3a79c224bbe5943fa08684e1f93a17881b0Christoph Lameter * * 298199d3a79c224bbe5943fa08684e1f93a17881b0Christoph Lameter * Authors: Dimitrios Michailidis <dm@chelsio.com> * 308199d3a79c224bbe5943fa08684e1f93a17881b0Christoph Lameter * Tina Yang <tainay@chelsio.com> * 318199d3a79c224bbe5943fa08684e1f93a17881b0Christoph Lameter * Felix Marti <felix@chelsio.com> * 328199d3a79c224bbe5943fa08684e1f93a17881b0Christoph Lameter * Scott Bardone <sbardone@chelsio.com> * 338199d3a79c224bbe5943fa08684e1f93a17881b0Christoph Lameter * Kurt Ottaway <kottaway@chelsio.com> * 348199d3a79c224bbe5943fa08684e1f93a17881b0Christoph Lameter * Frank DiMambro <frank@chelsio.com> * 358199d3a79c224bbe5943fa08684e1f93a17881b0Christoph Lameter * * 368199d3a79c224bbe5943fa08684e1f93a17881b0Christoph Lameter * History: * 378199d3a79c224bbe5943fa08684e1f93a17881b0Christoph Lameter * * 388199d3a79c224bbe5943fa08684e1f93a17881b0Christoph Lameter ****************************************************************************/ 398199d3a79c224bbe5943fa08684e1f93a17881b0Christoph Lameter 408199d3a79c224bbe5943fa08684e1f93a17881b0Christoph Lameter#include "common.h" 418199d3a79c224bbe5943fa08684e1f93a17881b0Christoph Lameter 428199d3a79c224bbe5943fa08684e1f93a17881b0Christoph Lameter#include <linux/types.h> 438199d3a79c224bbe5943fa08684e1f93a17881b0Christoph Lameter#include <linux/errno.h> 448199d3a79c224bbe5943fa08684e1f93a17881b0Christoph Lameter#include <linux/pci.h> 45f1d3d38af75789f1b82969b83b69cab540609789Stephen Hemminger#include <linux/ktime.h> 468199d3a79c224bbe5943fa08684e1f93a17881b0Christoph Lameter#include <linux/netdevice.h> 478199d3a79c224bbe5943fa08684e1f93a17881b0Christoph Lameter#include <linux/etherdevice.h> 488199d3a79c224bbe5943fa08684e1f93a17881b0Christoph Lameter#include <linux/if_vlan.h> 498199d3a79c224bbe5943fa08684e1f93a17881b0Christoph Lameter#include <linux/skbuff.h> 508199d3a79c224bbe5943fa08684e1f93a17881b0Christoph Lameter#include <linux/init.h> 518199d3a79c224bbe5943fa08684e1f93a17881b0Christoph Lameter#include <linux/mm.h> 52f1d3d38af75789f1b82969b83b69cab540609789Stephen Hemminger#include <linux/tcp.h> 538199d3a79c224bbe5943fa08684e1f93a17881b0Christoph Lameter#include <linux/ip.h> 548199d3a79c224bbe5943fa08684e1f93a17881b0Christoph Lameter#include <linux/in.h> 558199d3a79c224bbe5943fa08684e1f93a17881b0Christoph Lameter#include <linux/if_arp.h> 565a0e3ad6af8660be21ca98a971cd00f331318c05Tejun Heo#include <linux/slab.h> 5770c71606190e9115e5f8363bfcd164c582eb314aPaul Gortmaker#include <linux/prefetch.h> 588199d3a79c224bbe5943fa08684e1f93a17881b0Christoph Lameter 598199d3a79c224bbe5943fa08684e1f93a17881b0Christoph Lameter#include "cpl5_cmd.h" 608199d3a79c224bbe5943fa08684e1f93a17881b0Christoph Lameter#include "sge.h" 618199d3a79c224bbe5943fa08684e1f93a17881b0Christoph Lameter#include "regs.h" 628199d3a79c224bbe5943fa08684e1f93a17881b0Christoph Lameter#include "espi.h" 638199d3a79c224bbe5943fa08684e1f93a17881b0Christoph Lameter 64f1d3d38af75789f1b82969b83b69cab540609789Stephen Hemminger/* This belongs in if_ether.h */ 65f1d3d38af75789f1b82969b83b69cab540609789Stephen Hemminger#define ETH_P_CPL5 0xf 668199d3a79c224bbe5943fa08684e1f93a17881b0Christoph Lameter 678199d3a79c224bbe5943fa08684e1f93a17881b0Christoph Lameter#define SGE_CMDQ_N 2 688199d3a79c224bbe5943fa08684e1f93a17881b0Christoph Lameter#define SGE_FREELQ_N 2 69559fb51ba7e66fe298b8355fabde1275b7def35fScott Bardone#define SGE_CMDQ0_E_N 1024 708199d3a79c224bbe5943fa08684e1f93a17881b0Christoph Lameter#define SGE_CMDQ1_E_N 128 718199d3a79c224bbe5943fa08684e1f93a17881b0Christoph Lameter#define SGE_FREEL_SIZE 4096 728199d3a79c224bbe5943fa08684e1f93a17881b0Christoph Lameter#define SGE_JUMBO_FREEL_SIZE 512 738199d3a79c224bbe5943fa08684e1f93a17881b0Christoph Lameter#define SGE_FREEL_REFILL_THRESH 16 748199d3a79c224bbe5943fa08684e1f93a17881b0Christoph Lameter#define SGE_RESPQ_E_N 1024 75559fb51ba7e66fe298b8355fabde1275b7def35fScott Bardone#define SGE_INTRTIMER_NRES 1000 768199d3a79c224bbe5943fa08684e1f93a17881b0Christoph Lameter#define SGE_RX_SM_BUF_SIZE 1536 77f1d3d38af75789f1b82969b83b69cab540609789Stephen Hemminger#define SGE_TX_DESC_MAX_PLEN 16384 788199d3a79c224bbe5943fa08684e1f93a17881b0Christoph Lameter 79559fb51ba7e66fe298b8355fabde1275b7def35fScott Bardone#define SGE_RESPQ_REPLENISH_THRES (SGE_RESPQ_E_N / 4) 80559fb51ba7e66fe298b8355fabde1275b7def35fScott Bardone 81559fb51ba7e66fe298b8355fabde1275b7def35fScott Bardone/* 82559fb51ba7e66fe298b8355fabde1275b7def35fScott Bardone * Period of the TX buffer reclaim timer. This timer does not need to run 83559fb51ba7e66fe298b8355fabde1275b7def35fScott Bardone * frequently as TX buffers are usually reclaimed by new TX packets. 84559fb51ba7e66fe298b8355fabde1275b7def35fScott Bardone */ 85559fb51ba7e66fe298b8355fabde1275b7def35fScott Bardone#define TX_RECLAIM_PERIOD (HZ / 4) 868199d3a79c224bbe5943fa08684e1f93a17881b0Christoph Lameter 87559fb51ba7e66fe298b8355fabde1275b7def35fScott Bardone#define M_CMD_LEN 0x7fffffff 88559fb51ba7e66fe298b8355fabde1275b7def35fScott Bardone#define V_CMD_LEN(v) (v) 89559fb51ba7e66fe298b8355fabde1275b7def35fScott Bardone#define G_CMD_LEN(v) ((v) & M_CMD_LEN) 90559fb51ba7e66fe298b8355fabde1275b7def35fScott Bardone#define V_CMD_GEN1(v) ((v) << 31) 91559fb51ba7e66fe298b8355fabde1275b7def35fScott Bardone#define V_CMD_GEN2(v) (v) 92559fb51ba7e66fe298b8355fabde1275b7def35fScott Bardone#define F_CMD_DATAVALID (1 << 1) 93559fb51ba7e66fe298b8355fabde1275b7def35fScott Bardone#define F_CMD_SOP (1 << 2) 94559fb51ba7e66fe298b8355fabde1275b7def35fScott Bardone#define V_CMD_EOP(v) ((v) << 3) 95559fb51ba7e66fe298b8355fabde1275b7def35fScott Bardone 968199d3a79c224bbe5943fa08684e1f93a17881b0Christoph Lameter/* 97559fb51ba7e66fe298b8355fabde1275b7def35fScott Bardone * Command queue, receive buffer list, and response queue descriptors. 988199d3a79c224bbe5943fa08684e1f93a17881b0Christoph Lameter */ 998199d3a79c224bbe5943fa08684e1f93a17881b0Christoph Lameter#if defined(__BIG_ENDIAN_BITFIELD) 1008199d3a79c224bbe5943fa08684e1f93a17881b0Christoph Lameterstruct cmdQ_e { 101559fb51ba7e66fe298b8355fabde1275b7def35fScott Bardone u32 addr_lo; 102559fb51ba7e66fe298b8355fabde1275b7def35fScott Bardone u32 len_gen; 103559fb51ba7e66fe298b8355fabde1275b7def35fScott Bardone u32 flags; 104559fb51ba7e66fe298b8355fabde1275b7def35fScott Bardone u32 addr_hi; 1058199d3a79c224bbe5943fa08684e1f93a17881b0Christoph Lameter}; 1068199d3a79c224bbe5943fa08684e1f93a17881b0Christoph Lameter 1078199d3a79c224bbe5943fa08684e1f93a17881b0Christoph Lameterstruct freelQ_e { 108559fb51ba7e66fe298b8355fabde1275b7def35fScott Bardone u32 addr_lo; 109559fb51ba7e66fe298b8355fabde1275b7def35fScott Bardone u32 len_gen; 110559fb51ba7e66fe298b8355fabde1275b7def35fScott Bardone u32 gen2; 111559fb51ba7e66fe298b8355fabde1275b7def35fScott Bardone u32 addr_hi; 1128199d3a79c224bbe5943fa08684e1f93a17881b0Christoph Lameter}; 1138199d3a79c224bbe5943fa08684e1f93a17881b0Christoph Lameter 1148199d3a79c224bbe5943fa08684e1f93a17881b0Christoph Lameterstruct respQ_e { 1158199d3a79c224bbe5943fa08684e1f93a17881b0Christoph Lameter u32 Qsleeping : 4; 1168199d3a79c224bbe5943fa08684e1f93a17881b0Christoph Lameter u32 Cmdq1CreditReturn : 5; 1178199d3a79c224bbe5943fa08684e1f93a17881b0Christoph Lameter u32 Cmdq1DmaComplete : 5; 1188199d3a79c224bbe5943fa08684e1f93a17881b0Christoph Lameter u32 Cmdq0CreditReturn : 5; 1198199d3a79c224bbe5943fa08684e1f93a17881b0Christoph Lameter u32 Cmdq0DmaComplete : 5; 1208199d3a79c224bbe5943fa08684e1f93a17881b0Christoph Lameter u32 FreelistQid : 2; 1218199d3a79c224bbe5943fa08684e1f93a17881b0Christoph Lameter u32 CreditValid : 1; 1228199d3a79c224bbe5943fa08684e1f93a17881b0Christoph Lameter u32 DataValid : 1; 1238199d3a79c224bbe5943fa08684e1f93a17881b0Christoph Lameter u32 Offload : 1; 1248199d3a79c224bbe5943fa08684e1f93a17881b0Christoph Lameter u32 Eop : 1; 1258199d3a79c224bbe5943fa08684e1f93a17881b0Christoph Lameter u32 Sop : 1; 1268199d3a79c224bbe5943fa08684e1f93a17881b0Christoph Lameter u32 GenerationBit : 1; 1278199d3a79c224bbe5943fa08684e1f93a17881b0Christoph Lameter u32 BufferLength; 1288199d3a79c224bbe5943fa08684e1f93a17881b0Christoph Lameter}; 1298199d3a79c224bbe5943fa08684e1f93a17881b0Christoph Lameter#elif defined(__LITTLE_ENDIAN_BITFIELD) 1308199d3a79c224bbe5943fa08684e1f93a17881b0Christoph Lameterstruct cmdQ_e { 131559fb51ba7e66fe298b8355fabde1275b7def35fScott Bardone u32 len_gen; 132559fb51ba7e66fe298b8355fabde1275b7def35fScott Bardone u32 addr_lo; 133559fb51ba7e66fe298b8355fabde1275b7def35fScott Bardone u32 addr_hi; 134559fb51ba7e66fe298b8355fabde1275b7def35fScott Bardone u32 flags; 1358199d3a79c224bbe5943fa08684e1f93a17881b0Christoph Lameter}; 1368199d3a79c224bbe5943fa08684e1f93a17881b0Christoph Lameter 1378199d3a79c224bbe5943fa08684e1f93a17881b0Christoph Lameterstruct freelQ_e { 138559fb51ba7e66fe298b8355fabde1275b7def35fScott Bardone u32 len_gen; 139559fb51ba7e66fe298b8355fabde1275b7def35fScott Bardone u32 addr_lo; 140559fb51ba7e66fe298b8355fabde1275b7def35fScott Bardone u32 addr_hi; 141559fb51ba7e66fe298b8355fabde1275b7def35fScott Bardone u32 gen2; 1428199d3a79c224bbe5943fa08684e1f93a17881b0Christoph Lameter}; 1438199d3a79c224bbe5943fa08684e1f93a17881b0Christoph Lameter 1448199d3a79c224bbe5943fa08684e1f93a17881b0Christoph Lameterstruct respQ_e { 1458199d3a79c224bbe5943fa08684e1f93a17881b0Christoph Lameter u32 BufferLength; 1468199d3a79c224bbe5943fa08684e1f93a17881b0Christoph Lameter u32 GenerationBit : 1; 1478199d3a79c224bbe5943fa08684e1f93a17881b0Christoph Lameter u32 Sop : 1; 1488199d3a79c224bbe5943fa08684e1f93a17881b0Christoph Lameter u32 Eop : 1; 1498199d3a79c224bbe5943fa08684e1f93a17881b0Christoph Lameter u32 Offload : 1; 1508199d3a79c224bbe5943fa08684e1f93a17881b0Christoph Lameter u32 DataValid : 1; 1518199d3a79c224bbe5943fa08684e1f93a17881b0Christoph Lameter u32 CreditValid : 1; 1528199d3a79c224bbe5943fa08684e1f93a17881b0Christoph Lameter u32 FreelistQid : 2; 1538199d3a79c224bbe5943fa08684e1f93a17881b0Christoph Lameter u32 Cmdq0DmaComplete : 5; 1548199d3a79c224bbe5943fa08684e1f93a17881b0Christoph Lameter u32 Cmdq0CreditReturn : 5; 1558199d3a79c224bbe5943fa08684e1f93a17881b0Christoph Lameter u32 Cmdq1DmaComplete : 5; 1568199d3a79c224bbe5943fa08684e1f93a17881b0Christoph Lameter u32 Cmdq1CreditReturn : 5; 1578199d3a79c224bbe5943fa08684e1f93a17881b0Christoph Lameter u32 Qsleeping : 4; 1588199d3a79c224bbe5943fa08684e1f93a17881b0Christoph Lameter} ; 1598199d3a79c224bbe5943fa08684e1f93a17881b0Christoph Lameter#endif 1608199d3a79c224bbe5943fa08684e1f93a17881b0Christoph Lameter 1618199d3a79c224bbe5943fa08684e1f93a17881b0Christoph Lameter/* 1628199d3a79c224bbe5943fa08684e1f93a17881b0Christoph Lameter * SW Context Command and Freelist Queue Descriptors 1638199d3a79c224bbe5943fa08684e1f93a17881b0Christoph Lameter */ 1648199d3a79c224bbe5943fa08684e1f93a17881b0Christoph Lameterstruct cmdQ_ce { 1658199d3a79c224bbe5943fa08684e1f93a17881b0Christoph Lameter struct sk_buff *skb; 166094f92a61aa044142c231e04c35c00a9cc70adbcFUJITA Tomonori DEFINE_DMA_UNMAP_ADDR(dma_addr); 167094f92a61aa044142c231e04c35c00a9cc70adbcFUJITA Tomonori DEFINE_DMA_UNMAP_LEN(dma_len); 1688199d3a79c224bbe5943fa08684e1f93a17881b0Christoph Lameter}; 1698199d3a79c224bbe5943fa08684e1f93a17881b0Christoph Lameter 1708199d3a79c224bbe5943fa08684e1f93a17881b0Christoph Lameterstruct freelQ_ce { 1718199d3a79c224bbe5943fa08684e1f93a17881b0Christoph Lameter struct sk_buff *skb; 172094f92a61aa044142c231e04c35c00a9cc70adbcFUJITA Tomonori DEFINE_DMA_UNMAP_ADDR(dma_addr); 173094f92a61aa044142c231e04c35c00a9cc70adbcFUJITA Tomonori DEFINE_DMA_UNMAP_LEN(dma_len); 1748199d3a79c224bbe5943fa08684e1f93a17881b0Christoph Lameter}; 1758199d3a79c224bbe5943fa08684e1f93a17881b0Christoph Lameter 1768199d3a79c224bbe5943fa08684e1f93a17881b0Christoph Lameter/* 177559fb51ba7e66fe298b8355fabde1275b7def35fScott Bardone * SW command, freelist and response rings 1788199d3a79c224bbe5943fa08684e1f93a17881b0Christoph Lameter */ 1798199d3a79c224bbe5943fa08684e1f93a17881b0Christoph Lameterstruct cmdQ { 180559fb51ba7e66fe298b8355fabde1275b7def35fScott Bardone unsigned long status; /* HW DMA fetch status */ 181559fb51ba7e66fe298b8355fabde1275b7def35fScott Bardone unsigned int in_use; /* # of in-use command descriptors */ 182559fb51ba7e66fe298b8355fabde1275b7def35fScott Bardone unsigned int size; /* # of descriptors */ 183f1d3d38af75789f1b82969b83b69cab540609789Stephen Hemminger unsigned int processed; /* total # of descs HW has processed */ 184f1d3d38af75789f1b82969b83b69cab540609789Stephen Hemminger unsigned int cleaned; /* total # of descs SW has reclaimed */ 185f1d3d38af75789f1b82969b83b69cab540609789Stephen Hemminger unsigned int stop_thres; /* SW TX queue suspend threshold */ 186559fb51ba7e66fe298b8355fabde1275b7def35fScott Bardone u16 pidx; /* producer index (SW) */ 187559fb51ba7e66fe298b8355fabde1275b7def35fScott Bardone u16 cidx; /* consumer index (HW) */ 188559fb51ba7e66fe298b8355fabde1275b7def35fScott Bardone u8 genbit; /* current generation (=valid) bit */ 189f1d3d38af75789f1b82969b83b69cab540609789Stephen Hemminger u8 sop; /* is next entry start of packet? */ 190559fb51ba7e66fe298b8355fabde1275b7def35fScott Bardone struct cmdQ_e *entries; /* HW command descriptor Q */ 191559fb51ba7e66fe298b8355fabde1275b7def35fScott Bardone struct cmdQ_ce *centries; /* SW command context descriptor Q */ 192559fb51ba7e66fe298b8355fabde1275b7def35fScott Bardone dma_addr_t dma_addr; /* DMA addr HW command descriptor Q */ 193356bd1460d1e1c4e433e4114fdac02139bddf17cFrancois Romieu spinlock_t lock; /* Lock to protect cmdQ enqueuing */ 1948199d3a79c224bbe5943fa08684e1f93a17881b0Christoph Lameter}; 1958199d3a79c224bbe5943fa08684e1f93a17881b0Christoph Lameter 1968199d3a79c224bbe5943fa08684e1f93a17881b0Christoph Lameterstruct freelQ { 197559fb51ba7e66fe298b8355fabde1275b7def35fScott Bardone unsigned int credits; /* # of available RX buffers */ 198559fb51ba7e66fe298b8355fabde1275b7def35fScott Bardone unsigned int size; /* free list capacity */ 199559fb51ba7e66fe298b8355fabde1275b7def35fScott Bardone u16 pidx; /* producer index (SW) */ 200559fb51ba7e66fe298b8355fabde1275b7def35fScott Bardone u16 cidx; /* consumer index (HW) */ 2018199d3a79c224bbe5943fa08684e1f93a17881b0Christoph Lameter u16 rx_buffer_size; /* Buffer size on this free list */ 202f1d3d38af75789f1b82969b83b69cab540609789Stephen Hemminger u16 dma_offset; /* DMA offset to align IP headers */ 203f1d3d38af75789f1b82969b83b69cab540609789Stephen Hemminger u16 recycleq_idx; /* skb recycle q to use */ 204559fb51ba7e66fe298b8355fabde1275b7def35fScott Bardone u8 genbit; /* current generation (=valid) bit */ 205559fb51ba7e66fe298b8355fabde1275b7def35fScott Bardone struct freelQ_e *entries; /* HW freelist descriptor Q */ 206559fb51ba7e66fe298b8355fabde1275b7def35fScott Bardone struct freelQ_ce *centries; /* SW freelist context descriptor Q */ 207559fb51ba7e66fe298b8355fabde1275b7def35fScott Bardone dma_addr_t dma_addr; /* DMA addr HW freelist descriptor Q */ 2088199d3a79c224bbe5943fa08684e1f93a17881b0Christoph Lameter}; 2098199d3a79c224bbe5943fa08684e1f93a17881b0Christoph Lameter 2108199d3a79c224bbe5943fa08684e1f93a17881b0Christoph Lameterstruct respQ { 211559fb51ba7e66fe298b8355fabde1275b7def35fScott Bardone unsigned int credits; /* credits to be returned to SGE */ 212559fb51ba7e66fe298b8355fabde1275b7def35fScott Bardone unsigned int size; /* # of response Q descriptors */ 213559fb51ba7e66fe298b8355fabde1275b7def35fScott Bardone u16 cidx; /* consumer index (SW) */ 214559fb51ba7e66fe298b8355fabde1275b7def35fScott Bardone u8 genbit; /* current generation(=valid) bit */ 2158199d3a79c224bbe5943fa08684e1f93a17881b0Christoph Lameter struct respQ_e *entries; /* HW response descriptor Q */ 216559fb51ba7e66fe298b8355fabde1275b7def35fScott Bardone dma_addr_t dma_addr; /* DMA addr HW response descriptor Q */ 217559fb51ba7e66fe298b8355fabde1275b7def35fScott Bardone}; 218559fb51ba7e66fe298b8355fabde1275b7def35fScott Bardone 219559fb51ba7e66fe298b8355fabde1275b7def35fScott Bardone/* Bit flags for cmdQ.status */ 220559fb51ba7e66fe298b8355fabde1275b7def35fScott Bardoneenum { 221559fb51ba7e66fe298b8355fabde1275b7def35fScott Bardone CMDQ_STAT_RUNNING = 1, /* fetch engine is running */ 222559fb51ba7e66fe298b8355fabde1275b7def35fScott Bardone CMDQ_STAT_LAST_PKT_DB = 2 /* last packet rung the doorbell */ 2238199d3a79c224bbe5943fa08684e1f93a17881b0Christoph Lameter}; 2248199d3a79c224bbe5943fa08684e1f93a17881b0Christoph Lameter 225f1d3d38af75789f1b82969b83b69cab540609789Stephen Hemminger/* T204 TX SW scheduler */ 226f1d3d38af75789f1b82969b83b69cab540609789Stephen Hemminger 227f1d3d38af75789f1b82969b83b69cab540609789Stephen Hemminger/* Per T204 TX port */ 228f1d3d38af75789f1b82969b83b69cab540609789Stephen Hemmingerstruct sched_port { 229f1d3d38af75789f1b82969b83b69cab540609789Stephen Hemminger unsigned int avail; /* available bits - quota */ 230f1d3d38af75789f1b82969b83b69cab540609789Stephen Hemminger unsigned int drain_bits_per_1024ns; /* drain rate */ 231f1d3d38af75789f1b82969b83b69cab540609789Stephen Hemminger unsigned int speed; /* drain rate, mbps */ 232f1d3d38af75789f1b82969b83b69cab540609789Stephen Hemminger unsigned int mtu; /* mtu size */ 233f1d3d38af75789f1b82969b83b69cab540609789Stephen Hemminger struct sk_buff_head skbq; /* pending skbs */ 234f1d3d38af75789f1b82969b83b69cab540609789Stephen Hemminger}; 235f1d3d38af75789f1b82969b83b69cab540609789Stephen Hemminger 236f1d3d38af75789f1b82969b83b69cab540609789Stephen Hemminger/* Per T204 device */ 237f1d3d38af75789f1b82969b83b69cab540609789Stephen Hemmingerstruct sched { 238f1d3d38af75789f1b82969b83b69cab540609789Stephen Hemminger ktime_t last_updated; /* last time quotas were computed */ 239356bd1460d1e1c4e433e4114fdac02139bddf17cFrancois Romieu unsigned int max_avail; /* max bits to be sent to any port */ 240356bd1460d1e1c4e433e4114fdac02139bddf17cFrancois Romieu unsigned int port; /* port index (round robin ports) */ 241356bd1460d1e1c4e433e4114fdac02139bddf17cFrancois Romieu unsigned int num; /* num skbs in per port queues */ 242f1d3d38af75789f1b82969b83b69cab540609789Stephen Hemminger struct sched_port p[MAX_NPORTS]; 243f1d3d38af75789f1b82969b83b69cab540609789Stephen Hemminger struct tasklet_struct sched_tsk;/* tasklet used to run scheduler */ 244f1d3d38af75789f1b82969b83b69cab540609789Stephen Hemminger}; 245f1d3d38af75789f1b82969b83b69cab540609789Stephen Hemmingerstatic void restart_sched(unsigned long); 246f1d3d38af75789f1b82969b83b69cab540609789Stephen Hemminger 247f1d3d38af75789f1b82969b83b69cab540609789Stephen Hemminger 2488199d3a79c224bbe5943fa08684e1f93a17881b0Christoph Lameter/* 2498199d3a79c224bbe5943fa08684e1f93a17881b0Christoph Lameter * Main SGE data structure 2508199d3a79c224bbe5943fa08684e1f93a17881b0Christoph Lameter * 2518199d3a79c224bbe5943fa08684e1f93a17881b0Christoph Lameter * Interrupts are handled by a single CPU and it is likely that on a MP system 2528199d3a79c224bbe5943fa08684e1f93a17881b0Christoph Lameter * the application is migrated to another CPU. In that scenario, we try to 2533ad2f3fbb961429d2aa627465ae4829758bc7e07Daniel Mack * separate the RX(in irq context) and TX state in order to decrease memory 2548199d3a79c224bbe5943fa08684e1f93a17881b0Christoph Lameter * contention. 2558199d3a79c224bbe5943fa08684e1f93a17881b0Christoph Lameter */ 2568199d3a79c224bbe5943fa08684e1f93a17881b0Christoph Lameterstruct sge { 257356bd1460d1e1c4e433e4114fdac02139bddf17cFrancois Romieu struct adapter *adapter; /* adapter backpointer */ 258559fb51ba7e66fe298b8355fabde1275b7def35fScott Bardone struct net_device *netdev; /* netdevice backpointer */ 259356bd1460d1e1c4e433e4114fdac02139bddf17cFrancois Romieu struct freelQ freelQ[SGE_FREELQ_N]; /* buffer free lists */ 260356bd1460d1e1c4e433e4114fdac02139bddf17cFrancois Romieu struct respQ respQ; /* response Q */ 261559fb51ba7e66fe298b8355fabde1275b7def35fScott Bardone unsigned long stopped_tx_queues; /* bitmap of suspended Tx queues */ 2628199d3a79c224bbe5943fa08684e1f93a17881b0Christoph Lameter unsigned int rx_pkt_pad; /* RX padding for L2 packets */ 2638199d3a79c224bbe5943fa08684e1f93a17881b0Christoph Lameter unsigned int jumbo_fl; /* jumbo freelist Q index */ 264559fb51ba7e66fe298b8355fabde1275b7def35fScott Bardone unsigned int intrtimer_nres; /* no-resource interrupt timer */ 265f1d3d38af75789f1b82969b83b69cab540609789Stephen Hemminger unsigned int fixed_intrtimer;/* non-adaptive interrupt timer */ 266559fb51ba7e66fe298b8355fabde1275b7def35fScott Bardone struct timer_list tx_reclaim_timer; /* reclaims TX buffers */ 267559fb51ba7e66fe298b8355fabde1275b7def35fScott Bardone struct timer_list espibug_timer; 268f1d3d38af75789f1b82969b83b69cab540609789Stephen Hemminger unsigned long espibug_timeout; 269f1d3d38af75789f1b82969b83b69cab540609789Stephen Hemminger struct sk_buff *espibug_skb[MAX_NPORTS]; 270559fb51ba7e66fe298b8355fabde1275b7def35fScott Bardone u32 sge_control; /* shadow value of sge control reg */ 271559fb51ba7e66fe298b8355fabde1275b7def35fScott Bardone struct sge_intr_counts stats; 27247d742752df4c1088589d4424840bc761613ab2aTejun Heo struct sge_port_stats __percpu *port_stats[MAX_NPORTS]; 273f1d3d38af75789f1b82969b83b69cab540609789Stephen Hemminger struct sched *tx_sched; 274559fb51ba7e66fe298b8355fabde1275b7def35fScott Bardone struct cmdQ cmdQ[SGE_CMDQ_N] ____cacheline_aligned_in_smp; 2758199d3a79c224bbe5943fa08684e1f93a17881b0Christoph Lameter}; 2768199d3a79c224bbe5943fa08684e1f93a17881b0Christoph Lameter 277215faf9c5f6e319e97edea9e178123e07825c14dJoe Perchesstatic const u8 ch_mac_addr[ETH_ALEN] = { 278215faf9c5f6e319e97edea9e178123e07825c14dJoe Perches 0x0, 0x7, 0x43, 0x0, 0x0, 0x0 279215faf9c5f6e319e97edea9e178123e07825c14dJoe Perches}; 280215faf9c5f6e319e97edea9e178123e07825c14dJoe Perches 2818199d3a79c224bbe5943fa08684e1f93a17881b0Christoph Lameter/* 282f1d3d38af75789f1b82969b83b69cab540609789Stephen Hemminger * stop tasklet and free all pending skb's 283f1d3d38af75789f1b82969b83b69cab540609789Stephen Hemminger */ 284f1d3d38af75789f1b82969b83b69cab540609789Stephen Hemmingerstatic void tx_sched_stop(struct sge *sge) 285f1d3d38af75789f1b82969b83b69cab540609789Stephen Hemminger{ 286f1d3d38af75789f1b82969b83b69cab540609789Stephen Hemminger struct sched *s = sge->tx_sched; 287f1d3d38af75789f1b82969b83b69cab540609789Stephen Hemminger int i; 288f1d3d38af75789f1b82969b83b69cab540609789Stephen Hemminger 289f1d3d38af75789f1b82969b83b69cab540609789Stephen Hemminger tasklet_kill(&s->sched_tsk); 290f1d3d38af75789f1b82969b83b69cab540609789Stephen Hemminger 291f1d3d38af75789f1b82969b83b69cab540609789Stephen Hemminger for (i = 0; i < MAX_NPORTS; i++) 292f1d3d38af75789f1b82969b83b69cab540609789Stephen Hemminger __skb_queue_purge(&s->p[s->port].skbq); 293f1d3d38af75789f1b82969b83b69cab540609789Stephen Hemminger} 294f1d3d38af75789f1b82969b83b69cab540609789Stephen Hemminger 295f1d3d38af75789f1b82969b83b69cab540609789Stephen Hemminger/* 296f1d3d38af75789f1b82969b83b69cab540609789Stephen Hemminger * t1_sched_update_parms() is called when the MTU or link speed changes. It 297f1d3d38af75789f1b82969b83b69cab540609789Stephen Hemminger * re-computes scheduler parameters to scope with the change. 298f1d3d38af75789f1b82969b83b69cab540609789Stephen Hemminger */ 299f1d3d38af75789f1b82969b83b69cab540609789Stephen Hemmingerunsigned int t1_sched_update_parms(struct sge *sge, unsigned int port, 300f1d3d38af75789f1b82969b83b69cab540609789Stephen Hemminger unsigned int mtu, unsigned int speed) 301f1d3d38af75789f1b82969b83b69cab540609789Stephen Hemminger{ 302f1d3d38af75789f1b82969b83b69cab540609789Stephen Hemminger struct sched *s = sge->tx_sched; 303f1d3d38af75789f1b82969b83b69cab540609789Stephen Hemminger struct sched_port *p = &s->p[port]; 304f1d3d38af75789f1b82969b83b69cab540609789Stephen Hemminger unsigned int max_avail_segs; 305f1d3d38af75789f1b82969b83b69cab540609789Stephen Hemminger 306f1d3d38af75789f1b82969b83b69cab540609789Stephen Hemminger pr_debug("t1_sched_update_params mtu=%d speed=%d\n", mtu, speed); 307f1d3d38af75789f1b82969b83b69cab540609789Stephen Hemminger if (speed) 308f1d3d38af75789f1b82969b83b69cab540609789Stephen Hemminger p->speed = speed; 309f1d3d38af75789f1b82969b83b69cab540609789Stephen Hemminger if (mtu) 310f1d3d38af75789f1b82969b83b69cab540609789Stephen Hemminger p->mtu = mtu; 311f1d3d38af75789f1b82969b83b69cab540609789Stephen Hemminger 312f1d3d38af75789f1b82969b83b69cab540609789Stephen Hemminger if (speed || mtu) { 313f1d3d38af75789f1b82969b83b69cab540609789Stephen Hemminger unsigned long long drain = 1024ULL * p->speed * (p->mtu - 40); 314f1d3d38af75789f1b82969b83b69cab540609789Stephen Hemminger do_div(drain, (p->mtu + 50) * 1000); 315f1d3d38af75789f1b82969b83b69cab540609789Stephen Hemminger p->drain_bits_per_1024ns = (unsigned int) drain; 316f1d3d38af75789f1b82969b83b69cab540609789Stephen Hemminger 317f1d3d38af75789f1b82969b83b69cab540609789Stephen Hemminger if (p->speed < 1000) 318f1d3d38af75789f1b82969b83b69cab540609789Stephen Hemminger p->drain_bits_per_1024ns = 319f1d3d38af75789f1b82969b83b69cab540609789Stephen Hemminger 90 * p->drain_bits_per_1024ns / 100; 320f1d3d38af75789f1b82969b83b69cab540609789Stephen Hemminger } 321f1d3d38af75789f1b82969b83b69cab540609789Stephen Hemminger 322f1d3d38af75789f1b82969b83b69cab540609789Stephen Hemminger if (board_info(sge->adapter)->board == CHBT_BOARD_CHT204) { 323f1d3d38af75789f1b82969b83b69cab540609789Stephen Hemminger p->drain_bits_per_1024ns -= 16; 324f1d3d38af75789f1b82969b83b69cab540609789Stephen Hemminger s->max_avail = max(4096U, p->mtu + 16 + 14 + 4); 325f1d3d38af75789f1b82969b83b69cab540609789Stephen Hemminger max_avail_segs = max(1U, 4096 / (p->mtu - 40)); 326f1d3d38af75789f1b82969b83b69cab540609789Stephen Hemminger } else { 327f1d3d38af75789f1b82969b83b69cab540609789Stephen Hemminger s->max_avail = 16384; 328f1d3d38af75789f1b82969b83b69cab540609789Stephen Hemminger max_avail_segs = max(1U, 9000 / (p->mtu - 40)); 329f1d3d38af75789f1b82969b83b69cab540609789Stephen Hemminger } 330f1d3d38af75789f1b82969b83b69cab540609789Stephen Hemminger 331f1d3d38af75789f1b82969b83b69cab540609789Stephen Hemminger pr_debug("t1_sched_update_parms: mtu %u speed %u max_avail %u " 332f1d3d38af75789f1b82969b83b69cab540609789Stephen Hemminger "max_avail_segs %u drain_bits_per_1024ns %u\n", p->mtu, 333f1d3d38af75789f1b82969b83b69cab540609789Stephen Hemminger p->speed, s->max_avail, max_avail_segs, 334f1d3d38af75789f1b82969b83b69cab540609789Stephen Hemminger p->drain_bits_per_1024ns); 335f1d3d38af75789f1b82969b83b69cab540609789Stephen Hemminger 336f1d3d38af75789f1b82969b83b69cab540609789Stephen Hemminger return max_avail_segs * (p->mtu - 40); 337f1d3d38af75789f1b82969b83b69cab540609789Stephen Hemminger} 338f1d3d38af75789f1b82969b83b69cab540609789Stephen Hemminger 33968d579fb93df0db0442f90106b19a8dc701ecef1Adrian Bunk#if 0 34068d579fb93df0db0442f90106b19a8dc701ecef1Adrian Bunk 341f1d3d38af75789f1b82969b83b69cab540609789Stephen Hemminger/* 342f1d3d38af75789f1b82969b83b69cab540609789Stephen Hemminger * t1_sched_max_avail_bytes() tells the scheduler the maximum amount of 343f1d3d38af75789f1b82969b83b69cab540609789Stephen Hemminger * data that can be pushed per port. 344f1d3d38af75789f1b82969b83b69cab540609789Stephen Hemminger */ 345f1d3d38af75789f1b82969b83b69cab540609789Stephen Hemmingervoid t1_sched_set_max_avail_bytes(struct sge *sge, unsigned int val) 346f1d3d38af75789f1b82969b83b69cab540609789Stephen Hemminger{ 347f1d3d38af75789f1b82969b83b69cab540609789Stephen Hemminger struct sched *s = sge->tx_sched; 348f1d3d38af75789f1b82969b83b69cab540609789Stephen Hemminger unsigned int i; 349f1d3d38af75789f1b82969b83b69cab540609789Stephen Hemminger 350f1d3d38af75789f1b82969b83b69cab540609789Stephen Hemminger s->max_avail = val; 351f1d3d38af75789f1b82969b83b69cab540609789Stephen Hemminger for (i = 0; i < MAX_NPORTS; i++) 352f1d3d38af75789f1b82969b83b69cab540609789Stephen Hemminger t1_sched_update_parms(sge, i, 0, 0); 353f1d3d38af75789f1b82969b83b69cab540609789Stephen Hemminger} 354f1d3d38af75789f1b82969b83b69cab540609789Stephen Hemminger 355f1d3d38af75789f1b82969b83b69cab540609789Stephen Hemminger/* 356f1d3d38af75789f1b82969b83b69cab540609789Stephen Hemminger * t1_sched_set_drain_bits_per_us() tells the scheduler at which rate a port 357f1d3d38af75789f1b82969b83b69cab540609789Stephen Hemminger * is draining. 358f1d3d38af75789f1b82969b83b69cab540609789Stephen Hemminger */ 359f1d3d38af75789f1b82969b83b69cab540609789Stephen Hemmingervoid t1_sched_set_drain_bits_per_us(struct sge *sge, unsigned int port, 360f1d3d38af75789f1b82969b83b69cab540609789Stephen Hemminger unsigned int val) 361f1d3d38af75789f1b82969b83b69cab540609789Stephen Hemminger{ 362f1d3d38af75789f1b82969b83b69cab540609789Stephen Hemminger struct sched *s = sge->tx_sched; 363f1d3d38af75789f1b82969b83b69cab540609789Stephen Hemminger struct sched_port *p = &s->p[port]; 364f1d3d38af75789f1b82969b83b69cab540609789Stephen Hemminger p->drain_bits_per_1024ns = val * 1024 / 1000; 365f1d3d38af75789f1b82969b83b69cab540609789Stephen Hemminger t1_sched_update_parms(sge, port, 0, 0); 366f1d3d38af75789f1b82969b83b69cab540609789Stephen Hemminger} 367f1d3d38af75789f1b82969b83b69cab540609789Stephen Hemminger 36868d579fb93df0db0442f90106b19a8dc701ecef1Adrian Bunk#endif /* 0 */ 36968d579fb93df0db0442f90106b19a8dc701ecef1Adrian Bunk 370f1d3d38af75789f1b82969b83b69cab540609789Stephen Hemminger 371f1d3d38af75789f1b82969b83b69cab540609789Stephen Hemminger/* 372f1d3d38af75789f1b82969b83b69cab540609789Stephen Hemminger * get_clock() implements a ns clock (see ktime_get) 373f1d3d38af75789f1b82969b83b69cab540609789Stephen Hemminger */ 374f1d3d38af75789f1b82969b83b69cab540609789Stephen Hemmingerstatic inline ktime_t get_clock(void) 375f1d3d38af75789f1b82969b83b69cab540609789Stephen Hemminger{ 376f1d3d38af75789f1b82969b83b69cab540609789Stephen Hemminger struct timespec ts; 377f1d3d38af75789f1b82969b83b69cab540609789Stephen Hemminger 378f1d3d38af75789f1b82969b83b69cab540609789Stephen Hemminger ktime_get_ts(&ts); 379f1d3d38af75789f1b82969b83b69cab540609789Stephen Hemminger return timespec_to_ktime(ts); 380f1d3d38af75789f1b82969b83b69cab540609789Stephen Hemminger} 381f1d3d38af75789f1b82969b83b69cab540609789Stephen Hemminger 382f1d3d38af75789f1b82969b83b69cab540609789Stephen Hemminger/* 383f1d3d38af75789f1b82969b83b69cab540609789Stephen Hemminger * tx_sched_init() allocates resources and does basic initialization. 384f1d3d38af75789f1b82969b83b69cab540609789Stephen Hemminger */ 385f1d3d38af75789f1b82969b83b69cab540609789Stephen Hemmingerstatic int tx_sched_init(struct sge *sge) 386f1d3d38af75789f1b82969b83b69cab540609789Stephen Hemminger{ 387f1d3d38af75789f1b82969b83b69cab540609789Stephen Hemminger struct sched *s; 388f1d3d38af75789f1b82969b83b69cab540609789Stephen Hemminger int i; 389f1d3d38af75789f1b82969b83b69cab540609789Stephen Hemminger 390f1d3d38af75789f1b82969b83b69cab540609789Stephen Hemminger s = kzalloc(sizeof (struct sched), GFP_KERNEL); 391f1d3d38af75789f1b82969b83b69cab540609789Stephen Hemminger if (!s) 392f1d3d38af75789f1b82969b83b69cab540609789Stephen Hemminger return -ENOMEM; 393f1d3d38af75789f1b82969b83b69cab540609789Stephen Hemminger 394f1d3d38af75789f1b82969b83b69cab540609789Stephen Hemminger pr_debug("tx_sched_init\n"); 395f1d3d38af75789f1b82969b83b69cab540609789Stephen Hemminger tasklet_init(&s->sched_tsk, restart_sched, (unsigned long) sge); 396f1d3d38af75789f1b82969b83b69cab540609789Stephen Hemminger sge->tx_sched = s; 397f1d3d38af75789f1b82969b83b69cab540609789Stephen Hemminger 398f1d3d38af75789f1b82969b83b69cab540609789Stephen Hemminger for (i = 0; i < MAX_NPORTS; i++) { 399f1d3d38af75789f1b82969b83b69cab540609789Stephen Hemminger skb_queue_head_init(&s->p[i].skbq); 400f1d3d38af75789f1b82969b83b69cab540609789Stephen Hemminger t1_sched_update_parms(sge, i, 1500, 1000); 401f1d3d38af75789f1b82969b83b69cab540609789Stephen Hemminger } 402f1d3d38af75789f1b82969b83b69cab540609789Stephen Hemminger 403f1d3d38af75789f1b82969b83b69cab540609789Stephen Hemminger return 0; 404f1d3d38af75789f1b82969b83b69cab540609789Stephen Hemminger} 405f1d3d38af75789f1b82969b83b69cab540609789Stephen Hemminger 406f1d3d38af75789f1b82969b83b69cab540609789Stephen Hemminger/* 407f1d3d38af75789f1b82969b83b69cab540609789Stephen Hemminger * sched_update_avail() computes the delta since the last time it was called 408f1d3d38af75789f1b82969b83b69cab540609789Stephen Hemminger * and updates the per port quota (number of bits that can be sent to the any 409f1d3d38af75789f1b82969b83b69cab540609789Stephen Hemminger * port). 410f1d3d38af75789f1b82969b83b69cab540609789Stephen Hemminger */ 411f1d3d38af75789f1b82969b83b69cab540609789Stephen Hemmingerstatic inline int sched_update_avail(struct sge *sge) 412f1d3d38af75789f1b82969b83b69cab540609789Stephen Hemminger{ 413f1d3d38af75789f1b82969b83b69cab540609789Stephen Hemminger struct sched *s = sge->tx_sched; 414f1d3d38af75789f1b82969b83b69cab540609789Stephen Hemminger ktime_t now = get_clock(); 415f1d3d38af75789f1b82969b83b69cab540609789Stephen Hemminger unsigned int i; 416f1d3d38af75789f1b82969b83b69cab540609789Stephen Hemminger long long delta_time_ns; 417f1d3d38af75789f1b82969b83b69cab540609789Stephen Hemminger 418f1d3d38af75789f1b82969b83b69cab540609789Stephen Hemminger delta_time_ns = ktime_to_ns(ktime_sub(now, s->last_updated)); 419f1d3d38af75789f1b82969b83b69cab540609789Stephen Hemminger 420f1d3d38af75789f1b82969b83b69cab540609789Stephen Hemminger pr_debug("sched_update_avail delta=%lld\n", delta_time_ns); 421f1d3d38af75789f1b82969b83b69cab540609789Stephen Hemminger if (delta_time_ns < 15000) 422f1d3d38af75789f1b82969b83b69cab540609789Stephen Hemminger return 0; 423f1d3d38af75789f1b82969b83b69cab540609789Stephen Hemminger 424f1d3d38af75789f1b82969b83b69cab540609789Stephen Hemminger for (i = 0; i < MAX_NPORTS; i++) { 425f1d3d38af75789f1b82969b83b69cab540609789Stephen Hemminger struct sched_port *p = &s->p[i]; 426f1d3d38af75789f1b82969b83b69cab540609789Stephen Hemminger unsigned int delta_avail; 427f1d3d38af75789f1b82969b83b69cab540609789Stephen Hemminger 428f1d3d38af75789f1b82969b83b69cab540609789Stephen Hemminger delta_avail = (p->drain_bits_per_1024ns * delta_time_ns) >> 13; 429f1d3d38af75789f1b82969b83b69cab540609789Stephen Hemminger p->avail = min(p->avail + delta_avail, s->max_avail); 430f1d3d38af75789f1b82969b83b69cab540609789Stephen Hemminger } 431f1d3d38af75789f1b82969b83b69cab540609789Stephen Hemminger 432f1d3d38af75789f1b82969b83b69cab540609789Stephen Hemminger s->last_updated = now; 433f1d3d38af75789f1b82969b83b69cab540609789Stephen Hemminger 434f1d3d38af75789f1b82969b83b69cab540609789Stephen Hemminger return 1; 435f1d3d38af75789f1b82969b83b69cab540609789Stephen Hemminger} 436f1d3d38af75789f1b82969b83b69cab540609789Stephen Hemminger 437f1d3d38af75789f1b82969b83b69cab540609789Stephen Hemminger/* 438f1d3d38af75789f1b82969b83b69cab540609789Stephen Hemminger * sched_skb() is called from two different places. In the tx path, any 439f1d3d38af75789f1b82969b83b69cab540609789Stephen Hemminger * packet generating load on an output port will call sched_skb() 440f1d3d38af75789f1b82969b83b69cab540609789Stephen Hemminger * (skb != NULL). In addition, sched_skb() is called from the irq/soft irq 441f1d3d38af75789f1b82969b83b69cab540609789Stephen Hemminger * context (skb == NULL). 442f1d3d38af75789f1b82969b83b69cab540609789Stephen Hemminger * The scheduler only returns a skb (which will then be sent) if the 443f1d3d38af75789f1b82969b83b69cab540609789Stephen Hemminger * length of the skb is <= the current quota of the output port. 444f1d3d38af75789f1b82969b83b69cab540609789Stephen Hemminger */ 445f1d3d38af75789f1b82969b83b69cab540609789Stephen Hemmingerstatic struct sk_buff *sched_skb(struct sge *sge, struct sk_buff *skb, 446f1d3d38af75789f1b82969b83b69cab540609789Stephen Hemminger unsigned int credits) 447f1d3d38af75789f1b82969b83b69cab540609789Stephen Hemminger{ 448f1d3d38af75789f1b82969b83b69cab540609789Stephen Hemminger struct sched *s = sge->tx_sched; 449f1d3d38af75789f1b82969b83b69cab540609789Stephen Hemminger struct sk_buff_head *skbq; 450f1d3d38af75789f1b82969b83b69cab540609789Stephen Hemminger unsigned int i, len, update = 1; 451f1d3d38af75789f1b82969b83b69cab540609789Stephen Hemminger 452f1d3d38af75789f1b82969b83b69cab540609789Stephen Hemminger pr_debug("sched_skb %p\n", skb); 453f1d3d38af75789f1b82969b83b69cab540609789Stephen Hemminger if (!skb) { 454f1d3d38af75789f1b82969b83b69cab540609789Stephen Hemminger if (!s->num) 455f1d3d38af75789f1b82969b83b69cab540609789Stephen Hemminger return NULL; 456f1d3d38af75789f1b82969b83b69cab540609789Stephen Hemminger } else { 457f1d3d38af75789f1b82969b83b69cab540609789Stephen Hemminger skbq = &s->p[skb->dev->if_port].skbq; 458f1d3d38af75789f1b82969b83b69cab540609789Stephen Hemminger __skb_queue_tail(skbq, skb); 459f1d3d38af75789f1b82969b83b69cab540609789Stephen Hemminger s->num++; 460f1d3d38af75789f1b82969b83b69cab540609789Stephen Hemminger skb = NULL; 461f1d3d38af75789f1b82969b83b69cab540609789Stephen Hemminger } 462f1d3d38af75789f1b82969b83b69cab540609789Stephen Hemminger 463f1d3d38af75789f1b82969b83b69cab540609789Stephen Hemminger if (credits < MAX_SKB_FRAGS + 1) 464f1d3d38af75789f1b82969b83b69cab540609789Stephen Hemminger goto out; 465f1d3d38af75789f1b82969b83b69cab540609789Stephen Hemminger 466356bd1460d1e1c4e433e4114fdac02139bddf17cFrancois Romieuagain: 467f1d3d38af75789f1b82969b83b69cab540609789Stephen Hemminger for (i = 0; i < MAX_NPORTS; i++) { 46818d777a5bcf31491021f527d32b72dd16f911fe0David S. Miller s->port = (s->port + 1) & (MAX_NPORTS - 1); 469f1d3d38af75789f1b82969b83b69cab540609789Stephen Hemminger skbq = &s->p[s->port].skbq; 470f1d3d38af75789f1b82969b83b69cab540609789Stephen Hemminger 471f1d3d38af75789f1b82969b83b69cab540609789Stephen Hemminger skb = skb_peek(skbq); 472f1d3d38af75789f1b82969b83b69cab540609789Stephen Hemminger 473f1d3d38af75789f1b82969b83b69cab540609789Stephen Hemminger if (!skb) 474f1d3d38af75789f1b82969b83b69cab540609789Stephen Hemminger continue; 475f1d3d38af75789f1b82969b83b69cab540609789Stephen Hemminger 476f1d3d38af75789f1b82969b83b69cab540609789Stephen Hemminger len = skb->len; 477f1d3d38af75789f1b82969b83b69cab540609789Stephen Hemminger if (len <= s->p[s->port].avail) { 478f1d3d38af75789f1b82969b83b69cab540609789Stephen Hemminger s->p[s->port].avail -= len; 479f1d3d38af75789f1b82969b83b69cab540609789Stephen Hemminger s->num--; 480f1d3d38af75789f1b82969b83b69cab540609789Stephen Hemminger __skb_unlink(skb, skbq); 481f1d3d38af75789f1b82969b83b69cab540609789Stephen Hemminger goto out; 482f1d3d38af75789f1b82969b83b69cab540609789Stephen Hemminger } 483f1d3d38af75789f1b82969b83b69cab540609789Stephen Hemminger skb = NULL; 484f1d3d38af75789f1b82969b83b69cab540609789Stephen Hemminger } 485f1d3d38af75789f1b82969b83b69cab540609789Stephen Hemminger 486f1d3d38af75789f1b82969b83b69cab540609789Stephen Hemminger if (update-- && sched_update_avail(sge)) 487f1d3d38af75789f1b82969b83b69cab540609789Stephen Hemminger goto again; 488f1d3d38af75789f1b82969b83b69cab540609789Stephen Hemminger 489356bd1460d1e1c4e433e4114fdac02139bddf17cFrancois Romieuout: 490356bd1460d1e1c4e433e4114fdac02139bddf17cFrancois Romieu /* If there are more pending skbs, we use the hardware to schedule us 491f1d3d38af75789f1b82969b83b69cab540609789Stephen Hemminger * again. 492f1d3d38af75789f1b82969b83b69cab540609789Stephen Hemminger */ 493f1d3d38af75789f1b82969b83b69cab540609789Stephen Hemminger if (s->num && !skb) { 494f1d3d38af75789f1b82969b83b69cab540609789Stephen Hemminger struct cmdQ *q = &sge->cmdQ[0]; 495f1d3d38af75789f1b82969b83b69cab540609789Stephen Hemminger clear_bit(CMDQ_STAT_LAST_PKT_DB, &q->status); 496f1d3d38af75789f1b82969b83b69cab540609789Stephen Hemminger if (test_and_set_bit(CMDQ_STAT_RUNNING, &q->status) == 0) { 497f1d3d38af75789f1b82969b83b69cab540609789Stephen Hemminger set_bit(CMDQ_STAT_LAST_PKT_DB, &q->status); 498f1d3d38af75789f1b82969b83b69cab540609789Stephen Hemminger writel(F_CMDQ0_ENABLE, sge->adapter->regs + A_SG_DOORBELL); 499f1d3d38af75789f1b82969b83b69cab540609789Stephen Hemminger } 500f1d3d38af75789f1b82969b83b69cab540609789Stephen Hemminger } 501f1d3d38af75789f1b82969b83b69cab540609789Stephen Hemminger pr_debug("sched_skb ret %p\n", skb); 502f1d3d38af75789f1b82969b83b69cab540609789Stephen Hemminger 503f1d3d38af75789f1b82969b83b69cab540609789Stephen Hemminger return skb; 504f1d3d38af75789f1b82969b83b69cab540609789Stephen Hemminger} 505f1d3d38af75789f1b82969b83b69cab540609789Stephen Hemminger 506f1d3d38af75789f1b82969b83b69cab540609789Stephen Hemminger/* 5078199d3a79c224bbe5943fa08684e1f93a17881b0Christoph Lameter * PIO to indicate that memory mapped Q contains valid descriptor(s). 5088199d3a79c224bbe5943fa08684e1f93a17881b0Christoph Lameter */ 509559fb51ba7e66fe298b8355fabde1275b7def35fScott Bardonestatic inline void doorbell_pio(struct adapter *adapter, u32 val) 5108199d3a79c224bbe5943fa08684e1f93a17881b0Christoph Lameter{ 5118199d3a79c224bbe5943fa08684e1f93a17881b0Christoph Lameter wmb(); 512559fb51ba7e66fe298b8355fabde1275b7def35fScott Bardone writel(val, adapter->regs + A_SG_DOORBELL); 5138199d3a79c224bbe5943fa08684e1f93a17881b0Christoph Lameter} 5148199d3a79c224bbe5943fa08684e1f93a17881b0Christoph Lameter 5158199d3a79c224bbe5943fa08684e1f93a17881b0Christoph Lameter/* 5168199d3a79c224bbe5943fa08684e1f93a17881b0Christoph Lameter * Frees all RX buffers on the freelist Q. The caller must make sure that 5178199d3a79c224bbe5943fa08684e1f93a17881b0Christoph Lameter * the SGE is turned off before calling this function. 5188199d3a79c224bbe5943fa08684e1f93a17881b0Christoph Lameter */ 519559fb51ba7e66fe298b8355fabde1275b7def35fScott Bardonestatic void free_freelQ_buffers(struct pci_dev *pdev, struct freelQ *q) 5208199d3a79c224bbe5943fa08684e1f93a17881b0Christoph Lameter{ 521559fb51ba7e66fe298b8355fabde1275b7def35fScott Bardone unsigned int cidx = q->cidx; 5228199d3a79c224bbe5943fa08684e1f93a17881b0Christoph Lameter 523559fb51ba7e66fe298b8355fabde1275b7def35fScott Bardone while (q->credits--) { 524559fb51ba7e66fe298b8355fabde1275b7def35fScott Bardone struct freelQ_ce *ce = &q->centries[cidx]; 5258199d3a79c224bbe5943fa08684e1f93a17881b0Christoph Lameter 526094f92a61aa044142c231e04c35c00a9cc70adbcFUJITA Tomonori pci_unmap_single(pdev, dma_unmap_addr(ce, dma_addr), 527094f92a61aa044142c231e04c35c00a9cc70adbcFUJITA Tomonori dma_unmap_len(ce, dma_len), 5288199d3a79c224bbe5943fa08684e1f93a17881b0Christoph Lameter PCI_DMA_FROMDEVICE); 5298199d3a79c224bbe5943fa08684e1f93a17881b0Christoph Lameter dev_kfree_skb(ce->skb); 5308199d3a79c224bbe5943fa08684e1f93a17881b0Christoph Lameter ce->skb = NULL; 531559fb51ba7e66fe298b8355fabde1275b7def35fScott Bardone if (++cidx == q->size) 5328199d3a79c224bbe5943fa08684e1f93a17881b0Christoph Lameter cidx = 0; 5338199d3a79c224bbe5943fa08684e1f93a17881b0Christoph Lameter } 5348199d3a79c224bbe5943fa08684e1f93a17881b0Christoph Lameter} 5358199d3a79c224bbe5943fa08684e1f93a17881b0Christoph Lameter 5368199d3a79c224bbe5943fa08684e1f93a17881b0Christoph Lameter/* 5378199d3a79c224bbe5943fa08684e1f93a17881b0Christoph Lameter * Free RX free list and response queue resources. 5388199d3a79c224bbe5943fa08684e1f93a17881b0Christoph Lameter */ 5398199d3a79c224bbe5943fa08684e1f93a17881b0Christoph Lameterstatic void free_rx_resources(struct sge *sge) 5408199d3a79c224bbe5943fa08684e1f93a17881b0Christoph Lameter{ 5418199d3a79c224bbe5943fa08684e1f93a17881b0Christoph Lameter struct pci_dev *pdev = sge->adapter->pdev; 5428199d3a79c224bbe5943fa08684e1f93a17881b0Christoph Lameter unsigned int size, i; 5438199d3a79c224bbe5943fa08684e1f93a17881b0Christoph Lameter 5448199d3a79c224bbe5943fa08684e1f93a17881b0Christoph Lameter if (sge->respQ.entries) { 545559fb51ba7e66fe298b8355fabde1275b7def35fScott Bardone size = sizeof(struct respQ_e) * sge->respQ.size; 5468199d3a79c224bbe5943fa08684e1f93a17881b0Christoph Lameter pci_free_consistent(pdev, size, sge->respQ.entries, 5478199d3a79c224bbe5943fa08684e1f93a17881b0Christoph Lameter sge->respQ.dma_addr); 5488199d3a79c224bbe5943fa08684e1f93a17881b0Christoph Lameter } 5498199d3a79c224bbe5943fa08684e1f93a17881b0Christoph Lameter 5508199d3a79c224bbe5943fa08684e1f93a17881b0Christoph Lameter for (i = 0; i < SGE_FREELQ_N; i++) { 551559fb51ba7e66fe298b8355fabde1275b7def35fScott Bardone struct freelQ *q = &sge->freelQ[i]; 5528199d3a79c224bbe5943fa08684e1f93a17881b0Christoph Lameter 553559fb51ba7e66fe298b8355fabde1275b7def35fScott Bardone if (q->centries) { 554559fb51ba7e66fe298b8355fabde1275b7def35fScott Bardone free_freelQ_buffers(pdev, q); 555559fb51ba7e66fe298b8355fabde1275b7def35fScott Bardone kfree(q->centries); 5568199d3a79c224bbe5943fa08684e1f93a17881b0Christoph Lameter } 557559fb51ba7e66fe298b8355fabde1275b7def35fScott Bardone if (q->entries) { 558559fb51ba7e66fe298b8355fabde1275b7def35fScott Bardone size = sizeof(struct freelQ_e) * q->size; 559559fb51ba7e66fe298b8355fabde1275b7def35fScott Bardone pci_free_consistent(pdev, size, q->entries, 560559fb51ba7e66fe298b8355fabde1275b7def35fScott Bardone q->dma_addr); 5618199d3a79c224bbe5943fa08684e1f93a17881b0Christoph Lameter } 5628199d3a79c224bbe5943fa08684e1f93a17881b0Christoph Lameter } 5638199d3a79c224bbe5943fa08684e1f93a17881b0Christoph Lameter} 5648199d3a79c224bbe5943fa08684e1f93a17881b0Christoph Lameter 5658199d3a79c224bbe5943fa08684e1f93a17881b0Christoph Lameter/* 5668199d3a79c224bbe5943fa08684e1f93a17881b0Christoph Lameter * Allocates basic RX resources, consisting of memory mapped freelist Qs and a 567559fb51ba7e66fe298b8355fabde1275b7def35fScott Bardone * response queue. 5688199d3a79c224bbe5943fa08684e1f93a17881b0Christoph Lameter */ 5698199d3a79c224bbe5943fa08684e1f93a17881b0Christoph Lameterstatic int alloc_rx_resources(struct sge *sge, struct sge_params *p) 5708199d3a79c224bbe5943fa08684e1f93a17881b0Christoph Lameter{ 5718199d3a79c224bbe5943fa08684e1f93a17881b0Christoph Lameter struct pci_dev *pdev = sge->adapter->pdev; 5728199d3a79c224bbe5943fa08684e1f93a17881b0Christoph Lameter unsigned int size, i; 5738199d3a79c224bbe5943fa08684e1f93a17881b0Christoph Lameter 5748199d3a79c224bbe5943fa08684e1f93a17881b0Christoph Lameter for (i = 0; i < SGE_FREELQ_N; i++) { 575559fb51ba7e66fe298b8355fabde1275b7def35fScott Bardone struct freelQ *q = &sge->freelQ[i]; 576559fb51ba7e66fe298b8355fabde1275b7def35fScott Bardone 577559fb51ba7e66fe298b8355fabde1275b7def35fScott Bardone q->genbit = 1; 578559fb51ba7e66fe298b8355fabde1275b7def35fScott Bardone q->size = p->freelQ_size[i]; 579559fb51ba7e66fe298b8355fabde1275b7def35fScott Bardone q->dma_offset = sge->rx_pkt_pad ? 0 : NET_IP_ALIGN; 580559fb51ba7e66fe298b8355fabde1275b7def35fScott Bardone size = sizeof(struct freelQ_e) * q->size; 5813e0f75be52605a901165fa1d8acf4ffd37a4857bFrancois Romieu q->entries = pci_alloc_consistent(pdev, size, &q->dma_addr); 582559fb51ba7e66fe298b8355fabde1275b7def35fScott Bardone if (!q->entries) 5838199d3a79c224bbe5943fa08684e1f93a17881b0Christoph Lameter goto err_no_mem; 5843e0f75be52605a901165fa1d8acf4ffd37a4857bFrancois Romieu 585559fb51ba7e66fe298b8355fabde1275b7def35fScott Bardone size = sizeof(struct freelQ_ce) * q->size; 586cbee9f9115da7eac5dc44a450f2dab343236cb2cStephen Hemminger q->centries = kzalloc(size, GFP_KERNEL); 587559fb51ba7e66fe298b8355fabde1275b7def35fScott Bardone if (!q->centries) 5888199d3a79c224bbe5943fa08684e1f93a17881b0Christoph Lameter goto err_no_mem; 5898199d3a79c224bbe5943fa08684e1f93a17881b0Christoph Lameter } 5908199d3a79c224bbe5943fa08684e1f93a17881b0Christoph Lameter 5918199d3a79c224bbe5943fa08684e1f93a17881b0Christoph Lameter /* 5928199d3a79c224bbe5943fa08684e1f93a17881b0Christoph Lameter * Calculate the buffer sizes for the two free lists. FL0 accommodates 5938199d3a79c224bbe5943fa08684e1f93a17881b0Christoph Lameter * regular sized Ethernet frames, FL1 is sized not to exceed 16K, 5948199d3a79c224bbe5943fa08684e1f93a17881b0Christoph Lameter * including all the sk_buff overhead. 5958199d3a79c224bbe5943fa08684e1f93a17881b0Christoph Lameter * 5968199d3a79c224bbe5943fa08684e1f93a17881b0Christoph Lameter * Note: For T2 FL0 and FL1 are reversed. 5978199d3a79c224bbe5943fa08684e1f93a17881b0Christoph Lameter */ 5988199d3a79c224bbe5943fa08684e1f93a17881b0Christoph Lameter sge->freelQ[!sge->jumbo_fl].rx_buffer_size = SGE_RX_SM_BUF_SIZE + 5998199d3a79c224bbe5943fa08684e1f93a17881b0Christoph Lameter sizeof(struct cpl_rx_data) + 6008199d3a79c224bbe5943fa08684e1f93a17881b0Christoph Lameter sge->freelQ[!sge->jumbo_fl].dma_offset; 601f1d3d38af75789f1b82969b83b69cab540609789Stephen Hemminger 602f1d3d38af75789f1b82969b83b69cab540609789Stephen Hemminger size = (16 * 1024) - 603f1d3d38af75789f1b82969b83b69cab540609789Stephen Hemminger SKB_DATA_ALIGN(sizeof(struct skb_shared_info)); 604f1d3d38af75789f1b82969b83b69cab540609789Stephen Hemminger 605f1d3d38af75789f1b82969b83b69cab540609789Stephen Hemminger sge->freelQ[sge->jumbo_fl].rx_buffer_size = size; 6068199d3a79c224bbe5943fa08684e1f93a17881b0Christoph Lameter 607559fb51ba7e66fe298b8355fabde1275b7def35fScott Bardone /* 608559fb51ba7e66fe298b8355fabde1275b7def35fScott Bardone * Setup which skb recycle Q should be used when recycling buffers from 609559fb51ba7e66fe298b8355fabde1275b7def35fScott Bardone * each free list. 610559fb51ba7e66fe298b8355fabde1275b7def35fScott Bardone */ 611559fb51ba7e66fe298b8355fabde1275b7def35fScott Bardone sge->freelQ[!sge->jumbo_fl].recycleq_idx = 0; 612559fb51ba7e66fe298b8355fabde1275b7def35fScott Bardone sge->freelQ[sge->jumbo_fl].recycleq_idx = 1; 613559fb51ba7e66fe298b8355fabde1275b7def35fScott Bardone 6148199d3a79c224bbe5943fa08684e1f93a17881b0Christoph Lameter sge->respQ.genbit = 1; 615559fb51ba7e66fe298b8355fabde1275b7def35fScott Bardone sge->respQ.size = SGE_RESPQ_E_N; 616559fb51ba7e66fe298b8355fabde1275b7def35fScott Bardone sge->respQ.credits = 0; 617559fb51ba7e66fe298b8355fabde1275b7def35fScott Bardone size = sizeof(struct respQ_e) * sge->respQ.size; 6183e0f75be52605a901165fa1d8acf4ffd37a4857bFrancois Romieu sge->respQ.entries = 6198199d3a79c224bbe5943fa08684e1f93a17881b0Christoph Lameter pci_alloc_consistent(pdev, size, &sge->respQ.dma_addr); 6208199d3a79c224bbe5943fa08684e1f93a17881b0Christoph Lameter if (!sge->respQ.entries) 6218199d3a79c224bbe5943fa08684e1f93a17881b0Christoph Lameter goto err_no_mem; 6228199d3a79c224bbe5943fa08684e1f93a17881b0Christoph Lameter return 0; 6238199d3a79c224bbe5943fa08684e1f93a17881b0Christoph Lameter 6248199d3a79c224bbe5943fa08684e1f93a17881b0Christoph Lametererr_no_mem: 6258199d3a79c224bbe5943fa08684e1f93a17881b0Christoph Lameter free_rx_resources(sge); 6268199d3a79c224bbe5943fa08684e1f93a17881b0Christoph Lameter return -ENOMEM; 6278199d3a79c224bbe5943fa08684e1f93a17881b0Christoph Lameter} 6288199d3a79c224bbe5943fa08684e1f93a17881b0Christoph Lameter 6298199d3a79c224bbe5943fa08684e1f93a17881b0Christoph Lameter/* 630559fb51ba7e66fe298b8355fabde1275b7def35fScott Bardone * Reclaims n TX descriptors and frees the buffers associated with them. 6318199d3a79c224bbe5943fa08684e1f93a17881b0Christoph Lameter */ 632559fb51ba7e66fe298b8355fabde1275b7def35fScott Bardonestatic void free_cmdQ_buffers(struct sge *sge, struct cmdQ *q, unsigned int n) 6338199d3a79c224bbe5943fa08684e1f93a17881b0Christoph Lameter{ 634559fb51ba7e66fe298b8355fabde1275b7def35fScott Bardone struct cmdQ_ce *ce; 6358199d3a79c224bbe5943fa08684e1f93a17881b0Christoph Lameter struct pci_dev *pdev = sge->adapter->pdev; 636559fb51ba7e66fe298b8355fabde1275b7def35fScott Bardone unsigned int cidx = q->cidx; 6378199d3a79c224bbe5943fa08684e1f93a17881b0Christoph Lameter 638559fb51ba7e66fe298b8355fabde1275b7def35fScott Bardone q->in_use -= n; 639559fb51ba7e66fe298b8355fabde1275b7def35fScott Bardone ce = &q->centries[cidx]; 640559fb51ba7e66fe298b8355fabde1275b7def35fScott Bardone while (n--) { 641094f92a61aa044142c231e04c35c00a9cc70adbcFUJITA Tomonori if (likely(dma_unmap_len(ce, dma_len))) { 642094f92a61aa044142c231e04c35c00a9cc70adbcFUJITA Tomonori pci_unmap_single(pdev, dma_unmap_addr(ce, dma_addr), 643094f92a61aa044142c231e04c35c00a9cc70adbcFUJITA Tomonori dma_unmap_len(ce, dma_len), 6443e0f75be52605a901165fa1d8acf4ffd37a4857bFrancois Romieu PCI_DMA_TODEVICE); 6453e0f75be52605a901165fa1d8acf4ffd37a4857bFrancois Romieu if (q->sop) 646f1d3d38af75789f1b82969b83b69cab540609789Stephen Hemminger q->sop = 0; 647f1d3d38af75789f1b82969b83b69cab540609789Stephen Hemminger } 648559fb51ba7e66fe298b8355fabde1275b7def35fScott Bardone if (ce->skb) { 649f1d3d38af75789f1b82969b83b69cab540609789Stephen Hemminger dev_kfree_skb_any(ce->skb); 650559fb51ba7e66fe298b8355fabde1275b7def35fScott Bardone q->sop = 1; 651559fb51ba7e66fe298b8355fabde1275b7def35fScott Bardone } 6528199d3a79c224bbe5943fa08684e1f93a17881b0Christoph Lameter ce++; 653559fb51ba7e66fe298b8355fabde1275b7def35fScott Bardone if (++cidx == q->size) { 6548199d3a79c224bbe5943fa08684e1f93a17881b0Christoph Lameter cidx = 0; 655559fb51ba7e66fe298b8355fabde1275b7def35fScott Bardone ce = q->centries; 6568199d3a79c224bbe5943fa08684e1f93a17881b0Christoph Lameter } 6578199d3a79c224bbe5943fa08684e1f93a17881b0Christoph Lameter } 658559fb51ba7e66fe298b8355fabde1275b7def35fScott Bardone q->cidx = cidx; 6598199d3a79c224bbe5943fa08684e1f93a17881b0Christoph Lameter} 6608199d3a79c224bbe5943fa08684e1f93a17881b0Christoph Lameter 6618199d3a79c224bbe5943fa08684e1f93a17881b0Christoph Lameter/* 6628199d3a79c224bbe5943fa08684e1f93a17881b0Christoph Lameter * Free TX resources. 6638199d3a79c224bbe5943fa08684e1f93a17881b0Christoph Lameter * 6648199d3a79c224bbe5943fa08684e1f93a17881b0Christoph Lameter * Assumes that SGE is stopped and all interrupts are disabled. 6658199d3a79c224bbe5943fa08684e1f93a17881b0Christoph Lameter */ 6668199d3a79c224bbe5943fa08684e1f93a17881b0Christoph Lameterstatic void free_tx_resources(struct sge *sge) 6678199d3a79c224bbe5943fa08684e1f93a17881b0Christoph Lameter{ 6688199d3a79c224bbe5943fa08684e1f93a17881b0Christoph Lameter struct pci_dev *pdev = sge->adapter->pdev; 6698199d3a79c224bbe5943fa08684e1f93a17881b0Christoph Lameter unsigned int size, i; 6708199d3a79c224bbe5943fa08684e1f93a17881b0Christoph Lameter 6718199d3a79c224bbe5943fa08684e1f93a17881b0Christoph Lameter for (i = 0; i < SGE_CMDQ_N; i++) { 672559fb51ba7e66fe298b8355fabde1275b7def35fScott Bardone struct cmdQ *q = &sge->cmdQ[i]; 6738199d3a79c224bbe5943fa08684e1f93a17881b0Christoph Lameter 674559fb51ba7e66fe298b8355fabde1275b7def35fScott Bardone if (q->centries) { 675559fb51ba7e66fe298b8355fabde1275b7def35fScott Bardone if (q->in_use) 676559fb51ba7e66fe298b8355fabde1275b7def35fScott Bardone free_cmdQ_buffers(sge, q, q->in_use); 677559fb51ba7e66fe298b8355fabde1275b7def35fScott Bardone kfree(q->centries); 6788199d3a79c224bbe5943fa08684e1f93a17881b0Christoph Lameter } 679559fb51ba7e66fe298b8355fabde1275b7def35fScott Bardone if (q->entries) { 680559fb51ba7e66fe298b8355fabde1275b7def35fScott Bardone size = sizeof(struct cmdQ_e) * q->size; 681559fb51ba7e66fe298b8355fabde1275b7def35fScott Bardone pci_free_consistent(pdev, size, q->entries, 682559fb51ba7e66fe298b8355fabde1275b7def35fScott Bardone q->dma_addr); 6838199d3a79c224bbe5943fa08684e1f93a17881b0Christoph Lameter } 6848199d3a79c224bbe5943fa08684e1f93a17881b0Christoph Lameter } 6858199d3a79c224bbe5943fa08684e1f93a17881b0Christoph Lameter} 6868199d3a79c224bbe5943fa08684e1f93a17881b0Christoph Lameter 6878199d3a79c224bbe5943fa08684e1f93a17881b0Christoph Lameter/* 6888199d3a79c224bbe5943fa08684e1f93a17881b0Christoph Lameter * Allocates basic TX resources, consisting of memory mapped command Qs. 6898199d3a79c224bbe5943fa08684e1f93a17881b0Christoph Lameter */ 6908199d3a79c224bbe5943fa08684e1f93a17881b0Christoph Lameterstatic int alloc_tx_resources(struct sge *sge, struct sge_params *p) 6918199d3a79c224bbe5943fa08684e1f93a17881b0Christoph Lameter{ 6928199d3a79c224bbe5943fa08684e1f93a17881b0Christoph Lameter struct pci_dev *pdev = sge->adapter->pdev; 6938199d3a79c224bbe5943fa08684e1f93a17881b0Christoph Lameter unsigned int size, i; 6948199d3a79c224bbe5943fa08684e1f93a17881b0Christoph Lameter 6958199d3a79c224bbe5943fa08684e1f93a17881b0Christoph Lameter for (i = 0; i < SGE_CMDQ_N; i++) { 696559fb51ba7e66fe298b8355fabde1275b7def35fScott Bardone struct cmdQ *q = &sge->cmdQ[i]; 697559fb51ba7e66fe298b8355fabde1275b7def35fScott Bardone 698559fb51ba7e66fe298b8355fabde1275b7def35fScott Bardone q->genbit = 1; 699559fb51ba7e66fe298b8355fabde1275b7def35fScott Bardone q->sop = 1; 700559fb51ba7e66fe298b8355fabde1275b7def35fScott Bardone q->size = p->cmdQ_size[i]; 701559fb51ba7e66fe298b8355fabde1275b7def35fScott Bardone q->in_use = 0; 702559fb51ba7e66fe298b8355fabde1275b7def35fScott Bardone q->status = 0; 703559fb51ba7e66fe298b8355fabde1275b7def35fScott Bardone q->processed = q->cleaned = 0; 704559fb51ba7e66fe298b8355fabde1275b7def35fScott Bardone q->stop_thres = 0; 705559fb51ba7e66fe298b8355fabde1275b7def35fScott Bardone spin_lock_init(&q->lock); 706559fb51ba7e66fe298b8355fabde1275b7def35fScott Bardone size = sizeof(struct cmdQ_e) * q->size; 7073e0f75be52605a901165fa1d8acf4ffd37a4857bFrancois Romieu q->entries = pci_alloc_consistent(pdev, size, &q->dma_addr); 708559fb51ba7e66fe298b8355fabde1275b7def35fScott Bardone if (!q->entries) 7098199d3a79c224bbe5943fa08684e1f93a17881b0Christoph Lameter goto err_no_mem; 7103e0f75be52605a901165fa1d8acf4ffd37a4857bFrancois Romieu 711559fb51ba7e66fe298b8355fabde1275b7def35fScott Bardone size = sizeof(struct cmdQ_ce) * q->size; 712cbee9f9115da7eac5dc44a450f2dab343236cb2cStephen Hemminger q->centries = kzalloc(size, GFP_KERNEL); 713559fb51ba7e66fe298b8355fabde1275b7def35fScott Bardone if (!q->centries) 7148199d3a79c224bbe5943fa08684e1f93a17881b0Christoph Lameter goto err_no_mem; 7158199d3a79c224bbe5943fa08684e1f93a17881b0Christoph Lameter } 7168199d3a79c224bbe5943fa08684e1f93a17881b0Christoph Lameter 717559fb51ba7e66fe298b8355fabde1275b7def35fScott Bardone /* 718559fb51ba7e66fe298b8355fabde1275b7def35fScott Bardone * CommandQ 0 handles Ethernet and TOE packets, while queue 1 is TOE 719559fb51ba7e66fe298b8355fabde1275b7def35fScott Bardone * only. For queue 0 set the stop threshold so we can handle one more 720559fb51ba7e66fe298b8355fabde1275b7def35fScott Bardone * packet from each port, plus reserve an additional 24 entries for 721559fb51ba7e66fe298b8355fabde1275b7def35fScott Bardone * Ethernet packets only. Queue 1 never suspends nor do we reserve 722559fb51ba7e66fe298b8355fabde1275b7def35fScott Bardone * space for Ethernet packets. 723559fb51ba7e66fe298b8355fabde1275b7def35fScott Bardone */ 724559fb51ba7e66fe298b8355fabde1275b7def35fScott Bardone sge->cmdQ[0].stop_thres = sge->adapter->params.nports * 725559fb51ba7e66fe298b8355fabde1275b7def35fScott Bardone (MAX_SKB_FRAGS + 1); 7268199d3a79c224bbe5943fa08684e1f93a17881b0Christoph Lameter return 0; 7278199d3a79c224bbe5943fa08684e1f93a17881b0Christoph Lameter 7288199d3a79c224bbe5943fa08684e1f93a17881b0Christoph Lametererr_no_mem: 7298199d3a79c224bbe5943fa08684e1f93a17881b0Christoph Lameter free_tx_resources(sge); 7308199d3a79c224bbe5943fa08684e1f93a17881b0Christoph Lameter return -ENOMEM; 7318199d3a79c224bbe5943fa08684e1f93a17881b0Christoph Lameter} 7328199d3a79c224bbe5943fa08684e1f93a17881b0Christoph Lameter 7338199d3a79c224bbe5943fa08684e1f93a17881b0Christoph Lameterstatic inline void setup_ring_params(struct adapter *adapter, u64 addr, 7348199d3a79c224bbe5943fa08684e1f93a17881b0Christoph Lameter u32 size, int base_reg_lo, 7358199d3a79c224bbe5943fa08684e1f93a17881b0Christoph Lameter int base_reg_hi, int size_reg) 7368199d3a79c224bbe5943fa08684e1f93a17881b0Christoph Lameter{ 737559fb51ba7e66fe298b8355fabde1275b7def35fScott Bardone writel((u32)addr, adapter->regs + base_reg_lo); 738559fb51ba7e66fe298b8355fabde1275b7def35fScott Bardone writel(addr >> 32, adapter->regs + base_reg_hi); 739559fb51ba7e66fe298b8355fabde1275b7def35fScott Bardone writel(size, adapter->regs + size_reg); 7408199d3a79c224bbe5943fa08684e1f93a17881b0Christoph Lameter} 7418199d3a79c224bbe5943fa08684e1f93a17881b0Christoph Lameter 7428199d3a79c224bbe5943fa08684e1f93a17881b0Christoph Lameter/* 7438199d3a79c224bbe5943fa08684e1f93a17881b0Christoph Lameter * Enable/disable VLAN acceleration. 7448199d3a79c224bbe5943fa08684e1f93a17881b0Christoph Lameter */ 745c8f44affb7244f2ac3e703cab13d55ede27621bbMichał Mirosławvoid t1_vlan_mode(struct adapter *adapter, netdev_features_t features) 7468199d3a79c224bbe5943fa08684e1f93a17881b0Christoph Lameter{ 7478199d3a79c224bbe5943fa08684e1f93a17881b0Christoph Lameter struct sge *sge = adapter->sge; 7488199d3a79c224bbe5943fa08684e1f93a17881b0Christoph Lameter 749133b08513a99d44d9ed916bb83e46343f6ad2855Jiri Pirko if (features & NETIF_F_HW_VLAN_RX) 7508199d3a79c224bbe5943fa08684e1f93a17881b0Christoph Lameter sge->sge_control |= F_VLAN_XTRACT; 751133b08513a99d44d9ed916bb83e46343f6ad2855Jiri Pirko else 752133b08513a99d44d9ed916bb83e46343f6ad2855Jiri Pirko sge->sge_control &= ~F_VLAN_XTRACT; 7538199d3a79c224bbe5943fa08684e1f93a17881b0Christoph Lameter if (adapter->open_device_map) { 754559fb51ba7e66fe298b8355fabde1275b7def35fScott Bardone writel(sge->sge_control, adapter->regs + A_SG_CONTROL); 755f1d3d38af75789f1b82969b83b69cab540609789Stephen Hemminger readl(adapter->regs + A_SG_CONTROL); /* flush */ 7568199d3a79c224bbe5943fa08684e1f93a17881b0Christoph Lameter } 7578199d3a79c224bbe5943fa08684e1f93a17881b0Christoph Lameter} 7588199d3a79c224bbe5943fa08684e1f93a17881b0Christoph Lameter 7598199d3a79c224bbe5943fa08684e1f93a17881b0Christoph Lameter/* 7608199d3a79c224bbe5943fa08684e1f93a17881b0Christoph Lameter * Programs the various SGE registers. However, the engine is not yet enabled, 7618199d3a79c224bbe5943fa08684e1f93a17881b0Christoph Lameter * but sge->sge_control is setup and ready to go. 7628199d3a79c224bbe5943fa08684e1f93a17881b0Christoph Lameter */ 7638199d3a79c224bbe5943fa08684e1f93a17881b0Christoph Lameterstatic void configure_sge(struct sge *sge, struct sge_params *p) 7648199d3a79c224bbe5943fa08684e1f93a17881b0Christoph Lameter{ 7658199d3a79c224bbe5943fa08684e1f93a17881b0Christoph Lameter struct adapter *ap = sge->adapter; 766356bd1460d1e1c4e433e4114fdac02139bddf17cFrancois Romieu 767559fb51ba7e66fe298b8355fabde1275b7def35fScott Bardone writel(0, ap->regs + A_SG_CONTROL); 768559fb51ba7e66fe298b8355fabde1275b7def35fScott Bardone setup_ring_params(ap, sge->cmdQ[0].dma_addr, sge->cmdQ[0].size, 7698199d3a79c224bbe5943fa08684e1f93a17881b0Christoph Lameter A_SG_CMD0BASELWR, A_SG_CMD0BASEUPR, A_SG_CMD0SIZE); 770559fb51ba7e66fe298b8355fabde1275b7def35fScott Bardone setup_ring_params(ap, sge->cmdQ[1].dma_addr, sge->cmdQ[1].size, 7718199d3a79c224bbe5943fa08684e1f93a17881b0Christoph Lameter A_SG_CMD1BASELWR, A_SG_CMD1BASEUPR, A_SG_CMD1SIZE); 7728199d3a79c224bbe5943fa08684e1f93a17881b0Christoph Lameter setup_ring_params(ap, sge->freelQ[0].dma_addr, 773559fb51ba7e66fe298b8355fabde1275b7def35fScott Bardone sge->freelQ[0].size, A_SG_FL0BASELWR, 7748199d3a79c224bbe5943fa08684e1f93a17881b0Christoph Lameter A_SG_FL0BASEUPR, A_SG_FL0SIZE); 7758199d3a79c224bbe5943fa08684e1f93a17881b0Christoph Lameter setup_ring_params(ap, sge->freelQ[1].dma_addr, 776559fb51ba7e66fe298b8355fabde1275b7def35fScott Bardone sge->freelQ[1].size, A_SG_FL1BASELWR, 7778199d3a79c224bbe5943fa08684e1f93a17881b0Christoph Lameter A_SG_FL1BASEUPR, A_SG_FL1SIZE); 7788199d3a79c224bbe5943fa08684e1f93a17881b0Christoph Lameter 7798199d3a79c224bbe5943fa08684e1f93a17881b0Christoph Lameter /* The threshold comparison uses <. */ 780559fb51ba7e66fe298b8355fabde1275b7def35fScott Bardone writel(SGE_RX_SM_BUF_SIZE + 1, ap->regs + A_SG_FLTHRESHOLD); 7818199d3a79c224bbe5943fa08684e1f93a17881b0Christoph Lameter 782559fb51ba7e66fe298b8355fabde1275b7def35fScott Bardone setup_ring_params(ap, sge->respQ.dma_addr, sge->respQ.size, 783559fb51ba7e66fe298b8355fabde1275b7def35fScott Bardone A_SG_RSPBASELWR, A_SG_RSPBASEUPR, A_SG_RSPSIZE); 784559fb51ba7e66fe298b8355fabde1275b7def35fScott Bardone writel((u32)sge->respQ.size - 1, ap->regs + A_SG_RSPQUEUECREDIT); 7858199d3a79c224bbe5943fa08684e1f93a17881b0Christoph Lameter 7868199d3a79c224bbe5943fa08684e1f93a17881b0Christoph Lameter sge->sge_control = F_CMDQ0_ENABLE | F_CMDQ1_ENABLE | F_FL0_ENABLE | 7878199d3a79c224bbe5943fa08684e1f93a17881b0Christoph Lameter F_FL1_ENABLE | F_CPL_ENABLE | F_RESPONSE_QUEUE_ENABLE | 7888199d3a79c224bbe5943fa08684e1f93a17881b0Christoph Lameter V_CMDQ_PRIORITY(2) | F_DISABLE_CMDQ1_GTS | F_ISCSI_COALESCE | 7898199d3a79c224bbe5943fa08684e1f93a17881b0Christoph Lameter V_RX_PKT_OFFSET(sge->rx_pkt_pad); 7908199d3a79c224bbe5943fa08684e1f93a17881b0Christoph Lameter 7918199d3a79c224bbe5943fa08684e1f93a17881b0Christoph Lameter#if defined(__BIG_ENDIAN_BITFIELD) 7928199d3a79c224bbe5943fa08684e1f93a17881b0Christoph Lameter sge->sge_control |= F_ENABLE_BIG_ENDIAN; 7938199d3a79c224bbe5943fa08684e1f93a17881b0Christoph Lameter#endif 7948199d3a79c224bbe5943fa08684e1f93a17881b0Christoph Lameter 795559fb51ba7e66fe298b8355fabde1275b7def35fScott Bardone /* Initialize no-resource timer */ 796559fb51ba7e66fe298b8355fabde1275b7def35fScott Bardone sge->intrtimer_nres = SGE_INTRTIMER_NRES * core_ticks_per_usec(ap); 797559fb51ba7e66fe298b8355fabde1275b7def35fScott Bardone 798559fb51ba7e66fe298b8355fabde1275b7def35fScott Bardone t1_sge_set_coalesce_params(sge, p); 7998199d3a79c224bbe5943fa08684e1f93a17881b0Christoph Lameter} 8008199d3a79c224bbe5943fa08684e1f93a17881b0Christoph Lameter 8018199d3a79c224bbe5943fa08684e1f93a17881b0Christoph Lameter/* 8028199d3a79c224bbe5943fa08684e1f93a17881b0Christoph Lameter * Return the payload capacity of the jumbo free-list buffers. 8038199d3a79c224bbe5943fa08684e1f93a17881b0Christoph Lameter */ 8048199d3a79c224bbe5943fa08684e1f93a17881b0Christoph Lameterstatic inline unsigned int jumbo_payload_capacity(const struct sge *sge) 8058199d3a79c224bbe5943fa08684e1f93a17881b0Christoph Lameter{ 8068199d3a79c224bbe5943fa08684e1f93a17881b0Christoph Lameter return sge->freelQ[sge->jumbo_fl].rx_buffer_size - 807559fb51ba7e66fe298b8355fabde1275b7def35fScott Bardone sge->freelQ[sge->jumbo_fl].dma_offset - 808559fb51ba7e66fe298b8355fabde1275b7def35fScott Bardone sizeof(struct cpl_rx_data); 8098199d3a79c224bbe5943fa08684e1f93a17881b0Christoph Lameter} 8108199d3a79c224bbe5943fa08684e1f93a17881b0Christoph Lameter 8118199d3a79c224bbe5943fa08684e1f93a17881b0Christoph Lameter/* 8128199d3a79c224bbe5943fa08684e1f93a17881b0Christoph Lameter * Frees all SGE related resources and the sge structure itself 8138199d3a79c224bbe5943fa08684e1f93a17881b0Christoph Lameter */ 8148199d3a79c224bbe5943fa08684e1f93a17881b0Christoph Lametervoid t1_sge_destroy(struct sge *sge) 8158199d3a79c224bbe5943fa08684e1f93a17881b0Christoph Lameter{ 81656f643c28c5df63693d7c66e56f8e4767cfd7a65Stephen Hemminger int i; 81756f643c28c5df63693d7c66e56f8e4767cfd7a65Stephen Hemminger 81856f643c28c5df63693d7c66e56f8e4767cfd7a65Stephen Hemminger for_each_port(sge->adapter, i) 81956f643c28c5df63693d7c66e56f8e4767cfd7a65Stephen Hemminger free_percpu(sge->port_stats[i]); 82056f643c28c5df63693d7c66e56f8e4767cfd7a65Stephen Hemminger 821f1d3d38af75789f1b82969b83b69cab540609789Stephen Hemminger kfree(sge->tx_sched); 8228199d3a79c224bbe5943fa08684e1f93a17881b0Christoph Lameter free_tx_resources(sge); 8238199d3a79c224bbe5943fa08684e1f93a17881b0Christoph Lameter free_rx_resources(sge); 8248199d3a79c224bbe5943fa08684e1f93a17881b0Christoph Lameter kfree(sge); 8258199d3a79c224bbe5943fa08684e1f93a17881b0Christoph Lameter} 8268199d3a79c224bbe5943fa08684e1f93a17881b0Christoph Lameter 8278199d3a79c224bbe5943fa08684e1f93a17881b0Christoph Lameter/* 8288199d3a79c224bbe5943fa08684e1f93a17881b0Christoph Lameter * Allocates new RX buffers on the freelist Q (and tracks them on the freelist 8298199d3a79c224bbe5943fa08684e1f93a17881b0Christoph Lameter * context Q) until the Q is full or alloc_skb fails. 8308199d3a79c224bbe5943fa08684e1f93a17881b0Christoph Lameter * 8318199d3a79c224bbe5943fa08684e1f93a17881b0Christoph Lameter * It is possible that the generation bits already match, indicating that the 8328199d3a79c224bbe5943fa08684e1f93a17881b0Christoph Lameter * buffer is already valid and nothing needs to be done. This happens when we 8338199d3a79c224bbe5943fa08684e1f93a17881b0Christoph Lameter * copied a received buffer into a new sk_buff during the interrupt processing. 8348199d3a79c224bbe5943fa08684e1f93a17881b0Christoph Lameter * 8358199d3a79c224bbe5943fa08684e1f93a17881b0Christoph Lameter * If the SGE doesn't automatically align packets properly (!sge->rx_pkt_pad), 8368199d3a79c224bbe5943fa08684e1f93a17881b0Christoph Lameter * we specify a RX_OFFSET in order to make sure that the IP header is 4B 8378199d3a79c224bbe5943fa08684e1f93a17881b0Christoph Lameter * aligned. 8388199d3a79c224bbe5943fa08684e1f93a17881b0Christoph Lameter */ 839559fb51ba7e66fe298b8355fabde1275b7def35fScott Bardonestatic void refill_free_list(struct sge *sge, struct freelQ *q) 8408199d3a79c224bbe5943fa08684e1f93a17881b0Christoph Lameter{ 8418199d3a79c224bbe5943fa08684e1f93a17881b0Christoph Lameter struct pci_dev *pdev = sge->adapter->pdev; 842559fb51ba7e66fe298b8355fabde1275b7def35fScott Bardone struct freelQ_ce *ce = &q->centries[q->pidx]; 843559fb51ba7e66fe298b8355fabde1275b7def35fScott Bardone struct freelQ_e *e = &q->entries[q->pidx]; 844559fb51ba7e66fe298b8355fabde1275b7def35fScott Bardone unsigned int dma_len = q->rx_buffer_size - q->dma_offset; 8458199d3a79c224bbe5943fa08684e1f93a17881b0Christoph Lameter 846559fb51ba7e66fe298b8355fabde1275b7def35fScott Bardone while (q->credits < q->size) { 847559fb51ba7e66fe298b8355fabde1275b7def35fScott Bardone struct sk_buff *skb; 848559fb51ba7e66fe298b8355fabde1275b7def35fScott Bardone dma_addr_t mapping; 8498199d3a79c224bbe5943fa08684e1f93a17881b0Christoph Lameter 850559fb51ba7e66fe298b8355fabde1275b7def35fScott Bardone skb = alloc_skb(q->rx_buffer_size, GFP_ATOMIC); 851559fb51ba7e66fe298b8355fabde1275b7def35fScott Bardone if (!skb) 852559fb51ba7e66fe298b8355fabde1275b7def35fScott Bardone break; 853559fb51ba7e66fe298b8355fabde1275b7def35fScott Bardone 854559fb51ba7e66fe298b8355fabde1275b7def35fScott Bardone skb_reserve(skb, q->dma_offset); 855559fb51ba7e66fe298b8355fabde1275b7def35fScott Bardone mapping = pci_map_single(pdev, skb->data, dma_len, 856559fb51ba7e66fe298b8355fabde1275b7def35fScott Bardone PCI_DMA_FROMDEVICE); 85724a427cf76984726641ea0d8163e61e99119069dStephen Hemminger skb_reserve(skb, sge->rx_pkt_pad); 85824a427cf76984726641ea0d8163e61e99119069dStephen Hemminger 859559fb51ba7e66fe298b8355fabde1275b7def35fScott Bardone ce->skb = skb; 860094f92a61aa044142c231e04c35c00a9cc70adbcFUJITA Tomonori dma_unmap_addr_set(ce, dma_addr, mapping); 861094f92a61aa044142c231e04c35c00a9cc70adbcFUJITA Tomonori dma_unmap_len_set(ce, dma_len, dma_len); 862559fb51ba7e66fe298b8355fabde1275b7def35fScott Bardone e->addr_lo = (u32)mapping; 863559fb51ba7e66fe298b8355fabde1275b7def35fScott Bardone e->addr_hi = (u64)mapping >> 32; 864559fb51ba7e66fe298b8355fabde1275b7def35fScott Bardone e->len_gen = V_CMD_LEN(dma_len) | V_CMD_GEN1(q->genbit); 865559fb51ba7e66fe298b8355fabde1275b7def35fScott Bardone wmb(); 866559fb51ba7e66fe298b8355fabde1275b7def35fScott Bardone e->gen2 = V_CMD_GEN2(q->genbit); 8678199d3a79c224bbe5943fa08684e1f93a17881b0Christoph Lameter 8688199d3a79c224bbe5943fa08684e1f93a17881b0Christoph Lameter e++; 8698199d3a79c224bbe5943fa08684e1f93a17881b0Christoph Lameter ce++; 870559fb51ba7e66fe298b8355fabde1275b7def35fScott Bardone if (++q->pidx == q->size) { 871559fb51ba7e66fe298b8355fabde1275b7def35fScott Bardone q->pidx = 0; 872559fb51ba7e66fe298b8355fabde1275b7def35fScott Bardone q->genbit ^= 1; 873559fb51ba7e66fe298b8355fabde1275b7def35fScott Bardone ce = q->centries; 874559fb51ba7e66fe298b8355fabde1275b7def35fScott Bardone e = q->entries; 8758199d3a79c224bbe5943fa08684e1f93a17881b0Christoph Lameter } 876559fb51ba7e66fe298b8355fabde1275b7def35fScott Bardone q->credits++; 8778199d3a79c224bbe5943fa08684e1f93a17881b0Christoph Lameter } 8788199d3a79c224bbe5943fa08684e1f93a17881b0Christoph Lameter} 8798199d3a79c224bbe5943fa08684e1f93a17881b0Christoph Lameter 8808199d3a79c224bbe5943fa08684e1f93a17881b0Christoph Lameter/* 881559fb51ba7e66fe298b8355fabde1275b7def35fScott Bardone * Calls refill_free_list for both free lists. If we cannot fill at least 1/4 882559fb51ba7e66fe298b8355fabde1275b7def35fScott Bardone * of both rings, we go into 'few interrupt mode' in order to give the system 883559fb51ba7e66fe298b8355fabde1275b7def35fScott Bardone * time to free up resources. 8848199d3a79c224bbe5943fa08684e1f93a17881b0Christoph Lameter */ 8858199d3a79c224bbe5943fa08684e1f93a17881b0Christoph Lameterstatic void freelQs_empty(struct sge *sge) 8868199d3a79c224bbe5943fa08684e1f93a17881b0Christoph Lameter{ 887559fb51ba7e66fe298b8355fabde1275b7def35fScott Bardone struct adapter *adapter = sge->adapter; 888559fb51ba7e66fe298b8355fabde1275b7def35fScott Bardone u32 irq_reg = readl(adapter->regs + A_SG_INT_ENABLE); 8898199d3a79c224bbe5943fa08684e1f93a17881b0Christoph Lameter u32 irqholdoff_reg; 8908199d3a79c224bbe5943fa08684e1f93a17881b0Christoph Lameter 8918199d3a79c224bbe5943fa08684e1f93a17881b0Christoph Lameter refill_free_list(sge, &sge->freelQ[0]); 8928199d3a79c224bbe5943fa08684e1f93a17881b0Christoph Lameter refill_free_list(sge, &sge->freelQ[1]); 8938199d3a79c224bbe5943fa08684e1f93a17881b0Christoph Lameter 894559fb51ba7e66fe298b8355fabde1275b7def35fScott Bardone if (sge->freelQ[0].credits > (sge->freelQ[0].size >> 2) && 895559fb51ba7e66fe298b8355fabde1275b7def35fScott Bardone sge->freelQ[1].credits > (sge->freelQ[1].size >> 2)) { 8968199d3a79c224bbe5943fa08684e1f93a17881b0Christoph Lameter irq_reg |= F_FL_EXHAUSTED; 897559fb51ba7e66fe298b8355fabde1275b7def35fScott Bardone irqholdoff_reg = sge->fixed_intrtimer; 8988199d3a79c224bbe5943fa08684e1f93a17881b0Christoph Lameter } else { 8998199d3a79c224bbe5943fa08684e1f93a17881b0Christoph Lameter /* Clear the F_FL_EXHAUSTED interrupts for now */ 9008199d3a79c224bbe5943fa08684e1f93a17881b0Christoph Lameter irq_reg &= ~F_FL_EXHAUSTED; 9018199d3a79c224bbe5943fa08684e1f93a17881b0Christoph Lameter irqholdoff_reg = sge->intrtimer_nres; 9028199d3a79c224bbe5943fa08684e1f93a17881b0Christoph Lameter } 903559fb51ba7e66fe298b8355fabde1275b7def35fScott Bardone writel(irqholdoff_reg, adapter->regs + A_SG_INTRTIMER); 904559fb51ba7e66fe298b8355fabde1275b7def35fScott Bardone writel(irq_reg, adapter->regs + A_SG_INT_ENABLE); 9058199d3a79c224bbe5943fa08684e1f93a17881b0Christoph Lameter 9068199d3a79c224bbe5943fa08684e1f93a17881b0Christoph Lameter /* We reenable the Qs to force a freelist GTS interrupt later */ 907559fb51ba7e66fe298b8355fabde1275b7def35fScott Bardone doorbell_pio(adapter, F_FL0_ENABLE | F_FL1_ENABLE); 9088199d3a79c224bbe5943fa08684e1f93a17881b0Christoph Lameter} 9098199d3a79c224bbe5943fa08684e1f93a17881b0Christoph Lameter 9108199d3a79c224bbe5943fa08684e1f93a17881b0Christoph Lameter#define SGE_PL_INTR_MASK (F_PL_INTR_SGE_ERR | F_PL_INTR_SGE_DATA) 9118199d3a79c224bbe5943fa08684e1f93a17881b0Christoph Lameter#define SGE_INT_FATAL (F_RESPQ_OVERFLOW | F_PACKET_TOO_BIG | F_PACKET_MISMATCH) 9128199d3a79c224bbe5943fa08684e1f93a17881b0Christoph Lameter#define SGE_INT_ENABLE (F_RESPQ_EXHAUSTED | F_RESPQ_OVERFLOW | \ 9138199d3a79c224bbe5943fa08684e1f93a17881b0Christoph Lameter F_FL_EXHAUSTED | F_PACKET_TOO_BIG | F_PACKET_MISMATCH) 9148199d3a79c224bbe5943fa08684e1f93a17881b0Christoph Lameter 9158199d3a79c224bbe5943fa08684e1f93a17881b0Christoph Lameter/* 9168199d3a79c224bbe5943fa08684e1f93a17881b0Christoph Lameter * Disable SGE Interrupts 9178199d3a79c224bbe5943fa08684e1f93a17881b0Christoph Lameter */ 9188199d3a79c224bbe5943fa08684e1f93a17881b0Christoph Lametervoid t1_sge_intr_disable(struct sge *sge) 9198199d3a79c224bbe5943fa08684e1f93a17881b0Christoph Lameter{ 920559fb51ba7e66fe298b8355fabde1275b7def35fScott Bardone u32 val = readl(sge->adapter->regs + A_PL_ENABLE); 9218199d3a79c224bbe5943fa08684e1f93a17881b0Christoph Lameter 922559fb51ba7e66fe298b8355fabde1275b7def35fScott Bardone writel(val & ~SGE_PL_INTR_MASK, sge->adapter->regs + A_PL_ENABLE); 923559fb51ba7e66fe298b8355fabde1275b7def35fScott Bardone writel(0, sge->adapter->regs + A_SG_INT_ENABLE); 9248199d3a79c224bbe5943fa08684e1f93a17881b0Christoph Lameter} 9258199d3a79c224bbe5943fa08684e1f93a17881b0Christoph Lameter 9268199d3a79c224bbe5943fa08684e1f93a17881b0Christoph Lameter/* 9278199d3a79c224bbe5943fa08684e1f93a17881b0Christoph Lameter * Enable SGE interrupts. 9288199d3a79c224bbe5943fa08684e1f93a17881b0Christoph Lameter */ 9298199d3a79c224bbe5943fa08684e1f93a17881b0Christoph Lametervoid t1_sge_intr_enable(struct sge *sge) 9308199d3a79c224bbe5943fa08684e1f93a17881b0Christoph Lameter{ 9318199d3a79c224bbe5943fa08684e1f93a17881b0Christoph Lameter u32 en = SGE_INT_ENABLE; 932559fb51ba7e66fe298b8355fabde1275b7def35fScott Bardone u32 val = readl(sge->adapter->regs + A_PL_ENABLE); 9338199d3a79c224bbe5943fa08684e1f93a17881b0Christoph Lameter 93430f554f925335abad89aaa38eec6828242b27527Michał Mirosław if (sge->adapter->port[0].dev->hw_features & NETIF_F_TSO) 9358199d3a79c224bbe5943fa08684e1f93a17881b0Christoph Lameter en &= ~F_PACKET_TOO_BIG; 936559fb51ba7e66fe298b8355fabde1275b7def35fScott Bardone writel(en, sge->adapter->regs + A_SG_INT_ENABLE); 937559fb51ba7e66fe298b8355fabde1275b7def35fScott Bardone writel(val | SGE_PL_INTR_MASK, sge->adapter->regs + A_PL_ENABLE); 9388199d3a79c224bbe5943fa08684e1f93a17881b0Christoph Lameter} 9398199d3a79c224bbe5943fa08684e1f93a17881b0Christoph Lameter 9408199d3a79c224bbe5943fa08684e1f93a17881b0Christoph Lameter/* 9418199d3a79c224bbe5943fa08684e1f93a17881b0Christoph Lameter * Clear SGE interrupts. 9428199d3a79c224bbe5943fa08684e1f93a17881b0Christoph Lameter */ 9438199d3a79c224bbe5943fa08684e1f93a17881b0Christoph Lametervoid t1_sge_intr_clear(struct sge *sge) 9448199d3a79c224bbe5943fa08684e1f93a17881b0Christoph Lameter{ 945559fb51ba7e66fe298b8355fabde1275b7def35fScott Bardone writel(SGE_PL_INTR_MASK, sge->adapter->regs + A_PL_CAUSE); 946559fb51ba7e66fe298b8355fabde1275b7def35fScott Bardone writel(0xffffffff, sge->adapter->regs + A_SG_INT_CAUSE); 9478199d3a79c224bbe5943fa08684e1f93a17881b0Christoph Lameter} 9488199d3a79c224bbe5943fa08684e1f93a17881b0Christoph Lameter 9498199d3a79c224bbe5943fa08684e1f93a17881b0Christoph Lameter/* 9508199d3a79c224bbe5943fa08684e1f93a17881b0Christoph Lameter * SGE 'Error' interrupt handler 9518199d3a79c224bbe5943fa08684e1f93a17881b0Christoph Lameter */ 9528199d3a79c224bbe5943fa08684e1f93a17881b0Christoph Lameterint t1_sge_intr_error_handler(struct sge *sge) 9538199d3a79c224bbe5943fa08684e1f93a17881b0Christoph Lameter{ 9548199d3a79c224bbe5943fa08684e1f93a17881b0Christoph Lameter struct adapter *adapter = sge->adapter; 955559fb51ba7e66fe298b8355fabde1275b7def35fScott Bardone u32 cause = readl(adapter->regs + A_SG_INT_CAUSE); 9568199d3a79c224bbe5943fa08684e1f93a17881b0Christoph Lameter 95730f554f925335abad89aaa38eec6828242b27527Michał Mirosław if (adapter->port[0].dev->hw_features & NETIF_F_TSO) 9588199d3a79c224bbe5943fa08684e1f93a17881b0Christoph Lameter cause &= ~F_PACKET_TOO_BIG; 9598199d3a79c224bbe5943fa08684e1f93a17881b0Christoph Lameter if (cause & F_RESPQ_EXHAUSTED) 960559fb51ba7e66fe298b8355fabde1275b7def35fScott Bardone sge->stats.respQ_empty++; 9618199d3a79c224bbe5943fa08684e1f93a17881b0Christoph Lameter if (cause & F_RESPQ_OVERFLOW) { 962559fb51ba7e66fe298b8355fabde1275b7def35fScott Bardone sge->stats.respQ_overflow++; 963c1f51212eb809849bdc68a856ae5f424dcf20d9bJoe Perches pr_alert("%s: SGE response queue overflow\n", 9648199d3a79c224bbe5943fa08684e1f93a17881b0Christoph Lameter adapter->name); 9658199d3a79c224bbe5943fa08684e1f93a17881b0Christoph Lameter } 9668199d3a79c224bbe5943fa08684e1f93a17881b0Christoph Lameter if (cause & F_FL_EXHAUSTED) { 967559fb51ba7e66fe298b8355fabde1275b7def35fScott Bardone sge->stats.freelistQ_empty++; 9688199d3a79c224bbe5943fa08684e1f93a17881b0Christoph Lameter freelQs_empty(sge); 9698199d3a79c224bbe5943fa08684e1f93a17881b0Christoph Lameter } 9708199d3a79c224bbe5943fa08684e1f93a17881b0Christoph Lameter if (cause & F_PACKET_TOO_BIG) { 971559fb51ba7e66fe298b8355fabde1275b7def35fScott Bardone sge->stats.pkt_too_big++; 972c1f51212eb809849bdc68a856ae5f424dcf20d9bJoe Perches pr_alert("%s: SGE max packet size exceeded\n", 9738199d3a79c224bbe5943fa08684e1f93a17881b0Christoph Lameter adapter->name); 9748199d3a79c224bbe5943fa08684e1f93a17881b0Christoph Lameter } 9758199d3a79c224bbe5943fa08684e1f93a17881b0Christoph Lameter if (cause & F_PACKET_MISMATCH) { 976559fb51ba7e66fe298b8355fabde1275b7def35fScott Bardone sge->stats.pkt_mismatch++; 977c1f51212eb809849bdc68a856ae5f424dcf20d9bJoe Perches pr_alert("%s: SGE packet mismatch\n", adapter->name); 9788199d3a79c224bbe5943fa08684e1f93a17881b0Christoph Lameter } 9798199d3a79c224bbe5943fa08684e1f93a17881b0Christoph Lameter if (cause & SGE_INT_FATAL) 9808199d3a79c224bbe5943fa08684e1f93a17881b0Christoph Lameter t1_fatal_err(adapter); 9818199d3a79c224bbe5943fa08684e1f93a17881b0Christoph Lameter 982559fb51ba7e66fe298b8355fabde1275b7def35fScott Bardone writel(cause, adapter->regs + A_SG_INT_CAUSE); 9838199d3a79c224bbe5943fa08684e1f93a17881b0Christoph Lameter return 0; 9848199d3a79c224bbe5943fa08684e1f93a17881b0Christoph Lameter} 9858199d3a79c224bbe5943fa08684e1f93a17881b0Christoph Lameter 98656f643c28c5df63693d7c66e56f8e4767cfd7a65Stephen Hemmingerconst struct sge_intr_counts *t1_sge_get_intr_counts(const struct sge *sge) 987559fb51ba7e66fe298b8355fabde1275b7def35fScott Bardone{ 988559fb51ba7e66fe298b8355fabde1275b7def35fScott Bardone return &sge->stats; 989559fb51ba7e66fe298b8355fabde1275b7def35fScott Bardone} 990559fb51ba7e66fe298b8355fabde1275b7def35fScott Bardone 99156f643c28c5df63693d7c66e56f8e4767cfd7a65Stephen Hemmingervoid t1_sge_get_port_stats(const struct sge *sge, int port, 99256f643c28c5df63693d7c66e56f8e4767cfd7a65Stephen Hemminger struct sge_port_stats *ss) 993559fb51ba7e66fe298b8355fabde1275b7def35fScott Bardone{ 99456f643c28c5df63693d7c66e56f8e4767cfd7a65Stephen Hemminger int cpu; 99556f643c28c5df63693d7c66e56f8e4767cfd7a65Stephen Hemminger 99656f643c28c5df63693d7c66e56f8e4767cfd7a65Stephen Hemminger memset(ss, 0, sizeof(*ss)); 99756f643c28c5df63693d7c66e56f8e4767cfd7a65Stephen Hemminger for_each_possible_cpu(cpu) { 99856f643c28c5df63693d7c66e56f8e4767cfd7a65Stephen Hemminger struct sge_port_stats *st = per_cpu_ptr(sge->port_stats[port], cpu); 99956f643c28c5df63693d7c66e56f8e4767cfd7a65Stephen Hemminger 100056f643c28c5df63693d7c66e56f8e4767cfd7a65Stephen Hemminger ss->rx_cso_good += st->rx_cso_good; 100156f643c28c5df63693d7c66e56f8e4767cfd7a65Stephen Hemminger ss->tx_cso += st->tx_cso; 100256f643c28c5df63693d7c66e56f8e4767cfd7a65Stephen Hemminger ss->tx_tso += st->tx_tso; 10037832ee034b6ef78aab020c9ec1348544cd65ccbdDivy Le Ray ss->tx_need_hdrroom += st->tx_need_hdrroom; 100456f643c28c5df63693d7c66e56f8e4767cfd7a65Stephen Hemminger ss->vlan_xtract += st->vlan_xtract; 100556f643c28c5df63693d7c66e56f8e4767cfd7a65Stephen Hemminger ss->vlan_insert += st->vlan_insert; 100656f643c28c5df63693d7c66e56f8e4767cfd7a65Stephen Hemminger } 1007559fb51ba7e66fe298b8355fabde1275b7def35fScott Bardone} 1008559fb51ba7e66fe298b8355fabde1275b7def35fScott Bardone 1009559fb51ba7e66fe298b8355fabde1275b7def35fScott Bardone/** 1010559fb51ba7e66fe298b8355fabde1275b7def35fScott Bardone * recycle_fl_buf - recycle a free list buffer 1011559fb51ba7e66fe298b8355fabde1275b7def35fScott Bardone * @fl: the free list 1012559fb51ba7e66fe298b8355fabde1275b7def35fScott Bardone * @idx: index of buffer to recycle 10138199d3a79c224bbe5943fa08684e1f93a17881b0Christoph Lameter * 1014559fb51ba7e66fe298b8355fabde1275b7def35fScott Bardone * Recycles the specified buffer on the given free list by adding it at 1015559fb51ba7e66fe298b8355fabde1275b7def35fScott Bardone * the next available slot on the list. 10168199d3a79c224bbe5943fa08684e1f93a17881b0Christoph Lameter */ 1017559fb51ba7e66fe298b8355fabde1275b7def35fScott Bardonestatic void recycle_fl_buf(struct freelQ *fl, int idx) 10188199d3a79c224bbe5943fa08684e1f93a17881b0Christoph Lameter{ 1019559fb51ba7e66fe298b8355fabde1275b7def35fScott Bardone struct freelQ_e *from = &fl->entries[idx]; 1020559fb51ba7e66fe298b8355fabde1275b7def35fScott Bardone struct freelQ_e *to = &fl->entries[fl->pidx]; 10218199d3a79c224bbe5943fa08684e1f93a17881b0Christoph Lameter 1022559fb51ba7e66fe298b8355fabde1275b7def35fScott Bardone fl->centries[fl->pidx] = fl->centries[idx]; 1023559fb51ba7e66fe298b8355fabde1275b7def35fScott Bardone to->addr_lo = from->addr_lo; 1024559fb51ba7e66fe298b8355fabde1275b7def35fScott Bardone to->addr_hi = from->addr_hi; 1025559fb51ba7e66fe298b8355fabde1275b7def35fScott Bardone to->len_gen = G_CMD_LEN(from->len_gen) | V_CMD_GEN1(fl->genbit); 1026559fb51ba7e66fe298b8355fabde1275b7def35fScott Bardone wmb(); 1027559fb51ba7e66fe298b8355fabde1275b7def35fScott Bardone to->gen2 = V_CMD_GEN2(fl->genbit); 1028559fb51ba7e66fe298b8355fabde1275b7def35fScott Bardone fl->credits++; 10298199d3a79c224bbe5943fa08684e1f93a17881b0Christoph Lameter 1030559fb51ba7e66fe298b8355fabde1275b7def35fScott Bardone if (++fl->pidx == fl->size) { 1031559fb51ba7e66fe298b8355fabde1275b7def35fScott Bardone fl->pidx = 0; 1032559fb51ba7e66fe298b8355fabde1275b7def35fScott Bardone fl->genbit ^= 1; 10338199d3a79c224bbe5943fa08684e1f93a17881b0Christoph Lameter } 1034559fb51ba7e66fe298b8355fabde1275b7def35fScott Bardone} 10358199d3a79c224bbe5943fa08684e1f93a17881b0Christoph Lameter 103624a427cf76984726641ea0d8163e61e99119069dStephen Hemmingerstatic int copybreak __read_mostly = 256; 103724a427cf76984726641ea0d8163e61e99119069dStephen Hemmingermodule_param(copybreak, int, 0); 103824a427cf76984726641ea0d8163e61e99119069dStephen HemmingerMODULE_PARM_DESC(copybreak, "Receive copy threshold"); 103924a427cf76984726641ea0d8163e61e99119069dStephen Hemminger 1040559fb51ba7e66fe298b8355fabde1275b7def35fScott Bardone/** 1041559fb51ba7e66fe298b8355fabde1275b7def35fScott Bardone * get_packet - return the next ingress packet buffer 1042559fb51ba7e66fe298b8355fabde1275b7def35fScott Bardone * @pdev: the PCI device that received the packet 1043559fb51ba7e66fe298b8355fabde1275b7def35fScott Bardone * @fl: the SGE free list holding the packet 1044559fb51ba7e66fe298b8355fabde1275b7def35fScott Bardone * @len: the actual packet length, excluding any SGE padding 1045559fb51ba7e66fe298b8355fabde1275b7def35fScott Bardone * 1046559fb51ba7e66fe298b8355fabde1275b7def35fScott Bardone * Get the next packet from a free list and complete setup of the 1047559fb51ba7e66fe298b8355fabde1275b7def35fScott Bardone * sk_buff. If the packet is small we make a copy and recycle the 1048559fb51ba7e66fe298b8355fabde1275b7def35fScott Bardone * original buffer, otherwise we use the original buffer itself. If a 1049559fb51ba7e66fe298b8355fabde1275b7def35fScott Bardone * positive drop threshold is supplied packets are dropped and their 1050559fb51ba7e66fe298b8355fabde1275b7def35fScott Bardone * buffers recycled if (a) the number of remaining buffers is under the 1051559fb51ba7e66fe298b8355fabde1275b7def35fScott Bardone * threshold and the packet is too big to copy, or (b) the packet should 1052559fb51ba7e66fe298b8355fabde1275b7def35fScott Bardone * be copied but there is no memory for the copy. 1053559fb51ba7e66fe298b8355fabde1275b7def35fScott Bardone */ 1054559fb51ba7e66fe298b8355fabde1275b7def35fScott Bardonestatic inline struct sk_buff *get_packet(struct pci_dev *pdev, 105524a427cf76984726641ea0d8163e61e99119069dStephen Hemminger struct freelQ *fl, unsigned int len) 1056559fb51ba7e66fe298b8355fabde1275b7def35fScott Bardone{ 1057559fb51ba7e66fe298b8355fabde1275b7def35fScott Bardone struct sk_buff *skb; 105824a427cf76984726641ea0d8163e61e99119069dStephen Hemminger const struct freelQ_ce *ce = &fl->centries[fl->cidx]; 1059559fb51ba7e66fe298b8355fabde1275b7def35fScott Bardone 106024a427cf76984726641ea0d8163e61e99119069dStephen Hemminger if (len < copybreak) { 106124a427cf76984726641ea0d8163e61e99119069dStephen Hemminger skb = alloc_skb(len + 2, GFP_ATOMIC); 106224a427cf76984726641ea0d8163e61e99119069dStephen Hemminger if (!skb) 106324a427cf76984726641ea0d8163e61e99119069dStephen Hemminger goto use_orig_buf; 106424a427cf76984726641ea0d8163e61e99119069dStephen Hemminger 106524a427cf76984726641ea0d8163e61e99119069dStephen Hemminger skb_reserve(skb, 2); /* align IP header */ 106624a427cf76984726641ea0d8163e61e99119069dStephen Hemminger skb_put(skb, len); 106724a427cf76984726641ea0d8163e61e99119069dStephen Hemminger pci_dma_sync_single_for_cpu(pdev, 1068094f92a61aa044142c231e04c35c00a9cc70adbcFUJITA Tomonori dma_unmap_addr(ce, dma_addr), 1069094f92a61aa044142c231e04c35c00a9cc70adbcFUJITA Tomonori dma_unmap_len(ce, dma_len), 1070559fb51ba7e66fe298b8355fabde1275b7def35fScott Bardone PCI_DMA_FROMDEVICE); 1071d626f62b11e00c16e81e4308ab93d3f13551812aArnaldo Carvalho de Melo skb_copy_from_linear_data(ce->skb, skb->data, len); 107224a427cf76984726641ea0d8163e61e99119069dStephen Hemminger pci_dma_sync_single_for_device(pdev, 1073094f92a61aa044142c231e04c35c00a9cc70adbcFUJITA Tomonori dma_unmap_addr(ce, dma_addr), 1074094f92a61aa044142c231e04c35c00a9cc70adbcFUJITA Tomonori dma_unmap_len(ce, dma_len), 107524a427cf76984726641ea0d8163e61e99119069dStephen Hemminger PCI_DMA_FROMDEVICE); 1076559fb51ba7e66fe298b8355fabde1275b7def35fScott Bardone recycle_fl_buf(fl, fl->cidx); 1077559fb51ba7e66fe298b8355fabde1275b7def35fScott Bardone return skb; 10788199d3a79c224bbe5943fa08684e1f93a17881b0Christoph Lameter } 10798199d3a79c224bbe5943fa08684e1f93a17881b0Christoph Lameter 108024a427cf76984726641ea0d8163e61e99119069dStephen Hemmingeruse_orig_buf: 108124a427cf76984726641ea0d8163e61e99119069dStephen Hemminger if (fl->credits < 2) { 1082559fb51ba7e66fe298b8355fabde1275b7def35fScott Bardone recycle_fl_buf(fl, fl->cidx); 1083559fb51ba7e66fe298b8355fabde1275b7def35fScott Bardone return NULL; 1084559fb51ba7e66fe298b8355fabde1275b7def35fScott Bardone } 10858199d3a79c224bbe5943fa08684e1f93a17881b0Christoph Lameter 1086094f92a61aa044142c231e04c35c00a9cc70adbcFUJITA Tomonori pci_unmap_single(pdev, dma_unmap_addr(ce, dma_addr), 1087094f92a61aa044142c231e04c35c00a9cc70adbcFUJITA Tomonori dma_unmap_len(ce, dma_len), PCI_DMA_FROMDEVICE); 1088559fb51ba7e66fe298b8355fabde1275b7def35fScott Bardone skb = ce->skb; 108924a427cf76984726641ea0d8163e61e99119069dStephen Hemminger prefetch(skb->data); 109024a427cf76984726641ea0d8163e61e99119069dStephen Hemminger 1091559fb51ba7e66fe298b8355fabde1275b7def35fScott Bardone skb_put(skb, len); 1092559fb51ba7e66fe298b8355fabde1275b7def35fScott Bardone return skb; 1093559fb51ba7e66fe298b8355fabde1275b7def35fScott Bardone} 10948199d3a79c224bbe5943fa08684e1f93a17881b0Christoph Lameter 1095559fb51ba7e66fe298b8355fabde1275b7def35fScott Bardone/** 1096559fb51ba7e66fe298b8355fabde1275b7def35fScott Bardone * unexpected_offload - handle an unexpected offload packet 1097559fb51ba7e66fe298b8355fabde1275b7def35fScott Bardone * @adapter: the adapter 1098559fb51ba7e66fe298b8355fabde1275b7def35fScott Bardone * @fl: the free list that received the packet 1099559fb51ba7e66fe298b8355fabde1275b7def35fScott Bardone * 1100559fb51ba7e66fe298b8355fabde1275b7def35fScott Bardone * Called when we receive an unexpected offload packet (e.g., the TOE 1101559fb51ba7e66fe298b8355fabde1275b7def35fScott Bardone * function is disabled or the card is a NIC). Prints a message and 1102559fb51ba7e66fe298b8355fabde1275b7def35fScott Bardone * recycles the buffer. 1103559fb51ba7e66fe298b8355fabde1275b7def35fScott Bardone */ 1104559fb51ba7e66fe298b8355fabde1275b7def35fScott Bardonestatic void unexpected_offload(struct adapter *adapter, struct freelQ *fl) 1105559fb51ba7e66fe298b8355fabde1275b7def35fScott Bardone{ 1106559fb51ba7e66fe298b8355fabde1275b7def35fScott Bardone struct freelQ_ce *ce = &fl->centries[fl->cidx]; 1107559fb51ba7e66fe298b8355fabde1275b7def35fScott Bardone struct sk_buff *skb = ce->skb; 1108559fb51ba7e66fe298b8355fabde1275b7def35fScott Bardone 1109094f92a61aa044142c231e04c35c00a9cc70adbcFUJITA Tomonori pci_dma_sync_single_for_cpu(adapter->pdev, dma_unmap_addr(ce, dma_addr), 1110094f92a61aa044142c231e04c35c00a9cc70adbcFUJITA Tomonori dma_unmap_len(ce, dma_len), PCI_DMA_FROMDEVICE); 1111c1f51212eb809849bdc68a856ae5f424dcf20d9bJoe Perches pr_err("%s: unexpected offload packet, cmd %u\n", 1112559fb51ba7e66fe298b8355fabde1275b7def35fScott Bardone adapter->name, *skb->data); 1113559fb51ba7e66fe298b8355fabde1275b7def35fScott Bardone recycle_fl_buf(fl, fl->cidx); 11148199d3a79c224bbe5943fa08684e1f93a17881b0Christoph Lameter} 11158199d3a79c224bbe5943fa08684e1f93a17881b0Christoph Lameter 11168199d3a79c224bbe5943fa08684e1f93a17881b0Christoph Lameter/* 1117f1d3d38af75789f1b82969b83b69cab540609789Stephen Hemminger * T1/T2 SGE limits the maximum DMA size per TX descriptor to 1118f1d3d38af75789f1b82969b83b69cab540609789Stephen Hemminger * SGE_TX_DESC_MAX_PLEN (16KB). If the PAGE_SIZE is larger than 16KB, the 1119f1d3d38af75789f1b82969b83b69cab540609789Stephen Hemminger * stack might send more than SGE_TX_DESC_MAX_PLEN in a contiguous manner. 1120f1d3d38af75789f1b82969b83b69cab540609789Stephen Hemminger * Note that the *_large_page_tx_descs stuff will be optimized out when 1121f1d3d38af75789f1b82969b83b69cab540609789Stephen Hemminger * PAGE_SIZE <= SGE_TX_DESC_MAX_PLEN. 1122f1d3d38af75789f1b82969b83b69cab540609789Stephen Hemminger * 1123f1d3d38af75789f1b82969b83b69cab540609789Stephen Hemminger * compute_large_page_descs() computes how many additional descriptors are 1124f1d3d38af75789f1b82969b83b69cab540609789Stephen Hemminger * required to break down the stack's request. 1125f1d3d38af75789f1b82969b83b69cab540609789Stephen Hemminger */ 1126f1d3d38af75789f1b82969b83b69cab540609789Stephen Hemmingerstatic inline unsigned int compute_large_page_tx_descs(struct sk_buff *skb) 1127f1d3d38af75789f1b82969b83b69cab540609789Stephen Hemminger{ 1128f1d3d38af75789f1b82969b83b69cab540609789Stephen Hemminger unsigned int count = 0; 1129356bd1460d1e1c4e433e4114fdac02139bddf17cFrancois Romieu 1130f1d3d38af75789f1b82969b83b69cab540609789Stephen Hemminger if (PAGE_SIZE > SGE_TX_DESC_MAX_PLEN) { 1131f1d3d38af75789f1b82969b83b69cab540609789Stephen Hemminger unsigned int nfrags = skb_shinfo(skb)->nr_frags; 1132e743d31312d00932391b123dfac3324d2b9e8c81Eric Dumazet unsigned int i, len = skb_headlen(skb); 1133f1d3d38af75789f1b82969b83b69cab540609789Stephen Hemminger while (len > SGE_TX_DESC_MAX_PLEN) { 1134f1d3d38af75789f1b82969b83b69cab540609789Stephen Hemminger count++; 1135f1d3d38af75789f1b82969b83b69cab540609789Stephen Hemminger len -= SGE_TX_DESC_MAX_PLEN; 1136f1d3d38af75789f1b82969b83b69cab540609789Stephen Hemminger } 1137f1d3d38af75789f1b82969b83b69cab540609789Stephen Hemminger for (i = 0; nfrags--; i++) { 11389e903e085262ffbf1fc44a17ac06058aca03524aEric Dumazet const skb_frag_t *frag = &skb_shinfo(skb)->frags[i]; 11399e903e085262ffbf1fc44a17ac06058aca03524aEric Dumazet len = skb_frag_size(frag); 1140f1d3d38af75789f1b82969b83b69cab540609789Stephen Hemminger while (len > SGE_TX_DESC_MAX_PLEN) { 1141f1d3d38af75789f1b82969b83b69cab540609789Stephen Hemminger count++; 1142f1d3d38af75789f1b82969b83b69cab540609789Stephen Hemminger len -= SGE_TX_DESC_MAX_PLEN; 1143f1d3d38af75789f1b82969b83b69cab540609789Stephen Hemminger } 1144f1d3d38af75789f1b82969b83b69cab540609789Stephen Hemminger } 1145f1d3d38af75789f1b82969b83b69cab540609789Stephen Hemminger } 1146f1d3d38af75789f1b82969b83b69cab540609789Stephen Hemminger return count; 1147f1d3d38af75789f1b82969b83b69cab540609789Stephen Hemminger} 1148f1d3d38af75789f1b82969b83b69cab540609789Stephen Hemminger 1149f1d3d38af75789f1b82969b83b69cab540609789Stephen Hemminger/* 1150f1d3d38af75789f1b82969b83b69cab540609789Stephen Hemminger * Write a cmdQ entry. 1151f1d3d38af75789f1b82969b83b69cab540609789Stephen Hemminger * 1152f1d3d38af75789f1b82969b83b69cab540609789Stephen Hemminger * Since this function writes the 'flags' field, it must not be used to 1153f1d3d38af75789f1b82969b83b69cab540609789Stephen Hemminger * write the first cmdQ entry. 1154f1d3d38af75789f1b82969b83b69cab540609789Stephen Hemminger */ 1155f1d3d38af75789f1b82969b83b69cab540609789Stephen Hemmingerstatic inline void write_tx_desc(struct cmdQ_e *e, dma_addr_t mapping, 1156f1d3d38af75789f1b82969b83b69cab540609789Stephen Hemminger unsigned int len, unsigned int gen, 1157f1d3d38af75789f1b82969b83b69cab540609789Stephen Hemminger unsigned int eop) 1158f1d3d38af75789f1b82969b83b69cab540609789Stephen Hemminger{ 11590ee904c35cc3fdd26a9c76077d9692d458309186Alexander Beregalov BUG_ON(len > SGE_TX_DESC_MAX_PLEN); 11600ee904c35cc3fdd26a9c76077d9692d458309186Alexander Beregalov 1161f1d3d38af75789f1b82969b83b69cab540609789Stephen Hemminger e->addr_lo = (u32)mapping; 1162f1d3d38af75789f1b82969b83b69cab540609789Stephen Hemminger e->addr_hi = (u64)mapping >> 32; 1163f1d3d38af75789f1b82969b83b69cab540609789Stephen Hemminger e->len_gen = V_CMD_LEN(len) | V_CMD_GEN1(gen); 1164f1d3d38af75789f1b82969b83b69cab540609789Stephen Hemminger e->flags = F_CMD_DATAVALID | V_CMD_EOP(eop) | V_CMD_GEN2(gen); 1165f1d3d38af75789f1b82969b83b69cab540609789Stephen Hemminger} 1166f1d3d38af75789f1b82969b83b69cab540609789Stephen Hemminger 1167f1d3d38af75789f1b82969b83b69cab540609789Stephen Hemminger/* 1168f1d3d38af75789f1b82969b83b69cab540609789Stephen Hemminger * See comment for previous function. 1169f1d3d38af75789f1b82969b83b69cab540609789Stephen Hemminger * 1170f1d3d38af75789f1b82969b83b69cab540609789Stephen Hemminger * write_tx_descs_large_page() writes additional SGE tx descriptors if 1171f1d3d38af75789f1b82969b83b69cab540609789Stephen Hemminger * *desc_len exceeds HW's capability. 1172f1d3d38af75789f1b82969b83b69cab540609789Stephen Hemminger */ 1173f1d3d38af75789f1b82969b83b69cab540609789Stephen Hemmingerstatic inline unsigned int write_large_page_tx_descs(unsigned int pidx, 1174f1d3d38af75789f1b82969b83b69cab540609789Stephen Hemminger struct cmdQ_e **e, 1175f1d3d38af75789f1b82969b83b69cab540609789Stephen Hemminger struct cmdQ_ce **ce, 1176f1d3d38af75789f1b82969b83b69cab540609789Stephen Hemminger unsigned int *gen, 1177f1d3d38af75789f1b82969b83b69cab540609789Stephen Hemminger dma_addr_t *desc_mapping, 1178f1d3d38af75789f1b82969b83b69cab540609789Stephen Hemminger unsigned int *desc_len, 1179f1d3d38af75789f1b82969b83b69cab540609789Stephen Hemminger unsigned int nfrags, 1180f1d3d38af75789f1b82969b83b69cab540609789Stephen Hemminger struct cmdQ *q) 1181f1d3d38af75789f1b82969b83b69cab540609789Stephen Hemminger{ 1182f1d3d38af75789f1b82969b83b69cab540609789Stephen Hemminger if (PAGE_SIZE > SGE_TX_DESC_MAX_PLEN) { 1183f1d3d38af75789f1b82969b83b69cab540609789Stephen Hemminger struct cmdQ_e *e1 = *e; 1184f1d3d38af75789f1b82969b83b69cab540609789Stephen Hemminger struct cmdQ_ce *ce1 = *ce; 1185f1d3d38af75789f1b82969b83b69cab540609789Stephen Hemminger 1186f1d3d38af75789f1b82969b83b69cab540609789Stephen Hemminger while (*desc_len > SGE_TX_DESC_MAX_PLEN) { 1187f1d3d38af75789f1b82969b83b69cab540609789Stephen Hemminger *desc_len -= SGE_TX_DESC_MAX_PLEN; 1188f1d3d38af75789f1b82969b83b69cab540609789Stephen Hemminger write_tx_desc(e1, *desc_mapping, SGE_TX_DESC_MAX_PLEN, 1189f1d3d38af75789f1b82969b83b69cab540609789Stephen Hemminger *gen, nfrags == 0 && *desc_len == 0); 1190f1d3d38af75789f1b82969b83b69cab540609789Stephen Hemminger ce1->skb = NULL; 1191094f92a61aa044142c231e04c35c00a9cc70adbcFUJITA Tomonori dma_unmap_len_set(ce1, dma_len, 0); 1192f1d3d38af75789f1b82969b83b69cab540609789Stephen Hemminger *desc_mapping += SGE_TX_DESC_MAX_PLEN; 1193f1d3d38af75789f1b82969b83b69cab540609789Stephen Hemminger if (*desc_len) { 1194f1d3d38af75789f1b82969b83b69cab540609789Stephen Hemminger ce1++; 1195f1d3d38af75789f1b82969b83b69cab540609789Stephen Hemminger e1++; 1196f1d3d38af75789f1b82969b83b69cab540609789Stephen Hemminger if (++pidx == q->size) { 1197f1d3d38af75789f1b82969b83b69cab540609789Stephen Hemminger pidx = 0; 1198f1d3d38af75789f1b82969b83b69cab540609789Stephen Hemminger *gen ^= 1; 1199f1d3d38af75789f1b82969b83b69cab540609789Stephen Hemminger ce1 = q->centries; 1200f1d3d38af75789f1b82969b83b69cab540609789Stephen Hemminger e1 = q->entries; 1201f1d3d38af75789f1b82969b83b69cab540609789Stephen Hemminger } 1202f1d3d38af75789f1b82969b83b69cab540609789Stephen Hemminger } 1203f1d3d38af75789f1b82969b83b69cab540609789Stephen Hemminger } 1204f1d3d38af75789f1b82969b83b69cab540609789Stephen Hemminger *e = e1; 1205f1d3d38af75789f1b82969b83b69cab540609789Stephen Hemminger *ce = ce1; 1206f1d3d38af75789f1b82969b83b69cab540609789Stephen Hemminger } 1207f1d3d38af75789f1b82969b83b69cab540609789Stephen Hemminger return pidx; 1208f1d3d38af75789f1b82969b83b69cab540609789Stephen Hemminger} 1209f1d3d38af75789f1b82969b83b69cab540609789Stephen Hemminger 1210f1d3d38af75789f1b82969b83b69cab540609789Stephen Hemminger/* 1211559fb51ba7e66fe298b8355fabde1275b7def35fScott Bardone * Write the command descriptors to transmit the given skb starting at 1212559fb51ba7e66fe298b8355fabde1275b7def35fScott Bardone * descriptor pidx with the given generation. 12138199d3a79c224bbe5943fa08684e1f93a17881b0Christoph Lameter */ 1214559fb51ba7e66fe298b8355fabde1275b7def35fScott Bardonestatic inline void write_tx_descs(struct adapter *adapter, struct sk_buff *skb, 1215559fb51ba7e66fe298b8355fabde1275b7def35fScott Bardone unsigned int pidx, unsigned int gen, 1216559fb51ba7e66fe298b8355fabde1275b7def35fScott Bardone struct cmdQ *q) 12178199d3a79c224bbe5943fa08684e1f93a17881b0Christoph Lameter{ 1218f1d3d38af75789f1b82969b83b69cab540609789Stephen Hemminger dma_addr_t mapping, desc_mapping; 1219559fb51ba7e66fe298b8355fabde1275b7def35fScott Bardone struct cmdQ_e *e, *e1; 1220559fb51ba7e66fe298b8355fabde1275b7def35fScott Bardone struct cmdQ_ce *ce; 1221f1d3d38af75789f1b82969b83b69cab540609789Stephen Hemminger unsigned int i, flags, first_desc_len, desc_len, 1222f1d3d38af75789f1b82969b83b69cab540609789Stephen Hemminger nfrags = skb_shinfo(skb)->nr_frags; 1223559fb51ba7e66fe298b8355fabde1275b7def35fScott Bardone 1224f1d3d38af75789f1b82969b83b69cab540609789Stephen Hemminger e = e1 = &q->entries[pidx]; 1225559fb51ba7e66fe298b8355fabde1275b7def35fScott Bardone ce = &q->centries[pidx]; 1226f1d3d38af75789f1b82969b83b69cab540609789Stephen Hemminger 1227f1d3d38af75789f1b82969b83b69cab540609789Stephen Hemminger mapping = pci_map_single(adapter->pdev, skb->data, 1228e743d31312d00932391b123dfac3324d2b9e8c81Eric Dumazet skb_headlen(skb), PCI_DMA_TODEVICE); 1229f1d3d38af75789f1b82969b83b69cab540609789Stephen Hemminger 1230f1d3d38af75789f1b82969b83b69cab540609789Stephen Hemminger desc_mapping = mapping; 1231e743d31312d00932391b123dfac3324d2b9e8c81Eric Dumazet desc_len = skb_headlen(skb); 1232f1d3d38af75789f1b82969b83b69cab540609789Stephen Hemminger 1233f1d3d38af75789f1b82969b83b69cab540609789Stephen Hemminger flags = F_CMD_DATAVALID | F_CMD_SOP | 1234f1d3d38af75789f1b82969b83b69cab540609789Stephen Hemminger V_CMD_EOP(nfrags == 0 && desc_len <= SGE_TX_DESC_MAX_PLEN) | 1235f1d3d38af75789f1b82969b83b69cab540609789Stephen Hemminger V_CMD_GEN2(gen); 1236f1d3d38af75789f1b82969b83b69cab540609789Stephen Hemminger first_desc_len = (desc_len <= SGE_TX_DESC_MAX_PLEN) ? 1237f1d3d38af75789f1b82969b83b69cab540609789Stephen Hemminger desc_len : SGE_TX_DESC_MAX_PLEN; 1238f1d3d38af75789f1b82969b83b69cab540609789Stephen Hemminger e->addr_lo = (u32)desc_mapping; 1239f1d3d38af75789f1b82969b83b69cab540609789Stephen Hemminger e->addr_hi = (u64)desc_mapping >> 32; 1240f1d3d38af75789f1b82969b83b69cab540609789Stephen Hemminger e->len_gen = V_CMD_LEN(first_desc_len) | V_CMD_GEN1(gen); 1241f1d3d38af75789f1b82969b83b69cab540609789Stephen Hemminger ce->skb = NULL; 1242094f92a61aa044142c231e04c35c00a9cc70adbcFUJITA Tomonori dma_unmap_len_set(ce, dma_len, 0); 1243f1d3d38af75789f1b82969b83b69cab540609789Stephen Hemminger 1244f1d3d38af75789f1b82969b83b69cab540609789Stephen Hemminger if (PAGE_SIZE > SGE_TX_DESC_MAX_PLEN && 1245f1d3d38af75789f1b82969b83b69cab540609789Stephen Hemminger desc_len > SGE_TX_DESC_MAX_PLEN) { 1246f1d3d38af75789f1b82969b83b69cab540609789Stephen Hemminger desc_mapping += first_desc_len; 1247f1d3d38af75789f1b82969b83b69cab540609789Stephen Hemminger desc_len -= first_desc_len; 1248f1d3d38af75789f1b82969b83b69cab540609789Stephen Hemminger e1++; 1249f1d3d38af75789f1b82969b83b69cab540609789Stephen Hemminger ce++; 1250f1d3d38af75789f1b82969b83b69cab540609789Stephen Hemminger if (++pidx == q->size) { 1251f1d3d38af75789f1b82969b83b69cab540609789Stephen Hemminger pidx = 0; 1252f1d3d38af75789f1b82969b83b69cab540609789Stephen Hemminger gen ^= 1; 1253f1d3d38af75789f1b82969b83b69cab540609789Stephen Hemminger e1 = q->entries; 1254f1d3d38af75789f1b82969b83b69cab540609789Stephen Hemminger ce = q->centries; 1255f1d3d38af75789f1b82969b83b69cab540609789Stephen Hemminger } 1256f1d3d38af75789f1b82969b83b69cab540609789Stephen Hemminger pidx = write_large_page_tx_descs(pidx, &e1, &ce, &gen, 1257f1d3d38af75789f1b82969b83b69cab540609789Stephen Hemminger &desc_mapping, &desc_len, 1258f1d3d38af75789f1b82969b83b69cab540609789Stephen Hemminger nfrags, q); 1259f1d3d38af75789f1b82969b83b69cab540609789Stephen Hemminger 1260f1d3d38af75789f1b82969b83b69cab540609789Stephen Hemminger if (likely(desc_len)) 1261f1d3d38af75789f1b82969b83b69cab540609789Stephen Hemminger write_tx_desc(e1, desc_mapping, desc_len, gen, 1262f1d3d38af75789f1b82969b83b69cab540609789Stephen Hemminger nfrags == 0); 1263f1d3d38af75789f1b82969b83b69cab540609789Stephen Hemminger } 1264f1d3d38af75789f1b82969b83b69cab540609789Stephen Hemminger 1265559fb51ba7e66fe298b8355fabde1275b7def35fScott Bardone ce->skb = NULL; 1266094f92a61aa044142c231e04c35c00a9cc70adbcFUJITA Tomonori dma_unmap_addr_set(ce, dma_addr, mapping); 1267e743d31312d00932391b123dfac3324d2b9e8c81Eric Dumazet dma_unmap_len_set(ce, dma_len, skb_headlen(skb)); 12688199d3a79c224bbe5943fa08684e1f93a17881b0Christoph Lameter 1269f1d3d38af75789f1b82969b83b69cab540609789Stephen Hemminger for (i = 0; nfrags--; i++) { 1270559fb51ba7e66fe298b8355fabde1275b7def35fScott Bardone skb_frag_t *frag = &skb_shinfo(skb)->frags[i]; 1271559fb51ba7e66fe298b8355fabde1275b7def35fScott Bardone e1++; 1272f1d3d38af75789f1b82969b83b69cab540609789Stephen Hemminger ce++; 1273559fb51ba7e66fe298b8355fabde1275b7def35fScott Bardone if (++pidx == q->size) { 1274559fb51ba7e66fe298b8355fabde1275b7def35fScott Bardone pidx = 0; 1275559fb51ba7e66fe298b8355fabde1275b7def35fScott Bardone gen ^= 1; 1276559fb51ba7e66fe298b8355fabde1275b7def35fScott Bardone e1 = q->entries; 1277f1d3d38af75789f1b82969b83b69cab540609789Stephen Hemminger ce = q->centries; 12788199d3a79c224bbe5943fa08684e1f93a17881b0Christoph Lameter } 12798199d3a79c224bbe5943fa08684e1f93a17881b0Christoph Lameter 1280011392224b4f6019c3b737f3dcb7c8f6fba41246Ian Campbell mapping = skb_frag_dma_map(&adapter->pdev->dev, frag, 0, 12819e903e085262ffbf1fc44a17ac06058aca03524aEric Dumazet skb_frag_size(frag), DMA_TO_DEVICE); 1282f1d3d38af75789f1b82969b83b69cab540609789Stephen Hemminger desc_mapping = mapping; 12839e903e085262ffbf1fc44a17ac06058aca03524aEric Dumazet desc_len = skb_frag_size(frag); 1284f1d3d38af75789f1b82969b83b69cab540609789Stephen Hemminger 1285f1d3d38af75789f1b82969b83b69cab540609789Stephen Hemminger pidx = write_large_page_tx_descs(pidx, &e1, &ce, &gen, 1286f1d3d38af75789f1b82969b83b69cab540609789Stephen Hemminger &desc_mapping, &desc_len, 1287f1d3d38af75789f1b82969b83b69cab540609789Stephen Hemminger nfrags, q); 1288f1d3d38af75789f1b82969b83b69cab540609789Stephen Hemminger if (likely(desc_len)) 1289f1d3d38af75789f1b82969b83b69cab540609789Stephen Hemminger write_tx_desc(e1, desc_mapping, desc_len, gen, 1290f1d3d38af75789f1b82969b83b69cab540609789Stephen Hemminger nfrags == 0); 1291559fb51ba7e66fe298b8355fabde1275b7def35fScott Bardone ce->skb = NULL; 1292094f92a61aa044142c231e04c35c00a9cc70adbcFUJITA Tomonori dma_unmap_addr_set(ce, dma_addr, mapping); 12939e903e085262ffbf1fc44a17ac06058aca03524aEric Dumazet dma_unmap_len_set(ce, dma_len, skb_frag_size(frag)); 12948199d3a79c224bbe5943fa08684e1f93a17881b0Christoph Lameter } 1295559fb51ba7e66fe298b8355fabde1275b7def35fScott Bardone ce->skb = skb; 1296559fb51ba7e66fe298b8355fabde1275b7def35fScott Bardone wmb(); 1297559fb51ba7e66fe298b8355fabde1275b7def35fScott Bardone e->flags = flags; 1298559fb51ba7e66fe298b8355fabde1275b7def35fScott Bardone} 12998199d3a79c224bbe5943fa08684e1f93a17881b0Christoph Lameter 1300559fb51ba7e66fe298b8355fabde1275b7def35fScott Bardone/* 1301559fb51ba7e66fe298b8355fabde1275b7def35fScott Bardone * Clean up completed Tx buffers. 1302559fb51ba7e66fe298b8355fabde1275b7def35fScott Bardone */ 1303559fb51ba7e66fe298b8355fabde1275b7def35fScott Bardonestatic inline void reclaim_completed_tx(struct sge *sge, struct cmdQ *q) 1304559fb51ba7e66fe298b8355fabde1275b7def35fScott Bardone{ 1305559fb51ba7e66fe298b8355fabde1275b7def35fScott Bardone unsigned int reclaim = q->processed - q->cleaned; 13068199d3a79c224bbe5943fa08684e1f93a17881b0Christoph Lameter 1307559fb51ba7e66fe298b8355fabde1275b7def35fScott Bardone if (reclaim) { 1308f1d3d38af75789f1b82969b83b69cab540609789Stephen Hemminger pr_debug("reclaim_completed_tx processed:%d cleaned:%d\n", 1309f1d3d38af75789f1b82969b83b69cab540609789Stephen Hemminger q->processed, q->cleaned); 1310559fb51ba7e66fe298b8355fabde1275b7def35fScott Bardone free_cmdQ_buffers(sge, q, reclaim); 1311559fb51ba7e66fe298b8355fabde1275b7def35fScott Bardone q->cleaned += reclaim; 13128199d3a79c224bbe5943fa08684e1f93a17881b0Christoph Lameter } 1313559fb51ba7e66fe298b8355fabde1275b7def35fScott Bardone} 13148199d3a79c224bbe5943fa08684e1f93a17881b0Christoph Lameter 1315f1d3d38af75789f1b82969b83b69cab540609789Stephen Hemminger/* 1316f1d3d38af75789f1b82969b83b69cab540609789Stephen Hemminger * Called from tasklet. Checks the scheduler for any 1317f1d3d38af75789f1b82969b83b69cab540609789Stephen Hemminger * pending skbs that can be sent. 1318f1d3d38af75789f1b82969b83b69cab540609789Stephen Hemminger */ 1319f1d3d38af75789f1b82969b83b69cab540609789Stephen Hemmingerstatic void restart_sched(unsigned long arg) 1320f1d3d38af75789f1b82969b83b69cab540609789Stephen Hemminger{ 1321f1d3d38af75789f1b82969b83b69cab540609789Stephen Hemminger struct sge *sge = (struct sge *) arg; 1322f1d3d38af75789f1b82969b83b69cab540609789Stephen Hemminger struct adapter *adapter = sge->adapter; 1323f1d3d38af75789f1b82969b83b69cab540609789Stephen Hemminger struct cmdQ *q = &sge->cmdQ[0]; 1324f1d3d38af75789f1b82969b83b69cab540609789Stephen Hemminger struct sk_buff *skb; 1325f1d3d38af75789f1b82969b83b69cab540609789Stephen Hemminger unsigned int credits, queued_skb = 0; 1326f1d3d38af75789f1b82969b83b69cab540609789Stephen Hemminger 1327f1d3d38af75789f1b82969b83b69cab540609789Stephen Hemminger spin_lock(&q->lock); 1328f1d3d38af75789f1b82969b83b69cab540609789Stephen Hemminger reclaim_completed_tx(sge, q); 1329f1d3d38af75789f1b82969b83b69cab540609789Stephen Hemminger 1330f1d3d38af75789f1b82969b83b69cab540609789Stephen Hemminger credits = q->size - q->in_use; 1331f1d3d38af75789f1b82969b83b69cab540609789Stephen Hemminger pr_debug("restart_sched credits=%d\n", credits); 1332f1d3d38af75789f1b82969b83b69cab540609789Stephen Hemminger while ((skb = sched_skb(sge, NULL, credits)) != NULL) { 1333f1d3d38af75789f1b82969b83b69cab540609789Stephen Hemminger unsigned int genbit, pidx, count; 1334f1d3d38af75789f1b82969b83b69cab540609789Stephen Hemminger count = 1 + skb_shinfo(skb)->nr_frags; 1335356bd1460d1e1c4e433e4114fdac02139bddf17cFrancois Romieu count += compute_large_page_tx_descs(skb); 1336f1d3d38af75789f1b82969b83b69cab540609789Stephen Hemminger q->in_use += count; 1337f1d3d38af75789f1b82969b83b69cab540609789Stephen Hemminger genbit = q->genbit; 1338f1d3d38af75789f1b82969b83b69cab540609789Stephen Hemminger pidx = q->pidx; 1339f1d3d38af75789f1b82969b83b69cab540609789Stephen Hemminger q->pidx += count; 1340f1d3d38af75789f1b82969b83b69cab540609789Stephen Hemminger if (q->pidx >= q->size) { 1341f1d3d38af75789f1b82969b83b69cab540609789Stephen Hemminger q->pidx -= q->size; 1342f1d3d38af75789f1b82969b83b69cab540609789Stephen Hemminger q->genbit ^= 1; 1343f1d3d38af75789f1b82969b83b69cab540609789Stephen Hemminger } 1344f1d3d38af75789f1b82969b83b69cab540609789Stephen Hemminger write_tx_descs(adapter, skb, pidx, genbit, q); 1345f1d3d38af75789f1b82969b83b69cab540609789Stephen Hemminger credits = q->size - q->in_use; 1346f1d3d38af75789f1b82969b83b69cab540609789Stephen Hemminger queued_skb = 1; 1347f1d3d38af75789f1b82969b83b69cab540609789Stephen Hemminger } 1348f1d3d38af75789f1b82969b83b69cab540609789Stephen Hemminger 1349f1d3d38af75789f1b82969b83b69cab540609789Stephen Hemminger if (queued_skb) { 1350f1d3d38af75789f1b82969b83b69cab540609789Stephen Hemminger clear_bit(CMDQ_STAT_LAST_PKT_DB, &q->status); 1351f1d3d38af75789f1b82969b83b69cab540609789Stephen Hemminger if (test_and_set_bit(CMDQ_STAT_RUNNING, &q->status) == 0) { 1352f1d3d38af75789f1b82969b83b69cab540609789Stephen Hemminger set_bit(CMDQ_STAT_LAST_PKT_DB, &q->status); 1353f1d3d38af75789f1b82969b83b69cab540609789Stephen Hemminger writel(F_CMDQ0_ENABLE, adapter->regs + A_SG_DOORBELL); 1354f1d3d38af75789f1b82969b83b69cab540609789Stephen Hemminger } 1355f1d3d38af75789f1b82969b83b69cab540609789Stephen Hemminger } 1356f1d3d38af75789f1b82969b83b69cab540609789Stephen Hemminger spin_unlock(&q->lock); 1357f1d3d38af75789f1b82969b83b69cab540609789Stephen Hemminger} 13588199d3a79c224bbe5943fa08684e1f93a17881b0Christoph Lameter 1359559fb51ba7e66fe298b8355fabde1275b7def35fScott Bardone/** 1360559fb51ba7e66fe298b8355fabde1275b7def35fScott Bardone * sge_rx - process an ingress ethernet packet 1361559fb51ba7e66fe298b8355fabde1275b7def35fScott Bardone * @sge: the sge structure 1362559fb51ba7e66fe298b8355fabde1275b7def35fScott Bardone * @fl: the free list that contains the packet buffer 1363559fb51ba7e66fe298b8355fabde1275b7def35fScott Bardone * @len: the packet length 13648199d3a79c224bbe5943fa08684e1f93a17881b0Christoph Lameter * 1365559fb51ba7e66fe298b8355fabde1275b7def35fScott Bardone * Process an ingress ethernet pakcet and deliver it to the stack. 13668199d3a79c224bbe5943fa08684e1f93a17881b0Christoph Lameter */ 136724a427cf76984726641ea0d8163e61e99119069dStephen Hemmingerstatic void sge_rx(struct sge *sge, struct freelQ *fl, unsigned int len) 13688199d3a79c224bbe5943fa08684e1f93a17881b0Christoph Lameter{ 1369559fb51ba7e66fe298b8355fabde1275b7def35fScott Bardone struct sk_buff *skb; 137024a427cf76984726641ea0d8163e61e99119069dStephen Hemminger const struct cpl_rx_pkt *p; 1371559fb51ba7e66fe298b8355fabde1275b7def35fScott Bardone struct adapter *adapter = sge->adapter; 137256f643c28c5df63693d7c66e56f8e4767cfd7a65Stephen Hemminger struct sge_port_stats *st; 137330f554f925335abad89aaa38eec6828242b27527Michał Mirosław struct net_device *dev; 13748199d3a79c224bbe5943fa08684e1f93a17881b0Christoph Lameter 137524a427cf76984726641ea0d8163e61e99119069dStephen Hemminger skb = get_packet(adapter->pdev, fl, len - sge->rx_pkt_pad); 137656f643c28c5df63693d7c66e56f8e4767cfd7a65Stephen Hemminger if (unlikely(!skb)) { 137756f643c28c5df63693d7c66e56f8e4767cfd7a65Stephen Hemminger sge->stats.rx_drops++; 137824a427cf76984726641ea0d8163e61e99119069dStephen Hemminger return; 13798199d3a79c224bbe5943fa08684e1f93a17881b0Christoph Lameter } 1380559fb51ba7e66fe298b8355fabde1275b7def35fScott Bardone 138124a427cf76984726641ea0d8163e61e99119069dStephen Hemminger p = (const struct cpl_rx_pkt *) skb->data; 1382f1d3d38af75789f1b82969b83b69cab540609789Stephen Hemminger if (p->iff >= adapter->params.nports) { 1383f1d3d38af75789f1b82969b83b69cab540609789Stephen Hemminger kfree_skb(skb); 138424a427cf76984726641ea0d8163e61e99119069dStephen Hemminger return; 1385f1d3d38af75789f1b82969b83b69cab540609789Stephen Hemminger } 138624a427cf76984726641ea0d8163e61e99119069dStephen Hemminger __skb_pull(skb, sizeof(*p)); 1387f1d3d38af75789f1b82969b83b69cab540609789Stephen Hemminger 1388ca0c9584b1f16bd5911893647cb7f1be82e60554Christoph Lameter st = this_cpu_ptr(sge->port_stats[p->iff]); 138930f554f925335abad89aaa38eec6828242b27527Michał Mirosław dev = adapter->port[p->iff].dev; 139056f643c28c5df63693d7c66e56f8e4767cfd7a65Stephen Hemminger 139130f554f925335abad89aaa38eec6828242b27527Michał Mirosław skb->protocol = eth_type_trans(skb, dev); 139230f554f925335abad89aaa38eec6828242b27527Michał Mirosław if ((dev->features & NETIF_F_RXCSUM) && p->csum == 0xffff && 1393559fb51ba7e66fe298b8355fabde1275b7def35fScott Bardone skb->protocol == htons(ETH_P_IP) && 1394559fb51ba7e66fe298b8355fabde1275b7def35fScott Bardone (skb->data[9] == IPPROTO_TCP || skb->data[9] == IPPROTO_UDP)) { 139556f643c28c5df63693d7c66e56f8e4767cfd7a65Stephen Hemminger ++st->rx_cso_good; 1396559fb51ba7e66fe298b8355fabde1275b7def35fScott Bardone skb->ip_summed = CHECKSUM_UNNECESSARY; 1397559fb51ba7e66fe298b8355fabde1275b7def35fScott Bardone } else 1398bc8acf2c8c3e43fcc192762a9f964b3e9a17748bEric Dumazet skb_checksum_none_assert(skb); 1399559fb51ba7e66fe298b8355fabde1275b7def35fScott Bardone 1400133b08513a99d44d9ed916bb83e46343f6ad2855Jiri Pirko if (p->vlan_valid) { 140156f643c28c5df63693d7c66e56f8e4767cfd7a65Stephen Hemminger st->vlan_xtract++; 1402133b08513a99d44d9ed916bb83e46343f6ad2855Jiri Pirko __vlan_hwaccel_put_tag(skb, ntohs(p->vlan)); 1403133b08513a99d44d9ed916bb83e46343f6ad2855Jiri Pirko } 1404133b08513a99d44d9ed916bb83e46343f6ad2855Jiri Pirko netif_receive_skb(skb); 14058199d3a79c224bbe5943fa08684e1f93a17881b0Christoph Lameter} 14068199d3a79c224bbe5943fa08684e1f93a17881b0Christoph Lameter 14078199d3a79c224bbe5943fa08684e1f93a17881b0Christoph Lameter/* 1408559fb51ba7e66fe298b8355fabde1275b7def35fScott Bardone * Returns true if a command queue has enough available descriptors that 14098199d3a79c224bbe5943fa08684e1f93a17881b0Christoph Lameter * we can resume Tx operation after temporarily disabling its packet queue. 14108199d3a79c224bbe5943fa08684e1f93a17881b0Christoph Lameter */ 1411559fb51ba7e66fe298b8355fabde1275b7def35fScott Bardonestatic inline int enough_free_Tx_descs(const struct cmdQ *q) 14128199d3a79c224bbe5943fa08684e1f93a17881b0Christoph Lameter{ 1413559fb51ba7e66fe298b8355fabde1275b7def35fScott Bardone unsigned int r = q->processed - q->cleaned; 1414559fb51ba7e66fe298b8355fabde1275b7def35fScott Bardone 1415559fb51ba7e66fe298b8355fabde1275b7def35fScott Bardone return q->in_use - r < (q->size >> 1); 14168199d3a79c224bbe5943fa08684e1f93a17881b0Christoph Lameter} 14178199d3a79c224bbe5943fa08684e1f93a17881b0Christoph Lameter 14188199d3a79c224bbe5943fa08684e1f93a17881b0Christoph Lameter/* 1419559fb51ba7e66fe298b8355fabde1275b7def35fScott Bardone * Called when sufficient space has become available in the SGE command queues 1420559fb51ba7e66fe298b8355fabde1275b7def35fScott Bardone * after the Tx packet schedulers have been suspended to restart the Tx path. 14218199d3a79c224bbe5943fa08684e1f93a17881b0Christoph Lameter */ 1422559fb51ba7e66fe298b8355fabde1275b7def35fScott Bardonestatic void restart_tx_queues(struct sge *sge) 14238199d3a79c224bbe5943fa08684e1f93a17881b0Christoph Lameter{ 1424559fb51ba7e66fe298b8355fabde1275b7def35fScott Bardone struct adapter *adap = sge->adapter; 14253e0f75be52605a901165fa1d8acf4ffd37a4857bFrancois Romieu int i; 14268199d3a79c224bbe5943fa08684e1f93a17881b0Christoph Lameter 14273e0f75be52605a901165fa1d8acf4ffd37a4857bFrancois Romieu if (!enough_free_Tx_descs(&sge->cmdQ[0])) 14283e0f75be52605a901165fa1d8acf4ffd37a4857bFrancois Romieu return; 1429559fb51ba7e66fe298b8355fabde1275b7def35fScott Bardone 14303e0f75be52605a901165fa1d8acf4ffd37a4857bFrancois Romieu for_each_port(adap, i) { 14313e0f75be52605a901165fa1d8acf4ffd37a4857bFrancois Romieu struct net_device *nd = adap->port[i].dev; 1432559fb51ba7e66fe298b8355fabde1275b7def35fScott Bardone 14333e0f75be52605a901165fa1d8acf4ffd37a4857bFrancois Romieu if (test_and_clear_bit(nd->if_port, &sge->stopped_tx_queues) && 14343e0f75be52605a901165fa1d8acf4ffd37a4857bFrancois Romieu netif_running(nd)) { 14353e0f75be52605a901165fa1d8acf4ffd37a4857bFrancois Romieu sge->stats.cmdQ_restarted[2]++; 14363e0f75be52605a901165fa1d8acf4ffd37a4857bFrancois Romieu netif_wake_queue(nd); 1437559fb51ba7e66fe298b8355fabde1275b7def35fScott Bardone } 1438559fb51ba7e66fe298b8355fabde1275b7def35fScott Bardone } 1439559fb51ba7e66fe298b8355fabde1275b7def35fScott Bardone} 1440559fb51ba7e66fe298b8355fabde1275b7def35fScott Bardone 1441559fb51ba7e66fe298b8355fabde1275b7def35fScott Bardone/* 1442356bd1460d1e1c4e433e4114fdac02139bddf17cFrancois Romieu * update_tx_info is called from the interrupt handler/NAPI to return cmdQ0 1443559fb51ba7e66fe298b8355fabde1275b7def35fScott Bardone * information. 1444559fb51ba7e66fe298b8355fabde1275b7def35fScott Bardone */ 1445356bd1460d1e1c4e433e4114fdac02139bddf17cFrancois Romieustatic unsigned int update_tx_info(struct adapter *adapter, 1446356bd1460d1e1c4e433e4114fdac02139bddf17cFrancois Romieu unsigned int flags, 1447559fb51ba7e66fe298b8355fabde1275b7def35fScott Bardone unsigned int pr0) 1448559fb51ba7e66fe298b8355fabde1275b7def35fScott Bardone{ 1449559fb51ba7e66fe298b8355fabde1275b7def35fScott Bardone struct sge *sge = adapter->sge; 1450559fb51ba7e66fe298b8355fabde1275b7def35fScott Bardone struct cmdQ *cmdq = &sge->cmdQ[0]; 14518199d3a79c224bbe5943fa08684e1f93a17881b0Christoph Lameter 1452559fb51ba7e66fe298b8355fabde1275b7def35fScott Bardone cmdq->processed += pr0; 1453f1d3d38af75789f1b82969b83b69cab540609789Stephen Hemminger if (flags & (F_FL0_ENABLE | F_FL1_ENABLE)) { 1454f1d3d38af75789f1b82969b83b69cab540609789Stephen Hemminger freelQs_empty(sge); 1455f1d3d38af75789f1b82969b83b69cab540609789Stephen Hemminger flags &= ~(F_FL0_ENABLE | F_FL1_ENABLE); 1456f1d3d38af75789f1b82969b83b69cab540609789Stephen Hemminger } 1457559fb51ba7e66fe298b8355fabde1275b7def35fScott Bardone if (flags & F_CMDQ0_ENABLE) { 1458559fb51ba7e66fe298b8355fabde1275b7def35fScott Bardone clear_bit(CMDQ_STAT_RUNNING, &cmdq->status); 1459f1d3d38af75789f1b82969b83b69cab540609789Stephen Hemminger 1460559fb51ba7e66fe298b8355fabde1275b7def35fScott Bardone if (cmdq->cleaned + cmdq->in_use != cmdq->processed && 1461559fb51ba7e66fe298b8355fabde1275b7def35fScott Bardone !test_and_set_bit(CMDQ_STAT_LAST_PKT_DB, &cmdq->status)) { 1462559fb51ba7e66fe298b8355fabde1275b7def35fScott Bardone set_bit(CMDQ_STAT_RUNNING, &cmdq->status); 1463559fb51ba7e66fe298b8355fabde1275b7def35fScott Bardone writel(F_CMDQ0_ENABLE, adapter->regs + A_SG_DOORBELL); 1464559fb51ba7e66fe298b8355fabde1275b7def35fScott Bardone } 1465f1d3d38af75789f1b82969b83b69cab540609789Stephen Hemminger if (sge->tx_sched) 1466f1d3d38af75789f1b82969b83b69cab540609789Stephen Hemminger tasklet_hi_schedule(&sge->tx_sched->sched_tsk); 1467f1d3d38af75789f1b82969b83b69cab540609789Stephen Hemminger 1468f1d3d38af75789f1b82969b83b69cab540609789Stephen Hemminger flags &= ~F_CMDQ0_ENABLE; 1469559fb51ba7e66fe298b8355fabde1275b7def35fScott Bardone } 1470f1d3d38af75789f1b82969b83b69cab540609789Stephen Hemminger 1471559fb51ba7e66fe298b8355fabde1275b7def35fScott Bardone if (unlikely(sge->stopped_tx_queues != 0)) 1472559fb51ba7e66fe298b8355fabde1275b7def35fScott Bardone restart_tx_queues(sge); 14738199d3a79c224bbe5943fa08684e1f93a17881b0Christoph Lameter 1474559fb51ba7e66fe298b8355fabde1275b7def35fScott Bardone return flags; 1475559fb51ba7e66fe298b8355fabde1275b7def35fScott Bardone} 14768199d3a79c224bbe5943fa08684e1f93a17881b0Christoph Lameter 1477559fb51ba7e66fe298b8355fabde1275b7def35fScott Bardone/* 1478559fb51ba7e66fe298b8355fabde1275b7def35fScott Bardone * Process SGE responses, up to the supplied budget. Returns the number of 1479559fb51ba7e66fe298b8355fabde1275b7def35fScott Bardone * responses processed. A negative budget is effectively unlimited. 1480559fb51ba7e66fe298b8355fabde1275b7def35fScott Bardone */ 1481559fb51ba7e66fe298b8355fabde1275b7def35fScott Bardonestatic int process_responses(struct adapter *adapter, int budget) 1482559fb51ba7e66fe298b8355fabde1275b7def35fScott Bardone{ 1483559fb51ba7e66fe298b8355fabde1275b7def35fScott Bardone struct sge *sge = adapter->sge; 1484559fb51ba7e66fe298b8355fabde1275b7def35fScott Bardone struct respQ *q = &sge->respQ; 1485559fb51ba7e66fe298b8355fabde1275b7def35fScott Bardone struct respQ_e *e = &q->entries[q->cidx]; 148624a427cf76984726641ea0d8163e61e99119069dStephen Hemminger int done = 0; 1487559fb51ba7e66fe298b8355fabde1275b7def35fScott Bardone unsigned int flags = 0; 1488559fb51ba7e66fe298b8355fabde1275b7def35fScott Bardone unsigned int cmdq_processed[SGE_CMDQ_N] = {0, 0}; 1489356bd1460d1e1c4e433e4114fdac02139bddf17cFrancois Romieu 149024a427cf76984726641ea0d8163e61e99119069dStephen Hemminger while (done < budget && e->GenerationBit == q->genbit) { 1491559fb51ba7e66fe298b8355fabde1275b7def35fScott Bardone flags |= e->Qsleeping; 1492356bd1460d1e1c4e433e4114fdac02139bddf17cFrancois Romieu 1493559fb51ba7e66fe298b8355fabde1275b7def35fScott Bardone cmdq_processed[0] += e->Cmdq0CreditReturn; 1494559fb51ba7e66fe298b8355fabde1275b7def35fScott Bardone cmdq_processed[1] += e->Cmdq1CreditReturn; 1495356bd1460d1e1c4e433e4114fdac02139bddf17cFrancois Romieu 1496559fb51ba7e66fe298b8355fabde1275b7def35fScott Bardone /* We batch updates to the TX side to avoid cacheline 1497559fb51ba7e66fe298b8355fabde1275b7def35fScott Bardone * ping-pong of TX state information on MP where the sender 1498559fb51ba7e66fe298b8355fabde1275b7def35fScott Bardone * might run on a different CPU than this function... 1499559fb51ba7e66fe298b8355fabde1275b7def35fScott Bardone */ 150024a427cf76984726641ea0d8163e61e99119069dStephen Hemminger if (unlikely((flags & F_CMDQ0_ENABLE) || cmdq_processed[0] > 64)) { 1501559fb51ba7e66fe298b8355fabde1275b7def35fScott Bardone flags = update_tx_info(adapter, flags, cmdq_processed[0]); 1502559fb51ba7e66fe298b8355fabde1275b7def35fScott Bardone cmdq_processed[0] = 0; 1503559fb51ba7e66fe298b8355fabde1275b7def35fScott Bardone } 150424a427cf76984726641ea0d8163e61e99119069dStephen Hemminger 1505559fb51ba7e66fe298b8355fabde1275b7def35fScott Bardone if (unlikely(cmdq_processed[1] > 16)) { 1506559fb51ba7e66fe298b8355fabde1275b7def35fScott Bardone sge->cmdQ[1].processed += cmdq_processed[1]; 1507559fb51ba7e66fe298b8355fabde1275b7def35fScott Bardone cmdq_processed[1] = 0; 15088199d3a79c224bbe5943fa08684e1f93a17881b0Christoph Lameter } 150924a427cf76984726641ea0d8163e61e99119069dStephen Hemminger 15108199d3a79c224bbe5943fa08684e1f93a17881b0Christoph Lameter if (likely(e->DataValid)) { 1511559fb51ba7e66fe298b8355fabde1275b7def35fScott Bardone struct freelQ *fl = &sge->freelQ[e->FreelistQid]; 1512559fb51ba7e66fe298b8355fabde1275b7def35fScott Bardone 15135d9428de1a9785f10a4339f80b717be665ba51c7Eric Sesterhenn BUG_ON(!e->Sop || !e->Eop); 1514559fb51ba7e66fe298b8355fabde1275b7def35fScott Bardone if (unlikely(e->Offload)) 1515559fb51ba7e66fe298b8355fabde1275b7def35fScott Bardone unexpected_offload(adapter, fl); 1516559fb51ba7e66fe298b8355fabde1275b7def35fScott Bardone else 1517559fb51ba7e66fe298b8355fabde1275b7def35fScott Bardone sge_rx(sge, fl, e->BufferLength); 1518559fb51ba7e66fe298b8355fabde1275b7def35fScott Bardone 151924a427cf76984726641ea0d8163e61e99119069dStephen Hemminger ++done; 152024a427cf76984726641ea0d8163e61e99119069dStephen Hemminger 1521559fb51ba7e66fe298b8355fabde1275b7def35fScott Bardone /* 1522559fb51ba7e66fe298b8355fabde1275b7def35fScott Bardone * Note: this depends on each packet consuming a 1523559fb51ba7e66fe298b8355fabde1275b7def35fScott Bardone * single free-list buffer; cf. the BUG above. 1524559fb51ba7e66fe298b8355fabde1275b7def35fScott Bardone */ 1525559fb51ba7e66fe298b8355fabde1275b7def35fScott Bardone if (++fl->cidx == fl->size) 1526559fb51ba7e66fe298b8355fabde1275b7def35fScott Bardone fl->cidx = 0; 152724a427cf76984726641ea0d8163e61e99119069dStephen Hemminger prefetch(fl->centries[fl->cidx].skb); 152824a427cf76984726641ea0d8163e61e99119069dStephen Hemminger 1529559fb51ba7e66fe298b8355fabde1275b7def35fScott Bardone if (unlikely(--fl->credits < 1530559fb51ba7e66fe298b8355fabde1275b7def35fScott Bardone fl->size - SGE_FREEL_REFILL_THRESH)) 1531559fb51ba7e66fe298b8355fabde1275b7def35fScott Bardone refill_free_list(sge, fl); 1532559fb51ba7e66fe298b8355fabde1275b7def35fScott Bardone } else 1533559fb51ba7e66fe298b8355fabde1275b7def35fScott Bardone sge->stats.pure_rsps++; 15348199d3a79c224bbe5943fa08684e1f93a17881b0Christoph Lameter 15358199d3a79c224bbe5943fa08684e1f93a17881b0Christoph Lameter e++; 1536559fb51ba7e66fe298b8355fabde1275b7def35fScott Bardone if (unlikely(++q->cidx == q->size)) { 1537559fb51ba7e66fe298b8355fabde1275b7def35fScott Bardone q->cidx = 0; 1538559fb51ba7e66fe298b8355fabde1275b7def35fScott Bardone q->genbit ^= 1; 1539559fb51ba7e66fe298b8355fabde1275b7def35fScott Bardone e = q->entries; 1540559fb51ba7e66fe298b8355fabde1275b7def35fScott Bardone } 1541559fb51ba7e66fe298b8355fabde1275b7def35fScott Bardone prefetch(e); 1542559fb51ba7e66fe298b8355fabde1275b7def35fScott Bardone 1543559fb51ba7e66fe298b8355fabde1275b7def35fScott Bardone if (++q->credits > SGE_RESPQ_REPLENISH_THRES) { 1544559fb51ba7e66fe298b8355fabde1275b7def35fScott Bardone writel(q->credits, adapter->regs + A_SG_RSPQUEUECREDIT); 1545559fb51ba7e66fe298b8355fabde1275b7def35fScott Bardone q->credits = 0; 15468199d3a79c224bbe5943fa08684e1f93a17881b0Christoph Lameter } 15478199d3a79c224bbe5943fa08684e1f93a17881b0Christoph Lameter } 15488199d3a79c224bbe5943fa08684e1f93a17881b0Christoph Lameter 1549356bd1460d1e1c4e433e4114fdac02139bddf17cFrancois Romieu flags = update_tx_info(adapter, flags, cmdq_processed[0]); 1550559fb51ba7e66fe298b8355fabde1275b7def35fScott Bardone sge->cmdQ[1].processed += cmdq_processed[1]; 15518199d3a79c224bbe5943fa08684e1f93a17881b0Christoph Lameter 155224a427cf76984726641ea0d8163e61e99119069dStephen Hemminger return done; 1553559fb51ba7e66fe298b8355fabde1275b7def35fScott Bardone} 15548199d3a79c224bbe5943fa08684e1f93a17881b0Christoph Lameter 15553de00b8958b12d62712ae9500968c65d3b43bb27Stephen Hemmingerstatic inline int responses_pending(const struct adapter *adapter) 15563de00b8958b12d62712ae9500968c65d3b43bb27Stephen Hemminger{ 15573de00b8958b12d62712ae9500968c65d3b43bb27Stephen Hemminger const struct respQ *Q = &adapter->sge->respQ; 15583de00b8958b12d62712ae9500968c65d3b43bb27Stephen Hemminger const struct respQ_e *e = &Q->entries[Q->cidx]; 15593de00b8958b12d62712ae9500968c65d3b43bb27Stephen Hemminger 1560807540baae406c84dcb9c1c8ef07a56d2d2ae84aEric Dumazet return e->GenerationBit == Q->genbit; 15613de00b8958b12d62712ae9500968c65d3b43bb27Stephen Hemminger} 15623de00b8958b12d62712ae9500968c65d3b43bb27Stephen Hemminger 1563559fb51ba7e66fe298b8355fabde1275b7def35fScott Bardone/* 1564559fb51ba7e66fe298b8355fabde1275b7def35fScott Bardone * A simpler version of process_responses() that handles only pure (i.e., 1565559fb51ba7e66fe298b8355fabde1275b7def35fScott Bardone * non data-carrying) responses. Such respones are too light-weight to justify 1566559fb51ba7e66fe298b8355fabde1275b7def35fScott Bardone * calling a softirq when using NAPI, so we handle them specially in hard 1567559fb51ba7e66fe298b8355fabde1275b7def35fScott Bardone * interrupt context. The function is called with a pointer to a response, 1568559fb51ba7e66fe298b8355fabde1275b7def35fScott Bardone * which the caller must ensure is a valid pure response. Returns 1 if it 1569559fb51ba7e66fe298b8355fabde1275b7def35fScott Bardone * encounters a valid data-carrying response, 0 otherwise. 1570559fb51ba7e66fe298b8355fabde1275b7def35fScott Bardone */ 15713de00b8958b12d62712ae9500968c65d3b43bb27Stephen Hemmingerstatic int process_pure_responses(struct adapter *adapter) 1572559fb51ba7e66fe298b8355fabde1275b7def35fScott Bardone{ 1573559fb51ba7e66fe298b8355fabde1275b7def35fScott Bardone struct sge *sge = adapter->sge; 1574559fb51ba7e66fe298b8355fabde1275b7def35fScott Bardone struct respQ *q = &sge->respQ; 15753de00b8958b12d62712ae9500968c65d3b43bb27Stephen Hemminger struct respQ_e *e = &q->entries[q->cidx]; 157624a427cf76984726641ea0d8163e61e99119069dStephen Hemminger const struct freelQ *fl = &sge->freelQ[e->FreelistQid]; 1577559fb51ba7e66fe298b8355fabde1275b7def35fScott Bardone unsigned int flags = 0; 1578559fb51ba7e66fe298b8355fabde1275b7def35fScott Bardone unsigned int cmdq_processed[SGE_CMDQ_N] = {0, 0}; 15798199d3a79c224bbe5943fa08684e1f93a17881b0Christoph Lameter 158024a427cf76984726641ea0d8163e61e99119069dStephen Hemminger prefetch(fl->centries[fl->cidx].skb); 15813de00b8958b12d62712ae9500968c65d3b43bb27Stephen Hemminger if (e->DataValid) 15823de00b8958b12d62712ae9500968c65d3b43bb27Stephen Hemminger return 1; 158324a427cf76984726641ea0d8163e61e99119069dStephen Hemminger 1584559fb51ba7e66fe298b8355fabde1275b7def35fScott Bardone do { 1585559fb51ba7e66fe298b8355fabde1275b7def35fScott Bardone flags |= e->Qsleeping; 15868199d3a79c224bbe5943fa08684e1f93a17881b0Christoph Lameter 1587559fb51ba7e66fe298b8355fabde1275b7def35fScott Bardone cmdq_processed[0] += e->Cmdq0CreditReturn; 1588559fb51ba7e66fe298b8355fabde1275b7def35fScott Bardone cmdq_processed[1] += e->Cmdq1CreditReturn; 1589356bd1460d1e1c4e433e4114fdac02139bddf17cFrancois Romieu 1590559fb51ba7e66fe298b8355fabde1275b7def35fScott Bardone e++; 1591559fb51ba7e66fe298b8355fabde1275b7def35fScott Bardone if (unlikely(++q->cidx == q->size)) { 1592559fb51ba7e66fe298b8355fabde1275b7def35fScott Bardone q->cidx = 0; 1593559fb51ba7e66fe298b8355fabde1275b7def35fScott Bardone q->genbit ^= 1; 1594559fb51ba7e66fe298b8355fabde1275b7def35fScott Bardone e = q->entries; 15958199d3a79c224bbe5943fa08684e1f93a17881b0Christoph Lameter } 1596559fb51ba7e66fe298b8355fabde1275b7def35fScott Bardone prefetch(e); 15978199d3a79c224bbe5943fa08684e1f93a17881b0Christoph Lameter 1598559fb51ba7e66fe298b8355fabde1275b7def35fScott Bardone if (++q->credits > SGE_RESPQ_REPLENISH_THRES) { 1599559fb51ba7e66fe298b8355fabde1275b7def35fScott Bardone writel(q->credits, adapter->regs + A_SG_RSPQUEUECREDIT); 1600559fb51ba7e66fe298b8355fabde1275b7def35fScott Bardone q->credits = 0; 16018199d3a79c224bbe5943fa08684e1f93a17881b0Christoph Lameter } 1602559fb51ba7e66fe298b8355fabde1275b7def35fScott Bardone sge->stats.pure_rsps++; 1603559fb51ba7e66fe298b8355fabde1275b7def35fScott Bardone } while (e->GenerationBit == q->genbit && !e->DataValid); 16048199d3a79c224bbe5943fa08684e1f93a17881b0Christoph Lameter 1605356bd1460d1e1c4e433e4114fdac02139bddf17cFrancois Romieu flags = update_tx_info(adapter, flags, cmdq_processed[0]); 1606559fb51ba7e66fe298b8355fabde1275b7def35fScott Bardone sge->cmdQ[1].processed += cmdq_processed[1]; 16078199d3a79c224bbe5943fa08684e1f93a17881b0Christoph Lameter 1608559fb51ba7e66fe298b8355fabde1275b7def35fScott Bardone return e->GenerationBit == q->genbit; 16098199d3a79c224bbe5943fa08684e1f93a17881b0Christoph Lameter} 16108199d3a79c224bbe5943fa08684e1f93a17881b0Christoph Lameter 16118199d3a79c224bbe5943fa08684e1f93a17881b0Christoph Lameter/* 1612559fb51ba7e66fe298b8355fabde1275b7def35fScott Bardone * Handler for new data events when using NAPI. This does not need any locking 1613559fb51ba7e66fe298b8355fabde1275b7def35fScott Bardone * or protection from interrupts as data interrupts are off at this point and 1614559fb51ba7e66fe298b8355fabde1275b7def35fScott Bardone * other adapter interrupts do not interfere. 16158199d3a79c224bbe5943fa08684e1f93a17881b0Christoph Lameter */ 1616bea3348eef27e6044b6161fd04c3152215f96411Stephen Hemmingerint t1_poll(struct napi_struct *napi, int budget) 16178199d3a79c224bbe5943fa08684e1f93a17881b0Christoph Lameter{ 1618bea3348eef27e6044b6161fd04c3152215f96411Stephen Hemminger struct adapter *adapter = container_of(napi, struct adapter, napi); 1619445cf803ca757a5c43f4e11fdfba70355d510841Divy Le Ray int work_done = process_responses(adapter, budget); 16207fe26a60e08f38c797851fb3b444d753af616112Stephen Hemminger 1621445cf803ca757a5c43f4e11fdfba70355d510841Divy Le Ray if (likely(work_done < budget)) { 1622288379f050284087578b77e04f040b57db3db3f8Ben Hutchings napi_complete(napi); 1623bea3348eef27e6044b6161fd04c3152215f96411Stephen Hemminger writel(adapter->sge->respQ.cidx, 1624bea3348eef27e6044b6161fd04c3152215f96411Stephen Hemminger adapter->regs + A_SG_SLEEPING); 1625bea3348eef27e6044b6161fd04c3152215f96411Stephen Hemminger } 1626bea3348eef27e6044b6161fd04c3152215f96411Stephen Hemminger return work_done; 1627559fb51ba7e66fe298b8355fabde1275b7def35fScott Bardone} 16288199d3a79c224bbe5943fa08684e1f93a17881b0Christoph Lameter 16297fe26a60e08f38c797851fb3b444d753af616112Stephen Hemmingerirqreturn_t t1_interrupt(int irq, void *data) 1630559fb51ba7e66fe298b8355fabde1275b7def35fScott Bardone{ 1631559fb51ba7e66fe298b8355fabde1275b7def35fScott Bardone struct adapter *adapter = data; 1632559fb51ba7e66fe298b8355fabde1275b7def35fScott Bardone struct sge *sge = adapter->sge; 16333de00b8958b12d62712ae9500968c65d3b43bb27Stephen Hemminger int handled; 1634559fb51ba7e66fe298b8355fabde1275b7def35fScott Bardone 16353de00b8958b12d62712ae9500968c65d3b43bb27Stephen Hemminger if (likely(responses_pending(adapter))) { 1636356bd1460d1e1c4e433e4114fdac02139bddf17cFrancois Romieu writel(F_PL_INTR_SGE_DATA, adapter->regs + A_PL_CAUSE); 16377fe26a60e08f38c797851fb3b444d753af616112Stephen Hemminger 1638bea3348eef27e6044b6161fd04c3152215f96411Stephen Hemminger if (napi_schedule_prep(&adapter->napi)) { 16393de00b8958b12d62712ae9500968c65d3b43bb27Stephen Hemminger if (process_pure_responses(adapter)) 1640288379f050284087578b77e04f040b57db3db3f8Ben Hutchings __napi_schedule(&adapter->napi); 16413de00b8958b12d62712ae9500968c65d3b43bb27Stephen Hemminger else { 16423de00b8958b12d62712ae9500968c65d3b43bb27Stephen Hemminger /* no data, no NAPI needed */ 16433de00b8958b12d62712ae9500968c65d3b43bb27Stephen Hemminger writel(sge->respQ.cidx, adapter->regs + A_SG_SLEEPING); 16444422b00390749f8b877b2838a99ef2948ae08a58Francois Romieu /* undo schedule_prep */ 16454422b00390749f8b877b2838a99ef2948ae08a58Francois Romieu napi_enable(&adapter->napi); 16467fe26a60e08f38c797851fb3b444d753af616112Stephen Hemminger } 16477fe26a60e08f38c797851fb3b444d753af616112Stephen Hemminger } 16483de00b8958b12d62712ae9500968c65d3b43bb27Stephen Hemminger return IRQ_HANDLED; 16493de00b8958b12d62712ae9500968c65d3b43bb27Stephen Hemminger } 16503de00b8958b12d62712ae9500968c65d3b43bb27Stephen Hemminger 16513de00b8958b12d62712ae9500968c65d3b43bb27Stephen Hemminger spin_lock(&adapter->async_lock); 16523de00b8958b12d62712ae9500968c65d3b43bb27Stephen Hemminger handled = t1_slow_intr_handler(adapter); 16533de00b8958b12d62712ae9500968c65d3b43bb27Stephen Hemminger spin_unlock(&adapter->async_lock); 16547fe26a60e08f38c797851fb3b444d753af616112Stephen Hemminger 1655559fb51ba7e66fe298b8355fabde1275b7def35fScott Bardone if (!handled) 1656559fb51ba7e66fe298b8355fabde1275b7def35fScott Bardone sge->stats.unhandled_irqs++; 16573de00b8958b12d62712ae9500968c65d3b43bb27Stephen Hemminger 1658559fb51ba7e66fe298b8355fabde1275b7def35fScott Bardone return IRQ_RETVAL(handled != 0); 1659559fb51ba7e66fe298b8355fabde1275b7def35fScott Bardone} 16608199d3a79c224bbe5943fa08684e1f93a17881b0Christoph Lameter 1661559fb51ba7e66fe298b8355fabde1275b7def35fScott Bardone/* 1662559fb51ba7e66fe298b8355fabde1275b7def35fScott Bardone * Enqueues the sk_buff onto the cmdQ[qid] and has hardware fetch it. 1663559fb51ba7e66fe298b8355fabde1275b7def35fScott Bardone * 1664559fb51ba7e66fe298b8355fabde1275b7def35fScott Bardone * The code figures out how many entries the sk_buff will require in the 1665559fb51ba7e66fe298b8355fabde1275b7def35fScott Bardone * cmdQ and updates the cmdQ data structure with the state once the enqueue 1666559fb51ba7e66fe298b8355fabde1275b7def35fScott Bardone * has complete. Then, it doesn't access the global structure anymore, but 166725985edcedea6396277003854657b5f3cb31a628Lucas De Marchi * uses the corresponding fields on the stack. In conjunction with a spinlock 1668559fb51ba7e66fe298b8355fabde1275b7def35fScott Bardone * around that code, we can make the function reentrant without holding the 1669559fb51ba7e66fe298b8355fabde1275b7def35fScott Bardone * lock when we actually enqueue (which might be expensive, especially on 1670559fb51ba7e66fe298b8355fabde1275b7def35fScott Bardone * architectures with IO MMUs). 1671559fb51ba7e66fe298b8355fabde1275b7def35fScott Bardone * 1672559fb51ba7e66fe298b8355fabde1275b7def35fScott Bardone * This runs with softirqs disabled. 1673559fb51ba7e66fe298b8355fabde1275b7def35fScott Bardone */ 1674aa84505fb0fb9504c61d77e8e6930a417fc404d6Stephen Hemmingerstatic int t1_sge_tx(struct sk_buff *skb, struct adapter *adapter, 1675aa84505fb0fb9504c61d77e8e6930a417fc404d6Stephen Hemminger unsigned int qid, struct net_device *dev) 1676559fb51ba7e66fe298b8355fabde1275b7def35fScott Bardone{ 1677559fb51ba7e66fe298b8355fabde1275b7def35fScott Bardone struct sge *sge = adapter->sge; 1678559fb51ba7e66fe298b8355fabde1275b7def35fScott Bardone struct cmdQ *q = &sge->cmdQ[qid]; 1679f1d3d38af75789f1b82969b83b69cab540609789Stephen Hemminger unsigned int credits, pidx, genbit, count, use_sched_skb = 0; 1680559fb51ba7e66fe298b8355fabde1275b7def35fScott Bardone 1681cabdfb373ae74036225826ce260c16a8e260eb0bStephen Hemminger if (!spin_trylock(&q->lock)) 1682cabdfb373ae74036225826ce260c16a8e260eb0bStephen Hemminger return NETDEV_TX_LOCKED; 1683cabdfb373ae74036225826ce260c16a8e260eb0bStephen Hemminger 1684559fb51ba7e66fe298b8355fabde1275b7def35fScott Bardone reclaim_completed_tx(sge, q); 1685559fb51ba7e66fe298b8355fabde1275b7def35fScott Bardone 1686559fb51ba7e66fe298b8355fabde1275b7def35fScott Bardone pidx = q->pidx; 1687559fb51ba7e66fe298b8355fabde1275b7def35fScott Bardone credits = q->size - q->in_use; 1688559fb51ba7e66fe298b8355fabde1275b7def35fScott Bardone count = 1 + skb_shinfo(skb)->nr_frags; 1689f1d3d38af75789f1b82969b83b69cab540609789Stephen Hemminger count += compute_large_page_tx_descs(skb); 1690559fb51ba7e66fe298b8355fabde1275b7def35fScott Bardone 1691f1d3d38af75789f1b82969b83b69cab540609789Stephen Hemminger /* Ethernet packet */ 1692f1d3d38af75789f1b82969b83b69cab540609789Stephen Hemminger if (unlikely(credits < count)) { 1693f1d3d38af75789f1b82969b83b69cab540609789Stephen Hemminger if (!netif_queue_stopped(dev)) { 1694559fb51ba7e66fe298b8355fabde1275b7def35fScott Bardone netif_stop_queue(dev); 1695559fb51ba7e66fe298b8355fabde1275b7def35fScott Bardone set_bit(dev->if_port, &sge->stopped_tx_queues); 1696232a347a444e687b5f8cf0f6485704db1c6024d3Scott Bardone sge->stats.cmdQ_full[2]++; 1697c1f51212eb809849bdc68a856ae5f424dcf20d9bJoe Perches pr_err("%s: Tx ring full while queue awake!\n", 1698f1d3d38af75789f1b82969b83b69cab540609789Stephen Hemminger adapter->name); 16998199d3a79c224bbe5943fa08684e1f93a17881b0Christoph Lameter } 1700f1d3d38af75789f1b82969b83b69cab540609789Stephen Hemminger spin_unlock(&q->lock); 1701f1d3d38af75789f1b82969b83b69cab540609789Stephen Hemminger return NETDEV_TX_BUSY; 1702f1d3d38af75789f1b82969b83b69cab540609789Stephen Hemminger } 1703f1d3d38af75789f1b82969b83b69cab540609789Stephen Hemminger 1704f1d3d38af75789f1b82969b83b69cab540609789Stephen Hemminger if (unlikely(credits - count < q->stop_thres)) { 1705f1d3d38af75789f1b82969b83b69cab540609789Stephen Hemminger netif_stop_queue(dev); 1706f1d3d38af75789f1b82969b83b69cab540609789Stephen Hemminger set_bit(dev->if_port, &sge->stopped_tx_queues); 1707f1d3d38af75789f1b82969b83b69cab540609789Stephen Hemminger sge->stats.cmdQ_full[2]++; 1708f1d3d38af75789f1b82969b83b69cab540609789Stephen Hemminger } 1709f1d3d38af75789f1b82969b83b69cab540609789Stephen Hemminger 1710f1d3d38af75789f1b82969b83b69cab540609789Stephen Hemminger /* T204 cmdQ0 skbs that are destined for a certain port have to go 1711f1d3d38af75789f1b82969b83b69cab540609789Stephen Hemminger * through the scheduler. 1712f1d3d38af75789f1b82969b83b69cab540609789Stephen Hemminger */ 1713f1d3d38af75789f1b82969b83b69cab540609789Stephen Hemminger if (sge->tx_sched && !qid && skb->dev) { 1714356bd1460d1e1c4e433e4114fdac02139bddf17cFrancois Romieuuse_sched: 1715f1d3d38af75789f1b82969b83b69cab540609789Stephen Hemminger use_sched_skb = 1; 1716f1d3d38af75789f1b82969b83b69cab540609789Stephen Hemminger /* Note that the scheduler might return a different skb than 1717f1d3d38af75789f1b82969b83b69cab540609789Stephen Hemminger * the one passed in. 1718f1d3d38af75789f1b82969b83b69cab540609789Stephen Hemminger */ 1719f1d3d38af75789f1b82969b83b69cab540609789Stephen Hemminger skb = sched_skb(sge, skb, credits); 1720f1d3d38af75789f1b82969b83b69cab540609789Stephen Hemminger if (!skb) { 1721f1d3d38af75789f1b82969b83b69cab540609789Stephen Hemminger spin_unlock(&q->lock); 1722f1d3d38af75789f1b82969b83b69cab540609789Stephen Hemminger return NETDEV_TX_OK; 1723559fb51ba7e66fe298b8355fabde1275b7def35fScott Bardone } 1724f1d3d38af75789f1b82969b83b69cab540609789Stephen Hemminger pidx = q->pidx; 1725f1d3d38af75789f1b82969b83b69cab540609789Stephen Hemminger count = 1 + skb_shinfo(skb)->nr_frags; 1726f1d3d38af75789f1b82969b83b69cab540609789Stephen Hemminger count += compute_large_page_tx_descs(skb); 1727559fb51ba7e66fe298b8355fabde1275b7def35fScott Bardone } 1728f1d3d38af75789f1b82969b83b69cab540609789Stephen Hemminger 1729559fb51ba7e66fe298b8355fabde1275b7def35fScott Bardone q->in_use += count; 1730559fb51ba7e66fe298b8355fabde1275b7def35fScott Bardone genbit = q->genbit; 1731f1d3d38af75789f1b82969b83b69cab540609789Stephen Hemminger pidx = q->pidx; 1732559fb51ba7e66fe298b8355fabde1275b7def35fScott Bardone q->pidx += count; 1733559fb51ba7e66fe298b8355fabde1275b7def35fScott Bardone if (q->pidx >= q->size) { 1734559fb51ba7e66fe298b8355fabde1275b7def35fScott Bardone q->pidx -= q->size; 1735559fb51ba7e66fe298b8355fabde1275b7def35fScott Bardone q->genbit ^= 1; 17368199d3a79c224bbe5943fa08684e1f93a17881b0Christoph Lameter } 1737559fb51ba7e66fe298b8355fabde1275b7def35fScott Bardone spin_unlock(&q->lock); 17388199d3a79c224bbe5943fa08684e1f93a17881b0Christoph Lameter 1739559fb51ba7e66fe298b8355fabde1275b7def35fScott Bardone write_tx_descs(adapter, skb, pidx, genbit, q); 17408199d3a79c224bbe5943fa08684e1f93a17881b0Christoph Lameter 17418199d3a79c224bbe5943fa08684e1f93a17881b0Christoph Lameter /* 17428199d3a79c224bbe5943fa08684e1f93a17881b0Christoph Lameter * We always ring the doorbell for cmdQ1. For cmdQ0, we only ring 17438199d3a79c224bbe5943fa08684e1f93a17881b0Christoph Lameter * the doorbell if the Q is asleep. There is a natural race, where 17448199d3a79c224bbe5943fa08684e1f93a17881b0Christoph Lameter * the hardware is going to sleep just after we checked, however, 17458199d3a79c224bbe5943fa08684e1f93a17881b0Christoph Lameter * then the interrupt handler will detect the outstanding TX packet 17468199d3a79c224bbe5943fa08684e1f93a17881b0Christoph Lameter * and ring the doorbell for us. 17478199d3a79c224bbe5943fa08684e1f93a17881b0Christoph Lameter */ 1748559fb51ba7e66fe298b8355fabde1275b7def35fScott Bardone if (qid) 1749559fb51ba7e66fe298b8355fabde1275b7def35fScott Bardone doorbell_pio(adapter, F_CMDQ1_ENABLE); 1750559fb51ba7e66fe298b8355fabde1275b7def35fScott Bardone else { 1751559fb51ba7e66fe298b8355fabde1275b7def35fScott Bardone clear_bit(CMDQ_STAT_LAST_PKT_DB, &q->status); 1752559fb51ba7e66fe298b8355fabde1275b7def35fScott Bardone if (test_and_set_bit(CMDQ_STAT_RUNNING, &q->status) == 0) { 1753559fb51ba7e66fe298b8355fabde1275b7def35fScott Bardone set_bit(CMDQ_STAT_LAST_PKT_DB, &q->status); 1754559fb51ba7e66fe298b8355fabde1275b7def35fScott Bardone writel(F_CMDQ0_ENABLE, adapter->regs + A_SG_DOORBELL); 1755559fb51ba7e66fe298b8355fabde1275b7def35fScott Bardone } 17568199d3a79c224bbe5943fa08684e1f93a17881b0Christoph Lameter } 1757f1d3d38af75789f1b82969b83b69cab540609789Stephen Hemminger 1758f1d3d38af75789f1b82969b83b69cab540609789Stephen Hemminger if (use_sched_skb) { 1759f1d3d38af75789f1b82969b83b69cab540609789Stephen Hemminger if (spin_trylock(&q->lock)) { 1760f1d3d38af75789f1b82969b83b69cab540609789Stephen Hemminger credits = q->size - q->in_use; 1761f1d3d38af75789f1b82969b83b69cab540609789Stephen Hemminger skb = NULL; 1762f1d3d38af75789f1b82969b83b69cab540609789Stephen Hemminger goto use_sched; 1763f1d3d38af75789f1b82969b83b69cab540609789Stephen Hemminger } 1764f1d3d38af75789f1b82969b83b69cab540609789Stephen Hemminger } 1765aa84505fb0fb9504c61d77e8e6930a417fc404d6Stephen Hemminger return NETDEV_TX_OK; 17668199d3a79c224bbe5943fa08684e1f93a17881b0Christoph Lameter} 17678199d3a79c224bbe5943fa08684e1f93a17881b0Christoph Lameter 17688199d3a79c224bbe5943fa08684e1f93a17881b0Christoph Lameter#define MK_ETH_TYPE_MSS(type, mss) (((mss) & 0x3FFF) | ((type) << 14)) 17698199d3a79c224bbe5943fa08684e1f93a17881b0Christoph Lameter 17708199d3a79c224bbe5943fa08684e1f93a17881b0Christoph Lameter/* 1771559fb51ba7e66fe298b8355fabde1275b7def35fScott Bardone * eth_hdr_len - return the length of an Ethernet header 1772559fb51ba7e66fe298b8355fabde1275b7def35fScott Bardone * @data: pointer to the start of the Ethernet header 1773559fb51ba7e66fe298b8355fabde1275b7def35fScott Bardone * 1774559fb51ba7e66fe298b8355fabde1275b7def35fScott Bardone * Returns the length of an Ethernet header, including optional VLAN tag. 1775559fb51ba7e66fe298b8355fabde1275b7def35fScott Bardone */ 1776559fb51ba7e66fe298b8355fabde1275b7def35fScott Bardonestatic inline int eth_hdr_len(const void *data) 1777559fb51ba7e66fe298b8355fabde1275b7def35fScott Bardone{ 1778559fb51ba7e66fe298b8355fabde1275b7def35fScott Bardone const struct ethhdr *e = data; 1779559fb51ba7e66fe298b8355fabde1275b7def35fScott Bardone 1780559fb51ba7e66fe298b8355fabde1275b7def35fScott Bardone return e->h_proto == htons(ETH_P_8021Q) ? VLAN_ETH_HLEN : ETH_HLEN; 1781559fb51ba7e66fe298b8355fabde1275b7def35fScott Bardone} 1782559fb51ba7e66fe298b8355fabde1275b7def35fScott Bardone 1783559fb51ba7e66fe298b8355fabde1275b7def35fScott Bardone/* 17848199d3a79c224bbe5943fa08684e1f93a17881b0Christoph Lameter * Adds the CPL header to the sk_buff and passes it to t1_sge_tx. 17858199d3a79c224bbe5943fa08684e1f93a17881b0Christoph Lameter */ 178661357325f377889a1daffa14962d705dc814dd0eStephen Hemmingernetdev_tx_t t1_start_xmit(struct sk_buff *skb, struct net_device *dev) 17878199d3a79c224bbe5943fa08684e1f93a17881b0Christoph Lameter{ 1788c3ccc12339afa2633c72131e2aa97d52d9ca1b8aWang Chen struct adapter *adapter = dev->ml_priv; 1789559fb51ba7e66fe298b8355fabde1275b7def35fScott Bardone struct sge *sge = adapter->sge; 1790ca0c9584b1f16bd5911893647cb7f1be82e60554Christoph Lameter struct sge_port_stats *st = this_cpu_ptr(sge->port_stats[dev->if_port]); 17918199d3a79c224bbe5943fa08684e1f93a17881b0Christoph Lameter struct cpl_tx_pkt *cpl; 1792cabdfb373ae74036225826ce260c16a8e260eb0bStephen Hemminger struct sk_buff *orig_skb = skb; 1793cabdfb373ae74036225826ce260c16a8e260eb0bStephen Hemminger int ret; 17948199d3a79c224bbe5943fa08684e1f93a17881b0Christoph Lameter 1795f1d3d38af75789f1b82969b83b69cab540609789Stephen Hemminger if (skb->protocol == htons(ETH_P_CPL5)) 1796f1d3d38af75789f1b82969b83b69cab540609789Stephen Hemminger goto send; 1797f1d3d38af75789f1b82969b83b69cab540609789Stephen Hemminger 17987832ee034b6ef78aab020c9ec1348544cd65ccbdDivy Le Ray /* 17997832ee034b6ef78aab020c9ec1348544cd65ccbdDivy Le Ray * We are using a non-standard hard_header_len. 18007832ee034b6ef78aab020c9ec1348544cd65ccbdDivy Le Ray * Allocate more header room in the rare cases it is not big enough. 18017832ee034b6ef78aab020c9ec1348544cd65ccbdDivy Le Ray */ 18027832ee034b6ef78aab020c9ec1348544cd65ccbdDivy Le Ray if (unlikely(skb_headroom(skb) < dev->hard_header_len - ETH_HLEN)) { 18037832ee034b6ef78aab020c9ec1348544cd65ccbdDivy Le Ray skb = skb_realloc_headroom(skb, sizeof(struct cpl_tx_pkt_lso)); 18047832ee034b6ef78aab020c9ec1348544cd65ccbdDivy Le Ray ++st->tx_need_hdrroom; 18057832ee034b6ef78aab020c9ec1348544cd65ccbdDivy Le Ray dev_kfree_skb_any(orig_skb); 18067832ee034b6ef78aab020c9ec1348544cd65ccbdDivy Le Ray if (!skb) 18077832ee034b6ef78aab020c9ec1348544cd65ccbdDivy Le Ray return NETDEV_TX_OK; 18087832ee034b6ef78aab020c9ec1348544cd65ccbdDivy Le Ray } 18097832ee034b6ef78aab020c9ec1348544cd65ccbdDivy Le Ray 1810f1d3d38af75789f1b82969b83b69cab540609789Stephen Hemminger if (skb_shinfo(skb)->gso_size) { 18118199d3a79c224bbe5943fa08684e1f93a17881b0Christoph Lameter int eth_type; 18128199d3a79c224bbe5943fa08684e1f93a17881b0Christoph Lameter struct cpl_tx_pkt_lso *hdr; 18138199d3a79c224bbe5943fa08684e1f93a17881b0Christoph Lameter 181456f643c28c5df63693d7c66e56f8e4767cfd7a65Stephen Hemminger ++st->tx_tso; 1815559fb51ba7e66fe298b8355fabde1275b7def35fScott Bardone 1816bbe735e4247dba32568a305553b010081c8dea99Arnaldo Carvalho de Melo eth_type = skb_network_offset(skb) == ETH_HLEN ? 18178199d3a79c224bbe5943fa08684e1f93a17881b0Christoph Lameter CPL_ETH_II : CPL_ETH_II_VLAN; 18188199d3a79c224bbe5943fa08684e1f93a17881b0Christoph Lameter 18198199d3a79c224bbe5943fa08684e1f93a17881b0Christoph Lameter hdr = (struct cpl_tx_pkt_lso *)skb_push(skb, sizeof(*hdr)); 18208199d3a79c224bbe5943fa08684e1f93a17881b0Christoph Lameter hdr->opcode = CPL_TX_PKT_LSO; 18218199d3a79c224bbe5943fa08684e1f93a17881b0Christoph Lameter hdr->ip_csum_dis = hdr->l4_csum_dis = 0; 1822eddc9ec53be2ecdbf4efe0efd4a83052594f0ac0Arnaldo Carvalho de Melo hdr->ip_hdr_words = ip_hdr(skb)->ihl; 1823aa8223c7bb0b05183e1737881ed21827aa5b9e73Arnaldo Carvalho de Melo hdr->tcp_hdr_words = tcp_hdr(skb)->doff; 18248199d3a79c224bbe5943fa08684e1f93a17881b0Christoph Lameter hdr->eth_type_mss = htons(MK_ETH_TYPE_MSS(eth_type, 1825f1d3d38af75789f1b82969b83b69cab540609789Stephen Hemminger skb_shinfo(skb)->gso_size)); 18268199d3a79c224bbe5943fa08684e1f93a17881b0Christoph Lameter hdr->len = htonl(skb->len - sizeof(*hdr)); 18278199d3a79c224bbe5943fa08684e1f93a17881b0Christoph Lameter cpl = (struct cpl_tx_pkt *)hdr; 1828f1d3d38af75789f1b82969b83b69cab540609789Stephen Hemminger } else { 18298199d3a79c224bbe5943fa08684e1f93a17881b0Christoph Lameter /* 1830356bd1460d1e1c4e433e4114fdac02139bddf17cFrancois Romieu * Packets shorter than ETH_HLEN can break the MAC, drop them 1831559fb51ba7e66fe298b8355fabde1275b7def35fScott Bardone * early. Also, we may get oversized packets because some 1832559fb51ba7e66fe298b8355fabde1275b7def35fScott Bardone * parts of the kernel don't handle our unusual hard_header_len 1833559fb51ba7e66fe298b8355fabde1275b7def35fScott Bardone * right, drop those too. 18348199d3a79c224bbe5943fa08684e1f93a17881b0Christoph Lameter */ 1835559fb51ba7e66fe298b8355fabde1275b7def35fScott Bardone if (unlikely(skb->len < ETH_HLEN || 1836559fb51ba7e66fe298b8355fabde1275b7def35fScott Bardone skb->len > dev->mtu + eth_hdr_len(skb->data))) { 1837f1d3d38af75789f1b82969b83b69cab540609789Stephen Hemminger pr_debug("%s: packet size %d hdr %d mtu%d\n", dev->name, 1838f1d3d38af75789f1b82969b83b69cab540609789Stephen Hemminger skb->len, eth_hdr_len(skb->data), dev->mtu); 1839559fb51ba7e66fe298b8355fabde1275b7def35fScott Bardone dev_kfree_skb_any(skb); 1840aa84505fb0fb9504c61d77e8e6930a417fc404d6Stephen Hemminger return NETDEV_TX_OK; 1841559fb51ba7e66fe298b8355fabde1275b7def35fScott Bardone } 1842559fb51ba7e66fe298b8355fabde1275b7def35fScott Bardone 184330f554f925335abad89aaa38eec6828242b27527Michał Mirosław if (skb->ip_summed == CHECKSUM_PARTIAL && 1844eddc9ec53be2ecdbf4efe0efd4a83052594f0ac0Arnaldo Carvalho de Melo ip_hdr(skb)->protocol == IPPROTO_UDP) { 184584fa7933a33f806bbbaae6775e87459b1ec584c0Patrick McHardy if (unlikely(skb_checksum_help(skb))) { 1846f1d3d38af75789f1b82969b83b69cab540609789Stephen Hemminger pr_debug("%s: unable to do udp checksum\n", dev->name); 1847559fb51ba7e66fe298b8355fabde1275b7def35fScott Bardone dev_kfree_skb_any(skb); 1848aa84505fb0fb9504c61d77e8e6930a417fc404d6Stephen Hemminger return NETDEV_TX_OK; 1849559fb51ba7e66fe298b8355fabde1275b7def35fScott Bardone } 1850f1d3d38af75789f1b82969b83b69cab540609789Stephen Hemminger } 18518199d3a79c224bbe5943fa08684e1f93a17881b0Christoph Lameter 1852559fb51ba7e66fe298b8355fabde1275b7def35fScott Bardone /* Hmmm, assuming to catch the gratious arp... and we'll use 1853559fb51ba7e66fe298b8355fabde1275b7def35fScott Bardone * it to flush out stuck espi packets... 1854f1d3d38af75789f1b82969b83b69cab540609789Stephen Hemminger */ 1855f1d3d38af75789f1b82969b83b69cab540609789Stephen Hemminger if ((unlikely(!adapter->sge->espibug_skb[dev->if_port]))) { 18568199d3a79c224bbe5943fa08684e1f93a17881b0Christoph Lameter if (skb->protocol == htons(ETH_P_ARP) && 1857d0a92be05ed4aea7d35c2b257e3f9173565fe4ebArnaldo Carvalho de Melo arp_hdr(skb)->ar_op == htons(ARPOP_REQUEST)) { 1858f1d3d38af75789f1b82969b83b69cab540609789Stephen Hemminger adapter->sge->espibug_skb[dev->if_port] = skb; 1859559fb51ba7e66fe298b8355fabde1275b7def35fScott Bardone /* We want to re-use this skb later. We 1860559fb51ba7e66fe298b8355fabde1275b7def35fScott Bardone * simply bump the reference count and it 1861559fb51ba7e66fe298b8355fabde1275b7def35fScott Bardone * will not be freed... 1862559fb51ba7e66fe298b8355fabde1275b7def35fScott Bardone */ 1863559fb51ba7e66fe298b8355fabde1275b7def35fScott Bardone skb = skb_get(skb); 1864559fb51ba7e66fe298b8355fabde1275b7def35fScott Bardone } 18658199d3a79c224bbe5943fa08684e1f93a17881b0Christoph Lameter } 1866559fb51ba7e66fe298b8355fabde1275b7def35fScott Bardone 1867559fb51ba7e66fe298b8355fabde1275b7def35fScott Bardone cpl = (struct cpl_tx_pkt *)__skb_push(skb, sizeof(*cpl)); 18688199d3a79c224bbe5943fa08684e1f93a17881b0Christoph Lameter cpl->opcode = CPL_TX_PKT; 18698199d3a79c224bbe5943fa08684e1f93a17881b0Christoph Lameter cpl->ip_csum_dis = 1; /* SW calculates IP csum */ 187084fa7933a33f806bbbaae6775e87459b1ec584c0Patrick McHardy cpl->l4_csum_dis = skb->ip_summed == CHECKSUM_PARTIAL ? 0 : 1; 18718199d3a79c224bbe5943fa08684e1f93a17881b0Christoph Lameter /* the length field isn't used so don't bother setting it */ 1872559fb51ba7e66fe298b8355fabde1275b7def35fScott Bardone 187384fa7933a33f806bbbaae6775e87459b1ec584c0Patrick McHardy st->tx_cso += (skb->ip_summed == CHECKSUM_PARTIAL); 18748199d3a79c224bbe5943fa08684e1f93a17881b0Christoph Lameter } 18758199d3a79c224bbe5943fa08684e1f93a17881b0Christoph Lameter cpl->iff = dev->if_port; 18768199d3a79c224bbe5943fa08684e1f93a17881b0Christoph Lameter 1877eab6d18d20fc5b5ba04a7e7fcd6f357197870e51Jesse Gross if (vlan_tx_tag_present(skb)) { 18788199d3a79c224bbe5943fa08684e1f93a17881b0Christoph Lameter cpl->vlan_valid = 1; 18798199d3a79c224bbe5943fa08684e1f93a17881b0Christoph Lameter cpl->vlan = htons(vlan_tx_tag_get(skb)); 1880559fb51ba7e66fe298b8355fabde1275b7def35fScott Bardone st->vlan_insert++; 18818199d3a79c224bbe5943fa08684e1f93a17881b0Christoph Lameter } else 18828199d3a79c224bbe5943fa08684e1f93a17881b0Christoph Lameter cpl->vlan_valid = 0; 18838199d3a79c224bbe5943fa08684e1f93a17881b0Christoph Lameter 1884f1d3d38af75789f1b82969b83b69cab540609789Stephen Hemmingersend: 1885cabdfb373ae74036225826ce260c16a8e260eb0bStephen Hemminger ret = t1_sge_tx(skb, adapter, 0, dev); 1886cabdfb373ae74036225826ce260c16a8e260eb0bStephen Hemminger 1887cabdfb373ae74036225826ce260c16a8e260eb0bStephen Hemminger /* If transmit busy, and we reallocated skb's due to headroom limit, 1888cabdfb373ae74036225826ce260c16a8e260eb0bStephen Hemminger * then silently discard to avoid leak. 1889cabdfb373ae74036225826ce260c16a8e260eb0bStephen Hemminger */ 1890cabdfb373ae74036225826ce260c16a8e260eb0bStephen Hemminger if (unlikely(ret != NETDEV_TX_OK && skb != orig_skb)) { 1891356bd1460d1e1c4e433e4114fdac02139bddf17cFrancois Romieu dev_kfree_skb_any(skb); 1892cabdfb373ae74036225826ce260c16a8e260eb0bStephen Hemminger ret = NETDEV_TX_OK; 1893356bd1460d1e1c4e433e4114fdac02139bddf17cFrancois Romieu } 1894cabdfb373ae74036225826ce260c16a8e260eb0bStephen Hemminger return ret; 1895559fb51ba7e66fe298b8355fabde1275b7def35fScott Bardone} 18968199d3a79c224bbe5943fa08684e1f93a17881b0Christoph Lameter 1897559fb51ba7e66fe298b8355fabde1275b7def35fScott Bardone/* 1898559fb51ba7e66fe298b8355fabde1275b7def35fScott Bardone * Callback for the Tx buffer reclaim timer. Runs with softirqs disabled. 1899559fb51ba7e66fe298b8355fabde1275b7def35fScott Bardone */ 1900559fb51ba7e66fe298b8355fabde1275b7def35fScott Bardonestatic void sge_tx_reclaim_cb(unsigned long data) 1901559fb51ba7e66fe298b8355fabde1275b7def35fScott Bardone{ 1902559fb51ba7e66fe298b8355fabde1275b7def35fScott Bardone int i; 1903559fb51ba7e66fe298b8355fabde1275b7def35fScott Bardone struct sge *sge = (struct sge *)data; 1904559fb51ba7e66fe298b8355fabde1275b7def35fScott Bardone 1905559fb51ba7e66fe298b8355fabde1275b7def35fScott Bardone for (i = 0; i < SGE_CMDQ_N; ++i) { 1906559fb51ba7e66fe298b8355fabde1275b7def35fScott Bardone struct cmdQ *q = &sge->cmdQ[i]; 1907559fb51ba7e66fe298b8355fabde1275b7def35fScott Bardone 1908559fb51ba7e66fe298b8355fabde1275b7def35fScott Bardone if (!spin_trylock(&q->lock)) 1909559fb51ba7e66fe298b8355fabde1275b7def35fScott Bardone continue; 19108199d3a79c224bbe5943fa08684e1f93a17881b0Christoph Lameter 1911559fb51ba7e66fe298b8355fabde1275b7def35fScott Bardone reclaim_completed_tx(sge, q); 1912f1d3d38af75789f1b82969b83b69cab540609789Stephen Hemminger if (i == 0 && q->in_use) { /* flush pending credits */ 1913f1d3d38af75789f1b82969b83b69cab540609789Stephen Hemminger writel(F_CMDQ0_ENABLE, sge->adapter->regs + A_SG_DOORBELL); 1914f1d3d38af75789f1b82969b83b69cab540609789Stephen Hemminger } 1915559fb51ba7e66fe298b8355fabde1275b7def35fScott Bardone spin_unlock(&q->lock); 1916559fb51ba7e66fe298b8355fabde1275b7def35fScott Bardone } 1917559fb51ba7e66fe298b8355fabde1275b7def35fScott Bardone mod_timer(&sge->tx_reclaim_timer, jiffies + TX_RECLAIM_PERIOD); 1918559fb51ba7e66fe298b8355fabde1275b7def35fScott Bardone} 1919559fb51ba7e66fe298b8355fabde1275b7def35fScott Bardone 1920559fb51ba7e66fe298b8355fabde1275b7def35fScott Bardone/* 1921559fb51ba7e66fe298b8355fabde1275b7def35fScott Bardone * Propagate changes of the SGE coalescing parameters to the HW. 1922559fb51ba7e66fe298b8355fabde1275b7def35fScott Bardone */ 1923559fb51ba7e66fe298b8355fabde1275b7def35fScott Bardoneint t1_sge_set_coalesce_params(struct sge *sge, struct sge_params *p) 1924559fb51ba7e66fe298b8355fabde1275b7def35fScott Bardone{ 1925559fb51ba7e66fe298b8355fabde1275b7def35fScott Bardone sge->fixed_intrtimer = p->rx_coalesce_usecs * 1926559fb51ba7e66fe298b8355fabde1275b7def35fScott Bardone core_ticks_per_usec(sge->adapter); 1927559fb51ba7e66fe298b8355fabde1275b7def35fScott Bardone writel(sge->fixed_intrtimer, sge->adapter->regs + A_SG_INTRTIMER); 19288199d3a79c224bbe5943fa08684e1f93a17881b0Christoph Lameter return 0; 19298199d3a79c224bbe5943fa08684e1f93a17881b0Christoph Lameter} 19308199d3a79c224bbe5943fa08684e1f93a17881b0Christoph Lameter 1931559fb51ba7e66fe298b8355fabde1275b7def35fScott Bardone/* 1932559fb51ba7e66fe298b8355fabde1275b7def35fScott Bardone * Allocates both RX and TX resources and configures the SGE. However, 1933559fb51ba7e66fe298b8355fabde1275b7def35fScott Bardone * the hardware is not enabled yet. 1934559fb51ba7e66fe298b8355fabde1275b7def35fScott Bardone */ 1935559fb51ba7e66fe298b8355fabde1275b7def35fScott Bardoneint t1_sge_configure(struct sge *sge, struct sge_params *p) 19368199d3a79c224bbe5943fa08684e1f93a17881b0Christoph Lameter{ 1937559fb51ba7e66fe298b8355fabde1275b7def35fScott Bardone if (alloc_rx_resources(sge, p)) 1938559fb51ba7e66fe298b8355fabde1275b7def35fScott Bardone return -ENOMEM; 1939559fb51ba7e66fe298b8355fabde1275b7def35fScott Bardone if (alloc_tx_resources(sge, p)) { 1940559fb51ba7e66fe298b8355fabde1275b7def35fScott Bardone free_rx_resources(sge); 1941559fb51ba7e66fe298b8355fabde1275b7def35fScott Bardone return -ENOMEM; 1942559fb51ba7e66fe298b8355fabde1275b7def35fScott Bardone } 1943559fb51ba7e66fe298b8355fabde1275b7def35fScott Bardone configure_sge(sge, p); 1944559fb51ba7e66fe298b8355fabde1275b7def35fScott Bardone 1945559fb51ba7e66fe298b8355fabde1275b7def35fScott Bardone /* 1946559fb51ba7e66fe298b8355fabde1275b7def35fScott Bardone * Now that we have sized the free lists calculate the payload 1947559fb51ba7e66fe298b8355fabde1275b7def35fScott Bardone * capacity of the large buffers. Other parts of the driver use 1948559fb51ba7e66fe298b8355fabde1275b7def35fScott Bardone * this to set the max offload coalescing size so that RX packets 1949559fb51ba7e66fe298b8355fabde1275b7def35fScott Bardone * do not overflow our large buffers. 1950559fb51ba7e66fe298b8355fabde1275b7def35fScott Bardone */ 1951559fb51ba7e66fe298b8355fabde1275b7def35fScott Bardone p->large_buf_capacity = jumbo_payload_capacity(sge); 1952559fb51ba7e66fe298b8355fabde1275b7def35fScott Bardone return 0; 1953559fb51ba7e66fe298b8355fabde1275b7def35fScott Bardone} 19548199d3a79c224bbe5943fa08684e1f93a17881b0Christoph Lameter 1955559fb51ba7e66fe298b8355fabde1275b7def35fScott Bardone/* 1956559fb51ba7e66fe298b8355fabde1275b7def35fScott Bardone * Disables the DMA engine. 1957559fb51ba7e66fe298b8355fabde1275b7def35fScott Bardone */ 1958559fb51ba7e66fe298b8355fabde1275b7def35fScott Bardonevoid t1_sge_stop(struct sge *sge) 1959559fb51ba7e66fe298b8355fabde1275b7def35fScott Bardone{ 1960f1d3d38af75789f1b82969b83b69cab540609789Stephen Hemminger int i; 1961559fb51ba7e66fe298b8355fabde1275b7def35fScott Bardone writel(0, sge->adapter->regs + A_SG_CONTROL); 1962f1d3d38af75789f1b82969b83b69cab540609789Stephen Hemminger readl(sge->adapter->regs + A_SG_CONTROL); /* flush */ 1963f1d3d38af75789f1b82969b83b69cab540609789Stephen Hemminger 1964559fb51ba7e66fe298b8355fabde1275b7def35fScott Bardone if (is_T2(sge->adapter)) 1965559fb51ba7e66fe298b8355fabde1275b7def35fScott Bardone del_timer_sync(&sge->espibug_timer); 1966f1d3d38af75789f1b82969b83b69cab540609789Stephen Hemminger 1967559fb51ba7e66fe298b8355fabde1275b7def35fScott Bardone del_timer_sync(&sge->tx_reclaim_timer); 1968f1d3d38af75789f1b82969b83b69cab540609789Stephen Hemminger if (sge->tx_sched) 1969f1d3d38af75789f1b82969b83b69cab540609789Stephen Hemminger tx_sched_stop(sge); 1970f1d3d38af75789f1b82969b83b69cab540609789Stephen Hemminger 1971f1d3d38af75789f1b82969b83b69cab540609789Stephen Hemminger for (i = 0; i < MAX_NPORTS; i++) 1972f4fe5a9c6d49d760569feed12d54743e0c4fe6e8Wei Yongjun kfree_skb(sge->espibug_skb[i]); 19738199d3a79c224bbe5943fa08684e1f93a17881b0Christoph Lameter} 19748199d3a79c224bbe5943fa08684e1f93a17881b0Christoph Lameter 1975559fb51ba7e66fe298b8355fabde1275b7def35fScott Bardone/* 1976559fb51ba7e66fe298b8355fabde1275b7def35fScott Bardone * Enables the DMA engine. 1977559fb51ba7e66fe298b8355fabde1275b7def35fScott Bardone */ 1978559fb51ba7e66fe298b8355fabde1275b7def35fScott Bardonevoid t1_sge_start(struct sge *sge) 19798199d3a79c224bbe5943fa08684e1f93a17881b0Christoph Lameter{ 1980559fb51ba7e66fe298b8355fabde1275b7def35fScott Bardone refill_free_list(sge, &sge->freelQ[0]); 1981559fb51ba7e66fe298b8355fabde1275b7def35fScott Bardone refill_free_list(sge, &sge->freelQ[1]); 1982559fb51ba7e66fe298b8355fabde1275b7def35fScott Bardone 1983559fb51ba7e66fe298b8355fabde1275b7def35fScott Bardone writel(sge->sge_control, sge->adapter->regs + A_SG_CONTROL); 1984559fb51ba7e66fe298b8355fabde1275b7def35fScott Bardone doorbell_pio(sge->adapter, F_FL0_ENABLE | F_FL1_ENABLE); 1985f1d3d38af75789f1b82969b83b69cab540609789Stephen Hemminger readl(sge->adapter->regs + A_SG_CONTROL); /* flush */ 1986559fb51ba7e66fe298b8355fabde1275b7def35fScott Bardone 1987559fb51ba7e66fe298b8355fabde1275b7def35fScott Bardone mod_timer(&sge->tx_reclaim_timer, jiffies + TX_RECLAIM_PERIOD); 1988559fb51ba7e66fe298b8355fabde1275b7def35fScott Bardone 1989f1d3d38af75789f1b82969b83b69cab540609789Stephen Hemminger if (is_T2(sge->adapter)) 1990559fb51ba7e66fe298b8355fabde1275b7def35fScott Bardone mod_timer(&sge->espibug_timer, jiffies + sge->espibug_timeout); 1991559fb51ba7e66fe298b8355fabde1275b7def35fScott Bardone} 1992559fb51ba7e66fe298b8355fabde1275b7def35fScott Bardone 1993559fb51ba7e66fe298b8355fabde1275b7def35fScott Bardone/* 1994559fb51ba7e66fe298b8355fabde1275b7def35fScott Bardone * Callback for the T2 ESPI 'stuck packet feature' workaorund 1995559fb51ba7e66fe298b8355fabde1275b7def35fScott Bardone */ 1996f1d3d38af75789f1b82969b83b69cab540609789Stephen Hemmingerstatic void espibug_workaround_t204(unsigned long data) 1997559fb51ba7e66fe298b8355fabde1275b7def35fScott Bardone{ 1998559fb51ba7e66fe298b8355fabde1275b7def35fScott Bardone struct adapter *adapter = (struct adapter *)data; 19998199d3a79c224bbe5943fa08684e1f93a17881b0Christoph Lameter struct sge *sge = adapter->sge; 2000f1d3d38af75789f1b82969b83b69cab540609789Stephen Hemminger unsigned int nports = adapter->params.nports; 2001f1d3d38af75789f1b82969b83b69cab540609789Stephen Hemminger u32 seop[MAX_NPORTS]; 20028199d3a79c224bbe5943fa08684e1f93a17881b0Christoph Lameter 2003f1d3d38af75789f1b82969b83b69cab540609789Stephen Hemminger if (adapter->open_device_map & PORT_MASK) { 2004f1d3d38af75789f1b82969b83b69cab540609789Stephen Hemminger int i; 2005356bd1460d1e1c4e433e4114fdac02139bddf17cFrancois Romieu 2006356bd1460d1e1c4e433e4114fdac02139bddf17cFrancois Romieu if (t1_espi_get_mon_t204(adapter, &(seop[0]), 0) < 0) 2007f1d3d38af75789f1b82969b83b69cab540609789Stephen Hemminger return; 2008356bd1460d1e1c4e433e4114fdac02139bddf17cFrancois Romieu 2009f1d3d38af75789f1b82969b83b69cab540609789Stephen Hemminger for (i = 0; i < nports; i++) { 2010356bd1460d1e1c4e433e4114fdac02139bddf17cFrancois Romieu struct sk_buff *skb = sge->espibug_skb[i]; 2011356bd1460d1e1c4e433e4114fdac02139bddf17cFrancois Romieu 2012356bd1460d1e1c4e433e4114fdac02139bddf17cFrancois Romieu if (!netif_running(adapter->port[i].dev) || 2013356bd1460d1e1c4e433e4114fdac02139bddf17cFrancois Romieu netif_queue_stopped(adapter->port[i].dev) || 2014356bd1460d1e1c4e433e4114fdac02139bddf17cFrancois Romieu !seop[i] || ((seop[i] & 0xfff) != 0) || !skb) 2015356bd1460d1e1c4e433e4114fdac02139bddf17cFrancois Romieu continue; 2016356bd1460d1e1c4e433e4114fdac02139bddf17cFrancois Romieu 2017356bd1460d1e1c4e433e4114fdac02139bddf17cFrancois Romieu if (!skb->cb[0]) { 201827d7ff46a3498d3debc6ba68fb8014c702b81170Arnaldo Carvalho de Melo skb_copy_to_linear_data_offset(skb, 201927d7ff46a3498d3debc6ba68fb8014c702b81170Arnaldo Carvalho de Melo sizeof(struct cpl_tx_pkt), 202027d7ff46a3498d3debc6ba68fb8014c702b81170Arnaldo Carvalho de Melo ch_mac_addr, 202127d7ff46a3498d3debc6ba68fb8014c702b81170Arnaldo Carvalho de Melo ETH_ALEN); 202227d7ff46a3498d3debc6ba68fb8014c702b81170Arnaldo Carvalho de Melo skb_copy_to_linear_data_offset(skb, 202327d7ff46a3498d3debc6ba68fb8014c702b81170Arnaldo Carvalho de Melo skb->len - 10, 202427d7ff46a3498d3debc6ba68fb8014c702b81170Arnaldo Carvalho de Melo ch_mac_addr, 202527d7ff46a3498d3debc6ba68fb8014c702b81170Arnaldo Carvalho de Melo ETH_ALEN); 2026356bd1460d1e1c4e433e4114fdac02139bddf17cFrancois Romieu skb->cb[0] = 0xff; 2027559fb51ba7e66fe298b8355fabde1275b7def35fScott Bardone } 2028356bd1460d1e1c4e433e4114fdac02139bddf17cFrancois Romieu 2029356bd1460d1e1c4e433e4114fdac02139bddf17cFrancois Romieu /* bump the reference count to avoid freeing of 2030356bd1460d1e1c4e433e4114fdac02139bddf17cFrancois Romieu * the skb once the DMA has completed. 2031356bd1460d1e1c4e433e4114fdac02139bddf17cFrancois Romieu */ 2032356bd1460d1e1c4e433e4114fdac02139bddf17cFrancois Romieu skb = skb_get(skb); 2033356bd1460d1e1c4e433e4114fdac02139bddf17cFrancois Romieu t1_sge_tx(skb, adapter, 0, adapter->port[i].dev); 2034559fb51ba7e66fe298b8355fabde1275b7def35fScott Bardone } 2035559fb51ba7e66fe298b8355fabde1275b7def35fScott Bardone } 2036559fb51ba7e66fe298b8355fabde1275b7def35fScott Bardone mod_timer(&sge->espibug_timer, jiffies + sge->espibug_timeout); 20378199d3a79c224bbe5943fa08684e1f93a17881b0Christoph Lameter} 20388199d3a79c224bbe5943fa08684e1f93a17881b0Christoph Lameter 2039f1d3d38af75789f1b82969b83b69cab540609789Stephen Hemmingerstatic void espibug_workaround(unsigned long data) 2040f1d3d38af75789f1b82969b83b69cab540609789Stephen Hemminger{ 2041f1d3d38af75789f1b82969b83b69cab540609789Stephen Hemminger struct adapter *adapter = (struct adapter *)data; 2042f1d3d38af75789f1b82969b83b69cab540609789Stephen Hemminger struct sge *sge = adapter->sge; 2043f1d3d38af75789f1b82969b83b69cab540609789Stephen Hemminger 2044f1d3d38af75789f1b82969b83b69cab540609789Stephen Hemminger if (netif_running(adapter->port[0].dev)) { 2045f1d3d38af75789f1b82969b83b69cab540609789Stephen Hemminger struct sk_buff *skb = sge->espibug_skb[0]; 2046f1d3d38af75789f1b82969b83b69cab540609789Stephen Hemminger u32 seop = t1_espi_get_mon(adapter, 0x930, 0); 2047f1d3d38af75789f1b82969b83b69cab540609789Stephen Hemminger 2048f1d3d38af75789f1b82969b83b69cab540609789Stephen Hemminger if ((seop & 0xfff0fff) == 0xfff && skb) { 2049f1d3d38af75789f1b82969b83b69cab540609789Stephen Hemminger if (!skb->cb[0]) { 205027d7ff46a3498d3debc6ba68fb8014c702b81170Arnaldo Carvalho de Melo skb_copy_to_linear_data_offset(skb, 205127d7ff46a3498d3debc6ba68fb8014c702b81170Arnaldo Carvalho de Melo sizeof(struct cpl_tx_pkt), 205227d7ff46a3498d3debc6ba68fb8014c702b81170Arnaldo Carvalho de Melo ch_mac_addr, 205327d7ff46a3498d3debc6ba68fb8014c702b81170Arnaldo Carvalho de Melo ETH_ALEN); 205427d7ff46a3498d3debc6ba68fb8014c702b81170Arnaldo Carvalho de Melo skb_copy_to_linear_data_offset(skb, 205527d7ff46a3498d3debc6ba68fb8014c702b81170Arnaldo Carvalho de Melo skb->len - 10, 205627d7ff46a3498d3debc6ba68fb8014c702b81170Arnaldo Carvalho de Melo ch_mac_addr, 205727d7ff46a3498d3debc6ba68fb8014c702b81170Arnaldo Carvalho de Melo ETH_ALEN); 2058f1d3d38af75789f1b82969b83b69cab540609789Stephen Hemminger skb->cb[0] = 0xff; 2059f1d3d38af75789f1b82969b83b69cab540609789Stephen Hemminger } 2060f1d3d38af75789f1b82969b83b69cab540609789Stephen Hemminger 2061f1d3d38af75789f1b82969b83b69cab540609789Stephen Hemminger /* bump the reference count to avoid freeing of the 2062f1d3d38af75789f1b82969b83b69cab540609789Stephen Hemminger * skb once the DMA has completed. 2063f1d3d38af75789f1b82969b83b69cab540609789Stephen Hemminger */ 2064f1d3d38af75789f1b82969b83b69cab540609789Stephen Hemminger skb = skb_get(skb); 2065f1d3d38af75789f1b82969b83b69cab540609789Stephen Hemminger t1_sge_tx(skb, adapter, 0, adapter->port[0].dev); 2066f1d3d38af75789f1b82969b83b69cab540609789Stephen Hemminger } 2067f1d3d38af75789f1b82969b83b69cab540609789Stephen Hemminger } 2068f1d3d38af75789f1b82969b83b69cab540609789Stephen Hemminger mod_timer(&sge->espibug_timer, jiffies + sge->espibug_timeout); 2069f1d3d38af75789f1b82969b83b69cab540609789Stephen Hemminger} 2070f1d3d38af75789f1b82969b83b69cab540609789Stephen Hemminger 2071559fb51ba7e66fe298b8355fabde1275b7def35fScott Bardone/* 2072559fb51ba7e66fe298b8355fabde1275b7def35fScott Bardone * Creates a t1_sge structure and returns suggested resource parameters. 2073559fb51ba7e66fe298b8355fabde1275b7def35fScott Bardone */ 2074559fb51ba7e66fe298b8355fabde1275b7def35fScott Bardonestruct sge * __devinit t1_sge_create(struct adapter *adapter, 2075559fb51ba7e66fe298b8355fabde1275b7def35fScott Bardone struct sge_params *p) 2076559fb51ba7e66fe298b8355fabde1275b7def35fScott Bardone{ 2077cbee9f9115da7eac5dc44a450f2dab343236cb2cStephen Hemminger struct sge *sge = kzalloc(sizeof(*sge), GFP_KERNEL); 207856f643c28c5df63693d7c66e56f8e4767cfd7a65Stephen Hemminger int i; 2079559fb51ba7e66fe298b8355fabde1275b7def35fScott Bardone 2080559fb51ba7e66fe298b8355fabde1275b7def35fScott Bardone if (!sge) 2081559fb51ba7e66fe298b8355fabde1275b7def35fScott Bardone return NULL; 2082559fb51ba7e66fe298b8355fabde1275b7def35fScott Bardone 2083559fb51ba7e66fe298b8355fabde1275b7def35fScott Bardone sge->adapter = adapter; 2084559fb51ba7e66fe298b8355fabde1275b7def35fScott Bardone sge->netdev = adapter->port[0].dev; 2085559fb51ba7e66fe298b8355fabde1275b7def35fScott Bardone sge->rx_pkt_pad = t1_is_T1B(adapter) ? 0 : 2; 2086559fb51ba7e66fe298b8355fabde1275b7def35fScott Bardone sge->jumbo_fl = t1_is_T1B(adapter) ? 1 : 0; 2087559fb51ba7e66fe298b8355fabde1275b7def35fScott Bardone 208856f643c28c5df63693d7c66e56f8e4767cfd7a65Stephen Hemminger for_each_port(adapter, i) { 208956f643c28c5df63693d7c66e56f8e4767cfd7a65Stephen Hemminger sge->port_stats[i] = alloc_percpu(struct sge_port_stats); 209056f643c28c5df63693d7c66e56f8e4767cfd7a65Stephen Hemminger if (!sge->port_stats[i]) 209156f643c28c5df63693d7c66e56f8e4767cfd7a65Stephen Hemminger goto nomem_port; 209256f643c28c5df63693d7c66e56f8e4767cfd7a65Stephen Hemminger } 209356f643c28c5df63693d7c66e56f8e4767cfd7a65Stephen Hemminger 2094559fb51ba7e66fe298b8355fabde1275b7def35fScott Bardone init_timer(&sge->tx_reclaim_timer); 2095559fb51ba7e66fe298b8355fabde1275b7def35fScott Bardone sge->tx_reclaim_timer.data = (unsigned long)sge; 2096559fb51ba7e66fe298b8355fabde1275b7def35fScott Bardone sge->tx_reclaim_timer.function = sge_tx_reclaim_cb; 2097559fb51ba7e66fe298b8355fabde1275b7def35fScott Bardone 2098559fb51ba7e66fe298b8355fabde1275b7def35fScott Bardone if (is_T2(sge->adapter)) { 2099559fb51ba7e66fe298b8355fabde1275b7def35fScott Bardone init_timer(&sge->espibug_timer); 2100f1d3d38af75789f1b82969b83b69cab540609789Stephen Hemminger 2101f1d3d38af75789f1b82969b83b69cab540609789Stephen Hemminger if (adapter->params.nports > 1) { 2102f1d3d38af75789f1b82969b83b69cab540609789Stephen Hemminger tx_sched_init(sge); 2103f1d3d38af75789f1b82969b83b69cab540609789Stephen Hemminger sge->espibug_timer.function = espibug_workaround_t204; 2104d7487421b629c5ca71ce23b10461ef0c3ad2c741Francois Romieu } else 2105f1d3d38af75789f1b82969b83b69cab540609789Stephen Hemminger sge->espibug_timer.function = espibug_workaround; 2106559fb51ba7e66fe298b8355fabde1275b7def35fScott Bardone sge->espibug_timer.data = (unsigned long)sge->adapter; 2107f1d3d38af75789f1b82969b83b69cab540609789Stephen Hemminger 2108559fb51ba7e66fe298b8355fabde1275b7def35fScott Bardone sge->espibug_timeout = 1; 2109f1d3d38af75789f1b82969b83b69cab540609789Stephen Hemminger /* for T204, every 10ms */ 2110f1d3d38af75789f1b82969b83b69cab540609789Stephen Hemminger if (adapter->params.nports > 1) 2111f1d3d38af75789f1b82969b83b69cab540609789Stephen Hemminger sge->espibug_timeout = HZ/100; 2112559fb51ba7e66fe298b8355fabde1275b7def35fScott Bardone } 2113356bd1460d1e1c4e433e4114fdac02139bddf17cFrancois Romieu 2114559fb51ba7e66fe298b8355fabde1275b7def35fScott Bardone 2115559fb51ba7e66fe298b8355fabde1275b7def35fScott Bardone p->cmdQ_size[0] = SGE_CMDQ0_E_N; 2116559fb51ba7e66fe298b8355fabde1275b7def35fScott Bardone p->cmdQ_size[1] = SGE_CMDQ1_E_N; 2117559fb51ba7e66fe298b8355fabde1275b7def35fScott Bardone p->freelQ_size[!sge->jumbo_fl] = SGE_FREEL_SIZE; 2118559fb51ba7e66fe298b8355fabde1275b7def35fScott Bardone p->freelQ_size[sge->jumbo_fl] = SGE_JUMBO_FREEL_SIZE; 2119f1d3d38af75789f1b82969b83b69cab540609789Stephen Hemminger if (sge->tx_sched) { 2120f1d3d38af75789f1b82969b83b69cab540609789Stephen Hemminger if (board_info(sge->adapter)->board == CHBT_BOARD_CHT204) 2121f1d3d38af75789f1b82969b83b69cab540609789Stephen Hemminger p->rx_coalesce_usecs = 15; 2122f1d3d38af75789f1b82969b83b69cab540609789Stephen Hemminger else 2123f1d3d38af75789f1b82969b83b69cab540609789Stephen Hemminger p->rx_coalesce_usecs = 50; 2124f1d3d38af75789f1b82969b83b69cab540609789Stephen Hemminger } else 2125f1d3d38af75789f1b82969b83b69cab540609789Stephen Hemminger p->rx_coalesce_usecs = 50; 2126f1d3d38af75789f1b82969b83b69cab540609789Stephen Hemminger 2127559fb51ba7e66fe298b8355fabde1275b7def35fScott Bardone p->coalesce_enable = 0; 2128559fb51ba7e66fe298b8355fabde1275b7def35fScott Bardone p->sample_interval_usecs = 0; 2129559fb51ba7e66fe298b8355fabde1275b7def35fScott Bardone 2130559fb51ba7e66fe298b8355fabde1275b7def35fScott Bardone return sge; 213156f643c28c5df63693d7c66e56f8e4767cfd7a65Stephen Hemmingernomem_port: 213256f643c28c5df63693d7c66e56f8e4767cfd7a65Stephen Hemminger while (i >= 0) { 213356f643c28c5df63693d7c66e56f8e4767cfd7a65Stephen Hemminger free_percpu(sge->port_stats[i]); 213456f643c28c5df63693d7c66e56f8e4767cfd7a65Stephen Hemminger --i; 213556f643c28c5df63693d7c66e56f8e4767cfd7a65Stephen Hemminger } 213656f643c28c5df63693d7c66e56f8e4767cfd7a65Stephen Hemminger kfree(sge); 213756f643c28c5df63693d7c66e56f8e4767cfd7a65Stephen Hemminger return NULL; 213856f643c28c5df63693d7c66e56f8e4767cfd7a65Stephen Hemminger 2139559fb51ba7e66fe298b8355fabde1275b7def35fScott Bardone} 2140