1/******************************************************************************* 2 3 Intel 82599 Virtual Function driver 4 Copyright(c) 1999 - 2012 Intel Corporation. 5 6 This program is free software; you can redistribute it and/or modify it 7 under the terms and conditions of the GNU General Public License, 8 version 2, as published by the Free Software Foundation. 9 10 This program is distributed in the hope it will be useful, but WITHOUT 11 ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or 12 FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for 13 more details. 14 15 You should have received a copy of the GNU General Public License along with 16 this program; if not, write to the Free Software Foundation, Inc., 17 51 Franklin St - Fifth Floor, Boston, MA 02110-1301 USA. 18 19 The full GNU General Public License is included in this distribution in 20 the file called "COPYING". 21 22 Contact Information: 23 e1000-devel Mailing List <e1000-devel@lists.sourceforge.net> 24 Intel Corporation, 5200 N.E. Elam Young Parkway, Hillsboro, OR 97124-6497 25 26*******************************************************************************/ 27 28 29/****************************************************************************** 30 Copyright (c)2006 - 2007 Myricom, Inc. for some LRO specific code 31******************************************************************************/ 32 33#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt 34 35#include <linux/types.h> 36#include <linux/bitops.h> 37#include <linux/module.h> 38#include <linux/pci.h> 39#include <linux/netdevice.h> 40#include <linux/vmalloc.h> 41#include <linux/string.h> 42#include <linux/in.h> 43#include <linux/ip.h> 44#include <linux/tcp.h> 45#include <linux/ipv6.h> 46#include <linux/slab.h> 47#include <net/checksum.h> 48#include <net/ip6_checksum.h> 49#include <linux/ethtool.h> 50#include <linux/if.h> 51#include <linux/if_vlan.h> 52#include <linux/prefetch.h> 53 54#include "ixgbevf.h" 55 56const char ixgbevf_driver_name[] = "ixgbevf"; 57static const char ixgbevf_driver_string[] = 58 "Intel(R) 10 Gigabit PCI Express Virtual Function Network Driver"; 59 60#define DRV_VERSION "2.2.0-k" 61const char ixgbevf_driver_version[] = DRV_VERSION; 62static char ixgbevf_copyright[] = 63 "Copyright (c) 2009 - 2012 Intel Corporation."; 64 65static const struct ixgbevf_info *ixgbevf_info_tbl[] = { 66 [board_82599_vf] = &ixgbevf_82599_vf_info, 67 [board_X540_vf] = &ixgbevf_X540_vf_info, 68}; 69 70/* ixgbevf_pci_tbl - PCI Device ID Table 71 * 72 * Wildcard entries (PCI_ANY_ID) should come last 73 * Last entry must be all 0s 74 * 75 * { Vendor ID, Device ID, SubVendor ID, SubDevice ID, 76 * Class, Class Mask, private data (not used) } 77 */ 78static struct pci_device_id ixgbevf_pci_tbl[] = { 79 {PCI_VDEVICE(INTEL, IXGBE_DEV_ID_82599_VF), 80 board_82599_vf}, 81 {PCI_VDEVICE(INTEL, IXGBE_DEV_ID_X540_VF), 82 board_X540_vf}, 83 84 /* required last entry */ 85 {0, } 86}; 87MODULE_DEVICE_TABLE(pci, ixgbevf_pci_tbl); 88 89MODULE_AUTHOR("Intel Corporation, <linux.nics@intel.com>"); 90MODULE_DESCRIPTION("Intel(R) 82599 Virtual Function Driver"); 91MODULE_LICENSE("GPL"); 92MODULE_VERSION(DRV_VERSION); 93 94#define DEFAULT_DEBUG_LEVEL_SHIFT 3 95 96/* forward decls */ 97static void ixgbevf_set_itr_msix(struct ixgbevf_q_vector *q_vector); 98static void ixgbevf_write_eitr(struct ixgbevf_adapter *adapter, int v_idx, 99 u32 itr_reg); 100 101static inline void ixgbevf_release_rx_desc(struct ixgbe_hw *hw, 102 struct ixgbevf_ring *rx_ring, 103 u32 val) 104{ 105 /* 106 * Force memory writes to complete before letting h/w 107 * know there are new descriptors to fetch. (Only 108 * applicable for weak-ordered memory model archs, 109 * such as IA-64). 110 */ 111 wmb(); 112 IXGBE_WRITE_REG(hw, IXGBE_VFRDT(rx_ring->reg_idx), val); 113} 114 115/* 116 * ixgbevf_set_ivar - set IVAR registers - maps interrupt causes to vectors 117 * @adapter: pointer to adapter struct 118 * @direction: 0 for Rx, 1 for Tx, -1 for other causes 119 * @queue: queue to map the corresponding interrupt to 120 * @msix_vector: the vector to map to the corresponding queue 121 * 122 */ 123static void ixgbevf_set_ivar(struct ixgbevf_adapter *adapter, s8 direction, 124 u8 queue, u8 msix_vector) 125{ 126 u32 ivar, index; 127 struct ixgbe_hw *hw = &adapter->hw; 128 if (direction == -1) { 129 /* other causes */ 130 msix_vector |= IXGBE_IVAR_ALLOC_VAL; 131 ivar = IXGBE_READ_REG(hw, IXGBE_VTIVAR_MISC); 132 ivar &= ~0xFF; 133 ivar |= msix_vector; 134 IXGBE_WRITE_REG(hw, IXGBE_VTIVAR_MISC, ivar); 135 } else { 136 /* tx or rx causes */ 137 msix_vector |= IXGBE_IVAR_ALLOC_VAL; 138 index = ((16 * (queue & 1)) + (8 * direction)); 139 ivar = IXGBE_READ_REG(hw, IXGBE_VTIVAR(queue >> 1)); 140 ivar &= ~(0xFF << index); 141 ivar |= (msix_vector << index); 142 IXGBE_WRITE_REG(hw, IXGBE_VTIVAR(queue >> 1), ivar); 143 } 144} 145 146static void ixgbevf_unmap_and_free_tx_resource(struct ixgbevf_adapter *adapter, 147 struct ixgbevf_tx_buffer 148 *tx_buffer_info) 149{ 150 if (tx_buffer_info->dma) { 151 if (tx_buffer_info->mapped_as_page) 152 dma_unmap_page(&adapter->pdev->dev, 153 tx_buffer_info->dma, 154 tx_buffer_info->length, 155 DMA_TO_DEVICE); 156 else 157 dma_unmap_single(&adapter->pdev->dev, 158 tx_buffer_info->dma, 159 tx_buffer_info->length, 160 DMA_TO_DEVICE); 161 tx_buffer_info->dma = 0; 162 } 163 if (tx_buffer_info->skb) { 164 dev_kfree_skb_any(tx_buffer_info->skb); 165 tx_buffer_info->skb = NULL; 166 } 167 tx_buffer_info->time_stamp = 0; 168 /* tx_buffer_info must be completely set up in the transmit path */ 169} 170 171#define IXGBE_MAX_TXD_PWR 14 172#define IXGBE_MAX_DATA_PER_TXD (1 << IXGBE_MAX_TXD_PWR) 173 174/* Tx Descriptors needed, worst case */ 175#define TXD_USE_COUNT(S) (((S) >> IXGBE_MAX_TXD_PWR) + \ 176 (((S) & (IXGBE_MAX_DATA_PER_TXD - 1)) ? 1 : 0)) 177#ifdef MAX_SKB_FRAGS 178#define DESC_NEEDED (TXD_USE_COUNT(IXGBE_MAX_DATA_PER_TXD) /* skb->data */ + \ 179 MAX_SKB_FRAGS * TXD_USE_COUNT(PAGE_SIZE) + 1) /* for context */ 180#else 181#define DESC_NEEDED TXD_USE_COUNT(IXGBE_MAX_DATA_PER_TXD) 182#endif 183 184static void ixgbevf_tx_timeout(struct net_device *netdev); 185 186/** 187 * ixgbevf_clean_tx_irq - Reclaim resources after transmit completes 188 * @adapter: board private structure 189 * @tx_ring: tx ring to clean 190 **/ 191static bool ixgbevf_clean_tx_irq(struct ixgbevf_adapter *adapter, 192 struct ixgbevf_ring *tx_ring) 193{ 194 struct net_device *netdev = adapter->netdev; 195 struct ixgbe_hw *hw = &adapter->hw; 196 union ixgbe_adv_tx_desc *tx_desc, *eop_desc; 197 struct ixgbevf_tx_buffer *tx_buffer_info; 198 unsigned int i, eop, count = 0; 199 unsigned int total_bytes = 0, total_packets = 0; 200 201 i = tx_ring->next_to_clean; 202 eop = tx_ring->tx_buffer_info[i].next_to_watch; 203 eop_desc = IXGBE_TX_DESC_ADV(*tx_ring, eop); 204 205 while ((eop_desc->wb.status & cpu_to_le32(IXGBE_TXD_STAT_DD)) && 206 (count < tx_ring->work_limit)) { 207 bool cleaned = false; 208 rmb(); /* read buffer_info after eop_desc */ 209 /* eop could change between read and DD-check */ 210 if (unlikely(eop != tx_ring->tx_buffer_info[i].next_to_watch)) 211 goto cont_loop; 212 for ( ; !cleaned; count++) { 213 struct sk_buff *skb; 214 tx_desc = IXGBE_TX_DESC_ADV(*tx_ring, i); 215 tx_buffer_info = &tx_ring->tx_buffer_info[i]; 216 cleaned = (i == eop); 217 skb = tx_buffer_info->skb; 218 219 if (cleaned && skb) { 220 unsigned int segs, bytecount; 221 222 /* gso_segs is currently only valid for tcp */ 223 segs = skb_shinfo(skb)->gso_segs ?: 1; 224 /* multiply data chunks by size of headers */ 225 bytecount = ((segs - 1) * skb_headlen(skb)) + 226 skb->len; 227 total_packets += segs; 228 total_bytes += bytecount; 229 } 230 231 ixgbevf_unmap_and_free_tx_resource(adapter, 232 tx_buffer_info); 233 234 tx_desc->wb.status = 0; 235 236 i++; 237 if (i == tx_ring->count) 238 i = 0; 239 } 240 241cont_loop: 242 eop = tx_ring->tx_buffer_info[i].next_to_watch; 243 eop_desc = IXGBE_TX_DESC_ADV(*tx_ring, eop); 244 } 245 246 tx_ring->next_to_clean = i; 247 248#define TX_WAKE_THRESHOLD (DESC_NEEDED * 2) 249 if (unlikely(count && netif_carrier_ok(netdev) && 250 (IXGBE_DESC_UNUSED(tx_ring) >= TX_WAKE_THRESHOLD))) { 251 /* Make sure that anybody stopping the queue after this 252 * sees the new next_to_clean. 253 */ 254 smp_mb(); 255#ifdef HAVE_TX_MQ 256 if (__netif_subqueue_stopped(netdev, tx_ring->queue_index) && 257 !test_bit(__IXGBEVF_DOWN, &adapter->state)) { 258 netif_wake_subqueue(netdev, tx_ring->queue_index); 259 ++adapter->restart_queue; 260 } 261#else 262 if (netif_queue_stopped(netdev) && 263 !test_bit(__IXGBEVF_DOWN, &adapter->state)) { 264 netif_wake_queue(netdev); 265 ++adapter->restart_queue; 266 } 267#endif 268 } 269 270 /* re-arm the interrupt */ 271 if ((count >= tx_ring->work_limit) && 272 (!test_bit(__IXGBEVF_DOWN, &adapter->state))) { 273 IXGBE_WRITE_REG(hw, IXGBE_VTEICS, tx_ring->v_idx); 274 } 275 276 u64_stats_update_begin(&tx_ring->syncp); 277 tx_ring->total_bytes += total_bytes; 278 tx_ring->total_packets += total_packets; 279 u64_stats_update_end(&tx_ring->syncp); 280 281 return count < tx_ring->work_limit; 282} 283 284/** 285 * ixgbevf_receive_skb - Send a completed packet up the stack 286 * @q_vector: structure containing interrupt and ring information 287 * @skb: packet to send up 288 * @status: hardware indication of status of receive 289 * @rx_ring: rx descriptor ring (for a specific queue) to setup 290 * @rx_desc: rx descriptor 291 **/ 292static void ixgbevf_receive_skb(struct ixgbevf_q_vector *q_vector, 293 struct sk_buff *skb, u8 status, 294 struct ixgbevf_ring *ring, 295 union ixgbe_adv_rx_desc *rx_desc) 296{ 297 struct ixgbevf_adapter *adapter = q_vector->adapter; 298 bool is_vlan = (status & IXGBE_RXD_STAT_VP); 299 u16 tag = le16_to_cpu(rx_desc->wb.upper.vlan); 300 301 if (is_vlan && test_bit(tag, adapter->active_vlans)) 302 __vlan_hwaccel_put_tag(skb, tag); 303 304 if (!(adapter->flags & IXGBE_FLAG_IN_NETPOLL)) 305 napi_gro_receive(&q_vector->napi, skb); 306 else 307 netif_rx(skb); 308} 309 310/** 311 * ixgbevf_rx_checksum - indicate in skb if hw indicated a good cksum 312 * @adapter: address of board private structure 313 * @status_err: hardware indication of status of receive 314 * @skb: skb currently being received and modified 315 **/ 316static inline void ixgbevf_rx_checksum(struct ixgbevf_adapter *adapter, 317 u32 status_err, struct sk_buff *skb) 318{ 319 skb_checksum_none_assert(skb); 320 321 /* Rx csum disabled */ 322 if (!(adapter->flags & IXGBE_FLAG_RX_CSUM_ENABLED)) 323 return; 324 325 /* if IP and error */ 326 if ((status_err & IXGBE_RXD_STAT_IPCS) && 327 (status_err & IXGBE_RXDADV_ERR_IPE)) { 328 adapter->hw_csum_rx_error++; 329 return; 330 } 331 332 if (!(status_err & IXGBE_RXD_STAT_L4CS)) 333 return; 334 335 if (status_err & IXGBE_RXDADV_ERR_TCPE) { 336 adapter->hw_csum_rx_error++; 337 return; 338 } 339 340 /* It must be a TCP or UDP packet with a valid checksum */ 341 skb->ip_summed = CHECKSUM_UNNECESSARY; 342 adapter->hw_csum_rx_good++; 343} 344 345/** 346 * ixgbevf_alloc_rx_buffers - Replace used receive buffers; packet split 347 * @adapter: address of board private structure 348 **/ 349static void ixgbevf_alloc_rx_buffers(struct ixgbevf_adapter *adapter, 350 struct ixgbevf_ring *rx_ring, 351 int cleaned_count) 352{ 353 struct pci_dev *pdev = adapter->pdev; 354 union ixgbe_adv_rx_desc *rx_desc; 355 struct ixgbevf_rx_buffer *bi; 356 struct sk_buff *skb; 357 unsigned int i; 358 unsigned int bufsz = rx_ring->rx_buf_len + NET_IP_ALIGN; 359 360 i = rx_ring->next_to_use; 361 bi = &rx_ring->rx_buffer_info[i]; 362 363 while (cleaned_count--) { 364 rx_desc = IXGBE_RX_DESC_ADV(*rx_ring, i); 365 366 if (!bi->page_dma && 367 (adapter->flags & IXGBE_FLAG_RX_PS_ENABLED)) { 368 if (!bi->page) { 369 bi->page = alloc_page(GFP_ATOMIC | __GFP_COLD); 370 if (!bi->page) { 371 adapter->alloc_rx_page_failed++; 372 goto no_buffers; 373 } 374 bi->page_offset = 0; 375 } else { 376 /* use a half page if we're re-using */ 377 bi->page_offset ^= (PAGE_SIZE / 2); 378 } 379 380 bi->page_dma = dma_map_page(&pdev->dev, bi->page, 381 bi->page_offset, 382 (PAGE_SIZE / 2), 383 DMA_FROM_DEVICE); 384 } 385 386 skb = bi->skb; 387 if (!skb) { 388 skb = netdev_alloc_skb(adapter->netdev, 389 bufsz); 390 391 if (!skb) { 392 adapter->alloc_rx_buff_failed++; 393 goto no_buffers; 394 } 395 396 /* 397 * Make buffer alignment 2 beyond a 16 byte boundary 398 * this will result in a 16 byte aligned IP header after 399 * the 14 byte MAC header is removed 400 */ 401 skb_reserve(skb, NET_IP_ALIGN); 402 403 bi->skb = skb; 404 } 405 if (!bi->dma) { 406 bi->dma = dma_map_single(&pdev->dev, skb->data, 407 rx_ring->rx_buf_len, 408 DMA_FROM_DEVICE); 409 } 410 /* Refresh the desc even if buffer_addrs didn't change because 411 * each write-back erases this info. */ 412 if (adapter->flags & IXGBE_FLAG_RX_PS_ENABLED) { 413 rx_desc->read.pkt_addr = cpu_to_le64(bi->page_dma); 414 rx_desc->read.hdr_addr = cpu_to_le64(bi->dma); 415 } else { 416 rx_desc->read.pkt_addr = cpu_to_le64(bi->dma); 417 } 418 419 i++; 420 if (i == rx_ring->count) 421 i = 0; 422 bi = &rx_ring->rx_buffer_info[i]; 423 } 424 425no_buffers: 426 if (rx_ring->next_to_use != i) { 427 rx_ring->next_to_use = i; 428 if (i-- == 0) 429 i = (rx_ring->count - 1); 430 431 ixgbevf_release_rx_desc(&adapter->hw, rx_ring, i); 432 } 433} 434 435static inline void ixgbevf_irq_enable_queues(struct ixgbevf_adapter *adapter, 436 u64 qmask) 437{ 438 u32 mask; 439 struct ixgbe_hw *hw = &adapter->hw; 440 441 mask = (qmask & 0xFFFFFFFF); 442 IXGBE_WRITE_REG(hw, IXGBE_VTEIMS, mask); 443} 444 445static inline u16 ixgbevf_get_hdr_info(union ixgbe_adv_rx_desc *rx_desc) 446{ 447 return rx_desc->wb.lower.lo_dword.hs_rss.hdr_info; 448} 449 450static inline u16 ixgbevf_get_pkt_info(union ixgbe_adv_rx_desc *rx_desc) 451{ 452 return rx_desc->wb.lower.lo_dword.hs_rss.pkt_info; 453} 454 455static bool ixgbevf_clean_rx_irq(struct ixgbevf_q_vector *q_vector, 456 struct ixgbevf_ring *rx_ring, 457 int *work_done, int work_to_do) 458{ 459 struct ixgbevf_adapter *adapter = q_vector->adapter; 460 struct pci_dev *pdev = adapter->pdev; 461 union ixgbe_adv_rx_desc *rx_desc, *next_rxd; 462 struct ixgbevf_rx_buffer *rx_buffer_info, *next_buffer; 463 struct sk_buff *skb; 464 unsigned int i; 465 u32 len, staterr; 466 u16 hdr_info; 467 bool cleaned = false; 468 int cleaned_count = 0; 469 unsigned int total_rx_bytes = 0, total_rx_packets = 0; 470 471 i = rx_ring->next_to_clean; 472 rx_desc = IXGBE_RX_DESC_ADV(*rx_ring, i); 473 staterr = le32_to_cpu(rx_desc->wb.upper.status_error); 474 rx_buffer_info = &rx_ring->rx_buffer_info[i]; 475 476 while (staterr & IXGBE_RXD_STAT_DD) { 477 u32 upper_len = 0; 478 if (*work_done >= work_to_do) 479 break; 480 (*work_done)++; 481 482 rmb(); /* read descriptor and rx_buffer_info after status DD */ 483 if (adapter->flags & IXGBE_FLAG_RX_PS_ENABLED) { 484 hdr_info = le16_to_cpu(ixgbevf_get_hdr_info(rx_desc)); 485 len = (hdr_info & IXGBE_RXDADV_HDRBUFLEN_MASK) >> 486 IXGBE_RXDADV_HDRBUFLEN_SHIFT; 487 if (hdr_info & IXGBE_RXDADV_SPH) 488 adapter->rx_hdr_split++; 489 if (len > IXGBEVF_RX_HDR_SIZE) 490 len = IXGBEVF_RX_HDR_SIZE; 491 upper_len = le16_to_cpu(rx_desc->wb.upper.length); 492 } else { 493 len = le16_to_cpu(rx_desc->wb.upper.length); 494 } 495 cleaned = true; 496 skb = rx_buffer_info->skb; 497 prefetch(skb->data - NET_IP_ALIGN); 498 rx_buffer_info->skb = NULL; 499 500 if (rx_buffer_info->dma) { 501 dma_unmap_single(&pdev->dev, rx_buffer_info->dma, 502 rx_ring->rx_buf_len, 503 DMA_FROM_DEVICE); 504 rx_buffer_info->dma = 0; 505 skb_put(skb, len); 506 } 507 508 if (upper_len) { 509 dma_unmap_page(&pdev->dev, rx_buffer_info->page_dma, 510 PAGE_SIZE / 2, DMA_FROM_DEVICE); 511 rx_buffer_info->page_dma = 0; 512 skb_fill_page_desc(skb, skb_shinfo(skb)->nr_frags, 513 rx_buffer_info->page, 514 rx_buffer_info->page_offset, 515 upper_len); 516 517 if ((rx_ring->rx_buf_len > (PAGE_SIZE / 2)) || 518 (page_count(rx_buffer_info->page) != 1)) 519 rx_buffer_info->page = NULL; 520 else 521 get_page(rx_buffer_info->page); 522 523 skb->len += upper_len; 524 skb->data_len += upper_len; 525 skb->truesize += upper_len; 526 } 527 528 i++; 529 if (i == rx_ring->count) 530 i = 0; 531 532 next_rxd = IXGBE_RX_DESC_ADV(*rx_ring, i); 533 prefetch(next_rxd); 534 cleaned_count++; 535 536 next_buffer = &rx_ring->rx_buffer_info[i]; 537 538 if (!(staterr & IXGBE_RXD_STAT_EOP)) { 539 if (adapter->flags & IXGBE_FLAG_RX_PS_ENABLED) { 540 rx_buffer_info->skb = next_buffer->skb; 541 rx_buffer_info->dma = next_buffer->dma; 542 next_buffer->skb = skb; 543 next_buffer->dma = 0; 544 } else { 545 skb->next = next_buffer->skb; 546 skb->next->prev = skb; 547 } 548 adapter->non_eop_descs++; 549 goto next_desc; 550 } 551 552 /* ERR_MASK will only have valid bits if EOP set */ 553 if (unlikely(staterr & IXGBE_RXDADV_ERR_FRAME_ERR_MASK)) { 554 dev_kfree_skb_irq(skb); 555 goto next_desc; 556 } 557 558 ixgbevf_rx_checksum(adapter, staterr, skb); 559 560 /* probably a little skewed due to removing CRC */ 561 total_rx_bytes += skb->len; 562 total_rx_packets++; 563 564 /* 565 * Work around issue of some types of VM to VM loop back 566 * packets not getting split correctly 567 */ 568 if (staterr & IXGBE_RXD_STAT_LB) { 569 u32 header_fixup_len = skb_headlen(skb); 570 if (header_fixup_len < 14) 571 skb_push(skb, header_fixup_len); 572 } 573 skb->protocol = eth_type_trans(skb, adapter->netdev); 574 575 ixgbevf_receive_skb(q_vector, skb, staterr, rx_ring, rx_desc); 576 577next_desc: 578 rx_desc->wb.upper.status_error = 0; 579 580 /* return some buffers to hardware, one at a time is too slow */ 581 if (cleaned_count >= IXGBEVF_RX_BUFFER_WRITE) { 582 ixgbevf_alloc_rx_buffers(adapter, rx_ring, 583 cleaned_count); 584 cleaned_count = 0; 585 } 586 587 /* use prefetched values */ 588 rx_desc = next_rxd; 589 rx_buffer_info = &rx_ring->rx_buffer_info[i]; 590 591 staterr = le32_to_cpu(rx_desc->wb.upper.status_error); 592 } 593 594 rx_ring->next_to_clean = i; 595 cleaned_count = IXGBE_DESC_UNUSED(rx_ring); 596 597 if (cleaned_count) 598 ixgbevf_alloc_rx_buffers(adapter, rx_ring, cleaned_count); 599 600 u64_stats_update_begin(&rx_ring->syncp); 601 rx_ring->total_packets += total_rx_packets; 602 rx_ring->total_bytes += total_rx_bytes; 603 u64_stats_update_end(&rx_ring->syncp); 604 605 return cleaned; 606} 607 608/** 609 * ixgbevf_clean_rxonly - msix (aka one shot) rx clean routine 610 * @napi: napi struct with our devices info in it 611 * @budget: amount of work driver is allowed to do this pass, in packets 612 * 613 * This function is optimized for cleaning one queue only on a single 614 * q_vector!!! 615 **/ 616static int ixgbevf_clean_rxonly(struct napi_struct *napi, int budget) 617{ 618 struct ixgbevf_q_vector *q_vector = 619 container_of(napi, struct ixgbevf_q_vector, napi); 620 struct ixgbevf_adapter *adapter = q_vector->adapter; 621 struct ixgbevf_ring *rx_ring = NULL; 622 int work_done = 0; 623 long r_idx; 624 625 r_idx = find_first_bit(q_vector->rxr_idx, adapter->num_rx_queues); 626 rx_ring = &(adapter->rx_ring[r_idx]); 627 628 ixgbevf_clean_rx_irq(q_vector, rx_ring, &work_done, budget); 629 630 /* If all Rx work done, exit the polling mode */ 631 if (work_done < budget) { 632 napi_complete(napi); 633 if (adapter->itr_setting & 1) 634 ixgbevf_set_itr_msix(q_vector); 635 if (!test_bit(__IXGBEVF_DOWN, &adapter->state)) 636 ixgbevf_irq_enable_queues(adapter, rx_ring->v_idx); 637 } 638 639 return work_done; 640} 641 642/** 643 * ixgbevf_clean_rxonly_many - msix (aka one shot) rx clean routine 644 * @napi: napi struct with our devices info in it 645 * @budget: amount of work driver is allowed to do this pass, in packets 646 * 647 * This function will clean more than one rx queue associated with a 648 * q_vector. 649 **/ 650static int ixgbevf_clean_rxonly_many(struct napi_struct *napi, int budget) 651{ 652 struct ixgbevf_q_vector *q_vector = 653 container_of(napi, struct ixgbevf_q_vector, napi); 654 struct ixgbevf_adapter *adapter = q_vector->adapter; 655 struct ixgbevf_ring *rx_ring = NULL; 656 int work_done = 0, i; 657 long r_idx; 658 u64 enable_mask = 0; 659 660 /* attempt to distribute budget to each queue fairly, but don't allow 661 * the budget to go below 1 because we'll exit polling */ 662 budget /= (q_vector->rxr_count ?: 1); 663 budget = max(budget, 1); 664 r_idx = find_first_bit(q_vector->rxr_idx, adapter->num_rx_queues); 665 for (i = 0; i < q_vector->rxr_count; i++) { 666 rx_ring = &(adapter->rx_ring[r_idx]); 667 ixgbevf_clean_rx_irq(q_vector, rx_ring, &work_done, budget); 668 enable_mask |= rx_ring->v_idx; 669 r_idx = find_next_bit(q_vector->rxr_idx, adapter->num_rx_queues, 670 r_idx + 1); 671 } 672 673#ifndef HAVE_NETDEV_NAPI_LIST 674 if (!netif_running(adapter->netdev)) 675 work_done = 0; 676 677#endif 678 r_idx = find_first_bit(q_vector->rxr_idx, adapter->num_rx_queues); 679 rx_ring = &(adapter->rx_ring[r_idx]); 680 681 /* If all Rx work done, exit the polling mode */ 682 if (work_done < budget) { 683 napi_complete(napi); 684 if (adapter->itr_setting & 1) 685 ixgbevf_set_itr_msix(q_vector); 686 if (!test_bit(__IXGBEVF_DOWN, &adapter->state)) 687 ixgbevf_irq_enable_queues(adapter, enable_mask); 688 } 689 690 return work_done; 691} 692 693 694/** 695 * ixgbevf_configure_msix - Configure MSI-X hardware 696 * @adapter: board private structure 697 * 698 * ixgbevf_configure_msix sets up the hardware to properly generate MSI-X 699 * interrupts. 700 **/ 701static void ixgbevf_configure_msix(struct ixgbevf_adapter *adapter) 702{ 703 struct ixgbevf_q_vector *q_vector; 704 struct ixgbe_hw *hw = &adapter->hw; 705 int i, j, q_vectors, v_idx, r_idx; 706 u32 mask; 707 708 q_vectors = adapter->num_msix_vectors - NON_Q_VECTORS; 709 710 /* 711 * Populate the IVAR table and set the ITR values to the 712 * corresponding register. 713 */ 714 for (v_idx = 0; v_idx < q_vectors; v_idx++) { 715 q_vector = adapter->q_vector[v_idx]; 716 /* XXX for_each_set_bit(...) */ 717 r_idx = find_first_bit(q_vector->rxr_idx, 718 adapter->num_rx_queues); 719 720 for (i = 0; i < q_vector->rxr_count; i++) { 721 j = adapter->rx_ring[r_idx].reg_idx; 722 ixgbevf_set_ivar(adapter, 0, j, v_idx); 723 r_idx = find_next_bit(q_vector->rxr_idx, 724 adapter->num_rx_queues, 725 r_idx + 1); 726 } 727 r_idx = find_first_bit(q_vector->txr_idx, 728 adapter->num_tx_queues); 729 730 for (i = 0; i < q_vector->txr_count; i++) { 731 j = adapter->tx_ring[r_idx].reg_idx; 732 ixgbevf_set_ivar(adapter, 1, j, v_idx); 733 r_idx = find_next_bit(q_vector->txr_idx, 734 adapter->num_tx_queues, 735 r_idx + 1); 736 } 737 738 /* if this is a tx only vector halve the interrupt rate */ 739 if (q_vector->txr_count && !q_vector->rxr_count) 740 q_vector->eitr = (adapter->eitr_param >> 1); 741 else if (q_vector->rxr_count) 742 /* rx only */ 743 q_vector->eitr = adapter->eitr_param; 744 745 ixgbevf_write_eitr(adapter, v_idx, q_vector->eitr); 746 } 747 748 ixgbevf_set_ivar(adapter, -1, 1, v_idx); 749 750 /* set up to autoclear timer, and the vectors */ 751 mask = IXGBE_EIMS_ENABLE_MASK; 752 mask &= ~IXGBE_EIMS_OTHER; 753 IXGBE_WRITE_REG(hw, IXGBE_VTEIAC, mask); 754} 755 756enum latency_range { 757 lowest_latency = 0, 758 low_latency = 1, 759 bulk_latency = 2, 760 latency_invalid = 255 761}; 762 763/** 764 * ixgbevf_update_itr - update the dynamic ITR value based on statistics 765 * @adapter: pointer to adapter 766 * @eitr: eitr setting (ints per sec) to give last timeslice 767 * @itr_setting: current throttle rate in ints/second 768 * @packets: the number of packets during this measurement interval 769 * @bytes: the number of bytes during this measurement interval 770 * 771 * Stores a new ITR value based on packets and byte 772 * counts during the last interrupt. The advantage of per interrupt 773 * computation is faster updates and more accurate ITR for the current 774 * traffic pattern. Constants in this function were computed 775 * based on theoretical maximum wire speed and thresholds were set based 776 * on testing data as well as attempting to minimize response time 777 * while increasing bulk throughput. 778 **/ 779static u8 ixgbevf_update_itr(struct ixgbevf_adapter *adapter, 780 u32 eitr, u8 itr_setting, 781 int packets, int bytes) 782{ 783 unsigned int retval = itr_setting; 784 u32 timepassed_us; 785 u64 bytes_perint; 786 787 if (packets == 0) 788 goto update_itr_done; 789 790 791 /* simple throttlerate management 792 * 0-20MB/s lowest (100000 ints/s) 793 * 20-100MB/s low (20000 ints/s) 794 * 100-1249MB/s bulk (8000 ints/s) 795 */ 796 /* what was last interrupt timeslice? */ 797 timepassed_us = 1000000/eitr; 798 bytes_perint = bytes / timepassed_us; /* bytes/usec */ 799 800 switch (itr_setting) { 801 case lowest_latency: 802 if (bytes_perint > adapter->eitr_low) 803 retval = low_latency; 804 break; 805 case low_latency: 806 if (bytes_perint > adapter->eitr_high) 807 retval = bulk_latency; 808 else if (bytes_perint <= adapter->eitr_low) 809 retval = lowest_latency; 810 break; 811 case bulk_latency: 812 if (bytes_perint <= adapter->eitr_high) 813 retval = low_latency; 814 break; 815 } 816 817update_itr_done: 818 return retval; 819} 820 821/** 822 * ixgbevf_write_eitr - write VTEITR register in hardware specific way 823 * @adapter: pointer to adapter struct 824 * @v_idx: vector index into q_vector array 825 * @itr_reg: new value to be written in *register* format, not ints/s 826 * 827 * This function is made to be called by ethtool and by the driver 828 * when it needs to update VTEITR registers at runtime. Hardware 829 * specific quirks/differences are taken care of here. 830 */ 831static void ixgbevf_write_eitr(struct ixgbevf_adapter *adapter, int v_idx, 832 u32 itr_reg) 833{ 834 struct ixgbe_hw *hw = &adapter->hw; 835 836 itr_reg = EITR_INTS_PER_SEC_TO_REG(itr_reg); 837 838 /* 839 * set the WDIS bit to not clear the timer bits and cause an 840 * immediate assertion of the interrupt 841 */ 842 itr_reg |= IXGBE_EITR_CNT_WDIS; 843 844 IXGBE_WRITE_REG(hw, IXGBE_VTEITR(v_idx), itr_reg); 845} 846 847static void ixgbevf_set_itr_msix(struct ixgbevf_q_vector *q_vector) 848{ 849 struct ixgbevf_adapter *adapter = q_vector->adapter; 850 u32 new_itr; 851 u8 current_itr, ret_itr; 852 int i, r_idx, v_idx = q_vector->v_idx; 853 struct ixgbevf_ring *rx_ring, *tx_ring; 854 855 r_idx = find_first_bit(q_vector->txr_idx, adapter->num_tx_queues); 856 for (i = 0; i < q_vector->txr_count; i++) { 857 tx_ring = &(adapter->tx_ring[r_idx]); 858 ret_itr = ixgbevf_update_itr(adapter, q_vector->eitr, 859 q_vector->tx_itr, 860 tx_ring->total_packets, 861 tx_ring->total_bytes); 862 /* if the result for this queue would decrease interrupt 863 * rate for this vector then use that result */ 864 q_vector->tx_itr = ((q_vector->tx_itr > ret_itr) ? 865 q_vector->tx_itr - 1 : ret_itr); 866 r_idx = find_next_bit(q_vector->txr_idx, adapter->num_tx_queues, 867 r_idx + 1); 868 } 869 870 r_idx = find_first_bit(q_vector->rxr_idx, adapter->num_rx_queues); 871 for (i = 0; i < q_vector->rxr_count; i++) { 872 rx_ring = &(adapter->rx_ring[r_idx]); 873 ret_itr = ixgbevf_update_itr(adapter, q_vector->eitr, 874 q_vector->rx_itr, 875 rx_ring->total_packets, 876 rx_ring->total_bytes); 877 /* if the result for this queue would decrease interrupt 878 * rate for this vector then use that result */ 879 q_vector->rx_itr = ((q_vector->rx_itr > ret_itr) ? 880 q_vector->rx_itr - 1 : ret_itr); 881 r_idx = find_next_bit(q_vector->rxr_idx, adapter->num_rx_queues, 882 r_idx + 1); 883 } 884 885 current_itr = max(q_vector->rx_itr, q_vector->tx_itr); 886 887 switch (current_itr) { 888 /* counts and packets in update_itr are dependent on these numbers */ 889 case lowest_latency: 890 new_itr = 100000; 891 break; 892 case low_latency: 893 new_itr = 20000; /* aka hwitr = ~200 */ 894 break; 895 case bulk_latency: 896 default: 897 new_itr = 8000; 898 break; 899 } 900 901 if (new_itr != q_vector->eitr) { 902 u32 itr_reg; 903 904 /* save the algorithm value here, not the smoothed one */ 905 q_vector->eitr = new_itr; 906 /* do an exponential smoothing */ 907 new_itr = ((q_vector->eitr * 90)/100) + ((new_itr * 10)/100); 908 itr_reg = EITR_INTS_PER_SEC_TO_REG(new_itr); 909 ixgbevf_write_eitr(adapter, v_idx, itr_reg); 910 } 911} 912 913static irqreturn_t ixgbevf_msix_mbx(int irq, void *data) 914{ 915 struct net_device *netdev = data; 916 struct ixgbevf_adapter *adapter = netdev_priv(netdev); 917 struct ixgbe_hw *hw = &adapter->hw; 918 u32 eicr; 919 u32 msg; 920 bool got_ack = false; 921 922 eicr = IXGBE_READ_REG(hw, IXGBE_VTEICS); 923 IXGBE_WRITE_REG(hw, IXGBE_VTEICR, eicr); 924 925 if (!hw->mbx.ops.check_for_ack(hw)) 926 got_ack = true; 927 928 if (!hw->mbx.ops.check_for_msg(hw)) { 929 hw->mbx.ops.read(hw, &msg, 1); 930 931 if ((msg & IXGBE_MBVFICR_VFREQ_MASK) == IXGBE_PF_CONTROL_MSG) 932 mod_timer(&adapter->watchdog_timer, 933 round_jiffies(jiffies + 1)); 934 935 if (msg & IXGBE_VT_MSGTYPE_NACK) 936 pr_warn("Last Request of type %2.2x to PF Nacked\n", 937 msg & 0xFF); 938 /* 939 * Restore the PFSTS bit in case someone is polling for a 940 * return message from the PF 941 */ 942 hw->mbx.v2p_mailbox |= IXGBE_VFMAILBOX_PFSTS; 943 } 944 945 /* 946 * checking for the ack clears the PFACK bit. Place 947 * it back in the v2p_mailbox cache so that anyone 948 * polling for an ack will not miss it 949 */ 950 if (got_ack) 951 hw->mbx.v2p_mailbox |= IXGBE_VFMAILBOX_PFACK; 952 953 return IRQ_HANDLED; 954} 955 956static irqreturn_t ixgbevf_msix_clean_tx(int irq, void *data) 957{ 958 struct ixgbevf_q_vector *q_vector = data; 959 struct ixgbevf_adapter *adapter = q_vector->adapter; 960 struct ixgbevf_ring *tx_ring; 961 int i, r_idx; 962 963 if (!q_vector->txr_count) 964 return IRQ_HANDLED; 965 966 r_idx = find_first_bit(q_vector->txr_idx, adapter->num_tx_queues); 967 for (i = 0; i < q_vector->txr_count; i++) { 968 tx_ring = &(adapter->tx_ring[r_idx]); 969 tx_ring->total_bytes = 0; 970 tx_ring->total_packets = 0; 971 ixgbevf_clean_tx_irq(adapter, tx_ring); 972 r_idx = find_next_bit(q_vector->txr_idx, adapter->num_tx_queues, 973 r_idx + 1); 974 } 975 976 if (adapter->itr_setting & 1) 977 ixgbevf_set_itr_msix(q_vector); 978 979 return IRQ_HANDLED; 980} 981 982/** 983 * ixgbevf_msix_clean_rx - single unshared vector rx clean (all queues) 984 * @irq: unused 985 * @data: pointer to our q_vector struct for this interrupt vector 986 **/ 987static irqreturn_t ixgbevf_msix_clean_rx(int irq, void *data) 988{ 989 struct ixgbevf_q_vector *q_vector = data; 990 struct ixgbevf_adapter *adapter = q_vector->adapter; 991 struct ixgbe_hw *hw = &adapter->hw; 992 struct ixgbevf_ring *rx_ring; 993 int r_idx; 994 int i; 995 996 r_idx = find_first_bit(q_vector->rxr_idx, adapter->num_rx_queues); 997 for (i = 0; i < q_vector->rxr_count; i++) { 998 rx_ring = &(adapter->rx_ring[r_idx]); 999 rx_ring->total_bytes = 0; 1000 rx_ring->total_packets = 0; 1001 r_idx = find_next_bit(q_vector->rxr_idx, adapter->num_rx_queues, 1002 r_idx + 1); 1003 } 1004 1005 if (!q_vector->rxr_count) 1006 return IRQ_HANDLED; 1007 1008 r_idx = find_first_bit(q_vector->rxr_idx, adapter->num_rx_queues); 1009 rx_ring = &(adapter->rx_ring[r_idx]); 1010 /* disable interrupts on this vector only */ 1011 IXGBE_WRITE_REG(hw, IXGBE_VTEIMC, rx_ring->v_idx); 1012 napi_schedule(&q_vector->napi); 1013 1014 1015 return IRQ_HANDLED; 1016} 1017 1018static irqreturn_t ixgbevf_msix_clean_many(int irq, void *data) 1019{ 1020 ixgbevf_msix_clean_rx(irq, data); 1021 ixgbevf_msix_clean_tx(irq, data); 1022 1023 return IRQ_HANDLED; 1024} 1025 1026static inline void map_vector_to_rxq(struct ixgbevf_adapter *a, int v_idx, 1027 int r_idx) 1028{ 1029 struct ixgbevf_q_vector *q_vector = a->q_vector[v_idx]; 1030 1031 set_bit(r_idx, q_vector->rxr_idx); 1032 q_vector->rxr_count++; 1033 a->rx_ring[r_idx].v_idx = 1 << v_idx; 1034} 1035 1036static inline void map_vector_to_txq(struct ixgbevf_adapter *a, int v_idx, 1037 int t_idx) 1038{ 1039 struct ixgbevf_q_vector *q_vector = a->q_vector[v_idx]; 1040 1041 set_bit(t_idx, q_vector->txr_idx); 1042 q_vector->txr_count++; 1043 a->tx_ring[t_idx].v_idx = 1 << v_idx; 1044} 1045 1046/** 1047 * ixgbevf_map_rings_to_vectors - Maps descriptor rings to vectors 1048 * @adapter: board private structure to initialize 1049 * 1050 * This function maps descriptor rings to the queue-specific vectors 1051 * we were allotted through the MSI-X enabling code. Ideally, we'd have 1052 * one vector per ring/queue, but on a constrained vector budget, we 1053 * group the rings as "efficiently" as possible. You would add new 1054 * mapping configurations in here. 1055 **/ 1056static int ixgbevf_map_rings_to_vectors(struct ixgbevf_adapter *adapter) 1057{ 1058 int q_vectors; 1059 int v_start = 0; 1060 int rxr_idx = 0, txr_idx = 0; 1061 int rxr_remaining = adapter->num_rx_queues; 1062 int txr_remaining = adapter->num_tx_queues; 1063 int i, j; 1064 int rqpv, tqpv; 1065 int err = 0; 1066 1067 q_vectors = adapter->num_msix_vectors - NON_Q_VECTORS; 1068 1069 /* 1070 * The ideal configuration... 1071 * We have enough vectors to map one per queue. 1072 */ 1073 if (q_vectors == adapter->num_rx_queues + adapter->num_tx_queues) { 1074 for (; rxr_idx < rxr_remaining; v_start++, rxr_idx++) 1075 map_vector_to_rxq(adapter, v_start, rxr_idx); 1076 1077 for (; txr_idx < txr_remaining; v_start++, txr_idx++) 1078 map_vector_to_txq(adapter, v_start, txr_idx); 1079 goto out; 1080 } 1081 1082 /* 1083 * If we don't have enough vectors for a 1-to-1 1084 * mapping, we'll have to group them so there are 1085 * multiple queues per vector. 1086 */ 1087 /* Re-adjusting *qpv takes care of the remainder. */ 1088 for (i = v_start; i < q_vectors; i++) { 1089 rqpv = DIV_ROUND_UP(rxr_remaining, q_vectors - i); 1090 for (j = 0; j < rqpv; j++) { 1091 map_vector_to_rxq(adapter, i, rxr_idx); 1092 rxr_idx++; 1093 rxr_remaining--; 1094 } 1095 } 1096 for (i = v_start; i < q_vectors; i++) { 1097 tqpv = DIV_ROUND_UP(txr_remaining, q_vectors - i); 1098 for (j = 0; j < tqpv; j++) { 1099 map_vector_to_txq(adapter, i, txr_idx); 1100 txr_idx++; 1101 txr_remaining--; 1102 } 1103 } 1104 1105out: 1106 return err; 1107} 1108 1109/** 1110 * ixgbevf_request_msix_irqs - Initialize MSI-X interrupts 1111 * @adapter: board private structure 1112 * 1113 * ixgbevf_request_msix_irqs allocates MSI-X vectors and requests 1114 * interrupts from the kernel. 1115 **/ 1116static int ixgbevf_request_msix_irqs(struct ixgbevf_adapter *adapter) 1117{ 1118 struct net_device *netdev = adapter->netdev; 1119 irqreturn_t (*handler)(int, void *); 1120 int i, vector, q_vectors, err; 1121 int ri = 0, ti = 0; 1122 1123 /* Decrement for Other and TCP Timer vectors */ 1124 q_vectors = adapter->num_msix_vectors - NON_Q_VECTORS; 1125 1126#define SET_HANDLER(_v) (((_v)->rxr_count && (_v)->txr_count) \ 1127 ? &ixgbevf_msix_clean_many : \ 1128 (_v)->rxr_count ? &ixgbevf_msix_clean_rx : \ 1129 (_v)->txr_count ? &ixgbevf_msix_clean_tx : \ 1130 NULL) 1131 for (vector = 0; vector < q_vectors; vector++) { 1132 handler = SET_HANDLER(adapter->q_vector[vector]); 1133 1134 if (handler == &ixgbevf_msix_clean_rx) { 1135 sprintf(adapter->name[vector], "%s-%s-%d", 1136 netdev->name, "rx", ri++); 1137 } else if (handler == &ixgbevf_msix_clean_tx) { 1138 sprintf(adapter->name[vector], "%s-%s-%d", 1139 netdev->name, "tx", ti++); 1140 } else if (handler == &ixgbevf_msix_clean_many) { 1141 sprintf(adapter->name[vector], "%s-%s-%d", 1142 netdev->name, "TxRx", vector); 1143 } else { 1144 /* skip this unused q_vector */ 1145 continue; 1146 } 1147 err = request_irq(adapter->msix_entries[vector].vector, 1148 handler, 0, adapter->name[vector], 1149 adapter->q_vector[vector]); 1150 if (err) { 1151 hw_dbg(&adapter->hw, 1152 "request_irq failed for MSIX interrupt " 1153 "Error: %d\n", err); 1154 goto free_queue_irqs; 1155 } 1156 } 1157 1158 sprintf(adapter->name[vector], "%s:mbx", netdev->name); 1159 err = request_irq(adapter->msix_entries[vector].vector, 1160 &ixgbevf_msix_mbx, 0, adapter->name[vector], netdev); 1161 if (err) { 1162 hw_dbg(&adapter->hw, 1163 "request_irq for msix_mbx failed: %d\n", err); 1164 goto free_queue_irqs; 1165 } 1166 1167 return 0; 1168 1169free_queue_irqs: 1170 for (i = vector - 1; i >= 0; i--) 1171 free_irq(adapter->msix_entries[--vector].vector, 1172 &(adapter->q_vector[i])); 1173 pci_disable_msix(adapter->pdev); 1174 kfree(adapter->msix_entries); 1175 adapter->msix_entries = NULL; 1176 return err; 1177} 1178 1179static inline void ixgbevf_reset_q_vectors(struct ixgbevf_adapter *adapter) 1180{ 1181 int i, q_vectors = adapter->num_msix_vectors - NON_Q_VECTORS; 1182 1183 for (i = 0; i < q_vectors; i++) { 1184 struct ixgbevf_q_vector *q_vector = adapter->q_vector[i]; 1185 bitmap_zero(q_vector->rxr_idx, MAX_RX_QUEUES); 1186 bitmap_zero(q_vector->txr_idx, MAX_TX_QUEUES); 1187 q_vector->rxr_count = 0; 1188 q_vector->txr_count = 0; 1189 q_vector->eitr = adapter->eitr_param; 1190 } 1191} 1192 1193/** 1194 * ixgbevf_request_irq - initialize interrupts 1195 * @adapter: board private structure 1196 * 1197 * Attempts to configure interrupts using the best available 1198 * capabilities of the hardware and kernel. 1199 **/ 1200static int ixgbevf_request_irq(struct ixgbevf_adapter *adapter) 1201{ 1202 int err = 0; 1203 1204 err = ixgbevf_request_msix_irqs(adapter); 1205 1206 if (err) 1207 hw_dbg(&adapter->hw, 1208 "request_irq failed, Error %d\n", err); 1209 1210 return err; 1211} 1212 1213static void ixgbevf_free_irq(struct ixgbevf_adapter *adapter) 1214{ 1215 struct net_device *netdev = adapter->netdev; 1216 int i, q_vectors; 1217 1218 q_vectors = adapter->num_msix_vectors; 1219 1220 i = q_vectors - 1; 1221 1222 free_irq(adapter->msix_entries[i].vector, netdev); 1223 i--; 1224 1225 for (; i >= 0; i--) { 1226 free_irq(adapter->msix_entries[i].vector, 1227 adapter->q_vector[i]); 1228 } 1229 1230 ixgbevf_reset_q_vectors(adapter); 1231} 1232 1233/** 1234 * ixgbevf_irq_disable - Mask off interrupt generation on the NIC 1235 * @adapter: board private structure 1236 **/ 1237static inline void ixgbevf_irq_disable(struct ixgbevf_adapter *adapter) 1238{ 1239 int i; 1240 struct ixgbe_hw *hw = &adapter->hw; 1241 1242 IXGBE_WRITE_REG(hw, IXGBE_VTEIMC, ~0); 1243 1244 IXGBE_WRITE_FLUSH(hw); 1245 1246 for (i = 0; i < adapter->num_msix_vectors; i++) 1247 synchronize_irq(adapter->msix_entries[i].vector); 1248} 1249 1250/** 1251 * ixgbevf_irq_enable - Enable default interrupt generation settings 1252 * @adapter: board private structure 1253 **/ 1254static inline void ixgbevf_irq_enable(struct ixgbevf_adapter *adapter, 1255 bool queues, bool flush) 1256{ 1257 struct ixgbe_hw *hw = &adapter->hw; 1258 u32 mask; 1259 u64 qmask; 1260 1261 mask = (IXGBE_EIMS_ENABLE_MASK & ~IXGBE_EIMS_RTX_QUEUE); 1262 qmask = ~0; 1263 1264 IXGBE_WRITE_REG(hw, IXGBE_VTEIMS, mask); 1265 1266 if (queues) 1267 ixgbevf_irq_enable_queues(adapter, qmask); 1268 1269 if (flush) 1270 IXGBE_WRITE_FLUSH(hw); 1271} 1272 1273/** 1274 * ixgbevf_configure_tx - Configure 82599 VF Transmit Unit after Reset 1275 * @adapter: board private structure 1276 * 1277 * Configure the Tx unit of the MAC after a reset. 1278 **/ 1279static void ixgbevf_configure_tx(struct ixgbevf_adapter *adapter) 1280{ 1281 u64 tdba; 1282 struct ixgbe_hw *hw = &adapter->hw; 1283 u32 i, j, tdlen, txctrl; 1284 1285 /* Setup the HW Tx Head and Tail descriptor pointers */ 1286 for (i = 0; i < adapter->num_tx_queues; i++) { 1287 struct ixgbevf_ring *ring = &adapter->tx_ring[i]; 1288 j = ring->reg_idx; 1289 tdba = ring->dma; 1290 tdlen = ring->count * sizeof(union ixgbe_adv_tx_desc); 1291 IXGBE_WRITE_REG(hw, IXGBE_VFTDBAL(j), 1292 (tdba & DMA_BIT_MASK(32))); 1293 IXGBE_WRITE_REG(hw, IXGBE_VFTDBAH(j), (tdba >> 32)); 1294 IXGBE_WRITE_REG(hw, IXGBE_VFTDLEN(j), tdlen); 1295 IXGBE_WRITE_REG(hw, IXGBE_VFTDH(j), 0); 1296 IXGBE_WRITE_REG(hw, IXGBE_VFTDT(j), 0); 1297 adapter->tx_ring[i].head = IXGBE_VFTDH(j); 1298 adapter->tx_ring[i].tail = IXGBE_VFTDT(j); 1299 /* Disable Tx Head Writeback RO bit, since this hoses 1300 * bookkeeping if things aren't delivered in order. 1301 */ 1302 txctrl = IXGBE_READ_REG(hw, IXGBE_VFDCA_TXCTRL(j)); 1303 txctrl &= ~IXGBE_DCA_TXCTRL_TX_WB_RO_EN; 1304 IXGBE_WRITE_REG(hw, IXGBE_VFDCA_TXCTRL(j), txctrl); 1305 } 1306} 1307 1308#define IXGBE_SRRCTL_BSIZEHDRSIZE_SHIFT 2 1309 1310static void ixgbevf_configure_srrctl(struct ixgbevf_adapter *adapter, int index) 1311{ 1312 struct ixgbevf_ring *rx_ring; 1313 struct ixgbe_hw *hw = &adapter->hw; 1314 u32 srrctl; 1315 1316 rx_ring = &adapter->rx_ring[index]; 1317 1318 srrctl = IXGBE_SRRCTL_DROP_EN; 1319 1320 if (adapter->flags & IXGBE_FLAG_RX_PS_ENABLED) { 1321 u16 bufsz = IXGBEVF_RXBUFFER_2048; 1322 /* grow the amount we can receive on large page machines */ 1323 if (bufsz < (PAGE_SIZE / 2)) 1324 bufsz = (PAGE_SIZE / 2); 1325 /* cap the bufsz at our largest descriptor size */ 1326 bufsz = min((u16)IXGBEVF_MAX_RXBUFFER, bufsz); 1327 1328 srrctl |= bufsz >> IXGBE_SRRCTL_BSIZEPKT_SHIFT; 1329 srrctl |= IXGBE_SRRCTL_DESCTYPE_HDR_SPLIT_ALWAYS; 1330 srrctl |= ((IXGBEVF_RX_HDR_SIZE << 1331 IXGBE_SRRCTL_BSIZEHDRSIZE_SHIFT) & 1332 IXGBE_SRRCTL_BSIZEHDR_MASK); 1333 } else { 1334 srrctl |= IXGBE_SRRCTL_DESCTYPE_ADV_ONEBUF; 1335 1336 if (rx_ring->rx_buf_len == MAXIMUM_ETHERNET_VLAN_SIZE) 1337 srrctl |= IXGBEVF_RXBUFFER_2048 >> 1338 IXGBE_SRRCTL_BSIZEPKT_SHIFT; 1339 else 1340 srrctl |= rx_ring->rx_buf_len >> 1341 IXGBE_SRRCTL_BSIZEPKT_SHIFT; 1342 } 1343 IXGBE_WRITE_REG(hw, IXGBE_VFSRRCTL(index), srrctl); 1344} 1345 1346/** 1347 * ixgbevf_configure_rx - Configure 82599 VF Receive Unit after Reset 1348 * @adapter: board private structure 1349 * 1350 * Configure the Rx unit of the MAC after a reset. 1351 **/ 1352static void ixgbevf_configure_rx(struct ixgbevf_adapter *adapter) 1353{ 1354 u64 rdba; 1355 struct ixgbe_hw *hw = &adapter->hw; 1356 struct net_device *netdev = adapter->netdev; 1357 int max_frame = netdev->mtu + ETH_HLEN + ETH_FCS_LEN; 1358 int i, j; 1359 u32 rdlen; 1360 int rx_buf_len; 1361 1362 /* Decide whether to use packet split mode or not */ 1363 if (netdev->mtu > ETH_DATA_LEN) { 1364 if (adapter->flags & IXGBE_FLAG_RX_PS_CAPABLE) 1365 adapter->flags |= IXGBE_FLAG_RX_PS_ENABLED; 1366 else 1367 adapter->flags &= ~IXGBE_FLAG_RX_PS_ENABLED; 1368 } else { 1369 if (adapter->flags & IXGBE_FLAG_RX_1BUF_CAPABLE) 1370 adapter->flags &= ~IXGBE_FLAG_RX_PS_ENABLED; 1371 else 1372 adapter->flags |= IXGBE_FLAG_RX_PS_ENABLED; 1373 } 1374 1375 /* Set the RX buffer length according to the mode */ 1376 if (adapter->flags & IXGBE_FLAG_RX_PS_ENABLED) { 1377 /* PSRTYPE must be initialized in 82599 */ 1378 u32 psrtype = IXGBE_PSRTYPE_TCPHDR | 1379 IXGBE_PSRTYPE_UDPHDR | 1380 IXGBE_PSRTYPE_IPV4HDR | 1381 IXGBE_PSRTYPE_IPV6HDR | 1382 IXGBE_PSRTYPE_L2HDR; 1383 IXGBE_WRITE_REG(hw, IXGBE_VFPSRTYPE, psrtype); 1384 rx_buf_len = IXGBEVF_RX_HDR_SIZE; 1385 } else { 1386 IXGBE_WRITE_REG(hw, IXGBE_VFPSRTYPE, 0); 1387 if (netdev->mtu <= ETH_DATA_LEN) 1388 rx_buf_len = MAXIMUM_ETHERNET_VLAN_SIZE; 1389 else 1390 rx_buf_len = ALIGN(max_frame, 1024); 1391 } 1392 1393 rdlen = adapter->rx_ring[0].count * sizeof(union ixgbe_adv_rx_desc); 1394 /* Setup the HW Rx Head and Tail Descriptor Pointers and 1395 * the Base and Length of the Rx Descriptor Ring */ 1396 for (i = 0; i < adapter->num_rx_queues; i++) { 1397 rdba = adapter->rx_ring[i].dma; 1398 j = adapter->rx_ring[i].reg_idx; 1399 IXGBE_WRITE_REG(hw, IXGBE_VFRDBAL(j), 1400 (rdba & DMA_BIT_MASK(32))); 1401 IXGBE_WRITE_REG(hw, IXGBE_VFRDBAH(j), (rdba >> 32)); 1402 IXGBE_WRITE_REG(hw, IXGBE_VFRDLEN(j), rdlen); 1403 IXGBE_WRITE_REG(hw, IXGBE_VFRDH(j), 0); 1404 IXGBE_WRITE_REG(hw, IXGBE_VFRDT(j), 0); 1405 adapter->rx_ring[i].head = IXGBE_VFRDH(j); 1406 adapter->rx_ring[i].tail = IXGBE_VFRDT(j); 1407 adapter->rx_ring[i].rx_buf_len = rx_buf_len; 1408 1409 ixgbevf_configure_srrctl(adapter, j); 1410 } 1411} 1412 1413static int ixgbevf_vlan_rx_add_vid(struct net_device *netdev, u16 vid) 1414{ 1415 struct ixgbevf_adapter *adapter = netdev_priv(netdev); 1416 struct ixgbe_hw *hw = &adapter->hw; 1417 1418 /* add VID to filter table */ 1419 if (hw->mac.ops.set_vfta) 1420 hw->mac.ops.set_vfta(hw, vid, 0, true); 1421 set_bit(vid, adapter->active_vlans); 1422 1423 return 0; 1424} 1425 1426static int ixgbevf_vlan_rx_kill_vid(struct net_device *netdev, u16 vid) 1427{ 1428 struct ixgbevf_adapter *adapter = netdev_priv(netdev); 1429 struct ixgbe_hw *hw = &adapter->hw; 1430 1431 /* remove VID from filter table */ 1432 if (hw->mac.ops.set_vfta) 1433 hw->mac.ops.set_vfta(hw, vid, 0, false); 1434 clear_bit(vid, adapter->active_vlans); 1435 1436 return 0; 1437} 1438 1439static void ixgbevf_restore_vlan(struct ixgbevf_adapter *adapter) 1440{ 1441 u16 vid; 1442 1443 for_each_set_bit(vid, adapter->active_vlans, VLAN_N_VID) 1444 ixgbevf_vlan_rx_add_vid(adapter->netdev, vid); 1445} 1446 1447static int ixgbevf_write_uc_addr_list(struct net_device *netdev) 1448{ 1449 struct ixgbevf_adapter *adapter = netdev_priv(netdev); 1450 struct ixgbe_hw *hw = &adapter->hw; 1451 int count = 0; 1452 1453 if ((netdev_uc_count(netdev)) > 10) { 1454 pr_err("Too many unicast filters - No Space\n"); 1455 return -ENOSPC; 1456 } 1457 1458 if (!netdev_uc_empty(netdev)) { 1459 struct netdev_hw_addr *ha; 1460 netdev_for_each_uc_addr(ha, netdev) { 1461 hw->mac.ops.set_uc_addr(hw, ++count, ha->addr); 1462 udelay(200); 1463 } 1464 } else { 1465 /* 1466 * If the list is empty then send message to PF driver to 1467 * clear all macvlans on this VF. 1468 */ 1469 hw->mac.ops.set_uc_addr(hw, 0, NULL); 1470 } 1471 1472 return count; 1473} 1474 1475/** 1476 * ixgbevf_set_rx_mode - Multicast set 1477 * @netdev: network interface device structure 1478 * 1479 * The set_rx_method entry point is called whenever the multicast address 1480 * list or the network interface flags are updated. This routine is 1481 * responsible for configuring the hardware for proper multicast mode. 1482 **/ 1483static void ixgbevf_set_rx_mode(struct net_device *netdev) 1484{ 1485 struct ixgbevf_adapter *adapter = netdev_priv(netdev); 1486 struct ixgbe_hw *hw = &adapter->hw; 1487 1488 /* reprogram multicast list */ 1489 if (hw->mac.ops.update_mc_addr_list) 1490 hw->mac.ops.update_mc_addr_list(hw, netdev); 1491 1492 ixgbevf_write_uc_addr_list(netdev); 1493} 1494 1495static void ixgbevf_napi_enable_all(struct ixgbevf_adapter *adapter) 1496{ 1497 int q_idx; 1498 struct ixgbevf_q_vector *q_vector; 1499 int q_vectors = adapter->num_msix_vectors - NON_Q_VECTORS; 1500 1501 for (q_idx = 0; q_idx < q_vectors; q_idx++) { 1502 struct napi_struct *napi; 1503 q_vector = adapter->q_vector[q_idx]; 1504 if (!q_vector->rxr_count) 1505 continue; 1506 napi = &q_vector->napi; 1507 if (q_vector->rxr_count > 1) 1508 napi->poll = &ixgbevf_clean_rxonly_many; 1509 1510 napi_enable(napi); 1511 } 1512} 1513 1514static void ixgbevf_napi_disable_all(struct ixgbevf_adapter *adapter) 1515{ 1516 int q_idx; 1517 struct ixgbevf_q_vector *q_vector; 1518 int q_vectors = adapter->num_msix_vectors - NON_Q_VECTORS; 1519 1520 for (q_idx = 0; q_idx < q_vectors; q_idx++) { 1521 q_vector = adapter->q_vector[q_idx]; 1522 if (!q_vector->rxr_count) 1523 continue; 1524 napi_disable(&q_vector->napi); 1525 } 1526} 1527 1528static void ixgbevf_configure(struct ixgbevf_adapter *adapter) 1529{ 1530 struct net_device *netdev = adapter->netdev; 1531 int i; 1532 1533 ixgbevf_set_rx_mode(netdev); 1534 1535 ixgbevf_restore_vlan(adapter); 1536 1537 ixgbevf_configure_tx(adapter); 1538 ixgbevf_configure_rx(adapter); 1539 for (i = 0; i < adapter->num_rx_queues; i++) { 1540 struct ixgbevf_ring *ring = &adapter->rx_ring[i]; 1541 ixgbevf_alloc_rx_buffers(adapter, ring, ring->count); 1542 ring->next_to_use = ring->count - 1; 1543 writel(ring->next_to_use, adapter->hw.hw_addr + ring->tail); 1544 } 1545} 1546 1547#define IXGBE_MAX_RX_DESC_POLL 10 1548static inline void ixgbevf_rx_desc_queue_enable(struct ixgbevf_adapter *adapter, 1549 int rxr) 1550{ 1551 struct ixgbe_hw *hw = &adapter->hw; 1552 int j = adapter->rx_ring[rxr].reg_idx; 1553 int k; 1554 1555 for (k = 0; k < IXGBE_MAX_RX_DESC_POLL; k++) { 1556 if (IXGBE_READ_REG(hw, IXGBE_VFRXDCTL(j)) & IXGBE_RXDCTL_ENABLE) 1557 break; 1558 else 1559 msleep(1); 1560 } 1561 if (k >= IXGBE_MAX_RX_DESC_POLL) { 1562 hw_dbg(hw, "RXDCTL.ENABLE on Rx queue %d " 1563 "not set within the polling period\n", rxr); 1564 } 1565 1566 ixgbevf_release_rx_desc(&adapter->hw, &adapter->rx_ring[rxr], 1567 (adapter->rx_ring[rxr].count - 1)); 1568} 1569 1570static void ixgbevf_save_reset_stats(struct ixgbevf_adapter *adapter) 1571{ 1572 /* Only save pre-reset stats if there are some */ 1573 if (adapter->stats.vfgprc || adapter->stats.vfgptc) { 1574 adapter->stats.saved_reset_vfgprc += adapter->stats.vfgprc - 1575 adapter->stats.base_vfgprc; 1576 adapter->stats.saved_reset_vfgptc += adapter->stats.vfgptc - 1577 adapter->stats.base_vfgptc; 1578 adapter->stats.saved_reset_vfgorc += adapter->stats.vfgorc - 1579 adapter->stats.base_vfgorc; 1580 adapter->stats.saved_reset_vfgotc += adapter->stats.vfgotc - 1581 adapter->stats.base_vfgotc; 1582 adapter->stats.saved_reset_vfmprc += adapter->stats.vfmprc - 1583 adapter->stats.base_vfmprc; 1584 } 1585} 1586 1587static void ixgbevf_init_last_counter_stats(struct ixgbevf_adapter *adapter) 1588{ 1589 struct ixgbe_hw *hw = &adapter->hw; 1590 1591 adapter->stats.last_vfgprc = IXGBE_READ_REG(hw, IXGBE_VFGPRC); 1592 adapter->stats.last_vfgorc = IXGBE_READ_REG(hw, IXGBE_VFGORC_LSB); 1593 adapter->stats.last_vfgorc |= 1594 (((u64)(IXGBE_READ_REG(hw, IXGBE_VFGORC_MSB))) << 32); 1595 adapter->stats.last_vfgptc = IXGBE_READ_REG(hw, IXGBE_VFGPTC); 1596 adapter->stats.last_vfgotc = IXGBE_READ_REG(hw, IXGBE_VFGOTC_LSB); 1597 adapter->stats.last_vfgotc |= 1598 (((u64)(IXGBE_READ_REG(hw, IXGBE_VFGOTC_MSB))) << 32); 1599 adapter->stats.last_vfmprc = IXGBE_READ_REG(hw, IXGBE_VFMPRC); 1600 1601 adapter->stats.base_vfgprc = adapter->stats.last_vfgprc; 1602 adapter->stats.base_vfgorc = adapter->stats.last_vfgorc; 1603 adapter->stats.base_vfgptc = adapter->stats.last_vfgptc; 1604 adapter->stats.base_vfgotc = adapter->stats.last_vfgotc; 1605 adapter->stats.base_vfmprc = adapter->stats.last_vfmprc; 1606} 1607 1608static int ixgbevf_up_complete(struct ixgbevf_adapter *adapter) 1609{ 1610 struct net_device *netdev = adapter->netdev; 1611 struct ixgbe_hw *hw = &adapter->hw; 1612 int i, j = 0; 1613 int num_rx_rings = adapter->num_rx_queues; 1614 u32 txdctl, rxdctl; 1615 1616 for (i = 0; i < adapter->num_tx_queues; i++) { 1617 j = adapter->tx_ring[i].reg_idx; 1618 txdctl = IXGBE_READ_REG(hw, IXGBE_VFTXDCTL(j)); 1619 /* enable WTHRESH=8 descriptors, to encourage burst writeback */ 1620 txdctl |= (8 << 16); 1621 IXGBE_WRITE_REG(hw, IXGBE_VFTXDCTL(j), txdctl); 1622 } 1623 1624 for (i = 0; i < adapter->num_tx_queues; i++) { 1625 j = adapter->tx_ring[i].reg_idx; 1626 txdctl = IXGBE_READ_REG(hw, IXGBE_VFTXDCTL(j)); 1627 txdctl |= IXGBE_TXDCTL_ENABLE; 1628 IXGBE_WRITE_REG(hw, IXGBE_VFTXDCTL(j), txdctl); 1629 } 1630 1631 for (i = 0; i < num_rx_rings; i++) { 1632 j = adapter->rx_ring[i].reg_idx; 1633 rxdctl = IXGBE_READ_REG(hw, IXGBE_VFRXDCTL(j)); 1634 rxdctl |= IXGBE_RXDCTL_ENABLE | IXGBE_RXDCTL_VME; 1635 if (hw->mac.type == ixgbe_mac_X540_vf) { 1636 rxdctl &= ~IXGBE_RXDCTL_RLPMLMASK; 1637 rxdctl |= ((netdev->mtu + ETH_HLEN + ETH_FCS_LEN) | 1638 IXGBE_RXDCTL_RLPML_EN); 1639 } 1640 IXGBE_WRITE_REG(hw, IXGBE_VFRXDCTL(j), rxdctl); 1641 ixgbevf_rx_desc_queue_enable(adapter, i); 1642 } 1643 1644 ixgbevf_configure_msix(adapter); 1645 1646 if (hw->mac.ops.set_rar) { 1647 if (is_valid_ether_addr(hw->mac.addr)) 1648 hw->mac.ops.set_rar(hw, 0, hw->mac.addr, 0); 1649 else 1650 hw->mac.ops.set_rar(hw, 0, hw->mac.perm_addr, 0); 1651 } 1652 1653 clear_bit(__IXGBEVF_DOWN, &adapter->state); 1654 ixgbevf_napi_enable_all(adapter); 1655 1656 /* enable transmits */ 1657 netif_tx_start_all_queues(netdev); 1658 1659 ixgbevf_save_reset_stats(adapter); 1660 ixgbevf_init_last_counter_stats(adapter); 1661 1662 /* bring the link up in the watchdog, this could race with our first 1663 * link up interrupt but shouldn't be a problem */ 1664 adapter->flags |= IXGBE_FLAG_NEED_LINK_UPDATE; 1665 adapter->link_check_timeout = jiffies; 1666 mod_timer(&adapter->watchdog_timer, jiffies); 1667 return 0; 1668} 1669 1670int ixgbevf_up(struct ixgbevf_adapter *adapter) 1671{ 1672 int err; 1673 struct ixgbe_hw *hw = &adapter->hw; 1674 1675 ixgbevf_configure(adapter); 1676 1677 err = ixgbevf_up_complete(adapter); 1678 1679 /* clear any pending interrupts, may auto mask */ 1680 IXGBE_READ_REG(hw, IXGBE_VTEICR); 1681 1682 ixgbevf_irq_enable(adapter, true, true); 1683 1684 return err; 1685} 1686 1687/** 1688 * ixgbevf_clean_rx_ring - Free Rx Buffers per Queue 1689 * @adapter: board private structure 1690 * @rx_ring: ring to free buffers from 1691 **/ 1692static void ixgbevf_clean_rx_ring(struct ixgbevf_adapter *adapter, 1693 struct ixgbevf_ring *rx_ring) 1694{ 1695 struct pci_dev *pdev = adapter->pdev; 1696 unsigned long size; 1697 unsigned int i; 1698 1699 if (!rx_ring->rx_buffer_info) 1700 return; 1701 1702 /* Free all the Rx ring sk_buffs */ 1703 for (i = 0; i < rx_ring->count; i++) { 1704 struct ixgbevf_rx_buffer *rx_buffer_info; 1705 1706 rx_buffer_info = &rx_ring->rx_buffer_info[i]; 1707 if (rx_buffer_info->dma) { 1708 dma_unmap_single(&pdev->dev, rx_buffer_info->dma, 1709 rx_ring->rx_buf_len, 1710 DMA_FROM_DEVICE); 1711 rx_buffer_info->dma = 0; 1712 } 1713 if (rx_buffer_info->skb) { 1714 struct sk_buff *skb = rx_buffer_info->skb; 1715 rx_buffer_info->skb = NULL; 1716 do { 1717 struct sk_buff *this = skb; 1718 skb = skb->prev; 1719 dev_kfree_skb(this); 1720 } while (skb); 1721 } 1722 if (!rx_buffer_info->page) 1723 continue; 1724 dma_unmap_page(&pdev->dev, rx_buffer_info->page_dma, 1725 PAGE_SIZE / 2, DMA_FROM_DEVICE); 1726 rx_buffer_info->page_dma = 0; 1727 put_page(rx_buffer_info->page); 1728 rx_buffer_info->page = NULL; 1729 rx_buffer_info->page_offset = 0; 1730 } 1731 1732 size = sizeof(struct ixgbevf_rx_buffer) * rx_ring->count; 1733 memset(rx_ring->rx_buffer_info, 0, size); 1734 1735 /* Zero out the descriptor ring */ 1736 memset(rx_ring->desc, 0, rx_ring->size); 1737 1738 rx_ring->next_to_clean = 0; 1739 rx_ring->next_to_use = 0; 1740 1741 if (rx_ring->head) 1742 writel(0, adapter->hw.hw_addr + rx_ring->head); 1743 if (rx_ring->tail) 1744 writel(0, adapter->hw.hw_addr + rx_ring->tail); 1745} 1746 1747/** 1748 * ixgbevf_clean_tx_ring - Free Tx Buffers 1749 * @adapter: board private structure 1750 * @tx_ring: ring to be cleaned 1751 **/ 1752static void ixgbevf_clean_tx_ring(struct ixgbevf_adapter *adapter, 1753 struct ixgbevf_ring *tx_ring) 1754{ 1755 struct ixgbevf_tx_buffer *tx_buffer_info; 1756 unsigned long size; 1757 unsigned int i; 1758 1759 if (!tx_ring->tx_buffer_info) 1760 return; 1761 1762 /* Free all the Tx ring sk_buffs */ 1763 1764 for (i = 0; i < tx_ring->count; i++) { 1765 tx_buffer_info = &tx_ring->tx_buffer_info[i]; 1766 ixgbevf_unmap_and_free_tx_resource(adapter, tx_buffer_info); 1767 } 1768 1769 size = sizeof(struct ixgbevf_tx_buffer) * tx_ring->count; 1770 memset(tx_ring->tx_buffer_info, 0, size); 1771 1772 memset(tx_ring->desc, 0, tx_ring->size); 1773 1774 tx_ring->next_to_use = 0; 1775 tx_ring->next_to_clean = 0; 1776 1777 if (tx_ring->head) 1778 writel(0, adapter->hw.hw_addr + tx_ring->head); 1779 if (tx_ring->tail) 1780 writel(0, adapter->hw.hw_addr + tx_ring->tail); 1781} 1782 1783/** 1784 * ixgbevf_clean_all_rx_rings - Free Rx Buffers for all queues 1785 * @adapter: board private structure 1786 **/ 1787static void ixgbevf_clean_all_rx_rings(struct ixgbevf_adapter *adapter) 1788{ 1789 int i; 1790 1791 for (i = 0; i < adapter->num_rx_queues; i++) 1792 ixgbevf_clean_rx_ring(adapter, &adapter->rx_ring[i]); 1793} 1794 1795/** 1796 * ixgbevf_clean_all_tx_rings - Free Tx Buffers for all queues 1797 * @adapter: board private structure 1798 **/ 1799static void ixgbevf_clean_all_tx_rings(struct ixgbevf_adapter *adapter) 1800{ 1801 int i; 1802 1803 for (i = 0; i < adapter->num_tx_queues; i++) 1804 ixgbevf_clean_tx_ring(adapter, &adapter->tx_ring[i]); 1805} 1806 1807void ixgbevf_down(struct ixgbevf_adapter *adapter) 1808{ 1809 struct net_device *netdev = adapter->netdev; 1810 struct ixgbe_hw *hw = &adapter->hw; 1811 u32 txdctl; 1812 int i, j; 1813 1814 /* signal that we are down to the interrupt handler */ 1815 set_bit(__IXGBEVF_DOWN, &adapter->state); 1816 /* disable receives */ 1817 1818 netif_tx_disable(netdev); 1819 1820 msleep(10); 1821 1822 netif_tx_stop_all_queues(netdev); 1823 1824 ixgbevf_irq_disable(adapter); 1825 1826 ixgbevf_napi_disable_all(adapter); 1827 1828 del_timer_sync(&adapter->watchdog_timer); 1829 /* can't call flush scheduled work here because it can deadlock 1830 * if linkwatch_event tries to acquire the rtnl_lock which we are 1831 * holding */ 1832 while (adapter->flags & IXGBE_FLAG_IN_WATCHDOG_TASK) 1833 msleep(1); 1834 1835 /* disable transmits in the hardware now that interrupts are off */ 1836 for (i = 0; i < adapter->num_tx_queues; i++) { 1837 j = adapter->tx_ring[i].reg_idx; 1838 txdctl = IXGBE_READ_REG(hw, IXGBE_VFTXDCTL(j)); 1839 IXGBE_WRITE_REG(hw, IXGBE_VFTXDCTL(j), 1840 (txdctl & ~IXGBE_TXDCTL_ENABLE)); 1841 } 1842 1843 netif_carrier_off(netdev); 1844 1845 if (!pci_channel_offline(adapter->pdev)) 1846 ixgbevf_reset(adapter); 1847 1848 ixgbevf_clean_all_tx_rings(adapter); 1849 ixgbevf_clean_all_rx_rings(adapter); 1850} 1851 1852void ixgbevf_reinit_locked(struct ixgbevf_adapter *adapter) 1853{ 1854 struct ixgbe_hw *hw = &adapter->hw; 1855 1856 WARN_ON(in_interrupt()); 1857 1858 while (test_and_set_bit(__IXGBEVF_RESETTING, &adapter->state)) 1859 msleep(1); 1860 1861 /* 1862 * Check if PF is up before re-init. If not then skip until 1863 * later when the PF is up and ready to service requests from 1864 * the VF via mailbox. If the VF is up and running then the 1865 * watchdog task will continue to schedule reset tasks until 1866 * the PF is up and running. 1867 */ 1868 if (!hw->mac.ops.reset_hw(hw)) { 1869 ixgbevf_down(adapter); 1870 ixgbevf_up(adapter); 1871 } 1872 1873 clear_bit(__IXGBEVF_RESETTING, &adapter->state); 1874} 1875 1876void ixgbevf_reset(struct ixgbevf_adapter *adapter) 1877{ 1878 struct ixgbe_hw *hw = &adapter->hw; 1879 struct net_device *netdev = adapter->netdev; 1880 1881 if (hw->mac.ops.reset_hw(hw)) 1882 hw_dbg(hw, "PF still resetting\n"); 1883 else 1884 hw->mac.ops.init_hw(hw); 1885 1886 if (is_valid_ether_addr(adapter->hw.mac.addr)) { 1887 memcpy(netdev->dev_addr, adapter->hw.mac.addr, 1888 netdev->addr_len); 1889 memcpy(netdev->perm_addr, adapter->hw.mac.addr, 1890 netdev->addr_len); 1891 } 1892} 1893 1894static void ixgbevf_acquire_msix_vectors(struct ixgbevf_adapter *adapter, 1895 int vectors) 1896{ 1897 int err, vector_threshold; 1898 1899 /* We'll want at least 3 (vector_threshold): 1900 * 1) TxQ[0] Cleanup 1901 * 2) RxQ[0] Cleanup 1902 * 3) Other (Link Status Change, etc.) 1903 */ 1904 vector_threshold = MIN_MSIX_COUNT; 1905 1906 /* The more we get, the more we will assign to Tx/Rx Cleanup 1907 * for the separate queues...where Rx Cleanup >= Tx Cleanup. 1908 * Right now, we simply care about how many we'll get; we'll 1909 * set them up later while requesting irq's. 1910 */ 1911 while (vectors >= vector_threshold) { 1912 err = pci_enable_msix(adapter->pdev, adapter->msix_entries, 1913 vectors); 1914 if (!err) /* Success in acquiring all requested vectors. */ 1915 break; 1916 else if (err < 0) 1917 vectors = 0; /* Nasty failure, quit now */ 1918 else /* err == number of vectors we should try again with */ 1919 vectors = err; 1920 } 1921 1922 if (vectors < vector_threshold) { 1923 /* Can't allocate enough MSI-X interrupts? Oh well. 1924 * This just means we'll go with either a single MSI 1925 * vector or fall back to legacy interrupts. 1926 */ 1927 hw_dbg(&adapter->hw, 1928 "Unable to allocate MSI-X interrupts\n"); 1929 kfree(adapter->msix_entries); 1930 adapter->msix_entries = NULL; 1931 } else { 1932 /* 1933 * Adjust for only the vectors we'll use, which is minimum 1934 * of max_msix_q_vectors + NON_Q_VECTORS, or the number of 1935 * vectors we were allocated. 1936 */ 1937 adapter->num_msix_vectors = vectors; 1938 } 1939} 1940 1941/* 1942 * ixgbevf_set_num_queues: Allocate queues for device, feature dependent 1943 * @adapter: board private structure to initialize 1944 * 1945 * This is the top level queue allocation routine. The order here is very 1946 * important, starting with the "most" number of features turned on at once, 1947 * and ending with the smallest set of features. This way large combinations 1948 * can be allocated if they're turned on, and smaller combinations are the 1949 * fallthrough conditions. 1950 * 1951 **/ 1952static void ixgbevf_set_num_queues(struct ixgbevf_adapter *adapter) 1953{ 1954 /* Start with base case */ 1955 adapter->num_rx_queues = 1; 1956 adapter->num_tx_queues = 1; 1957 adapter->num_rx_pools = adapter->num_rx_queues; 1958 adapter->num_rx_queues_per_pool = 1; 1959} 1960 1961/** 1962 * ixgbevf_alloc_queues - Allocate memory for all rings 1963 * @adapter: board private structure to initialize 1964 * 1965 * We allocate one ring per queue at run-time since we don't know the 1966 * number of queues at compile-time. The polling_netdev array is 1967 * intended for Multiqueue, but should work fine with a single queue. 1968 **/ 1969static int ixgbevf_alloc_queues(struct ixgbevf_adapter *adapter) 1970{ 1971 int i; 1972 1973 adapter->tx_ring = kcalloc(adapter->num_tx_queues, 1974 sizeof(struct ixgbevf_ring), GFP_KERNEL); 1975 if (!adapter->tx_ring) 1976 goto err_tx_ring_allocation; 1977 1978 adapter->rx_ring = kcalloc(adapter->num_rx_queues, 1979 sizeof(struct ixgbevf_ring), GFP_KERNEL); 1980 if (!adapter->rx_ring) 1981 goto err_rx_ring_allocation; 1982 1983 for (i = 0; i < adapter->num_tx_queues; i++) { 1984 adapter->tx_ring[i].count = adapter->tx_ring_count; 1985 adapter->tx_ring[i].queue_index = i; 1986 adapter->tx_ring[i].reg_idx = i; 1987 } 1988 1989 for (i = 0; i < adapter->num_rx_queues; i++) { 1990 adapter->rx_ring[i].count = adapter->rx_ring_count; 1991 adapter->rx_ring[i].queue_index = i; 1992 adapter->rx_ring[i].reg_idx = i; 1993 } 1994 1995 return 0; 1996 1997err_rx_ring_allocation: 1998 kfree(adapter->tx_ring); 1999err_tx_ring_allocation: 2000 return -ENOMEM; 2001} 2002 2003/** 2004 * ixgbevf_set_interrupt_capability - set MSI-X or FAIL if not supported 2005 * @adapter: board private structure to initialize 2006 * 2007 * Attempt to configure the interrupts using the best available 2008 * capabilities of the hardware and the kernel. 2009 **/ 2010static int ixgbevf_set_interrupt_capability(struct ixgbevf_adapter *adapter) 2011{ 2012 int err = 0; 2013 int vector, v_budget; 2014 2015 /* 2016 * It's easy to be greedy for MSI-X vectors, but it really 2017 * doesn't do us much good if we have a lot more vectors 2018 * than CPU's. So let's be conservative and only ask for 2019 * (roughly) twice the number of vectors as there are CPU's. 2020 */ 2021 v_budget = min(adapter->num_rx_queues + adapter->num_tx_queues, 2022 (int)(num_online_cpus() * 2)) + NON_Q_VECTORS; 2023 2024 /* A failure in MSI-X entry allocation isn't fatal, but it does 2025 * mean we disable MSI-X capabilities of the adapter. */ 2026 adapter->msix_entries = kcalloc(v_budget, 2027 sizeof(struct msix_entry), GFP_KERNEL); 2028 if (!adapter->msix_entries) { 2029 err = -ENOMEM; 2030 goto out; 2031 } 2032 2033 for (vector = 0; vector < v_budget; vector++) 2034 adapter->msix_entries[vector].entry = vector; 2035 2036 ixgbevf_acquire_msix_vectors(adapter, v_budget); 2037 2038out: 2039 return err; 2040} 2041 2042/** 2043 * ixgbevf_alloc_q_vectors - Allocate memory for interrupt vectors 2044 * @adapter: board private structure to initialize 2045 * 2046 * We allocate one q_vector per queue interrupt. If allocation fails we 2047 * return -ENOMEM. 2048 **/ 2049static int ixgbevf_alloc_q_vectors(struct ixgbevf_adapter *adapter) 2050{ 2051 int q_idx, num_q_vectors; 2052 struct ixgbevf_q_vector *q_vector; 2053 int napi_vectors; 2054 int (*poll)(struct napi_struct *, int); 2055 2056 num_q_vectors = adapter->num_msix_vectors - NON_Q_VECTORS; 2057 napi_vectors = adapter->num_rx_queues; 2058 poll = &ixgbevf_clean_rxonly; 2059 2060 for (q_idx = 0; q_idx < num_q_vectors; q_idx++) { 2061 q_vector = kzalloc(sizeof(struct ixgbevf_q_vector), GFP_KERNEL); 2062 if (!q_vector) 2063 goto err_out; 2064 q_vector->adapter = adapter; 2065 q_vector->v_idx = q_idx; 2066 q_vector->eitr = adapter->eitr_param; 2067 if (q_idx < napi_vectors) 2068 netif_napi_add(adapter->netdev, &q_vector->napi, 2069 (*poll), 64); 2070 adapter->q_vector[q_idx] = q_vector; 2071 } 2072 2073 return 0; 2074 2075err_out: 2076 while (q_idx) { 2077 q_idx--; 2078 q_vector = adapter->q_vector[q_idx]; 2079 netif_napi_del(&q_vector->napi); 2080 kfree(q_vector); 2081 adapter->q_vector[q_idx] = NULL; 2082 } 2083 return -ENOMEM; 2084} 2085 2086/** 2087 * ixgbevf_free_q_vectors - Free memory allocated for interrupt vectors 2088 * @adapter: board private structure to initialize 2089 * 2090 * This function frees the memory allocated to the q_vectors. In addition if 2091 * NAPI is enabled it will delete any references to the NAPI struct prior 2092 * to freeing the q_vector. 2093 **/ 2094static void ixgbevf_free_q_vectors(struct ixgbevf_adapter *adapter) 2095{ 2096 int q_idx, num_q_vectors; 2097 int napi_vectors; 2098 2099 num_q_vectors = adapter->num_msix_vectors - NON_Q_VECTORS; 2100 napi_vectors = adapter->num_rx_queues; 2101 2102 for (q_idx = 0; q_idx < num_q_vectors; q_idx++) { 2103 struct ixgbevf_q_vector *q_vector = adapter->q_vector[q_idx]; 2104 2105 adapter->q_vector[q_idx] = NULL; 2106 if (q_idx < napi_vectors) 2107 netif_napi_del(&q_vector->napi); 2108 kfree(q_vector); 2109 } 2110} 2111 2112/** 2113 * ixgbevf_reset_interrupt_capability - Reset MSIX setup 2114 * @adapter: board private structure 2115 * 2116 **/ 2117static void ixgbevf_reset_interrupt_capability(struct ixgbevf_adapter *adapter) 2118{ 2119 pci_disable_msix(adapter->pdev); 2120 kfree(adapter->msix_entries); 2121 adapter->msix_entries = NULL; 2122} 2123 2124/** 2125 * ixgbevf_init_interrupt_scheme - Determine if MSIX is supported and init 2126 * @adapter: board private structure to initialize 2127 * 2128 **/ 2129static int ixgbevf_init_interrupt_scheme(struct ixgbevf_adapter *adapter) 2130{ 2131 int err; 2132 2133 /* Number of supported queues */ 2134 ixgbevf_set_num_queues(adapter); 2135 2136 err = ixgbevf_set_interrupt_capability(adapter); 2137 if (err) { 2138 hw_dbg(&adapter->hw, 2139 "Unable to setup interrupt capabilities\n"); 2140 goto err_set_interrupt; 2141 } 2142 2143 err = ixgbevf_alloc_q_vectors(adapter); 2144 if (err) { 2145 hw_dbg(&adapter->hw, "Unable to allocate memory for queue " 2146 "vectors\n"); 2147 goto err_alloc_q_vectors; 2148 } 2149 2150 err = ixgbevf_alloc_queues(adapter); 2151 if (err) { 2152 pr_err("Unable to allocate memory for queues\n"); 2153 goto err_alloc_queues; 2154 } 2155 2156 hw_dbg(&adapter->hw, "Multiqueue %s: Rx Queue count = %u, " 2157 "Tx Queue count = %u\n", 2158 (adapter->num_rx_queues > 1) ? "Enabled" : 2159 "Disabled", adapter->num_rx_queues, adapter->num_tx_queues); 2160 2161 set_bit(__IXGBEVF_DOWN, &adapter->state); 2162 2163 return 0; 2164err_alloc_queues: 2165 ixgbevf_free_q_vectors(adapter); 2166err_alloc_q_vectors: 2167 ixgbevf_reset_interrupt_capability(adapter); 2168err_set_interrupt: 2169 return err; 2170} 2171 2172/** 2173 * ixgbevf_sw_init - Initialize general software structures 2174 * (struct ixgbevf_adapter) 2175 * @adapter: board private structure to initialize 2176 * 2177 * ixgbevf_sw_init initializes the Adapter private data structure. 2178 * Fields are initialized based on PCI device information and 2179 * OS network device settings (MTU size). 2180 **/ 2181static int __devinit ixgbevf_sw_init(struct ixgbevf_adapter *adapter) 2182{ 2183 struct ixgbe_hw *hw = &adapter->hw; 2184 struct pci_dev *pdev = adapter->pdev; 2185 int err; 2186 2187 /* PCI config space info */ 2188 2189 hw->vendor_id = pdev->vendor; 2190 hw->device_id = pdev->device; 2191 hw->revision_id = pdev->revision; 2192 hw->subsystem_vendor_id = pdev->subsystem_vendor; 2193 hw->subsystem_device_id = pdev->subsystem_device; 2194 2195 hw->mbx.ops.init_params(hw); 2196 hw->mac.max_tx_queues = MAX_TX_QUEUES; 2197 hw->mac.max_rx_queues = MAX_RX_QUEUES; 2198 err = hw->mac.ops.reset_hw(hw); 2199 if (err) { 2200 dev_info(&pdev->dev, 2201 "PF still in reset state, assigning new address\n"); 2202 dev_hw_addr_random(adapter->netdev, hw->mac.addr); 2203 } else { 2204 err = hw->mac.ops.init_hw(hw); 2205 if (err) { 2206 pr_err("init_shared_code failed: %d\n", err); 2207 goto out; 2208 } 2209 } 2210 2211 /* Enable dynamic interrupt throttling rates */ 2212 adapter->eitr_param = 20000; 2213 adapter->itr_setting = 1; 2214 2215 /* set defaults for eitr in MegaBytes */ 2216 adapter->eitr_low = 10; 2217 adapter->eitr_high = 20; 2218 2219 /* set default ring sizes */ 2220 adapter->tx_ring_count = IXGBEVF_DEFAULT_TXD; 2221 adapter->rx_ring_count = IXGBEVF_DEFAULT_RXD; 2222 2223 /* enable rx csum by default */ 2224 adapter->flags |= IXGBE_FLAG_RX_CSUM_ENABLED; 2225 2226 set_bit(__IXGBEVF_DOWN, &adapter->state); 2227 2228out: 2229 return err; 2230} 2231 2232#define UPDATE_VF_COUNTER_32bit(reg, last_counter, counter) \ 2233 { \ 2234 u32 current_counter = IXGBE_READ_REG(hw, reg); \ 2235 if (current_counter < last_counter) \ 2236 counter += 0x100000000LL; \ 2237 last_counter = current_counter; \ 2238 counter &= 0xFFFFFFFF00000000LL; \ 2239 counter |= current_counter; \ 2240 } 2241 2242#define UPDATE_VF_COUNTER_36bit(reg_lsb, reg_msb, last_counter, counter) \ 2243 { \ 2244 u64 current_counter_lsb = IXGBE_READ_REG(hw, reg_lsb); \ 2245 u64 current_counter_msb = IXGBE_READ_REG(hw, reg_msb); \ 2246 u64 current_counter = (current_counter_msb << 32) | \ 2247 current_counter_lsb; \ 2248 if (current_counter < last_counter) \ 2249 counter += 0x1000000000LL; \ 2250 last_counter = current_counter; \ 2251 counter &= 0xFFFFFFF000000000LL; \ 2252 counter |= current_counter; \ 2253 } 2254/** 2255 * ixgbevf_update_stats - Update the board statistics counters. 2256 * @adapter: board private structure 2257 **/ 2258void ixgbevf_update_stats(struct ixgbevf_adapter *adapter) 2259{ 2260 struct ixgbe_hw *hw = &adapter->hw; 2261 2262 UPDATE_VF_COUNTER_32bit(IXGBE_VFGPRC, adapter->stats.last_vfgprc, 2263 adapter->stats.vfgprc); 2264 UPDATE_VF_COUNTER_32bit(IXGBE_VFGPTC, adapter->stats.last_vfgptc, 2265 adapter->stats.vfgptc); 2266 UPDATE_VF_COUNTER_36bit(IXGBE_VFGORC_LSB, IXGBE_VFGORC_MSB, 2267 adapter->stats.last_vfgorc, 2268 adapter->stats.vfgorc); 2269 UPDATE_VF_COUNTER_36bit(IXGBE_VFGOTC_LSB, IXGBE_VFGOTC_MSB, 2270 adapter->stats.last_vfgotc, 2271 adapter->stats.vfgotc); 2272 UPDATE_VF_COUNTER_32bit(IXGBE_VFMPRC, adapter->stats.last_vfmprc, 2273 adapter->stats.vfmprc); 2274} 2275 2276/** 2277 * ixgbevf_watchdog - Timer Call-back 2278 * @data: pointer to adapter cast into an unsigned long 2279 **/ 2280static void ixgbevf_watchdog(unsigned long data) 2281{ 2282 struct ixgbevf_adapter *adapter = (struct ixgbevf_adapter *)data; 2283 struct ixgbe_hw *hw = &adapter->hw; 2284 u64 eics = 0; 2285 int i; 2286 2287 /* 2288 * Do the watchdog outside of interrupt context due to the lovely 2289 * delays that some of the newer hardware requires 2290 */ 2291 2292 if (test_bit(__IXGBEVF_DOWN, &adapter->state)) 2293 goto watchdog_short_circuit; 2294 2295 /* get one bit for every active tx/rx interrupt vector */ 2296 for (i = 0; i < adapter->num_msix_vectors - NON_Q_VECTORS; i++) { 2297 struct ixgbevf_q_vector *qv = adapter->q_vector[i]; 2298 if (qv->rxr_count || qv->txr_count) 2299 eics |= (1 << i); 2300 } 2301 2302 IXGBE_WRITE_REG(hw, IXGBE_VTEICS, (u32)eics); 2303 2304watchdog_short_circuit: 2305 schedule_work(&adapter->watchdog_task); 2306} 2307 2308/** 2309 * ixgbevf_tx_timeout - Respond to a Tx Hang 2310 * @netdev: network interface device structure 2311 **/ 2312static void ixgbevf_tx_timeout(struct net_device *netdev) 2313{ 2314 struct ixgbevf_adapter *adapter = netdev_priv(netdev); 2315 2316 /* Do the reset outside of interrupt context */ 2317 schedule_work(&adapter->reset_task); 2318} 2319 2320static void ixgbevf_reset_task(struct work_struct *work) 2321{ 2322 struct ixgbevf_adapter *adapter; 2323 adapter = container_of(work, struct ixgbevf_adapter, reset_task); 2324 2325 /* If we're already down or resetting, just bail */ 2326 if (test_bit(__IXGBEVF_DOWN, &adapter->state) || 2327 test_bit(__IXGBEVF_RESETTING, &adapter->state)) 2328 return; 2329 2330 adapter->tx_timeout_count++; 2331 2332 ixgbevf_reinit_locked(adapter); 2333} 2334 2335/** 2336 * ixgbevf_watchdog_task - worker thread to bring link up 2337 * @work: pointer to work_struct containing our data 2338 **/ 2339static void ixgbevf_watchdog_task(struct work_struct *work) 2340{ 2341 struct ixgbevf_adapter *adapter = container_of(work, 2342 struct ixgbevf_adapter, 2343 watchdog_task); 2344 struct net_device *netdev = adapter->netdev; 2345 struct ixgbe_hw *hw = &adapter->hw; 2346 u32 link_speed = adapter->link_speed; 2347 bool link_up = adapter->link_up; 2348 2349 adapter->flags |= IXGBE_FLAG_IN_WATCHDOG_TASK; 2350 2351 /* 2352 * Always check the link on the watchdog because we have 2353 * no LSC interrupt 2354 */ 2355 if (hw->mac.ops.check_link) { 2356 if ((hw->mac.ops.check_link(hw, &link_speed, 2357 &link_up, false)) != 0) { 2358 adapter->link_up = link_up; 2359 adapter->link_speed = link_speed; 2360 netif_carrier_off(netdev); 2361 netif_tx_stop_all_queues(netdev); 2362 schedule_work(&adapter->reset_task); 2363 goto pf_has_reset; 2364 } 2365 } else { 2366 /* always assume link is up, if no check link 2367 * function */ 2368 link_speed = IXGBE_LINK_SPEED_10GB_FULL; 2369 link_up = true; 2370 } 2371 adapter->link_up = link_up; 2372 adapter->link_speed = link_speed; 2373 2374 if (link_up) { 2375 if (!netif_carrier_ok(netdev)) { 2376 hw_dbg(&adapter->hw, "NIC Link is Up, %u Gbps\n", 2377 (link_speed == IXGBE_LINK_SPEED_10GB_FULL) ? 2378 10 : 1); 2379 netif_carrier_on(netdev); 2380 netif_tx_wake_all_queues(netdev); 2381 } 2382 } else { 2383 adapter->link_up = false; 2384 adapter->link_speed = 0; 2385 if (netif_carrier_ok(netdev)) { 2386 hw_dbg(&adapter->hw, "NIC Link is Down\n"); 2387 netif_carrier_off(netdev); 2388 netif_tx_stop_all_queues(netdev); 2389 } 2390 } 2391 2392 ixgbevf_update_stats(adapter); 2393 2394pf_has_reset: 2395 /* Reset the timer */ 2396 if (!test_bit(__IXGBEVF_DOWN, &adapter->state)) 2397 mod_timer(&adapter->watchdog_timer, 2398 round_jiffies(jiffies + (2 * HZ))); 2399 2400 adapter->flags &= ~IXGBE_FLAG_IN_WATCHDOG_TASK; 2401} 2402 2403/** 2404 * ixgbevf_free_tx_resources - Free Tx Resources per Queue 2405 * @adapter: board private structure 2406 * @tx_ring: Tx descriptor ring for a specific queue 2407 * 2408 * Free all transmit software resources 2409 **/ 2410void ixgbevf_free_tx_resources(struct ixgbevf_adapter *adapter, 2411 struct ixgbevf_ring *tx_ring) 2412{ 2413 struct pci_dev *pdev = adapter->pdev; 2414 2415 ixgbevf_clean_tx_ring(adapter, tx_ring); 2416 2417 vfree(tx_ring->tx_buffer_info); 2418 tx_ring->tx_buffer_info = NULL; 2419 2420 dma_free_coherent(&pdev->dev, tx_ring->size, tx_ring->desc, 2421 tx_ring->dma); 2422 2423 tx_ring->desc = NULL; 2424} 2425 2426/** 2427 * ixgbevf_free_all_tx_resources - Free Tx Resources for All Queues 2428 * @adapter: board private structure 2429 * 2430 * Free all transmit software resources 2431 **/ 2432static void ixgbevf_free_all_tx_resources(struct ixgbevf_adapter *adapter) 2433{ 2434 int i; 2435 2436 for (i = 0; i < adapter->num_tx_queues; i++) 2437 if (adapter->tx_ring[i].desc) 2438 ixgbevf_free_tx_resources(adapter, 2439 &adapter->tx_ring[i]); 2440 2441} 2442 2443/** 2444 * ixgbevf_setup_tx_resources - allocate Tx resources (Descriptors) 2445 * @adapter: board private structure 2446 * @tx_ring: tx descriptor ring (for a specific queue) to setup 2447 * 2448 * Return 0 on success, negative on failure 2449 **/ 2450int ixgbevf_setup_tx_resources(struct ixgbevf_adapter *adapter, 2451 struct ixgbevf_ring *tx_ring) 2452{ 2453 struct pci_dev *pdev = adapter->pdev; 2454 int size; 2455 2456 size = sizeof(struct ixgbevf_tx_buffer) * tx_ring->count; 2457 tx_ring->tx_buffer_info = vzalloc(size); 2458 if (!tx_ring->tx_buffer_info) 2459 goto err; 2460 2461 /* round up to nearest 4K */ 2462 tx_ring->size = tx_ring->count * sizeof(union ixgbe_adv_tx_desc); 2463 tx_ring->size = ALIGN(tx_ring->size, 4096); 2464 2465 tx_ring->desc = dma_alloc_coherent(&pdev->dev, tx_ring->size, 2466 &tx_ring->dma, GFP_KERNEL); 2467 if (!tx_ring->desc) 2468 goto err; 2469 2470 tx_ring->next_to_use = 0; 2471 tx_ring->next_to_clean = 0; 2472 tx_ring->work_limit = tx_ring->count; 2473 return 0; 2474 2475err: 2476 vfree(tx_ring->tx_buffer_info); 2477 tx_ring->tx_buffer_info = NULL; 2478 hw_dbg(&adapter->hw, "Unable to allocate memory for the transmit " 2479 "descriptor ring\n"); 2480 return -ENOMEM; 2481} 2482 2483/** 2484 * ixgbevf_setup_all_tx_resources - allocate all queues Tx resources 2485 * @adapter: board private structure 2486 * 2487 * If this function returns with an error, then it's possible one or 2488 * more of the rings is populated (while the rest are not). It is the 2489 * callers duty to clean those orphaned rings. 2490 * 2491 * Return 0 on success, negative on failure 2492 **/ 2493static int ixgbevf_setup_all_tx_resources(struct ixgbevf_adapter *adapter) 2494{ 2495 int i, err = 0; 2496 2497 for (i = 0; i < adapter->num_tx_queues; i++) { 2498 err = ixgbevf_setup_tx_resources(adapter, &adapter->tx_ring[i]); 2499 if (!err) 2500 continue; 2501 hw_dbg(&adapter->hw, 2502 "Allocation for Tx Queue %u failed\n", i); 2503 break; 2504 } 2505 2506 return err; 2507} 2508 2509/** 2510 * ixgbevf_setup_rx_resources - allocate Rx resources (Descriptors) 2511 * @adapter: board private structure 2512 * @rx_ring: rx descriptor ring (for a specific queue) to setup 2513 * 2514 * Returns 0 on success, negative on failure 2515 **/ 2516int ixgbevf_setup_rx_resources(struct ixgbevf_adapter *adapter, 2517 struct ixgbevf_ring *rx_ring) 2518{ 2519 struct pci_dev *pdev = adapter->pdev; 2520 int size; 2521 2522 size = sizeof(struct ixgbevf_rx_buffer) * rx_ring->count; 2523 rx_ring->rx_buffer_info = vzalloc(size); 2524 if (!rx_ring->rx_buffer_info) { 2525 hw_dbg(&adapter->hw, 2526 "Unable to vmalloc buffer memory for " 2527 "the receive descriptor ring\n"); 2528 goto alloc_failed; 2529 } 2530 2531 /* Round up to nearest 4K */ 2532 rx_ring->size = rx_ring->count * sizeof(union ixgbe_adv_rx_desc); 2533 rx_ring->size = ALIGN(rx_ring->size, 4096); 2534 2535 rx_ring->desc = dma_alloc_coherent(&pdev->dev, rx_ring->size, 2536 &rx_ring->dma, GFP_KERNEL); 2537 2538 if (!rx_ring->desc) { 2539 hw_dbg(&adapter->hw, 2540 "Unable to allocate memory for " 2541 "the receive descriptor ring\n"); 2542 vfree(rx_ring->rx_buffer_info); 2543 rx_ring->rx_buffer_info = NULL; 2544 goto alloc_failed; 2545 } 2546 2547 rx_ring->next_to_clean = 0; 2548 rx_ring->next_to_use = 0; 2549 2550 return 0; 2551alloc_failed: 2552 return -ENOMEM; 2553} 2554 2555/** 2556 * ixgbevf_setup_all_rx_resources - allocate all queues Rx resources 2557 * @adapter: board private structure 2558 * 2559 * If this function returns with an error, then it's possible one or 2560 * more of the rings is populated (while the rest are not). It is the 2561 * callers duty to clean those orphaned rings. 2562 * 2563 * Return 0 on success, negative on failure 2564 **/ 2565static int ixgbevf_setup_all_rx_resources(struct ixgbevf_adapter *adapter) 2566{ 2567 int i, err = 0; 2568 2569 for (i = 0; i < adapter->num_rx_queues; i++) { 2570 err = ixgbevf_setup_rx_resources(adapter, &adapter->rx_ring[i]); 2571 if (!err) 2572 continue; 2573 hw_dbg(&adapter->hw, 2574 "Allocation for Rx Queue %u failed\n", i); 2575 break; 2576 } 2577 return err; 2578} 2579 2580/** 2581 * ixgbevf_free_rx_resources - Free Rx Resources 2582 * @adapter: board private structure 2583 * @rx_ring: ring to clean the resources from 2584 * 2585 * Free all receive software resources 2586 **/ 2587void ixgbevf_free_rx_resources(struct ixgbevf_adapter *adapter, 2588 struct ixgbevf_ring *rx_ring) 2589{ 2590 struct pci_dev *pdev = adapter->pdev; 2591 2592 ixgbevf_clean_rx_ring(adapter, rx_ring); 2593 2594 vfree(rx_ring->rx_buffer_info); 2595 rx_ring->rx_buffer_info = NULL; 2596 2597 dma_free_coherent(&pdev->dev, rx_ring->size, rx_ring->desc, 2598 rx_ring->dma); 2599 2600 rx_ring->desc = NULL; 2601} 2602 2603/** 2604 * ixgbevf_free_all_rx_resources - Free Rx Resources for All Queues 2605 * @adapter: board private structure 2606 * 2607 * Free all receive software resources 2608 **/ 2609static void ixgbevf_free_all_rx_resources(struct ixgbevf_adapter *adapter) 2610{ 2611 int i; 2612 2613 for (i = 0; i < adapter->num_rx_queues; i++) 2614 if (adapter->rx_ring[i].desc) 2615 ixgbevf_free_rx_resources(adapter, 2616 &adapter->rx_ring[i]); 2617} 2618 2619/** 2620 * ixgbevf_open - Called when a network interface is made active 2621 * @netdev: network interface device structure 2622 * 2623 * Returns 0 on success, negative value on failure 2624 * 2625 * The open entry point is called when a network interface is made 2626 * active by the system (IFF_UP). At this point all resources needed 2627 * for transmit and receive operations are allocated, the interrupt 2628 * handler is registered with the OS, the watchdog timer is started, 2629 * and the stack is notified that the interface is ready. 2630 **/ 2631static int ixgbevf_open(struct net_device *netdev) 2632{ 2633 struct ixgbevf_adapter *adapter = netdev_priv(netdev); 2634 struct ixgbe_hw *hw = &adapter->hw; 2635 int err; 2636 2637 /* disallow open during test */ 2638 if (test_bit(__IXGBEVF_TESTING, &adapter->state)) 2639 return -EBUSY; 2640 2641 if (hw->adapter_stopped) { 2642 ixgbevf_reset(adapter); 2643 /* if adapter is still stopped then PF isn't up and 2644 * the vf can't start. */ 2645 if (hw->adapter_stopped) { 2646 err = IXGBE_ERR_MBX; 2647 pr_err("Unable to start - perhaps the PF Driver isn't " 2648 "up yet\n"); 2649 goto err_setup_reset; 2650 } 2651 } 2652 2653 /* allocate transmit descriptors */ 2654 err = ixgbevf_setup_all_tx_resources(adapter); 2655 if (err) 2656 goto err_setup_tx; 2657 2658 /* allocate receive descriptors */ 2659 err = ixgbevf_setup_all_rx_resources(adapter); 2660 if (err) 2661 goto err_setup_rx; 2662 2663 ixgbevf_configure(adapter); 2664 2665 /* 2666 * Map the Tx/Rx rings to the vectors we were allotted. 2667 * if request_irq will be called in this function map_rings 2668 * must be called *before* up_complete 2669 */ 2670 ixgbevf_map_rings_to_vectors(adapter); 2671 2672 err = ixgbevf_up_complete(adapter); 2673 if (err) 2674 goto err_up; 2675 2676 /* clear any pending interrupts, may auto mask */ 2677 IXGBE_READ_REG(hw, IXGBE_VTEICR); 2678 err = ixgbevf_request_irq(adapter); 2679 if (err) 2680 goto err_req_irq; 2681 2682 ixgbevf_irq_enable(adapter, true, true); 2683 2684 return 0; 2685 2686err_req_irq: 2687 ixgbevf_down(adapter); 2688err_up: 2689 ixgbevf_free_irq(adapter); 2690err_setup_rx: 2691 ixgbevf_free_all_rx_resources(adapter); 2692err_setup_tx: 2693 ixgbevf_free_all_tx_resources(adapter); 2694 ixgbevf_reset(adapter); 2695 2696err_setup_reset: 2697 2698 return err; 2699} 2700 2701/** 2702 * ixgbevf_close - Disables a network interface 2703 * @netdev: network interface device structure 2704 * 2705 * Returns 0, this is not allowed to fail 2706 * 2707 * The close entry point is called when an interface is de-activated 2708 * by the OS. The hardware is still under the drivers control, but 2709 * needs to be disabled. A global MAC reset is issued to stop the 2710 * hardware, and all transmit and receive resources are freed. 2711 **/ 2712static int ixgbevf_close(struct net_device *netdev) 2713{ 2714 struct ixgbevf_adapter *adapter = netdev_priv(netdev); 2715 2716 ixgbevf_down(adapter); 2717 ixgbevf_free_irq(adapter); 2718 2719 ixgbevf_free_all_tx_resources(adapter); 2720 ixgbevf_free_all_rx_resources(adapter); 2721 2722 return 0; 2723} 2724 2725static int ixgbevf_tso(struct ixgbevf_adapter *adapter, 2726 struct ixgbevf_ring *tx_ring, 2727 struct sk_buff *skb, u32 tx_flags, u8 *hdr_len) 2728{ 2729 struct ixgbe_adv_tx_context_desc *context_desc; 2730 unsigned int i; 2731 int err; 2732 struct ixgbevf_tx_buffer *tx_buffer_info; 2733 u32 vlan_macip_lens = 0, type_tucmd_mlhl; 2734 u32 mss_l4len_idx, l4len; 2735 2736 if (skb_is_gso(skb)) { 2737 if (skb_header_cloned(skb)) { 2738 err = pskb_expand_head(skb, 0, 0, GFP_ATOMIC); 2739 if (err) 2740 return err; 2741 } 2742 l4len = tcp_hdrlen(skb); 2743 *hdr_len += l4len; 2744 2745 if (skb->protocol == htons(ETH_P_IP)) { 2746 struct iphdr *iph = ip_hdr(skb); 2747 iph->tot_len = 0; 2748 iph->check = 0; 2749 tcp_hdr(skb)->check = ~csum_tcpudp_magic(iph->saddr, 2750 iph->daddr, 0, 2751 IPPROTO_TCP, 2752 0); 2753 adapter->hw_tso_ctxt++; 2754 } else if (skb_is_gso_v6(skb)) { 2755 ipv6_hdr(skb)->payload_len = 0; 2756 tcp_hdr(skb)->check = 2757 ~csum_ipv6_magic(&ipv6_hdr(skb)->saddr, 2758 &ipv6_hdr(skb)->daddr, 2759 0, IPPROTO_TCP, 0); 2760 adapter->hw_tso6_ctxt++; 2761 } 2762 2763 i = tx_ring->next_to_use; 2764 2765 tx_buffer_info = &tx_ring->tx_buffer_info[i]; 2766 context_desc = IXGBE_TX_CTXTDESC_ADV(*tx_ring, i); 2767 2768 /* VLAN MACLEN IPLEN */ 2769 if (tx_flags & IXGBE_TX_FLAGS_VLAN) 2770 vlan_macip_lens |= 2771 (tx_flags & IXGBE_TX_FLAGS_VLAN_MASK); 2772 vlan_macip_lens |= ((skb_network_offset(skb)) << 2773 IXGBE_ADVTXD_MACLEN_SHIFT); 2774 *hdr_len += skb_network_offset(skb); 2775 vlan_macip_lens |= 2776 (skb_transport_header(skb) - skb_network_header(skb)); 2777 *hdr_len += 2778 (skb_transport_header(skb) - skb_network_header(skb)); 2779 context_desc->vlan_macip_lens = cpu_to_le32(vlan_macip_lens); 2780 context_desc->seqnum_seed = 0; 2781 2782 /* ADV DTYP TUCMD MKRLOC/ISCSIHEDLEN */ 2783 type_tucmd_mlhl = (IXGBE_TXD_CMD_DEXT | 2784 IXGBE_ADVTXD_DTYP_CTXT); 2785 2786 if (skb->protocol == htons(ETH_P_IP)) 2787 type_tucmd_mlhl |= IXGBE_ADVTXD_TUCMD_IPV4; 2788 type_tucmd_mlhl |= IXGBE_ADVTXD_TUCMD_L4T_TCP; 2789 context_desc->type_tucmd_mlhl = cpu_to_le32(type_tucmd_mlhl); 2790 2791 /* MSS L4LEN IDX */ 2792 mss_l4len_idx = 2793 (skb_shinfo(skb)->gso_size << IXGBE_ADVTXD_MSS_SHIFT); 2794 mss_l4len_idx |= (l4len << IXGBE_ADVTXD_L4LEN_SHIFT); 2795 /* use index 1 for TSO */ 2796 mss_l4len_idx |= (1 << IXGBE_ADVTXD_IDX_SHIFT); 2797 context_desc->mss_l4len_idx = cpu_to_le32(mss_l4len_idx); 2798 2799 tx_buffer_info->time_stamp = jiffies; 2800 tx_buffer_info->next_to_watch = i; 2801 2802 i++; 2803 if (i == tx_ring->count) 2804 i = 0; 2805 tx_ring->next_to_use = i; 2806 2807 return true; 2808 } 2809 2810 return false; 2811} 2812 2813static bool ixgbevf_tx_csum(struct ixgbevf_adapter *adapter, 2814 struct ixgbevf_ring *tx_ring, 2815 struct sk_buff *skb, u32 tx_flags) 2816{ 2817 struct ixgbe_adv_tx_context_desc *context_desc; 2818 unsigned int i; 2819 struct ixgbevf_tx_buffer *tx_buffer_info; 2820 u32 vlan_macip_lens = 0, type_tucmd_mlhl = 0; 2821 2822 if (skb->ip_summed == CHECKSUM_PARTIAL || 2823 (tx_flags & IXGBE_TX_FLAGS_VLAN)) { 2824 i = tx_ring->next_to_use; 2825 tx_buffer_info = &tx_ring->tx_buffer_info[i]; 2826 context_desc = IXGBE_TX_CTXTDESC_ADV(*tx_ring, i); 2827 2828 if (tx_flags & IXGBE_TX_FLAGS_VLAN) 2829 vlan_macip_lens |= (tx_flags & 2830 IXGBE_TX_FLAGS_VLAN_MASK); 2831 vlan_macip_lens |= (skb_network_offset(skb) << 2832 IXGBE_ADVTXD_MACLEN_SHIFT); 2833 if (skb->ip_summed == CHECKSUM_PARTIAL) 2834 vlan_macip_lens |= (skb_transport_header(skb) - 2835 skb_network_header(skb)); 2836 2837 context_desc->vlan_macip_lens = cpu_to_le32(vlan_macip_lens); 2838 context_desc->seqnum_seed = 0; 2839 2840 type_tucmd_mlhl |= (IXGBE_TXD_CMD_DEXT | 2841 IXGBE_ADVTXD_DTYP_CTXT); 2842 2843 if (skb->ip_summed == CHECKSUM_PARTIAL) { 2844 switch (skb->protocol) { 2845 case __constant_htons(ETH_P_IP): 2846 type_tucmd_mlhl |= IXGBE_ADVTXD_TUCMD_IPV4; 2847 if (ip_hdr(skb)->protocol == IPPROTO_TCP) 2848 type_tucmd_mlhl |= 2849 IXGBE_ADVTXD_TUCMD_L4T_TCP; 2850 break; 2851 case __constant_htons(ETH_P_IPV6): 2852 /* XXX what about other V6 headers?? */ 2853 if (ipv6_hdr(skb)->nexthdr == IPPROTO_TCP) 2854 type_tucmd_mlhl |= 2855 IXGBE_ADVTXD_TUCMD_L4T_TCP; 2856 break; 2857 default: 2858 if (unlikely(net_ratelimit())) { 2859 pr_warn("partial checksum but " 2860 "proto=%x!\n", skb->protocol); 2861 } 2862 break; 2863 } 2864 } 2865 2866 context_desc->type_tucmd_mlhl = cpu_to_le32(type_tucmd_mlhl); 2867 /* use index zero for tx checksum offload */ 2868 context_desc->mss_l4len_idx = 0; 2869 2870 tx_buffer_info->time_stamp = jiffies; 2871 tx_buffer_info->next_to_watch = i; 2872 2873 adapter->hw_csum_tx_good++; 2874 i++; 2875 if (i == tx_ring->count) 2876 i = 0; 2877 tx_ring->next_to_use = i; 2878 2879 return true; 2880 } 2881 2882 return false; 2883} 2884 2885static int ixgbevf_tx_map(struct ixgbevf_adapter *adapter, 2886 struct ixgbevf_ring *tx_ring, 2887 struct sk_buff *skb, u32 tx_flags, 2888 unsigned int first) 2889{ 2890 struct pci_dev *pdev = adapter->pdev; 2891 struct ixgbevf_tx_buffer *tx_buffer_info; 2892 unsigned int len; 2893 unsigned int total = skb->len; 2894 unsigned int offset = 0, size; 2895 int count = 0; 2896 unsigned int nr_frags = skb_shinfo(skb)->nr_frags; 2897 unsigned int f; 2898 int i; 2899 2900 i = tx_ring->next_to_use; 2901 2902 len = min(skb_headlen(skb), total); 2903 while (len) { 2904 tx_buffer_info = &tx_ring->tx_buffer_info[i]; 2905 size = min(len, (unsigned int)IXGBE_MAX_DATA_PER_TXD); 2906 2907 tx_buffer_info->length = size; 2908 tx_buffer_info->mapped_as_page = false; 2909 tx_buffer_info->dma = dma_map_single(&adapter->pdev->dev, 2910 skb->data + offset, 2911 size, DMA_TO_DEVICE); 2912 if (dma_mapping_error(&pdev->dev, tx_buffer_info->dma)) 2913 goto dma_error; 2914 tx_buffer_info->time_stamp = jiffies; 2915 tx_buffer_info->next_to_watch = i; 2916 2917 len -= size; 2918 total -= size; 2919 offset += size; 2920 count++; 2921 i++; 2922 if (i == tx_ring->count) 2923 i = 0; 2924 } 2925 2926 for (f = 0; f < nr_frags; f++) { 2927 const struct skb_frag_struct *frag; 2928 2929 frag = &skb_shinfo(skb)->frags[f]; 2930 len = min((unsigned int)skb_frag_size(frag), total); 2931 offset = 0; 2932 2933 while (len) { 2934 tx_buffer_info = &tx_ring->tx_buffer_info[i]; 2935 size = min(len, (unsigned int)IXGBE_MAX_DATA_PER_TXD); 2936 2937 tx_buffer_info->length = size; 2938 tx_buffer_info->dma = 2939 skb_frag_dma_map(&adapter->pdev->dev, frag, 2940 offset, size, DMA_TO_DEVICE); 2941 tx_buffer_info->mapped_as_page = true; 2942 if (dma_mapping_error(&pdev->dev, tx_buffer_info->dma)) 2943 goto dma_error; 2944 tx_buffer_info->time_stamp = jiffies; 2945 tx_buffer_info->next_to_watch = i; 2946 2947 len -= size; 2948 total -= size; 2949 offset += size; 2950 count++; 2951 i++; 2952 if (i == tx_ring->count) 2953 i = 0; 2954 } 2955 if (total == 0) 2956 break; 2957 } 2958 2959 if (i == 0) 2960 i = tx_ring->count - 1; 2961 else 2962 i = i - 1; 2963 tx_ring->tx_buffer_info[i].skb = skb; 2964 tx_ring->tx_buffer_info[first].next_to_watch = i; 2965 2966 return count; 2967 2968dma_error: 2969 dev_err(&pdev->dev, "TX DMA map failed\n"); 2970 2971 /* clear timestamp and dma mappings for failed tx_buffer_info map */ 2972 tx_buffer_info->dma = 0; 2973 tx_buffer_info->time_stamp = 0; 2974 tx_buffer_info->next_to_watch = 0; 2975 count--; 2976 2977 /* clear timestamp and dma mappings for remaining portion of packet */ 2978 while (count >= 0) { 2979 count--; 2980 i--; 2981 if (i < 0) 2982 i += tx_ring->count; 2983 tx_buffer_info = &tx_ring->tx_buffer_info[i]; 2984 ixgbevf_unmap_and_free_tx_resource(adapter, tx_buffer_info); 2985 } 2986 2987 return count; 2988} 2989 2990static void ixgbevf_tx_queue(struct ixgbevf_adapter *adapter, 2991 struct ixgbevf_ring *tx_ring, int tx_flags, 2992 int count, u32 paylen, u8 hdr_len) 2993{ 2994 union ixgbe_adv_tx_desc *tx_desc = NULL; 2995 struct ixgbevf_tx_buffer *tx_buffer_info; 2996 u32 olinfo_status = 0, cmd_type_len = 0; 2997 unsigned int i; 2998 2999 u32 txd_cmd = IXGBE_TXD_CMD_EOP | IXGBE_TXD_CMD_RS | IXGBE_TXD_CMD_IFCS; 3000 3001 cmd_type_len |= IXGBE_ADVTXD_DTYP_DATA; 3002 3003 cmd_type_len |= IXGBE_ADVTXD_DCMD_IFCS | IXGBE_ADVTXD_DCMD_DEXT; 3004 3005 if (tx_flags & IXGBE_TX_FLAGS_VLAN) 3006 cmd_type_len |= IXGBE_ADVTXD_DCMD_VLE; 3007 3008 if (tx_flags & IXGBE_TX_FLAGS_TSO) { 3009 cmd_type_len |= IXGBE_ADVTXD_DCMD_TSE; 3010 3011 olinfo_status |= IXGBE_TXD_POPTS_TXSM << 3012 IXGBE_ADVTXD_POPTS_SHIFT; 3013 3014 /* use index 1 context for tso */ 3015 olinfo_status |= (1 << IXGBE_ADVTXD_IDX_SHIFT); 3016 if (tx_flags & IXGBE_TX_FLAGS_IPV4) 3017 olinfo_status |= IXGBE_TXD_POPTS_IXSM << 3018 IXGBE_ADVTXD_POPTS_SHIFT; 3019 3020 } else if (tx_flags & IXGBE_TX_FLAGS_CSUM) 3021 olinfo_status |= IXGBE_TXD_POPTS_TXSM << 3022 IXGBE_ADVTXD_POPTS_SHIFT; 3023 3024 olinfo_status |= ((paylen - hdr_len) << IXGBE_ADVTXD_PAYLEN_SHIFT); 3025 3026 i = tx_ring->next_to_use; 3027 while (count--) { 3028 tx_buffer_info = &tx_ring->tx_buffer_info[i]; 3029 tx_desc = IXGBE_TX_DESC_ADV(*tx_ring, i); 3030 tx_desc->read.buffer_addr = cpu_to_le64(tx_buffer_info->dma); 3031 tx_desc->read.cmd_type_len = 3032 cpu_to_le32(cmd_type_len | tx_buffer_info->length); 3033 tx_desc->read.olinfo_status = cpu_to_le32(olinfo_status); 3034 i++; 3035 if (i == tx_ring->count) 3036 i = 0; 3037 } 3038 3039 tx_desc->read.cmd_type_len |= cpu_to_le32(txd_cmd); 3040 3041 /* 3042 * Force memory writes to complete before letting h/w 3043 * know there are new descriptors to fetch. (Only 3044 * applicable for weak-ordered memory model archs, 3045 * such as IA-64). 3046 */ 3047 wmb(); 3048 3049 tx_ring->next_to_use = i; 3050 writel(i, adapter->hw.hw_addr + tx_ring->tail); 3051} 3052 3053static int __ixgbevf_maybe_stop_tx(struct net_device *netdev, 3054 struct ixgbevf_ring *tx_ring, int size) 3055{ 3056 struct ixgbevf_adapter *adapter = netdev_priv(netdev); 3057 3058 netif_stop_subqueue(netdev, tx_ring->queue_index); 3059 /* Herbert's original patch had: 3060 * smp_mb__after_netif_stop_queue(); 3061 * but since that doesn't exist yet, just open code it. */ 3062 smp_mb(); 3063 3064 /* We need to check again in a case another CPU has just 3065 * made room available. */ 3066 if (likely(IXGBE_DESC_UNUSED(tx_ring) < size)) 3067 return -EBUSY; 3068 3069 /* A reprieve! - use start_queue because it doesn't call schedule */ 3070 netif_start_subqueue(netdev, tx_ring->queue_index); 3071 ++adapter->restart_queue; 3072 return 0; 3073} 3074 3075static int ixgbevf_maybe_stop_tx(struct net_device *netdev, 3076 struct ixgbevf_ring *tx_ring, int size) 3077{ 3078 if (likely(IXGBE_DESC_UNUSED(tx_ring) >= size)) 3079 return 0; 3080 return __ixgbevf_maybe_stop_tx(netdev, tx_ring, size); 3081} 3082 3083static int ixgbevf_xmit_frame(struct sk_buff *skb, struct net_device *netdev) 3084{ 3085 struct ixgbevf_adapter *adapter = netdev_priv(netdev); 3086 struct ixgbevf_ring *tx_ring; 3087 unsigned int first; 3088 unsigned int tx_flags = 0; 3089 u8 hdr_len = 0; 3090 int r_idx = 0, tso; 3091 int count = 0; 3092 3093 unsigned int f; 3094 3095 tx_ring = &adapter->tx_ring[r_idx]; 3096 3097 if (vlan_tx_tag_present(skb)) { 3098 tx_flags |= vlan_tx_tag_get(skb); 3099 tx_flags <<= IXGBE_TX_FLAGS_VLAN_SHIFT; 3100 tx_flags |= IXGBE_TX_FLAGS_VLAN; 3101 } 3102 3103 /* four things can cause us to need a context descriptor */ 3104 if (skb_is_gso(skb) || 3105 (skb->ip_summed == CHECKSUM_PARTIAL) || 3106 (tx_flags & IXGBE_TX_FLAGS_VLAN)) 3107 count++; 3108 3109 count += TXD_USE_COUNT(skb_headlen(skb)); 3110 for (f = 0; f < skb_shinfo(skb)->nr_frags; f++) 3111 count += TXD_USE_COUNT(skb_frag_size(&skb_shinfo(skb)->frags[f])); 3112 3113 if (ixgbevf_maybe_stop_tx(netdev, tx_ring, count)) { 3114 adapter->tx_busy++; 3115 return NETDEV_TX_BUSY; 3116 } 3117 3118 first = tx_ring->next_to_use; 3119 3120 if (skb->protocol == htons(ETH_P_IP)) 3121 tx_flags |= IXGBE_TX_FLAGS_IPV4; 3122 tso = ixgbevf_tso(adapter, tx_ring, skb, tx_flags, &hdr_len); 3123 if (tso < 0) { 3124 dev_kfree_skb_any(skb); 3125 return NETDEV_TX_OK; 3126 } 3127 3128 if (tso) 3129 tx_flags |= IXGBE_TX_FLAGS_TSO; 3130 else if (ixgbevf_tx_csum(adapter, tx_ring, skb, tx_flags) && 3131 (skb->ip_summed == CHECKSUM_PARTIAL)) 3132 tx_flags |= IXGBE_TX_FLAGS_CSUM; 3133 3134 ixgbevf_tx_queue(adapter, tx_ring, tx_flags, 3135 ixgbevf_tx_map(adapter, tx_ring, skb, tx_flags, first), 3136 skb->len, hdr_len); 3137 3138 ixgbevf_maybe_stop_tx(netdev, tx_ring, DESC_NEEDED); 3139 3140 return NETDEV_TX_OK; 3141} 3142 3143/** 3144 * ixgbevf_set_mac - Change the Ethernet Address of the NIC 3145 * @netdev: network interface device structure 3146 * @p: pointer to an address structure 3147 * 3148 * Returns 0 on success, negative on failure 3149 **/ 3150static int ixgbevf_set_mac(struct net_device *netdev, void *p) 3151{ 3152 struct ixgbevf_adapter *adapter = netdev_priv(netdev); 3153 struct ixgbe_hw *hw = &adapter->hw; 3154 struct sockaddr *addr = p; 3155 3156 if (!is_valid_ether_addr(addr->sa_data)) 3157 return -EADDRNOTAVAIL; 3158 3159 memcpy(netdev->dev_addr, addr->sa_data, netdev->addr_len); 3160 memcpy(hw->mac.addr, addr->sa_data, netdev->addr_len); 3161 3162 if (hw->mac.ops.set_rar) 3163 hw->mac.ops.set_rar(hw, 0, hw->mac.addr, 0); 3164 3165 return 0; 3166} 3167 3168/** 3169 * ixgbevf_change_mtu - Change the Maximum Transfer Unit 3170 * @netdev: network interface device structure 3171 * @new_mtu: new value for maximum frame size 3172 * 3173 * Returns 0 on success, negative on failure 3174 **/ 3175static int ixgbevf_change_mtu(struct net_device *netdev, int new_mtu) 3176{ 3177 struct ixgbevf_adapter *adapter = netdev_priv(netdev); 3178 struct ixgbe_hw *hw = &adapter->hw; 3179 int max_frame = new_mtu + ETH_HLEN + ETH_FCS_LEN; 3180 int max_possible_frame = MAXIMUM_ETHERNET_VLAN_SIZE; 3181 u32 msg[2]; 3182 3183 if (adapter->hw.mac.type == ixgbe_mac_X540_vf) 3184 max_possible_frame = IXGBE_MAX_JUMBO_FRAME_SIZE; 3185 3186 /* MTU < 68 is an error and causes problems on some kernels */ 3187 if ((new_mtu < 68) || (max_frame > max_possible_frame)) 3188 return -EINVAL; 3189 3190 hw_dbg(&adapter->hw, "changing MTU from %d to %d\n", 3191 netdev->mtu, new_mtu); 3192 /* must set new MTU before calling down or up */ 3193 netdev->mtu = new_mtu; 3194 3195 msg[0] = IXGBE_VF_SET_LPE; 3196 msg[1] = max_frame; 3197 hw->mbx.ops.write_posted(hw, msg, 2); 3198 3199 if (netif_running(netdev)) 3200 ixgbevf_reinit_locked(adapter); 3201 3202 return 0; 3203} 3204 3205static void ixgbevf_shutdown(struct pci_dev *pdev) 3206{ 3207 struct net_device *netdev = pci_get_drvdata(pdev); 3208 struct ixgbevf_adapter *adapter = netdev_priv(netdev); 3209 3210 netif_device_detach(netdev); 3211 3212 if (netif_running(netdev)) { 3213 ixgbevf_down(adapter); 3214 ixgbevf_free_irq(adapter); 3215 ixgbevf_free_all_tx_resources(adapter); 3216 ixgbevf_free_all_rx_resources(adapter); 3217 } 3218 3219#ifdef CONFIG_PM 3220 pci_save_state(pdev); 3221#endif 3222 3223 pci_disable_device(pdev); 3224} 3225 3226static struct rtnl_link_stats64 *ixgbevf_get_stats(struct net_device *netdev, 3227 struct rtnl_link_stats64 *stats) 3228{ 3229 struct ixgbevf_adapter *adapter = netdev_priv(netdev); 3230 unsigned int start; 3231 u64 bytes, packets; 3232 const struct ixgbevf_ring *ring; 3233 int i; 3234 3235 ixgbevf_update_stats(adapter); 3236 3237 stats->multicast = adapter->stats.vfmprc - adapter->stats.base_vfmprc; 3238 3239 for (i = 0; i < adapter->num_rx_queues; i++) { 3240 ring = &adapter->rx_ring[i]; 3241 do { 3242 start = u64_stats_fetch_begin_bh(&ring->syncp); 3243 bytes = ring->total_bytes; 3244 packets = ring->total_packets; 3245 } while (u64_stats_fetch_retry_bh(&ring->syncp, start)); 3246 stats->rx_bytes += bytes; 3247 stats->rx_packets += packets; 3248 } 3249 3250 for (i = 0; i < adapter->num_tx_queues; i++) { 3251 ring = &adapter->tx_ring[i]; 3252 do { 3253 start = u64_stats_fetch_begin_bh(&ring->syncp); 3254 bytes = ring->total_bytes; 3255 packets = ring->total_packets; 3256 } while (u64_stats_fetch_retry_bh(&ring->syncp, start)); 3257 stats->tx_bytes += bytes; 3258 stats->tx_packets += packets; 3259 } 3260 3261 return stats; 3262} 3263 3264static int ixgbevf_set_features(struct net_device *netdev, 3265 netdev_features_t features) 3266{ 3267 struct ixgbevf_adapter *adapter = netdev_priv(netdev); 3268 3269 if (features & NETIF_F_RXCSUM) 3270 adapter->flags |= IXGBE_FLAG_RX_CSUM_ENABLED; 3271 else 3272 adapter->flags &= ~IXGBE_FLAG_RX_CSUM_ENABLED; 3273 3274 return 0; 3275} 3276 3277static const struct net_device_ops ixgbe_netdev_ops = { 3278 .ndo_open = ixgbevf_open, 3279 .ndo_stop = ixgbevf_close, 3280 .ndo_start_xmit = ixgbevf_xmit_frame, 3281 .ndo_set_rx_mode = ixgbevf_set_rx_mode, 3282 .ndo_get_stats64 = ixgbevf_get_stats, 3283 .ndo_validate_addr = eth_validate_addr, 3284 .ndo_set_mac_address = ixgbevf_set_mac, 3285 .ndo_change_mtu = ixgbevf_change_mtu, 3286 .ndo_tx_timeout = ixgbevf_tx_timeout, 3287 .ndo_vlan_rx_add_vid = ixgbevf_vlan_rx_add_vid, 3288 .ndo_vlan_rx_kill_vid = ixgbevf_vlan_rx_kill_vid, 3289 .ndo_set_features = ixgbevf_set_features, 3290}; 3291 3292static void ixgbevf_assign_netdev_ops(struct net_device *dev) 3293{ 3294 dev->netdev_ops = &ixgbe_netdev_ops; 3295 ixgbevf_set_ethtool_ops(dev); 3296 dev->watchdog_timeo = 5 * HZ; 3297} 3298 3299/** 3300 * ixgbevf_probe - Device Initialization Routine 3301 * @pdev: PCI device information struct 3302 * @ent: entry in ixgbevf_pci_tbl 3303 * 3304 * Returns 0 on success, negative on failure 3305 * 3306 * ixgbevf_probe initializes an adapter identified by a pci_dev structure. 3307 * The OS initialization, configuring of the adapter private structure, 3308 * and a hardware reset occur. 3309 **/ 3310static int __devinit ixgbevf_probe(struct pci_dev *pdev, 3311 const struct pci_device_id *ent) 3312{ 3313 struct net_device *netdev; 3314 struct ixgbevf_adapter *adapter = NULL; 3315 struct ixgbe_hw *hw = NULL; 3316 const struct ixgbevf_info *ii = ixgbevf_info_tbl[ent->driver_data]; 3317 static int cards_found; 3318 int err, pci_using_dac; 3319 3320 err = pci_enable_device(pdev); 3321 if (err) 3322 return err; 3323 3324 if (!dma_set_mask(&pdev->dev, DMA_BIT_MASK(64)) && 3325 !dma_set_coherent_mask(&pdev->dev, DMA_BIT_MASK(64))) { 3326 pci_using_dac = 1; 3327 } else { 3328 err = dma_set_mask(&pdev->dev, DMA_BIT_MASK(32)); 3329 if (err) { 3330 err = dma_set_coherent_mask(&pdev->dev, 3331 DMA_BIT_MASK(32)); 3332 if (err) { 3333 dev_err(&pdev->dev, "No usable DMA " 3334 "configuration, aborting\n"); 3335 goto err_dma; 3336 } 3337 } 3338 pci_using_dac = 0; 3339 } 3340 3341 err = pci_request_regions(pdev, ixgbevf_driver_name); 3342 if (err) { 3343 dev_err(&pdev->dev, "pci_request_regions failed 0x%x\n", err); 3344 goto err_pci_reg; 3345 } 3346 3347 pci_set_master(pdev); 3348 3349#ifdef HAVE_TX_MQ 3350 netdev = alloc_etherdev_mq(sizeof(struct ixgbevf_adapter), 3351 MAX_TX_QUEUES); 3352#else 3353 netdev = alloc_etherdev(sizeof(struct ixgbevf_adapter)); 3354#endif 3355 if (!netdev) { 3356 err = -ENOMEM; 3357 goto err_alloc_etherdev; 3358 } 3359 3360 SET_NETDEV_DEV(netdev, &pdev->dev); 3361 3362 pci_set_drvdata(pdev, netdev); 3363 adapter = netdev_priv(netdev); 3364 3365 adapter->netdev = netdev; 3366 adapter->pdev = pdev; 3367 hw = &adapter->hw; 3368 hw->back = adapter; 3369 adapter->msg_enable = (1 << DEFAULT_DEBUG_LEVEL_SHIFT) - 1; 3370 3371 /* 3372 * call save state here in standalone driver because it relies on 3373 * adapter struct to exist, and needs to call netdev_priv 3374 */ 3375 pci_save_state(pdev); 3376 3377 hw->hw_addr = ioremap(pci_resource_start(pdev, 0), 3378 pci_resource_len(pdev, 0)); 3379 if (!hw->hw_addr) { 3380 err = -EIO; 3381 goto err_ioremap; 3382 } 3383 3384 ixgbevf_assign_netdev_ops(netdev); 3385 3386 adapter->bd_number = cards_found; 3387 3388 /* Setup hw api */ 3389 memcpy(&hw->mac.ops, ii->mac_ops, sizeof(hw->mac.ops)); 3390 hw->mac.type = ii->mac; 3391 3392 memcpy(&hw->mbx.ops, &ixgbevf_mbx_ops, 3393 sizeof(struct ixgbe_mbx_operations)); 3394 3395 adapter->flags &= ~IXGBE_FLAG_RX_PS_CAPABLE; 3396 adapter->flags &= ~IXGBE_FLAG_RX_PS_ENABLED; 3397 adapter->flags |= IXGBE_FLAG_RX_1BUF_CAPABLE; 3398 3399 /* setup the private structure */ 3400 err = ixgbevf_sw_init(adapter); 3401 3402 netdev->hw_features = NETIF_F_SG | 3403 NETIF_F_IP_CSUM | 3404 NETIF_F_IPV6_CSUM | 3405 NETIF_F_TSO | 3406 NETIF_F_TSO6 | 3407 NETIF_F_RXCSUM; 3408 3409 netdev->features = netdev->hw_features | 3410 NETIF_F_HW_VLAN_TX | 3411 NETIF_F_HW_VLAN_RX | 3412 NETIF_F_HW_VLAN_FILTER; 3413 3414 netdev->vlan_features |= NETIF_F_TSO; 3415 netdev->vlan_features |= NETIF_F_TSO6; 3416 netdev->vlan_features |= NETIF_F_IP_CSUM; 3417 netdev->vlan_features |= NETIF_F_IPV6_CSUM; 3418 netdev->vlan_features |= NETIF_F_SG; 3419 3420 if (pci_using_dac) 3421 netdev->features |= NETIF_F_HIGHDMA; 3422 3423 netdev->priv_flags |= IFF_UNICAST_FLT; 3424 3425 /* The HW MAC address was set and/or determined in sw_init */ 3426 memcpy(netdev->dev_addr, adapter->hw.mac.addr, netdev->addr_len); 3427 memcpy(netdev->perm_addr, adapter->hw.mac.addr, netdev->addr_len); 3428 3429 if (!is_valid_ether_addr(netdev->dev_addr)) { 3430 pr_err("invalid MAC address\n"); 3431 err = -EIO; 3432 goto err_sw_init; 3433 } 3434 3435 init_timer(&adapter->watchdog_timer); 3436 adapter->watchdog_timer.function = ixgbevf_watchdog; 3437 adapter->watchdog_timer.data = (unsigned long)adapter; 3438 3439 INIT_WORK(&adapter->reset_task, ixgbevf_reset_task); 3440 INIT_WORK(&adapter->watchdog_task, ixgbevf_watchdog_task); 3441 3442 err = ixgbevf_init_interrupt_scheme(adapter); 3443 if (err) 3444 goto err_sw_init; 3445 3446 /* pick up the PCI bus settings for reporting later */ 3447 if (hw->mac.ops.get_bus_info) 3448 hw->mac.ops.get_bus_info(hw); 3449 3450 strcpy(netdev->name, "eth%d"); 3451 3452 err = register_netdev(netdev); 3453 if (err) 3454 goto err_register; 3455 3456 adapter->netdev_registered = true; 3457 3458 netif_carrier_off(netdev); 3459 3460 ixgbevf_init_last_counter_stats(adapter); 3461 3462 /* print the MAC address */ 3463 hw_dbg(hw, "%2.2x:%2.2x:%2.2x:%2.2x:%2.2x:%2.2x\n", 3464 netdev->dev_addr[0], 3465 netdev->dev_addr[1], 3466 netdev->dev_addr[2], 3467 netdev->dev_addr[3], 3468 netdev->dev_addr[4], 3469 netdev->dev_addr[5]); 3470 3471 hw_dbg(hw, "MAC: %d\n", hw->mac.type); 3472 3473 hw_dbg(hw, "LRO is disabled\n"); 3474 3475 hw_dbg(hw, "Intel(R) 82599 Virtual Function\n"); 3476 cards_found++; 3477 return 0; 3478 3479err_register: 3480err_sw_init: 3481 ixgbevf_reset_interrupt_capability(adapter); 3482 iounmap(hw->hw_addr); 3483err_ioremap: 3484 free_netdev(netdev); 3485err_alloc_etherdev: 3486 pci_release_regions(pdev); 3487err_pci_reg: 3488err_dma: 3489 pci_disable_device(pdev); 3490 return err; 3491} 3492 3493/** 3494 * ixgbevf_remove - Device Removal Routine 3495 * @pdev: PCI device information struct 3496 * 3497 * ixgbevf_remove is called by the PCI subsystem to alert the driver 3498 * that it should release a PCI device. The could be caused by a 3499 * Hot-Plug event, or because the driver is going to be removed from 3500 * memory. 3501 **/ 3502static void __devexit ixgbevf_remove(struct pci_dev *pdev) 3503{ 3504 struct net_device *netdev = pci_get_drvdata(pdev); 3505 struct ixgbevf_adapter *adapter = netdev_priv(netdev); 3506 3507 set_bit(__IXGBEVF_DOWN, &adapter->state); 3508 3509 del_timer_sync(&adapter->watchdog_timer); 3510 3511 cancel_work_sync(&adapter->reset_task); 3512 cancel_work_sync(&adapter->watchdog_task); 3513 3514 if (adapter->netdev_registered) { 3515 unregister_netdev(netdev); 3516 adapter->netdev_registered = false; 3517 } 3518 3519 ixgbevf_reset_interrupt_capability(adapter); 3520 3521 iounmap(adapter->hw.hw_addr); 3522 pci_release_regions(pdev); 3523 3524 hw_dbg(&adapter->hw, "Remove complete\n"); 3525 3526 kfree(adapter->tx_ring); 3527 kfree(adapter->rx_ring); 3528 3529 free_netdev(netdev); 3530 3531 pci_disable_device(pdev); 3532} 3533 3534static struct pci_driver ixgbevf_driver = { 3535 .name = ixgbevf_driver_name, 3536 .id_table = ixgbevf_pci_tbl, 3537 .probe = ixgbevf_probe, 3538 .remove = __devexit_p(ixgbevf_remove), 3539 .shutdown = ixgbevf_shutdown, 3540}; 3541 3542/** 3543 * ixgbevf_init_module - Driver Registration Routine 3544 * 3545 * ixgbevf_init_module is the first routine called when the driver is 3546 * loaded. All it does is register with the PCI subsystem. 3547 **/ 3548static int __init ixgbevf_init_module(void) 3549{ 3550 int ret; 3551 pr_info("%s - version %s\n", ixgbevf_driver_string, 3552 ixgbevf_driver_version); 3553 3554 pr_info("%s\n", ixgbevf_copyright); 3555 3556 ret = pci_register_driver(&ixgbevf_driver); 3557 return ret; 3558} 3559 3560module_init(ixgbevf_init_module); 3561 3562/** 3563 * ixgbevf_exit_module - Driver Exit Cleanup Routine 3564 * 3565 * ixgbevf_exit_module is called just before the driver is removed 3566 * from memory. 3567 **/ 3568static void __exit ixgbevf_exit_module(void) 3569{ 3570 pci_unregister_driver(&ixgbevf_driver); 3571} 3572 3573#ifdef DEBUG 3574/** 3575 * ixgbevf_get_hw_dev_name - return device name string 3576 * used by hardware layer to print debugging information 3577 **/ 3578char *ixgbevf_get_hw_dev_name(struct ixgbe_hw *hw) 3579{ 3580 struct ixgbevf_adapter *adapter = hw->back; 3581 return adapter->netdev->name; 3582} 3583 3584#endif 3585module_exit(ixgbevf_exit_module); 3586 3587/* ixgbevf_main.c */ 3588