ixgbevf_main.c revision dd1ed3b7bfed15f6162f63840941e9cf4f3611a1
1/******************************************************************************* 2 3 Intel 82599 Virtual Function driver 4 Copyright(c) 1999 - 2010 Intel Corporation. 5 6 This program is free software; you can redistribute it and/or modify it 7 under the terms and conditions of the GNU General Public License, 8 version 2, as published by the Free Software Foundation. 9 10 This program is distributed in the hope it will be useful, but WITHOUT 11 ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or 12 FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for 13 more details. 14 15 You should have received a copy of the GNU General Public License along with 16 this program; if not, write to the Free Software Foundation, Inc., 17 51 Franklin St - Fifth Floor, Boston, MA 02110-1301 USA. 18 19 The full GNU General Public License is included in this distribution in 20 the file called "COPYING". 21 22 Contact Information: 23 e1000-devel Mailing List <e1000-devel@lists.sourceforge.net> 24 Intel Corporation, 5200 N.E. Elam Young Parkway, Hillsboro, OR 97124-6497 25 26*******************************************************************************/ 27 28 29/****************************************************************************** 30 Copyright (c)2006 - 2007 Myricom, Inc. for some LRO specific code 31******************************************************************************/ 32#include <linux/types.h> 33#include <linux/bitops.h> 34#include <linux/module.h> 35#include <linux/pci.h> 36#include <linux/netdevice.h> 37#include <linux/vmalloc.h> 38#include <linux/string.h> 39#include <linux/in.h> 40#include <linux/ip.h> 41#include <linux/tcp.h> 42#include <linux/ipv6.h> 43#include <linux/slab.h> 44#include <net/checksum.h> 45#include <net/ip6_checksum.h> 46#include <linux/ethtool.h> 47#include <linux/if.h> 48#include <linux/if_vlan.h> 49#include <linux/prefetch.h> 50 51#include "ixgbevf.h" 52 53char ixgbevf_driver_name[] = "ixgbevf"; 54static const char ixgbevf_driver_string[] = 55 "Intel(R) 10 Gigabit PCI Express Virtual Function Network Driver"; 56 57#define DRV_VERSION "2.1.0-k" 58const char ixgbevf_driver_version[] = DRV_VERSION; 59static char ixgbevf_copyright[] = 60 "Copyright (c) 2009 - 2010 Intel Corporation."; 61 62static const struct ixgbevf_info *ixgbevf_info_tbl[] = { 63 [board_82599_vf] = &ixgbevf_82599_vf_info, 64 [board_X540_vf] = &ixgbevf_X540_vf_info, 65}; 66 67/* ixgbevf_pci_tbl - PCI Device ID Table 68 * 69 * Wildcard entries (PCI_ANY_ID) should come last 70 * Last entry must be all 0s 71 * 72 * { Vendor ID, Device ID, SubVendor ID, SubDevice ID, 73 * Class, Class Mask, private data (not used) } 74 */ 75static struct pci_device_id ixgbevf_pci_tbl[] = { 76 {PCI_VDEVICE(INTEL, IXGBE_DEV_ID_82599_VF), 77 board_82599_vf}, 78 {PCI_VDEVICE(INTEL, IXGBE_DEV_ID_X540_VF), 79 board_X540_vf}, 80 81 /* required last entry */ 82 {0, } 83}; 84MODULE_DEVICE_TABLE(pci, ixgbevf_pci_tbl); 85 86MODULE_AUTHOR("Intel Corporation, <linux.nics@intel.com>"); 87MODULE_DESCRIPTION("Intel(R) 82599 Virtual Function Driver"); 88MODULE_LICENSE("GPL"); 89MODULE_VERSION(DRV_VERSION); 90 91#define DEFAULT_DEBUG_LEVEL_SHIFT 3 92 93/* forward decls */ 94static void ixgbevf_set_itr_msix(struct ixgbevf_q_vector *q_vector); 95static void ixgbevf_write_eitr(struct ixgbevf_adapter *adapter, int v_idx, 96 u32 itr_reg); 97 98static inline void ixgbevf_release_rx_desc(struct ixgbe_hw *hw, 99 struct ixgbevf_ring *rx_ring, 100 u32 val) 101{ 102 /* 103 * Force memory writes to complete before letting h/w 104 * know there are new descriptors to fetch. (Only 105 * applicable for weak-ordered memory model archs, 106 * such as IA-64). 107 */ 108 wmb(); 109 IXGBE_WRITE_REG(hw, IXGBE_VFRDT(rx_ring->reg_idx), val); 110} 111 112/* 113 * ixgbevf_set_ivar - set IVAR registers - maps interrupt causes to vectors 114 * @adapter: pointer to adapter struct 115 * @direction: 0 for Rx, 1 for Tx, -1 for other causes 116 * @queue: queue to map the corresponding interrupt to 117 * @msix_vector: the vector to map to the corresponding queue 118 * 119 */ 120static void ixgbevf_set_ivar(struct ixgbevf_adapter *adapter, s8 direction, 121 u8 queue, u8 msix_vector) 122{ 123 u32 ivar, index; 124 struct ixgbe_hw *hw = &adapter->hw; 125 if (direction == -1) { 126 /* other causes */ 127 msix_vector |= IXGBE_IVAR_ALLOC_VAL; 128 ivar = IXGBE_READ_REG(hw, IXGBE_VTIVAR_MISC); 129 ivar &= ~0xFF; 130 ivar |= msix_vector; 131 IXGBE_WRITE_REG(hw, IXGBE_VTIVAR_MISC, ivar); 132 } else { 133 /* tx or rx causes */ 134 msix_vector |= IXGBE_IVAR_ALLOC_VAL; 135 index = ((16 * (queue & 1)) + (8 * direction)); 136 ivar = IXGBE_READ_REG(hw, IXGBE_VTIVAR(queue >> 1)); 137 ivar &= ~(0xFF << index); 138 ivar |= (msix_vector << index); 139 IXGBE_WRITE_REG(hw, IXGBE_VTIVAR(queue >> 1), ivar); 140 } 141} 142 143static void ixgbevf_unmap_and_free_tx_resource(struct ixgbevf_adapter *adapter, 144 struct ixgbevf_tx_buffer 145 *tx_buffer_info) 146{ 147 if (tx_buffer_info->dma) { 148 if (tx_buffer_info->mapped_as_page) 149 dma_unmap_page(&adapter->pdev->dev, 150 tx_buffer_info->dma, 151 tx_buffer_info->length, 152 DMA_TO_DEVICE); 153 else 154 dma_unmap_single(&adapter->pdev->dev, 155 tx_buffer_info->dma, 156 tx_buffer_info->length, 157 DMA_TO_DEVICE); 158 tx_buffer_info->dma = 0; 159 } 160 if (tx_buffer_info->skb) { 161 dev_kfree_skb_any(tx_buffer_info->skb); 162 tx_buffer_info->skb = NULL; 163 } 164 tx_buffer_info->time_stamp = 0; 165 /* tx_buffer_info must be completely set up in the transmit path */ 166} 167 168#define IXGBE_MAX_TXD_PWR 14 169#define IXGBE_MAX_DATA_PER_TXD (1 << IXGBE_MAX_TXD_PWR) 170 171/* Tx Descriptors needed, worst case */ 172#define TXD_USE_COUNT(S) (((S) >> IXGBE_MAX_TXD_PWR) + \ 173 (((S) & (IXGBE_MAX_DATA_PER_TXD - 1)) ? 1 : 0)) 174#ifdef MAX_SKB_FRAGS 175#define DESC_NEEDED (TXD_USE_COUNT(IXGBE_MAX_DATA_PER_TXD) /* skb->data */ + \ 176 MAX_SKB_FRAGS * TXD_USE_COUNT(PAGE_SIZE) + 1) /* for context */ 177#else 178#define DESC_NEEDED TXD_USE_COUNT(IXGBE_MAX_DATA_PER_TXD) 179#endif 180 181static void ixgbevf_tx_timeout(struct net_device *netdev); 182 183/** 184 * ixgbevf_clean_tx_irq - Reclaim resources after transmit completes 185 * @adapter: board private structure 186 * @tx_ring: tx ring to clean 187 **/ 188static bool ixgbevf_clean_tx_irq(struct ixgbevf_adapter *adapter, 189 struct ixgbevf_ring *tx_ring) 190{ 191 struct net_device *netdev = adapter->netdev; 192 struct ixgbe_hw *hw = &adapter->hw; 193 union ixgbe_adv_tx_desc *tx_desc, *eop_desc; 194 struct ixgbevf_tx_buffer *tx_buffer_info; 195 unsigned int i, eop, count = 0; 196 unsigned int total_bytes = 0, total_packets = 0; 197 198 i = tx_ring->next_to_clean; 199 eop = tx_ring->tx_buffer_info[i].next_to_watch; 200 eop_desc = IXGBE_TX_DESC_ADV(*tx_ring, eop); 201 202 while ((eop_desc->wb.status & cpu_to_le32(IXGBE_TXD_STAT_DD)) && 203 (count < tx_ring->work_limit)) { 204 bool cleaned = false; 205 rmb(); /* read buffer_info after eop_desc */ 206 /* eop could change between read and DD-check */ 207 if (unlikely(eop != tx_ring->tx_buffer_info[i].next_to_watch)) 208 goto cont_loop; 209 for ( ; !cleaned; count++) { 210 struct sk_buff *skb; 211 tx_desc = IXGBE_TX_DESC_ADV(*tx_ring, i); 212 tx_buffer_info = &tx_ring->tx_buffer_info[i]; 213 cleaned = (i == eop); 214 skb = tx_buffer_info->skb; 215 216 if (cleaned && skb) { 217 unsigned int segs, bytecount; 218 219 /* gso_segs is currently only valid for tcp */ 220 segs = skb_shinfo(skb)->gso_segs ?: 1; 221 /* multiply data chunks by size of headers */ 222 bytecount = ((segs - 1) * skb_headlen(skb)) + 223 skb->len; 224 total_packets += segs; 225 total_bytes += bytecount; 226 } 227 228 ixgbevf_unmap_and_free_tx_resource(adapter, 229 tx_buffer_info); 230 231 tx_desc->wb.status = 0; 232 233 i++; 234 if (i == tx_ring->count) 235 i = 0; 236 } 237 238cont_loop: 239 eop = tx_ring->tx_buffer_info[i].next_to_watch; 240 eop_desc = IXGBE_TX_DESC_ADV(*tx_ring, eop); 241 } 242 243 tx_ring->next_to_clean = i; 244 245#define TX_WAKE_THRESHOLD (DESC_NEEDED * 2) 246 if (unlikely(count && netif_carrier_ok(netdev) && 247 (IXGBE_DESC_UNUSED(tx_ring) >= TX_WAKE_THRESHOLD))) { 248 /* Make sure that anybody stopping the queue after this 249 * sees the new next_to_clean. 250 */ 251 smp_mb(); 252#ifdef HAVE_TX_MQ 253 if (__netif_subqueue_stopped(netdev, tx_ring->queue_index) && 254 !test_bit(__IXGBEVF_DOWN, &adapter->state)) { 255 netif_wake_subqueue(netdev, tx_ring->queue_index); 256 ++adapter->restart_queue; 257 } 258#else 259 if (netif_queue_stopped(netdev) && 260 !test_bit(__IXGBEVF_DOWN, &adapter->state)) { 261 netif_wake_queue(netdev); 262 ++adapter->restart_queue; 263 } 264#endif 265 } 266 267 /* re-arm the interrupt */ 268 if ((count >= tx_ring->work_limit) && 269 (!test_bit(__IXGBEVF_DOWN, &adapter->state))) { 270 IXGBE_WRITE_REG(hw, IXGBE_VTEICS, tx_ring->v_idx); 271 } 272 273 u64_stats_update_begin(&tx_ring->syncp); 274 tx_ring->total_bytes += total_bytes; 275 tx_ring->total_packets += total_packets; 276 u64_stats_update_end(&tx_ring->syncp); 277 278 return count < tx_ring->work_limit; 279} 280 281/** 282 * ixgbevf_receive_skb - Send a completed packet up the stack 283 * @q_vector: structure containing interrupt and ring information 284 * @skb: packet to send up 285 * @status: hardware indication of status of receive 286 * @rx_ring: rx descriptor ring (for a specific queue) to setup 287 * @rx_desc: rx descriptor 288 **/ 289static void ixgbevf_receive_skb(struct ixgbevf_q_vector *q_vector, 290 struct sk_buff *skb, u8 status, 291 struct ixgbevf_ring *ring, 292 union ixgbe_adv_rx_desc *rx_desc) 293{ 294 struct ixgbevf_adapter *adapter = q_vector->adapter; 295 bool is_vlan = (status & IXGBE_RXD_STAT_VP); 296 u16 tag = le16_to_cpu(rx_desc->wb.upper.vlan); 297 298 if (is_vlan && test_bit(tag, adapter->active_vlans)) 299 __vlan_hwaccel_put_tag(skb, tag); 300 301 if (!(adapter->flags & IXGBE_FLAG_IN_NETPOLL)) 302 napi_gro_receive(&q_vector->napi, skb); 303 else 304 netif_rx(skb); 305} 306 307/** 308 * ixgbevf_rx_checksum - indicate in skb if hw indicated a good cksum 309 * @adapter: address of board private structure 310 * @status_err: hardware indication of status of receive 311 * @skb: skb currently being received and modified 312 **/ 313static inline void ixgbevf_rx_checksum(struct ixgbevf_adapter *adapter, 314 u32 status_err, struct sk_buff *skb) 315{ 316 skb_checksum_none_assert(skb); 317 318 /* Rx csum disabled */ 319 if (!(adapter->flags & IXGBE_FLAG_RX_CSUM_ENABLED)) 320 return; 321 322 /* if IP and error */ 323 if ((status_err & IXGBE_RXD_STAT_IPCS) && 324 (status_err & IXGBE_RXDADV_ERR_IPE)) { 325 adapter->hw_csum_rx_error++; 326 return; 327 } 328 329 if (!(status_err & IXGBE_RXD_STAT_L4CS)) 330 return; 331 332 if (status_err & IXGBE_RXDADV_ERR_TCPE) { 333 adapter->hw_csum_rx_error++; 334 return; 335 } 336 337 /* It must be a TCP or UDP packet with a valid checksum */ 338 skb->ip_summed = CHECKSUM_UNNECESSARY; 339 adapter->hw_csum_rx_good++; 340} 341 342/** 343 * ixgbevf_alloc_rx_buffers - Replace used receive buffers; packet split 344 * @adapter: address of board private structure 345 **/ 346static void ixgbevf_alloc_rx_buffers(struct ixgbevf_adapter *adapter, 347 struct ixgbevf_ring *rx_ring, 348 int cleaned_count) 349{ 350 struct pci_dev *pdev = adapter->pdev; 351 union ixgbe_adv_rx_desc *rx_desc; 352 struct ixgbevf_rx_buffer *bi; 353 struct sk_buff *skb; 354 unsigned int i; 355 unsigned int bufsz = rx_ring->rx_buf_len + NET_IP_ALIGN; 356 357 i = rx_ring->next_to_use; 358 bi = &rx_ring->rx_buffer_info[i]; 359 360 while (cleaned_count--) { 361 rx_desc = IXGBE_RX_DESC_ADV(*rx_ring, i); 362 363 if (!bi->page_dma && 364 (adapter->flags & IXGBE_FLAG_RX_PS_ENABLED)) { 365 if (!bi->page) { 366 bi->page = netdev_alloc_page(adapter->netdev); 367 if (!bi->page) { 368 adapter->alloc_rx_page_failed++; 369 goto no_buffers; 370 } 371 bi->page_offset = 0; 372 } else { 373 /* use a half page if we're re-using */ 374 bi->page_offset ^= (PAGE_SIZE / 2); 375 } 376 377 bi->page_dma = dma_map_page(&pdev->dev, bi->page, 378 bi->page_offset, 379 (PAGE_SIZE / 2), 380 DMA_FROM_DEVICE); 381 } 382 383 skb = bi->skb; 384 if (!skb) { 385 skb = netdev_alloc_skb(adapter->netdev, 386 bufsz); 387 388 if (!skb) { 389 adapter->alloc_rx_buff_failed++; 390 goto no_buffers; 391 } 392 393 /* 394 * Make buffer alignment 2 beyond a 16 byte boundary 395 * this will result in a 16 byte aligned IP header after 396 * the 14 byte MAC header is removed 397 */ 398 skb_reserve(skb, NET_IP_ALIGN); 399 400 bi->skb = skb; 401 } 402 if (!bi->dma) { 403 bi->dma = dma_map_single(&pdev->dev, skb->data, 404 rx_ring->rx_buf_len, 405 DMA_FROM_DEVICE); 406 } 407 /* Refresh the desc even if buffer_addrs didn't change because 408 * each write-back erases this info. */ 409 if (adapter->flags & IXGBE_FLAG_RX_PS_ENABLED) { 410 rx_desc->read.pkt_addr = cpu_to_le64(bi->page_dma); 411 rx_desc->read.hdr_addr = cpu_to_le64(bi->dma); 412 } else { 413 rx_desc->read.pkt_addr = cpu_to_le64(bi->dma); 414 } 415 416 i++; 417 if (i == rx_ring->count) 418 i = 0; 419 bi = &rx_ring->rx_buffer_info[i]; 420 } 421 422no_buffers: 423 if (rx_ring->next_to_use != i) { 424 rx_ring->next_to_use = i; 425 if (i-- == 0) 426 i = (rx_ring->count - 1); 427 428 ixgbevf_release_rx_desc(&adapter->hw, rx_ring, i); 429 } 430} 431 432static inline void ixgbevf_irq_enable_queues(struct ixgbevf_adapter *adapter, 433 u64 qmask) 434{ 435 u32 mask; 436 struct ixgbe_hw *hw = &adapter->hw; 437 438 mask = (qmask & 0xFFFFFFFF); 439 IXGBE_WRITE_REG(hw, IXGBE_VTEIMS, mask); 440} 441 442static inline u16 ixgbevf_get_hdr_info(union ixgbe_adv_rx_desc *rx_desc) 443{ 444 return rx_desc->wb.lower.lo_dword.hs_rss.hdr_info; 445} 446 447static inline u16 ixgbevf_get_pkt_info(union ixgbe_adv_rx_desc *rx_desc) 448{ 449 return rx_desc->wb.lower.lo_dword.hs_rss.pkt_info; 450} 451 452static bool ixgbevf_clean_rx_irq(struct ixgbevf_q_vector *q_vector, 453 struct ixgbevf_ring *rx_ring, 454 int *work_done, int work_to_do) 455{ 456 struct ixgbevf_adapter *adapter = q_vector->adapter; 457 struct pci_dev *pdev = adapter->pdev; 458 union ixgbe_adv_rx_desc *rx_desc, *next_rxd; 459 struct ixgbevf_rx_buffer *rx_buffer_info, *next_buffer; 460 struct sk_buff *skb; 461 unsigned int i; 462 u32 len, staterr; 463 u16 hdr_info; 464 bool cleaned = false; 465 int cleaned_count = 0; 466 unsigned int total_rx_bytes = 0, total_rx_packets = 0; 467 468 i = rx_ring->next_to_clean; 469 rx_desc = IXGBE_RX_DESC_ADV(*rx_ring, i); 470 staterr = le32_to_cpu(rx_desc->wb.upper.status_error); 471 rx_buffer_info = &rx_ring->rx_buffer_info[i]; 472 473 while (staterr & IXGBE_RXD_STAT_DD) { 474 u32 upper_len = 0; 475 if (*work_done >= work_to_do) 476 break; 477 (*work_done)++; 478 479 rmb(); /* read descriptor and rx_buffer_info after status DD */ 480 if (adapter->flags & IXGBE_FLAG_RX_PS_ENABLED) { 481 hdr_info = le16_to_cpu(ixgbevf_get_hdr_info(rx_desc)); 482 len = (hdr_info & IXGBE_RXDADV_HDRBUFLEN_MASK) >> 483 IXGBE_RXDADV_HDRBUFLEN_SHIFT; 484 if (hdr_info & IXGBE_RXDADV_SPH) 485 adapter->rx_hdr_split++; 486 if (len > IXGBEVF_RX_HDR_SIZE) 487 len = IXGBEVF_RX_HDR_SIZE; 488 upper_len = le16_to_cpu(rx_desc->wb.upper.length); 489 } else { 490 len = le16_to_cpu(rx_desc->wb.upper.length); 491 } 492 cleaned = true; 493 skb = rx_buffer_info->skb; 494 prefetch(skb->data - NET_IP_ALIGN); 495 rx_buffer_info->skb = NULL; 496 497 if (rx_buffer_info->dma) { 498 dma_unmap_single(&pdev->dev, rx_buffer_info->dma, 499 rx_ring->rx_buf_len, 500 DMA_FROM_DEVICE); 501 rx_buffer_info->dma = 0; 502 skb_put(skb, len); 503 } 504 505 if (upper_len) { 506 dma_unmap_page(&pdev->dev, rx_buffer_info->page_dma, 507 PAGE_SIZE / 2, DMA_FROM_DEVICE); 508 rx_buffer_info->page_dma = 0; 509 skb_fill_page_desc(skb, skb_shinfo(skb)->nr_frags, 510 rx_buffer_info->page, 511 rx_buffer_info->page_offset, 512 upper_len); 513 514 if ((rx_ring->rx_buf_len > (PAGE_SIZE / 2)) || 515 (page_count(rx_buffer_info->page) != 1)) 516 rx_buffer_info->page = NULL; 517 else 518 get_page(rx_buffer_info->page); 519 520 skb->len += upper_len; 521 skb->data_len += upper_len; 522 skb->truesize += upper_len; 523 } 524 525 i++; 526 if (i == rx_ring->count) 527 i = 0; 528 529 next_rxd = IXGBE_RX_DESC_ADV(*rx_ring, i); 530 prefetch(next_rxd); 531 cleaned_count++; 532 533 next_buffer = &rx_ring->rx_buffer_info[i]; 534 535 if (!(staterr & IXGBE_RXD_STAT_EOP)) { 536 if (adapter->flags & IXGBE_FLAG_RX_PS_ENABLED) { 537 rx_buffer_info->skb = next_buffer->skb; 538 rx_buffer_info->dma = next_buffer->dma; 539 next_buffer->skb = skb; 540 next_buffer->dma = 0; 541 } else { 542 skb->next = next_buffer->skb; 543 skb->next->prev = skb; 544 } 545 adapter->non_eop_descs++; 546 goto next_desc; 547 } 548 549 /* ERR_MASK will only have valid bits if EOP set */ 550 if (unlikely(staterr & IXGBE_RXDADV_ERR_FRAME_ERR_MASK)) { 551 dev_kfree_skb_irq(skb); 552 goto next_desc; 553 } 554 555 ixgbevf_rx_checksum(adapter, staterr, skb); 556 557 /* probably a little skewed due to removing CRC */ 558 total_rx_bytes += skb->len; 559 total_rx_packets++; 560 561 /* 562 * Work around issue of some types of VM to VM loop back 563 * packets not getting split correctly 564 */ 565 if (staterr & IXGBE_RXD_STAT_LB) { 566 u32 header_fixup_len = skb_headlen(skb); 567 if (header_fixup_len < 14) 568 skb_push(skb, header_fixup_len); 569 } 570 skb->protocol = eth_type_trans(skb, adapter->netdev); 571 572 ixgbevf_receive_skb(q_vector, skb, staterr, rx_ring, rx_desc); 573 574next_desc: 575 rx_desc->wb.upper.status_error = 0; 576 577 /* return some buffers to hardware, one at a time is too slow */ 578 if (cleaned_count >= IXGBEVF_RX_BUFFER_WRITE) { 579 ixgbevf_alloc_rx_buffers(adapter, rx_ring, 580 cleaned_count); 581 cleaned_count = 0; 582 } 583 584 /* use prefetched values */ 585 rx_desc = next_rxd; 586 rx_buffer_info = &rx_ring->rx_buffer_info[i]; 587 588 staterr = le32_to_cpu(rx_desc->wb.upper.status_error); 589 } 590 591 rx_ring->next_to_clean = i; 592 cleaned_count = IXGBE_DESC_UNUSED(rx_ring); 593 594 if (cleaned_count) 595 ixgbevf_alloc_rx_buffers(adapter, rx_ring, cleaned_count); 596 597 u64_stats_update_begin(&rx_ring->syncp); 598 rx_ring->total_packets += total_rx_packets; 599 rx_ring->total_bytes += total_rx_bytes; 600 u64_stats_update_end(&rx_ring->syncp); 601 602 return cleaned; 603} 604 605/** 606 * ixgbevf_clean_rxonly - msix (aka one shot) rx clean routine 607 * @napi: napi struct with our devices info in it 608 * @budget: amount of work driver is allowed to do this pass, in packets 609 * 610 * This function is optimized for cleaning one queue only on a single 611 * q_vector!!! 612 **/ 613static int ixgbevf_clean_rxonly(struct napi_struct *napi, int budget) 614{ 615 struct ixgbevf_q_vector *q_vector = 616 container_of(napi, struct ixgbevf_q_vector, napi); 617 struct ixgbevf_adapter *adapter = q_vector->adapter; 618 struct ixgbevf_ring *rx_ring = NULL; 619 int work_done = 0; 620 long r_idx; 621 622 r_idx = find_first_bit(q_vector->rxr_idx, adapter->num_rx_queues); 623 rx_ring = &(adapter->rx_ring[r_idx]); 624 625 ixgbevf_clean_rx_irq(q_vector, rx_ring, &work_done, budget); 626 627 /* If all Rx work done, exit the polling mode */ 628 if (work_done < budget) { 629 napi_complete(napi); 630 if (adapter->itr_setting & 1) 631 ixgbevf_set_itr_msix(q_vector); 632 if (!test_bit(__IXGBEVF_DOWN, &adapter->state)) 633 ixgbevf_irq_enable_queues(adapter, rx_ring->v_idx); 634 } 635 636 return work_done; 637} 638 639/** 640 * ixgbevf_clean_rxonly_many - msix (aka one shot) rx clean routine 641 * @napi: napi struct with our devices info in it 642 * @budget: amount of work driver is allowed to do this pass, in packets 643 * 644 * This function will clean more than one rx queue associated with a 645 * q_vector. 646 **/ 647static int ixgbevf_clean_rxonly_many(struct napi_struct *napi, int budget) 648{ 649 struct ixgbevf_q_vector *q_vector = 650 container_of(napi, struct ixgbevf_q_vector, napi); 651 struct ixgbevf_adapter *adapter = q_vector->adapter; 652 struct ixgbevf_ring *rx_ring = NULL; 653 int work_done = 0, i; 654 long r_idx; 655 u64 enable_mask = 0; 656 657 /* attempt to distribute budget to each queue fairly, but don't allow 658 * the budget to go below 1 because we'll exit polling */ 659 budget /= (q_vector->rxr_count ?: 1); 660 budget = max(budget, 1); 661 r_idx = find_first_bit(q_vector->rxr_idx, adapter->num_rx_queues); 662 for (i = 0; i < q_vector->rxr_count; i++) { 663 rx_ring = &(adapter->rx_ring[r_idx]); 664 ixgbevf_clean_rx_irq(q_vector, rx_ring, &work_done, budget); 665 enable_mask |= rx_ring->v_idx; 666 r_idx = find_next_bit(q_vector->rxr_idx, adapter->num_rx_queues, 667 r_idx + 1); 668 } 669 670#ifndef HAVE_NETDEV_NAPI_LIST 671 if (!netif_running(adapter->netdev)) 672 work_done = 0; 673 674#endif 675 r_idx = find_first_bit(q_vector->rxr_idx, adapter->num_rx_queues); 676 rx_ring = &(adapter->rx_ring[r_idx]); 677 678 /* If all Rx work done, exit the polling mode */ 679 if (work_done < budget) { 680 napi_complete(napi); 681 if (adapter->itr_setting & 1) 682 ixgbevf_set_itr_msix(q_vector); 683 if (!test_bit(__IXGBEVF_DOWN, &adapter->state)) 684 ixgbevf_irq_enable_queues(adapter, enable_mask); 685 } 686 687 return work_done; 688} 689 690 691/** 692 * ixgbevf_configure_msix - Configure MSI-X hardware 693 * @adapter: board private structure 694 * 695 * ixgbevf_configure_msix sets up the hardware to properly generate MSI-X 696 * interrupts. 697 **/ 698static void ixgbevf_configure_msix(struct ixgbevf_adapter *adapter) 699{ 700 struct ixgbevf_q_vector *q_vector; 701 struct ixgbe_hw *hw = &adapter->hw; 702 int i, j, q_vectors, v_idx, r_idx; 703 u32 mask; 704 705 q_vectors = adapter->num_msix_vectors - NON_Q_VECTORS; 706 707 /* 708 * Populate the IVAR table and set the ITR values to the 709 * corresponding register. 710 */ 711 for (v_idx = 0; v_idx < q_vectors; v_idx++) { 712 q_vector = adapter->q_vector[v_idx]; 713 /* XXX for_each_set_bit(...) */ 714 r_idx = find_first_bit(q_vector->rxr_idx, 715 adapter->num_rx_queues); 716 717 for (i = 0; i < q_vector->rxr_count; i++) { 718 j = adapter->rx_ring[r_idx].reg_idx; 719 ixgbevf_set_ivar(adapter, 0, j, v_idx); 720 r_idx = find_next_bit(q_vector->rxr_idx, 721 adapter->num_rx_queues, 722 r_idx + 1); 723 } 724 r_idx = find_first_bit(q_vector->txr_idx, 725 adapter->num_tx_queues); 726 727 for (i = 0; i < q_vector->txr_count; i++) { 728 j = adapter->tx_ring[r_idx].reg_idx; 729 ixgbevf_set_ivar(adapter, 1, j, v_idx); 730 r_idx = find_next_bit(q_vector->txr_idx, 731 adapter->num_tx_queues, 732 r_idx + 1); 733 } 734 735 /* if this is a tx only vector halve the interrupt rate */ 736 if (q_vector->txr_count && !q_vector->rxr_count) 737 q_vector->eitr = (adapter->eitr_param >> 1); 738 else if (q_vector->rxr_count) 739 /* rx only */ 740 q_vector->eitr = adapter->eitr_param; 741 742 ixgbevf_write_eitr(adapter, v_idx, q_vector->eitr); 743 } 744 745 ixgbevf_set_ivar(adapter, -1, 1, v_idx); 746 747 /* set up to autoclear timer, and the vectors */ 748 mask = IXGBE_EIMS_ENABLE_MASK; 749 mask &= ~IXGBE_EIMS_OTHER; 750 IXGBE_WRITE_REG(hw, IXGBE_VTEIAC, mask); 751} 752 753enum latency_range { 754 lowest_latency = 0, 755 low_latency = 1, 756 bulk_latency = 2, 757 latency_invalid = 255 758}; 759 760/** 761 * ixgbevf_update_itr - update the dynamic ITR value based on statistics 762 * @adapter: pointer to adapter 763 * @eitr: eitr setting (ints per sec) to give last timeslice 764 * @itr_setting: current throttle rate in ints/second 765 * @packets: the number of packets during this measurement interval 766 * @bytes: the number of bytes during this measurement interval 767 * 768 * Stores a new ITR value based on packets and byte 769 * counts during the last interrupt. The advantage of per interrupt 770 * computation is faster updates and more accurate ITR for the current 771 * traffic pattern. Constants in this function were computed 772 * based on theoretical maximum wire speed and thresholds were set based 773 * on testing data as well as attempting to minimize response time 774 * while increasing bulk throughput. 775 **/ 776static u8 ixgbevf_update_itr(struct ixgbevf_adapter *adapter, 777 u32 eitr, u8 itr_setting, 778 int packets, int bytes) 779{ 780 unsigned int retval = itr_setting; 781 u32 timepassed_us; 782 u64 bytes_perint; 783 784 if (packets == 0) 785 goto update_itr_done; 786 787 788 /* simple throttlerate management 789 * 0-20MB/s lowest (100000 ints/s) 790 * 20-100MB/s low (20000 ints/s) 791 * 100-1249MB/s bulk (8000 ints/s) 792 */ 793 /* what was last interrupt timeslice? */ 794 timepassed_us = 1000000/eitr; 795 bytes_perint = bytes / timepassed_us; /* bytes/usec */ 796 797 switch (itr_setting) { 798 case lowest_latency: 799 if (bytes_perint > adapter->eitr_low) 800 retval = low_latency; 801 break; 802 case low_latency: 803 if (bytes_perint > adapter->eitr_high) 804 retval = bulk_latency; 805 else if (bytes_perint <= adapter->eitr_low) 806 retval = lowest_latency; 807 break; 808 case bulk_latency: 809 if (bytes_perint <= adapter->eitr_high) 810 retval = low_latency; 811 break; 812 } 813 814update_itr_done: 815 return retval; 816} 817 818/** 819 * ixgbevf_write_eitr - write VTEITR register in hardware specific way 820 * @adapter: pointer to adapter struct 821 * @v_idx: vector index into q_vector array 822 * @itr_reg: new value to be written in *register* format, not ints/s 823 * 824 * This function is made to be called by ethtool and by the driver 825 * when it needs to update VTEITR registers at runtime. Hardware 826 * specific quirks/differences are taken care of here. 827 */ 828static void ixgbevf_write_eitr(struct ixgbevf_adapter *adapter, int v_idx, 829 u32 itr_reg) 830{ 831 struct ixgbe_hw *hw = &adapter->hw; 832 833 itr_reg = EITR_INTS_PER_SEC_TO_REG(itr_reg); 834 835 /* 836 * set the WDIS bit to not clear the timer bits and cause an 837 * immediate assertion of the interrupt 838 */ 839 itr_reg |= IXGBE_EITR_CNT_WDIS; 840 841 IXGBE_WRITE_REG(hw, IXGBE_VTEITR(v_idx), itr_reg); 842} 843 844static void ixgbevf_set_itr_msix(struct ixgbevf_q_vector *q_vector) 845{ 846 struct ixgbevf_adapter *adapter = q_vector->adapter; 847 u32 new_itr; 848 u8 current_itr, ret_itr; 849 int i, r_idx, v_idx = q_vector->v_idx; 850 struct ixgbevf_ring *rx_ring, *tx_ring; 851 852 r_idx = find_first_bit(q_vector->txr_idx, adapter->num_tx_queues); 853 for (i = 0; i < q_vector->txr_count; i++) { 854 tx_ring = &(adapter->tx_ring[r_idx]); 855 ret_itr = ixgbevf_update_itr(adapter, q_vector->eitr, 856 q_vector->tx_itr, 857 tx_ring->total_packets, 858 tx_ring->total_bytes); 859 /* if the result for this queue would decrease interrupt 860 * rate for this vector then use that result */ 861 q_vector->tx_itr = ((q_vector->tx_itr > ret_itr) ? 862 q_vector->tx_itr - 1 : ret_itr); 863 r_idx = find_next_bit(q_vector->txr_idx, adapter->num_tx_queues, 864 r_idx + 1); 865 } 866 867 r_idx = find_first_bit(q_vector->rxr_idx, adapter->num_rx_queues); 868 for (i = 0; i < q_vector->rxr_count; i++) { 869 rx_ring = &(adapter->rx_ring[r_idx]); 870 ret_itr = ixgbevf_update_itr(adapter, q_vector->eitr, 871 q_vector->rx_itr, 872 rx_ring->total_packets, 873 rx_ring->total_bytes); 874 /* if the result for this queue would decrease interrupt 875 * rate for this vector then use that result */ 876 q_vector->rx_itr = ((q_vector->rx_itr > ret_itr) ? 877 q_vector->rx_itr - 1 : ret_itr); 878 r_idx = find_next_bit(q_vector->rxr_idx, adapter->num_rx_queues, 879 r_idx + 1); 880 } 881 882 current_itr = max(q_vector->rx_itr, q_vector->tx_itr); 883 884 switch (current_itr) { 885 /* counts and packets in update_itr are dependent on these numbers */ 886 case lowest_latency: 887 new_itr = 100000; 888 break; 889 case low_latency: 890 new_itr = 20000; /* aka hwitr = ~200 */ 891 break; 892 case bulk_latency: 893 default: 894 new_itr = 8000; 895 break; 896 } 897 898 if (new_itr != q_vector->eitr) { 899 u32 itr_reg; 900 901 /* save the algorithm value here, not the smoothed one */ 902 q_vector->eitr = new_itr; 903 /* do an exponential smoothing */ 904 new_itr = ((q_vector->eitr * 90)/100) + ((new_itr * 10)/100); 905 itr_reg = EITR_INTS_PER_SEC_TO_REG(new_itr); 906 ixgbevf_write_eitr(adapter, v_idx, itr_reg); 907 } 908} 909 910static irqreturn_t ixgbevf_msix_mbx(int irq, void *data) 911{ 912 struct net_device *netdev = data; 913 struct ixgbevf_adapter *adapter = netdev_priv(netdev); 914 struct ixgbe_hw *hw = &adapter->hw; 915 u32 eicr; 916 u32 msg; 917 918 eicr = IXGBE_READ_REG(hw, IXGBE_VTEICS); 919 IXGBE_WRITE_REG(hw, IXGBE_VTEICR, eicr); 920 921 if (!hw->mbx.ops.check_for_ack(hw)) { 922 /* 923 * checking for the ack clears the PFACK bit. Place 924 * it back in the v2p_mailbox cache so that anyone 925 * polling for an ack will not miss it. Also 926 * avoid the read below because the code to read 927 * the mailbox will also clear the ack bit. This was 928 * causing lost acks. Just cache the bit and exit 929 * the IRQ handler. 930 */ 931 hw->mbx.v2p_mailbox |= IXGBE_VFMAILBOX_PFACK; 932 goto out; 933 } 934 935 /* Not an ack interrupt, go ahead and read the message */ 936 hw->mbx.ops.read(hw, &msg, 1); 937 938 if ((msg & IXGBE_MBVFICR_VFREQ_MASK) == IXGBE_PF_CONTROL_MSG) 939 mod_timer(&adapter->watchdog_timer, 940 round_jiffies(jiffies + 1)); 941 942out: 943 return IRQ_HANDLED; 944} 945 946static irqreturn_t ixgbevf_msix_clean_tx(int irq, void *data) 947{ 948 struct ixgbevf_q_vector *q_vector = data; 949 struct ixgbevf_adapter *adapter = q_vector->adapter; 950 struct ixgbevf_ring *tx_ring; 951 int i, r_idx; 952 953 if (!q_vector->txr_count) 954 return IRQ_HANDLED; 955 956 r_idx = find_first_bit(q_vector->txr_idx, adapter->num_tx_queues); 957 for (i = 0; i < q_vector->txr_count; i++) { 958 tx_ring = &(adapter->tx_ring[r_idx]); 959 tx_ring->total_bytes = 0; 960 tx_ring->total_packets = 0; 961 ixgbevf_clean_tx_irq(adapter, tx_ring); 962 r_idx = find_next_bit(q_vector->txr_idx, adapter->num_tx_queues, 963 r_idx + 1); 964 } 965 966 if (adapter->itr_setting & 1) 967 ixgbevf_set_itr_msix(q_vector); 968 969 return IRQ_HANDLED; 970} 971 972/** 973 * ixgbevf_msix_clean_rx - single unshared vector rx clean (all queues) 974 * @irq: unused 975 * @data: pointer to our q_vector struct for this interrupt vector 976 **/ 977static irqreturn_t ixgbevf_msix_clean_rx(int irq, void *data) 978{ 979 struct ixgbevf_q_vector *q_vector = data; 980 struct ixgbevf_adapter *adapter = q_vector->adapter; 981 struct ixgbe_hw *hw = &adapter->hw; 982 struct ixgbevf_ring *rx_ring; 983 int r_idx; 984 int i; 985 986 r_idx = find_first_bit(q_vector->rxr_idx, adapter->num_rx_queues); 987 for (i = 0; i < q_vector->rxr_count; i++) { 988 rx_ring = &(adapter->rx_ring[r_idx]); 989 rx_ring->total_bytes = 0; 990 rx_ring->total_packets = 0; 991 r_idx = find_next_bit(q_vector->rxr_idx, adapter->num_rx_queues, 992 r_idx + 1); 993 } 994 995 if (!q_vector->rxr_count) 996 return IRQ_HANDLED; 997 998 r_idx = find_first_bit(q_vector->rxr_idx, adapter->num_rx_queues); 999 rx_ring = &(adapter->rx_ring[r_idx]); 1000 /* disable interrupts on this vector only */ 1001 IXGBE_WRITE_REG(hw, IXGBE_VTEIMC, rx_ring->v_idx); 1002 napi_schedule(&q_vector->napi); 1003 1004 1005 return IRQ_HANDLED; 1006} 1007 1008static irqreturn_t ixgbevf_msix_clean_many(int irq, void *data) 1009{ 1010 ixgbevf_msix_clean_rx(irq, data); 1011 ixgbevf_msix_clean_tx(irq, data); 1012 1013 return IRQ_HANDLED; 1014} 1015 1016static inline void map_vector_to_rxq(struct ixgbevf_adapter *a, int v_idx, 1017 int r_idx) 1018{ 1019 struct ixgbevf_q_vector *q_vector = a->q_vector[v_idx]; 1020 1021 set_bit(r_idx, q_vector->rxr_idx); 1022 q_vector->rxr_count++; 1023 a->rx_ring[r_idx].v_idx = 1 << v_idx; 1024} 1025 1026static inline void map_vector_to_txq(struct ixgbevf_adapter *a, int v_idx, 1027 int t_idx) 1028{ 1029 struct ixgbevf_q_vector *q_vector = a->q_vector[v_idx]; 1030 1031 set_bit(t_idx, q_vector->txr_idx); 1032 q_vector->txr_count++; 1033 a->tx_ring[t_idx].v_idx = 1 << v_idx; 1034} 1035 1036/** 1037 * ixgbevf_map_rings_to_vectors - Maps descriptor rings to vectors 1038 * @adapter: board private structure to initialize 1039 * 1040 * This function maps descriptor rings to the queue-specific vectors 1041 * we were allotted through the MSI-X enabling code. Ideally, we'd have 1042 * one vector per ring/queue, but on a constrained vector budget, we 1043 * group the rings as "efficiently" as possible. You would add new 1044 * mapping configurations in here. 1045 **/ 1046static int ixgbevf_map_rings_to_vectors(struct ixgbevf_adapter *adapter) 1047{ 1048 int q_vectors; 1049 int v_start = 0; 1050 int rxr_idx = 0, txr_idx = 0; 1051 int rxr_remaining = adapter->num_rx_queues; 1052 int txr_remaining = adapter->num_tx_queues; 1053 int i, j; 1054 int rqpv, tqpv; 1055 int err = 0; 1056 1057 q_vectors = adapter->num_msix_vectors - NON_Q_VECTORS; 1058 1059 /* 1060 * The ideal configuration... 1061 * We have enough vectors to map one per queue. 1062 */ 1063 if (q_vectors == adapter->num_rx_queues + adapter->num_tx_queues) { 1064 for (; rxr_idx < rxr_remaining; v_start++, rxr_idx++) 1065 map_vector_to_rxq(adapter, v_start, rxr_idx); 1066 1067 for (; txr_idx < txr_remaining; v_start++, txr_idx++) 1068 map_vector_to_txq(adapter, v_start, txr_idx); 1069 goto out; 1070 } 1071 1072 /* 1073 * If we don't have enough vectors for a 1-to-1 1074 * mapping, we'll have to group them so there are 1075 * multiple queues per vector. 1076 */ 1077 /* Re-adjusting *qpv takes care of the remainder. */ 1078 for (i = v_start; i < q_vectors; i++) { 1079 rqpv = DIV_ROUND_UP(rxr_remaining, q_vectors - i); 1080 for (j = 0; j < rqpv; j++) { 1081 map_vector_to_rxq(adapter, i, rxr_idx); 1082 rxr_idx++; 1083 rxr_remaining--; 1084 } 1085 } 1086 for (i = v_start; i < q_vectors; i++) { 1087 tqpv = DIV_ROUND_UP(txr_remaining, q_vectors - i); 1088 for (j = 0; j < tqpv; j++) { 1089 map_vector_to_txq(adapter, i, txr_idx); 1090 txr_idx++; 1091 txr_remaining--; 1092 } 1093 } 1094 1095out: 1096 return err; 1097} 1098 1099/** 1100 * ixgbevf_request_msix_irqs - Initialize MSI-X interrupts 1101 * @adapter: board private structure 1102 * 1103 * ixgbevf_request_msix_irqs allocates MSI-X vectors and requests 1104 * interrupts from the kernel. 1105 **/ 1106static int ixgbevf_request_msix_irqs(struct ixgbevf_adapter *adapter) 1107{ 1108 struct net_device *netdev = adapter->netdev; 1109 irqreturn_t (*handler)(int, void *); 1110 int i, vector, q_vectors, err; 1111 int ri = 0, ti = 0; 1112 1113 /* Decrement for Other and TCP Timer vectors */ 1114 q_vectors = adapter->num_msix_vectors - NON_Q_VECTORS; 1115 1116#define SET_HANDLER(_v) (((_v)->rxr_count && (_v)->txr_count) \ 1117 ? &ixgbevf_msix_clean_many : \ 1118 (_v)->rxr_count ? &ixgbevf_msix_clean_rx : \ 1119 (_v)->txr_count ? &ixgbevf_msix_clean_tx : \ 1120 NULL) 1121 for (vector = 0; vector < q_vectors; vector++) { 1122 handler = SET_HANDLER(adapter->q_vector[vector]); 1123 1124 if (handler == &ixgbevf_msix_clean_rx) { 1125 sprintf(adapter->name[vector], "%s-%s-%d", 1126 netdev->name, "rx", ri++); 1127 } else if (handler == &ixgbevf_msix_clean_tx) { 1128 sprintf(adapter->name[vector], "%s-%s-%d", 1129 netdev->name, "tx", ti++); 1130 } else if (handler == &ixgbevf_msix_clean_many) { 1131 sprintf(adapter->name[vector], "%s-%s-%d", 1132 netdev->name, "TxRx", vector); 1133 } else { 1134 /* skip this unused q_vector */ 1135 continue; 1136 } 1137 err = request_irq(adapter->msix_entries[vector].vector, 1138 handler, 0, adapter->name[vector], 1139 adapter->q_vector[vector]); 1140 if (err) { 1141 hw_dbg(&adapter->hw, 1142 "request_irq failed for MSIX interrupt " 1143 "Error: %d\n", err); 1144 goto free_queue_irqs; 1145 } 1146 } 1147 1148 sprintf(adapter->name[vector], "%s:mbx", netdev->name); 1149 err = request_irq(adapter->msix_entries[vector].vector, 1150 &ixgbevf_msix_mbx, 0, adapter->name[vector], netdev); 1151 if (err) { 1152 hw_dbg(&adapter->hw, 1153 "request_irq for msix_mbx failed: %d\n", err); 1154 goto free_queue_irqs; 1155 } 1156 1157 return 0; 1158 1159free_queue_irqs: 1160 for (i = vector - 1; i >= 0; i--) 1161 free_irq(adapter->msix_entries[--vector].vector, 1162 &(adapter->q_vector[i])); 1163 pci_disable_msix(adapter->pdev); 1164 kfree(adapter->msix_entries); 1165 adapter->msix_entries = NULL; 1166 return err; 1167} 1168 1169static inline void ixgbevf_reset_q_vectors(struct ixgbevf_adapter *adapter) 1170{ 1171 int i, q_vectors = adapter->num_msix_vectors - NON_Q_VECTORS; 1172 1173 for (i = 0; i < q_vectors; i++) { 1174 struct ixgbevf_q_vector *q_vector = adapter->q_vector[i]; 1175 bitmap_zero(q_vector->rxr_idx, MAX_RX_QUEUES); 1176 bitmap_zero(q_vector->txr_idx, MAX_TX_QUEUES); 1177 q_vector->rxr_count = 0; 1178 q_vector->txr_count = 0; 1179 q_vector->eitr = adapter->eitr_param; 1180 } 1181} 1182 1183/** 1184 * ixgbevf_request_irq - initialize interrupts 1185 * @adapter: board private structure 1186 * 1187 * Attempts to configure interrupts using the best available 1188 * capabilities of the hardware and kernel. 1189 **/ 1190static int ixgbevf_request_irq(struct ixgbevf_adapter *adapter) 1191{ 1192 int err = 0; 1193 1194 err = ixgbevf_request_msix_irqs(adapter); 1195 1196 if (err) 1197 hw_dbg(&adapter->hw, 1198 "request_irq failed, Error %d\n", err); 1199 1200 return err; 1201} 1202 1203static void ixgbevf_free_irq(struct ixgbevf_adapter *adapter) 1204{ 1205 struct net_device *netdev = adapter->netdev; 1206 int i, q_vectors; 1207 1208 q_vectors = adapter->num_msix_vectors; 1209 1210 i = q_vectors - 1; 1211 1212 free_irq(adapter->msix_entries[i].vector, netdev); 1213 i--; 1214 1215 for (; i >= 0; i--) { 1216 free_irq(adapter->msix_entries[i].vector, 1217 adapter->q_vector[i]); 1218 } 1219 1220 ixgbevf_reset_q_vectors(adapter); 1221} 1222 1223/** 1224 * ixgbevf_irq_disable - Mask off interrupt generation on the NIC 1225 * @adapter: board private structure 1226 **/ 1227static inline void ixgbevf_irq_disable(struct ixgbevf_adapter *adapter) 1228{ 1229 int i; 1230 struct ixgbe_hw *hw = &adapter->hw; 1231 1232 IXGBE_WRITE_REG(hw, IXGBE_VTEIMC, ~0); 1233 1234 IXGBE_WRITE_FLUSH(hw); 1235 1236 for (i = 0; i < adapter->num_msix_vectors; i++) 1237 synchronize_irq(adapter->msix_entries[i].vector); 1238} 1239 1240/** 1241 * ixgbevf_irq_enable - Enable default interrupt generation settings 1242 * @adapter: board private structure 1243 **/ 1244static inline void ixgbevf_irq_enable(struct ixgbevf_adapter *adapter, 1245 bool queues, bool flush) 1246{ 1247 struct ixgbe_hw *hw = &adapter->hw; 1248 u32 mask; 1249 u64 qmask; 1250 1251 mask = (IXGBE_EIMS_ENABLE_MASK & ~IXGBE_EIMS_RTX_QUEUE); 1252 qmask = ~0; 1253 1254 IXGBE_WRITE_REG(hw, IXGBE_VTEIMS, mask); 1255 1256 if (queues) 1257 ixgbevf_irq_enable_queues(adapter, qmask); 1258 1259 if (flush) 1260 IXGBE_WRITE_FLUSH(hw); 1261} 1262 1263/** 1264 * ixgbevf_configure_tx - Configure 82599 VF Transmit Unit after Reset 1265 * @adapter: board private structure 1266 * 1267 * Configure the Tx unit of the MAC after a reset. 1268 **/ 1269static void ixgbevf_configure_tx(struct ixgbevf_adapter *adapter) 1270{ 1271 u64 tdba; 1272 struct ixgbe_hw *hw = &adapter->hw; 1273 u32 i, j, tdlen, txctrl; 1274 1275 /* Setup the HW Tx Head and Tail descriptor pointers */ 1276 for (i = 0; i < adapter->num_tx_queues; i++) { 1277 struct ixgbevf_ring *ring = &adapter->tx_ring[i]; 1278 j = ring->reg_idx; 1279 tdba = ring->dma; 1280 tdlen = ring->count * sizeof(union ixgbe_adv_tx_desc); 1281 IXGBE_WRITE_REG(hw, IXGBE_VFTDBAL(j), 1282 (tdba & DMA_BIT_MASK(32))); 1283 IXGBE_WRITE_REG(hw, IXGBE_VFTDBAH(j), (tdba >> 32)); 1284 IXGBE_WRITE_REG(hw, IXGBE_VFTDLEN(j), tdlen); 1285 IXGBE_WRITE_REG(hw, IXGBE_VFTDH(j), 0); 1286 IXGBE_WRITE_REG(hw, IXGBE_VFTDT(j), 0); 1287 adapter->tx_ring[i].head = IXGBE_VFTDH(j); 1288 adapter->tx_ring[i].tail = IXGBE_VFTDT(j); 1289 /* Disable Tx Head Writeback RO bit, since this hoses 1290 * bookkeeping if things aren't delivered in order. 1291 */ 1292 txctrl = IXGBE_READ_REG(hw, IXGBE_VFDCA_TXCTRL(j)); 1293 txctrl &= ~IXGBE_DCA_TXCTRL_TX_WB_RO_EN; 1294 IXGBE_WRITE_REG(hw, IXGBE_VFDCA_TXCTRL(j), txctrl); 1295 } 1296} 1297 1298#define IXGBE_SRRCTL_BSIZEHDRSIZE_SHIFT 2 1299 1300static void ixgbevf_configure_srrctl(struct ixgbevf_adapter *adapter, int index) 1301{ 1302 struct ixgbevf_ring *rx_ring; 1303 struct ixgbe_hw *hw = &adapter->hw; 1304 u32 srrctl; 1305 1306 rx_ring = &adapter->rx_ring[index]; 1307 1308 srrctl = IXGBE_SRRCTL_DROP_EN; 1309 1310 if (adapter->flags & IXGBE_FLAG_RX_PS_ENABLED) { 1311 u16 bufsz = IXGBEVF_RXBUFFER_2048; 1312 /* grow the amount we can receive on large page machines */ 1313 if (bufsz < (PAGE_SIZE / 2)) 1314 bufsz = (PAGE_SIZE / 2); 1315 /* cap the bufsz at our largest descriptor size */ 1316 bufsz = min((u16)IXGBEVF_MAX_RXBUFFER, bufsz); 1317 1318 srrctl |= bufsz >> IXGBE_SRRCTL_BSIZEPKT_SHIFT; 1319 srrctl |= IXGBE_SRRCTL_DESCTYPE_HDR_SPLIT_ALWAYS; 1320 srrctl |= ((IXGBEVF_RX_HDR_SIZE << 1321 IXGBE_SRRCTL_BSIZEHDRSIZE_SHIFT) & 1322 IXGBE_SRRCTL_BSIZEHDR_MASK); 1323 } else { 1324 srrctl |= IXGBE_SRRCTL_DESCTYPE_ADV_ONEBUF; 1325 1326 if (rx_ring->rx_buf_len == MAXIMUM_ETHERNET_VLAN_SIZE) 1327 srrctl |= IXGBEVF_RXBUFFER_2048 >> 1328 IXGBE_SRRCTL_BSIZEPKT_SHIFT; 1329 else 1330 srrctl |= rx_ring->rx_buf_len >> 1331 IXGBE_SRRCTL_BSIZEPKT_SHIFT; 1332 } 1333 IXGBE_WRITE_REG(hw, IXGBE_VFSRRCTL(index), srrctl); 1334} 1335 1336/** 1337 * ixgbevf_configure_rx - Configure 82599 VF Receive Unit after Reset 1338 * @adapter: board private structure 1339 * 1340 * Configure the Rx unit of the MAC after a reset. 1341 **/ 1342static void ixgbevf_configure_rx(struct ixgbevf_adapter *adapter) 1343{ 1344 u64 rdba; 1345 struct ixgbe_hw *hw = &adapter->hw; 1346 struct net_device *netdev = adapter->netdev; 1347 int max_frame = netdev->mtu + ETH_HLEN + ETH_FCS_LEN; 1348 int i, j; 1349 u32 rdlen; 1350 int rx_buf_len; 1351 1352 /* Decide whether to use packet split mode or not */ 1353 if (netdev->mtu > ETH_DATA_LEN) { 1354 if (adapter->flags & IXGBE_FLAG_RX_PS_CAPABLE) 1355 adapter->flags |= IXGBE_FLAG_RX_PS_ENABLED; 1356 else 1357 adapter->flags &= ~IXGBE_FLAG_RX_PS_ENABLED; 1358 } else { 1359 if (adapter->flags & IXGBE_FLAG_RX_1BUF_CAPABLE) 1360 adapter->flags &= ~IXGBE_FLAG_RX_PS_ENABLED; 1361 else 1362 adapter->flags |= IXGBE_FLAG_RX_PS_ENABLED; 1363 } 1364 1365 /* Set the RX buffer length according to the mode */ 1366 if (adapter->flags & IXGBE_FLAG_RX_PS_ENABLED) { 1367 /* PSRTYPE must be initialized in 82599 */ 1368 u32 psrtype = IXGBE_PSRTYPE_TCPHDR | 1369 IXGBE_PSRTYPE_UDPHDR | 1370 IXGBE_PSRTYPE_IPV4HDR | 1371 IXGBE_PSRTYPE_IPV6HDR | 1372 IXGBE_PSRTYPE_L2HDR; 1373 IXGBE_WRITE_REG(hw, IXGBE_VFPSRTYPE, psrtype); 1374 rx_buf_len = IXGBEVF_RX_HDR_SIZE; 1375 } else { 1376 IXGBE_WRITE_REG(hw, IXGBE_VFPSRTYPE, 0); 1377 if (netdev->mtu <= ETH_DATA_LEN) 1378 rx_buf_len = MAXIMUM_ETHERNET_VLAN_SIZE; 1379 else 1380 rx_buf_len = ALIGN(max_frame, 1024); 1381 } 1382 1383 rdlen = adapter->rx_ring[0].count * sizeof(union ixgbe_adv_rx_desc); 1384 /* Setup the HW Rx Head and Tail Descriptor Pointers and 1385 * the Base and Length of the Rx Descriptor Ring */ 1386 for (i = 0; i < adapter->num_rx_queues; i++) { 1387 rdba = adapter->rx_ring[i].dma; 1388 j = adapter->rx_ring[i].reg_idx; 1389 IXGBE_WRITE_REG(hw, IXGBE_VFRDBAL(j), 1390 (rdba & DMA_BIT_MASK(32))); 1391 IXGBE_WRITE_REG(hw, IXGBE_VFRDBAH(j), (rdba >> 32)); 1392 IXGBE_WRITE_REG(hw, IXGBE_VFRDLEN(j), rdlen); 1393 IXGBE_WRITE_REG(hw, IXGBE_VFRDH(j), 0); 1394 IXGBE_WRITE_REG(hw, IXGBE_VFRDT(j), 0); 1395 adapter->rx_ring[i].head = IXGBE_VFRDH(j); 1396 adapter->rx_ring[i].tail = IXGBE_VFRDT(j); 1397 adapter->rx_ring[i].rx_buf_len = rx_buf_len; 1398 1399 ixgbevf_configure_srrctl(adapter, j); 1400 } 1401} 1402 1403static void ixgbevf_vlan_rx_add_vid(struct net_device *netdev, u16 vid) 1404{ 1405 struct ixgbevf_adapter *adapter = netdev_priv(netdev); 1406 struct ixgbe_hw *hw = &adapter->hw; 1407 1408 /* add VID to filter table */ 1409 if (hw->mac.ops.set_vfta) 1410 hw->mac.ops.set_vfta(hw, vid, 0, true); 1411 set_bit(vid, adapter->active_vlans); 1412} 1413 1414static void ixgbevf_vlan_rx_kill_vid(struct net_device *netdev, u16 vid) 1415{ 1416 struct ixgbevf_adapter *adapter = netdev_priv(netdev); 1417 struct ixgbe_hw *hw = &adapter->hw; 1418 1419 /* remove VID from filter table */ 1420 if (hw->mac.ops.set_vfta) 1421 hw->mac.ops.set_vfta(hw, vid, 0, false); 1422 clear_bit(vid, adapter->active_vlans); 1423} 1424 1425static void ixgbevf_restore_vlan(struct ixgbevf_adapter *adapter) 1426{ 1427 u16 vid; 1428 1429 for_each_set_bit(vid, adapter->active_vlans, VLAN_N_VID) 1430 ixgbevf_vlan_rx_add_vid(adapter->netdev, vid); 1431} 1432 1433static int ixgbevf_write_uc_addr_list(struct net_device *netdev) 1434{ 1435 struct ixgbevf_adapter *adapter = netdev_priv(netdev); 1436 struct ixgbe_hw *hw = &adapter->hw; 1437 int count = 0; 1438 1439 if ((netdev_uc_count(netdev)) > 10) { 1440 printk(KERN_ERR "Too many unicast filters - No Space\n"); 1441 return -ENOSPC; 1442 } 1443 1444 if (!netdev_uc_empty(netdev)) { 1445 struct netdev_hw_addr *ha; 1446 netdev_for_each_uc_addr(ha, netdev) { 1447 hw->mac.ops.set_uc_addr(hw, ++count, ha->addr); 1448 udelay(200); 1449 } 1450 } else { 1451 /* 1452 * If the list is empty then send message to PF driver to 1453 * clear all macvlans on this VF. 1454 */ 1455 hw->mac.ops.set_uc_addr(hw, 0, NULL); 1456 } 1457 1458 return count; 1459} 1460 1461/** 1462 * ixgbevf_set_rx_mode - Multicast set 1463 * @netdev: network interface device structure 1464 * 1465 * The set_rx_method entry point is called whenever the multicast address 1466 * list or the network interface flags are updated. This routine is 1467 * responsible for configuring the hardware for proper multicast mode. 1468 **/ 1469static void ixgbevf_set_rx_mode(struct net_device *netdev) 1470{ 1471 struct ixgbevf_adapter *adapter = netdev_priv(netdev); 1472 struct ixgbe_hw *hw = &adapter->hw; 1473 1474 /* reprogram multicast list */ 1475 if (hw->mac.ops.update_mc_addr_list) 1476 hw->mac.ops.update_mc_addr_list(hw, netdev); 1477 1478 ixgbevf_write_uc_addr_list(netdev); 1479} 1480 1481static void ixgbevf_napi_enable_all(struct ixgbevf_adapter *adapter) 1482{ 1483 int q_idx; 1484 struct ixgbevf_q_vector *q_vector; 1485 int q_vectors = adapter->num_msix_vectors - NON_Q_VECTORS; 1486 1487 for (q_idx = 0; q_idx < q_vectors; q_idx++) { 1488 struct napi_struct *napi; 1489 q_vector = adapter->q_vector[q_idx]; 1490 if (!q_vector->rxr_count) 1491 continue; 1492 napi = &q_vector->napi; 1493 if (q_vector->rxr_count > 1) 1494 napi->poll = &ixgbevf_clean_rxonly_many; 1495 1496 napi_enable(napi); 1497 } 1498} 1499 1500static void ixgbevf_napi_disable_all(struct ixgbevf_adapter *adapter) 1501{ 1502 int q_idx; 1503 struct ixgbevf_q_vector *q_vector; 1504 int q_vectors = adapter->num_msix_vectors - NON_Q_VECTORS; 1505 1506 for (q_idx = 0; q_idx < q_vectors; q_idx++) { 1507 q_vector = adapter->q_vector[q_idx]; 1508 if (!q_vector->rxr_count) 1509 continue; 1510 napi_disable(&q_vector->napi); 1511 } 1512} 1513 1514static void ixgbevf_configure(struct ixgbevf_adapter *adapter) 1515{ 1516 struct net_device *netdev = adapter->netdev; 1517 int i; 1518 1519 ixgbevf_set_rx_mode(netdev); 1520 1521 ixgbevf_restore_vlan(adapter); 1522 1523 ixgbevf_configure_tx(adapter); 1524 ixgbevf_configure_rx(adapter); 1525 for (i = 0; i < adapter->num_rx_queues; i++) { 1526 struct ixgbevf_ring *ring = &adapter->rx_ring[i]; 1527 ixgbevf_alloc_rx_buffers(adapter, ring, ring->count); 1528 ring->next_to_use = ring->count - 1; 1529 writel(ring->next_to_use, adapter->hw.hw_addr + ring->tail); 1530 } 1531} 1532 1533#define IXGBE_MAX_RX_DESC_POLL 10 1534static inline void ixgbevf_rx_desc_queue_enable(struct ixgbevf_adapter *adapter, 1535 int rxr) 1536{ 1537 struct ixgbe_hw *hw = &adapter->hw; 1538 int j = adapter->rx_ring[rxr].reg_idx; 1539 int k; 1540 1541 for (k = 0; k < IXGBE_MAX_RX_DESC_POLL; k++) { 1542 if (IXGBE_READ_REG(hw, IXGBE_VFRXDCTL(j)) & IXGBE_RXDCTL_ENABLE) 1543 break; 1544 else 1545 msleep(1); 1546 } 1547 if (k >= IXGBE_MAX_RX_DESC_POLL) { 1548 hw_dbg(hw, "RXDCTL.ENABLE on Rx queue %d " 1549 "not set within the polling period\n", rxr); 1550 } 1551 1552 ixgbevf_release_rx_desc(&adapter->hw, &adapter->rx_ring[rxr], 1553 (adapter->rx_ring[rxr].count - 1)); 1554} 1555 1556static void ixgbevf_save_reset_stats(struct ixgbevf_adapter *adapter) 1557{ 1558 /* Only save pre-reset stats if there are some */ 1559 if (adapter->stats.vfgprc || adapter->stats.vfgptc) { 1560 adapter->stats.saved_reset_vfgprc += adapter->stats.vfgprc - 1561 adapter->stats.base_vfgprc; 1562 adapter->stats.saved_reset_vfgptc += adapter->stats.vfgptc - 1563 adapter->stats.base_vfgptc; 1564 adapter->stats.saved_reset_vfgorc += adapter->stats.vfgorc - 1565 adapter->stats.base_vfgorc; 1566 adapter->stats.saved_reset_vfgotc += adapter->stats.vfgotc - 1567 adapter->stats.base_vfgotc; 1568 adapter->stats.saved_reset_vfmprc += adapter->stats.vfmprc - 1569 adapter->stats.base_vfmprc; 1570 } 1571} 1572 1573static void ixgbevf_init_last_counter_stats(struct ixgbevf_adapter *adapter) 1574{ 1575 struct ixgbe_hw *hw = &adapter->hw; 1576 1577 adapter->stats.last_vfgprc = IXGBE_READ_REG(hw, IXGBE_VFGPRC); 1578 adapter->stats.last_vfgorc = IXGBE_READ_REG(hw, IXGBE_VFGORC_LSB); 1579 adapter->stats.last_vfgorc |= 1580 (((u64)(IXGBE_READ_REG(hw, IXGBE_VFGORC_MSB))) << 32); 1581 adapter->stats.last_vfgptc = IXGBE_READ_REG(hw, IXGBE_VFGPTC); 1582 adapter->stats.last_vfgotc = IXGBE_READ_REG(hw, IXGBE_VFGOTC_LSB); 1583 adapter->stats.last_vfgotc |= 1584 (((u64)(IXGBE_READ_REG(hw, IXGBE_VFGOTC_MSB))) << 32); 1585 adapter->stats.last_vfmprc = IXGBE_READ_REG(hw, IXGBE_VFMPRC); 1586 1587 adapter->stats.base_vfgprc = adapter->stats.last_vfgprc; 1588 adapter->stats.base_vfgorc = adapter->stats.last_vfgorc; 1589 adapter->stats.base_vfgptc = adapter->stats.last_vfgptc; 1590 adapter->stats.base_vfgotc = adapter->stats.last_vfgotc; 1591 adapter->stats.base_vfmprc = adapter->stats.last_vfmprc; 1592} 1593 1594static int ixgbevf_up_complete(struct ixgbevf_adapter *adapter) 1595{ 1596 struct net_device *netdev = adapter->netdev; 1597 struct ixgbe_hw *hw = &adapter->hw; 1598 int i, j = 0; 1599 int num_rx_rings = adapter->num_rx_queues; 1600 u32 txdctl, rxdctl; 1601 1602 for (i = 0; i < adapter->num_tx_queues; i++) { 1603 j = adapter->tx_ring[i].reg_idx; 1604 txdctl = IXGBE_READ_REG(hw, IXGBE_VFTXDCTL(j)); 1605 /* enable WTHRESH=8 descriptors, to encourage burst writeback */ 1606 txdctl |= (8 << 16); 1607 IXGBE_WRITE_REG(hw, IXGBE_VFTXDCTL(j), txdctl); 1608 } 1609 1610 for (i = 0; i < adapter->num_tx_queues; i++) { 1611 j = adapter->tx_ring[i].reg_idx; 1612 txdctl = IXGBE_READ_REG(hw, IXGBE_VFTXDCTL(j)); 1613 txdctl |= IXGBE_TXDCTL_ENABLE; 1614 IXGBE_WRITE_REG(hw, IXGBE_VFTXDCTL(j), txdctl); 1615 } 1616 1617 for (i = 0; i < num_rx_rings; i++) { 1618 j = adapter->rx_ring[i].reg_idx; 1619 rxdctl = IXGBE_READ_REG(hw, IXGBE_VFRXDCTL(j)); 1620 rxdctl |= IXGBE_RXDCTL_ENABLE | IXGBE_RXDCTL_VME; 1621 if (hw->mac.type == ixgbe_mac_X540_vf) { 1622 rxdctl &= ~IXGBE_RXDCTL_RLPMLMASK; 1623 rxdctl |= ((netdev->mtu + ETH_HLEN + ETH_FCS_LEN) | 1624 IXGBE_RXDCTL_RLPML_EN); 1625 } 1626 IXGBE_WRITE_REG(hw, IXGBE_VFRXDCTL(j), rxdctl); 1627 ixgbevf_rx_desc_queue_enable(adapter, i); 1628 } 1629 1630 ixgbevf_configure_msix(adapter); 1631 1632 if (hw->mac.ops.set_rar) { 1633 if (is_valid_ether_addr(hw->mac.addr)) 1634 hw->mac.ops.set_rar(hw, 0, hw->mac.addr, 0); 1635 else 1636 hw->mac.ops.set_rar(hw, 0, hw->mac.perm_addr, 0); 1637 } 1638 1639 clear_bit(__IXGBEVF_DOWN, &adapter->state); 1640 ixgbevf_napi_enable_all(adapter); 1641 1642 /* enable transmits */ 1643 netif_tx_start_all_queues(netdev); 1644 1645 ixgbevf_save_reset_stats(adapter); 1646 ixgbevf_init_last_counter_stats(adapter); 1647 1648 /* bring the link up in the watchdog, this could race with our first 1649 * link up interrupt but shouldn't be a problem */ 1650 adapter->flags |= IXGBE_FLAG_NEED_LINK_UPDATE; 1651 adapter->link_check_timeout = jiffies; 1652 mod_timer(&adapter->watchdog_timer, jiffies); 1653 return 0; 1654} 1655 1656int ixgbevf_up(struct ixgbevf_adapter *adapter) 1657{ 1658 int err; 1659 struct ixgbe_hw *hw = &adapter->hw; 1660 1661 ixgbevf_configure(adapter); 1662 1663 err = ixgbevf_up_complete(adapter); 1664 1665 /* clear any pending interrupts, may auto mask */ 1666 IXGBE_READ_REG(hw, IXGBE_VTEICR); 1667 1668 ixgbevf_irq_enable(adapter, true, true); 1669 1670 return err; 1671} 1672 1673/** 1674 * ixgbevf_clean_rx_ring - Free Rx Buffers per Queue 1675 * @adapter: board private structure 1676 * @rx_ring: ring to free buffers from 1677 **/ 1678static void ixgbevf_clean_rx_ring(struct ixgbevf_adapter *adapter, 1679 struct ixgbevf_ring *rx_ring) 1680{ 1681 struct pci_dev *pdev = adapter->pdev; 1682 unsigned long size; 1683 unsigned int i; 1684 1685 if (!rx_ring->rx_buffer_info) 1686 return; 1687 1688 /* Free all the Rx ring sk_buffs */ 1689 for (i = 0; i < rx_ring->count; i++) { 1690 struct ixgbevf_rx_buffer *rx_buffer_info; 1691 1692 rx_buffer_info = &rx_ring->rx_buffer_info[i]; 1693 if (rx_buffer_info->dma) { 1694 dma_unmap_single(&pdev->dev, rx_buffer_info->dma, 1695 rx_ring->rx_buf_len, 1696 DMA_FROM_DEVICE); 1697 rx_buffer_info->dma = 0; 1698 } 1699 if (rx_buffer_info->skb) { 1700 struct sk_buff *skb = rx_buffer_info->skb; 1701 rx_buffer_info->skb = NULL; 1702 do { 1703 struct sk_buff *this = skb; 1704 skb = skb->prev; 1705 dev_kfree_skb(this); 1706 } while (skb); 1707 } 1708 if (!rx_buffer_info->page) 1709 continue; 1710 dma_unmap_page(&pdev->dev, rx_buffer_info->page_dma, 1711 PAGE_SIZE / 2, DMA_FROM_DEVICE); 1712 rx_buffer_info->page_dma = 0; 1713 put_page(rx_buffer_info->page); 1714 rx_buffer_info->page = NULL; 1715 rx_buffer_info->page_offset = 0; 1716 } 1717 1718 size = sizeof(struct ixgbevf_rx_buffer) * rx_ring->count; 1719 memset(rx_ring->rx_buffer_info, 0, size); 1720 1721 /* Zero out the descriptor ring */ 1722 memset(rx_ring->desc, 0, rx_ring->size); 1723 1724 rx_ring->next_to_clean = 0; 1725 rx_ring->next_to_use = 0; 1726 1727 if (rx_ring->head) 1728 writel(0, adapter->hw.hw_addr + rx_ring->head); 1729 if (rx_ring->tail) 1730 writel(0, adapter->hw.hw_addr + rx_ring->tail); 1731} 1732 1733/** 1734 * ixgbevf_clean_tx_ring - Free Tx Buffers 1735 * @adapter: board private structure 1736 * @tx_ring: ring to be cleaned 1737 **/ 1738static void ixgbevf_clean_tx_ring(struct ixgbevf_adapter *adapter, 1739 struct ixgbevf_ring *tx_ring) 1740{ 1741 struct ixgbevf_tx_buffer *tx_buffer_info; 1742 unsigned long size; 1743 unsigned int i; 1744 1745 if (!tx_ring->tx_buffer_info) 1746 return; 1747 1748 /* Free all the Tx ring sk_buffs */ 1749 1750 for (i = 0; i < tx_ring->count; i++) { 1751 tx_buffer_info = &tx_ring->tx_buffer_info[i]; 1752 ixgbevf_unmap_and_free_tx_resource(adapter, tx_buffer_info); 1753 } 1754 1755 size = sizeof(struct ixgbevf_tx_buffer) * tx_ring->count; 1756 memset(tx_ring->tx_buffer_info, 0, size); 1757 1758 memset(tx_ring->desc, 0, tx_ring->size); 1759 1760 tx_ring->next_to_use = 0; 1761 tx_ring->next_to_clean = 0; 1762 1763 if (tx_ring->head) 1764 writel(0, adapter->hw.hw_addr + tx_ring->head); 1765 if (tx_ring->tail) 1766 writel(0, adapter->hw.hw_addr + tx_ring->tail); 1767} 1768 1769/** 1770 * ixgbevf_clean_all_rx_rings - Free Rx Buffers for all queues 1771 * @adapter: board private structure 1772 **/ 1773static void ixgbevf_clean_all_rx_rings(struct ixgbevf_adapter *adapter) 1774{ 1775 int i; 1776 1777 for (i = 0; i < adapter->num_rx_queues; i++) 1778 ixgbevf_clean_rx_ring(adapter, &adapter->rx_ring[i]); 1779} 1780 1781/** 1782 * ixgbevf_clean_all_tx_rings - Free Tx Buffers for all queues 1783 * @adapter: board private structure 1784 **/ 1785static void ixgbevf_clean_all_tx_rings(struct ixgbevf_adapter *adapter) 1786{ 1787 int i; 1788 1789 for (i = 0; i < adapter->num_tx_queues; i++) 1790 ixgbevf_clean_tx_ring(adapter, &adapter->tx_ring[i]); 1791} 1792 1793void ixgbevf_down(struct ixgbevf_adapter *adapter) 1794{ 1795 struct net_device *netdev = adapter->netdev; 1796 struct ixgbe_hw *hw = &adapter->hw; 1797 u32 txdctl; 1798 int i, j; 1799 1800 /* signal that we are down to the interrupt handler */ 1801 set_bit(__IXGBEVF_DOWN, &adapter->state); 1802 /* disable receives */ 1803 1804 netif_tx_disable(netdev); 1805 1806 msleep(10); 1807 1808 netif_tx_stop_all_queues(netdev); 1809 1810 ixgbevf_irq_disable(adapter); 1811 1812 ixgbevf_napi_disable_all(adapter); 1813 1814 del_timer_sync(&adapter->watchdog_timer); 1815 /* can't call flush scheduled work here because it can deadlock 1816 * if linkwatch_event tries to acquire the rtnl_lock which we are 1817 * holding */ 1818 while (adapter->flags & IXGBE_FLAG_IN_WATCHDOG_TASK) 1819 msleep(1); 1820 1821 /* disable transmits in the hardware now that interrupts are off */ 1822 for (i = 0; i < adapter->num_tx_queues; i++) { 1823 j = adapter->tx_ring[i].reg_idx; 1824 txdctl = IXGBE_READ_REG(hw, IXGBE_VFTXDCTL(j)); 1825 IXGBE_WRITE_REG(hw, IXGBE_VFTXDCTL(j), 1826 (txdctl & ~IXGBE_TXDCTL_ENABLE)); 1827 } 1828 1829 netif_carrier_off(netdev); 1830 1831 if (!pci_channel_offline(adapter->pdev)) 1832 ixgbevf_reset(adapter); 1833 1834 ixgbevf_clean_all_tx_rings(adapter); 1835 ixgbevf_clean_all_rx_rings(adapter); 1836} 1837 1838void ixgbevf_reinit_locked(struct ixgbevf_adapter *adapter) 1839{ 1840 struct ixgbe_hw *hw = &adapter->hw; 1841 1842 WARN_ON(in_interrupt()); 1843 1844 while (test_and_set_bit(__IXGBEVF_RESETTING, &adapter->state)) 1845 msleep(1); 1846 1847 /* 1848 * Check if PF is up before re-init. If not then skip until 1849 * later when the PF is up and ready to service requests from 1850 * the VF via mailbox. If the VF is up and running then the 1851 * watchdog task will continue to schedule reset tasks until 1852 * the PF is up and running. 1853 */ 1854 if (!hw->mac.ops.reset_hw(hw)) { 1855 ixgbevf_down(adapter); 1856 ixgbevf_up(adapter); 1857 } 1858 1859 clear_bit(__IXGBEVF_RESETTING, &adapter->state); 1860} 1861 1862void ixgbevf_reset(struct ixgbevf_adapter *adapter) 1863{ 1864 struct ixgbe_hw *hw = &adapter->hw; 1865 struct net_device *netdev = adapter->netdev; 1866 1867 if (hw->mac.ops.reset_hw(hw)) 1868 hw_dbg(hw, "PF still resetting\n"); 1869 else 1870 hw->mac.ops.init_hw(hw); 1871 1872 if (is_valid_ether_addr(adapter->hw.mac.addr)) { 1873 memcpy(netdev->dev_addr, adapter->hw.mac.addr, 1874 netdev->addr_len); 1875 memcpy(netdev->perm_addr, adapter->hw.mac.addr, 1876 netdev->addr_len); 1877 } 1878} 1879 1880static void ixgbevf_acquire_msix_vectors(struct ixgbevf_adapter *adapter, 1881 int vectors) 1882{ 1883 int err, vector_threshold; 1884 1885 /* We'll want at least 3 (vector_threshold): 1886 * 1) TxQ[0] Cleanup 1887 * 2) RxQ[0] Cleanup 1888 * 3) Other (Link Status Change, etc.) 1889 */ 1890 vector_threshold = MIN_MSIX_COUNT; 1891 1892 /* The more we get, the more we will assign to Tx/Rx Cleanup 1893 * for the separate queues...where Rx Cleanup >= Tx Cleanup. 1894 * Right now, we simply care about how many we'll get; we'll 1895 * set them up later while requesting irq's. 1896 */ 1897 while (vectors >= vector_threshold) { 1898 err = pci_enable_msix(adapter->pdev, adapter->msix_entries, 1899 vectors); 1900 if (!err) /* Success in acquiring all requested vectors. */ 1901 break; 1902 else if (err < 0) 1903 vectors = 0; /* Nasty failure, quit now */ 1904 else /* err == number of vectors we should try again with */ 1905 vectors = err; 1906 } 1907 1908 if (vectors < vector_threshold) { 1909 /* Can't allocate enough MSI-X interrupts? Oh well. 1910 * This just means we'll go with either a single MSI 1911 * vector or fall back to legacy interrupts. 1912 */ 1913 hw_dbg(&adapter->hw, 1914 "Unable to allocate MSI-X interrupts\n"); 1915 kfree(adapter->msix_entries); 1916 adapter->msix_entries = NULL; 1917 } else { 1918 /* 1919 * Adjust for only the vectors we'll use, which is minimum 1920 * of max_msix_q_vectors + NON_Q_VECTORS, or the number of 1921 * vectors we were allocated. 1922 */ 1923 adapter->num_msix_vectors = vectors; 1924 } 1925} 1926 1927/* 1928 * ixgbevf_set_num_queues: Allocate queues for device, feature dependent 1929 * @adapter: board private structure to initialize 1930 * 1931 * This is the top level queue allocation routine. The order here is very 1932 * important, starting with the "most" number of features turned on at once, 1933 * and ending with the smallest set of features. This way large combinations 1934 * can be allocated if they're turned on, and smaller combinations are the 1935 * fallthrough conditions. 1936 * 1937 **/ 1938static void ixgbevf_set_num_queues(struct ixgbevf_adapter *adapter) 1939{ 1940 /* Start with base case */ 1941 adapter->num_rx_queues = 1; 1942 adapter->num_tx_queues = 1; 1943 adapter->num_rx_pools = adapter->num_rx_queues; 1944 adapter->num_rx_queues_per_pool = 1; 1945} 1946 1947/** 1948 * ixgbevf_alloc_queues - Allocate memory for all rings 1949 * @adapter: board private structure to initialize 1950 * 1951 * We allocate one ring per queue at run-time since we don't know the 1952 * number of queues at compile-time. The polling_netdev array is 1953 * intended for Multiqueue, but should work fine with a single queue. 1954 **/ 1955static int ixgbevf_alloc_queues(struct ixgbevf_adapter *adapter) 1956{ 1957 int i; 1958 1959 adapter->tx_ring = kcalloc(adapter->num_tx_queues, 1960 sizeof(struct ixgbevf_ring), GFP_KERNEL); 1961 if (!adapter->tx_ring) 1962 goto err_tx_ring_allocation; 1963 1964 adapter->rx_ring = kcalloc(adapter->num_rx_queues, 1965 sizeof(struct ixgbevf_ring), GFP_KERNEL); 1966 if (!adapter->rx_ring) 1967 goto err_rx_ring_allocation; 1968 1969 for (i = 0; i < adapter->num_tx_queues; i++) { 1970 adapter->tx_ring[i].count = adapter->tx_ring_count; 1971 adapter->tx_ring[i].queue_index = i; 1972 adapter->tx_ring[i].reg_idx = i; 1973 } 1974 1975 for (i = 0; i < adapter->num_rx_queues; i++) { 1976 adapter->rx_ring[i].count = adapter->rx_ring_count; 1977 adapter->rx_ring[i].queue_index = i; 1978 adapter->rx_ring[i].reg_idx = i; 1979 } 1980 1981 return 0; 1982 1983err_rx_ring_allocation: 1984 kfree(adapter->tx_ring); 1985err_tx_ring_allocation: 1986 return -ENOMEM; 1987} 1988 1989/** 1990 * ixgbevf_set_interrupt_capability - set MSI-X or FAIL if not supported 1991 * @adapter: board private structure to initialize 1992 * 1993 * Attempt to configure the interrupts using the best available 1994 * capabilities of the hardware and the kernel. 1995 **/ 1996static int ixgbevf_set_interrupt_capability(struct ixgbevf_adapter *adapter) 1997{ 1998 int err = 0; 1999 int vector, v_budget; 2000 2001 /* 2002 * It's easy to be greedy for MSI-X vectors, but it really 2003 * doesn't do us much good if we have a lot more vectors 2004 * than CPU's. So let's be conservative and only ask for 2005 * (roughly) twice the number of vectors as there are CPU's. 2006 */ 2007 v_budget = min(adapter->num_rx_queues + adapter->num_tx_queues, 2008 (int)(num_online_cpus() * 2)) + NON_Q_VECTORS; 2009 2010 /* A failure in MSI-X entry allocation isn't fatal, but it does 2011 * mean we disable MSI-X capabilities of the adapter. */ 2012 adapter->msix_entries = kcalloc(v_budget, 2013 sizeof(struct msix_entry), GFP_KERNEL); 2014 if (!adapter->msix_entries) { 2015 err = -ENOMEM; 2016 goto out; 2017 } 2018 2019 for (vector = 0; vector < v_budget; vector++) 2020 adapter->msix_entries[vector].entry = vector; 2021 2022 ixgbevf_acquire_msix_vectors(adapter, v_budget); 2023 2024out: 2025 return err; 2026} 2027 2028/** 2029 * ixgbevf_alloc_q_vectors - Allocate memory for interrupt vectors 2030 * @adapter: board private structure to initialize 2031 * 2032 * We allocate one q_vector per queue interrupt. If allocation fails we 2033 * return -ENOMEM. 2034 **/ 2035static int ixgbevf_alloc_q_vectors(struct ixgbevf_adapter *adapter) 2036{ 2037 int q_idx, num_q_vectors; 2038 struct ixgbevf_q_vector *q_vector; 2039 int napi_vectors; 2040 int (*poll)(struct napi_struct *, int); 2041 2042 num_q_vectors = adapter->num_msix_vectors - NON_Q_VECTORS; 2043 napi_vectors = adapter->num_rx_queues; 2044 poll = &ixgbevf_clean_rxonly; 2045 2046 for (q_idx = 0; q_idx < num_q_vectors; q_idx++) { 2047 q_vector = kzalloc(sizeof(struct ixgbevf_q_vector), GFP_KERNEL); 2048 if (!q_vector) 2049 goto err_out; 2050 q_vector->adapter = adapter; 2051 q_vector->v_idx = q_idx; 2052 q_vector->eitr = adapter->eitr_param; 2053 if (q_idx < napi_vectors) 2054 netif_napi_add(adapter->netdev, &q_vector->napi, 2055 (*poll), 64); 2056 adapter->q_vector[q_idx] = q_vector; 2057 } 2058 2059 return 0; 2060 2061err_out: 2062 while (q_idx) { 2063 q_idx--; 2064 q_vector = adapter->q_vector[q_idx]; 2065 netif_napi_del(&q_vector->napi); 2066 kfree(q_vector); 2067 adapter->q_vector[q_idx] = NULL; 2068 } 2069 return -ENOMEM; 2070} 2071 2072/** 2073 * ixgbevf_free_q_vectors - Free memory allocated for interrupt vectors 2074 * @adapter: board private structure to initialize 2075 * 2076 * This function frees the memory allocated to the q_vectors. In addition if 2077 * NAPI is enabled it will delete any references to the NAPI struct prior 2078 * to freeing the q_vector. 2079 **/ 2080static void ixgbevf_free_q_vectors(struct ixgbevf_adapter *adapter) 2081{ 2082 int q_idx, num_q_vectors; 2083 int napi_vectors; 2084 2085 num_q_vectors = adapter->num_msix_vectors - NON_Q_VECTORS; 2086 napi_vectors = adapter->num_rx_queues; 2087 2088 for (q_idx = 0; q_idx < num_q_vectors; q_idx++) { 2089 struct ixgbevf_q_vector *q_vector = adapter->q_vector[q_idx]; 2090 2091 adapter->q_vector[q_idx] = NULL; 2092 if (q_idx < napi_vectors) 2093 netif_napi_del(&q_vector->napi); 2094 kfree(q_vector); 2095 } 2096} 2097 2098/** 2099 * ixgbevf_reset_interrupt_capability - Reset MSIX setup 2100 * @adapter: board private structure 2101 * 2102 **/ 2103static void ixgbevf_reset_interrupt_capability(struct ixgbevf_adapter *adapter) 2104{ 2105 pci_disable_msix(adapter->pdev); 2106 kfree(adapter->msix_entries); 2107 adapter->msix_entries = NULL; 2108} 2109 2110/** 2111 * ixgbevf_init_interrupt_scheme - Determine if MSIX is supported and init 2112 * @adapter: board private structure to initialize 2113 * 2114 **/ 2115static int ixgbevf_init_interrupt_scheme(struct ixgbevf_adapter *adapter) 2116{ 2117 int err; 2118 2119 /* Number of supported queues */ 2120 ixgbevf_set_num_queues(adapter); 2121 2122 err = ixgbevf_set_interrupt_capability(adapter); 2123 if (err) { 2124 hw_dbg(&adapter->hw, 2125 "Unable to setup interrupt capabilities\n"); 2126 goto err_set_interrupt; 2127 } 2128 2129 err = ixgbevf_alloc_q_vectors(adapter); 2130 if (err) { 2131 hw_dbg(&adapter->hw, "Unable to allocate memory for queue " 2132 "vectors\n"); 2133 goto err_alloc_q_vectors; 2134 } 2135 2136 err = ixgbevf_alloc_queues(adapter); 2137 if (err) { 2138 printk(KERN_ERR "Unable to allocate memory for queues\n"); 2139 goto err_alloc_queues; 2140 } 2141 2142 hw_dbg(&adapter->hw, "Multiqueue %s: Rx Queue count = %u, " 2143 "Tx Queue count = %u\n", 2144 (adapter->num_rx_queues > 1) ? "Enabled" : 2145 "Disabled", adapter->num_rx_queues, adapter->num_tx_queues); 2146 2147 set_bit(__IXGBEVF_DOWN, &adapter->state); 2148 2149 return 0; 2150err_alloc_queues: 2151 ixgbevf_free_q_vectors(adapter); 2152err_alloc_q_vectors: 2153 ixgbevf_reset_interrupt_capability(adapter); 2154err_set_interrupt: 2155 return err; 2156} 2157 2158/** 2159 * ixgbevf_sw_init - Initialize general software structures 2160 * (struct ixgbevf_adapter) 2161 * @adapter: board private structure to initialize 2162 * 2163 * ixgbevf_sw_init initializes the Adapter private data structure. 2164 * Fields are initialized based on PCI device information and 2165 * OS network device settings (MTU size). 2166 **/ 2167static int __devinit ixgbevf_sw_init(struct ixgbevf_adapter *adapter) 2168{ 2169 struct ixgbe_hw *hw = &adapter->hw; 2170 struct pci_dev *pdev = adapter->pdev; 2171 int err; 2172 2173 /* PCI config space info */ 2174 2175 hw->vendor_id = pdev->vendor; 2176 hw->device_id = pdev->device; 2177 hw->revision_id = pdev->revision; 2178 hw->subsystem_vendor_id = pdev->subsystem_vendor; 2179 hw->subsystem_device_id = pdev->subsystem_device; 2180 2181 hw->mbx.ops.init_params(hw); 2182 hw->mac.max_tx_queues = MAX_TX_QUEUES; 2183 hw->mac.max_rx_queues = MAX_RX_QUEUES; 2184 err = hw->mac.ops.reset_hw(hw); 2185 if (err) { 2186 dev_info(&pdev->dev, 2187 "PF still in reset state, assigning new address\n"); 2188 dev_hw_addr_random(adapter->netdev, hw->mac.addr); 2189 } else { 2190 err = hw->mac.ops.init_hw(hw); 2191 if (err) { 2192 printk(KERN_ERR "init_shared_code failed: %d\n", err); 2193 goto out; 2194 } 2195 } 2196 2197 /* Enable dynamic interrupt throttling rates */ 2198 adapter->eitr_param = 20000; 2199 adapter->itr_setting = 1; 2200 2201 /* set defaults for eitr in MegaBytes */ 2202 adapter->eitr_low = 10; 2203 adapter->eitr_high = 20; 2204 2205 /* set default ring sizes */ 2206 adapter->tx_ring_count = IXGBEVF_DEFAULT_TXD; 2207 adapter->rx_ring_count = IXGBEVF_DEFAULT_RXD; 2208 2209 /* enable rx csum by default */ 2210 adapter->flags |= IXGBE_FLAG_RX_CSUM_ENABLED; 2211 2212 set_bit(__IXGBEVF_DOWN, &adapter->state); 2213 2214out: 2215 return err; 2216} 2217 2218#define UPDATE_VF_COUNTER_32bit(reg, last_counter, counter) \ 2219 { \ 2220 u32 current_counter = IXGBE_READ_REG(hw, reg); \ 2221 if (current_counter < last_counter) \ 2222 counter += 0x100000000LL; \ 2223 last_counter = current_counter; \ 2224 counter &= 0xFFFFFFFF00000000LL; \ 2225 counter |= current_counter; \ 2226 } 2227 2228#define UPDATE_VF_COUNTER_36bit(reg_lsb, reg_msb, last_counter, counter) \ 2229 { \ 2230 u64 current_counter_lsb = IXGBE_READ_REG(hw, reg_lsb); \ 2231 u64 current_counter_msb = IXGBE_READ_REG(hw, reg_msb); \ 2232 u64 current_counter = (current_counter_msb << 32) | \ 2233 current_counter_lsb; \ 2234 if (current_counter < last_counter) \ 2235 counter += 0x1000000000LL; \ 2236 last_counter = current_counter; \ 2237 counter &= 0xFFFFFFF000000000LL; \ 2238 counter |= current_counter; \ 2239 } 2240/** 2241 * ixgbevf_update_stats - Update the board statistics counters. 2242 * @adapter: board private structure 2243 **/ 2244void ixgbevf_update_stats(struct ixgbevf_adapter *adapter) 2245{ 2246 struct ixgbe_hw *hw = &adapter->hw; 2247 2248 UPDATE_VF_COUNTER_32bit(IXGBE_VFGPRC, adapter->stats.last_vfgprc, 2249 adapter->stats.vfgprc); 2250 UPDATE_VF_COUNTER_32bit(IXGBE_VFGPTC, adapter->stats.last_vfgptc, 2251 adapter->stats.vfgptc); 2252 UPDATE_VF_COUNTER_36bit(IXGBE_VFGORC_LSB, IXGBE_VFGORC_MSB, 2253 adapter->stats.last_vfgorc, 2254 adapter->stats.vfgorc); 2255 UPDATE_VF_COUNTER_36bit(IXGBE_VFGOTC_LSB, IXGBE_VFGOTC_MSB, 2256 adapter->stats.last_vfgotc, 2257 adapter->stats.vfgotc); 2258 UPDATE_VF_COUNTER_32bit(IXGBE_VFMPRC, adapter->stats.last_vfmprc, 2259 adapter->stats.vfmprc); 2260} 2261 2262/** 2263 * ixgbevf_watchdog - Timer Call-back 2264 * @data: pointer to adapter cast into an unsigned long 2265 **/ 2266static void ixgbevf_watchdog(unsigned long data) 2267{ 2268 struct ixgbevf_adapter *adapter = (struct ixgbevf_adapter *)data; 2269 struct ixgbe_hw *hw = &adapter->hw; 2270 u64 eics = 0; 2271 int i; 2272 2273 /* 2274 * Do the watchdog outside of interrupt context due to the lovely 2275 * delays that some of the newer hardware requires 2276 */ 2277 2278 if (test_bit(__IXGBEVF_DOWN, &adapter->state)) 2279 goto watchdog_short_circuit; 2280 2281 /* get one bit for every active tx/rx interrupt vector */ 2282 for (i = 0; i < adapter->num_msix_vectors - NON_Q_VECTORS; i++) { 2283 struct ixgbevf_q_vector *qv = adapter->q_vector[i]; 2284 if (qv->rxr_count || qv->txr_count) 2285 eics |= (1 << i); 2286 } 2287 2288 IXGBE_WRITE_REG(hw, IXGBE_VTEICS, (u32)eics); 2289 2290watchdog_short_circuit: 2291 schedule_work(&adapter->watchdog_task); 2292} 2293 2294/** 2295 * ixgbevf_tx_timeout - Respond to a Tx Hang 2296 * @netdev: network interface device structure 2297 **/ 2298static void ixgbevf_tx_timeout(struct net_device *netdev) 2299{ 2300 struct ixgbevf_adapter *adapter = netdev_priv(netdev); 2301 2302 /* Do the reset outside of interrupt context */ 2303 schedule_work(&adapter->reset_task); 2304} 2305 2306static void ixgbevf_reset_task(struct work_struct *work) 2307{ 2308 struct ixgbevf_adapter *adapter; 2309 adapter = container_of(work, struct ixgbevf_adapter, reset_task); 2310 2311 /* If we're already down or resetting, just bail */ 2312 if (test_bit(__IXGBEVF_DOWN, &adapter->state) || 2313 test_bit(__IXGBEVF_RESETTING, &adapter->state)) 2314 return; 2315 2316 adapter->tx_timeout_count++; 2317 2318 ixgbevf_reinit_locked(adapter); 2319} 2320 2321/** 2322 * ixgbevf_watchdog_task - worker thread to bring link up 2323 * @work: pointer to work_struct containing our data 2324 **/ 2325static void ixgbevf_watchdog_task(struct work_struct *work) 2326{ 2327 struct ixgbevf_adapter *adapter = container_of(work, 2328 struct ixgbevf_adapter, 2329 watchdog_task); 2330 struct net_device *netdev = adapter->netdev; 2331 struct ixgbe_hw *hw = &adapter->hw; 2332 u32 link_speed = adapter->link_speed; 2333 bool link_up = adapter->link_up; 2334 2335 adapter->flags |= IXGBE_FLAG_IN_WATCHDOG_TASK; 2336 2337 /* 2338 * Always check the link on the watchdog because we have 2339 * no LSC interrupt 2340 */ 2341 if (hw->mac.ops.check_link) { 2342 if ((hw->mac.ops.check_link(hw, &link_speed, 2343 &link_up, false)) != 0) { 2344 adapter->link_up = link_up; 2345 adapter->link_speed = link_speed; 2346 netif_carrier_off(netdev); 2347 netif_tx_stop_all_queues(netdev); 2348 schedule_work(&adapter->reset_task); 2349 goto pf_has_reset; 2350 } 2351 } else { 2352 /* always assume link is up, if no check link 2353 * function */ 2354 link_speed = IXGBE_LINK_SPEED_10GB_FULL; 2355 link_up = true; 2356 } 2357 adapter->link_up = link_up; 2358 adapter->link_speed = link_speed; 2359 2360 if (link_up) { 2361 if (!netif_carrier_ok(netdev)) { 2362 hw_dbg(&adapter->hw, "NIC Link is Up, %u Gbps\n", 2363 (link_speed == IXGBE_LINK_SPEED_10GB_FULL) ? 2364 10 : 1); 2365 netif_carrier_on(netdev); 2366 netif_tx_wake_all_queues(netdev); 2367 } 2368 } else { 2369 adapter->link_up = false; 2370 adapter->link_speed = 0; 2371 if (netif_carrier_ok(netdev)) { 2372 hw_dbg(&adapter->hw, "NIC Link is Down\n"); 2373 netif_carrier_off(netdev); 2374 netif_tx_stop_all_queues(netdev); 2375 } 2376 } 2377 2378 ixgbevf_update_stats(adapter); 2379 2380pf_has_reset: 2381 /* Reset the timer */ 2382 if (!test_bit(__IXGBEVF_DOWN, &adapter->state)) 2383 mod_timer(&adapter->watchdog_timer, 2384 round_jiffies(jiffies + (2 * HZ))); 2385 2386 adapter->flags &= ~IXGBE_FLAG_IN_WATCHDOG_TASK; 2387} 2388 2389/** 2390 * ixgbevf_free_tx_resources - Free Tx Resources per Queue 2391 * @adapter: board private structure 2392 * @tx_ring: Tx descriptor ring for a specific queue 2393 * 2394 * Free all transmit software resources 2395 **/ 2396void ixgbevf_free_tx_resources(struct ixgbevf_adapter *adapter, 2397 struct ixgbevf_ring *tx_ring) 2398{ 2399 struct pci_dev *pdev = adapter->pdev; 2400 2401 ixgbevf_clean_tx_ring(adapter, tx_ring); 2402 2403 vfree(tx_ring->tx_buffer_info); 2404 tx_ring->tx_buffer_info = NULL; 2405 2406 dma_free_coherent(&pdev->dev, tx_ring->size, tx_ring->desc, 2407 tx_ring->dma); 2408 2409 tx_ring->desc = NULL; 2410} 2411 2412/** 2413 * ixgbevf_free_all_tx_resources - Free Tx Resources for All Queues 2414 * @adapter: board private structure 2415 * 2416 * Free all transmit software resources 2417 **/ 2418static void ixgbevf_free_all_tx_resources(struct ixgbevf_adapter *adapter) 2419{ 2420 int i; 2421 2422 for (i = 0; i < adapter->num_tx_queues; i++) 2423 if (adapter->tx_ring[i].desc) 2424 ixgbevf_free_tx_resources(adapter, 2425 &adapter->tx_ring[i]); 2426 2427} 2428 2429/** 2430 * ixgbevf_setup_tx_resources - allocate Tx resources (Descriptors) 2431 * @adapter: board private structure 2432 * @tx_ring: tx descriptor ring (for a specific queue) to setup 2433 * 2434 * Return 0 on success, negative on failure 2435 **/ 2436int ixgbevf_setup_tx_resources(struct ixgbevf_adapter *adapter, 2437 struct ixgbevf_ring *tx_ring) 2438{ 2439 struct pci_dev *pdev = adapter->pdev; 2440 int size; 2441 2442 size = sizeof(struct ixgbevf_tx_buffer) * tx_ring->count; 2443 tx_ring->tx_buffer_info = vzalloc(size); 2444 if (!tx_ring->tx_buffer_info) 2445 goto err; 2446 2447 /* round up to nearest 4K */ 2448 tx_ring->size = tx_ring->count * sizeof(union ixgbe_adv_tx_desc); 2449 tx_ring->size = ALIGN(tx_ring->size, 4096); 2450 2451 tx_ring->desc = dma_alloc_coherent(&pdev->dev, tx_ring->size, 2452 &tx_ring->dma, GFP_KERNEL); 2453 if (!tx_ring->desc) 2454 goto err; 2455 2456 tx_ring->next_to_use = 0; 2457 tx_ring->next_to_clean = 0; 2458 tx_ring->work_limit = tx_ring->count; 2459 return 0; 2460 2461err: 2462 vfree(tx_ring->tx_buffer_info); 2463 tx_ring->tx_buffer_info = NULL; 2464 hw_dbg(&adapter->hw, "Unable to allocate memory for the transmit " 2465 "descriptor ring\n"); 2466 return -ENOMEM; 2467} 2468 2469/** 2470 * ixgbevf_setup_all_tx_resources - allocate all queues Tx resources 2471 * @adapter: board private structure 2472 * 2473 * If this function returns with an error, then it's possible one or 2474 * more of the rings is populated (while the rest are not). It is the 2475 * callers duty to clean those orphaned rings. 2476 * 2477 * Return 0 on success, negative on failure 2478 **/ 2479static int ixgbevf_setup_all_tx_resources(struct ixgbevf_adapter *adapter) 2480{ 2481 int i, err = 0; 2482 2483 for (i = 0; i < adapter->num_tx_queues; i++) { 2484 err = ixgbevf_setup_tx_resources(adapter, &adapter->tx_ring[i]); 2485 if (!err) 2486 continue; 2487 hw_dbg(&adapter->hw, 2488 "Allocation for Tx Queue %u failed\n", i); 2489 break; 2490 } 2491 2492 return err; 2493} 2494 2495/** 2496 * ixgbevf_setup_rx_resources - allocate Rx resources (Descriptors) 2497 * @adapter: board private structure 2498 * @rx_ring: rx descriptor ring (for a specific queue) to setup 2499 * 2500 * Returns 0 on success, negative on failure 2501 **/ 2502int ixgbevf_setup_rx_resources(struct ixgbevf_adapter *adapter, 2503 struct ixgbevf_ring *rx_ring) 2504{ 2505 struct pci_dev *pdev = adapter->pdev; 2506 int size; 2507 2508 size = sizeof(struct ixgbevf_rx_buffer) * rx_ring->count; 2509 rx_ring->rx_buffer_info = vzalloc(size); 2510 if (!rx_ring->rx_buffer_info) { 2511 hw_dbg(&adapter->hw, 2512 "Unable to vmalloc buffer memory for " 2513 "the receive descriptor ring\n"); 2514 goto alloc_failed; 2515 } 2516 2517 /* Round up to nearest 4K */ 2518 rx_ring->size = rx_ring->count * sizeof(union ixgbe_adv_rx_desc); 2519 rx_ring->size = ALIGN(rx_ring->size, 4096); 2520 2521 rx_ring->desc = dma_alloc_coherent(&pdev->dev, rx_ring->size, 2522 &rx_ring->dma, GFP_KERNEL); 2523 2524 if (!rx_ring->desc) { 2525 hw_dbg(&adapter->hw, 2526 "Unable to allocate memory for " 2527 "the receive descriptor ring\n"); 2528 vfree(rx_ring->rx_buffer_info); 2529 rx_ring->rx_buffer_info = NULL; 2530 goto alloc_failed; 2531 } 2532 2533 rx_ring->next_to_clean = 0; 2534 rx_ring->next_to_use = 0; 2535 2536 return 0; 2537alloc_failed: 2538 return -ENOMEM; 2539} 2540 2541/** 2542 * ixgbevf_setup_all_rx_resources - allocate all queues Rx resources 2543 * @adapter: board private structure 2544 * 2545 * If this function returns with an error, then it's possible one or 2546 * more of the rings is populated (while the rest are not). It is the 2547 * callers duty to clean those orphaned rings. 2548 * 2549 * Return 0 on success, negative on failure 2550 **/ 2551static int ixgbevf_setup_all_rx_resources(struct ixgbevf_adapter *adapter) 2552{ 2553 int i, err = 0; 2554 2555 for (i = 0; i < adapter->num_rx_queues; i++) { 2556 err = ixgbevf_setup_rx_resources(adapter, &adapter->rx_ring[i]); 2557 if (!err) 2558 continue; 2559 hw_dbg(&adapter->hw, 2560 "Allocation for Rx Queue %u failed\n", i); 2561 break; 2562 } 2563 return err; 2564} 2565 2566/** 2567 * ixgbevf_free_rx_resources - Free Rx Resources 2568 * @adapter: board private structure 2569 * @rx_ring: ring to clean the resources from 2570 * 2571 * Free all receive software resources 2572 **/ 2573void ixgbevf_free_rx_resources(struct ixgbevf_adapter *adapter, 2574 struct ixgbevf_ring *rx_ring) 2575{ 2576 struct pci_dev *pdev = adapter->pdev; 2577 2578 ixgbevf_clean_rx_ring(adapter, rx_ring); 2579 2580 vfree(rx_ring->rx_buffer_info); 2581 rx_ring->rx_buffer_info = NULL; 2582 2583 dma_free_coherent(&pdev->dev, rx_ring->size, rx_ring->desc, 2584 rx_ring->dma); 2585 2586 rx_ring->desc = NULL; 2587} 2588 2589/** 2590 * ixgbevf_free_all_rx_resources - Free Rx Resources for All Queues 2591 * @adapter: board private structure 2592 * 2593 * Free all receive software resources 2594 **/ 2595static void ixgbevf_free_all_rx_resources(struct ixgbevf_adapter *adapter) 2596{ 2597 int i; 2598 2599 for (i = 0; i < adapter->num_rx_queues; i++) 2600 if (adapter->rx_ring[i].desc) 2601 ixgbevf_free_rx_resources(adapter, 2602 &adapter->rx_ring[i]); 2603} 2604 2605/** 2606 * ixgbevf_open - Called when a network interface is made active 2607 * @netdev: network interface device structure 2608 * 2609 * Returns 0 on success, negative value on failure 2610 * 2611 * The open entry point is called when a network interface is made 2612 * active by the system (IFF_UP). At this point all resources needed 2613 * for transmit and receive operations are allocated, the interrupt 2614 * handler is registered with the OS, the watchdog timer is started, 2615 * and the stack is notified that the interface is ready. 2616 **/ 2617static int ixgbevf_open(struct net_device *netdev) 2618{ 2619 struct ixgbevf_adapter *adapter = netdev_priv(netdev); 2620 struct ixgbe_hw *hw = &adapter->hw; 2621 int err; 2622 2623 /* disallow open during test */ 2624 if (test_bit(__IXGBEVF_TESTING, &adapter->state)) 2625 return -EBUSY; 2626 2627 if (hw->adapter_stopped) { 2628 ixgbevf_reset(adapter); 2629 /* if adapter is still stopped then PF isn't up and 2630 * the vf can't start. */ 2631 if (hw->adapter_stopped) { 2632 err = IXGBE_ERR_MBX; 2633 printk(KERN_ERR "Unable to start - perhaps the PF" 2634 " Driver isn't up yet\n"); 2635 goto err_setup_reset; 2636 } 2637 } 2638 2639 /* allocate transmit descriptors */ 2640 err = ixgbevf_setup_all_tx_resources(adapter); 2641 if (err) 2642 goto err_setup_tx; 2643 2644 /* allocate receive descriptors */ 2645 err = ixgbevf_setup_all_rx_resources(adapter); 2646 if (err) 2647 goto err_setup_rx; 2648 2649 ixgbevf_configure(adapter); 2650 2651 /* 2652 * Map the Tx/Rx rings to the vectors we were allotted. 2653 * if request_irq will be called in this function map_rings 2654 * must be called *before* up_complete 2655 */ 2656 ixgbevf_map_rings_to_vectors(adapter); 2657 2658 err = ixgbevf_up_complete(adapter); 2659 if (err) 2660 goto err_up; 2661 2662 /* clear any pending interrupts, may auto mask */ 2663 IXGBE_READ_REG(hw, IXGBE_VTEICR); 2664 err = ixgbevf_request_irq(adapter); 2665 if (err) 2666 goto err_req_irq; 2667 2668 ixgbevf_irq_enable(adapter, true, true); 2669 2670 return 0; 2671 2672err_req_irq: 2673 ixgbevf_down(adapter); 2674err_up: 2675 ixgbevf_free_irq(adapter); 2676err_setup_rx: 2677 ixgbevf_free_all_rx_resources(adapter); 2678err_setup_tx: 2679 ixgbevf_free_all_tx_resources(adapter); 2680 ixgbevf_reset(adapter); 2681 2682err_setup_reset: 2683 2684 return err; 2685} 2686 2687/** 2688 * ixgbevf_close - Disables a network interface 2689 * @netdev: network interface device structure 2690 * 2691 * Returns 0, this is not allowed to fail 2692 * 2693 * The close entry point is called when an interface is de-activated 2694 * by the OS. The hardware is still under the drivers control, but 2695 * needs to be disabled. A global MAC reset is issued to stop the 2696 * hardware, and all transmit and receive resources are freed. 2697 **/ 2698static int ixgbevf_close(struct net_device *netdev) 2699{ 2700 struct ixgbevf_adapter *adapter = netdev_priv(netdev); 2701 2702 ixgbevf_down(adapter); 2703 ixgbevf_free_irq(adapter); 2704 2705 ixgbevf_free_all_tx_resources(adapter); 2706 ixgbevf_free_all_rx_resources(adapter); 2707 2708 return 0; 2709} 2710 2711static int ixgbevf_tso(struct ixgbevf_adapter *adapter, 2712 struct ixgbevf_ring *tx_ring, 2713 struct sk_buff *skb, u32 tx_flags, u8 *hdr_len) 2714{ 2715 struct ixgbe_adv_tx_context_desc *context_desc; 2716 unsigned int i; 2717 int err; 2718 struct ixgbevf_tx_buffer *tx_buffer_info; 2719 u32 vlan_macip_lens = 0, type_tucmd_mlhl; 2720 u32 mss_l4len_idx, l4len; 2721 2722 if (skb_is_gso(skb)) { 2723 if (skb_header_cloned(skb)) { 2724 err = pskb_expand_head(skb, 0, 0, GFP_ATOMIC); 2725 if (err) 2726 return err; 2727 } 2728 l4len = tcp_hdrlen(skb); 2729 *hdr_len += l4len; 2730 2731 if (skb->protocol == htons(ETH_P_IP)) { 2732 struct iphdr *iph = ip_hdr(skb); 2733 iph->tot_len = 0; 2734 iph->check = 0; 2735 tcp_hdr(skb)->check = ~csum_tcpudp_magic(iph->saddr, 2736 iph->daddr, 0, 2737 IPPROTO_TCP, 2738 0); 2739 adapter->hw_tso_ctxt++; 2740 } else if (skb_is_gso_v6(skb)) { 2741 ipv6_hdr(skb)->payload_len = 0; 2742 tcp_hdr(skb)->check = 2743 ~csum_ipv6_magic(&ipv6_hdr(skb)->saddr, 2744 &ipv6_hdr(skb)->daddr, 2745 0, IPPROTO_TCP, 0); 2746 adapter->hw_tso6_ctxt++; 2747 } 2748 2749 i = tx_ring->next_to_use; 2750 2751 tx_buffer_info = &tx_ring->tx_buffer_info[i]; 2752 context_desc = IXGBE_TX_CTXTDESC_ADV(*tx_ring, i); 2753 2754 /* VLAN MACLEN IPLEN */ 2755 if (tx_flags & IXGBE_TX_FLAGS_VLAN) 2756 vlan_macip_lens |= 2757 (tx_flags & IXGBE_TX_FLAGS_VLAN_MASK); 2758 vlan_macip_lens |= ((skb_network_offset(skb)) << 2759 IXGBE_ADVTXD_MACLEN_SHIFT); 2760 *hdr_len += skb_network_offset(skb); 2761 vlan_macip_lens |= 2762 (skb_transport_header(skb) - skb_network_header(skb)); 2763 *hdr_len += 2764 (skb_transport_header(skb) - skb_network_header(skb)); 2765 context_desc->vlan_macip_lens = cpu_to_le32(vlan_macip_lens); 2766 context_desc->seqnum_seed = 0; 2767 2768 /* ADV DTYP TUCMD MKRLOC/ISCSIHEDLEN */ 2769 type_tucmd_mlhl = (IXGBE_TXD_CMD_DEXT | 2770 IXGBE_ADVTXD_DTYP_CTXT); 2771 2772 if (skb->protocol == htons(ETH_P_IP)) 2773 type_tucmd_mlhl |= IXGBE_ADVTXD_TUCMD_IPV4; 2774 type_tucmd_mlhl |= IXGBE_ADVTXD_TUCMD_L4T_TCP; 2775 context_desc->type_tucmd_mlhl = cpu_to_le32(type_tucmd_mlhl); 2776 2777 /* MSS L4LEN IDX */ 2778 mss_l4len_idx = 2779 (skb_shinfo(skb)->gso_size << IXGBE_ADVTXD_MSS_SHIFT); 2780 mss_l4len_idx |= (l4len << IXGBE_ADVTXD_L4LEN_SHIFT); 2781 /* use index 1 for TSO */ 2782 mss_l4len_idx |= (1 << IXGBE_ADVTXD_IDX_SHIFT); 2783 context_desc->mss_l4len_idx = cpu_to_le32(mss_l4len_idx); 2784 2785 tx_buffer_info->time_stamp = jiffies; 2786 tx_buffer_info->next_to_watch = i; 2787 2788 i++; 2789 if (i == tx_ring->count) 2790 i = 0; 2791 tx_ring->next_to_use = i; 2792 2793 return true; 2794 } 2795 2796 return false; 2797} 2798 2799static bool ixgbevf_tx_csum(struct ixgbevf_adapter *adapter, 2800 struct ixgbevf_ring *tx_ring, 2801 struct sk_buff *skb, u32 tx_flags) 2802{ 2803 struct ixgbe_adv_tx_context_desc *context_desc; 2804 unsigned int i; 2805 struct ixgbevf_tx_buffer *tx_buffer_info; 2806 u32 vlan_macip_lens = 0, type_tucmd_mlhl = 0; 2807 2808 if (skb->ip_summed == CHECKSUM_PARTIAL || 2809 (tx_flags & IXGBE_TX_FLAGS_VLAN)) { 2810 i = tx_ring->next_to_use; 2811 tx_buffer_info = &tx_ring->tx_buffer_info[i]; 2812 context_desc = IXGBE_TX_CTXTDESC_ADV(*tx_ring, i); 2813 2814 if (tx_flags & IXGBE_TX_FLAGS_VLAN) 2815 vlan_macip_lens |= (tx_flags & 2816 IXGBE_TX_FLAGS_VLAN_MASK); 2817 vlan_macip_lens |= (skb_network_offset(skb) << 2818 IXGBE_ADVTXD_MACLEN_SHIFT); 2819 if (skb->ip_summed == CHECKSUM_PARTIAL) 2820 vlan_macip_lens |= (skb_transport_header(skb) - 2821 skb_network_header(skb)); 2822 2823 context_desc->vlan_macip_lens = cpu_to_le32(vlan_macip_lens); 2824 context_desc->seqnum_seed = 0; 2825 2826 type_tucmd_mlhl |= (IXGBE_TXD_CMD_DEXT | 2827 IXGBE_ADVTXD_DTYP_CTXT); 2828 2829 if (skb->ip_summed == CHECKSUM_PARTIAL) { 2830 switch (skb->protocol) { 2831 case __constant_htons(ETH_P_IP): 2832 type_tucmd_mlhl |= IXGBE_ADVTXD_TUCMD_IPV4; 2833 if (ip_hdr(skb)->protocol == IPPROTO_TCP) 2834 type_tucmd_mlhl |= 2835 IXGBE_ADVTXD_TUCMD_L4T_TCP; 2836 break; 2837 case __constant_htons(ETH_P_IPV6): 2838 /* XXX what about other V6 headers?? */ 2839 if (ipv6_hdr(skb)->nexthdr == IPPROTO_TCP) 2840 type_tucmd_mlhl |= 2841 IXGBE_ADVTXD_TUCMD_L4T_TCP; 2842 break; 2843 default: 2844 if (unlikely(net_ratelimit())) { 2845 printk(KERN_WARNING 2846 "partial checksum but " 2847 "proto=%x!\n", 2848 skb->protocol); 2849 } 2850 break; 2851 } 2852 } 2853 2854 context_desc->type_tucmd_mlhl = cpu_to_le32(type_tucmd_mlhl); 2855 /* use index zero for tx checksum offload */ 2856 context_desc->mss_l4len_idx = 0; 2857 2858 tx_buffer_info->time_stamp = jiffies; 2859 tx_buffer_info->next_to_watch = i; 2860 2861 adapter->hw_csum_tx_good++; 2862 i++; 2863 if (i == tx_ring->count) 2864 i = 0; 2865 tx_ring->next_to_use = i; 2866 2867 return true; 2868 } 2869 2870 return false; 2871} 2872 2873static int ixgbevf_tx_map(struct ixgbevf_adapter *adapter, 2874 struct ixgbevf_ring *tx_ring, 2875 struct sk_buff *skb, u32 tx_flags, 2876 unsigned int first) 2877{ 2878 struct pci_dev *pdev = adapter->pdev; 2879 struct ixgbevf_tx_buffer *tx_buffer_info; 2880 unsigned int len; 2881 unsigned int total = skb->len; 2882 unsigned int offset = 0, size; 2883 int count = 0; 2884 unsigned int nr_frags = skb_shinfo(skb)->nr_frags; 2885 unsigned int f; 2886 int i; 2887 2888 i = tx_ring->next_to_use; 2889 2890 len = min(skb_headlen(skb), total); 2891 while (len) { 2892 tx_buffer_info = &tx_ring->tx_buffer_info[i]; 2893 size = min(len, (unsigned int)IXGBE_MAX_DATA_PER_TXD); 2894 2895 tx_buffer_info->length = size; 2896 tx_buffer_info->mapped_as_page = false; 2897 tx_buffer_info->dma = dma_map_single(&adapter->pdev->dev, 2898 skb->data + offset, 2899 size, DMA_TO_DEVICE); 2900 if (dma_mapping_error(&pdev->dev, tx_buffer_info->dma)) 2901 goto dma_error; 2902 tx_buffer_info->time_stamp = jiffies; 2903 tx_buffer_info->next_to_watch = i; 2904 2905 len -= size; 2906 total -= size; 2907 offset += size; 2908 count++; 2909 i++; 2910 if (i == tx_ring->count) 2911 i = 0; 2912 } 2913 2914 for (f = 0; f < nr_frags; f++) { 2915 struct skb_frag_struct *frag; 2916 2917 frag = &skb_shinfo(skb)->frags[f]; 2918 len = min((unsigned int)frag->size, total); 2919 offset = 0; 2920 2921 while (len) { 2922 tx_buffer_info = &tx_ring->tx_buffer_info[i]; 2923 size = min(len, (unsigned int)IXGBE_MAX_DATA_PER_TXD); 2924 2925 tx_buffer_info->length = size; 2926 tx_buffer_info->dma = 2927 skb_frag_dma_map(&adapter->pdev->dev, frag, 2928 offset, size, DMA_TO_DEVICE); 2929 tx_buffer_info->mapped_as_page = true; 2930 if (dma_mapping_error(&pdev->dev, tx_buffer_info->dma)) 2931 goto dma_error; 2932 tx_buffer_info->time_stamp = jiffies; 2933 tx_buffer_info->next_to_watch = i; 2934 2935 len -= size; 2936 total -= size; 2937 offset += size; 2938 count++; 2939 i++; 2940 if (i == tx_ring->count) 2941 i = 0; 2942 } 2943 if (total == 0) 2944 break; 2945 } 2946 2947 if (i == 0) 2948 i = tx_ring->count - 1; 2949 else 2950 i = i - 1; 2951 tx_ring->tx_buffer_info[i].skb = skb; 2952 tx_ring->tx_buffer_info[first].next_to_watch = i; 2953 2954 return count; 2955 2956dma_error: 2957 dev_err(&pdev->dev, "TX DMA map failed\n"); 2958 2959 /* clear timestamp and dma mappings for failed tx_buffer_info map */ 2960 tx_buffer_info->dma = 0; 2961 tx_buffer_info->time_stamp = 0; 2962 tx_buffer_info->next_to_watch = 0; 2963 count--; 2964 2965 /* clear timestamp and dma mappings for remaining portion of packet */ 2966 while (count >= 0) { 2967 count--; 2968 i--; 2969 if (i < 0) 2970 i += tx_ring->count; 2971 tx_buffer_info = &tx_ring->tx_buffer_info[i]; 2972 ixgbevf_unmap_and_free_tx_resource(adapter, tx_buffer_info); 2973 } 2974 2975 return count; 2976} 2977 2978static void ixgbevf_tx_queue(struct ixgbevf_adapter *adapter, 2979 struct ixgbevf_ring *tx_ring, int tx_flags, 2980 int count, u32 paylen, u8 hdr_len) 2981{ 2982 union ixgbe_adv_tx_desc *tx_desc = NULL; 2983 struct ixgbevf_tx_buffer *tx_buffer_info; 2984 u32 olinfo_status = 0, cmd_type_len = 0; 2985 unsigned int i; 2986 2987 u32 txd_cmd = IXGBE_TXD_CMD_EOP | IXGBE_TXD_CMD_RS | IXGBE_TXD_CMD_IFCS; 2988 2989 cmd_type_len |= IXGBE_ADVTXD_DTYP_DATA; 2990 2991 cmd_type_len |= IXGBE_ADVTXD_DCMD_IFCS | IXGBE_ADVTXD_DCMD_DEXT; 2992 2993 if (tx_flags & IXGBE_TX_FLAGS_VLAN) 2994 cmd_type_len |= IXGBE_ADVTXD_DCMD_VLE; 2995 2996 if (tx_flags & IXGBE_TX_FLAGS_TSO) { 2997 cmd_type_len |= IXGBE_ADVTXD_DCMD_TSE; 2998 2999 olinfo_status |= IXGBE_TXD_POPTS_TXSM << 3000 IXGBE_ADVTXD_POPTS_SHIFT; 3001 3002 /* use index 1 context for tso */ 3003 olinfo_status |= (1 << IXGBE_ADVTXD_IDX_SHIFT); 3004 if (tx_flags & IXGBE_TX_FLAGS_IPV4) 3005 olinfo_status |= IXGBE_TXD_POPTS_IXSM << 3006 IXGBE_ADVTXD_POPTS_SHIFT; 3007 3008 } else if (tx_flags & IXGBE_TX_FLAGS_CSUM) 3009 olinfo_status |= IXGBE_TXD_POPTS_TXSM << 3010 IXGBE_ADVTXD_POPTS_SHIFT; 3011 3012 olinfo_status |= ((paylen - hdr_len) << IXGBE_ADVTXD_PAYLEN_SHIFT); 3013 3014 i = tx_ring->next_to_use; 3015 while (count--) { 3016 tx_buffer_info = &tx_ring->tx_buffer_info[i]; 3017 tx_desc = IXGBE_TX_DESC_ADV(*tx_ring, i); 3018 tx_desc->read.buffer_addr = cpu_to_le64(tx_buffer_info->dma); 3019 tx_desc->read.cmd_type_len = 3020 cpu_to_le32(cmd_type_len | tx_buffer_info->length); 3021 tx_desc->read.olinfo_status = cpu_to_le32(olinfo_status); 3022 i++; 3023 if (i == tx_ring->count) 3024 i = 0; 3025 } 3026 3027 tx_desc->read.cmd_type_len |= cpu_to_le32(txd_cmd); 3028 3029 /* 3030 * Force memory writes to complete before letting h/w 3031 * know there are new descriptors to fetch. (Only 3032 * applicable for weak-ordered memory model archs, 3033 * such as IA-64). 3034 */ 3035 wmb(); 3036 3037 tx_ring->next_to_use = i; 3038 writel(i, adapter->hw.hw_addr + tx_ring->tail); 3039} 3040 3041static int __ixgbevf_maybe_stop_tx(struct net_device *netdev, 3042 struct ixgbevf_ring *tx_ring, int size) 3043{ 3044 struct ixgbevf_adapter *adapter = netdev_priv(netdev); 3045 3046 netif_stop_subqueue(netdev, tx_ring->queue_index); 3047 /* Herbert's original patch had: 3048 * smp_mb__after_netif_stop_queue(); 3049 * but since that doesn't exist yet, just open code it. */ 3050 smp_mb(); 3051 3052 /* We need to check again in a case another CPU has just 3053 * made room available. */ 3054 if (likely(IXGBE_DESC_UNUSED(tx_ring) < size)) 3055 return -EBUSY; 3056 3057 /* A reprieve! - use start_queue because it doesn't call schedule */ 3058 netif_start_subqueue(netdev, tx_ring->queue_index); 3059 ++adapter->restart_queue; 3060 return 0; 3061} 3062 3063static int ixgbevf_maybe_stop_tx(struct net_device *netdev, 3064 struct ixgbevf_ring *tx_ring, int size) 3065{ 3066 if (likely(IXGBE_DESC_UNUSED(tx_ring) >= size)) 3067 return 0; 3068 return __ixgbevf_maybe_stop_tx(netdev, tx_ring, size); 3069} 3070 3071static int ixgbevf_xmit_frame(struct sk_buff *skb, struct net_device *netdev) 3072{ 3073 struct ixgbevf_adapter *adapter = netdev_priv(netdev); 3074 struct ixgbevf_ring *tx_ring; 3075 unsigned int first; 3076 unsigned int tx_flags = 0; 3077 u8 hdr_len = 0; 3078 int r_idx = 0, tso; 3079 int count = 0; 3080 3081 unsigned int f; 3082 3083 tx_ring = &adapter->tx_ring[r_idx]; 3084 3085 if (vlan_tx_tag_present(skb)) { 3086 tx_flags |= vlan_tx_tag_get(skb); 3087 tx_flags <<= IXGBE_TX_FLAGS_VLAN_SHIFT; 3088 tx_flags |= IXGBE_TX_FLAGS_VLAN; 3089 } 3090 3091 /* four things can cause us to need a context descriptor */ 3092 if (skb_is_gso(skb) || 3093 (skb->ip_summed == CHECKSUM_PARTIAL) || 3094 (tx_flags & IXGBE_TX_FLAGS_VLAN)) 3095 count++; 3096 3097 count += TXD_USE_COUNT(skb_headlen(skb)); 3098 for (f = 0; f < skb_shinfo(skb)->nr_frags; f++) 3099 count += TXD_USE_COUNT(skb_shinfo(skb)->frags[f].size); 3100 3101 if (ixgbevf_maybe_stop_tx(netdev, tx_ring, count)) { 3102 adapter->tx_busy++; 3103 return NETDEV_TX_BUSY; 3104 } 3105 3106 first = tx_ring->next_to_use; 3107 3108 if (skb->protocol == htons(ETH_P_IP)) 3109 tx_flags |= IXGBE_TX_FLAGS_IPV4; 3110 tso = ixgbevf_tso(adapter, tx_ring, skb, tx_flags, &hdr_len); 3111 if (tso < 0) { 3112 dev_kfree_skb_any(skb); 3113 return NETDEV_TX_OK; 3114 } 3115 3116 if (tso) 3117 tx_flags |= IXGBE_TX_FLAGS_TSO; 3118 else if (ixgbevf_tx_csum(adapter, tx_ring, skb, tx_flags) && 3119 (skb->ip_summed == CHECKSUM_PARTIAL)) 3120 tx_flags |= IXGBE_TX_FLAGS_CSUM; 3121 3122 ixgbevf_tx_queue(adapter, tx_ring, tx_flags, 3123 ixgbevf_tx_map(adapter, tx_ring, skb, tx_flags, first), 3124 skb->len, hdr_len); 3125 3126 ixgbevf_maybe_stop_tx(netdev, tx_ring, DESC_NEEDED); 3127 3128 return NETDEV_TX_OK; 3129} 3130 3131/** 3132 * ixgbevf_set_mac - Change the Ethernet Address of the NIC 3133 * @netdev: network interface device structure 3134 * @p: pointer to an address structure 3135 * 3136 * Returns 0 on success, negative on failure 3137 **/ 3138static int ixgbevf_set_mac(struct net_device *netdev, void *p) 3139{ 3140 struct ixgbevf_adapter *adapter = netdev_priv(netdev); 3141 struct ixgbe_hw *hw = &adapter->hw; 3142 struct sockaddr *addr = p; 3143 3144 if (!is_valid_ether_addr(addr->sa_data)) 3145 return -EADDRNOTAVAIL; 3146 3147 memcpy(netdev->dev_addr, addr->sa_data, netdev->addr_len); 3148 memcpy(hw->mac.addr, addr->sa_data, netdev->addr_len); 3149 3150 if (hw->mac.ops.set_rar) 3151 hw->mac.ops.set_rar(hw, 0, hw->mac.addr, 0); 3152 3153 return 0; 3154} 3155 3156/** 3157 * ixgbevf_change_mtu - Change the Maximum Transfer Unit 3158 * @netdev: network interface device structure 3159 * @new_mtu: new value for maximum frame size 3160 * 3161 * Returns 0 on success, negative on failure 3162 **/ 3163static int ixgbevf_change_mtu(struct net_device *netdev, int new_mtu) 3164{ 3165 struct ixgbevf_adapter *adapter = netdev_priv(netdev); 3166 struct ixgbe_hw *hw = &adapter->hw; 3167 int max_frame = new_mtu + ETH_HLEN + ETH_FCS_LEN; 3168 int max_possible_frame = MAXIMUM_ETHERNET_VLAN_SIZE; 3169 u32 msg[2]; 3170 3171 if (adapter->hw.mac.type == ixgbe_mac_X540_vf) 3172 max_possible_frame = IXGBE_MAX_JUMBO_FRAME_SIZE; 3173 3174 /* MTU < 68 is an error and causes problems on some kernels */ 3175 if ((new_mtu < 68) || (max_frame > max_possible_frame)) 3176 return -EINVAL; 3177 3178 hw_dbg(&adapter->hw, "changing MTU from %d to %d\n", 3179 netdev->mtu, new_mtu); 3180 /* must set new MTU before calling down or up */ 3181 netdev->mtu = new_mtu; 3182 3183 msg[0] = IXGBE_VF_SET_LPE; 3184 msg[1] = max_frame; 3185 hw->mbx.ops.write_posted(hw, msg, 2); 3186 3187 if (netif_running(netdev)) 3188 ixgbevf_reinit_locked(adapter); 3189 3190 return 0; 3191} 3192 3193static void ixgbevf_shutdown(struct pci_dev *pdev) 3194{ 3195 struct net_device *netdev = pci_get_drvdata(pdev); 3196 struct ixgbevf_adapter *adapter = netdev_priv(netdev); 3197 3198 netif_device_detach(netdev); 3199 3200 if (netif_running(netdev)) { 3201 ixgbevf_down(adapter); 3202 ixgbevf_free_irq(adapter); 3203 ixgbevf_free_all_tx_resources(adapter); 3204 ixgbevf_free_all_rx_resources(adapter); 3205 } 3206 3207#ifdef CONFIG_PM 3208 pci_save_state(pdev); 3209#endif 3210 3211 pci_disable_device(pdev); 3212} 3213 3214static struct rtnl_link_stats64 *ixgbevf_get_stats(struct net_device *netdev, 3215 struct rtnl_link_stats64 *stats) 3216{ 3217 struct ixgbevf_adapter *adapter = netdev_priv(netdev); 3218 unsigned int start; 3219 u64 bytes, packets; 3220 const struct ixgbevf_ring *ring; 3221 int i; 3222 3223 ixgbevf_update_stats(adapter); 3224 3225 stats->multicast = adapter->stats.vfmprc - adapter->stats.base_vfmprc; 3226 3227 for (i = 0; i < adapter->num_rx_queues; i++) { 3228 ring = &adapter->rx_ring[i]; 3229 do { 3230 start = u64_stats_fetch_begin_bh(&ring->syncp); 3231 bytes = ring->total_bytes; 3232 packets = ring->total_packets; 3233 } while (u64_stats_fetch_retry_bh(&ring->syncp, start)); 3234 stats->rx_bytes += bytes; 3235 stats->rx_packets += packets; 3236 } 3237 3238 for (i = 0; i < adapter->num_tx_queues; i++) { 3239 ring = &adapter->tx_ring[i]; 3240 do { 3241 start = u64_stats_fetch_begin_bh(&ring->syncp); 3242 bytes = ring->total_bytes; 3243 packets = ring->total_packets; 3244 } while (u64_stats_fetch_retry_bh(&ring->syncp, start)); 3245 stats->tx_bytes += bytes; 3246 stats->tx_packets += packets; 3247 } 3248 3249 return stats; 3250} 3251 3252static int ixgbevf_set_features(struct net_device *netdev, u32 features) 3253{ 3254 struct ixgbevf_adapter *adapter = netdev_priv(netdev); 3255 3256 if (features & NETIF_F_RXCSUM) 3257 adapter->flags |= IXGBE_FLAG_RX_CSUM_ENABLED; 3258 else 3259 adapter->flags &= ~IXGBE_FLAG_RX_CSUM_ENABLED; 3260 3261 return 0; 3262} 3263 3264static const struct net_device_ops ixgbe_netdev_ops = { 3265 .ndo_open = ixgbevf_open, 3266 .ndo_stop = ixgbevf_close, 3267 .ndo_start_xmit = ixgbevf_xmit_frame, 3268 .ndo_set_rx_mode = ixgbevf_set_rx_mode, 3269 .ndo_get_stats64 = ixgbevf_get_stats, 3270 .ndo_validate_addr = eth_validate_addr, 3271 .ndo_set_mac_address = ixgbevf_set_mac, 3272 .ndo_change_mtu = ixgbevf_change_mtu, 3273 .ndo_tx_timeout = ixgbevf_tx_timeout, 3274 .ndo_vlan_rx_add_vid = ixgbevf_vlan_rx_add_vid, 3275 .ndo_vlan_rx_kill_vid = ixgbevf_vlan_rx_kill_vid, 3276 .ndo_set_features = ixgbevf_set_features, 3277}; 3278 3279static void ixgbevf_assign_netdev_ops(struct net_device *dev) 3280{ 3281 dev->netdev_ops = &ixgbe_netdev_ops; 3282 ixgbevf_set_ethtool_ops(dev); 3283 dev->watchdog_timeo = 5 * HZ; 3284} 3285 3286/** 3287 * ixgbevf_probe - Device Initialization Routine 3288 * @pdev: PCI device information struct 3289 * @ent: entry in ixgbevf_pci_tbl 3290 * 3291 * Returns 0 on success, negative on failure 3292 * 3293 * ixgbevf_probe initializes an adapter identified by a pci_dev structure. 3294 * The OS initialization, configuring of the adapter private structure, 3295 * and a hardware reset occur. 3296 **/ 3297static int __devinit ixgbevf_probe(struct pci_dev *pdev, 3298 const struct pci_device_id *ent) 3299{ 3300 struct net_device *netdev; 3301 struct ixgbevf_adapter *adapter = NULL; 3302 struct ixgbe_hw *hw = NULL; 3303 const struct ixgbevf_info *ii = ixgbevf_info_tbl[ent->driver_data]; 3304 static int cards_found; 3305 int err, pci_using_dac; 3306 3307 err = pci_enable_device(pdev); 3308 if (err) 3309 return err; 3310 3311 if (!dma_set_mask(&pdev->dev, DMA_BIT_MASK(64)) && 3312 !dma_set_coherent_mask(&pdev->dev, DMA_BIT_MASK(64))) { 3313 pci_using_dac = 1; 3314 } else { 3315 err = dma_set_mask(&pdev->dev, DMA_BIT_MASK(32)); 3316 if (err) { 3317 err = dma_set_coherent_mask(&pdev->dev, 3318 DMA_BIT_MASK(32)); 3319 if (err) { 3320 dev_err(&pdev->dev, "No usable DMA " 3321 "configuration, aborting\n"); 3322 goto err_dma; 3323 } 3324 } 3325 pci_using_dac = 0; 3326 } 3327 3328 err = pci_request_regions(pdev, ixgbevf_driver_name); 3329 if (err) { 3330 dev_err(&pdev->dev, "pci_request_regions failed 0x%x\n", err); 3331 goto err_pci_reg; 3332 } 3333 3334 pci_set_master(pdev); 3335 3336#ifdef HAVE_TX_MQ 3337 netdev = alloc_etherdev_mq(sizeof(struct ixgbevf_adapter), 3338 MAX_TX_QUEUES); 3339#else 3340 netdev = alloc_etherdev(sizeof(struct ixgbevf_adapter)); 3341#endif 3342 if (!netdev) { 3343 err = -ENOMEM; 3344 goto err_alloc_etherdev; 3345 } 3346 3347 SET_NETDEV_DEV(netdev, &pdev->dev); 3348 3349 pci_set_drvdata(pdev, netdev); 3350 adapter = netdev_priv(netdev); 3351 3352 adapter->netdev = netdev; 3353 adapter->pdev = pdev; 3354 hw = &adapter->hw; 3355 hw->back = adapter; 3356 adapter->msg_enable = (1 << DEFAULT_DEBUG_LEVEL_SHIFT) - 1; 3357 3358 /* 3359 * call save state here in standalone driver because it relies on 3360 * adapter struct to exist, and needs to call netdev_priv 3361 */ 3362 pci_save_state(pdev); 3363 3364 hw->hw_addr = ioremap(pci_resource_start(pdev, 0), 3365 pci_resource_len(pdev, 0)); 3366 if (!hw->hw_addr) { 3367 err = -EIO; 3368 goto err_ioremap; 3369 } 3370 3371 ixgbevf_assign_netdev_ops(netdev); 3372 3373 adapter->bd_number = cards_found; 3374 3375 /* Setup hw api */ 3376 memcpy(&hw->mac.ops, ii->mac_ops, sizeof(hw->mac.ops)); 3377 hw->mac.type = ii->mac; 3378 3379 memcpy(&hw->mbx.ops, &ixgbevf_mbx_ops, 3380 sizeof(struct ixgbe_mbx_operations)); 3381 3382 adapter->flags &= ~IXGBE_FLAG_RX_PS_CAPABLE; 3383 adapter->flags &= ~IXGBE_FLAG_RX_PS_ENABLED; 3384 adapter->flags |= IXGBE_FLAG_RX_1BUF_CAPABLE; 3385 3386 /* setup the private structure */ 3387 err = ixgbevf_sw_init(adapter); 3388 3389 netdev->hw_features = NETIF_F_SG | 3390 NETIF_F_IP_CSUM | 3391 NETIF_F_IPV6_CSUM | 3392 NETIF_F_TSO | 3393 NETIF_F_TSO6 | 3394 NETIF_F_RXCSUM; 3395 3396 netdev->features = netdev->hw_features | 3397 NETIF_F_HW_VLAN_TX | 3398 NETIF_F_HW_VLAN_RX | 3399 NETIF_F_HW_VLAN_FILTER; 3400 3401 netdev->vlan_features |= NETIF_F_TSO; 3402 netdev->vlan_features |= NETIF_F_TSO6; 3403 netdev->vlan_features |= NETIF_F_IP_CSUM; 3404 netdev->vlan_features |= NETIF_F_IPV6_CSUM; 3405 netdev->vlan_features |= NETIF_F_SG; 3406 3407 if (pci_using_dac) 3408 netdev->features |= NETIF_F_HIGHDMA; 3409 3410 netdev->priv_flags |= IFF_UNICAST_FLT; 3411 3412 /* The HW MAC address was set and/or determined in sw_init */ 3413 memcpy(netdev->dev_addr, adapter->hw.mac.addr, netdev->addr_len); 3414 memcpy(netdev->perm_addr, adapter->hw.mac.addr, netdev->addr_len); 3415 3416 if (!is_valid_ether_addr(netdev->dev_addr)) { 3417 printk(KERN_ERR "invalid MAC address\n"); 3418 err = -EIO; 3419 goto err_sw_init; 3420 } 3421 3422 init_timer(&adapter->watchdog_timer); 3423 adapter->watchdog_timer.function = ixgbevf_watchdog; 3424 adapter->watchdog_timer.data = (unsigned long)adapter; 3425 3426 INIT_WORK(&adapter->reset_task, ixgbevf_reset_task); 3427 INIT_WORK(&adapter->watchdog_task, ixgbevf_watchdog_task); 3428 3429 err = ixgbevf_init_interrupt_scheme(adapter); 3430 if (err) 3431 goto err_sw_init; 3432 3433 /* pick up the PCI bus settings for reporting later */ 3434 if (hw->mac.ops.get_bus_info) 3435 hw->mac.ops.get_bus_info(hw); 3436 3437 strcpy(netdev->name, "eth%d"); 3438 3439 err = register_netdev(netdev); 3440 if (err) 3441 goto err_register; 3442 3443 adapter->netdev_registered = true; 3444 3445 netif_carrier_off(netdev); 3446 3447 ixgbevf_init_last_counter_stats(adapter); 3448 3449 /* print the MAC address */ 3450 hw_dbg(hw, "%2.2x:%2.2x:%2.2x:%2.2x:%2.2x:%2.2x\n", 3451 netdev->dev_addr[0], 3452 netdev->dev_addr[1], 3453 netdev->dev_addr[2], 3454 netdev->dev_addr[3], 3455 netdev->dev_addr[4], 3456 netdev->dev_addr[5]); 3457 3458 hw_dbg(hw, "MAC: %d\n", hw->mac.type); 3459 3460 hw_dbg(hw, "LRO is disabled\n"); 3461 3462 hw_dbg(hw, "Intel(R) 82599 Virtual Function\n"); 3463 cards_found++; 3464 return 0; 3465 3466err_register: 3467err_sw_init: 3468 ixgbevf_reset_interrupt_capability(adapter); 3469 iounmap(hw->hw_addr); 3470err_ioremap: 3471 free_netdev(netdev); 3472err_alloc_etherdev: 3473 pci_release_regions(pdev); 3474err_pci_reg: 3475err_dma: 3476 pci_disable_device(pdev); 3477 return err; 3478} 3479 3480/** 3481 * ixgbevf_remove - Device Removal Routine 3482 * @pdev: PCI device information struct 3483 * 3484 * ixgbevf_remove is called by the PCI subsystem to alert the driver 3485 * that it should release a PCI device. The could be caused by a 3486 * Hot-Plug event, or because the driver is going to be removed from 3487 * memory. 3488 **/ 3489static void __devexit ixgbevf_remove(struct pci_dev *pdev) 3490{ 3491 struct net_device *netdev = pci_get_drvdata(pdev); 3492 struct ixgbevf_adapter *adapter = netdev_priv(netdev); 3493 3494 set_bit(__IXGBEVF_DOWN, &adapter->state); 3495 3496 del_timer_sync(&adapter->watchdog_timer); 3497 3498 cancel_work_sync(&adapter->reset_task); 3499 cancel_work_sync(&adapter->watchdog_task); 3500 3501 if (adapter->netdev_registered) { 3502 unregister_netdev(netdev); 3503 adapter->netdev_registered = false; 3504 } 3505 3506 ixgbevf_reset_interrupt_capability(adapter); 3507 3508 iounmap(adapter->hw.hw_addr); 3509 pci_release_regions(pdev); 3510 3511 hw_dbg(&adapter->hw, "Remove complete\n"); 3512 3513 kfree(adapter->tx_ring); 3514 kfree(adapter->rx_ring); 3515 3516 free_netdev(netdev); 3517 3518 pci_disable_device(pdev); 3519} 3520 3521static struct pci_driver ixgbevf_driver = { 3522 .name = ixgbevf_driver_name, 3523 .id_table = ixgbevf_pci_tbl, 3524 .probe = ixgbevf_probe, 3525 .remove = __devexit_p(ixgbevf_remove), 3526 .shutdown = ixgbevf_shutdown, 3527}; 3528 3529/** 3530 * ixgbevf_init_module - Driver Registration Routine 3531 * 3532 * ixgbevf_init_module is the first routine called when the driver is 3533 * loaded. All it does is register with the PCI subsystem. 3534 **/ 3535static int __init ixgbevf_init_module(void) 3536{ 3537 int ret; 3538 printk(KERN_INFO "ixgbevf: %s - version %s\n", ixgbevf_driver_string, 3539 ixgbevf_driver_version); 3540 3541 printk(KERN_INFO "%s\n", ixgbevf_copyright); 3542 3543 ret = pci_register_driver(&ixgbevf_driver); 3544 return ret; 3545} 3546 3547module_init(ixgbevf_init_module); 3548 3549/** 3550 * ixgbevf_exit_module - Driver Exit Cleanup Routine 3551 * 3552 * ixgbevf_exit_module is called just before the driver is removed 3553 * from memory. 3554 **/ 3555static void __exit ixgbevf_exit_module(void) 3556{ 3557 pci_unregister_driver(&ixgbevf_driver); 3558} 3559 3560#ifdef DEBUG 3561/** 3562 * ixgbevf_get_hw_dev_name - return device name string 3563 * used by hardware layer to print debugging information 3564 **/ 3565char *ixgbevf_get_hw_dev_name(struct ixgbe_hw *hw) 3566{ 3567 struct ixgbevf_adapter *adapter = hw->back; 3568 return adapter->netdev->name; 3569} 3570 3571#endif 3572module_exit(ixgbevf_exit_module); 3573 3574/* ixgbevf_main.c */ 3575