ixgbevf_main.c revision 56e94095efb3d4f749212bf7c0b151843d157f49
1/******************************************************************************* 2 3 Intel 82599 Virtual Function driver 4 Copyright(c) 1999 - 2012 Intel Corporation. 5 6 This program is free software; you can redistribute it and/or modify it 7 under the terms and conditions of the GNU General Public License, 8 version 2, as published by the Free Software Foundation. 9 10 This program is distributed in the hope it will be useful, but WITHOUT 11 ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or 12 FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for 13 more details. 14 15 You should have received a copy of the GNU General Public License along with 16 this program; if not, write to the Free Software Foundation, Inc., 17 51 Franklin St - Fifth Floor, Boston, MA 02110-1301 USA. 18 19 The full GNU General Public License is included in this distribution in 20 the file called "COPYING". 21 22 Contact Information: 23 e1000-devel Mailing List <e1000-devel@lists.sourceforge.net> 24 Intel Corporation, 5200 N.E. Elam Young Parkway, Hillsboro, OR 97124-6497 25 26*******************************************************************************/ 27 28 29/****************************************************************************** 30 Copyright (c)2006 - 2007 Myricom, Inc. for some LRO specific code 31******************************************************************************/ 32 33#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt 34 35#include <linux/types.h> 36#include <linux/bitops.h> 37#include <linux/module.h> 38#include <linux/pci.h> 39#include <linux/netdevice.h> 40#include <linux/vmalloc.h> 41#include <linux/string.h> 42#include <linux/in.h> 43#include <linux/ip.h> 44#include <linux/tcp.h> 45#include <linux/sctp.h> 46#include <linux/ipv6.h> 47#include <linux/slab.h> 48#include <net/checksum.h> 49#include <net/ip6_checksum.h> 50#include <linux/ethtool.h> 51#include <linux/if.h> 52#include <linux/if_vlan.h> 53#include <linux/prefetch.h> 54 55#include "ixgbevf.h" 56 57const char ixgbevf_driver_name[] = "ixgbevf"; 58static const char ixgbevf_driver_string[] = 59 "Intel(R) 10 Gigabit PCI Express Virtual Function Network Driver"; 60 61#define DRV_VERSION "2.6.0-k" 62const char ixgbevf_driver_version[] = DRV_VERSION; 63static char ixgbevf_copyright[] = 64 "Copyright (c) 2009 - 2012 Intel Corporation."; 65 66static const struct ixgbevf_info *ixgbevf_info_tbl[] = { 67 [board_82599_vf] = &ixgbevf_82599_vf_info, 68 [board_X540_vf] = &ixgbevf_X540_vf_info, 69}; 70 71/* ixgbevf_pci_tbl - PCI Device ID Table 72 * 73 * Wildcard entries (PCI_ANY_ID) should come last 74 * Last entry must be all 0s 75 * 76 * { Vendor ID, Device ID, SubVendor ID, SubDevice ID, 77 * Class, Class Mask, private data (not used) } 78 */ 79static struct pci_device_id ixgbevf_pci_tbl[] = { 80 {PCI_VDEVICE(INTEL, IXGBE_DEV_ID_82599_VF), 81 board_82599_vf}, 82 {PCI_VDEVICE(INTEL, IXGBE_DEV_ID_X540_VF), 83 board_X540_vf}, 84 85 /* required last entry */ 86 {0, } 87}; 88MODULE_DEVICE_TABLE(pci, ixgbevf_pci_tbl); 89 90MODULE_AUTHOR("Intel Corporation, <linux.nics@intel.com>"); 91MODULE_DESCRIPTION("Intel(R) 82599 Virtual Function Driver"); 92MODULE_LICENSE("GPL"); 93MODULE_VERSION(DRV_VERSION); 94 95#define DEFAULT_MSG_ENABLE (NETIF_MSG_DRV|NETIF_MSG_PROBE|NETIF_MSG_LINK) 96static int debug = -1; 97module_param(debug, int, 0); 98MODULE_PARM_DESC(debug, "Debug level (0=none,...,16=all)"); 99 100/* forward decls */ 101static void ixgbevf_set_itr(struct ixgbevf_q_vector *q_vector); 102static void ixgbevf_free_all_rx_resources(struct ixgbevf_adapter *adapter); 103 104static inline void ixgbevf_release_rx_desc(struct ixgbe_hw *hw, 105 struct ixgbevf_ring *rx_ring, 106 u32 val) 107{ 108 /* 109 * Force memory writes to complete before letting h/w 110 * know there are new descriptors to fetch. (Only 111 * applicable for weak-ordered memory model archs, 112 * such as IA-64). 113 */ 114 wmb(); 115 IXGBE_WRITE_REG(hw, IXGBE_VFRDT(rx_ring->reg_idx), val); 116} 117 118/** 119 * ixgbevf_set_ivar - set IVAR registers - maps interrupt causes to vectors 120 * @adapter: pointer to adapter struct 121 * @direction: 0 for Rx, 1 for Tx, -1 for other causes 122 * @queue: queue to map the corresponding interrupt to 123 * @msix_vector: the vector to map to the corresponding queue 124 * 125 */ 126static void ixgbevf_set_ivar(struct ixgbevf_adapter *adapter, s8 direction, 127 u8 queue, u8 msix_vector) 128{ 129 u32 ivar, index; 130 struct ixgbe_hw *hw = &adapter->hw; 131 if (direction == -1) { 132 /* other causes */ 133 msix_vector |= IXGBE_IVAR_ALLOC_VAL; 134 ivar = IXGBE_READ_REG(hw, IXGBE_VTIVAR_MISC); 135 ivar &= ~0xFF; 136 ivar |= msix_vector; 137 IXGBE_WRITE_REG(hw, IXGBE_VTIVAR_MISC, ivar); 138 } else { 139 /* tx or rx causes */ 140 msix_vector |= IXGBE_IVAR_ALLOC_VAL; 141 index = ((16 * (queue & 1)) + (8 * direction)); 142 ivar = IXGBE_READ_REG(hw, IXGBE_VTIVAR(queue >> 1)); 143 ivar &= ~(0xFF << index); 144 ivar |= (msix_vector << index); 145 IXGBE_WRITE_REG(hw, IXGBE_VTIVAR(queue >> 1), ivar); 146 } 147} 148 149static void ixgbevf_unmap_and_free_tx_resource(struct ixgbevf_ring *tx_ring, 150 struct ixgbevf_tx_buffer 151 *tx_buffer_info) 152{ 153 if (tx_buffer_info->dma) { 154 if (tx_buffer_info->mapped_as_page) 155 dma_unmap_page(tx_ring->dev, 156 tx_buffer_info->dma, 157 tx_buffer_info->length, 158 DMA_TO_DEVICE); 159 else 160 dma_unmap_single(tx_ring->dev, 161 tx_buffer_info->dma, 162 tx_buffer_info->length, 163 DMA_TO_DEVICE); 164 tx_buffer_info->dma = 0; 165 } 166 if (tx_buffer_info->skb) { 167 dev_kfree_skb_any(tx_buffer_info->skb); 168 tx_buffer_info->skb = NULL; 169 } 170 tx_buffer_info->time_stamp = 0; 171 /* tx_buffer_info must be completely set up in the transmit path */ 172} 173 174#define IXGBE_MAX_TXD_PWR 14 175#define IXGBE_MAX_DATA_PER_TXD (1 << IXGBE_MAX_TXD_PWR) 176 177/* Tx Descriptors needed, worst case */ 178#define TXD_USE_COUNT(S) DIV_ROUND_UP((S), IXGBE_MAX_DATA_PER_TXD) 179#define DESC_NEEDED (MAX_SKB_FRAGS + 4) 180 181static void ixgbevf_tx_timeout(struct net_device *netdev); 182 183/** 184 * ixgbevf_clean_tx_irq - Reclaim resources after transmit completes 185 * @q_vector: board private structure 186 * @tx_ring: tx ring to clean 187 **/ 188static bool ixgbevf_clean_tx_irq(struct ixgbevf_q_vector *q_vector, 189 struct ixgbevf_ring *tx_ring) 190{ 191 struct ixgbevf_adapter *adapter = q_vector->adapter; 192 union ixgbe_adv_tx_desc *tx_desc, *eop_desc; 193 struct ixgbevf_tx_buffer *tx_buffer_info; 194 unsigned int i, eop, count = 0; 195 unsigned int total_bytes = 0, total_packets = 0; 196 197 if (test_bit(__IXGBEVF_DOWN, &adapter->state)) 198 return true; 199 200 i = tx_ring->next_to_clean; 201 eop = tx_ring->tx_buffer_info[i].next_to_watch; 202 eop_desc = IXGBEVF_TX_DESC(tx_ring, eop); 203 204 while ((eop_desc->wb.status & cpu_to_le32(IXGBE_TXD_STAT_DD)) && 205 (count < tx_ring->count)) { 206 bool cleaned = false; 207 rmb(); /* read buffer_info after eop_desc */ 208 /* eop could change between read and DD-check */ 209 if (unlikely(eop != tx_ring->tx_buffer_info[i].next_to_watch)) 210 goto cont_loop; 211 for ( ; !cleaned; count++) { 212 struct sk_buff *skb; 213 tx_desc = IXGBEVF_TX_DESC(tx_ring, i); 214 tx_buffer_info = &tx_ring->tx_buffer_info[i]; 215 cleaned = (i == eop); 216 skb = tx_buffer_info->skb; 217 218 if (cleaned && skb) { 219 unsigned int segs, bytecount; 220 221 /* gso_segs is currently only valid for tcp */ 222 segs = skb_shinfo(skb)->gso_segs ?: 1; 223 /* multiply data chunks by size of headers */ 224 bytecount = ((segs - 1) * skb_headlen(skb)) + 225 skb->len; 226 total_packets += segs; 227 total_bytes += bytecount; 228 } 229 230 ixgbevf_unmap_and_free_tx_resource(tx_ring, 231 tx_buffer_info); 232 233 tx_desc->wb.status = 0; 234 235 i++; 236 if (i == tx_ring->count) 237 i = 0; 238 } 239 240cont_loop: 241 eop = tx_ring->tx_buffer_info[i].next_to_watch; 242 eop_desc = IXGBEVF_TX_DESC(tx_ring, eop); 243 } 244 245 tx_ring->next_to_clean = i; 246 247#define TX_WAKE_THRESHOLD (DESC_NEEDED * 2) 248 if (unlikely(count && netif_carrier_ok(tx_ring->netdev) && 249 (IXGBE_DESC_UNUSED(tx_ring) >= TX_WAKE_THRESHOLD))) { 250 /* Make sure that anybody stopping the queue after this 251 * sees the new next_to_clean. 252 */ 253 smp_mb(); 254 if (__netif_subqueue_stopped(tx_ring->netdev, 255 tx_ring->queue_index) && 256 !test_bit(__IXGBEVF_DOWN, &adapter->state)) { 257 netif_wake_subqueue(tx_ring->netdev, 258 tx_ring->queue_index); 259 ++adapter->restart_queue; 260 } 261 } 262 263 u64_stats_update_begin(&tx_ring->syncp); 264 tx_ring->total_bytes += total_bytes; 265 tx_ring->total_packets += total_packets; 266 u64_stats_update_end(&tx_ring->syncp); 267 q_vector->tx.total_bytes += total_bytes; 268 q_vector->tx.total_packets += total_packets; 269 270 return count < tx_ring->count; 271} 272 273/** 274 * ixgbevf_receive_skb - Send a completed packet up the stack 275 * @q_vector: structure containing interrupt and ring information 276 * @skb: packet to send up 277 * @status: hardware indication of status of receive 278 * @rx_desc: rx descriptor 279 **/ 280static void ixgbevf_receive_skb(struct ixgbevf_q_vector *q_vector, 281 struct sk_buff *skb, u8 status, 282 union ixgbe_adv_rx_desc *rx_desc) 283{ 284 struct ixgbevf_adapter *adapter = q_vector->adapter; 285 bool is_vlan = (status & IXGBE_RXD_STAT_VP); 286 u16 tag = le16_to_cpu(rx_desc->wb.upper.vlan); 287 288 if (is_vlan && test_bit(tag & VLAN_VID_MASK, adapter->active_vlans)) 289 __vlan_hwaccel_put_tag(skb, tag); 290 291 napi_gro_receive(&q_vector->napi, skb); 292} 293 294/** 295 * ixgbevf_rx_checksum - indicate in skb if hw indicated a good cksum 296 * @adapter: address of board private structure 297 * @status_err: hardware indication of status of receive 298 * @skb: skb currently being received and modified 299 **/ 300static inline void ixgbevf_rx_checksum(struct ixgbevf_adapter *adapter, 301 struct ixgbevf_ring *ring, 302 u32 status_err, struct sk_buff *skb) 303{ 304 skb_checksum_none_assert(skb); 305 306 /* Rx csum disabled */ 307 if (!(ring->netdev->features & NETIF_F_RXCSUM)) 308 return; 309 310 /* if IP and error */ 311 if ((status_err & IXGBE_RXD_STAT_IPCS) && 312 (status_err & IXGBE_RXDADV_ERR_IPE)) { 313 adapter->hw_csum_rx_error++; 314 return; 315 } 316 317 if (!(status_err & IXGBE_RXD_STAT_L4CS)) 318 return; 319 320 if (status_err & IXGBE_RXDADV_ERR_TCPE) { 321 adapter->hw_csum_rx_error++; 322 return; 323 } 324 325 /* It must be a TCP or UDP packet with a valid checksum */ 326 skb->ip_summed = CHECKSUM_UNNECESSARY; 327 adapter->hw_csum_rx_good++; 328} 329 330/** 331 * ixgbevf_alloc_rx_buffers - Replace used receive buffers; packet split 332 * @adapter: address of board private structure 333 **/ 334static void ixgbevf_alloc_rx_buffers(struct ixgbevf_adapter *adapter, 335 struct ixgbevf_ring *rx_ring, 336 int cleaned_count) 337{ 338 struct pci_dev *pdev = adapter->pdev; 339 union ixgbe_adv_rx_desc *rx_desc; 340 struct ixgbevf_rx_buffer *bi; 341 struct sk_buff *skb; 342 unsigned int i = rx_ring->next_to_use; 343 344 bi = &rx_ring->rx_buffer_info[i]; 345 346 while (cleaned_count--) { 347 rx_desc = IXGBEVF_RX_DESC(rx_ring, i); 348 skb = bi->skb; 349 if (!skb) { 350 skb = netdev_alloc_skb_ip_align(rx_ring->netdev, 351 rx_ring->rx_buf_len); 352 if (!skb) { 353 adapter->alloc_rx_buff_failed++; 354 goto no_buffers; 355 } 356 bi->skb = skb; 357 } 358 if (!bi->dma) { 359 bi->dma = dma_map_single(&pdev->dev, skb->data, 360 rx_ring->rx_buf_len, 361 DMA_FROM_DEVICE); 362 } 363 rx_desc->read.pkt_addr = cpu_to_le64(bi->dma); 364 365 i++; 366 if (i == rx_ring->count) 367 i = 0; 368 bi = &rx_ring->rx_buffer_info[i]; 369 } 370 371no_buffers: 372 if (rx_ring->next_to_use != i) { 373 rx_ring->next_to_use = i; 374 375 ixgbevf_release_rx_desc(&adapter->hw, rx_ring, i); 376 } 377} 378 379static inline void ixgbevf_irq_enable_queues(struct ixgbevf_adapter *adapter, 380 u32 qmask) 381{ 382 struct ixgbe_hw *hw = &adapter->hw; 383 384 IXGBE_WRITE_REG(hw, IXGBE_VTEIMS, qmask); 385} 386 387static bool ixgbevf_clean_rx_irq(struct ixgbevf_q_vector *q_vector, 388 struct ixgbevf_ring *rx_ring, 389 int budget) 390{ 391 struct ixgbevf_adapter *adapter = q_vector->adapter; 392 struct pci_dev *pdev = adapter->pdev; 393 union ixgbe_adv_rx_desc *rx_desc, *next_rxd; 394 struct ixgbevf_rx_buffer *rx_buffer_info, *next_buffer; 395 struct sk_buff *skb; 396 unsigned int i; 397 u32 len, staterr; 398 int cleaned_count = 0; 399 unsigned int total_rx_bytes = 0, total_rx_packets = 0; 400 401 i = rx_ring->next_to_clean; 402 rx_desc = IXGBEVF_RX_DESC(rx_ring, i); 403 staterr = le32_to_cpu(rx_desc->wb.upper.status_error); 404 rx_buffer_info = &rx_ring->rx_buffer_info[i]; 405 406 while (staterr & IXGBE_RXD_STAT_DD) { 407 if (!budget) 408 break; 409 budget--; 410 411 rmb(); /* read descriptor and rx_buffer_info after status DD */ 412 len = le16_to_cpu(rx_desc->wb.upper.length); 413 skb = rx_buffer_info->skb; 414 prefetch(skb->data - NET_IP_ALIGN); 415 rx_buffer_info->skb = NULL; 416 417 if (rx_buffer_info->dma) { 418 dma_unmap_single(&pdev->dev, rx_buffer_info->dma, 419 rx_ring->rx_buf_len, 420 DMA_FROM_DEVICE); 421 rx_buffer_info->dma = 0; 422 skb_put(skb, len); 423 } 424 425 i++; 426 if (i == rx_ring->count) 427 i = 0; 428 429 next_rxd = IXGBEVF_RX_DESC(rx_ring, i); 430 prefetch(next_rxd); 431 cleaned_count++; 432 433 next_buffer = &rx_ring->rx_buffer_info[i]; 434 435 if (!(staterr & IXGBE_RXD_STAT_EOP)) { 436 skb->next = next_buffer->skb; 437 IXGBE_CB(skb->next)->prev = skb; 438 adapter->non_eop_descs++; 439 goto next_desc; 440 } 441 442 /* we should not be chaining buffers, if we did drop the skb */ 443 if (IXGBE_CB(skb)->prev) { 444 do { 445 struct sk_buff *this = skb; 446 skb = IXGBE_CB(skb)->prev; 447 dev_kfree_skb(this); 448 } while (skb); 449 goto next_desc; 450 } 451 452 /* ERR_MASK will only have valid bits if EOP set */ 453 if (unlikely(staterr & IXGBE_RXDADV_ERR_FRAME_ERR_MASK)) { 454 dev_kfree_skb_irq(skb); 455 goto next_desc; 456 } 457 458 ixgbevf_rx_checksum(adapter, rx_ring, staterr, skb); 459 460 /* probably a little skewed due to removing CRC */ 461 total_rx_bytes += skb->len; 462 total_rx_packets++; 463 464 /* 465 * Work around issue of some types of VM to VM loop back 466 * packets not getting split correctly 467 */ 468 if (staterr & IXGBE_RXD_STAT_LB) { 469 u32 header_fixup_len = skb_headlen(skb); 470 if (header_fixup_len < 14) 471 skb_push(skb, header_fixup_len); 472 } 473 skb->protocol = eth_type_trans(skb, rx_ring->netdev); 474 475 ixgbevf_receive_skb(q_vector, skb, staterr, rx_desc); 476 477next_desc: 478 rx_desc->wb.upper.status_error = 0; 479 480 /* return some buffers to hardware, one at a time is too slow */ 481 if (cleaned_count >= IXGBEVF_RX_BUFFER_WRITE) { 482 ixgbevf_alloc_rx_buffers(adapter, rx_ring, 483 cleaned_count); 484 cleaned_count = 0; 485 } 486 487 /* use prefetched values */ 488 rx_desc = next_rxd; 489 rx_buffer_info = &rx_ring->rx_buffer_info[i]; 490 491 staterr = le32_to_cpu(rx_desc->wb.upper.status_error); 492 } 493 494 rx_ring->next_to_clean = i; 495 cleaned_count = IXGBE_DESC_UNUSED(rx_ring); 496 497 if (cleaned_count) 498 ixgbevf_alloc_rx_buffers(adapter, rx_ring, cleaned_count); 499 500 u64_stats_update_begin(&rx_ring->syncp); 501 rx_ring->total_packets += total_rx_packets; 502 rx_ring->total_bytes += total_rx_bytes; 503 u64_stats_update_end(&rx_ring->syncp); 504 q_vector->rx.total_packets += total_rx_packets; 505 q_vector->rx.total_bytes += total_rx_bytes; 506 507 return !!budget; 508} 509 510/** 511 * ixgbevf_poll - NAPI polling calback 512 * @napi: napi struct with our devices info in it 513 * @budget: amount of work driver is allowed to do this pass, in packets 514 * 515 * This function will clean more than one or more rings associated with a 516 * q_vector. 517 **/ 518static int ixgbevf_poll(struct napi_struct *napi, int budget) 519{ 520 struct ixgbevf_q_vector *q_vector = 521 container_of(napi, struct ixgbevf_q_vector, napi); 522 struct ixgbevf_adapter *adapter = q_vector->adapter; 523 struct ixgbevf_ring *ring; 524 int per_ring_budget; 525 bool clean_complete = true; 526 527 ixgbevf_for_each_ring(ring, q_vector->tx) 528 clean_complete &= ixgbevf_clean_tx_irq(q_vector, ring); 529 530 /* attempt to distribute budget to each queue fairly, but don't allow 531 * the budget to go below 1 because we'll exit polling */ 532 if (q_vector->rx.count > 1) 533 per_ring_budget = max(budget/q_vector->rx.count, 1); 534 else 535 per_ring_budget = budget; 536 537 ixgbevf_for_each_ring(ring, q_vector->rx) 538 clean_complete &= ixgbevf_clean_rx_irq(q_vector, ring, 539 per_ring_budget); 540 541 /* If all work not completed, return budget and keep polling */ 542 if (!clean_complete) 543 return budget; 544 /* all work done, exit the polling mode */ 545 napi_complete(napi); 546 if (adapter->rx_itr_setting & 1) 547 ixgbevf_set_itr(q_vector); 548 if (!test_bit(__IXGBEVF_DOWN, &adapter->state)) 549 ixgbevf_irq_enable_queues(adapter, 550 1 << q_vector->v_idx); 551 552 return 0; 553} 554 555/** 556 * ixgbevf_write_eitr - write VTEITR register in hardware specific way 557 * @q_vector: structure containing interrupt and ring information 558 */ 559static void ixgbevf_write_eitr(struct ixgbevf_q_vector *q_vector) 560{ 561 struct ixgbevf_adapter *adapter = q_vector->adapter; 562 struct ixgbe_hw *hw = &adapter->hw; 563 int v_idx = q_vector->v_idx; 564 u32 itr_reg = q_vector->itr & IXGBE_MAX_EITR; 565 566 /* 567 * set the WDIS bit to not clear the timer bits and cause an 568 * immediate assertion of the interrupt 569 */ 570 itr_reg |= IXGBE_EITR_CNT_WDIS; 571 572 IXGBE_WRITE_REG(hw, IXGBE_VTEITR(v_idx), itr_reg); 573} 574 575/** 576 * ixgbevf_configure_msix - Configure MSI-X hardware 577 * @adapter: board private structure 578 * 579 * ixgbevf_configure_msix sets up the hardware to properly generate MSI-X 580 * interrupts. 581 **/ 582static void ixgbevf_configure_msix(struct ixgbevf_adapter *adapter) 583{ 584 struct ixgbevf_q_vector *q_vector; 585 int q_vectors, v_idx; 586 587 q_vectors = adapter->num_msix_vectors - NON_Q_VECTORS; 588 adapter->eims_enable_mask = 0; 589 590 /* 591 * Populate the IVAR table and set the ITR values to the 592 * corresponding register. 593 */ 594 for (v_idx = 0; v_idx < q_vectors; v_idx++) { 595 struct ixgbevf_ring *ring; 596 q_vector = adapter->q_vector[v_idx]; 597 598 ixgbevf_for_each_ring(ring, q_vector->rx) 599 ixgbevf_set_ivar(adapter, 0, ring->reg_idx, v_idx); 600 601 ixgbevf_for_each_ring(ring, q_vector->tx) 602 ixgbevf_set_ivar(adapter, 1, ring->reg_idx, v_idx); 603 604 if (q_vector->tx.ring && !q_vector->rx.ring) { 605 /* tx only vector */ 606 if (adapter->tx_itr_setting == 1) 607 q_vector->itr = IXGBE_10K_ITR; 608 else 609 q_vector->itr = adapter->tx_itr_setting; 610 } else { 611 /* rx or rx/tx vector */ 612 if (adapter->rx_itr_setting == 1) 613 q_vector->itr = IXGBE_20K_ITR; 614 else 615 q_vector->itr = adapter->rx_itr_setting; 616 } 617 618 /* add q_vector eims value to global eims_enable_mask */ 619 adapter->eims_enable_mask |= 1 << v_idx; 620 621 ixgbevf_write_eitr(q_vector); 622 } 623 624 ixgbevf_set_ivar(adapter, -1, 1, v_idx); 625 /* setup eims_other and add value to global eims_enable_mask */ 626 adapter->eims_other = 1 << v_idx; 627 adapter->eims_enable_mask |= adapter->eims_other; 628} 629 630enum latency_range { 631 lowest_latency = 0, 632 low_latency = 1, 633 bulk_latency = 2, 634 latency_invalid = 255 635}; 636 637/** 638 * ixgbevf_update_itr - update the dynamic ITR value based on statistics 639 * @q_vector: structure containing interrupt and ring information 640 * @ring_container: structure containing ring performance data 641 * 642 * Stores a new ITR value based on packets and byte 643 * counts during the last interrupt. The advantage of per interrupt 644 * computation is faster updates and more accurate ITR for the current 645 * traffic pattern. Constants in this function were computed 646 * based on theoretical maximum wire speed and thresholds were set based 647 * on testing data as well as attempting to minimize response time 648 * while increasing bulk throughput. 649 **/ 650static void ixgbevf_update_itr(struct ixgbevf_q_vector *q_vector, 651 struct ixgbevf_ring_container *ring_container) 652{ 653 int bytes = ring_container->total_bytes; 654 int packets = ring_container->total_packets; 655 u32 timepassed_us; 656 u64 bytes_perint; 657 u8 itr_setting = ring_container->itr; 658 659 if (packets == 0) 660 return; 661 662 /* simple throttlerate management 663 * 0-20MB/s lowest (100000 ints/s) 664 * 20-100MB/s low (20000 ints/s) 665 * 100-1249MB/s bulk (8000 ints/s) 666 */ 667 /* what was last interrupt timeslice? */ 668 timepassed_us = q_vector->itr >> 2; 669 bytes_perint = bytes / timepassed_us; /* bytes/usec */ 670 671 switch (itr_setting) { 672 case lowest_latency: 673 if (bytes_perint > 10) 674 itr_setting = low_latency; 675 break; 676 case low_latency: 677 if (bytes_perint > 20) 678 itr_setting = bulk_latency; 679 else if (bytes_perint <= 10) 680 itr_setting = lowest_latency; 681 break; 682 case bulk_latency: 683 if (bytes_perint <= 20) 684 itr_setting = low_latency; 685 break; 686 } 687 688 /* clear work counters since we have the values we need */ 689 ring_container->total_bytes = 0; 690 ring_container->total_packets = 0; 691 692 /* write updated itr to ring container */ 693 ring_container->itr = itr_setting; 694} 695 696static void ixgbevf_set_itr(struct ixgbevf_q_vector *q_vector) 697{ 698 u32 new_itr = q_vector->itr; 699 u8 current_itr; 700 701 ixgbevf_update_itr(q_vector, &q_vector->tx); 702 ixgbevf_update_itr(q_vector, &q_vector->rx); 703 704 current_itr = max(q_vector->rx.itr, q_vector->tx.itr); 705 706 switch (current_itr) { 707 /* counts and packets in update_itr are dependent on these numbers */ 708 case lowest_latency: 709 new_itr = IXGBE_100K_ITR; 710 break; 711 case low_latency: 712 new_itr = IXGBE_20K_ITR; 713 break; 714 case bulk_latency: 715 default: 716 new_itr = IXGBE_8K_ITR; 717 break; 718 } 719 720 if (new_itr != q_vector->itr) { 721 /* do an exponential smoothing */ 722 new_itr = (10 * new_itr * q_vector->itr) / 723 ((9 * new_itr) + q_vector->itr); 724 725 /* save the algorithm value here */ 726 q_vector->itr = new_itr; 727 728 ixgbevf_write_eitr(q_vector); 729 } 730} 731 732static irqreturn_t ixgbevf_msix_other(int irq, void *data) 733{ 734 struct ixgbevf_adapter *adapter = data; 735 struct ixgbe_hw *hw = &adapter->hw; 736 737 hw->mac.get_link_status = 1; 738 739 if (!test_bit(__IXGBEVF_DOWN, &adapter->state)) 740 mod_timer(&adapter->watchdog_timer, jiffies); 741 742 IXGBE_WRITE_REG(hw, IXGBE_VTEIMS, adapter->eims_other); 743 744 return IRQ_HANDLED; 745} 746 747 748/** 749 * ixgbevf_msix_clean_rings - single unshared vector rx clean (all queues) 750 * @irq: unused 751 * @data: pointer to our q_vector struct for this interrupt vector 752 **/ 753static irqreturn_t ixgbevf_msix_clean_rings(int irq, void *data) 754{ 755 struct ixgbevf_q_vector *q_vector = data; 756 757 /* EIAM disabled interrupts (on this vector) for us */ 758 if (q_vector->rx.ring || q_vector->tx.ring) 759 napi_schedule(&q_vector->napi); 760 761 return IRQ_HANDLED; 762} 763 764static inline void map_vector_to_rxq(struct ixgbevf_adapter *a, int v_idx, 765 int r_idx) 766{ 767 struct ixgbevf_q_vector *q_vector = a->q_vector[v_idx]; 768 769 a->rx_ring[r_idx].next = q_vector->rx.ring; 770 q_vector->rx.ring = &a->rx_ring[r_idx]; 771 q_vector->rx.count++; 772} 773 774static inline void map_vector_to_txq(struct ixgbevf_adapter *a, int v_idx, 775 int t_idx) 776{ 777 struct ixgbevf_q_vector *q_vector = a->q_vector[v_idx]; 778 779 a->tx_ring[t_idx].next = q_vector->tx.ring; 780 q_vector->tx.ring = &a->tx_ring[t_idx]; 781 q_vector->tx.count++; 782} 783 784/** 785 * ixgbevf_map_rings_to_vectors - Maps descriptor rings to vectors 786 * @adapter: board private structure to initialize 787 * 788 * This function maps descriptor rings to the queue-specific vectors 789 * we were allotted through the MSI-X enabling code. Ideally, we'd have 790 * one vector per ring/queue, but on a constrained vector budget, we 791 * group the rings as "efficiently" as possible. You would add new 792 * mapping configurations in here. 793 **/ 794static int ixgbevf_map_rings_to_vectors(struct ixgbevf_adapter *adapter) 795{ 796 int q_vectors; 797 int v_start = 0; 798 int rxr_idx = 0, txr_idx = 0; 799 int rxr_remaining = adapter->num_rx_queues; 800 int txr_remaining = adapter->num_tx_queues; 801 int i, j; 802 int rqpv, tqpv; 803 int err = 0; 804 805 q_vectors = adapter->num_msix_vectors - NON_Q_VECTORS; 806 807 /* 808 * The ideal configuration... 809 * We have enough vectors to map one per queue. 810 */ 811 if (q_vectors == adapter->num_rx_queues + adapter->num_tx_queues) { 812 for (; rxr_idx < rxr_remaining; v_start++, rxr_idx++) 813 map_vector_to_rxq(adapter, v_start, rxr_idx); 814 815 for (; txr_idx < txr_remaining; v_start++, txr_idx++) 816 map_vector_to_txq(adapter, v_start, txr_idx); 817 goto out; 818 } 819 820 /* 821 * If we don't have enough vectors for a 1-to-1 822 * mapping, we'll have to group them so there are 823 * multiple queues per vector. 824 */ 825 /* Re-adjusting *qpv takes care of the remainder. */ 826 for (i = v_start; i < q_vectors; i++) { 827 rqpv = DIV_ROUND_UP(rxr_remaining, q_vectors - i); 828 for (j = 0; j < rqpv; j++) { 829 map_vector_to_rxq(adapter, i, rxr_idx); 830 rxr_idx++; 831 rxr_remaining--; 832 } 833 } 834 for (i = v_start; i < q_vectors; i++) { 835 tqpv = DIV_ROUND_UP(txr_remaining, q_vectors - i); 836 for (j = 0; j < tqpv; j++) { 837 map_vector_to_txq(adapter, i, txr_idx); 838 txr_idx++; 839 txr_remaining--; 840 } 841 } 842 843out: 844 return err; 845} 846 847/** 848 * ixgbevf_request_msix_irqs - Initialize MSI-X interrupts 849 * @adapter: board private structure 850 * 851 * ixgbevf_request_msix_irqs allocates MSI-X vectors and requests 852 * interrupts from the kernel. 853 **/ 854static int ixgbevf_request_msix_irqs(struct ixgbevf_adapter *adapter) 855{ 856 struct net_device *netdev = adapter->netdev; 857 int q_vectors = adapter->num_msix_vectors - NON_Q_VECTORS; 858 int vector, err; 859 int ri = 0, ti = 0; 860 861 for (vector = 0; vector < q_vectors; vector++) { 862 struct ixgbevf_q_vector *q_vector = adapter->q_vector[vector]; 863 struct msix_entry *entry = &adapter->msix_entries[vector]; 864 865 if (q_vector->tx.ring && q_vector->rx.ring) { 866 snprintf(q_vector->name, sizeof(q_vector->name) - 1, 867 "%s-%s-%d", netdev->name, "TxRx", ri++); 868 ti++; 869 } else if (q_vector->rx.ring) { 870 snprintf(q_vector->name, sizeof(q_vector->name) - 1, 871 "%s-%s-%d", netdev->name, "rx", ri++); 872 } else if (q_vector->tx.ring) { 873 snprintf(q_vector->name, sizeof(q_vector->name) - 1, 874 "%s-%s-%d", netdev->name, "tx", ti++); 875 } else { 876 /* skip this unused q_vector */ 877 continue; 878 } 879 err = request_irq(entry->vector, &ixgbevf_msix_clean_rings, 0, 880 q_vector->name, q_vector); 881 if (err) { 882 hw_dbg(&adapter->hw, 883 "request_irq failed for MSIX interrupt " 884 "Error: %d\n", err); 885 goto free_queue_irqs; 886 } 887 } 888 889 err = request_irq(adapter->msix_entries[vector].vector, 890 &ixgbevf_msix_other, 0, netdev->name, adapter); 891 if (err) { 892 hw_dbg(&adapter->hw, 893 "request_irq for msix_other failed: %d\n", err); 894 goto free_queue_irqs; 895 } 896 897 return 0; 898 899free_queue_irqs: 900 while (vector) { 901 vector--; 902 free_irq(adapter->msix_entries[vector].vector, 903 adapter->q_vector[vector]); 904 } 905 pci_disable_msix(adapter->pdev); 906 kfree(adapter->msix_entries); 907 adapter->msix_entries = NULL; 908 return err; 909} 910 911static inline void ixgbevf_reset_q_vectors(struct ixgbevf_adapter *adapter) 912{ 913 int i, q_vectors = adapter->num_msix_vectors - NON_Q_VECTORS; 914 915 for (i = 0; i < q_vectors; i++) { 916 struct ixgbevf_q_vector *q_vector = adapter->q_vector[i]; 917 q_vector->rx.ring = NULL; 918 q_vector->tx.ring = NULL; 919 q_vector->rx.count = 0; 920 q_vector->tx.count = 0; 921 } 922} 923 924/** 925 * ixgbevf_request_irq - initialize interrupts 926 * @adapter: board private structure 927 * 928 * Attempts to configure interrupts using the best available 929 * capabilities of the hardware and kernel. 930 **/ 931static int ixgbevf_request_irq(struct ixgbevf_adapter *adapter) 932{ 933 int err = 0; 934 935 err = ixgbevf_request_msix_irqs(adapter); 936 937 if (err) 938 hw_dbg(&adapter->hw, 939 "request_irq failed, Error %d\n", err); 940 941 return err; 942} 943 944static void ixgbevf_free_irq(struct ixgbevf_adapter *adapter) 945{ 946 int i, q_vectors; 947 948 q_vectors = adapter->num_msix_vectors; 949 i = q_vectors - 1; 950 951 free_irq(adapter->msix_entries[i].vector, adapter); 952 i--; 953 954 for (; i >= 0; i--) { 955 /* free only the irqs that were actually requested */ 956 if (!adapter->q_vector[i]->rx.ring && 957 !adapter->q_vector[i]->tx.ring) 958 continue; 959 960 free_irq(adapter->msix_entries[i].vector, 961 adapter->q_vector[i]); 962 } 963 964 ixgbevf_reset_q_vectors(adapter); 965} 966 967/** 968 * ixgbevf_irq_disable - Mask off interrupt generation on the NIC 969 * @adapter: board private structure 970 **/ 971static inline void ixgbevf_irq_disable(struct ixgbevf_adapter *adapter) 972{ 973 struct ixgbe_hw *hw = &adapter->hw; 974 int i; 975 976 IXGBE_WRITE_REG(hw, IXGBE_VTEIAM, 0); 977 IXGBE_WRITE_REG(hw, IXGBE_VTEIMC, ~0); 978 IXGBE_WRITE_REG(hw, IXGBE_VTEIAC, 0); 979 980 IXGBE_WRITE_FLUSH(hw); 981 982 for (i = 0; i < adapter->num_msix_vectors; i++) 983 synchronize_irq(adapter->msix_entries[i].vector); 984} 985 986/** 987 * ixgbevf_irq_enable - Enable default interrupt generation settings 988 * @adapter: board private structure 989 **/ 990static inline void ixgbevf_irq_enable(struct ixgbevf_adapter *adapter) 991{ 992 struct ixgbe_hw *hw = &adapter->hw; 993 994 IXGBE_WRITE_REG(hw, IXGBE_VTEIAM, adapter->eims_enable_mask); 995 IXGBE_WRITE_REG(hw, IXGBE_VTEIAC, adapter->eims_enable_mask); 996 IXGBE_WRITE_REG(hw, IXGBE_VTEIMS, adapter->eims_enable_mask); 997} 998 999/** 1000 * ixgbevf_configure_tx - Configure 82599 VF Transmit Unit after Reset 1001 * @adapter: board private structure 1002 * 1003 * Configure the Tx unit of the MAC after a reset. 1004 **/ 1005static void ixgbevf_configure_tx(struct ixgbevf_adapter *adapter) 1006{ 1007 u64 tdba; 1008 struct ixgbe_hw *hw = &adapter->hw; 1009 u32 i, j, tdlen, txctrl; 1010 1011 /* Setup the HW Tx Head and Tail descriptor pointers */ 1012 for (i = 0; i < adapter->num_tx_queues; i++) { 1013 struct ixgbevf_ring *ring = &adapter->tx_ring[i]; 1014 j = ring->reg_idx; 1015 tdba = ring->dma; 1016 tdlen = ring->count * sizeof(union ixgbe_adv_tx_desc); 1017 IXGBE_WRITE_REG(hw, IXGBE_VFTDBAL(j), 1018 (tdba & DMA_BIT_MASK(32))); 1019 IXGBE_WRITE_REG(hw, IXGBE_VFTDBAH(j), (tdba >> 32)); 1020 IXGBE_WRITE_REG(hw, IXGBE_VFTDLEN(j), tdlen); 1021 IXGBE_WRITE_REG(hw, IXGBE_VFTDH(j), 0); 1022 IXGBE_WRITE_REG(hw, IXGBE_VFTDT(j), 0); 1023 adapter->tx_ring[i].head = IXGBE_VFTDH(j); 1024 adapter->tx_ring[i].tail = IXGBE_VFTDT(j); 1025 /* Disable Tx Head Writeback RO bit, since this hoses 1026 * bookkeeping if things aren't delivered in order. 1027 */ 1028 txctrl = IXGBE_READ_REG(hw, IXGBE_VFDCA_TXCTRL(j)); 1029 txctrl &= ~IXGBE_DCA_TXCTRL_TX_WB_RO_EN; 1030 IXGBE_WRITE_REG(hw, IXGBE_VFDCA_TXCTRL(j), txctrl); 1031 } 1032} 1033 1034#define IXGBE_SRRCTL_BSIZEHDRSIZE_SHIFT 2 1035 1036static void ixgbevf_configure_srrctl(struct ixgbevf_adapter *adapter, int index) 1037{ 1038 struct ixgbevf_ring *rx_ring; 1039 struct ixgbe_hw *hw = &adapter->hw; 1040 u32 srrctl; 1041 1042 rx_ring = &adapter->rx_ring[index]; 1043 1044 srrctl = IXGBE_SRRCTL_DROP_EN; 1045 1046 srrctl |= IXGBE_SRRCTL_DESCTYPE_ADV_ONEBUF; 1047 1048 srrctl |= ALIGN(rx_ring->rx_buf_len, 1024) >> 1049 IXGBE_SRRCTL_BSIZEPKT_SHIFT; 1050 1051 IXGBE_WRITE_REG(hw, IXGBE_VFSRRCTL(index), srrctl); 1052} 1053 1054static void ixgbevf_set_rx_buffer_len(struct ixgbevf_adapter *adapter) 1055{ 1056 struct ixgbe_hw *hw = &adapter->hw; 1057 struct net_device *netdev = adapter->netdev; 1058 int max_frame = netdev->mtu + ETH_HLEN + ETH_FCS_LEN; 1059 int i; 1060 u16 rx_buf_len; 1061 1062 /* notify the PF of our intent to use this size of frame */ 1063 ixgbevf_rlpml_set_vf(hw, max_frame); 1064 1065 /* PF will allow an extra 4 bytes past for vlan tagged frames */ 1066 max_frame += VLAN_HLEN; 1067 1068 /* 1069 * Make best use of allocation by using all but 1K of a 1070 * power of 2 allocation that will be used for skb->head. 1071 */ 1072 if ((hw->mac.type == ixgbe_mac_X540_vf) && 1073 (max_frame <= MAXIMUM_ETHERNET_VLAN_SIZE)) 1074 rx_buf_len = MAXIMUM_ETHERNET_VLAN_SIZE; 1075 else if (max_frame <= IXGBEVF_RXBUFFER_3K) 1076 rx_buf_len = IXGBEVF_RXBUFFER_3K; 1077 else if (max_frame <= IXGBEVF_RXBUFFER_7K) 1078 rx_buf_len = IXGBEVF_RXBUFFER_7K; 1079 else if (max_frame <= IXGBEVF_RXBUFFER_15K) 1080 rx_buf_len = IXGBEVF_RXBUFFER_15K; 1081 else 1082 rx_buf_len = IXGBEVF_MAX_RXBUFFER; 1083 1084 for (i = 0; i < adapter->num_rx_queues; i++) 1085 adapter->rx_ring[i].rx_buf_len = rx_buf_len; 1086} 1087 1088/** 1089 * ixgbevf_configure_rx - Configure 82599 VF Receive Unit after Reset 1090 * @adapter: board private structure 1091 * 1092 * Configure the Rx unit of the MAC after a reset. 1093 **/ 1094static void ixgbevf_configure_rx(struct ixgbevf_adapter *adapter) 1095{ 1096 u64 rdba; 1097 struct ixgbe_hw *hw = &adapter->hw; 1098 int i, j; 1099 u32 rdlen; 1100 1101 /* PSRTYPE must be initialized in 82599 */ 1102 IXGBE_WRITE_REG(hw, IXGBE_VFPSRTYPE, 0); 1103 1104 /* set_rx_buffer_len must be called before ring initialization */ 1105 ixgbevf_set_rx_buffer_len(adapter); 1106 1107 rdlen = adapter->rx_ring[0].count * sizeof(union ixgbe_adv_rx_desc); 1108 /* Setup the HW Rx Head and Tail Descriptor Pointers and 1109 * the Base and Length of the Rx Descriptor Ring */ 1110 for (i = 0; i < adapter->num_rx_queues; i++) { 1111 rdba = adapter->rx_ring[i].dma; 1112 j = adapter->rx_ring[i].reg_idx; 1113 IXGBE_WRITE_REG(hw, IXGBE_VFRDBAL(j), 1114 (rdba & DMA_BIT_MASK(32))); 1115 IXGBE_WRITE_REG(hw, IXGBE_VFRDBAH(j), (rdba >> 32)); 1116 IXGBE_WRITE_REG(hw, IXGBE_VFRDLEN(j), rdlen); 1117 IXGBE_WRITE_REG(hw, IXGBE_VFRDH(j), 0); 1118 IXGBE_WRITE_REG(hw, IXGBE_VFRDT(j), 0); 1119 adapter->rx_ring[i].head = IXGBE_VFRDH(j); 1120 adapter->rx_ring[i].tail = IXGBE_VFRDT(j); 1121 1122 ixgbevf_configure_srrctl(adapter, j); 1123 } 1124} 1125 1126static int ixgbevf_vlan_rx_add_vid(struct net_device *netdev, u16 vid) 1127{ 1128 struct ixgbevf_adapter *adapter = netdev_priv(netdev); 1129 struct ixgbe_hw *hw = &adapter->hw; 1130 int err; 1131 1132 if (!hw->mac.ops.set_vfta) 1133 return -EOPNOTSUPP; 1134 1135 spin_lock(&adapter->mbx_lock); 1136 1137 /* add VID to filter table */ 1138 err = hw->mac.ops.set_vfta(hw, vid, 0, true); 1139 1140 spin_unlock(&adapter->mbx_lock); 1141 1142 /* translate error return types so error makes sense */ 1143 if (err == IXGBE_ERR_MBX) 1144 return -EIO; 1145 1146 if (err == IXGBE_ERR_INVALID_ARGUMENT) 1147 return -EACCES; 1148 1149 set_bit(vid, adapter->active_vlans); 1150 1151 return err; 1152} 1153 1154static int ixgbevf_vlan_rx_kill_vid(struct net_device *netdev, u16 vid) 1155{ 1156 struct ixgbevf_adapter *adapter = netdev_priv(netdev); 1157 struct ixgbe_hw *hw = &adapter->hw; 1158 int err = -EOPNOTSUPP; 1159 1160 spin_lock(&adapter->mbx_lock); 1161 1162 /* remove VID from filter table */ 1163 if (hw->mac.ops.set_vfta) 1164 err = hw->mac.ops.set_vfta(hw, vid, 0, false); 1165 1166 spin_unlock(&adapter->mbx_lock); 1167 1168 clear_bit(vid, adapter->active_vlans); 1169 1170 return err; 1171} 1172 1173static void ixgbevf_restore_vlan(struct ixgbevf_adapter *adapter) 1174{ 1175 u16 vid; 1176 1177 for_each_set_bit(vid, adapter->active_vlans, VLAN_N_VID) 1178 ixgbevf_vlan_rx_add_vid(adapter->netdev, vid); 1179} 1180 1181static int ixgbevf_write_uc_addr_list(struct net_device *netdev) 1182{ 1183 struct ixgbevf_adapter *adapter = netdev_priv(netdev); 1184 struct ixgbe_hw *hw = &adapter->hw; 1185 int count = 0; 1186 1187 if ((netdev_uc_count(netdev)) > 10) { 1188 pr_err("Too many unicast filters - No Space\n"); 1189 return -ENOSPC; 1190 } 1191 1192 if (!netdev_uc_empty(netdev)) { 1193 struct netdev_hw_addr *ha; 1194 netdev_for_each_uc_addr(ha, netdev) { 1195 hw->mac.ops.set_uc_addr(hw, ++count, ha->addr); 1196 udelay(200); 1197 } 1198 } else { 1199 /* 1200 * If the list is empty then send message to PF driver to 1201 * clear all macvlans on this VF. 1202 */ 1203 hw->mac.ops.set_uc_addr(hw, 0, NULL); 1204 } 1205 1206 return count; 1207} 1208 1209/** 1210 * ixgbevf_set_rx_mode - Multicast set 1211 * @netdev: network interface device structure 1212 * 1213 * The set_rx_method entry point is called whenever the multicast address 1214 * list or the network interface flags are updated. This routine is 1215 * responsible for configuring the hardware for proper multicast mode. 1216 **/ 1217static void ixgbevf_set_rx_mode(struct net_device *netdev) 1218{ 1219 struct ixgbevf_adapter *adapter = netdev_priv(netdev); 1220 struct ixgbe_hw *hw = &adapter->hw; 1221 1222 spin_lock(&adapter->mbx_lock); 1223 1224 /* reprogram multicast list */ 1225 if (hw->mac.ops.update_mc_addr_list) 1226 hw->mac.ops.update_mc_addr_list(hw, netdev); 1227 1228 ixgbevf_write_uc_addr_list(netdev); 1229 1230 spin_unlock(&adapter->mbx_lock); 1231} 1232 1233static void ixgbevf_napi_enable_all(struct ixgbevf_adapter *adapter) 1234{ 1235 int q_idx; 1236 struct ixgbevf_q_vector *q_vector; 1237 int q_vectors = adapter->num_msix_vectors - NON_Q_VECTORS; 1238 1239 for (q_idx = 0; q_idx < q_vectors; q_idx++) { 1240 q_vector = adapter->q_vector[q_idx]; 1241 napi_enable(&q_vector->napi); 1242 } 1243} 1244 1245static void ixgbevf_napi_disable_all(struct ixgbevf_adapter *adapter) 1246{ 1247 int q_idx; 1248 struct ixgbevf_q_vector *q_vector; 1249 int q_vectors = adapter->num_msix_vectors - NON_Q_VECTORS; 1250 1251 for (q_idx = 0; q_idx < q_vectors; q_idx++) { 1252 q_vector = adapter->q_vector[q_idx]; 1253 napi_disable(&q_vector->napi); 1254 } 1255} 1256 1257static void ixgbevf_configure(struct ixgbevf_adapter *adapter) 1258{ 1259 struct net_device *netdev = adapter->netdev; 1260 int i; 1261 1262 ixgbevf_set_rx_mode(netdev); 1263 1264 ixgbevf_restore_vlan(adapter); 1265 1266 ixgbevf_configure_tx(adapter); 1267 ixgbevf_configure_rx(adapter); 1268 for (i = 0; i < adapter->num_rx_queues; i++) { 1269 struct ixgbevf_ring *ring = &adapter->rx_ring[i]; 1270 ixgbevf_alloc_rx_buffers(adapter, ring, 1271 IXGBE_DESC_UNUSED(ring)); 1272 } 1273} 1274 1275#define IXGBE_MAX_RX_DESC_POLL 10 1276static inline void ixgbevf_rx_desc_queue_enable(struct ixgbevf_adapter *adapter, 1277 int rxr) 1278{ 1279 struct ixgbe_hw *hw = &adapter->hw; 1280 int j = adapter->rx_ring[rxr].reg_idx; 1281 int k; 1282 1283 for (k = 0; k < IXGBE_MAX_RX_DESC_POLL; k++) { 1284 if (IXGBE_READ_REG(hw, IXGBE_VFRXDCTL(j)) & IXGBE_RXDCTL_ENABLE) 1285 break; 1286 else 1287 msleep(1); 1288 } 1289 if (k >= IXGBE_MAX_RX_DESC_POLL) { 1290 hw_dbg(hw, "RXDCTL.ENABLE on Rx queue %d " 1291 "not set within the polling period\n", rxr); 1292 } 1293 1294 ixgbevf_release_rx_desc(&adapter->hw, &adapter->rx_ring[rxr], 1295 (adapter->rx_ring[rxr].count - 1)); 1296} 1297 1298static void ixgbevf_save_reset_stats(struct ixgbevf_adapter *adapter) 1299{ 1300 /* Only save pre-reset stats if there are some */ 1301 if (adapter->stats.vfgprc || adapter->stats.vfgptc) { 1302 adapter->stats.saved_reset_vfgprc += adapter->stats.vfgprc - 1303 adapter->stats.base_vfgprc; 1304 adapter->stats.saved_reset_vfgptc += adapter->stats.vfgptc - 1305 adapter->stats.base_vfgptc; 1306 adapter->stats.saved_reset_vfgorc += adapter->stats.vfgorc - 1307 adapter->stats.base_vfgorc; 1308 adapter->stats.saved_reset_vfgotc += adapter->stats.vfgotc - 1309 adapter->stats.base_vfgotc; 1310 adapter->stats.saved_reset_vfmprc += adapter->stats.vfmprc - 1311 adapter->stats.base_vfmprc; 1312 } 1313} 1314 1315static void ixgbevf_init_last_counter_stats(struct ixgbevf_adapter *adapter) 1316{ 1317 struct ixgbe_hw *hw = &adapter->hw; 1318 1319 adapter->stats.last_vfgprc = IXGBE_READ_REG(hw, IXGBE_VFGPRC); 1320 adapter->stats.last_vfgorc = IXGBE_READ_REG(hw, IXGBE_VFGORC_LSB); 1321 adapter->stats.last_vfgorc |= 1322 (((u64)(IXGBE_READ_REG(hw, IXGBE_VFGORC_MSB))) << 32); 1323 adapter->stats.last_vfgptc = IXGBE_READ_REG(hw, IXGBE_VFGPTC); 1324 adapter->stats.last_vfgotc = IXGBE_READ_REG(hw, IXGBE_VFGOTC_LSB); 1325 adapter->stats.last_vfgotc |= 1326 (((u64)(IXGBE_READ_REG(hw, IXGBE_VFGOTC_MSB))) << 32); 1327 adapter->stats.last_vfmprc = IXGBE_READ_REG(hw, IXGBE_VFMPRC); 1328 1329 adapter->stats.base_vfgprc = adapter->stats.last_vfgprc; 1330 adapter->stats.base_vfgorc = adapter->stats.last_vfgorc; 1331 adapter->stats.base_vfgptc = adapter->stats.last_vfgptc; 1332 adapter->stats.base_vfgotc = adapter->stats.last_vfgotc; 1333 adapter->stats.base_vfmprc = adapter->stats.last_vfmprc; 1334} 1335 1336static void ixgbevf_negotiate_api(struct ixgbevf_adapter *adapter) 1337{ 1338 struct ixgbe_hw *hw = &adapter->hw; 1339 int api[] = { ixgbe_mbox_api_11, 1340 ixgbe_mbox_api_10, 1341 ixgbe_mbox_api_unknown }; 1342 int err = 0, idx = 0; 1343 1344 spin_lock(&adapter->mbx_lock); 1345 1346 while (api[idx] != ixgbe_mbox_api_unknown) { 1347 err = ixgbevf_negotiate_api_version(hw, api[idx]); 1348 if (!err) 1349 break; 1350 idx++; 1351 } 1352 1353 spin_unlock(&adapter->mbx_lock); 1354} 1355 1356static void ixgbevf_up_complete(struct ixgbevf_adapter *adapter) 1357{ 1358 struct net_device *netdev = adapter->netdev; 1359 struct ixgbe_hw *hw = &adapter->hw; 1360 int i, j = 0; 1361 int num_rx_rings = adapter->num_rx_queues; 1362 u32 txdctl, rxdctl; 1363 1364 for (i = 0; i < adapter->num_tx_queues; i++) { 1365 j = adapter->tx_ring[i].reg_idx; 1366 txdctl = IXGBE_READ_REG(hw, IXGBE_VFTXDCTL(j)); 1367 /* enable WTHRESH=8 descriptors, to encourage burst writeback */ 1368 txdctl |= (8 << 16); 1369 IXGBE_WRITE_REG(hw, IXGBE_VFTXDCTL(j), txdctl); 1370 } 1371 1372 for (i = 0; i < adapter->num_tx_queues; i++) { 1373 j = adapter->tx_ring[i].reg_idx; 1374 txdctl = IXGBE_READ_REG(hw, IXGBE_VFTXDCTL(j)); 1375 txdctl |= IXGBE_TXDCTL_ENABLE; 1376 IXGBE_WRITE_REG(hw, IXGBE_VFTXDCTL(j), txdctl); 1377 } 1378 1379 for (i = 0; i < num_rx_rings; i++) { 1380 j = adapter->rx_ring[i].reg_idx; 1381 rxdctl = IXGBE_READ_REG(hw, IXGBE_VFRXDCTL(j)); 1382 rxdctl |= IXGBE_RXDCTL_ENABLE | IXGBE_RXDCTL_VME; 1383 if (hw->mac.type == ixgbe_mac_X540_vf) { 1384 rxdctl &= ~IXGBE_RXDCTL_RLPMLMASK; 1385 rxdctl |= ((netdev->mtu + ETH_HLEN + ETH_FCS_LEN) | 1386 IXGBE_RXDCTL_RLPML_EN); 1387 } 1388 IXGBE_WRITE_REG(hw, IXGBE_VFRXDCTL(j), rxdctl); 1389 ixgbevf_rx_desc_queue_enable(adapter, i); 1390 } 1391 1392 ixgbevf_configure_msix(adapter); 1393 1394 spin_lock(&adapter->mbx_lock); 1395 1396 if (hw->mac.ops.set_rar) { 1397 if (is_valid_ether_addr(hw->mac.addr)) 1398 hw->mac.ops.set_rar(hw, 0, hw->mac.addr, 0); 1399 else 1400 hw->mac.ops.set_rar(hw, 0, hw->mac.perm_addr, 0); 1401 } 1402 1403 spin_unlock(&adapter->mbx_lock); 1404 1405 clear_bit(__IXGBEVF_DOWN, &adapter->state); 1406 ixgbevf_napi_enable_all(adapter); 1407 1408 /* enable transmits */ 1409 netif_tx_start_all_queues(netdev); 1410 1411 ixgbevf_save_reset_stats(adapter); 1412 ixgbevf_init_last_counter_stats(adapter); 1413 1414 hw->mac.get_link_status = 1; 1415 mod_timer(&adapter->watchdog_timer, jiffies); 1416} 1417 1418static int ixgbevf_reset_queues(struct ixgbevf_adapter *adapter) 1419{ 1420 struct ixgbe_hw *hw = &adapter->hw; 1421 struct ixgbevf_ring *rx_ring; 1422 unsigned int def_q = 0; 1423 unsigned int num_tcs = 0; 1424 unsigned int num_rx_queues = 1; 1425 int err, i; 1426 1427 spin_lock(&adapter->mbx_lock); 1428 1429 /* fetch queue configuration from the PF */ 1430 err = ixgbevf_get_queues(hw, &num_tcs, &def_q); 1431 1432 spin_unlock(&adapter->mbx_lock); 1433 1434 if (err) 1435 return err; 1436 1437 if (num_tcs > 1) { 1438 /* update default Tx ring register index */ 1439 adapter->tx_ring[0].reg_idx = def_q; 1440 1441 /* we need as many queues as traffic classes */ 1442 num_rx_queues = num_tcs; 1443 } 1444 1445 /* nothing to do if we have the correct number of queues */ 1446 if (adapter->num_rx_queues == num_rx_queues) 1447 return 0; 1448 1449 /* allocate new rings */ 1450 rx_ring = kcalloc(num_rx_queues, 1451 sizeof(struct ixgbevf_ring), GFP_KERNEL); 1452 if (!rx_ring) 1453 return -ENOMEM; 1454 1455 /* setup ring fields */ 1456 for (i = 0; i < num_rx_queues; i++) { 1457 rx_ring[i].count = adapter->rx_ring_count; 1458 rx_ring[i].queue_index = i; 1459 rx_ring[i].reg_idx = i; 1460 rx_ring[i].dev = &adapter->pdev->dev; 1461 rx_ring[i].netdev = adapter->netdev; 1462 1463 /* allocate resources on the ring */ 1464 err = ixgbevf_setup_rx_resources(adapter, &rx_ring[i]); 1465 if (err) { 1466 while (i) { 1467 i--; 1468 ixgbevf_free_rx_resources(adapter, &rx_ring[i]); 1469 } 1470 kfree(rx_ring); 1471 return err; 1472 } 1473 } 1474 1475 /* free the existing rings and queues */ 1476 ixgbevf_free_all_rx_resources(adapter); 1477 adapter->num_rx_queues = 0; 1478 kfree(adapter->rx_ring); 1479 1480 /* move new rings into position on the adapter struct */ 1481 adapter->rx_ring = rx_ring; 1482 adapter->num_rx_queues = num_rx_queues; 1483 1484 /* reset ring to vector mapping */ 1485 ixgbevf_reset_q_vectors(adapter); 1486 ixgbevf_map_rings_to_vectors(adapter); 1487 1488 return 0; 1489} 1490 1491void ixgbevf_up(struct ixgbevf_adapter *adapter) 1492{ 1493 struct ixgbe_hw *hw = &adapter->hw; 1494 1495 ixgbevf_negotiate_api(adapter); 1496 1497 ixgbevf_reset_queues(adapter); 1498 1499 ixgbevf_configure(adapter); 1500 1501 ixgbevf_up_complete(adapter); 1502 1503 /* clear any pending interrupts, may auto mask */ 1504 IXGBE_READ_REG(hw, IXGBE_VTEICR); 1505 1506 ixgbevf_irq_enable(adapter); 1507} 1508 1509/** 1510 * ixgbevf_clean_rx_ring - Free Rx Buffers per Queue 1511 * @adapter: board private structure 1512 * @rx_ring: ring to free buffers from 1513 **/ 1514static void ixgbevf_clean_rx_ring(struct ixgbevf_adapter *adapter, 1515 struct ixgbevf_ring *rx_ring) 1516{ 1517 struct pci_dev *pdev = adapter->pdev; 1518 unsigned long size; 1519 unsigned int i; 1520 1521 if (!rx_ring->rx_buffer_info) 1522 return; 1523 1524 /* Free all the Rx ring sk_buffs */ 1525 for (i = 0; i < rx_ring->count; i++) { 1526 struct ixgbevf_rx_buffer *rx_buffer_info; 1527 1528 rx_buffer_info = &rx_ring->rx_buffer_info[i]; 1529 if (rx_buffer_info->dma) { 1530 dma_unmap_single(&pdev->dev, rx_buffer_info->dma, 1531 rx_ring->rx_buf_len, 1532 DMA_FROM_DEVICE); 1533 rx_buffer_info->dma = 0; 1534 } 1535 if (rx_buffer_info->skb) { 1536 struct sk_buff *skb = rx_buffer_info->skb; 1537 rx_buffer_info->skb = NULL; 1538 do { 1539 struct sk_buff *this = skb; 1540 skb = IXGBE_CB(skb)->prev; 1541 dev_kfree_skb(this); 1542 } while (skb); 1543 } 1544 } 1545 1546 size = sizeof(struct ixgbevf_rx_buffer) * rx_ring->count; 1547 memset(rx_ring->rx_buffer_info, 0, size); 1548 1549 /* Zero out the descriptor ring */ 1550 memset(rx_ring->desc, 0, rx_ring->size); 1551 1552 rx_ring->next_to_clean = 0; 1553 rx_ring->next_to_use = 0; 1554 1555 if (rx_ring->head) 1556 writel(0, adapter->hw.hw_addr + rx_ring->head); 1557 if (rx_ring->tail) 1558 writel(0, adapter->hw.hw_addr + rx_ring->tail); 1559} 1560 1561/** 1562 * ixgbevf_clean_tx_ring - Free Tx Buffers 1563 * @adapter: board private structure 1564 * @tx_ring: ring to be cleaned 1565 **/ 1566static void ixgbevf_clean_tx_ring(struct ixgbevf_adapter *adapter, 1567 struct ixgbevf_ring *tx_ring) 1568{ 1569 struct ixgbevf_tx_buffer *tx_buffer_info; 1570 unsigned long size; 1571 unsigned int i; 1572 1573 if (!tx_ring->tx_buffer_info) 1574 return; 1575 1576 /* Free all the Tx ring sk_buffs */ 1577 1578 for (i = 0; i < tx_ring->count; i++) { 1579 tx_buffer_info = &tx_ring->tx_buffer_info[i]; 1580 ixgbevf_unmap_and_free_tx_resource(tx_ring, tx_buffer_info); 1581 } 1582 1583 size = sizeof(struct ixgbevf_tx_buffer) * tx_ring->count; 1584 memset(tx_ring->tx_buffer_info, 0, size); 1585 1586 memset(tx_ring->desc, 0, tx_ring->size); 1587 1588 tx_ring->next_to_use = 0; 1589 tx_ring->next_to_clean = 0; 1590 1591 if (tx_ring->head) 1592 writel(0, adapter->hw.hw_addr + tx_ring->head); 1593 if (tx_ring->tail) 1594 writel(0, adapter->hw.hw_addr + tx_ring->tail); 1595} 1596 1597/** 1598 * ixgbevf_clean_all_rx_rings - Free Rx Buffers for all queues 1599 * @adapter: board private structure 1600 **/ 1601static void ixgbevf_clean_all_rx_rings(struct ixgbevf_adapter *adapter) 1602{ 1603 int i; 1604 1605 for (i = 0; i < adapter->num_rx_queues; i++) 1606 ixgbevf_clean_rx_ring(adapter, &adapter->rx_ring[i]); 1607} 1608 1609/** 1610 * ixgbevf_clean_all_tx_rings - Free Tx Buffers for all queues 1611 * @adapter: board private structure 1612 **/ 1613static void ixgbevf_clean_all_tx_rings(struct ixgbevf_adapter *adapter) 1614{ 1615 int i; 1616 1617 for (i = 0; i < adapter->num_tx_queues; i++) 1618 ixgbevf_clean_tx_ring(adapter, &adapter->tx_ring[i]); 1619} 1620 1621void ixgbevf_down(struct ixgbevf_adapter *adapter) 1622{ 1623 struct net_device *netdev = adapter->netdev; 1624 struct ixgbe_hw *hw = &adapter->hw; 1625 u32 txdctl; 1626 int i, j; 1627 1628 /* signal that we are down to the interrupt handler */ 1629 set_bit(__IXGBEVF_DOWN, &adapter->state); 1630 /* disable receives */ 1631 1632 netif_tx_disable(netdev); 1633 1634 msleep(10); 1635 1636 netif_tx_stop_all_queues(netdev); 1637 1638 ixgbevf_irq_disable(adapter); 1639 1640 ixgbevf_napi_disable_all(adapter); 1641 1642 del_timer_sync(&adapter->watchdog_timer); 1643 /* can't call flush scheduled work here because it can deadlock 1644 * if linkwatch_event tries to acquire the rtnl_lock which we are 1645 * holding */ 1646 while (adapter->flags & IXGBE_FLAG_IN_WATCHDOG_TASK) 1647 msleep(1); 1648 1649 /* disable transmits in the hardware now that interrupts are off */ 1650 for (i = 0; i < adapter->num_tx_queues; i++) { 1651 j = adapter->tx_ring[i].reg_idx; 1652 txdctl = IXGBE_READ_REG(hw, IXGBE_VFTXDCTL(j)); 1653 IXGBE_WRITE_REG(hw, IXGBE_VFTXDCTL(j), 1654 (txdctl & ~IXGBE_TXDCTL_ENABLE)); 1655 } 1656 1657 netif_carrier_off(netdev); 1658 1659 if (!pci_channel_offline(adapter->pdev)) 1660 ixgbevf_reset(adapter); 1661 1662 ixgbevf_clean_all_tx_rings(adapter); 1663 ixgbevf_clean_all_rx_rings(adapter); 1664} 1665 1666void ixgbevf_reinit_locked(struct ixgbevf_adapter *adapter) 1667{ 1668 WARN_ON(in_interrupt()); 1669 1670 while (test_and_set_bit(__IXGBEVF_RESETTING, &adapter->state)) 1671 msleep(1); 1672 1673 /* 1674 * Check if PF is up before re-init. If not then skip until 1675 * later when the PF is up and ready to service requests from 1676 * the VF via mailbox. If the VF is up and running then the 1677 * watchdog task will continue to schedule reset tasks until 1678 * the PF is up and running. 1679 */ 1680 ixgbevf_down(adapter); 1681 ixgbevf_up(adapter); 1682 1683 clear_bit(__IXGBEVF_RESETTING, &adapter->state); 1684} 1685 1686void ixgbevf_reset(struct ixgbevf_adapter *adapter) 1687{ 1688 struct ixgbe_hw *hw = &adapter->hw; 1689 struct net_device *netdev = adapter->netdev; 1690 1691 spin_lock(&adapter->mbx_lock); 1692 1693 if (hw->mac.ops.reset_hw(hw)) 1694 hw_dbg(hw, "PF still resetting\n"); 1695 else 1696 hw->mac.ops.init_hw(hw); 1697 1698 spin_unlock(&adapter->mbx_lock); 1699 1700 if (is_valid_ether_addr(adapter->hw.mac.addr)) { 1701 memcpy(netdev->dev_addr, adapter->hw.mac.addr, 1702 netdev->addr_len); 1703 memcpy(netdev->perm_addr, adapter->hw.mac.addr, 1704 netdev->addr_len); 1705 } 1706} 1707 1708static void ixgbevf_acquire_msix_vectors(struct ixgbevf_adapter *adapter, 1709 int vectors) 1710{ 1711 int err, vector_threshold; 1712 1713 /* We'll want at least 2 (vector_threshold): 1714 * 1) TxQ[0] + RxQ[0] handler 1715 * 2) Other (Link Status Change, etc.) 1716 */ 1717 vector_threshold = MIN_MSIX_COUNT; 1718 1719 /* The more we get, the more we will assign to Tx/Rx Cleanup 1720 * for the separate queues...where Rx Cleanup >= Tx Cleanup. 1721 * Right now, we simply care about how many we'll get; we'll 1722 * set them up later while requesting irq's. 1723 */ 1724 while (vectors >= vector_threshold) { 1725 err = pci_enable_msix(adapter->pdev, adapter->msix_entries, 1726 vectors); 1727 if (!err) /* Success in acquiring all requested vectors. */ 1728 break; 1729 else if (err < 0) 1730 vectors = 0; /* Nasty failure, quit now */ 1731 else /* err == number of vectors we should try again with */ 1732 vectors = err; 1733 } 1734 1735 if (vectors < vector_threshold) { 1736 /* Can't allocate enough MSI-X interrupts? Oh well. 1737 * This just means we'll go with either a single MSI 1738 * vector or fall back to legacy interrupts. 1739 */ 1740 hw_dbg(&adapter->hw, 1741 "Unable to allocate MSI-X interrupts\n"); 1742 kfree(adapter->msix_entries); 1743 adapter->msix_entries = NULL; 1744 } else { 1745 /* 1746 * Adjust for only the vectors we'll use, which is minimum 1747 * of max_msix_q_vectors + NON_Q_VECTORS, or the number of 1748 * vectors we were allocated. 1749 */ 1750 adapter->num_msix_vectors = vectors; 1751 } 1752} 1753 1754/** 1755 * ixgbevf_set_num_queues - Allocate queues for device, feature dependent 1756 * @adapter: board private structure to initialize 1757 * 1758 * This is the top level queue allocation routine. The order here is very 1759 * important, starting with the "most" number of features turned on at once, 1760 * and ending with the smallest set of features. This way large combinations 1761 * can be allocated if they're turned on, and smaller combinations are the 1762 * fallthrough conditions. 1763 * 1764 **/ 1765static void ixgbevf_set_num_queues(struct ixgbevf_adapter *adapter) 1766{ 1767 /* Start with base case */ 1768 adapter->num_rx_queues = 1; 1769 adapter->num_tx_queues = 1; 1770} 1771 1772/** 1773 * ixgbevf_alloc_queues - Allocate memory for all rings 1774 * @adapter: board private structure to initialize 1775 * 1776 * We allocate one ring per queue at run-time since we don't know the 1777 * number of queues at compile-time. The polling_netdev array is 1778 * intended for Multiqueue, but should work fine with a single queue. 1779 **/ 1780static int ixgbevf_alloc_queues(struct ixgbevf_adapter *adapter) 1781{ 1782 int i; 1783 1784 adapter->tx_ring = kcalloc(adapter->num_tx_queues, 1785 sizeof(struct ixgbevf_ring), GFP_KERNEL); 1786 if (!adapter->tx_ring) 1787 goto err_tx_ring_allocation; 1788 1789 adapter->rx_ring = kcalloc(adapter->num_rx_queues, 1790 sizeof(struct ixgbevf_ring), GFP_KERNEL); 1791 if (!adapter->rx_ring) 1792 goto err_rx_ring_allocation; 1793 1794 for (i = 0; i < adapter->num_tx_queues; i++) { 1795 adapter->tx_ring[i].count = adapter->tx_ring_count; 1796 adapter->tx_ring[i].queue_index = i; 1797 /* reg_idx may be remapped later by DCB config */ 1798 adapter->tx_ring[i].reg_idx = i; 1799 adapter->tx_ring[i].dev = &adapter->pdev->dev; 1800 adapter->tx_ring[i].netdev = adapter->netdev; 1801 } 1802 1803 for (i = 0; i < adapter->num_rx_queues; i++) { 1804 adapter->rx_ring[i].count = adapter->rx_ring_count; 1805 adapter->rx_ring[i].queue_index = i; 1806 adapter->rx_ring[i].reg_idx = i; 1807 adapter->rx_ring[i].dev = &adapter->pdev->dev; 1808 adapter->rx_ring[i].netdev = adapter->netdev; 1809 } 1810 1811 return 0; 1812 1813err_rx_ring_allocation: 1814 kfree(adapter->tx_ring); 1815err_tx_ring_allocation: 1816 return -ENOMEM; 1817} 1818 1819/** 1820 * ixgbevf_set_interrupt_capability - set MSI-X or FAIL if not supported 1821 * @adapter: board private structure to initialize 1822 * 1823 * Attempt to configure the interrupts using the best available 1824 * capabilities of the hardware and the kernel. 1825 **/ 1826static int ixgbevf_set_interrupt_capability(struct ixgbevf_adapter *adapter) 1827{ 1828 struct net_device *netdev = adapter->netdev; 1829 int err = 0; 1830 int vector, v_budget; 1831 1832 /* 1833 * It's easy to be greedy for MSI-X vectors, but it really 1834 * doesn't do us much good if we have a lot more vectors 1835 * than CPU's. So let's be conservative and only ask for 1836 * (roughly) the same number of vectors as there are CPU's. 1837 * The default is to use pairs of vectors. 1838 */ 1839 v_budget = max(adapter->num_rx_queues, adapter->num_tx_queues); 1840 v_budget = min_t(int, v_budget, num_online_cpus()); 1841 v_budget += NON_Q_VECTORS; 1842 1843 /* A failure in MSI-X entry allocation isn't fatal, but it does 1844 * mean we disable MSI-X capabilities of the adapter. */ 1845 adapter->msix_entries = kcalloc(v_budget, 1846 sizeof(struct msix_entry), GFP_KERNEL); 1847 if (!adapter->msix_entries) { 1848 err = -ENOMEM; 1849 goto out; 1850 } 1851 1852 for (vector = 0; vector < v_budget; vector++) 1853 adapter->msix_entries[vector].entry = vector; 1854 1855 ixgbevf_acquire_msix_vectors(adapter, v_budget); 1856 1857 err = netif_set_real_num_tx_queues(netdev, adapter->num_tx_queues); 1858 if (err) 1859 goto out; 1860 1861 err = netif_set_real_num_rx_queues(netdev, adapter->num_rx_queues); 1862 1863out: 1864 return err; 1865} 1866 1867/** 1868 * ixgbevf_alloc_q_vectors - Allocate memory for interrupt vectors 1869 * @adapter: board private structure to initialize 1870 * 1871 * We allocate one q_vector per queue interrupt. If allocation fails we 1872 * return -ENOMEM. 1873 **/ 1874static int ixgbevf_alloc_q_vectors(struct ixgbevf_adapter *adapter) 1875{ 1876 int q_idx, num_q_vectors; 1877 struct ixgbevf_q_vector *q_vector; 1878 1879 num_q_vectors = adapter->num_msix_vectors - NON_Q_VECTORS; 1880 1881 for (q_idx = 0; q_idx < num_q_vectors; q_idx++) { 1882 q_vector = kzalloc(sizeof(struct ixgbevf_q_vector), GFP_KERNEL); 1883 if (!q_vector) 1884 goto err_out; 1885 q_vector->adapter = adapter; 1886 q_vector->v_idx = q_idx; 1887 netif_napi_add(adapter->netdev, &q_vector->napi, 1888 ixgbevf_poll, 64); 1889 adapter->q_vector[q_idx] = q_vector; 1890 } 1891 1892 return 0; 1893 1894err_out: 1895 while (q_idx) { 1896 q_idx--; 1897 q_vector = adapter->q_vector[q_idx]; 1898 netif_napi_del(&q_vector->napi); 1899 kfree(q_vector); 1900 adapter->q_vector[q_idx] = NULL; 1901 } 1902 return -ENOMEM; 1903} 1904 1905/** 1906 * ixgbevf_free_q_vectors - Free memory allocated for interrupt vectors 1907 * @adapter: board private structure to initialize 1908 * 1909 * This function frees the memory allocated to the q_vectors. In addition if 1910 * NAPI is enabled it will delete any references to the NAPI struct prior 1911 * to freeing the q_vector. 1912 **/ 1913static void ixgbevf_free_q_vectors(struct ixgbevf_adapter *adapter) 1914{ 1915 int q_idx, num_q_vectors; 1916 int napi_vectors; 1917 1918 num_q_vectors = adapter->num_msix_vectors - NON_Q_VECTORS; 1919 napi_vectors = adapter->num_rx_queues; 1920 1921 for (q_idx = 0; q_idx < num_q_vectors; q_idx++) { 1922 struct ixgbevf_q_vector *q_vector = adapter->q_vector[q_idx]; 1923 1924 adapter->q_vector[q_idx] = NULL; 1925 if (q_idx < napi_vectors) 1926 netif_napi_del(&q_vector->napi); 1927 kfree(q_vector); 1928 } 1929} 1930 1931/** 1932 * ixgbevf_reset_interrupt_capability - Reset MSIX setup 1933 * @adapter: board private structure 1934 * 1935 **/ 1936static void ixgbevf_reset_interrupt_capability(struct ixgbevf_adapter *adapter) 1937{ 1938 pci_disable_msix(adapter->pdev); 1939 kfree(adapter->msix_entries); 1940 adapter->msix_entries = NULL; 1941} 1942 1943/** 1944 * ixgbevf_init_interrupt_scheme - Determine if MSIX is supported and init 1945 * @adapter: board private structure to initialize 1946 * 1947 **/ 1948static int ixgbevf_init_interrupt_scheme(struct ixgbevf_adapter *adapter) 1949{ 1950 int err; 1951 1952 /* Number of supported queues */ 1953 ixgbevf_set_num_queues(adapter); 1954 1955 err = ixgbevf_set_interrupt_capability(adapter); 1956 if (err) { 1957 hw_dbg(&adapter->hw, 1958 "Unable to setup interrupt capabilities\n"); 1959 goto err_set_interrupt; 1960 } 1961 1962 err = ixgbevf_alloc_q_vectors(adapter); 1963 if (err) { 1964 hw_dbg(&adapter->hw, "Unable to allocate memory for queue " 1965 "vectors\n"); 1966 goto err_alloc_q_vectors; 1967 } 1968 1969 err = ixgbevf_alloc_queues(adapter); 1970 if (err) { 1971 pr_err("Unable to allocate memory for queues\n"); 1972 goto err_alloc_queues; 1973 } 1974 1975 hw_dbg(&adapter->hw, "Multiqueue %s: Rx Queue count = %u, " 1976 "Tx Queue count = %u\n", 1977 (adapter->num_rx_queues > 1) ? "Enabled" : 1978 "Disabled", adapter->num_rx_queues, adapter->num_tx_queues); 1979 1980 set_bit(__IXGBEVF_DOWN, &adapter->state); 1981 1982 return 0; 1983err_alloc_queues: 1984 ixgbevf_free_q_vectors(adapter); 1985err_alloc_q_vectors: 1986 ixgbevf_reset_interrupt_capability(adapter); 1987err_set_interrupt: 1988 return err; 1989} 1990 1991/** 1992 * ixgbevf_clear_interrupt_scheme - Clear the current interrupt scheme settings 1993 * @adapter: board private structure to clear interrupt scheme on 1994 * 1995 * We go through and clear interrupt specific resources and reset the structure 1996 * to pre-load conditions 1997 **/ 1998static void ixgbevf_clear_interrupt_scheme(struct ixgbevf_adapter *adapter) 1999{ 2000 adapter->num_tx_queues = 0; 2001 adapter->num_rx_queues = 0; 2002 2003 ixgbevf_free_q_vectors(adapter); 2004 ixgbevf_reset_interrupt_capability(adapter); 2005} 2006 2007/** 2008 * ixgbevf_sw_init - Initialize general software structures 2009 * (struct ixgbevf_adapter) 2010 * @adapter: board private structure to initialize 2011 * 2012 * ixgbevf_sw_init initializes the Adapter private data structure. 2013 * Fields are initialized based on PCI device information and 2014 * OS network device settings (MTU size). 2015 **/ 2016static int __devinit ixgbevf_sw_init(struct ixgbevf_adapter *adapter) 2017{ 2018 struct ixgbe_hw *hw = &adapter->hw; 2019 struct pci_dev *pdev = adapter->pdev; 2020 int err; 2021 2022 /* PCI config space info */ 2023 2024 hw->vendor_id = pdev->vendor; 2025 hw->device_id = pdev->device; 2026 hw->revision_id = pdev->revision; 2027 hw->subsystem_vendor_id = pdev->subsystem_vendor; 2028 hw->subsystem_device_id = pdev->subsystem_device; 2029 2030 hw->mbx.ops.init_params(hw); 2031 2032 /* assume legacy case in which PF would only give VF 2 queues */ 2033 hw->mac.max_tx_queues = 2; 2034 hw->mac.max_rx_queues = 2; 2035 2036 err = hw->mac.ops.reset_hw(hw); 2037 if (err) { 2038 dev_info(&pdev->dev, 2039 "PF still in reset state, assigning new address\n"); 2040 eth_hw_addr_random(adapter->netdev); 2041 memcpy(adapter->hw.mac.addr, adapter->netdev->dev_addr, 2042 adapter->netdev->addr_len); 2043 } else { 2044 err = hw->mac.ops.init_hw(hw); 2045 if (err) { 2046 pr_err("init_shared_code failed: %d\n", err); 2047 goto out; 2048 } 2049 memcpy(adapter->netdev->dev_addr, adapter->hw.mac.addr, 2050 adapter->netdev->addr_len); 2051 } 2052 2053 /* lock to protect mailbox accesses */ 2054 spin_lock_init(&adapter->mbx_lock); 2055 2056 /* Enable dynamic interrupt throttling rates */ 2057 adapter->rx_itr_setting = 1; 2058 adapter->tx_itr_setting = 1; 2059 2060 /* set default ring sizes */ 2061 adapter->tx_ring_count = IXGBEVF_DEFAULT_TXD; 2062 adapter->rx_ring_count = IXGBEVF_DEFAULT_RXD; 2063 2064 set_bit(__IXGBEVF_DOWN, &adapter->state); 2065 return 0; 2066 2067out: 2068 return err; 2069} 2070 2071#define UPDATE_VF_COUNTER_32bit(reg, last_counter, counter) \ 2072 { \ 2073 u32 current_counter = IXGBE_READ_REG(hw, reg); \ 2074 if (current_counter < last_counter) \ 2075 counter += 0x100000000LL; \ 2076 last_counter = current_counter; \ 2077 counter &= 0xFFFFFFFF00000000LL; \ 2078 counter |= current_counter; \ 2079 } 2080 2081#define UPDATE_VF_COUNTER_36bit(reg_lsb, reg_msb, last_counter, counter) \ 2082 { \ 2083 u64 current_counter_lsb = IXGBE_READ_REG(hw, reg_lsb); \ 2084 u64 current_counter_msb = IXGBE_READ_REG(hw, reg_msb); \ 2085 u64 current_counter = (current_counter_msb << 32) | \ 2086 current_counter_lsb; \ 2087 if (current_counter < last_counter) \ 2088 counter += 0x1000000000LL; \ 2089 last_counter = current_counter; \ 2090 counter &= 0xFFFFFFF000000000LL; \ 2091 counter |= current_counter; \ 2092 } 2093/** 2094 * ixgbevf_update_stats - Update the board statistics counters. 2095 * @adapter: board private structure 2096 **/ 2097void ixgbevf_update_stats(struct ixgbevf_adapter *adapter) 2098{ 2099 struct ixgbe_hw *hw = &adapter->hw; 2100 2101 UPDATE_VF_COUNTER_32bit(IXGBE_VFGPRC, adapter->stats.last_vfgprc, 2102 adapter->stats.vfgprc); 2103 UPDATE_VF_COUNTER_32bit(IXGBE_VFGPTC, adapter->stats.last_vfgptc, 2104 adapter->stats.vfgptc); 2105 UPDATE_VF_COUNTER_36bit(IXGBE_VFGORC_LSB, IXGBE_VFGORC_MSB, 2106 adapter->stats.last_vfgorc, 2107 adapter->stats.vfgorc); 2108 UPDATE_VF_COUNTER_36bit(IXGBE_VFGOTC_LSB, IXGBE_VFGOTC_MSB, 2109 adapter->stats.last_vfgotc, 2110 adapter->stats.vfgotc); 2111 UPDATE_VF_COUNTER_32bit(IXGBE_VFMPRC, adapter->stats.last_vfmprc, 2112 adapter->stats.vfmprc); 2113} 2114 2115/** 2116 * ixgbevf_watchdog - Timer Call-back 2117 * @data: pointer to adapter cast into an unsigned long 2118 **/ 2119static void ixgbevf_watchdog(unsigned long data) 2120{ 2121 struct ixgbevf_adapter *adapter = (struct ixgbevf_adapter *)data; 2122 struct ixgbe_hw *hw = &adapter->hw; 2123 u32 eics = 0; 2124 int i; 2125 2126 /* 2127 * Do the watchdog outside of interrupt context due to the lovely 2128 * delays that some of the newer hardware requires 2129 */ 2130 2131 if (test_bit(__IXGBEVF_DOWN, &adapter->state)) 2132 goto watchdog_short_circuit; 2133 2134 /* get one bit for every active tx/rx interrupt vector */ 2135 for (i = 0; i < adapter->num_msix_vectors - NON_Q_VECTORS; i++) { 2136 struct ixgbevf_q_vector *qv = adapter->q_vector[i]; 2137 if (qv->rx.ring || qv->tx.ring) 2138 eics |= 1 << i; 2139 } 2140 2141 IXGBE_WRITE_REG(hw, IXGBE_VTEICS, eics); 2142 2143watchdog_short_circuit: 2144 schedule_work(&adapter->watchdog_task); 2145} 2146 2147/** 2148 * ixgbevf_tx_timeout - Respond to a Tx Hang 2149 * @netdev: network interface device structure 2150 **/ 2151static void ixgbevf_tx_timeout(struct net_device *netdev) 2152{ 2153 struct ixgbevf_adapter *adapter = netdev_priv(netdev); 2154 2155 /* Do the reset outside of interrupt context */ 2156 schedule_work(&adapter->reset_task); 2157} 2158 2159static void ixgbevf_reset_task(struct work_struct *work) 2160{ 2161 struct ixgbevf_adapter *adapter; 2162 adapter = container_of(work, struct ixgbevf_adapter, reset_task); 2163 2164 /* If we're already down or resetting, just bail */ 2165 if (test_bit(__IXGBEVF_DOWN, &adapter->state) || 2166 test_bit(__IXGBEVF_RESETTING, &adapter->state)) 2167 return; 2168 2169 adapter->tx_timeout_count++; 2170 2171 ixgbevf_reinit_locked(adapter); 2172} 2173 2174/** 2175 * ixgbevf_watchdog_task - worker thread to bring link up 2176 * @work: pointer to work_struct containing our data 2177 **/ 2178static void ixgbevf_watchdog_task(struct work_struct *work) 2179{ 2180 struct ixgbevf_adapter *adapter = container_of(work, 2181 struct ixgbevf_adapter, 2182 watchdog_task); 2183 struct net_device *netdev = adapter->netdev; 2184 struct ixgbe_hw *hw = &adapter->hw; 2185 u32 link_speed = adapter->link_speed; 2186 bool link_up = adapter->link_up; 2187 2188 adapter->flags |= IXGBE_FLAG_IN_WATCHDOG_TASK; 2189 2190 /* 2191 * Always check the link on the watchdog because we have 2192 * no LSC interrupt 2193 */ 2194 if (hw->mac.ops.check_link) { 2195 s32 need_reset; 2196 2197 spin_lock(&adapter->mbx_lock); 2198 2199 need_reset = hw->mac.ops.check_link(hw, &link_speed, 2200 &link_up, false); 2201 2202 spin_unlock(&adapter->mbx_lock); 2203 2204 if (need_reset) { 2205 adapter->link_up = link_up; 2206 adapter->link_speed = link_speed; 2207 netif_carrier_off(netdev); 2208 netif_tx_stop_all_queues(netdev); 2209 schedule_work(&adapter->reset_task); 2210 goto pf_has_reset; 2211 } 2212 } else { 2213 /* always assume link is up, if no check link 2214 * function */ 2215 link_speed = IXGBE_LINK_SPEED_10GB_FULL; 2216 link_up = true; 2217 } 2218 adapter->link_up = link_up; 2219 adapter->link_speed = link_speed; 2220 2221 if (link_up) { 2222 if (!netif_carrier_ok(netdev)) { 2223 hw_dbg(&adapter->hw, "NIC Link is Up, %u Gbps\n", 2224 (link_speed == IXGBE_LINK_SPEED_10GB_FULL) ? 2225 10 : 1); 2226 netif_carrier_on(netdev); 2227 netif_tx_wake_all_queues(netdev); 2228 } 2229 } else { 2230 adapter->link_up = false; 2231 adapter->link_speed = 0; 2232 if (netif_carrier_ok(netdev)) { 2233 hw_dbg(&adapter->hw, "NIC Link is Down\n"); 2234 netif_carrier_off(netdev); 2235 netif_tx_stop_all_queues(netdev); 2236 } 2237 } 2238 2239 ixgbevf_update_stats(adapter); 2240 2241pf_has_reset: 2242 /* Reset the timer */ 2243 if (!test_bit(__IXGBEVF_DOWN, &adapter->state)) 2244 mod_timer(&adapter->watchdog_timer, 2245 round_jiffies(jiffies + (2 * HZ))); 2246 2247 adapter->flags &= ~IXGBE_FLAG_IN_WATCHDOG_TASK; 2248} 2249 2250/** 2251 * ixgbevf_free_tx_resources - Free Tx Resources per Queue 2252 * @adapter: board private structure 2253 * @tx_ring: Tx descriptor ring for a specific queue 2254 * 2255 * Free all transmit software resources 2256 **/ 2257void ixgbevf_free_tx_resources(struct ixgbevf_adapter *adapter, 2258 struct ixgbevf_ring *tx_ring) 2259{ 2260 struct pci_dev *pdev = adapter->pdev; 2261 2262 ixgbevf_clean_tx_ring(adapter, tx_ring); 2263 2264 vfree(tx_ring->tx_buffer_info); 2265 tx_ring->tx_buffer_info = NULL; 2266 2267 dma_free_coherent(&pdev->dev, tx_ring->size, tx_ring->desc, 2268 tx_ring->dma); 2269 2270 tx_ring->desc = NULL; 2271} 2272 2273/** 2274 * ixgbevf_free_all_tx_resources - Free Tx Resources for All Queues 2275 * @adapter: board private structure 2276 * 2277 * Free all transmit software resources 2278 **/ 2279static void ixgbevf_free_all_tx_resources(struct ixgbevf_adapter *adapter) 2280{ 2281 int i; 2282 2283 for (i = 0; i < adapter->num_tx_queues; i++) 2284 if (adapter->tx_ring[i].desc) 2285 ixgbevf_free_tx_resources(adapter, 2286 &adapter->tx_ring[i]); 2287 2288} 2289 2290/** 2291 * ixgbevf_setup_tx_resources - allocate Tx resources (Descriptors) 2292 * @adapter: board private structure 2293 * @tx_ring: tx descriptor ring (for a specific queue) to setup 2294 * 2295 * Return 0 on success, negative on failure 2296 **/ 2297int ixgbevf_setup_tx_resources(struct ixgbevf_adapter *adapter, 2298 struct ixgbevf_ring *tx_ring) 2299{ 2300 struct pci_dev *pdev = adapter->pdev; 2301 int size; 2302 2303 size = sizeof(struct ixgbevf_tx_buffer) * tx_ring->count; 2304 tx_ring->tx_buffer_info = vzalloc(size); 2305 if (!tx_ring->tx_buffer_info) 2306 goto err; 2307 2308 /* round up to nearest 4K */ 2309 tx_ring->size = tx_ring->count * sizeof(union ixgbe_adv_tx_desc); 2310 tx_ring->size = ALIGN(tx_ring->size, 4096); 2311 2312 tx_ring->desc = dma_alloc_coherent(&pdev->dev, tx_ring->size, 2313 &tx_ring->dma, GFP_KERNEL); 2314 if (!tx_ring->desc) 2315 goto err; 2316 2317 tx_ring->next_to_use = 0; 2318 tx_ring->next_to_clean = 0; 2319 return 0; 2320 2321err: 2322 vfree(tx_ring->tx_buffer_info); 2323 tx_ring->tx_buffer_info = NULL; 2324 hw_dbg(&adapter->hw, "Unable to allocate memory for the transmit " 2325 "descriptor ring\n"); 2326 return -ENOMEM; 2327} 2328 2329/** 2330 * ixgbevf_setup_all_tx_resources - allocate all queues Tx resources 2331 * @adapter: board private structure 2332 * 2333 * If this function returns with an error, then it's possible one or 2334 * more of the rings is populated (while the rest are not). It is the 2335 * callers duty to clean those orphaned rings. 2336 * 2337 * Return 0 on success, negative on failure 2338 **/ 2339static int ixgbevf_setup_all_tx_resources(struct ixgbevf_adapter *adapter) 2340{ 2341 int i, err = 0; 2342 2343 for (i = 0; i < adapter->num_tx_queues; i++) { 2344 err = ixgbevf_setup_tx_resources(adapter, &adapter->tx_ring[i]); 2345 if (!err) 2346 continue; 2347 hw_dbg(&adapter->hw, 2348 "Allocation for Tx Queue %u failed\n", i); 2349 break; 2350 } 2351 2352 return err; 2353} 2354 2355/** 2356 * ixgbevf_setup_rx_resources - allocate Rx resources (Descriptors) 2357 * @adapter: board private structure 2358 * @rx_ring: rx descriptor ring (for a specific queue) to setup 2359 * 2360 * Returns 0 on success, negative on failure 2361 **/ 2362int ixgbevf_setup_rx_resources(struct ixgbevf_adapter *adapter, 2363 struct ixgbevf_ring *rx_ring) 2364{ 2365 struct pci_dev *pdev = adapter->pdev; 2366 int size; 2367 2368 size = sizeof(struct ixgbevf_rx_buffer) * rx_ring->count; 2369 rx_ring->rx_buffer_info = vzalloc(size); 2370 if (!rx_ring->rx_buffer_info) 2371 goto alloc_failed; 2372 2373 /* Round up to nearest 4K */ 2374 rx_ring->size = rx_ring->count * sizeof(union ixgbe_adv_rx_desc); 2375 rx_ring->size = ALIGN(rx_ring->size, 4096); 2376 2377 rx_ring->desc = dma_alloc_coherent(&pdev->dev, rx_ring->size, 2378 &rx_ring->dma, GFP_KERNEL); 2379 2380 if (!rx_ring->desc) { 2381 hw_dbg(&adapter->hw, 2382 "Unable to allocate memory for " 2383 "the receive descriptor ring\n"); 2384 vfree(rx_ring->rx_buffer_info); 2385 rx_ring->rx_buffer_info = NULL; 2386 goto alloc_failed; 2387 } 2388 2389 rx_ring->next_to_clean = 0; 2390 rx_ring->next_to_use = 0; 2391 2392 return 0; 2393alloc_failed: 2394 return -ENOMEM; 2395} 2396 2397/** 2398 * ixgbevf_setup_all_rx_resources - allocate all queues Rx resources 2399 * @adapter: board private structure 2400 * 2401 * If this function returns with an error, then it's possible one or 2402 * more of the rings is populated (while the rest are not). It is the 2403 * callers duty to clean those orphaned rings. 2404 * 2405 * Return 0 on success, negative on failure 2406 **/ 2407static int ixgbevf_setup_all_rx_resources(struct ixgbevf_adapter *adapter) 2408{ 2409 int i, err = 0; 2410 2411 for (i = 0; i < adapter->num_rx_queues; i++) { 2412 err = ixgbevf_setup_rx_resources(adapter, &adapter->rx_ring[i]); 2413 if (!err) 2414 continue; 2415 hw_dbg(&adapter->hw, 2416 "Allocation for Rx Queue %u failed\n", i); 2417 break; 2418 } 2419 return err; 2420} 2421 2422/** 2423 * ixgbevf_free_rx_resources - Free Rx Resources 2424 * @adapter: board private structure 2425 * @rx_ring: ring to clean the resources from 2426 * 2427 * Free all receive software resources 2428 **/ 2429void ixgbevf_free_rx_resources(struct ixgbevf_adapter *adapter, 2430 struct ixgbevf_ring *rx_ring) 2431{ 2432 struct pci_dev *pdev = adapter->pdev; 2433 2434 ixgbevf_clean_rx_ring(adapter, rx_ring); 2435 2436 vfree(rx_ring->rx_buffer_info); 2437 rx_ring->rx_buffer_info = NULL; 2438 2439 dma_free_coherent(&pdev->dev, rx_ring->size, rx_ring->desc, 2440 rx_ring->dma); 2441 2442 rx_ring->desc = NULL; 2443} 2444 2445/** 2446 * ixgbevf_free_all_rx_resources - Free Rx Resources for All Queues 2447 * @adapter: board private structure 2448 * 2449 * Free all receive software resources 2450 **/ 2451static void ixgbevf_free_all_rx_resources(struct ixgbevf_adapter *adapter) 2452{ 2453 int i; 2454 2455 for (i = 0; i < adapter->num_rx_queues; i++) 2456 if (adapter->rx_ring[i].desc) 2457 ixgbevf_free_rx_resources(adapter, 2458 &adapter->rx_ring[i]); 2459} 2460 2461static int ixgbevf_setup_queues(struct ixgbevf_adapter *adapter) 2462{ 2463 struct ixgbe_hw *hw = &adapter->hw; 2464 struct ixgbevf_ring *rx_ring; 2465 unsigned int def_q = 0; 2466 unsigned int num_tcs = 0; 2467 unsigned int num_rx_queues = 1; 2468 int err, i; 2469 2470 spin_lock(&adapter->mbx_lock); 2471 2472 /* fetch queue configuration from the PF */ 2473 err = ixgbevf_get_queues(hw, &num_tcs, &def_q); 2474 2475 spin_unlock(&adapter->mbx_lock); 2476 2477 if (err) 2478 return err; 2479 2480 if (num_tcs > 1) { 2481 /* update default Tx ring register index */ 2482 adapter->tx_ring[0].reg_idx = def_q; 2483 2484 /* we need as many queues as traffic classes */ 2485 num_rx_queues = num_tcs; 2486 } 2487 2488 /* nothing to do if we have the correct number of queues */ 2489 if (adapter->num_rx_queues == num_rx_queues) 2490 return 0; 2491 2492 /* allocate new rings */ 2493 rx_ring = kcalloc(num_rx_queues, 2494 sizeof(struct ixgbevf_ring), GFP_KERNEL); 2495 if (!rx_ring) 2496 return -ENOMEM; 2497 2498 /* setup ring fields */ 2499 for (i = 0; i < num_rx_queues; i++) { 2500 rx_ring[i].count = adapter->rx_ring_count; 2501 rx_ring[i].queue_index = i; 2502 rx_ring[i].reg_idx = i; 2503 rx_ring[i].dev = &adapter->pdev->dev; 2504 rx_ring[i].netdev = adapter->netdev; 2505 } 2506 2507 /* free the existing ring and queues */ 2508 adapter->num_rx_queues = 0; 2509 kfree(adapter->rx_ring); 2510 2511 /* move new rings into position on the adapter struct */ 2512 adapter->rx_ring = rx_ring; 2513 adapter->num_rx_queues = num_rx_queues; 2514 2515 return 0; 2516} 2517 2518/** 2519 * ixgbevf_open - Called when a network interface is made active 2520 * @netdev: network interface device structure 2521 * 2522 * Returns 0 on success, negative value on failure 2523 * 2524 * The open entry point is called when a network interface is made 2525 * active by the system (IFF_UP). At this point all resources needed 2526 * for transmit and receive operations are allocated, the interrupt 2527 * handler is registered with the OS, the watchdog timer is started, 2528 * and the stack is notified that the interface is ready. 2529 **/ 2530static int ixgbevf_open(struct net_device *netdev) 2531{ 2532 struct ixgbevf_adapter *adapter = netdev_priv(netdev); 2533 struct ixgbe_hw *hw = &adapter->hw; 2534 int err; 2535 2536 /* disallow open during test */ 2537 if (test_bit(__IXGBEVF_TESTING, &adapter->state)) 2538 return -EBUSY; 2539 2540 if (hw->adapter_stopped) { 2541 ixgbevf_reset(adapter); 2542 /* if adapter is still stopped then PF isn't up and 2543 * the vf can't start. */ 2544 if (hw->adapter_stopped) { 2545 err = IXGBE_ERR_MBX; 2546 pr_err("Unable to start - perhaps the PF Driver isn't " 2547 "up yet\n"); 2548 goto err_setup_reset; 2549 } 2550 } 2551 2552 ixgbevf_negotiate_api(adapter); 2553 2554 /* setup queue reg_idx and Rx queue count */ 2555 err = ixgbevf_setup_queues(adapter); 2556 if (err) 2557 goto err_setup_queues; 2558 2559 /* allocate transmit descriptors */ 2560 err = ixgbevf_setup_all_tx_resources(adapter); 2561 if (err) 2562 goto err_setup_tx; 2563 2564 /* allocate receive descriptors */ 2565 err = ixgbevf_setup_all_rx_resources(adapter); 2566 if (err) 2567 goto err_setup_rx; 2568 2569 ixgbevf_configure(adapter); 2570 2571 /* 2572 * Map the Tx/Rx rings to the vectors we were allotted. 2573 * if request_irq will be called in this function map_rings 2574 * must be called *before* up_complete 2575 */ 2576 ixgbevf_map_rings_to_vectors(adapter); 2577 2578 ixgbevf_up_complete(adapter); 2579 2580 /* clear any pending interrupts, may auto mask */ 2581 IXGBE_READ_REG(hw, IXGBE_VTEICR); 2582 err = ixgbevf_request_irq(adapter); 2583 if (err) 2584 goto err_req_irq; 2585 2586 ixgbevf_irq_enable(adapter); 2587 2588 return 0; 2589 2590err_req_irq: 2591 ixgbevf_down(adapter); 2592 ixgbevf_free_irq(adapter); 2593err_setup_rx: 2594 ixgbevf_free_all_rx_resources(adapter); 2595err_setup_tx: 2596 ixgbevf_free_all_tx_resources(adapter); 2597err_setup_queues: 2598 ixgbevf_reset(adapter); 2599 2600err_setup_reset: 2601 2602 return err; 2603} 2604 2605/** 2606 * ixgbevf_close - Disables a network interface 2607 * @netdev: network interface device structure 2608 * 2609 * Returns 0, this is not allowed to fail 2610 * 2611 * The close entry point is called when an interface is de-activated 2612 * by the OS. The hardware is still under the drivers control, but 2613 * needs to be disabled. A global MAC reset is issued to stop the 2614 * hardware, and all transmit and receive resources are freed. 2615 **/ 2616static int ixgbevf_close(struct net_device *netdev) 2617{ 2618 struct ixgbevf_adapter *adapter = netdev_priv(netdev); 2619 2620 ixgbevf_down(adapter); 2621 ixgbevf_free_irq(adapter); 2622 2623 ixgbevf_free_all_tx_resources(adapter); 2624 ixgbevf_free_all_rx_resources(adapter); 2625 2626 return 0; 2627} 2628 2629static void ixgbevf_tx_ctxtdesc(struct ixgbevf_ring *tx_ring, 2630 u32 vlan_macip_lens, u32 type_tucmd, 2631 u32 mss_l4len_idx) 2632{ 2633 struct ixgbe_adv_tx_context_desc *context_desc; 2634 u16 i = tx_ring->next_to_use; 2635 2636 context_desc = IXGBEVF_TX_CTXTDESC(tx_ring, i); 2637 2638 i++; 2639 tx_ring->next_to_use = (i < tx_ring->count) ? i : 0; 2640 2641 /* set bits to identify this as an advanced context descriptor */ 2642 type_tucmd |= IXGBE_TXD_CMD_DEXT | IXGBE_ADVTXD_DTYP_CTXT; 2643 2644 context_desc->vlan_macip_lens = cpu_to_le32(vlan_macip_lens); 2645 context_desc->seqnum_seed = 0; 2646 context_desc->type_tucmd_mlhl = cpu_to_le32(type_tucmd); 2647 context_desc->mss_l4len_idx = cpu_to_le32(mss_l4len_idx); 2648} 2649 2650static int ixgbevf_tso(struct ixgbevf_ring *tx_ring, 2651 struct sk_buff *skb, u32 tx_flags, u8 *hdr_len) 2652{ 2653 u32 vlan_macip_lens, type_tucmd; 2654 u32 mss_l4len_idx, l4len; 2655 2656 if (!skb_is_gso(skb)) 2657 return 0; 2658 2659 if (skb_header_cloned(skb)) { 2660 int err = pskb_expand_head(skb, 0, 0, GFP_ATOMIC); 2661 if (err) 2662 return err; 2663 } 2664 2665 /* ADV DTYP TUCMD MKRLOC/ISCSIHEDLEN */ 2666 type_tucmd = IXGBE_ADVTXD_TUCMD_L4T_TCP; 2667 2668 if (skb->protocol == htons(ETH_P_IP)) { 2669 struct iphdr *iph = ip_hdr(skb); 2670 iph->tot_len = 0; 2671 iph->check = 0; 2672 tcp_hdr(skb)->check = ~csum_tcpudp_magic(iph->saddr, 2673 iph->daddr, 0, 2674 IPPROTO_TCP, 2675 0); 2676 type_tucmd |= IXGBE_ADVTXD_TUCMD_IPV4; 2677 } else if (skb_is_gso_v6(skb)) { 2678 ipv6_hdr(skb)->payload_len = 0; 2679 tcp_hdr(skb)->check = 2680 ~csum_ipv6_magic(&ipv6_hdr(skb)->saddr, 2681 &ipv6_hdr(skb)->daddr, 2682 0, IPPROTO_TCP, 0); 2683 } 2684 2685 /* compute header lengths */ 2686 l4len = tcp_hdrlen(skb); 2687 *hdr_len += l4len; 2688 *hdr_len = skb_transport_offset(skb) + l4len; 2689 2690 /* mss_l4len_id: use 1 as index for TSO */ 2691 mss_l4len_idx = l4len << IXGBE_ADVTXD_L4LEN_SHIFT; 2692 mss_l4len_idx |= skb_shinfo(skb)->gso_size << IXGBE_ADVTXD_MSS_SHIFT; 2693 mss_l4len_idx |= 1 << IXGBE_ADVTXD_IDX_SHIFT; 2694 2695 /* vlan_macip_lens: HEADLEN, MACLEN, VLAN tag */ 2696 vlan_macip_lens = skb_network_header_len(skb); 2697 vlan_macip_lens |= skb_network_offset(skb) << IXGBE_ADVTXD_MACLEN_SHIFT; 2698 vlan_macip_lens |= tx_flags & IXGBE_TX_FLAGS_VLAN_MASK; 2699 2700 ixgbevf_tx_ctxtdesc(tx_ring, vlan_macip_lens, 2701 type_tucmd, mss_l4len_idx); 2702 2703 return 1; 2704} 2705 2706static bool ixgbevf_tx_csum(struct ixgbevf_ring *tx_ring, 2707 struct sk_buff *skb, u32 tx_flags) 2708{ 2709 2710 2711 2712 u32 vlan_macip_lens = 0; 2713 u32 mss_l4len_idx = 0; 2714 u32 type_tucmd = 0; 2715 2716 if (skb->ip_summed == CHECKSUM_PARTIAL) { 2717 u8 l4_hdr = 0; 2718 switch (skb->protocol) { 2719 case __constant_htons(ETH_P_IP): 2720 vlan_macip_lens |= skb_network_header_len(skb); 2721 type_tucmd |= IXGBE_ADVTXD_TUCMD_IPV4; 2722 l4_hdr = ip_hdr(skb)->protocol; 2723 break; 2724 case __constant_htons(ETH_P_IPV6): 2725 vlan_macip_lens |= skb_network_header_len(skb); 2726 l4_hdr = ipv6_hdr(skb)->nexthdr; 2727 break; 2728 default: 2729 if (unlikely(net_ratelimit())) { 2730 dev_warn(tx_ring->dev, 2731 "partial checksum but proto=%x!\n", 2732 skb->protocol); 2733 } 2734 break; 2735 } 2736 2737 switch (l4_hdr) { 2738 case IPPROTO_TCP: 2739 type_tucmd |= IXGBE_ADVTXD_TUCMD_L4T_TCP; 2740 mss_l4len_idx = tcp_hdrlen(skb) << 2741 IXGBE_ADVTXD_L4LEN_SHIFT; 2742 break; 2743 case IPPROTO_SCTP: 2744 type_tucmd |= IXGBE_ADVTXD_TUCMD_L4T_SCTP; 2745 mss_l4len_idx = sizeof(struct sctphdr) << 2746 IXGBE_ADVTXD_L4LEN_SHIFT; 2747 break; 2748 case IPPROTO_UDP: 2749 mss_l4len_idx = sizeof(struct udphdr) << 2750 IXGBE_ADVTXD_L4LEN_SHIFT; 2751 break; 2752 default: 2753 if (unlikely(net_ratelimit())) { 2754 dev_warn(tx_ring->dev, 2755 "partial checksum but l4 proto=%x!\n", 2756 l4_hdr); 2757 } 2758 break; 2759 } 2760 } 2761 2762 /* vlan_macip_lens: MACLEN, VLAN tag */ 2763 vlan_macip_lens |= skb_network_offset(skb) << IXGBE_ADVTXD_MACLEN_SHIFT; 2764 vlan_macip_lens |= tx_flags & IXGBE_TX_FLAGS_VLAN_MASK; 2765 2766 ixgbevf_tx_ctxtdesc(tx_ring, vlan_macip_lens, 2767 type_tucmd, mss_l4len_idx); 2768 2769 return (skb->ip_summed == CHECKSUM_PARTIAL); 2770} 2771 2772static int ixgbevf_tx_map(struct ixgbevf_ring *tx_ring, 2773 struct sk_buff *skb, u32 tx_flags, 2774 unsigned int first) 2775{ 2776 struct ixgbevf_tx_buffer *tx_buffer_info; 2777 unsigned int len; 2778 unsigned int total = skb->len; 2779 unsigned int offset = 0, size; 2780 int count = 0; 2781 unsigned int nr_frags = skb_shinfo(skb)->nr_frags; 2782 unsigned int f; 2783 int i; 2784 2785 i = tx_ring->next_to_use; 2786 2787 len = min(skb_headlen(skb), total); 2788 while (len) { 2789 tx_buffer_info = &tx_ring->tx_buffer_info[i]; 2790 size = min(len, (unsigned int)IXGBE_MAX_DATA_PER_TXD); 2791 2792 tx_buffer_info->length = size; 2793 tx_buffer_info->mapped_as_page = false; 2794 tx_buffer_info->dma = dma_map_single(tx_ring->dev, 2795 skb->data + offset, 2796 size, DMA_TO_DEVICE); 2797 if (dma_mapping_error(tx_ring->dev, tx_buffer_info->dma)) 2798 goto dma_error; 2799 tx_buffer_info->next_to_watch = i; 2800 2801 len -= size; 2802 total -= size; 2803 offset += size; 2804 count++; 2805 i++; 2806 if (i == tx_ring->count) 2807 i = 0; 2808 } 2809 2810 for (f = 0; f < nr_frags; f++) { 2811 const struct skb_frag_struct *frag; 2812 2813 frag = &skb_shinfo(skb)->frags[f]; 2814 len = min((unsigned int)skb_frag_size(frag), total); 2815 offset = 0; 2816 2817 while (len) { 2818 tx_buffer_info = &tx_ring->tx_buffer_info[i]; 2819 size = min(len, (unsigned int)IXGBE_MAX_DATA_PER_TXD); 2820 2821 tx_buffer_info->length = size; 2822 tx_buffer_info->dma = 2823 skb_frag_dma_map(tx_ring->dev, frag, 2824 offset, size, DMA_TO_DEVICE); 2825 tx_buffer_info->mapped_as_page = true; 2826 if (dma_mapping_error(tx_ring->dev, 2827 tx_buffer_info->dma)) 2828 goto dma_error; 2829 tx_buffer_info->next_to_watch = i; 2830 2831 len -= size; 2832 total -= size; 2833 offset += size; 2834 count++; 2835 i++; 2836 if (i == tx_ring->count) 2837 i = 0; 2838 } 2839 if (total == 0) 2840 break; 2841 } 2842 2843 if (i == 0) 2844 i = tx_ring->count - 1; 2845 else 2846 i = i - 1; 2847 tx_ring->tx_buffer_info[i].skb = skb; 2848 tx_ring->tx_buffer_info[first].next_to_watch = i; 2849 tx_ring->tx_buffer_info[first].time_stamp = jiffies; 2850 2851 return count; 2852 2853dma_error: 2854 dev_err(tx_ring->dev, "TX DMA map failed\n"); 2855 2856 /* clear timestamp and dma mappings for failed tx_buffer_info map */ 2857 tx_buffer_info->dma = 0; 2858 tx_buffer_info->next_to_watch = 0; 2859 count--; 2860 2861 /* clear timestamp and dma mappings for remaining portion of packet */ 2862 while (count >= 0) { 2863 count--; 2864 i--; 2865 if (i < 0) 2866 i += tx_ring->count; 2867 tx_buffer_info = &tx_ring->tx_buffer_info[i]; 2868 ixgbevf_unmap_and_free_tx_resource(tx_ring, tx_buffer_info); 2869 } 2870 2871 return count; 2872} 2873 2874static void ixgbevf_tx_queue(struct ixgbevf_ring *tx_ring, int tx_flags, 2875 int count, u32 paylen, u8 hdr_len) 2876{ 2877 union ixgbe_adv_tx_desc *tx_desc = NULL; 2878 struct ixgbevf_tx_buffer *tx_buffer_info; 2879 u32 olinfo_status = 0, cmd_type_len = 0; 2880 unsigned int i; 2881 2882 u32 txd_cmd = IXGBE_TXD_CMD_EOP | IXGBE_TXD_CMD_RS | IXGBE_TXD_CMD_IFCS; 2883 2884 cmd_type_len |= IXGBE_ADVTXD_DTYP_DATA; 2885 2886 cmd_type_len |= IXGBE_ADVTXD_DCMD_IFCS | IXGBE_ADVTXD_DCMD_DEXT; 2887 2888 if (tx_flags & IXGBE_TX_FLAGS_VLAN) 2889 cmd_type_len |= IXGBE_ADVTXD_DCMD_VLE; 2890 2891 if (tx_flags & IXGBE_TX_FLAGS_CSUM) 2892 olinfo_status |= IXGBE_ADVTXD_POPTS_TXSM; 2893 2894 if (tx_flags & IXGBE_TX_FLAGS_TSO) { 2895 cmd_type_len |= IXGBE_ADVTXD_DCMD_TSE; 2896 2897 /* use index 1 context for tso */ 2898 olinfo_status |= (1 << IXGBE_ADVTXD_IDX_SHIFT); 2899 if (tx_flags & IXGBE_TX_FLAGS_IPV4) 2900 olinfo_status |= IXGBE_ADVTXD_POPTS_IXSM; 2901 2902 } 2903 2904 /* 2905 * Check Context must be set if Tx switch is enabled, which it 2906 * always is for case where virtual functions are running 2907 */ 2908 olinfo_status |= IXGBE_ADVTXD_CC; 2909 2910 olinfo_status |= ((paylen - hdr_len) << IXGBE_ADVTXD_PAYLEN_SHIFT); 2911 2912 i = tx_ring->next_to_use; 2913 while (count--) { 2914 tx_buffer_info = &tx_ring->tx_buffer_info[i]; 2915 tx_desc = IXGBEVF_TX_DESC(tx_ring, i); 2916 tx_desc->read.buffer_addr = cpu_to_le64(tx_buffer_info->dma); 2917 tx_desc->read.cmd_type_len = 2918 cpu_to_le32(cmd_type_len | tx_buffer_info->length); 2919 tx_desc->read.olinfo_status = cpu_to_le32(olinfo_status); 2920 i++; 2921 if (i == tx_ring->count) 2922 i = 0; 2923 } 2924 2925 tx_desc->read.cmd_type_len |= cpu_to_le32(txd_cmd); 2926 2927 tx_ring->next_to_use = i; 2928} 2929 2930static int __ixgbevf_maybe_stop_tx(struct ixgbevf_ring *tx_ring, int size) 2931{ 2932 struct ixgbevf_adapter *adapter = netdev_priv(tx_ring->netdev); 2933 2934 netif_stop_subqueue(tx_ring->netdev, tx_ring->queue_index); 2935 /* Herbert's original patch had: 2936 * smp_mb__after_netif_stop_queue(); 2937 * but since that doesn't exist yet, just open code it. */ 2938 smp_mb(); 2939 2940 /* We need to check again in a case another CPU has just 2941 * made room available. */ 2942 if (likely(IXGBE_DESC_UNUSED(tx_ring) < size)) 2943 return -EBUSY; 2944 2945 /* A reprieve! - use start_queue because it doesn't call schedule */ 2946 netif_start_subqueue(tx_ring->netdev, tx_ring->queue_index); 2947 ++adapter->restart_queue; 2948 return 0; 2949} 2950 2951static int ixgbevf_maybe_stop_tx(struct ixgbevf_ring *tx_ring, int size) 2952{ 2953 if (likely(IXGBE_DESC_UNUSED(tx_ring) >= size)) 2954 return 0; 2955 return __ixgbevf_maybe_stop_tx(tx_ring, size); 2956} 2957 2958static int ixgbevf_xmit_frame(struct sk_buff *skb, struct net_device *netdev) 2959{ 2960 struct ixgbevf_adapter *adapter = netdev_priv(netdev); 2961 struct ixgbevf_ring *tx_ring; 2962 unsigned int first; 2963 unsigned int tx_flags = 0; 2964 u8 hdr_len = 0; 2965 int r_idx = 0, tso; 2966 u16 count = TXD_USE_COUNT(skb_headlen(skb)); 2967#if PAGE_SIZE > IXGBE_MAX_DATA_PER_TXD 2968 unsigned short f; 2969#endif 2970 2971 tx_ring = &adapter->tx_ring[r_idx]; 2972 2973 /* 2974 * need: 1 descriptor per page * PAGE_SIZE/IXGBE_MAX_DATA_PER_TXD, 2975 * + 1 desc for skb_headlen/IXGBE_MAX_DATA_PER_TXD, 2976 * + 2 desc gap to keep tail from touching head, 2977 * + 1 desc for context descriptor, 2978 * otherwise try next time 2979 */ 2980#if PAGE_SIZE > IXGBE_MAX_DATA_PER_TXD 2981 for (f = 0; f < skb_shinfo(skb)->nr_frags; f++) 2982 count += TXD_USE_COUNT(skb_shinfo(skb)->frags[f].size); 2983#else 2984 count += skb_shinfo(skb)->nr_frags; 2985#endif 2986 if (ixgbevf_maybe_stop_tx(tx_ring, count + 3)) { 2987 adapter->tx_busy++; 2988 return NETDEV_TX_BUSY; 2989 } 2990 2991 if (vlan_tx_tag_present(skb)) { 2992 tx_flags |= vlan_tx_tag_get(skb); 2993 tx_flags <<= IXGBE_TX_FLAGS_VLAN_SHIFT; 2994 tx_flags |= IXGBE_TX_FLAGS_VLAN; 2995 } 2996 2997 first = tx_ring->next_to_use; 2998 2999 if (skb->protocol == htons(ETH_P_IP)) 3000 tx_flags |= IXGBE_TX_FLAGS_IPV4; 3001 tso = ixgbevf_tso(tx_ring, skb, tx_flags, &hdr_len); 3002 if (tso < 0) { 3003 dev_kfree_skb_any(skb); 3004 return NETDEV_TX_OK; 3005 } 3006 3007 if (tso) 3008 tx_flags |= IXGBE_TX_FLAGS_TSO | IXGBE_TX_FLAGS_CSUM; 3009 else if (ixgbevf_tx_csum(tx_ring, skb, tx_flags)) 3010 tx_flags |= IXGBE_TX_FLAGS_CSUM; 3011 3012 ixgbevf_tx_queue(tx_ring, tx_flags, 3013 ixgbevf_tx_map(tx_ring, skb, tx_flags, first), 3014 skb->len, hdr_len); 3015 /* 3016 * Force memory writes to complete before letting h/w 3017 * know there are new descriptors to fetch. (Only 3018 * applicable for weak-ordered memory model archs, 3019 * such as IA-64). 3020 */ 3021 wmb(); 3022 3023 writel(tx_ring->next_to_use, adapter->hw.hw_addr + tx_ring->tail); 3024 3025 ixgbevf_maybe_stop_tx(tx_ring, DESC_NEEDED); 3026 3027 return NETDEV_TX_OK; 3028} 3029 3030/** 3031 * ixgbevf_set_mac - Change the Ethernet Address of the NIC 3032 * @netdev: network interface device structure 3033 * @p: pointer to an address structure 3034 * 3035 * Returns 0 on success, negative on failure 3036 **/ 3037static int ixgbevf_set_mac(struct net_device *netdev, void *p) 3038{ 3039 struct ixgbevf_adapter *adapter = netdev_priv(netdev); 3040 struct ixgbe_hw *hw = &adapter->hw; 3041 struct sockaddr *addr = p; 3042 3043 if (!is_valid_ether_addr(addr->sa_data)) 3044 return -EADDRNOTAVAIL; 3045 3046 memcpy(netdev->dev_addr, addr->sa_data, netdev->addr_len); 3047 memcpy(hw->mac.addr, addr->sa_data, netdev->addr_len); 3048 3049 spin_lock(&adapter->mbx_lock); 3050 3051 if (hw->mac.ops.set_rar) 3052 hw->mac.ops.set_rar(hw, 0, hw->mac.addr, 0); 3053 3054 spin_unlock(&adapter->mbx_lock); 3055 3056 return 0; 3057} 3058 3059/** 3060 * ixgbevf_change_mtu - Change the Maximum Transfer Unit 3061 * @netdev: network interface device structure 3062 * @new_mtu: new value for maximum frame size 3063 * 3064 * Returns 0 on success, negative on failure 3065 **/ 3066static int ixgbevf_change_mtu(struct net_device *netdev, int new_mtu) 3067{ 3068 struct ixgbevf_adapter *adapter = netdev_priv(netdev); 3069 int max_frame = new_mtu + ETH_HLEN + ETH_FCS_LEN; 3070 int max_possible_frame = MAXIMUM_ETHERNET_VLAN_SIZE; 3071 3072 switch (adapter->hw.api_version) { 3073 case ixgbe_mbox_api_11: 3074 max_possible_frame = IXGBE_MAX_JUMBO_FRAME_SIZE; 3075 break; 3076 default: 3077 if (adapter->hw.mac.type == ixgbe_mac_X540_vf) 3078 max_possible_frame = IXGBE_MAX_JUMBO_FRAME_SIZE; 3079 break; 3080 } 3081 3082 /* MTU < 68 is an error and causes problems on some kernels */ 3083 if ((new_mtu < 68) || (max_frame > max_possible_frame)) 3084 return -EINVAL; 3085 3086 hw_dbg(&adapter->hw, "changing MTU from %d to %d\n", 3087 netdev->mtu, new_mtu); 3088 /* must set new MTU before calling down or up */ 3089 netdev->mtu = new_mtu; 3090 3091 if (netif_running(netdev)) 3092 ixgbevf_reinit_locked(adapter); 3093 3094 return 0; 3095} 3096 3097static int ixgbevf_suspend(struct pci_dev *pdev, pm_message_t state) 3098{ 3099 struct net_device *netdev = pci_get_drvdata(pdev); 3100 struct ixgbevf_adapter *adapter = netdev_priv(netdev); 3101#ifdef CONFIG_PM 3102 int retval = 0; 3103#endif 3104 3105 netif_device_detach(netdev); 3106 3107 if (netif_running(netdev)) { 3108 rtnl_lock(); 3109 ixgbevf_down(adapter); 3110 ixgbevf_free_irq(adapter); 3111 ixgbevf_free_all_tx_resources(adapter); 3112 ixgbevf_free_all_rx_resources(adapter); 3113 rtnl_unlock(); 3114 } 3115 3116 ixgbevf_clear_interrupt_scheme(adapter); 3117 3118#ifdef CONFIG_PM 3119 retval = pci_save_state(pdev); 3120 if (retval) 3121 return retval; 3122 3123#endif 3124 pci_disable_device(pdev); 3125 3126 return 0; 3127} 3128 3129#ifdef CONFIG_PM 3130static int ixgbevf_resume(struct pci_dev *pdev) 3131{ 3132 struct ixgbevf_adapter *adapter = pci_get_drvdata(pdev); 3133 struct net_device *netdev = adapter->netdev; 3134 u32 err; 3135 3136 pci_set_power_state(pdev, PCI_D0); 3137 pci_restore_state(pdev); 3138 /* 3139 * pci_restore_state clears dev->state_saved so call 3140 * pci_save_state to restore it. 3141 */ 3142 pci_save_state(pdev); 3143 3144 err = pci_enable_device_mem(pdev); 3145 if (err) { 3146 dev_err(&pdev->dev, "Cannot enable PCI device from suspend\n"); 3147 return err; 3148 } 3149 pci_set_master(pdev); 3150 3151 rtnl_lock(); 3152 err = ixgbevf_init_interrupt_scheme(adapter); 3153 rtnl_unlock(); 3154 if (err) { 3155 dev_err(&pdev->dev, "Cannot initialize interrupts\n"); 3156 return err; 3157 } 3158 3159 ixgbevf_reset(adapter); 3160 3161 if (netif_running(netdev)) { 3162 err = ixgbevf_open(netdev); 3163 if (err) 3164 return err; 3165 } 3166 3167 netif_device_attach(netdev); 3168 3169 return err; 3170} 3171 3172#endif /* CONFIG_PM */ 3173static void ixgbevf_shutdown(struct pci_dev *pdev) 3174{ 3175 ixgbevf_suspend(pdev, PMSG_SUSPEND); 3176} 3177 3178static struct rtnl_link_stats64 *ixgbevf_get_stats(struct net_device *netdev, 3179 struct rtnl_link_stats64 *stats) 3180{ 3181 struct ixgbevf_adapter *adapter = netdev_priv(netdev); 3182 unsigned int start; 3183 u64 bytes, packets; 3184 const struct ixgbevf_ring *ring; 3185 int i; 3186 3187 ixgbevf_update_stats(adapter); 3188 3189 stats->multicast = adapter->stats.vfmprc - adapter->stats.base_vfmprc; 3190 3191 for (i = 0; i < adapter->num_rx_queues; i++) { 3192 ring = &adapter->rx_ring[i]; 3193 do { 3194 start = u64_stats_fetch_begin_bh(&ring->syncp); 3195 bytes = ring->total_bytes; 3196 packets = ring->total_packets; 3197 } while (u64_stats_fetch_retry_bh(&ring->syncp, start)); 3198 stats->rx_bytes += bytes; 3199 stats->rx_packets += packets; 3200 } 3201 3202 for (i = 0; i < adapter->num_tx_queues; i++) { 3203 ring = &adapter->tx_ring[i]; 3204 do { 3205 start = u64_stats_fetch_begin_bh(&ring->syncp); 3206 bytes = ring->total_bytes; 3207 packets = ring->total_packets; 3208 } while (u64_stats_fetch_retry_bh(&ring->syncp, start)); 3209 stats->tx_bytes += bytes; 3210 stats->tx_packets += packets; 3211 } 3212 3213 return stats; 3214} 3215 3216static const struct net_device_ops ixgbevf_netdev_ops = { 3217 .ndo_open = ixgbevf_open, 3218 .ndo_stop = ixgbevf_close, 3219 .ndo_start_xmit = ixgbevf_xmit_frame, 3220 .ndo_set_rx_mode = ixgbevf_set_rx_mode, 3221 .ndo_get_stats64 = ixgbevf_get_stats, 3222 .ndo_validate_addr = eth_validate_addr, 3223 .ndo_set_mac_address = ixgbevf_set_mac, 3224 .ndo_change_mtu = ixgbevf_change_mtu, 3225 .ndo_tx_timeout = ixgbevf_tx_timeout, 3226 .ndo_vlan_rx_add_vid = ixgbevf_vlan_rx_add_vid, 3227 .ndo_vlan_rx_kill_vid = ixgbevf_vlan_rx_kill_vid, 3228}; 3229 3230static void ixgbevf_assign_netdev_ops(struct net_device *dev) 3231{ 3232 dev->netdev_ops = &ixgbevf_netdev_ops; 3233 ixgbevf_set_ethtool_ops(dev); 3234 dev->watchdog_timeo = 5 * HZ; 3235} 3236 3237/** 3238 * ixgbevf_probe - Device Initialization Routine 3239 * @pdev: PCI device information struct 3240 * @ent: entry in ixgbevf_pci_tbl 3241 * 3242 * Returns 0 on success, negative on failure 3243 * 3244 * ixgbevf_probe initializes an adapter identified by a pci_dev structure. 3245 * The OS initialization, configuring of the adapter private structure, 3246 * and a hardware reset occur. 3247 **/ 3248static int __devinit ixgbevf_probe(struct pci_dev *pdev, 3249 const struct pci_device_id *ent) 3250{ 3251 struct net_device *netdev; 3252 struct ixgbevf_adapter *adapter = NULL; 3253 struct ixgbe_hw *hw = NULL; 3254 const struct ixgbevf_info *ii = ixgbevf_info_tbl[ent->driver_data]; 3255 static int cards_found; 3256 int err, pci_using_dac; 3257 3258 err = pci_enable_device(pdev); 3259 if (err) 3260 return err; 3261 3262 if (!dma_set_mask(&pdev->dev, DMA_BIT_MASK(64)) && 3263 !dma_set_coherent_mask(&pdev->dev, DMA_BIT_MASK(64))) { 3264 pci_using_dac = 1; 3265 } else { 3266 err = dma_set_mask(&pdev->dev, DMA_BIT_MASK(32)); 3267 if (err) { 3268 err = dma_set_coherent_mask(&pdev->dev, 3269 DMA_BIT_MASK(32)); 3270 if (err) { 3271 dev_err(&pdev->dev, "No usable DMA " 3272 "configuration, aborting\n"); 3273 goto err_dma; 3274 } 3275 } 3276 pci_using_dac = 0; 3277 } 3278 3279 err = pci_request_regions(pdev, ixgbevf_driver_name); 3280 if (err) { 3281 dev_err(&pdev->dev, "pci_request_regions failed 0x%x\n", err); 3282 goto err_pci_reg; 3283 } 3284 3285 pci_set_master(pdev); 3286 3287 netdev = alloc_etherdev_mq(sizeof(struct ixgbevf_adapter), 3288 MAX_TX_QUEUES); 3289 if (!netdev) { 3290 err = -ENOMEM; 3291 goto err_alloc_etherdev; 3292 } 3293 3294 SET_NETDEV_DEV(netdev, &pdev->dev); 3295 3296 pci_set_drvdata(pdev, netdev); 3297 adapter = netdev_priv(netdev); 3298 3299 adapter->netdev = netdev; 3300 adapter->pdev = pdev; 3301 hw = &adapter->hw; 3302 hw->back = adapter; 3303 adapter->msg_enable = netif_msg_init(debug, DEFAULT_MSG_ENABLE); 3304 3305 /* 3306 * call save state here in standalone driver because it relies on 3307 * adapter struct to exist, and needs to call netdev_priv 3308 */ 3309 pci_save_state(pdev); 3310 3311 hw->hw_addr = ioremap(pci_resource_start(pdev, 0), 3312 pci_resource_len(pdev, 0)); 3313 if (!hw->hw_addr) { 3314 err = -EIO; 3315 goto err_ioremap; 3316 } 3317 3318 ixgbevf_assign_netdev_ops(netdev); 3319 3320 adapter->bd_number = cards_found; 3321 3322 /* Setup hw api */ 3323 memcpy(&hw->mac.ops, ii->mac_ops, sizeof(hw->mac.ops)); 3324 hw->mac.type = ii->mac; 3325 3326 memcpy(&hw->mbx.ops, &ixgbevf_mbx_ops, 3327 sizeof(struct ixgbe_mbx_operations)); 3328 3329 /* setup the private structure */ 3330 err = ixgbevf_sw_init(adapter); 3331 if (err) 3332 goto err_sw_init; 3333 3334 /* The HW MAC address was set and/or determined in sw_init */ 3335 memcpy(netdev->perm_addr, adapter->hw.mac.addr, netdev->addr_len); 3336 3337 if (!is_valid_ether_addr(netdev->dev_addr)) { 3338 pr_err("invalid MAC address\n"); 3339 err = -EIO; 3340 goto err_sw_init; 3341 } 3342 3343 netdev->hw_features = NETIF_F_SG | 3344 NETIF_F_IP_CSUM | 3345 NETIF_F_IPV6_CSUM | 3346 NETIF_F_TSO | 3347 NETIF_F_TSO6 | 3348 NETIF_F_RXCSUM; 3349 3350 netdev->features = netdev->hw_features | 3351 NETIF_F_HW_VLAN_TX | 3352 NETIF_F_HW_VLAN_RX | 3353 NETIF_F_HW_VLAN_FILTER; 3354 3355 netdev->vlan_features |= NETIF_F_TSO; 3356 netdev->vlan_features |= NETIF_F_TSO6; 3357 netdev->vlan_features |= NETIF_F_IP_CSUM; 3358 netdev->vlan_features |= NETIF_F_IPV6_CSUM; 3359 netdev->vlan_features |= NETIF_F_SG; 3360 3361 if (pci_using_dac) 3362 netdev->features |= NETIF_F_HIGHDMA; 3363 3364 netdev->priv_flags |= IFF_UNICAST_FLT; 3365 3366 init_timer(&adapter->watchdog_timer); 3367 adapter->watchdog_timer.function = ixgbevf_watchdog; 3368 adapter->watchdog_timer.data = (unsigned long)adapter; 3369 3370 INIT_WORK(&adapter->reset_task, ixgbevf_reset_task); 3371 INIT_WORK(&adapter->watchdog_task, ixgbevf_watchdog_task); 3372 3373 err = ixgbevf_init_interrupt_scheme(adapter); 3374 if (err) 3375 goto err_sw_init; 3376 3377 /* pick up the PCI bus settings for reporting later */ 3378 if (hw->mac.ops.get_bus_info) 3379 hw->mac.ops.get_bus_info(hw); 3380 3381 strcpy(netdev->name, "eth%d"); 3382 3383 err = register_netdev(netdev); 3384 if (err) 3385 goto err_register; 3386 3387 netif_carrier_off(netdev); 3388 3389 ixgbevf_init_last_counter_stats(adapter); 3390 3391 /* print the MAC address */ 3392 hw_dbg(hw, "%pM\n", netdev->dev_addr); 3393 3394 hw_dbg(hw, "MAC: %d\n", hw->mac.type); 3395 3396 hw_dbg(hw, "Intel(R) 82599 Virtual Function\n"); 3397 cards_found++; 3398 return 0; 3399 3400err_register: 3401 ixgbevf_clear_interrupt_scheme(adapter); 3402err_sw_init: 3403 ixgbevf_reset_interrupt_capability(adapter); 3404 iounmap(hw->hw_addr); 3405err_ioremap: 3406 free_netdev(netdev); 3407err_alloc_etherdev: 3408 pci_release_regions(pdev); 3409err_pci_reg: 3410err_dma: 3411 pci_disable_device(pdev); 3412 return err; 3413} 3414 3415/** 3416 * ixgbevf_remove - Device Removal Routine 3417 * @pdev: PCI device information struct 3418 * 3419 * ixgbevf_remove is called by the PCI subsystem to alert the driver 3420 * that it should release a PCI device. The could be caused by a 3421 * Hot-Plug event, or because the driver is going to be removed from 3422 * memory. 3423 **/ 3424static void __devexit ixgbevf_remove(struct pci_dev *pdev) 3425{ 3426 struct net_device *netdev = pci_get_drvdata(pdev); 3427 struct ixgbevf_adapter *adapter = netdev_priv(netdev); 3428 3429 set_bit(__IXGBEVF_DOWN, &adapter->state); 3430 3431 del_timer_sync(&adapter->watchdog_timer); 3432 3433 cancel_work_sync(&adapter->reset_task); 3434 cancel_work_sync(&adapter->watchdog_task); 3435 3436 if (netdev->reg_state == NETREG_REGISTERED) 3437 unregister_netdev(netdev); 3438 3439 ixgbevf_clear_interrupt_scheme(adapter); 3440 ixgbevf_reset_interrupt_capability(adapter); 3441 3442 iounmap(adapter->hw.hw_addr); 3443 pci_release_regions(pdev); 3444 3445 hw_dbg(&adapter->hw, "Remove complete\n"); 3446 3447 kfree(adapter->tx_ring); 3448 kfree(adapter->rx_ring); 3449 3450 free_netdev(netdev); 3451 3452 pci_disable_device(pdev); 3453} 3454 3455/** 3456 * ixgbevf_io_error_detected - called when PCI error is detected 3457 * @pdev: Pointer to PCI device 3458 * @state: The current pci connection state 3459 * 3460 * This function is called after a PCI bus error affecting 3461 * this device has been detected. 3462 */ 3463static pci_ers_result_t ixgbevf_io_error_detected(struct pci_dev *pdev, 3464 pci_channel_state_t state) 3465{ 3466 struct net_device *netdev = pci_get_drvdata(pdev); 3467 struct ixgbevf_adapter *adapter = netdev_priv(netdev); 3468 3469 netif_device_detach(netdev); 3470 3471 if (state == pci_channel_io_perm_failure) 3472 return PCI_ERS_RESULT_DISCONNECT; 3473 3474 if (netif_running(netdev)) 3475 ixgbevf_down(adapter); 3476 3477 pci_disable_device(pdev); 3478 3479 /* Request a slot slot reset. */ 3480 return PCI_ERS_RESULT_NEED_RESET; 3481} 3482 3483/** 3484 * ixgbevf_io_slot_reset - called after the pci bus has been reset. 3485 * @pdev: Pointer to PCI device 3486 * 3487 * Restart the card from scratch, as if from a cold-boot. Implementation 3488 * resembles the first-half of the ixgbevf_resume routine. 3489 */ 3490static pci_ers_result_t ixgbevf_io_slot_reset(struct pci_dev *pdev) 3491{ 3492 struct net_device *netdev = pci_get_drvdata(pdev); 3493 struct ixgbevf_adapter *adapter = netdev_priv(netdev); 3494 3495 if (pci_enable_device_mem(pdev)) { 3496 dev_err(&pdev->dev, 3497 "Cannot re-enable PCI device after reset.\n"); 3498 return PCI_ERS_RESULT_DISCONNECT; 3499 } 3500 3501 pci_set_master(pdev); 3502 3503 ixgbevf_reset(adapter); 3504 3505 return PCI_ERS_RESULT_RECOVERED; 3506} 3507 3508/** 3509 * ixgbevf_io_resume - called when traffic can start flowing again. 3510 * @pdev: Pointer to PCI device 3511 * 3512 * This callback is called when the error recovery driver tells us that 3513 * its OK to resume normal operation. Implementation resembles the 3514 * second-half of the ixgbevf_resume routine. 3515 */ 3516static void ixgbevf_io_resume(struct pci_dev *pdev) 3517{ 3518 struct net_device *netdev = pci_get_drvdata(pdev); 3519 struct ixgbevf_adapter *adapter = netdev_priv(netdev); 3520 3521 if (netif_running(netdev)) 3522 ixgbevf_up(adapter); 3523 3524 netif_device_attach(netdev); 3525} 3526 3527/* PCI Error Recovery (ERS) */ 3528static const struct pci_error_handlers ixgbevf_err_handler = { 3529 .error_detected = ixgbevf_io_error_detected, 3530 .slot_reset = ixgbevf_io_slot_reset, 3531 .resume = ixgbevf_io_resume, 3532}; 3533 3534static struct pci_driver ixgbevf_driver = { 3535 .name = ixgbevf_driver_name, 3536 .id_table = ixgbevf_pci_tbl, 3537 .probe = ixgbevf_probe, 3538 .remove = __devexit_p(ixgbevf_remove), 3539#ifdef CONFIG_PM 3540 /* Power Management Hooks */ 3541 .suspend = ixgbevf_suspend, 3542 .resume = ixgbevf_resume, 3543#endif 3544 .shutdown = ixgbevf_shutdown, 3545 .err_handler = &ixgbevf_err_handler 3546}; 3547 3548/** 3549 * ixgbevf_init_module - Driver Registration Routine 3550 * 3551 * ixgbevf_init_module is the first routine called when the driver is 3552 * loaded. All it does is register with the PCI subsystem. 3553 **/ 3554static int __init ixgbevf_init_module(void) 3555{ 3556 int ret; 3557 pr_info("%s - version %s\n", ixgbevf_driver_string, 3558 ixgbevf_driver_version); 3559 3560 pr_info("%s\n", ixgbevf_copyright); 3561 3562 ret = pci_register_driver(&ixgbevf_driver); 3563 return ret; 3564} 3565 3566module_init(ixgbevf_init_module); 3567 3568/** 3569 * ixgbevf_exit_module - Driver Exit Cleanup Routine 3570 * 3571 * ixgbevf_exit_module is called just before the driver is removed 3572 * from memory. 3573 **/ 3574static void __exit ixgbevf_exit_module(void) 3575{ 3576 pci_unregister_driver(&ixgbevf_driver); 3577} 3578 3579#ifdef DEBUG 3580/** 3581 * ixgbevf_get_hw_dev_name - return device name string 3582 * used by hardware layer to print debugging information 3583 **/ 3584char *ixgbevf_get_hw_dev_name(struct ixgbe_hw *hw) 3585{ 3586 struct ixgbevf_adapter *adapter = hw->back; 3587 return adapter->netdev->name; 3588} 3589 3590#endif 3591module_exit(ixgbevf_exit_module); 3592 3593/* ixgbevf_main.c */ 3594