ixgb_main.c revision 590990362b939bc6edf8e05bd0f3146b33a42870
1/******************************************************************************* 2 3 Intel PRO/10GbE Linux driver 4 Copyright(c) 1999 - 2008 Intel Corporation. 5 6 This program is free software; you can redistribute it and/or modify it 7 under the terms and conditions of the GNU General Public License, 8 version 2, as published by the Free Software Foundation. 9 10 This program is distributed in the hope it will be useful, but WITHOUT 11 ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or 12 FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for 13 more details. 14 15 You should have received a copy of the GNU General Public License along with 16 this program; if not, write to the Free Software Foundation, Inc., 17 51 Franklin St - Fifth Floor, Boston, MA 02110-1301 USA. 18 19 The full GNU General Public License is included in this distribution in 20 the file called "COPYING". 21 22 Contact Information: 23 Linux NICS <linux.nics@intel.com> 24 e1000-devel Mailing List <e1000-devel@lists.sourceforge.net> 25 Intel Corporation, 5200 N.E. Elam Young Parkway, Hillsboro, OR 97124-6497 26 27*******************************************************************************/ 28 29#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt 30 31#include <linux/prefetch.h> 32#include "ixgb.h" 33 34char ixgb_driver_name[] = "ixgb"; 35static char ixgb_driver_string[] = "Intel(R) PRO/10GbE Network Driver"; 36 37#define DRIVERNAPI "-NAPI" 38#define DRV_VERSION "1.0.135-k2" DRIVERNAPI 39const char ixgb_driver_version[] = DRV_VERSION; 40static const char ixgb_copyright[] = "Copyright (c) 1999-2008 Intel Corporation."; 41 42#define IXGB_CB_LENGTH 256 43static unsigned int copybreak __read_mostly = IXGB_CB_LENGTH; 44module_param(copybreak, uint, 0644); 45MODULE_PARM_DESC(copybreak, 46 "Maximum size of packet that is copied to a new buffer on receive"); 47 48/* ixgb_pci_tbl - PCI Device ID Table 49 * 50 * Wildcard entries (PCI_ANY_ID) should come last 51 * Last entry must be all 0s 52 * 53 * { Vendor ID, Device ID, SubVendor ID, SubDevice ID, 54 * Class, Class Mask, private data (not used) } 55 */ 56static DEFINE_PCI_DEVICE_TABLE(ixgb_pci_tbl) = { 57 {PCI_VENDOR_ID_INTEL, IXGB_DEVICE_ID_82597EX, 58 PCI_ANY_ID, PCI_ANY_ID, 0, 0, 0}, 59 {PCI_VENDOR_ID_INTEL, IXGB_DEVICE_ID_82597EX_CX4, 60 PCI_ANY_ID, PCI_ANY_ID, 0, 0, 0}, 61 {PCI_VENDOR_ID_INTEL, IXGB_DEVICE_ID_82597EX_SR, 62 PCI_ANY_ID, PCI_ANY_ID, 0, 0, 0}, 63 {PCI_VENDOR_ID_INTEL, IXGB_DEVICE_ID_82597EX_LR, 64 PCI_ANY_ID, PCI_ANY_ID, 0, 0, 0}, 65 66 /* required last entry */ 67 {0,} 68}; 69 70MODULE_DEVICE_TABLE(pci, ixgb_pci_tbl); 71 72/* Local Function Prototypes */ 73static int ixgb_init_module(void); 74static void ixgb_exit_module(void); 75static int ixgb_probe(struct pci_dev *pdev, const struct pci_device_id *ent); 76static void ixgb_remove(struct pci_dev *pdev); 77static int ixgb_sw_init(struct ixgb_adapter *adapter); 78static int ixgb_open(struct net_device *netdev); 79static int ixgb_close(struct net_device *netdev); 80static void ixgb_configure_tx(struct ixgb_adapter *adapter); 81static void ixgb_configure_rx(struct ixgb_adapter *adapter); 82static void ixgb_setup_rctl(struct ixgb_adapter *adapter); 83static void ixgb_clean_tx_ring(struct ixgb_adapter *adapter); 84static void ixgb_clean_rx_ring(struct ixgb_adapter *adapter); 85static void ixgb_set_multi(struct net_device *netdev); 86static void ixgb_watchdog(unsigned long data); 87static netdev_tx_t ixgb_xmit_frame(struct sk_buff *skb, 88 struct net_device *netdev); 89static struct net_device_stats *ixgb_get_stats(struct net_device *netdev); 90static int ixgb_change_mtu(struct net_device *netdev, int new_mtu); 91static int ixgb_set_mac(struct net_device *netdev, void *p); 92static irqreturn_t ixgb_intr(int irq, void *data); 93static bool ixgb_clean_tx_irq(struct ixgb_adapter *adapter); 94 95static int ixgb_clean(struct napi_struct *, int); 96static bool ixgb_clean_rx_irq(struct ixgb_adapter *, int *, int); 97static void ixgb_alloc_rx_buffers(struct ixgb_adapter *, int); 98 99static void ixgb_tx_timeout(struct net_device *dev); 100static void ixgb_tx_timeout_task(struct work_struct *work); 101 102static void ixgb_vlan_strip_enable(struct ixgb_adapter *adapter); 103static void ixgb_vlan_strip_disable(struct ixgb_adapter *adapter); 104static int ixgb_vlan_rx_add_vid(struct net_device *netdev, 105 __be16 proto, u16 vid); 106static int ixgb_vlan_rx_kill_vid(struct net_device *netdev, 107 __be16 proto, u16 vid); 108static void ixgb_restore_vlan(struct ixgb_adapter *adapter); 109 110#ifdef CONFIG_NET_POLL_CONTROLLER 111/* for netdump / net console */ 112static void ixgb_netpoll(struct net_device *dev); 113#endif 114 115static pci_ers_result_t ixgb_io_error_detected (struct pci_dev *pdev, 116 enum pci_channel_state state); 117static pci_ers_result_t ixgb_io_slot_reset (struct pci_dev *pdev); 118static void ixgb_io_resume (struct pci_dev *pdev); 119 120static const struct pci_error_handlers ixgb_err_handler = { 121 .error_detected = ixgb_io_error_detected, 122 .slot_reset = ixgb_io_slot_reset, 123 .resume = ixgb_io_resume, 124}; 125 126static struct pci_driver ixgb_driver = { 127 .name = ixgb_driver_name, 128 .id_table = ixgb_pci_tbl, 129 .probe = ixgb_probe, 130 .remove = ixgb_remove, 131 .err_handler = &ixgb_err_handler 132}; 133 134MODULE_AUTHOR("Intel Corporation, <linux.nics@intel.com>"); 135MODULE_DESCRIPTION("Intel(R) PRO/10GbE Network Driver"); 136MODULE_LICENSE("GPL"); 137MODULE_VERSION(DRV_VERSION); 138 139#define DEFAULT_MSG_ENABLE (NETIF_MSG_DRV|NETIF_MSG_PROBE|NETIF_MSG_LINK) 140static int debug = -1; 141module_param(debug, int, 0); 142MODULE_PARM_DESC(debug, "Debug level (0=none,...,16=all)"); 143 144/** 145 * ixgb_init_module - Driver Registration Routine 146 * 147 * ixgb_init_module is the first routine called when the driver is 148 * loaded. All it does is register with the PCI subsystem. 149 **/ 150 151static int __init 152ixgb_init_module(void) 153{ 154 pr_info("%s - version %s\n", ixgb_driver_string, ixgb_driver_version); 155 pr_info("%s\n", ixgb_copyright); 156 157 return pci_register_driver(&ixgb_driver); 158} 159 160module_init(ixgb_init_module); 161 162/** 163 * ixgb_exit_module - Driver Exit Cleanup Routine 164 * 165 * ixgb_exit_module is called just before the driver is removed 166 * from memory. 167 **/ 168 169static void __exit 170ixgb_exit_module(void) 171{ 172 pci_unregister_driver(&ixgb_driver); 173} 174 175module_exit(ixgb_exit_module); 176 177/** 178 * ixgb_irq_disable - Mask off interrupt generation on the NIC 179 * @adapter: board private structure 180 **/ 181 182static void 183ixgb_irq_disable(struct ixgb_adapter *adapter) 184{ 185 IXGB_WRITE_REG(&adapter->hw, IMC, ~0); 186 IXGB_WRITE_FLUSH(&adapter->hw); 187 synchronize_irq(adapter->pdev->irq); 188} 189 190/** 191 * ixgb_irq_enable - Enable default interrupt generation settings 192 * @adapter: board private structure 193 **/ 194 195static void 196ixgb_irq_enable(struct ixgb_adapter *adapter) 197{ 198 u32 val = IXGB_INT_RXT0 | IXGB_INT_RXDMT0 | 199 IXGB_INT_TXDW | IXGB_INT_LSC; 200 if (adapter->hw.subsystem_vendor_id == PCI_VENDOR_ID_SUN) 201 val |= IXGB_INT_GPI0; 202 IXGB_WRITE_REG(&adapter->hw, IMS, val); 203 IXGB_WRITE_FLUSH(&adapter->hw); 204} 205 206int 207ixgb_up(struct ixgb_adapter *adapter) 208{ 209 struct net_device *netdev = adapter->netdev; 210 int err, irq_flags = IRQF_SHARED; 211 int max_frame = netdev->mtu + ENET_HEADER_SIZE + ENET_FCS_LENGTH; 212 struct ixgb_hw *hw = &adapter->hw; 213 214 /* hardware has been reset, we need to reload some things */ 215 216 ixgb_rar_set(hw, netdev->dev_addr, 0); 217 ixgb_set_multi(netdev); 218 219 ixgb_restore_vlan(adapter); 220 221 ixgb_configure_tx(adapter); 222 ixgb_setup_rctl(adapter); 223 ixgb_configure_rx(adapter); 224 ixgb_alloc_rx_buffers(adapter, IXGB_DESC_UNUSED(&adapter->rx_ring)); 225 226 /* disable interrupts and get the hardware into a known state */ 227 IXGB_WRITE_REG(&adapter->hw, IMC, 0xffffffff); 228 229 /* only enable MSI if bus is in PCI-X mode */ 230 if (IXGB_READ_REG(&adapter->hw, STATUS) & IXGB_STATUS_PCIX_MODE) { 231 err = pci_enable_msi(adapter->pdev); 232 if (!err) { 233 adapter->have_msi = true; 234 irq_flags = 0; 235 } 236 /* proceed to try to request regular interrupt */ 237 } 238 239 err = request_irq(adapter->pdev->irq, ixgb_intr, irq_flags, 240 netdev->name, netdev); 241 if (err) { 242 if (adapter->have_msi) 243 pci_disable_msi(adapter->pdev); 244 netif_err(adapter, probe, adapter->netdev, 245 "Unable to allocate interrupt Error: %d\n", err); 246 return err; 247 } 248 249 if ((hw->max_frame_size != max_frame) || 250 (hw->max_frame_size != 251 (IXGB_READ_REG(hw, MFS) >> IXGB_MFS_SHIFT))) { 252 253 hw->max_frame_size = max_frame; 254 255 IXGB_WRITE_REG(hw, MFS, hw->max_frame_size << IXGB_MFS_SHIFT); 256 257 if (hw->max_frame_size > 258 IXGB_MAX_ENET_FRAME_SIZE_WITHOUT_FCS + ENET_FCS_LENGTH) { 259 u32 ctrl0 = IXGB_READ_REG(hw, CTRL0); 260 261 if (!(ctrl0 & IXGB_CTRL0_JFE)) { 262 ctrl0 |= IXGB_CTRL0_JFE; 263 IXGB_WRITE_REG(hw, CTRL0, ctrl0); 264 } 265 } 266 } 267 268 clear_bit(__IXGB_DOWN, &adapter->flags); 269 270 napi_enable(&adapter->napi); 271 ixgb_irq_enable(adapter); 272 273 netif_wake_queue(netdev); 274 275 mod_timer(&adapter->watchdog_timer, jiffies); 276 277 return 0; 278} 279 280void 281ixgb_down(struct ixgb_adapter *adapter, bool kill_watchdog) 282{ 283 struct net_device *netdev = adapter->netdev; 284 285 /* prevent the interrupt handler from restarting watchdog */ 286 set_bit(__IXGB_DOWN, &adapter->flags); 287 288 napi_disable(&adapter->napi); 289 /* waiting for NAPI to complete can re-enable interrupts */ 290 ixgb_irq_disable(adapter); 291 free_irq(adapter->pdev->irq, netdev); 292 293 if (adapter->have_msi) 294 pci_disable_msi(adapter->pdev); 295 296 if (kill_watchdog) 297 del_timer_sync(&adapter->watchdog_timer); 298 299 adapter->link_speed = 0; 300 adapter->link_duplex = 0; 301 netif_carrier_off(netdev); 302 netif_stop_queue(netdev); 303 304 ixgb_reset(adapter); 305 ixgb_clean_tx_ring(adapter); 306 ixgb_clean_rx_ring(adapter); 307} 308 309void 310ixgb_reset(struct ixgb_adapter *adapter) 311{ 312 struct ixgb_hw *hw = &adapter->hw; 313 314 ixgb_adapter_stop(hw); 315 if (!ixgb_init_hw(hw)) 316 netif_err(adapter, probe, adapter->netdev, "ixgb_init_hw failed\n"); 317 318 /* restore frame size information */ 319 IXGB_WRITE_REG(hw, MFS, hw->max_frame_size << IXGB_MFS_SHIFT); 320 if (hw->max_frame_size > 321 IXGB_MAX_ENET_FRAME_SIZE_WITHOUT_FCS + ENET_FCS_LENGTH) { 322 u32 ctrl0 = IXGB_READ_REG(hw, CTRL0); 323 if (!(ctrl0 & IXGB_CTRL0_JFE)) { 324 ctrl0 |= IXGB_CTRL0_JFE; 325 IXGB_WRITE_REG(hw, CTRL0, ctrl0); 326 } 327 } 328} 329 330static netdev_features_t 331ixgb_fix_features(struct net_device *netdev, netdev_features_t features) 332{ 333 /* 334 * Tx VLAN insertion does not work per HW design when Rx stripping is 335 * disabled. 336 */ 337 if (!(features & NETIF_F_HW_VLAN_CTAG_RX)) 338 features &= ~NETIF_F_HW_VLAN_CTAG_TX; 339 340 return features; 341} 342 343static int 344ixgb_set_features(struct net_device *netdev, netdev_features_t features) 345{ 346 struct ixgb_adapter *adapter = netdev_priv(netdev); 347 netdev_features_t changed = features ^ netdev->features; 348 349 if (!(changed & (NETIF_F_RXCSUM|NETIF_F_HW_VLAN_CTAG_RX))) 350 return 0; 351 352 adapter->rx_csum = !!(features & NETIF_F_RXCSUM); 353 354 if (netif_running(netdev)) { 355 ixgb_down(adapter, true); 356 ixgb_up(adapter); 357 ixgb_set_speed_duplex(netdev); 358 } else 359 ixgb_reset(adapter); 360 361 return 0; 362} 363 364 365static const struct net_device_ops ixgb_netdev_ops = { 366 .ndo_open = ixgb_open, 367 .ndo_stop = ixgb_close, 368 .ndo_start_xmit = ixgb_xmit_frame, 369 .ndo_get_stats = ixgb_get_stats, 370 .ndo_set_rx_mode = ixgb_set_multi, 371 .ndo_validate_addr = eth_validate_addr, 372 .ndo_set_mac_address = ixgb_set_mac, 373 .ndo_change_mtu = ixgb_change_mtu, 374 .ndo_tx_timeout = ixgb_tx_timeout, 375 .ndo_vlan_rx_add_vid = ixgb_vlan_rx_add_vid, 376 .ndo_vlan_rx_kill_vid = ixgb_vlan_rx_kill_vid, 377#ifdef CONFIG_NET_POLL_CONTROLLER 378 .ndo_poll_controller = ixgb_netpoll, 379#endif 380 .ndo_fix_features = ixgb_fix_features, 381 .ndo_set_features = ixgb_set_features, 382}; 383 384/** 385 * ixgb_probe - Device Initialization Routine 386 * @pdev: PCI device information struct 387 * @ent: entry in ixgb_pci_tbl 388 * 389 * Returns 0 on success, negative on failure 390 * 391 * ixgb_probe initializes an adapter identified by a pci_dev structure. 392 * The OS initialization, configuring of the adapter private structure, 393 * and a hardware reset occur. 394 **/ 395 396static int 397ixgb_probe(struct pci_dev *pdev, const struct pci_device_id *ent) 398{ 399 struct net_device *netdev = NULL; 400 struct ixgb_adapter *adapter; 401 static int cards_found = 0; 402 int pci_using_dac; 403 int i; 404 int err; 405 406 err = pci_enable_device(pdev); 407 if (err) 408 return err; 409 410 pci_using_dac = 0; 411 err = dma_set_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(64)); 412 if (!err) { 413 pci_using_dac = 1; 414 } else { 415 err = dma_set_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(32)); 416 if (err) { 417 pr_err("No usable DMA configuration, aborting\n"); 418 goto err_dma_mask; 419 } 420 } 421 422 err = pci_request_regions(pdev, ixgb_driver_name); 423 if (err) 424 goto err_request_regions; 425 426 pci_set_master(pdev); 427 428 netdev = alloc_etherdev(sizeof(struct ixgb_adapter)); 429 if (!netdev) { 430 err = -ENOMEM; 431 goto err_alloc_etherdev; 432 } 433 434 SET_NETDEV_DEV(netdev, &pdev->dev); 435 436 pci_set_drvdata(pdev, netdev); 437 adapter = netdev_priv(netdev); 438 adapter->netdev = netdev; 439 adapter->pdev = pdev; 440 adapter->hw.back = adapter; 441 adapter->msg_enable = netif_msg_init(debug, DEFAULT_MSG_ENABLE); 442 443 adapter->hw.hw_addr = pci_ioremap_bar(pdev, BAR_0); 444 if (!adapter->hw.hw_addr) { 445 err = -EIO; 446 goto err_ioremap; 447 } 448 449 for (i = BAR_1; i <= BAR_5; i++) { 450 if (pci_resource_len(pdev, i) == 0) 451 continue; 452 if (pci_resource_flags(pdev, i) & IORESOURCE_IO) { 453 adapter->hw.io_base = pci_resource_start(pdev, i); 454 break; 455 } 456 } 457 458 netdev->netdev_ops = &ixgb_netdev_ops; 459 ixgb_set_ethtool_ops(netdev); 460 netdev->watchdog_timeo = 5 * HZ; 461 netif_napi_add(netdev, &adapter->napi, ixgb_clean, 64); 462 463 strncpy(netdev->name, pci_name(pdev), sizeof(netdev->name) - 1); 464 465 adapter->bd_number = cards_found; 466 adapter->link_speed = 0; 467 adapter->link_duplex = 0; 468 469 /* setup the private structure */ 470 471 err = ixgb_sw_init(adapter); 472 if (err) 473 goto err_sw_init; 474 475 netdev->hw_features = NETIF_F_SG | 476 NETIF_F_TSO | 477 NETIF_F_HW_CSUM | 478 NETIF_F_HW_VLAN_CTAG_TX | 479 NETIF_F_HW_VLAN_CTAG_RX; 480 netdev->features = netdev->hw_features | 481 NETIF_F_HW_VLAN_CTAG_FILTER; 482 netdev->hw_features |= NETIF_F_RXCSUM; 483 484 if (pci_using_dac) { 485 netdev->features |= NETIF_F_HIGHDMA; 486 netdev->vlan_features |= NETIF_F_HIGHDMA; 487 } 488 489 /* make sure the EEPROM is good */ 490 491 if (!ixgb_validate_eeprom_checksum(&adapter->hw)) { 492 netif_err(adapter, probe, adapter->netdev, 493 "The EEPROM Checksum Is Not Valid\n"); 494 err = -EIO; 495 goto err_eeprom; 496 } 497 498 ixgb_get_ee_mac_addr(&adapter->hw, netdev->dev_addr); 499 500 if (!is_valid_ether_addr(netdev->dev_addr)) { 501 netif_err(adapter, probe, adapter->netdev, "Invalid MAC Address\n"); 502 err = -EIO; 503 goto err_eeprom; 504 } 505 506 adapter->part_num = ixgb_get_ee_pba_number(&adapter->hw); 507 508 init_timer(&adapter->watchdog_timer); 509 adapter->watchdog_timer.function = ixgb_watchdog; 510 adapter->watchdog_timer.data = (unsigned long)adapter; 511 512 INIT_WORK(&adapter->tx_timeout_task, ixgb_tx_timeout_task); 513 514 strcpy(netdev->name, "eth%d"); 515 err = register_netdev(netdev); 516 if (err) 517 goto err_register; 518 519 /* carrier off reporting is important to ethtool even BEFORE open */ 520 netif_carrier_off(netdev); 521 522 netif_info(adapter, probe, adapter->netdev, 523 "Intel(R) PRO/10GbE Network Connection\n"); 524 ixgb_check_options(adapter); 525 /* reset the hardware with the new settings */ 526 527 ixgb_reset(adapter); 528 529 cards_found++; 530 return 0; 531 532err_register: 533err_sw_init: 534err_eeprom: 535 iounmap(adapter->hw.hw_addr); 536err_ioremap: 537 free_netdev(netdev); 538err_alloc_etherdev: 539 pci_release_regions(pdev); 540err_request_regions: 541err_dma_mask: 542 pci_disable_device(pdev); 543 return err; 544} 545 546/** 547 * ixgb_remove - Device Removal Routine 548 * @pdev: PCI device information struct 549 * 550 * ixgb_remove is called by the PCI subsystem to alert the driver 551 * that it should release a PCI device. The could be caused by a 552 * Hot-Plug event, or because the driver is going to be removed from 553 * memory. 554 **/ 555 556static void 557ixgb_remove(struct pci_dev *pdev) 558{ 559 struct net_device *netdev = pci_get_drvdata(pdev); 560 struct ixgb_adapter *adapter = netdev_priv(netdev); 561 562 cancel_work_sync(&adapter->tx_timeout_task); 563 564 unregister_netdev(netdev); 565 566 iounmap(adapter->hw.hw_addr); 567 pci_release_regions(pdev); 568 569 free_netdev(netdev); 570 pci_disable_device(pdev); 571} 572 573/** 574 * ixgb_sw_init - Initialize general software structures (struct ixgb_adapter) 575 * @adapter: board private structure to initialize 576 * 577 * ixgb_sw_init initializes the Adapter private data structure. 578 * Fields are initialized based on PCI device information and 579 * OS network device settings (MTU size). 580 **/ 581 582static int 583ixgb_sw_init(struct ixgb_adapter *adapter) 584{ 585 struct ixgb_hw *hw = &adapter->hw; 586 struct net_device *netdev = adapter->netdev; 587 struct pci_dev *pdev = adapter->pdev; 588 589 /* PCI config space info */ 590 591 hw->vendor_id = pdev->vendor; 592 hw->device_id = pdev->device; 593 hw->subsystem_vendor_id = pdev->subsystem_vendor; 594 hw->subsystem_id = pdev->subsystem_device; 595 596 hw->max_frame_size = netdev->mtu + ENET_HEADER_SIZE + ENET_FCS_LENGTH; 597 adapter->rx_buffer_len = hw->max_frame_size + 8; /* + 8 for errata */ 598 599 if ((hw->device_id == IXGB_DEVICE_ID_82597EX) || 600 (hw->device_id == IXGB_DEVICE_ID_82597EX_CX4) || 601 (hw->device_id == IXGB_DEVICE_ID_82597EX_LR) || 602 (hw->device_id == IXGB_DEVICE_ID_82597EX_SR)) 603 hw->mac_type = ixgb_82597; 604 else { 605 /* should never have loaded on this device */ 606 netif_err(adapter, probe, adapter->netdev, "unsupported device id\n"); 607 } 608 609 /* enable flow control to be programmed */ 610 hw->fc.send_xon = 1; 611 612 set_bit(__IXGB_DOWN, &adapter->flags); 613 return 0; 614} 615 616/** 617 * ixgb_open - Called when a network interface is made active 618 * @netdev: network interface device structure 619 * 620 * Returns 0 on success, negative value on failure 621 * 622 * The open entry point is called when a network interface is made 623 * active by the system (IFF_UP). At this point all resources needed 624 * for transmit and receive operations are allocated, the interrupt 625 * handler is registered with the OS, the watchdog timer is started, 626 * and the stack is notified that the interface is ready. 627 **/ 628 629static int 630ixgb_open(struct net_device *netdev) 631{ 632 struct ixgb_adapter *adapter = netdev_priv(netdev); 633 int err; 634 635 /* allocate transmit descriptors */ 636 err = ixgb_setup_tx_resources(adapter); 637 if (err) 638 goto err_setup_tx; 639 640 netif_carrier_off(netdev); 641 642 /* allocate receive descriptors */ 643 644 err = ixgb_setup_rx_resources(adapter); 645 if (err) 646 goto err_setup_rx; 647 648 err = ixgb_up(adapter); 649 if (err) 650 goto err_up; 651 652 netif_start_queue(netdev); 653 654 return 0; 655 656err_up: 657 ixgb_free_rx_resources(adapter); 658err_setup_rx: 659 ixgb_free_tx_resources(adapter); 660err_setup_tx: 661 ixgb_reset(adapter); 662 663 return err; 664} 665 666/** 667 * ixgb_close - Disables a network interface 668 * @netdev: network interface device structure 669 * 670 * Returns 0, this is not allowed to fail 671 * 672 * The close entry point is called when an interface is de-activated 673 * by the OS. The hardware is still under the drivers control, but 674 * needs to be disabled. A global MAC reset is issued to stop the 675 * hardware, and all transmit and receive resources are freed. 676 **/ 677 678static int 679ixgb_close(struct net_device *netdev) 680{ 681 struct ixgb_adapter *adapter = netdev_priv(netdev); 682 683 ixgb_down(adapter, true); 684 685 ixgb_free_tx_resources(adapter); 686 ixgb_free_rx_resources(adapter); 687 688 return 0; 689} 690 691/** 692 * ixgb_setup_tx_resources - allocate Tx resources (Descriptors) 693 * @adapter: board private structure 694 * 695 * Return 0 on success, negative on failure 696 **/ 697 698int 699ixgb_setup_tx_resources(struct ixgb_adapter *adapter) 700{ 701 struct ixgb_desc_ring *txdr = &adapter->tx_ring; 702 struct pci_dev *pdev = adapter->pdev; 703 int size; 704 705 size = sizeof(struct ixgb_buffer) * txdr->count; 706 txdr->buffer_info = vzalloc(size); 707 if (!txdr->buffer_info) 708 return -ENOMEM; 709 710 /* round up to nearest 4K */ 711 712 txdr->size = txdr->count * sizeof(struct ixgb_tx_desc); 713 txdr->size = ALIGN(txdr->size, 4096); 714 715 txdr->desc = dma_zalloc_coherent(&pdev->dev, txdr->size, &txdr->dma, 716 GFP_KERNEL); 717 if (!txdr->desc) { 718 vfree(txdr->buffer_info); 719 return -ENOMEM; 720 } 721 722 txdr->next_to_use = 0; 723 txdr->next_to_clean = 0; 724 725 return 0; 726} 727 728/** 729 * ixgb_configure_tx - Configure 82597 Transmit Unit after Reset. 730 * @adapter: board private structure 731 * 732 * Configure the Tx unit of the MAC after a reset. 733 **/ 734 735static void 736ixgb_configure_tx(struct ixgb_adapter *adapter) 737{ 738 u64 tdba = adapter->tx_ring.dma; 739 u32 tdlen = adapter->tx_ring.count * sizeof(struct ixgb_tx_desc); 740 u32 tctl; 741 struct ixgb_hw *hw = &adapter->hw; 742 743 /* Setup the Base and Length of the Tx Descriptor Ring 744 * tx_ring.dma can be either a 32 or 64 bit value 745 */ 746 747 IXGB_WRITE_REG(hw, TDBAL, (tdba & 0x00000000ffffffffULL)); 748 IXGB_WRITE_REG(hw, TDBAH, (tdba >> 32)); 749 750 IXGB_WRITE_REG(hw, TDLEN, tdlen); 751 752 /* Setup the HW Tx Head and Tail descriptor pointers */ 753 754 IXGB_WRITE_REG(hw, TDH, 0); 755 IXGB_WRITE_REG(hw, TDT, 0); 756 757 /* don't set up txdctl, it induces performance problems if configured 758 * incorrectly */ 759 /* Set the Tx Interrupt Delay register */ 760 761 IXGB_WRITE_REG(hw, TIDV, adapter->tx_int_delay); 762 763 /* Program the Transmit Control Register */ 764 765 tctl = IXGB_TCTL_TCE | IXGB_TCTL_TXEN | IXGB_TCTL_TPDE; 766 IXGB_WRITE_REG(hw, TCTL, tctl); 767 768 /* Setup Transmit Descriptor Settings for this adapter */ 769 adapter->tx_cmd_type = 770 IXGB_TX_DESC_TYPE | 771 (adapter->tx_int_delay_enable ? IXGB_TX_DESC_CMD_IDE : 0); 772} 773 774/** 775 * ixgb_setup_rx_resources - allocate Rx resources (Descriptors) 776 * @adapter: board private structure 777 * 778 * Returns 0 on success, negative on failure 779 **/ 780 781int 782ixgb_setup_rx_resources(struct ixgb_adapter *adapter) 783{ 784 struct ixgb_desc_ring *rxdr = &adapter->rx_ring; 785 struct pci_dev *pdev = adapter->pdev; 786 int size; 787 788 size = sizeof(struct ixgb_buffer) * rxdr->count; 789 rxdr->buffer_info = vzalloc(size); 790 if (!rxdr->buffer_info) 791 return -ENOMEM; 792 793 /* Round up to nearest 4K */ 794 795 rxdr->size = rxdr->count * sizeof(struct ixgb_rx_desc); 796 rxdr->size = ALIGN(rxdr->size, 4096); 797 798 rxdr->desc = dma_alloc_coherent(&pdev->dev, rxdr->size, &rxdr->dma, 799 GFP_KERNEL); 800 801 if (!rxdr->desc) { 802 vfree(rxdr->buffer_info); 803 return -ENOMEM; 804 } 805 memset(rxdr->desc, 0, rxdr->size); 806 807 rxdr->next_to_clean = 0; 808 rxdr->next_to_use = 0; 809 810 return 0; 811} 812 813/** 814 * ixgb_setup_rctl - configure the receive control register 815 * @adapter: Board private structure 816 **/ 817 818static void 819ixgb_setup_rctl(struct ixgb_adapter *adapter) 820{ 821 u32 rctl; 822 823 rctl = IXGB_READ_REG(&adapter->hw, RCTL); 824 825 rctl &= ~(3 << IXGB_RCTL_MO_SHIFT); 826 827 rctl |= 828 IXGB_RCTL_BAM | IXGB_RCTL_RDMTS_1_2 | 829 IXGB_RCTL_RXEN | IXGB_RCTL_CFF | 830 (adapter->hw.mc_filter_type << IXGB_RCTL_MO_SHIFT); 831 832 rctl |= IXGB_RCTL_SECRC; 833 834 if (adapter->rx_buffer_len <= IXGB_RXBUFFER_2048) 835 rctl |= IXGB_RCTL_BSIZE_2048; 836 else if (adapter->rx_buffer_len <= IXGB_RXBUFFER_4096) 837 rctl |= IXGB_RCTL_BSIZE_4096; 838 else if (adapter->rx_buffer_len <= IXGB_RXBUFFER_8192) 839 rctl |= IXGB_RCTL_BSIZE_8192; 840 else if (adapter->rx_buffer_len <= IXGB_RXBUFFER_16384) 841 rctl |= IXGB_RCTL_BSIZE_16384; 842 843 IXGB_WRITE_REG(&adapter->hw, RCTL, rctl); 844} 845 846/** 847 * ixgb_configure_rx - Configure 82597 Receive Unit after Reset. 848 * @adapter: board private structure 849 * 850 * Configure the Rx unit of the MAC after a reset. 851 **/ 852 853static void 854ixgb_configure_rx(struct ixgb_adapter *adapter) 855{ 856 u64 rdba = adapter->rx_ring.dma; 857 u32 rdlen = adapter->rx_ring.count * sizeof(struct ixgb_rx_desc); 858 struct ixgb_hw *hw = &adapter->hw; 859 u32 rctl; 860 u32 rxcsum; 861 862 /* make sure receives are disabled while setting up the descriptors */ 863 864 rctl = IXGB_READ_REG(hw, RCTL); 865 IXGB_WRITE_REG(hw, RCTL, rctl & ~IXGB_RCTL_RXEN); 866 867 /* set the Receive Delay Timer Register */ 868 869 IXGB_WRITE_REG(hw, RDTR, adapter->rx_int_delay); 870 871 /* Setup the Base and Length of the Rx Descriptor Ring */ 872 873 IXGB_WRITE_REG(hw, RDBAL, (rdba & 0x00000000ffffffffULL)); 874 IXGB_WRITE_REG(hw, RDBAH, (rdba >> 32)); 875 876 IXGB_WRITE_REG(hw, RDLEN, rdlen); 877 878 /* Setup the HW Rx Head and Tail Descriptor Pointers */ 879 IXGB_WRITE_REG(hw, RDH, 0); 880 IXGB_WRITE_REG(hw, RDT, 0); 881 882 /* due to the hardware errata with RXDCTL, we are unable to use any of 883 * the performance enhancing features of it without causing other 884 * subtle bugs, some of the bugs could include receive length 885 * corruption at high data rates (WTHRESH > 0) and/or receive 886 * descriptor ring irregularites (particularly in hardware cache) */ 887 IXGB_WRITE_REG(hw, RXDCTL, 0); 888 889 /* Enable Receive Checksum Offload for TCP and UDP */ 890 if (adapter->rx_csum) { 891 rxcsum = IXGB_READ_REG(hw, RXCSUM); 892 rxcsum |= IXGB_RXCSUM_TUOFL; 893 IXGB_WRITE_REG(hw, RXCSUM, rxcsum); 894 } 895 896 /* Enable Receives */ 897 898 IXGB_WRITE_REG(hw, RCTL, rctl); 899} 900 901/** 902 * ixgb_free_tx_resources - Free Tx Resources 903 * @adapter: board private structure 904 * 905 * Free all transmit software resources 906 **/ 907 908void 909ixgb_free_tx_resources(struct ixgb_adapter *adapter) 910{ 911 struct pci_dev *pdev = adapter->pdev; 912 913 ixgb_clean_tx_ring(adapter); 914 915 vfree(adapter->tx_ring.buffer_info); 916 adapter->tx_ring.buffer_info = NULL; 917 918 dma_free_coherent(&pdev->dev, adapter->tx_ring.size, 919 adapter->tx_ring.desc, adapter->tx_ring.dma); 920 921 adapter->tx_ring.desc = NULL; 922} 923 924static void 925ixgb_unmap_and_free_tx_resource(struct ixgb_adapter *adapter, 926 struct ixgb_buffer *buffer_info) 927{ 928 if (buffer_info->dma) { 929 if (buffer_info->mapped_as_page) 930 dma_unmap_page(&adapter->pdev->dev, buffer_info->dma, 931 buffer_info->length, DMA_TO_DEVICE); 932 else 933 dma_unmap_single(&adapter->pdev->dev, buffer_info->dma, 934 buffer_info->length, DMA_TO_DEVICE); 935 buffer_info->dma = 0; 936 } 937 938 if (buffer_info->skb) { 939 dev_kfree_skb_any(buffer_info->skb); 940 buffer_info->skb = NULL; 941 } 942 buffer_info->time_stamp = 0; 943 /* these fields must always be initialized in tx 944 * buffer_info->length = 0; 945 * buffer_info->next_to_watch = 0; */ 946} 947 948/** 949 * ixgb_clean_tx_ring - Free Tx Buffers 950 * @adapter: board private structure 951 **/ 952 953static void 954ixgb_clean_tx_ring(struct ixgb_adapter *adapter) 955{ 956 struct ixgb_desc_ring *tx_ring = &adapter->tx_ring; 957 struct ixgb_buffer *buffer_info; 958 unsigned long size; 959 unsigned int i; 960 961 /* Free all the Tx ring sk_buffs */ 962 963 for (i = 0; i < tx_ring->count; i++) { 964 buffer_info = &tx_ring->buffer_info[i]; 965 ixgb_unmap_and_free_tx_resource(adapter, buffer_info); 966 } 967 968 size = sizeof(struct ixgb_buffer) * tx_ring->count; 969 memset(tx_ring->buffer_info, 0, size); 970 971 /* Zero out the descriptor ring */ 972 973 memset(tx_ring->desc, 0, tx_ring->size); 974 975 tx_ring->next_to_use = 0; 976 tx_ring->next_to_clean = 0; 977 978 IXGB_WRITE_REG(&adapter->hw, TDH, 0); 979 IXGB_WRITE_REG(&adapter->hw, TDT, 0); 980} 981 982/** 983 * ixgb_free_rx_resources - Free Rx Resources 984 * @adapter: board private structure 985 * 986 * Free all receive software resources 987 **/ 988 989void 990ixgb_free_rx_resources(struct ixgb_adapter *adapter) 991{ 992 struct ixgb_desc_ring *rx_ring = &adapter->rx_ring; 993 struct pci_dev *pdev = adapter->pdev; 994 995 ixgb_clean_rx_ring(adapter); 996 997 vfree(rx_ring->buffer_info); 998 rx_ring->buffer_info = NULL; 999 1000 dma_free_coherent(&pdev->dev, rx_ring->size, rx_ring->desc, 1001 rx_ring->dma); 1002 1003 rx_ring->desc = NULL; 1004} 1005 1006/** 1007 * ixgb_clean_rx_ring - Free Rx Buffers 1008 * @adapter: board private structure 1009 **/ 1010 1011static void 1012ixgb_clean_rx_ring(struct ixgb_adapter *adapter) 1013{ 1014 struct ixgb_desc_ring *rx_ring = &adapter->rx_ring; 1015 struct ixgb_buffer *buffer_info; 1016 struct pci_dev *pdev = adapter->pdev; 1017 unsigned long size; 1018 unsigned int i; 1019 1020 /* Free all the Rx ring sk_buffs */ 1021 1022 for (i = 0; i < rx_ring->count; i++) { 1023 buffer_info = &rx_ring->buffer_info[i]; 1024 if (buffer_info->dma) { 1025 dma_unmap_single(&pdev->dev, 1026 buffer_info->dma, 1027 buffer_info->length, 1028 DMA_FROM_DEVICE); 1029 buffer_info->dma = 0; 1030 buffer_info->length = 0; 1031 } 1032 1033 if (buffer_info->skb) { 1034 dev_kfree_skb(buffer_info->skb); 1035 buffer_info->skb = NULL; 1036 } 1037 } 1038 1039 size = sizeof(struct ixgb_buffer) * rx_ring->count; 1040 memset(rx_ring->buffer_info, 0, size); 1041 1042 /* Zero out the descriptor ring */ 1043 1044 memset(rx_ring->desc, 0, rx_ring->size); 1045 1046 rx_ring->next_to_clean = 0; 1047 rx_ring->next_to_use = 0; 1048 1049 IXGB_WRITE_REG(&adapter->hw, RDH, 0); 1050 IXGB_WRITE_REG(&adapter->hw, RDT, 0); 1051} 1052 1053/** 1054 * ixgb_set_mac - Change the Ethernet Address of the NIC 1055 * @netdev: network interface device structure 1056 * @p: pointer to an address structure 1057 * 1058 * Returns 0 on success, negative on failure 1059 **/ 1060 1061static int 1062ixgb_set_mac(struct net_device *netdev, void *p) 1063{ 1064 struct ixgb_adapter *adapter = netdev_priv(netdev); 1065 struct sockaddr *addr = p; 1066 1067 if (!is_valid_ether_addr(addr->sa_data)) 1068 return -EADDRNOTAVAIL; 1069 1070 memcpy(netdev->dev_addr, addr->sa_data, netdev->addr_len); 1071 1072 ixgb_rar_set(&adapter->hw, addr->sa_data, 0); 1073 1074 return 0; 1075} 1076 1077/** 1078 * ixgb_set_multi - Multicast and Promiscuous mode set 1079 * @netdev: network interface device structure 1080 * 1081 * The set_multi entry point is called whenever the multicast address 1082 * list or the network interface flags are updated. This routine is 1083 * responsible for configuring the hardware for proper multicast, 1084 * promiscuous mode, and all-multi behavior. 1085 **/ 1086 1087static void 1088ixgb_set_multi(struct net_device *netdev) 1089{ 1090 struct ixgb_adapter *adapter = netdev_priv(netdev); 1091 struct ixgb_hw *hw = &adapter->hw; 1092 struct netdev_hw_addr *ha; 1093 u32 rctl; 1094 1095 /* Check for Promiscuous and All Multicast modes */ 1096 1097 rctl = IXGB_READ_REG(hw, RCTL); 1098 1099 if (netdev->flags & IFF_PROMISC) { 1100 rctl |= (IXGB_RCTL_UPE | IXGB_RCTL_MPE); 1101 /* disable VLAN filtering */ 1102 rctl &= ~IXGB_RCTL_CFIEN; 1103 rctl &= ~IXGB_RCTL_VFE; 1104 } else { 1105 if (netdev->flags & IFF_ALLMULTI) { 1106 rctl |= IXGB_RCTL_MPE; 1107 rctl &= ~IXGB_RCTL_UPE; 1108 } else { 1109 rctl &= ~(IXGB_RCTL_UPE | IXGB_RCTL_MPE); 1110 } 1111 /* enable VLAN filtering */ 1112 rctl |= IXGB_RCTL_VFE; 1113 rctl &= ~IXGB_RCTL_CFIEN; 1114 } 1115 1116 if (netdev_mc_count(netdev) > IXGB_MAX_NUM_MULTICAST_ADDRESSES) { 1117 rctl |= IXGB_RCTL_MPE; 1118 IXGB_WRITE_REG(hw, RCTL, rctl); 1119 } else { 1120 u8 *mta = kmalloc(IXGB_MAX_NUM_MULTICAST_ADDRESSES * 1121 ETH_ALEN, GFP_ATOMIC); 1122 u8 *addr; 1123 if (!mta) 1124 goto alloc_failed; 1125 1126 IXGB_WRITE_REG(hw, RCTL, rctl); 1127 1128 addr = mta; 1129 netdev_for_each_mc_addr(ha, netdev) { 1130 memcpy(addr, ha->addr, ETH_ALEN); 1131 addr += ETH_ALEN; 1132 } 1133 1134 ixgb_mc_addr_list_update(hw, mta, netdev_mc_count(netdev), 0); 1135 kfree(mta); 1136 } 1137 1138alloc_failed: 1139 if (netdev->features & NETIF_F_HW_VLAN_CTAG_RX) 1140 ixgb_vlan_strip_enable(adapter); 1141 else 1142 ixgb_vlan_strip_disable(adapter); 1143 1144} 1145 1146/** 1147 * ixgb_watchdog - Timer Call-back 1148 * @data: pointer to netdev cast into an unsigned long 1149 **/ 1150 1151static void 1152ixgb_watchdog(unsigned long data) 1153{ 1154 struct ixgb_adapter *adapter = (struct ixgb_adapter *)data; 1155 struct net_device *netdev = adapter->netdev; 1156 struct ixgb_desc_ring *txdr = &adapter->tx_ring; 1157 1158 ixgb_check_for_link(&adapter->hw); 1159 1160 if (ixgb_check_for_bad_link(&adapter->hw)) { 1161 /* force the reset path */ 1162 netif_stop_queue(netdev); 1163 } 1164 1165 if (adapter->hw.link_up) { 1166 if (!netif_carrier_ok(netdev)) { 1167 netdev_info(netdev, 1168 "NIC Link is Up 10 Gbps Full Duplex, Flow Control: %s\n", 1169 (adapter->hw.fc.type == ixgb_fc_full) ? 1170 "RX/TX" : 1171 (adapter->hw.fc.type == ixgb_fc_rx_pause) ? 1172 "RX" : 1173 (adapter->hw.fc.type == ixgb_fc_tx_pause) ? 1174 "TX" : "None"); 1175 adapter->link_speed = 10000; 1176 adapter->link_duplex = FULL_DUPLEX; 1177 netif_carrier_on(netdev); 1178 } 1179 } else { 1180 if (netif_carrier_ok(netdev)) { 1181 adapter->link_speed = 0; 1182 adapter->link_duplex = 0; 1183 netdev_info(netdev, "NIC Link is Down\n"); 1184 netif_carrier_off(netdev); 1185 } 1186 } 1187 1188 ixgb_update_stats(adapter); 1189 1190 if (!netif_carrier_ok(netdev)) { 1191 if (IXGB_DESC_UNUSED(txdr) + 1 < txdr->count) { 1192 /* We've lost link, so the controller stops DMA, 1193 * but we've got queued Tx work that's never going 1194 * to get done, so reset controller to flush Tx. 1195 * (Do the reset outside of interrupt context). */ 1196 schedule_work(&adapter->tx_timeout_task); 1197 /* return immediately since reset is imminent */ 1198 return; 1199 } 1200 } 1201 1202 /* Force detection of hung controller every watchdog period */ 1203 adapter->detect_tx_hung = true; 1204 1205 /* generate an interrupt to force clean up of any stragglers */ 1206 IXGB_WRITE_REG(&adapter->hw, ICS, IXGB_INT_TXDW); 1207 1208 /* Reset the timer */ 1209 mod_timer(&adapter->watchdog_timer, jiffies + 2 * HZ); 1210} 1211 1212#define IXGB_TX_FLAGS_CSUM 0x00000001 1213#define IXGB_TX_FLAGS_VLAN 0x00000002 1214#define IXGB_TX_FLAGS_TSO 0x00000004 1215 1216static int 1217ixgb_tso(struct ixgb_adapter *adapter, struct sk_buff *skb) 1218{ 1219 struct ixgb_context_desc *context_desc; 1220 unsigned int i; 1221 u8 ipcss, ipcso, tucss, tucso, hdr_len; 1222 u16 ipcse, tucse, mss; 1223 int err; 1224 1225 if (likely(skb_is_gso(skb))) { 1226 struct ixgb_buffer *buffer_info; 1227 struct iphdr *iph; 1228 1229 if (skb_header_cloned(skb)) { 1230 err = pskb_expand_head(skb, 0, 0, GFP_ATOMIC); 1231 if (err) 1232 return err; 1233 } 1234 1235 hdr_len = skb_transport_offset(skb) + tcp_hdrlen(skb); 1236 mss = skb_shinfo(skb)->gso_size; 1237 iph = ip_hdr(skb); 1238 iph->tot_len = 0; 1239 iph->check = 0; 1240 tcp_hdr(skb)->check = ~csum_tcpudp_magic(iph->saddr, 1241 iph->daddr, 0, 1242 IPPROTO_TCP, 0); 1243 ipcss = skb_network_offset(skb); 1244 ipcso = (void *)&(iph->check) - (void *)skb->data; 1245 ipcse = skb_transport_offset(skb) - 1; 1246 tucss = skb_transport_offset(skb); 1247 tucso = (void *)&(tcp_hdr(skb)->check) - (void *)skb->data; 1248 tucse = 0; 1249 1250 i = adapter->tx_ring.next_to_use; 1251 context_desc = IXGB_CONTEXT_DESC(adapter->tx_ring, i); 1252 buffer_info = &adapter->tx_ring.buffer_info[i]; 1253 WARN_ON(buffer_info->dma != 0); 1254 1255 context_desc->ipcss = ipcss; 1256 context_desc->ipcso = ipcso; 1257 context_desc->ipcse = cpu_to_le16(ipcse); 1258 context_desc->tucss = tucss; 1259 context_desc->tucso = tucso; 1260 context_desc->tucse = cpu_to_le16(tucse); 1261 context_desc->mss = cpu_to_le16(mss); 1262 context_desc->hdr_len = hdr_len; 1263 context_desc->status = 0; 1264 context_desc->cmd_type_len = cpu_to_le32( 1265 IXGB_CONTEXT_DESC_TYPE 1266 | IXGB_CONTEXT_DESC_CMD_TSE 1267 | IXGB_CONTEXT_DESC_CMD_IP 1268 | IXGB_CONTEXT_DESC_CMD_TCP 1269 | IXGB_CONTEXT_DESC_CMD_IDE 1270 | (skb->len - (hdr_len))); 1271 1272 1273 if (++i == adapter->tx_ring.count) i = 0; 1274 adapter->tx_ring.next_to_use = i; 1275 1276 return 1; 1277 } 1278 1279 return 0; 1280} 1281 1282static bool 1283ixgb_tx_csum(struct ixgb_adapter *adapter, struct sk_buff *skb) 1284{ 1285 struct ixgb_context_desc *context_desc; 1286 unsigned int i; 1287 u8 css, cso; 1288 1289 if (likely(skb->ip_summed == CHECKSUM_PARTIAL)) { 1290 struct ixgb_buffer *buffer_info; 1291 css = skb_checksum_start_offset(skb); 1292 cso = css + skb->csum_offset; 1293 1294 i = adapter->tx_ring.next_to_use; 1295 context_desc = IXGB_CONTEXT_DESC(adapter->tx_ring, i); 1296 buffer_info = &adapter->tx_ring.buffer_info[i]; 1297 WARN_ON(buffer_info->dma != 0); 1298 1299 context_desc->tucss = css; 1300 context_desc->tucso = cso; 1301 context_desc->tucse = 0; 1302 /* zero out any previously existing data in one instruction */ 1303 *(u32 *)&(context_desc->ipcss) = 0; 1304 context_desc->status = 0; 1305 context_desc->hdr_len = 0; 1306 context_desc->mss = 0; 1307 context_desc->cmd_type_len = 1308 cpu_to_le32(IXGB_CONTEXT_DESC_TYPE 1309 | IXGB_TX_DESC_CMD_IDE); 1310 1311 if (++i == adapter->tx_ring.count) i = 0; 1312 adapter->tx_ring.next_to_use = i; 1313 1314 return true; 1315 } 1316 1317 return false; 1318} 1319 1320#define IXGB_MAX_TXD_PWR 14 1321#define IXGB_MAX_DATA_PER_TXD (1<<IXGB_MAX_TXD_PWR) 1322 1323static int 1324ixgb_tx_map(struct ixgb_adapter *adapter, struct sk_buff *skb, 1325 unsigned int first) 1326{ 1327 struct ixgb_desc_ring *tx_ring = &adapter->tx_ring; 1328 struct pci_dev *pdev = adapter->pdev; 1329 struct ixgb_buffer *buffer_info; 1330 int len = skb_headlen(skb); 1331 unsigned int offset = 0, size, count = 0, i; 1332 unsigned int mss = skb_shinfo(skb)->gso_size; 1333 unsigned int nr_frags = skb_shinfo(skb)->nr_frags; 1334 unsigned int f; 1335 1336 i = tx_ring->next_to_use; 1337 1338 while (len) { 1339 buffer_info = &tx_ring->buffer_info[i]; 1340 size = min(len, IXGB_MAX_DATA_PER_TXD); 1341 /* Workaround for premature desc write-backs 1342 * in TSO mode. Append 4-byte sentinel desc */ 1343 if (unlikely(mss && !nr_frags && size == len && size > 8)) 1344 size -= 4; 1345 1346 buffer_info->length = size; 1347 WARN_ON(buffer_info->dma != 0); 1348 buffer_info->time_stamp = jiffies; 1349 buffer_info->mapped_as_page = false; 1350 buffer_info->dma = dma_map_single(&pdev->dev, 1351 skb->data + offset, 1352 size, DMA_TO_DEVICE); 1353 if (dma_mapping_error(&pdev->dev, buffer_info->dma)) 1354 goto dma_error; 1355 buffer_info->next_to_watch = 0; 1356 1357 len -= size; 1358 offset += size; 1359 count++; 1360 if (len) { 1361 i++; 1362 if (i == tx_ring->count) 1363 i = 0; 1364 } 1365 } 1366 1367 for (f = 0; f < nr_frags; f++) { 1368 const struct skb_frag_struct *frag; 1369 1370 frag = &skb_shinfo(skb)->frags[f]; 1371 len = skb_frag_size(frag); 1372 offset = 0; 1373 1374 while (len) { 1375 i++; 1376 if (i == tx_ring->count) 1377 i = 0; 1378 1379 buffer_info = &tx_ring->buffer_info[i]; 1380 size = min(len, IXGB_MAX_DATA_PER_TXD); 1381 1382 /* Workaround for premature desc write-backs 1383 * in TSO mode. Append 4-byte sentinel desc */ 1384 if (unlikely(mss && (f == (nr_frags - 1)) 1385 && size == len && size > 8)) 1386 size -= 4; 1387 1388 buffer_info->length = size; 1389 buffer_info->time_stamp = jiffies; 1390 buffer_info->mapped_as_page = true; 1391 buffer_info->dma = 1392 skb_frag_dma_map(&pdev->dev, frag, offset, size, 1393 DMA_TO_DEVICE); 1394 if (dma_mapping_error(&pdev->dev, buffer_info->dma)) 1395 goto dma_error; 1396 buffer_info->next_to_watch = 0; 1397 1398 len -= size; 1399 offset += size; 1400 count++; 1401 } 1402 } 1403 tx_ring->buffer_info[i].skb = skb; 1404 tx_ring->buffer_info[first].next_to_watch = i; 1405 1406 return count; 1407 1408dma_error: 1409 dev_err(&pdev->dev, "TX DMA map failed\n"); 1410 buffer_info->dma = 0; 1411 if (count) 1412 count--; 1413 1414 while (count--) { 1415 if (i==0) 1416 i += tx_ring->count; 1417 i--; 1418 buffer_info = &tx_ring->buffer_info[i]; 1419 ixgb_unmap_and_free_tx_resource(adapter, buffer_info); 1420 } 1421 1422 return 0; 1423} 1424 1425static void 1426ixgb_tx_queue(struct ixgb_adapter *adapter, int count, int vlan_id,int tx_flags) 1427{ 1428 struct ixgb_desc_ring *tx_ring = &adapter->tx_ring; 1429 struct ixgb_tx_desc *tx_desc = NULL; 1430 struct ixgb_buffer *buffer_info; 1431 u32 cmd_type_len = adapter->tx_cmd_type; 1432 u8 status = 0; 1433 u8 popts = 0; 1434 unsigned int i; 1435 1436 if (tx_flags & IXGB_TX_FLAGS_TSO) { 1437 cmd_type_len |= IXGB_TX_DESC_CMD_TSE; 1438 popts |= (IXGB_TX_DESC_POPTS_IXSM | IXGB_TX_DESC_POPTS_TXSM); 1439 } 1440 1441 if (tx_flags & IXGB_TX_FLAGS_CSUM) 1442 popts |= IXGB_TX_DESC_POPTS_TXSM; 1443 1444 if (tx_flags & IXGB_TX_FLAGS_VLAN) 1445 cmd_type_len |= IXGB_TX_DESC_CMD_VLE; 1446 1447 i = tx_ring->next_to_use; 1448 1449 while (count--) { 1450 buffer_info = &tx_ring->buffer_info[i]; 1451 tx_desc = IXGB_TX_DESC(*tx_ring, i); 1452 tx_desc->buff_addr = cpu_to_le64(buffer_info->dma); 1453 tx_desc->cmd_type_len = 1454 cpu_to_le32(cmd_type_len | buffer_info->length); 1455 tx_desc->status = status; 1456 tx_desc->popts = popts; 1457 tx_desc->vlan = cpu_to_le16(vlan_id); 1458 1459 if (++i == tx_ring->count) i = 0; 1460 } 1461 1462 tx_desc->cmd_type_len |= 1463 cpu_to_le32(IXGB_TX_DESC_CMD_EOP | IXGB_TX_DESC_CMD_RS); 1464 1465 /* Force memory writes to complete before letting h/w 1466 * know there are new descriptors to fetch. (Only 1467 * applicable for weak-ordered memory model archs, 1468 * such as IA-64). */ 1469 wmb(); 1470 1471 tx_ring->next_to_use = i; 1472 IXGB_WRITE_REG(&adapter->hw, TDT, i); 1473} 1474 1475static int __ixgb_maybe_stop_tx(struct net_device *netdev, int size) 1476{ 1477 struct ixgb_adapter *adapter = netdev_priv(netdev); 1478 struct ixgb_desc_ring *tx_ring = &adapter->tx_ring; 1479 1480 netif_stop_queue(netdev); 1481 /* Herbert's original patch had: 1482 * smp_mb__after_netif_stop_queue(); 1483 * but since that doesn't exist yet, just open code it. */ 1484 smp_mb(); 1485 1486 /* We need to check again in a case another CPU has just 1487 * made room available. */ 1488 if (likely(IXGB_DESC_UNUSED(tx_ring) < size)) 1489 return -EBUSY; 1490 1491 /* A reprieve! */ 1492 netif_start_queue(netdev); 1493 ++adapter->restart_queue; 1494 return 0; 1495} 1496 1497static int ixgb_maybe_stop_tx(struct net_device *netdev, 1498 struct ixgb_desc_ring *tx_ring, int size) 1499{ 1500 if (likely(IXGB_DESC_UNUSED(tx_ring) >= size)) 1501 return 0; 1502 return __ixgb_maybe_stop_tx(netdev, size); 1503} 1504 1505 1506/* Tx Descriptors needed, worst case */ 1507#define TXD_USE_COUNT(S) (((S) >> IXGB_MAX_TXD_PWR) + \ 1508 (((S) & (IXGB_MAX_DATA_PER_TXD - 1)) ? 1 : 0)) 1509#define DESC_NEEDED TXD_USE_COUNT(IXGB_MAX_DATA_PER_TXD) /* skb->date */ + \ 1510 MAX_SKB_FRAGS * TXD_USE_COUNT(PAGE_SIZE) + 1 /* for context */ \ 1511 + 1 /* one more needed for sentinel TSO workaround */ 1512 1513static netdev_tx_t 1514ixgb_xmit_frame(struct sk_buff *skb, struct net_device *netdev) 1515{ 1516 struct ixgb_adapter *adapter = netdev_priv(netdev); 1517 unsigned int first; 1518 unsigned int tx_flags = 0; 1519 int vlan_id = 0; 1520 int count = 0; 1521 int tso; 1522 1523 if (test_bit(__IXGB_DOWN, &adapter->flags)) { 1524 dev_kfree_skb(skb); 1525 return NETDEV_TX_OK; 1526 } 1527 1528 if (skb->len <= 0) { 1529 dev_kfree_skb(skb); 1530 return NETDEV_TX_OK; 1531 } 1532 1533 if (unlikely(ixgb_maybe_stop_tx(netdev, &adapter->tx_ring, 1534 DESC_NEEDED))) 1535 return NETDEV_TX_BUSY; 1536 1537 if (vlan_tx_tag_present(skb)) { 1538 tx_flags |= IXGB_TX_FLAGS_VLAN; 1539 vlan_id = vlan_tx_tag_get(skb); 1540 } 1541 1542 first = adapter->tx_ring.next_to_use; 1543 1544 tso = ixgb_tso(adapter, skb); 1545 if (tso < 0) { 1546 dev_kfree_skb(skb); 1547 return NETDEV_TX_OK; 1548 } 1549 1550 if (likely(tso)) 1551 tx_flags |= IXGB_TX_FLAGS_TSO; 1552 else if (ixgb_tx_csum(adapter, skb)) 1553 tx_flags |= IXGB_TX_FLAGS_CSUM; 1554 1555 count = ixgb_tx_map(adapter, skb, first); 1556 1557 if (count) { 1558 ixgb_tx_queue(adapter, count, vlan_id, tx_flags); 1559 /* Make sure there is space in the ring for the next send. */ 1560 ixgb_maybe_stop_tx(netdev, &adapter->tx_ring, DESC_NEEDED); 1561 1562 } else { 1563 dev_kfree_skb_any(skb); 1564 adapter->tx_ring.buffer_info[first].time_stamp = 0; 1565 adapter->tx_ring.next_to_use = first; 1566 } 1567 1568 return NETDEV_TX_OK; 1569} 1570 1571/** 1572 * ixgb_tx_timeout - Respond to a Tx Hang 1573 * @netdev: network interface device structure 1574 **/ 1575 1576static void 1577ixgb_tx_timeout(struct net_device *netdev) 1578{ 1579 struct ixgb_adapter *adapter = netdev_priv(netdev); 1580 1581 /* Do the reset outside of interrupt context */ 1582 schedule_work(&adapter->tx_timeout_task); 1583} 1584 1585static void 1586ixgb_tx_timeout_task(struct work_struct *work) 1587{ 1588 struct ixgb_adapter *adapter = 1589 container_of(work, struct ixgb_adapter, tx_timeout_task); 1590 1591 adapter->tx_timeout_count++; 1592 ixgb_down(adapter, true); 1593 ixgb_up(adapter); 1594} 1595 1596/** 1597 * ixgb_get_stats - Get System Network Statistics 1598 * @netdev: network interface device structure 1599 * 1600 * Returns the address of the device statistics structure. 1601 * The statistics are actually updated from the timer callback. 1602 **/ 1603 1604static struct net_device_stats * 1605ixgb_get_stats(struct net_device *netdev) 1606{ 1607 return &netdev->stats; 1608} 1609 1610/** 1611 * ixgb_change_mtu - Change the Maximum Transfer Unit 1612 * @netdev: network interface device structure 1613 * @new_mtu: new value for maximum frame size 1614 * 1615 * Returns 0 on success, negative on failure 1616 **/ 1617 1618static int 1619ixgb_change_mtu(struct net_device *netdev, int new_mtu) 1620{ 1621 struct ixgb_adapter *adapter = netdev_priv(netdev); 1622 int max_frame = new_mtu + ENET_HEADER_SIZE + ENET_FCS_LENGTH; 1623 int old_max_frame = netdev->mtu + ENET_HEADER_SIZE + ENET_FCS_LENGTH; 1624 1625 /* MTU < 68 is an error for IPv4 traffic, just don't allow it */ 1626 if ((new_mtu < 68) || 1627 (max_frame > IXGB_MAX_JUMBO_FRAME_SIZE + ENET_FCS_LENGTH)) { 1628 netif_err(adapter, probe, adapter->netdev, 1629 "Invalid MTU setting %d\n", new_mtu); 1630 return -EINVAL; 1631 } 1632 1633 if (old_max_frame == max_frame) 1634 return 0; 1635 1636 if (netif_running(netdev)) 1637 ixgb_down(adapter, true); 1638 1639 adapter->rx_buffer_len = max_frame + 8; /* + 8 for errata */ 1640 1641 netdev->mtu = new_mtu; 1642 1643 if (netif_running(netdev)) 1644 ixgb_up(adapter); 1645 1646 return 0; 1647} 1648 1649/** 1650 * ixgb_update_stats - Update the board statistics counters. 1651 * @adapter: board private structure 1652 **/ 1653 1654void 1655ixgb_update_stats(struct ixgb_adapter *adapter) 1656{ 1657 struct net_device *netdev = adapter->netdev; 1658 struct pci_dev *pdev = adapter->pdev; 1659 1660 /* Prevent stats update while adapter is being reset */ 1661 if (pci_channel_offline(pdev)) 1662 return; 1663 1664 if ((netdev->flags & IFF_PROMISC) || (netdev->flags & IFF_ALLMULTI) || 1665 (netdev_mc_count(netdev) > IXGB_MAX_NUM_MULTICAST_ADDRESSES)) { 1666 u64 multi = IXGB_READ_REG(&adapter->hw, MPRCL); 1667 u32 bcast_l = IXGB_READ_REG(&adapter->hw, BPRCL); 1668 u32 bcast_h = IXGB_READ_REG(&adapter->hw, BPRCH); 1669 u64 bcast = ((u64)bcast_h << 32) | bcast_l; 1670 1671 multi |= ((u64)IXGB_READ_REG(&adapter->hw, MPRCH) << 32); 1672 /* fix up multicast stats by removing broadcasts */ 1673 if (multi >= bcast) 1674 multi -= bcast; 1675 1676 adapter->stats.mprcl += (multi & 0xFFFFFFFF); 1677 adapter->stats.mprch += (multi >> 32); 1678 adapter->stats.bprcl += bcast_l; 1679 adapter->stats.bprch += bcast_h; 1680 } else { 1681 adapter->stats.mprcl += IXGB_READ_REG(&adapter->hw, MPRCL); 1682 adapter->stats.mprch += IXGB_READ_REG(&adapter->hw, MPRCH); 1683 adapter->stats.bprcl += IXGB_READ_REG(&adapter->hw, BPRCL); 1684 adapter->stats.bprch += IXGB_READ_REG(&adapter->hw, BPRCH); 1685 } 1686 adapter->stats.tprl += IXGB_READ_REG(&adapter->hw, TPRL); 1687 adapter->stats.tprh += IXGB_READ_REG(&adapter->hw, TPRH); 1688 adapter->stats.gprcl += IXGB_READ_REG(&adapter->hw, GPRCL); 1689 adapter->stats.gprch += IXGB_READ_REG(&adapter->hw, GPRCH); 1690 adapter->stats.uprcl += IXGB_READ_REG(&adapter->hw, UPRCL); 1691 adapter->stats.uprch += IXGB_READ_REG(&adapter->hw, UPRCH); 1692 adapter->stats.vprcl += IXGB_READ_REG(&adapter->hw, VPRCL); 1693 adapter->stats.vprch += IXGB_READ_REG(&adapter->hw, VPRCH); 1694 adapter->stats.jprcl += IXGB_READ_REG(&adapter->hw, JPRCL); 1695 adapter->stats.jprch += IXGB_READ_REG(&adapter->hw, JPRCH); 1696 adapter->stats.gorcl += IXGB_READ_REG(&adapter->hw, GORCL); 1697 adapter->stats.gorch += IXGB_READ_REG(&adapter->hw, GORCH); 1698 adapter->stats.torl += IXGB_READ_REG(&adapter->hw, TORL); 1699 adapter->stats.torh += IXGB_READ_REG(&adapter->hw, TORH); 1700 adapter->stats.rnbc += IXGB_READ_REG(&adapter->hw, RNBC); 1701 adapter->stats.ruc += IXGB_READ_REG(&adapter->hw, RUC); 1702 adapter->stats.roc += IXGB_READ_REG(&adapter->hw, ROC); 1703 adapter->stats.rlec += IXGB_READ_REG(&adapter->hw, RLEC); 1704 adapter->stats.crcerrs += IXGB_READ_REG(&adapter->hw, CRCERRS); 1705 adapter->stats.icbc += IXGB_READ_REG(&adapter->hw, ICBC); 1706 adapter->stats.ecbc += IXGB_READ_REG(&adapter->hw, ECBC); 1707 adapter->stats.mpc += IXGB_READ_REG(&adapter->hw, MPC); 1708 adapter->stats.tptl += IXGB_READ_REG(&adapter->hw, TPTL); 1709 adapter->stats.tpth += IXGB_READ_REG(&adapter->hw, TPTH); 1710 adapter->stats.gptcl += IXGB_READ_REG(&adapter->hw, GPTCL); 1711 adapter->stats.gptch += IXGB_READ_REG(&adapter->hw, GPTCH); 1712 adapter->stats.bptcl += IXGB_READ_REG(&adapter->hw, BPTCL); 1713 adapter->stats.bptch += IXGB_READ_REG(&adapter->hw, BPTCH); 1714 adapter->stats.mptcl += IXGB_READ_REG(&adapter->hw, MPTCL); 1715 adapter->stats.mptch += IXGB_READ_REG(&adapter->hw, MPTCH); 1716 adapter->stats.uptcl += IXGB_READ_REG(&adapter->hw, UPTCL); 1717 adapter->stats.uptch += IXGB_READ_REG(&adapter->hw, UPTCH); 1718 adapter->stats.vptcl += IXGB_READ_REG(&adapter->hw, VPTCL); 1719 adapter->stats.vptch += IXGB_READ_REG(&adapter->hw, VPTCH); 1720 adapter->stats.jptcl += IXGB_READ_REG(&adapter->hw, JPTCL); 1721 adapter->stats.jptch += IXGB_READ_REG(&adapter->hw, JPTCH); 1722 adapter->stats.gotcl += IXGB_READ_REG(&adapter->hw, GOTCL); 1723 adapter->stats.gotch += IXGB_READ_REG(&adapter->hw, GOTCH); 1724 adapter->stats.totl += IXGB_READ_REG(&adapter->hw, TOTL); 1725 adapter->stats.toth += IXGB_READ_REG(&adapter->hw, TOTH); 1726 adapter->stats.dc += IXGB_READ_REG(&adapter->hw, DC); 1727 adapter->stats.plt64c += IXGB_READ_REG(&adapter->hw, PLT64C); 1728 adapter->stats.tsctc += IXGB_READ_REG(&adapter->hw, TSCTC); 1729 adapter->stats.tsctfc += IXGB_READ_REG(&adapter->hw, TSCTFC); 1730 adapter->stats.ibic += IXGB_READ_REG(&adapter->hw, IBIC); 1731 adapter->stats.rfc += IXGB_READ_REG(&adapter->hw, RFC); 1732 adapter->stats.lfc += IXGB_READ_REG(&adapter->hw, LFC); 1733 adapter->stats.pfrc += IXGB_READ_REG(&adapter->hw, PFRC); 1734 adapter->stats.pftc += IXGB_READ_REG(&adapter->hw, PFTC); 1735 adapter->stats.mcfrc += IXGB_READ_REG(&adapter->hw, MCFRC); 1736 adapter->stats.mcftc += IXGB_READ_REG(&adapter->hw, MCFTC); 1737 adapter->stats.xonrxc += IXGB_READ_REG(&adapter->hw, XONRXC); 1738 adapter->stats.xontxc += IXGB_READ_REG(&adapter->hw, XONTXC); 1739 adapter->stats.xoffrxc += IXGB_READ_REG(&adapter->hw, XOFFRXC); 1740 adapter->stats.xofftxc += IXGB_READ_REG(&adapter->hw, XOFFTXC); 1741 adapter->stats.rjc += IXGB_READ_REG(&adapter->hw, RJC); 1742 1743 /* Fill out the OS statistics structure */ 1744 1745 netdev->stats.rx_packets = adapter->stats.gprcl; 1746 netdev->stats.tx_packets = adapter->stats.gptcl; 1747 netdev->stats.rx_bytes = adapter->stats.gorcl; 1748 netdev->stats.tx_bytes = adapter->stats.gotcl; 1749 netdev->stats.multicast = adapter->stats.mprcl; 1750 netdev->stats.collisions = 0; 1751 1752 /* ignore RLEC as it reports errors for padded (<64bytes) frames 1753 * with a length in the type/len field */ 1754 netdev->stats.rx_errors = 1755 /* adapter->stats.rnbc + */ adapter->stats.crcerrs + 1756 adapter->stats.ruc + 1757 adapter->stats.roc /*+ adapter->stats.rlec */ + 1758 adapter->stats.icbc + 1759 adapter->stats.ecbc + adapter->stats.mpc; 1760 1761 /* see above 1762 * netdev->stats.rx_length_errors = adapter->stats.rlec; 1763 */ 1764 1765 netdev->stats.rx_crc_errors = adapter->stats.crcerrs; 1766 netdev->stats.rx_fifo_errors = adapter->stats.mpc; 1767 netdev->stats.rx_missed_errors = adapter->stats.mpc; 1768 netdev->stats.rx_over_errors = adapter->stats.mpc; 1769 1770 netdev->stats.tx_errors = 0; 1771 netdev->stats.rx_frame_errors = 0; 1772 netdev->stats.tx_aborted_errors = 0; 1773 netdev->stats.tx_carrier_errors = 0; 1774 netdev->stats.tx_fifo_errors = 0; 1775 netdev->stats.tx_heartbeat_errors = 0; 1776 netdev->stats.tx_window_errors = 0; 1777} 1778 1779#define IXGB_MAX_INTR 10 1780/** 1781 * ixgb_intr - Interrupt Handler 1782 * @irq: interrupt number 1783 * @data: pointer to a network interface device structure 1784 **/ 1785 1786static irqreturn_t 1787ixgb_intr(int irq, void *data) 1788{ 1789 struct net_device *netdev = data; 1790 struct ixgb_adapter *adapter = netdev_priv(netdev); 1791 struct ixgb_hw *hw = &adapter->hw; 1792 u32 icr = IXGB_READ_REG(hw, ICR); 1793 1794 if (unlikely(!icr)) 1795 return IRQ_NONE; /* Not our interrupt */ 1796 1797 if (unlikely(icr & (IXGB_INT_RXSEQ | IXGB_INT_LSC))) 1798 if (!test_bit(__IXGB_DOWN, &adapter->flags)) 1799 mod_timer(&adapter->watchdog_timer, jiffies); 1800 1801 if (napi_schedule_prep(&adapter->napi)) { 1802 1803 /* Disable interrupts and register for poll. The flush 1804 of the posted write is intentionally left out. 1805 */ 1806 1807 IXGB_WRITE_REG(&adapter->hw, IMC, ~0); 1808 __napi_schedule(&adapter->napi); 1809 } 1810 return IRQ_HANDLED; 1811} 1812 1813/** 1814 * ixgb_clean - NAPI Rx polling callback 1815 * @adapter: board private structure 1816 **/ 1817 1818static int 1819ixgb_clean(struct napi_struct *napi, int budget) 1820{ 1821 struct ixgb_adapter *adapter = container_of(napi, struct ixgb_adapter, napi); 1822 int work_done = 0; 1823 1824 ixgb_clean_tx_irq(adapter); 1825 ixgb_clean_rx_irq(adapter, &work_done, budget); 1826 1827 /* If budget not fully consumed, exit the polling mode */ 1828 if (work_done < budget) { 1829 napi_complete(napi); 1830 if (!test_bit(__IXGB_DOWN, &adapter->flags)) 1831 ixgb_irq_enable(adapter); 1832 } 1833 1834 return work_done; 1835} 1836 1837/** 1838 * ixgb_clean_tx_irq - Reclaim resources after transmit completes 1839 * @adapter: board private structure 1840 **/ 1841 1842static bool 1843ixgb_clean_tx_irq(struct ixgb_adapter *adapter) 1844{ 1845 struct ixgb_desc_ring *tx_ring = &adapter->tx_ring; 1846 struct net_device *netdev = adapter->netdev; 1847 struct ixgb_tx_desc *tx_desc, *eop_desc; 1848 struct ixgb_buffer *buffer_info; 1849 unsigned int i, eop; 1850 bool cleaned = false; 1851 1852 i = tx_ring->next_to_clean; 1853 eop = tx_ring->buffer_info[i].next_to_watch; 1854 eop_desc = IXGB_TX_DESC(*tx_ring, eop); 1855 1856 while (eop_desc->status & IXGB_TX_DESC_STATUS_DD) { 1857 1858 rmb(); /* read buffer_info after eop_desc */ 1859 for (cleaned = false; !cleaned; ) { 1860 tx_desc = IXGB_TX_DESC(*tx_ring, i); 1861 buffer_info = &tx_ring->buffer_info[i]; 1862 1863 if (tx_desc->popts & 1864 (IXGB_TX_DESC_POPTS_TXSM | 1865 IXGB_TX_DESC_POPTS_IXSM)) 1866 adapter->hw_csum_tx_good++; 1867 1868 ixgb_unmap_and_free_tx_resource(adapter, buffer_info); 1869 1870 *(u32 *)&(tx_desc->status) = 0; 1871 1872 cleaned = (i == eop); 1873 if (++i == tx_ring->count) i = 0; 1874 } 1875 1876 eop = tx_ring->buffer_info[i].next_to_watch; 1877 eop_desc = IXGB_TX_DESC(*tx_ring, eop); 1878 } 1879 1880 tx_ring->next_to_clean = i; 1881 1882 if (unlikely(cleaned && netif_carrier_ok(netdev) && 1883 IXGB_DESC_UNUSED(tx_ring) >= DESC_NEEDED)) { 1884 /* Make sure that anybody stopping the queue after this 1885 * sees the new next_to_clean. */ 1886 smp_mb(); 1887 1888 if (netif_queue_stopped(netdev) && 1889 !(test_bit(__IXGB_DOWN, &adapter->flags))) { 1890 netif_wake_queue(netdev); 1891 ++adapter->restart_queue; 1892 } 1893 } 1894 1895 if (adapter->detect_tx_hung) { 1896 /* detect a transmit hang in hardware, this serializes the 1897 * check with the clearing of time_stamp and movement of i */ 1898 adapter->detect_tx_hung = false; 1899 if (tx_ring->buffer_info[eop].time_stamp && 1900 time_after(jiffies, tx_ring->buffer_info[eop].time_stamp + HZ) 1901 && !(IXGB_READ_REG(&adapter->hw, STATUS) & 1902 IXGB_STATUS_TXOFF)) { 1903 /* detected Tx unit hang */ 1904 netif_err(adapter, drv, adapter->netdev, 1905 "Detected Tx Unit Hang\n" 1906 " TDH <%x>\n" 1907 " TDT <%x>\n" 1908 " next_to_use <%x>\n" 1909 " next_to_clean <%x>\n" 1910 "buffer_info[next_to_clean]\n" 1911 " time_stamp <%lx>\n" 1912 " next_to_watch <%x>\n" 1913 " jiffies <%lx>\n" 1914 " next_to_watch.status <%x>\n", 1915 IXGB_READ_REG(&adapter->hw, TDH), 1916 IXGB_READ_REG(&adapter->hw, TDT), 1917 tx_ring->next_to_use, 1918 tx_ring->next_to_clean, 1919 tx_ring->buffer_info[eop].time_stamp, 1920 eop, 1921 jiffies, 1922 eop_desc->status); 1923 netif_stop_queue(netdev); 1924 } 1925 } 1926 1927 return cleaned; 1928} 1929 1930/** 1931 * ixgb_rx_checksum - Receive Checksum Offload for 82597. 1932 * @adapter: board private structure 1933 * @rx_desc: receive descriptor 1934 * @sk_buff: socket buffer with received data 1935 **/ 1936 1937static void 1938ixgb_rx_checksum(struct ixgb_adapter *adapter, 1939 struct ixgb_rx_desc *rx_desc, 1940 struct sk_buff *skb) 1941{ 1942 /* Ignore Checksum bit is set OR 1943 * TCP Checksum has not been calculated 1944 */ 1945 if ((rx_desc->status & IXGB_RX_DESC_STATUS_IXSM) || 1946 (!(rx_desc->status & IXGB_RX_DESC_STATUS_TCPCS))) { 1947 skb_checksum_none_assert(skb); 1948 return; 1949 } 1950 1951 /* At this point we know the hardware did the TCP checksum */ 1952 /* now look at the TCP checksum error bit */ 1953 if (rx_desc->errors & IXGB_RX_DESC_ERRORS_TCPE) { 1954 /* let the stack verify checksum errors */ 1955 skb_checksum_none_assert(skb); 1956 adapter->hw_csum_rx_error++; 1957 } else { 1958 /* TCP checksum is good */ 1959 skb->ip_summed = CHECKSUM_UNNECESSARY; 1960 adapter->hw_csum_rx_good++; 1961 } 1962} 1963 1964/* 1965 * this should improve performance for small packets with large amounts 1966 * of reassembly being done in the stack 1967 */ 1968static void ixgb_check_copybreak(struct net_device *netdev, 1969 struct ixgb_buffer *buffer_info, 1970 u32 length, struct sk_buff **skb) 1971{ 1972 struct sk_buff *new_skb; 1973 1974 if (length > copybreak) 1975 return; 1976 1977 new_skb = netdev_alloc_skb_ip_align(netdev, length); 1978 if (!new_skb) 1979 return; 1980 1981 skb_copy_to_linear_data_offset(new_skb, -NET_IP_ALIGN, 1982 (*skb)->data - NET_IP_ALIGN, 1983 length + NET_IP_ALIGN); 1984 /* save the skb in buffer_info as good */ 1985 buffer_info->skb = *skb; 1986 *skb = new_skb; 1987} 1988 1989/** 1990 * ixgb_clean_rx_irq - Send received data up the network stack, 1991 * @adapter: board private structure 1992 **/ 1993 1994static bool 1995ixgb_clean_rx_irq(struct ixgb_adapter *adapter, int *work_done, int work_to_do) 1996{ 1997 struct ixgb_desc_ring *rx_ring = &adapter->rx_ring; 1998 struct net_device *netdev = adapter->netdev; 1999 struct pci_dev *pdev = adapter->pdev; 2000 struct ixgb_rx_desc *rx_desc, *next_rxd; 2001 struct ixgb_buffer *buffer_info, *next_buffer, *next2_buffer; 2002 u32 length; 2003 unsigned int i, j; 2004 int cleaned_count = 0; 2005 bool cleaned = false; 2006 2007 i = rx_ring->next_to_clean; 2008 rx_desc = IXGB_RX_DESC(*rx_ring, i); 2009 buffer_info = &rx_ring->buffer_info[i]; 2010 2011 while (rx_desc->status & IXGB_RX_DESC_STATUS_DD) { 2012 struct sk_buff *skb; 2013 u8 status; 2014 2015 if (*work_done >= work_to_do) 2016 break; 2017 2018 (*work_done)++; 2019 rmb(); /* read descriptor and rx_buffer_info after status DD */ 2020 status = rx_desc->status; 2021 skb = buffer_info->skb; 2022 buffer_info->skb = NULL; 2023 2024 prefetch(skb->data - NET_IP_ALIGN); 2025 2026 if (++i == rx_ring->count) 2027 i = 0; 2028 next_rxd = IXGB_RX_DESC(*rx_ring, i); 2029 prefetch(next_rxd); 2030 2031 j = i + 1; 2032 if (j == rx_ring->count) 2033 j = 0; 2034 next2_buffer = &rx_ring->buffer_info[j]; 2035 prefetch(next2_buffer); 2036 2037 next_buffer = &rx_ring->buffer_info[i]; 2038 2039 cleaned = true; 2040 cleaned_count++; 2041 2042 dma_unmap_single(&pdev->dev, 2043 buffer_info->dma, 2044 buffer_info->length, 2045 DMA_FROM_DEVICE); 2046 buffer_info->dma = 0; 2047 2048 length = le16_to_cpu(rx_desc->length); 2049 rx_desc->length = 0; 2050 2051 if (unlikely(!(status & IXGB_RX_DESC_STATUS_EOP))) { 2052 2053 /* All receives must fit into a single buffer */ 2054 2055 pr_debug("Receive packet consumed multiple buffers length<%x>\n", 2056 length); 2057 2058 dev_kfree_skb_irq(skb); 2059 goto rxdesc_done; 2060 } 2061 2062 if (unlikely(rx_desc->errors & 2063 (IXGB_RX_DESC_ERRORS_CE | IXGB_RX_DESC_ERRORS_SE | 2064 IXGB_RX_DESC_ERRORS_P | IXGB_RX_DESC_ERRORS_RXE))) { 2065 dev_kfree_skb_irq(skb); 2066 goto rxdesc_done; 2067 } 2068 2069 ixgb_check_copybreak(netdev, buffer_info, length, &skb); 2070 2071 /* Good Receive */ 2072 skb_put(skb, length); 2073 2074 /* Receive Checksum Offload */ 2075 ixgb_rx_checksum(adapter, rx_desc, skb); 2076 2077 skb->protocol = eth_type_trans(skb, netdev); 2078 if (status & IXGB_RX_DESC_STATUS_VP) 2079 __vlan_hwaccel_put_tag(skb, htons(ETH_P_8021Q), 2080 le16_to_cpu(rx_desc->special)); 2081 2082 netif_receive_skb(skb); 2083 2084rxdesc_done: 2085 /* clean up descriptor, might be written over by hw */ 2086 rx_desc->status = 0; 2087 2088 /* return some buffers to hardware, one at a time is too slow */ 2089 if (unlikely(cleaned_count >= IXGB_RX_BUFFER_WRITE)) { 2090 ixgb_alloc_rx_buffers(adapter, cleaned_count); 2091 cleaned_count = 0; 2092 } 2093 2094 /* use prefetched values */ 2095 rx_desc = next_rxd; 2096 buffer_info = next_buffer; 2097 } 2098 2099 rx_ring->next_to_clean = i; 2100 2101 cleaned_count = IXGB_DESC_UNUSED(rx_ring); 2102 if (cleaned_count) 2103 ixgb_alloc_rx_buffers(adapter, cleaned_count); 2104 2105 return cleaned; 2106} 2107 2108/** 2109 * ixgb_alloc_rx_buffers - Replace used receive buffers 2110 * @adapter: address of board private structure 2111 **/ 2112 2113static void 2114ixgb_alloc_rx_buffers(struct ixgb_adapter *adapter, int cleaned_count) 2115{ 2116 struct ixgb_desc_ring *rx_ring = &adapter->rx_ring; 2117 struct net_device *netdev = adapter->netdev; 2118 struct pci_dev *pdev = adapter->pdev; 2119 struct ixgb_rx_desc *rx_desc; 2120 struct ixgb_buffer *buffer_info; 2121 struct sk_buff *skb; 2122 unsigned int i; 2123 long cleancount; 2124 2125 i = rx_ring->next_to_use; 2126 buffer_info = &rx_ring->buffer_info[i]; 2127 cleancount = IXGB_DESC_UNUSED(rx_ring); 2128 2129 2130 /* leave three descriptors unused */ 2131 while (--cleancount > 2 && cleaned_count--) { 2132 /* recycle! its good for you */ 2133 skb = buffer_info->skb; 2134 if (skb) { 2135 skb_trim(skb, 0); 2136 goto map_skb; 2137 } 2138 2139 skb = netdev_alloc_skb_ip_align(netdev, adapter->rx_buffer_len); 2140 if (unlikely(!skb)) { 2141 /* Better luck next round */ 2142 adapter->alloc_rx_buff_failed++; 2143 break; 2144 } 2145 2146 buffer_info->skb = skb; 2147 buffer_info->length = adapter->rx_buffer_len; 2148map_skb: 2149 buffer_info->dma = dma_map_single(&pdev->dev, 2150 skb->data, 2151 adapter->rx_buffer_len, 2152 DMA_FROM_DEVICE); 2153 if (dma_mapping_error(&pdev->dev, buffer_info->dma)) { 2154 adapter->alloc_rx_buff_failed++; 2155 break; 2156 } 2157 2158 rx_desc = IXGB_RX_DESC(*rx_ring, i); 2159 rx_desc->buff_addr = cpu_to_le64(buffer_info->dma); 2160 /* guarantee DD bit not set now before h/w gets descriptor 2161 * this is the rest of the workaround for h/w double 2162 * writeback. */ 2163 rx_desc->status = 0; 2164 2165 2166 if (++i == rx_ring->count) 2167 i = 0; 2168 buffer_info = &rx_ring->buffer_info[i]; 2169 } 2170 2171 if (likely(rx_ring->next_to_use != i)) { 2172 rx_ring->next_to_use = i; 2173 if (unlikely(i-- == 0)) 2174 i = (rx_ring->count - 1); 2175 2176 /* Force memory writes to complete before letting h/w 2177 * know there are new descriptors to fetch. (Only 2178 * applicable for weak-ordered memory model archs, such 2179 * as IA-64). */ 2180 wmb(); 2181 IXGB_WRITE_REG(&adapter->hw, RDT, i); 2182 } 2183} 2184 2185static void 2186ixgb_vlan_strip_enable(struct ixgb_adapter *adapter) 2187{ 2188 u32 ctrl; 2189 2190 /* enable VLAN tag insert/strip */ 2191 ctrl = IXGB_READ_REG(&adapter->hw, CTRL0); 2192 ctrl |= IXGB_CTRL0_VME; 2193 IXGB_WRITE_REG(&adapter->hw, CTRL0, ctrl); 2194} 2195 2196static void 2197ixgb_vlan_strip_disable(struct ixgb_adapter *adapter) 2198{ 2199 u32 ctrl; 2200 2201 /* disable VLAN tag insert/strip */ 2202 ctrl = IXGB_READ_REG(&adapter->hw, CTRL0); 2203 ctrl &= ~IXGB_CTRL0_VME; 2204 IXGB_WRITE_REG(&adapter->hw, CTRL0, ctrl); 2205} 2206 2207static int 2208ixgb_vlan_rx_add_vid(struct net_device *netdev, __be16 proto, u16 vid) 2209{ 2210 struct ixgb_adapter *adapter = netdev_priv(netdev); 2211 u32 vfta, index; 2212 2213 /* add VID to filter table */ 2214 2215 index = (vid >> 5) & 0x7F; 2216 vfta = IXGB_READ_REG_ARRAY(&adapter->hw, VFTA, index); 2217 vfta |= (1 << (vid & 0x1F)); 2218 ixgb_write_vfta(&adapter->hw, index, vfta); 2219 set_bit(vid, adapter->active_vlans); 2220 2221 return 0; 2222} 2223 2224static int 2225ixgb_vlan_rx_kill_vid(struct net_device *netdev, __be16 proto, u16 vid) 2226{ 2227 struct ixgb_adapter *adapter = netdev_priv(netdev); 2228 u32 vfta, index; 2229 2230 /* remove VID from filter table */ 2231 2232 index = (vid >> 5) & 0x7F; 2233 vfta = IXGB_READ_REG_ARRAY(&adapter->hw, VFTA, index); 2234 vfta &= ~(1 << (vid & 0x1F)); 2235 ixgb_write_vfta(&adapter->hw, index, vfta); 2236 clear_bit(vid, adapter->active_vlans); 2237 2238 return 0; 2239} 2240 2241static void 2242ixgb_restore_vlan(struct ixgb_adapter *adapter) 2243{ 2244 u16 vid; 2245 2246 for_each_set_bit(vid, adapter->active_vlans, VLAN_N_VID) 2247 ixgb_vlan_rx_add_vid(adapter->netdev, htons(ETH_P_8021Q), vid); 2248} 2249 2250#ifdef CONFIG_NET_POLL_CONTROLLER 2251/* 2252 * Polling 'interrupt' - used by things like netconsole to send skbs 2253 * without having to re-enable interrupts. It's not called while 2254 * the interrupt routine is executing. 2255 */ 2256 2257static void ixgb_netpoll(struct net_device *dev) 2258{ 2259 struct ixgb_adapter *adapter = netdev_priv(dev); 2260 2261 disable_irq(adapter->pdev->irq); 2262 ixgb_intr(adapter->pdev->irq, dev); 2263 enable_irq(adapter->pdev->irq); 2264} 2265#endif 2266 2267/** 2268 * ixgb_io_error_detected - called when PCI error is detected 2269 * @pdev: pointer to pci device with error 2270 * @state: pci channel state after error 2271 * 2272 * This callback is called by the PCI subsystem whenever 2273 * a PCI bus error is detected. 2274 */ 2275static pci_ers_result_t ixgb_io_error_detected(struct pci_dev *pdev, 2276 enum pci_channel_state state) 2277{ 2278 struct net_device *netdev = pci_get_drvdata(pdev); 2279 struct ixgb_adapter *adapter = netdev_priv(netdev); 2280 2281 netif_device_detach(netdev); 2282 2283 if (state == pci_channel_io_perm_failure) 2284 return PCI_ERS_RESULT_DISCONNECT; 2285 2286 if (netif_running(netdev)) 2287 ixgb_down(adapter, true); 2288 2289 pci_disable_device(pdev); 2290 2291 /* Request a slot reset. */ 2292 return PCI_ERS_RESULT_NEED_RESET; 2293} 2294 2295/** 2296 * ixgb_io_slot_reset - called after the pci bus has been reset. 2297 * @pdev pointer to pci device with error 2298 * 2299 * This callback is called after the PCI bus has been reset. 2300 * Basically, this tries to restart the card from scratch. 2301 * This is a shortened version of the device probe/discovery code, 2302 * it resembles the first-half of the ixgb_probe() routine. 2303 */ 2304static pci_ers_result_t ixgb_io_slot_reset(struct pci_dev *pdev) 2305{ 2306 struct net_device *netdev = pci_get_drvdata(pdev); 2307 struct ixgb_adapter *adapter = netdev_priv(netdev); 2308 2309 if (pci_enable_device(pdev)) { 2310 netif_err(adapter, probe, adapter->netdev, 2311 "Cannot re-enable PCI device after reset\n"); 2312 return PCI_ERS_RESULT_DISCONNECT; 2313 } 2314 2315 /* Perform card reset only on one instance of the card */ 2316 if (0 != PCI_FUNC (pdev->devfn)) 2317 return PCI_ERS_RESULT_RECOVERED; 2318 2319 pci_set_master(pdev); 2320 2321 netif_carrier_off(netdev); 2322 netif_stop_queue(netdev); 2323 ixgb_reset(adapter); 2324 2325 /* Make sure the EEPROM is good */ 2326 if (!ixgb_validate_eeprom_checksum(&adapter->hw)) { 2327 netif_err(adapter, probe, adapter->netdev, 2328 "After reset, the EEPROM checksum is not valid\n"); 2329 return PCI_ERS_RESULT_DISCONNECT; 2330 } 2331 ixgb_get_ee_mac_addr(&adapter->hw, netdev->dev_addr); 2332 memcpy(netdev->perm_addr, netdev->dev_addr, netdev->addr_len); 2333 2334 if (!is_valid_ether_addr(netdev->perm_addr)) { 2335 netif_err(adapter, probe, adapter->netdev, 2336 "After reset, invalid MAC address\n"); 2337 return PCI_ERS_RESULT_DISCONNECT; 2338 } 2339 2340 return PCI_ERS_RESULT_RECOVERED; 2341} 2342 2343/** 2344 * ixgb_io_resume - called when its OK to resume normal operations 2345 * @pdev pointer to pci device with error 2346 * 2347 * The error recovery driver tells us that its OK to resume 2348 * normal operation. Implementation resembles the second-half 2349 * of the ixgb_probe() routine. 2350 */ 2351static void ixgb_io_resume(struct pci_dev *pdev) 2352{ 2353 struct net_device *netdev = pci_get_drvdata(pdev); 2354 struct ixgb_adapter *adapter = netdev_priv(netdev); 2355 2356 pci_set_master(pdev); 2357 2358 if (netif_running(netdev)) { 2359 if (ixgb_up(adapter)) { 2360 pr_err("can't bring device back up after reset\n"); 2361 return; 2362 } 2363 } 2364 2365 netif_device_attach(netdev); 2366 mod_timer(&adapter->watchdog_timer, jiffies); 2367} 2368 2369/* ixgb_main.c */ 2370