1/* 2 3 Broadcom B43 wireless driver 4 5 DMA ringbuffer and descriptor allocation/management 6 7 Copyright (c) 2005, 2006 Michael Buesch <m@bues.ch> 8 9 Some code in this file is derived from the b44.c driver 10 Copyright (C) 2002 David S. Miller 11 Copyright (C) Pekka Pietikainen 12 13 This program is free software; you can redistribute it and/or modify 14 it under the terms of the GNU General Public License as published by 15 the Free Software Foundation; either version 2 of the License, or 16 (at your option) any later version. 17 18 This program is distributed in the hope that it will be useful, 19 but WITHOUT ANY WARRANTY; without even the implied warranty of 20 MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 21 GNU General Public License for more details. 22 23 You should have received a copy of the GNU General Public License 24 along with this program; see the file COPYING. If not, write to 25 the Free Software Foundation, Inc., 51 Franklin Steet, Fifth Floor, 26 Boston, MA 02110-1301, USA. 27 28*/ 29 30#include "b43.h" 31#include "dma.h" 32#include "main.h" 33#include "debugfs.h" 34#include "xmit.h" 35 36#include <linux/dma-mapping.h> 37#include <linux/pci.h> 38#include <linux/delay.h> 39#include <linux/skbuff.h> 40#include <linux/etherdevice.h> 41#include <linux/slab.h> 42#include <asm/div64.h> 43 44 45/* Required number of TX DMA slots per TX frame. 46 * This currently is 2, because we put the header and the ieee80211 frame 47 * into separate slots. */ 48#define TX_SLOTS_PER_FRAME 2 49 50static u32 b43_dma_address(struct b43_dma *dma, dma_addr_t dmaaddr, 51 enum b43_addrtype addrtype) 52{ 53 u32 uninitialized_var(addr); 54 55 switch (addrtype) { 56 case B43_DMA_ADDR_LOW: 57 addr = lower_32_bits(dmaaddr); 58 if (dma->translation_in_low) { 59 addr &= ~SSB_DMA_TRANSLATION_MASK; 60 addr |= dma->translation; 61 } 62 break; 63 case B43_DMA_ADDR_HIGH: 64 addr = upper_32_bits(dmaaddr); 65 if (!dma->translation_in_low) { 66 addr &= ~SSB_DMA_TRANSLATION_MASK; 67 addr |= dma->translation; 68 } 69 break; 70 case B43_DMA_ADDR_EXT: 71 if (dma->translation_in_low) 72 addr = lower_32_bits(dmaaddr); 73 else 74 addr = upper_32_bits(dmaaddr); 75 addr &= SSB_DMA_TRANSLATION_MASK; 76 addr >>= SSB_DMA_TRANSLATION_SHIFT; 77 break; 78 } 79 80 return addr; 81} 82 83/* 32bit DMA ops. */ 84static 85struct b43_dmadesc_generic *op32_idx2desc(struct b43_dmaring *ring, 86 int slot, 87 struct b43_dmadesc_meta **meta) 88{ 89 struct b43_dmadesc32 *desc; 90 91 *meta = &(ring->meta[slot]); 92 desc = ring->descbase; 93 desc = &(desc[slot]); 94 95 return (struct b43_dmadesc_generic *)desc; 96} 97 98static void op32_fill_descriptor(struct b43_dmaring *ring, 99 struct b43_dmadesc_generic *desc, 100 dma_addr_t dmaaddr, u16 bufsize, 101 int start, int end, int irq) 102{ 103 struct b43_dmadesc32 *descbase = ring->descbase; 104 int slot; 105 u32 ctl; 106 u32 addr; 107 u32 addrext; 108 109 slot = (int)(&(desc->dma32) - descbase); 110 B43_WARN_ON(!(slot >= 0 && slot < ring->nr_slots)); 111 112 addr = b43_dma_address(&ring->dev->dma, dmaaddr, B43_DMA_ADDR_LOW); 113 addrext = b43_dma_address(&ring->dev->dma, dmaaddr, B43_DMA_ADDR_EXT); 114 115 ctl = bufsize & B43_DMA32_DCTL_BYTECNT; 116 if (slot == ring->nr_slots - 1) 117 ctl |= B43_DMA32_DCTL_DTABLEEND; 118 if (start) 119 ctl |= B43_DMA32_DCTL_FRAMESTART; 120 if (end) 121 ctl |= B43_DMA32_DCTL_FRAMEEND; 122 if (irq) 123 ctl |= B43_DMA32_DCTL_IRQ; 124 ctl |= (addrext << B43_DMA32_DCTL_ADDREXT_SHIFT) 125 & B43_DMA32_DCTL_ADDREXT_MASK; 126 127 desc->dma32.control = cpu_to_le32(ctl); 128 desc->dma32.address = cpu_to_le32(addr); 129} 130 131static void op32_poke_tx(struct b43_dmaring *ring, int slot) 132{ 133 b43_dma_write(ring, B43_DMA32_TXINDEX, 134 (u32) (slot * sizeof(struct b43_dmadesc32))); 135} 136 137static void op32_tx_suspend(struct b43_dmaring *ring) 138{ 139 b43_dma_write(ring, B43_DMA32_TXCTL, b43_dma_read(ring, B43_DMA32_TXCTL) 140 | B43_DMA32_TXSUSPEND); 141} 142 143static void op32_tx_resume(struct b43_dmaring *ring) 144{ 145 b43_dma_write(ring, B43_DMA32_TXCTL, b43_dma_read(ring, B43_DMA32_TXCTL) 146 & ~B43_DMA32_TXSUSPEND); 147} 148 149static int op32_get_current_rxslot(struct b43_dmaring *ring) 150{ 151 u32 val; 152 153 val = b43_dma_read(ring, B43_DMA32_RXSTATUS); 154 val &= B43_DMA32_RXDPTR; 155 156 return (val / sizeof(struct b43_dmadesc32)); 157} 158 159static void op32_set_current_rxslot(struct b43_dmaring *ring, int slot) 160{ 161 b43_dma_write(ring, B43_DMA32_RXINDEX, 162 (u32) (slot * sizeof(struct b43_dmadesc32))); 163} 164 165static const struct b43_dma_ops dma32_ops = { 166 .idx2desc = op32_idx2desc, 167 .fill_descriptor = op32_fill_descriptor, 168 .poke_tx = op32_poke_tx, 169 .tx_suspend = op32_tx_suspend, 170 .tx_resume = op32_tx_resume, 171 .get_current_rxslot = op32_get_current_rxslot, 172 .set_current_rxslot = op32_set_current_rxslot, 173}; 174 175/* 64bit DMA ops. */ 176static 177struct b43_dmadesc_generic *op64_idx2desc(struct b43_dmaring *ring, 178 int slot, 179 struct b43_dmadesc_meta **meta) 180{ 181 struct b43_dmadesc64 *desc; 182 183 *meta = &(ring->meta[slot]); 184 desc = ring->descbase; 185 desc = &(desc[slot]); 186 187 return (struct b43_dmadesc_generic *)desc; 188} 189 190static void op64_fill_descriptor(struct b43_dmaring *ring, 191 struct b43_dmadesc_generic *desc, 192 dma_addr_t dmaaddr, u16 bufsize, 193 int start, int end, int irq) 194{ 195 struct b43_dmadesc64 *descbase = ring->descbase; 196 int slot; 197 u32 ctl0 = 0, ctl1 = 0; 198 u32 addrlo, addrhi; 199 u32 addrext; 200 201 slot = (int)(&(desc->dma64) - descbase); 202 B43_WARN_ON(!(slot >= 0 && slot < ring->nr_slots)); 203 204 addrlo = b43_dma_address(&ring->dev->dma, dmaaddr, B43_DMA_ADDR_LOW); 205 addrhi = b43_dma_address(&ring->dev->dma, dmaaddr, B43_DMA_ADDR_HIGH); 206 addrext = b43_dma_address(&ring->dev->dma, dmaaddr, B43_DMA_ADDR_EXT); 207 208 if (slot == ring->nr_slots - 1) 209 ctl0 |= B43_DMA64_DCTL0_DTABLEEND; 210 if (start) 211 ctl0 |= B43_DMA64_DCTL0_FRAMESTART; 212 if (end) 213 ctl0 |= B43_DMA64_DCTL0_FRAMEEND; 214 if (irq) 215 ctl0 |= B43_DMA64_DCTL0_IRQ; 216 ctl1 |= bufsize & B43_DMA64_DCTL1_BYTECNT; 217 ctl1 |= (addrext << B43_DMA64_DCTL1_ADDREXT_SHIFT) 218 & B43_DMA64_DCTL1_ADDREXT_MASK; 219 220 desc->dma64.control0 = cpu_to_le32(ctl0); 221 desc->dma64.control1 = cpu_to_le32(ctl1); 222 desc->dma64.address_low = cpu_to_le32(addrlo); 223 desc->dma64.address_high = cpu_to_le32(addrhi); 224} 225 226static void op64_poke_tx(struct b43_dmaring *ring, int slot) 227{ 228 b43_dma_write(ring, B43_DMA64_TXINDEX, 229 (u32) (slot * sizeof(struct b43_dmadesc64))); 230} 231 232static void op64_tx_suspend(struct b43_dmaring *ring) 233{ 234 b43_dma_write(ring, B43_DMA64_TXCTL, b43_dma_read(ring, B43_DMA64_TXCTL) 235 | B43_DMA64_TXSUSPEND); 236} 237 238static void op64_tx_resume(struct b43_dmaring *ring) 239{ 240 b43_dma_write(ring, B43_DMA64_TXCTL, b43_dma_read(ring, B43_DMA64_TXCTL) 241 & ~B43_DMA64_TXSUSPEND); 242} 243 244static int op64_get_current_rxslot(struct b43_dmaring *ring) 245{ 246 u32 val; 247 248 val = b43_dma_read(ring, B43_DMA64_RXSTATUS); 249 val &= B43_DMA64_RXSTATDPTR; 250 251 return (val / sizeof(struct b43_dmadesc64)); 252} 253 254static void op64_set_current_rxslot(struct b43_dmaring *ring, int slot) 255{ 256 b43_dma_write(ring, B43_DMA64_RXINDEX, 257 (u32) (slot * sizeof(struct b43_dmadesc64))); 258} 259 260static const struct b43_dma_ops dma64_ops = { 261 .idx2desc = op64_idx2desc, 262 .fill_descriptor = op64_fill_descriptor, 263 .poke_tx = op64_poke_tx, 264 .tx_suspend = op64_tx_suspend, 265 .tx_resume = op64_tx_resume, 266 .get_current_rxslot = op64_get_current_rxslot, 267 .set_current_rxslot = op64_set_current_rxslot, 268}; 269 270static inline int free_slots(struct b43_dmaring *ring) 271{ 272 return (ring->nr_slots - ring->used_slots); 273} 274 275static inline int next_slot(struct b43_dmaring *ring, int slot) 276{ 277 B43_WARN_ON(!(slot >= -1 && slot <= ring->nr_slots - 1)); 278 if (slot == ring->nr_slots - 1) 279 return 0; 280 return slot + 1; 281} 282 283static inline int prev_slot(struct b43_dmaring *ring, int slot) 284{ 285 B43_WARN_ON(!(slot >= 0 && slot <= ring->nr_slots - 1)); 286 if (slot == 0) 287 return ring->nr_slots - 1; 288 return slot - 1; 289} 290 291#ifdef CONFIG_B43_DEBUG 292static void update_max_used_slots(struct b43_dmaring *ring, 293 int current_used_slots) 294{ 295 if (current_used_slots <= ring->max_used_slots) 296 return; 297 ring->max_used_slots = current_used_slots; 298 if (b43_debug(ring->dev, B43_DBG_DMAVERBOSE)) { 299 b43dbg(ring->dev->wl, 300 "max_used_slots increased to %d on %s ring %d\n", 301 ring->max_used_slots, 302 ring->tx ? "TX" : "RX", ring->index); 303 } 304} 305#else 306static inline 307 void update_max_used_slots(struct b43_dmaring *ring, int current_used_slots) 308{ 309} 310#endif /* DEBUG */ 311 312/* Request a slot for usage. */ 313static inline int request_slot(struct b43_dmaring *ring) 314{ 315 int slot; 316 317 B43_WARN_ON(!ring->tx); 318 B43_WARN_ON(ring->stopped); 319 B43_WARN_ON(free_slots(ring) == 0); 320 321 slot = next_slot(ring, ring->current_slot); 322 ring->current_slot = slot; 323 ring->used_slots++; 324 325 update_max_used_slots(ring, ring->used_slots); 326 327 return slot; 328} 329 330static u16 b43_dmacontroller_base(enum b43_dmatype type, int controller_idx) 331{ 332 static const u16 map64[] = { 333 B43_MMIO_DMA64_BASE0, 334 B43_MMIO_DMA64_BASE1, 335 B43_MMIO_DMA64_BASE2, 336 B43_MMIO_DMA64_BASE3, 337 B43_MMIO_DMA64_BASE4, 338 B43_MMIO_DMA64_BASE5, 339 }; 340 static const u16 map32[] = { 341 B43_MMIO_DMA32_BASE0, 342 B43_MMIO_DMA32_BASE1, 343 B43_MMIO_DMA32_BASE2, 344 B43_MMIO_DMA32_BASE3, 345 B43_MMIO_DMA32_BASE4, 346 B43_MMIO_DMA32_BASE5, 347 }; 348 349 if (type == B43_DMA_64BIT) { 350 B43_WARN_ON(!(controller_idx >= 0 && 351 controller_idx < ARRAY_SIZE(map64))); 352 return map64[controller_idx]; 353 } 354 B43_WARN_ON(!(controller_idx >= 0 && 355 controller_idx < ARRAY_SIZE(map32))); 356 return map32[controller_idx]; 357} 358 359static inline 360 dma_addr_t map_descbuffer(struct b43_dmaring *ring, 361 unsigned char *buf, size_t len, int tx) 362{ 363 dma_addr_t dmaaddr; 364 365 if (tx) { 366 dmaaddr = dma_map_single(ring->dev->dev->dma_dev, 367 buf, len, DMA_TO_DEVICE); 368 } else { 369 dmaaddr = dma_map_single(ring->dev->dev->dma_dev, 370 buf, len, DMA_FROM_DEVICE); 371 } 372 373 return dmaaddr; 374} 375 376static inline 377 void unmap_descbuffer(struct b43_dmaring *ring, 378 dma_addr_t addr, size_t len, int tx) 379{ 380 if (tx) { 381 dma_unmap_single(ring->dev->dev->dma_dev, 382 addr, len, DMA_TO_DEVICE); 383 } else { 384 dma_unmap_single(ring->dev->dev->dma_dev, 385 addr, len, DMA_FROM_DEVICE); 386 } 387} 388 389static inline 390 void sync_descbuffer_for_cpu(struct b43_dmaring *ring, 391 dma_addr_t addr, size_t len) 392{ 393 B43_WARN_ON(ring->tx); 394 dma_sync_single_for_cpu(ring->dev->dev->dma_dev, 395 addr, len, DMA_FROM_DEVICE); 396} 397 398static inline 399 void sync_descbuffer_for_device(struct b43_dmaring *ring, 400 dma_addr_t addr, size_t len) 401{ 402 B43_WARN_ON(ring->tx); 403 dma_sync_single_for_device(ring->dev->dev->dma_dev, 404 addr, len, DMA_FROM_DEVICE); 405} 406 407static inline 408 void free_descriptor_buffer(struct b43_dmaring *ring, 409 struct b43_dmadesc_meta *meta) 410{ 411 if (meta->skb) { 412 dev_kfree_skb_any(meta->skb); 413 meta->skb = NULL; 414 } 415} 416 417static int alloc_ringmemory(struct b43_dmaring *ring) 418{ 419 gfp_t flags = GFP_KERNEL; 420 421 /* The specs call for 4K buffers for 30- and 32-bit DMA with 4K 422 * alignment and 8K buffers for 64-bit DMA with 8K alignment. 423 * In practice we could use smaller buffers for the latter, but the 424 * alignment is really important because of the hardware bug. If bit 425 * 0x00001000 is used in DMA address, some hardware (like BCM4331) 426 * copies that bit into B43_DMA64_RXSTATUS and we get false values from 427 * B43_DMA64_RXSTATDPTR. Let's just use 8K buffers even if we don't use 428 * more than 256 slots for ring. 429 */ 430 u16 ring_mem_size = (ring->type == B43_DMA_64BIT) ? 431 B43_DMA64_RINGMEMSIZE : B43_DMA32_RINGMEMSIZE; 432 433 ring->descbase = dma_alloc_coherent(ring->dev->dev->dma_dev, 434 ring_mem_size, &(ring->dmabase), 435 flags); 436 if (!ring->descbase) { 437 b43err(ring->dev->wl, "DMA ringmemory allocation failed\n"); 438 return -ENOMEM; 439 } 440 memset(ring->descbase, 0, ring_mem_size); 441 442 return 0; 443} 444 445static void free_ringmemory(struct b43_dmaring *ring) 446{ 447 u16 ring_mem_size = (ring->type == B43_DMA_64BIT) ? 448 B43_DMA64_RINGMEMSIZE : B43_DMA32_RINGMEMSIZE; 449 dma_free_coherent(ring->dev->dev->dma_dev, ring_mem_size, 450 ring->descbase, ring->dmabase); 451} 452 453/* Reset the RX DMA channel */ 454static int b43_dmacontroller_rx_reset(struct b43_wldev *dev, u16 mmio_base, 455 enum b43_dmatype type) 456{ 457 int i; 458 u32 value; 459 u16 offset; 460 461 might_sleep(); 462 463 offset = (type == B43_DMA_64BIT) ? B43_DMA64_RXCTL : B43_DMA32_RXCTL; 464 b43_write32(dev, mmio_base + offset, 0); 465 for (i = 0; i < 10; i++) { 466 offset = (type == B43_DMA_64BIT) ? B43_DMA64_RXSTATUS : 467 B43_DMA32_RXSTATUS; 468 value = b43_read32(dev, mmio_base + offset); 469 if (type == B43_DMA_64BIT) { 470 value &= B43_DMA64_RXSTAT; 471 if (value == B43_DMA64_RXSTAT_DISABLED) { 472 i = -1; 473 break; 474 } 475 } else { 476 value &= B43_DMA32_RXSTATE; 477 if (value == B43_DMA32_RXSTAT_DISABLED) { 478 i = -1; 479 break; 480 } 481 } 482 msleep(1); 483 } 484 if (i != -1) { 485 b43err(dev->wl, "DMA RX reset timed out\n"); 486 return -ENODEV; 487 } 488 489 return 0; 490} 491 492/* Reset the TX DMA channel */ 493static int b43_dmacontroller_tx_reset(struct b43_wldev *dev, u16 mmio_base, 494 enum b43_dmatype type) 495{ 496 int i; 497 u32 value; 498 u16 offset; 499 500 might_sleep(); 501 502 for (i = 0; i < 10; i++) { 503 offset = (type == B43_DMA_64BIT) ? B43_DMA64_TXSTATUS : 504 B43_DMA32_TXSTATUS; 505 value = b43_read32(dev, mmio_base + offset); 506 if (type == B43_DMA_64BIT) { 507 value &= B43_DMA64_TXSTAT; 508 if (value == B43_DMA64_TXSTAT_DISABLED || 509 value == B43_DMA64_TXSTAT_IDLEWAIT || 510 value == B43_DMA64_TXSTAT_STOPPED) 511 break; 512 } else { 513 value &= B43_DMA32_TXSTATE; 514 if (value == B43_DMA32_TXSTAT_DISABLED || 515 value == B43_DMA32_TXSTAT_IDLEWAIT || 516 value == B43_DMA32_TXSTAT_STOPPED) 517 break; 518 } 519 msleep(1); 520 } 521 offset = (type == B43_DMA_64BIT) ? B43_DMA64_TXCTL : B43_DMA32_TXCTL; 522 b43_write32(dev, mmio_base + offset, 0); 523 for (i = 0; i < 10; i++) { 524 offset = (type == B43_DMA_64BIT) ? B43_DMA64_TXSTATUS : 525 B43_DMA32_TXSTATUS; 526 value = b43_read32(dev, mmio_base + offset); 527 if (type == B43_DMA_64BIT) { 528 value &= B43_DMA64_TXSTAT; 529 if (value == B43_DMA64_TXSTAT_DISABLED) { 530 i = -1; 531 break; 532 } 533 } else { 534 value &= B43_DMA32_TXSTATE; 535 if (value == B43_DMA32_TXSTAT_DISABLED) { 536 i = -1; 537 break; 538 } 539 } 540 msleep(1); 541 } 542 if (i != -1) { 543 b43err(dev->wl, "DMA TX reset timed out\n"); 544 return -ENODEV; 545 } 546 /* ensure the reset is completed. */ 547 msleep(1); 548 549 return 0; 550} 551 552/* Check if a DMA mapping address is invalid. */ 553static bool b43_dma_mapping_error(struct b43_dmaring *ring, 554 dma_addr_t addr, 555 size_t buffersize, bool dma_to_device) 556{ 557 if (unlikely(dma_mapping_error(ring->dev->dev->dma_dev, addr))) 558 return 1; 559 560 switch (ring->type) { 561 case B43_DMA_30BIT: 562 if ((u64)addr + buffersize > (1ULL << 30)) 563 goto address_error; 564 break; 565 case B43_DMA_32BIT: 566 if ((u64)addr + buffersize > (1ULL << 32)) 567 goto address_error; 568 break; 569 case B43_DMA_64BIT: 570 /* Currently we can't have addresses beyond 571 * 64bit in the kernel. */ 572 break; 573 } 574 575 /* The address is OK. */ 576 return 0; 577 578address_error: 579 /* We can't support this address. Unmap it again. */ 580 unmap_descbuffer(ring, addr, buffersize, dma_to_device); 581 582 return 1; 583} 584 585static bool b43_rx_buffer_is_poisoned(struct b43_dmaring *ring, struct sk_buff *skb) 586{ 587 unsigned char *f = skb->data + ring->frameoffset; 588 589 return ((f[0] & f[1] & f[2] & f[3] & f[4] & f[5] & f[6] & f[7]) == 0xFF); 590} 591 592static void b43_poison_rx_buffer(struct b43_dmaring *ring, struct sk_buff *skb) 593{ 594 struct b43_rxhdr_fw4 *rxhdr; 595 unsigned char *frame; 596 597 /* This poisons the RX buffer to detect DMA failures. */ 598 599 rxhdr = (struct b43_rxhdr_fw4 *)(skb->data); 600 rxhdr->frame_len = 0; 601 602 B43_WARN_ON(ring->rx_buffersize < ring->frameoffset + sizeof(struct b43_plcp_hdr6) + 2); 603 frame = skb->data + ring->frameoffset; 604 memset(frame, 0xFF, sizeof(struct b43_plcp_hdr6) + 2 /* padding */); 605} 606 607static int setup_rx_descbuffer(struct b43_dmaring *ring, 608 struct b43_dmadesc_generic *desc, 609 struct b43_dmadesc_meta *meta, gfp_t gfp_flags) 610{ 611 dma_addr_t dmaaddr; 612 struct sk_buff *skb; 613 614 B43_WARN_ON(ring->tx); 615 616 skb = __dev_alloc_skb(ring->rx_buffersize, gfp_flags); 617 if (unlikely(!skb)) 618 return -ENOMEM; 619 b43_poison_rx_buffer(ring, skb); 620 dmaaddr = map_descbuffer(ring, skb->data, ring->rx_buffersize, 0); 621 if (b43_dma_mapping_error(ring, dmaaddr, ring->rx_buffersize, 0)) { 622 /* ugh. try to realloc in zone_dma */ 623 gfp_flags |= GFP_DMA; 624 625 dev_kfree_skb_any(skb); 626 627 skb = __dev_alloc_skb(ring->rx_buffersize, gfp_flags); 628 if (unlikely(!skb)) 629 return -ENOMEM; 630 b43_poison_rx_buffer(ring, skb); 631 dmaaddr = map_descbuffer(ring, skb->data, 632 ring->rx_buffersize, 0); 633 if (b43_dma_mapping_error(ring, dmaaddr, ring->rx_buffersize, 0)) { 634 b43err(ring->dev->wl, "RX DMA buffer allocation failed\n"); 635 dev_kfree_skb_any(skb); 636 return -EIO; 637 } 638 } 639 640 meta->skb = skb; 641 meta->dmaaddr = dmaaddr; 642 ring->ops->fill_descriptor(ring, desc, dmaaddr, 643 ring->rx_buffersize, 0, 0, 0); 644 645 return 0; 646} 647 648/* Allocate the initial descbuffers. 649 * This is used for an RX ring only. 650 */ 651static int alloc_initial_descbuffers(struct b43_dmaring *ring) 652{ 653 int i, err = -ENOMEM; 654 struct b43_dmadesc_generic *desc; 655 struct b43_dmadesc_meta *meta; 656 657 for (i = 0; i < ring->nr_slots; i++) { 658 desc = ring->ops->idx2desc(ring, i, &meta); 659 660 err = setup_rx_descbuffer(ring, desc, meta, GFP_KERNEL); 661 if (err) { 662 b43err(ring->dev->wl, 663 "Failed to allocate initial descbuffers\n"); 664 goto err_unwind; 665 } 666 } 667 mb(); 668 ring->used_slots = ring->nr_slots; 669 err = 0; 670 out: 671 return err; 672 673 err_unwind: 674 for (i--; i >= 0; i--) { 675 desc = ring->ops->idx2desc(ring, i, &meta); 676 677 unmap_descbuffer(ring, meta->dmaaddr, ring->rx_buffersize, 0); 678 dev_kfree_skb(meta->skb); 679 } 680 goto out; 681} 682 683/* Do initial setup of the DMA controller. 684 * Reset the controller, write the ring busaddress 685 * and switch the "enable" bit on. 686 */ 687static int dmacontroller_setup(struct b43_dmaring *ring) 688{ 689 int err = 0; 690 u32 value; 691 u32 addrext; 692 bool parity = ring->dev->dma.parity; 693 u32 addrlo; 694 u32 addrhi; 695 696 if (ring->tx) { 697 if (ring->type == B43_DMA_64BIT) { 698 u64 ringbase = (u64) (ring->dmabase); 699 addrext = b43_dma_address(&ring->dev->dma, ringbase, B43_DMA_ADDR_EXT); 700 addrlo = b43_dma_address(&ring->dev->dma, ringbase, B43_DMA_ADDR_LOW); 701 addrhi = b43_dma_address(&ring->dev->dma, ringbase, B43_DMA_ADDR_HIGH); 702 703 value = B43_DMA64_TXENABLE; 704 value |= (addrext << B43_DMA64_TXADDREXT_SHIFT) 705 & B43_DMA64_TXADDREXT_MASK; 706 if (!parity) 707 value |= B43_DMA64_TXPARITYDISABLE; 708 b43_dma_write(ring, B43_DMA64_TXCTL, value); 709 b43_dma_write(ring, B43_DMA64_TXRINGLO, addrlo); 710 b43_dma_write(ring, B43_DMA64_TXRINGHI, addrhi); 711 } else { 712 u32 ringbase = (u32) (ring->dmabase); 713 addrext = b43_dma_address(&ring->dev->dma, ringbase, B43_DMA_ADDR_EXT); 714 addrlo = b43_dma_address(&ring->dev->dma, ringbase, B43_DMA_ADDR_LOW); 715 716 value = B43_DMA32_TXENABLE; 717 value |= (addrext << B43_DMA32_TXADDREXT_SHIFT) 718 & B43_DMA32_TXADDREXT_MASK; 719 if (!parity) 720 value |= B43_DMA32_TXPARITYDISABLE; 721 b43_dma_write(ring, B43_DMA32_TXCTL, value); 722 b43_dma_write(ring, B43_DMA32_TXRING, addrlo); 723 } 724 } else { 725 err = alloc_initial_descbuffers(ring); 726 if (err) 727 goto out; 728 if (ring->type == B43_DMA_64BIT) { 729 u64 ringbase = (u64) (ring->dmabase); 730 addrext = b43_dma_address(&ring->dev->dma, ringbase, B43_DMA_ADDR_EXT); 731 addrlo = b43_dma_address(&ring->dev->dma, ringbase, B43_DMA_ADDR_LOW); 732 addrhi = b43_dma_address(&ring->dev->dma, ringbase, B43_DMA_ADDR_HIGH); 733 734 value = (ring->frameoffset << B43_DMA64_RXFROFF_SHIFT); 735 value |= B43_DMA64_RXENABLE; 736 value |= (addrext << B43_DMA64_RXADDREXT_SHIFT) 737 & B43_DMA64_RXADDREXT_MASK; 738 if (!parity) 739 value |= B43_DMA64_RXPARITYDISABLE; 740 b43_dma_write(ring, B43_DMA64_RXCTL, value); 741 b43_dma_write(ring, B43_DMA64_RXRINGLO, addrlo); 742 b43_dma_write(ring, B43_DMA64_RXRINGHI, addrhi); 743 b43_dma_write(ring, B43_DMA64_RXINDEX, ring->nr_slots * 744 sizeof(struct b43_dmadesc64)); 745 } else { 746 u32 ringbase = (u32) (ring->dmabase); 747 addrext = b43_dma_address(&ring->dev->dma, ringbase, B43_DMA_ADDR_EXT); 748 addrlo = b43_dma_address(&ring->dev->dma, ringbase, B43_DMA_ADDR_LOW); 749 750 value = (ring->frameoffset << B43_DMA32_RXFROFF_SHIFT); 751 value |= B43_DMA32_RXENABLE; 752 value |= (addrext << B43_DMA32_RXADDREXT_SHIFT) 753 & B43_DMA32_RXADDREXT_MASK; 754 if (!parity) 755 value |= B43_DMA32_RXPARITYDISABLE; 756 b43_dma_write(ring, B43_DMA32_RXCTL, value); 757 b43_dma_write(ring, B43_DMA32_RXRING, addrlo); 758 b43_dma_write(ring, B43_DMA32_RXINDEX, ring->nr_slots * 759 sizeof(struct b43_dmadesc32)); 760 } 761 } 762 763out: 764 return err; 765} 766 767/* Shutdown the DMA controller. */ 768static void dmacontroller_cleanup(struct b43_dmaring *ring) 769{ 770 if (ring->tx) { 771 b43_dmacontroller_tx_reset(ring->dev, ring->mmio_base, 772 ring->type); 773 if (ring->type == B43_DMA_64BIT) { 774 b43_dma_write(ring, B43_DMA64_TXRINGLO, 0); 775 b43_dma_write(ring, B43_DMA64_TXRINGHI, 0); 776 } else 777 b43_dma_write(ring, B43_DMA32_TXRING, 0); 778 } else { 779 b43_dmacontroller_rx_reset(ring->dev, ring->mmio_base, 780 ring->type); 781 if (ring->type == B43_DMA_64BIT) { 782 b43_dma_write(ring, B43_DMA64_RXRINGLO, 0); 783 b43_dma_write(ring, B43_DMA64_RXRINGHI, 0); 784 } else 785 b43_dma_write(ring, B43_DMA32_RXRING, 0); 786 } 787} 788 789static void free_all_descbuffers(struct b43_dmaring *ring) 790{ 791 struct b43_dmadesc_meta *meta; 792 int i; 793 794 if (!ring->used_slots) 795 return; 796 for (i = 0; i < ring->nr_slots; i++) { 797 /* get meta - ignore returned value */ 798 ring->ops->idx2desc(ring, i, &meta); 799 800 if (!meta->skb || b43_dma_ptr_is_poisoned(meta->skb)) { 801 B43_WARN_ON(!ring->tx); 802 continue; 803 } 804 if (ring->tx) { 805 unmap_descbuffer(ring, meta->dmaaddr, 806 meta->skb->len, 1); 807 } else { 808 unmap_descbuffer(ring, meta->dmaaddr, 809 ring->rx_buffersize, 0); 810 } 811 free_descriptor_buffer(ring, meta); 812 } 813} 814 815static u64 supported_dma_mask(struct b43_wldev *dev) 816{ 817 u32 tmp; 818 u16 mmio_base; 819 820 switch (dev->dev->bus_type) { 821#ifdef CONFIG_B43_BCMA 822 case B43_BUS_BCMA: 823 tmp = bcma_aread32(dev->dev->bdev, BCMA_IOST); 824 if (tmp & BCMA_IOST_DMA64) 825 return DMA_BIT_MASK(64); 826 break; 827#endif 828#ifdef CONFIG_B43_SSB 829 case B43_BUS_SSB: 830 tmp = ssb_read32(dev->dev->sdev, SSB_TMSHIGH); 831 if (tmp & SSB_TMSHIGH_DMA64) 832 return DMA_BIT_MASK(64); 833 break; 834#endif 835 } 836 837 mmio_base = b43_dmacontroller_base(0, 0); 838 b43_write32(dev, mmio_base + B43_DMA32_TXCTL, B43_DMA32_TXADDREXT_MASK); 839 tmp = b43_read32(dev, mmio_base + B43_DMA32_TXCTL); 840 if (tmp & B43_DMA32_TXADDREXT_MASK) 841 return DMA_BIT_MASK(32); 842 843 return DMA_BIT_MASK(30); 844} 845 846static enum b43_dmatype dma_mask_to_engine_type(u64 dmamask) 847{ 848 if (dmamask == DMA_BIT_MASK(30)) 849 return B43_DMA_30BIT; 850 if (dmamask == DMA_BIT_MASK(32)) 851 return B43_DMA_32BIT; 852 if (dmamask == DMA_BIT_MASK(64)) 853 return B43_DMA_64BIT; 854 B43_WARN_ON(1); 855 return B43_DMA_30BIT; 856} 857 858/* Main initialization function. */ 859static 860struct b43_dmaring *b43_setup_dmaring(struct b43_wldev *dev, 861 int controller_index, 862 int for_tx, 863 enum b43_dmatype type) 864{ 865 struct b43_dmaring *ring; 866 int i, err; 867 dma_addr_t dma_test; 868 869 ring = kzalloc(sizeof(*ring), GFP_KERNEL); 870 if (!ring) 871 goto out; 872 873 ring->nr_slots = B43_RXRING_SLOTS; 874 if (for_tx) 875 ring->nr_slots = B43_TXRING_SLOTS; 876 877 ring->meta = kcalloc(ring->nr_slots, sizeof(struct b43_dmadesc_meta), 878 GFP_KERNEL); 879 if (!ring->meta) 880 goto err_kfree_ring; 881 for (i = 0; i < ring->nr_slots; i++) 882 ring->meta->skb = B43_DMA_PTR_POISON; 883 884 ring->type = type; 885 ring->dev = dev; 886 ring->mmio_base = b43_dmacontroller_base(type, controller_index); 887 ring->index = controller_index; 888 if (type == B43_DMA_64BIT) 889 ring->ops = &dma64_ops; 890 else 891 ring->ops = &dma32_ops; 892 if (for_tx) { 893 ring->tx = true; 894 ring->current_slot = -1; 895 } else { 896 if (ring->index == 0) { 897 switch (dev->fw.hdr_format) { 898 case B43_FW_HDR_598: 899 ring->rx_buffersize = B43_DMA0_RX_FW598_BUFSIZE; 900 ring->frameoffset = B43_DMA0_RX_FW598_FO; 901 break; 902 case B43_FW_HDR_410: 903 case B43_FW_HDR_351: 904 ring->rx_buffersize = B43_DMA0_RX_FW351_BUFSIZE; 905 ring->frameoffset = B43_DMA0_RX_FW351_FO; 906 break; 907 } 908 } else 909 B43_WARN_ON(1); 910 } 911#ifdef CONFIG_B43_DEBUG 912 ring->last_injected_overflow = jiffies; 913#endif 914 915 if (for_tx) { 916 /* Assumption: B43_TXRING_SLOTS can be divided by TX_SLOTS_PER_FRAME */ 917 BUILD_BUG_ON(B43_TXRING_SLOTS % TX_SLOTS_PER_FRAME != 0); 918 919 ring->txhdr_cache = kcalloc(ring->nr_slots / TX_SLOTS_PER_FRAME, 920 b43_txhdr_size(dev), 921 GFP_KERNEL); 922 if (!ring->txhdr_cache) 923 goto err_kfree_meta; 924 925 /* test for ability to dma to txhdr_cache */ 926 dma_test = dma_map_single(dev->dev->dma_dev, 927 ring->txhdr_cache, 928 b43_txhdr_size(dev), 929 DMA_TO_DEVICE); 930 931 if (b43_dma_mapping_error(ring, dma_test, 932 b43_txhdr_size(dev), 1)) { 933 /* ugh realloc */ 934 kfree(ring->txhdr_cache); 935 ring->txhdr_cache = kcalloc(ring->nr_slots / TX_SLOTS_PER_FRAME, 936 b43_txhdr_size(dev), 937 GFP_KERNEL | GFP_DMA); 938 if (!ring->txhdr_cache) 939 goto err_kfree_meta; 940 941 dma_test = dma_map_single(dev->dev->dma_dev, 942 ring->txhdr_cache, 943 b43_txhdr_size(dev), 944 DMA_TO_DEVICE); 945 946 if (b43_dma_mapping_error(ring, dma_test, 947 b43_txhdr_size(dev), 1)) { 948 949 b43err(dev->wl, 950 "TXHDR DMA allocation failed\n"); 951 goto err_kfree_txhdr_cache; 952 } 953 } 954 955 dma_unmap_single(dev->dev->dma_dev, 956 dma_test, b43_txhdr_size(dev), 957 DMA_TO_DEVICE); 958 } 959 960 err = alloc_ringmemory(ring); 961 if (err) 962 goto err_kfree_txhdr_cache; 963 err = dmacontroller_setup(ring); 964 if (err) 965 goto err_free_ringmemory; 966 967 out: 968 return ring; 969 970 err_free_ringmemory: 971 free_ringmemory(ring); 972 err_kfree_txhdr_cache: 973 kfree(ring->txhdr_cache); 974 err_kfree_meta: 975 kfree(ring->meta); 976 err_kfree_ring: 977 kfree(ring); 978 ring = NULL; 979 goto out; 980} 981 982#define divide(a, b) ({ \ 983 typeof(a) __a = a; \ 984 do_div(__a, b); \ 985 __a; \ 986 }) 987 988#define modulo(a, b) ({ \ 989 typeof(a) __a = a; \ 990 do_div(__a, b); \ 991 }) 992 993/* Main cleanup function. */ 994static void b43_destroy_dmaring(struct b43_dmaring *ring, 995 const char *ringname) 996{ 997 if (!ring) 998 return; 999 1000#ifdef CONFIG_B43_DEBUG 1001 { 1002 /* Print some statistics. */ 1003 u64 failed_packets = ring->nr_failed_tx_packets; 1004 u64 succeed_packets = ring->nr_succeed_tx_packets; 1005 u64 nr_packets = failed_packets + succeed_packets; 1006 u64 permille_failed = 0, average_tries = 0; 1007 1008 if (nr_packets) 1009 permille_failed = divide(failed_packets * 1000, nr_packets); 1010 if (nr_packets) 1011 average_tries = divide(ring->nr_total_packet_tries * 100, nr_packets); 1012 1013 b43dbg(ring->dev->wl, "DMA-%u %s: " 1014 "Used slots %d/%d, Failed frames %llu/%llu = %llu.%01llu%%, " 1015 "Average tries %llu.%02llu\n", 1016 (unsigned int)(ring->type), ringname, 1017 ring->max_used_slots, 1018 ring->nr_slots, 1019 (unsigned long long)failed_packets, 1020 (unsigned long long)nr_packets, 1021 (unsigned long long)divide(permille_failed, 10), 1022 (unsigned long long)modulo(permille_failed, 10), 1023 (unsigned long long)divide(average_tries, 100), 1024 (unsigned long long)modulo(average_tries, 100)); 1025 } 1026#endif /* DEBUG */ 1027 1028 /* Device IRQs are disabled prior entering this function, 1029 * so no need to take care of concurrency with rx handler stuff. 1030 */ 1031 dmacontroller_cleanup(ring); 1032 free_all_descbuffers(ring); 1033 free_ringmemory(ring); 1034 1035 kfree(ring->txhdr_cache); 1036 kfree(ring->meta); 1037 kfree(ring); 1038} 1039 1040#define destroy_ring(dma, ring) do { \ 1041 b43_destroy_dmaring((dma)->ring, __stringify(ring)); \ 1042 (dma)->ring = NULL; \ 1043 } while (0) 1044 1045void b43_dma_free(struct b43_wldev *dev) 1046{ 1047 struct b43_dma *dma; 1048 1049 if (b43_using_pio_transfers(dev)) 1050 return; 1051 dma = &dev->dma; 1052 1053 destroy_ring(dma, rx_ring); 1054 destroy_ring(dma, tx_ring_AC_BK); 1055 destroy_ring(dma, tx_ring_AC_BE); 1056 destroy_ring(dma, tx_ring_AC_VI); 1057 destroy_ring(dma, tx_ring_AC_VO); 1058 destroy_ring(dma, tx_ring_mcast); 1059} 1060 1061static int b43_dma_set_mask(struct b43_wldev *dev, u64 mask) 1062{ 1063 u64 orig_mask = mask; 1064 bool fallback = false; 1065 int err; 1066 1067 /* Try to set the DMA mask. If it fails, try falling back to a 1068 * lower mask, as we can always also support a lower one. */ 1069 while (1) { 1070 err = dma_set_mask(dev->dev->dma_dev, mask); 1071 if (!err) { 1072 err = dma_set_coherent_mask(dev->dev->dma_dev, mask); 1073 if (!err) 1074 break; 1075 } 1076 if (mask == DMA_BIT_MASK(64)) { 1077 mask = DMA_BIT_MASK(32); 1078 fallback = true; 1079 continue; 1080 } 1081 if (mask == DMA_BIT_MASK(32)) { 1082 mask = DMA_BIT_MASK(30); 1083 fallback = true; 1084 continue; 1085 } 1086 b43err(dev->wl, "The machine/kernel does not support " 1087 "the required %u-bit DMA mask\n", 1088 (unsigned int)dma_mask_to_engine_type(orig_mask)); 1089 return -EOPNOTSUPP; 1090 } 1091 if (fallback) { 1092 b43info(dev->wl, "DMA mask fallback from %u-bit to %u-bit\n", 1093 (unsigned int)dma_mask_to_engine_type(orig_mask), 1094 (unsigned int)dma_mask_to_engine_type(mask)); 1095 } 1096 1097 return 0; 1098} 1099 1100/* Some hardware with 64-bit DMA seems to be bugged and looks for translation 1101 * bit in low address word instead of high one. 1102 */ 1103static bool b43_dma_translation_in_low_word(struct b43_wldev *dev, 1104 enum b43_dmatype type) 1105{ 1106 if (type != B43_DMA_64BIT) 1107 return 1; 1108 1109#ifdef CONFIG_B43_SSB 1110 if (dev->dev->bus_type == B43_BUS_SSB && 1111 dev->dev->sdev->bus->bustype == SSB_BUSTYPE_PCI && 1112 !(dev->dev->sdev->bus->host_pci->is_pcie && 1113 ssb_read32(dev->dev->sdev, SSB_TMSHIGH) & SSB_TMSHIGH_DMA64)) 1114 return 1; 1115#endif 1116 return 0; 1117} 1118 1119int b43_dma_init(struct b43_wldev *dev) 1120{ 1121 struct b43_dma *dma = &dev->dma; 1122 int err; 1123 u64 dmamask; 1124 enum b43_dmatype type; 1125 1126 dmamask = supported_dma_mask(dev); 1127 type = dma_mask_to_engine_type(dmamask); 1128 err = b43_dma_set_mask(dev, dmamask); 1129 if (err) 1130 return err; 1131 1132 switch (dev->dev->bus_type) { 1133#ifdef CONFIG_B43_BCMA 1134 case B43_BUS_BCMA: 1135 dma->translation = bcma_core_dma_translation(dev->dev->bdev); 1136 break; 1137#endif 1138#ifdef CONFIG_B43_SSB 1139 case B43_BUS_SSB: 1140 dma->translation = ssb_dma_translation(dev->dev->sdev); 1141 break; 1142#endif 1143 } 1144 dma->translation_in_low = b43_dma_translation_in_low_word(dev, type); 1145 1146 dma->parity = true; 1147#ifdef CONFIG_B43_BCMA 1148 /* TODO: find out which SSB devices need disabling parity */ 1149 if (dev->dev->bus_type == B43_BUS_BCMA) 1150 dma->parity = false; 1151#endif 1152 1153 err = -ENOMEM; 1154 /* setup TX DMA channels. */ 1155 dma->tx_ring_AC_BK = b43_setup_dmaring(dev, 0, 1, type); 1156 if (!dma->tx_ring_AC_BK) 1157 goto out; 1158 1159 dma->tx_ring_AC_BE = b43_setup_dmaring(dev, 1, 1, type); 1160 if (!dma->tx_ring_AC_BE) 1161 goto err_destroy_bk; 1162 1163 dma->tx_ring_AC_VI = b43_setup_dmaring(dev, 2, 1, type); 1164 if (!dma->tx_ring_AC_VI) 1165 goto err_destroy_be; 1166 1167 dma->tx_ring_AC_VO = b43_setup_dmaring(dev, 3, 1, type); 1168 if (!dma->tx_ring_AC_VO) 1169 goto err_destroy_vi; 1170 1171 dma->tx_ring_mcast = b43_setup_dmaring(dev, 4, 1, type); 1172 if (!dma->tx_ring_mcast) 1173 goto err_destroy_vo; 1174 1175 /* setup RX DMA channel. */ 1176 dma->rx_ring = b43_setup_dmaring(dev, 0, 0, type); 1177 if (!dma->rx_ring) 1178 goto err_destroy_mcast; 1179 1180 /* No support for the TX status DMA ring. */ 1181 B43_WARN_ON(dev->dev->core_rev < 5); 1182 1183 b43dbg(dev->wl, "%u-bit DMA initialized\n", 1184 (unsigned int)type); 1185 err = 0; 1186out: 1187 return err; 1188 1189err_destroy_mcast: 1190 destroy_ring(dma, tx_ring_mcast); 1191err_destroy_vo: 1192 destroy_ring(dma, tx_ring_AC_VO); 1193err_destroy_vi: 1194 destroy_ring(dma, tx_ring_AC_VI); 1195err_destroy_be: 1196 destroy_ring(dma, tx_ring_AC_BE); 1197err_destroy_bk: 1198 destroy_ring(dma, tx_ring_AC_BK); 1199 return err; 1200} 1201 1202/* Generate a cookie for the TX header. */ 1203static u16 generate_cookie(struct b43_dmaring *ring, int slot) 1204{ 1205 u16 cookie; 1206 1207 /* Use the upper 4 bits of the cookie as 1208 * DMA controller ID and store the slot number 1209 * in the lower 12 bits. 1210 * Note that the cookie must never be 0, as this 1211 * is a special value used in RX path. 1212 * It can also not be 0xFFFF because that is special 1213 * for multicast frames. 1214 */ 1215 cookie = (((u16)ring->index + 1) << 12); 1216 B43_WARN_ON(slot & ~0x0FFF); 1217 cookie |= (u16)slot; 1218 1219 return cookie; 1220} 1221 1222/* Inspect a cookie and find out to which controller/slot it belongs. */ 1223static 1224struct b43_dmaring *parse_cookie(struct b43_wldev *dev, u16 cookie, int *slot) 1225{ 1226 struct b43_dma *dma = &dev->dma; 1227 struct b43_dmaring *ring = NULL; 1228 1229 switch (cookie & 0xF000) { 1230 case 0x1000: 1231 ring = dma->tx_ring_AC_BK; 1232 break; 1233 case 0x2000: 1234 ring = dma->tx_ring_AC_BE; 1235 break; 1236 case 0x3000: 1237 ring = dma->tx_ring_AC_VI; 1238 break; 1239 case 0x4000: 1240 ring = dma->tx_ring_AC_VO; 1241 break; 1242 case 0x5000: 1243 ring = dma->tx_ring_mcast; 1244 break; 1245 } 1246 *slot = (cookie & 0x0FFF); 1247 if (unlikely(!ring || *slot < 0 || *slot >= ring->nr_slots)) { 1248 b43dbg(dev->wl, "TX-status contains " 1249 "invalid cookie: 0x%04X\n", cookie); 1250 return NULL; 1251 } 1252 1253 return ring; 1254} 1255 1256static int dma_tx_fragment(struct b43_dmaring *ring, 1257 struct sk_buff *skb) 1258{ 1259 const struct b43_dma_ops *ops = ring->ops; 1260 struct ieee80211_tx_info *info = IEEE80211_SKB_CB(skb); 1261 struct b43_private_tx_info *priv_info = b43_get_priv_tx_info(info); 1262 u8 *header; 1263 int slot, old_top_slot, old_used_slots; 1264 int err; 1265 struct b43_dmadesc_generic *desc; 1266 struct b43_dmadesc_meta *meta; 1267 struct b43_dmadesc_meta *meta_hdr; 1268 u16 cookie; 1269 size_t hdrsize = b43_txhdr_size(ring->dev); 1270 1271 /* Important note: If the number of used DMA slots per TX frame 1272 * is changed here, the TX_SLOTS_PER_FRAME definition at the top of 1273 * the file has to be updated, too! 1274 */ 1275 1276 old_top_slot = ring->current_slot; 1277 old_used_slots = ring->used_slots; 1278 1279 /* Get a slot for the header. */ 1280 slot = request_slot(ring); 1281 desc = ops->idx2desc(ring, slot, &meta_hdr); 1282 memset(meta_hdr, 0, sizeof(*meta_hdr)); 1283 1284 header = &(ring->txhdr_cache[(slot / TX_SLOTS_PER_FRAME) * hdrsize]); 1285 cookie = generate_cookie(ring, slot); 1286 err = b43_generate_txhdr(ring->dev, header, 1287 skb, info, cookie); 1288 if (unlikely(err)) { 1289 ring->current_slot = old_top_slot; 1290 ring->used_slots = old_used_slots; 1291 return err; 1292 } 1293 1294 meta_hdr->dmaaddr = map_descbuffer(ring, (unsigned char *)header, 1295 hdrsize, 1); 1296 if (b43_dma_mapping_error(ring, meta_hdr->dmaaddr, hdrsize, 1)) { 1297 ring->current_slot = old_top_slot; 1298 ring->used_slots = old_used_slots; 1299 return -EIO; 1300 } 1301 ops->fill_descriptor(ring, desc, meta_hdr->dmaaddr, 1302 hdrsize, 1, 0, 0); 1303 1304 /* Get a slot for the payload. */ 1305 slot = request_slot(ring); 1306 desc = ops->idx2desc(ring, slot, &meta); 1307 memset(meta, 0, sizeof(*meta)); 1308 1309 meta->skb = skb; 1310 meta->is_last_fragment = true; 1311 priv_info->bouncebuffer = NULL; 1312 1313 meta->dmaaddr = map_descbuffer(ring, skb->data, skb->len, 1); 1314 /* create a bounce buffer in zone_dma on mapping failure. */ 1315 if (b43_dma_mapping_error(ring, meta->dmaaddr, skb->len, 1)) { 1316 priv_info->bouncebuffer = kmemdup(skb->data, skb->len, 1317 GFP_ATOMIC | GFP_DMA); 1318 if (!priv_info->bouncebuffer) { 1319 ring->current_slot = old_top_slot; 1320 ring->used_slots = old_used_slots; 1321 err = -ENOMEM; 1322 goto out_unmap_hdr; 1323 } 1324 1325 meta->dmaaddr = map_descbuffer(ring, priv_info->bouncebuffer, skb->len, 1); 1326 if (b43_dma_mapping_error(ring, meta->dmaaddr, skb->len, 1)) { 1327 kfree(priv_info->bouncebuffer); 1328 priv_info->bouncebuffer = NULL; 1329 ring->current_slot = old_top_slot; 1330 ring->used_slots = old_used_slots; 1331 err = -EIO; 1332 goto out_unmap_hdr; 1333 } 1334 } 1335 1336 ops->fill_descriptor(ring, desc, meta->dmaaddr, skb->len, 0, 1, 1); 1337 1338 if (info->flags & IEEE80211_TX_CTL_SEND_AFTER_DTIM) { 1339 /* Tell the firmware about the cookie of the last 1340 * mcast frame, so it can clear the more-data bit in it. */ 1341 b43_shm_write16(ring->dev, B43_SHM_SHARED, 1342 B43_SHM_SH_MCASTCOOKIE, cookie); 1343 } 1344 /* Now transfer the whole frame. */ 1345 wmb(); 1346 ops->poke_tx(ring, next_slot(ring, slot)); 1347 return 0; 1348 1349out_unmap_hdr: 1350 unmap_descbuffer(ring, meta_hdr->dmaaddr, 1351 hdrsize, 1); 1352 return err; 1353} 1354 1355static inline int should_inject_overflow(struct b43_dmaring *ring) 1356{ 1357#ifdef CONFIG_B43_DEBUG 1358 if (unlikely(b43_debug(ring->dev, B43_DBG_DMAOVERFLOW))) { 1359 /* Check if we should inject another ringbuffer overflow 1360 * to test handling of this situation in the stack. */ 1361 unsigned long next_overflow; 1362 1363 next_overflow = ring->last_injected_overflow + HZ; 1364 if (time_after(jiffies, next_overflow)) { 1365 ring->last_injected_overflow = jiffies; 1366 b43dbg(ring->dev->wl, 1367 "Injecting TX ring overflow on " 1368 "DMA controller %d\n", ring->index); 1369 return 1; 1370 } 1371 } 1372#endif /* CONFIG_B43_DEBUG */ 1373 return 0; 1374} 1375 1376/* Static mapping of mac80211's queues (priorities) to b43 DMA rings. */ 1377static struct b43_dmaring *select_ring_by_priority(struct b43_wldev *dev, 1378 u8 queue_prio) 1379{ 1380 struct b43_dmaring *ring; 1381 1382 if (dev->qos_enabled) { 1383 /* 0 = highest priority */ 1384 switch (queue_prio) { 1385 default: 1386 B43_WARN_ON(1); 1387 /* fallthrough */ 1388 case 0: 1389 ring = dev->dma.tx_ring_AC_VO; 1390 break; 1391 case 1: 1392 ring = dev->dma.tx_ring_AC_VI; 1393 break; 1394 case 2: 1395 ring = dev->dma.tx_ring_AC_BE; 1396 break; 1397 case 3: 1398 ring = dev->dma.tx_ring_AC_BK; 1399 break; 1400 } 1401 } else 1402 ring = dev->dma.tx_ring_AC_BE; 1403 1404 return ring; 1405} 1406 1407int b43_dma_tx(struct b43_wldev *dev, struct sk_buff *skb) 1408{ 1409 struct b43_dmaring *ring; 1410 struct ieee80211_hdr *hdr; 1411 int err = 0; 1412 struct ieee80211_tx_info *info = IEEE80211_SKB_CB(skb); 1413 1414 hdr = (struct ieee80211_hdr *)skb->data; 1415 if (info->flags & IEEE80211_TX_CTL_SEND_AFTER_DTIM) { 1416 /* The multicast ring will be sent after the DTIM */ 1417 ring = dev->dma.tx_ring_mcast; 1418 /* Set the more-data bit. Ucode will clear it on 1419 * the last frame for us. */ 1420 hdr->frame_control |= cpu_to_le16(IEEE80211_FCTL_MOREDATA); 1421 } else { 1422 /* Decide by priority where to put this frame. */ 1423 ring = select_ring_by_priority( 1424 dev, skb_get_queue_mapping(skb)); 1425 } 1426 1427 B43_WARN_ON(!ring->tx); 1428 1429 if (unlikely(ring->stopped)) { 1430 /* We get here only because of a bug in mac80211. 1431 * Because of a race, one packet may be queued after 1432 * the queue is stopped, thus we got called when we shouldn't. 1433 * For now, just refuse the transmit. */ 1434 if (b43_debug(dev, B43_DBG_DMAVERBOSE)) 1435 b43err(dev->wl, "Packet after queue stopped\n"); 1436 err = -ENOSPC; 1437 goto out; 1438 } 1439 1440 if (unlikely(WARN_ON(free_slots(ring) < TX_SLOTS_PER_FRAME))) { 1441 /* If we get here, we have a real error with the queue 1442 * full, but queues not stopped. */ 1443 b43err(dev->wl, "DMA queue overflow\n"); 1444 err = -ENOSPC; 1445 goto out; 1446 } 1447 1448 /* Assign the queue number to the ring (if not already done before) 1449 * so TX status handling can use it. The queue to ring mapping is 1450 * static, so we don't need to store it per frame. */ 1451 ring->queue_prio = skb_get_queue_mapping(skb); 1452 1453 err = dma_tx_fragment(ring, skb); 1454 if (unlikely(err == -ENOKEY)) { 1455 /* Drop this packet, as we don't have the encryption key 1456 * anymore and must not transmit it unencrypted. */ 1457 dev_kfree_skb_any(skb); 1458 err = 0; 1459 goto out; 1460 } 1461 if (unlikely(err)) { 1462 b43err(dev->wl, "DMA tx mapping failure\n"); 1463 goto out; 1464 } 1465 if ((free_slots(ring) < TX_SLOTS_PER_FRAME) || 1466 should_inject_overflow(ring)) { 1467 /* This TX ring is full. */ 1468 unsigned int skb_mapping = skb_get_queue_mapping(skb); 1469 ieee80211_stop_queue(dev->wl->hw, skb_mapping); 1470 dev->wl->tx_queue_stopped[skb_mapping] = 1; 1471 ring->stopped = true; 1472 if (b43_debug(dev, B43_DBG_DMAVERBOSE)) { 1473 b43dbg(dev->wl, "Stopped TX ring %d\n", ring->index); 1474 } 1475 } 1476out: 1477 1478 return err; 1479} 1480 1481void b43_dma_handle_txstatus(struct b43_wldev *dev, 1482 const struct b43_txstatus *status) 1483{ 1484 const struct b43_dma_ops *ops; 1485 struct b43_dmaring *ring; 1486 struct b43_dmadesc_meta *meta; 1487 int slot, firstused; 1488 bool frame_succeed; 1489 1490 ring = parse_cookie(dev, status->cookie, &slot); 1491 if (unlikely(!ring)) 1492 return; 1493 B43_WARN_ON(!ring->tx); 1494 1495 /* Sanity check: TX packets are processed in-order on one ring. 1496 * Check if the slot deduced from the cookie really is the first 1497 * used slot. */ 1498 firstused = ring->current_slot - ring->used_slots + 1; 1499 if (firstused < 0) 1500 firstused = ring->nr_slots + firstused; 1501 if (unlikely(slot != firstused)) { 1502 /* This possibly is a firmware bug and will result in 1503 * malfunction, memory leaks and/or stall of DMA functionality. */ 1504 b43dbg(dev->wl, "Out of order TX status report on DMA ring %d. " 1505 "Expected %d, but got %d\n", 1506 ring->index, firstused, slot); 1507 return; 1508 } 1509 1510 ops = ring->ops; 1511 while (1) { 1512 B43_WARN_ON(slot < 0 || slot >= ring->nr_slots); 1513 /* get meta - ignore returned value */ 1514 ops->idx2desc(ring, slot, &meta); 1515 1516 if (b43_dma_ptr_is_poisoned(meta->skb)) { 1517 b43dbg(dev->wl, "Poisoned TX slot %d (first=%d) " 1518 "on ring %d\n", 1519 slot, firstused, ring->index); 1520 break; 1521 } 1522 if (meta->skb) { 1523 struct b43_private_tx_info *priv_info = 1524 b43_get_priv_tx_info(IEEE80211_SKB_CB(meta->skb)); 1525 1526 unmap_descbuffer(ring, meta->dmaaddr, meta->skb->len, 1); 1527 kfree(priv_info->bouncebuffer); 1528 priv_info->bouncebuffer = NULL; 1529 } else { 1530 unmap_descbuffer(ring, meta->dmaaddr, 1531 b43_txhdr_size(dev), 1); 1532 } 1533 1534 if (meta->is_last_fragment) { 1535 struct ieee80211_tx_info *info; 1536 1537 if (unlikely(!meta->skb)) { 1538 /* This is a scatter-gather fragment of a frame, so 1539 * the skb pointer must not be NULL. */ 1540 b43dbg(dev->wl, "TX status unexpected NULL skb " 1541 "at slot %d (first=%d) on ring %d\n", 1542 slot, firstused, ring->index); 1543 break; 1544 } 1545 1546 info = IEEE80211_SKB_CB(meta->skb); 1547 1548 /* 1549 * Call back to inform the ieee80211 subsystem about 1550 * the status of the transmission. 1551 */ 1552 frame_succeed = b43_fill_txstatus_report(dev, info, status); 1553#ifdef CONFIG_B43_DEBUG 1554 if (frame_succeed) 1555 ring->nr_succeed_tx_packets++; 1556 else 1557 ring->nr_failed_tx_packets++; 1558 ring->nr_total_packet_tries += status->frame_count; 1559#endif /* DEBUG */ 1560 ieee80211_tx_status(dev->wl->hw, meta->skb); 1561 1562 /* skb will be freed by ieee80211_tx_status(). 1563 * Poison our pointer. */ 1564 meta->skb = B43_DMA_PTR_POISON; 1565 } else { 1566 /* No need to call free_descriptor_buffer here, as 1567 * this is only the txhdr, which is not allocated. 1568 */ 1569 if (unlikely(meta->skb)) { 1570 b43dbg(dev->wl, "TX status unexpected non-NULL skb " 1571 "at slot %d (first=%d) on ring %d\n", 1572 slot, firstused, ring->index); 1573 break; 1574 } 1575 } 1576 1577 /* Everything unmapped and free'd. So it's not used anymore. */ 1578 ring->used_slots--; 1579 1580 if (meta->is_last_fragment) { 1581 /* This is the last scatter-gather 1582 * fragment of the frame. We are done. */ 1583 break; 1584 } 1585 slot = next_slot(ring, slot); 1586 } 1587 if (ring->stopped) { 1588 B43_WARN_ON(free_slots(ring) < TX_SLOTS_PER_FRAME); 1589 ring->stopped = false; 1590 } 1591 1592 if (dev->wl->tx_queue_stopped[ring->queue_prio]) { 1593 dev->wl->tx_queue_stopped[ring->queue_prio] = 0; 1594 } else { 1595 /* If the driver queue is running wake the corresponding 1596 * mac80211 queue. */ 1597 ieee80211_wake_queue(dev->wl->hw, ring->queue_prio); 1598 if (b43_debug(dev, B43_DBG_DMAVERBOSE)) { 1599 b43dbg(dev->wl, "Woke up TX ring %d\n", ring->index); 1600 } 1601 } 1602 /* Add work to the queue. */ 1603 ieee80211_queue_work(dev->wl->hw, &dev->wl->tx_work); 1604} 1605 1606static void dma_rx(struct b43_dmaring *ring, int *slot) 1607{ 1608 const struct b43_dma_ops *ops = ring->ops; 1609 struct b43_dmadesc_generic *desc; 1610 struct b43_dmadesc_meta *meta; 1611 struct b43_rxhdr_fw4 *rxhdr; 1612 struct sk_buff *skb; 1613 u16 len; 1614 int err; 1615 dma_addr_t dmaaddr; 1616 1617 desc = ops->idx2desc(ring, *slot, &meta); 1618 1619 sync_descbuffer_for_cpu(ring, meta->dmaaddr, ring->rx_buffersize); 1620 skb = meta->skb; 1621 1622 rxhdr = (struct b43_rxhdr_fw4 *)skb->data; 1623 len = le16_to_cpu(rxhdr->frame_len); 1624 if (len == 0) { 1625 int i = 0; 1626 1627 do { 1628 udelay(2); 1629 barrier(); 1630 len = le16_to_cpu(rxhdr->frame_len); 1631 } while (len == 0 && i++ < 5); 1632 if (unlikely(len == 0)) { 1633 dmaaddr = meta->dmaaddr; 1634 goto drop_recycle_buffer; 1635 } 1636 } 1637 if (unlikely(b43_rx_buffer_is_poisoned(ring, skb))) { 1638 /* Something went wrong with the DMA. 1639 * The device did not touch the buffer and did not overwrite the poison. */ 1640 b43dbg(ring->dev->wl, "DMA RX: Dropping poisoned buffer.\n"); 1641 dmaaddr = meta->dmaaddr; 1642 goto drop_recycle_buffer; 1643 } 1644 if (unlikely(len + ring->frameoffset > ring->rx_buffersize)) { 1645 /* The data did not fit into one descriptor buffer 1646 * and is split over multiple buffers. 1647 * This should never happen, as we try to allocate buffers 1648 * big enough. So simply ignore this packet. 1649 */ 1650 int cnt = 0; 1651 s32 tmp = len; 1652 1653 while (1) { 1654 desc = ops->idx2desc(ring, *slot, &meta); 1655 /* recycle the descriptor buffer. */ 1656 b43_poison_rx_buffer(ring, meta->skb); 1657 sync_descbuffer_for_device(ring, meta->dmaaddr, 1658 ring->rx_buffersize); 1659 *slot = next_slot(ring, *slot); 1660 cnt++; 1661 tmp -= ring->rx_buffersize; 1662 if (tmp <= 0) 1663 break; 1664 } 1665 b43err(ring->dev->wl, "DMA RX buffer too small " 1666 "(len: %u, buffer: %u, nr-dropped: %d)\n", 1667 len, ring->rx_buffersize, cnt); 1668 goto drop; 1669 } 1670 1671 dmaaddr = meta->dmaaddr; 1672 err = setup_rx_descbuffer(ring, desc, meta, GFP_ATOMIC); 1673 if (unlikely(err)) { 1674 b43dbg(ring->dev->wl, "DMA RX: setup_rx_descbuffer() failed\n"); 1675 goto drop_recycle_buffer; 1676 } 1677 1678 unmap_descbuffer(ring, dmaaddr, ring->rx_buffersize, 0); 1679 skb_put(skb, len + ring->frameoffset); 1680 skb_pull(skb, ring->frameoffset); 1681 1682 b43_rx(ring->dev, skb, rxhdr); 1683drop: 1684 return; 1685 1686drop_recycle_buffer: 1687 /* Poison and recycle the RX buffer. */ 1688 b43_poison_rx_buffer(ring, skb); 1689 sync_descbuffer_for_device(ring, dmaaddr, ring->rx_buffersize); 1690} 1691 1692void b43_dma_rx(struct b43_dmaring *ring) 1693{ 1694 const struct b43_dma_ops *ops = ring->ops; 1695 int slot, current_slot; 1696 int used_slots = 0; 1697 1698 B43_WARN_ON(ring->tx); 1699 current_slot = ops->get_current_rxslot(ring); 1700 B43_WARN_ON(!(current_slot >= 0 && current_slot < ring->nr_slots)); 1701 1702 slot = ring->current_slot; 1703 for (; slot != current_slot; slot = next_slot(ring, slot)) { 1704 dma_rx(ring, &slot); 1705 update_max_used_slots(ring, ++used_slots); 1706 } 1707 wmb(); 1708 ops->set_current_rxslot(ring, slot); 1709 ring->current_slot = slot; 1710} 1711 1712static void b43_dma_tx_suspend_ring(struct b43_dmaring *ring) 1713{ 1714 B43_WARN_ON(!ring->tx); 1715 ring->ops->tx_suspend(ring); 1716} 1717 1718static void b43_dma_tx_resume_ring(struct b43_dmaring *ring) 1719{ 1720 B43_WARN_ON(!ring->tx); 1721 ring->ops->tx_resume(ring); 1722} 1723 1724void b43_dma_tx_suspend(struct b43_wldev *dev) 1725{ 1726 b43_power_saving_ctl_bits(dev, B43_PS_AWAKE); 1727 b43_dma_tx_suspend_ring(dev->dma.tx_ring_AC_BK); 1728 b43_dma_tx_suspend_ring(dev->dma.tx_ring_AC_BE); 1729 b43_dma_tx_suspend_ring(dev->dma.tx_ring_AC_VI); 1730 b43_dma_tx_suspend_ring(dev->dma.tx_ring_AC_VO); 1731 b43_dma_tx_suspend_ring(dev->dma.tx_ring_mcast); 1732} 1733 1734void b43_dma_tx_resume(struct b43_wldev *dev) 1735{ 1736 b43_dma_tx_resume_ring(dev->dma.tx_ring_mcast); 1737 b43_dma_tx_resume_ring(dev->dma.tx_ring_AC_VO); 1738 b43_dma_tx_resume_ring(dev->dma.tx_ring_AC_VI); 1739 b43_dma_tx_resume_ring(dev->dma.tx_ring_AC_BE); 1740 b43_dma_tx_resume_ring(dev->dma.tx_ring_AC_BK); 1741 b43_power_saving_ctl_bits(dev, 0); 1742} 1743 1744static void direct_fifo_rx(struct b43_wldev *dev, enum b43_dmatype type, 1745 u16 mmio_base, bool enable) 1746{ 1747 u32 ctl; 1748 1749 if (type == B43_DMA_64BIT) { 1750 ctl = b43_read32(dev, mmio_base + B43_DMA64_RXCTL); 1751 ctl &= ~B43_DMA64_RXDIRECTFIFO; 1752 if (enable) 1753 ctl |= B43_DMA64_RXDIRECTFIFO; 1754 b43_write32(dev, mmio_base + B43_DMA64_RXCTL, ctl); 1755 } else { 1756 ctl = b43_read32(dev, mmio_base + B43_DMA32_RXCTL); 1757 ctl &= ~B43_DMA32_RXDIRECTFIFO; 1758 if (enable) 1759 ctl |= B43_DMA32_RXDIRECTFIFO; 1760 b43_write32(dev, mmio_base + B43_DMA32_RXCTL, ctl); 1761 } 1762} 1763 1764/* Enable/Disable Direct FIFO Receive Mode (PIO) on a RX engine. 1765 * This is called from PIO code, so DMA structures are not available. */ 1766void b43_dma_direct_fifo_rx(struct b43_wldev *dev, 1767 unsigned int engine_index, bool enable) 1768{ 1769 enum b43_dmatype type; 1770 u16 mmio_base; 1771 1772 type = dma_mask_to_engine_type(supported_dma_mask(dev)); 1773 1774 mmio_base = b43_dmacontroller_base(type, engine_index); 1775 direct_fifo_rx(dev, type, mmio_base, enable); 1776} 1777