ce.c revision 2e761b5a5222071d55ebccf65f93d281e4c11958
1/* 2 * Copyright (c) 2005-2011 Atheros Communications Inc. 3 * Copyright (c) 2011-2013 Qualcomm Atheros, Inc. 4 * 5 * Permission to use, copy, modify, and/or distribute this software for any 6 * purpose with or without fee is hereby granted, provided that the above 7 * copyright notice and this permission notice appear in all copies. 8 * 9 * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES 10 * WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF 11 * MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR 12 * ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES 13 * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN 14 * ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF 15 * OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. 16 */ 17 18#include "hif.h" 19#include "pci.h" 20#include "ce.h" 21#include "debug.h" 22 23/* 24 * Support for Copy Engine hardware, which is mainly used for 25 * communication between Host and Target over a PCIe interconnect. 26 */ 27 28/* 29 * A single CopyEngine (CE) comprises two "rings": 30 * a source ring 31 * a destination ring 32 * 33 * Each ring consists of a number of descriptors which specify 34 * an address, length, and meta-data. 35 * 36 * Typically, one side of the PCIe interconnect (Host or Target) 37 * controls one ring and the other side controls the other ring. 38 * The source side chooses when to initiate a transfer and it 39 * chooses what to send (buffer address, length). The destination 40 * side keeps a supply of "anonymous receive buffers" available and 41 * it handles incoming data as it arrives (when the destination 42 * recieves an interrupt). 43 * 44 * The sender may send a simple buffer (address/length) or it may 45 * send a small list of buffers. When a small list is sent, hardware 46 * "gathers" these and they end up in a single destination buffer 47 * with a single interrupt. 48 * 49 * There are several "contexts" managed by this layer -- more, it 50 * may seem -- than should be needed. These are provided mainly for 51 * maximum flexibility and especially to facilitate a simpler HIF 52 * implementation. There are per-CopyEngine recv, send, and watermark 53 * contexts. These are supplied by the caller when a recv, send, 54 * or watermark handler is established and they are echoed back to 55 * the caller when the respective callbacks are invoked. There is 56 * also a per-transfer context supplied by the caller when a buffer 57 * (or sendlist) is sent and when a buffer is enqueued for recv. 58 * These per-transfer contexts are echoed back to the caller when 59 * the buffer is sent/received. 60 */ 61 62static inline void ath10k_ce_dest_ring_write_index_set(struct ath10k *ar, 63 u32 ce_ctrl_addr, 64 unsigned int n) 65{ 66 ath10k_pci_write32(ar, ce_ctrl_addr + DST_WR_INDEX_ADDRESS, n); 67} 68 69static inline u32 ath10k_ce_dest_ring_write_index_get(struct ath10k *ar, 70 u32 ce_ctrl_addr) 71{ 72 return ath10k_pci_read32(ar, ce_ctrl_addr + DST_WR_INDEX_ADDRESS); 73} 74 75static inline void ath10k_ce_src_ring_write_index_set(struct ath10k *ar, 76 u32 ce_ctrl_addr, 77 unsigned int n) 78{ 79 ath10k_pci_write32(ar, ce_ctrl_addr + SR_WR_INDEX_ADDRESS, n); 80} 81 82static inline u32 ath10k_ce_src_ring_write_index_get(struct ath10k *ar, 83 u32 ce_ctrl_addr) 84{ 85 return ath10k_pci_read32(ar, ce_ctrl_addr + SR_WR_INDEX_ADDRESS); 86} 87 88static inline u32 ath10k_ce_src_ring_read_index_get(struct ath10k *ar, 89 u32 ce_ctrl_addr) 90{ 91 return ath10k_pci_read32(ar, ce_ctrl_addr + CURRENT_SRRI_ADDRESS); 92} 93 94static inline void ath10k_ce_src_ring_base_addr_set(struct ath10k *ar, 95 u32 ce_ctrl_addr, 96 unsigned int addr) 97{ 98 ath10k_pci_write32(ar, ce_ctrl_addr + SR_BA_ADDRESS, addr); 99} 100 101static inline void ath10k_ce_src_ring_size_set(struct ath10k *ar, 102 u32 ce_ctrl_addr, 103 unsigned int n) 104{ 105 ath10k_pci_write32(ar, ce_ctrl_addr + SR_SIZE_ADDRESS, n); 106} 107 108static inline void ath10k_ce_src_ring_dmax_set(struct ath10k *ar, 109 u32 ce_ctrl_addr, 110 unsigned int n) 111{ 112 u32 ctrl1_addr = ath10k_pci_read32((ar), 113 (ce_ctrl_addr) + CE_CTRL1_ADDRESS); 114 115 ath10k_pci_write32(ar, ce_ctrl_addr + CE_CTRL1_ADDRESS, 116 (ctrl1_addr & ~CE_CTRL1_DMAX_LENGTH_MASK) | 117 CE_CTRL1_DMAX_LENGTH_SET(n)); 118} 119 120static inline void ath10k_ce_src_ring_byte_swap_set(struct ath10k *ar, 121 u32 ce_ctrl_addr, 122 unsigned int n) 123{ 124 u32 ctrl1_addr = ath10k_pci_read32(ar, ce_ctrl_addr + CE_CTRL1_ADDRESS); 125 126 ath10k_pci_write32(ar, ce_ctrl_addr + CE_CTRL1_ADDRESS, 127 (ctrl1_addr & ~CE_CTRL1_SRC_RING_BYTE_SWAP_EN_MASK) | 128 CE_CTRL1_SRC_RING_BYTE_SWAP_EN_SET(n)); 129} 130 131static inline void ath10k_ce_dest_ring_byte_swap_set(struct ath10k *ar, 132 u32 ce_ctrl_addr, 133 unsigned int n) 134{ 135 u32 ctrl1_addr = ath10k_pci_read32(ar, ce_ctrl_addr + CE_CTRL1_ADDRESS); 136 137 ath10k_pci_write32(ar, ce_ctrl_addr + CE_CTRL1_ADDRESS, 138 (ctrl1_addr & ~CE_CTRL1_DST_RING_BYTE_SWAP_EN_MASK) | 139 CE_CTRL1_DST_RING_BYTE_SWAP_EN_SET(n)); 140} 141 142static inline u32 ath10k_ce_dest_ring_read_index_get(struct ath10k *ar, 143 u32 ce_ctrl_addr) 144{ 145 return ath10k_pci_read32(ar, ce_ctrl_addr + CURRENT_DRRI_ADDRESS); 146} 147 148static inline void ath10k_ce_dest_ring_base_addr_set(struct ath10k *ar, 149 u32 ce_ctrl_addr, 150 u32 addr) 151{ 152 ath10k_pci_write32(ar, ce_ctrl_addr + DR_BA_ADDRESS, addr); 153} 154 155static inline void ath10k_ce_dest_ring_size_set(struct ath10k *ar, 156 u32 ce_ctrl_addr, 157 unsigned int n) 158{ 159 ath10k_pci_write32(ar, ce_ctrl_addr + DR_SIZE_ADDRESS, n); 160} 161 162static inline void ath10k_ce_src_ring_highmark_set(struct ath10k *ar, 163 u32 ce_ctrl_addr, 164 unsigned int n) 165{ 166 u32 addr = ath10k_pci_read32(ar, ce_ctrl_addr + SRC_WATERMARK_ADDRESS); 167 168 ath10k_pci_write32(ar, ce_ctrl_addr + SRC_WATERMARK_ADDRESS, 169 (addr & ~SRC_WATERMARK_HIGH_MASK) | 170 SRC_WATERMARK_HIGH_SET(n)); 171} 172 173static inline void ath10k_ce_src_ring_lowmark_set(struct ath10k *ar, 174 u32 ce_ctrl_addr, 175 unsigned int n) 176{ 177 u32 addr = ath10k_pci_read32(ar, ce_ctrl_addr + SRC_WATERMARK_ADDRESS); 178 179 ath10k_pci_write32(ar, ce_ctrl_addr + SRC_WATERMARK_ADDRESS, 180 (addr & ~SRC_WATERMARK_LOW_MASK) | 181 SRC_WATERMARK_LOW_SET(n)); 182} 183 184static inline void ath10k_ce_dest_ring_highmark_set(struct ath10k *ar, 185 u32 ce_ctrl_addr, 186 unsigned int n) 187{ 188 u32 addr = ath10k_pci_read32(ar, ce_ctrl_addr + DST_WATERMARK_ADDRESS); 189 190 ath10k_pci_write32(ar, ce_ctrl_addr + DST_WATERMARK_ADDRESS, 191 (addr & ~DST_WATERMARK_HIGH_MASK) | 192 DST_WATERMARK_HIGH_SET(n)); 193} 194 195static inline void ath10k_ce_dest_ring_lowmark_set(struct ath10k *ar, 196 u32 ce_ctrl_addr, 197 unsigned int n) 198{ 199 u32 addr = ath10k_pci_read32(ar, ce_ctrl_addr + DST_WATERMARK_ADDRESS); 200 201 ath10k_pci_write32(ar, ce_ctrl_addr + DST_WATERMARK_ADDRESS, 202 (addr & ~DST_WATERMARK_LOW_MASK) | 203 DST_WATERMARK_LOW_SET(n)); 204} 205 206static inline void ath10k_ce_copy_complete_inter_enable(struct ath10k *ar, 207 u32 ce_ctrl_addr) 208{ 209 u32 host_ie_addr = ath10k_pci_read32(ar, 210 ce_ctrl_addr + HOST_IE_ADDRESS); 211 212 ath10k_pci_write32(ar, ce_ctrl_addr + HOST_IE_ADDRESS, 213 host_ie_addr | HOST_IE_COPY_COMPLETE_MASK); 214} 215 216static inline void ath10k_ce_copy_complete_intr_disable(struct ath10k *ar, 217 u32 ce_ctrl_addr) 218{ 219 u32 host_ie_addr = ath10k_pci_read32(ar, 220 ce_ctrl_addr + HOST_IE_ADDRESS); 221 222 ath10k_pci_write32(ar, ce_ctrl_addr + HOST_IE_ADDRESS, 223 host_ie_addr & ~HOST_IE_COPY_COMPLETE_MASK); 224} 225 226static inline void ath10k_ce_watermark_intr_disable(struct ath10k *ar, 227 u32 ce_ctrl_addr) 228{ 229 u32 host_ie_addr = ath10k_pci_read32(ar, 230 ce_ctrl_addr + HOST_IE_ADDRESS); 231 232 ath10k_pci_write32(ar, ce_ctrl_addr + HOST_IE_ADDRESS, 233 host_ie_addr & ~CE_WATERMARK_MASK); 234} 235 236static inline void ath10k_ce_error_intr_enable(struct ath10k *ar, 237 u32 ce_ctrl_addr) 238{ 239 u32 misc_ie_addr = ath10k_pci_read32(ar, 240 ce_ctrl_addr + MISC_IE_ADDRESS); 241 242 ath10k_pci_write32(ar, ce_ctrl_addr + MISC_IE_ADDRESS, 243 misc_ie_addr | CE_ERROR_MASK); 244} 245 246static inline void ath10k_ce_engine_int_status_clear(struct ath10k *ar, 247 u32 ce_ctrl_addr, 248 unsigned int mask) 249{ 250 ath10k_pci_write32(ar, ce_ctrl_addr + HOST_IS_ADDRESS, mask); 251} 252 253 254/* 255 * Guts of ath10k_ce_send, used by both ath10k_ce_send and 256 * ath10k_ce_sendlist_send. 257 * The caller takes responsibility for any needed locking. 258 */ 259static int ath10k_ce_send_nolock(struct ath10k_ce_pipe *ce_state, 260 void *per_transfer_context, 261 u32 buffer, 262 unsigned int nbytes, 263 unsigned int transfer_id, 264 unsigned int flags) 265{ 266 struct ath10k *ar = ce_state->ar; 267 struct ath10k_ce_ring *src_ring = ce_state->src_ring; 268 struct ce_desc *desc, *sdesc; 269 unsigned int nentries_mask = src_ring->nentries_mask; 270 unsigned int sw_index = src_ring->sw_index; 271 unsigned int write_index = src_ring->write_index; 272 u32 ctrl_addr = ce_state->ctrl_addr; 273 u32 desc_flags = 0; 274 int ret = 0; 275 276 if (nbytes > ce_state->src_sz_max) 277 ath10k_warn("%s: send more we can (nbytes: %d, max: %d)\n", 278 __func__, nbytes, ce_state->src_sz_max); 279 280 ret = ath10k_pci_wake(ar); 281 if (ret) 282 return ret; 283 284 if (unlikely(CE_RING_DELTA(nentries_mask, 285 write_index, sw_index - 1) <= 0)) { 286 ret = -EIO; 287 goto exit; 288 } 289 290 desc = CE_SRC_RING_TO_DESC(src_ring->base_addr_owner_space, 291 write_index); 292 sdesc = CE_SRC_RING_TO_DESC(src_ring->shadow_base, write_index); 293 294 desc_flags |= SM(transfer_id, CE_DESC_FLAGS_META_DATA); 295 296 if (flags & CE_SEND_FLAG_GATHER) 297 desc_flags |= CE_DESC_FLAGS_GATHER; 298 if (flags & CE_SEND_FLAG_BYTE_SWAP) 299 desc_flags |= CE_DESC_FLAGS_BYTE_SWAP; 300 301 sdesc->addr = __cpu_to_le32(buffer); 302 sdesc->nbytes = __cpu_to_le16(nbytes); 303 sdesc->flags = __cpu_to_le16(desc_flags); 304 305 *desc = *sdesc; 306 307 src_ring->per_transfer_context[write_index] = per_transfer_context; 308 309 /* Update Source Ring Write Index */ 310 write_index = CE_RING_IDX_INCR(nentries_mask, write_index); 311 312 /* WORKAROUND */ 313 if (!(flags & CE_SEND_FLAG_GATHER)) 314 ath10k_ce_src_ring_write_index_set(ar, ctrl_addr, write_index); 315 316 src_ring->write_index = write_index; 317exit: 318 ath10k_pci_sleep(ar); 319 return ret; 320} 321 322int ath10k_ce_send(struct ath10k_ce_pipe *ce_state, 323 void *per_transfer_context, 324 u32 buffer, 325 unsigned int nbytes, 326 unsigned int transfer_id, 327 unsigned int flags) 328{ 329 struct ath10k *ar = ce_state->ar; 330 struct ath10k_pci *ar_pci = ath10k_pci_priv(ar); 331 int ret; 332 333 spin_lock_bh(&ar_pci->ce_lock); 334 ret = ath10k_ce_send_nolock(ce_state, per_transfer_context, 335 buffer, nbytes, transfer_id, flags); 336 spin_unlock_bh(&ar_pci->ce_lock); 337 338 return ret; 339} 340 341int ath10k_ce_recv_buf_enqueue(struct ath10k_ce_pipe *ce_state, 342 void *per_recv_context, 343 u32 buffer) 344{ 345 struct ath10k_ce_ring *dest_ring = ce_state->dest_ring; 346 u32 ctrl_addr = ce_state->ctrl_addr; 347 struct ath10k *ar = ce_state->ar; 348 struct ath10k_pci *ar_pci = ath10k_pci_priv(ar); 349 unsigned int nentries_mask = dest_ring->nentries_mask; 350 unsigned int write_index; 351 unsigned int sw_index; 352 int ret; 353 354 spin_lock_bh(&ar_pci->ce_lock); 355 write_index = dest_ring->write_index; 356 sw_index = dest_ring->sw_index; 357 358 ret = ath10k_pci_wake(ar); 359 if (ret) 360 goto out; 361 362 if (CE_RING_DELTA(nentries_mask, write_index, sw_index - 1) > 0) { 363 struct ce_desc *base = dest_ring->base_addr_owner_space; 364 struct ce_desc *desc = CE_DEST_RING_TO_DESC(base, write_index); 365 366 /* Update destination descriptor */ 367 desc->addr = __cpu_to_le32(buffer); 368 desc->nbytes = 0; 369 370 dest_ring->per_transfer_context[write_index] = 371 per_recv_context; 372 373 /* Update Destination Ring Write Index */ 374 write_index = CE_RING_IDX_INCR(nentries_mask, write_index); 375 ath10k_ce_dest_ring_write_index_set(ar, ctrl_addr, write_index); 376 dest_ring->write_index = write_index; 377 ret = 0; 378 } else { 379 ret = -EIO; 380 } 381 ath10k_pci_sleep(ar); 382 383out: 384 spin_unlock_bh(&ar_pci->ce_lock); 385 386 return ret; 387} 388 389/* 390 * Guts of ath10k_ce_completed_recv_next. 391 * The caller takes responsibility for any necessary locking. 392 */ 393static int ath10k_ce_completed_recv_next_nolock(struct ath10k_ce_pipe *ce_state, 394 void **per_transfer_contextp, 395 u32 *bufferp, 396 unsigned int *nbytesp, 397 unsigned int *transfer_idp, 398 unsigned int *flagsp) 399{ 400 struct ath10k_ce_ring *dest_ring = ce_state->dest_ring; 401 unsigned int nentries_mask = dest_ring->nentries_mask; 402 unsigned int sw_index = dest_ring->sw_index; 403 404 struct ce_desc *base = dest_ring->base_addr_owner_space; 405 struct ce_desc *desc = CE_DEST_RING_TO_DESC(base, sw_index); 406 struct ce_desc sdesc; 407 u16 nbytes; 408 409 /* Copy in one go for performance reasons */ 410 sdesc = *desc; 411 412 nbytes = __le16_to_cpu(sdesc.nbytes); 413 if (nbytes == 0) { 414 /* 415 * This closes a relatively unusual race where the Host 416 * sees the updated DRRI before the update to the 417 * corresponding descriptor has completed. We treat this 418 * as a descriptor that is not yet done. 419 */ 420 return -EIO; 421 } 422 423 desc->nbytes = 0; 424 425 /* Return data from completed destination descriptor */ 426 *bufferp = __le32_to_cpu(sdesc.addr); 427 *nbytesp = nbytes; 428 *transfer_idp = MS(__le16_to_cpu(sdesc.flags), CE_DESC_FLAGS_META_DATA); 429 430 if (__le16_to_cpu(sdesc.flags) & CE_DESC_FLAGS_BYTE_SWAP) 431 *flagsp = CE_RECV_FLAG_SWAPPED; 432 else 433 *flagsp = 0; 434 435 if (per_transfer_contextp) 436 *per_transfer_contextp = 437 dest_ring->per_transfer_context[sw_index]; 438 439 /* sanity */ 440 dest_ring->per_transfer_context[sw_index] = NULL; 441 442 /* Update sw_index */ 443 sw_index = CE_RING_IDX_INCR(nentries_mask, sw_index); 444 dest_ring->sw_index = sw_index; 445 446 return 0; 447} 448 449int ath10k_ce_completed_recv_next(struct ath10k_ce_pipe *ce_state, 450 void **per_transfer_contextp, 451 u32 *bufferp, 452 unsigned int *nbytesp, 453 unsigned int *transfer_idp, 454 unsigned int *flagsp) 455{ 456 struct ath10k *ar = ce_state->ar; 457 struct ath10k_pci *ar_pci = ath10k_pci_priv(ar); 458 int ret; 459 460 spin_lock_bh(&ar_pci->ce_lock); 461 ret = ath10k_ce_completed_recv_next_nolock(ce_state, 462 per_transfer_contextp, 463 bufferp, nbytesp, 464 transfer_idp, flagsp); 465 spin_unlock_bh(&ar_pci->ce_lock); 466 467 return ret; 468} 469 470int ath10k_ce_revoke_recv_next(struct ath10k_ce_pipe *ce_state, 471 void **per_transfer_contextp, 472 u32 *bufferp) 473{ 474 struct ath10k_ce_ring *dest_ring; 475 unsigned int nentries_mask; 476 unsigned int sw_index; 477 unsigned int write_index; 478 int ret; 479 struct ath10k *ar; 480 struct ath10k_pci *ar_pci; 481 482 dest_ring = ce_state->dest_ring; 483 484 if (!dest_ring) 485 return -EIO; 486 487 ar = ce_state->ar; 488 ar_pci = ath10k_pci_priv(ar); 489 490 spin_lock_bh(&ar_pci->ce_lock); 491 492 nentries_mask = dest_ring->nentries_mask; 493 sw_index = dest_ring->sw_index; 494 write_index = dest_ring->write_index; 495 if (write_index != sw_index) { 496 struct ce_desc *base = dest_ring->base_addr_owner_space; 497 struct ce_desc *desc = CE_DEST_RING_TO_DESC(base, sw_index); 498 499 /* Return data from completed destination descriptor */ 500 *bufferp = __le32_to_cpu(desc->addr); 501 502 if (per_transfer_contextp) 503 *per_transfer_contextp = 504 dest_ring->per_transfer_context[sw_index]; 505 506 /* sanity */ 507 dest_ring->per_transfer_context[sw_index] = NULL; 508 509 /* Update sw_index */ 510 sw_index = CE_RING_IDX_INCR(nentries_mask, sw_index); 511 dest_ring->sw_index = sw_index; 512 ret = 0; 513 } else { 514 ret = -EIO; 515 } 516 517 spin_unlock_bh(&ar_pci->ce_lock); 518 519 return ret; 520} 521 522/* 523 * Guts of ath10k_ce_completed_send_next. 524 * The caller takes responsibility for any necessary locking. 525 */ 526static int ath10k_ce_completed_send_next_nolock(struct ath10k_ce_pipe *ce_state, 527 void **per_transfer_contextp, 528 u32 *bufferp, 529 unsigned int *nbytesp, 530 unsigned int *transfer_idp) 531{ 532 struct ath10k_ce_ring *src_ring = ce_state->src_ring; 533 u32 ctrl_addr = ce_state->ctrl_addr; 534 struct ath10k *ar = ce_state->ar; 535 unsigned int nentries_mask = src_ring->nentries_mask; 536 unsigned int sw_index = src_ring->sw_index; 537 struct ce_desc *sdesc, *sbase; 538 unsigned int read_index; 539 int ret; 540 541 if (src_ring->hw_index == sw_index) { 542 /* 543 * The SW completion index has caught up with the cached 544 * version of the HW completion index. 545 * Update the cached HW completion index to see whether 546 * the SW has really caught up to the HW, or if the cached 547 * value of the HW index has become stale. 548 */ 549 550 ret = ath10k_pci_wake(ar); 551 if (ret) 552 return ret; 553 554 src_ring->hw_index = 555 ath10k_ce_src_ring_read_index_get(ar, ctrl_addr); 556 src_ring->hw_index &= nentries_mask; 557 558 ath10k_pci_sleep(ar); 559 } 560 561 read_index = src_ring->hw_index; 562 563 if ((read_index == sw_index) || (read_index == 0xffffffff)) 564 return -EIO; 565 566 sbase = src_ring->shadow_base; 567 sdesc = CE_SRC_RING_TO_DESC(sbase, sw_index); 568 569 /* Return data from completed source descriptor */ 570 *bufferp = __le32_to_cpu(sdesc->addr); 571 *nbytesp = __le16_to_cpu(sdesc->nbytes); 572 *transfer_idp = MS(__le16_to_cpu(sdesc->flags), 573 CE_DESC_FLAGS_META_DATA); 574 575 if (per_transfer_contextp) 576 *per_transfer_contextp = 577 src_ring->per_transfer_context[sw_index]; 578 579 /* sanity */ 580 src_ring->per_transfer_context[sw_index] = NULL; 581 582 /* Update sw_index */ 583 sw_index = CE_RING_IDX_INCR(nentries_mask, sw_index); 584 src_ring->sw_index = sw_index; 585 586 return 0; 587} 588 589/* NB: Modeled after ath10k_ce_completed_send_next */ 590int ath10k_ce_cancel_send_next(struct ath10k_ce_pipe *ce_state, 591 void **per_transfer_contextp, 592 u32 *bufferp, 593 unsigned int *nbytesp, 594 unsigned int *transfer_idp) 595{ 596 struct ath10k_ce_ring *src_ring; 597 unsigned int nentries_mask; 598 unsigned int sw_index; 599 unsigned int write_index; 600 int ret; 601 struct ath10k *ar; 602 struct ath10k_pci *ar_pci; 603 604 src_ring = ce_state->src_ring; 605 606 if (!src_ring) 607 return -EIO; 608 609 ar = ce_state->ar; 610 ar_pci = ath10k_pci_priv(ar); 611 612 spin_lock_bh(&ar_pci->ce_lock); 613 614 nentries_mask = src_ring->nentries_mask; 615 sw_index = src_ring->sw_index; 616 write_index = src_ring->write_index; 617 618 if (write_index != sw_index) { 619 struct ce_desc *base = src_ring->base_addr_owner_space; 620 struct ce_desc *desc = CE_SRC_RING_TO_DESC(base, sw_index); 621 622 /* Return data from completed source descriptor */ 623 *bufferp = __le32_to_cpu(desc->addr); 624 *nbytesp = __le16_to_cpu(desc->nbytes); 625 *transfer_idp = MS(__le16_to_cpu(desc->flags), 626 CE_DESC_FLAGS_META_DATA); 627 628 if (per_transfer_contextp) 629 *per_transfer_contextp = 630 src_ring->per_transfer_context[sw_index]; 631 632 /* sanity */ 633 src_ring->per_transfer_context[sw_index] = NULL; 634 635 /* Update sw_index */ 636 sw_index = CE_RING_IDX_INCR(nentries_mask, sw_index); 637 src_ring->sw_index = sw_index; 638 ret = 0; 639 } else { 640 ret = -EIO; 641 } 642 643 spin_unlock_bh(&ar_pci->ce_lock); 644 645 return ret; 646} 647 648int ath10k_ce_completed_send_next(struct ath10k_ce_pipe *ce_state, 649 void **per_transfer_contextp, 650 u32 *bufferp, 651 unsigned int *nbytesp, 652 unsigned int *transfer_idp) 653{ 654 struct ath10k *ar = ce_state->ar; 655 struct ath10k_pci *ar_pci = ath10k_pci_priv(ar); 656 int ret; 657 658 spin_lock_bh(&ar_pci->ce_lock); 659 ret = ath10k_ce_completed_send_next_nolock(ce_state, 660 per_transfer_contextp, 661 bufferp, nbytesp, 662 transfer_idp); 663 spin_unlock_bh(&ar_pci->ce_lock); 664 665 return ret; 666} 667 668/* 669 * Guts of interrupt handler for per-engine interrupts on a particular CE. 670 * 671 * Invokes registered callbacks for recv_complete, 672 * send_complete, and watermarks. 673 */ 674void ath10k_ce_per_engine_service(struct ath10k *ar, unsigned int ce_id) 675{ 676 struct ath10k_pci *ar_pci = ath10k_pci_priv(ar); 677 struct ath10k_ce_pipe *ce_state = &ar_pci->ce_states[ce_id]; 678 u32 ctrl_addr = ce_state->ctrl_addr; 679 int ret; 680 681 ret = ath10k_pci_wake(ar); 682 if (ret) 683 return; 684 685 spin_lock_bh(&ar_pci->ce_lock); 686 687 /* Clear the copy-complete interrupts that will be handled here. */ 688 ath10k_ce_engine_int_status_clear(ar, ctrl_addr, 689 HOST_IS_COPY_COMPLETE_MASK); 690 691 spin_unlock_bh(&ar_pci->ce_lock); 692 693 if (ce_state->recv_cb) 694 ce_state->recv_cb(ce_state); 695 696 if (ce_state->send_cb) 697 ce_state->send_cb(ce_state); 698 699 spin_lock_bh(&ar_pci->ce_lock); 700 701 /* 702 * Misc CE interrupts are not being handled, but still need 703 * to be cleared. 704 */ 705 ath10k_ce_engine_int_status_clear(ar, ctrl_addr, CE_WATERMARK_MASK); 706 707 spin_unlock_bh(&ar_pci->ce_lock); 708 ath10k_pci_sleep(ar); 709} 710 711/* 712 * Handler for per-engine interrupts on ALL active CEs. 713 * This is used in cases where the system is sharing a 714 * single interrput for all CEs 715 */ 716 717void ath10k_ce_per_engine_service_any(struct ath10k *ar) 718{ 719 struct ath10k_pci *ar_pci = ath10k_pci_priv(ar); 720 int ce_id, ret; 721 u32 intr_summary; 722 723 ret = ath10k_pci_wake(ar); 724 if (ret) 725 return; 726 727 intr_summary = CE_INTERRUPT_SUMMARY(ar); 728 729 for (ce_id = 0; intr_summary && (ce_id < ar_pci->ce_count); ce_id++) { 730 if (intr_summary & (1 << ce_id)) 731 intr_summary &= ~(1 << ce_id); 732 else 733 /* no intr pending on this CE */ 734 continue; 735 736 ath10k_ce_per_engine_service(ar, ce_id); 737 } 738 739 ath10k_pci_sleep(ar); 740} 741 742/* 743 * Adjust interrupts for the copy complete handler. 744 * If it's needed for either send or recv, then unmask 745 * this interrupt; otherwise, mask it. 746 * 747 * Called with ce_lock held. 748 */ 749static void ath10k_ce_per_engine_handler_adjust(struct ath10k_ce_pipe *ce_state, 750 int disable_copy_compl_intr) 751{ 752 u32 ctrl_addr = ce_state->ctrl_addr; 753 struct ath10k *ar = ce_state->ar; 754 int ret; 755 756 ret = ath10k_pci_wake(ar); 757 if (ret) 758 return; 759 760 if ((!disable_copy_compl_intr) && 761 (ce_state->send_cb || ce_state->recv_cb)) 762 ath10k_ce_copy_complete_inter_enable(ar, ctrl_addr); 763 else 764 ath10k_ce_copy_complete_intr_disable(ar, ctrl_addr); 765 766 ath10k_ce_watermark_intr_disable(ar, ctrl_addr); 767 768 ath10k_pci_sleep(ar); 769} 770 771void ath10k_ce_disable_interrupts(struct ath10k *ar) 772{ 773 struct ath10k_pci *ar_pci = ath10k_pci_priv(ar); 774 int ce_id, ret; 775 776 ret = ath10k_pci_wake(ar); 777 if (ret) 778 return; 779 780 for (ce_id = 0; ce_id < ar_pci->ce_count; ce_id++) { 781 struct ath10k_ce_pipe *ce_state = &ar_pci->ce_states[ce_id]; 782 u32 ctrl_addr = ce_state->ctrl_addr; 783 784 ath10k_ce_copy_complete_intr_disable(ar, ctrl_addr); 785 } 786 ath10k_pci_sleep(ar); 787} 788 789void ath10k_ce_send_cb_register(struct ath10k_ce_pipe *ce_state, 790 void (*send_cb)(struct ath10k_ce_pipe *), 791 int disable_interrupts) 792{ 793 struct ath10k *ar = ce_state->ar; 794 struct ath10k_pci *ar_pci = ath10k_pci_priv(ar); 795 796 spin_lock_bh(&ar_pci->ce_lock); 797 ce_state->send_cb = send_cb; 798 ath10k_ce_per_engine_handler_adjust(ce_state, disable_interrupts); 799 spin_unlock_bh(&ar_pci->ce_lock); 800} 801 802void ath10k_ce_recv_cb_register(struct ath10k_ce_pipe *ce_state, 803 void (*recv_cb)(struct ath10k_ce_pipe *)) 804{ 805 struct ath10k *ar = ce_state->ar; 806 struct ath10k_pci *ar_pci = ath10k_pci_priv(ar); 807 808 spin_lock_bh(&ar_pci->ce_lock); 809 ce_state->recv_cb = recv_cb; 810 ath10k_ce_per_engine_handler_adjust(ce_state, 0); 811 spin_unlock_bh(&ar_pci->ce_lock); 812} 813 814static int ath10k_ce_init_src_ring(struct ath10k *ar, 815 unsigned int ce_id, 816 struct ath10k_ce_pipe *ce_state, 817 const struct ce_attr *attr) 818{ 819 struct ath10k_pci *ar_pci = ath10k_pci_priv(ar); 820 struct ath10k_ce_ring *src_ring; 821 unsigned int nentries = attr->src_nentries; 822 unsigned int ce_nbytes; 823 u32 ctrl_addr = ath10k_ce_base_address(ce_id); 824 dma_addr_t base_addr; 825 char *ptr; 826 827 nentries = roundup_pow_of_two(nentries); 828 829 if (ce_state->src_ring) { 830 WARN_ON(ce_state->src_ring->nentries != nentries); 831 return 0; 832 } 833 834 ce_nbytes = sizeof(struct ath10k_ce_ring) + (nentries * sizeof(void *)); 835 ptr = kzalloc(ce_nbytes, GFP_KERNEL); 836 if (ptr == NULL) 837 return -ENOMEM; 838 839 ce_state->src_ring = (struct ath10k_ce_ring *)ptr; 840 src_ring = ce_state->src_ring; 841 842 ptr += sizeof(struct ath10k_ce_ring); 843 src_ring->nentries = nentries; 844 src_ring->nentries_mask = nentries - 1; 845 846 src_ring->sw_index = ath10k_ce_src_ring_read_index_get(ar, ctrl_addr); 847 src_ring->sw_index &= src_ring->nentries_mask; 848 src_ring->hw_index = src_ring->sw_index; 849 850 src_ring->write_index = 851 ath10k_ce_src_ring_write_index_get(ar, ctrl_addr); 852 src_ring->write_index &= src_ring->nentries_mask; 853 854 src_ring->per_transfer_context = (void **)ptr; 855 856 /* 857 * Legacy platforms that do not support cache 858 * coherent DMA are unsupported 859 */ 860 src_ring->base_addr_owner_space_unaligned = 861 pci_alloc_consistent(ar_pci->pdev, 862 (nentries * sizeof(struct ce_desc) + 863 CE_DESC_RING_ALIGN), 864 &base_addr); 865 if (!src_ring->base_addr_owner_space_unaligned) { 866 kfree(ce_state->src_ring); 867 ce_state->src_ring = NULL; 868 return -ENOMEM; 869 } 870 871 src_ring->base_addr_ce_space_unaligned = base_addr; 872 873 src_ring->base_addr_owner_space = PTR_ALIGN( 874 src_ring->base_addr_owner_space_unaligned, 875 CE_DESC_RING_ALIGN); 876 src_ring->base_addr_ce_space = ALIGN( 877 src_ring->base_addr_ce_space_unaligned, 878 CE_DESC_RING_ALIGN); 879 880 /* 881 * Also allocate a shadow src ring in regular 882 * mem to use for faster access. 883 */ 884 src_ring->shadow_base_unaligned = 885 kmalloc((nentries * sizeof(struct ce_desc) + 886 CE_DESC_RING_ALIGN), GFP_KERNEL); 887 if (!src_ring->shadow_base_unaligned) { 888 pci_free_consistent(ar_pci->pdev, 889 (nentries * sizeof(struct ce_desc) + 890 CE_DESC_RING_ALIGN), 891 src_ring->base_addr_owner_space, 892 src_ring->base_addr_ce_space); 893 kfree(ce_state->src_ring); 894 ce_state->src_ring = NULL; 895 return -ENOMEM; 896 } 897 898 src_ring->shadow_base = PTR_ALIGN( 899 src_ring->shadow_base_unaligned, 900 CE_DESC_RING_ALIGN); 901 902 ath10k_ce_src_ring_base_addr_set(ar, ctrl_addr, 903 src_ring->base_addr_ce_space); 904 ath10k_ce_src_ring_size_set(ar, ctrl_addr, nentries); 905 ath10k_ce_src_ring_dmax_set(ar, ctrl_addr, attr->src_sz_max); 906 ath10k_ce_src_ring_byte_swap_set(ar, ctrl_addr, 0); 907 ath10k_ce_src_ring_lowmark_set(ar, ctrl_addr, 0); 908 ath10k_ce_src_ring_highmark_set(ar, ctrl_addr, nentries); 909 910 ath10k_dbg(ATH10K_DBG_BOOT, 911 "boot ce src ring id %d entries %d base_addr %p\n", 912 ce_id, nentries, src_ring->base_addr_owner_space); 913 914 return 0; 915} 916 917static int ath10k_ce_init_dest_ring(struct ath10k *ar, 918 unsigned int ce_id, 919 struct ath10k_ce_pipe *ce_state, 920 const struct ce_attr *attr) 921{ 922 struct ath10k_pci *ar_pci = ath10k_pci_priv(ar); 923 struct ath10k_ce_ring *dest_ring; 924 unsigned int nentries = attr->dest_nentries; 925 unsigned int ce_nbytes; 926 u32 ctrl_addr = ath10k_ce_base_address(ce_id); 927 dma_addr_t base_addr; 928 char *ptr; 929 930 nentries = roundup_pow_of_two(nentries); 931 932 if (ce_state->dest_ring) { 933 WARN_ON(ce_state->dest_ring->nentries != nentries); 934 return 0; 935 } 936 937 ce_nbytes = sizeof(struct ath10k_ce_ring) + (nentries * sizeof(void *)); 938 ptr = kzalloc(ce_nbytes, GFP_KERNEL); 939 if (ptr == NULL) 940 return -ENOMEM; 941 942 ce_state->dest_ring = (struct ath10k_ce_ring *)ptr; 943 dest_ring = ce_state->dest_ring; 944 945 ptr += sizeof(struct ath10k_ce_ring); 946 dest_ring->nentries = nentries; 947 dest_ring->nentries_mask = nentries - 1; 948 949 dest_ring->sw_index = ath10k_ce_dest_ring_read_index_get(ar, ctrl_addr); 950 dest_ring->sw_index &= dest_ring->nentries_mask; 951 dest_ring->write_index = 952 ath10k_ce_dest_ring_write_index_get(ar, ctrl_addr); 953 dest_ring->write_index &= dest_ring->nentries_mask; 954 955 dest_ring->per_transfer_context = (void **)ptr; 956 957 /* 958 * Legacy platforms that do not support cache 959 * coherent DMA are unsupported 960 */ 961 dest_ring->base_addr_owner_space_unaligned = 962 pci_alloc_consistent(ar_pci->pdev, 963 (nentries * sizeof(struct ce_desc) + 964 CE_DESC_RING_ALIGN), 965 &base_addr); 966 if (!dest_ring->base_addr_owner_space_unaligned) { 967 kfree(ce_state->dest_ring); 968 ce_state->dest_ring = NULL; 969 return -ENOMEM; 970 } 971 972 dest_ring->base_addr_ce_space_unaligned = base_addr; 973 974 /* 975 * Correctly initialize memory to 0 to prevent garbage 976 * data crashing system when download firmware 977 */ 978 memset(dest_ring->base_addr_owner_space_unaligned, 0, 979 nentries * sizeof(struct ce_desc) + CE_DESC_RING_ALIGN); 980 981 dest_ring->base_addr_owner_space = PTR_ALIGN( 982 dest_ring->base_addr_owner_space_unaligned, 983 CE_DESC_RING_ALIGN); 984 dest_ring->base_addr_ce_space = ALIGN( 985 dest_ring->base_addr_ce_space_unaligned, 986 CE_DESC_RING_ALIGN); 987 988 ath10k_ce_dest_ring_base_addr_set(ar, ctrl_addr, 989 dest_ring->base_addr_ce_space); 990 ath10k_ce_dest_ring_size_set(ar, ctrl_addr, nentries); 991 ath10k_ce_dest_ring_byte_swap_set(ar, ctrl_addr, 0); 992 ath10k_ce_dest_ring_lowmark_set(ar, ctrl_addr, 0); 993 ath10k_ce_dest_ring_highmark_set(ar, ctrl_addr, nentries); 994 995 ath10k_dbg(ATH10K_DBG_BOOT, 996 "boot ce dest ring id %d entries %d base_addr %p\n", 997 ce_id, nentries, dest_ring->base_addr_owner_space); 998 999 return 0; 1000} 1001 1002static struct ath10k_ce_pipe *ath10k_ce_init_state(struct ath10k *ar, 1003 unsigned int ce_id, 1004 const struct ce_attr *attr) 1005{ 1006 struct ath10k_pci *ar_pci = ath10k_pci_priv(ar); 1007 struct ath10k_ce_pipe *ce_state = &ar_pci->ce_states[ce_id]; 1008 u32 ctrl_addr = ath10k_ce_base_address(ce_id); 1009 1010 spin_lock_bh(&ar_pci->ce_lock); 1011 1012 ce_state->ar = ar; 1013 ce_state->id = ce_id; 1014 ce_state->ctrl_addr = ctrl_addr; 1015 ce_state->attr_flags = attr->flags; 1016 ce_state->src_sz_max = attr->src_sz_max; 1017 1018 spin_unlock_bh(&ar_pci->ce_lock); 1019 1020 return ce_state; 1021} 1022 1023/* 1024 * Initialize a Copy Engine based on caller-supplied attributes. 1025 * This may be called once to initialize both source and destination 1026 * rings or it may be called twice for separate source and destination 1027 * initialization. It may be that only one side or the other is 1028 * initialized by software/firmware. 1029 */ 1030struct ath10k_ce_pipe *ath10k_ce_init(struct ath10k *ar, 1031 unsigned int ce_id, 1032 const struct ce_attr *attr) 1033{ 1034 struct ath10k_ce_pipe *ce_state; 1035 u32 ctrl_addr = ath10k_ce_base_address(ce_id); 1036 int ret; 1037 1038 ret = ath10k_pci_wake(ar); 1039 if (ret) 1040 return NULL; 1041 1042 ce_state = ath10k_ce_init_state(ar, ce_id, attr); 1043 if (!ce_state) { 1044 ath10k_err("Failed to initialize CE state for ID: %d\n", ce_id); 1045 return NULL; 1046 } 1047 1048 if (attr->src_nentries) { 1049 ret = ath10k_ce_init_src_ring(ar, ce_id, ce_state, attr); 1050 if (ret) { 1051 ath10k_err("Failed to initialize CE src ring for ID: %d (%d)\n", 1052 ce_id, ret); 1053 ath10k_ce_deinit(ce_state); 1054 return NULL; 1055 } 1056 } 1057 1058 if (attr->dest_nentries) { 1059 ret = ath10k_ce_init_dest_ring(ar, ce_id, ce_state, attr); 1060 if (ret) { 1061 ath10k_err("Failed to initialize CE dest ring for ID: %d (%d)\n", 1062 ce_id, ret); 1063 ath10k_ce_deinit(ce_state); 1064 return NULL; 1065 } 1066 } 1067 1068 /* Enable CE error interrupts */ 1069 ath10k_ce_error_intr_enable(ar, ctrl_addr); 1070 1071 ath10k_pci_sleep(ar); 1072 1073 return ce_state; 1074} 1075 1076void ath10k_ce_deinit(struct ath10k_ce_pipe *ce_state) 1077{ 1078 struct ath10k *ar = ce_state->ar; 1079 struct ath10k_pci *ar_pci = ath10k_pci_priv(ar); 1080 1081 if (ce_state->src_ring) { 1082 kfree(ce_state->src_ring->shadow_base_unaligned); 1083 pci_free_consistent(ar_pci->pdev, 1084 (ce_state->src_ring->nentries * 1085 sizeof(struct ce_desc) + 1086 CE_DESC_RING_ALIGN), 1087 ce_state->src_ring->base_addr_owner_space, 1088 ce_state->src_ring->base_addr_ce_space); 1089 kfree(ce_state->src_ring); 1090 } 1091 1092 if (ce_state->dest_ring) { 1093 pci_free_consistent(ar_pci->pdev, 1094 (ce_state->dest_ring->nentries * 1095 sizeof(struct ce_desc) + 1096 CE_DESC_RING_ALIGN), 1097 ce_state->dest_ring->base_addr_owner_space, 1098 ce_state->dest_ring->base_addr_ce_space); 1099 kfree(ce_state->dest_ring); 1100 } 1101 1102 ce_state->src_ring = NULL; 1103 ce_state->dest_ring = NULL; 1104} 1105