ce.c revision 39e4086a7365b933265839090b3468c9ecc52a42
1/* 2 * Copyright (c) 2005-2011 Atheros Communications Inc. 3 * Copyright (c) 2011-2013 Qualcomm Atheros, Inc. 4 * 5 * Permission to use, copy, modify, and/or distribute this software for any 6 * purpose with or without fee is hereby granted, provided that the above 7 * copyright notice and this permission notice appear in all copies. 8 * 9 * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES 10 * WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF 11 * MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR 12 * ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES 13 * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN 14 * ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF 15 * OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. 16 */ 17 18#include "hif.h" 19#include "pci.h" 20#include "ce.h" 21#include "debug.h" 22 23/* 24 * Support for Copy Engine hardware, which is mainly used for 25 * communication between Host and Target over a PCIe interconnect. 26 */ 27 28/* 29 * A single CopyEngine (CE) comprises two "rings": 30 * a source ring 31 * a destination ring 32 * 33 * Each ring consists of a number of descriptors which specify 34 * an address, length, and meta-data. 35 * 36 * Typically, one side of the PCIe interconnect (Host or Target) 37 * controls one ring and the other side controls the other ring. 38 * The source side chooses when to initiate a transfer and it 39 * chooses what to send (buffer address, length). The destination 40 * side keeps a supply of "anonymous receive buffers" available and 41 * it handles incoming data as it arrives (when the destination 42 * recieves an interrupt). 43 * 44 * The sender may send a simple buffer (address/length) or it may 45 * send a small list of buffers. When a small list is sent, hardware 46 * "gathers" these and they end up in a single destination buffer 47 * with a single interrupt. 48 * 49 * There are several "contexts" managed by this layer -- more, it 50 * may seem -- than should be needed. These are provided mainly for 51 * maximum flexibility and especially to facilitate a simpler HIF 52 * implementation. There are per-CopyEngine recv, send, and watermark 53 * contexts. These are supplied by the caller when a recv, send, 54 * or watermark handler is established and they are echoed back to 55 * the caller when the respective callbacks are invoked. There is 56 * also a per-transfer context supplied by the caller when a buffer 57 * (or sendlist) is sent and when a buffer is enqueued for recv. 58 * These per-transfer contexts are echoed back to the caller when 59 * the buffer is sent/received. 60 */ 61 62static inline void ath10k_ce_dest_ring_write_index_set(struct ath10k *ar, 63 u32 ce_ctrl_addr, 64 unsigned int n) 65{ 66 ath10k_pci_write32(ar, ce_ctrl_addr + DST_WR_INDEX_ADDRESS, n); 67} 68 69static inline u32 ath10k_ce_dest_ring_write_index_get(struct ath10k *ar, 70 u32 ce_ctrl_addr) 71{ 72 return ath10k_pci_read32(ar, ce_ctrl_addr + DST_WR_INDEX_ADDRESS); 73} 74 75static inline void ath10k_ce_src_ring_write_index_set(struct ath10k *ar, 76 u32 ce_ctrl_addr, 77 unsigned int n) 78{ 79 ath10k_pci_write32(ar, ce_ctrl_addr + SR_WR_INDEX_ADDRESS, n); 80} 81 82static inline u32 ath10k_ce_src_ring_write_index_get(struct ath10k *ar, 83 u32 ce_ctrl_addr) 84{ 85 return ath10k_pci_read32(ar, ce_ctrl_addr + SR_WR_INDEX_ADDRESS); 86} 87 88static inline u32 ath10k_ce_src_ring_read_index_get(struct ath10k *ar, 89 u32 ce_ctrl_addr) 90{ 91 return ath10k_pci_read32(ar, ce_ctrl_addr + CURRENT_SRRI_ADDRESS); 92} 93 94static inline void ath10k_ce_src_ring_base_addr_set(struct ath10k *ar, 95 u32 ce_ctrl_addr, 96 unsigned int addr) 97{ 98 ath10k_pci_write32(ar, ce_ctrl_addr + SR_BA_ADDRESS, addr); 99} 100 101static inline void ath10k_ce_src_ring_size_set(struct ath10k *ar, 102 u32 ce_ctrl_addr, 103 unsigned int n) 104{ 105 ath10k_pci_write32(ar, ce_ctrl_addr + SR_SIZE_ADDRESS, n); 106} 107 108static inline void ath10k_ce_src_ring_dmax_set(struct ath10k *ar, 109 u32 ce_ctrl_addr, 110 unsigned int n) 111{ 112 u32 ctrl1_addr = ath10k_pci_read32((ar), 113 (ce_ctrl_addr) + CE_CTRL1_ADDRESS); 114 115 ath10k_pci_write32(ar, ce_ctrl_addr + CE_CTRL1_ADDRESS, 116 (ctrl1_addr & ~CE_CTRL1_DMAX_LENGTH_MASK) | 117 CE_CTRL1_DMAX_LENGTH_SET(n)); 118} 119 120static inline void ath10k_ce_src_ring_byte_swap_set(struct ath10k *ar, 121 u32 ce_ctrl_addr, 122 unsigned int n) 123{ 124 u32 ctrl1_addr = ath10k_pci_read32(ar, ce_ctrl_addr + CE_CTRL1_ADDRESS); 125 126 ath10k_pci_write32(ar, ce_ctrl_addr + CE_CTRL1_ADDRESS, 127 (ctrl1_addr & ~CE_CTRL1_SRC_RING_BYTE_SWAP_EN_MASK) | 128 CE_CTRL1_SRC_RING_BYTE_SWAP_EN_SET(n)); 129} 130 131static inline void ath10k_ce_dest_ring_byte_swap_set(struct ath10k *ar, 132 u32 ce_ctrl_addr, 133 unsigned int n) 134{ 135 u32 ctrl1_addr = ath10k_pci_read32(ar, ce_ctrl_addr + CE_CTRL1_ADDRESS); 136 137 ath10k_pci_write32(ar, ce_ctrl_addr + CE_CTRL1_ADDRESS, 138 (ctrl1_addr & ~CE_CTRL1_DST_RING_BYTE_SWAP_EN_MASK) | 139 CE_CTRL1_DST_RING_BYTE_SWAP_EN_SET(n)); 140} 141 142static inline u32 ath10k_ce_dest_ring_read_index_get(struct ath10k *ar, 143 u32 ce_ctrl_addr) 144{ 145 return ath10k_pci_read32(ar, ce_ctrl_addr + CURRENT_DRRI_ADDRESS); 146} 147 148static inline void ath10k_ce_dest_ring_base_addr_set(struct ath10k *ar, 149 u32 ce_ctrl_addr, 150 u32 addr) 151{ 152 ath10k_pci_write32(ar, ce_ctrl_addr + DR_BA_ADDRESS, addr); 153} 154 155static inline void ath10k_ce_dest_ring_size_set(struct ath10k *ar, 156 u32 ce_ctrl_addr, 157 unsigned int n) 158{ 159 ath10k_pci_write32(ar, ce_ctrl_addr + DR_SIZE_ADDRESS, n); 160} 161 162static inline void ath10k_ce_src_ring_highmark_set(struct ath10k *ar, 163 u32 ce_ctrl_addr, 164 unsigned int n) 165{ 166 u32 addr = ath10k_pci_read32(ar, ce_ctrl_addr + SRC_WATERMARK_ADDRESS); 167 168 ath10k_pci_write32(ar, ce_ctrl_addr + SRC_WATERMARK_ADDRESS, 169 (addr & ~SRC_WATERMARK_HIGH_MASK) | 170 SRC_WATERMARK_HIGH_SET(n)); 171} 172 173static inline void ath10k_ce_src_ring_lowmark_set(struct ath10k *ar, 174 u32 ce_ctrl_addr, 175 unsigned int n) 176{ 177 u32 addr = ath10k_pci_read32(ar, ce_ctrl_addr + SRC_WATERMARK_ADDRESS); 178 179 ath10k_pci_write32(ar, ce_ctrl_addr + SRC_WATERMARK_ADDRESS, 180 (addr & ~SRC_WATERMARK_LOW_MASK) | 181 SRC_WATERMARK_LOW_SET(n)); 182} 183 184static inline void ath10k_ce_dest_ring_highmark_set(struct ath10k *ar, 185 u32 ce_ctrl_addr, 186 unsigned int n) 187{ 188 u32 addr = ath10k_pci_read32(ar, ce_ctrl_addr + DST_WATERMARK_ADDRESS); 189 190 ath10k_pci_write32(ar, ce_ctrl_addr + DST_WATERMARK_ADDRESS, 191 (addr & ~DST_WATERMARK_HIGH_MASK) | 192 DST_WATERMARK_HIGH_SET(n)); 193} 194 195static inline void ath10k_ce_dest_ring_lowmark_set(struct ath10k *ar, 196 u32 ce_ctrl_addr, 197 unsigned int n) 198{ 199 u32 addr = ath10k_pci_read32(ar, ce_ctrl_addr + DST_WATERMARK_ADDRESS); 200 201 ath10k_pci_write32(ar, ce_ctrl_addr + DST_WATERMARK_ADDRESS, 202 (addr & ~DST_WATERMARK_LOW_MASK) | 203 DST_WATERMARK_LOW_SET(n)); 204} 205 206static inline void ath10k_ce_copy_complete_inter_enable(struct ath10k *ar, 207 u32 ce_ctrl_addr) 208{ 209 u32 host_ie_addr = ath10k_pci_read32(ar, 210 ce_ctrl_addr + HOST_IE_ADDRESS); 211 212 ath10k_pci_write32(ar, ce_ctrl_addr + HOST_IE_ADDRESS, 213 host_ie_addr | HOST_IE_COPY_COMPLETE_MASK); 214} 215 216static inline void ath10k_ce_copy_complete_intr_disable(struct ath10k *ar, 217 u32 ce_ctrl_addr) 218{ 219 u32 host_ie_addr = ath10k_pci_read32(ar, 220 ce_ctrl_addr + HOST_IE_ADDRESS); 221 222 ath10k_pci_write32(ar, ce_ctrl_addr + HOST_IE_ADDRESS, 223 host_ie_addr & ~HOST_IE_COPY_COMPLETE_MASK); 224} 225 226static inline void ath10k_ce_watermark_intr_disable(struct ath10k *ar, 227 u32 ce_ctrl_addr) 228{ 229 u32 host_ie_addr = ath10k_pci_read32(ar, 230 ce_ctrl_addr + HOST_IE_ADDRESS); 231 232 ath10k_pci_write32(ar, ce_ctrl_addr + HOST_IE_ADDRESS, 233 host_ie_addr & ~CE_WATERMARK_MASK); 234} 235 236static inline void ath10k_ce_error_intr_enable(struct ath10k *ar, 237 u32 ce_ctrl_addr) 238{ 239 u32 misc_ie_addr = ath10k_pci_read32(ar, 240 ce_ctrl_addr + MISC_IE_ADDRESS); 241 242 ath10k_pci_write32(ar, ce_ctrl_addr + MISC_IE_ADDRESS, 243 misc_ie_addr | CE_ERROR_MASK); 244} 245 246static inline void ath10k_ce_engine_int_status_clear(struct ath10k *ar, 247 u32 ce_ctrl_addr, 248 unsigned int mask) 249{ 250 ath10k_pci_write32(ar, ce_ctrl_addr + HOST_IS_ADDRESS, mask); 251} 252 253 254/* 255 * Guts of ath10k_ce_send, used by both ath10k_ce_send and 256 * ath10k_ce_sendlist_send. 257 * The caller takes responsibility for any needed locking. 258 */ 259static int ath10k_ce_send_nolock(struct ce_state *ce_state, 260 void *per_transfer_context, 261 u32 buffer, 262 unsigned int nbytes, 263 unsigned int transfer_id, 264 unsigned int flags) 265{ 266 struct ath10k *ar = ce_state->ar; 267 struct ce_ring_state *src_ring = ce_state->src_ring; 268 struct ce_desc *desc, *sdesc; 269 unsigned int nentries_mask = src_ring->nentries_mask; 270 unsigned int sw_index = src_ring->sw_index; 271 unsigned int write_index = src_ring->write_index; 272 u32 ctrl_addr = ce_state->ctrl_addr; 273 u32 desc_flags = 0; 274 int ret = 0; 275 276 if (nbytes > ce_state->src_sz_max) 277 ath10k_warn("%s: send more we can (nbytes: %d, max: %d)\n", 278 __func__, nbytes, ce_state->src_sz_max); 279 280 ath10k_pci_wake(ar); 281 282 if (unlikely(CE_RING_DELTA(nentries_mask, 283 write_index, sw_index - 1) <= 0)) { 284 ret = -EIO; 285 goto exit; 286 } 287 288 desc = CE_SRC_RING_TO_DESC(src_ring->base_addr_owner_space, 289 write_index); 290 sdesc = CE_SRC_RING_TO_DESC(src_ring->shadow_base, write_index); 291 292 desc_flags |= SM(transfer_id, CE_DESC_FLAGS_META_DATA); 293 294 if (flags & CE_SEND_FLAG_GATHER) 295 desc_flags |= CE_DESC_FLAGS_GATHER; 296 if (flags & CE_SEND_FLAG_BYTE_SWAP) 297 desc_flags |= CE_DESC_FLAGS_BYTE_SWAP; 298 299 sdesc->addr = __cpu_to_le32(buffer); 300 sdesc->nbytes = __cpu_to_le16(nbytes); 301 sdesc->flags = __cpu_to_le16(desc_flags); 302 303 *desc = *sdesc; 304 305 src_ring->per_transfer_context[write_index] = per_transfer_context; 306 307 /* Update Source Ring Write Index */ 308 write_index = CE_RING_IDX_INCR(nentries_mask, write_index); 309 310 /* WORKAROUND */ 311 if (!(flags & CE_SEND_FLAG_GATHER)) 312 ath10k_ce_src_ring_write_index_set(ar, ctrl_addr, write_index); 313 314 src_ring->write_index = write_index; 315exit: 316 ath10k_pci_sleep(ar); 317 return ret; 318} 319 320int ath10k_ce_send(struct ce_state *ce_state, 321 void *per_transfer_context, 322 u32 buffer, 323 unsigned int nbytes, 324 unsigned int transfer_id, 325 unsigned int flags) 326{ 327 struct ath10k *ar = ce_state->ar; 328 struct ath10k_pci *ar_pci = ath10k_pci_priv(ar); 329 int ret; 330 331 spin_lock_bh(&ar_pci->ce_lock); 332 ret = ath10k_ce_send_nolock(ce_state, per_transfer_context, 333 buffer, nbytes, transfer_id, flags); 334 spin_unlock_bh(&ar_pci->ce_lock); 335 336 return ret; 337} 338 339void ath10k_ce_sendlist_buf_add(struct ce_sendlist *sendlist, u32 buffer, 340 unsigned int nbytes, u32 flags) 341{ 342 unsigned int num_items = sendlist->num_items; 343 struct ce_sendlist_item *item; 344 345 item = &sendlist->item[num_items]; 346 item->data = buffer; 347 item->u.nbytes = nbytes; 348 item->flags = flags; 349 sendlist->num_items++; 350} 351 352int ath10k_ce_sendlist_send(struct ce_state *ce_state, 353 void *per_transfer_context, 354 struct ce_sendlist *sendlist, 355 unsigned int transfer_id) 356{ 357 struct ce_ring_state *src_ring = ce_state->src_ring; 358 struct ce_sendlist_item *item; 359 struct ath10k *ar = ce_state->ar; 360 struct ath10k_pci *ar_pci = ath10k_pci_priv(ar); 361 unsigned int nentries_mask = src_ring->nentries_mask; 362 unsigned int num_items = sendlist->num_items; 363 unsigned int sw_index; 364 unsigned int write_index; 365 int i, delta, ret = -ENOMEM; 366 367 spin_lock_bh(&ar_pci->ce_lock); 368 369 sw_index = src_ring->sw_index; 370 write_index = src_ring->write_index; 371 372 delta = CE_RING_DELTA(nentries_mask, write_index, sw_index - 1); 373 374 if (delta >= num_items) { 375 /* 376 * Handle all but the last item uniformly. 377 */ 378 for (i = 0; i < num_items - 1; i++) { 379 item = &sendlist->item[i]; 380 ret = ath10k_ce_send_nolock(ce_state, 381 CE_SENDLIST_ITEM_CTXT, 382 (u32) item->data, 383 item->u.nbytes, transfer_id, 384 item->flags | 385 CE_SEND_FLAG_GATHER); 386 if (ret) 387 ath10k_warn("CE send failed for item: %d\n", i); 388 } 389 /* 390 * Provide valid context pointer for final item. 391 */ 392 item = &sendlist->item[i]; 393 ret = ath10k_ce_send_nolock(ce_state, per_transfer_context, 394 (u32) item->data, item->u.nbytes, 395 transfer_id, item->flags); 396 if (ret) 397 ath10k_warn("CE send failed for last item: %d\n", i); 398 } 399 400 spin_unlock_bh(&ar_pci->ce_lock); 401 402 return ret; 403} 404 405int ath10k_ce_recv_buf_enqueue(struct ce_state *ce_state, 406 void *per_recv_context, 407 u32 buffer) 408{ 409 struct ce_ring_state *dest_ring = ce_state->dest_ring; 410 u32 ctrl_addr = ce_state->ctrl_addr; 411 struct ath10k *ar = ce_state->ar; 412 struct ath10k_pci *ar_pci = ath10k_pci_priv(ar); 413 unsigned int nentries_mask = dest_ring->nentries_mask; 414 unsigned int write_index; 415 unsigned int sw_index; 416 int ret; 417 418 spin_lock_bh(&ar_pci->ce_lock); 419 write_index = dest_ring->write_index; 420 sw_index = dest_ring->sw_index; 421 422 ath10k_pci_wake(ar); 423 424 if (CE_RING_DELTA(nentries_mask, write_index, sw_index - 1) > 0) { 425 struct ce_desc *base = dest_ring->base_addr_owner_space; 426 struct ce_desc *desc = CE_DEST_RING_TO_DESC(base, write_index); 427 428 /* Update destination descriptor */ 429 desc->addr = __cpu_to_le32(buffer); 430 desc->nbytes = 0; 431 432 dest_ring->per_transfer_context[write_index] = 433 per_recv_context; 434 435 /* Update Destination Ring Write Index */ 436 write_index = CE_RING_IDX_INCR(nentries_mask, write_index); 437 ath10k_ce_dest_ring_write_index_set(ar, ctrl_addr, write_index); 438 dest_ring->write_index = write_index; 439 ret = 0; 440 } else { 441 ret = -EIO; 442 } 443 ath10k_pci_sleep(ar); 444 spin_unlock_bh(&ar_pci->ce_lock); 445 446 return ret; 447} 448 449/* 450 * Guts of ath10k_ce_completed_recv_next. 451 * The caller takes responsibility for any necessary locking. 452 */ 453static int ath10k_ce_completed_recv_next_nolock(struct ce_state *ce_state, 454 void **per_transfer_contextp, 455 u32 *bufferp, 456 unsigned int *nbytesp, 457 unsigned int *transfer_idp, 458 unsigned int *flagsp) 459{ 460 struct ce_ring_state *dest_ring = ce_state->dest_ring; 461 unsigned int nentries_mask = dest_ring->nentries_mask; 462 unsigned int sw_index = dest_ring->sw_index; 463 464 struct ce_desc *base = dest_ring->base_addr_owner_space; 465 struct ce_desc *desc = CE_DEST_RING_TO_DESC(base, sw_index); 466 struct ce_desc sdesc; 467 u16 nbytes; 468 469 /* Copy in one go for performance reasons */ 470 sdesc = *desc; 471 472 nbytes = __le16_to_cpu(sdesc.nbytes); 473 if (nbytes == 0) { 474 /* 475 * This closes a relatively unusual race where the Host 476 * sees the updated DRRI before the update to the 477 * corresponding descriptor has completed. We treat this 478 * as a descriptor that is not yet done. 479 */ 480 return -EIO; 481 } 482 483 desc->nbytes = 0; 484 485 /* Return data from completed destination descriptor */ 486 *bufferp = __le32_to_cpu(sdesc.addr); 487 *nbytesp = nbytes; 488 *transfer_idp = MS(__le16_to_cpu(sdesc.flags), CE_DESC_FLAGS_META_DATA); 489 490 if (__le16_to_cpu(sdesc.flags) & CE_DESC_FLAGS_BYTE_SWAP) 491 *flagsp = CE_RECV_FLAG_SWAPPED; 492 else 493 *flagsp = 0; 494 495 if (per_transfer_contextp) 496 *per_transfer_contextp = 497 dest_ring->per_transfer_context[sw_index]; 498 499 /* sanity */ 500 dest_ring->per_transfer_context[sw_index] = NULL; 501 502 /* Update sw_index */ 503 sw_index = CE_RING_IDX_INCR(nentries_mask, sw_index); 504 dest_ring->sw_index = sw_index; 505 506 return 0; 507} 508 509int ath10k_ce_completed_recv_next(struct ce_state *ce_state, 510 void **per_transfer_contextp, 511 u32 *bufferp, 512 unsigned int *nbytesp, 513 unsigned int *transfer_idp, 514 unsigned int *flagsp) 515{ 516 struct ath10k *ar = ce_state->ar; 517 struct ath10k_pci *ar_pci = ath10k_pci_priv(ar); 518 int ret; 519 520 spin_lock_bh(&ar_pci->ce_lock); 521 ret = ath10k_ce_completed_recv_next_nolock(ce_state, 522 per_transfer_contextp, 523 bufferp, nbytesp, 524 transfer_idp, flagsp); 525 spin_unlock_bh(&ar_pci->ce_lock); 526 527 return ret; 528} 529 530int ath10k_ce_revoke_recv_next(struct ce_state *ce_state, 531 void **per_transfer_contextp, 532 u32 *bufferp) 533{ 534 struct ce_ring_state *dest_ring; 535 unsigned int nentries_mask; 536 unsigned int sw_index; 537 unsigned int write_index; 538 int ret; 539 struct ath10k *ar; 540 struct ath10k_pci *ar_pci; 541 542 dest_ring = ce_state->dest_ring; 543 544 if (!dest_ring) 545 return -EIO; 546 547 ar = ce_state->ar; 548 ar_pci = ath10k_pci_priv(ar); 549 550 spin_lock_bh(&ar_pci->ce_lock); 551 552 nentries_mask = dest_ring->nentries_mask; 553 sw_index = dest_ring->sw_index; 554 write_index = dest_ring->write_index; 555 if (write_index != sw_index) { 556 struct ce_desc *base = dest_ring->base_addr_owner_space; 557 struct ce_desc *desc = CE_DEST_RING_TO_DESC(base, sw_index); 558 559 /* Return data from completed destination descriptor */ 560 *bufferp = __le32_to_cpu(desc->addr); 561 562 if (per_transfer_contextp) 563 *per_transfer_contextp = 564 dest_ring->per_transfer_context[sw_index]; 565 566 /* sanity */ 567 dest_ring->per_transfer_context[sw_index] = NULL; 568 569 /* Update sw_index */ 570 sw_index = CE_RING_IDX_INCR(nentries_mask, sw_index); 571 dest_ring->sw_index = sw_index; 572 ret = 0; 573 } else { 574 ret = -EIO; 575 } 576 577 spin_unlock_bh(&ar_pci->ce_lock); 578 579 return ret; 580} 581 582/* 583 * Guts of ath10k_ce_completed_send_next. 584 * The caller takes responsibility for any necessary locking. 585 */ 586static int ath10k_ce_completed_send_next_nolock(struct ce_state *ce_state, 587 void **per_transfer_contextp, 588 u32 *bufferp, 589 unsigned int *nbytesp, 590 unsigned int *transfer_idp) 591{ 592 struct ce_ring_state *src_ring = ce_state->src_ring; 593 u32 ctrl_addr = ce_state->ctrl_addr; 594 struct ath10k *ar = ce_state->ar; 595 unsigned int nentries_mask = src_ring->nentries_mask; 596 unsigned int sw_index = src_ring->sw_index; 597 unsigned int read_index; 598 int ret = -EIO; 599 600 if (src_ring->hw_index == sw_index) { 601 /* 602 * The SW completion index has caught up with the cached 603 * version of the HW completion index. 604 * Update the cached HW completion index to see whether 605 * the SW has really caught up to the HW, or if the cached 606 * value of the HW index has become stale. 607 */ 608 ath10k_pci_wake(ar); 609 src_ring->hw_index = 610 ath10k_ce_src_ring_read_index_get(ar, ctrl_addr); 611 src_ring->hw_index &= nentries_mask; 612 ath10k_pci_sleep(ar); 613 } 614 read_index = src_ring->hw_index; 615 616 if ((read_index != sw_index) && (read_index != 0xffffffff)) { 617 struct ce_desc *sbase = src_ring->shadow_base; 618 struct ce_desc *sdesc = CE_SRC_RING_TO_DESC(sbase, sw_index); 619 620 /* Return data from completed source descriptor */ 621 *bufferp = __le32_to_cpu(sdesc->addr); 622 *nbytesp = __le16_to_cpu(sdesc->nbytes); 623 *transfer_idp = MS(__le16_to_cpu(sdesc->flags), 624 CE_DESC_FLAGS_META_DATA); 625 626 if (per_transfer_contextp) 627 *per_transfer_contextp = 628 src_ring->per_transfer_context[sw_index]; 629 630 /* sanity */ 631 src_ring->per_transfer_context[sw_index] = NULL; 632 633 /* Update sw_index */ 634 sw_index = CE_RING_IDX_INCR(nentries_mask, sw_index); 635 src_ring->sw_index = sw_index; 636 ret = 0; 637 } 638 639 return ret; 640} 641 642/* NB: Modeled after ath10k_ce_completed_send_next */ 643int ath10k_ce_cancel_send_next(struct ce_state *ce_state, 644 void **per_transfer_contextp, 645 u32 *bufferp, 646 unsigned int *nbytesp, 647 unsigned int *transfer_idp) 648{ 649 struct ce_ring_state *src_ring; 650 unsigned int nentries_mask; 651 unsigned int sw_index; 652 unsigned int write_index; 653 int ret; 654 struct ath10k *ar; 655 struct ath10k_pci *ar_pci; 656 657 src_ring = ce_state->src_ring; 658 659 if (!src_ring) 660 return -EIO; 661 662 ar = ce_state->ar; 663 ar_pci = ath10k_pci_priv(ar); 664 665 spin_lock_bh(&ar_pci->ce_lock); 666 667 nentries_mask = src_ring->nentries_mask; 668 sw_index = src_ring->sw_index; 669 write_index = src_ring->write_index; 670 671 if (write_index != sw_index) { 672 struct ce_desc *base = src_ring->base_addr_owner_space; 673 struct ce_desc *desc = CE_SRC_RING_TO_DESC(base, sw_index); 674 675 /* Return data from completed source descriptor */ 676 *bufferp = __le32_to_cpu(desc->addr); 677 *nbytesp = __le16_to_cpu(desc->nbytes); 678 *transfer_idp = MS(__le16_to_cpu(desc->flags), 679 CE_DESC_FLAGS_META_DATA); 680 681 if (per_transfer_contextp) 682 *per_transfer_contextp = 683 src_ring->per_transfer_context[sw_index]; 684 685 /* sanity */ 686 src_ring->per_transfer_context[sw_index] = NULL; 687 688 /* Update sw_index */ 689 sw_index = CE_RING_IDX_INCR(nentries_mask, sw_index); 690 src_ring->sw_index = sw_index; 691 ret = 0; 692 } else { 693 ret = -EIO; 694 } 695 696 spin_unlock_bh(&ar_pci->ce_lock); 697 698 return ret; 699} 700 701int ath10k_ce_completed_send_next(struct ce_state *ce_state, 702 void **per_transfer_contextp, 703 u32 *bufferp, 704 unsigned int *nbytesp, 705 unsigned int *transfer_idp) 706{ 707 struct ath10k *ar = ce_state->ar; 708 struct ath10k_pci *ar_pci = ath10k_pci_priv(ar); 709 int ret; 710 711 spin_lock_bh(&ar_pci->ce_lock); 712 ret = ath10k_ce_completed_send_next_nolock(ce_state, 713 per_transfer_contextp, 714 bufferp, nbytesp, 715 transfer_idp); 716 spin_unlock_bh(&ar_pci->ce_lock); 717 718 return ret; 719} 720 721/* 722 * Guts of interrupt handler for per-engine interrupts on a particular CE. 723 * 724 * Invokes registered callbacks for recv_complete, 725 * send_complete, and watermarks. 726 */ 727void ath10k_ce_per_engine_service(struct ath10k *ar, unsigned int ce_id) 728{ 729 struct ath10k_pci *ar_pci = ath10k_pci_priv(ar); 730 struct ce_state *ce_state = &ar_pci->ce_states[ce_id]; 731 u32 ctrl_addr = ce_state->ctrl_addr; 732 void *transfer_context; 733 u32 buf; 734 unsigned int nbytes; 735 unsigned int id; 736 unsigned int flags; 737 738 ath10k_pci_wake(ar); 739 spin_lock_bh(&ar_pci->ce_lock); 740 741 /* Clear the copy-complete interrupts that will be handled here. */ 742 ath10k_ce_engine_int_status_clear(ar, ctrl_addr, 743 HOST_IS_COPY_COMPLETE_MASK); 744 745 if (ce_state->recv_cb) { 746 /* 747 * Pop completed recv buffers and call the registered 748 * recv callback for each 749 */ 750 while (ath10k_ce_completed_recv_next_nolock(ce_state, 751 &transfer_context, 752 &buf, &nbytes, 753 &id, &flags) == 0) { 754 spin_unlock_bh(&ar_pci->ce_lock); 755 ce_state->recv_cb(ce_state, transfer_context, buf, 756 nbytes, id, flags); 757 spin_lock_bh(&ar_pci->ce_lock); 758 } 759 } 760 761 if (ce_state->send_cb) { 762 /* 763 * Pop completed send buffers and call the registered 764 * send callback for each 765 */ 766 while (ath10k_ce_completed_send_next_nolock(ce_state, 767 &transfer_context, 768 &buf, 769 &nbytes, 770 &id) == 0) { 771 spin_unlock_bh(&ar_pci->ce_lock); 772 ce_state->send_cb(ce_state, transfer_context, 773 buf, nbytes, id); 774 spin_lock_bh(&ar_pci->ce_lock); 775 } 776 } 777 778 /* 779 * Misc CE interrupts are not being handled, but still need 780 * to be cleared. 781 */ 782 ath10k_ce_engine_int_status_clear(ar, ctrl_addr, CE_WATERMARK_MASK); 783 784 spin_unlock_bh(&ar_pci->ce_lock); 785 ath10k_pci_sleep(ar); 786} 787 788/* 789 * Handler for per-engine interrupts on ALL active CEs. 790 * This is used in cases where the system is sharing a 791 * single interrput for all CEs 792 */ 793 794void ath10k_ce_per_engine_service_any(struct ath10k *ar) 795{ 796 struct ath10k_pci *ar_pci = ath10k_pci_priv(ar); 797 int ce_id; 798 u32 intr_summary; 799 800 ath10k_pci_wake(ar); 801 intr_summary = CE_INTERRUPT_SUMMARY(ar); 802 803 for (ce_id = 0; intr_summary && (ce_id < ar_pci->ce_count); ce_id++) { 804 if (intr_summary & (1 << ce_id)) 805 intr_summary &= ~(1 << ce_id); 806 else 807 /* no intr pending on this CE */ 808 continue; 809 810 ath10k_ce_per_engine_service(ar, ce_id); 811 } 812 813 ath10k_pci_sleep(ar); 814} 815 816/* 817 * Adjust interrupts for the copy complete handler. 818 * If it's needed for either send or recv, then unmask 819 * this interrupt; otherwise, mask it. 820 * 821 * Called with ce_lock held. 822 */ 823static void ath10k_ce_per_engine_handler_adjust(struct ce_state *ce_state, 824 int disable_copy_compl_intr) 825{ 826 u32 ctrl_addr = ce_state->ctrl_addr; 827 struct ath10k *ar = ce_state->ar; 828 829 ath10k_pci_wake(ar); 830 831 if ((!disable_copy_compl_intr) && 832 (ce_state->send_cb || ce_state->recv_cb)) 833 ath10k_ce_copy_complete_inter_enable(ar, ctrl_addr); 834 else 835 ath10k_ce_copy_complete_intr_disable(ar, ctrl_addr); 836 837 ath10k_ce_watermark_intr_disable(ar, ctrl_addr); 838 839 ath10k_pci_sleep(ar); 840} 841 842void ath10k_ce_disable_interrupts(struct ath10k *ar) 843{ 844 struct ath10k_pci *ar_pci = ath10k_pci_priv(ar); 845 int ce_id; 846 847 ath10k_pci_wake(ar); 848 for (ce_id = 0; ce_id < ar_pci->ce_count; ce_id++) { 849 struct ce_state *ce_state = &ar_pci->ce_states[ce_id]; 850 u32 ctrl_addr = ce_state->ctrl_addr; 851 852 ath10k_ce_copy_complete_intr_disable(ar, ctrl_addr); 853 } 854 ath10k_pci_sleep(ar); 855} 856 857void ath10k_ce_send_cb_register(struct ce_state *ce_state, 858 void (*send_cb) (struct ce_state *ce_state, 859 void *transfer_context, 860 u32 buffer, 861 unsigned int nbytes, 862 unsigned int transfer_id), 863 int disable_interrupts) 864{ 865 struct ath10k *ar = ce_state->ar; 866 struct ath10k_pci *ar_pci = ath10k_pci_priv(ar); 867 868 spin_lock_bh(&ar_pci->ce_lock); 869 ce_state->send_cb = send_cb; 870 ath10k_ce_per_engine_handler_adjust(ce_state, disable_interrupts); 871 spin_unlock_bh(&ar_pci->ce_lock); 872} 873 874void ath10k_ce_recv_cb_register(struct ce_state *ce_state, 875 void (*recv_cb) (struct ce_state *ce_state, 876 void *transfer_context, 877 u32 buffer, 878 unsigned int nbytes, 879 unsigned int transfer_id, 880 unsigned int flags)) 881{ 882 struct ath10k *ar = ce_state->ar; 883 struct ath10k_pci *ar_pci = ath10k_pci_priv(ar); 884 885 spin_lock_bh(&ar_pci->ce_lock); 886 ce_state->recv_cb = recv_cb; 887 ath10k_ce_per_engine_handler_adjust(ce_state, 0); 888 spin_unlock_bh(&ar_pci->ce_lock); 889} 890 891static int ath10k_ce_init_src_ring(struct ath10k *ar, 892 unsigned int ce_id, 893 struct ce_state *ce_state, 894 const struct ce_attr *attr) 895{ 896 struct ath10k_pci *ar_pci = ath10k_pci_priv(ar); 897 struct ce_ring_state *src_ring; 898 unsigned int nentries = attr->src_nentries; 899 unsigned int ce_nbytes; 900 u32 ctrl_addr = ath10k_ce_base_address(ce_id); 901 dma_addr_t base_addr; 902 char *ptr; 903 904 nentries = roundup_pow_of_two(nentries); 905 906 if (ce_state->src_ring) { 907 WARN_ON(ce_state->src_ring->nentries != nentries); 908 return 0; 909 } 910 911 ce_nbytes = sizeof(struct ce_ring_state) + (nentries * sizeof(void *)); 912 ptr = kzalloc(ce_nbytes, GFP_KERNEL); 913 if (ptr == NULL) 914 return -ENOMEM; 915 916 ce_state->src_ring = (struct ce_ring_state *)ptr; 917 src_ring = ce_state->src_ring; 918 919 ptr += sizeof(struct ce_ring_state); 920 src_ring->nentries = nentries; 921 src_ring->nentries_mask = nentries - 1; 922 923 ath10k_pci_wake(ar); 924 src_ring->sw_index = ath10k_ce_src_ring_read_index_get(ar, ctrl_addr); 925 src_ring->sw_index &= src_ring->nentries_mask; 926 src_ring->hw_index = src_ring->sw_index; 927 928 src_ring->write_index = 929 ath10k_ce_src_ring_write_index_get(ar, ctrl_addr); 930 src_ring->write_index &= src_ring->nentries_mask; 931 ath10k_pci_sleep(ar); 932 933 src_ring->per_transfer_context = (void **)ptr; 934 935 /* 936 * Legacy platforms that do not support cache 937 * coherent DMA are unsupported 938 */ 939 src_ring->base_addr_owner_space_unaligned = 940 pci_alloc_consistent(ar_pci->pdev, 941 (nentries * sizeof(struct ce_desc) + 942 CE_DESC_RING_ALIGN), 943 &base_addr); 944 if (!src_ring->base_addr_owner_space_unaligned) { 945 kfree(ce_state->src_ring); 946 ce_state->src_ring = NULL; 947 return -ENOMEM; 948 } 949 950 src_ring->base_addr_ce_space_unaligned = base_addr; 951 952 src_ring->base_addr_owner_space = PTR_ALIGN( 953 src_ring->base_addr_owner_space_unaligned, 954 CE_DESC_RING_ALIGN); 955 src_ring->base_addr_ce_space = ALIGN( 956 src_ring->base_addr_ce_space_unaligned, 957 CE_DESC_RING_ALIGN); 958 959 /* 960 * Also allocate a shadow src ring in regular 961 * mem to use for faster access. 962 */ 963 src_ring->shadow_base_unaligned = 964 kmalloc((nentries * sizeof(struct ce_desc) + 965 CE_DESC_RING_ALIGN), GFP_KERNEL); 966 if (!src_ring->shadow_base_unaligned) { 967 pci_free_consistent(ar_pci->pdev, 968 (nentries * sizeof(struct ce_desc) + 969 CE_DESC_RING_ALIGN), 970 src_ring->base_addr_owner_space, 971 src_ring->base_addr_ce_space); 972 kfree(ce_state->src_ring); 973 ce_state->src_ring = NULL; 974 return -ENOMEM; 975 } 976 977 src_ring->shadow_base = PTR_ALIGN( 978 src_ring->shadow_base_unaligned, 979 CE_DESC_RING_ALIGN); 980 981 ath10k_pci_wake(ar); 982 ath10k_ce_src_ring_base_addr_set(ar, ctrl_addr, 983 src_ring->base_addr_ce_space); 984 ath10k_ce_src_ring_size_set(ar, ctrl_addr, nentries); 985 ath10k_ce_src_ring_dmax_set(ar, ctrl_addr, attr->src_sz_max); 986 ath10k_ce_src_ring_byte_swap_set(ar, ctrl_addr, 0); 987 ath10k_ce_src_ring_lowmark_set(ar, ctrl_addr, 0); 988 ath10k_ce_src_ring_highmark_set(ar, ctrl_addr, nentries); 989 ath10k_pci_sleep(ar); 990 991 return 0; 992} 993 994static int ath10k_ce_init_dest_ring(struct ath10k *ar, 995 unsigned int ce_id, 996 struct ce_state *ce_state, 997 const struct ce_attr *attr) 998{ 999 struct ath10k_pci *ar_pci = ath10k_pci_priv(ar); 1000 struct ce_ring_state *dest_ring; 1001 unsigned int nentries = attr->dest_nentries; 1002 unsigned int ce_nbytes; 1003 u32 ctrl_addr = ath10k_ce_base_address(ce_id); 1004 dma_addr_t base_addr; 1005 char *ptr; 1006 1007 nentries = roundup_pow_of_two(nentries); 1008 1009 if (ce_state->dest_ring) { 1010 WARN_ON(ce_state->dest_ring->nentries != nentries); 1011 return 0; 1012 } 1013 1014 ce_nbytes = sizeof(struct ce_ring_state) + (nentries * sizeof(void *)); 1015 ptr = kzalloc(ce_nbytes, GFP_KERNEL); 1016 if (ptr == NULL) 1017 return -ENOMEM; 1018 1019 ce_state->dest_ring = (struct ce_ring_state *)ptr; 1020 dest_ring = ce_state->dest_ring; 1021 1022 ptr += sizeof(struct ce_ring_state); 1023 dest_ring->nentries = nentries; 1024 dest_ring->nentries_mask = nentries - 1; 1025 1026 ath10k_pci_wake(ar); 1027 dest_ring->sw_index = ath10k_ce_dest_ring_read_index_get(ar, ctrl_addr); 1028 dest_ring->sw_index &= dest_ring->nentries_mask; 1029 dest_ring->write_index = 1030 ath10k_ce_dest_ring_write_index_get(ar, ctrl_addr); 1031 dest_ring->write_index &= dest_ring->nentries_mask; 1032 ath10k_pci_sleep(ar); 1033 1034 dest_ring->per_transfer_context = (void **)ptr; 1035 1036 /* 1037 * Legacy platforms that do not support cache 1038 * coherent DMA are unsupported 1039 */ 1040 dest_ring->base_addr_owner_space_unaligned = 1041 pci_alloc_consistent(ar_pci->pdev, 1042 (nentries * sizeof(struct ce_desc) + 1043 CE_DESC_RING_ALIGN), 1044 &base_addr); 1045 if (!dest_ring->base_addr_owner_space_unaligned) { 1046 kfree(ce_state->dest_ring); 1047 ce_state->dest_ring = NULL; 1048 return -ENOMEM; 1049 } 1050 1051 dest_ring->base_addr_ce_space_unaligned = base_addr; 1052 1053 /* 1054 * Correctly initialize memory to 0 to prevent garbage 1055 * data crashing system when download firmware 1056 */ 1057 memset(dest_ring->base_addr_owner_space_unaligned, 0, 1058 nentries * sizeof(struct ce_desc) + CE_DESC_RING_ALIGN); 1059 1060 dest_ring->base_addr_owner_space = PTR_ALIGN( 1061 dest_ring->base_addr_owner_space_unaligned, 1062 CE_DESC_RING_ALIGN); 1063 dest_ring->base_addr_ce_space = ALIGN( 1064 dest_ring->base_addr_ce_space_unaligned, 1065 CE_DESC_RING_ALIGN); 1066 1067 ath10k_pci_wake(ar); 1068 ath10k_ce_dest_ring_base_addr_set(ar, ctrl_addr, 1069 dest_ring->base_addr_ce_space); 1070 ath10k_ce_dest_ring_size_set(ar, ctrl_addr, nentries); 1071 ath10k_ce_dest_ring_byte_swap_set(ar, ctrl_addr, 0); 1072 ath10k_ce_dest_ring_lowmark_set(ar, ctrl_addr, 0); 1073 ath10k_ce_dest_ring_highmark_set(ar, ctrl_addr, nentries); 1074 ath10k_pci_sleep(ar); 1075 1076 return 0; 1077} 1078 1079static struct ce_state *ath10k_ce_init_state(struct ath10k *ar, 1080 unsigned int ce_id, 1081 const struct ce_attr *attr) 1082{ 1083 struct ath10k_pci *ar_pci = ath10k_pci_priv(ar); 1084 struct ce_state *ce_state = &ar_pci->ce_states[ce_id]; 1085 u32 ctrl_addr = ath10k_ce_base_address(ce_id); 1086 1087 spin_lock_bh(&ar_pci->ce_lock); 1088 1089 ce_state->ar = ar; 1090 ce_state->id = ce_id; 1091 ce_state->ctrl_addr = ctrl_addr; 1092 ce_state->state = CE_RUNNING; 1093 ce_state->attr_flags = attr->flags; 1094 ce_state->src_sz_max = attr->src_sz_max; 1095 1096 spin_unlock_bh(&ar_pci->ce_lock); 1097 1098 return ce_state; 1099} 1100 1101/* 1102 * Initialize a Copy Engine based on caller-supplied attributes. 1103 * This may be called once to initialize both source and destination 1104 * rings or it may be called twice for separate source and destination 1105 * initialization. It may be that only one side or the other is 1106 * initialized by software/firmware. 1107 */ 1108struct ce_state *ath10k_ce_init(struct ath10k *ar, 1109 unsigned int ce_id, 1110 const struct ce_attr *attr) 1111{ 1112 struct ce_state *ce_state; 1113 u32 ctrl_addr = ath10k_ce_base_address(ce_id); 1114 int ret; 1115 1116 ce_state = ath10k_ce_init_state(ar, ce_id, attr); 1117 if (!ce_state) { 1118 ath10k_err("Failed to initialize CE state for ID: %d\n", ce_id); 1119 return NULL; 1120 } 1121 1122 if (attr->src_nentries) { 1123 ret = ath10k_ce_init_src_ring(ar, ce_id, ce_state, attr); 1124 if (ret) { 1125 ath10k_err("Failed to initialize CE src ring for ID: %d (%d)\n", 1126 ce_id, ret); 1127 ath10k_ce_deinit(ce_state); 1128 return NULL; 1129 } 1130 } 1131 1132 if (attr->dest_nentries) { 1133 ret = ath10k_ce_init_dest_ring(ar, ce_id, ce_state, attr); 1134 if (ret) { 1135 ath10k_err("Failed to initialize CE dest ring for ID: %d (%d)\n", 1136 ce_id, ret); 1137 ath10k_ce_deinit(ce_state); 1138 return NULL; 1139 } 1140 } 1141 1142 /* Enable CE error interrupts */ 1143 ath10k_pci_wake(ar); 1144 ath10k_ce_error_intr_enable(ar, ctrl_addr); 1145 ath10k_pci_sleep(ar); 1146 1147 return ce_state; 1148} 1149 1150void ath10k_ce_deinit(struct ce_state *ce_state) 1151{ 1152 struct ath10k *ar = ce_state->ar; 1153 struct ath10k_pci *ar_pci = ath10k_pci_priv(ar); 1154 1155 if (ce_state->src_ring) { 1156 kfree(ce_state->src_ring->shadow_base_unaligned); 1157 pci_free_consistent(ar_pci->pdev, 1158 (ce_state->src_ring->nentries * 1159 sizeof(struct ce_desc) + 1160 CE_DESC_RING_ALIGN), 1161 ce_state->src_ring->base_addr_owner_space, 1162 ce_state->src_ring->base_addr_ce_space); 1163 kfree(ce_state->src_ring); 1164 } 1165 1166 if (ce_state->dest_ring) { 1167 pci_free_consistent(ar_pci->pdev, 1168 (ce_state->dest_ring->nentries * 1169 sizeof(struct ce_desc) + 1170 CE_DESC_RING_ALIGN), 1171 ce_state->dest_ring->base_addr_owner_space, 1172 ce_state->dest_ring->base_addr_ce_space); 1173 kfree(ce_state->dest_ring); 1174 } 1175 1176 ce_state->state = CE_UNUSED; 1177 ce_state->src_ring = NULL; 1178 ce_state->dest_ring = NULL; 1179} 1180