1/* 2 * Copyright (c) 2007, 2008 QLogic Corporation. All rights reserved. 3 * 4 * This software is available to you under a choice of one of two 5 * licenses. You may choose to be licensed under the terms of the GNU 6 * General Public License (GPL) Version 2, available from the file 7 * COPYING in the main directory of this source tree, or the 8 * OpenIB.org BSD license below: 9 * 10 * Redistribution and use in source and binary forms, with or 11 * without modification, are permitted provided that the following 12 * conditions are met: 13 * 14 * - Redistributions of source code must retain the above 15 * copyright notice, this list of conditions and the following 16 * disclaimer. 17 * 18 * - Redistributions in binary form must reproduce the above 19 * copyright notice, this list of conditions and the following 20 * disclaimer in the documentation and/or other materials 21 * provided with the distribution. 22 * 23 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, 24 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF 25 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND 26 * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS 27 * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN 28 * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN 29 * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 30 * SOFTWARE. 31 */ 32 33#include <linux/spinlock.h> 34#include <linux/gfp.h> 35 36#include "ipath_kernel.h" 37#include "ipath_verbs.h" 38#include "ipath_common.h" 39 40#define SDMA_DESCQ_SZ PAGE_SIZE /* 256 entries per 4KB page */ 41 42static void vl15_watchdog_enq(struct ipath_devdata *dd) 43{ 44 /* ipath_sdma_lock must already be held */ 45 if (atomic_inc_return(&dd->ipath_sdma_vl15_count) == 1) { 46 unsigned long interval = (HZ + 19) / 20; 47 dd->ipath_sdma_vl15_timer.expires = jiffies + interval; 48 add_timer(&dd->ipath_sdma_vl15_timer); 49 } 50} 51 52static void vl15_watchdog_deq(struct ipath_devdata *dd) 53{ 54 /* ipath_sdma_lock must already be held */ 55 if (atomic_dec_return(&dd->ipath_sdma_vl15_count) != 0) { 56 unsigned long interval = (HZ + 19) / 20; 57 mod_timer(&dd->ipath_sdma_vl15_timer, jiffies + interval); 58 } else { 59 del_timer(&dd->ipath_sdma_vl15_timer); 60 } 61} 62 63static void vl15_watchdog_timeout(unsigned long opaque) 64{ 65 struct ipath_devdata *dd = (struct ipath_devdata *)opaque; 66 67 if (atomic_read(&dd->ipath_sdma_vl15_count) != 0) { 68 ipath_dbg("vl15 watchdog timeout - clearing\n"); 69 ipath_cancel_sends(dd, 1); 70 ipath_hol_down(dd); 71 } else { 72 ipath_dbg("vl15 watchdog timeout - " 73 "condition already cleared\n"); 74 } 75} 76 77static void unmap_desc(struct ipath_devdata *dd, unsigned head) 78{ 79 __le64 *descqp = &dd->ipath_sdma_descq[head].qw[0]; 80 u64 desc[2]; 81 dma_addr_t addr; 82 size_t len; 83 84 desc[0] = le64_to_cpu(descqp[0]); 85 desc[1] = le64_to_cpu(descqp[1]); 86 87 addr = (desc[1] << 32) | (desc[0] >> 32); 88 len = (desc[0] >> 14) & (0x7ffULL << 2); 89 dma_unmap_single(&dd->pcidev->dev, addr, len, DMA_TO_DEVICE); 90} 91 92/* 93 * ipath_sdma_lock should be locked before calling this. 94 */ 95int ipath_sdma_make_progress(struct ipath_devdata *dd) 96{ 97 struct list_head *lp = NULL; 98 struct ipath_sdma_txreq *txp = NULL; 99 u16 dmahead; 100 u16 start_idx = 0; 101 int progress = 0; 102 103 if (!list_empty(&dd->ipath_sdma_activelist)) { 104 lp = dd->ipath_sdma_activelist.next; 105 txp = list_entry(lp, struct ipath_sdma_txreq, list); 106 start_idx = txp->start_idx; 107 } 108 109 /* 110 * Read the SDMA head register in order to know that the 111 * interrupt clear has been written to the chip. 112 * Otherwise, we may not get an interrupt for the last 113 * descriptor in the queue. 114 */ 115 dmahead = (u16)ipath_read_kreg32(dd, dd->ipath_kregs->kr_senddmahead); 116 /* sanity check return value for error handling (chip reset, etc.) */ 117 if (dmahead >= dd->ipath_sdma_descq_cnt) 118 goto done; 119 120 while (dd->ipath_sdma_descq_head != dmahead) { 121 if (txp && txp->flags & IPATH_SDMA_TXREQ_F_FREEDESC && 122 dd->ipath_sdma_descq_head == start_idx) { 123 unmap_desc(dd, dd->ipath_sdma_descq_head); 124 start_idx++; 125 if (start_idx == dd->ipath_sdma_descq_cnt) 126 start_idx = 0; 127 } 128 129 /* increment free count and head */ 130 dd->ipath_sdma_descq_removed++; 131 if (++dd->ipath_sdma_descq_head == dd->ipath_sdma_descq_cnt) 132 dd->ipath_sdma_descq_head = 0; 133 134 if (txp && txp->next_descq_idx == dd->ipath_sdma_descq_head) { 135 /* move to notify list */ 136 if (txp->flags & IPATH_SDMA_TXREQ_F_VL15) 137 vl15_watchdog_deq(dd); 138 list_move_tail(lp, &dd->ipath_sdma_notifylist); 139 if (!list_empty(&dd->ipath_sdma_activelist)) { 140 lp = dd->ipath_sdma_activelist.next; 141 txp = list_entry(lp, struct ipath_sdma_txreq, 142 list); 143 start_idx = txp->start_idx; 144 } else { 145 lp = NULL; 146 txp = NULL; 147 } 148 } 149 progress = 1; 150 } 151 152 if (progress) 153 tasklet_hi_schedule(&dd->ipath_sdma_notify_task); 154 155done: 156 return progress; 157} 158 159static void ipath_sdma_notify(struct ipath_devdata *dd, struct list_head *list) 160{ 161 struct ipath_sdma_txreq *txp, *txp_next; 162 163 list_for_each_entry_safe(txp, txp_next, list, list) { 164 list_del_init(&txp->list); 165 166 if (txp->callback) 167 (*txp->callback)(txp->callback_cookie, 168 txp->callback_status); 169 } 170} 171 172static void sdma_notify_taskbody(struct ipath_devdata *dd) 173{ 174 unsigned long flags; 175 struct list_head list; 176 177 INIT_LIST_HEAD(&list); 178 179 spin_lock_irqsave(&dd->ipath_sdma_lock, flags); 180 181 list_splice_init(&dd->ipath_sdma_notifylist, &list); 182 183 spin_unlock_irqrestore(&dd->ipath_sdma_lock, flags); 184 185 ipath_sdma_notify(dd, &list); 186 187 /* 188 * The IB verbs layer needs to see the callback before getting 189 * the call to ipath_ib_piobufavail() because the callback 190 * handles releasing resources the next send will need. 191 * Otherwise, we could do these calls in 192 * ipath_sdma_make_progress(). 193 */ 194 ipath_ib_piobufavail(dd->verbs_dev); 195} 196 197static void sdma_notify_task(unsigned long opaque) 198{ 199 struct ipath_devdata *dd = (struct ipath_devdata *)opaque; 200 201 if (!test_bit(IPATH_SDMA_SHUTDOWN, &dd->ipath_sdma_status)) 202 sdma_notify_taskbody(dd); 203} 204 205static void dump_sdma_state(struct ipath_devdata *dd) 206{ 207 unsigned long reg; 208 209 reg = ipath_read_kreg64(dd, dd->ipath_kregs->kr_senddmastatus); 210 ipath_cdbg(VERBOSE, "kr_senddmastatus: 0x%016lx\n", reg); 211 212 reg = ipath_read_kreg64(dd, dd->ipath_kregs->kr_sendctrl); 213 ipath_cdbg(VERBOSE, "kr_sendctrl: 0x%016lx\n", reg); 214 215 reg = ipath_read_kreg64(dd, dd->ipath_kregs->kr_senddmabufmask0); 216 ipath_cdbg(VERBOSE, "kr_senddmabufmask0: 0x%016lx\n", reg); 217 218 reg = ipath_read_kreg64(dd, dd->ipath_kregs->kr_senddmabufmask1); 219 ipath_cdbg(VERBOSE, "kr_senddmabufmask1: 0x%016lx\n", reg); 220 221 reg = ipath_read_kreg64(dd, dd->ipath_kregs->kr_senddmabufmask2); 222 ipath_cdbg(VERBOSE, "kr_senddmabufmask2: 0x%016lx\n", reg); 223 224 reg = ipath_read_kreg64(dd, dd->ipath_kregs->kr_senddmatail); 225 ipath_cdbg(VERBOSE, "kr_senddmatail: 0x%016lx\n", reg); 226 227 reg = ipath_read_kreg64(dd, dd->ipath_kregs->kr_senddmahead); 228 ipath_cdbg(VERBOSE, "kr_senddmahead: 0x%016lx\n", reg); 229} 230 231static void sdma_abort_task(unsigned long opaque) 232{ 233 struct ipath_devdata *dd = (struct ipath_devdata *) opaque; 234 u64 status; 235 unsigned long flags; 236 237 if (test_bit(IPATH_SDMA_SHUTDOWN, &dd->ipath_sdma_status)) 238 return; 239 240 spin_lock_irqsave(&dd->ipath_sdma_lock, flags); 241 242 status = dd->ipath_sdma_status & IPATH_SDMA_ABORT_MASK; 243 244 /* nothing to do */ 245 if (status == IPATH_SDMA_ABORT_NONE) 246 goto unlock; 247 248 /* ipath_sdma_abort() is done, waiting for interrupt */ 249 if (status == IPATH_SDMA_ABORT_DISARMED) { 250 if (time_before(jiffies, dd->ipath_sdma_abort_intr_timeout)) 251 goto resched_noprint; 252 /* give up, intr got lost somewhere */ 253 ipath_dbg("give up waiting for SDMADISABLED intr\n"); 254 __set_bit(IPATH_SDMA_DISABLED, &dd->ipath_sdma_status); 255 status = IPATH_SDMA_ABORT_ABORTED; 256 } 257 258 /* everything is stopped, time to clean up and restart */ 259 if (status == IPATH_SDMA_ABORT_ABORTED) { 260 struct ipath_sdma_txreq *txp, *txpnext; 261 u64 hwstatus; 262 int notify = 0; 263 264 hwstatus = ipath_read_kreg64(dd, 265 dd->ipath_kregs->kr_senddmastatus); 266 267 if ((hwstatus & (IPATH_SDMA_STATUS_SCORE_BOARD_DRAIN_IN_PROG | 268 IPATH_SDMA_STATUS_ABORT_IN_PROG | 269 IPATH_SDMA_STATUS_INTERNAL_SDMA_ENABLE)) || 270 !(hwstatus & IPATH_SDMA_STATUS_SCB_EMPTY)) { 271 if (dd->ipath_sdma_reset_wait > 0) { 272 /* not done shutting down sdma */ 273 --dd->ipath_sdma_reset_wait; 274 goto resched; 275 } 276 ipath_cdbg(VERBOSE, "gave up waiting for quiescent " 277 "status after SDMA reset, continuing\n"); 278 dump_sdma_state(dd); 279 } 280 281 /* dequeue all "sent" requests */ 282 list_for_each_entry_safe(txp, txpnext, 283 &dd->ipath_sdma_activelist, list) { 284 txp->callback_status = IPATH_SDMA_TXREQ_S_ABORTED; 285 if (txp->flags & IPATH_SDMA_TXREQ_F_VL15) 286 vl15_watchdog_deq(dd); 287 list_move_tail(&txp->list, &dd->ipath_sdma_notifylist); 288 notify = 1; 289 } 290 if (notify) 291 tasklet_hi_schedule(&dd->ipath_sdma_notify_task); 292 293 /* reset our notion of head and tail */ 294 dd->ipath_sdma_descq_tail = 0; 295 dd->ipath_sdma_descq_head = 0; 296 dd->ipath_sdma_head_dma[0] = 0; 297 dd->ipath_sdma_generation = 0; 298 dd->ipath_sdma_descq_removed = dd->ipath_sdma_descq_added; 299 300 /* Reset SendDmaLenGen */ 301 ipath_write_kreg(dd, dd->ipath_kregs->kr_senddmalengen, 302 (u64) dd->ipath_sdma_descq_cnt | (1ULL << 18)); 303 304 /* done with sdma state for a bit */ 305 spin_unlock_irqrestore(&dd->ipath_sdma_lock, flags); 306 307 /* 308 * Don't restart sdma here (with the exception 309 * below). Wait until link is up to ACTIVE. VL15 MADs 310 * used to bring the link up use PIO, and multiple link 311 * transitions otherwise cause the sdma engine to be 312 * stopped and started multiple times. 313 * The disable is done here, including the shadow, 314 * so the state is kept consistent. 315 * See ipath_restart_sdma() for the actual starting 316 * of sdma. 317 */ 318 spin_lock_irqsave(&dd->ipath_sendctrl_lock, flags); 319 dd->ipath_sendctrl &= ~INFINIPATH_S_SDMAENABLE; 320 ipath_write_kreg(dd, dd->ipath_kregs->kr_sendctrl, 321 dd->ipath_sendctrl); 322 ipath_read_kreg64(dd, dd->ipath_kregs->kr_scratch); 323 spin_unlock_irqrestore(&dd->ipath_sendctrl_lock, flags); 324 325 /* make sure I see next message */ 326 dd->ipath_sdma_abort_jiffies = 0; 327 328 /* 329 * Not everything that takes SDMA offline is a link 330 * status change. If the link was up, restart SDMA. 331 */ 332 if (dd->ipath_flags & IPATH_LINKACTIVE) 333 ipath_restart_sdma(dd); 334 335 goto done; 336 } 337 338resched: 339 /* 340 * for now, keep spinning 341 * JAG - this is bad to just have default be a loop without 342 * state change 343 */ 344 if (time_after(jiffies, dd->ipath_sdma_abort_jiffies)) { 345 ipath_dbg("looping with status 0x%08lx\n", 346 dd->ipath_sdma_status); 347 dd->ipath_sdma_abort_jiffies = jiffies + 5 * HZ; 348 } 349resched_noprint: 350 spin_unlock_irqrestore(&dd->ipath_sdma_lock, flags); 351 if (!test_bit(IPATH_SDMA_SHUTDOWN, &dd->ipath_sdma_status)) 352 tasklet_hi_schedule(&dd->ipath_sdma_abort_task); 353 return; 354 355unlock: 356 spin_unlock_irqrestore(&dd->ipath_sdma_lock, flags); 357done: 358 return; 359} 360 361/* 362 * This is called from interrupt context. 363 */ 364void ipath_sdma_intr(struct ipath_devdata *dd) 365{ 366 unsigned long flags; 367 368 spin_lock_irqsave(&dd->ipath_sdma_lock, flags); 369 370 (void) ipath_sdma_make_progress(dd); 371 372 spin_unlock_irqrestore(&dd->ipath_sdma_lock, flags); 373} 374 375static int alloc_sdma(struct ipath_devdata *dd) 376{ 377 int ret = 0; 378 379 /* Allocate memory for SendDMA descriptor FIFO */ 380 dd->ipath_sdma_descq = dma_alloc_coherent(&dd->pcidev->dev, 381 SDMA_DESCQ_SZ, &dd->ipath_sdma_descq_phys, GFP_KERNEL); 382 383 if (!dd->ipath_sdma_descq) { 384 ipath_dev_err(dd, "failed to allocate SendDMA descriptor " 385 "FIFO memory\n"); 386 ret = -ENOMEM; 387 goto done; 388 } 389 390 dd->ipath_sdma_descq_cnt = 391 SDMA_DESCQ_SZ / sizeof(struct ipath_sdma_desc); 392 393 /* Allocate memory for DMA of head register to memory */ 394 dd->ipath_sdma_head_dma = dma_alloc_coherent(&dd->pcidev->dev, 395 PAGE_SIZE, &dd->ipath_sdma_head_phys, GFP_KERNEL); 396 if (!dd->ipath_sdma_head_dma) { 397 ipath_dev_err(dd, "failed to allocate SendDMA head memory\n"); 398 ret = -ENOMEM; 399 goto cleanup_descq; 400 } 401 dd->ipath_sdma_head_dma[0] = 0; 402 403 init_timer(&dd->ipath_sdma_vl15_timer); 404 dd->ipath_sdma_vl15_timer.function = vl15_watchdog_timeout; 405 dd->ipath_sdma_vl15_timer.data = (unsigned long)dd; 406 atomic_set(&dd->ipath_sdma_vl15_count, 0); 407 408 goto done; 409 410cleanup_descq: 411 dma_free_coherent(&dd->pcidev->dev, SDMA_DESCQ_SZ, 412 (void *)dd->ipath_sdma_descq, dd->ipath_sdma_descq_phys); 413 dd->ipath_sdma_descq = NULL; 414 dd->ipath_sdma_descq_phys = 0; 415done: 416 return ret; 417} 418 419int setup_sdma(struct ipath_devdata *dd) 420{ 421 int ret = 0; 422 unsigned i, n; 423 u64 tmp64; 424 u64 senddmabufmask[3] = { 0 }; 425 unsigned long flags; 426 427 ret = alloc_sdma(dd); 428 if (ret) 429 goto done; 430 431 if (!dd->ipath_sdma_descq) { 432 ipath_dev_err(dd, "SendDMA memory not allocated\n"); 433 goto done; 434 } 435 436 /* 437 * Set initial status as if we had been up, then gone down. 438 * This lets initial start on transition to ACTIVE be the 439 * same as restart after link flap. 440 */ 441 dd->ipath_sdma_status = IPATH_SDMA_ABORT_ABORTED; 442 dd->ipath_sdma_abort_jiffies = 0; 443 dd->ipath_sdma_generation = 0; 444 dd->ipath_sdma_descq_tail = 0; 445 dd->ipath_sdma_descq_head = 0; 446 dd->ipath_sdma_descq_removed = 0; 447 dd->ipath_sdma_descq_added = 0; 448 449 /* Set SendDmaBase */ 450 ipath_write_kreg(dd, dd->ipath_kregs->kr_senddmabase, 451 dd->ipath_sdma_descq_phys); 452 /* Set SendDmaLenGen */ 453 tmp64 = dd->ipath_sdma_descq_cnt; 454 tmp64 |= 1<<18; /* enable generation checking */ 455 ipath_write_kreg(dd, dd->ipath_kregs->kr_senddmalengen, tmp64); 456 /* Set SendDmaTail */ 457 ipath_write_kreg(dd, dd->ipath_kregs->kr_senddmatail, 458 dd->ipath_sdma_descq_tail); 459 /* Set SendDmaHeadAddr */ 460 ipath_write_kreg(dd, dd->ipath_kregs->kr_senddmaheadaddr, 461 dd->ipath_sdma_head_phys); 462 463 /* 464 * Reserve all the former "kernel" piobufs, using high number range 465 * so we get as many 4K buffers as possible 466 */ 467 n = dd->ipath_piobcnt2k + dd->ipath_piobcnt4k; 468 i = dd->ipath_lastport_piobuf + dd->ipath_pioreserved; 469 ipath_chg_pioavailkernel(dd, i, n - i , 0); 470 for (; i < n; ++i) { 471 unsigned word = i / 64; 472 unsigned bit = i & 63; 473 BUG_ON(word >= 3); 474 senddmabufmask[word] |= 1ULL << bit; 475 } 476 ipath_write_kreg(dd, dd->ipath_kregs->kr_senddmabufmask0, 477 senddmabufmask[0]); 478 ipath_write_kreg(dd, dd->ipath_kregs->kr_senddmabufmask1, 479 senddmabufmask[1]); 480 ipath_write_kreg(dd, dd->ipath_kregs->kr_senddmabufmask2, 481 senddmabufmask[2]); 482 483 INIT_LIST_HEAD(&dd->ipath_sdma_activelist); 484 INIT_LIST_HEAD(&dd->ipath_sdma_notifylist); 485 486 tasklet_init(&dd->ipath_sdma_notify_task, sdma_notify_task, 487 (unsigned long) dd); 488 tasklet_init(&dd->ipath_sdma_abort_task, sdma_abort_task, 489 (unsigned long) dd); 490 491 /* 492 * No use to turn on SDMA here, as link is probably not ACTIVE 493 * Just mark it RUNNING and enable the interrupt, and let the 494 * ipath_restart_sdma() on link transition to ACTIVE actually 495 * enable it. 496 */ 497 spin_lock_irqsave(&dd->ipath_sendctrl_lock, flags); 498 dd->ipath_sendctrl |= INFINIPATH_S_SDMAINTENABLE; 499 ipath_write_kreg(dd, dd->ipath_kregs->kr_sendctrl, dd->ipath_sendctrl); 500 ipath_read_kreg64(dd, dd->ipath_kregs->kr_scratch); 501 __set_bit(IPATH_SDMA_RUNNING, &dd->ipath_sdma_status); 502 spin_unlock_irqrestore(&dd->ipath_sendctrl_lock, flags); 503 504done: 505 return ret; 506} 507 508void teardown_sdma(struct ipath_devdata *dd) 509{ 510 struct ipath_sdma_txreq *txp, *txpnext; 511 unsigned long flags; 512 dma_addr_t sdma_head_phys = 0; 513 dma_addr_t sdma_descq_phys = 0; 514 void *sdma_descq = NULL; 515 void *sdma_head_dma = NULL; 516 517 spin_lock_irqsave(&dd->ipath_sdma_lock, flags); 518 __clear_bit(IPATH_SDMA_RUNNING, &dd->ipath_sdma_status); 519 __set_bit(IPATH_SDMA_ABORTING, &dd->ipath_sdma_status); 520 __set_bit(IPATH_SDMA_SHUTDOWN, &dd->ipath_sdma_status); 521 spin_unlock_irqrestore(&dd->ipath_sdma_lock, flags); 522 523 tasklet_kill(&dd->ipath_sdma_abort_task); 524 tasklet_kill(&dd->ipath_sdma_notify_task); 525 526 /* turn off sdma */ 527 spin_lock_irqsave(&dd->ipath_sendctrl_lock, flags); 528 dd->ipath_sendctrl &= ~INFINIPATH_S_SDMAENABLE; 529 ipath_write_kreg(dd, dd->ipath_kregs->kr_sendctrl, 530 dd->ipath_sendctrl); 531 ipath_read_kreg64(dd, dd->ipath_kregs->kr_scratch); 532 spin_unlock_irqrestore(&dd->ipath_sendctrl_lock, flags); 533 534 spin_lock_irqsave(&dd->ipath_sdma_lock, flags); 535 /* dequeue all "sent" requests */ 536 list_for_each_entry_safe(txp, txpnext, &dd->ipath_sdma_activelist, 537 list) { 538 txp->callback_status = IPATH_SDMA_TXREQ_S_SHUTDOWN; 539 if (txp->flags & IPATH_SDMA_TXREQ_F_VL15) 540 vl15_watchdog_deq(dd); 541 list_move_tail(&txp->list, &dd->ipath_sdma_notifylist); 542 } 543 spin_unlock_irqrestore(&dd->ipath_sdma_lock, flags); 544 545 sdma_notify_taskbody(dd); 546 547 del_timer_sync(&dd->ipath_sdma_vl15_timer); 548 549 spin_lock_irqsave(&dd->ipath_sdma_lock, flags); 550 551 dd->ipath_sdma_abort_jiffies = 0; 552 553 ipath_write_kreg(dd, dd->ipath_kregs->kr_senddmabase, 0); 554 ipath_write_kreg(dd, dd->ipath_kregs->kr_senddmalengen, 0); 555 ipath_write_kreg(dd, dd->ipath_kregs->kr_senddmatail, 0); 556 ipath_write_kreg(dd, dd->ipath_kregs->kr_senddmaheadaddr, 0); 557 ipath_write_kreg(dd, dd->ipath_kregs->kr_senddmabufmask0, 0); 558 ipath_write_kreg(dd, dd->ipath_kregs->kr_senddmabufmask1, 0); 559 ipath_write_kreg(dd, dd->ipath_kregs->kr_senddmabufmask2, 0); 560 561 if (dd->ipath_sdma_head_dma) { 562 sdma_head_dma = (void *) dd->ipath_sdma_head_dma; 563 sdma_head_phys = dd->ipath_sdma_head_phys; 564 dd->ipath_sdma_head_dma = NULL; 565 dd->ipath_sdma_head_phys = 0; 566 } 567 568 if (dd->ipath_sdma_descq) { 569 sdma_descq = dd->ipath_sdma_descq; 570 sdma_descq_phys = dd->ipath_sdma_descq_phys; 571 dd->ipath_sdma_descq = NULL; 572 dd->ipath_sdma_descq_phys = 0; 573 } 574 575 spin_unlock_irqrestore(&dd->ipath_sdma_lock, flags); 576 577 if (sdma_head_dma) 578 dma_free_coherent(&dd->pcidev->dev, PAGE_SIZE, 579 sdma_head_dma, sdma_head_phys); 580 581 if (sdma_descq) 582 dma_free_coherent(&dd->pcidev->dev, SDMA_DESCQ_SZ, 583 sdma_descq, sdma_descq_phys); 584} 585 586/* 587 * [Re]start SDMA, if we use it, and it's not already OK. 588 * This is called on transition to link ACTIVE, either the first or 589 * subsequent times. 590 */ 591void ipath_restart_sdma(struct ipath_devdata *dd) 592{ 593 unsigned long flags; 594 int needed = 1; 595 596 if (!(dd->ipath_flags & IPATH_HAS_SEND_DMA)) 597 goto bail; 598 599 /* 600 * First, make sure we should, which is to say, 601 * check that we are "RUNNING" (not in teardown) 602 * and not "SHUTDOWN" 603 */ 604 spin_lock_irqsave(&dd->ipath_sdma_lock, flags); 605 if (!test_bit(IPATH_SDMA_RUNNING, &dd->ipath_sdma_status) 606 || test_bit(IPATH_SDMA_SHUTDOWN, &dd->ipath_sdma_status)) 607 needed = 0; 608 else { 609 __clear_bit(IPATH_SDMA_DISABLED, &dd->ipath_sdma_status); 610 __clear_bit(IPATH_SDMA_DISARMED, &dd->ipath_sdma_status); 611 __clear_bit(IPATH_SDMA_ABORTING, &dd->ipath_sdma_status); 612 } 613 spin_unlock_irqrestore(&dd->ipath_sdma_lock, flags); 614 if (!needed) { 615 ipath_dbg("invalid attempt to restart SDMA, status 0x%08lx\n", 616 dd->ipath_sdma_status); 617 goto bail; 618 } 619 spin_lock_irqsave(&dd->ipath_sendctrl_lock, flags); 620 /* 621 * First clear, just to be safe. Enable is only done 622 * in chip on 0->1 transition 623 */ 624 dd->ipath_sendctrl &= ~INFINIPATH_S_SDMAENABLE; 625 ipath_write_kreg(dd, dd->ipath_kregs->kr_sendctrl, dd->ipath_sendctrl); 626 ipath_read_kreg64(dd, dd->ipath_kregs->kr_scratch); 627 dd->ipath_sendctrl |= INFINIPATH_S_SDMAENABLE; 628 ipath_write_kreg(dd, dd->ipath_kregs->kr_sendctrl, dd->ipath_sendctrl); 629 ipath_read_kreg64(dd, dd->ipath_kregs->kr_scratch); 630 spin_unlock_irqrestore(&dd->ipath_sendctrl_lock, flags); 631 632 /* notify upper layers */ 633 ipath_ib_piobufavail(dd->verbs_dev); 634 635bail: 636 return; 637} 638 639static inline void make_sdma_desc(struct ipath_devdata *dd, 640 u64 *sdmadesc, u64 addr, u64 dwlen, u64 dwoffset) 641{ 642 WARN_ON(addr & 3); 643 /* SDmaPhyAddr[47:32] */ 644 sdmadesc[1] = addr >> 32; 645 /* SDmaPhyAddr[31:0] */ 646 sdmadesc[0] = (addr & 0xfffffffcULL) << 32; 647 /* SDmaGeneration[1:0] */ 648 sdmadesc[0] |= (dd->ipath_sdma_generation & 3ULL) << 30; 649 /* SDmaDwordCount[10:0] */ 650 sdmadesc[0] |= (dwlen & 0x7ffULL) << 16; 651 /* SDmaBufOffset[12:2] */ 652 sdmadesc[0] |= dwoffset & 0x7ffULL; 653} 654 655/* 656 * This function queues one IB packet onto the send DMA queue per call. 657 * The caller is responsible for checking: 658 * 1) The number of send DMA descriptor entries is less than the size of 659 * the descriptor queue. 660 * 2) The IB SGE addresses and lengths are 32-bit aligned 661 * (except possibly the last SGE's length) 662 * 3) The SGE addresses are suitable for passing to dma_map_single(). 663 */ 664int ipath_sdma_verbs_send(struct ipath_devdata *dd, 665 struct ipath_sge_state *ss, u32 dwords, 666 struct ipath_verbs_txreq *tx) 667{ 668 669 unsigned long flags; 670 struct ipath_sge *sge; 671 int ret = 0; 672 u16 tail; 673 __le64 *descqp; 674 u64 sdmadesc[2]; 675 u32 dwoffset; 676 dma_addr_t addr; 677 678 if ((tx->map_len + (dwords<<2)) > dd->ipath_ibmaxlen) { 679 ipath_dbg("packet size %X > ibmax %X, fail\n", 680 tx->map_len + (dwords<<2), dd->ipath_ibmaxlen); 681 ret = -EMSGSIZE; 682 goto fail; 683 } 684 685 spin_lock_irqsave(&dd->ipath_sdma_lock, flags); 686 687retry: 688 if (unlikely(test_bit(IPATH_SDMA_ABORTING, &dd->ipath_sdma_status))) { 689 ret = -EBUSY; 690 goto unlock; 691 } 692 693 if (tx->txreq.sg_count > ipath_sdma_descq_freecnt(dd)) { 694 if (ipath_sdma_make_progress(dd)) 695 goto retry; 696 ret = -ENOBUFS; 697 goto unlock; 698 } 699 700 addr = dma_map_single(&dd->pcidev->dev, tx->txreq.map_addr, 701 tx->map_len, DMA_TO_DEVICE); 702 if (dma_mapping_error(&dd->pcidev->dev, addr)) 703 goto ioerr; 704 705 dwoffset = tx->map_len >> 2; 706 make_sdma_desc(dd, sdmadesc, (u64) addr, dwoffset, 0); 707 708 /* SDmaFirstDesc */ 709 sdmadesc[0] |= 1ULL << 12; 710 if (tx->txreq.flags & IPATH_SDMA_TXREQ_F_USELARGEBUF) 711 sdmadesc[0] |= 1ULL << 14; /* SDmaUseLargeBuf */ 712 713 /* write to the descq */ 714 tail = dd->ipath_sdma_descq_tail; 715 descqp = &dd->ipath_sdma_descq[tail].qw[0]; 716 *descqp++ = cpu_to_le64(sdmadesc[0]); 717 *descqp++ = cpu_to_le64(sdmadesc[1]); 718 719 if (tx->txreq.flags & IPATH_SDMA_TXREQ_F_FREEDESC) 720 tx->txreq.start_idx = tail; 721 722 /* increment the tail */ 723 if (++tail == dd->ipath_sdma_descq_cnt) { 724 tail = 0; 725 descqp = &dd->ipath_sdma_descq[0].qw[0]; 726 ++dd->ipath_sdma_generation; 727 } 728 729 sge = &ss->sge; 730 while (dwords) { 731 u32 dw; 732 u32 len; 733 734 len = dwords << 2; 735 if (len > sge->length) 736 len = sge->length; 737 if (len > sge->sge_length) 738 len = sge->sge_length; 739 BUG_ON(len == 0); 740 dw = (len + 3) >> 2; 741 addr = dma_map_single(&dd->pcidev->dev, sge->vaddr, dw << 2, 742 DMA_TO_DEVICE); 743 if (dma_mapping_error(&dd->pcidev->dev, addr)) 744 goto unmap; 745 make_sdma_desc(dd, sdmadesc, (u64) addr, dw, dwoffset); 746 /* SDmaUseLargeBuf has to be set in every descriptor */ 747 if (tx->txreq.flags & IPATH_SDMA_TXREQ_F_USELARGEBUF) 748 sdmadesc[0] |= 1ULL << 14; 749 /* write to the descq */ 750 *descqp++ = cpu_to_le64(sdmadesc[0]); 751 *descqp++ = cpu_to_le64(sdmadesc[1]); 752 753 /* increment the tail */ 754 if (++tail == dd->ipath_sdma_descq_cnt) { 755 tail = 0; 756 descqp = &dd->ipath_sdma_descq[0].qw[0]; 757 ++dd->ipath_sdma_generation; 758 } 759 sge->vaddr += len; 760 sge->length -= len; 761 sge->sge_length -= len; 762 if (sge->sge_length == 0) { 763 if (--ss->num_sge) 764 *sge = *ss->sg_list++; 765 } else if (sge->length == 0 && sge->mr != NULL) { 766 if (++sge->n >= IPATH_SEGSZ) { 767 if (++sge->m >= sge->mr->mapsz) 768 break; 769 sge->n = 0; 770 } 771 sge->vaddr = 772 sge->mr->map[sge->m]->segs[sge->n].vaddr; 773 sge->length = 774 sge->mr->map[sge->m]->segs[sge->n].length; 775 } 776 777 dwoffset += dw; 778 dwords -= dw; 779 } 780 781 if (!tail) 782 descqp = &dd->ipath_sdma_descq[dd->ipath_sdma_descq_cnt].qw[0]; 783 descqp -= 2; 784 /* SDmaLastDesc */ 785 descqp[0] |= cpu_to_le64(1ULL << 11); 786 if (tx->txreq.flags & IPATH_SDMA_TXREQ_F_INTREQ) { 787 /* SDmaIntReq */ 788 descqp[0] |= cpu_to_le64(1ULL << 15); 789 } 790 791 /* Commit writes to memory and advance the tail on the chip */ 792 wmb(); 793 ipath_write_kreg(dd, dd->ipath_kregs->kr_senddmatail, tail); 794 795 tx->txreq.next_descq_idx = tail; 796 tx->txreq.callback_status = IPATH_SDMA_TXREQ_S_OK; 797 dd->ipath_sdma_descq_tail = tail; 798 dd->ipath_sdma_descq_added += tx->txreq.sg_count; 799 list_add_tail(&tx->txreq.list, &dd->ipath_sdma_activelist); 800 if (tx->txreq.flags & IPATH_SDMA_TXREQ_F_VL15) 801 vl15_watchdog_enq(dd); 802 goto unlock; 803 804unmap: 805 while (tail != dd->ipath_sdma_descq_tail) { 806 if (!tail) 807 tail = dd->ipath_sdma_descq_cnt - 1; 808 else 809 tail--; 810 unmap_desc(dd, tail); 811 } 812ioerr: 813 ret = -EIO; 814unlock: 815 spin_unlock_irqrestore(&dd->ipath_sdma_lock, flags); 816fail: 817 return ret; 818} 819