1/* 2 * Copyright (c) 2001-2004 by David Brownell 3 * Copyright (c) 2003 Michal Sojka, for high-speed iso transfers 4 * 5 * This program is free software; you can redistribute it and/or modify it 6 * under the terms of the GNU General Public License as published by the 7 * Free Software Foundation; either version 2 of the License, or (at your 8 * option) any later version. 9 * 10 * This program is distributed in the hope that it will be useful, but 11 * WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY 12 * or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License 13 * for more details. 14 * 15 * You should have received a copy of the GNU General Public License 16 * along with this program; if not, write to the Free Software Foundation, 17 * Inc., 675 Mass Ave, Cambridge, MA 02139, USA. 18 */ 19 20/* this file is part of ehci-hcd.c */ 21 22/*-------------------------------------------------------------------------*/ 23 24/* 25 * EHCI scheduled transaction support: interrupt, iso, split iso 26 * These are called "periodic" transactions in the EHCI spec. 27 * 28 * Note that for interrupt transfers, the QH/QTD manipulation is shared 29 * with the "asynchronous" transaction support (control/bulk transfers). 30 * The only real difference is in how interrupt transfers are scheduled. 31 * 32 * For ISO, we make an "iso_stream" head to serve the same role as a QH. 33 * It keeps track of every ITD (or SITD) that's linked, and holds enough 34 * pre-calculated schedule data to make appending to the queue be quick. 35 */ 36 37static int ehci_get_frame (struct usb_hcd *hcd); 38 39#ifdef CONFIG_PCI 40 41static unsigned ehci_read_frame_index(struct ehci_hcd *ehci) 42{ 43 unsigned uf; 44 45 /* 46 * The MosChip MCS9990 controller updates its microframe counter 47 * a little before the frame counter, and occasionally we will read 48 * the invalid intermediate value. Avoid problems by checking the 49 * microframe number (the low-order 3 bits); if they are 0 then 50 * re-read the register to get the correct value. 51 */ 52 uf = ehci_readl(ehci, &ehci->regs->frame_index); 53 if (unlikely(ehci->frame_index_bug && ((uf & 7) == 0))) 54 uf = ehci_readl(ehci, &ehci->regs->frame_index); 55 return uf; 56} 57 58#endif 59 60/*-------------------------------------------------------------------------*/ 61 62/* 63 * periodic_next_shadow - return "next" pointer on shadow list 64 * @periodic: host pointer to qh/itd/sitd 65 * @tag: hardware tag for type of this record 66 */ 67static union ehci_shadow * 68periodic_next_shadow(struct ehci_hcd *ehci, union ehci_shadow *periodic, 69 __hc32 tag) 70{ 71 switch (hc32_to_cpu(ehci, tag)) { 72 case Q_TYPE_QH: 73 return &periodic->qh->qh_next; 74 case Q_TYPE_FSTN: 75 return &periodic->fstn->fstn_next; 76 case Q_TYPE_ITD: 77 return &periodic->itd->itd_next; 78 // case Q_TYPE_SITD: 79 default: 80 return &periodic->sitd->sitd_next; 81 } 82} 83 84static __hc32 * 85shadow_next_periodic(struct ehci_hcd *ehci, union ehci_shadow *periodic, 86 __hc32 tag) 87{ 88 switch (hc32_to_cpu(ehci, tag)) { 89 /* our ehci_shadow.qh is actually software part */ 90 case Q_TYPE_QH: 91 return &periodic->qh->hw->hw_next; 92 /* others are hw parts */ 93 default: 94 return periodic->hw_next; 95 } 96} 97 98/* caller must hold ehci->lock */ 99static void periodic_unlink (struct ehci_hcd *ehci, unsigned frame, void *ptr) 100{ 101 union ehci_shadow *prev_p = &ehci->pshadow[frame]; 102 __hc32 *hw_p = &ehci->periodic[frame]; 103 union ehci_shadow here = *prev_p; 104 105 /* find predecessor of "ptr"; hw and shadow lists are in sync */ 106 while (here.ptr && here.ptr != ptr) { 107 prev_p = periodic_next_shadow(ehci, prev_p, 108 Q_NEXT_TYPE(ehci, *hw_p)); 109 hw_p = shadow_next_periodic(ehci, &here, 110 Q_NEXT_TYPE(ehci, *hw_p)); 111 here = *prev_p; 112 } 113 /* an interrupt entry (at list end) could have been shared */ 114 if (!here.ptr) 115 return; 116 117 /* update shadow and hardware lists ... the old "next" pointers 118 * from ptr may still be in use, the caller updates them. 119 */ 120 *prev_p = *periodic_next_shadow(ehci, &here, 121 Q_NEXT_TYPE(ehci, *hw_p)); 122 123 if (!ehci->use_dummy_qh || 124 *shadow_next_periodic(ehci, &here, Q_NEXT_TYPE(ehci, *hw_p)) 125 != EHCI_LIST_END(ehci)) 126 *hw_p = *shadow_next_periodic(ehci, &here, 127 Q_NEXT_TYPE(ehci, *hw_p)); 128 else 129 *hw_p = ehci->dummy->qh_dma; 130} 131 132/* how many of the uframe's 125 usecs are allocated? */ 133static unsigned short 134periodic_usecs (struct ehci_hcd *ehci, unsigned frame, unsigned uframe) 135{ 136 __hc32 *hw_p = &ehci->periodic [frame]; 137 union ehci_shadow *q = &ehci->pshadow [frame]; 138 unsigned usecs = 0; 139 struct ehci_qh_hw *hw; 140 141 while (q->ptr) { 142 switch (hc32_to_cpu(ehci, Q_NEXT_TYPE(ehci, *hw_p))) { 143 case Q_TYPE_QH: 144 hw = q->qh->hw; 145 /* is it in the S-mask? */ 146 if (hw->hw_info2 & cpu_to_hc32(ehci, 1 << uframe)) 147 usecs += q->qh->usecs; 148 /* ... or C-mask? */ 149 if (hw->hw_info2 & cpu_to_hc32(ehci, 150 1 << (8 + uframe))) 151 usecs += q->qh->c_usecs; 152 hw_p = &hw->hw_next; 153 q = &q->qh->qh_next; 154 break; 155 // case Q_TYPE_FSTN: 156 default: 157 /* for "save place" FSTNs, count the relevant INTR 158 * bandwidth from the previous frame 159 */ 160 if (q->fstn->hw_prev != EHCI_LIST_END(ehci)) { 161 ehci_dbg (ehci, "ignoring FSTN cost ...\n"); 162 } 163 hw_p = &q->fstn->hw_next; 164 q = &q->fstn->fstn_next; 165 break; 166 case Q_TYPE_ITD: 167 if (q->itd->hw_transaction[uframe]) 168 usecs += q->itd->stream->usecs; 169 hw_p = &q->itd->hw_next; 170 q = &q->itd->itd_next; 171 break; 172 case Q_TYPE_SITD: 173 /* is it in the S-mask? (count SPLIT, DATA) */ 174 if (q->sitd->hw_uframe & cpu_to_hc32(ehci, 175 1 << uframe)) { 176 if (q->sitd->hw_fullspeed_ep & 177 cpu_to_hc32(ehci, 1<<31)) 178 usecs += q->sitd->stream->usecs; 179 else /* worst case for OUT start-split */ 180 usecs += HS_USECS_ISO (188); 181 } 182 183 /* ... C-mask? (count CSPLIT, DATA) */ 184 if (q->sitd->hw_uframe & 185 cpu_to_hc32(ehci, 1 << (8 + uframe))) { 186 /* worst case for IN complete-split */ 187 usecs += q->sitd->stream->c_usecs; 188 } 189 190 hw_p = &q->sitd->hw_next; 191 q = &q->sitd->sitd_next; 192 break; 193 } 194 } 195#ifdef DEBUG 196 if (usecs > ehci->uframe_periodic_max) 197 ehci_err (ehci, "uframe %d sched overrun: %d usecs\n", 198 frame * 8 + uframe, usecs); 199#endif 200 return usecs; 201} 202 203/*-------------------------------------------------------------------------*/ 204 205static int same_tt (struct usb_device *dev1, struct usb_device *dev2) 206{ 207 if (!dev1->tt || !dev2->tt) 208 return 0; 209 if (dev1->tt != dev2->tt) 210 return 0; 211 if (dev1->tt->multi) 212 return dev1->ttport == dev2->ttport; 213 else 214 return 1; 215} 216 217#ifdef CONFIG_USB_EHCI_TT_NEWSCHED 218 219/* Which uframe does the low/fullspeed transfer start in? 220 * 221 * The parameter is the mask of ssplits in "H-frame" terms 222 * and this returns the transfer start uframe in "B-frame" terms, 223 * which allows both to match, e.g. a ssplit in "H-frame" uframe 0 224 * will cause a transfer in "B-frame" uframe 0. "B-frames" lag 225 * "H-frames" by 1 uframe. See the EHCI spec sec 4.5 and figure 4.7. 226 */ 227static inline unsigned char tt_start_uframe(struct ehci_hcd *ehci, __hc32 mask) 228{ 229 unsigned char smask = QH_SMASK & hc32_to_cpu(ehci, mask); 230 if (!smask) { 231 ehci_err(ehci, "invalid empty smask!\n"); 232 /* uframe 7 can't have bw so this will indicate failure */ 233 return 7; 234 } 235 return ffs(smask) - 1; 236} 237 238static const unsigned char 239max_tt_usecs[] = { 125, 125, 125, 125, 125, 125, 30, 0 }; 240 241/* carryover low/fullspeed bandwidth that crosses uframe boundries */ 242static inline void carryover_tt_bandwidth(unsigned short tt_usecs[8]) 243{ 244 int i; 245 for (i=0; i<7; i++) { 246 if (max_tt_usecs[i] < tt_usecs[i]) { 247 tt_usecs[i+1] += tt_usecs[i] - max_tt_usecs[i]; 248 tt_usecs[i] = max_tt_usecs[i]; 249 } 250 } 251} 252 253/* How many of the tt's periodic downstream 1000 usecs are allocated? 254 * 255 * While this measures the bandwidth in terms of usecs/uframe, 256 * the low/fullspeed bus has no notion of uframes, so any particular 257 * low/fullspeed transfer can "carry over" from one uframe to the next, 258 * since the TT just performs downstream transfers in sequence. 259 * 260 * For example two separate 100 usec transfers can start in the same uframe, 261 * and the second one would "carry over" 75 usecs into the next uframe. 262 */ 263static void 264periodic_tt_usecs ( 265 struct ehci_hcd *ehci, 266 struct usb_device *dev, 267 unsigned frame, 268 unsigned short tt_usecs[8] 269) 270{ 271 __hc32 *hw_p = &ehci->periodic [frame]; 272 union ehci_shadow *q = &ehci->pshadow [frame]; 273 unsigned char uf; 274 275 memset(tt_usecs, 0, 16); 276 277 while (q->ptr) { 278 switch (hc32_to_cpu(ehci, Q_NEXT_TYPE(ehci, *hw_p))) { 279 case Q_TYPE_ITD: 280 hw_p = &q->itd->hw_next; 281 q = &q->itd->itd_next; 282 continue; 283 case Q_TYPE_QH: 284 if (same_tt(dev, q->qh->dev)) { 285 uf = tt_start_uframe(ehci, q->qh->hw->hw_info2); 286 tt_usecs[uf] += q->qh->tt_usecs; 287 } 288 hw_p = &q->qh->hw->hw_next; 289 q = &q->qh->qh_next; 290 continue; 291 case Q_TYPE_SITD: 292 if (same_tt(dev, q->sitd->urb->dev)) { 293 uf = tt_start_uframe(ehci, q->sitd->hw_uframe); 294 tt_usecs[uf] += q->sitd->stream->tt_usecs; 295 } 296 hw_p = &q->sitd->hw_next; 297 q = &q->sitd->sitd_next; 298 continue; 299 // case Q_TYPE_FSTN: 300 default: 301 ehci_dbg(ehci, "ignoring periodic frame %d FSTN\n", 302 frame); 303 hw_p = &q->fstn->hw_next; 304 q = &q->fstn->fstn_next; 305 } 306 } 307 308 carryover_tt_bandwidth(tt_usecs); 309 310 if (max_tt_usecs[7] < tt_usecs[7]) 311 ehci_err(ehci, "frame %d tt sched overrun: %d usecs\n", 312 frame, tt_usecs[7] - max_tt_usecs[7]); 313} 314 315/* 316 * Return true if the device's tt's downstream bus is available for a 317 * periodic transfer of the specified length (usecs), starting at the 318 * specified frame/uframe. Note that (as summarized in section 11.19 319 * of the usb 2.0 spec) TTs can buffer multiple transactions for each 320 * uframe. 321 * 322 * The uframe parameter is when the fullspeed/lowspeed transfer 323 * should be executed in "B-frame" terms, which is the same as the 324 * highspeed ssplit's uframe (which is in "H-frame" terms). For example 325 * a ssplit in "H-frame" 0 causes a transfer in "B-frame" 0. 326 * See the EHCI spec sec 4.5 and fig 4.7. 327 * 328 * This checks if the full/lowspeed bus, at the specified starting uframe, 329 * has the specified bandwidth available, according to rules listed 330 * in USB 2.0 spec section 11.18.1 fig 11-60. 331 * 332 * This does not check if the transfer would exceed the max ssplit 333 * limit of 16, specified in USB 2.0 spec section 11.18.4 requirement #4, 334 * since proper scheduling limits ssplits to less than 16 per uframe. 335 */ 336static int tt_available ( 337 struct ehci_hcd *ehci, 338 unsigned period, 339 struct usb_device *dev, 340 unsigned frame, 341 unsigned uframe, 342 u16 usecs 343) 344{ 345 if ((period == 0) || (uframe >= 7)) /* error */ 346 return 0; 347 348 for (; frame < ehci->periodic_size; frame += period) { 349 unsigned short tt_usecs[8]; 350 351 periodic_tt_usecs (ehci, dev, frame, tt_usecs); 352 353 ehci_vdbg(ehci, "tt frame %d check %d usecs start uframe %d in" 354 " schedule %d/%d/%d/%d/%d/%d/%d/%d\n", 355 frame, usecs, uframe, 356 tt_usecs[0], tt_usecs[1], tt_usecs[2], tt_usecs[3], 357 tt_usecs[4], tt_usecs[5], tt_usecs[6], tt_usecs[7]); 358 359 if (max_tt_usecs[uframe] <= tt_usecs[uframe]) { 360 ehci_vdbg(ehci, "frame %d uframe %d fully scheduled\n", 361 frame, uframe); 362 return 0; 363 } 364 365 /* special case for isoc transfers larger than 125us: 366 * the first and each subsequent fully used uframe 367 * must be empty, so as to not illegally delay 368 * already scheduled transactions 369 */ 370 if (125 < usecs) { 371 int ufs = (usecs / 125); 372 int i; 373 for (i = uframe; i < (uframe + ufs) && i < 8; i++) 374 if (0 < tt_usecs[i]) { 375 ehci_vdbg(ehci, 376 "multi-uframe xfer can't fit " 377 "in frame %d uframe %d\n", 378 frame, i); 379 return 0; 380 } 381 } 382 383 tt_usecs[uframe] += usecs; 384 385 carryover_tt_bandwidth(tt_usecs); 386 387 /* fail if the carryover pushed bw past the last uframe's limit */ 388 if (max_tt_usecs[7] < tt_usecs[7]) { 389 ehci_vdbg(ehci, 390 "tt unavailable usecs %d frame %d uframe %d\n", 391 usecs, frame, uframe); 392 return 0; 393 } 394 } 395 396 return 1; 397} 398 399#else 400 401/* return true iff the device's transaction translator is available 402 * for a periodic transfer starting at the specified frame, using 403 * all the uframes in the mask. 404 */ 405static int tt_no_collision ( 406 struct ehci_hcd *ehci, 407 unsigned period, 408 struct usb_device *dev, 409 unsigned frame, 410 u32 uf_mask 411) 412{ 413 if (period == 0) /* error */ 414 return 0; 415 416 /* note bandwidth wastage: split never follows csplit 417 * (different dev or endpoint) until the next uframe. 418 * calling convention doesn't make that distinction. 419 */ 420 for (; frame < ehci->periodic_size; frame += period) { 421 union ehci_shadow here; 422 __hc32 type; 423 struct ehci_qh_hw *hw; 424 425 here = ehci->pshadow [frame]; 426 type = Q_NEXT_TYPE(ehci, ehci->periodic [frame]); 427 while (here.ptr) { 428 switch (hc32_to_cpu(ehci, type)) { 429 case Q_TYPE_ITD: 430 type = Q_NEXT_TYPE(ehci, here.itd->hw_next); 431 here = here.itd->itd_next; 432 continue; 433 case Q_TYPE_QH: 434 hw = here.qh->hw; 435 if (same_tt (dev, here.qh->dev)) { 436 u32 mask; 437 438 mask = hc32_to_cpu(ehci, 439 hw->hw_info2); 440 /* "knows" no gap is needed */ 441 mask |= mask >> 8; 442 if (mask & uf_mask) 443 break; 444 } 445 type = Q_NEXT_TYPE(ehci, hw->hw_next); 446 here = here.qh->qh_next; 447 continue; 448 case Q_TYPE_SITD: 449 if (same_tt (dev, here.sitd->urb->dev)) { 450 u16 mask; 451 452 mask = hc32_to_cpu(ehci, here.sitd 453 ->hw_uframe); 454 /* FIXME assumes no gap for IN! */ 455 mask |= mask >> 8; 456 if (mask & uf_mask) 457 break; 458 } 459 type = Q_NEXT_TYPE(ehci, here.sitd->hw_next); 460 here = here.sitd->sitd_next; 461 continue; 462 // case Q_TYPE_FSTN: 463 default: 464 ehci_dbg (ehci, 465 "periodic frame %d bogus type %d\n", 466 frame, type); 467 } 468 469 /* collision or error */ 470 return 0; 471 } 472 } 473 474 /* no collision */ 475 return 1; 476} 477 478#endif /* CONFIG_USB_EHCI_TT_NEWSCHED */ 479 480/*-------------------------------------------------------------------------*/ 481 482static int enable_periodic (struct ehci_hcd *ehci) 483{ 484 u32 cmd; 485 int status; 486 487 if (ehci->periodic_sched++) 488 return 0; 489 490 /* did clearing PSE did take effect yet? 491 * takes effect only at frame boundaries... 492 */ 493 status = handshake_on_error_set_halt(ehci, &ehci->regs->status, 494 STS_PSS, 0, 9 * 125); 495 if (status) { 496 usb_hc_died(ehci_to_hcd(ehci)); 497 return status; 498 } 499 500 cmd = ehci_readl(ehci, &ehci->regs->command) | CMD_PSE; 501 ehci_writel(ehci, cmd, &ehci->regs->command); 502 /* posted write ... PSS happens later */ 503 504 /* make sure ehci_work scans these */ 505 ehci->next_uframe = ehci_read_frame_index(ehci) 506 % (ehci->periodic_size << 3); 507 if (unlikely(ehci->broken_periodic)) 508 ehci->last_periodic_enable = ktime_get_real(); 509 return 0; 510} 511 512static int disable_periodic (struct ehci_hcd *ehci) 513{ 514 u32 cmd; 515 int status; 516 517 if (--ehci->periodic_sched) 518 return 0; 519 520 if (unlikely(ehci->broken_periodic)) { 521 /* delay experimentally determined */ 522 ktime_t safe = ktime_add_us(ehci->last_periodic_enable, 1000); 523 ktime_t now = ktime_get_real(); 524 s64 delay = ktime_us_delta(safe, now); 525 526 if (unlikely(delay > 0)) 527 udelay(delay); 528 } 529 530 /* did setting PSE not take effect yet? 531 * takes effect only at frame boundaries... 532 */ 533 status = handshake_on_error_set_halt(ehci, &ehci->regs->status, 534 STS_PSS, STS_PSS, 9 * 125); 535 if (status) { 536 usb_hc_died(ehci_to_hcd(ehci)); 537 return status; 538 } 539 540 cmd = ehci_readl(ehci, &ehci->regs->command) & ~CMD_PSE; 541 ehci_writel(ehci, cmd, &ehci->regs->command); 542 /* posted write ... */ 543 544 free_cached_lists(ehci); 545 546 ehci->next_uframe = -1; 547 return 0; 548} 549 550/*-------------------------------------------------------------------------*/ 551 552/* periodic schedule slots have iso tds (normal or split) first, then a 553 * sparse tree for active interrupt transfers. 554 * 555 * this just links in a qh; caller guarantees uframe masks are set right. 556 * no FSTN support (yet; ehci 0.96+) 557 */ 558static int qh_link_periodic (struct ehci_hcd *ehci, struct ehci_qh *qh) 559{ 560 unsigned i; 561 unsigned period = qh->period; 562 563 dev_dbg (&qh->dev->dev, 564 "link qh%d-%04x/%p start %d [%d/%d us]\n", 565 period, hc32_to_cpup(ehci, &qh->hw->hw_info2) 566 & (QH_CMASK | QH_SMASK), 567 qh, qh->start, qh->usecs, qh->c_usecs); 568 569 /* high bandwidth, or otherwise every microframe */ 570 if (period == 0) 571 period = 1; 572 573 for (i = qh->start; i < ehci->periodic_size; i += period) { 574 union ehci_shadow *prev = &ehci->pshadow[i]; 575 __hc32 *hw_p = &ehci->periodic[i]; 576 union ehci_shadow here = *prev; 577 __hc32 type = 0; 578 579 /* skip the iso nodes at list head */ 580 while (here.ptr) { 581 type = Q_NEXT_TYPE(ehci, *hw_p); 582 if (type == cpu_to_hc32(ehci, Q_TYPE_QH)) 583 break; 584 prev = periodic_next_shadow(ehci, prev, type); 585 hw_p = shadow_next_periodic(ehci, &here, type); 586 here = *prev; 587 } 588 589 /* sorting each branch by period (slow-->fast) 590 * enables sharing interior tree nodes 591 */ 592 while (here.ptr && qh != here.qh) { 593 if (qh->period > here.qh->period) 594 break; 595 prev = &here.qh->qh_next; 596 hw_p = &here.qh->hw->hw_next; 597 here = *prev; 598 } 599 /* link in this qh, unless some earlier pass did that */ 600 if (qh != here.qh) { 601 qh->qh_next = here; 602 if (here.qh) 603 qh->hw->hw_next = *hw_p; 604 wmb (); 605 prev->qh = qh; 606 *hw_p = QH_NEXT (ehci, qh->qh_dma); 607 } 608 } 609 qh->qh_state = QH_STATE_LINKED; 610 qh->xacterrs = 0; 611 qh_get (qh); 612 613 /* update per-qh bandwidth for usbfs */ 614 ehci_to_hcd(ehci)->self.bandwidth_allocated += qh->period 615 ? ((qh->usecs + qh->c_usecs) / qh->period) 616 : (qh->usecs * 8); 617 618 /* maybe enable periodic schedule processing */ 619 return enable_periodic(ehci); 620} 621 622static int qh_unlink_periodic(struct ehci_hcd *ehci, struct ehci_qh *qh) 623{ 624 unsigned i; 625 unsigned period; 626 627 // FIXME: 628 // IF this isn't high speed 629 // and this qh is active in the current uframe 630 // (and overlay token SplitXstate is false?) 631 // THEN 632 // qh->hw_info1 |= cpu_to_hc32(1 << 7 /* "ignore" */); 633 634 /* high bandwidth, or otherwise part of every microframe */ 635 if ((period = qh->period) == 0) 636 period = 1; 637 638 for (i = qh->start; i < ehci->periodic_size; i += period) 639 periodic_unlink (ehci, i, qh); 640 641 /* update per-qh bandwidth for usbfs */ 642 ehci_to_hcd(ehci)->self.bandwidth_allocated -= qh->period 643 ? ((qh->usecs + qh->c_usecs) / qh->period) 644 : (qh->usecs * 8); 645 646 dev_dbg (&qh->dev->dev, 647 "unlink qh%d-%04x/%p start %d [%d/%d us]\n", 648 qh->period, 649 hc32_to_cpup(ehci, &qh->hw->hw_info2) & (QH_CMASK | QH_SMASK), 650 qh, qh->start, qh->usecs, qh->c_usecs); 651 652 /* qh->qh_next still "live" to HC */ 653 qh->qh_state = QH_STATE_UNLINK; 654 qh->qh_next.ptr = NULL; 655 qh_put (qh); 656 657 /* maybe turn off periodic schedule */ 658 return disable_periodic(ehci); 659} 660 661static void intr_deschedule (struct ehci_hcd *ehci, struct ehci_qh *qh) 662{ 663 unsigned wait; 664 struct ehci_qh_hw *hw = qh->hw; 665 int rc; 666 667 /* If the QH isn't linked then there's nothing we can do 668 * unless we were called during a giveback, in which case 669 * qh_completions() has to deal with it. 670 */ 671 if (qh->qh_state != QH_STATE_LINKED) { 672 if (qh->qh_state == QH_STATE_COMPLETING) 673 qh->needs_rescan = 1; 674 return; 675 } 676 677 qh_unlink_periodic (ehci, qh); 678 679 /* simple/paranoid: always delay, expecting the HC needs to read 680 * qh->hw_next or finish a writeback after SPLIT/CSPLIT ... and 681 * expect khubd to clean up after any CSPLITs we won't issue. 682 * active high speed queues may need bigger delays... 683 */ 684 if (list_empty (&qh->qtd_list) 685 || (cpu_to_hc32(ehci, QH_CMASK) 686 & hw->hw_info2) != 0) 687 wait = 2; 688 else 689 wait = 55; /* worst case: 3 * 1024 */ 690 691 udelay (wait); 692 qh->qh_state = QH_STATE_IDLE; 693 hw->hw_next = EHCI_LIST_END(ehci); 694 wmb (); 695 696 qh_completions(ehci, qh); 697 698 /* reschedule QH iff another request is queued */ 699 if (!list_empty(&qh->qtd_list) && 700 ehci->rh_state == EHCI_RH_RUNNING) { 701 rc = qh_schedule(ehci, qh); 702 703 /* An error here likely indicates handshake failure 704 * or no space left in the schedule. Neither fault 705 * should happen often ... 706 * 707 * FIXME kill the now-dysfunctional queued urbs 708 */ 709 if (rc != 0) 710 ehci_err(ehci, "can't reschedule qh %p, err %d\n", 711 qh, rc); 712 } 713} 714 715/*-------------------------------------------------------------------------*/ 716 717static int check_period ( 718 struct ehci_hcd *ehci, 719 unsigned frame, 720 unsigned uframe, 721 unsigned period, 722 unsigned usecs 723) { 724 int claimed; 725 726 /* complete split running into next frame? 727 * given FSTN support, we could sometimes check... 728 */ 729 if (uframe >= 8) 730 return 0; 731 732 /* convert "usecs we need" to "max already claimed" */ 733 usecs = ehci->uframe_periodic_max - usecs; 734 735 /* we "know" 2 and 4 uframe intervals were rejected; so 736 * for period 0, check _every_ microframe in the schedule. 737 */ 738 if (unlikely (period == 0)) { 739 do { 740 for (uframe = 0; uframe < 7; uframe++) { 741 claimed = periodic_usecs (ehci, frame, uframe); 742 if (claimed > usecs) 743 return 0; 744 } 745 } while ((frame += 1) < ehci->periodic_size); 746 747 /* just check the specified uframe, at that period */ 748 } else { 749 do { 750 claimed = periodic_usecs (ehci, frame, uframe); 751 if (claimed > usecs) 752 return 0; 753 } while ((frame += period) < ehci->periodic_size); 754 } 755 756 // success! 757 return 1; 758} 759 760static int check_intr_schedule ( 761 struct ehci_hcd *ehci, 762 unsigned frame, 763 unsigned uframe, 764 const struct ehci_qh *qh, 765 __hc32 *c_maskp 766) 767{ 768 int retval = -ENOSPC; 769 u8 mask = 0; 770 771 if (qh->c_usecs && uframe >= 6) /* FSTN territory? */ 772 goto done; 773 774 if (!check_period (ehci, frame, uframe, qh->period, qh->usecs)) 775 goto done; 776 if (!qh->c_usecs) { 777 retval = 0; 778 *c_maskp = 0; 779 goto done; 780 } 781 782#ifdef CONFIG_USB_EHCI_TT_NEWSCHED 783 if (tt_available (ehci, qh->period, qh->dev, frame, uframe, 784 qh->tt_usecs)) { 785 unsigned i; 786 787 /* TODO : this may need FSTN for SSPLIT in uframe 5. */ 788 for (i=uframe+1; i<8 && i<uframe+4; i++) 789 if (!check_period (ehci, frame, i, 790 qh->period, qh->c_usecs)) 791 goto done; 792 else 793 mask |= 1 << i; 794 795 retval = 0; 796 797 *c_maskp = cpu_to_hc32(ehci, mask << 8); 798 } 799#else 800 /* Make sure this tt's buffer is also available for CSPLITs. 801 * We pessimize a bit; probably the typical full speed case 802 * doesn't need the second CSPLIT. 803 * 804 * NOTE: both SPLIT and CSPLIT could be checked in just 805 * one smart pass... 806 */ 807 mask = 0x03 << (uframe + qh->gap_uf); 808 *c_maskp = cpu_to_hc32(ehci, mask << 8); 809 810 mask |= 1 << uframe; 811 if (tt_no_collision (ehci, qh->period, qh->dev, frame, mask)) { 812 if (!check_period (ehci, frame, uframe + qh->gap_uf + 1, 813 qh->period, qh->c_usecs)) 814 goto done; 815 if (!check_period (ehci, frame, uframe + qh->gap_uf, 816 qh->period, qh->c_usecs)) 817 goto done; 818 retval = 0; 819 } 820#endif 821done: 822 return retval; 823} 824 825/* "first fit" scheduling policy used the first time through, 826 * or when the previous schedule slot can't be re-used. 827 */ 828static int qh_schedule(struct ehci_hcd *ehci, struct ehci_qh *qh) 829{ 830 int status; 831 unsigned uframe; 832 __hc32 c_mask; 833 unsigned frame; /* 0..(qh->period - 1), or NO_FRAME */ 834 struct ehci_qh_hw *hw = qh->hw; 835 836 qh_refresh(ehci, qh); 837 hw->hw_next = EHCI_LIST_END(ehci); 838 frame = qh->start; 839 840 /* reuse the previous schedule slots, if we can */ 841 if (frame < qh->period) { 842 uframe = ffs(hc32_to_cpup(ehci, &hw->hw_info2) & QH_SMASK); 843 status = check_intr_schedule (ehci, frame, --uframe, 844 qh, &c_mask); 845 } else { 846 uframe = 0; 847 c_mask = 0; 848 status = -ENOSPC; 849 } 850 851 /* else scan the schedule to find a group of slots such that all 852 * uframes have enough periodic bandwidth available. 853 */ 854 if (status) { 855 /* "normal" case, uframing flexible except with splits */ 856 if (qh->period) { 857 int i; 858 859 for (i = qh->period; status && i > 0; --i) { 860 frame = ++ehci->random_frame % qh->period; 861 for (uframe = 0; uframe < 8; uframe++) { 862 status = check_intr_schedule (ehci, 863 frame, uframe, qh, 864 &c_mask); 865 if (status == 0) 866 break; 867 } 868 } 869 870 /* qh->period == 0 means every uframe */ 871 } else { 872 frame = 0; 873 status = check_intr_schedule (ehci, 0, 0, qh, &c_mask); 874 } 875 if (status) 876 goto done; 877 qh->start = frame; 878 879 /* reset S-frame and (maybe) C-frame masks */ 880 hw->hw_info2 &= cpu_to_hc32(ehci, ~(QH_CMASK | QH_SMASK)); 881 hw->hw_info2 |= qh->period 882 ? cpu_to_hc32(ehci, 1 << uframe) 883 : cpu_to_hc32(ehci, QH_SMASK); 884 hw->hw_info2 |= c_mask; 885 } else 886 ehci_dbg (ehci, "reused qh %p schedule\n", qh); 887 888 /* stuff into the periodic schedule */ 889 status = qh_link_periodic (ehci, qh); 890done: 891 return status; 892} 893 894static int intr_submit ( 895 struct ehci_hcd *ehci, 896 struct urb *urb, 897 struct list_head *qtd_list, 898 gfp_t mem_flags 899) { 900 unsigned epnum; 901 unsigned long flags; 902 struct ehci_qh *qh; 903 int status; 904 struct list_head empty; 905 906 /* get endpoint and transfer/schedule data */ 907 epnum = urb->ep->desc.bEndpointAddress; 908 909 spin_lock_irqsave (&ehci->lock, flags); 910 911 if (unlikely(!HCD_HW_ACCESSIBLE(ehci_to_hcd(ehci)))) { 912 status = -ESHUTDOWN; 913 goto done_not_linked; 914 } 915 status = usb_hcd_link_urb_to_ep(ehci_to_hcd(ehci), urb); 916 if (unlikely(status)) 917 goto done_not_linked; 918 919 /* get qh and force any scheduling errors */ 920 INIT_LIST_HEAD (&empty); 921 qh = qh_append_tds(ehci, urb, &empty, epnum, &urb->ep->hcpriv); 922 if (qh == NULL) { 923 status = -ENOMEM; 924 goto done; 925 } 926 if (qh->qh_state == QH_STATE_IDLE) { 927 if ((status = qh_schedule (ehci, qh)) != 0) 928 goto done; 929 } 930 931 /* then queue the urb's tds to the qh */ 932 qh = qh_append_tds(ehci, urb, qtd_list, epnum, &urb->ep->hcpriv); 933 BUG_ON (qh == NULL); 934 935 /* ... update usbfs periodic stats */ 936 ehci_to_hcd(ehci)->self.bandwidth_int_reqs++; 937 938done: 939 if (unlikely(status)) 940 usb_hcd_unlink_urb_from_ep(ehci_to_hcd(ehci), urb); 941done_not_linked: 942 spin_unlock_irqrestore (&ehci->lock, flags); 943 if (status) 944 qtd_list_free (ehci, urb, qtd_list); 945 946 return status; 947} 948 949/*-------------------------------------------------------------------------*/ 950 951/* ehci_iso_stream ops work with both ITD and SITD */ 952 953static struct ehci_iso_stream * 954iso_stream_alloc (gfp_t mem_flags) 955{ 956 struct ehci_iso_stream *stream; 957 958 stream = kzalloc(sizeof *stream, mem_flags); 959 if (likely (stream != NULL)) { 960 INIT_LIST_HEAD(&stream->td_list); 961 INIT_LIST_HEAD(&stream->free_list); 962 stream->next_uframe = -1; 963 stream->refcount = 1; 964 } 965 return stream; 966} 967 968static void 969iso_stream_init ( 970 struct ehci_hcd *ehci, 971 struct ehci_iso_stream *stream, 972 struct usb_device *dev, 973 int pipe, 974 unsigned interval 975) 976{ 977 static const u8 smask_out [] = { 0x01, 0x03, 0x07, 0x0f, 0x1f, 0x3f }; 978 979 u32 buf1; 980 unsigned epnum, maxp; 981 int is_input; 982 long bandwidth; 983 984 /* 985 * this might be a "high bandwidth" highspeed endpoint, 986 * as encoded in the ep descriptor's wMaxPacket field 987 */ 988 epnum = usb_pipeendpoint (pipe); 989 is_input = usb_pipein (pipe) ? USB_DIR_IN : 0; 990 maxp = usb_maxpacket(dev, pipe, !is_input); 991 if (is_input) { 992 buf1 = (1 << 11); 993 } else { 994 buf1 = 0; 995 } 996 997 /* knows about ITD vs SITD */ 998 if (dev->speed == USB_SPEED_HIGH) { 999 unsigned multi = hb_mult(maxp); 1000 1001 stream->highspeed = 1; 1002 1003 maxp = max_packet(maxp); 1004 buf1 |= maxp; 1005 maxp *= multi; 1006 1007 stream->buf0 = cpu_to_hc32(ehci, (epnum << 8) | dev->devnum); 1008 stream->buf1 = cpu_to_hc32(ehci, buf1); 1009 stream->buf2 = cpu_to_hc32(ehci, multi); 1010 1011 /* usbfs wants to report the average usecs per frame tied up 1012 * when transfers on this endpoint are scheduled ... 1013 */ 1014 stream->usecs = HS_USECS_ISO (maxp); 1015 bandwidth = stream->usecs * 8; 1016 bandwidth /= interval; 1017 1018 } else { 1019 u32 addr; 1020 int think_time; 1021 int hs_transfers; 1022 1023 addr = dev->ttport << 24; 1024 if (!ehci_is_TDI(ehci) 1025 || (dev->tt->hub != 1026 ehci_to_hcd(ehci)->self.root_hub)) 1027 addr |= dev->tt->hub->devnum << 16; 1028 addr |= epnum << 8; 1029 addr |= dev->devnum; 1030 stream->usecs = HS_USECS_ISO (maxp); 1031 think_time = dev->tt ? dev->tt->think_time : 0; 1032 stream->tt_usecs = NS_TO_US (think_time + usb_calc_bus_time ( 1033 dev->speed, is_input, 1, maxp)); 1034 hs_transfers = max (1u, (maxp + 187) / 188); 1035 if (is_input) { 1036 u32 tmp; 1037 1038 addr |= 1 << 31; 1039 stream->c_usecs = stream->usecs; 1040 stream->usecs = HS_USECS_ISO (1); 1041 stream->raw_mask = 1; 1042 1043 /* c-mask as specified in USB 2.0 11.18.4 3.c */ 1044 tmp = (1 << (hs_transfers + 2)) - 1; 1045 stream->raw_mask |= tmp << (8 + 2); 1046 } else 1047 stream->raw_mask = smask_out [hs_transfers - 1]; 1048 bandwidth = stream->usecs + stream->c_usecs; 1049 bandwidth /= interval << 3; 1050 1051 /* stream->splits gets created from raw_mask later */ 1052 stream->address = cpu_to_hc32(ehci, addr); 1053 } 1054 stream->bandwidth = bandwidth; 1055 1056 stream->udev = dev; 1057 1058 stream->bEndpointAddress = is_input | epnum; 1059 stream->interval = interval; 1060 stream->maxp = maxp; 1061} 1062 1063static void 1064iso_stream_put(struct ehci_hcd *ehci, struct ehci_iso_stream *stream) 1065{ 1066 stream->refcount--; 1067 1068 /* free whenever just a dev->ep reference remains. 1069 * not like a QH -- no persistent state (toggle, halt) 1070 */ 1071 if (stream->refcount == 1) { 1072 // BUG_ON (!list_empty(&stream->td_list)); 1073 1074 while (!list_empty (&stream->free_list)) { 1075 struct list_head *entry; 1076 1077 entry = stream->free_list.next; 1078 list_del (entry); 1079 1080 /* knows about ITD vs SITD */ 1081 if (stream->highspeed) { 1082 struct ehci_itd *itd; 1083 1084 itd = list_entry (entry, struct ehci_itd, 1085 itd_list); 1086 dma_pool_free (ehci->itd_pool, itd, 1087 itd->itd_dma); 1088 } else { 1089 struct ehci_sitd *sitd; 1090 1091 sitd = list_entry (entry, struct ehci_sitd, 1092 sitd_list); 1093 dma_pool_free (ehci->sitd_pool, sitd, 1094 sitd->sitd_dma); 1095 } 1096 } 1097 1098 stream->bEndpointAddress &= 0x0f; 1099 if (stream->ep) 1100 stream->ep->hcpriv = NULL; 1101 1102 kfree(stream); 1103 } 1104} 1105 1106static inline struct ehci_iso_stream * 1107iso_stream_get (struct ehci_iso_stream *stream) 1108{ 1109 if (likely (stream != NULL)) 1110 stream->refcount++; 1111 return stream; 1112} 1113 1114static struct ehci_iso_stream * 1115iso_stream_find (struct ehci_hcd *ehci, struct urb *urb) 1116{ 1117 unsigned epnum; 1118 struct ehci_iso_stream *stream; 1119 struct usb_host_endpoint *ep; 1120 unsigned long flags; 1121 1122 epnum = usb_pipeendpoint (urb->pipe); 1123 if (usb_pipein(urb->pipe)) 1124 ep = urb->dev->ep_in[epnum]; 1125 else 1126 ep = urb->dev->ep_out[epnum]; 1127 1128 spin_lock_irqsave (&ehci->lock, flags); 1129 stream = ep->hcpriv; 1130 1131 if (unlikely (stream == NULL)) { 1132 stream = iso_stream_alloc(GFP_ATOMIC); 1133 if (likely (stream != NULL)) { 1134 /* dev->ep owns the initial refcount */ 1135 ep->hcpriv = stream; 1136 stream->ep = ep; 1137 iso_stream_init(ehci, stream, urb->dev, urb->pipe, 1138 urb->interval); 1139 } 1140 1141 /* if dev->ep [epnum] is a QH, hw is set */ 1142 } else if (unlikely (stream->hw != NULL)) { 1143 ehci_dbg (ehci, "dev %s ep%d%s, not iso??\n", 1144 urb->dev->devpath, epnum, 1145 usb_pipein(urb->pipe) ? "in" : "out"); 1146 stream = NULL; 1147 } 1148 1149 /* caller guarantees an eventual matching iso_stream_put */ 1150 stream = iso_stream_get (stream); 1151 1152 spin_unlock_irqrestore (&ehci->lock, flags); 1153 return stream; 1154} 1155 1156/*-------------------------------------------------------------------------*/ 1157 1158/* ehci_iso_sched ops can be ITD-only or SITD-only */ 1159 1160static struct ehci_iso_sched * 1161iso_sched_alloc (unsigned packets, gfp_t mem_flags) 1162{ 1163 struct ehci_iso_sched *iso_sched; 1164 int size = sizeof *iso_sched; 1165 1166 size += packets * sizeof (struct ehci_iso_packet); 1167 iso_sched = kzalloc(size, mem_flags); 1168 if (likely (iso_sched != NULL)) { 1169 INIT_LIST_HEAD (&iso_sched->td_list); 1170 } 1171 return iso_sched; 1172} 1173 1174static inline void 1175itd_sched_init( 1176 struct ehci_hcd *ehci, 1177 struct ehci_iso_sched *iso_sched, 1178 struct ehci_iso_stream *stream, 1179 struct urb *urb 1180) 1181{ 1182 unsigned i; 1183 dma_addr_t dma = urb->transfer_dma; 1184 1185 /* how many uframes are needed for these transfers */ 1186 iso_sched->span = urb->number_of_packets * stream->interval; 1187 1188 /* figure out per-uframe itd fields that we'll need later 1189 * when we fit new itds into the schedule. 1190 */ 1191 for (i = 0; i < urb->number_of_packets; i++) { 1192 struct ehci_iso_packet *uframe = &iso_sched->packet [i]; 1193 unsigned length; 1194 dma_addr_t buf; 1195 u32 trans; 1196 1197 length = urb->iso_frame_desc [i].length; 1198 buf = dma + urb->iso_frame_desc [i].offset; 1199 1200 trans = EHCI_ISOC_ACTIVE; 1201 trans |= buf & 0x0fff; 1202 if (unlikely (((i + 1) == urb->number_of_packets)) 1203 && !(urb->transfer_flags & URB_NO_INTERRUPT)) 1204 trans |= EHCI_ITD_IOC; 1205 trans |= length << 16; 1206 uframe->transaction = cpu_to_hc32(ehci, trans); 1207 1208 /* might need to cross a buffer page within a uframe */ 1209 uframe->bufp = (buf & ~(u64)0x0fff); 1210 buf += length; 1211 if (unlikely ((uframe->bufp != (buf & ~(u64)0x0fff)))) 1212 uframe->cross = 1; 1213 } 1214} 1215 1216static void 1217iso_sched_free ( 1218 struct ehci_iso_stream *stream, 1219 struct ehci_iso_sched *iso_sched 1220) 1221{ 1222 if (!iso_sched) 1223 return; 1224 // caller must hold ehci->lock! 1225 list_splice (&iso_sched->td_list, &stream->free_list); 1226 kfree (iso_sched); 1227} 1228 1229static int 1230itd_urb_transaction ( 1231 struct ehci_iso_stream *stream, 1232 struct ehci_hcd *ehci, 1233 struct urb *urb, 1234 gfp_t mem_flags 1235) 1236{ 1237 struct ehci_itd *itd; 1238 dma_addr_t itd_dma; 1239 int i; 1240 unsigned num_itds; 1241 struct ehci_iso_sched *sched; 1242 unsigned long flags; 1243 1244 sched = iso_sched_alloc (urb->number_of_packets, mem_flags); 1245 if (unlikely (sched == NULL)) 1246 return -ENOMEM; 1247 1248 itd_sched_init(ehci, sched, stream, urb); 1249 1250 if (urb->interval < 8) 1251 num_itds = 1 + (sched->span + 7) / 8; 1252 else 1253 num_itds = urb->number_of_packets; 1254 1255 /* allocate/init ITDs */ 1256 spin_lock_irqsave (&ehci->lock, flags); 1257 for (i = 0; i < num_itds; i++) { 1258 1259 /* free_list.next might be cache-hot ... but maybe 1260 * the HC caches it too. avoid that issue for now. 1261 */ 1262 1263 /* prefer previously-allocated itds */ 1264 if (likely (!list_empty(&stream->free_list))) { 1265 itd = list_entry (stream->free_list.prev, 1266 struct ehci_itd, itd_list); 1267 list_del (&itd->itd_list); 1268 itd_dma = itd->itd_dma; 1269 } else { 1270 spin_unlock_irqrestore (&ehci->lock, flags); 1271 itd = dma_pool_alloc (ehci->itd_pool, mem_flags, 1272 &itd_dma); 1273 spin_lock_irqsave (&ehci->lock, flags); 1274 if (!itd) { 1275 iso_sched_free(stream, sched); 1276 spin_unlock_irqrestore(&ehci->lock, flags); 1277 return -ENOMEM; 1278 } 1279 } 1280 1281 memset (itd, 0, sizeof *itd); 1282 itd->itd_dma = itd_dma; 1283 list_add (&itd->itd_list, &sched->td_list); 1284 } 1285 spin_unlock_irqrestore (&ehci->lock, flags); 1286 1287 /* temporarily store schedule info in hcpriv */ 1288 urb->hcpriv = sched; 1289 urb->error_count = 0; 1290 return 0; 1291} 1292 1293/*-------------------------------------------------------------------------*/ 1294 1295static inline int 1296itd_slot_ok ( 1297 struct ehci_hcd *ehci, 1298 u32 mod, 1299 u32 uframe, 1300 u8 usecs, 1301 u32 period 1302) 1303{ 1304 uframe %= period; 1305 do { 1306 /* can't commit more than uframe_periodic_max usec */ 1307 if (periodic_usecs (ehci, uframe >> 3, uframe & 0x7) 1308 > (ehci->uframe_periodic_max - usecs)) 1309 return 0; 1310 1311 /* we know urb->interval is 2^N uframes */ 1312 uframe += period; 1313 } while (uframe < mod); 1314 return 1; 1315} 1316 1317static inline int 1318sitd_slot_ok ( 1319 struct ehci_hcd *ehci, 1320 u32 mod, 1321 struct ehci_iso_stream *stream, 1322 u32 uframe, 1323 struct ehci_iso_sched *sched, 1324 u32 period_uframes 1325) 1326{ 1327 u32 mask, tmp; 1328 u32 frame, uf; 1329 1330 mask = stream->raw_mask << (uframe & 7); 1331 1332 /* for IN, don't wrap CSPLIT into the next frame */ 1333 if (mask & ~0xffff) 1334 return 0; 1335 1336 /* this multi-pass logic is simple, but performance may 1337 * suffer when the schedule data isn't cached. 1338 */ 1339 1340 /* check bandwidth */ 1341 uframe %= period_uframes; 1342 do { 1343 u32 max_used; 1344 1345 frame = uframe >> 3; 1346 uf = uframe & 7; 1347 1348#ifdef CONFIG_USB_EHCI_TT_NEWSCHED 1349 /* The tt's fullspeed bus bandwidth must be available. 1350 * tt_available scheduling guarantees 10+% for control/bulk. 1351 */ 1352 if (!tt_available (ehci, period_uframes << 3, 1353 stream->udev, frame, uf, stream->tt_usecs)) 1354 return 0; 1355#else 1356 /* tt must be idle for start(s), any gap, and csplit. 1357 * assume scheduling slop leaves 10+% for control/bulk. 1358 */ 1359 if (!tt_no_collision (ehci, period_uframes << 3, 1360 stream->udev, frame, mask)) 1361 return 0; 1362#endif 1363 1364 /* check starts (OUT uses more than one) */ 1365 max_used = ehci->uframe_periodic_max - stream->usecs; 1366 for (tmp = stream->raw_mask & 0xff; tmp; tmp >>= 1, uf++) { 1367 if (periodic_usecs (ehci, frame, uf) > max_used) 1368 return 0; 1369 } 1370 1371 /* for IN, check CSPLIT */ 1372 if (stream->c_usecs) { 1373 uf = uframe & 7; 1374 max_used = ehci->uframe_periodic_max - stream->c_usecs; 1375 do { 1376 tmp = 1 << uf; 1377 tmp <<= 8; 1378 if ((stream->raw_mask & tmp) == 0) 1379 continue; 1380 if (periodic_usecs (ehci, frame, uf) 1381 > max_used) 1382 return 0; 1383 } while (++uf < 8); 1384 } 1385 1386 /* we know urb->interval is 2^N uframes */ 1387 uframe += period_uframes; 1388 } while (uframe < mod); 1389 1390 stream->splits = cpu_to_hc32(ehci, stream->raw_mask << (uframe & 7)); 1391 return 1; 1392} 1393 1394/* 1395 * This scheduler plans almost as far into the future as it has actual 1396 * periodic schedule slots. (Affected by TUNE_FLS, which defaults to 1397 * "as small as possible" to be cache-friendlier.) That limits the size 1398 * transfers you can stream reliably; avoid more than 64 msec per urb. 1399 * Also avoid queue depths of less than ehci's worst irq latency (affected 1400 * by the per-urb URB_NO_INTERRUPT hint, the log2_irq_thresh module parameter, 1401 * and other factors); or more than about 230 msec total (for portability, 1402 * given EHCI_TUNE_FLS and the slop). Or, write a smarter scheduler! 1403 */ 1404 1405#define SCHEDULE_SLOP 80 /* microframes */ 1406 1407static int 1408iso_stream_schedule ( 1409 struct ehci_hcd *ehci, 1410 struct urb *urb, 1411 struct ehci_iso_stream *stream 1412) 1413{ 1414 u32 now, next, start, period, span; 1415 int status; 1416 unsigned mod = ehci->periodic_size << 3; 1417 struct ehci_iso_sched *sched = urb->hcpriv; 1418 1419 period = urb->interval; 1420 span = sched->span; 1421 if (!stream->highspeed) { 1422 period <<= 3; 1423 span <<= 3; 1424 } 1425 1426 if (span > mod - SCHEDULE_SLOP) { 1427 ehci_dbg (ehci, "iso request %p too long\n", urb); 1428 status = -EFBIG; 1429 goto fail; 1430 } 1431 1432 now = ehci_read_frame_index(ehci) & (mod - 1); 1433 1434 /* Typical case: reuse current schedule, stream is still active. 1435 * Hopefully there are no gaps from the host falling behind 1436 * (irq delays etc), but if there are we'll take the next 1437 * slot in the schedule, implicitly assuming URB_ISO_ASAP. 1438 */ 1439 if (likely (!list_empty (&stream->td_list))) { 1440 u32 excess; 1441 1442 /* For high speed devices, allow scheduling within the 1443 * isochronous scheduling threshold. For full speed devices 1444 * and Intel PCI-based controllers, don't (work around for 1445 * Intel ICH9 bug). 1446 */ 1447 if (!stream->highspeed && ehci->fs_i_thresh) 1448 next = now + ehci->i_thresh; 1449 else 1450 next = now; 1451 1452 /* Fell behind (by up to twice the slop amount)? 1453 * We decide based on the time of the last currently-scheduled 1454 * slot, not the time of the next available slot. 1455 */ 1456 excess = (stream->next_uframe - period - next) & (mod - 1); 1457 if (excess >= mod - 2 * SCHEDULE_SLOP) 1458 start = next + excess - mod + period * 1459 DIV_ROUND_UP(mod - excess, period); 1460 else 1461 start = next + excess + period; 1462 if (start - now >= mod) { 1463 ehci_dbg(ehci, "request %p would overflow (%d+%d >= %d)\n", 1464 urb, start - now - period, period, 1465 mod); 1466 status = -EFBIG; 1467 goto fail; 1468 } 1469 } 1470 1471 /* need to schedule; when's the next (u)frame we could start? 1472 * this is bigger than ehci->i_thresh allows; scheduling itself 1473 * isn't free, the slop should handle reasonably slow cpus. it 1474 * can also help high bandwidth if the dma and irq loads don't 1475 * jump until after the queue is primed. 1476 */ 1477 else { 1478 int done = 0; 1479 start = SCHEDULE_SLOP + (now & ~0x07); 1480 1481 /* NOTE: assumes URB_ISO_ASAP, to limit complexity/bugs */ 1482 1483 /* find a uframe slot with enough bandwidth. 1484 * Early uframes are more precious because full-speed 1485 * iso IN transfers can't use late uframes, 1486 * and therefore they should be allocated last. 1487 */ 1488 next = start; 1489 start += period; 1490 do { 1491 start--; 1492 /* check schedule: enough space? */ 1493 if (stream->highspeed) { 1494 if (itd_slot_ok(ehci, mod, start, 1495 stream->usecs, period)) 1496 done = 1; 1497 } else { 1498 if ((start % 8) >= 6) 1499 continue; 1500 if (sitd_slot_ok(ehci, mod, stream, 1501 start, sched, period)) 1502 done = 1; 1503 } 1504 } while (start > next && !done); 1505 1506 /* no room in the schedule */ 1507 if (!done) { 1508 ehci_dbg(ehci, "iso resched full %p (now %d max %d)\n", 1509 urb, now, now + mod); 1510 status = -ENOSPC; 1511 goto fail; 1512 } 1513 } 1514 1515 /* Tried to schedule too far into the future? */ 1516 if (unlikely(start - now + span - period 1517 >= mod - 2 * SCHEDULE_SLOP)) { 1518 ehci_dbg(ehci, "request %p would overflow (%d+%d >= %d)\n", 1519 urb, start - now, span - period, 1520 mod - 2 * SCHEDULE_SLOP); 1521 status = -EFBIG; 1522 goto fail; 1523 } 1524 1525 stream->next_uframe = start & (mod - 1); 1526 1527 /* report high speed start in uframes; full speed, in frames */ 1528 urb->start_frame = stream->next_uframe; 1529 if (!stream->highspeed) 1530 urb->start_frame >>= 3; 1531 return 0; 1532 1533 fail: 1534 iso_sched_free(stream, sched); 1535 urb->hcpriv = NULL; 1536 return status; 1537} 1538 1539/*-------------------------------------------------------------------------*/ 1540 1541static inline void 1542itd_init(struct ehci_hcd *ehci, struct ehci_iso_stream *stream, 1543 struct ehci_itd *itd) 1544{ 1545 int i; 1546 1547 /* it's been recently zeroed */ 1548 itd->hw_next = EHCI_LIST_END(ehci); 1549 itd->hw_bufp [0] = stream->buf0; 1550 itd->hw_bufp [1] = stream->buf1; 1551 itd->hw_bufp [2] = stream->buf2; 1552 1553 for (i = 0; i < 8; i++) 1554 itd->index[i] = -1; 1555 1556 /* All other fields are filled when scheduling */ 1557} 1558 1559static inline void 1560itd_patch( 1561 struct ehci_hcd *ehci, 1562 struct ehci_itd *itd, 1563 struct ehci_iso_sched *iso_sched, 1564 unsigned index, 1565 u16 uframe 1566) 1567{ 1568 struct ehci_iso_packet *uf = &iso_sched->packet [index]; 1569 unsigned pg = itd->pg; 1570 1571 // BUG_ON (pg == 6 && uf->cross); 1572 1573 uframe &= 0x07; 1574 itd->index [uframe] = index; 1575 1576 itd->hw_transaction[uframe] = uf->transaction; 1577 itd->hw_transaction[uframe] |= cpu_to_hc32(ehci, pg << 12); 1578 itd->hw_bufp[pg] |= cpu_to_hc32(ehci, uf->bufp & ~(u32)0); 1579 itd->hw_bufp_hi[pg] |= cpu_to_hc32(ehci, (u32)(uf->bufp >> 32)); 1580 1581 /* iso_frame_desc[].offset must be strictly increasing */ 1582 if (unlikely (uf->cross)) { 1583 u64 bufp = uf->bufp + 4096; 1584 1585 itd->pg = ++pg; 1586 itd->hw_bufp[pg] |= cpu_to_hc32(ehci, bufp & ~(u32)0); 1587 itd->hw_bufp_hi[pg] |= cpu_to_hc32(ehci, (u32)(bufp >> 32)); 1588 } 1589} 1590 1591static inline void 1592itd_link (struct ehci_hcd *ehci, unsigned frame, struct ehci_itd *itd) 1593{ 1594 union ehci_shadow *prev = &ehci->pshadow[frame]; 1595 __hc32 *hw_p = &ehci->periodic[frame]; 1596 union ehci_shadow here = *prev; 1597 __hc32 type = 0; 1598 1599 /* skip any iso nodes which might belong to previous microframes */ 1600 while (here.ptr) { 1601 type = Q_NEXT_TYPE(ehci, *hw_p); 1602 if (type == cpu_to_hc32(ehci, Q_TYPE_QH)) 1603 break; 1604 prev = periodic_next_shadow(ehci, prev, type); 1605 hw_p = shadow_next_periodic(ehci, &here, type); 1606 here = *prev; 1607 } 1608 1609 itd->itd_next = here; 1610 itd->hw_next = *hw_p; 1611 prev->itd = itd; 1612 itd->frame = frame; 1613 wmb (); 1614 *hw_p = cpu_to_hc32(ehci, itd->itd_dma | Q_TYPE_ITD); 1615} 1616 1617/* fit urb's itds into the selected schedule slot; activate as needed */ 1618static int 1619itd_link_urb ( 1620 struct ehci_hcd *ehci, 1621 struct urb *urb, 1622 unsigned mod, 1623 struct ehci_iso_stream *stream 1624) 1625{ 1626 int packet; 1627 unsigned next_uframe, uframe, frame; 1628 struct ehci_iso_sched *iso_sched = urb->hcpriv; 1629 struct ehci_itd *itd; 1630 1631 next_uframe = stream->next_uframe & (mod - 1); 1632 1633 if (unlikely (list_empty(&stream->td_list))) { 1634 ehci_to_hcd(ehci)->self.bandwidth_allocated 1635 += stream->bandwidth; 1636 ehci_vdbg (ehci, 1637 "schedule devp %s ep%d%s-iso period %d start %d.%d\n", 1638 urb->dev->devpath, stream->bEndpointAddress & 0x0f, 1639 (stream->bEndpointAddress & USB_DIR_IN) ? "in" : "out", 1640 urb->interval, 1641 next_uframe >> 3, next_uframe & 0x7); 1642 } 1643 1644 if (ehci_to_hcd(ehci)->self.bandwidth_isoc_reqs == 0) { 1645 if (ehci->amd_pll_fix == 1) 1646 usb_amd_quirk_pll_disable(); 1647 } 1648 1649 ehci_to_hcd(ehci)->self.bandwidth_isoc_reqs++; 1650 1651 /* fill iTDs uframe by uframe */ 1652 for (packet = 0, itd = NULL; packet < urb->number_of_packets; ) { 1653 if (itd == NULL) { 1654 /* ASSERT: we have all necessary itds */ 1655 // BUG_ON (list_empty (&iso_sched->td_list)); 1656 1657 /* ASSERT: no itds for this endpoint in this uframe */ 1658 1659 itd = list_entry (iso_sched->td_list.next, 1660 struct ehci_itd, itd_list); 1661 list_move_tail (&itd->itd_list, &stream->td_list); 1662 itd->stream = iso_stream_get (stream); 1663 itd->urb = urb; 1664 itd_init (ehci, stream, itd); 1665 } 1666 1667 uframe = next_uframe & 0x07; 1668 frame = next_uframe >> 3; 1669 1670 itd_patch(ehci, itd, iso_sched, packet, uframe); 1671 1672 next_uframe += stream->interval; 1673 next_uframe &= mod - 1; 1674 packet++; 1675 1676 /* link completed itds into the schedule */ 1677 if (((next_uframe >> 3) != frame) 1678 || packet == urb->number_of_packets) { 1679 itd_link(ehci, frame & (ehci->periodic_size - 1), itd); 1680 itd = NULL; 1681 } 1682 } 1683 stream->next_uframe = next_uframe; 1684 1685 /* don't need that schedule data any more */ 1686 iso_sched_free (stream, iso_sched); 1687 urb->hcpriv = NULL; 1688 1689 timer_action (ehci, TIMER_IO_WATCHDOG); 1690 return enable_periodic(ehci); 1691} 1692 1693#define ISO_ERRS (EHCI_ISOC_BUF_ERR | EHCI_ISOC_BABBLE | EHCI_ISOC_XACTERR) 1694 1695/* Process and recycle a completed ITD. Return true iff its urb completed, 1696 * and hence its completion callback probably added things to the hardware 1697 * schedule. 1698 * 1699 * Note that we carefully avoid recycling this descriptor until after any 1700 * completion callback runs, so that it won't be reused quickly. That is, 1701 * assuming (a) no more than two urbs per frame on this endpoint, and also 1702 * (b) only this endpoint's completions submit URBs. It seems some silicon 1703 * corrupts things if you reuse completed descriptors very quickly... 1704 */ 1705static unsigned 1706itd_complete ( 1707 struct ehci_hcd *ehci, 1708 struct ehci_itd *itd 1709) { 1710 struct urb *urb = itd->urb; 1711 struct usb_iso_packet_descriptor *desc; 1712 u32 t; 1713 unsigned uframe; 1714 int urb_index = -1; 1715 struct ehci_iso_stream *stream = itd->stream; 1716 struct usb_device *dev; 1717 unsigned retval = false; 1718 1719 /* for each uframe with a packet */ 1720 for (uframe = 0; uframe < 8; uframe++) { 1721 if (likely (itd->index[uframe] == -1)) 1722 continue; 1723 urb_index = itd->index[uframe]; 1724 desc = &urb->iso_frame_desc [urb_index]; 1725 1726 t = hc32_to_cpup(ehci, &itd->hw_transaction [uframe]); 1727 itd->hw_transaction [uframe] = 0; 1728 1729 /* report transfer status */ 1730 if (unlikely (t & ISO_ERRS)) { 1731 urb->error_count++; 1732 if (t & EHCI_ISOC_BUF_ERR) 1733 desc->status = usb_pipein (urb->pipe) 1734 ? -ENOSR /* hc couldn't read */ 1735 : -ECOMM; /* hc couldn't write */ 1736 else if (t & EHCI_ISOC_BABBLE) 1737 desc->status = -EOVERFLOW; 1738 else /* (t & EHCI_ISOC_XACTERR) */ 1739 desc->status = -EPROTO; 1740 1741 /* HC need not update length with this error */ 1742 if (!(t & EHCI_ISOC_BABBLE)) { 1743 desc->actual_length = EHCI_ITD_LENGTH(t); 1744 urb->actual_length += desc->actual_length; 1745 } 1746 } else if (likely ((t & EHCI_ISOC_ACTIVE) == 0)) { 1747 desc->status = 0; 1748 desc->actual_length = EHCI_ITD_LENGTH(t); 1749 urb->actual_length += desc->actual_length; 1750 } else { 1751 /* URB was too late */ 1752 desc->status = -EXDEV; 1753 } 1754 } 1755 1756 /* handle completion now? */ 1757 if (likely ((urb_index + 1) != urb->number_of_packets)) 1758 goto done; 1759 1760 /* ASSERT: it's really the last itd for this urb 1761 list_for_each_entry (itd, &stream->td_list, itd_list) 1762 BUG_ON (itd->urb == urb); 1763 */ 1764 1765 /* give urb back to the driver; completion often (re)submits */ 1766 dev = urb->dev; 1767 ehci_urb_done(ehci, urb, 0); 1768 retval = true; 1769 urb = NULL; 1770 (void) disable_periodic(ehci); 1771 ehci_to_hcd(ehci)->self.bandwidth_isoc_reqs--; 1772 1773 if (ehci_to_hcd(ehci)->self.bandwidth_isoc_reqs == 0) { 1774 if (ehci->amd_pll_fix == 1) 1775 usb_amd_quirk_pll_enable(); 1776 } 1777 1778 if (unlikely(list_is_singular(&stream->td_list))) { 1779 ehci_to_hcd(ehci)->self.bandwidth_allocated 1780 -= stream->bandwidth; 1781 ehci_vdbg (ehci, 1782 "deschedule devp %s ep%d%s-iso\n", 1783 dev->devpath, stream->bEndpointAddress & 0x0f, 1784 (stream->bEndpointAddress & USB_DIR_IN) ? "in" : "out"); 1785 } 1786 iso_stream_put (ehci, stream); 1787 1788done: 1789 itd->urb = NULL; 1790 if (ehci->clock_frame != itd->frame || itd->index[7] != -1) { 1791 /* OK to recycle this ITD now. */ 1792 itd->stream = NULL; 1793 list_move(&itd->itd_list, &stream->free_list); 1794 iso_stream_put(ehci, stream); 1795 } else { 1796 /* HW might remember this ITD, so we can't recycle it yet. 1797 * Move it to a safe place until a new frame starts. 1798 */ 1799 list_move(&itd->itd_list, &ehci->cached_itd_list); 1800 if (stream->refcount == 2) { 1801 /* If iso_stream_put() were called here, stream 1802 * would be freed. Instead, just prevent reuse. 1803 */ 1804 stream->ep->hcpriv = NULL; 1805 stream->ep = NULL; 1806 } 1807 } 1808 return retval; 1809} 1810 1811/*-------------------------------------------------------------------------*/ 1812 1813static int itd_submit (struct ehci_hcd *ehci, struct urb *urb, 1814 gfp_t mem_flags) 1815{ 1816 int status = -EINVAL; 1817 unsigned long flags; 1818 struct ehci_iso_stream *stream; 1819 1820 /* Get iso_stream head */ 1821 stream = iso_stream_find (ehci, urb); 1822 if (unlikely (stream == NULL)) { 1823 ehci_dbg (ehci, "can't get iso stream\n"); 1824 return -ENOMEM; 1825 } 1826 if (unlikely (urb->interval != stream->interval)) { 1827 ehci_dbg (ehci, "can't change iso interval %d --> %d\n", 1828 stream->interval, urb->interval); 1829 goto done; 1830 } 1831 1832#ifdef EHCI_URB_TRACE 1833 ehci_dbg (ehci, 1834 "%s %s urb %p ep%d%s len %d, %d pkts %d uframes [%p]\n", 1835 __func__, urb->dev->devpath, urb, 1836 usb_pipeendpoint (urb->pipe), 1837 usb_pipein (urb->pipe) ? "in" : "out", 1838 urb->transfer_buffer_length, 1839 urb->number_of_packets, urb->interval, 1840 stream); 1841#endif 1842 1843 /* allocate ITDs w/o locking anything */ 1844 status = itd_urb_transaction (stream, ehci, urb, mem_flags); 1845 if (unlikely (status < 0)) { 1846 ehci_dbg (ehci, "can't init itds\n"); 1847 goto done; 1848 } 1849 1850 /* schedule ... need to lock */ 1851 spin_lock_irqsave (&ehci->lock, flags); 1852 if (unlikely(!HCD_HW_ACCESSIBLE(ehci_to_hcd(ehci)))) { 1853 status = -ESHUTDOWN; 1854 goto done_not_linked; 1855 } 1856 status = usb_hcd_link_urb_to_ep(ehci_to_hcd(ehci), urb); 1857 if (unlikely(status)) 1858 goto done_not_linked; 1859 status = iso_stream_schedule(ehci, urb, stream); 1860 if (likely (status == 0)) 1861 itd_link_urb (ehci, urb, ehci->periodic_size << 3, stream); 1862 else 1863 usb_hcd_unlink_urb_from_ep(ehci_to_hcd(ehci), urb); 1864done_not_linked: 1865 spin_unlock_irqrestore (&ehci->lock, flags); 1866 1867done: 1868 if (unlikely (status < 0)) 1869 iso_stream_put (ehci, stream); 1870 return status; 1871} 1872 1873/*-------------------------------------------------------------------------*/ 1874 1875/* 1876 * "Split ISO TDs" ... used for USB 1.1 devices going through the 1877 * TTs in USB 2.0 hubs. These need microframe scheduling. 1878 */ 1879 1880static inline void 1881sitd_sched_init( 1882 struct ehci_hcd *ehci, 1883 struct ehci_iso_sched *iso_sched, 1884 struct ehci_iso_stream *stream, 1885 struct urb *urb 1886) 1887{ 1888 unsigned i; 1889 dma_addr_t dma = urb->transfer_dma; 1890 1891 /* how many frames are needed for these transfers */ 1892 iso_sched->span = urb->number_of_packets * stream->interval; 1893 1894 /* figure out per-frame sitd fields that we'll need later 1895 * when we fit new sitds into the schedule. 1896 */ 1897 for (i = 0; i < urb->number_of_packets; i++) { 1898 struct ehci_iso_packet *packet = &iso_sched->packet [i]; 1899 unsigned length; 1900 dma_addr_t buf; 1901 u32 trans; 1902 1903 length = urb->iso_frame_desc [i].length & 0x03ff; 1904 buf = dma + urb->iso_frame_desc [i].offset; 1905 1906 trans = SITD_STS_ACTIVE; 1907 if (((i + 1) == urb->number_of_packets) 1908 && !(urb->transfer_flags & URB_NO_INTERRUPT)) 1909 trans |= SITD_IOC; 1910 trans |= length << 16; 1911 packet->transaction = cpu_to_hc32(ehci, trans); 1912 1913 /* might need to cross a buffer page within a td */ 1914 packet->bufp = buf; 1915 packet->buf1 = (buf + length) & ~0x0fff; 1916 if (packet->buf1 != (buf & ~(u64)0x0fff)) 1917 packet->cross = 1; 1918 1919 /* OUT uses multiple start-splits */ 1920 if (stream->bEndpointAddress & USB_DIR_IN) 1921 continue; 1922 length = (length + 187) / 188; 1923 if (length > 1) /* BEGIN vs ALL */ 1924 length |= 1 << 3; 1925 packet->buf1 |= length; 1926 } 1927} 1928 1929static int 1930sitd_urb_transaction ( 1931 struct ehci_iso_stream *stream, 1932 struct ehci_hcd *ehci, 1933 struct urb *urb, 1934 gfp_t mem_flags 1935) 1936{ 1937 struct ehci_sitd *sitd; 1938 dma_addr_t sitd_dma; 1939 int i; 1940 struct ehci_iso_sched *iso_sched; 1941 unsigned long flags; 1942 1943 iso_sched = iso_sched_alloc (urb->number_of_packets, mem_flags); 1944 if (iso_sched == NULL) 1945 return -ENOMEM; 1946 1947 sitd_sched_init(ehci, iso_sched, stream, urb); 1948 1949 /* allocate/init sITDs */ 1950 spin_lock_irqsave (&ehci->lock, flags); 1951 for (i = 0; i < urb->number_of_packets; i++) { 1952 1953 /* NOTE: for now, we don't try to handle wraparound cases 1954 * for IN (using sitd->hw_backpointer, like a FSTN), which 1955 * means we never need two sitds for full speed packets. 1956 */ 1957 1958 /* free_list.next might be cache-hot ... but maybe 1959 * the HC caches it too. avoid that issue for now. 1960 */ 1961 1962 /* prefer previously-allocated sitds */ 1963 if (!list_empty(&stream->free_list)) { 1964 sitd = list_entry (stream->free_list.prev, 1965 struct ehci_sitd, sitd_list); 1966 list_del (&sitd->sitd_list); 1967 sitd_dma = sitd->sitd_dma; 1968 } else { 1969 spin_unlock_irqrestore (&ehci->lock, flags); 1970 sitd = dma_pool_alloc (ehci->sitd_pool, mem_flags, 1971 &sitd_dma); 1972 spin_lock_irqsave (&ehci->lock, flags); 1973 if (!sitd) { 1974 iso_sched_free(stream, iso_sched); 1975 spin_unlock_irqrestore(&ehci->lock, flags); 1976 return -ENOMEM; 1977 } 1978 } 1979 1980 memset (sitd, 0, sizeof *sitd); 1981 sitd->sitd_dma = sitd_dma; 1982 list_add (&sitd->sitd_list, &iso_sched->td_list); 1983 } 1984 1985 /* temporarily store schedule info in hcpriv */ 1986 urb->hcpriv = iso_sched; 1987 urb->error_count = 0; 1988 1989 spin_unlock_irqrestore (&ehci->lock, flags); 1990 return 0; 1991} 1992 1993/*-------------------------------------------------------------------------*/ 1994 1995static inline void 1996sitd_patch( 1997 struct ehci_hcd *ehci, 1998 struct ehci_iso_stream *stream, 1999 struct ehci_sitd *sitd, 2000 struct ehci_iso_sched *iso_sched, 2001 unsigned index 2002) 2003{ 2004 struct ehci_iso_packet *uf = &iso_sched->packet [index]; 2005 u64 bufp = uf->bufp; 2006 2007 sitd->hw_next = EHCI_LIST_END(ehci); 2008 sitd->hw_fullspeed_ep = stream->address; 2009 sitd->hw_uframe = stream->splits; 2010 sitd->hw_results = uf->transaction; 2011 sitd->hw_backpointer = EHCI_LIST_END(ehci); 2012 2013 bufp = uf->bufp; 2014 sitd->hw_buf[0] = cpu_to_hc32(ehci, bufp); 2015 sitd->hw_buf_hi[0] = cpu_to_hc32(ehci, bufp >> 32); 2016 2017 sitd->hw_buf[1] = cpu_to_hc32(ehci, uf->buf1); 2018 if (uf->cross) 2019 bufp += 4096; 2020 sitd->hw_buf_hi[1] = cpu_to_hc32(ehci, bufp >> 32); 2021 sitd->index = index; 2022} 2023 2024static inline void 2025sitd_link (struct ehci_hcd *ehci, unsigned frame, struct ehci_sitd *sitd) 2026{ 2027 /* note: sitd ordering could matter (CSPLIT then SSPLIT) */ 2028 sitd->sitd_next = ehci->pshadow [frame]; 2029 sitd->hw_next = ehci->periodic [frame]; 2030 ehci->pshadow [frame].sitd = sitd; 2031 sitd->frame = frame; 2032 wmb (); 2033 ehci->periodic[frame] = cpu_to_hc32(ehci, sitd->sitd_dma | Q_TYPE_SITD); 2034} 2035 2036/* fit urb's sitds into the selected schedule slot; activate as needed */ 2037static int 2038sitd_link_urb ( 2039 struct ehci_hcd *ehci, 2040 struct urb *urb, 2041 unsigned mod, 2042 struct ehci_iso_stream *stream 2043) 2044{ 2045 int packet; 2046 unsigned next_uframe; 2047 struct ehci_iso_sched *sched = urb->hcpriv; 2048 struct ehci_sitd *sitd; 2049 2050 next_uframe = stream->next_uframe; 2051 2052 if (list_empty(&stream->td_list)) { 2053 /* usbfs ignores TT bandwidth */ 2054 ehci_to_hcd(ehci)->self.bandwidth_allocated 2055 += stream->bandwidth; 2056 ehci_vdbg (ehci, 2057 "sched devp %s ep%d%s-iso [%d] %dms/%04x\n", 2058 urb->dev->devpath, stream->bEndpointAddress & 0x0f, 2059 (stream->bEndpointAddress & USB_DIR_IN) ? "in" : "out", 2060 (next_uframe >> 3) & (ehci->periodic_size - 1), 2061 stream->interval, hc32_to_cpu(ehci, stream->splits)); 2062 } 2063 2064 if (ehci_to_hcd(ehci)->self.bandwidth_isoc_reqs == 0) { 2065 if (ehci->amd_pll_fix == 1) 2066 usb_amd_quirk_pll_disable(); 2067 } 2068 2069 ehci_to_hcd(ehci)->self.bandwidth_isoc_reqs++; 2070 2071 /* fill sITDs frame by frame */ 2072 for (packet = 0, sitd = NULL; 2073 packet < urb->number_of_packets; 2074 packet++) { 2075 2076 /* ASSERT: we have all necessary sitds */ 2077 BUG_ON (list_empty (&sched->td_list)); 2078 2079 /* ASSERT: no itds for this endpoint in this frame */ 2080 2081 sitd = list_entry (sched->td_list.next, 2082 struct ehci_sitd, sitd_list); 2083 list_move_tail (&sitd->sitd_list, &stream->td_list); 2084 sitd->stream = iso_stream_get (stream); 2085 sitd->urb = urb; 2086 2087 sitd_patch(ehci, stream, sitd, sched, packet); 2088 sitd_link(ehci, (next_uframe >> 3) & (ehci->periodic_size - 1), 2089 sitd); 2090 2091 next_uframe += stream->interval << 3; 2092 } 2093 stream->next_uframe = next_uframe & (mod - 1); 2094 2095 /* don't need that schedule data any more */ 2096 iso_sched_free (stream, sched); 2097 urb->hcpriv = NULL; 2098 2099 timer_action (ehci, TIMER_IO_WATCHDOG); 2100 return enable_periodic(ehci); 2101} 2102 2103/*-------------------------------------------------------------------------*/ 2104 2105#define SITD_ERRS (SITD_STS_ERR | SITD_STS_DBE | SITD_STS_BABBLE \ 2106 | SITD_STS_XACT | SITD_STS_MMF) 2107 2108/* Process and recycle a completed SITD. Return true iff its urb completed, 2109 * and hence its completion callback probably added things to the hardware 2110 * schedule. 2111 * 2112 * Note that we carefully avoid recycling this descriptor until after any 2113 * completion callback runs, so that it won't be reused quickly. That is, 2114 * assuming (a) no more than two urbs per frame on this endpoint, and also 2115 * (b) only this endpoint's completions submit URBs. It seems some silicon 2116 * corrupts things if you reuse completed descriptors very quickly... 2117 */ 2118static unsigned 2119sitd_complete ( 2120 struct ehci_hcd *ehci, 2121 struct ehci_sitd *sitd 2122) { 2123 struct urb *urb = sitd->urb; 2124 struct usb_iso_packet_descriptor *desc; 2125 u32 t; 2126 int urb_index = -1; 2127 struct ehci_iso_stream *stream = sitd->stream; 2128 struct usb_device *dev; 2129 unsigned retval = false; 2130 2131 urb_index = sitd->index; 2132 desc = &urb->iso_frame_desc [urb_index]; 2133 t = hc32_to_cpup(ehci, &sitd->hw_results); 2134 2135 /* report transfer status */ 2136 if (t & SITD_ERRS) { 2137 urb->error_count++; 2138 if (t & SITD_STS_DBE) 2139 desc->status = usb_pipein (urb->pipe) 2140 ? -ENOSR /* hc couldn't read */ 2141 : -ECOMM; /* hc couldn't write */ 2142 else if (t & SITD_STS_BABBLE) 2143 desc->status = -EOVERFLOW; 2144 else /* XACT, MMF, etc */ 2145 desc->status = -EPROTO; 2146 } else { 2147 desc->status = 0; 2148 desc->actual_length = desc->length - SITD_LENGTH(t); 2149 urb->actual_length += desc->actual_length; 2150 } 2151 2152 /* handle completion now? */ 2153 if ((urb_index + 1) != urb->number_of_packets) 2154 goto done; 2155 2156 /* ASSERT: it's really the last sitd for this urb 2157 list_for_each_entry (sitd, &stream->td_list, sitd_list) 2158 BUG_ON (sitd->urb == urb); 2159 */ 2160 2161 /* give urb back to the driver; completion often (re)submits */ 2162 dev = urb->dev; 2163 ehci_urb_done(ehci, urb, 0); 2164 retval = true; 2165 urb = NULL; 2166 (void) disable_periodic(ehci); 2167 ehci_to_hcd(ehci)->self.bandwidth_isoc_reqs--; 2168 2169 if (ehci_to_hcd(ehci)->self.bandwidth_isoc_reqs == 0) { 2170 if (ehci->amd_pll_fix == 1) 2171 usb_amd_quirk_pll_enable(); 2172 } 2173 2174 if (list_is_singular(&stream->td_list)) { 2175 ehci_to_hcd(ehci)->self.bandwidth_allocated 2176 -= stream->bandwidth; 2177 ehci_vdbg (ehci, 2178 "deschedule devp %s ep%d%s-iso\n", 2179 dev->devpath, stream->bEndpointAddress & 0x0f, 2180 (stream->bEndpointAddress & USB_DIR_IN) ? "in" : "out"); 2181 } 2182 iso_stream_put (ehci, stream); 2183 2184done: 2185 sitd->urb = NULL; 2186 if (ehci->clock_frame != sitd->frame) { 2187 /* OK to recycle this SITD now. */ 2188 sitd->stream = NULL; 2189 list_move(&sitd->sitd_list, &stream->free_list); 2190 iso_stream_put(ehci, stream); 2191 } else { 2192 /* HW might remember this SITD, so we can't recycle it yet. 2193 * Move it to a safe place until a new frame starts. 2194 */ 2195 list_move(&sitd->sitd_list, &ehci->cached_sitd_list); 2196 if (stream->refcount == 2) { 2197 /* If iso_stream_put() were called here, stream 2198 * would be freed. Instead, just prevent reuse. 2199 */ 2200 stream->ep->hcpriv = NULL; 2201 stream->ep = NULL; 2202 } 2203 } 2204 return retval; 2205} 2206 2207 2208static int sitd_submit (struct ehci_hcd *ehci, struct urb *urb, 2209 gfp_t mem_flags) 2210{ 2211 int status = -EINVAL; 2212 unsigned long flags; 2213 struct ehci_iso_stream *stream; 2214 2215 /* Get iso_stream head */ 2216 stream = iso_stream_find (ehci, urb); 2217 if (stream == NULL) { 2218 ehci_dbg (ehci, "can't get iso stream\n"); 2219 return -ENOMEM; 2220 } 2221 if (urb->interval != stream->interval) { 2222 ehci_dbg (ehci, "can't change iso interval %d --> %d\n", 2223 stream->interval, urb->interval); 2224 goto done; 2225 } 2226 2227#ifdef EHCI_URB_TRACE 2228 ehci_dbg (ehci, 2229 "submit %p dev%s ep%d%s-iso len %d\n", 2230 urb, urb->dev->devpath, 2231 usb_pipeendpoint (urb->pipe), 2232 usb_pipein (urb->pipe) ? "in" : "out", 2233 urb->transfer_buffer_length); 2234#endif 2235 2236 /* allocate SITDs */ 2237 status = sitd_urb_transaction (stream, ehci, urb, mem_flags); 2238 if (status < 0) { 2239 ehci_dbg (ehci, "can't init sitds\n"); 2240 goto done; 2241 } 2242 2243 /* schedule ... need to lock */ 2244 spin_lock_irqsave (&ehci->lock, flags); 2245 if (unlikely(!HCD_HW_ACCESSIBLE(ehci_to_hcd(ehci)))) { 2246 status = -ESHUTDOWN; 2247 goto done_not_linked; 2248 } 2249 status = usb_hcd_link_urb_to_ep(ehci_to_hcd(ehci), urb); 2250 if (unlikely(status)) 2251 goto done_not_linked; 2252 status = iso_stream_schedule(ehci, urb, stream); 2253 if (status == 0) 2254 sitd_link_urb (ehci, urb, ehci->periodic_size << 3, stream); 2255 else 2256 usb_hcd_unlink_urb_from_ep(ehci_to_hcd(ehci), urb); 2257done_not_linked: 2258 spin_unlock_irqrestore (&ehci->lock, flags); 2259 2260done: 2261 if (status < 0) 2262 iso_stream_put (ehci, stream); 2263 return status; 2264} 2265 2266/*-------------------------------------------------------------------------*/ 2267 2268static void free_cached_lists(struct ehci_hcd *ehci) 2269{ 2270 struct ehci_itd *itd, *n; 2271 struct ehci_sitd *sitd, *sn; 2272 2273 list_for_each_entry_safe(itd, n, &ehci->cached_itd_list, itd_list) { 2274 struct ehci_iso_stream *stream = itd->stream; 2275 itd->stream = NULL; 2276 list_move(&itd->itd_list, &stream->free_list); 2277 iso_stream_put(ehci, stream); 2278 } 2279 2280 list_for_each_entry_safe(sitd, sn, &ehci->cached_sitd_list, sitd_list) { 2281 struct ehci_iso_stream *stream = sitd->stream; 2282 sitd->stream = NULL; 2283 list_move(&sitd->sitd_list, &stream->free_list); 2284 iso_stream_put(ehci, stream); 2285 } 2286} 2287 2288/*-------------------------------------------------------------------------*/ 2289 2290static void 2291scan_periodic (struct ehci_hcd *ehci) 2292{ 2293 unsigned now_uframe, frame, clock, clock_frame, mod; 2294 unsigned modified; 2295 2296 mod = ehci->periodic_size << 3; 2297 2298 /* 2299 * When running, scan from last scan point up to "now" 2300 * else clean up by scanning everything that's left. 2301 * Touches as few pages as possible: cache-friendly. 2302 */ 2303 now_uframe = ehci->next_uframe; 2304 if (ehci->rh_state == EHCI_RH_RUNNING) { 2305 clock = ehci_read_frame_index(ehci); 2306 clock_frame = (clock >> 3) & (ehci->periodic_size - 1); 2307 } else { 2308 clock = now_uframe + mod - 1; 2309 clock_frame = -1; 2310 } 2311 if (ehci->clock_frame != clock_frame) { 2312 free_cached_lists(ehci); 2313 ehci->clock_frame = clock_frame; 2314 } 2315 clock &= mod - 1; 2316 clock_frame = clock >> 3; 2317 ++ehci->periodic_stamp; 2318 2319 for (;;) { 2320 union ehci_shadow q, *q_p; 2321 __hc32 type, *hw_p; 2322 unsigned incomplete = false; 2323 2324 frame = now_uframe >> 3; 2325 2326restart: 2327 /* scan each element in frame's queue for completions */ 2328 q_p = &ehci->pshadow [frame]; 2329 hw_p = &ehci->periodic [frame]; 2330 q.ptr = q_p->ptr; 2331 type = Q_NEXT_TYPE(ehci, *hw_p); 2332 modified = 0; 2333 2334 while (q.ptr != NULL) { 2335 unsigned uf; 2336 union ehci_shadow temp; 2337 int live; 2338 2339 live = (ehci->rh_state == EHCI_RH_RUNNING); 2340 switch (hc32_to_cpu(ehci, type)) { 2341 case Q_TYPE_QH: 2342 /* handle any completions */ 2343 temp.qh = qh_get (q.qh); 2344 type = Q_NEXT_TYPE(ehci, q.qh->hw->hw_next); 2345 q = q.qh->qh_next; 2346 if (temp.qh->stamp != ehci->periodic_stamp) { 2347 modified = qh_completions(ehci, temp.qh); 2348 if (!modified) 2349 temp.qh->stamp = ehci->periodic_stamp; 2350 if (unlikely(list_empty(&temp.qh->qtd_list) || 2351 temp.qh->needs_rescan)) 2352 intr_deschedule(ehci, temp.qh); 2353 } 2354 qh_put (temp.qh); 2355 break; 2356 case Q_TYPE_FSTN: 2357 /* for "save place" FSTNs, look at QH entries 2358 * in the previous frame for completions. 2359 */ 2360 if (q.fstn->hw_prev != EHCI_LIST_END(ehci)) { 2361 dbg ("ignoring completions from FSTNs"); 2362 } 2363 type = Q_NEXT_TYPE(ehci, q.fstn->hw_next); 2364 q = q.fstn->fstn_next; 2365 break; 2366 case Q_TYPE_ITD: 2367 /* If this ITD is still active, leave it for 2368 * later processing ... check the next entry. 2369 * No need to check for activity unless the 2370 * frame is current. 2371 */ 2372 if (frame == clock_frame && live) { 2373 rmb(); 2374 for (uf = 0; uf < 8; uf++) { 2375 if (q.itd->hw_transaction[uf] & 2376 ITD_ACTIVE(ehci)) 2377 break; 2378 } 2379 if (uf < 8) { 2380 incomplete = true; 2381 q_p = &q.itd->itd_next; 2382 hw_p = &q.itd->hw_next; 2383 type = Q_NEXT_TYPE(ehci, 2384 q.itd->hw_next); 2385 q = *q_p; 2386 break; 2387 } 2388 } 2389 2390 /* Take finished ITDs out of the schedule 2391 * and process them: recycle, maybe report 2392 * URB completion. HC won't cache the 2393 * pointer for much longer, if at all. 2394 */ 2395 *q_p = q.itd->itd_next; 2396 if (!ehci->use_dummy_qh || 2397 q.itd->hw_next != EHCI_LIST_END(ehci)) 2398 *hw_p = q.itd->hw_next; 2399 else 2400 *hw_p = ehci->dummy->qh_dma; 2401 type = Q_NEXT_TYPE(ehci, q.itd->hw_next); 2402 wmb(); 2403 modified = itd_complete (ehci, q.itd); 2404 q = *q_p; 2405 break; 2406 case Q_TYPE_SITD: 2407 /* If this SITD is still active, leave it for 2408 * later processing ... check the next entry. 2409 * No need to check for activity unless the 2410 * frame is current. 2411 */ 2412 if (((frame == clock_frame) || 2413 (((frame + 1) & (ehci->periodic_size - 1)) 2414 == clock_frame)) 2415 && live 2416 && (q.sitd->hw_results & 2417 SITD_ACTIVE(ehci))) { 2418 2419 incomplete = true; 2420 q_p = &q.sitd->sitd_next; 2421 hw_p = &q.sitd->hw_next; 2422 type = Q_NEXT_TYPE(ehci, 2423 q.sitd->hw_next); 2424 q = *q_p; 2425 break; 2426 } 2427 2428 /* Take finished SITDs out of the schedule 2429 * and process them: recycle, maybe report 2430 * URB completion. 2431 */ 2432 *q_p = q.sitd->sitd_next; 2433 if (!ehci->use_dummy_qh || 2434 q.sitd->hw_next != EHCI_LIST_END(ehci)) 2435 *hw_p = q.sitd->hw_next; 2436 else 2437 *hw_p = ehci->dummy->qh_dma; 2438 type = Q_NEXT_TYPE(ehci, q.sitd->hw_next); 2439 wmb(); 2440 modified = sitd_complete (ehci, q.sitd); 2441 q = *q_p; 2442 break; 2443 default: 2444 dbg ("corrupt type %d frame %d shadow %p", 2445 type, frame, q.ptr); 2446 // BUG (); 2447 q.ptr = NULL; 2448 } 2449 2450 /* assume completion callbacks modify the queue */ 2451 if (unlikely (modified)) { 2452 if (likely(ehci->periodic_sched > 0)) 2453 goto restart; 2454 /* short-circuit this scan */ 2455 now_uframe = clock; 2456 break; 2457 } 2458 } 2459 2460 /* If we can tell we caught up to the hardware, stop now. 2461 * We can't advance our scan without collecting the ISO 2462 * transfers that are still pending in this frame. 2463 */ 2464 if (incomplete && ehci->rh_state == EHCI_RH_RUNNING) { 2465 ehci->next_uframe = now_uframe; 2466 break; 2467 } 2468 2469 // FIXME: this assumes we won't get lapped when 2470 // latencies climb; that should be rare, but... 2471 // detect it, and just go all the way around. 2472 // FLR might help detect this case, so long as latencies 2473 // don't exceed periodic_size msec (default 1.024 sec). 2474 2475 // FIXME: likewise assumes HC doesn't halt mid-scan 2476 2477 if (now_uframe == clock) { 2478 unsigned now; 2479 2480 if (ehci->rh_state != EHCI_RH_RUNNING 2481 || ehci->periodic_sched == 0) 2482 break; 2483 ehci->next_uframe = now_uframe; 2484 now = ehci_read_frame_index(ehci) & (mod - 1); 2485 if (now_uframe == now) 2486 break; 2487 2488 /* rescan the rest of this frame, then ... */ 2489 clock = now; 2490 clock_frame = clock >> 3; 2491 if (ehci->clock_frame != clock_frame) { 2492 free_cached_lists(ehci); 2493 ehci->clock_frame = clock_frame; 2494 ++ehci->periodic_stamp; 2495 } 2496 } else { 2497 now_uframe++; 2498 now_uframe &= mod - 1; 2499 } 2500 } 2501} 2502