smtc.c revision ec43c01420fc1da8bf0b19f0ceb24d7d3c7f47f3
1/* Copyright (C) 2004 Mips Technologies, Inc */ 2 3#include <linux/kernel.h> 4#include <linux/sched.h> 5#include <linux/cpumask.h> 6#include <linux/interrupt.h> 7#include <linux/module.h> 8 9#include <asm/cpu.h> 10#include <asm/processor.h> 11#include <asm/atomic.h> 12#include <asm/system.h> 13#include <asm/hardirq.h> 14#include <asm/hazards.h> 15#include <asm/mmu_context.h> 16#include <asm/smp.h> 17#include <asm/mipsregs.h> 18#include <asm/cacheflush.h> 19#include <asm/time.h> 20#include <asm/addrspace.h> 21#include <asm/smtc.h> 22#include <asm/smtc_ipi.h> 23#include <asm/smtc_proc.h> 24 25/* 26 * This file should be built into the kernel only if CONFIG_MIPS_MT_SMTC is set. 27 */ 28 29/* 30 * MIPSCPU_INT_BASE is identically defined in both 31 * asm-mips/mips-boards/maltaint.h and asm-mips/mips-boards/simint.h, 32 * but as yet there's no properly organized include structure that 33 * will ensure that the right *int.h file will be included for a 34 * given platform build. 35 */ 36 37#define MIPSCPU_INT_BASE 16 38 39#define MIPS_CPU_IPI_IRQ 1 40 41#define LOCK_MT_PRA() \ 42 local_irq_save(flags); \ 43 mtflags = dmt() 44 45#define UNLOCK_MT_PRA() \ 46 emt(mtflags); \ 47 local_irq_restore(flags) 48 49#define LOCK_CORE_PRA() \ 50 local_irq_save(flags); \ 51 mtflags = dvpe() 52 53#define UNLOCK_CORE_PRA() \ 54 evpe(mtflags); \ 55 local_irq_restore(flags) 56 57/* 58 * Data structures purely associated with SMTC parallelism 59 */ 60 61 62/* 63 * Table for tracking ASIDs whose lifetime is prolonged. 64 */ 65 66asiduse smtc_live_asid[MAX_SMTC_TLBS][MAX_SMTC_ASIDS]; 67 68/* 69 * Clock interrupt "latch" buffers, per "CPU" 70 */ 71 72unsigned int ipi_timer_latch[NR_CPUS]; 73 74/* 75 * Number of InterProcessor Interupt (IPI) message buffers to allocate 76 */ 77 78#define IPIBUF_PER_CPU 4 79 80struct smtc_ipi_q IPIQ[NR_CPUS]; 81struct smtc_ipi_q freeIPIq; 82 83 84/* Forward declarations */ 85 86void ipi_decode(struct smtc_ipi *); 87void post_direct_ipi(int cpu, struct smtc_ipi *pipi); 88void setup_cross_vpe_interrupts(void); 89void init_smtc_stats(void); 90 91/* Global SMTC Status */ 92 93unsigned int smtc_status = 0; 94 95/* Boot command line configuration overrides */ 96 97static int vpelimit = 0; 98static int tclimit = 0; 99static int ipibuffers = 0; 100static int nostlb = 0; 101static int asidmask = 0; 102unsigned long smtc_asid_mask = 0xff; 103 104static int __init maxvpes(char *str) 105{ 106 get_option(&str, &vpelimit); 107 return 1; 108} 109 110static int __init maxtcs(char *str) 111{ 112 get_option(&str, &tclimit); 113 return 1; 114} 115 116static int __init ipibufs(char *str) 117{ 118 get_option(&str, &ipibuffers); 119 return 1; 120} 121 122static int __init stlb_disable(char *s) 123{ 124 nostlb = 1; 125 return 1; 126} 127 128static int __init asidmask_set(char *str) 129{ 130 get_option(&str, &asidmask); 131 switch (asidmask) { 132 case 0x1: 133 case 0x3: 134 case 0x7: 135 case 0xf: 136 case 0x1f: 137 case 0x3f: 138 case 0x7f: 139 case 0xff: 140 smtc_asid_mask = (unsigned long)asidmask; 141 break; 142 default: 143 printk("ILLEGAL ASID mask 0x%x from command line\n", asidmask); 144 } 145 return 1; 146} 147 148__setup("maxvpes=", maxvpes); 149__setup("maxtcs=", maxtcs); 150__setup("ipibufs=", ipibufs); 151__setup("nostlb", stlb_disable); 152__setup("asidmask=", asidmask_set); 153 154/* Enable additional debug checks before going into CPU idle loop */ 155#define SMTC_IDLE_HOOK_DEBUG 156 157#ifdef SMTC_IDLE_HOOK_DEBUG 158 159static int hang_trig = 0; 160 161static int __init hangtrig_enable(char *s) 162{ 163 hang_trig = 1; 164 return 1; 165} 166 167 168__setup("hangtrig", hangtrig_enable); 169 170#define DEFAULT_BLOCKED_IPI_LIMIT 32 171 172static int timerq_limit = DEFAULT_BLOCKED_IPI_LIMIT; 173 174static int __init tintq(char *str) 175{ 176 get_option(&str, &timerq_limit); 177 return 1; 178} 179 180__setup("tintq=", tintq); 181 182int imstuckcount[2][8]; 183/* vpemask represents IM/IE bits of per-VPE Status registers, low-to-high */ 184int vpemask[2][8] = {{0,1,1,0,0,0,0,1},{0,1,0,0,0,0,0,1}}; 185int tcnoprog[NR_CPUS]; 186static atomic_t idle_hook_initialized = {0}; 187static int clock_hang_reported[NR_CPUS]; 188 189#endif /* SMTC_IDLE_HOOK_DEBUG */ 190 191/* Initialize shared TLB - the should probably migrate to smtc_setup_cpus() */ 192 193void __init sanitize_tlb_entries(void) 194{ 195 printk("Deprecated sanitize_tlb_entries() invoked\n"); 196} 197 198 199/* 200 * Configure shared TLB - VPC configuration bit must be set by caller 201 */ 202 203void smtc_configure_tlb(void) 204{ 205 int i,tlbsiz,vpes; 206 unsigned long mvpconf0; 207 unsigned long config1val; 208 209 /* Set up ASID preservation table */ 210 for (vpes=0; vpes<MAX_SMTC_TLBS; vpes++) { 211 for(i = 0; i < MAX_SMTC_ASIDS; i++) { 212 smtc_live_asid[vpes][i] = 0; 213 } 214 } 215 mvpconf0 = read_c0_mvpconf0(); 216 217 if ((vpes = ((mvpconf0 & MVPCONF0_PVPE) 218 >> MVPCONF0_PVPE_SHIFT) + 1) > 1) { 219 /* If we have multiple VPEs, try to share the TLB */ 220 if ((mvpconf0 & MVPCONF0_TLBS) && !nostlb) { 221 /* 222 * If TLB sizing is programmable, shared TLB 223 * size is the total available complement. 224 * Otherwise, we have to take the sum of all 225 * static VPE TLB entries. 226 */ 227 if ((tlbsiz = ((mvpconf0 & MVPCONF0_PTLBE) 228 >> MVPCONF0_PTLBE_SHIFT)) == 0) { 229 /* 230 * If there's more than one VPE, there had better 231 * be more than one TC, because we need one to bind 232 * to each VPE in turn to be able to read 233 * its configuration state! 234 */ 235 settc(1); 236 /* Stop the TC from doing anything foolish */ 237 write_tc_c0_tchalt(TCHALT_H); 238 mips_ihb(); 239 /* No need to un-Halt - that happens later anyway */ 240 for (i=0; i < vpes; i++) { 241 write_tc_c0_tcbind(i); 242 /* 243 * To be 100% sure we're really getting the right 244 * information, we exit the configuration state 245 * and do an IHB after each rebinding. 246 */ 247 write_c0_mvpcontrol( 248 read_c0_mvpcontrol() & ~ MVPCONTROL_VPC ); 249 mips_ihb(); 250 /* 251 * Only count if the MMU Type indicated is TLB 252 */ 253 if (((read_vpe_c0_config() & MIPS_CONF_MT) >> 7) == 1) { 254 config1val = read_vpe_c0_config1(); 255 tlbsiz += ((config1val >> 25) & 0x3f) + 1; 256 } 257 258 /* Put core back in configuration state */ 259 write_c0_mvpcontrol( 260 read_c0_mvpcontrol() | MVPCONTROL_VPC ); 261 mips_ihb(); 262 } 263 } 264 write_c0_mvpcontrol(read_c0_mvpcontrol() | MVPCONTROL_STLB); 265 ehb(); 266 267 /* 268 * Setup kernel data structures to use software total, 269 * rather than read the per-VPE Config1 value. The values 270 * for "CPU 0" gets copied to all the other CPUs as part 271 * of their initialization in smtc_cpu_setup(). 272 */ 273 274 /* MIPS32 limits TLB indices to 64 */ 275 if (tlbsiz > 64) 276 tlbsiz = 64; 277 cpu_data[0].tlbsize = current_cpu_data.tlbsize = tlbsiz; 278 smtc_status |= SMTC_TLB_SHARED; 279 local_flush_tlb_all(); 280 281 printk("TLB of %d entry pairs shared by %d VPEs\n", 282 tlbsiz, vpes); 283 } else { 284 printk("WARNING: TLB Not Sharable on SMTC Boot!\n"); 285 } 286 } 287} 288 289 290/* 291 * Incrementally build the CPU map out of constituent MIPS MT cores, 292 * using the specified available VPEs and TCs. Plaform code needs 293 * to ensure that each MIPS MT core invokes this routine on reset, 294 * one at a time(!). 295 * 296 * This version of the build_cpu_map and prepare_cpus routines assumes 297 * that *all* TCs of a MIPS MT core will be used for Linux, and that 298 * they will be spread across *all* available VPEs (to minimise the 299 * loss of efficiency due to exception service serialization). 300 * An improved version would pick up configuration information and 301 * possibly leave some TCs/VPEs as "slave" processors. 302 * 303 * Use c0_MVPConf0 to find out how many TCs are available, setting up 304 * phys_cpu_present_map and the logical/physical mappings. 305 */ 306 307int __init mipsmt_build_cpu_map(int start_cpu_slot) 308{ 309 int i, ntcs; 310 311 /* 312 * The CPU map isn't actually used for anything at this point, 313 * so it's not clear what else we should do apart from set 314 * everything up so that "logical" = "physical". 315 */ 316 ntcs = ((read_c0_mvpconf0() & MVPCONF0_PTC) >> MVPCONF0_PTC_SHIFT) + 1; 317 for (i=start_cpu_slot; i<NR_CPUS && i<ntcs; i++) { 318 cpu_set(i, phys_cpu_present_map); 319 __cpu_number_map[i] = i; 320 __cpu_logical_map[i] = i; 321 } 322 /* Initialize map of CPUs with FPUs */ 323 cpus_clear(mt_fpu_cpumask); 324 325 /* One of those TC's is the one booting, and not a secondary... */ 326 printk("%i available secondary CPU TC(s)\n", i - 1); 327 328 return i; 329} 330 331/* 332 * Common setup before any secondaries are started 333 * Make sure all CPU's are in a sensible state before we boot any of the 334 * secondaries. 335 * 336 * For MIPS MT "SMTC" operation, we set up all TCs, spread as evenly 337 * as possible across the available VPEs. 338 */ 339 340static void smtc_tc_setup(int vpe, int tc, int cpu) 341{ 342 settc(tc); 343 write_tc_c0_tchalt(TCHALT_H); 344 mips_ihb(); 345 write_tc_c0_tcstatus((read_tc_c0_tcstatus() 346 & ~(TCSTATUS_TKSU | TCSTATUS_DA | TCSTATUS_IXMT)) 347 | TCSTATUS_A); 348 write_tc_c0_tccontext(0); 349 /* Bind tc to vpe */ 350 write_tc_c0_tcbind(vpe); 351 /* In general, all TCs should have the same cpu_data indications */ 352 memcpy(&cpu_data[cpu], &cpu_data[0], sizeof(struct cpuinfo_mips)); 353 /* For 34Kf, start with TC/CPU 0 as sole owner of single FPU context */ 354 if (cpu_data[0].cputype == CPU_34K) 355 cpu_data[cpu].options &= ~MIPS_CPU_FPU; 356 cpu_data[cpu].vpe_id = vpe; 357 cpu_data[cpu].tc_id = tc; 358} 359 360 361void mipsmt_prepare_cpus(void) 362{ 363 int i, vpe, tc, ntc, nvpe, tcpervpe, slop, cpu; 364 unsigned long flags; 365 unsigned long val; 366 int nipi; 367 struct smtc_ipi *pipi; 368 369 /* disable interrupts so we can disable MT */ 370 local_irq_save(flags); 371 /* disable MT so we can configure */ 372 dvpe(); 373 dmt(); 374 375 spin_lock_init(&freeIPIq.lock); 376 377 /* 378 * We probably don't have as many VPEs as we do SMP "CPUs", 379 * but it's possible - and in any case we'll never use more! 380 */ 381 for (i=0; i<NR_CPUS; i++) { 382 IPIQ[i].head = IPIQ[i].tail = NULL; 383 spin_lock_init(&IPIQ[i].lock); 384 IPIQ[i].depth = 0; 385 ipi_timer_latch[i] = 0; 386 } 387 388 /* cpu_data index starts at zero */ 389 cpu = 0; 390 cpu_data[cpu].vpe_id = 0; 391 cpu_data[cpu].tc_id = 0; 392 cpu++; 393 394 /* Report on boot-time options */ 395 mips_mt_set_cpuoptions (); 396 if (vpelimit > 0) 397 printk("Limit of %d VPEs set\n", vpelimit); 398 if (tclimit > 0) 399 printk("Limit of %d TCs set\n", tclimit); 400 if (nostlb) { 401 printk("Shared TLB Use Inhibited - UNSAFE for Multi-VPE Operation\n"); 402 } 403 if (asidmask) 404 printk("ASID mask value override to 0x%x\n", asidmask); 405 406 /* Temporary */ 407#ifdef SMTC_IDLE_HOOK_DEBUG 408 if (hang_trig) 409 printk("Logic Analyser Trigger on suspected TC hang\n"); 410#endif /* SMTC_IDLE_HOOK_DEBUG */ 411 412 /* Put MVPE's into 'configuration state' */ 413 write_c0_mvpcontrol( read_c0_mvpcontrol() | MVPCONTROL_VPC ); 414 415 val = read_c0_mvpconf0(); 416 nvpe = ((val & MVPCONF0_PVPE) >> MVPCONF0_PVPE_SHIFT) + 1; 417 if (vpelimit > 0 && nvpe > vpelimit) 418 nvpe = vpelimit; 419 ntc = ((val & MVPCONF0_PTC) >> MVPCONF0_PTC_SHIFT) + 1; 420 if (ntc > NR_CPUS) 421 ntc = NR_CPUS; 422 if (tclimit > 0 && ntc > tclimit) 423 ntc = tclimit; 424 tcpervpe = ntc / nvpe; 425 slop = ntc % nvpe; /* Residual TCs, < NVPE */ 426 427 /* Set up shared TLB */ 428 smtc_configure_tlb(); 429 430 for (tc = 0, vpe = 0 ; (vpe < nvpe) && (tc < ntc) ; vpe++) { 431 /* 432 * Set the MVP bits. 433 */ 434 settc(tc); 435 write_vpe_c0_vpeconf0(read_vpe_c0_vpeconf0() | VPECONF0_MVP); 436 if (vpe != 0) 437 printk(", "); 438 printk("VPE %d: TC", vpe); 439 for (i = 0; i < tcpervpe; i++) { 440 /* 441 * TC 0 is bound to VPE 0 at reset, 442 * and is presumably executing this 443 * code. Leave it alone! 444 */ 445 if (tc != 0) { 446 smtc_tc_setup(vpe,tc, cpu); 447 cpu++; 448 } 449 printk(" %d", tc); 450 tc++; 451 } 452 if (slop) { 453 if (tc != 0) { 454 smtc_tc_setup(vpe,tc, cpu); 455 cpu++; 456 } 457 printk(" %d", tc); 458 tc++; 459 slop--; 460 } 461 if (vpe != 0) { 462 /* 463 * Clear any stale software interrupts from VPE's Cause 464 */ 465 write_vpe_c0_cause(0); 466 467 /* 468 * Clear ERL/EXL of VPEs other than 0 469 * and set restricted interrupt enable/mask. 470 */ 471 write_vpe_c0_status((read_vpe_c0_status() 472 & ~(ST0_BEV | ST0_ERL | ST0_EXL | ST0_IM)) 473 | (STATUSF_IP0 | STATUSF_IP1 | STATUSF_IP7 474 | ST0_IE)); 475 /* 476 * set config to be the same as vpe0, 477 * particularly kseg0 coherency alg 478 */ 479 write_vpe_c0_config(read_c0_config()); 480 /* Clear any pending timer interrupt */ 481 write_vpe_c0_compare(0); 482 /* Propagate Config7 */ 483 write_vpe_c0_config7(read_c0_config7()); 484 write_vpe_c0_count(read_c0_count()); 485 } 486 /* enable multi-threading within VPE */ 487 write_vpe_c0_vpecontrol(read_vpe_c0_vpecontrol() | VPECONTROL_TE); 488 /* enable the VPE */ 489 write_vpe_c0_vpeconf0(read_vpe_c0_vpeconf0() | VPECONF0_VPA); 490 } 491 492 /* 493 * Pull any physically present but unused TCs out of circulation. 494 */ 495 while (tc < (((val & MVPCONF0_PTC) >> MVPCONF0_PTC_SHIFT) + 1)) { 496 cpu_clear(tc, phys_cpu_present_map); 497 cpu_clear(tc, cpu_present_map); 498 tc++; 499 } 500 501 /* release config state */ 502 write_c0_mvpcontrol( read_c0_mvpcontrol() & ~ MVPCONTROL_VPC ); 503 504 printk("\n"); 505 506 /* Set up coprocessor affinity CPU mask(s) */ 507 508 for (tc = 0; tc < ntc; tc++) { 509 if (cpu_data[tc].options & MIPS_CPU_FPU) 510 cpu_set(tc, mt_fpu_cpumask); 511 } 512 513 /* set up ipi interrupts... */ 514 515 /* If we have multiple VPEs running, set up the cross-VPE interrupt */ 516 517 if (nvpe > 1) 518 setup_cross_vpe_interrupts(); 519 520 /* Set up queue of free IPI "messages". */ 521 nipi = NR_CPUS * IPIBUF_PER_CPU; 522 if (ipibuffers > 0) 523 nipi = ipibuffers; 524 525 pipi = kmalloc(nipi *sizeof(struct smtc_ipi), GFP_KERNEL); 526 if (pipi == NULL) 527 panic("kmalloc of IPI message buffers failed\n"); 528 else 529 printk("IPI buffer pool of %d buffers\n", nipi); 530 for (i = 0; i < nipi; i++) { 531 smtc_ipi_nq(&freeIPIq, pipi); 532 pipi++; 533 } 534 535 /* Arm multithreading and enable other VPEs - but all TCs are Halted */ 536 emt(EMT_ENABLE); 537 evpe(EVPE_ENABLE); 538 local_irq_restore(flags); 539 /* Initialize SMTC /proc statistics/diagnostics */ 540 init_smtc_stats(); 541} 542 543 544/* 545 * Setup the PC, SP, and GP of a secondary processor and start it 546 * running! 547 * smp_bootstrap is the place to resume from 548 * __KSTK_TOS(idle) is apparently the stack pointer 549 * (unsigned long)idle->thread_info the gp 550 * 551 */ 552void smtc_boot_secondary(int cpu, struct task_struct *idle) 553{ 554 extern u32 kernelsp[NR_CPUS]; 555 long flags; 556 int mtflags; 557 558 LOCK_MT_PRA(); 559 if (cpu_data[cpu].vpe_id != cpu_data[smp_processor_id()].vpe_id) { 560 dvpe(); 561 } 562 settc(cpu_data[cpu].tc_id); 563 564 /* pc */ 565 write_tc_c0_tcrestart((unsigned long)&smp_bootstrap); 566 567 /* stack pointer */ 568 kernelsp[cpu] = __KSTK_TOS(idle); 569 write_tc_gpr_sp(__KSTK_TOS(idle)); 570 571 /* global pointer */ 572 write_tc_gpr_gp((unsigned long)idle->thread_info); 573 574 smtc_status |= SMTC_MTC_ACTIVE; 575 write_tc_c0_tchalt(0); 576 if (cpu_data[cpu].vpe_id != cpu_data[smp_processor_id()].vpe_id) { 577 evpe(EVPE_ENABLE); 578 } 579 UNLOCK_MT_PRA(); 580} 581 582void smtc_init_secondary(void) 583{ 584 /* 585 * Start timer on secondary VPEs if necessary. 586 * plat_timer_setup has already have been invoked by init/main 587 * on "boot" TC. Like per_cpu_trap_init() hack, this assumes that 588 * SMTC init code assigns TCs consdecutively and in ascending order 589 * to across available VPEs. 590 */ 591 if (((read_c0_tcbind() & TCBIND_CURTC) != 0) && 592 ((read_c0_tcbind() & TCBIND_CURVPE) 593 != cpu_data[smp_processor_id() - 1].vpe_id)){ 594 write_c0_compare (read_c0_count() + mips_hpt_frequency/HZ); 595 } 596 597 local_irq_enable(); 598} 599 600void smtc_smp_finish(void) 601{ 602 printk("TC %d going on-line as CPU %d\n", 603 cpu_data[smp_processor_id()].tc_id, smp_processor_id()); 604} 605 606void smtc_cpus_done(void) 607{ 608} 609 610/* 611 * Support for SMTC-optimized driver IRQ registration 612 */ 613 614/* 615 * SMTC Kernel needs to manipulate low-level CPU interrupt mask 616 * in do_IRQ. These are passed in setup_irq_smtc() and stored 617 * in this table. 618 */ 619 620int setup_irq_smtc(unsigned int irq, struct irqaction * new, 621 unsigned long hwmask) 622{ 623 irq_hwmask[irq] = hwmask; 624 625 return setup_irq(irq, new); 626} 627 628/* 629 * IPI model for SMTC is tricky, because interrupts aren't TC-specific. 630 * Within a VPE one TC can interrupt another by different approaches. 631 * The easiest to get right would probably be to make all TCs except 632 * the target IXMT and set a software interrupt, but an IXMT-based 633 * scheme requires that a handler must run before a new IPI could 634 * be sent, which would break the "broadcast" loops in MIPS MT. 635 * A more gonzo approach within a VPE is to halt the TC, extract 636 * its Restart, Status, and a couple of GPRs, and program the Restart 637 * address to emulate an interrupt. 638 * 639 * Within a VPE, one can be confident that the target TC isn't in 640 * a critical EXL state when halted, since the write to the Halt 641 * register could not have issued on the writing thread if the 642 * halting thread had EXL set. So k0 and k1 of the target TC 643 * can be used by the injection code. Across VPEs, one can't 644 * be certain that the target TC isn't in a critical exception 645 * state. So we try a two-step process of sending a software 646 * interrupt to the target VPE, which either handles the event 647 * itself (if it was the target) or injects the event within 648 * the VPE. 649 */ 650 651void smtc_ipi_qdump(void) 652{ 653 int i; 654 655 for (i = 0; i < NR_CPUS ;i++) { 656 printk("IPIQ[%d]: head = 0x%x, tail = 0x%x, depth = %d\n", 657 i, (unsigned)IPIQ[i].head, (unsigned)IPIQ[i].tail, 658 IPIQ[i].depth); 659 } 660} 661 662/* 663 * The standard atomic.h primitives don't quite do what we want 664 * here: We need an atomic add-and-return-previous-value (which 665 * could be done with atomic_add_return and a decrement) and an 666 * atomic set/zero-and-return-previous-value (which can't really 667 * be done with the atomic.h primitives). And since this is 668 * MIPS MT, we can assume that we have LL/SC. 669 */ 670static __inline__ int atomic_postincrement(unsigned int *pv) 671{ 672 unsigned long result; 673 674 unsigned long temp; 675 676 __asm__ __volatile__( 677 "1: ll %0, %2 \n" 678 " addu %1, %0, 1 \n" 679 " sc %1, %2 \n" 680 " beqz %1, 1b \n" 681 " sync \n" 682 : "=&r" (result), "=&r" (temp), "=m" (*pv) 683 : "m" (*pv) 684 : "memory"); 685 686 return result; 687} 688 689/* No longer used in IPI dispatch, but retained for future recycling */ 690 691static __inline__ int atomic_postclear(unsigned int *pv) 692{ 693 unsigned long result; 694 695 unsigned long temp; 696 697 __asm__ __volatile__( 698 "1: ll %0, %2 \n" 699 " or %1, $0, $0 \n" 700 " sc %1, %2 \n" 701 " beqz %1, 1b \n" 702 " sync \n" 703 : "=&r" (result), "=&r" (temp), "=m" (*pv) 704 : "m" (*pv) 705 : "memory"); 706 707 return result; 708} 709 710 711void smtc_send_ipi(int cpu, int type, unsigned int action) 712{ 713 int tcstatus; 714 struct smtc_ipi *pipi; 715 long flags; 716 int mtflags; 717 718 if (cpu == smp_processor_id()) { 719 printk("Cannot Send IPI to self!\n"); 720 return; 721 } 722 /* Set up a descriptor, to be delivered either promptly or queued */ 723 pipi = smtc_ipi_dq(&freeIPIq); 724 if (pipi == NULL) { 725 bust_spinlocks(1); 726 mips_mt_regdump(dvpe()); 727 panic("IPI Msg. Buffers Depleted\n"); 728 } 729 pipi->type = type; 730 pipi->arg = (void *)action; 731 pipi->dest = cpu; 732 if (cpu_data[cpu].vpe_id != cpu_data[smp_processor_id()].vpe_id) { 733 /* If not on same VPE, enqueue and send cross-VPE interupt */ 734 smtc_ipi_nq(&IPIQ[cpu], pipi); 735 LOCK_CORE_PRA(); 736 settc(cpu_data[cpu].tc_id); 737 write_vpe_c0_cause(read_vpe_c0_cause() | C_SW1); 738 UNLOCK_CORE_PRA(); 739 } else { 740 /* 741 * Not sufficient to do a LOCK_MT_PRA (dmt) here, 742 * since ASID shootdown on the other VPE may 743 * collide with this operation. 744 */ 745 LOCK_CORE_PRA(); 746 settc(cpu_data[cpu].tc_id); 747 /* Halt the targeted TC */ 748 write_tc_c0_tchalt(TCHALT_H); 749 mips_ihb(); 750 751 /* 752 * Inspect TCStatus - if IXMT is set, we have to queue 753 * a message. Otherwise, we set up the "interrupt" 754 * of the other TC 755 */ 756 tcstatus = read_tc_c0_tcstatus(); 757 758 if ((tcstatus & TCSTATUS_IXMT) != 0) { 759 /* 760 * Spin-waiting here can deadlock, 761 * so we queue the message for the target TC. 762 */ 763 write_tc_c0_tchalt(0); 764 UNLOCK_CORE_PRA(); 765 /* Try to reduce redundant timer interrupt messages */ 766 if (type == SMTC_CLOCK_TICK) { 767 if (atomic_postincrement(&ipi_timer_latch[cpu])!=0){ 768 smtc_ipi_nq(&freeIPIq, pipi); 769 return; 770 } 771 } 772 smtc_ipi_nq(&IPIQ[cpu], pipi); 773 } else { 774 post_direct_ipi(cpu, pipi); 775 write_tc_c0_tchalt(0); 776 UNLOCK_CORE_PRA(); 777 } 778 } 779} 780 781/* 782 * Send IPI message to Halted TC, TargTC/TargVPE already having been set 783 */ 784void post_direct_ipi(int cpu, struct smtc_ipi *pipi) 785{ 786 struct pt_regs *kstack; 787 unsigned long tcstatus; 788 unsigned long tcrestart; 789 extern u32 kernelsp[NR_CPUS]; 790 extern void __smtc_ipi_vector(void); 791 792 /* Extract Status, EPC from halted TC */ 793 tcstatus = read_tc_c0_tcstatus(); 794 tcrestart = read_tc_c0_tcrestart(); 795 /* If TCRestart indicates a WAIT instruction, advance the PC */ 796 if ((tcrestart & 0x80000000) 797 && ((*(unsigned int *)tcrestart & 0xfe00003f) == 0x42000020)) { 798 tcrestart += 4; 799 } 800 /* 801 * Save on TC's future kernel stack 802 * 803 * CU bit of Status is indicator that TC was 804 * already running on a kernel stack... 805 */ 806 if (tcstatus & ST0_CU0) { 807 /* Note that this "- 1" is pointer arithmetic */ 808 kstack = ((struct pt_regs *)read_tc_gpr_sp()) - 1; 809 } else { 810 kstack = ((struct pt_regs *)kernelsp[cpu]) - 1; 811 } 812 813 kstack->cp0_epc = (long)tcrestart; 814 /* Save TCStatus */ 815 kstack->cp0_tcstatus = tcstatus; 816 /* Pass token of operation to be performed kernel stack pad area */ 817 kstack->pad0[4] = (unsigned long)pipi; 818 /* Pass address of function to be called likewise */ 819 kstack->pad0[5] = (unsigned long)&ipi_decode; 820 /* Set interrupt exempt and kernel mode */ 821 tcstatus |= TCSTATUS_IXMT; 822 tcstatus &= ~TCSTATUS_TKSU; 823 write_tc_c0_tcstatus(tcstatus); 824 ehb(); 825 /* Set TC Restart address to be SMTC IPI vector */ 826 write_tc_c0_tcrestart(__smtc_ipi_vector); 827} 828 829static void ipi_resched_interrupt(void) 830{ 831 /* Return from interrupt should be enough to cause scheduler check */ 832} 833 834 835static void ipi_call_interrupt(void) 836{ 837 /* Invoke generic function invocation code in smp.c */ 838 smp_call_function_interrupt(); 839} 840 841void ipi_decode(struct smtc_ipi *pipi) 842{ 843 void *arg_copy = pipi->arg; 844 int type_copy = pipi->type; 845 int dest_copy = pipi->dest; 846 847 smtc_ipi_nq(&freeIPIq, pipi); 848 switch (type_copy) { 849 case SMTC_CLOCK_TICK: 850 /* Invoke Clock "Interrupt" */ 851 ipi_timer_latch[dest_copy] = 0; 852#ifdef SMTC_IDLE_HOOK_DEBUG 853 clock_hang_reported[dest_copy] = 0; 854#endif /* SMTC_IDLE_HOOK_DEBUG */ 855 local_timer_interrupt(0, NULL); 856 break; 857 case LINUX_SMP_IPI: 858 switch ((int)arg_copy) { 859 case SMP_RESCHEDULE_YOURSELF: 860 ipi_resched_interrupt(); 861 break; 862 case SMP_CALL_FUNCTION: 863 ipi_call_interrupt(); 864 break; 865 default: 866 printk("Impossible SMTC IPI Argument 0x%x\n", 867 (int)arg_copy); 868 break; 869 } 870 break; 871 default: 872 printk("Impossible SMTC IPI Type 0x%x\n", type_copy); 873 break; 874 } 875} 876 877void deferred_smtc_ipi(void) 878{ 879 struct smtc_ipi *pipi; 880 unsigned long flags; 881/* DEBUG */ 882 int q = smp_processor_id(); 883 884 /* 885 * Test is not atomic, but much faster than a dequeue, 886 * and the vast majority of invocations will have a null queue. 887 */ 888 if (IPIQ[q].head != NULL) { 889 while((pipi = smtc_ipi_dq(&IPIQ[q])) != NULL) { 890 /* ipi_decode() should be called with interrupts off */ 891 local_irq_save(flags); 892 ipi_decode(pipi); 893 local_irq_restore(flags); 894 } 895 } 896} 897 898/* 899 * Send clock tick to all TCs except the one executing the funtion 900 */ 901 902void smtc_timer_broadcast(int vpe) 903{ 904 int cpu; 905 int myTC = cpu_data[smp_processor_id()].tc_id; 906 int myVPE = cpu_data[smp_processor_id()].vpe_id; 907 908 smtc_cpu_stats[smp_processor_id()].timerints++; 909 910 for_each_online_cpu(cpu) { 911 if (cpu_data[cpu].vpe_id == myVPE && 912 cpu_data[cpu].tc_id != myTC) 913 smtc_send_ipi(cpu, SMTC_CLOCK_TICK, 0); 914 } 915} 916 917/* 918 * Cross-VPE interrupts in the SMTC prototype use "software interrupts" 919 * set via cross-VPE MTTR manipulation of the Cause register. It would be 920 * in some regards preferable to have external logic for "doorbell" hardware 921 * interrupts. 922 */ 923 924static int cpu_ipi_irq = MIPSCPU_INT_BASE + MIPS_CPU_IPI_IRQ; 925 926static irqreturn_t ipi_interrupt(int irq, void *dev_idm) 927{ 928 int my_vpe = cpu_data[smp_processor_id()].vpe_id; 929 int my_tc = cpu_data[smp_processor_id()].tc_id; 930 int cpu; 931 struct smtc_ipi *pipi; 932 unsigned long tcstatus; 933 int sent; 934 long flags; 935 unsigned int mtflags; 936 unsigned int vpflags; 937 938 /* 939 * So long as cross-VPE interrupts are done via 940 * MFTR/MTTR read-modify-writes of Cause, we need 941 * to stop other VPEs whenever the local VPE does 942 * anything similar. 943 */ 944 local_irq_save(flags); 945 vpflags = dvpe(); 946 clear_c0_cause(0x100 << MIPS_CPU_IPI_IRQ); 947 set_c0_status(0x100 << MIPS_CPU_IPI_IRQ); 948 irq_enable_hazard(); 949 evpe(vpflags); 950 local_irq_restore(flags); 951 952 /* 953 * Cross-VPE Interrupt handler: Try to directly deliver IPIs 954 * queued for TCs on this VPE other than the current one. 955 * Return-from-interrupt should cause us to drain the queue 956 * for the current TC, so we ought not to have to do it explicitly here. 957 */ 958 959 for_each_online_cpu(cpu) { 960 if (cpu_data[cpu].vpe_id != my_vpe) 961 continue; 962 963 pipi = smtc_ipi_dq(&IPIQ[cpu]); 964 if (pipi != NULL) { 965 if (cpu_data[cpu].tc_id != my_tc) { 966 sent = 0; 967 LOCK_MT_PRA(); 968 settc(cpu_data[cpu].tc_id); 969 write_tc_c0_tchalt(TCHALT_H); 970 mips_ihb(); 971 tcstatus = read_tc_c0_tcstatus(); 972 if ((tcstatus & TCSTATUS_IXMT) == 0) { 973 post_direct_ipi(cpu, pipi); 974 sent = 1; 975 } 976 write_tc_c0_tchalt(0); 977 UNLOCK_MT_PRA(); 978 if (!sent) { 979 smtc_ipi_req(&IPIQ[cpu], pipi); 980 } 981 } else { 982 /* 983 * ipi_decode() should be called 984 * with interrupts off 985 */ 986 local_irq_save(flags); 987 ipi_decode(pipi); 988 local_irq_restore(flags); 989 } 990 } 991 } 992 993 return IRQ_HANDLED; 994} 995 996static void ipi_irq_dispatch(void) 997{ 998 do_IRQ(cpu_ipi_irq); 999} 1000 1001static struct irqaction irq_ipi; 1002 1003void setup_cross_vpe_interrupts(void) 1004{ 1005 if (!cpu_has_vint) 1006 panic("SMTC Kernel requires Vectored Interupt support"); 1007 1008 set_vi_handler(MIPS_CPU_IPI_IRQ, ipi_irq_dispatch); 1009 1010 irq_ipi.handler = ipi_interrupt; 1011 irq_ipi.flags = IRQF_DISABLED; 1012 irq_ipi.name = "SMTC_IPI"; 1013 1014 setup_irq_smtc(cpu_ipi_irq, &irq_ipi, (0x100 << MIPS_CPU_IPI_IRQ)); 1015 1016 irq_desc[cpu_ipi_irq].status |= IRQ_PER_CPU; 1017 set_irq_handler(cpu_ipi_irq, handle_percpu_irq); 1018} 1019 1020/* 1021 * SMTC-specific hacks invoked from elsewhere in the kernel. 1022 */ 1023 1024void smtc_ipi_replay(void) 1025{ 1026 /* 1027 * To the extent that we've ever turned interrupts off, 1028 * we may have accumulated deferred IPIs. This is subtle. 1029 * If we use the smtc_ipi_qdepth() macro, we'll get an 1030 * exact number - but we'll also disable interrupts 1031 * and create a window of failure where a new IPI gets 1032 * queued after we test the depth but before we re-enable 1033 * interrupts. So long as IXMT never gets set, however, 1034 * we should be OK: If we pick up something and dispatch 1035 * it here, that's great. If we see nothing, but concurrent 1036 * with this operation, another TC sends us an IPI, IXMT 1037 * is clear, and we'll handle it as a real pseudo-interrupt 1038 * and not a pseudo-pseudo interrupt. 1039 */ 1040 if (IPIQ[smp_processor_id()].depth > 0) { 1041 struct smtc_ipi *pipi; 1042 extern void self_ipi(struct smtc_ipi *); 1043 1044 while ((pipi = smtc_ipi_dq(&IPIQ[smp_processor_id()]))) { 1045 self_ipi(pipi); 1046 smtc_cpu_stats[smp_processor_id()].selfipis++; 1047 } 1048 } 1049} 1050 1051EXPORT_SYMBOL(smtc_ipi_replay); 1052 1053void smtc_idle_loop_hook(void) 1054{ 1055#ifdef SMTC_IDLE_HOOK_DEBUG 1056 int im; 1057 int flags; 1058 int mtflags; 1059 int bit; 1060 int vpe; 1061 int tc; 1062 int hook_ntcs; 1063 /* 1064 * printk within DMT-protected regions can deadlock, 1065 * so buffer diagnostic messages for later output. 1066 */ 1067 char *pdb_msg; 1068 char id_ho_db_msg[768]; /* worst-case use should be less than 700 */ 1069 1070 if (atomic_read(&idle_hook_initialized) == 0) { /* fast test */ 1071 if (atomic_add_return(1, &idle_hook_initialized) == 1) { 1072 int mvpconf0; 1073 /* Tedious stuff to just do once */ 1074 mvpconf0 = read_c0_mvpconf0(); 1075 hook_ntcs = ((mvpconf0 & MVPCONF0_PTC) >> MVPCONF0_PTC_SHIFT) + 1; 1076 if (hook_ntcs > NR_CPUS) 1077 hook_ntcs = NR_CPUS; 1078 for (tc = 0; tc < hook_ntcs; tc++) { 1079 tcnoprog[tc] = 0; 1080 clock_hang_reported[tc] = 0; 1081 } 1082 for (vpe = 0; vpe < 2; vpe++) 1083 for (im = 0; im < 8; im++) 1084 imstuckcount[vpe][im] = 0; 1085 printk("Idle loop test hook initialized for %d TCs\n", hook_ntcs); 1086 atomic_set(&idle_hook_initialized, 1000); 1087 } else { 1088 /* Someone else is initializing in parallel - let 'em finish */ 1089 while (atomic_read(&idle_hook_initialized) < 1000) 1090 ; 1091 } 1092 } 1093 1094 /* Have we stupidly left IXMT set somewhere? */ 1095 if (read_c0_tcstatus() & 0x400) { 1096 write_c0_tcstatus(read_c0_tcstatus() & ~0x400); 1097 ehb(); 1098 printk("Dangling IXMT in cpu_idle()\n"); 1099 } 1100 1101 /* Have we stupidly left an IM bit turned off? */ 1102#define IM_LIMIT 2000 1103 local_irq_save(flags); 1104 mtflags = dmt(); 1105 pdb_msg = &id_ho_db_msg[0]; 1106 im = read_c0_status(); 1107 vpe = cpu_data[smp_processor_id()].vpe_id; 1108 for (bit = 0; bit < 8; bit++) { 1109 /* 1110 * In current prototype, I/O interrupts 1111 * are masked for VPE > 0 1112 */ 1113 if (vpemask[vpe][bit]) { 1114 if (!(im & (0x100 << bit))) 1115 imstuckcount[vpe][bit]++; 1116 else 1117 imstuckcount[vpe][bit] = 0; 1118 if (imstuckcount[vpe][bit] > IM_LIMIT) { 1119 set_c0_status(0x100 << bit); 1120 ehb(); 1121 imstuckcount[vpe][bit] = 0; 1122 pdb_msg += sprintf(pdb_msg, 1123 "Dangling IM %d fixed for VPE %d\n", bit, 1124 vpe); 1125 } 1126 } 1127 } 1128 1129 /* 1130 * Now that we limit outstanding timer IPIs, check for hung TC 1131 */ 1132 for (tc = 0; tc < NR_CPUS; tc++) { 1133 /* Don't check ourself - we'll dequeue IPIs just below */ 1134 if ((tc != smp_processor_id()) && 1135 ipi_timer_latch[tc] > timerq_limit) { 1136 if (clock_hang_reported[tc] == 0) { 1137 pdb_msg += sprintf(pdb_msg, 1138 "TC %d looks hung with timer latch at %d\n", 1139 tc, ipi_timer_latch[tc]); 1140 clock_hang_reported[tc]++; 1141 } 1142 } 1143 } 1144 emt(mtflags); 1145 local_irq_restore(flags); 1146 if (pdb_msg != &id_ho_db_msg[0]) 1147 printk("CPU%d: %s", smp_processor_id(), id_ho_db_msg); 1148#endif /* SMTC_IDLE_HOOK_DEBUG */ 1149 1150 /* 1151 * Replay any accumulated deferred IPIs. If "Instant Replay" 1152 * is in use, there should never be any. 1153 */ 1154#ifndef CONFIG_MIPS_MT_SMTC_INSTANT_REPLAY 1155 smtc_ipi_replay(); 1156#endif /* CONFIG_MIPS_MT_SMTC_INSTANT_REPLAY */ 1157} 1158 1159void smtc_soft_dump(void) 1160{ 1161 int i; 1162 1163 printk("Counter Interrupts taken per CPU (TC)\n"); 1164 for (i=0; i < NR_CPUS; i++) { 1165 printk("%d: %ld\n", i, smtc_cpu_stats[i].timerints); 1166 } 1167 printk("Self-IPI invocations:\n"); 1168 for (i=0; i < NR_CPUS; i++) { 1169 printk("%d: %ld\n", i, smtc_cpu_stats[i].selfipis); 1170 } 1171 smtc_ipi_qdump(); 1172 printk("Timer IPI Backlogs:\n"); 1173 for (i=0; i < NR_CPUS; i++) { 1174 printk("%d: %d\n", i, ipi_timer_latch[i]); 1175 } 1176 printk("%d Recoveries of \"stolen\" FPU\n", 1177 atomic_read(&smtc_fpu_recoveries)); 1178} 1179 1180 1181/* 1182 * TLB management routines special to SMTC 1183 */ 1184 1185void smtc_get_new_mmu_context(struct mm_struct *mm, unsigned long cpu) 1186{ 1187 unsigned long flags, mtflags, tcstat, prevhalt, asid; 1188 int tlb, i; 1189 1190 /* 1191 * It would be nice to be able to use a spinlock here, 1192 * but this is invoked from within TLB flush routines 1193 * that protect themselves with DVPE, so if a lock is 1194 * held by another TC, it'll never be freed. 1195 * 1196 * DVPE/DMT must not be done with interrupts enabled, 1197 * so even so most callers will already have disabled 1198 * them, let's be really careful... 1199 */ 1200 1201 local_irq_save(flags); 1202 if (smtc_status & SMTC_TLB_SHARED) { 1203 mtflags = dvpe(); 1204 tlb = 0; 1205 } else { 1206 mtflags = dmt(); 1207 tlb = cpu_data[cpu].vpe_id; 1208 } 1209 asid = asid_cache(cpu); 1210 1211 do { 1212 if (!((asid += ASID_INC) & ASID_MASK) ) { 1213 if (cpu_has_vtag_icache) 1214 flush_icache_all(); 1215 /* Traverse all online CPUs (hack requires contigous range) */ 1216 for (i = 0; i < num_online_cpus(); i++) { 1217 /* 1218 * We don't need to worry about our own CPU, nor those of 1219 * CPUs who don't share our TLB. 1220 */ 1221 if ((i != smp_processor_id()) && 1222 ((smtc_status & SMTC_TLB_SHARED) || 1223 (cpu_data[i].vpe_id == cpu_data[cpu].vpe_id))) { 1224 settc(cpu_data[i].tc_id); 1225 prevhalt = read_tc_c0_tchalt() & TCHALT_H; 1226 if (!prevhalt) { 1227 write_tc_c0_tchalt(TCHALT_H); 1228 mips_ihb(); 1229 } 1230 tcstat = read_tc_c0_tcstatus(); 1231 smtc_live_asid[tlb][(tcstat & ASID_MASK)] |= (asiduse)(0x1 << i); 1232 if (!prevhalt) 1233 write_tc_c0_tchalt(0); 1234 } 1235 } 1236 if (!asid) /* fix version if needed */ 1237 asid = ASID_FIRST_VERSION; 1238 local_flush_tlb_all(); /* start new asid cycle */ 1239 } 1240 } while (smtc_live_asid[tlb][(asid & ASID_MASK)]); 1241 1242 /* 1243 * SMTC shares the TLB within VPEs and possibly across all VPEs. 1244 */ 1245 for (i = 0; i < num_online_cpus(); i++) { 1246 if ((smtc_status & SMTC_TLB_SHARED) || 1247 (cpu_data[i].vpe_id == cpu_data[cpu].vpe_id)) 1248 cpu_context(i, mm) = asid_cache(i) = asid; 1249 } 1250 1251 if (smtc_status & SMTC_TLB_SHARED) 1252 evpe(mtflags); 1253 else 1254 emt(mtflags); 1255 local_irq_restore(flags); 1256} 1257 1258/* 1259 * Invoked from macros defined in mmu_context.h 1260 * which must already have disabled interrupts 1261 * and done a DVPE or DMT as appropriate. 1262 */ 1263 1264void smtc_flush_tlb_asid(unsigned long asid) 1265{ 1266 int entry; 1267 unsigned long ehi; 1268 1269 entry = read_c0_wired(); 1270 1271 /* Traverse all non-wired entries */ 1272 while (entry < current_cpu_data.tlbsize) { 1273 write_c0_index(entry); 1274 ehb(); 1275 tlb_read(); 1276 ehb(); 1277 ehi = read_c0_entryhi(); 1278 if ((ehi & ASID_MASK) == asid) { 1279 /* 1280 * Invalidate only entries with specified ASID, 1281 * makiing sure all entries differ. 1282 */ 1283 write_c0_entryhi(CKSEG0 + (entry << (PAGE_SHIFT + 1))); 1284 write_c0_entrylo0(0); 1285 write_c0_entrylo1(0); 1286 mtc0_tlbw_hazard(); 1287 tlb_write_indexed(); 1288 } 1289 entry++; 1290 } 1291 write_c0_index(PARKED_INDEX); 1292 tlbw_use_hazard(); 1293} 1294 1295/* 1296 * Support for single-threading cache flush operations. 1297 */ 1298 1299int halt_state_save[NR_CPUS]; 1300 1301/* 1302 * To really, really be sure that nothing is being done 1303 * by other TCs, halt them all. This code assumes that 1304 * a DVPE has already been done, so while their Halted 1305 * state is theoretically architecturally unstable, in 1306 * practice, it's not going to change while we're looking 1307 * at it. 1308 */ 1309 1310void smtc_cflush_lockdown(void) 1311{ 1312 int cpu; 1313 1314 for_each_online_cpu(cpu) { 1315 if (cpu != smp_processor_id()) { 1316 settc(cpu_data[cpu].tc_id); 1317 halt_state_save[cpu] = read_tc_c0_tchalt(); 1318 write_tc_c0_tchalt(TCHALT_H); 1319 } 1320 } 1321 mips_ihb(); 1322} 1323 1324/* It would be cheating to change the cpu_online states during a flush! */ 1325 1326void smtc_cflush_release(void) 1327{ 1328 int cpu; 1329 1330 /* 1331 * Start with a hazard barrier to ensure 1332 * that all CACHE ops have played through. 1333 */ 1334 mips_ihb(); 1335 1336 for_each_online_cpu(cpu) { 1337 if (cpu != smp_processor_id()) { 1338 settc(cpu_data[cpu].tc_id); 1339 write_tc_c0_tchalt(halt_state_save[cpu]); 1340 } 1341 } 1342 mips_ihb(); 1343} 1344