smtc.c revision e0daad449c5195fa4552c60392eeee4e5c58d31c
1/* Copyright (C) 2004 Mips Technologies, Inc */ 2 3#include <linux/kernel.h> 4#include <linux/sched.h> 5#include <linux/cpumask.h> 6#include <linux/interrupt.h> 7#include <linux/module.h> 8 9#include <asm/cpu.h> 10#include <asm/processor.h> 11#include <asm/atomic.h> 12#include <asm/system.h> 13#include <asm/hardirq.h> 14#include <asm/hazards.h> 15#include <asm/mmu_context.h> 16#include <asm/smp.h> 17#include <asm/mipsregs.h> 18#include <asm/cacheflush.h> 19#include <asm/time.h> 20#include <asm/addrspace.h> 21#include <asm/smtc.h> 22#include <asm/smtc_ipi.h> 23#include <asm/smtc_proc.h> 24 25/* 26 * This file should be built into the kernel only if CONFIG_MIPS_MT_SMTC is set. 27 */ 28 29#define MIPS_CPU_IPI_IRQ 1 30 31#define LOCK_MT_PRA() \ 32 local_irq_save(flags); \ 33 mtflags = dmt() 34 35#define UNLOCK_MT_PRA() \ 36 emt(mtflags); \ 37 local_irq_restore(flags) 38 39#define LOCK_CORE_PRA() \ 40 local_irq_save(flags); \ 41 mtflags = dvpe() 42 43#define UNLOCK_CORE_PRA() \ 44 evpe(mtflags); \ 45 local_irq_restore(flags) 46 47/* 48 * Data structures purely associated with SMTC parallelism 49 */ 50 51 52/* 53 * Table for tracking ASIDs whose lifetime is prolonged. 54 */ 55 56asiduse smtc_live_asid[MAX_SMTC_TLBS][MAX_SMTC_ASIDS]; 57 58/* 59 * Clock interrupt "latch" buffers, per "CPU" 60 */ 61 62unsigned int ipi_timer_latch[NR_CPUS]; 63 64/* 65 * Number of InterProcessor Interupt (IPI) message buffers to allocate 66 */ 67 68#define IPIBUF_PER_CPU 4 69 70struct smtc_ipi_q IPIQ[NR_CPUS]; 71struct smtc_ipi_q freeIPIq; 72 73 74/* Forward declarations */ 75 76void ipi_decode(struct smtc_ipi *); 77void post_direct_ipi(int cpu, struct smtc_ipi *pipi); 78void setup_cross_vpe_interrupts(void); 79void init_smtc_stats(void); 80 81/* Global SMTC Status */ 82 83unsigned int smtc_status = 0; 84 85/* Boot command line configuration overrides */ 86 87static int vpelimit = 0; 88static int tclimit = 0; 89static int ipibuffers = 0; 90static int nostlb = 0; 91static int asidmask = 0; 92unsigned long smtc_asid_mask = 0xff; 93 94static int __init maxvpes(char *str) 95{ 96 get_option(&str, &vpelimit); 97 return 1; 98} 99 100static int __init maxtcs(char *str) 101{ 102 get_option(&str, &tclimit); 103 return 1; 104} 105 106static int __init ipibufs(char *str) 107{ 108 get_option(&str, &ipibuffers); 109 return 1; 110} 111 112static int __init stlb_disable(char *s) 113{ 114 nostlb = 1; 115 return 1; 116} 117 118static int __init asidmask_set(char *str) 119{ 120 get_option(&str, &asidmask); 121 switch (asidmask) { 122 case 0x1: 123 case 0x3: 124 case 0x7: 125 case 0xf: 126 case 0x1f: 127 case 0x3f: 128 case 0x7f: 129 case 0xff: 130 smtc_asid_mask = (unsigned long)asidmask; 131 break; 132 default: 133 printk("ILLEGAL ASID mask 0x%x from command line\n", asidmask); 134 } 135 return 1; 136} 137 138__setup("maxvpes=", maxvpes); 139__setup("maxtcs=", maxtcs); 140__setup("ipibufs=", ipibufs); 141__setup("nostlb", stlb_disable); 142__setup("asidmask=", asidmask_set); 143 144/* Enable additional debug checks before going into CPU idle loop */ 145#define SMTC_IDLE_HOOK_DEBUG 146 147#ifdef SMTC_IDLE_HOOK_DEBUG 148 149static int hang_trig = 0; 150 151static int __init hangtrig_enable(char *s) 152{ 153 hang_trig = 1; 154 return 1; 155} 156 157 158__setup("hangtrig", hangtrig_enable); 159 160#define DEFAULT_BLOCKED_IPI_LIMIT 32 161 162static int timerq_limit = DEFAULT_BLOCKED_IPI_LIMIT; 163 164static int __init tintq(char *str) 165{ 166 get_option(&str, &timerq_limit); 167 return 1; 168} 169 170__setup("tintq=", tintq); 171 172int imstuckcount[2][8]; 173/* vpemask represents IM/IE bits of per-VPE Status registers, low-to-high */ 174int vpemask[2][8] = {{0,1,1,0,0,0,0,1},{0,1,0,0,0,0,0,1}}; 175int tcnoprog[NR_CPUS]; 176static atomic_t idle_hook_initialized = {0}; 177static int clock_hang_reported[NR_CPUS]; 178 179#endif /* SMTC_IDLE_HOOK_DEBUG */ 180 181/* Initialize shared TLB - the should probably migrate to smtc_setup_cpus() */ 182 183void __init sanitize_tlb_entries(void) 184{ 185 printk("Deprecated sanitize_tlb_entries() invoked\n"); 186} 187 188 189/* 190 * Configure shared TLB - VPC configuration bit must be set by caller 191 */ 192 193void smtc_configure_tlb(void) 194{ 195 int i,tlbsiz,vpes; 196 unsigned long mvpconf0; 197 unsigned long config1val; 198 199 /* Set up ASID preservation table */ 200 for (vpes=0; vpes<MAX_SMTC_TLBS; vpes++) { 201 for(i = 0; i < MAX_SMTC_ASIDS; i++) { 202 smtc_live_asid[vpes][i] = 0; 203 } 204 } 205 mvpconf0 = read_c0_mvpconf0(); 206 207 if ((vpes = ((mvpconf0 & MVPCONF0_PVPE) 208 >> MVPCONF0_PVPE_SHIFT) + 1) > 1) { 209 /* If we have multiple VPEs, try to share the TLB */ 210 if ((mvpconf0 & MVPCONF0_TLBS) && !nostlb) { 211 /* 212 * If TLB sizing is programmable, shared TLB 213 * size is the total available complement. 214 * Otherwise, we have to take the sum of all 215 * static VPE TLB entries. 216 */ 217 if ((tlbsiz = ((mvpconf0 & MVPCONF0_PTLBE) 218 >> MVPCONF0_PTLBE_SHIFT)) == 0) { 219 /* 220 * If there's more than one VPE, there had better 221 * be more than one TC, because we need one to bind 222 * to each VPE in turn to be able to read 223 * its configuration state! 224 */ 225 settc(1); 226 /* Stop the TC from doing anything foolish */ 227 write_tc_c0_tchalt(TCHALT_H); 228 mips_ihb(); 229 /* No need to un-Halt - that happens later anyway */ 230 for (i=0; i < vpes; i++) { 231 write_tc_c0_tcbind(i); 232 /* 233 * To be 100% sure we're really getting the right 234 * information, we exit the configuration state 235 * and do an IHB after each rebinding. 236 */ 237 write_c0_mvpcontrol( 238 read_c0_mvpcontrol() & ~ MVPCONTROL_VPC ); 239 mips_ihb(); 240 /* 241 * Only count if the MMU Type indicated is TLB 242 */ 243 if (((read_vpe_c0_config() & MIPS_CONF_MT) >> 7) == 1) { 244 config1val = read_vpe_c0_config1(); 245 tlbsiz += ((config1val >> 25) & 0x3f) + 1; 246 } 247 248 /* Put core back in configuration state */ 249 write_c0_mvpcontrol( 250 read_c0_mvpcontrol() | MVPCONTROL_VPC ); 251 mips_ihb(); 252 } 253 } 254 write_c0_mvpcontrol(read_c0_mvpcontrol() | MVPCONTROL_STLB); 255 ehb(); 256 257 /* 258 * Setup kernel data structures to use software total, 259 * rather than read the per-VPE Config1 value. The values 260 * for "CPU 0" gets copied to all the other CPUs as part 261 * of their initialization in smtc_cpu_setup(). 262 */ 263 264 /* MIPS32 limits TLB indices to 64 */ 265 if (tlbsiz > 64) 266 tlbsiz = 64; 267 cpu_data[0].tlbsize = current_cpu_data.tlbsize = tlbsiz; 268 smtc_status |= SMTC_TLB_SHARED; 269 local_flush_tlb_all(); 270 271 printk("TLB of %d entry pairs shared by %d VPEs\n", 272 tlbsiz, vpes); 273 } else { 274 printk("WARNING: TLB Not Sharable on SMTC Boot!\n"); 275 } 276 } 277} 278 279 280/* 281 * Incrementally build the CPU map out of constituent MIPS MT cores, 282 * using the specified available VPEs and TCs. Plaform code needs 283 * to ensure that each MIPS MT core invokes this routine on reset, 284 * one at a time(!). 285 * 286 * This version of the build_cpu_map and prepare_cpus routines assumes 287 * that *all* TCs of a MIPS MT core will be used for Linux, and that 288 * they will be spread across *all* available VPEs (to minimise the 289 * loss of efficiency due to exception service serialization). 290 * An improved version would pick up configuration information and 291 * possibly leave some TCs/VPEs as "slave" processors. 292 * 293 * Use c0_MVPConf0 to find out how many TCs are available, setting up 294 * phys_cpu_present_map and the logical/physical mappings. 295 */ 296 297int __init mipsmt_build_cpu_map(int start_cpu_slot) 298{ 299 int i, ntcs; 300 301 /* 302 * The CPU map isn't actually used for anything at this point, 303 * so it's not clear what else we should do apart from set 304 * everything up so that "logical" = "physical". 305 */ 306 ntcs = ((read_c0_mvpconf0() & MVPCONF0_PTC) >> MVPCONF0_PTC_SHIFT) + 1; 307 for (i=start_cpu_slot; i<NR_CPUS && i<ntcs; i++) { 308 cpu_set(i, phys_cpu_present_map); 309 __cpu_number_map[i] = i; 310 __cpu_logical_map[i] = i; 311 } 312 /* Initialize map of CPUs with FPUs */ 313 cpus_clear(mt_fpu_cpumask); 314 315 /* One of those TC's is the one booting, and not a secondary... */ 316 printk("%i available secondary CPU TC(s)\n", i - 1); 317 318 return i; 319} 320 321/* 322 * Common setup before any secondaries are started 323 * Make sure all CPU's are in a sensible state before we boot any of the 324 * secondaries. 325 * 326 * For MIPS MT "SMTC" operation, we set up all TCs, spread as evenly 327 * as possible across the available VPEs. 328 */ 329 330static void smtc_tc_setup(int vpe, int tc, int cpu) 331{ 332 settc(tc); 333 write_tc_c0_tchalt(TCHALT_H); 334 mips_ihb(); 335 write_tc_c0_tcstatus((read_tc_c0_tcstatus() 336 & ~(TCSTATUS_TKSU | TCSTATUS_DA | TCSTATUS_IXMT)) 337 | TCSTATUS_A); 338 write_tc_c0_tccontext(0); 339 /* Bind tc to vpe */ 340 write_tc_c0_tcbind(vpe); 341 /* In general, all TCs should have the same cpu_data indications */ 342 memcpy(&cpu_data[cpu], &cpu_data[0], sizeof(struct cpuinfo_mips)); 343 /* For 34Kf, start with TC/CPU 0 as sole owner of single FPU context */ 344 if (cpu_data[0].cputype == CPU_34K) 345 cpu_data[cpu].options &= ~MIPS_CPU_FPU; 346 cpu_data[cpu].vpe_id = vpe; 347 cpu_data[cpu].tc_id = tc; 348} 349 350 351void mipsmt_prepare_cpus(void) 352{ 353 int i, vpe, tc, ntc, nvpe, tcpervpe, slop, cpu; 354 unsigned long flags; 355 unsigned long val; 356 int nipi; 357 struct smtc_ipi *pipi; 358 359 /* disable interrupts so we can disable MT */ 360 local_irq_save(flags); 361 /* disable MT so we can configure */ 362 dvpe(); 363 dmt(); 364 365 spin_lock_init(&freeIPIq.lock); 366 367 /* 368 * We probably don't have as many VPEs as we do SMP "CPUs", 369 * but it's possible - and in any case we'll never use more! 370 */ 371 for (i=0; i<NR_CPUS; i++) { 372 IPIQ[i].head = IPIQ[i].tail = NULL; 373 spin_lock_init(&IPIQ[i].lock); 374 IPIQ[i].depth = 0; 375 ipi_timer_latch[i] = 0; 376 } 377 378 /* cpu_data index starts at zero */ 379 cpu = 0; 380 cpu_data[cpu].vpe_id = 0; 381 cpu_data[cpu].tc_id = 0; 382 cpu++; 383 384 /* Report on boot-time options */ 385 mips_mt_set_cpuoptions (); 386 if (vpelimit > 0) 387 printk("Limit of %d VPEs set\n", vpelimit); 388 if (tclimit > 0) 389 printk("Limit of %d TCs set\n", tclimit); 390 if (nostlb) { 391 printk("Shared TLB Use Inhibited - UNSAFE for Multi-VPE Operation\n"); 392 } 393 if (asidmask) 394 printk("ASID mask value override to 0x%x\n", asidmask); 395 396 /* Temporary */ 397#ifdef SMTC_IDLE_HOOK_DEBUG 398 if (hang_trig) 399 printk("Logic Analyser Trigger on suspected TC hang\n"); 400#endif /* SMTC_IDLE_HOOK_DEBUG */ 401 402 /* Put MVPE's into 'configuration state' */ 403 write_c0_mvpcontrol( read_c0_mvpcontrol() | MVPCONTROL_VPC ); 404 405 val = read_c0_mvpconf0(); 406 nvpe = ((val & MVPCONF0_PVPE) >> MVPCONF0_PVPE_SHIFT) + 1; 407 if (vpelimit > 0 && nvpe > vpelimit) 408 nvpe = vpelimit; 409 ntc = ((val & MVPCONF0_PTC) >> MVPCONF0_PTC_SHIFT) + 1; 410 if (ntc > NR_CPUS) 411 ntc = NR_CPUS; 412 if (tclimit > 0 && ntc > tclimit) 413 ntc = tclimit; 414 tcpervpe = ntc / nvpe; 415 slop = ntc % nvpe; /* Residual TCs, < NVPE */ 416 417 /* Set up shared TLB */ 418 smtc_configure_tlb(); 419 420 for (tc = 0, vpe = 0 ; (vpe < nvpe) && (tc < ntc) ; vpe++) { 421 /* 422 * Set the MVP bits. 423 */ 424 settc(tc); 425 write_vpe_c0_vpeconf0(read_vpe_c0_vpeconf0() | VPECONF0_MVP); 426 if (vpe != 0) 427 printk(", "); 428 printk("VPE %d: TC", vpe); 429 for (i = 0; i < tcpervpe; i++) { 430 /* 431 * TC 0 is bound to VPE 0 at reset, 432 * and is presumably executing this 433 * code. Leave it alone! 434 */ 435 if (tc != 0) { 436 smtc_tc_setup(vpe,tc, cpu); 437 cpu++; 438 } 439 printk(" %d", tc); 440 tc++; 441 } 442 if (slop) { 443 if (tc != 0) { 444 smtc_tc_setup(vpe,tc, cpu); 445 cpu++; 446 } 447 printk(" %d", tc); 448 tc++; 449 slop--; 450 } 451 if (vpe != 0) { 452 /* 453 * Clear any stale software interrupts from VPE's Cause 454 */ 455 write_vpe_c0_cause(0); 456 457 /* 458 * Clear ERL/EXL of VPEs other than 0 459 * and set restricted interrupt enable/mask. 460 */ 461 write_vpe_c0_status((read_vpe_c0_status() 462 & ~(ST0_BEV | ST0_ERL | ST0_EXL | ST0_IM)) 463 | (STATUSF_IP0 | STATUSF_IP1 | STATUSF_IP7 464 | ST0_IE)); 465 /* 466 * set config to be the same as vpe0, 467 * particularly kseg0 coherency alg 468 */ 469 write_vpe_c0_config(read_c0_config()); 470 /* Clear any pending timer interrupt */ 471 write_vpe_c0_compare(0); 472 /* Propagate Config7 */ 473 write_vpe_c0_config7(read_c0_config7()); 474 write_vpe_c0_count(read_c0_count()); 475 } 476 /* enable multi-threading within VPE */ 477 write_vpe_c0_vpecontrol(read_vpe_c0_vpecontrol() | VPECONTROL_TE); 478 /* enable the VPE */ 479 write_vpe_c0_vpeconf0(read_vpe_c0_vpeconf0() | VPECONF0_VPA); 480 } 481 482 /* 483 * Pull any physically present but unused TCs out of circulation. 484 */ 485 while (tc < (((val & MVPCONF0_PTC) >> MVPCONF0_PTC_SHIFT) + 1)) { 486 cpu_clear(tc, phys_cpu_present_map); 487 cpu_clear(tc, cpu_present_map); 488 tc++; 489 } 490 491 /* release config state */ 492 write_c0_mvpcontrol( read_c0_mvpcontrol() & ~ MVPCONTROL_VPC ); 493 494 printk("\n"); 495 496 /* Set up coprocessor affinity CPU mask(s) */ 497 498 for (tc = 0; tc < ntc; tc++) { 499 if (cpu_data[tc].options & MIPS_CPU_FPU) 500 cpu_set(tc, mt_fpu_cpumask); 501 } 502 503 /* set up ipi interrupts... */ 504 505 /* If we have multiple VPEs running, set up the cross-VPE interrupt */ 506 507 if (nvpe > 1) 508 setup_cross_vpe_interrupts(); 509 510 /* Set up queue of free IPI "messages". */ 511 nipi = NR_CPUS * IPIBUF_PER_CPU; 512 if (ipibuffers > 0) 513 nipi = ipibuffers; 514 515 pipi = kmalloc(nipi *sizeof(struct smtc_ipi), GFP_KERNEL); 516 if (pipi == NULL) 517 panic("kmalloc of IPI message buffers failed\n"); 518 else 519 printk("IPI buffer pool of %d buffers\n", nipi); 520 for (i = 0; i < nipi; i++) { 521 smtc_ipi_nq(&freeIPIq, pipi); 522 pipi++; 523 } 524 525 /* Arm multithreading and enable other VPEs - but all TCs are Halted */ 526 emt(EMT_ENABLE); 527 evpe(EVPE_ENABLE); 528 local_irq_restore(flags); 529 /* Initialize SMTC /proc statistics/diagnostics */ 530 init_smtc_stats(); 531} 532 533 534/* 535 * Setup the PC, SP, and GP of a secondary processor and start it 536 * running! 537 * smp_bootstrap is the place to resume from 538 * __KSTK_TOS(idle) is apparently the stack pointer 539 * (unsigned long)idle->thread_info the gp 540 * 541 */ 542void smtc_boot_secondary(int cpu, struct task_struct *idle) 543{ 544 extern u32 kernelsp[NR_CPUS]; 545 long flags; 546 int mtflags; 547 548 LOCK_MT_PRA(); 549 if (cpu_data[cpu].vpe_id != cpu_data[smp_processor_id()].vpe_id) { 550 dvpe(); 551 } 552 settc(cpu_data[cpu].tc_id); 553 554 /* pc */ 555 write_tc_c0_tcrestart((unsigned long)&smp_bootstrap); 556 557 /* stack pointer */ 558 kernelsp[cpu] = __KSTK_TOS(idle); 559 write_tc_gpr_sp(__KSTK_TOS(idle)); 560 561 /* global pointer */ 562 write_tc_gpr_gp((unsigned long)idle->thread_info); 563 564 smtc_status |= SMTC_MTC_ACTIVE; 565 write_tc_c0_tchalt(0); 566 if (cpu_data[cpu].vpe_id != cpu_data[smp_processor_id()].vpe_id) { 567 evpe(EVPE_ENABLE); 568 } 569 UNLOCK_MT_PRA(); 570} 571 572void smtc_init_secondary(void) 573{ 574 /* 575 * Start timer on secondary VPEs if necessary. 576 * plat_timer_setup has already have been invoked by init/main 577 * on "boot" TC. Like per_cpu_trap_init() hack, this assumes that 578 * SMTC init code assigns TCs consdecutively and in ascending order 579 * to across available VPEs. 580 */ 581 if (((read_c0_tcbind() & TCBIND_CURTC) != 0) && 582 ((read_c0_tcbind() & TCBIND_CURVPE) 583 != cpu_data[smp_processor_id() - 1].vpe_id)){ 584 write_c0_compare (read_c0_count() + mips_hpt_frequency/HZ); 585 } 586 587 local_irq_enable(); 588} 589 590void smtc_smp_finish(void) 591{ 592 printk("TC %d going on-line as CPU %d\n", 593 cpu_data[smp_processor_id()].tc_id, smp_processor_id()); 594} 595 596void smtc_cpus_done(void) 597{ 598} 599 600/* 601 * Support for SMTC-optimized driver IRQ registration 602 */ 603 604/* 605 * SMTC Kernel needs to manipulate low-level CPU interrupt mask 606 * in do_IRQ. These are passed in setup_irq_smtc() and stored 607 * in this table. 608 */ 609 610int setup_irq_smtc(unsigned int irq, struct irqaction * new, 611 unsigned long hwmask) 612{ 613 irq_hwmask[irq] = hwmask; 614 615 return setup_irq(irq, new); 616} 617 618/* 619 * IPI model for SMTC is tricky, because interrupts aren't TC-specific. 620 * Within a VPE one TC can interrupt another by different approaches. 621 * The easiest to get right would probably be to make all TCs except 622 * the target IXMT and set a software interrupt, but an IXMT-based 623 * scheme requires that a handler must run before a new IPI could 624 * be sent, which would break the "broadcast" loops in MIPS MT. 625 * A more gonzo approach within a VPE is to halt the TC, extract 626 * its Restart, Status, and a couple of GPRs, and program the Restart 627 * address to emulate an interrupt. 628 * 629 * Within a VPE, one can be confident that the target TC isn't in 630 * a critical EXL state when halted, since the write to the Halt 631 * register could not have issued on the writing thread if the 632 * halting thread had EXL set. So k0 and k1 of the target TC 633 * can be used by the injection code. Across VPEs, one can't 634 * be certain that the target TC isn't in a critical exception 635 * state. So we try a two-step process of sending a software 636 * interrupt to the target VPE, which either handles the event 637 * itself (if it was the target) or injects the event within 638 * the VPE. 639 */ 640 641void smtc_ipi_qdump(void) 642{ 643 int i; 644 645 for (i = 0; i < NR_CPUS ;i++) { 646 printk("IPIQ[%d]: head = 0x%x, tail = 0x%x, depth = %d\n", 647 i, (unsigned)IPIQ[i].head, (unsigned)IPIQ[i].tail, 648 IPIQ[i].depth); 649 } 650} 651 652/* 653 * The standard atomic.h primitives don't quite do what we want 654 * here: We need an atomic add-and-return-previous-value (which 655 * could be done with atomic_add_return and a decrement) and an 656 * atomic set/zero-and-return-previous-value (which can't really 657 * be done with the atomic.h primitives). And since this is 658 * MIPS MT, we can assume that we have LL/SC. 659 */ 660static __inline__ int atomic_postincrement(unsigned int *pv) 661{ 662 unsigned long result; 663 664 unsigned long temp; 665 666 __asm__ __volatile__( 667 "1: ll %0, %2 \n" 668 " addu %1, %0, 1 \n" 669 " sc %1, %2 \n" 670 " beqz %1, 1b \n" 671 " sync \n" 672 : "=&r" (result), "=&r" (temp), "=m" (*pv) 673 : "m" (*pv) 674 : "memory"); 675 676 return result; 677} 678 679void smtc_send_ipi(int cpu, int type, unsigned int action) 680{ 681 int tcstatus; 682 struct smtc_ipi *pipi; 683 long flags; 684 int mtflags; 685 686 if (cpu == smp_processor_id()) { 687 printk("Cannot Send IPI to self!\n"); 688 return; 689 } 690 /* Set up a descriptor, to be delivered either promptly or queued */ 691 pipi = smtc_ipi_dq(&freeIPIq); 692 if (pipi == NULL) { 693 bust_spinlocks(1); 694 mips_mt_regdump(dvpe()); 695 panic("IPI Msg. Buffers Depleted\n"); 696 } 697 pipi->type = type; 698 pipi->arg = (void *)action; 699 pipi->dest = cpu; 700 if (cpu_data[cpu].vpe_id != cpu_data[smp_processor_id()].vpe_id) { 701 /* If not on same VPE, enqueue and send cross-VPE interupt */ 702 smtc_ipi_nq(&IPIQ[cpu], pipi); 703 LOCK_CORE_PRA(); 704 settc(cpu_data[cpu].tc_id); 705 write_vpe_c0_cause(read_vpe_c0_cause() | C_SW1); 706 UNLOCK_CORE_PRA(); 707 } else { 708 /* 709 * Not sufficient to do a LOCK_MT_PRA (dmt) here, 710 * since ASID shootdown on the other VPE may 711 * collide with this operation. 712 */ 713 LOCK_CORE_PRA(); 714 settc(cpu_data[cpu].tc_id); 715 /* Halt the targeted TC */ 716 write_tc_c0_tchalt(TCHALT_H); 717 mips_ihb(); 718 719 /* 720 * Inspect TCStatus - if IXMT is set, we have to queue 721 * a message. Otherwise, we set up the "interrupt" 722 * of the other TC 723 */ 724 tcstatus = read_tc_c0_tcstatus(); 725 726 if ((tcstatus & TCSTATUS_IXMT) != 0) { 727 /* 728 * Spin-waiting here can deadlock, 729 * so we queue the message for the target TC. 730 */ 731 write_tc_c0_tchalt(0); 732 UNLOCK_CORE_PRA(); 733 /* Try to reduce redundant timer interrupt messages */ 734 if (type == SMTC_CLOCK_TICK) { 735 if (atomic_postincrement(&ipi_timer_latch[cpu])!=0){ 736 smtc_ipi_nq(&freeIPIq, pipi); 737 return; 738 } 739 } 740 smtc_ipi_nq(&IPIQ[cpu], pipi); 741 } else { 742 post_direct_ipi(cpu, pipi); 743 write_tc_c0_tchalt(0); 744 UNLOCK_CORE_PRA(); 745 } 746 } 747} 748 749/* 750 * Send IPI message to Halted TC, TargTC/TargVPE already having been set 751 */ 752void post_direct_ipi(int cpu, struct smtc_ipi *pipi) 753{ 754 struct pt_regs *kstack; 755 unsigned long tcstatus; 756 unsigned long tcrestart; 757 extern u32 kernelsp[NR_CPUS]; 758 extern void __smtc_ipi_vector(void); 759 760 /* Extract Status, EPC from halted TC */ 761 tcstatus = read_tc_c0_tcstatus(); 762 tcrestart = read_tc_c0_tcrestart(); 763 /* If TCRestart indicates a WAIT instruction, advance the PC */ 764 if ((tcrestart & 0x80000000) 765 && ((*(unsigned int *)tcrestart & 0xfe00003f) == 0x42000020)) { 766 tcrestart += 4; 767 } 768 /* 769 * Save on TC's future kernel stack 770 * 771 * CU bit of Status is indicator that TC was 772 * already running on a kernel stack... 773 */ 774 if (tcstatus & ST0_CU0) { 775 /* Note that this "- 1" is pointer arithmetic */ 776 kstack = ((struct pt_regs *)read_tc_gpr_sp()) - 1; 777 } else { 778 kstack = ((struct pt_regs *)kernelsp[cpu]) - 1; 779 } 780 781 kstack->cp0_epc = (long)tcrestart; 782 /* Save TCStatus */ 783 kstack->cp0_tcstatus = tcstatus; 784 /* Pass token of operation to be performed kernel stack pad area */ 785 kstack->pad0[4] = (unsigned long)pipi; 786 /* Pass address of function to be called likewise */ 787 kstack->pad0[5] = (unsigned long)&ipi_decode; 788 /* Set interrupt exempt and kernel mode */ 789 tcstatus |= TCSTATUS_IXMT; 790 tcstatus &= ~TCSTATUS_TKSU; 791 write_tc_c0_tcstatus(tcstatus); 792 ehb(); 793 /* Set TC Restart address to be SMTC IPI vector */ 794 write_tc_c0_tcrestart(__smtc_ipi_vector); 795} 796 797static void ipi_resched_interrupt(void) 798{ 799 /* Return from interrupt should be enough to cause scheduler check */ 800} 801 802 803static void ipi_call_interrupt(void) 804{ 805 /* Invoke generic function invocation code in smp.c */ 806 smp_call_function_interrupt(); 807} 808 809void ipi_decode(struct smtc_ipi *pipi) 810{ 811 void *arg_copy = pipi->arg; 812 int type_copy = pipi->type; 813 int dest_copy = pipi->dest; 814 815 smtc_ipi_nq(&freeIPIq, pipi); 816 switch (type_copy) { 817 case SMTC_CLOCK_TICK: 818 /* Invoke Clock "Interrupt" */ 819 ipi_timer_latch[dest_copy] = 0; 820#ifdef SMTC_IDLE_HOOK_DEBUG 821 clock_hang_reported[dest_copy] = 0; 822#endif /* SMTC_IDLE_HOOK_DEBUG */ 823 local_timer_interrupt(0, NULL); 824 break; 825 case LINUX_SMP_IPI: 826 switch ((int)arg_copy) { 827 case SMP_RESCHEDULE_YOURSELF: 828 ipi_resched_interrupt(); 829 break; 830 case SMP_CALL_FUNCTION: 831 ipi_call_interrupt(); 832 break; 833 default: 834 printk("Impossible SMTC IPI Argument 0x%x\n", 835 (int)arg_copy); 836 break; 837 } 838 break; 839 default: 840 printk("Impossible SMTC IPI Type 0x%x\n", type_copy); 841 break; 842 } 843} 844 845void deferred_smtc_ipi(void) 846{ 847 struct smtc_ipi *pipi; 848 unsigned long flags; 849/* DEBUG */ 850 int q = smp_processor_id(); 851 852 /* 853 * Test is not atomic, but much faster than a dequeue, 854 * and the vast majority of invocations will have a null queue. 855 */ 856 if (IPIQ[q].head != NULL) { 857 while((pipi = smtc_ipi_dq(&IPIQ[q])) != NULL) { 858 /* ipi_decode() should be called with interrupts off */ 859 local_irq_save(flags); 860 ipi_decode(pipi); 861 local_irq_restore(flags); 862 } 863 } 864} 865 866/* 867 * Send clock tick to all TCs except the one executing the funtion 868 */ 869 870void smtc_timer_broadcast(int vpe) 871{ 872 int cpu; 873 int myTC = cpu_data[smp_processor_id()].tc_id; 874 int myVPE = cpu_data[smp_processor_id()].vpe_id; 875 876 smtc_cpu_stats[smp_processor_id()].timerints++; 877 878 for_each_online_cpu(cpu) { 879 if (cpu_data[cpu].vpe_id == myVPE && 880 cpu_data[cpu].tc_id != myTC) 881 smtc_send_ipi(cpu, SMTC_CLOCK_TICK, 0); 882 } 883} 884 885/* 886 * Cross-VPE interrupts in the SMTC prototype use "software interrupts" 887 * set via cross-VPE MTTR manipulation of the Cause register. It would be 888 * in some regards preferable to have external logic for "doorbell" hardware 889 * interrupts. 890 */ 891 892static int cpu_ipi_irq = MIPS_CPU_IRQ_BASE + MIPS_CPU_IPI_IRQ; 893 894static irqreturn_t ipi_interrupt(int irq, void *dev_idm) 895{ 896 int my_vpe = cpu_data[smp_processor_id()].vpe_id; 897 int my_tc = cpu_data[smp_processor_id()].tc_id; 898 int cpu; 899 struct smtc_ipi *pipi; 900 unsigned long tcstatus; 901 int sent; 902 long flags; 903 unsigned int mtflags; 904 unsigned int vpflags; 905 906 /* 907 * So long as cross-VPE interrupts are done via 908 * MFTR/MTTR read-modify-writes of Cause, we need 909 * to stop other VPEs whenever the local VPE does 910 * anything similar. 911 */ 912 local_irq_save(flags); 913 vpflags = dvpe(); 914 clear_c0_cause(0x100 << MIPS_CPU_IPI_IRQ); 915 set_c0_status(0x100 << MIPS_CPU_IPI_IRQ); 916 irq_enable_hazard(); 917 evpe(vpflags); 918 local_irq_restore(flags); 919 920 /* 921 * Cross-VPE Interrupt handler: Try to directly deliver IPIs 922 * queued for TCs on this VPE other than the current one. 923 * Return-from-interrupt should cause us to drain the queue 924 * for the current TC, so we ought not to have to do it explicitly here. 925 */ 926 927 for_each_online_cpu(cpu) { 928 if (cpu_data[cpu].vpe_id != my_vpe) 929 continue; 930 931 pipi = smtc_ipi_dq(&IPIQ[cpu]); 932 if (pipi != NULL) { 933 if (cpu_data[cpu].tc_id != my_tc) { 934 sent = 0; 935 LOCK_MT_PRA(); 936 settc(cpu_data[cpu].tc_id); 937 write_tc_c0_tchalt(TCHALT_H); 938 mips_ihb(); 939 tcstatus = read_tc_c0_tcstatus(); 940 if ((tcstatus & TCSTATUS_IXMT) == 0) { 941 post_direct_ipi(cpu, pipi); 942 sent = 1; 943 } 944 write_tc_c0_tchalt(0); 945 UNLOCK_MT_PRA(); 946 if (!sent) { 947 smtc_ipi_req(&IPIQ[cpu], pipi); 948 } 949 } else { 950 /* 951 * ipi_decode() should be called 952 * with interrupts off 953 */ 954 local_irq_save(flags); 955 ipi_decode(pipi); 956 local_irq_restore(flags); 957 } 958 } 959 } 960 961 return IRQ_HANDLED; 962} 963 964static void ipi_irq_dispatch(void) 965{ 966 do_IRQ(cpu_ipi_irq); 967} 968 969static struct irqaction irq_ipi; 970 971void setup_cross_vpe_interrupts(void) 972{ 973 if (!cpu_has_vint) 974 panic("SMTC Kernel requires Vectored Interupt support"); 975 976 set_vi_handler(MIPS_CPU_IPI_IRQ, ipi_irq_dispatch); 977 978 irq_ipi.handler = ipi_interrupt; 979 irq_ipi.flags = IRQF_DISABLED; 980 irq_ipi.name = "SMTC_IPI"; 981 982 setup_irq_smtc(cpu_ipi_irq, &irq_ipi, (0x100 << MIPS_CPU_IPI_IRQ)); 983 984 irq_desc[cpu_ipi_irq].status |= IRQ_PER_CPU; 985 set_irq_handler(cpu_ipi_irq, handle_percpu_irq); 986} 987 988/* 989 * SMTC-specific hacks invoked from elsewhere in the kernel. 990 */ 991 992void smtc_ipi_replay(void) 993{ 994 /* 995 * To the extent that we've ever turned interrupts off, 996 * we may have accumulated deferred IPIs. This is subtle. 997 * If we use the smtc_ipi_qdepth() macro, we'll get an 998 * exact number - but we'll also disable interrupts 999 * and create a window of failure where a new IPI gets 1000 * queued after we test the depth but before we re-enable 1001 * interrupts. So long as IXMT never gets set, however, 1002 * we should be OK: If we pick up something and dispatch 1003 * it here, that's great. If we see nothing, but concurrent 1004 * with this operation, another TC sends us an IPI, IXMT 1005 * is clear, and we'll handle it as a real pseudo-interrupt 1006 * and not a pseudo-pseudo interrupt. 1007 */ 1008 if (IPIQ[smp_processor_id()].depth > 0) { 1009 struct smtc_ipi *pipi; 1010 extern void self_ipi(struct smtc_ipi *); 1011 1012 while ((pipi = smtc_ipi_dq(&IPIQ[smp_processor_id()]))) { 1013 self_ipi(pipi); 1014 smtc_cpu_stats[smp_processor_id()].selfipis++; 1015 } 1016 } 1017} 1018 1019EXPORT_SYMBOL(smtc_ipi_replay); 1020 1021void smtc_idle_loop_hook(void) 1022{ 1023#ifdef SMTC_IDLE_HOOK_DEBUG 1024 int im; 1025 int flags; 1026 int mtflags; 1027 int bit; 1028 int vpe; 1029 int tc; 1030 int hook_ntcs; 1031 /* 1032 * printk within DMT-protected regions can deadlock, 1033 * so buffer diagnostic messages for later output. 1034 */ 1035 char *pdb_msg; 1036 char id_ho_db_msg[768]; /* worst-case use should be less than 700 */ 1037 1038 if (atomic_read(&idle_hook_initialized) == 0) { /* fast test */ 1039 if (atomic_add_return(1, &idle_hook_initialized) == 1) { 1040 int mvpconf0; 1041 /* Tedious stuff to just do once */ 1042 mvpconf0 = read_c0_mvpconf0(); 1043 hook_ntcs = ((mvpconf0 & MVPCONF0_PTC) >> MVPCONF0_PTC_SHIFT) + 1; 1044 if (hook_ntcs > NR_CPUS) 1045 hook_ntcs = NR_CPUS; 1046 for (tc = 0; tc < hook_ntcs; tc++) { 1047 tcnoprog[tc] = 0; 1048 clock_hang_reported[tc] = 0; 1049 } 1050 for (vpe = 0; vpe < 2; vpe++) 1051 for (im = 0; im < 8; im++) 1052 imstuckcount[vpe][im] = 0; 1053 printk("Idle loop test hook initialized for %d TCs\n", hook_ntcs); 1054 atomic_set(&idle_hook_initialized, 1000); 1055 } else { 1056 /* Someone else is initializing in parallel - let 'em finish */ 1057 while (atomic_read(&idle_hook_initialized) < 1000) 1058 ; 1059 } 1060 } 1061 1062 /* Have we stupidly left IXMT set somewhere? */ 1063 if (read_c0_tcstatus() & 0x400) { 1064 write_c0_tcstatus(read_c0_tcstatus() & ~0x400); 1065 ehb(); 1066 printk("Dangling IXMT in cpu_idle()\n"); 1067 } 1068 1069 /* Have we stupidly left an IM bit turned off? */ 1070#define IM_LIMIT 2000 1071 local_irq_save(flags); 1072 mtflags = dmt(); 1073 pdb_msg = &id_ho_db_msg[0]; 1074 im = read_c0_status(); 1075 vpe = cpu_data[smp_processor_id()].vpe_id; 1076 for (bit = 0; bit < 8; bit++) { 1077 /* 1078 * In current prototype, I/O interrupts 1079 * are masked for VPE > 0 1080 */ 1081 if (vpemask[vpe][bit]) { 1082 if (!(im & (0x100 << bit))) 1083 imstuckcount[vpe][bit]++; 1084 else 1085 imstuckcount[vpe][bit] = 0; 1086 if (imstuckcount[vpe][bit] > IM_LIMIT) { 1087 set_c0_status(0x100 << bit); 1088 ehb(); 1089 imstuckcount[vpe][bit] = 0; 1090 pdb_msg += sprintf(pdb_msg, 1091 "Dangling IM %d fixed for VPE %d\n", bit, 1092 vpe); 1093 } 1094 } 1095 } 1096 1097 /* 1098 * Now that we limit outstanding timer IPIs, check for hung TC 1099 */ 1100 for (tc = 0; tc < NR_CPUS; tc++) { 1101 /* Don't check ourself - we'll dequeue IPIs just below */ 1102 if ((tc != smp_processor_id()) && 1103 ipi_timer_latch[tc] > timerq_limit) { 1104 if (clock_hang_reported[tc] == 0) { 1105 pdb_msg += sprintf(pdb_msg, 1106 "TC %d looks hung with timer latch at %d\n", 1107 tc, ipi_timer_latch[tc]); 1108 clock_hang_reported[tc]++; 1109 } 1110 } 1111 } 1112 emt(mtflags); 1113 local_irq_restore(flags); 1114 if (pdb_msg != &id_ho_db_msg[0]) 1115 printk("CPU%d: %s", smp_processor_id(), id_ho_db_msg); 1116#endif /* SMTC_IDLE_HOOK_DEBUG */ 1117 1118 /* 1119 * Replay any accumulated deferred IPIs. If "Instant Replay" 1120 * is in use, there should never be any. 1121 */ 1122#ifndef CONFIG_MIPS_MT_SMTC_INSTANT_REPLAY 1123 smtc_ipi_replay(); 1124#endif /* CONFIG_MIPS_MT_SMTC_INSTANT_REPLAY */ 1125} 1126 1127void smtc_soft_dump(void) 1128{ 1129 int i; 1130 1131 printk("Counter Interrupts taken per CPU (TC)\n"); 1132 for (i=0; i < NR_CPUS; i++) { 1133 printk("%d: %ld\n", i, smtc_cpu_stats[i].timerints); 1134 } 1135 printk("Self-IPI invocations:\n"); 1136 for (i=0; i < NR_CPUS; i++) { 1137 printk("%d: %ld\n", i, smtc_cpu_stats[i].selfipis); 1138 } 1139 smtc_ipi_qdump(); 1140 printk("Timer IPI Backlogs:\n"); 1141 for (i=0; i < NR_CPUS; i++) { 1142 printk("%d: %d\n", i, ipi_timer_latch[i]); 1143 } 1144 printk("%d Recoveries of \"stolen\" FPU\n", 1145 atomic_read(&smtc_fpu_recoveries)); 1146} 1147 1148 1149/* 1150 * TLB management routines special to SMTC 1151 */ 1152 1153void smtc_get_new_mmu_context(struct mm_struct *mm, unsigned long cpu) 1154{ 1155 unsigned long flags, mtflags, tcstat, prevhalt, asid; 1156 int tlb, i; 1157 1158 /* 1159 * It would be nice to be able to use a spinlock here, 1160 * but this is invoked from within TLB flush routines 1161 * that protect themselves with DVPE, so if a lock is 1162 * held by another TC, it'll never be freed. 1163 * 1164 * DVPE/DMT must not be done with interrupts enabled, 1165 * so even so most callers will already have disabled 1166 * them, let's be really careful... 1167 */ 1168 1169 local_irq_save(flags); 1170 if (smtc_status & SMTC_TLB_SHARED) { 1171 mtflags = dvpe(); 1172 tlb = 0; 1173 } else { 1174 mtflags = dmt(); 1175 tlb = cpu_data[cpu].vpe_id; 1176 } 1177 asid = asid_cache(cpu); 1178 1179 do { 1180 if (!((asid += ASID_INC) & ASID_MASK) ) { 1181 if (cpu_has_vtag_icache) 1182 flush_icache_all(); 1183 /* Traverse all online CPUs (hack requires contigous range) */ 1184 for (i = 0; i < num_online_cpus(); i++) { 1185 /* 1186 * We don't need to worry about our own CPU, nor those of 1187 * CPUs who don't share our TLB. 1188 */ 1189 if ((i != smp_processor_id()) && 1190 ((smtc_status & SMTC_TLB_SHARED) || 1191 (cpu_data[i].vpe_id == cpu_data[cpu].vpe_id))) { 1192 settc(cpu_data[i].tc_id); 1193 prevhalt = read_tc_c0_tchalt() & TCHALT_H; 1194 if (!prevhalt) { 1195 write_tc_c0_tchalt(TCHALT_H); 1196 mips_ihb(); 1197 } 1198 tcstat = read_tc_c0_tcstatus(); 1199 smtc_live_asid[tlb][(tcstat & ASID_MASK)] |= (asiduse)(0x1 << i); 1200 if (!prevhalt) 1201 write_tc_c0_tchalt(0); 1202 } 1203 } 1204 if (!asid) /* fix version if needed */ 1205 asid = ASID_FIRST_VERSION; 1206 local_flush_tlb_all(); /* start new asid cycle */ 1207 } 1208 } while (smtc_live_asid[tlb][(asid & ASID_MASK)]); 1209 1210 /* 1211 * SMTC shares the TLB within VPEs and possibly across all VPEs. 1212 */ 1213 for (i = 0; i < num_online_cpus(); i++) { 1214 if ((smtc_status & SMTC_TLB_SHARED) || 1215 (cpu_data[i].vpe_id == cpu_data[cpu].vpe_id)) 1216 cpu_context(i, mm) = asid_cache(i) = asid; 1217 } 1218 1219 if (smtc_status & SMTC_TLB_SHARED) 1220 evpe(mtflags); 1221 else 1222 emt(mtflags); 1223 local_irq_restore(flags); 1224} 1225 1226/* 1227 * Invoked from macros defined in mmu_context.h 1228 * which must already have disabled interrupts 1229 * and done a DVPE or DMT as appropriate. 1230 */ 1231 1232void smtc_flush_tlb_asid(unsigned long asid) 1233{ 1234 int entry; 1235 unsigned long ehi; 1236 1237 entry = read_c0_wired(); 1238 1239 /* Traverse all non-wired entries */ 1240 while (entry < current_cpu_data.tlbsize) { 1241 write_c0_index(entry); 1242 ehb(); 1243 tlb_read(); 1244 ehb(); 1245 ehi = read_c0_entryhi(); 1246 if ((ehi & ASID_MASK) == asid) { 1247 /* 1248 * Invalidate only entries with specified ASID, 1249 * makiing sure all entries differ. 1250 */ 1251 write_c0_entryhi(CKSEG0 + (entry << (PAGE_SHIFT + 1))); 1252 write_c0_entrylo0(0); 1253 write_c0_entrylo1(0); 1254 mtc0_tlbw_hazard(); 1255 tlb_write_indexed(); 1256 } 1257 entry++; 1258 } 1259 write_c0_index(PARKED_INDEX); 1260 tlbw_use_hazard(); 1261} 1262 1263/* 1264 * Support for single-threading cache flush operations. 1265 */ 1266 1267int halt_state_save[NR_CPUS]; 1268 1269/* 1270 * To really, really be sure that nothing is being done 1271 * by other TCs, halt them all. This code assumes that 1272 * a DVPE has already been done, so while their Halted 1273 * state is theoretically architecturally unstable, in 1274 * practice, it's not going to change while we're looking 1275 * at it. 1276 */ 1277 1278void smtc_cflush_lockdown(void) 1279{ 1280 int cpu; 1281 1282 for_each_online_cpu(cpu) { 1283 if (cpu != smp_processor_id()) { 1284 settc(cpu_data[cpu].tc_id); 1285 halt_state_save[cpu] = read_tc_c0_tchalt(); 1286 write_tc_c0_tchalt(TCHALT_H); 1287 } 1288 } 1289 mips_ihb(); 1290} 1291 1292/* It would be cheating to change the cpu_online states during a flush! */ 1293 1294void smtc_cflush_release(void) 1295{ 1296 int cpu; 1297 1298 /* 1299 * Start with a hazard barrier to ensure 1300 * that all CACHE ops have played through. 1301 */ 1302 mips_ihb(); 1303 1304 for_each_online_cpu(cpu) { 1305 if (cpu != smp_processor_id()) { 1306 settc(cpu_data[cpu].tc_id); 1307 write_tc_c0_tchalt(halt_state_save[cpu]); 1308 } 1309 } 1310 mips_ihb(); 1311} 1312