smtc.c revision 930cd54b3bd78e52991a89b39b5ef58355ad2b6d
1/* 2 * This program is free software; you can redistribute it and/or 3 * modify it under the terms of the GNU General Public License 4 * as published by the Free Software Foundation; either version 2 5 * of the License, or (at your option) any later version. 6 * 7 * This program is distributed in the hope that it will be useful, 8 * but WITHOUT ANY WARRANTY; without even the implied warranty of 9 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 10 * GNU General Public License for more details. 11 * 12 * You should have received a copy of the GNU General Public License 13 * along with this program; if not, write to the Free Software 14 * Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA. 15 * 16 * Copyright (C) 2004 Mips Technologies, Inc 17 * Copyright (C) 2008 Kevin D. Kissell 18 */ 19 20#include <linux/clockchips.h> 21#include <linux/kernel.h> 22#include <linux/sched.h> 23#include <linux/smp.h> 24#include <linux/cpumask.h> 25#include <linux/interrupt.h> 26#include <linux/kernel_stat.h> 27#include <linux/module.h> 28#include <linux/ftrace.h> 29#include <linux/slab.h> 30 31#include <asm/cpu.h> 32#include <asm/processor.h> 33#include <asm/atomic.h> 34#include <asm/system.h> 35#include <asm/hardirq.h> 36#include <asm/hazards.h> 37#include <asm/irq.h> 38#include <asm/mmu_context.h> 39#include <asm/mipsregs.h> 40#include <asm/cacheflush.h> 41#include <asm/time.h> 42#include <asm/addrspace.h> 43#include <asm/smtc.h> 44#include <asm/smtc_proc.h> 45 46/* 47 * SMTC Kernel needs to manipulate low-level CPU interrupt mask 48 * in do_IRQ. These are passed in setup_irq_smtc() and stored 49 * in this table. 50 */ 51unsigned long irq_hwmask[NR_IRQS]; 52 53#define LOCK_MT_PRA() \ 54 local_irq_save(flags); \ 55 mtflags = dmt() 56 57#define UNLOCK_MT_PRA() \ 58 emt(mtflags); \ 59 local_irq_restore(flags) 60 61#define LOCK_CORE_PRA() \ 62 local_irq_save(flags); \ 63 mtflags = dvpe() 64 65#define UNLOCK_CORE_PRA() \ 66 evpe(mtflags); \ 67 local_irq_restore(flags) 68 69/* 70 * Data structures purely associated with SMTC parallelism 71 */ 72 73 74/* 75 * Table for tracking ASIDs whose lifetime is prolonged. 76 */ 77 78asiduse smtc_live_asid[MAX_SMTC_TLBS][MAX_SMTC_ASIDS]; 79 80/* 81 * Number of InterProcessor Interrupt (IPI) message buffers to allocate 82 */ 83 84#define IPIBUF_PER_CPU 4 85 86struct smtc_ipi_q IPIQ[NR_CPUS]; 87static struct smtc_ipi_q freeIPIq; 88 89 90/* Forward declarations */ 91 92void ipi_decode(struct smtc_ipi *); 93static void post_direct_ipi(int cpu, struct smtc_ipi *pipi); 94static void setup_cross_vpe_interrupts(unsigned int nvpe); 95void init_smtc_stats(void); 96 97/* Global SMTC Status */ 98 99unsigned int smtc_status; 100 101/* Boot command line configuration overrides */ 102 103static int vpe0limit; 104static int ipibuffers; 105static int nostlb; 106static int asidmask; 107unsigned long smtc_asid_mask = 0xff; 108 109static int __init vpe0tcs(char *str) 110{ 111 get_option(&str, &vpe0limit); 112 113 return 1; 114} 115 116static int __init ipibufs(char *str) 117{ 118 get_option(&str, &ipibuffers); 119 return 1; 120} 121 122static int __init stlb_disable(char *s) 123{ 124 nostlb = 1; 125 return 1; 126} 127 128static int __init asidmask_set(char *str) 129{ 130 get_option(&str, &asidmask); 131 switch (asidmask) { 132 case 0x1: 133 case 0x3: 134 case 0x7: 135 case 0xf: 136 case 0x1f: 137 case 0x3f: 138 case 0x7f: 139 case 0xff: 140 smtc_asid_mask = (unsigned long)asidmask; 141 break; 142 default: 143 printk("ILLEGAL ASID mask 0x%x from command line\n", asidmask); 144 } 145 return 1; 146} 147 148__setup("vpe0tcs=", vpe0tcs); 149__setup("ipibufs=", ipibufs); 150__setup("nostlb", stlb_disable); 151__setup("asidmask=", asidmask_set); 152 153#ifdef CONFIG_SMTC_IDLE_HOOK_DEBUG 154 155static int hang_trig; 156 157static int __init hangtrig_enable(char *s) 158{ 159 hang_trig = 1; 160 return 1; 161} 162 163 164__setup("hangtrig", hangtrig_enable); 165 166#define DEFAULT_BLOCKED_IPI_LIMIT 32 167 168static int timerq_limit = DEFAULT_BLOCKED_IPI_LIMIT; 169 170static int __init tintq(char *str) 171{ 172 get_option(&str, &timerq_limit); 173 return 1; 174} 175 176__setup("tintq=", tintq); 177 178static int imstuckcount[2][8]; 179/* vpemask represents IM/IE bits of per-VPE Status registers, low-to-high */ 180static int vpemask[2][8] = { 181 {0, 0, 1, 0, 0, 0, 0, 1}, 182 {0, 0, 0, 0, 0, 0, 0, 1} 183}; 184int tcnoprog[NR_CPUS]; 185static atomic_t idle_hook_initialized = ATOMIC_INIT(0); 186static int clock_hang_reported[NR_CPUS]; 187 188#endif /* CONFIG_SMTC_IDLE_HOOK_DEBUG */ 189 190/* 191 * Configure shared TLB - VPC configuration bit must be set by caller 192 */ 193 194static void smtc_configure_tlb(void) 195{ 196 int i, tlbsiz, vpes; 197 unsigned long mvpconf0; 198 unsigned long config1val; 199 200 /* Set up ASID preservation table */ 201 for (vpes=0; vpes<MAX_SMTC_TLBS; vpes++) { 202 for(i = 0; i < MAX_SMTC_ASIDS; i++) { 203 smtc_live_asid[vpes][i] = 0; 204 } 205 } 206 mvpconf0 = read_c0_mvpconf0(); 207 208 if ((vpes = ((mvpconf0 & MVPCONF0_PVPE) 209 >> MVPCONF0_PVPE_SHIFT) + 1) > 1) { 210 /* If we have multiple VPEs, try to share the TLB */ 211 if ((mvpconf0 & MVPCONF0_TLBS) && !nostlb) { 212 /* 213 * If TLB sizing is programmable, shared TLB 214 * size is the total available complement. 215 * Otherwise, we have to take the sum of all 216 * static VPE TLB entries. 217 */ 218 if ((tlbsiz = ((mvpconf0 & MVPCONF0_PTLBE) 219 >> MVPCONF0_PTLBE_SHIFT)) == 0) { 220 /* 221 * If there's more than one VPE, there had better 222 * be more than one TC, because we need one to bind 223 * to each VPE in turn to be able to read 224 * its configuration state! 225 */ 226 settc(1); 227 /* Stop the TC from doing anything foolish */ 228 write_tc_c0_tchalt(TCHALT_H); 229 mips_ihb(); 230 /* No need to un-Halt - that happens later anyway */ 231 for (i=0; i < vpes; i++) { 232 write_tc_c0_tcbind(i); 233 /* 234 * To be 100% sure we're really getting the right 235 * information, we exit the configuration state 236 * and do an IHB after each rebinding. 237 */ 238 write_c0_mvpcontrol( 239 read_c0_mvpcontrol() & ~ MVPCONTROL_VPC ); 240 mips_ihb(); 241 /* 242 * Only count if the MMU Type indicated is TLB 243 */ 244 if (((read_vpe_c0_config() & MIPS_CONF_MT) >> 7) == 1) { 245 config1val = read_vpe_c0_config1(); 246 tlbsiz += ((config1val >> 25) & 0x3f) + 1; 247 } 248 249 /* Put core back in configuration state */ 250 write_c0_mvpcontrol( 251 read_c0_mvpcontrol() | MVPCONTROL_VPC ); 252 mips_ihb(); 253 } 254 } 255 write_c0_mvpcontrol(read_c0_mvpcontrol() | MVPCONTROL_STLB); 256 ehb(); 257 258 /* 259 * Setup kernel data structures to use software total, 260 * rather than read the per-VPE Config1 value. The values 261 * for "CPU 0" gets copied to all the other CPUs as part 262 * of their initialization in smtc_cpu_setup(). 263 */ 264 265 /* MIPS32 limits TLB indices to 64 */ 266 if (tlbsiz > 64) 267 tlbsiz = 64; 268 cpu_data[0].tlbsize = current_cpu_data.tlbsize = tlbsiz; 269 smtc_status |= SMTC_TLB_SHARED; 270 local_flush_tlb_all(); 271 272 printk("TLB of %d entry pairs shared by %d VPEs\n", 273 tlbsiz, vpes); 274 } else { 275 printk("WARNING: TLB Not Sharable on SMTC Boot!\n"); 276 } 277 } 278} 279 280 281/* 282 * Incrementally build the CPU map out of constituent MIPS MT cores, 283 * using the specified available VPEs and TCs. Plaform code needs 284 * to ensure that each MIPS MT core invokes this routine on reset, 285 * one at a time(!). 286 * 287 * This version of the build_cpu_map and prepare_cpus routines assumes 288 * that *all* TCs of a MIPS MT core will be used for Linux, and that 289 * they will be spread across *all* available VPEs (to minimise the 290 * loss of efficiency due to exception service serialization). 291 * An improved version would pick up configuration information and 292 * possibly leave some TCs/VPEs as "slave" processors. 293 * 294 * Use c0_MVPConf0 to find out how many TCs are available, setting up 295 * cpu_possible_map and the logical/physical mappings. 296 */ 297 298int __init smtc_build_cpu_map(int start_cpu_slot) 299{ 300 int i, ntcs; 301 302 /* 303 * The CPU map isn't actually used for anything at this point, 304 * so it's not clear what else we should do apart from set 305 * everything up so that "logical" = "physical". 306 */ 307 ntcs = ((read_c0_mvpconf0() & MVPCONF0_PTC) >> MVPCONF0_PTC_SHIFT) + 1; 308 for (i=start_cpu_slot; i<NR_CPUS && i<ntcs; i++) { 309 set_cpu_possible(i, true); 310 __cpu_number_map[i] = i; 311 __cpu_logical_map[i] = i; 312 } 313#ifdef CONFIG_MIPS_MT_FPAFF 314 /* Initialize map of CPUs with FPUs */ 315 cpus_clear(mt_fpu_cpumask); 316#endif 317 318 /* One of those TC's is the one booting, and not a secondary... */ 319 printk("%i available secondary CPU TC(s)\n", i - 1); 320 321 return i; 322} 323 324/* 325 * Common setup before any secondaries are started 326 * Make sure all CPU's are in a sensible state before we boot any of the 327 * secondaries. 328 * 329 * For MIPS MT "SMTC" operation, we set up all TCs, spread as evenly 330 * as possible across the available VPEs. 331 */ 332 333static void smtc_tc_setup(int vpe, int tc, int cpu) 334{ 335 settc(tc); 336 write_tc_c0_tchalt(TCHALT_H); 337 mips_ihb(); 338 write_tc_c0_tcstatus((read_tc_c0_tcstatus() 339 & ~(TCSTATUS_TKSU | TCSTATUS_DA | TCSTATUS_IXMT)) 340 | TCSTATUS_A); 341 /* 342 * TCContext gets an offset from the base of the IPIQ array 343 * to be used in low-level code to detect the presence of 344 * an active IPI queue 345 */ 346 write_tc_c0_tccontext((sizeof(struct smtc_ipi_q) * cpu) << 16); 347 /* Bind tc to vpe */ 348 write_tc_c0_tcbind(vpe); 349 /* In general, all TCs should have the same cpu_data indications */ 350 memcpy(&cpu_data[cpu], &cpu_data[0], sizeof(struct cpuinfo_mips)); 351 /* For 34Kf, start with TC/CPU 0 as sole owner of single FPU context */ 352 if (cpu_data[0].cputype == CPU_34K || 353 cpu_data[0].cputype == CPU_1004K) 354 cpu_data[cpu].options &= ~MIPS_CPU_FPU; 355 cpu_data[cpu].vpe_id = vpe; 356 cpu_data[cpu].tc_id = tc; 357 /* Multi-core SMTC hasn't been tested, but be prepared */ 358 cpu_data[cpu].core = (read_vpe_c0_ebase() >> 1) & 0xff; 359} 360 361/* 362 * Tweak to get Count registes in as close a sync as possible. 363 * Value seems good for 34K-class cores. 364 */ 365 366#define CP0_SKEW 8 367 368void smtc_prepare_cpus(int cpus) 369{ 370 int i, vpe, tc, ntc, nvpe, tcpervpe[NR_CPUS], slop, cpu; 371 unsigned long flags; 372 unsigned long val; 373 int nipi; 374 struct smtc_ipi *pipi; 375 376 /* disable interrupts so we can disable MT */ 377 local_irq_save(flags); 378 /* disable MT so we can configure */ 379 dvpe(); 380 dmt(); 381 382 spin_lock_init(&freeIPIq.lock); 383 384 /* 385 * We probably don't have as many VPEs as we do SMP "CPUs", 386 * but it's possible - and in any case we'll never use more! 387 */ 388 for (i=0; i<NR_CPUS; i++) { 389 IPIQ[i].head = IPIQ[i].tail = NULL; 390 spin_lock_init(&IPIQ[i].lock); 391 IPIQ[i].depth = 0; 392 IPIQ[i].resched_flag = 0; /* No reschedules queued initially */ 393 } 394 395 /* cpu_data index starts at zero */ 396 cpu = 0; 397 cpu_data[cpu].vpe_id = 0; 398 cpu_data[cpu].tc_id = 0; 399 cpu_data[cpu].core = (read_c0_ebase() >> 1) & 0xff; 400 cpu++; 401 402 /* Report on boot-time options */ 403 mips_mt_set_cpuoptions(); 404 if (vpelimit > 0) 405 printk("Limit of %d VPEs set\n", vpelimit); 406 if (tclimit > 0) 407 printk("Limit of %d TCs set\n", tclimit); 408 if (nostlb) { 409 printk("Shared TLB Use Inhibited - UNSAFE for Multi-VPE Operation\n"); 410 } 411 if (asidmask) 412 printk("ASID mask value override to 0x%x\n", asidmask); 413 414 /* Temporary */ 415#ifdef CONFIG_SMTC_IDLE_HOOK_DEBUG 416 if (hang_trig) 417 printk("Logic Analyser Trigger on suspected TC hang\n"); 418#endif /* CONFIG_SMTC_IDLE_HOOK_DEBUG */ 419 420 /* Put MVPE's into 'configuration state' */ 421 write_c0_mvpcontrol( read_c0_mvpcontrol() | MVPCONTROL_VPC ); 422 423 val = read_c0_mvpconf0(); 424 nvpe = ((val & MVPCONF0_PVPE) >> MVPCONF0_PVPE_SHIFT) + 1; 425 if (vpelimit > 0 && nvpe > vpelimit) 426 nvpe = vpelimit; 427 ntc = ((val & MVPCONF0_PTC) >> MVPCONF0_PTC_SHIFT) + 1; 428 if (ntc > NR_CPUS) 429 ntc = NR_CPUS; 430 if (tclimit > 0 && ntc > tclimit) 431 ntc = tclimit; 432 slop = ntc % nvpe; 433 for (i = 0; i < nvpe; i++) { 434 tcpervpe[i] = ntc / nvpe; 435 if (slop) { 436 if((slop - i) > 0) tcpervpe[i]++; 437 } 438 } 439 /* Handle command line override for VPE0 */ 440 if (vpe0limit > ntc) vpe0limit = ntc; 441 if (vpe0limit > 0) { 442 int slopslop; 443 if (vpe0limit < tcpervpe[0]) { 444 /* Reducing TC count - distribute to others */ 445 slop = tcpervpe[0] - vpe0limit; 446 slopslop = slop % (nvpe - 1); 447 tcpervpe[0] = vpe0limit; 448 for (i = 1; i < nvpe; i++) { 449 tcpervpe[i] += slop / (nvpe - 1); 450 if(slopslop && ((slopslop - (i - 1) > 0))) 451 tcpervpe[i]++; 452 } 453 } else if (vpe0limit > tcpervpe[0]) { 454 /* Increasing TC count - steal from others */ 455 slop = vpe0limit - tcpervpe[0]; 456 slopslop = slop % (nvpe - 1); 457 tcpervpe[0] = vpe0limit; 458 for (i = 1; i < nvpe; i++) { 459 tcpervpe[i] -= slop / (nvpe - 1); 460 if(slopslop && ((slopslop - (i - 1) > 0))) 461 tcpervpe[i]--; 462 } 463 } 464 } 465 466 /* Set up shared TLB */ 467 smtc_configure_tlb(); 468 469 for (tc = 0, vpe = 0 ; (vpe < nvpe) && (tc < ntc) ; vpe++) { 470 if (tcpervpe[vpe] == 0) 471 continue; 472 if (vpe != 0) 473 printk(", "); 474 printk("VPE %d: TC", vpe); 475 for (i = 0; i < tcpervpe[vpe]; i++) { 476 /* 477 * TC 0 is bound to VPE 0 at reset, 478 * and is presumably executing this 479 * code. Leave it alone! 480 */ 481 if (tc != 0) { 482 smtc_tc_setup(vpe, tc, cpu); 483 cpu++; 484 } 485 printk(" %d", tc); 486 tc++; 487 } 488 if (vpe != 0) { 489 /* 490 * Allow this VPE to control others. 491 */ 492 write_vpe_c0_vpeconf0(read_vpe_c0_vpeconf0() | 493 VPECONF0_MVP); 494 495 /* 496 * Clear any stale software interrupts from VPE's Cause 497 */ 498 write_vpe_c0_cause(0); 499 500 /* 501 * Clear ERL/EXL of VPEs other than 0 502 * and set restricted interrupt enable/mask. 503 */ 504 write_vpe_c0_status((read_vpe_c0_status() 505 & ~(ST0_BEV | ST0_ERL | ST0_EXL | ST0_IM)) 506 | (STATUSF_IP0 | STATUSF_IP1 | STATUSF_IP7 507 | ST0_IE)); 508 /* 509 * set config to be the same as vpe0, 510 * particularly kseg0 coherency alg 511 */ 512 write_vpe_c0_config(read_c0_config()); 513 /* Clear any pending timer interrupt */ 514 write_vpe_c0_compare(0); 515 /* Propagate Config7 */ 516 write_vpe_c0_config7(read_c0_config7()); 517 write_vpe_c0_count(read_c0_count() + CP0_SKEW); 518 ehb(); 519 } 520 /* enable multi-threading within VPE */ 521 write_vpe_c0_vpecontrol(read_vpe_c0_vpecontrol() | VPECONTROL_TE); 522 /* enable the VPE */ 523 write_vpe_c0_vpeconf0(read_vpe_c0_vpeconf0() | VPECONF0_VPA); 524 } 525 526 /* 527 * Pull any physically present but unused TCs out of circulation. 528 */ 529 while (tc < (((val & MVPCONF0_PTC) >> MVPCONF0_PTC_SHIFT) + 1)) { 530 set_cpu_possible(tc, false); 531 set_cpu_present(tc, false); 532 tc++; 533 } 534 535 /* release config state */ 536 write_c0_mvpcontrol( read_c0_mvpcontrol() & ~ MVPCONTROL_VPC ); 537 538 printk("\n"); 539 540 /* Set up coprocessor affinity CPU mask(s) */ 541 542#ifdef CONFIG_MIPS_MT_FPAFF 543 for (tc = 0; tc < ntc; tc++) { 544 if (cpu_data[tc].options & MIPS_CPU_FPU) 545 cpu_set(tc, mt_fpu_cpumask); 546 } 547#endif 548 549 /* set up ipi interrupts... */ 550 551 /* If we have multiple VPEs running, set up the cross-VPE interrupt */ 552 553 setup_cross_vpe_interrupts(nvpe); 554 555 /* Set up queue of free IPI "messages". */ 556 nipi = NR_CPUS * IPIBUF_PER_CPU; 557 if (ipibuffers > 0) 558 nipi = ipibuffers; 559 560 pipi = kmalloc(nipi *sizeof(struct smtc_ipi), GFP_KERNEL); 561 if (pipi == NULL) 562 panic("kmalloc of IPI message buffers failed\n"); 563 else 564 printk("IPI buffer pool of %d buffers\n", nipi); 565 for (i = 0; i < nipi; i++) { 566 smtc_ipi_nq(&freeIPIq, pipi); 567 pipi++; 568 } 569 570 /* Arm multithreading and enable other VPEs - but all TCs are Halted */ 571 emt(EMT_ENABLE); 572 evpe(EVPE_ENABLE); 573 local_irq_restore(flags); 574 /* Initialize SMTC /proc statistics/diagnostics */ 575 init_smtc_stats(); 576} 577 578 579/* 580 * Setup the PC, SP, and GP of a secondary processor and start it 581 * running! 582 * smp_bootstrap is the place to resume from 583 * __KSTK_TOS(idle) is apparently the stack pointer 584 * (unsigned long)idle->thread_info the gp 585 * 586 */ 587void __cpuinit smtc_boot_secondary(int cpu, struct task_struct *idle) 588{ 589 extern u32 kernelsp[NR_CPUS]; 590 unsigned long flags; 591 int mtflags; 592 593 LOCK_MT_PRA(); 594 if (cpu_data[cpu].vpe_id != cpu_data[smp_processor_id()].vpe_id) { 595 dvpe(); 596 } 597 settc(cpu_data[cpu].tc_id); 598 599 /* pc */ 600 write_tc_c0_tcrestart((unsigned long)&smp_bootstrap); 601 602 /* stack pointer */ 603 kernelsp[cpu] = __KSTK_TOS(idle); 604 write_tc_gpr_sp(__KSTK_TOS(idle)); 605 606 /* global pointer */ 607 write_tc_gpr_gp((unsigned long)task_thread_info(idle)); 608 609 smtc_status |= SMTC_MTC_ACTIVE; 610 write_tc_c0_tchalt(0); 611 if (cpu_data[cpu].vpe_id != cpu_data[smp_processor_id()].vpe_id) { 612 evpe(EVPE_ENABLE); 613 } 614 UNLOCK_MT_PRA(); 615} 616 617void smtc_init_secondary(void) 618{ 619 local_irq_enable(); 620} 621 622void smtc_smp_finish(void) 623{ 624 int cpu = smp_processor_id(); 625 626 /* 627 * Lowest-numbered CPU per VPE starts a clock tick. 628 * Like per_cpu_trap_init() hack, this assumes that 629 * SMTC init code assigns TCs consdecutively and 630 * in ascending order across available VPEs. 631 */ 632 if (cpu > 0 && (cpu_data[cpu].vpe_id != cpu_data[cpu - 1].vpe_id)) 633 write_c0_compare(read_c0_count() + mips_hpt_frequency/HZ); 634 635 printk("TC %d going on-line as CPU %d\n", 636 cpu_data[smp_processor_id()].tc_id, smp_processor_id()); 637} 638 639void smtc_cpus_done(void) 640{ 641} 642 643/* 644 * Support for SMTC-optimized driver IRQ registration 645 */ 646 647/* 648 * SMTC Kernel needs to manipulate low-level CPU interrupt mask 649 * in do_IRQ. These are passed in setup_irq_smtc() and stored 650 * in this table. 651 */ 652 653int setup_irq_smtc(unsigned int irq, struct irqaction * new, 654 unsigned long hwmask) 655{ 656#ifdef CONFIG_SMTC_IDLE_HOOK_DEBUG 657 unsigned int vpe = current_cpu_data.vpe_id; 658 659 vpemask[vpe][irq - MIPS_CPU_IRQ_BASE] = 1; 660#endif 661 irq_hwmask[irq] = hwmask; 662 663 return setup_irq(irq, new); 664} 665 666#ifdef CONFIG_MIPS_MT_SMTC_IRQAFF 667/* 668 * Support for IRQ affinity to TCs 669 */ 670 671void smtc_set_irq_affinity(unsigned int irq, cpumask_t affinity) 672{ 673 /* 674 * If a "fast path" cache of quickly decodable affinity state 675 * is maintained, this is where it gets done, on a call up 676 * from the platform affinity code. 677 */ 678} 679 680void smtc_forward_irq(struct irq_data *d) 681{ 682 unsigned int irq = d->irq; 683 int target; 684 685 /* 686 * OK wise guy, now figure out how to get the IRQ 687 * to be serviced on an authorized "CPU". 688 * 689 * Ideally, to handle the situation where an IRQ has multiple 690 * eligible CPUS, we would maintain state per IRQ that would 691 * allow a fair distribution of service requests. Since the 692 * expected use model is any-or-only-one, for simplicity 693 * and efficiency, we just pick the easiest one to find. 694 */ 695 696 target = cpumask_first(d->affinity); 697 698 /* 699 * We depend on the platform code to have correctly processed 700 * IRQ affinity change requests to ensure that the IRQ affinity 701 * mask has been purged of bits corresponding to nonexistent and 702 * offline "CPUs", and to TCs bound to VPEs other than the VPE 703 * connected to the physical interrupt input for the interrupt 704 * in question. Otherwise we have a nasty problem with interrupt 705 * mask management. This is best handled in non-performance-critical 706 * platform IRQ affinity setting code, to minimize interrupt-time 707 * checks. 708 */ 709 710 /* If no one is eligible, service locally */ 711 if (target >= NR_CPUS) 712 do_IRQ_no_affinity(irq); 713 else 714 smtc_send_ipi(target, IRQ_AFFINITY_IPI, irq); 715} 716 717#endif /* CONFIG_MIPS_MT_SMTC_IRQAFF */ 718 719/* 720 * IPI model for SMTC is tricky, because interrupts aren't TC-specific. 721 * Within a VPE one TC can interrupt another by different approaches. 722 * The easiest to get right would probably be to make all TCs except 723 * the target IXMT and set a software interrupt, but an IXMT-based 724 * scheme requires that a handler must run before a new IPI could 725 * be sent, which would break the "broadcast" loops in MIPS MT. 726 * A more gonzo approach within a VPE is to halt the TC, extract 727 * its Restart, Status, and a couple of GPRs, and program the Restart 728 * address to emulate an interrupt. 729 * 730 * Within a VPE, one can be confident that the target TC isn't in 731 * a critical EXL state when halted, since the write to the Halt 732 * register could not have issued on the writing thread if the 733 * halting thread had EXL set. So k0 and k1 of the target TC 734 * can be used by the injection code. Across VPEs, one can't 735 * be certain that the target TC isn't in a critical exception 736 * state. So we try a two-step process of sending a software 737 * interrupt to the target VPE, which either handles the event 738 * itself (if it was the target) or injects the event within 739 * the VPE. 740 */ 741 742static void smtc_ipi_qdump(void) 743{ 744 int i; 745 struct smtc_ipi *temp; 746 747 for (i = 0; i < NR_CPUS ;i++) { 748 pr_info("IPIQ[%d]: head = 0x%x, tail = 0x%x, depth = %d\n", 749 i, (unsigned)IPIQ[i].head, (unsigned)IPIQ[i].tail, 750 IPIQ[i].depth); 751 temp = IPIQ[i].head; 752 753 while (temp != IPIQ[i].tail) { 754 pr_debug("%d %d %d: ", temp->type, temp->dest, 755 (int)temp->arg); 756#ifdef SMTC_IPI_DEBUG 757 pr_debug("%u %lu\n", temp->sender, temp->stamp); 758#else 759 pr_debug("\n"); 760#endif 761 temp = temp->flink; 762 } 763 } 764} 765 766/* 767 * The standard atomic.h primitives don't quite do what we want 768 * here: We need an atomic add-and-return-previous-value (which 769 * could be done with atomic_add_return and a decrement) and an 770 * atomic set/zero-and-return-previous-value (which can't really 771 * be done with the atomic.h primitives). And since this is 772 * MIPS MT, we can assume that we have LL/SC. 773 */ 774static inline int atomic_postincrement(atomic_t *v) 775{ 776 unsigned long result; 777 778 unsigned long temp; 779 780 __asm__ __volatile__( 781 "1: ll %0, %2 \n" 782 " addu %1, %0, 1 \n" 783 " sc %1, %2 \n" 784 " beqz %1, 1b \n" 785 __WEAK_LLSC_MB 786 : "=&r" (result), "=&r" (temp), "=m" (v->counter) 787 : "m" (v->counter) 788 : "memory"); 789 790 return result; 791} 792 793void smtc_send_ipi(int cpu, int type, unsigned int action) 794{ 795 int tcstatus; 796 struct smtc_ipi *pipi; 797 unsigned long flags; 798 int mtflags; 799 unsigned long tcrestart; 800 extern void r4k_wait_irqoff(void), __pastwait(void); 801 int set_resched_flag = (type == LINUX_SMP_IPI && 802 action == SMP_RESCHEDULE_YOURSELF); 803 804 if (cpu == smp_processor_id()) { 805 printk("Cannot Send IPI to self!\n"); 806 return; 807 } 808 if (set_resched_flag && IPIQ[cpu].resched_flag != 0) 809 return; /* There is a reschedule queued already */ 810 811 /* Set up a descriptor, to be delivered either promptly or queued */ 812 pipi = smtc_ipi_dq(&freeIPIq); 813 if (pipi == NULL) { 814 bust_spinlocks(1); 815 mips_mt_regdump(dvpe()); 816 panic("IPI Msg. Buffers Depleted\n"); 817 } 818 pipi->type = type; 819 pipi->arg = (void *)action; 820 pipi->dest = cpu; 821 if (cpu_data[cpu].vpe_id != cpu_data[smp_processor_id()].vpe_id) { 822 /* If not on same VPE, enqueue and send cross-VPE interrupt */ 823 IPIQ[cpu].resched_flag |= set_resched_flag; 824 smtc_ipi_nq(&IPIQ[cpu], pipi); 825 LOCK_CORE_PRA(); 826 settc(cpu_data[cpu].tc_id); 827 write_vpe_c0_cause(read_vpe_c0_cause() | C_SW1); 828 UNLOCK_CORE_PRA(); 829 } else { 830 /* 831 * Not sufficient to do a LOCK_MT_PRA (dmt) here, 832 * since ASID shootdown on the other VPE may 833 * collide with this operation. 834 */ 835 LOCK_CORE_PRA(); 836 settc(cpu_data[cpu].tc_id); 837 /* Halt the targeted TC */ 838 write_tc_c0_tchalt(TCHALT_H); 839 mips_ihb(); 840 841 /* 842 * Inspect TCStatus - if IXMT is set, we have to queue 843 * a message. Otherwise, we set up the "interrupt" 844 * of the other TC 845 */ 846 tcstatus = read_tc_c0_tcstatus(); 847 848 if ((tcstatus & TCSTATUS_IXMT) != 0) { 849 /* 850 * If we're in the the irq-off version of the wait 851 * loop, we need to force exit from the wait and 852 * do a direct post of the IPI. 853 */ 854 if (cpu_wait == r4k_wait_irqoff) { 855 tcrestart = read_tc_c0_tcrestart(); 856 if (tcrestart >= (unsigned long)r4k_wait_irqoff 857 && tcrestart < (unsigned long)__pastwait) { 858 write_tc_c0_tcrestart(__pastwait); 859 tcstatus &= ~TCSTATUS_IXMT; 860 write_tc_c0_tcstatus(tcstatus); 861 goto postdirect; 862 } 863 } 864 /* 865 * Otherwise we queue the message for the target TC 866 * to pick up when he does a local_irq_restore() 867 */ 868 write_tc_c0_tchalt(0); 869 UNLOCK_CORE_PRA(); 870 IPIQ[cpu].resched_flag |= set_resched_flag; 871 smtc_ipi_nq(&IPIQ[cpu], pipi); 872 } else { 873postdirect: 874 post_direct_ipi(cpu, pipi); 875 write_tc_c0_tchalt(0); 876 UNLOCK_CORE_PRA(); 877 } 878 } 879} 880 881/* 882 * Send IPI message to Halted TC, TargTC/TargVPE already having been set 883 */ 884static void post_direct_ipi(int cpu, struct smtc_ipi *pipi) 885{ 886 struct pt_regs *kstack; 887 unsigned long tcstatus; 888 unsigned long tcrestart; 889 extern u32 kernelsp[NR_CPUS]; 890 extern void __smtc_ipi_vector(void); 891//printk("%s: on %d for %d\n", __func__, smp_processor_id(), cpu); 892 893 /* Extract Status, EPC from halted TC */ 894 tcstatus = read_tc_c0_tcstatus(); 895 tcrestart = read_tc_c0_tcrestart(); 896 /* If TCRestart indicates a WAIT instruction, advance the PC */ 897 if ((tcrestart & 0x80000000) 898 && ((*(unsigned int *)tcrestart & 0xfe00003f) == 0x42000020)) { 899 tcrestart += 4; 900 } 901 /* 902 * Save on TC's future kernel stack 903 * 904 * CU bit of Status is indicator that TC was 905 * already running on a kernel stack... 906 */ 907 if (tcstatus & ST0_CU0) { 908 /* Note that this "- 1" is pointer arithmetic */ 909 kstack = ((struct pt_regs *)read_tc_gpr_sp()) - 1; 910 } else { 911 kstack = ((struct pt_regs *)kernelsp[cpu]) - 1; 912 } 913 914 kstack->cp0_epc = (long)tcrestart; 915 /* Save TCStatus */ 916 kstack->cp0_tcstatus = tcstatus; 917 /* Pass token of operation to be performed kernel stack pad area */ 918 kstack->pad0[4] = (unsigned long)pipi; 919 /* Pass address of function to be called likewise */ 920 kstack->pad0[5] = (unsigned long)&ipi_decode; 921 /* Set interrupt exempt and kernel mode */ 922 tcstatus |= TCSTATUS_IXMT; 923 tcstatus &= ~TCSTATUS_TKSU; 924 write_tc_c0_tcstatus(tcstatus); 925 ehb(); 926 /* Set TC Restart address to be SMTC IPI vector */ 927 write_tc_c0_tcrestart(__smtc_ipi_vector); 928} 929 930static void ipi_resched_interrupt(void) 931{ 932 /* Return from interrupt should be enough to cause scheduler check */ 933} 934 935static void ipi_call_interrupt(void) 936{ 937 /* Invoke generic function invocation code in smp.c */ 938 smp_call_function_interrupt(); 939} 940 941DECLARE_PER_CPU(struct clock_event_device, mips_clockevent_device); 942 943static void __irq_entry smtc_clock_tick_interrupt(void) 944{ 945 unsigned int cpu = smp_processor_id(); 946 struct clock_event_device *cd; 947 int irq = MIPS_CPU_IRQ_BASE + 1; 948 949 irq_enter(); 950 kstat_incr_irqs_this_cpu(irq, irq_to_desc(irq)); 951 cd = &per_cpu(mips_clockevent_device, cpu); 952 cd->event_handler(cd); 953 irq_exit(); 954} 955 956void ipi_decode(struct smtc_ipi *pipi) 957{ 958 void *arg_copy = pipi->arg; 959 int type_copy = pipi->type; 960 961 smtc_ipi_nq(&freeIPIq, pipi); 962 963 switch (type_copy) { 964 case SMTC_CLOCK_TICK: 965 smtc_clock_tick_interrupt(); 966 break; 967 968 case LINUX_SMP_IPI: 969 switch ((int)arg_copy) { 970 case SMP_RESCHEDULE_YOURSELF: 971 ipi_resched_interrupt(); 972 break; 973 case SMP_CALL_FUNCTION: 974 ipi_call_interrupt(); 975 break; 976 default: 977 printk("Impossible SMTC IPI Argument %p\n", arg_copy); 978 break; 979 } 980 break; 981#ifdef CONFIG_MIPS_MT_SMTC_IRQAFF 982 case IRQ_AFFINITY_IPI: 983 /* 984 * Accept a "forwarded" interrupt that was initially 985 * taken by a TC who doesn't have affinity for the IRQ. 986 */ 987 do_IRQ_no_affinity((int)arg_copy); 988 break; 989#endif /* CONFIG_MIPS_MT_SMTC_IRQAFF */ 990 default: 991 printk("Impossible SMTC IPI Type 0x%x\n", type_copy); 992 break; 993 } 994} 995 996/* 997 * Similar to smtc_ipi_replay(), but invoked from context restore, 998 * so it reuses the current exception frame rather than set up a 999 * new one with self_ipi. 1000 */ 1001 1002void deferred_smtc_ipi(void) 1003{ 1004 int cpu = smp_processor_id(); 1005 1006 /* 1007 * Test is not atomic, but much faster than a dequeue, 1008 * and the vast majority of invocations will have a null queue. 1009 * If irq_disabled when this was called, then any IPIs queued 1010 * after we test last will be taken on the next irq_enable/restore. 1011 * If interrupts were enabled, then any IPIs added after the 1012 * last test will be taken directly. 1013 */ 1014 1015 while (IPIQ[cpu].head != NULL) { 1016 struct smtc_ipi_q *q = &IPIQ[cpu]; 1017 struct smtc_ipi *pipi; 1018 unsigned long flags; 1019 1020 /* 1021 * It may be possible we'll come in with interrupts 1022 * already enabled. 1023 */ 1024 local_irq_save(flags); 1025 spin_lock(&q->lock); 1026 pipi = __smtc_ipi_dq(q); 1027 spin_unlock(&q->lock); 1028 if (pipi != NULL) { 1029 if (pipi->type == LINUX_SMP_IPI && 1030 (int)pipi->arg == SMP_RESCHEDULE_YOURSELF) 1031 IPIQ[cpu].resched_flag = 0; 1032 ipi_decode(pipi); 1033 } 1034 /* 1035 * The use of the __raw_local restore isn't 1036 * as obviously necessary here as in smtc_ipi_replay(), 1037 * but it's more efficient, given that we're already 1038 * running down the IPI queue. 1039 */ 1040 __arch_local_irq_restore(flags); 1041 } 1042} 1043 1044/* 1045 * Cross-VPE interrupts in the SMTC prototype use "software interrupts" 1046 * set via cross-VPE MTTR manipulation of the Cause register. It would be 1047 * in some regards preferable to have external logic for "doorbell" hardware 1048 * interrupts. 1049 */ 1050 1051static int cpu_ipi_irq = MIPS_CPU_IRQ_BASE + MIPS_CPU_IPI_IRQ; 1052 1053static irqreturn_t ipi_interrupt(int irq, void *dev_idm) 1054{ 1055 int my_vpe = cpu_data[smp_processor_id()].vpe_id; 1056 int my_tc = cpu_data[smp_processor_id()].tc_id; 1057 int cpu; 1058 struct smtc_ipi *pipi; 1059 unsigned long tcstatus; 1060 int sent; 1061 unsigned long flags; 1062 unsigned int mtflags; 1063 unsigned int vpflags; 1064 1065 /* 1066 * So long as cross-VPE interrupts are done via 1067 * MFTR/MTTR read-modify-writes of Cause, we need 1068 * to stop other VPEs whenever the local VPE does 1069 * anything similar. 1070 */ 1071 local_irq_save(flags); 1072 vpflags = dvpe(); 1073 clear_c0_cause(0x100 << MIPS_CPU_IPI_IRQ); 1074 set_c0_status(0x100 << MIPS_CPU_IPI_IRQ); 1075 irq_enable_hazard(); 1076 evpe(vpflags); 1077 local_irq_restore(flags); 1078 1079 /* 1080 * Cross-VPE Interrupt handler: Try to directly deliver IPIs 1081 * queued for TCs on this VPE other than the current one. 1082 * Return-from-interrupt should cause us to drain the queue 1083 * for the current TC, so we ought not to have to do it explicitly here. 1084 */ 1085 1086 for_each_online_cpu(cpu) { 1087 if (cpu_data[cpu].vpe_id != my_vpe) 1088 continue; 1089 1090 pipi = smtc_ipi_dq(&IPIQ[cpu]); 1091 if (pipi != NULL) { 1092 if (cpu_data[cpu].tc_id != my_tc) { 1093 sent = 0; 1094 LOCK_MT_PRA(); 1095 settc(cpu_data[cpu].tc_id); 1096 write_tc_c0_tchalt(TCHALT_H); 1097 mips_ihb(); 1098 tcstatus = read_tc_c0_tcstatus(); 1099 if ((tcstatus & TCSTATUS_IXMT) == 0) { 1100 post_direct_ipi(cpu, pipi); 1101 sent = 1; 1102 } 1103 write_tc_c0_tchalt(0); 1104 UNLOCK_MT_PRA(); 1105 if (!sent) { 1106 smtc_ipi_req(&IPIQ[cpu], pipi); 1107 } 1108 } else { 1109 /* 1110 * ipi_decode() should be called 1111 * with interrupts off 1112 */ 1113 local_irq_save(flags); 1114 if (pipi->type == LINUX_SMP_IPI && 1115 (int)pipi->arg == SMP_RESCHEDULE_YOURSELF) 1116 IPIQ[cpu].resched_flag = 0; 1117 ipi_decode(pipi); 1118 local_irq_restore(flags); 1119 } 1120 } 1121 } 1122 1123 return IRQ_HANDLED; 1124} 1125 1126static void ipi_irq_dispatch(void) 1127{ 1128 do_IRQ(cpu_ipi_irq); 1129} 1130 1131static struct irqaction irq_ipi = { 1132 .handler = ipi_interrupt, 1133 .flags = IRQF_DISABLED | IRQF_PERCPU, 1134 .name = "SMTC_IPI" 1135}; 1136 1137static void setup_cross_vpe_interrupts(unsigned int nvpe) 1138{ 1139 if (nvpe < 1) 1140 return; 1141 1142 if (!cpu_has_vint) 1143 panic("SMTC Kernel requires Vectored Interrupt support"); 1144 1145 set_vi_handler(MIPS_CPU_IPI_IRQ, ipi_irq_dispatch); 1146 1147 setup_irq_smtc(cpu_ipi_irq, &irq_ipi, (0x100 << MIPS_CPU_IPI_IRQ)); 1148 1149 set_irq_handler(cpu_ipi_irq, handle_percpu_irq); 1150} 1151 1152/* 1153 * SMTC-specific hacks invoked from elsewhere in the kernel. 1154 */ 1155 1156 /* 1157 * smtc_ipi_replay is called from raw_local_irq_restore 1158 */ 1159 1160void smtc_ipi_replay(void) 1161{ 1162 unsigned int cpu = smp_processor_id(); 1163 1164 /* 1165 * To the extent that we've ever turned interrupts off, 1166 * we may have accumulated deferred IPIs. This is subtle. 1167 * we should be OK: If we pick up something and dispatch 1168 * it here, that's great. If we see nothing, but concurrent 1169 * with this operation, another TC sends us an IPI, IXMT 1170 * is clear, and we'll handle it as a real pseudo-interrupt 1171 * and not a pseudo-pseudo interrupt. The important thing 1172 * is to do the last check for queued message *after* the 1173 * re-enabling of interrupts. 1174 */ 1175 while (IPIQ[cpu].head != NULL) { 1176 struct smtc_ipi_q *q = &IPIQ[cpu]; 1177 struct smtc_ipi *pipi; 1178 unsigned long flags; 1179 1180 /* 1181 * It's just possible we'll come in with interrupts 1182 * already enabled. 1183 */ 1184 local_irq_save(flags); 1185 1186 spin_lock(&q->lock); 1187 pipi = __smtc_ipi_dq(q); 1188 spin_unlock(&q->lock); 1189 /* 1190 ** But use a raw restore here to avoid recursion. 1191 */ 1192 __arch_local_irq_restore(flags); 1193 1194 if (pipi) { 1195 self_ipi(pipi); 1196 smtc_cpu_stats[cpu].selfipis++; 1197 } 1198 } 1199} 1200 1201EXPORT_SYMBOL(smtc_ipi_replay); 1202 1203void smtc_idle_loop_hook(void) 1204{ 1205#ifdef CONFIG_SMTC_IDLE_HOOK_DEBUG 1206 int im; 1207 int flags; 1208 int mtflags; 1209 int bit; 1210 int vpe; 1211 int tc; 1212 int hook_ntcs; 1213 /* 1214 * printk within DMT-protected regions can deadlock, 1215 * so buffer diagnostic messages for later output. 1216 */ 1217 char *pdb_msg; 1218 char id_ho_db_msg[768]; /* worst-case use should be less than 700 */ 1219 1220 if (atomic_read(&idle_hook_initialized) == 0) { /* fast test */ 1221 if (atomic_add_return(1, &idle_hook_initialized) == 1) { 1222 int mvpconf0; 1223 /* Tedious stuff to just do once */ 1224 mvpconf0 = read_c0_mvpconf0(); 1225 hook_ntcs = ((mvpconf0 & MVPCONF0_PTC) >> MVPCONF0_PTC_SHIFT) + 1; 1226 if (hook_ntcs > NR_CPUS) 1227 hook_ntcs = NR_CPUS; 1228 for (tc = 0; tc < hook_ntcs; tc++) { 1229 tcnoprog[tc] = 0; 1230 clock_hang_reported[tc] = 0; 1231 } 1232 for (vpe = 0; vpe < 2; vpe++) 1233 for (im = 0; im < 8; im++) 1234 imstuckcount[vpe][im] = 0; 1235 printk("Idle loop test hook initialized for %d TCs\n", hook_ntcs); 1236 atomic_set(&idle_hook_initialized, 1000); 1237 } else { 1238 /* Someone else is initializing in parallel - let 'em finish */ 1239 while (atomic_read(&idle_hook_initialized) < 1000) 1240 ; 1241 } 1242 } 1243 1244 /* Have we stupidly left IXMT set somewhere? */ 1245 if (read_c0_tcstatus() & 0x400) { 1246 write_c0_tcstatus(read_c0_tcstatus() & ~0x400); 1247 ehb(); 1248 printk("Dangling IXMT in cpu_idle()\n"); 1249 } 1250 1251 /* Have we stupidly left an IM bit turned off? */ 1252#define IM_LIMIT 2000 1253 local_irq_save(flags); 1254 mtflags = dmt(); 1255 pdb_msg = &id_ho_db_msg[0]; 1256 im = read_c0_status(); 1257 vpe = current_cpu_data.vpe_id; 1258 for (bit = 0; bit < 8; bit++) { 1259 /* 1260 * In current prototype, I/O interrupts 1261 * are masked for VPE > 0 1262 */ 1263 if (vpemask[vpe][bit]) { 1264 if (!(im & (0x100 << bit))) 1265 imstuckcount[vpe][bit]++; 1266 else 1267 imstuckcount[vpe][bit] = 0; 1268 if (imstuckcount[vpe][bit] > IM_LIMIT) { 1269 set_c0_status(0x100 << bit); 1270 ehb(); 1271 imstuckcount[vpe][bit] = 0; 1272 pdb_msg += sprintf(pdb_msg, 1273 "Dangling IM %d fixed for VPE %d\n", bit, 1274 vpe); 1275 } 1276 } 1277 } 1278 1279 emt(mtflags); 1280 local_irq_restore(flags); 1281 if (pdb_msg != &id_ho_db_msg[0]) 1282 printk("CPU%d: %s", smp_processor_id(), id_ho_db_msg); 1283#endif /* CONFIG_SMTC_IDLE_HOOK_DEBUG */ 1284 1285 smtc_ipi_replay(); 1286} 1287 1288void smtc_soft_dump(void) 1289{ 1290 int i; 1291 1292 printk("Counter Interrupts taken per CPU (TC)\n"); 1293 for (i=0; i < NR_CPUS; i++) { 1294 printk("%d: %ld\n", i, smtc_cpu_stats[i].timerints); 1295 } 1296 printk("Self-IPI invocations:\n"); 1297 for (i=0; i < NR_CPUS; i++) { 1298 printk("%d: %ld\n", i, smtc_cpu_stats[i].selfipis); 1299 } 1300 smtc_ipi_qdump(); 1301 printk("%d Recoveries of \"stolen\" FPU\n", 1302 atomic_read(&smtc_fpu_recoveries)); 1303} 1304 1305 1306/* 1307 * TLB management routines special to SMTC 1308 */ 1309 1310void smtc_get_new_mmu_context(struct mm_struct *mm, unsigned long cpu) 1311{ 1312 unsigned long flags, mtflags, tcstat, prevhalt, asid; 1313 int tlb, i; 1314 1315 /* 1316 * It would be nice to be able to use a spinlock here, 1317 * but this is invoked from within TLB flush routines 1318 * that protect themselves with DVPE, so if a lock is 1319 * held by another TC, it'll never be freed. 1320 * 1321 * DVPE/DMT must not be done with interrupts enabled, 1322 * so even so most callers will already have disabled 1323 * them, let's be really careful... 1324 */ 1325 1326 local_irq_save(flags); 1327 if (smtc_status & SMTC_TLB_SHARED) { 1328 mtflags = dvpe(); 1329 tlb = 0; 1330 } else { 1331 mtflags = dmt(); 1332 tlb = cpu_data[cpu].vpe_id; 1333 } 1334 asid = asid_cache(cpu); 1335 1336 do { 1337 if (!((asid += ASID_INC) & ASID_MASK) ) { 1338 if (cpu_has_vtag_icache) 1339 flush_icache_all(); 1340 /* Traverse all online CPUs (hack requires contiguous range) */ 1341 for_each_online_cpu(i) { 1342 /* 1343 * We don't need to worry about our own CPU, nor those of 1344 * CPUs who don't share our TLB. 1345 */ 1346 if ((i != smp_processor_id()) && 1347 ((smtc_status & SMTC_TLB_SHARED) || 1348 (cpu_data[i].vpe_id == cpu_data[cpu].vpe_id))) { 1349 settc(cpu_data[i].tc_id); 1350 prevhalt = read_tc_c0_tchalt() & TCHALT_H; 1351 if (!prevhalt) { 1352 write_tc_c0_tchalt(TCHALT_H); 1353 mips_ihb(); 1354 } 1355 tcstat = read_tc_c0_tcstatus(); 1356 smtc_live_asid[tlb][(tcstat & ASID_MASK)] |= (asiduse)(0x1 << i); 1357 if (!prevhalt) 1358 write_tc_c0_tchalt(0); 1359 } 1360 } 1361 if (!asid) /* fix version if needed */ 1362 asid = ASID_FIRST_VERSION; 1363 local_flush_tlb_all(); /* start new asid cycle */ 1364 } 1365 } while (smtc_live_asid[tlb][(asid & ASID_MASK)]); 1366 1367 /* 1368 * SMTC shares the TLB within VPEs and possibly across all VPEs. 1369 */ 1370 for_each_online_cpu(i) { 1371 if ((smtc_status & SMTC_TLB_SHARED) || 1372 (cpu_data[i].vpe_id == cpu_data[cpu].vpe_id)) 1373 cpu_context(i, mm) = asid_cache(i) = asid; 1374 } 1375 1376 if (smtc_status & SMTC_TLB_SHARED) 1377 evpe(mtflags); 1378 else 1379 emt(mtflags); 1380 local_irq_restore(flags); 1381} 1382 1383/* 1384 * Invoked from macros defined in mmu_context.h 1385 * which must already have disabled interrupts 1386 * and done a DVPE or DMT as appropriate. 1387 */ 1388 1389void smtc_flush_tlb_asid(unsigned long asid) 1390{ 1391 int entry; 1392 unsigned long ehi; 1393 1394 entry = read_c0_wired(); 1395 1396 /* Traverse all non-wired entries */ 1397 while (entry < current_cpu_data.tlbsize) { 1398 write_c0_index(entry); 1399 ehb(); 1400 tlb_read(); 1401 ehb(); 1402 ehi = read_c0_entryhi(); 1403 if ((ehi & ASID_MASK) == asid) { 1404 /* 1405 * Invalidate only entries with specified ASID, 1406 * makiing sure all entries differ. 1407 */ 1408 write_c0_entryhi(CKSEG0 + (entry << (PAGE_SHIFT + 1))); 1409 write_c0_entrylo0(0); 1410 write_c0_entrylo1(0); 1411 mtc0_tlbw_hazard(); 1412 tlb_write_indexed(); 1413 } 1414 entry++; 1415 } 1416 write_c0_index(PARKED_INDEX); 1417 tlbw_use_hazard(); 1418} 1419 1420/* 1421 * Support for single-threading cache flush operations. 1422 */ 1423 1424static int halt_state_save[NR_CPUS]; 1425 1426/* 1427 * To really, really be sure that nothing is being done 1428 * by other TCs, halt them all. This code assumes that 1429 * a DVPE has already been done, so while their Halted 1430 * state is theoretically architecturally unstable, in 1431 * practice, it's not going to change while we're looking 1432 * at it. 1433 */ 1434 1435void smtc_cflush_lockdown(void) 1436{ 1437 int cpu; 1438 1439 for_each_online_cpu(cpu) { 1440 if (cpu != smp_processor_id()) { 1441 settc(cpu_data[cpu].tc_id); 1442 halt_state_save[cpu] = read_tc_c0_tchalt(); 1443 write_tc_c0_tchalt(TCHALT_H); 1444 } 1445 } 1446 mips_ihb(); 1447} 1448 1449/* It would be cheating to change the cpu_online states during a flush! */ 1450 1451void smtc_cflush_release(void) 1452{ 1453 int cpu; 1454 1455 /* 1456 * Start with a hazard barrier to ensure 1457 * that all CACHE ops have played through. 1458 */ 1459 mips_ihb(); 1460 1461 for_each_online_cpu(cpu) { 1462 if (cpu != smp_processor_id()) { 1463 settc(cpu_data[cpu].tc_id); 1464 write_tc_c0_tchalt(halt_state_save[cpu]); 1465 } 1466 } 1467 mips_ihb(); 1468} 1469