lpar.c revision f5339277eb8d3aed37f12a27988366f68ab68930
1/* 2 * pSeries_lpar.c 3 * Copyright (C) 2001 Todd Inglett, IBM Corporation 4 * 5 * pSeries LPAR support. 6 * 7 * This program is free software; you can redistribute it and/or modify 8 * it under the terms of the GNU General Public License as published by 9 * the Free Software Foundation; either version 2 of the License, or 10 * (at your option) any later version. 11 * 12 * This program is distributed in the hope that it will be useful, 13 * but WITHOUT ANY WARRANTY; without even the implied warranty of 14 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 15 * GNU General Public License for more details. 16 * 17 * You should have received a copy of the GNU General Public License 18 * along with this program; if not, write to the Free Software 19 * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA 20 */ 21 22/* Enables debugging of low-level hash table routines - careful! */ 23#undef DEBUG 24 25#include <linux/kernel.h> 26#include <linux/dma-mapping.h> 27#include <linux/console.h> 28#include <linux/export.h> 29#include <asm/processor.h> 30#include <asm/mmu.h> 31#include <asm/page.h> 32#include <asm/pgtable.h> 33#include <asm/machdep.h> 34#include <asm/abs_addr.h> 35#include <asm/mmu_context.h> 36#include <asm/iommu.h> 37#include <asm/tlbflush.h> 38#include <asm/tlb.h> 39#include <asm/prom.h> 40#include <asm/cputable.h> 41#include <asm/udbg.h> 42#include <asm/smp.h> 43#include <asm/trace.h> 44#include <asm/firmware.h> 45 46#include "plpar_wrappers.h" 47#include "pseries.h" 48 49 50/* in hvCall.S */ 51EXPORT_SYMBOL(plpar_hcall); 52EXPORT_SYMBOL(plpar_hcall9); 53EXPORT_SYMBOL(plpar_hcall_norets); 54 55extern void pSeries_find_serial_port(void); 56 57void vpa_init(int cpu) 58{ 59 int hwcpu = get_hard_smp_processor_id(cpu); 60 unsigned long addr; 61 long ret; 62 struct paca_struct *pp; 63 struct dtl_entry *dtl; 64 65 if (cpu_has_feature(CPU_FTR_ALTIVEC)) 66 lppaca_of(cpu).vmxregs_in_use = 1; 67 68 addr = __pa(&lppaca_of(cpu)); 69 ret = register_vpa(hwcpu, addr); 70 71 if (ret) { 72 pr_err("WARNING: VPA registration for cpu %d (hw %d) of area " 73 "%lx failed with %ld\n", cpu, hwcpu, addr, ret); 74 return; 75 } 76 /* 77 * PAPR says this feature is SLB-Buffer but firmware never 78 * reports that. All SPLPAR support SLB shadow buffer. 79 */ 80 addr = __pa(&slb_shadow[cpu]); 81 if (firmware_has_feature(FW_FEATURE_SPLPAR)) { 82 ret = register_slb_shadow(hwcpu, addr); 83 if (ret) 84 pr_err("WARNING: SLB shadow buffer registration for " 85 "cpu %d (hw %d) of area %lx failed with %ld\n", 86 cpu, hwcpu, addr, ret); 87 } 88 89 /* 90 * Register dispatch trace log, if one has been allocated. 91 */ 92 pp = &paca[cpu]; 93 dtl = pp->dispatch_log; 94 if (dtl) { 95 pp->dtl_ridx = 0; 96 pp->dtl_curr = dtl; 97 lppaca_of(cpu).dtl_idx = 0; 98 99 /* hypervisor reads buffer length from this field */ 100 dtl->enqueue_to_dispatch_time = DISPATCH_LOG_BYTES; 101 ret = register_dtl(hwcpu, __pa(dtl)); 102 if (ret) 103 pr_err("WARNING: DTL registration of cpu %d (hw %d) " 104 "failed with %ld\n", smp_processor_id(), 105 hwcpu, ret); 106 lppaca_of(cpu).dtl_enable_mask = 2; 107 } 108} 109 110static long pSeries_lpar_hpte_insert(unsigned long hpte_group, 111 unsigned long va, unsigned long pa, 112 unsigned long rflags, unsigned long vflags, 113 int psize, int ssize) 114{ 115 unsigned long lpar_rc; 116 unsigned long flags; 117 unsigned long slot; 118 unsigned long hpte_v, hpte_r; 119 120 if (!(vflags & HPTE_V_BOLTED)) 121 pr_devel("hpte_insert(group=%lx, va=%016lx, pa=%016lx, " 122 "rflags=%lx, vflags=%lx, psize=%d)\n", 123 hpte_group, va, pa, rflags, vflags, psize); 124 125 hpte_v = hpte_encode_v(va, psize, ssize) | vflags | HPTE_V_VALID; 126 hpte_r = hpte_encode_r(pa, psize) | rflags; 127 128 if (!(vflags & HPTE_V_BOLTED)) 129 pr_devel(" hpte_v=%016lx, hpte_r=%016lx\n", hpte_v, hpte_r); 130 131 /* Now fill in the actual HPTE */ 132 /* Set CEC cookie to 0 */ 133 /* Zero page = 0 */ 134 /* I-cache Invalidate = 0 */ 135 /* I-cache synchronize = 0 */ 136 /* Exact = 0 */ 137 flags = 0; 138 139 /* Make pHyp happy */ 140 if ((rflags & _PAGE_NO_CACHE) & !(rflags & _PAGE_WRITETHRU)) 141 hpte_r &= ~_PAGE_COHERENT; 142 if (firmware_has_feature(FW_FEATURE_XCMO) && !(hpte_r & HPTE_R_N)) 143 flags |= H_COALESCE_CAND; 144 145 lpar_rc = plpar_pte_enter(flags, hpte_group, hpte_v, hpte_r, &slot); 146 if (unlikely(lpar_rc == H_PTEG_FULL)) { 147 if (!(vflags & HPTE_V_BOLTED)) 148 pr_devel(" full\n"); 149 return -1; 150 } 151 152 /* 153 * Since we try and ioremap PHBs we don't own, the pte insert 154 * will fail. However we must catch the failure in hash_page 155 * or we will loop forever, so return -2 in this case. 156 */ 157 if (unlikely(lpar_rc != H_SUCCESS)) { 158 if (!(vflags & HPTE_V_BOLTED)) 159 pr_devel(" lpar err %lu\n", lpar_rc); 160 return -2; 161 } 162 if (!(vflags & HPTE_V_BOLTED)) 163 pr_devel(" -> slot: %lu\n", slot & 7); 164 165 /* Because of iSeries, we have to pass down the secondary 166 * bucket bit here as well 167 */ 168 return (slot & 7) | (!!(vflags & HPTE_V_SECONDARY) << 3); 169} 170 171static DEFINE_SPINLOCK(pSeries_lpar_tlbie_lock); 172 173static long pSeries_lpar_hpte_remove(unsigned long hpte_group) 174{ 175 unsigned long slot_offset; 176 unsigned long lpar_rc; 177 int i; 178 unsigned long dummy1, dummy2; 179 180 /* pick a random slot to start at */ 181 slot_offset = mftb() & 0x7; 182 183 for (i = 0; i < HPTES_PER_GROUP; i++) { 184 185 /* don't remove a bolted entry */ 186 lpar_rc = plpar_pte_remove(H_ANDCOND, hpte_group + slot_offset, 187 (0x1UL << 4), &dummy1, &dummy2); 188 if (lpar_rc == H_SUCCESS) 189 return i; 190 BUG_ON(lpar_rc != H_NOT_FOUND); 191 192 slot_offset++; 193 slot_offset &= 0x7; 194 } 195 196 return -1; 197} 198 199static void pSeries_lpar_hptab_clear(void) 200{ 201 unsigned long size_bytes = 1UL << ppc64_pft_size; 202 unsigned long hpte_count = size_bytes >> 4; 203 struct { 204 unsigned long pteh; 205 unsigned long ptel; 206 } ptes[4]; 207 long lpar_rc; 208 unsigned long i, j; 209 210 /* Read in batches of 4, 211 * invalidate only valid entries not in the VRMA 212 * hpte_count will be a multiple of 4 213 */ 214 for (i = 0; i < hpte_count; i += 4) { 215 lpar_rc = plpar_pte_read_4_raw(0, i, (void *)ptes); 216 if (lpar_rc != H_SUCCESS) 217 continue; 218 for (j = 0; j < 4; j++){ 219 if ((ptes[j].pteh & HPTE_V_VRMA_MASK) == 220 HPTE_V_VRMA_MASK) 221 continue; 222 if (ptes[j].pteh & HPTE_V_VALID) 223 plpar_pte_remove_raw(0, i + j, 0, 224 &(ptes[j].pteh), &(ptes[j].ptel)); 225 } 226 } 227} 228 229/* 230 * This computes the AVPN and B fields of the first dword of a HPTE, 231 * for use when we want to match an existing PTE. The bottom 7 bits 232 * of the returned value are zero. 233 */ 234static inline unsigned long hpte_encode_avpn(unsigned long va, int psize, 235 int ssize) 236{ 237 unsigned long v; 238 239 v = (va >> 23) & ~(mmu_psize_defs[psize].avpnm); 240 v <<= HPTE_V_AVPN_SHIFT; 241 v |= ((unsigned long) ssize) << HPTE_V_SSIZE_SHIFT; 242 return v; 243} 244 245/* 246 * NOTE: for updatepp ops we are fortunate that the linux "newpp" bits and 247 * the low 3 bits of flags happen to line up. So no transform is needed. 248 * We can probably optimize here and assume the high bits of newpp are 249 * already zero. For now I am paranoid. 250 */ 251static long pSeries_lpar_hpte_updatepp(unsigned long slot, 252 unsigned long newpp, 253 unsigned long va, 254 int psize, int ssize, int local) 255{ 256 unsigned long lpar_rc; 257 unsigned long flags = (newpp & 7) | H_AVPN; 258 unsigned long want_v; 259 260 want_v = hpte_encode_avpn(va, psize, ssize); 261 262 pr_devel(" update: avpnv=%016lx, hash=%016lx, f=%lx, psize: %d ...", 263 want_v, slot, flags, psize); 264 265 lpar_rc = plpar_pte_protect(flags, slot, want_v); 266 267 if (lpar_rc == H_NOT_FOUND) { 268 pr_devel("not found !\n"); 269 return -1; 270 } 271 272 pr_devel("ok\n"); 273 274 BUG_ON(lpar_rc != H_SUCCESS); 275 276 return 0; 277} 278 279static unsigned long pSeries_lpar_hpte_getword0(unsigned long slot) 280{ 281 unsigned long dword0; 282 unsigned long lpar_rc; 283 unsigned long dummy_word1; 284 unsigned long flags; 285 286 /* Read 1 pte at a time */ 287 /* Do not need RPN to logical page translation */ 288 /* No cross CEC PFT access */ 289 flags = 0; 290 291 lpar_rc = plpar_pte_read(flags, slot, &dword0, &dummy_word1); 292 293 BUG_ON(lpar_rc != H_SUCCESS); 294 295 return dword0; 296} 297 298static long pSeries_lpar_hpte_find(unsigned long va, int psize, int ssize) 299{ 300 unsigned long hash; 301 unsigned long i; 302 long slot; 303 unsigned long want_v, hpte_v; 304 305 hash = hpt_hash(va, mmu_psize_defs[psize].shift, ssize); 306 want_v = hpte_encode_avpn(va, psize, ssize); 307 308 /* Bolted entries are always in the primary group */ 309 slot = (hash & htab_hash_mask) * HPTES_PER_GROUP; 310 for (i = 0; i < HPTES_PER_GROUP; i++) { 311 hpte_v = pSeries_lpar_hpte_getword0(slot); 312 313 if (HPTE_V_COMPARE(hpte_v, want_v) && (hpte_v & HPTE_V_VALID)) 314 /* HPTE matches */ 315 return slot; 316 ++slot; 317 } 318 319 return -1; 320} 321 322static void pSeries_lpar_hpte_updateboltedpp(unsigned long newpp, 323 unsigned long ea, 324 int psize, int ssize) 325{ 326 unsigned long lpar_rc, slot, vsid, va, flags; 327 328 vsid = get_kernel_vsid(ea, ssize); 329 va = hpt_va(ea, vsid, ssize); 330 331 slot = pSeries_lpar_hpte_find(va, psize, ssize); 332 BUG_ON(slot == -1); 333 334 flags = newpp & 7; 335 lpar_rc = plpar_pte_protect(flags, slot, 0); 336 337 BUG_ON(lpar_rc != H_SUCCESS); 338} 339 340static void pSeries_lpar_hpte_invalidate(unsigned long slot, unsigned long va, 341 int psize, int ssize, int local) 342{ 343 unsigned long want_v; 344 unsigned long lpar_rc; 345 unsigned long dummy1, dummy2; 346 347 pr_devel(" inval : slot=%lx, va=%016lx, psize: %d, local: %d\n", 348 slot, va, psize, local); 349 350 want_v = hpte_encode_avpn(va, psize, ssize); 351 lpar_rc = plpar_pte_remove(H_AVPN, slot, want_v, &dummy1, &dummy2); 352 if (lpar_rc == H_NOT_FOUND) 353 return; 354 355 BUG_ON(lpar_rc != H_SUCCESS); 356} 357 358static void pSeries_lpar_hpte_removebolted(unsigned long ea, 359 int psize, int ssize) 360{ 361 unsigned long slot, vsid, va; 362 363 vsid = get_kernel_vsid(ea, ssize); 364 va = hpt_va(ea, vsid, ssize); 365 366 slot = pSeries_lpar_hpte_find(va, psize, ssize); 367 BUG_ON(slot == -1); 368 369 pSeries_lpar_hpte_invalidate(slot, va, psize, ssize, 0); 370} 371 372/* Flag bits for H_BULK_REMOVE */ 373#define HBR_REQUEST 0x4000000000000000UL 374#define HBR_RESPONSE 0x8000000000000000UL 375#define HBR_END 0xc000000000000000UL 376#define HBR_AVPN 0x0200000000000000UL 377#define HBR_ANDCOND 0x0100000000000000UL 378 379/* 380 * Take a spinlock around flushes to avoid bouncing the hypervisor tlbie 381 * lock. 382 */ 383static void pSeries_lpar_flush_hash_range(unsigned long number, int local) 384{ 385 unsigned long i, pix, rc; 386 unsigned long flags = 0; 387 struct ppc64_tlb_batch *batch = &__get_cpu_var(ppc64_tlb_batch); 388 int lock_tlbie = !mmu_has_feature(MMU_FTR_LOCKLESS_TLBIE); 389 unsigned long param[9]; 390 unsigned long va; 391 unsigned long hash, index, shift, hidx, slot; 392 real_pte_t pte; 393 int psize, ssize; 394 395 if (lock_tlbie) 396 spin_lock_irqsave(&pSeries_lpar_tlbie_lock, flags); 397 398 psize = batch->psize; 399 ssize = batch->ssize; 400 pix = 0; 401 for (i = 0; i < number; i++) { 402 va = batch->vaddr[i]; 403 pte = batch->pte[i]; 404 pte_iterate_hashed_subpages(pte, psize, va, index, shift) { 405 hash = hpt_hash(va, shift, ssize); 406 hidx = __rpte_to_hidx(pte, index); 407 if (hidx & _PTEIDX_SECONDARY) 408 hash = ~hash; 409 slot = (hash & htab_hash_mask) * HPTES_PER_GROUP; 410 slot += hidx & _PTEIDX_GROUP_IX; 411 if (!firmware_has_feature(FW_FEATURE_BULK_REMOVE)) { 412 pSeries_lpar_hpte_invalidate(slot, va, psize, 413 ssize, local); 414 } else { 415 param[pix] = HBR_REQUEST | HBR_AVPN | slot; 416 param[pix+1] = hpte_encode_avpn(va, psize, 417 ssize); 418 pix += 2; 419 if (pix == 8) { 420 rc = plpar_hcall9(H_BULK_REMOVE, param, 421 param[0], param[1], param[2], 422 param[3], param[4], param[5], 423 param[6], param[7]); 424 BUG_ON(rc != H_SUCCESS); 425 pix = 0; 426 } 427 } 428 } pte_iterate_hashed_end(); 429 } 430 if (pix) { 431 param[pix] = HBR_END; 432 rc = plpar_hcall9(H_BULK_REMOVE, param, param[0], param[1], 433 param[2], param[3], param[4], param[5], 434 param[6], param[7]); 435 BUG_ON(rc != H_SUCCESS); 436 } 437 438 if (lock_tlbie) 439 spin_unlock_irqrestore(&pSeries_lpar_tlbie_lock, flags); 440} 441 442static int __init disable_bulk_remove(char *str) 443{ 444 if (strcmp(str, "off") == 0 && 445 firmware_has_feature(FW_FEATURE_BULK_REMOVE)) { 446 printk(KERN_INFO "Disabling BULK_REMOVE firmware feature"); 447 powerpc_firmware_features &= ~FW_FEATURE_BULK_REMOVE; 448 } 449 return 1; 450} 451 452__setup("bulk_remove=", disable_bulk_remove); 453 454void __init hpte_init_lpar(void) 455{ 456 ppc_md.hpte_invalidate = pSeries_lpar_hpte_invalidate; 457 ppc_md.hpte_updatepp = pSeries_lpar_hpte_updatepp; 458 ppc_md.hpte_updateboltedpp = pSeries_lpar_hpte_updateboltedpp; 459 ppc_md.hpte_insert = pSeries_lpar_hpte_insert; 460 ppc_md.hpte_remove = pSeries_lpar_hpte_remove; 461 ppc_md.hpte_removebolted = pSeries_lpar_hpte_removebolted; 462 ppc_md.flush_hash_range = pSeries_lpar_flush_hash_range; 463 ppc_md.hpte_clear_all = pSeries_lpar_hptab_clear; 464} 465 466#ifdef CONFIG_PPC_SMLPAR 467#define CMO_FREE_HINT_DEFAULT 1 468static int cmo_free_hint_flag = CMO_FREE_HINT_DEFAULT; 469 470static int __init cmo_free_hint(char *str) 471{ 472 char *parm; 473 parm = strstrip(str); 474 475 if (strcasecmp(parm, "no") == 0 || strcasecmp(parm, "off") == 0) { 476 printk(KERN_INFO "cmo_free_hint: CMO free page hinting is not active.\n"); 477 cmo_free_hint_flag = 0; 478 return 1; 479 } 480 481 cmo_free_hint_flag = 1; 482 printk(KERN_INFO "cmo_free_hint: CMO free page hinting is active.\n"); 483 484 if (strcasecmp(parm, "yes") == 0 || strcasecmp(parm, "on") == 0) 485 return 1; 486 487 return 0; 488} 489 490__setup("cmo_free_hint=", cmo_free_hint); 491 492static void pSeries_set_page_state(struct page *page, int order, 493 unsigned long state) 494{ 495 int i, j; 496 unsigned long cmo_page_sz, addr; 497 498 cmo_page_sz = cmo_get_page_size(); 499 addr = __pa((unsigned long)page_address(page)); 500 501 for (i = 0; i < (1 << order); i++, addr += PAGE_SIZE) { 502 for (j = 0; j < PAGE_SIZE; j += cmo_page_sz) 503 plpar_hcall_norets(H_PAGE_INIT, state, addr + j, 0); 504 } 505} 506 507void arch_free_page(struct page *page, int order) 508{ 509 if (!cmo_free_hint_flag || !firmware_has_feature(FW_FEATURE_CMO)) 510 return; 511 512 pSeries_set_page_state(page, order, H_PAGE_SET_UNUSED); 513} 514EXPORT_SYMBOL(arch_free_page); 515 516#endif 517 518#ifdef CONFIG_TRACEPOINTS 519/* 520 * We optimise our hcall path by placing hcall_tracepoint_refcount 521 * directly in the TOC so we can check if the hcall tracepoints are 522 * enabled via a single load. 523 */ 524 525/* NB: reg/unreg are called while guarded with the tracepoints_mutex */ 526extern long hcall_tracepoint_refcount; 527 528/* 529 * Since the tracing code might execute hcalls we need to guard against 530 * recursion. One example of this are spinlocks calling H_YIELD on 531 * shared processor partitions. 532 */ 533static DEFINE_PER_CPU(unsigned int, hcall_trace_depth); 534 535void hcall_tracepoint_regfunc(void) 536{ 537 hcall_tracepoint_refcount++; 538} 539 540void hcall_tracepoint_unregfunc(void) 541{ 542 hcall_tracepoint_refcount--; 543} 544 545void __trace_hcall_entry(unsigned long opcode, unsigned long *args) 546{ 547 unsigned long flags; 548 unsigned int *depth; 549 550 /* 551 * We cannot call tracepoints inside RCU idle regions which 552 * means we must not trace H_CEDE. 553 */ 554 if (opcode == H_CEDE) 555 return; 556 557 local_irq_save(flags); 558 559 depth = &__get_cpu_var(hcall_trace_depth); 560 561 if (*depth) 562 goto out; 563 564 (*depth)++; 565 preempt_disable(); 566 trace_hcall_entry(opcode, args); 567 (*depth)--; 568 569out: 570 local_irq_restore(flags); 571} 572 573void __trace_hcall_exit(long opcode, unsigned long retval, 574 unsigned long *retbuf) 575{ 576 unsigned long flags; 577 unsigned int *depth; 578 579 if (opcode == H_CEDE) 580 return; 581 582 local_irq_save(flags); 583 584 depth = &__get_cpu_var(hcall_trace_depth); 585 586 if (*depth) 587 goto out; 588 589 (*depth)++; 590 trace_hcall_exit(opcode, retval, retbuf); 591 preempt_enable(); 592 (*depth)--; 593 594out: 595 local_irq_restore(flags); 596} 597#endif 598 599/** 600 * h_get_mpp 601 * H_GET_MPP hcall returns info in 7 parms 602 */ 603int h_get_mpp(struct hvcall_mpp_data *mpp_data) 604{ 605 int rc; 606 unsigned long retbuf[PLPAR_HCALL9_BUFSIZE]; 607 608 rc = plpar_hcall9(H_GET_MPP, retbuf); 609 610 mpp_data->entitled_mem = retbuf[0]; 611 mpp_data->mapped_mem = retbuf[1]; 612 613 mpp_data->group_num = (retbuf[2] >> 2 * 8) & 0xffff; 614 mpp_data->pool_num = retbuf[2] & 0xffff; 615 616 mpp_data->mem_weight = (retbuf[3] >> 7 * 8) & 0xff; 617 mpp_data->unallocated_mem_weight = (retbuf[3] >> 6 * 8) & 0xff; 618 mpp_data->unallocated_entitlement = retbuf[3] & 0xffffffffffff; 619 620 mpp_data->pool_size = retbuf[4]; 621 mpp_data->loan_request = retbuf[5]; 622 mpp_data->backing_mem = retbuf[6]; 623 624 return rc; 625} 626EXPORT_SYMBOL(h_get_mpp); 627 628int h_get_mpp_x(struct hvcall_mpp_x_data *mpp_x_data) 629{ 630 int rc; 631 unsigned long retbuf[PLPAR_HCALL9_BUFSIZE] = { 0 }; 632 633 rc = plpar_hcall9(H_GET_MPP_X, retbuf); 634 635 mpp_x_data->coalesced_bytes = retbuf[0]; 636 mpp_x_data->pool_coalesced_bytes = retbuf[1]; 637 mpp_x_data->pool_purr_cycles = retbuf[2]; 638 mpp_x_data->pool_spurr_cycles = retbuf[3]; 639 640 return rc; 641} 642