paravirt_patch.c revision bf7ab02f620c1020c869fc71a2c855918b6a5375
1/****************************************************************************** 2 * linux/arch/ia64/xen/paravirt_patch.c 3 * 4 * Copyright (c) 2008 Isaku Yamahata <yamahata at valinux co jp> 5 * VA Linux Systems Japan K.K. 6 * 7 * This program is free software; you can redistribute it and/or modify 8 * it under the terms of the GNU General Public License as published by 9 * the Free Software Foundation; either version 2 of the License, or 10 * (at your option) any later version. 11 * 12 * This program is distributed in the hope that it will be useful, 13 * but WITHOUT ANY WARRANTY; without even the implied warranty of 14 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 15 * GNU General Public License for more details. 16 * 17 * You should have received a copy of the GNU General Public License 18 * along with this program; if not, write to the Free Software 19 * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA 20 * 21 */ 22 23#include <linux/init.h> 24#include <asm/intrinsics.h> 25#include <asm/kprobes.h> 26#include <asm/paravirt.h> 27#include <asm/paravirt_patch.h> 28 29typedef union ia64_inst { 30 struct { 31 unsigned long long qp : 6; 32 unsigned long long : 31; 33 unsigned long long opcode : 4; 34 unsigned long long reserved : 23; 35 } generic; 36 unsigned long long l; 37} ia64_inst_t; 38 39/* 40 * flush_icache_range() can't be used here. 41 * we are here before cpu_init() which initializes 42 * ia64_i_cache_stride_shift. flush_icache_range() uses it. 43 */ 44void __init_or_module 45paravirt_flush_i_cache_range(const void *instr, unsigned long size) 46{ 47 extern void paravirt_fc_i(const void *addr); 48 unsigned long i; 49 50 for (i = 0; i < size; i += sizeof(bundle_t)) 51 paravirt_fc_i(instr + i); 52} 53 54bundle_t* __init_or_module 55paravirt_get_bundle(unsigned long tag) 56{ 57 return (bundle_t *)(tag & ~3UL); 58} 59 60unsigned long __init_or_module 61paravirt_get_slot(unsigned long tag) 62{ 63 return tag & 3UL; 64} 65 66unsigned long __init_or_module 67paravirt_get_num_inst(unsigned long stag, unsigned long etag) 68{ 69 bundle_t *sbundle = paravirt_get_bundle(stag); 70 unsigned long sslot = paravirt_get_slot(stag); 71 bundle_t *ebundle = paravirt_get_bundle(etag); 72 unsigned long eslot = paravirt_get_slot(etag); 73 74 return (ebundle - sbundle) * 3 + eslot - sslot + 1; 75} 76 77unsigned long __init_or_module 78paravirt_get_next_tag(unsigned long tag) 79{ 80 unsigned long slot = paravirt_get_slot(tag); 81 82 switch (slot) { 83 case 0: 84 case 1: 85 return tag + 1; 86 case 2: { 87 bundle_t *bundle = paravirt_get_bundle(tag); 88 return (unsigned long)(bundle + 1); 89 } 90 default: 91 BUG(); 92 } 93 /* NOTREACHED */ 94} 95 96ia64_inst_t __init_or_module 97paravirt_read_slot0(const bundle_t *bundle) 98{ 99 ia64_inst_t inst; 100 inst.l = bundle->quad0.slot0; 101 return inst; 102} 103 104ia64_inst_t __init_or_module 105paravirt_read_slot1(const bundle_t *bundle) 106{ 107 ia64_inst_t inst; 108 inst.l = bundle->quad0.slot1_p0 | 109 ((unsigned long long)bundle->quad1.slot1_p1 << 18UL); 110 return inst; 111} 112 113ia64_inst_t __init_or_module 114paravirt_read_slot2(const bundle_t *bundle) 115{ 116 ia64_inst_t inst; 117 inst.l = bundle->quad1.slot2; 118 return inst; 119} 120 121ia64_inst_t __init_or_module 122paravirt_read_inst(unsigned long tag) 123{ 124 bundle_t *bundle = paravirt_get_bundle(tag); 125 unsigned long slot = paravirt_get_slot(tag); 126 127 switch (slot) { 128 case 0: 129 return paravirt_read_slot0(bundle); 130 case 1: 131 return paravirt_read_slot1(bundle); 132 case 2: 133 return paravirt_read_slot2(bundle); 134 default: 135 BUG(); 136 } 137 /* NOTREACHED */ 138} 139 140void __init_or_module 141paravirt_write_slot0(bundle_t *bundle, ia64_inst_t inst) 142{ 143 bundle->quad0.slot0 = inst.l; 144} 145 146void __init_or_module 147paravirt_write_slot1(bundle_t *bundle, ia64_inst_t inst) 148{ 149 bundle->quad0.slot1_p0 = inst.l; 150 bundle->quad1.slot1_p1 = inst.l >> 18UL; 151} 152 153void __init_or_module 154paravirt_write_slot2(bundle_t *bundle, ia64_inst_t inst) 155{ 156 bundle->quad1.slot2 = inst.l; 157} 158 159void __init_or_module 160paravirt_write_inst(unsigned long tag, ia64_inst_t inst) 161{ 162 bundle_t *bundle = paravirt_get_bundle(tag); 163 unsigned long slot = paravirt_get_slot(tag); 164 165 switch (slot) { 166 case 0: 167 paravirt_write_slot0(bundle, inst); 168 break; 169 case 1: 170 paravirt_write_slot1(bundle, inst); 171 break; 172 case 2: 173 paravirt_write_slot2(bundle, inst); 174 break; 175 default: 176 BUG(); 177 break; 178 } 179 paravirt_flush_i_cache_range(bundle, sizeof(*bundle)); 180} 181 182/* for debug */ 183void 184paravirt_print_bundle(const bundle_t *bundle) 185{ 186 const unsigned long *quad = (const unsigned long *)bundle; 187 ia64_inst_t slot0 = paravirt_read_slot0(bundle); 188 ia64_inst_t slot1 = paravirt_read_slot1(bundle); 189 ia64_inst_t slot2 = paravirt_read_slot2(bundle); 190 191 printk(KERN_DEBUG 192 "bundle 0x%p 0x%016lx 0x%016lx\n", bundle, quad[0], quad[1]); 193 printk(KERN_DEBUG 194 "bundle template 0x%x\n", 195 bundle->quad0.template); 196 printk(KERN_DEBUG 197 "slot0 0x%lx slot1_p0 0x%lx slot1_p1 0x%lx slot2 0x%lx\n", 198 (unsigned long)bundle->quad0.slot0, 199 (unsigned long)bundle->quad0.slot1_p0, 200 (unsigned long)bundle->quad1.slot1_p1, 201 (unsigned long)bundle->quad1.slot2); 202 printk(KERN_DEBUG 203 "slot0 0x%016llx slot1 0x%016llx slot2 0x%016llx\n", 204 slot0.l, slot1.l, slot2.l); 205} 206 207static int noreplace_paravirt __init_or_module = 0; 208 209static int __init setup_noreplace_paravirt(char *str) 210{ 211 noreplace_paravirt = 1; 212 return 1; 213} 214__setup("noreplace-paravirt", setup_noreplace_paravirt); 215 216#ifdef ASM_SUPPORTED 217static void __init_or_module 218fill_nop_bundle(void *sbundle, void *ebundle) 219{ 220 extern const char paravirt_nop_bundle[]; 221 extern const unsigned long paravirt_nop_bundle_size; 222 223 void *bundle = sbundle; 224 225 BUG_ON((((unsigned long)sbundle) % sizeof(bundle_t)) != 0); 226 BUG_ON((((unsigned long)ebundle) % sizeof(bundle_t)) != 0); 227 228 while (bundle < ebundle) { 229 memcpy(bundle, paravirt_nop_bundle, paravirt_nop_bundle_size); 230 231 bundle += paravirt_nop_bundle_size; 232 } 233} 234 235/* helper function */ 236unsigned long __init_or_module 237__paravirt_patch_apply_bundle(void *sbundle, void *ebundle, unsigned long type, 238 const struct paravirt_patch_bundle_elem *elems, 239 unsigned long nelems, 240 const struct paravirt_patch_bundle_elem **found) 241{ 242 unsigned long used = 0; 243 unsigned long i; 244 245 BUG_ON((((unsigned long)sbundle) % sizeof(bundle_t)) != 0); 246 BUG_ON((((unsigned long)ebundle) % sizeof(bundle_t)) != 0); 247 248 found = NULL; 249 for (i = 0; i < nelems; i++) { 250 const struct paravirt_patch_bundle_elem *p = &elems[i]; 251 if (p->type == type) { 252 unsigned long need = p->ebundle - p->sbundle; 253 unsigned long room = ebundle - sbundle; 254 255 if (found != NULL) 256 *found = p; 257 258 if (room < need) { 259 /* no room to replace. skip it */ 260 printk(KERN_DEBUG 261 "the space is too small to put " 262 "bundles. type %ld need %ld room %ld\n", 263 type, need, room); 264 break; 265 } 266 267 used = need; 268 memcpy(sbundle, p->sbundle, used); 269 break; 270 } 271 } 272 273 return used; 274} 275 276void __init_or_module 277paravirt_patch_apply_bundle(const struct paravirt_patch_site_bundle *start, 278 const struct paravirt_patch_site_bundle *end) 279{ 280 const struct paravirt_patch_site_bundle *p; 281 282 if (noreplace_paravirt) 283 return; 284 if (pv_init_ops.patch_bundle == NULL) 285 return; 286 287 for (p = start; p < end; p++) { 288 unsigned long used; 289 290 used = (*pv_init_ops.patch_bundle)(p->sbundle, p->ebundle, 291 p->type); 292 if (used == 0) 293 continue; 294 295 fill_nop_bundle(p->sbundle + used, p->ebundle); 296 paravirt_flush_i_cache_range(p->sbundle, 297 p->ebundle - p->sbundle); 298 } 299 ia64_sync_i(); 300 ia64_srlz_i(); 301} 302 303/* 304 * nop.i, nop.m, nop.f instruction are same format. 305 * but nop.b has differennt format. 306 * This doesn't support nop.b for now. 307 */ 308static void __init_or_module 309fill_nop_inst(unsigned long stag, unsigned long etag) 310{ 311 extern const bundle_t paravirt_nop_mfi_inst_bundle[]; 312 unsigned long tag; 313 const ia64_inst_t nop_inst = 314 paravirt_read_slot0(paravirt_nop_mfi_inst_bundle); 315 316 for (tag = stag; tag < etag; tag = paravirt_get_next_tag(tag)) 317 paravirt_write_inst(tag, nop_inst); 318} 319 320void __init_or_module 321paravirt_patch_apply_inst(const struct paravirt_patch_site_inst *start, 322 const struct paravirt_patch_site_inst *end) 323{ 324 const struct paravirt_patch_site_inst *p; 325 326 if (noreplace_paravirt) 327 return; 328 if (pv_init_ops.patch_inst == NULL) 329 return; 330 331 for (p = start; p < end; p++) { 332 unsigned long tag; 333 bundle_t *sbundle; 334 bundle_t *ebundle; 335 336 tag = (*pv_init_ops.patch_inst)(p->stag, p->etag, p->type); 337 if (tag == p->stag) 338 continue; 339 340 fill_nop_inst(tag, p->etag); 341 sbundle = paravirt_get_bundle(p->stag); 342 ebundle = paravirt_get_bundle(p->etag) + 1; 343 paravirt_flush_i_cache_range(sbundle, (ebundle - sbundle) * 344 sizeof(bundle_t)); 345 } 346 ia64_sync_i(); 347 ia64_srlz_i(); 348} 349#endif /* ASM_SUPPOTED */ 350 351/* brl.cond.sptk.many <target64> X3 */ 352typedef union inst_x3_op { 353 ia64_inst_t inst; 354 struct { 355 unsigned long qp: 6; 356 unsigned long btyp: 3; 357 unsigned long unused: 3; 358 unsigned long p: 1; 359 unsigned long imm20b: 20; 360 unsigned long wh: 2; 361 unsigned long d: 1; 362 unsigned long i: 1; 363 unsigned long opcode: 4; 364 }; 365 unsigned long l; 366} inst_x3_op_t; 367 368typedef union inst_x3_imm { 369 ia64_inst_t inst; 370 struct { 371 unsigned long unused: 2; 372 unsigned long imm39: 39; 373 }; 374 unsigned long l; 375} inst_x3_imm_t; 376 377void __init_or_module 378paravirt_patch_reloc_brl(unsigned long tag, const void *target) 379{ 380 unsigned long tag_op = paravirt_get_next_tag(tag); 381 unsigned long tag_imm = tag; 382 bundle_t *bundle = paravirt_get_bundle(tag); 383 384 ia64_inst_t inst_op = paravirt_read_inst(tag_op); 385 ia64_inst_t inst_imm = paravirt_read_inst(tag_imm); 386 387 inst_x3_op_t inst_x3_op = { .l = inst_op.l }; 388 inst_x3_imm_t inst_x3_imm = { .l = inst_imm.l }; 389 390 unsigned long imm60 = 391 ((unsigned long)target - (unsigned long)bundle) >> 4; 392 393 BUG_ON(paravirt_get_slot(tag) != 1); /* MLX */ 394 BUG_ON(((unsigned long)target & (sizeof(bundle_t) - 1)) != 0); 395 396 /* imm60[59] 1bit */ 397 inst_x3_op.i = (imm60 >> 59) & 1; 398 /* imm60[19:0] 20bit */ 399 inst_x3_op.imm20b = imm60 & ((1UL << 20) - 1); 400 /* imm60[58:20] 39bit */ 401 inst_x3_imm.imm39 = (imm60 >> 20) & ((1UL << 39) - 1); 402 403 inst_op.l = inst_x3_op.l; 404 inst_imm.l = inst_x3_imm.l; 405 406 paravirt_write_inst(tag_op, inst_op); 407 paravirt_write_inst(tag_imm, inst_imm); 408} 409 410/* br.cond.sptk.many <target25> B1 */ 411typedef union inst_b1 { 412 ia64_inst_t inst; 413 struct { 414 unsigned long qp: 6; 415 unsigned long btype: 3; 416 unsigned long unused: 3; 417 unsigned long p: 1; 418 unsigned long imm20b: 20; 419 unsigned long wh: 2; 420 unsigned long d: 1; 421 unsigned long s: 1; 422 unsigned long opcode: 4; 423 }; 424 unsigned long l; 425} inst_b1_t; 426 427void __init 428paravirt_patch_reloc_br(unsigned long tag, const void *target) 429{ 430 bundle_t *bundle = paravirt_get_bundle(tag); 431 ia64_inst_t inst = paravirt_read_inst(tag); 432 unsigned long target25 = (unsigned long)target - (unsigned long)bundle; 433 inst_b1_t inst_b1; 434 435 BUG_ON(((unsigned long)target & (sizeof(bundle_t) - 1)) != 0); 436 437 inst_b1.l = inst.l; 438 if (target25 & (1UL << 63)) 439 inst_b1.s = 1; 440 else 441 inst_b1.s = 0; 442 443 inst_b1.imm20b = target25 >> 4; 444 inst.l = inst_b1.l; 445 446 paravirt_write_inst(tag, inst); 447} 448 449void __init 450__paravirt_patch_apply_branch( 451 unsigned long tag, unsigned long type, 452 const struct paravirt_patch_branch_target *entries, 453 unsigned int nr_entries) 454{ 455 unsigned int i; 456 for (i = 0; i < nr_entries; i++) { 457 if (entries[i].type == type) { 458 paravirt_patch_reloc_br(tag, entries[i].entry); 459 break; 460 } 461 } 462} 463 464static void __init 465paravirt_patch_apply_branch(const struct paravirt_patch_site_branch *start, 466 const struct paravirt_patch_site_branch *end) 467{ 468 const struct paravirt_patch_site_branch *p; 469 470 if (noreplace_paravirt) 471 return; 472 if (pv_init_ops.patch_branch == NULL) 473 return; 474 475 for (p = start; p < end; p++) 476 (*pv_init_ops.patch_branch)(p->tag, p->type); 477 478 ia64_sync_i(); 479 ia64_srlz_i(); 480} 481 482void __init 483paravirt_patch_apply(void) 484{ 485 extern const char __start_paravirt_bundles[]; 486 extern const char __stop_paravirt_bundles[]; 487 extern const char __start_paravirt_insts[]; 488 extern const char __stop_paravirt_insts[]; 489 extern const char __start_paravirt_branches[]; 490 extern const char __stop_paravirt_branches[]; 491 492 paravirt_patch_apply_bundle((const struct paravirt_patch_site_bundle *) 493 __start_paravirt_bundles, 494 (const struct paravirt_patch_site_bundle *) 495 __stop_paravirt_bundles); 496 paravirt_patch_apply_inst((const struct paravirt_patch_site_inst *) 497 __start_paravirt_insts, 498 (const struct paravirt_patch_site_inst *) 499 __stop_paravirt_insts); 500 paravirt_patch_apply_branch((const struct paravirt_patch_site_branch *) 501 __start_paravirt_branches, 502 (const struct paravirt_patch_site_branch *) 503 __stop_paravirt_branches); 504} 505 506/* 507 * Local variables: 508 * mode: C 509 * c-set-style: "linux" 510 * c-basic-offset: 8 511 * tab-width: 8 512 * indent-tabs-mode: t 513 * End: 514 */ 515