kprobes.c revision 3a872d89baae821a0f6e2c1055d4b47650661137
1/* 2 * Kernel Probes (KProbes) 3 * kernel/kprobes.c 4 * 5 * This program is free software; you can redistribute it and/or modify 6 * it under the terms of the GNU General Public License as published by 7 * the Free Software Foundation; either version 2 of the License, or 8 * (at your option) any later version. 9 * 10 * This program is distributed in the hope that it will be useful, 11 * but WITHOUT ANY WARRANTY; without even the implied warranty of 12 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 13 * GNU General Public License for more details. 14 * 15 * You should have received a copy of the GNU General Public License 16 * along with this program; if not, write to the Free Software 17 * Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA. 18 * 19 * Copyright (C) IBM Corporation, 2002, 2004 20 * 21 * 2002-Oct Created by Vamsi Krishna S <vamsi_krishna@in.ibm.com> Kernel 22 * Probes initial implementation (includes suggestions from 23 * Rusty Russell). 24 * 2004-Aug Updated by Prasanna S Panchamukhi <prasanna@in.ibm.com> with 25 * hlists and exceptions notifier as suggested by Andi Kleen. 26 * 2004-July Suparna Bhattacharya <suparna@in.ibm.com> added jumper probes 27 * interface to access function arguments. 28 * 2004-Sep Prasanna S Panchamukhi <prasanna@in.ibm.com> Changed Kprobes 29 * exceptions notifier to be first on the priority list. 30 * 2005-May Hien Nguyen <hien@us.ibm.com>, Jim Keniston 31 * <jkenisto@us.ibm.com> and Prasanna S Panchamukhi 32 * <prasanna@in.ibm.com> added function-return probes. 33 */ 34#include <linux/kprobes.h> 35#include <linux/hash.h> 36#include <linux/init.h> 37#include <linux/slab.h> 38#include <linux/module.h> 39#include <linux/moduleloader.h> 40#include <linux/kallsyms.h> 41#include <asm-generic/sections.h> 42#include <asm/cacheflush.h> 43#include <asm/errno.h> 44#include <asm/kdebug.h> 45 46#define KPROBE_HASH_BITS 6 47#define KPROBE_TABLE_SIZE (1 << KPROBE_HASH_BITS) 48 49 50/* 51 * Some oddball architectures like 64bit powerpc have function descriptors 52 * so this must be overridable. 53 */ 54#ifndef kprobe_lookup_name 55#define kprobe_lookup_name(name, addr) \ 56 addr = ((kprobe_opcode_t *)(kallsyms_lookup_name(name))) 57#endif 58 59static struct hlist_head kprobe_table[KPROBE_TABLE_SIZE]; 60static struct hlist_head kretprobe_inst_table[KPROBE_TABLE_SIZE]; 61static atomic_t kprobe_count; 62 63DEFINE_MUTEX(kprobe_mutex); /* Protects kprobe_table */ 64DEFINE_SPINLOCK(kretprobe_lock); /* Protects kretprobe_inst_table */ 65static DEFINE_PER_CPU(struct kprobe *, kprobe_instance) = NULL; 66 67static struct notifier_block kprobe_page_fault_nb = { 68 .notifier_call = kprobe_exceptions_notify, 69 .priority = 0x7fffffff /* we need to notified first */ 70}; 71 72#ifdef __ARCH_WANT_KPROBES_INSN_SLOT 73/* 74 * kprobe->ainsn.insn points to the copy of the instruction to be 75 * single-stepped. x86_64, POWER4 and above have no-exec support and 76 * stepping on the instruction on a vmalloced/kmalloced/data page 77 * is a recipe for disaster 78 */ 79#define INSNS_PER_PAGE (PAGE_SIZE/(MAX_INSN_SIZE * sizeof(kprobe_opcode_t))) 80 81struct kprobe_insn_page { 82 struct hlist_node hlist; 83 kprobe_opcode_t *insns; /* Page of instruction slots */ 84 char slot_used[INSNS_PER_PAGE]; 85 int nused; 86}; 87 88static struct hlist_head kprobe_insn_pages; 89 90/** 91 * get_insn_slot() - Find a slot on an executable page for an instruction. 92 * We allocate an executable page if there's no room on existing ones. 93 */ 94kprobe_opcode_t __kprobes *get_insn_slot(void) 95{ 96 struct kprobe_insn_page *kip; 97 struct hlist_node *pos; 98 99 hlist_for_each(pos, &kprobe_insn_pages) { 100 kip = hlist_entry(pos, struct kprobe_insn_page, hlist); 101 if (kip->nused < INSNS_PER_PAGE) { 102 int i; 103 for (i = 0; i < INSNS_PER_PAGE; i++) { 104 if (!kip->slot_used[i]) { 105 kip->slot_used[i] = 1; 106 kip->nused++; 107 return kip->insns + (i * MAX_INSN_SIZE); 108 } 109 } 110 /* Surprise! No unused slots. Fix kip->nused. */ 111 kip->nused = INSNS_PER_PAGE; 112 } 113 } 114 115 /* All out of space. Need to allocate a new page. Use slot 0.*/ 116 kip = kmalloc(sizeof(struct kprobe_insn_page), GFP_KERNEL); 117 if (!kip) { 118 return NULL; 119 } 120 121 /* 122 * Use module_alloc so this page is within +/- 2GB of where the 123 * kernel image and loaded module images reside. This is required 124 * so x86_64 can correctly handle the %rip-relative fixups. 125 */ 126 kip->insns = module_alloc(PAGE_SIZE); 127 if (!kip->insns) { 128 kfree(kip); 129 return NULL; 130 } 131 INIT_HLIST_NODE(&kip->hlist); 132 hlist_add_head(&kip->hlist, &kprobe_insn_pages); 133 memset(kip->slot_used, 0, INSNS_PER_PAGE); 134 kip->slot_used[0] = 1; 135 kip->nused = 1; 136 return kip->insns; 137} 138 139void __kprobes free_insn_slot(kprobe_opcode_t *slot) 140{ 141 struct kprobe_insn_page *kip; 142 struct hlist_node *pos; 143 144 hlist_for_each(pos, &kprobe_insn_pages) { 145 kip = hlist_entry(pos, struct kprobe_insn_page, hlist); 146 if (kip->insns <= slot && 147 slot < kip->insns + (INSNS_PER_PAGE * MAX_INSN_SIZE)) { 148 int i = (slot - kip->insns) / MAX_INSN_SIZE; 149 kip->slot_used[i] = 0; 150 kip->nused--; 151 if (kip->nused == 0) { 152 /* 153 * Page is no longer in use. Free it unless 154 * it's the last one. We keep the last one 155 * so as not to have to set it up again the 156 * next time somebody inserts a probe. 157 */ 158 hlist_del(&kip->hlist); 159 if (hlist_empty(&kprobe_insn_pages)) { 160 INIT_HLIST_NODE(&kip->hlist); 161 hlist_add_head(&kip->hlist, 162 &kprobe_insn_pages); 163 } else { 164 module_free(NULL, kip->insns); 165 kfree(kip); 166 } 167 } 168 return; 169 } 170 } 171} 172#endif 173 174/* We have preemption disabled.. so it is safe to use __ versions */ 175static inline void set_kprobe_instance(struct kprobe *kp) 176{ 177 __get_cpu_var(kprobe_instance) = kp; 178} 179 180static inline void reset_kprobe_instance(void) 181{ 182 __get_cpu_var(kprobe_instance) = NULL; 183} 184 185/* 186 * This routine is called either: 187 * - under the kprobe_mutex - during kprobe_[un]register() 188 * OR 189 * - with preemption disabled - from arch/xxx/kernel/kprobes.c 190 */ 191struct kprobe __kprobes *get_kprobe(void *addr) 192{ 193 struct hlist_head *head; 194 struct hlist_node *node; 195 struct kprobe *p; 196 197 head = &kprobe_table[hash_ptr(addr, KPROBE_HASH_BITS)]; 198 hlist_for_each_entry_rcu(p, node, head, hlist) { 199 if (p->addr == addr) 200 return p; 201 } 202 return NULL; 203} 204 205/* 206 * Aggregate handlers for multiple kprobes support - these handlers 207 * take care of invoking the individual kprobe handlers on p->list 208 */ 209static int __kprobes aggr_pre_handler(struct kprobe *p, struct pt_regs *regs) 210{ 211 struct kprobe *kp; 212 213 list_for_each_entry_rcu(kp, &p->list, list) { 214 if (kp->pre_handler) { 215 set_kprobe_instance(kp); 216 if (kp->pre_handler(kp, regs)) 217 return 1; 218 } 219 reset_kprobe_instance(); 220 } 221 return 0; 222} 223 224static void __kprobes aggr_post_handler(struct kprobe *p, struct pt_regs *regs, 225 unsigned long flags) 226{ 227 struct kprobe *kp; 228 229 list_for_each_entry_rcu(kp, &p->list, list) { 230 if (kp->post_handler) { 231 set_kprobe_instance(kp); 232 kp->post_handler(kp, regs, flags); 233 reset_kprobe_instance(); 234 } 235 } 236 return; 237} 238 239static int __kprobes aggr_fault_handler(struct kprobe *p, struct pt_regs *regs, 240 int trapnr) 241{ 242 struct kprobe *cur = __get_cpu_var(kprobe_instance); 243 244 /* 245 * if we faulted "during" the execution of a user specified 246 * probe handler, invoke just that probe's fault handler 247 */ 248 if (cur && cur->fault_handler) { 249 if (cur->fault_handler(cur, regs, trapnr)) 250 return 1; 251 } 252 return 0; 253} 254 255static int __kprobes aggr_break_handler(struct kprobe *p, struct pt_regs *regs) 256{ 257 struct kprobe *cur = __get_cpu_var(kprobe_instance); 258 int ret = 0; 259 260 if (cur && cur->break_handler) { 261 if (cur->break_handler(cur, regs)) 262 ret = 1; 263 } 264 reset_kprobe_instance(); 265 return ret; 266} 267 268/* Walks the list and increments nmissed count for multiprobe case */ 269void __kprobes kprobes_inc_nmissed_count(struct kprobe *p) 270{ 271 struct kprobe *kp; 272 if (p->pre_handler != aggr_pre_handler) { 273 p->nmissed++; 274 } else { 275 list_for_each_entry_rcu(kp, &p->list, list) 276 kp->nmissed++; 277 } 278 return; 279} 280 281/* Called with kretprobe_lock held */ 282struct kretprobe_instance __kprobes *get_free_rp_inst(struct kretprobe *rp) 283{ 284 struct hlist_node *node; 285 struct kretprobe_instance *ri; 286 hlist_for_each_entry(ri, node, &rp->free_instances, uflist) 287 return ri; 288 return NULL; 289} 290 291/* Called with kretprobe_lock held */ 292static struct kretprobe_instance __kprobes *get_used_rp_inst(struct kretprobe 293 *rp) 294{ 295 struct hlist_node *node; 296 struct kretprobe_instance *ri; 297 hlist_for_each_entry(ri, node, &rp->used_instances, uflist) 298 return ri; 299 return NULL; 300} 301 302/* Called with kretprobe_lock held */ 303void __kprobes add_rp_inst(struct kretprobe_instance *ri) 304{ 305 /* 306 * Remove rp inst off the free list - 307 * Add it back when probed function returns 308 */ 309 hlist_del(&ri->uflist); 310 311 /* Add rp inst onto table */ 312 INIT_HLIST_NODE(&ri->hlist); 313 hlist_add_head(&ri->hlist, 314 &kretprobe_inst_table[hash_ptr(ri->task, KPROBE_HASH_BITS)]); 315 316 /* Also add this rp inst to the used list. */ 317 INIT_HLIST_NODE(&ri->uflist); 318 hlist_add_head(&ri->uflist, &ri->rp->used_instances); 319} 320 321/* Called with kretprobe_lock held */ 322void __kprobes recycle_rp_inst(struct kretprobe_instance *ri) 323{ 324 /* remove rp inst off the rprobe_inst_table */ 325 hlist_del(&ri->hlist); 326 if (ri->rp) { 327 /* remove rp inst off the used list */ 328 hlist_del(&ri->uflist); 329 /* put rp inst back onto the free list */ 330 INIT_HLIST_NODE(&ri->uflist); 331 hlist_add_head(&ri->uflist, &ri->rp->free_instances); 332 } else 333 /* Unregistering */ 334 kfree(ri); 335} 336 337struct hlist_head __kprobes *kretprobe_inst_table_head(struct task_struct *tsk) 338{ 339 return &kretprobe_inst_table[hash_ptr(tsk, KPROBE_HASH_BITS)]; 340} 341 342/* 343 * This function is called from finish_task_switch when task tk becomes dead, 344 * so that we can recycle any function-return probe instances associated 345 * with this task. These left over instances represent probed functions 346 * that have been called but will never return. 347 */ 348void __kprobes kprobe_flush_task(struct task_struct *tk) 349{ 350 struct kretprobe_instance *ri; 351 struct hlist_head *head; 352 struct hlist_node *node, *tmp; 353 unsigned long flags = 0; 354 355 spin_lock_irqsave(&kretprobe_lock, flags); 356 head = kretprobe_inst_table_head(tk); 357 hlist_for_each_entry_safe(ri, node, tmp, head, hlist) { 358 if (ri->task == tk) 359 recycle_rp_inst(ri); 360 } 361 spin_unlock_irqrestore(&kretprobe_lock, flags); 362} 363 364static inline void free_rp_inst(struct kretprobe *rp) 365{ 366 struct kretprobe_instance *ri; 367 while ((ri = get_free_rp_inst(rp)) != NULL) { 368 hlist_del(&ri->uflist); 369 kfree(ri); 370 } 371} 372 373/* 374 * Keep all fields in the kprobe consistent 375 */ 376static inline void copy_kprobe(struct kprobe *old_p, struct kprobe *p) 377{ 378 memcpy(&p->opcode, &old_p->opcode, sizeof(kprobe_opcode_t)); 379 memcpy(&p->ainsn, &old_p->ainsn, sizeof(struct arch_specific_insn)); 380} 381 382/* 383* Add the new probe to old_p->list. Fail if this is the 384* second jprobe at the address - two jprobes can't coexist 385*/ 386static int __kprobes add_new_kprobe(struct kprobe *old_p, struct kprobe *p) 387{ 388 if (p->break_handler) { 389 if (old_p->break_handler) 390 return -EEXIST; 391 list_add_tail_rcu(&p->list, &old_p->list); 392 old_p->break_handler = aggr_break_handler; 393 } else 394 list_add_rcu(&p->list, &old_p->list); 395 if (p->post_handler && !old_p->post_handler) 396 old_p->post_handler = aggr_post_handler; 397 return 0; 398} 399 400/* 401 * Fill in the required fields of the "manager kprobe". Replace the 402 * earlier kprobe in the hlist with the manager kprobe 403 */ 404static inline void add_aggr_kprobe(struct kprobe *ap, struct kprobe *p) 405{ 406 copy_kprobe(p, ap); 407 flush_insn_slot(ap); 408 ap->addr = p->addr; 409 ap->pre_handler = aggr_pre_handler; 410 ap->fault_handler = aggr_fault_handler; 411 if (p->post_handler) 412 ap->post_handler = aggr_post_handler; 413 if (p->break_handler) 414 ap->break_handler = aggr_break_handler; 415 416 INIT_LIST_HEAD(&ap->list); 417 list_add_rcu(&p->list, &ap->list); 418 419 hlist_replace_rcu(&p->hlist, &ap->hlist); 420} 421 422/* 423 * This is the second or subsequent kprobe at the address - handle 424 * the intricacies 425 */ 426static int __kprobes register_aggr_kprobe(struct kprobe *old_p, 427 struct kprobe *p) 428{ 429 int ret = 0; 430 struct kprobe *ap; 431 432 if (old_p->pre_handler == aggr_pre_handler) { 433 copy_kprobe(old_p, p); 434 ret = add_new_kprobe(old_p, p); 435 } else { 436 ap = kzalloc(sizeof(struct kprobe), GFP_KERNEL); 437 if (!ap) 438 return -ENOMEM; 439 add_aggr_kprobe(ap, old_p); 440 copy_kprobe(ap, p); 441 ret = add_new_kprobe(ap, p); 442 } 443 return ret; 444} 445 446static int __kprobes in_kprobes_functions(unsigned long addr) 447{ 448 if (addr >= (unsigned long)__kprobes_text_start 449 && addr < (unsigned long)__kprobes_text_end) 450 return -EINVAL; 451 return 0; 452} 453 454static int __kprobes __register_kprobe(struct kprobe *p, 455 unsigned long called_from) 456{ 457 int ret = 0; 458 struct kprobe *old_p; 459 struct module *probed_mod; 460 461 /* 462 * If we have a symbol_name argument look it up, 463 * and add it to the address. That way the addr 464 * field can either be global or relative to a symbol. 465 */ 466 if (p->symbol_name) { 467 if (p->addr) 468 return -EINVAL; 469 kprobe_lookup_name(p->symbol_name, p->addr); 470 } 471 472 if (!p->addr) 473 return -EINVAL; 474 p->addr = (kprobe_opcode_t *)(((char *)p->addr)+ p->offset); 475 476 if ((!kernel_text_address((unsigned long) p->addr)) || 477 in_kprobes_functions((unsigned long) p->addr)) 478 return -EINVAL; 479 480 p->mod_refcounted = 0; 481 /* Check are we probing a module */ 482 if ((probed_mod = module_text_address((unsigned long) p->addr))) { 483 struct module *calling_mod = module_text_address(called_from); 484 /* We must allow modules to probe themself and 485 * in this case avoid incrementing the module refcount, 486 * so as to allow unloading of self probing modules. 487 */ 488 if (calling_mod && (calling_mod != probed_mod)) { 489 if (unlikely(!try_module_get(probed_mod))) 490 return -EINVAL; 491 p->mod_refcounted = 1; 492 } else 493 probed_mod = NULL; 494 } 495 496 p->nmissed = 0; 497 mutex_lock(&kprobe_mutex); 498 old_p = get_kprobe(p->addr); 499 if (old_p) { 500 ret = register_aggr_kprobe(old_p, p); 501 if (!ret) 502 atomic_inc(&kprobe_count); 503 goto out; 504 } 505 506 if ((ret = arch_prepare_kprobe(p)) != 0) 507 goto out; 508 509 INIT_HLIST_NODE(&p->hlist); 510 hlist_add_head_rcu(&p->hlist, 511 &kprobe_table[hash_ptr(p->addr, KPROBE_HASH_BITS)]); 512 513 if (atomic_add_return(1, &kprobe_count) == \ 514 (ARCH_INACTIVE_KPROBE_COUNT + 1)) 515 register_page_fault_notifier(&kprobe_page_fault_nb); 516 517 arch_arm_kprobe(p); 518 519out: 520 mutex_unlock(&kprobe_mutex); 521 522 if (ret && probed_mod) 523 module_put(probed_mod); 524 return ret; 525} 526 527int __kprobes register_kprobe(struct kprobe *p) 528{ 529 return __register_kprobe(p, 530 (unsigned long)__builtin_return_address(0)); 531} 532 533void __kprobes unregister_kprobe(struct kprobe *p) 534{ 535 struct module *mod; 536 struct kprobe *old_p, *list_p; 537 int cleanup_p; 538 539 mutex_lock(&kprobe_mutex); 540 old_p = get_kprobe(p->addr); 541 if (unlikely(!old_p)) { 542 mutex_unlock(&kprobe_mutex); 543 return; 544 } 545 if (p != old_p) { 546 list_for_each_entry_rcu(list_p, &old_p->list, list) 547 if (list_p == p) 548 /* kprobe p is a valid probe */ 549 goto valid_p; 550 mutex_unlock(&kprobe_mutex); 551 return; 552 } 553valid_p: 554 if ((old_p == p) || ((old_p->pre_handler == aggr_pre_handler) && 555 (p->list.next == &old_p->list) && 556 (p->list.prev == &old_p->list))) { 557 /* Only probe on the hash list */ 558 arch_disarm_kprobe(p); 559 hlist_del_rcu(&old_p->hlist); 560 cleanup_p = 1; 561 } else { 562 list_del_rcu(&p->list); 563 cleanup_p = 0; 564 } 565 566 mutex_unlock(&kprobe_mutex); 567 568 synchronize_sched(); 569 if (p->mod_refcounted && 570 (mod = module_text_address((unsigned long)p->addr))) 571 module_put(mod); 572 573 if (cleanup_p) { 574 if (p != old_p) { 575 list_del_rcu(&p->list); 576 kfree(old_p); 577 } 578 arch_remove_kprobe(p); 579 } else { 580 mutex_lock(&kprobe_mutex); 581 if (p->break_handler) 582 old_p->break_handler = NULL; 583 if (p->post_handler){ 584 list_for_each_entry_rcu(list_p, &old_p->list, list){ 585 if (list_p->post_handler){ 586 cleanup_p = 2; 587 break; 588 } 589 } 590 if (cleanup_p == 0) 591 old_p->post_handler = NULL; 592 } 593 mutex_unlock(&kprobe_mutex); 594 } 595 596 /* Call unregister_page_fault_notifier() 597 * if no probes are active 598 */ 599 mutex_lock(&kprobe_mutex); 600 if (atomic_add_return(-1, &kprobe_count) == \ 601 ARCH_INACTIVE_KPROBE_COUNT) 602 unregister_page_fault_notifier(&kprobe_page_fault_nb); 603 mutex_unlock(&kprobe_mutex); 604 return; 605} 606 607static struct notifier_block kprobe_exceptions_nb = { 608 .notifier_call = kprobe_exceptions_notify, 609 .priority = 0x7fffffff /* we need to be notified first */ 610}; 611 612 613int __kprobes register_jprobe(struct jprobe *jp) 614{ 615 /* Todo: Verify probepoint is a function entry point */ 616 jp->kp.pre_handler = setjmp_pre_handler; 617 jp->kp.break_handler = longjmp_break_handler; 618 619 return __register_kprobe(&jp->kp, 620 (unsigned long)__builtin_return_address(0)); 621} 622 623void __kprobes unregister_jprobe(struct jprobe *jp) 624{ 625 unregister_kprobe(&jp->kp); 626} 627 628#ifdef ARCH_SUPPORTS_KRETPROBES 629 630/* 631 * This kprobe pre_handler is registered with every kretprobe. When probe 632 * hits it will set up the return probe. 633 */ 634static int __kprobes pre_handler_kretprobe(struct kprobe *p, 635 struct pt_regs *regs) 636{ 637 struct kretprobe *rp = container_of(p, struct kretprobe, kp); 638 unsigned long flags = 0; 639 640 /*TODO: consider to only swap the RA after the last pre_handler fired */ 641 spin_lock_irqsave(&kretprobe_lock, flags); 642 arch_prepare_kretprobe(rp, regs); 643 spin_unlock_irqrestore(&kretprobe_lock, flags); 644 return 0; 645} 646 647int __kprobes register_kretprobe(struct kretprobe *rp) 648{ 649 int ret = 0; 650 struct kretprobe_instance *inst; 651 int i; 652 653 rp->kp.pre_handler = pre_handler_kretprobe; 654 rp->kp.post_handler = NULL; 655 rp->kp.fault_handler = NULL; 656 rp->kp.break_handler = NULL; 657 658 /* Pre-allocate memory for max kretprobe instances */ 659 if (rp->maxactive <= 0) { 660#ifdef CONFIG_PREEMPT 661 rp->maxactive = max(10, 2 * NR_CPUS); 662#else 663 rp->maxactive = NR_CPUS; 664#endif 665 } 666 INIT_HLIST_HEAD(&rp->used_instances); 667 INIT_HLIST_HEAD(&rp->free_instances); 668 for (i = 0; i < rp->maxactive; i++) { 669 inst = kmalloc(sizeof(struct kretprobe_instance), GFP_KERNEL); 670 if (inst == NULL) { 671 free_rp_inst(rp); 672 return -ENOMEM; 673 } 674 INIT_HLIST_NODE(&inst->uflist); 675 hlist_add_head(&inst->uflist, &rp->free_instances); 676 } 677 678 rp->nmissed = 0; 679 /* Establish function entry probe point */ 680 if ((ret = __register_kprobe(&rp->kp, 681 (unsigned long)__builtin_return_address(0))) != 0) 682 free_rp_inst(rp); 683 return ret; 684} 685 686#else /* ARCH_SUPPORTS_KRETPROBES */ 687 688int __kprobes register_kretprobe(struct kretprobe *rp) 689{ 690 return -ENOSYS; 691} 692 693#endif /* ARCH_SUPPORTS_KRETPROBES */ 694 695void __kprobes unregister_kretprobe(struct kretprobe *rp) 696{ 697 unsigned long flags; 698 struct kretprobe_instance *ri; 699 700 unregister_kprobe(&rp->kp); 701 /* No race here */ 702 spin_lock_irqsave(&kretprobe_lock, flags); 703 while ((ri = get_used_rp_inst(rp)) != NULL) { 704 ri->rp = NULL; 705 hlist_del(&ri->uflist); 706 } 707 spin_unlock_irqrestore(&kretprobe_lock, flags); 708 free_rp_inst(rp); 709} 710 711static int __init init_kprobes(void) 712{ 713 int i, err = 0; 714 715 /* FIXME allocate the probe table, currently defined statically */ 716 /* initialize all list heads */ 717 for (i = 0; i < KPROBE_TABLE_SIZE; i++) { 718 INIT_HLIST_HEAD(&kprobe_table[i]); 719 INIT_HLIST_HEAD(&kretprobe_inst_table[i]); 720 } 721 atomic_set(&kprobe_count, 0); 722 723 err = arch_init_kprobes(); 724 if (!err) 725 err = register_die_notifier(&kprobe_exceptions_nb); 726 727 return err; 728} 729 730__initcall(init_kprobes); 731 732EXPORT_SYMBOL_GPL(register_kprobe); 733EXPORT_SYMBOL_GPL(unregister_kprobe); 734EXPORT_SYMBOL_GPL(register_jprobe); 735EXPORT_SYMBOL_GPL(unregister_jprobe); 736EXPORT_SYMBOL_GPL(jprobe_return); 737EXPORT_SYMBOL_GPL(register_kretprobe); 738EXPORT_SYMBOL_GPL(unregister_kretprobe); 739 740