kprobes.c revision b76834bc1b6db0a0923eed85c81b1113021b0612
1/* 2 * Kernel Probes (KProbes) 3 * kernel/kprobes.c 4 * 5 * This program is free software; you can redistribute it and/or modify 6 * it under the terms of the GNU General Public License as published by 7 * the Free Software Foundation; either version 2 of the License, or 8 * (at your option) any later version. 9 * 10 * This program is distributed in the hope that it will be useful, 11 * but WITHOUT ANY WARRANTY; without even the implied warranty of 12 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 13 * GNU General Public License for more details. 14 * 15 * You should have received a copy of the GNU General Public License 16 * along with this program; if not, write to the Free Software 17 * Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA. 18 * 19 * Copyright (C) IBM Corporation, 2002, 2004 20 * 21 * 2002-Oct Created by Vamsi Krishna S <vamsi_krishna@in.ibm.com> Kernel 22 * Probes initial implementation (includes suggestions from 23 * Rusty Russell). 24 * 2004-Aug Updated by Prasanna S Panchamukhi <prasanna@in.ibm.com> with 25 * hlists and exceptions notifier as suggested by Andi Kleen. 26 * 2004-July Suparna Bhattacharya <suparna@in.ibm.com> added jumper probes 27 * interface to access function arguments. 28 * 2004-Sep Prasanna S Panchamukhi <prasanna@in.ibm.com> Changed Kprobes 29 * exceptions notifier to be first on the priority list. 30 * 2005-May Hien Nguyen <hien@us.ibm.com>, Jim Keniston 31 * <jkenisto@us.ibm.com> and Prasanna S Panchamukhi 32 * <prasanna@in.ibm.com> added function-return probes. 33 */ 34#include <linux/kprobes.h> 35#include <linux/hash.h> 36#include <linux/init.h> 37#include <linux/slab.h> 38#include <linux/stddef.h> 39#include <linux/module.h> 40#include <linux/moduleloader.h> 41#include <linux/kallsyms.h> 42#include <linux/freezer.h> 43#include <linux/seq_file.h> 44#include <linux/debugfs.h> 45#include <linux/sysctl.h> 46#include <linux/kdebug.h> 47#include <linux/memory.h> 48#include <linux/ftrace.h> 49#include <linux/cpu.h> 50#include <linux/jump_label.h> 51 52#include <asm-generic/sections.h> 53#include <asm/cacheflush.h> 54#include <asm/errno.h> 55#include <asm/uaccess.h> 56 57#define KPROBE_HASH_BITS 6 58#define KPROBE_TABLE_SIZE (1 << KPROBE_HASH_BITS) 59 60 61/* 62 * Some oddball architectures like 64bit powerpc have function descriptors 63 * so this must be overridable. 64 */ 65#ifndef kprobe_lookup_name 66#define kprobe_lookup_name(name, addr) \ 67 addr = ((kprobe_opcode_t *)(kallsyms_lookup_name(name))) 68#endif 69 70static int kprobes_initialized; 71static struct hlist_head kprobe_table[KPROBE_TABLE_SIZE]; 72static struct hlist_head kretprobe_inst_table[KPROBE_TABLE_SIZE]; 73 74/* NOTE: change this value only with kprobe_mutex held */ 75static bool kprobes_all_disarmed; 76 77/* This protects kprobe_table and optimizing_list */ 78static DEFINE_MUTEX(kprobe_mutex); 79static DEFINE_PER_CPU(struct kprobe *, kprobe_instance) = NULL; 80static struct { 81 spinlock_t lock ____cacheline_aligned_in_smp; 82} kretprobe_table_locks[KPROBE_TABLE_SIZE]; 83 84static spinlock_t *kretprobe_table_lock_ptr(unsigned long hash) 85{ 86 return &(kretprobe_table_locks[hash].lock); 87} 88 89/* 90 * Normally, functions that we'd want to prohibit kprobes in, are marked 91 * __kprobes. But, there are cases where such functions already belong to 92 * a different section (__sched for preempt_schedule) 93 * 94 * For such cases, we now have a blacklist 95 */ 96static struct kprobe_blackpoint kprobe_blacklist[] = { 97 {"preempt_schedule",}, 98 {"native_get_debugreg",}, 99 {"irq_entries_start",}, 100 {"common_interrupt",}, 101 {"mcount",}, /* mcount can be called from everywhere */ 102 {NULL} /* Terminator */ 103}; 104 105#ifdef __ARCH_WANT_KPROBES_INSN_SLOT 106/* 107 * kprobe->ainsn.insn points to the copy of the instruction to be 108 * single-stepped. x86_64, POWER4 and above have no-exec support and 109 * stepping on the instruction on a vmalloced/kmalloced/data page 110 * is a recipe for disaster 111 */ 112struct kprobe_insn_page { 113 struct list_head list; 114 kprobe_opcode_t *insns; /* Page of instruction slots */ 115 int nused; 116 int ngarbage; 117 char slot_used[]; 118}; 119 120#define KPROBE_INSN_PAGE_SIZE(slots) \ 121 (offsetof(struct kprobe_insn_page, slot_used) + \ 122 (sizeof(char) * (slots))) 123 124struct kprobe_insn_cache { 125 struct list_head pages; /* list of kprobe_insn_page */ 126 size_t insn_size; /* size of instruction slot */ 127 int nr_garbage; 128}; 129 130static int slots_per_page(struct kprobe_insn_cache *c) 131{ 132 return PAGE_SIZE/(c->insn_size * sizeof(kprobe_opcode_t)); 133} 134 135enum kprobe_slot_state { 136 SLOT_CLEAN = 0, 137 SLOT_DIRTY = 1, 138 SLOT_USED = 2, 139}; 140 141static DEFINE_MUTEX(kprobe_insn_mutex); /* Protects kprobe_insn_slots */ 142static struct kprobe_insn_cache kprobe_insn_slots = { 143 .pages = LIST_HEAD_INIT(kprobe_insn_slots.pages), 144 .insn_size = MAX_INSN_SIZE, 145 .nr_garbage = 0, 146}; 147static int __kprobes collect_garbage_slots(struct kprobe_insn_cache *c); 148 149/** 150 * __get_insn_slot() - Find a slot on an executable page for an instruction. 151 * We allocate an executable page if there's no room on existing ones. 152 */ 153static kprobe_opcode_t __kprobes *__get_insn_slot(struct kprobe_insn_cache *c) 154{ 155 struct kprobe_insn_page *kip; 156 157 retry: 158 list_for_each_entry(kip, &c->pages, list) { 159 if (kip->nused < slots_per_page(c)) { 160 int i; 161 for (i = 0; i < slots_per_page(c); i++) { 162 if (kip->slot_used[i] == SLOT_CLEAN) { 163 kip->slot_used[i] = SLOT_USED; 164 kip->nused++; 165 return kip->insns + (i * c->insn_size); 166 } 167 } 168 /* kip->nused is broken. Fix it. */ 169 kip->nused = slots_per_page(c); 170 WARN_ON(1); 171 } 172 } 173 174 /* If there are any garbage slots, collect it and try again. */ 175 if (c->nr_garbage && collect_garbage_slots(c) == 0) 176 goto retry; 177 178 /* All out of space. Need to allocate a new page. */ 179 kip = kmalloc(KPROBE_INSN_PAGE_SIZE(slots_per_page(c)), GFP_KERNEL); 180 if (!kip) 181 return NULL; 182 183 /* 184 * Use module_alloc so this page is within +/- 2GB of where the 185 * kernel image and loaded module images reside. This is required 186 * so x86_64 can correctly handle the %rip-relative fixups. 187 */ 188 kip->insns = module_alloc(PAGE_SIZE); 189 if (!kip->insns) { 190 kfree(kip); 191 return NULL; 192 } 193 INIT_LIST_HEAD(&kip->list); 194 memset(kip->slot_used, SLOT_CLEAN, slots_per_page(c)); 195 kip->slot_used[0] = SLOT_USED; 196 kip->nused = 1; 197 kip->ngarbage = 0; 198 list_add(&kip->list, &c->pages); 199 return kip->insns; 200} 201 202 203kprobe_opcode_t __kprobes *get_insn_slot(void) 204{ 205 kprobe_opcode_t *ret = NULL; 206 207 mutex_lock(&kprobe_insn_mutex); 208 ret = __get_insn_slot(&kprobe_insn_slots); 209 mutex_unlock(&kprobe_insn_mutex); 210 211 return ret; 212} 213 214/* Return 1 if all garbages are collected, otherwise 0. */ 215static int __kprobes collect_one_slot(struct kprobe_insn_page *kip, int idx) 216{ 217 kip->slot_used[idx] = SLOT_CLEAN; 218 kip->nused--; 219 if (kip->nused == 0) { 220 /* 221 * Page is no longer in use. Free it unless 222 * it's the last one. We keep the last one 223 * so as not to have to set it up again the 224 * next time somebody inserts a probe. 225 */ 226 if (!list_is_singular(&kip->list)) { 227 list_del(&kip->list); 228 module_free(NULL, kip->insns); 229 kfree(kip); 230 } 231 return 1; 232 } 233 return 0; 234} 235 236static int __kprobes collect_garbage_slots(struct kprobe_insn_cache *c) 237{ 238 struct kprobe_insn_page *kip, *next; 239 240 /* Ensure no-one is interrupted on the garbages */ 241 synchronize_sched(); 242 243 list_for_each_entry_safe(kip, next, &c->pages, list) { 244 int i; 245 if (kip->ngarbage == 0) 246 continue; 247 kip->ngarbage = 0; /* we will collect all garbages */ 248 for (i = 0; i < slots_per_page(c); i++) { 249 if (kip->slot_used[i] == SLOT_DIRTY && 250 collect_one_slot(kip, i)) 251 break; 252 } 253 } 254 c->nr_garbage = 0; 255 return 0; 256} 257 258static void __kprobes __free_insn_slot(struct kprobe_insn_cache *c, 259 kprobe_opcode_t *slot, int dirty) 260{ 261 struct kprobe_insn_page *kip; 262 263 list_for_each_entry(kip, &c->pages, list) { 264 long idx = ((long)slot - (long)kip->insns) / 265 (c->insn_size * sizeof(kprobe_opcode_t)); 266 if (idx >= 0 && idx < slots_per_page(c)) { 267 WARN_ON(kip->slot_used[idx] != SLOT_USED); 268 if (dirty) { 269 kip->slot_used[idx] = SLOT_DIRTY; 270 kip->ngarbage++; 271 if (++c->nr_garbage > slots_per_page(c)) 272 collect_garbage_slots(c); 273 } else 274 collect_one_slot(kip, idx); 275 return; 276 } 277 } 278 /* Could not free this slot. */ 279 WARN_ON(1); 280} 281 282void __kprobes free_insn_slot(kprobe_opcode_t * slot, int dirty) 283{ 284 mutex_lock(&kprobe_insn_mutex); 285 __free_insn_slot(&kprobe_insn_slots, slot, dirty); 286 mutex_unlock(&kprobe_insn_mutex); 287} 288#ifdef CONFIG_OPTPROBES 289/* For optimized_kprobe buffer */ 290static DEFINE_MUTEX(kprobe_optinsn_mutex); /* Protects kprobe_optinsn_slots */ 291static struct kprobe_insn_cache kprobe_optinsn_slots = { 292 .pages = LIST_HEAD_INIT(kprobe_optinsn_slots.pages), 293 /* .insn_size is initialized later */ 294 .nr_garbage = 0, 295}; 296/* Get a slot for optimized_kprobe buffer */ 297kprobe_opcode_t __kprobes *get_optinsn_slot(void) 298{ 299 kprobe_opcode_t *ret = NULL; 300 301 mutex_lock(&kprobe_optinsn_mutex); 302 ret = __get_insn_slot(&kprobe_optinsn_slots); 303 mutex_unlock(&kprobe_optinsn_mutex); 304 305 return ret; 306} 307 308void __kprobes free_optinsn_slot(kprobe_opcode_t * slot, int dirty) 309{ 310 mutex_lock(&kprobe_optinsn_mutex); 311 __free_insn_slot(&kprobe_optinsn_slots, slot, dirty); 312 mutex_unlock(&kprobe_optinsn_mutex); 313} 314#endif 315#endif 316 317/* We have preemption disabled.. so it is safe to use __ versions */ 318static inline void set_kprobe_instance(struct kprobe *kp) 319{ 320 __this_cpu_write(kprobe_instance, kp); 321} 322 323static inline void reset_kprobe_instance(void) 324{ 325 __this_cpu_write(kprobe_instance, NULL); 326} 327 328/* 329 * This routine is called either: 330 * - under the kprobe_mutex - during kprobe_[un]register() 331 * OR 332 * - with preemption disabled - from arch/xxx/kernel/kprobes.c 333 */ 334struct kprobe __kprobes *get_kprobe(void *addr) 335{ 336 struct hlist_head *head; 337 struct hlist_node *node; 338 struct kprobe *p; 339 340 head = &kprobe_table[hash_ptr(addr, KPROBE_HASH_BITS)]; 341 hlist_for_each_entry_rcu(p, node, head, hlist) { 342 if (p->addr == addr) 343 return p; 344 } 345 346 return NULL; 347} 348 349static int __kprobes aggr_pre_handler(struct kprobe *p, struct pt_regs *regs); 350 351/* Return true if the kprobe is an aggregator */ 352static inline int kprobe_aggrprobe(struct kprobe *p) 353{ 354 return p->pre_handler == aggr_pre_handler; 355} 356 357/* 358 * Keep all fields in the kprobe consistent 359 */ 360static inline void copy_kprobe(struct kprobe *old_p, struct kprobe *p) 361{ 362 memcpy(&p->opcode, &old_p->opcode, sizeof(kprobe_opcode_t)); 363 memcpy(&p->ainsn, &old_p->ainsn, sizeof(struct arch_specific_insn)); 364} 365 366#ifdef CONFIG_OPTPROBES 367/* NOTE: change this value only with kprobe_mutex held */ 368static bool kprobes_allow_optimization; 369 370/* 371 * Call all pre_handler on the list, but ignores its return value. 372 * This must be called from arch-dep optimized caller. 373 */ 374void __kprobes opt_pre_handler(struct kprobe *p, struct pt_regs *regs) 375{ 376 struct kprobe *kp; 377 378 list_for_each_entry_rcu(kp, &p->list, list) { 379 if (kp->pre_handler && likely(!kprobe_disabled(kp))) { 380 set_kprobe_instance(kp); 381 kp->pre_handler(kp, regs); 382 } 383 reset_kprobe_instance(); 384 } 385} 386 387/* Return true(!0) if the kprobe is ready for optimization. */ 388static inline int kprobe_optready(struct kprobe *p) 389{ 390 struct optimized_kprobe *op; 391 392 if (kprobe_aggrprobe(p)) { 393 op = container_of(p, struct optimized_kprobe, kp); 394 return arch_prepared_optinsn(&op->optinsn); 395 } 396 397 return 0; 398} 399 400/* 401 * Return an optimized kprobe whose optimizing code replaces 402 * instructions including addr (exclude breakpoint). 403 */ 404static struct kprobe *__kprobes get_optimized_kprobe(unsigned long addr) 405{ 406 int i; 407 struct kprobe *p = NULL; 408 struct optimized_kprobe *op; 409 410 /* Don't check i == 0, since that is a breakpoint case. */ 411 for (i = 1; !p && i < MAX_OPTIMIZED_LENGTH; i++) 412 p = get_kprobe((void *)(addr - i)); 413 414 if (p && kprobe_optready(p)) { 415 op = container_of(p, struct optimized_kprobe, kp); 416 if (arch_within_optimized_kprobe(op, addr)) 417 return p; 418 } 419 420 return NULL; 421} 422 423/* Optimization staging list, protected by kprobe_mutex */ 424static LIST_HEAD(optimizing_list); 425 426static void kprobe_optimizer(struct work_struct *work); 427static DECLARE_DELAYED_WORK(optimizing_work, kprobe_optimizer); 428#define OPTIMIZE_DELAY 5 429 430/* Kprobe jump optimizer */ 431static __kprobes void kprobe_optimizer(struct work_struct *work) 432{ 433 struct optimized_kprobe *op, *tmp; 434 435 /* Lock modules while optimizing kprobes */ 436 mutex_lock(&module_mutex); 437 mutex_lock(&kprobe_mutex); 438 if (kprobes_all_disarmed || !kprobes_allow_optimization) 439 goto end; 440 441 /* 442 * Wait for quiesence period to ensure all running interrupts 443 * are done. Because optprobe may modify multiple instructions 444 * there is a chance that Nth instruction is interrupted. In that 445 * case, running interrupt can return to 2nd-Nth byte of jump 446 * instruction. This wait is for avoiding it. 447 */ 448 synchronize_sched(); 449 450 /* 451 * The optimization/unoptimization refers online_cpus via 452 * stop_machine() and cpu-hotplug modifies online_cpus. 453 * And same time, text_mutex will be held in cpu-hotplug and here. 454 * This combination can cause a deadlock (cpu-hotplug try to lock 455 * text_mutex but stop_machine can not be done because online_cpus 456 * has been changed) 457 * To avoid this deadlock, we need to call get_online_cpus() 458 * for preventing cpu-hotplug outside of text_mutex locking. 459 */ 460 get_online_cpus(); 461 mutex_lock(&text_mutex); 462 list_for_each_entry_safe(op, tmp, &optimizing_list, list) { 463 WARN_ON(kprobe_disabled(&op->kp)); 464 if (arch_optimize_kprobe(op) < 0) 465 op->kp.flags &= ~KPROBE_FLAG_OPTIMIZED; 466 list_del_init(&op->list); 467 } 468 mutex_unlock(&text_mutex); 469 put_online_cpus(); 470end: 471 mutex_unlock(&kprobe_mutex); 472 mutex_unlock(&module_mutex); 473} 474 475/* Optimize kprobe if p is ready to be optimized */ 476static __kprobes void optimize_kprobe(struct kprobe *p) 477{ 478 struct optimized_kprobe *op; 479 480 /* Check if the kprobe is disabled or not ready for optimization. */ 481 if (!kprobe_optready(p) || !kprobes_allow_optimization || 482 (kprobe_disabled(p) || kprobes_all_disarmed)) 483 return; 484 485 /* Both of break_handler and post_handler are not supported. */ 486 if (p->break_handler || p->post_handler) 487 return; 488 489 op = container_of(p, struct optimized_kprobe, kp); 490 491 /* Check there is no other kprobes at the optimized instructions */ 492 if (arch_check_optimized_kprobe(op) < 0) 493 return; 494 495 /* Check if it is already optimized. */ 496 if (op->kp.flags & KPROBE_FLAG_OPTIMIZED) 497 return; 498 499 op->kp.flags |= KPROBE_FLAG_OPTIMIZED; 500 list_add(&op->list, &optimizing_list); 501 if (!delayed_work_pending(&optimizing_work)) 502 schedule_delayed_work(&optimizing_work, OPTIMIZE_DELAY); 503} 504 505/* Unoptimize a kprobe if p is optimized */ 506static __kprobes void unoptimize_kprobe(struct kprobe *p) 507{ 508 struct optimized_kprobe *op; 509 510 if ((p->flags & KPROBE_FLAG_OPTIMIZED) && kprobe_aggrprobe(p)) { 511 op = container_of(p, struct optimized_kprobe, kp); 512 if (!list_empty(&op->list)) 513 /* Dequeue from the optimization queue */ 514 list_del_init(&op->list); 515 else 516 /* Replace jump with break */ 517 arch_unoptimize_kprobe(op); 518 op->kp.flags &= ~KPROBE_FLAG_OPTIMIZED; 519 } 520} 521 522/* Remove optimized instructions */ 523static void __kprobes kill_optimized_kprobe(struct kprobe *p) 524{ 525 struct optimized_kprobe *op; 526 527 op = container_of(p, struct optimized_kprobe, kp); 528 if (!list_empty(&op->list)) { 529 /* Dequeue from the optimization queue */ 530 list_del_init(&op->list); 531 op->kp.flags &= ~KPROBE_FLAG_OPTIMIZED; 532 } 533 /* Don't unoptimize, because the target code will be freed. */ 534 arch_remove_optimized_kprobe(op); 535} 536 537/* Try to prepare optimized instructions */ 538static __kprobes void prepare_optimized_kprobe(struct kprobe *p) 539{ 540 struct optimized_kprobe *op; 541 542 op = container_of(p, struct optimized_kprobe, kp); 543 arch_prepare_optimized_kprobe(op); 544} 545 546/* Free optimized instructions and optimized_kprobe */ 547static __kprobes void free_aggr_kprobe(struct kprobe *p) 548{ 549 struct optimized_kprobe *op; 550 551 op = container_of(p, struct optimized_kprobe, kp); 552 arch_remove_optimized_kprobe(op); 553 kfree(op); 554} 555 556/* Allocate new optimized_kprobe and try to prepare optimized instructions */ 557static __kprobes struct kprobe *alloc_aggr_kprobe(struct kprobe *p) 558{ 559 struct optimized_kprobe *op; 560 561 op = kzalloc(sizeof(struct optimized_kprobe), GFP_KERNEL); 562 if (!op) 563 return NULL; 564 565 INIT_LIST_HEAD(&op->list); 566 op->kp.addr = p->addr; 567 arch_prepare_optimized_kprobe(op); 568 569 return &op->kp; 570} 571 572static void __kprobes init_aggr_kprobe(struct kprobe *ap, struct kprobe *p); 573 574/* 575 * Prepare an optimized_kprobe and optimize it 576 * NOTE: p must be a normal registered kprobe 577 */ 578static __kprobes void try_to_optimize_kprobe(struct kprobe *p) 579{ 580 struct kprobe *ap; 581 struct optimized_kprobe *op; 582 583 ap = alloc_aggr_kprobe(p); 584 if (!ap) 585 return; 586 587 op = container_of(ap, struct optimized_kprobe, kp); 588 if (!arch_prepared_optinsn(&op->optinsn)) { 589 /* If failed to setup optimizing, fallback to kprobe */ 590 free_aggr_kprobe(ap); 591 return; 592 } 593 594 init_aggr_kprobe(ap, p); 595 optimize_kprobe(ap); 596} 597 598#ifdef CONFIG_SYSCTL 599/* This should be called with kprobe_mutex locked */ 600static void __kprobes optimize_all_kprobes(void) 601{ 602 struct hlist_head *head; 603 struct hlist_node *node; 604 struct kprobe *p; 605 unsigned int i; 606 607 /* If optimization is already allowed, just return */ 608 if (kprobes_allow_optimization) 609 return; 610 611 kprobes_allow_optimization = true; 612 for (i = 0; i < KPROBE_TABLE_SIZE; i++) { 613 head = &kprobe_table[i]; 614 hlist_for_each_entry_rcu(p, node, head, hlist) 615 if (!kprobe_disabled(p)) 616 optimize_kprobe(p); 617 } 618 printk(KERN_INFO "Kprobes globally optimized\n"); 619} 620 621/* This should be called with kprobe_mutex locked */ 622static void __kprobes unoptimize_all_kprobes(void) 623{ 624 struct hlist_head *head; 625 struct hlist_node *node; 626 struct kprobe *p; 627 unsigned int i; 628 629 /* If optimization is already prohibited, just return */ 630 if (!kprobes_allow_optimization) 631 return; 632 633 kprobes_allow_optimization = false; 634 printk(KERN_INFO "Kprobes globally unoptimized\n"); 635 get_online_cpus(); /* For avoiding text_mutex deadlock */ 636 mutex_lock(&text_mutex); 637 for (i = 0; i < KPROBE_TABLE_SIZE; i++) { 638 head = &kprobe_table[i]; 639 hlist_for_each_entry_rcu(p, node, head, hlist) { 640 if (!kprobe_disabled(p)) 641 unoptimize_kprobe(p); 642 } 643 } 644 645 mutex_unlock(&text_mutex); 646 put_online_cpus(); 647 /* Allow all currently running kprobes to complete */ 648 synchronize_sched(); 649} 650 651int sysctl_kprobes_optimization; 652int proc_kprobes_optimization_handler(struct ctl_table *table, int write, 653 void __user *buffer, size_t *length, 654 loff_t *ppos) 655{ 656 int ret; 657 658 mutex_lock(&kprobe_mutex); 659 sysctl_kprobes_optimization = kprobes_allow_optimization ? 1 : 0; 660 ret = proc_dointvec_minmax(table, write, buffer, length, ppos); 661 662 if (sysctl_kprobes_optimization) 663 optimize_all_kprobes(); 664 else 665 unoptimize_all_kprobes(); 666 mutex_unlock(&kprobe_mutex); 667 668 return ret; 669} 670#endif /* CONFIG_SYSCTL */ 671 672static void __kprobes __arm_kprobe(struct kprobe *p) 673{ 674 struct kprobe *old_p; 675 676 /* Check collision with other optimized kprobes */ 677 old_p = get_optimized_kprobe((unsigned long)p->addr); 678 if (unlikely(old_p)) 679 unoptimize_kprobe(old_p); /* Fallback to unoptimized kprobe */ 680 681 arch_arm_kprobe(p); 682 optimize_kprobe(p); /* Try to optimize (add kprobe to a list) */ 683} 684 685static void __kprobes __disarm_kprobe(struct kprobe *p) 686{ 687 struct kprobe *old_p; 688 689 unoptimize_kprobe(p); /* Try to unoptimize */ 690 arch_disarm_kprobe(p); 691 692 /* If another kprobe was blocked, optimize it. */ 693 old_p = get_optimized_kprobe((unsigned long)p->addr); 694 if (unlikely(old_p)) 695 optimize_kprobe(old_p); 696} 697 698#else /* !CONFIG_OPTPROBES */ 699 700#define optimize_kprobe(p) do {} while (0) 701#define unoptimize_kprobe(p) do {} while (0) 702#define kill_optimized_kprobe(p) do {} while (0) 703#define prepare_optimized_kprobe(p) do {} while (0) 704#define try_to_optimize_kprobe(p) do {} while (0) 705#define __arm_kprobe(p) arch_arm_kprobe(p) 706#define __disarm_kprobe(p) arch_disarm_kprobe(p) 707 708static __kprobes void free_aggr_kprobe(struct kprobe *p) 709{ 710 kfree(p); 711} 712 713static __kprobes struct kprobe *alloc_aggr_kprobe(struct kprobe *p) 714{ 715 return kzalloc(sizeof(struct kprobe), GFP_KERNEL); 716} 717#endif /* CONFIG_OPTPROBES */ 718 719/* Arm a kprobe with text_mutex */ 720static void __kprobes arm_kprobe(struct kprobe *kp) 721{ 722 /* 723 * Here, since __arm_kprobe() doesn't use stop_machine(), 724 * this doesn't cause deadlock on text_mutex. So, we don't 725 * need get_online_cpus(). 726 */ 727 mutex_lock(&text_mutex); 728 __arm_kprobe(kp); 729 mutex_unlock(&text_mutex); 730} 731 732/* Disarm a kprobe with text_mutex */ 733static void __kprobes disarm_kprobe(struct kprobe *kp) 734{ 735 get_online_cpus(); /* For avoiding text_mutex deadlock */ 736 mutex_lock(&text_mutex); 737 __disarm_kprobe(kp); 738 mutex_unlock(&text_mutex); 739 put_online_cpus(); 740} 741 742/* 743 * Aggregate handlers for multiple kprobes support - these handlers 744 * take care of invoking the individual kprobe handlers on p->list 745 */ 746static int __kprobes aggr_pre_handler(struct kprobe *p, struct pt_regs *regs) 747{ 748 struct kprobe *kp; 749 750 list_for_each_entry_rcu(kp, &p->list, list) { 751 if (kp->pre_handler && likely(!kprobe_disabled(kp))) { 752 set_kprobe_instance(kp); 753 if (kp->pre_handler(kp, regs)) 754 return 1; 755 } 756 reset_kprobe_instance(); 757 } 758 return 0; 759} 760 761static void __kprobes aggr_post_handler(struct kprobe *p, struct pt_regs *regs, 762 unsigned long flags) 763{ 764 struct kprobe *kp; 765 766 list_for_each_entry_rcu(kp, &p->list, list) { 767 if (kp->post_handler && likely(!kprobe_disabled(kp))) { 768 set_kprobe_instance(kp); 769 kp->post_handler(kp, regs, flags); 770 reset_kprobe_instance(); 771 } 772 } 773} 774 775static int __kprobes aggr_fault_handler(struct kprobe *p, struct pt_regs *regs, 776 int trapnr) 777{ 778 struct kprobe *cur = __this_cpu_read(kprobe_instance); 779 780 /* 781 * if we faulted "during" the execution of a user specified 782 * probe handler, invoke just that probe's fault handler 783 */ 784 if (cur && cur->fault_handler) { 785 if (cur->fault_handler(cur, regs, trapnr)) 786 return 1; 787 } 788 return 0; 789} 790 791static int __kprobes aggr_break_handler(struct kprobe *p, struct pt_regs *regs) 792{ 793 struct kprobe *cur = __this_cpu_read(kprobe_instance); 794 int ret = 0; 795 796 if (cur && cur->break_handler) { 797 if (cur->break_handler(cur, regs)) 798 ret = 1; 799 } 800 reset_kprobe_instance(); 801 return ret; 802} 803 804/* Walks the list and increments nmissed count for multiprobe case */ 805void __kprobes kprobes_inc_nmissed_count(struct kprobe *p) 806{ 807 struct kprobe *kp; 808 if (!kprobe_aggrprobe(p)) { 809 p->nmissed++; 810 } else { 811 list_for_each_entry_rcu(kp, &p->list, list) 812 kp->nmissed++; 813 } 814 return; 815} 816 817void __kprobes recycle_rp_inst(struct kretprobe_instance *ri, 818 struct hlist_head *head) 819{ 820 struct kretprobe *rp = ri->rp; 821 822 /* remove rp inst off the rprobe_inst_table */ 823 hlist_del(&ri->hlist); 824 INIT_HLIST_NODE(&ri->hlist); 825 if (likely(rp)) { 826 spin_lock(&rp->lock); 827 hlist_add_head(&ri->hlist, &rp->free_instances); 828 spin_unlock(&rp->lock); 829 } else 830 /* Unregistering */ 831 hlist_add_head(&ri->hlist, head); 832} 833 834void __kprobes kretprobe_hash_lock(struct task_struct *tsk, 835 struct hlist_head **head, unsigned long *flags) 836__acquires(hlist_lock) 837{ 838 unsigned long hash = hash_ptr(tsk, KPROBE_HASH_BITS); 839 spinlock_t *hlist_lock; 840 841 *head = &kretprobe_inst_table[hash]; 842 hlist_lock = kretprobe_table_lock_ptr(hash); 843 spin_lock_irqsave(hlist_lock, *flags); 844} 845 846static void __kprobes kretprobe_table_lock(unsigned long hash, 847 unsigned long *flags) 848__acquires(hlist_lock) 849{ 850 spinlock_t *hlist_lock = kretprobe_table_lock_ptr(hash); 851 spin_lock_irqsave(hlist_lock, *flags); 852} 853 854void __kprobes kretprobe_hash_unlock(struct task_struct *tsk, 855 unsigned long *flags) 856__releases(hlist_lock) 857{ 858 unsigned long hash = hash_ptr(tsk, KPROBE_HASH_BITS); 859 spinlock_t *hlist_lock; 860 861 hlist_lock = kretprobe_table_lock_ptr(hash); 862 spin_unlock_irqrestore(hlist_lock, *flags); 863} 864 865static void __kprobes kretprobe_table_unlock(unsigned long hash, 866 unsigned long *flags) 867__releases(hlist_lock) 868{ 869 spinlock_t *hlist_lock = kretprobe_table_lock_ptr(hash); 870 spin_unlock_irqrestore(hlist_lock, *flags); 871} 872 873/* 874 * This function is called from finish_task_switch when task tk becomes dead, 875 * so that we can recycle any function-return probe instances associated 876 * with this task. These left over instances represent probed functions 877 * that have been called but will never return. 878 */ 879void __kprobes kprobe_flush_task(struct task_struct *tk) 880{ 881 struct kretprobe_instance *ri; 882 struct hlist_head *head, empty_rp; 883 struct hlist_node *node, *tmp; 884 unsigned long hash, flags = 0; 885 886 if (unlikely(!kprobes_initialized)) 887 /* Early boot. kretprobe_table_locks not yet initialized. */ 888 return; 889 890 hash = hash_ptr(tk, KPROBE_HASH_BITS); 891 head = &kretprobe_inst_table[hash]; 892 kretprobe_table_lock(hash, &flags); 893 hlist_for_each_entry_safe(ri, node, tmp, head, hlist) { 894 if (ri->task == tk) 895 recycle_rp_inst(ri, &empty_rp); 896 } 897 kretprobe_table_unlock(hash, &flags); 898 INIT_HLIST_HEAD(&empty_rp); 899 hlist_for_each_entry_safe(ri, node, tmp, &empty_rp, hlist) { 900 hlist_del(&ri->hlist); 901 kfree(ri); 902 } 903} 904 905static inline void free_rp_inst(struct kretprobe *rp) 906{ 907 struct kretprobe_instance *ri; 908 struct hlist_node *pos, *next; 909 910 hlist_for_each_entry_safe(ri, pos, next, &rp->free_instances, hlist) { 911 hlist_del(&ri->hlist); 912 kfree(ri); 913 } 914} 915 916static void __kprobes cleanup_rp_inst(struct kretprobe *rp) 917{ 918 unsigned long flags, hash; 919 struct kretprobe_instance *ri; 920 struct hlist_node *pos, *next; 921 struct hlist_head *head; 922 923 /* No race here */ 924 for (hash = 0; hash < KPROBE_TABLE_SIZE; hash++) { 925 kretprobe_table_lock(hash, &flags); 926 head = &kretprobe_inst_table[hash]; 927 hlist_for_each_entry_safe(ri, pos, next, head, hlist) { 928 if (ri->rp == rp) 929 ri->rp = NULL; 930 } 931 kretprobe_table_unlock(hash, &flags); 932 } 933 free_rp_inst(rp); 934} 935 936/* 937* Add the new probe to ap->list. Fail if this is the 938* second jprobe at the address - two jprobes can't coexist 939*/ 940static int __kprobes add_new_kprobe(struct kprobe *ap, struct kprobe *p) 941{ 942 BUG_ON(kprobe_gone(ap) || kprobe_gone(p)); 943 944 if (p->break_handler || p->post_handler) 945 unoptimize_kprobe(ap); /* Fall back to normal kprobe */ 946 947 if (p->break_handler) { 948 if (ap->break_handler) 949 return -EEXIST; 950 list_add_tail_rcu(&p->list, &ap->list); 951 ap->break_handler = aggr_break_handler; 952 } else 953 list_add_rcu(&p->list, &ap->list); 954 if (p->post_handler && !ap->post_handler) 955 ap->post_handler = aggr_post_handler; 956 957 if (kprobe_disabled(ap) && !kprobe_disabled(p)) { 958 ap->flags &= ~KPROBE_FLAG_DISABLED; 959 if (!kprobes_all_disarmed) 960 /* Arm the breakpoint again. */ 961 __arm_kprobe(ap); 962 } 963 return 0; 964} 965 966/* 967 * Fill in the required fields of the "manager kprobe". Replace the 968 * earlier kprobe in the hlist with the manager kprobe 969 */ 970static void __kprobes init_aggr_kprobe(struct kprobe *ap, struct kprobe *p) 971{ 972 /* Copy p's insn slot to ap */ 973 copy_kprobe(p, ap); 974 flush_insn_slot(ap); 975 ap->addr = p->addr; 976 ap->flags = p->flags & ~KPROBE_FLAG_OPTIMIZED; 977 ap->pre_handler = aggr_pre_handler; 978 ap->fault_handler = aggr_fault_handler; 979 /* We don't care the kprobe which has gone. */ 980 if (p->post_handler && !kprobe_gone(p)) 981 ap->post_handler = aggr_post_handler; 982 if (p->break_handler && !kprobe_gone(p)) 983 ap->break_handler = aggr_break_handler; 984 985 INIT_LIST_HEAD(&ap->list); 986 INIT_HLIST_NODE(&ap->hlist); 987 988 list_add_rcu(&p->list, &ap->list); 989 hlist_replace_rcu(&p->hlist, &ap->hlist); 990} 991 992/* 993 * This is the second or subsequent kprobe at the address - handle 994 * the intricacies 995 */ 996static int __kprobes register_aggr_kprobe(struct kprobe *old_p, 997 struct kprobe *p) 998{ 999 int ret = 0; 1000 struct kprobe *ap = old_p; 1001 1002 if (!kprobe_aggrprobe(old_p)) { 1003 /* If old_p is not an aggr_kprobe, create new aggr_kprobe. */ 1004 ap = alloc_aggr_kprobe(old_p); 1005 if (!ap) 1006 return -ENOMEM; 1007 init_aggr_kprobe(ap, old_p); 1008 } 1009 1010 if (kprobe_gone(ap)) { 1011 /* 1012 * Attempting to insert new probe at the same location that 1013 * had a probe in the module vaddr area which already 1014 * freed. So, the instruction slot has already been 1015 * released. We need a new slot for the new probe. 1016 */ 1017 ret = arch_prepare_kprobe(ap); 1018 if (ret) 1019 /* 1020 * Even if fail to allocate new slot, don't need to 1021 * free aggr_probe. It will be used next time, or 1022 * freed by unregister_kprobe. 1023 */ 1024 return ret; 1025 1026 /* Prepare optimized instructions if possible. */ 1027 prepare_optimized_kprobe(ap); 1028 1029 /* 1030 * Clear gone flag to prevent allocating new slot again, and 1031 * set disabled flag because it is not armed yet. 1032 */ 1033 ap->flags = (ap->flags & ~KPROBE_FLAG_GONE) 1034 | KPROBE_FLAG_DISABLED; 1035 } 1036 1037 /* Copy ap's insn slot to p */ 1038 copy_kprobe(ap, p); 1039 return add_new_kprobe(ap, p); 1040} 1041 1042/* Try to disable aggr_kprobe, and return 1 if succeeded.*/ 1043static int __kprobes try_to_disable_aggr_kprobe(struct kprobe *p) 1044{ 1045 struct kprobe *kp; 1046 1047 list_for_each_entry_rcu(kp, &p->list, list) { 1048 if (!kprobe_disabled(kp)) 1049 /* 1050 * There is an active probe on the list. 1051 * We can't disable aggr_kprobe. 1052 */ 1053 return 0; 1054 } 1055 p->flags |= KPROBE_FLAG_DISABLED; 1056 return 1; 1057} 1058 1059static int __kprobes in_kprobes_functions(unsigned long addr) 1060{ 1061 struct kprobe_blackpoint *kb; 1062 1063 if (addr >= (unsigned long)__kprobes_text_start && 1064 addr < (unsigned long)__kprobes_text_end) 1065 return -EINVAL; 1066 /* 1067 * If there exists a kprobe_blacklist, verify and 1068 * fail any probe registration in the prohibited area 1069 */ 1070 for (kb = kprobe_blacklist; kb->name != NULL; kb++) { 1071 if (kb->start_addr) { 1072 if (addr >= kb->start_addr && 1073 addr < (kb->start_addr + kb->range)) 1074 return -EINVAL; 1075 } 1076 } 1077 return 0; 1078} 1079 1080/* 1081 * If we have a symbol_name argument, look it up and add the offset field 1082 * to it. This way, we can specify a relative address to a symbol. 1083 */ 1084static kprobe_opcode_t __kprobes *kprobe_addr(struct kprobe *p) 1085{ 1086 kprobe_opcode_t *addr = p->addr; 1087 if (p->symbol_name) { 1088 if (addr) 1089 return NULL; 1090 kprobe_lookup_name(p->symbol_name, addr); 1091 } 1092 1093 if (!addr) 1094 return NULL; 1095 return (kprobe_opcode_t *)(((char *)addr) + p->offset); 1096} 1097 1098/* Check passed kprobe is valid and return kprobe in kprobe_table. */ 1099static struct kprobe * __kprobes __get_valid_kprobe(struct kprobe *p) 1100{ 1101 struct kprobe *old_p, *list_p; 1102 1103 old_p = get_kprobe(p->addr); 1104 if (unlikely(!old_p)) 1105 return NULL; 1106 1107 if (p != old_p) { 1108 list_for_each_entry_rcu(list_p, &old_p->list, list) 1109 if (list_p == p) 1110 /* kprobe p is a valid probe */ 1111 goto valid; 1112 return NULL; 1113 } 1114valid: 1115 return old_p; 1116} 1117 1118/* Return error if the kprobe is being re-registered */ 1119static inline int check_kprobe_rereg(struct kprobe *p) 1120{ 1121 int ret = 0; 1122 struct kprobe *old_p; 1123 1124 mutex_lock(&kprobe_mutex); 1125 old_p = __get_valid_kprobe(p); 1126 if (old_p) 1127 ret = -EINVAL; 1128 mutex_unlock(&kprobe_mutex); 1129 return ret; 1130} 1131 1132int __kprobes register_kprobe(struct kprobe *p) 1133{ 1134 int ret = 0; 1135 struct kprobe *old_p; 1136 struct module *probed_mod; 1137 kprobe_opcode_t *addr; 1138 1139 addr = kprobe_addr(p); 1140 if (!addr) 1141 return -EINVAL; 1142 p->addr = addr; 1143 1144 ret = check_kprobe_rereg(p); 1145 if (ret) 1146 return ret; 1147 1148 jump_label_lock(); 1149 preempt_disable(); 1150 if (!kernel_text_address((unsigned long) p->addr) || 1151 in_kprobes_functions((unsigned long) p->addr) || 1152 ftrace_text_reserved(p->addr, p->addr) || 1153 jump_label_text_reserved(p->addr, p->addr)) 1154 goto fail_with_jump_label; 1155 1156 /* User can pass only KPROBE_FLAG_DISABLED to register_kprobe */ 1157 p->flags &= KPROBE_FLAG_DISABLED; 1158 1159 /* 1160 * Check if are we probing a module. 1161 */ 1162 probed_mod = __module_text_address((unsigned long) p->addr); 1163 if (probed_mod) { 1164 /* 1165 * We must hold a refcount of the probed module while updating 1166 * its code to prohibit unexpected unloading. 1167 */ 1168 if (unlikely(!try_module_get(probed_mod))) 1169 goto fail_with_jump_label; 1170 1171 /* 1172 * If the module freed .init.text, we couldn't insert 1173 * kprobes in there. 1174 */ 1175 if (within_module_init((unsigned long)p->addr, probed_mod) && 1176 probed_mod->state != MODULE_STATE_COMING) { 1177 module_put(probed_mod); 1178 goto fail_with_jump_label; 1179 } 1180 } 1181 preempt_enable(); 1182 jump_label_unlock(); 1183 1184 p->nmissed = 0; 1185 INIT_LIST_HEAD(&p->list); 1186 mutex_lock(&kprobe_mutex); 1187 1188 jump_label_lock(); /* needed to call jump_label_text_reserved() */ 1189 1190 get_online_cpus(); /* For avoiding text_mutex deadlock. */ 1191 mutex_lock(&text_mutex); 1192 1193 old_p = get_kprobe(p->addr); 1194 if (old_p) { 1195 /* Since this may unoptimize old_p, locking text_mutex. */ 1196 ret = register_aggr_kprobe(old_p, p); 1197 goto out; 1198 } 1199 1200 ret = arch_prepare_kprobe(p); 1201 if (ret) 1202 goto out; 1203 1204 INIT_HLIST_NODE(&p->hlist); 1205 hlist_add_head_rcu(&p->hlist, 1206 &kprobe_table[hash_ptr(p->addr, KPROBE_HASH_BITS)]); 1207 1208 if (!kprobes_all_disarmed && !kprobe_disabled(p)) 1209 __arm_kprobe(p); 1210 1211 /* Try to optimize kprobe */ 1212 try_to_optimize_kprobe(p); 1213 1214out: 1215 mutex_unlock(&text_mutex); 1216 put_online_cpus(); 1217 jump_label_unlock(); 1218 mutex_unlock(&kprobe_mutex); 1219 1220 if (probed_mod) 1221 module_put(probed_mod); 1222 1223 return ret; 1224 1225fail_with_jump_label: 1226 preempt_enable(); 1227 jump_label_unlock(); 1228 return -EINVAL; 1229} 1230EXPORT_SYMBOL_GPL(register_kprobe); 1231 1232/* 1233 * Unregister a kprobe without a scheduler synchronization. 1234 */ 1235static int __kprobes __unregister_kprobe_top(struct kprobe *p) 1236{ 1237 struct kprobe *old_p, *list_p; 1238 1239 old_p = __get_valid_kprobe(p); 1240 if (old_p == NULL) 1241 return -EINVAL; 1242 1243 if (old_p == p || 1244 (kprobe_aggrprobe(old_p) && 1245 list_is_singular(&old_p->list))) { 1246 /* 1247 * Only probe on the hash list. Disarm only if kprobes are 1248 * enabled and not gone - otherwise, the breakpoint would 1249 * already have been removed. We save on flushing icache. 1250 */ 1251 if (!kprobes_all_disarmed && !kprobe_disabled(old_p)) 1252 disarm_kprobe(old_p); 1253 hlist_del_rcu(&old_p->hlist); 1254 } else { 1255 if (p->break_handler && !kprobe_gone(p)) 1256 old_p->break_handler = NULL; 1257 if (p->post_handler && !kprobe_gone(p)) { 1258 list_for_each_entry_rcu(list_p, &old_p->list, list) { 1259 if ((list_p != p) && (list_p->post_handler)) 1260 goto noclean; 1261 } 1262 old_p->post_handler = NULL; 1263 } 1264noclean: 1265 list_del_rcu(&p->list); 1266 if (!kprobe_disabled(old_p)) { 1267 try_to_disable_aggr_kprobe(old_p); 1268 if (!kprobes_all_disarmed) { 1269 if (kprobe_disabled(old_p)) 1270 disarm_kprobe(old_p); 1271 else 1272 /* Try to optimize this probe again */ 1273 optimize_kprobe(old_p); 1274 } 1275 } 1276 } 1277 return 0; 1278} 1279 1280static void __kprobes __unregister_kprobe_bottom(struct kprobe *p) 1281{ 1282 struct kprobe *old_p; 1283 1284 if (list_empty(&p->list)) 1285 arch_remove_kprobe(p); 1286 else if (list_is_singular(&p->list)) { 1287 /* "p" is the last child of an aggr_kprobe */ 1288 old_p = list_entry(p->list.next, struct kprobe, list); 1289 list_del(&p->list); 1290 arch_remove_kprobe(old_p); 1291 free_aggr_kprobe(old_p); 1292 } 1293} 1294 1295int __kprobes register_kprobes(struct kprobe **kps, int num) 1296{ 1297 int i, ret = 0; 1298 1299 if (num <= 0) 1300 return -EINVAL; 1301 for (i = 0; i < num; i++) { 1302 ret = register_kprobe(kps[i]); 1303 if (ret < 0) { 1304 if (i > 0) 1305 unregister_kprobes(kps, i); 1306 break; 1307 } 1308 } 1309 return ret; 1310} 1311EXPORT_SYMBOL_GPL(register_kprobes); 1312 1313void __kprobes unregister_kprobe(struct kprobe *p) 1314{ 1315 unregister_kprobes(&p, 1); 1316} 1317EXPORT_SYMBOL_GPL(unregister_kprobe); 1318 1319void __kprobes unregister_kprobes(struct kprobe **kps, int num) 1320{ 1321 int i; 1322 1323 if (num <= 0) 1324 return; 1325 mutex_lock(&kprobe_mutex); 1326 for (i = 0; i < num; i++) 1327 if (__unregister_kprobe_top(kps[i]) < 0) 1328 kps[i]->addr = NULL; 1329 mutex_unlock(&kprobe_mutex); 1330 1331 synchronize_sched(); 1332 for (i = 0; i < num; i++) 1333 if (kps[i]->addr) 1334 __unregister_kprobe_bottom(kps[i]); 1335} 1336EXPORT_SYMBOL_GPL(unregister_kprobes); 1337 1338static struct notifier_block kprobe_exceptions_nb = { 1339 .notifier_call = kprobe_exceptions_notify, 1340 .priority = 0x7fffffff /* we need to be notified first */ 1341}; 1342 1343unsigned long __weak arch_deref_entry_point(void *entry) 1344{ 1345 return (unsigned long)entry; 1346} 1347 1348int __kprobes register_jprobes(struct jprobe **jps, int num) 1349{ 1350 struct jprobe *jp; 1351 int ret = 0, i; 1352 1353 if (num <= 0) 1354 return -EINVAL; 1355 for (i = 0; i < num; i++) { 1356 unsigned long addr, offset; 1357 jp = jps[i]; 1358 addr = arch_deref_entry_point(jp->entry); 1359 1360 /* Verify probepoint is a function entry point */ 1361 if (kallsyms_lookup_size_offset(addr, NULL, &offset) && 1362 offset == 0) { 1363 jp->kp.pre_handler = setjmp_pre_handler; 1364 jp->kp.break_handler = longjmp_break_handler; 1365 ret = register_kprobe(&jp->kp); 1366 } else 1367 ret = -EINVAL; 1368 1369 if (ret < 0) { 1370 if (i > 0) 1371 unregister_jprobes(jps, i); 1372 break; 1373 } 1374 } 1375 return ret; 1376} 1377EXPORT_SYMBOL_GPL(register_jprobes); 1378 1379int __kprobes register_jprobe(struct jprobe *jp) 1380{ 1381 return register_jprobes(&jp, 1); 1382} 1383EXPORT_SYMBOL_GPL(register_jprobe); 1384 1385void __kprobes unregister_jprobe(struct jprobe *jp) 1386{ 1387 unregister_jprobes(&jp, 1); 1388} 1389EXPORT_SYMBOL_GPL(unregister_jprobe); 1390 1391void __kprobes unregister_jprobes(struct jprobe **jps, int num) 1392{ 1393 int i; 1394 1395 if (num <= 0) 1396 return; 1397 mutex_lock(&kprobe_mutex); 1398 for (i = 0; i < num; i++) 1399 if (__unregister_kprobe_top(&jps[i]->kp) < 0) 1400 jps[i]->kp.addr = NULL; 1401 mutex_unlock(&kprobe_mutex); 1402 1403 synchronize_sched(); 1404 for (i = 0; i < num; i++) { 1405 if (jps[i]->kp.addr) 1406 __unregister_kprobe_bottom(&jps[i]->kp); 1407 } 1408} 1409EXPORT_SYMBOL_GPL(unregister_jprobes); 1410 1411#ifdef CONFIG_KRETPROBES 1412/* 1413 * This kprobe pre_handler is registered with every kretprobe. When probe 1414 * hits it will set up the return probe. 1415 */ 1416static int __kprobes pre_handler_kretprobe(struct kprobe *p, 1417 struct pt_regs *regs) 1418{ 1419 struct kretprobe *rp = container_of(p, struct kretprobe, kp); 1420 unsigned long hash, flags = 0; 1421 struct kretprobe_instance *ri; 1422 1423 /*TODO: consider to only swap the RA after the last pre_handler fired */ 1424 hash = hash_ptr(current, KPROBE_HASH_BITS); 1425 spin_lock_irqsave(&rp->lock, flags); 1426 if (!hlist_empty(&rp->free_instances)) { 1427 ri = hlist_entry(rp->free_instances.first, 1428 struct kretprobe_instance, hlist); 1429 hlist_del(&ri->hlist); 1430 spin_unlock_irqrestore(&rp->lock, flags); 1431 1432 ri->rp = rp; 1433 ri->task = current; 1434 1435 if (rp->entry_handler && rp->entry_handler(ri, regs)) 1436 return 0; 1437 1438 arch_prepare_kretprobe(ri, regs); 1439 1440 /* XXX(hch): why is there no hlist_move_head? */ 1441 INIT_HLIST_NODE(&ri->hlist); 1442 kretprobe_table_lock(hash, &flags); 1443 hlist_add_head(&ri->hlist, &kretprobe_inst_table[hash]); 1444 kretprobe_table_unlock(hash, &flags); 1445 } else { 1446 rp->nmissed++; 1447 spin_unlock_irqrestore(&rp->lock, flags); 1448 } 1449 return 0; 1450} 1451 1452int __kprobes register_kretprobe(struct kretprobe *rp) 1453{ 1454 int ret = 0; 1455 struct kretprobe_instance *inst; 1456 int i; 1457 void *addr; 1458 1459 if (kretprobe_blacklist_size) { 1460 addr = kprobe_addr(&rp->kp); 1461 if (!addr) 1462 return -EINVAL; 1463 1464 for (i = 0; kretprobe_blacklist[i].name != NULL; i++) { 1465 if (kretprobe_blacklist[i].addr == addr) 1466 return -EINVAL; 1467 } 1468 } 1469 1470 rp->kp.pre_handler = pre_handler_kretprobe; 1471 rp->kp.post_handler = NULL; 1472 rp->kp.fault_handler = NULL; 1473 rp->kp.break_handler = NULL; 1474 1475 /* Pre-allocate memory for max kretprobe instances */ 1476 if (rp->maxactive <= 0) { 1477#ifdef CONFIG_PREEMPT 1478 rp->maxactive = max_t(unsigned int, 10, 2*num_possible_cpus()); 1479#else 1480 rp->maxactive = num_possible_cpus(); 1481#endif 1482 } 1483 spin_lock_init(&rp->lock); 1484 INIT_HLIST_HEAD(&rp->free_instances); 1485 for (i = 0; i < rp->maxactive; i++) { 1486 inst = kmalloc(sizeof(struct kretprobe_instance) + 1487 rp->data_size, GFP_KERNEL); 1488 if (inst == NULL) { 1489 free_rp_inst(rp); 1490 return -ENOMEM; 1491 } 1492 INIT_HLIST_NODE(&inst->hlist); 1493 hlist_add_head(&inst->hlist, &rp->free_instances); 1494 } 1495 1496 rp->nmissed = 0; 1497 /* Establish function entry probe point */ 1498 ret = register_kprobe(&rp->kp); 1499 if (ret != 0) 1500 free_rp_inst(rp); 1501 return ret; 1502} 1503EXPORT_SYMBOL_GPL(register_kretprobe); 1504 1505int __kprobes register_kretprobes(struct kretprobe **rps, int num) 1506{ 1507 int ret = 0, i; 1508 1509 if (num <= 0) 1510 return -EINVAL; 1511 for (i = 0; i < num; i++) { 1512 ret = register_kretprobe(rps[i]); 1513 if (ret < 0) { 1514 if (i > 0) 1515 unregister_kretprobes(rps, i); 1516 break; 1517 } 1518 } 1519 return ret; 1520} 1521EXPORT_SYMBOL_GPL(register_kretprobes); 1522 1523void __kprobes unregister_kretprobe(struct kretprobe *rp) 1524{ 1525 unregister_kretprobes(&rp, 1); 1526} 1527EXPORT_SYMBOL_GPL(unregister_kretprobe); 1528 1529void __kprobes unregister_kretprobes(struct kretprobe **rps, int num) 1530{ 1531 int i; 1532 1533 if (num <= 0) 1534 return; 1535 mutex_lock(&kprobe_mutex); 1536 for (i = 0; i < num; i++) 1537 if (__unregister_kprobe_top(&rps[i]->kp) < 0) 1538 rps[i]->kp.addr = NULL; 1539 mutex_unlock(&kprobe_mutex); 1540 1541 synchronize_sched(); 1542 for (i = 0; i < num; i++) { 1543 if (rps[i]->kp.addr) { 1544 __unregister_kprobe_bottom(&rps[i]->kp); 1545 cleanup_rp_inst(rps[i]); 1546 } 1547 } 1548} 1549EXPORT_SYMBOL_GPL(unregister_kretprobes); 1550 1551#else /* CONFIG_KRETPROBES */ 1552int __kprobes register_kretprobe(struct kretprobe *rp) 1553{ 1554 return -ENOSYS; 1555} 1556EXPORT_SYMBOL_GPL(register_kretprobe); 1557 1558int __kprobes register_kretprobes(struct kretprobe **rps, int num) 1559{ 1560 return -ENOSYS; 1561} 1562EXPORT_SYMBOL_GPL(register_kretprobes); 1563 1564void __kprobes unregister_kretprobe(struct kretprobe *rp) 1565{ 1566} 1567EXPORT_SYMBOL_GPL(unregister_kretprobe); 1568 1569void __kprobes unregister_kretprobes(struct kretprobe **rps, int num) 1570{ 1571} 1572EXPORT_SYMBOL_GPL(unregister_kretprobes); 1573 1574static int __kprobes pre_handler_kretprobe(struct kprobe *p, 1575 struct pt_regs *regs) 1576{ 1577 return 0; 1578} 1579 1580#endif /* CONFIG_KRETPROBES */ 1581 1582/* Set the kprobe gone and remove its instruction buffer. */ 1583static void __kprobes kill_kprobe(struct kprobe *p) 1584{ 1585 struct kprobe *kp; 1586 1587 p->flags |= KPROBE_FLAG_GONE; 1588 if (kprobe_aggrprobe(p)) { 1589 /* 1590 * If this is an aggr_kprobe, we have to list all the 1591 * chained probes and mark them GONE. 1592 */ 1593 list_for_each_entry_rcu(kp, &p->list, list) 1594 kp->flags |= KPROBE_FLAG_GONE; 1595 p->post_handler = NULL; 1596 p->break_handler = NULL; 1597 kill_optimized_kprobe(p); 1598 } 1599 /* 1600 * Here, we can remove insn_slot safely, because no thread calls 1601 * the original probed function (which will be freed soon) any more. 1602 */ 1603 arch_remove_kprobe(p); 1604} 1605 1606/* Disable one kprobe */ 1607int __kprobes disable_kprobe(struct kprobe *kp) 1608{ 1609 int ret = 0; 1610 struct kprobe *p; 1611 1612 mutex_lock(&kprobe_mutex); 1613 1614 /* Check whether specified probe is valid. */ 1615 p = __get_valid_kprobe(kp); 1616 if (unlikely(p == NULL)) { 1617 ret = -EINVAL; 1618 goto out; 1619 } 1620 1621 /* If the probe is already disabled (or gone), just return */ 1622 if (kprobe_disabled(kp)) 1623 goto out; 1624 1625 kp->flags |= KPROBE_FLAG_DISABLED; 1626 if (p != kp) 1627 /* When kp != p, p is always enabled. */ 1628 try_to_disable_aggr_kprobe(p); 1629 1630 if (!kprobes_all_disarmed && kprobe_disabled(p)) 1631 disarm_kprobe(p); 1632out: 1633 mutex_unlock(&kprobe_mutex); 1634 return ret; 1635} 1636EXPORT_SYMBOL_GPL(disable_kprobe); 1637 1638/* Enable one kprobe */ 1639int __kprobes enable_kprobe(struct kprobe *kp) 1640{ 1641 int ret = 0; 1642 struct kprobe *p; 1643 1644 mutex_lock(&kprobe_mutex); 1645 1646 /* Check whether specified probe is valid. */ 1647 p = __get_valid_kprobe(kp); 1648 if (unlikely(p == NULL)) { 1649 ret = -EINVAL; 1650 goto out; 1651 } 1652 1653 if (kprobe_gone(kp)) { 1654 /* This kprobe has gone, we couldn't enable it. */ 1655 ret = -EINVAL; 1656 goto out; 1657 } 1658 1659 if (p != kp) 1660 kp->flags &= ~KPROBE_FLAG_DISABLED; 1661 1662 if (!kprobes_all_disarmed && kprobe_disabled(p)) { 1663 p->flags &= ~KPROBE_FLAG_DISABLED; 1664 arm_kprobe(p); 1665 } 1666out: 1667 mutex_unlock(&kprobe_mutex); 1668 return ret; 1669} 1670EXPORT_SYMBOL_GPL(enable_kprobe); 1671 1672void __kprobes dump_kprobe(struct kprobe *kp) 1673{ 1674 printk(KERN_WARNING "Dumping kprobe:\n"); 1675 printk(KERN_WARNING "Name: %s\nAddress: %p\nOffset: %x\n", 1676 kp->symbol_name, kp->addr, kp->offset); 1677} 1678 1679/* Module notifier call back, checking kprobes on the module */ 1680static int __kprobes kprobes_module_callback(struct notifier_block *nb, 1681 unsigned long val, void *data) 1682{ 1683 struct module *mod = data; 1684 struct hlist_head *head; 1685 struct hlist_node *node; 1686 struct kprobe *p; 1687 unsigned int i; 1688 int checkcore = (val == MODULE_STATE_GOING); 1689 1690 if (val != MODULE_STATE_GOING && val != MODULE_STATE_LIVE) 1691 return NOTIFY_DONE; 1692 1693 /* 1694 * When MODULE_STATE_GOING was notified, both of module .text and 1695 * .init.text sections would be freed. When MODULE_STATE_LIVE was 1696 * notified, only .init.text section would be freed. We need to 1697 * disable kprobes which have been inserted in the sections. 1698 */ 1699 mutex_lock(&kprobe_mutex); 1700 for (i = 0; i < KPROBE_TABLE_SIZE; i++) { 1701 head = &kprobe_table[i]; 1702 hlist_for_each_entry_rcu(p, node, head, hlist) 1703 if (within_module_init((unsigned long)p->addr, mod) || 1704 (checkcore && 1705 within_module_core((unsigned long)p->addr, mod))) { 1706 /* 1707 * The vaddr this probe is installed will soon 1708 * be vfreed buy not synced to disk. Hence, 1709 * disarming the breakpoint isn't needed. 1710 */ 1711 kill_kprobe(p); 1712 } 1713 } 1714 mutex_unlock(&kprobe_mutex); 1715 return NOTIFY_DONE; 1716} 1717 1718static struct notifier_block kprobe_module_nb = { 1719 .notifier_call = kprobes_module_callback, 1720 .priority = 0 1721}; 1722 1723static int __init init_kprobes(void) 1724{ 1725 int i, err = 0; 1726 unsigned long offset = 0, size = 0; 1727 char *modname, namebuf[128]; 1728 const char *symbol_name; 1729 void *addr; 1730 struct kprobe_blackpoint *kb; 1731 1732 /* FIXME allocate the probe table, currently defined statically */ 1733 /* initialize all list heads */ 1734 for (i = 0; i < KPROBE_TABLE_SIZE; i++) { 1735 INIT_HLIST_HEAD(&kprobe_table[i]); 1736 INIT_HLIST_HEAD(&kretprobe_inst_table[i]); 1737 spin_lock_init(&(kretprobe_table_locks[i].lock)); 1738 } 1739 1740 /* 1741 * Lookup and populate the kprobe_blacklist. 1742 * 1743 * Unlike the kretprobe blacklist, we'll need to determine 1744 * the range of addresses that belong to the said functions, 1745 * since a kprobe need not necessarily be at the beginning 1746 * of a function. 1747 */ 1748 for (kb = kprobe_blacklist; kb->name != NULL; kb++) { 1749 kprobe_lookup_name(kb->name, addr); 1750 if (!addr) 1751 continue; 1752 1753 kb->start_addr = (unsigned long)addr; 1754 symbol_name = kallsyms_lookup(kb->start_addr, 1755 &size, &offset, &modname, namebuf); 1756 if (!symbol_name) 1757 kb->range = 0; 1758 else 1759 kb->range = size; 1760 } 1761 1762 if (kretprobe_blacklist_size) { 1763 /* lookup the function address from its name */ 1764 for (i = 0; kretprobe_blacklist[i].name != NULL; i++) { 1765 kprobe_lookup_name(kretprobe_blacklist[i].name, 1766 kretprobe_blacklist[i].addr); 1767 if (!kretprobe_blacklist[i].addr) 1768 printk("kretprobe: lookup failed: %s\n", 1769 kretprobe_blacklist[i].name); 1770 } 1771 } 1772 1773#if defined(CONFIG_OPTPROBES) 1774#if defined(__ARCH_WANT_KPROBES_INSN_SLOT) 1775 /* Init kprobe_optinsn_slots */ 1776 kprobe_optinsn_slots.insn_size = MAX_OPTINSN_SIZE; 1777#endif 1778 /* By default, kprobes can be optimized */ 1779 kprobes_allow_optimization = true; 1780#endif 1781 1782 /* By default, kprobes are armed */ 1783 kprobes_all_disarmed = false; 1784 1785 err = arch_init_kprobes(); 1786 if (!err) 1787 err = register_die_notifier(&kprobe_exceptions_nb); 1788 if (!err) 1789 err = register_module_notifier(&kprobe_module_nb); 1790 1791 kprobes_initialized = (err == 0); 1792 1793 if (!err) 1794 init_test_probes(); 1795 return err; 1796} 1797 1798#ifdef CONFIG_DEBUG_FS 1799static void __kprobes report_probe(struct seq_file *pi, struct kprobe *p, 1800 const char *sym, int offset, char *modname, struct kprobe *pp) 1801{ 1802 char *kprobe_type; 1803 1804 if (p->pre_handler == pre_handler_kretprobe) 1805 kprobe_type = "r"; 1806 else if (p->pre_handler == setjmp_pre_handler) 1807 kprobe_type = "j"; 1808 else 1809 kprobe_type = "k"; 1810 1811 if (sym) 1812 seq_printf(pi, "%p %s %s+0x%x %s ", 1813 p->addr, kprobe_type, sym, offset, 1814 (modname ? modname : " ")); 1815 else 1816 seq_printf(pi, "%p %s %p ", 1817 p->addr, kprobe_type, p->addr); 1818 1819 if (!pp) 1820 pp = p; 1821 seq_printf(pi, "%s%s%s\n", 1822 (kprobe_gone(p) ? "[GONE]" : ""), 1823 ((kprobe_disabled(p) && !kprobe_gone(p)) ? "[DISABLED]" : ""), 1824 (kprobe_optimized(pp) ? "[OPTIMIZED]" : "")); 1825} 1826 1827static void __kprobes *kprobe_seq_start(struct seq_file *f, loff_t *pos) 1828{ 1829 return (*pos < KPROBE_TABLE_SIZE) ? pos : NULL; 1830} 1831 1832static void __kprobes *kprobe_seq_next(struct seq_file *f, void *v, loff_t *pos) 1833{ 1834 (*pos)++; 1835 if (*pos >= KPROBE_TABLE_SIZE) 1836 return NULL; 1837 return pos; 1838} 1839 1840static void __kprobes kprobe_seq_stop(struct seq_file *f, void *v) 1841{ 1842 /* Nothing to do */ 1843} 1844 1845static int __kprobes show_kprobe_addr(struct seq_file *pi, void *v) 1846{ 1847 struct hlist_head *head; 1848 struct hlist_node *node; 1849 struct kprobe *p, *kp; 1850 const char *sym = NULL; 1851 unsigned int i = *(loff_t *) v; 1852 unsigned long offset = 0; 1853 char *modname, namebuf[128]; 1854 1855 head = &kprobe_table[i]; 1856 preempt_disable(); 1857 hlist_for_each_entry_rcu(p, node, head, hlist) { 1858 sym = kallsyms_lookup((unsigned long)p->addr, NULL, 1859 &offset, &modname, namebuf); 1860 if (kprobe_aggrprobe(p)) { 1861 list_for_each_entry_rcu(kp, &p->list, list) 1862 report_probe(pi, kp, sym, offset, modname, p); 1863 } else 1864 report_probe(pi, p, sym, offset, modname, NULL); 1865 } 1866 preempt_enable(); 1867 return 0; 1868} 1869 1870static const struct seq_operations kprobes_seq_ops = { 1871 .start = kprobe_seq_start, 1872 .next = kprobe_seq_next, 1873 .stop = kprobe_seq_stop, 1874 .show = show_kprobe_addr 1875}; 1876 1877static int __kprobes kprobes_open(struct inode *inode, struct file *filp) 1878{ 1879 return seq_open(filp, &kprobes_seq_ops); 1880} 1881 1882static const struct file_operations debugfs_kprobes_operations = { 1883 .open = kprobes_open, 1884 .read = seq_read, 1885 .llseek = seq_lseek, 1886 .release = seq_release, 1887}; 1888 1889static void __kprobes arm_all_kprobes(void) 1890{ 1891 struct hlist_head *head; 1892 struct hlist_node *node; 1893 struct kprobe *p; 1894 unsigned int i; 1895 1896 mutex_lock(&kprobe_mutex); 1897 1898 /* If kprobes are armed, just return */ 1899 if (!kprobes_all_disarmed) 1900 goto already_enabled; 1901 1902 /* Arming kprobes doesn't optimize kprobe itself */ 1903 mutex_lock(&text_mutex); 1904 for (i = 0; i < KPROBE_TABLE_SIZE; i++) { 1905 head = &kprobe_table[i]; 1906 hlist_for_each_entry_rcu(p, node, head, hlist) 1907 if (!kprobe_disabled(p)) 1908 __arm_kprobe(p); 1909 } 1910 mutex_unlock(&text_mutex); 1911 1912 kprobes_all_disarmed = false; 1913 printk(KERN_INFO "Kprobes globally enabled\n"); 1914 1915already_enabled: 1916 mutex_unlock(&kprobe_mutex); 1917 return; 1918} 1919 1920static void __kprobes disarm_all_kprobes(void) 1921{ 1922 struct hlist_head *head; 1923 struct hlist_node *node; 1924 struct kprobe *p; 1925 unsigned int i; 1926 1927 mutex_lock(&kprobe_mutex); 1928 1929 /* If kprobes are already disarmed, just return */ 1930 if (kprobes_all_disarmed) 1931 goto already_disabled; 1932 1933 kprobes_all_disarmed = true; 1934 printk(KERN_INFO "Kprobes globally disabled\n"); 1935 1936 /* 1937 * Here we call get_online_cpus() for avoiding text_mutex deadlock, 1938 * because disarming may also unoptimize kprobes. 1939 */ 1940 get_online_cpus(); 1941 mutex_lock(&text_mutex); 1942 for (i = 0; i < KPROBE_TABLE_SIZE; i++) { 1943 head = &kprobe_table[i]; 1944 hlist_for_each_entry_rcu(p, node, head, hlist) { 1945 if (!arch_trampoline_kprobe(p) && !kprobe_disabled(p)) 1946 __disarm_kprobe(p); 1947 } 1948 } 1949 1950 mutex_unlock(&text_mutex); 1951 put_online_cpus(); 1952 mutex_unlock(&kprobe_mutex); 1953 /* Allow all currently running kprobes to complete */ 1954 synchronize_sched(); 1955 return; 1956 1957already_disabled: 1958 mutex_unlock(&kprobe_mutex); 1959 return; 1960} 1961 1962/* 1963 * XXX: The debugfs bool file interface doesn't allow for callbacks 1964 * when the bool state is switched. We can reuse that facility when 1965 * available 1966 */ 1967static ssize_t read_enabled_file_bool(struct file *file, 1968 char __user *user_buf, size_t count, loff_t *ppos) 1969{ 1970 char buf[3]; 1971 1972 if (!kprobes_all_disarmed) 1973 buf[0] = '1'; 1974 else 1975 buf[0] = '0'; 1976 buf[1] = '\n'; 1977 buf[2] = 0x00; 1978 return simple_read_from_buffer(user_buf, count, ppos, buf, 2); 1979} 1980 1981static ssize_t write_enabled_file_bool(struct file *file, 1982 const char __user *user_buf, size_t count, loff_t *ppos) 1983{ 1984 char buf[32]; 1985 int buf_size; 1986 1987 buf_size = min(count, (sizeof(buf)-1)); 1988 if (copy_from_user(buf, user_buf, buf_size)) 1989 return -EFAULT; 1990 1991 switch (buf[0]) { 1992 case 'y': 1993 case 'Y': 1994 case '1': 1995 arm_all_kprobes(); 1996 break; 1997 case 'n': 1998 case 'N': 1999 case '0': 2000 disarm_all_kprobes(); 2001 break; 2002 } 2003 2004 return count; 2005} 2006 2007static const struct file_operations fops_kp = { 2008 .read = read_enabled_file_bool, 2009 .write = write_enabled_file_bool, 2010 .llseek = default_llseek, 2011}; 2012 2013static int __kprobes debugfs_kprobe_init(void) 2014{ 2015 struct dentry *dir, *file; 2016 unsigned int value = 1; 2017 2018 dir = debugfs_create_dir("kprobes", NULL); 2019 if (!dir) 2020 return -ENOMEM; 2021 2022 file = debugfs_create_file("list", 0444, dir, NULL, 2023 &debugfs_kprobes_operations); 2024 if (!file) { 2025 debugfs_remove(dir); 2026 return -ENOMEM; 2027 } 2028 2029 file = debugfs_create_file("enabled", 0600, dir, 2030 &value, &fops_kp); 2031 if (!file) { 2032 debugfs_remove(dir); 2033 return -ENOMEM; 2034 } 2035 2036 return 0; 2037} 2038 2039late_initcall(debugfs_kprobe_init); 2040#endif /* CONFIG_DEBUG_FS */ 2041 2042module_init(init_kprobes); 2043 2044/* defined in arch/.../kernel/kprobes.c */ 2045EXPORT_SYMBOL_GPL(jprobe_return); 2046