kprobes.c revision e52538965119319447c0800c534da73142c27be2
1/* 2 * Kernel Probes (KProbes) 3 * kernel/kprobes.c 4 * 5 * This program is free software; you can redistribute it and/or modify 6 * it under the terms of the GNU General Public License as published by 7 * the Free Software Foundation; either version 2 of the License, or 8 * (at your option) any later version. 9 * 10 * This program is distributed in the hope that it will be useful, 11 * but WITHOUT ANY WARRANTY; without even the implied warranty of 12 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 13 * GNU General Public License for more details. 14 * 15 * You should have received a copy of the GNU General Public License 16 * along with this program; if not, write to the Free Software 17 * Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA. 18 * 19 * Copyright (C) IBM Corporation, 2002, 2004 20 * 21 * 2002-Oct Created by Vamsi Krishna S <vamsi_krishna@in.ibm.com> Kernel 22 * Probes initial implementation (includes suggestions from 23 * Rusty Russell). 24 * 2004-Aug Updated by Prasanna S Panchamukhi <prasanna@in.ibm.com> with 25 * hlists and exceptions notifier as suggested by Andi Kleen. 26 * 2004-July Suparna Bhattacharya <suparna@in.ibm.com> added jumper probes 27 * interface to access function arguments. 28 * 2004-Sep Prasanna S Panchamukhi <prasanna@in.ibm.com> Changed Kprobes 29 * exceptions notifier to be first on the priority list. 30 * 2005-May Hien Nguyen <hien@us.ibm.com>, Jim Keniston 31 * <jkenisto@us.ibm.com> and Prasanna S Panchamukhi 32 * <prasanna@in.ibm.com> added function-return probes. 33 */ 34#include <linux/kprobes.h> 35#include <linux/hash.h> 36#include <linux/init.h> 37#include <linux/slab.h> 38#include <linux/stddef.h> 39#include <linux/export.h> 40#include <linux/moduleloader.h> 41#include <linux/kallsyms.h> 42#include <linux/freezer.h> 43#include <linux/seq_file.h> 44#include <linux/debugfs.h> 45#include <linux/sysctl.h> 46#include <linux/kdebug.h> 47#include <linux/memory.h> 48#include <linux/ftrace.h> 49#include <linux/cpu.h> 50#include <linux/jump_label.h> 51 52#include <asm-generic/sections.h> 53#include <asm/cacheflush.h> 54#include <asm/errno.h> 55#include <asm/uaccess.h> 56 57#define KPROBE_HASH_BITS 6 58#define KPROBE_TABLE_SIZE (1 << KPROBE_HASH_BITS) 59 60 61/* 62 * Some oddball architectures like 64bit powerpc have function descriptors 63 * so this must be overridable. 64 */ 65#ifndef kprobe_lookup_name 66#define kprobe_lookup_name(name, addr) \ 67 addr = ((kprobe_opcode_t *)(kallsyms_lookup_name(name))) 68#endif 69 70static int kprobes_initialized; 71static struct hlist_head kprobe_table[KPROBE_TABLE_SIZE]; 72static struct hlist_head kretprobe_inst_table[KPROBE_TABLE_SIZE]; 73 74/* NOTE: change this value only with kprobe_mutex held */ 75static bool kprobes_all_disarmed; 76 77/* This protects kprobe_table and optimizing_list */ 78static DEFINE_MUTEX(kprobe_mutex); 79static DEFINE_PER_CPU(struct kprobe *, kprobe_instance) = NULL; 80static struct { 81 raw_spinlock_t lock ____cacheline_aligned_in_smp; 82} kretprobe_table_locks[KPROBE_TABLE_SIZE]; 83 84static raw_spinlock_t *kretprobe_table_lock_ptr(unsigned long hash) 85{ 86 return &(kretprobe_table_locks[hash].lock); 87} 88 89/* 90 * Normally, functions that we'd want to prohibit kprobes in, are marked 91 * __kprobes. But, there are cases where such functions already belong to 92 * a different section (__sched for preempt_schedule) 93 * 94 * For such cases, we now have a blacklist 95 */ 96static struct kprobe_blackpoint kprobe_blacklist[] = { 97 {"preempt_schedule",}, 98 {"native_get_debugreg",}, 99 {"irq_entries_start",}, 100 {"common_interrupt",}, 101 {"mcount",}, /* mcount can be called from everywhere */ 102 {NULL} /* Terminator */ 103}; 104 105#ifdef __ARCH_WANT_KPROBES_INSN_SLOT 106/* 107 * kprobe->ainsn.insn points to the copy of the instruction to be 108 * single-stepped. x86_64, POWER4 and above have no-exec support and 109 * stepping on the instruction on a vmalloced/kmalloced/data page 110 * is a recipe for disaster 111 */ 112struct kprobe_insn_page { 113 struct list_head list; 114 kprobe_opcode_t *insns; /* Page of instruction slots */ 115 int nused; 116 int ngarbage; 117 char slot_used[]; 118}; 119 120#define KPROBE_INSN_PAGE_SIZE(slots) \ 121 (offsetof(struct kprobe_insn_page, slot_used) + \ 122 (sizeof(char) * (slots))) 123 124struct kprobe_insn_cache { 125 struct list_head pages; /* list of kprobe_insn_page */ 126 size_t insn_size; /* size of instruction slot */ 127 int nr_garbage; 128}; 129 130static int slots_per_page(struct kprobe_insn_cache *c) 131{ 132 return PAGE_SIZE/(c->insn_size * sizeof(kprobe_opcode_t)); 133} 134 135enum kprobe_slot_state { 136 SLOT_CLEAN = 0, 137 SLOT_DIRTY = 1, 138 SLOT_USED = 2, 139}; 140 141static DEFINE_MUTEX(kprobe_insn_mutex); /* Protects kprobe_insn_slots */ 142static struct kprobe_insn_cache kprobe_insn_slots = { 143 .pages = LIST_HEAD_INIT(kprobe_insn_slots.pages), 144 .insn_size = MAX_INSN_SIZE, 145 .nr_garbage = 0, 146}; 147static int __kprobes collect_garbage_slots(struct kprobe_insn_cache *c); 148 149/** 150 * __get_insn_slot() - Find a slot on an executable page for an instruction. 151 * We allocate an executable page if there's no room on existing ones. 152 */ 153static kprobe_opcode_t __kprobes *__get_insn_slot(struct kprobe_insn_cache *c) 154{ 155 struct kprobe_insn_page *kip; 156 157 retry: 158 list_for_each_entry(kip, &c->pages, list) { 159 if (kip->nused < slots_per_page(c)) { 160 int i; 161 for (i = 0; i < slots_per_page(c); i++) { 162 if (kip->slot_used[i] == SLOT_CLEAN) { 163 kip->slot_used[i] = SLOT_USED; 164 kip->nused++; 165 return kip->insns + (i * c->insn_size); 166 } 167 } 168 /* kip->nused is broken. Fix it. */ 169 kip->nused = slots_per_page(c); 170 WARN_ON(1); 171 } 172 } 173 174 /* If there are any garbage slots, collect it and try again. */ 175 if (c->nr_garbage && collect_garbage_slots(c) == 0) 176 goto retry; 177 178 /* All out of space. Need to allocate a new page. */ 179 kip = kmalloc(KPROBE_INSN_PAGE_SIZE(slots_per_page(c)), GFP_KERNEL); 180 if (!kip) 181 return NULL; 182 183 /* 184 * Use module_alloc so this page is within +/- 2GB of where the 185 * kernel image and loaded module images reside. This is required 186 * so x86_64 can correctly handle the %rip-relative fixups. 187 */ 188 kip->insns = module_alloc(PAGE_SIZE); 189 if (!kip->insns) { 190 kfree(kip); 191 return NULL; 192 } 193 INIT_LIST_HEAD(&kip->list); 194 memset(kip->slot_used, SLOT_CLEAN, slots_per_page(c)); 195 kip->slot_used[0] = SLOT_USED; 196 kip->nused = 1; 197 kip->ngarbage = 0; 198 list_add(&kip->list, &c->pages); 199 return kip->insns; 200} 201 202 203kprobe_opcode_t __kprobes *get_insn_slot(void) 204{ 205 kprobe_opcode_t *ret = NULL; 206 207 mutex_lock(&kprobe_insn_mutex); 208 ret = __get_insn_slot(&kprobe_insn_slots); 209 mutex_unlock(&kprobe_insn_mutex); 210 211 return ret; 212} 213 214/* Return 1 if all garbages are collected, otherwise 0. */ 215static int __kprobes collect_one_slot(struct kprobe_insn_page *kip, int idx) 216{ 217 kip->slot_used[idx] = SLOT_CLEAN; 218 kip->nused--; 219 if (kip->nused == 0) { 220 /* 221 * Page is no longer in use. Free it unless 222 * it's the last one. We keep the last one 223 * so as not to have to set it up again the 224 * next time somebody inserts a probe. 225 */ 226 if (!list_is_singular(&kip->list)) { 227 list_del(&kip->list); 228 module_free(NULL, kip->insns); 229 kfree(kip); 230 } 231 return 1; 232 } 233 return 0; 234} 235 236static int __kprobes collect_garbage_slots(struct kprobe_insn_cache *c) 237{ 238 struct kprobe_insn_page *kip, *next; 239 240 /* Ensure no-one is interrupted on the garbages */ 241 synchronize_sched(); 242 243 list_for_each_entry_safe(kip, next, &c->pages, list) { 244 int i; 245 if (kip->ngarbage == 0) 246 continue; 247 kip->ngarbage = 0; /* we will collect all garbages */ 248 for (i = 0; i < slots_per_page(c); i++) { 249 if (kip->slot_used[i] == SLOT_DIRTY && 250 collect_one_slot(kip, i)) 251 break; 252 } 253 } 254 c->nr_garbage = 0; 255 return 0; 256} 257 258static void __kprobes __free_insn_slot(struct kprobe_insn_cache *c, 259 kprobe_opcode_t *slot, int dirty) 260{ 261 struct kprobe_insn_page *kip; 262 263 list_for_each_entry(kip, &c->pages, list) { 264 long idx = ((long)slot - (long)kip->insns) / 265 (c->insn_size * sizeof(kprobe_opcode_t)); 266 if (idx >= 0 && idx < slots_per_page(c)) { 267 WARN_ON(kip->slot_used[idx] != SLOT_USED); 268 if (dirty) { 269 kip->slot_used[idx] = SLOT_DIRTY; 270 kip->ngarbage++; 271 if (++c->nr_garbage > slots_per_page(c)) 272 collect_garbage_slots(c); 273 } else 274 collect_one_slot(kip, idx); 275 return; 276 } 277 } 278 /* Could not free this slot. */ 279 WARN_ON(1); 280} 281 282void __kprobes free_insn_slot(kprobe_opcode_t * slot, int dirty) 283{ 284 mutex_lock(&kprobe_insn_mutex); 285 __free_insn_slot(&kprobe_insn_slots, slot, dirty); 286 mutex_unlock(&kprobe_insn_mutex); 287} 288#ifdef CONFIG_OPTPROBES 289/* For optimized_kprobe buffer */ 290static DEFINE_MUTEX(kprobe_optinsn_mutex); /* Protects kprobe_optinsn_slots */ 291static struct kprobe_insn_cache kprobe_optinsn_slots = { 292 .pages = LIST_HEAD_INIT(kprobe_optinsn_slots.pages), 293 /* .insn_size is initialized later */ 294 .nr_garbage = 0, 295}; 296/* Get a slot for optimized_kprobe buffer */ 297kprobe_opcode_t __kprobes *get_optinsn_slot(void) 298{ 299 kprobe_opcode_t *ret = NULL; 300 301 mutex_lock(&kprobe_optinsn_mutex); 302 ret = __get_insn_slot(&kprobe_optinsn_slots); 303 mutex_unlock(&kprobe_optinsn_mutex); 304 305 return ret; 306} 307 308void __kprobes free_optinsn_slot(kprobe_opcode_t * slot, int dirty) 309{ 310 mutex_lock(&kprobe_optinsn_mutex); 311 __free_insn_slot(&kprobe_optinsn_slots, slot, dirty); 312 mutex_unlock(&kprobe_optinsn_mutex); 313} 314#endif 315#endif 316 317/* We have preemption disabled.. so it is safe to use __ versions */ 318static inline void set_kprobe_instance(struct kprobe *kp) 319{ 320 __this_cpu_write(kprobe_instance, kp); 321} 322 323static inline void reset_kprobe_instance(void) 324{ 325 __this_cpu_write(kprobe_instance, NULL); 326} 327 328/* 329 * This routine is called either: 330 * - under the kprobe_mutex - during kprobe_[un]register() 331 * OR 332 * - with preemption disabled - from arch/xxx/kernel/kprobes.c 333 */ 334struct kprobe __kprobes *get_kprobe(void *addr) 335{ 336 struct hlist_head *head; 337 struct hlist_node *node; 338 struct kprobe *p; 339 340 head = &kprobe_table[hash_ptr(addr, KPROBE_HASH_BITS)]; 341 hlist_for_each_entry_rcu(p, node, head, hlist) { 342 if (p->addr == addr) 343 return p; 344 } 345 346 return NULL; 347} 348 349static int __kprobes aggr_pre_handler(struct kprobe *p, struct pt_regs *regs); 350 351/* Return true if the kprobe is an aggregator */ 352static inline int kprobe_aggrprobe(struct kprobe *p) 353{ 354 return p->pre_handler == aggr_pre_handler; 355} 356 357/* Return true(!0) if the kprobe is unused */ 358static inline int kprobe_unused(struct kprobe *p) 359{ 360 return kprobe_aggrprobe(p) && kprobe_disabled(p) && 361 list_empty(&p->list); 362} 363 364/* 365 * Keep all fields in the kprobe consistent 366 */ 367static inline void copy_kprobe(struct kprobe *ap, struct kprobe *p) 368{ 369 memcpy(&p->opcode, &ap->opcode, sizeof(kprobe_opcode_t)); 370 memcpy(&p->ainsn, &ap->ainsn, sizeof(struct arch_specific_insn)); 371} 372 373#ifdef CONFIG_OPTPROBES 374/* NOTE: change this value only with kprobe_mutex held */ 375static bool kprobes_allow_optimization; 376 377/* 378 * Call all pre_handler on the list, but ignores its return value. 379 * This must be called from arch-dep optimized caller. 380 */ 381void __kprobes opt_pre_handler(struct kprobe *p, struct pt_regs *regs) 382{ 383 struct kprobe *kp; 384 385 list_for_each_entry_rcu(kp, &p->list, list) { 386 if (kp->pre_handler && likely(!kprobe_disabled(kp))) { 387 set_kprobe_instance(kp); 388 kp->pre_handler(kp, regs); 389 } 390 reset_kprobe_instance(); 391 } 392} 393 394/* Free optimized instructions and optimized_kprobe */ 395static __kprobes void free_aggr_kprobe(struct kprobe *p) 396{ 397 struct optimized_kprobe *op; 398 399 op = container_of(p, struct optimized_kprobe, kp); 400 arch_remove_optimized_kprobe(op); 401 arch_remove_kprobe(p); 402 kfree(op); 403} 404 405/* Return true(!0) if the kprobe is ready for optimization. */ 406static inline int kprobe_optready(struct kprobe *p) 407{ 408 struct optimized_kprobe *op; 409 410 if (kprobe_aggrprobe(p)) { 411 op = container_of(p, struct optimized_kprobe, kp); 412 return arch_prepared_optinsn(&op->optinsn); 413 } 414 415 return 0; 416} 417 418/* Return true(!0) if the kprobe is disarmed. Note: p must be on hash list */ 419static inline int kprobe_disarmed(struct kprobe *p) 420{ 421 struct optimized_kprobe *op; 422 423 /* If kprobe is not aggr/opt probe, just return kprobe is disabled */ 424 if (!kprobe_aggrprobe(p)) 425 return kprobe_disabled(p); 426 427 op = container_of(p, struct optimized_kprobe, kp); 428 429 return kprobe_disabled(p) && list_empty(&op->list); 430} 431 432/* Return true(!0) if the probe is queued on (un)optimizing lists */ 433static int __kprobes kprobe_queued(struct kprobe *p) 434{ 435 struct optimized_kprobe *op; 436 437 if (kprobe_aggrprobe(p)) { 438 op = container_of(p, struct optimized_kprobe, kp); 439 if (!list_empty(&op->list)) 440 return 1; 441 } 442 return 0; 443} 444 445/* 446 * Return an optimized kprobe whose optimizing code replaces 447 * instructions including addr (exclude breakpoint). 448 */ 449static struct kprobe *__kprobes get_optimized_kprobe(unsigned long addr) 450{ 451 int i; 452 struct kprobe *p = NULL; 453 struct optimized_kprobe *op; 454 455 /* Don't check i == 0, since that is a breakpoint case. */ 456 for (i = 1; !p && i < MAX_OPTIMIZED_LENGTH; i++) 457 p = get_kprobe((void *)(addr - i)); 458 459 if (p && kprobe_optready(p)) { 460 op = container_of(p, struct optimized_kprobe, kp); 461 if (arch_within_optimized_kprobe(op, addr)) 462 return p; 463 } 464 465 return NULL; 466} 467 468/* Optimization staging list, protected by kprobe_mutex */ 469static LIST_HEAD(optimizing_list); 470static LIST_HEAD(unoptimizing_list); 471 472static void kprobe_optimizer(struct work_struct *work); 473static DECLARE_DELAYED_WORK(optimizing_work, kprobe_optimizer); 474static DECLARE_COMPLETION(optimizer_comp); 475#define OPTIMIZE_DELAY 5 476 477/* 478 * Optimize (replace a breakpoint with a jump) kprobes listed on 479 * optimizing_list. 480 */ 481static __kprobes void do_optimize_kprobes(void) 482{ 483 /* Optimization never be done when disarmed */ 484 if (kprobes_all_disarmed || !kprobes_allow_optimization || 485 list_empty(&optimizing_list)) 486 return; 487 488 /* 489 * The optimization/unoptimization refers online_cpus via 490 * stop_machine() and cpu-hotplug modifies online_cpus. 491 * And same time, text_mutex will be held in cpu-hotplug and here. 492 * This combination can cause a deadlock (cpu-hotplug try to lock 493 * text_mutex but stop_machine can not be done because online_cpus 494 * has been changed) 495 * To avoid this deadlock, we need to call get_online_cpus() 496 * for preventing cpu-hotplug outside of text_mutex locking. 497 */ 498 get_online_cpus(); 499 mutex_lock(&text_mutex); 500 arch_optimize_kprobes(&optimizing_list); 501 mutex_unlock(&text_mutex); 502 put_online_cpus(); 503} 504 505/* 506 * Unoptimize (replace a jump with a breakpoint and remove the breakpoint 507 * if need) kprobes listed on unoptimizing_list. 508 */ 509static __kprobes void do_unoptimize_kprobes(struct list_head *free_list) 510{ 511 struct optimized_kprobe *op, *tmp; 512 513 /* Unoptimization must be done anytime */ 514 if (list_empty(&unoptimizing_list)) 515 return; 516 517 /* Ditto to do_optimize_kprobes */ 518 get_online_cpus(); 519 mutex_lock(&text_mutex); 520 arch_unoptimize_kprobes(&unoptimizing_list, free_list); 521 /* Loop free_list for disarming */ 522 list_for_each_entry_safe(op, tmp, free_list, list) { 523 /* Disarm probes if marked disabled */ 524 if (kprobe_disabled(&op->kp)) 525 arch_disarm_kprobe(&op->kp); 526 if (kprobe_unused(&op->kp)) { 527 /* 528 * Remove unused probes from hash list. After waiting 529 * for synchronization, these probes are reclaimed. 530 * (reclaiming is done by do_free_cleaned_kprobes.) 531 */ 532 hlist_del_rcu(&op->kp.hlist); 533 } else 534 list_del_init(&op->list); 535 } 536 mutex_unlock(&text_mutex); 537 put_online_cpus(); 538} 539 540/* Reclaim all kprobes on the free_list */ 541static __kprobes void do_free_cleaned_kprobes(struct list_head *free_list) 542{ 543 struct optimized_kprobe *op, *tmp; 544 545 list_for_each_entry_safe(op, tmp, free_list, list) { 546 BUG_ON(!kprobe_unused(&op->kp)); 547 list_del_init(&op->list); 548 free_aggr_kprobe(&op->kp); 549 } 550} 551 552/* Start optimizer after OPTIMIZE_DELAY passed */ 553static __kprobes void kick_kprobe_optimizer(void) 554{ 555 if (!delayed_work_pending(&optimizing_work)) 556 schedule_delayed_work(&optimizing_work, OPTIMIZE_DELAY); 557} 558 559/* Kprobe jump optimizer */ 560static __kprobes void kprobe_optimizer(struct work_struct *work) 561{ 562 LIST_HEAD(free_list); 563 564 mutex_lock(&kprobe_mutex); 565 /* Lock modules while optimizing kprobes */ 566 mutex_lock(&module_mutex); 567 568 /* 569 * Step 1: Unoptimize kprobes and collect cleaned (unused and disarmed) 570 * kprobes before waiting for quiesence period. 571 */ 572 do_unoptimize_kprobes(&free_list); 573 574 /* 575 * Step 2: Wait for quiesence period to ensure all running interrupts 576 * are done. Because optprobe may modify multiple instructions 577 * there is a chance that Nth instruction is interrupted. In that 578 * case, running interrupt can return to 2nd-Nth byte of jump 579 * instruction. This wait is for avoiding it. 580 */ 581 synchronize_sched(); 582 583 /* Step 3: Optimize kprobes after quiesence period */ 584 do_optimize_kprobes(); 585 586 /* Step 4: Free cleaned kprobes after quiesence period */ 587 do_free_cleaned_kprobes(&free_list); 588 589 mutex_unlock(&module_mutex); 590 mutex_unlock(&kprobe_mutex); 591 592 /* Step 5: Kick optimizer again if needed */ 593 if (!list_empty(&optimizing_list) || !list_empty(&unoptimizing_list)) 594 kick_kprobe_optimizer(); 595 else 596 /* Wake up all waiters */ 597 complete_all(&optimizer_comp); 598} 599 600/* Wait for completing optimization and unoptimization */ 601static __kprobes void wait_for_kprobe_optimizer(void) 602{ 603 if (delayed_work_pending(&optimizing_work)) 604 wait_for_completion(&optimizer_comp); 605} 606 607/* Optimize kprobe if p is ready to be optimized */ 608static __kprobes void optimize_kprobe(struct kprobe *p) 609{ 610 struct optimized_kprobe *op; 611 612 /* Check if the kprobe is disabled or not ready for optimization. */ 613 if (!kprobe_optready(p) || !kprobes_allow_optimization || 614 (kprobe_disabled(p) || kprobes_all_disarmed)) 615 return; 616 617 /* Both of break_handler and post_handler are not supported. */ 618 if (p->break_handler || p->post_handler) 619 return; 620 621 op = container_of(p, struct optimized_kprobe, kp); 622 623 /* Check there is no other kprobes at the optimized instructions */ 624 if (arch_check_optimized_kprobe(op) < 0) 625 return; 626 627 /* Check if it is already optimized. */ 628 if (op->kp.flags & KPROBE_FLAG_OPTIMIZED) 629 return; 630 op->kp.flags |= KPROBE_FLAG_OPTIMIZED; 631 632 if (!list_empty(&op->list)) 633 /* This is under unoptimizing. Just dequeue the probe */ 634 list_del_init(&op->list); 635 else { 636 list_add(&op->list, &optimizing_list); 637 kick_kprobe_optimizer(); 638 } 639} 640 641/* Short cut to direct unoptimizing */ 642static __kprobes void force_unoptimize_kprobe(struct optimized_kprobe *op) 643{ 644 get_online_cpus(); 645 arch_unoptimize_kprobe(op); 646 put_online_cpus(); 647 if (kprobe_disabled(&op->kp)) 648 arch_disarm_kprobe(&op->kp); 649} 650 651/* Unoptimize a kprobe if p is optimized */ 652static __kprobes void unoptimize_kprobe(struct kprobe *p, bool force) 653{ 654 struct optimized_kprobe *op; 655 656 if (!kprobe_aggrprobe(p) || kprobe_disarmed(p)) 657 return; /* This is not an optprobe nor optimized */ 658 659 op = container_of(p, struct optimized_kprobe, kp); 660 if (!kprobe_optimized(p)) { 661 /* Unoptimized or unoptimizing case */ 662 if (force && !list_empty(&op->list)) { 663 /* 664 * Only if this is unoptimizing kprobe and forced, 665 * forcibly unoptimize it. (No need to unoptimize 666 * unoptimized kprobe again :) 667 */ 668 list_del_init(&op->list); 669 force_unoptimize_kprobe(op); 670 } 671 return; 672 } 673 674 op->kp.flags &= ~KPROBE_FLAG_OPTIMIZED; 675 if (!list_empty(&op->list)) { 676 /* Dequeue from the optimization queue */ 677 list_del_init(&op->list); 678 return; 679 } 680 /* Optimized kprobe case */ 681 if (force) 682 /* Forcibly update the code: this is a special case */ 683 force_unoptimize_kprobe(op); 684 else { 685 list_add(&op->list, &unoptimizing_list); 686 kick_kprobe_optimizer(); 687 } 688} 689 690/* Cancel unoptimizing for reusing */ 691static void reuse_unused_kprobe(struct kprobe *ap) 692{ 693 struct optimized_kprobe *op; 694 695 BUG_ON(!kprobe_unused(ap)); 696 /* 697 * Unused kprobe MUST be on the way of delayed unoptimizing (means 698 * there is still a relative jump) and disabled. 699 */ 700 op = container_of(ap, struct optimized_kprobe, kp); 701 if (unlikely(list_empty(&op->list))) 702 printk(KERN_WARNING "Warning: found a stray unused " 703 "aggrprobe@%p\n", ap->addr); 704 /* Enable the probe again */ 705 ap->flags &= ~KPROBE_FLAG_DISABLED; 706 /* Optimize it again (remove from op->list) */ 707 BUG_ON(!kprobe_optready(ap)); 708 optimize_kprobe(ap); 709} 710 711/* Remove optimized instructions */ 712static void __kprobes kill_optimized_kprobe(struct kprobe *p) 713{ 714 struct optimized_kprobe *op; 715 716 op = container_of(p, struct optimized_kprobe, kp); 717 if (!list_empty(&op->list)) 718 /* Dequeue from the (un)optimization queue */ 719 list_del_init(&op->list); 720 721 op->kp.flags &= ~KPROBE_FLAG_OPTIMIZED; 722 /* Don't touch the code, because it is already freed. */ 723 arch_remove_optimized_kprobe(op); 724} 725 726/* Try to prepare optimized instructions */ 727static __kprobes void prepare_optimized_kprobe(struct kprobe *p) 728{ 729 struct optimized_kprobe *op; 730 731 op = container_of(p, struct optimized_kprobe, kp); 732 arch_prepare_optimized_kprobe(op); 733} 734 735/* Allocate new optimized_kprobe and try to prepare optimized instructions */ 736static __kprobes struct kprobe *alloc_aggr_kprobe(struct kprobe *p) 737{ 738 struct optimized_kprobe *op; 739 740 op = kzalloc(sizeof(struct optimized_kprobe), GFP_KERNEL); 741 if (!op) 742 return NULL; 743 744 INIT_LIST_HEAD(&op->list); 745 op->kp.addr = p->addr; 746 arch_prepare_optimized_kprobe(op); 747 748 return &op->kp; 749} 750 751static void __kprobes init_aggr_kprobe(struct kprobe *ap, struct kprobe *p); 752 753/* 754 * Prepare an optimized_kprobe and optimize it 755 * NOTE: p must be a normal registered kprobe 756 */ 757static __kprobes void try_to_optimize_kprobe(struct kprobe *p) 758{ 759 struct kprobe *ap; 760 struct optimized_kprobe *op; 761 762 /* Impossible to optimize ftrace-based kprobe */ 763 if (kprobe_ftrace(p)) 764 return; 765 766 /* For preparing optimization, jump_label_text_reserved() is called */ 767 jump_label_lock(); 768 mutex_lock(&text_mutex); 769 770 ap = alloc_aggr_kprobe(p); 771 if (!ap) 772 goto out; 773 774 op = container_of(ap, struct optimized_kprobe, kp); 775 if (!arch_prepared_optinsn(&op->optinsn)) { 776 /* If failed to setup optimizing, fallback to kprobe */ 777 arch_remove_optimized_kprobe(op); 778 kfree(op); 779 goto out; 780 } 781 782 init_aggr_kprobe(ap, p); 783 optimize_kprobe(ap); /* This just kicks optimizer thread */ 784 785out: 786 mutex_unlock(&text_mutex); 787 jump_label_unlock(); 788} 789 790#ifdef CONFIG_SYSCTL 791/* This should be called with kprobe_mutex locked */ 792static void __kprobes optimize_all_kprobes(void) 793{ 794 struct hlist_head *head; 795 struct hlist_node *node; 796 struct kprobe *p; 797 unsigned int i; 798 799 /* If optimization is already allowed, just return */ 800 if (kprobes_allow_optimization) 801 return; 802 803 kprobes_allow_optimization = true; 804 for (i = 0; i < KPROBE_TABLE_SIZE; i++) { 805 head = &kprobe_table[i]; 806 hlist_for_each_entry_rcu(p, node, head, hlist) 807 if (!kprobe_disabled(p)) 808 optimize_kprobe(p); 809 } 810 printk(KERN_INFO "Kprobes globally optimized\n"); 811} 812 813/* This should be called with kprobe_mutex locked */ 814static void __kprobes unoptimize_all_kprobes(void) 815{ 816 struct hlist_head *head; 817 struct hlist_node *node; 818 struct kprobe *p; 819 unsigned int i; 820 821 /* If optimization is already prohibited, just return */ 822 if (!kprobes_allow_optimization) 823 return; 824 825 kprobes_allow_optimization = false; 826 for (i = 0; i < KPROBE_TABLE_SIZE; i++) { 827 head = &kprobe_table[i]; 828 hlist_for_each_entry_rcu(p, node, head, hlist) { 829 if (!kprobe_disabled(p)) 830 unoptimize_kprobe(p, false); 831 } 832 } 833 /* Wait for unoptimizing completion */ 834 wait_for_kprobe_optimizer(); 835 printk(KERN_INFO "Kprobes globally unoptimized\n"); 836} 837 838int sysctl_kprobes_optimization; 839int proc_kprobes_optimization_handler(struct ctl_table *table, int write, 840 void __user *buffer, size_t *length, 841 loff_t *ppos) 842{ 843 int ret; 844 845 mutex_lock(&kprobe_mutex); 846 sysctl_kprobes_optimization = kprobes_allow_optimization ? 1 : 0; 847 ret = proc_dointvec_minmax(table, write, buffer, length, ppos); 848 849 if (sysctl_kprobes_optimization) 850 optimize_all_kprobes(); 851 else 852 unoptimize_all_kprobes(); 853 mutex_unlock(&kprobe_mutex); 854 855 return ret; 856} 857#endif /* CONFIG_SYSCTL */ 858 859/* Put a breakpoint for a probe. Must be called with text_mutex locked */ 860static void __kprobes __arm_kprobe(struct kprobe *p) 861{ 862 struct kprobe *_p; 863 864 /* Check collision with other optimized kprobes */ 865 _p = get_optimized_kprobe((unsigned long)p->addr); 866 if (unlikely(_p)) 867 /* Fallback to unoptimized kprobe */ 868 unoptimize_kprobe(_p, true); 869 870 arch_arm_kprobe(p); 871 optimize_kprobe(p); /* Try to optimize (add kprobe to a list) */ 872} 873 874/* Remove the breakpoint of a probe. Must be called with text_mutex locked */ 875static void __kprobes __disarm_kprobe(struct kprobe *p, bool reopt) 876{ 877 struct kprobe *_p; 878 879 unoptimize_kprobe(p, false); /* Try to unoptimize */ 880 881 if (!kprobe_queued(p)) { 882 arch_disarm_kprobe(p); 883 /* If another kprobe was blocked, optimize it. */ 884 _p = get_optimized_kprobe((unsigned long)p->addr); 885 if (unlikely(_p) && reopt) 886 optimize_kprobe(_p); 887 } 888 /* TODO: reoptimize others after unoptimized this probe */ 889} 890 891#else /* !CONFIG_OPTPROBES */ 892 893#define optimize_kprobe(p) do {} while (0) 894#define unoptimize_kprobe(p, f) do {} while (0) 895#define kill_optimized_kprobe(p) do {} while (0) 896#define prepare_optimized_kprobe(p) do {} while (0) 897#define try_to_optimize_kprobe(p) do {} while (0) 898#define __arm_kprobe(p) arch_arm_kprobe(p) 899#define __disarm_kprobe(p, o) arch_disarm_kprobe(p) 900#define kprobe_disarmed(p) kprobe_disabled(p) 901#define wait_for_kprobe_optimizer() do {} while (0) 902 903/* There should be no unused kprobes can be reused without optimization */ 904static void reuse_unused_kprobe(struct kprobe *ap) 905{ 906 printk(KERN_ERR "Error: There should be no unused kprobe here.\n"); 907 BUG_ON(kprobe_unused(ap)); 908} 909 910static __kprobes void free_aggr_kprobe(struct kprobe *p) 911{ 912 arch_remove_kprobe(p); 913 kfree(p); 914} 915 916static __kprobes struct kprobe *alloc_aggr_kprobe(struct kprobe *p) 917{ 918 return kzalloc(sizeof(struct kprobe), GFP_KERNEL); 919} 920#endif /* CONFIG_OPTPROBES */ 921 922#ifdef KPROBES_CAN_USE_FTRACE 923static struct ftrace_ops kprobe_ftrace_ops __read_mostly = { 924 .func = kprobe_ftrace_handler, 925 .flags = FTRACE_OPS_FL_SAVE_REGS, 926}; 927static int kprobe_ftrace_enabled; 928 929/* Must ensure p->addr is really on ftrace */ 930static int __kprobes prepare_kprobe(struct kprobe *p) 931{ 932 if (!kprobe_ftrace(p)) 933 return arch_prepare_kprobe(p); 934 935 return arch_prepare_kprobe_ftrace(p); 936} 937 938/* Caller must lock kprobe_mutex */ 939static void __kprobes arm_kprobe_ftrace(struct kprobe *p) 940{ 941 int ret; 942 943 ret = ftrace_set_filter_ip(&kprobe_ftrace_ops, 944 (unsigned long)p->addr, 0, 0); 945 WARN(ret < 0, "Failed to arm kprobe-ftrace at %p (%d)\n", p->addr, ret); 946 kprobe_ftrace_enabled++; 947 if (kprobe_ftrace_enabled == 1) { 948 ret = register_ftrace_function(&kprobe_ftrace_ops); 949 WARN(ret < 0, "Failed to init kprobe-ftrace (%d)\n", ret); 950 } 951} 952 953/* Caller must lock kprobe_mutex */ 954static void __kprobes disarm_kprobe_ftrace(struct kprobe *p) 955{ 956 int ret; 957 958 kprobe_ftrace_enabled--; 959 if (kprobe_ftrace_enabled == 0) { 960 ret = unregister_ftrace_function(&kprobe_ftrace_ops); 961 WARN(ret < 0, "Failed to init kprobe-ftrace (%d)\n", ret); 962 } 963 ret = ftrace_set_filter_ip(&kprobe_ftrace_ops, 964 (unsigned long)p->addr, 1, 0); 965 WARN(ret < 0, "Failed to disarm kprobe-ftrace at %p (%d)\n", p->addr, ret); 966} 967#else /* !KPROBES_CAN_USE_FTRACE */ 968#define prepare_kprobe(p) arch_prepare_kprobe(p) 969#define arm_kprobe_ftrace(p) do {} while (0) 970#define disarm_kprobe_ftrace(p) do {} while (0) 971#endif 972 973/* Arm a kprobe with text_mutex */ 974static void __kprobes arm_kprobe(struct kprobe *kp) 975{ 976 if (unlikely(kprobe_ftrace(kp))) { 977 arm_kprobe_ftrace(kp); 978 return; 979 } 980 /* 981 * Here, since __arm_kprobe() doesn't use stop_machine(), 982 * this doesn't cause deadlock on text_mutex. So, we don't 983 * need get_online_cpus(). 984 */ 985 mutex_lock(&text_mutex); 986 __arm_kprobe(kp); 987 mutex_unlock(&text_mutex); 988} 989 990/* Disarm a kprobe with text_mutex */ 991static void __kprobes disarm_kprobe(struct kprobe *kp, bool reopt) 992{ 993 if (unlikely(kprobe_ftrace(kp))) { 994 disarm_kprobe_ftrace(kp); 995 return; 996 } 997 /* Ditto */ 998 mutex_lock(&text_mutex); 999 __disarm_kprobe(kp, reopt); 1000 mutex_unlock(&text_mutex); 1001} 1002 1003/* 1004 * Aggregate handlers for multiple kprobes support - these handlers 1005 * take care of invoking the individual kprobe handlers on p->list 1006 */ 1007static int __kprobes aggr_pre_handler(struct kprobe *p, struct pt_regs *regs) 1008{ 1009 struct kprobe *kp; 1010 1011 list_for_each_entry_rcu(kp, &p->list, list) { 1012 if (kp->pre_handler && likely(!kprobe_disabled(kp))) { 1013 set_kprobe_instance(kp); 1014 if (kp->pre_handler(kp, regs)) 1015 return 1; 1016 } 1017 reset_kprobe_instance(); 1018 } 1019 return 0; 1020} 1021 1022static void __kprobes aggr_post_handler(struct kprobe *p, struct pt_regs *regs, 1023 unsigned long flags) 1024{ 1025 struct kprobe *kp; 1026 1027 list_for_each_entry_rcu(kp, &p->list, list) { 1028 if (kp->post_handler && likely(!kprobe_disabled(kp))) { 1029 set_kprobe_instance(kp); 1030 kp->post_handler(kp, regs, flags); 1031 reset_kprobe_instance(); 1032 } 1033 } 1034} 1035 1036static int __kprobes aggr_fault_handler(struct kprobe *p, struct pt_regs *regs, 1037 int trapnr) 1038{ 1039 struct kprobe *cur = __this_cpu_read(kprobe_instance); 1040 1041 /* 1042 * if we faulted "during" the execution of a user specified 1043 * probe handler, invoke just that probe's fault handler 1044 */ 1045 if (cur && cur->fault_handler) { 1046 if (cur->fault_handler(cur, regs, trapnr)) 1047 return 1; 1048 } 1049 return 0; 1050} 1051 1052static int __kprobes aggr_break_handler(struct kprobe *p, struct pt_regs *regs) 1053{ 1054 struct kprobe *cur = __this_cpu_read(kprobe_instance); 1055 int ret = 0; 1056 1057 if (cur && cur->break_handler) { 1058 if (cur->break_handler(cur, regs)) 1059 ret = 1; 1060 } 1061 reset_kprobe_instance(); 1062 return ret; 1063} 1064 1065/* Walks the list and increments nmissed count for multiprobe case */ 1066void __kprobes kprobes_inc_nmissed_count(struct kprobe *p) 1067{ 1068 struct kprobe *kp; 1069 if (!kprobe_aggrprobe(p)) { 1070 p->nmissed++; 1071 } else { 1072 list_for_each_entry_rcu(kp, &p->list, list) 1073 kp->nmissed++; 1074 } 1075 return; 1076} 1077 1078void __kprobes recycle_rp_inst(struct kretprobe_instance *ri, 1079 struct hlist_head *head) 1080{ 1081 struct kretprobe *rp = ri->rp; 1082 1083 /* remove rp inst off the rprobe_inst_table */ 1084 hlist_del(&ri->hlist); 1085 INIT_HLIST_NODE(&ri->hlist); 1086 if (likely(rp)) { 1087 raw_spin_lock(&rp->lock); 1088 hlist_add_head(&ri->hlist, &rp->free_instances); 1089 raw_spin_unlock(&rp->lock); 1090 } else 1091 /* Unregistering */ 1092 hlist_add_head(&ri->hlist, head); 1093} 1094 1095void __kprobes kretprobe_hash_lock(struct task_struct *tsk, 1096 struct hlist_head **head, unsigned long *flags) 1097__acquires(hlist_lock) 1098{ 1099 unsigned long hash = hash_ptr(tsk, KPROBE_HASH_BITS); 1100 raw_spinlock_t *hlist_lock; 1101 1102 *head = &kretprobe_inst_table[hash]; 1103 hlist_lock = kretprobe_table_lock_ptr(hash); 1104 raw_spin_lock_irqsave(hlist_lock, *flags); 1105} 1106 1107static void __kprobes kretprobe_table_lock(unsigned long hash, 1108 unsigned long *flags) 1109__acquires(hlist_lock) 1110{ 1111 raw_spinlock_t *hlist_lock = kretprobe_table_lock_ptr(hash); 1112 raw_spin_lock_irqsave(hlist_lock, *flags); 1113} 1114 1115void __kprobes kretprobe_hash_unlock(struct task_struct *tsk, 1116 unsigned long *flags) 1117__releases(hlist_lock) 1118{ 1119 unsigned long hash = hash_ptr(tsk, KPROBE_HASH_BITS); 1120 raw_spinlock_t *hlist_lock; 1121 1122 hlist_lock = kretprobe_table_lock_ptr(hash); 1123 raw_spin_unlock_irqrestore(hlist_lock, *flags); 1124} 1125 1126static void __kprobes kretprobe_table_unlock(unsigned long hash, 1127 unsigned long *flags) 1128__releases(hlist_lock) 1129{ 1130 raw_spinlock_t *hlist_lock = kretprobe_table_lock_ptr(hash); 1131 raw_spin_unlock_irqrestore(hlist_lock, *flags); 1132} 1133 1134/* 1135 * This function is called from finish_task_switch when task tk becomes dead, 1136 * so that we can recycle any function-return probe instances associated 1137 * with this task. These left over instances represent probed functions 1138 * that have been called but will never return. 1139 */ 1140void __kprobes kprobe_flush_task(struct task_struct *tk) 1141{ 1142 struct kretprobe_instance *ri; 1143 struct hlist_head *head, empty_rp; 1144 struct hlist_node *node, *tmp; 1145 unsigned long hash, flags = 0; 1146 1147 if (unlikely(!kprobes_initialized)) 1148 /* Early boot. kretprobe_table_locks not yet initialized. */ 1149 return; 1150 1151 INIT_HLIST_HEAD(&empty_rp); 1152 hash = hash_ptr(tk, KPROBE_HASH_BITS); 1153 head = &kretprobe_inst_table[hash]; 1154 kretprobe_table_lock(hash, &flags); 1155 hlist_for_each_entry_safe(ri, node, tmp, head, hlist) { 1156 if (ri->task == tk) 1157 recycle_rp_inst(ri, &empty_rp); 1158 } 1159 kretprobe_table_unlock(hash, &flags); 1160 hlist_for_each_entry_safe(ri, node, tmp, &empty_rp, hlist) { 1161 hlist_del(&ri->hlist); 1162 kfree(ri); 1163 } 1164} 1165 1166static inline void free_rp_inst(struct kretprobe *rp) 1167{ 1168 struct kretprobe_instance *ri; 1169 struct hlist_node *pos, *next; 1170 1171 hlist_for_each_entry_safe(ri, pos, next, &rp->free_instances, hlist) { 1172 hlist_del(&ri->hlist); 1173 kfree(ri); 1174 } 1175} 1176 1177static void __kprobes cleanup_rp_inst(struct kretprobe *rp) 1178{ 1179 unsigned long flags, hash; 1180 struct kretprobe_instance *ri; 1181 struct hlist_node *pos, *next; 1182 struct hlist_head *head; 1183 1184 /* No race here */ 1185 for (hash = 0; hash < KPROBE_TABLE_SIZE; hash++) { 1186 kretprobe_table_lock(hash, &flags); 1187 head = &kretprobe_inst_table[hash]; 1188 hlist_for_each_entry_safe(ri, pos, next, head, hlist) { 1189 if (ri->rp == rp) 1190 ri->rp = NULL; 1191 } 1192 kretprobe_table_unlock(hash, &flags); 1193 } 1194 free_rp_inst(rp); 1195} 1196 1197/* 1198* Add the new probe to ap->list. Fail if this is the 1199* second jprobe at the address - two jprobes can't coexist 1200*/ 1201static int __kprobes add_new_kprobe(struct kprobe *ap, struct kprobe *p) 1202{ 1203 BUG_ON(kprobe_gone(ap) || kprobe_gone(p)); 1204 1205 if (p->break_handler || p->post_handler) 1206 unoptimize_kprobe(ap, true); /* Fall back to normal kprobe */ 1207 1208 if (p->break_handler) { 1209 if (ap->break_handler) 1210 return -EEXIST; 1211 list_add_tail_rcu(&p->list, &ap->list); 1212 ap->break_handler = aggr_break_handler; 1213 } else 1214 list_add_rcu(&p->list, &ap->list); 1215 if (p->post_handler && !ap->post_handler) 1216 ap->post_handler = aggr_post_handler; 1217 1218 return 0; 1219} 1220 1221/* 1222 * Fill in the required fields of the "manager kprobe". Replace the 1223 * earlier kprobe in the hlist with the manager kprobe 1224 */ 1225static void __kprobes init_aggr_kprobe(struct kprobe *ap, struct kprobe *p) 1226{ 1227 /* Copy p's insn slot to ap */ 1228 copy_kprobe(p, ap); 1229 flush_insn_slot(ap); 1230 ap->addr = p->addr; 1231 ap->flags = p->flags & ~KPROBE_FLAG_OPTIMIZED; 1232 ap->pre_handler = aggr_pre_handler; 1233 ap->fault_handler = aggr_fault_handler; 1234 /* We don't care the kprobe which has gone. */ 1235 if (p->post_handler && !kprobe_gone(p)) 1236 ap->post_handler = aggr_post_handler; 1237 if (p->break_handler && !kprobe_gone(p)) 1238 ap->break_handler = aggr_break_handler; 1239 1240 INIT_LIST_HEAD(&ap->list); 1241 INIT_HLIST_NODE(&ap->hlist); 1242 1243 list_add_rcu(&p->list, &ap->list); 1244 hlist_replace_rcu(&p->hlist, &ap->hlist); 1245} 1246 1247/* 1248 * This is the second or subsequent kprobe at the address - handle 1249 * the intricacies 1250 */ 1251static int __kprobes register_aggr_kprobe(struct kprobe *orig_p, 1252 struct kprobe *p) 1253{ 1254 int ret = 0; 1255 struct kprobe *ap = orig_p; 1256 1257 /* For preparing optimization, jump_label_text_reserved() is called */ 1258 jump_label_lock(); 1259 /* 1260 * Get online CPUs to avoid text_mutex deadlock.with stop machine, 1261 * which is invoked by unoptimize_kprobe() in add_new_kprobe() 1262 */ 1263 get_online_cpus(); 1264 mutex_lock(&text_mutex); 1265 1266 if (!kprobe_aggrprobe(orig_p)) { 1267 /* If orig_p is not an aggr_kprobe, create new aggr_kprobe. */ 1268 ap = alloc_aggr_kprobe(orig_p); 1269 if (!ap) { 1270 ret = -ENOMEM; 1271 goto out; 1272 } 1273 init_aggr_kprobe(ap, orig_p); 1274 } else if (kprobe_unused(ap)) 1275 /* This probe is going to die. Rescue it */ 1276 reuse_unused_kprobe(ap); 1277 1278 if (kprobe_gone(ap)) { 1279 /* 1280 * Attempting to insert new probe at the same location that 1281 * had a probe in the module vaddr area which already 1282 * freed. So, the instruction slot has already been 1283 * released. We need a new slot for the new probe. 1284 */ 1285 ret = arch_prepare_kprobe(ap); 1286 if (ret) 1287 /* 1288 * Even if fail to allocate new slot, don't need to 1289 * free aggr_probe. It will be used next time, or 1290 * freed by unregister_kprobe. 1291 */ 1292 goto out; 1293 1294 /* Prepare optimized instructions if possible. */ 1295 prepare_optimized_kprobe(ap); 1296 1297 /* 1298 * Clear gone flag to prevent allocating new slot again, and 1299 * set disabled flag because it is not armed yet. 1300 */ 1301 ap->flags = (ap->flags & ~KPROBE_FLAG_GONE) 1302 | KPROBE_FLAG_DISABLED; 1303 } 1304 1305 /* Copy ap's insn slot to p */ 1306 copy_kprobe(ap, p); 1307 ret = add_new_kprobe(ap, p); 1308 1309out: 1310 mutex_unlock(&text_mutex); 1311 put_online_cpus(); 1312 jump_label_unlock(); 1313 1314 if (ret == 0 && kprobe_disabled(ap) && !kprobe_disabled(p)) { 1315 ap->flags &= ~KPROBE_FLAG_DISABLED; 1316 if (!kprobes_all_disarmed) 1317 /* Arm the breakpoint again. */ 1318 arm_kprobe(ap); 1319 } 1320 return ret; 1321} 1322 1323static int __kprobes in_kprobes_functions(unsigned long addr) 1324{ 1325 struct kprobe_blackpoint *kb; 1326 1327 if (addr >= (unsigned long)__kprobes_text_start && 1328 addr < (unsigned long)__kprobes_text_end) 1329 return -EINVAL; 1330 /* 1331 * If there exists a kprobe_blacklist, verify and 1332 * fail any probe registration in the prohibited area 1333 */ 1334 for (kb = kprobe_blacklist; kb->name != NULL; kb++) { 1335 if (kb->start_addr) { 1336 if (addr >= kb->start_addr && 1337 addr < (kb->start_addr + kb->range)) 1338 return -EINVAL; 1339 } 1340 } 1341 return 0; 1342} 1343 1344/* 1345 * If we have a symbol_name argument, look it up and add the offset field 1346 * to it. This way, we can specify a relative address to a symbol. 1347 * This returns encoded errors if it fails to look up symbol or invalid 1348 * combination of parameters. 1349 */ 1350static kprobe_opcode_t __kprobes *kprobe_addr(struct kprobe *p) 1351{ 1352 kprobe_opcode_t *addr = p->addr; 1353 1354 if ((p->symbol_name && p->addr) || 1355 (!p->symbol_name && !p->addr)) 1356 goto invalid; 1357 1358 if (p->symbol_name) { 1359 kprobe_lookup_name(p->symbol_name, addr); 1360 if (!addr) 1361 return ERR_PTR(-ENOENT); 1362 } 1363 1364 addr = (kprobe_opcode_t *)(((char *)addr) + p->offset); 1365 if (addr) 1366 return addr; 1367 1368invalid: 1369 return ERR_PTR(-EINVAL); 1370} 1371 1372/* Check passed kprobe is valid and return kprobe in kprobe_table. */ 1373static struct kprobe * __kprobes __get_valid_kprobe(struct kprobe *p) 1374{ 1375 struct kprobe *ap, *list_p; 1376 1377 ap = get_kprobe(p->addr); 1378 if (unlikely(!ap)) 1379 return NULL; 1380 1381 if (p != ap) { 1382 list_for_each_entry_rcu(list_p, &ap->list, list) 1383 if (list_p == p) 1384 /* kprobe p is a valid probe */ 1385 goto valid; 1386 return NULL; 1387 } 1388valid: 1389 return ap; 1390} 1391 1392/* Return error if the kprobe is being re-registered */ 1393static inline int check_kprobe_rereg(struct kprobe *p) 1394{ 1395 int ret = 0; 1396 1397 mutex_lock(&kprobe_mutex); 1398 if (__get_valid_kprobe(p)) 1399 ret = -EINVAL; 1400 mutex_unlock(&kprobe_mutex); 1401 1402 return ret; 1403} 1404 1405static __kprobes int check_kprobe_address_safe(struct kprobe *p, 1406 struct module **probed_mod) 1407{ 1408 int ret = 0; 1409 unsigned long ftrace_addr; 1410 1411 /* 1412 * If the address is located on a ftrace nop, set the 1413 * breakpoint to the following instruction. 1414 */ 1415 ftrace_addr = ftrace_location((unsigned long)p->addr); 1416 if (ftrace_addr) { 1417#ifdef KPROBES_CAN_USE_FTRACE 1418 /* Given address is not on the instruction boundary */ 1419 if ((unsigned long)p->addr != ftrace_addr) 1420 return -EILSEQ; 1421 /* break_handler (jprobe) can not work with ftrace */ 1422 if (p->break_handler) 1423 return -EINVAL; 1424 p->flags |= KPROBE_FLAG_FTRACE; 1425#else /* !KPROBES_CAN_USE_FTRACE */ 1426 return -EINVAL; 1427#endif 1428 } 1429 1430 jump_label_lock(); 1431 preempt_disable(); 1432 1433 /* Ensure it is not in reserved area nor out of text */ 1434 if (!kernel_text_address((unsigned long) p->addr) || 1435 in_kprobes_functions((unsigned long) p->addr) || 1436 jump_label_text_reserved(p->addr, p->addr)) { 1437 ret = -EINVAL; 1438 goto out; 1439 } 1440 1441 /* Check if are we probing a module */ 1442 *probed_mod = __module_text_address((unsigned long) p->addr); 1443 if (*probed_mod) { 1444 /* 1445 * We must hold a refcount of the probed module while updating 1446 * its code to prohibit unexpected unloading. 1447 */ 1448 if (unlikely(!try_module_get(*probed_mod))) { 1449 ret = -ENOENT; 1450 goto out; 1451 } 1452 1453 /* 1454 * If the module freed .init.text, we couldn't insert 1455 * kprobes in there. 1456 */ 1457 if (within_module_init((unsigned long)p->addr, *probed_mod) && 1458 (*probed_mod)->state != MODULE_STATE_COMING) { 1459 module_put(*probed_mod); 1460 *probed_mod = NULL; 1461 ret = -ENOENT; 1462 } 1463 } 1464out: 1465 preempt_enable(); 1466 jump_label_unlock(); 1467 1468 return ret; 1469} 1470 1471int __kprobes register_kprobe(struct kprobe *p) 1472{ 1473 int ret; 1474 struct kprobe *old_p; 1475 struct module *probed_mod; 1476 kprobe_opcode_t *addr; 1477 1478 /* Adjust probe address from symbol */ 1479 addr = kprobe_addr(p); 1480 if (IS_ERR(addr)) 1481 return PTR_ERR(addr); 1482 p->addr = addr; 1483 1484 ret = check_kprobe_rereg(p); 1485 if (ret) 1486 return ret; 1487 1488 /* User can pass only KPROBE_FLAG_DISABLED to register_kprobe */ 1489 p->flags &= KPROBE_FLAG_DISABLED; 1490 p->nmissed = 0; 1491 INIT_LIST_HEAD(&p->list); 1492 1493 ret = check_kprobe_address_safe(p, &probed_mod); 1494 if (ret) 1495 return ret; 1496 1497 mutex_lock(&kprobe_mutex); 1498 1499 old_p = get_kprobe(p->addr); 1500 if (old_p) { 1501 /* Since this may unoptimize old_p, locking text_mutex. */ 1502 ret = register_aggr_kprobe(old_p, p); 1503 goto out; 1504 } 1505 1506 mutex_lock(&text_mutex); /* Avoiding text modification */ 1507 ret = prepare_kprobe(p); 1508 mutex_unlock(&text_mutex); 1509 if (ret) 1510 goto out; 1511 1512 INIT_HLIST_NODE(&p->hlist); 1513 hlist_add_head_rcu(&p->hlist, 1514 &kprobe_table[hash_ptr(p->addr, KPROBE_HASH_BITS)]); 1515 1516 if (!kprobes_all_disarmed && !kprobe_disabled(p)) 1517 arm_kprobe(p); 1518 1519 /* Try to optimize kprobe */ 1520 try_to_optimize_kprobe(p); 1521 1522out: 1523 mutex_unlock(&kprobe_mutex); 1524 1525 if (probed_mod) 1526 module_put(probed_mod); 1527 1528 return ret; 1529} 1530EXPORT_SYMBOL_GPL(register_kprobe); 1531 1532/* Check if all probes on the aggrprobe are disabled */ 1533static int __kprobes aggr_kprobe_disabled(struct kprobe *ap) 1534{ 1535 struct kprobe *kp; 1536 1537 list_for_each_entry_rcu(kp, &ap->list, list) 1538 if (!kprobe_disabled(kp)) 1539 /* 1540 * There is an active probe on the list. 1541 * We can't disable this ap. 1542 */ 1543 return 0; 1544 1545 return 1; 1546} 1547 1548/* Disable one kprobe: Make sure called under kprobe_mutex is locked */ 1549static struct kprobe *__kprobes __disable_kprobe(struct kprobe *p) 1550{ 1551 struct kprobe *orig_p; 1552 1553 /* Get an original kprobe for return */ 1554 orig_p = __get_valid_kprobe(p); 1555 if (unlikely(orig_p == NULL)) 1556 return NULL; 1557 1558 if (!kprobe_disabled(p)) { 1559 /* Disable probe if it is a child probe */ 1560 if (p != orig_p) 1561 p->flags |= KPROBE_FLAG_DISABLED; 1562 1563 /* Try to disarm and disable this/parent probe */ 1564 if (p == orig_p || aggr_kprobe_disabled(orig_p)) { 1565 disarm_kprobe(orig_p, true); 1566 orig_p->flags |= KPROBE_FLAG_DISABLED; 1567 } 1568 } 1569 1570 return orig_p; 1571} 1572 1573/* 1574 * Unregister a kprobe without a scheduler synchronization. 1575 */ 1576static int __kprobes __unregister_kprobe_top(struct kprobe *p) 1577{ 1578 struct kprobe *ap, *list_p; 1579 1580 /* Disable kprobe. This will disarm it if needed. */ 1581 ap = __disable_kprobe(p); 1582 if (ap == NULL) 1583 return -EINVAL; 1584 1585 if (ap == p) 1586 /* 1587 * This probe is an independent(and non-optimized) kprobe 1588 * (not an aggrprobe). Remove from the hash list. 1589 */ 1590 goto disarmed; 1591 1592 /* Following process expects this probe is an aggrprobe */ 1593 WARN_ON(!kprobe_aggrprobe(ap)); 1594 1595 if (list_is_singular(&ap->list) && kprobe_disarmed(ap)) 1596 /* 1597 * !disarmed could be happen if the probe is under delayed 1598 * unoptimizing. 1599 */ 1600 goto disarmed; 1601 else { 1602 /* If disabling probe has special handlers, update aggrprobe */ 1603 if (p->break_handler && !kprobe_gone(p)) 1604 ap->break_handler = NULL; 1605 if (p->post_handler && !kprobe_gone(p)) { 1606 list_for_each_entry_rcu(list_p, &ap->list, list) { 1607 if ((list_p != p) && (list_p->post_handler)) 1608 goto noclean; 1609 } 1610 ap->post_handler = NULL; 1611 } 1612noclean: 1613 /* 1614 * Remove from the aggrprobe: this path will do nothing in 1615 * __unregister_kprobe_bottom(). 1616 */ 1617 list_del_rcu(&p->list); 1618 if (!kprobe_disabled(ap) && !kprobes_all_disarmed) 1619 /* 1620 * Try to optimize this probe again, because post 1621 * handler may have been changed. 1622 */ 1623 optimize_kprobe(ap); 1624 } 1625 return 0; 1626 1627disarmed: 1628 BUG_ON(!kprobe_disarmed(ap)); 1629 hlist_del_rcu(&ap->hlist); 1630 return 0; 1631} 1632 1633static void __kprobes __unregister_kprobe_bottom(struct kprobe *p) 1634{ 1635 struct kprobe *ap; 1636 1637 if (list_empty(&p->list)) 1638 /* This is an independent kprobe */ 1639 arch_remove_kprobe(p); 1640 else if (list_is_singular(&p->list)) { 1641 /* This is the last child of an aggrprobe */ 1642 ap = list_entry(p->list.next, struct kprobe, list); 1643 list_del(&p->list); 1644 free_aggr_kprobe(ap); 1645 } 1646 /* Otherwise, do nothing. */ 1647} 1648 1649int __kprobes register_kprobes(struct kprobe **kps, int num) 1650{ 1651 int i, ret = 0; 1652 1653 if (num <= 0) 1654 return -EINVAL; 1655 for (i = 0; i < num; i++) { 1656 ret = register_kprobe(kps[i]); 1657 if (ret < 0) { 1658 if (i > 0) 1659 unregister_kprobes(kps, i); 1660 break; 1661 } 1662 } 1663 return ret; 1664} 1665EXPORT_SYMBOL_GPL(register_kprobes); 1666 1667void __kprobes unregister_kprobe(struct kprobe *p) 1668{ 1669 unregister_kprobes(&p, 1); 1670} 1671EXPORT_SYMBOL_GPL(unregister_kprobe); 1672 1673void __kprobes unregister_kprobes(struct kprobe **kps, int num) 1674{ 1675 int i; 1676 1677 if (num <= 0) 1678 return; 1679 mutex_lock(&kprobe_mutex); 1680 for (i = 0; i < num; i++) 1681 if (__unregister_kprobe_top(kps[i]) < 0) 1682 kps[i]->addr = NULL; 1683 mutex_unlock(&kprobe_mutex); 1684 1685 synchronize_sched(); 1686 for (i = 0; i < num; i++) 1687 if (kps[i]->addr) 1688 __unregister_kprobe_bottom(kps[i]); 1689} 1690EXPORT_SYMBOL_GPL(unregister_kprobes); 1691 1692static struct notifier_block kprobe_exceptions_nb = { 1693 .notifier_call = kprobe_exceptions_notify, 1694 .priority = 0x7fffffff /* we need to be notified first */ 1695}; 1696 1697unsigned long __weak arch_deref_entry_point(void *entry) 1698{ 1699 return (unsigned long)entry; 1700} 1701 1702int __kprobes register_jprobes(struct jprobe **jps, int num) 1703{ 1704 struct jprobe *jp; 1705 int ret = 0, i; 1706 1707 if (num <= 0) 1708 return -EINVAL; 1709 for (i = 0; i < num; i++) { 1710 unsigned long addr, offset; 1711 jp = jps[i]; 1712 addr = arch_deref_entry_point(jp->entry); 1713 1714 /* Verify probepoint is a function entry point */ 1715 if (kallsyms_lookup_size_offset(addr, NULL, &offset) && 1716 offset == 0) { 1717 jp->kp.pre_handler = setjmp_pre_handler; 1718 jp->kp.break_handler = longjmp_break_handler; 1719 ret = register_kprobe(&jp->kp); 1720 } else 1721 ret = -EINVAL; 1722 1723 if (ret < 0) { 1724 if (i > 0) 1725 unregister_jprobes(jps, i); 1726 break; 1727 } 1728 } 1729 return ret; 1730} 1731EXPORT_SYMBOL_GPL(register_jprobes); 1732 1733int __kprobes register_jprobe(struct jprobe *jp) 1734{ 1735 return register_jprobes(&jp, 1); 1736} 1737EXPORT_SYMBOL_GPL(register_jprobe); 1738 1739void __kprobes unregister_jprobe(struct jprobe *jp) 1740{ 1741 unregister_jprobes(&jp, 1); 1742} 1743EXPORT_SYMBOL_GPL(unregister_jprobe); 1744 1745void __kprobes unregister_jprobes(struct jprobe **jps, int num) 1746{ 1747 int i; 1748 1749 if (num <= 0) 1750 return; 1751 mutex_lock(&kprobe_mutex); 1752 for (i = 0; i < num; i++) 1753 if (__unregister_kprobe_top(&jps[i]->kp) < 0) 1754 jps[i]->kp.addr = NULL; 1755 mutex_unlock(&kprobe_mutex); 1756 1757 synchronize_sched(); 1758 for (i = 0; i < num; i++) { 1759 if (jps[i]->kp.addr) 1760 __unregister_kprobe_bottom(&jps[i]->kp); 1761 } 1762} 1763EXPORT_SYMBOL_GPL(unregister_jprobes); 1764 1765#ifdef CONFIG_KRETPROBES 1766/* 1767 * This kprobe pre_handler is registered with every kretprobe. When probe 1768 * hits it will set up the return probe. 1769 */ 1770static int __kprobes pre_handler_kretprobe(struct kprobe *p, 1771 struct pt_regs *regs) 1772{ 1773 struct kretprobe *rp = container_of(p, struct kretprobe, kp); 1774 unsigned long hash, flags = 0; 1775 struct kretprobe_instance *ri; 1776 1777 /*TODO: consider to only swap the RA after the last pre_handler fired */ 1778 hash = hash_ptr(current, KPROBE_HASH_BITS); 1779 raw_spin_lock_irqsave(&rp->lock, flags); 1780 if (!hlist_empty(&rp->free_instances)) { 1781 ri = hlist_entry(rp->free_instances.first, 1782 struct kretprobe_instance, hlist); 1783 hlist_del(&ri->hlist); 1784 raw_spin_unlock_irqrestore(&rp->lock, flags); 1785 1786 ri->rp = rp; 1787 ri->task = current; 1788 1789 if (rp->entry_handler && rp->entry_handler(ri, regs)) { 1790 raw_spin_lock_irqsave(&rp->lock, flags); 1791 hlist_add_head(&ri->hlist, &rp->free_instances); 1792 raw_spin_unlock_irqrestore(&rp->lock, flags); 1793 return 0; 1794 } 1795 1796 arch_prepare_kretprobe(ri, regs); 1797 1798 /* XXX(hch): why is there no hlist_move_head? */ 1799 INIT_HLIST_NODE(&ri->hlist); 1800 kretprobe_table_lock(hash, &flags); 1801 hlist_add_head(&ri->hlist, &kretprobe_inst_table[hash]); 1802 kretprobe_table_unlock(hash, &flags); 1803 } else { 1804 rp->nmissed++; 1805 raw_spin_unlock_irqrestore(&rp->lock, flags); 1806 } 1807 return 0; 1808} 1809 1810int __kprobes register_kretprobe(struct kretprobe *rp) 1811{ 1812 int ret = 0; 1813 struct kretprobe_instance *inst; 1814 int i; 1815 void *addr; 1816 1817 if (kretprobe_blacklist_size) { 1818 addr = kprobe_addr(&rp->kp); 1819 if (IS_ERR(addr)) 1820 return PTR_ERR(addr); 1821 1822 for (i = 0; kretprobe_blacklist[i].name != NULL; i++) { 1823 if (kretprobe_blacklist[i].addr == addr) 1824 return -EINVAL; 1825 } 1826 } 1827 1828 rp->kp.pre_handler = pre_handler_kretprobe; 1829 rp->kp.post_handler = NULL; 1830 rp->kp.fault_handler = NULL; 1831 rp->kp.break_handler = NULL; 1832 1833 /* Pre-allocate memory for max kretprobe instances */ 1834 if (rp->maxactive <= 0) { 1835#ifdef CONFIG_PREEMPT 1836 rp->maxactive = max_t(unsigned int, 10, 2*num_possible_cpus()); 1837#else 1838 rp->maxactive = num_possible_cpus(); 1839#endif 1840 } 1841 raw_spin_lock_init(&rp->lock); 1842 INIT_HLIST_HEAD(&rp->free_instances); 1843 for (i = 0; i < rp->maxactive; i++) { 1844 inst = kmalloc(sizeof(struct kretprobe_instance) + 1845 rp->data_size, GFP_KERNEL); 1846 if (inst == NULL) { 1847 free_rp_inst(rp); 1848 return -ENOMEM; 1849 } 1850 INIT_HLIST_NODE(&inst->hlist); 1851 hlist_add_head(&inst->hlist, &rp->free_instances); 1852 } 1853 1854 rp->nmissed = 0; 1855 /* Establish function entry probe point */ 1856 ret = register_kprobe(&rp->kp); 1857 if (ret != 0) 1858 free_rp_inst(rp); 1859 return ret; 1860} 1861EXPORT_SYMBOL_GPL(register_kretprobe); 1862 1863int __kprobes register_kretprobes(struct kretprobe **rps, int num) 1864{ 1865 int ret = 0, i; 1866 1867 if (num <= 0) 1868 return -EINVAL; 1869 for (i = 0; i < num; i++) { 1870 ret = register_kretprobe(rps[i]); 1871 if (ret < 0) { 1872 if (i > 0) 1873 unregister_kretprobes(rps, i); 1874 break; 1875 } 1876 } 1877 return ret; 1878} 1879EXPORT_SYMBOL_GPL(register_kretprobes); 1880 1881void __kprobes unregister_kretprobe(struct kretprobe *rp) 1882{ 1883 unregister_kretprobes(&rp, 1); 1884} 1885EXPORT_SYMBOL_GPL(unregister_kretprobe); 1886 1887void __kprobes unregister_kretprobes(struct kretprobe **rps, int num) 1888{ 1889 int i; 1890 1891 if (num <= 0) 1892 return; 1893 mutex_lock(&kprobe_mutex); 1894 for (i = 0; i < num; i++) 1895 if (__unregister_kprobe_top(&rps[i]->kp) < 0) 1896 rps[i]->kp.addr = NULL; 1897 mutex_unlock(&kprobe_mutex); 1898 1899 synchronize_sched(); 1900 for (i = 0; i < num; i++) { 1901 if (rps[i]->kp.addr) { 1902 __unregister_kprobe_bottom(&rps[i]->kp); 1903 cleanup_rp_inst(rps[i]); 1904 } 1905 } 1906} 1907EXPORT_SYMBOL_GPL(unregister_kretprobes); 1908 1909#else /* CONFIG_KRETPROBES */ 1910int __kprobes register_kretprobe(struct kretprobe *rp) 1911{ 1912 return -ENOSYS; 1913} 1914EXPORT_SYMBOL_GPL(register_kretprobe); 1915 1916int __kprobes register_kretprobes(struct kretprobe **rps, int num) 1917{ 1918 return -ENOSYS; 1919} 1920EXPORT_SYMBOL_GPL(register_kretprobes); 1921 1922void __kprobes unregister_kretprobe(struct kretprobe *rp) 1923{ 1924} 1925EXPORT_SYMBOL_GPL(unregister_kretprobe); 1926 1927void __kprobes unregister_kretprobes(struct kretprobe **rps, int num) 1928{ 1929} 1930EXPORT_SYMBOL_GPL(unregister_kretprobes); 1931 1932static int __kprobes pre_handler_kretprobe(struct kprobe *p, 1933 struct pt_regs *regs) 1934{ 1935 return 0; 1936} 1937 1938#endif /* CONFIG_KRETPROBES */ 1939 1940/* Set the kprobe gone and remove its instruction buffer. */ 1941static void __kprobes kill_kprobe(struct kprobe *p) 1942{ 1943 struct kprobe *kp; 1944 1945 p->flags |= KPROBE_FLAG_GONE; 1946 if (kprobe_aggrprobe(p)) { 1947 /* 1948 * If this is an aggr_kprobe, we have to list all the 1949 * chained probes and mark them GONE. 1950 */ 1951 list_for_each_entry_rcu(kp, &p->list, list) 1952 kp->flags |= KPROBE_FLAG_GONE; 1953 p->post_handler = NULL; 1954 p->break_handler = NULL; 1955 kill_optimized_kprobe(p); 1956 } 1957 /* 1958 * Here, we can remove insn_slot safely, because no thread calls 1959 * the original probed function (which will be freed soon) any more. 1960 */ 1961 arch_remove_kprobe(p); 1962} 1963 1964/* Disable one kprobe */ 1965int __kprobes disable_kprobe(struct kprobe *kp) 1966{ 1967 int ret = 0; 1968 1969 mutex_lock(&kprobe_mutex); 1970 1971 /* Disable this kprobe */ 1972 if (__disable_kprobe(kp) == NULL) 1973 ret = -EINVAL; 1974 1975 mutex_unlock(&kprobe_mutex); 1976 return ret; 1977} 1978EXPORT_SYMBOL_GPL(disable_kprobe); 1979 1980/* Enable one kprobe */ 1981int __kprobes enable_kprobe(struct kprobe *kp) 1982{ 1983 int ret = 0; 1984 struct kprobe *p; 1985 1986 mutex_lock(&kprobe_mutex); 1987 1988 /* Check whether specified probe is valid. */ 1989 p = __get_valid_kprobe(kp); 1990 if (unlikely(p == NULL)) { 1991 ret = -EINVAL; 1992 goto out; 1993 } 1994 1995 if (kprobe_gone(kp)) { 1996 /* This kprobe has gone, we couldn't enable it. */ 1997 ret = -EINVAL; 1998 goto out; 1999 } 2000 2001 if (p != kp) 2002 kp->flags &= ~KPROBE_FLAG_DISABLED; 2003 2004 if (!kprobes_all_disarmed && kprobe_disabled(p)) { 2005 p->flags &= ~KPROBE_FLAG_DISABLED; 2006 arm_kprobe(p); 2007 } 2008out: 2009 mutex_unlock(&kprobe_mutex); 2010 return ret; 2011} 2012EXPORT_SYMBOL_GPL(enable_kprobe); 2013 2014void __kprobes dump_kprobe(struct kprobe *kp) 2015{ 2016 printk(KERN_WARNING "Dumping kprobe:\n"); 2017 printk(KERN_WARNING "Name: %s\nAddress: %p\nOffset: %x\n", 2018 kp->symbol_name, kp->addr, kp->offset); 2019} 2020 2021/* Module notifier call back, checking kprobes on the module */ 2022static int __kprobes kprobes_module_callback(struct notifier_block *nb, 2023 unsigned long val, void *data) 2024{ 2025 struct module *mod = data; 2026 struct hlist_head *head; 2027 struct hlist_node *node; 2028 struct kprobe *p; 2029 unsigned int i; 2030 int checkcore = (val == MODULE_STATE_GOING); 2031 2032 if (val != MODULE_STATE_GOING && val != MODULE_STATE_LIVE) 2033 return NOTIFY_DONE; 2034 2035 /* 2036 * When MODULE_STATE_GOING was notified, both of module .text and 2037 * .init.text sections would be freed. When MODULE_STATE_LIVE was 2038 * notified, only .init.text section would be freed. We need to 2039 * disable kprobes which have been inserted in the sections. 2040 */ 2041 mutex_lock(&kprobe_mutex); 2042 for (i = 0; i < KPROBE_TABLE_SIZE; i++) { 2043 head = &kprobe_table[i]; 2044 hlist_for_each_entry_rcu(p, node, head, hlist) 2045 if (within_module_init((unsigned long)p->addr, mod) || 2046 (checkcore && 2047 within_module_core((unsigned long)p->addr, mod))) { 2048 /* 2049 * The vaddr this probe is installed will soon 2050 * be vfreed buy not synced to disk. Hence, 2051 * disarming the breakpoint isn't needed. 2052 */ 2053 kill_kprobe(p); 2054 } 2055 } 2056 mutex_unlock(&kprobe_mutex); 2057 return NOTIFY_DONE; 2058} 2059 2060static struct notifier_block kprobe_module_nb = { 2061 .notifier_call = kprobes_module_callback, 2062 .priority = 0 2063}; 2064 2065static int __init init_kprobes(void) 2066{ 2067 int i, err = 0; 2068 unsigned long offset = 0, size = 0; 2069 char *modname, namebuf[128]; 2070 const char *symbol_name; 2071 void *addr; 2072 struct kprobe_blackpoint *kb; 2073 2074 /* FIXME allocate the probe table, currently defined statically */ 2075 /* initialize all list heads */ 2076 for (i = 0; i < KPROBE_TABLE_SIZE; i++) { 2077 INIT_HLIST_HEAD(&kprobe_table[i]); 2078 INIT_HLIST_HEAD(&kretprobe_inst_table[i]); 2079 raw_spin_lock_init(&(kretprobe_table_locks[i].lock)); 2080 } 2081 2082 /* 2083 * Lookup and populate the kprobe_blacklist. 2084 * 2085 * Unlike the kretprobe blacklist, we'll need to determine 2086 * the range of addresses that belong to the said functions, 2087 * since a kprobe need not necessarily be at the beginning 2088 * of a function. 2089 */ 2090 for (kb = kprobe_blacklist; kb->name != NULL; kb++) { 2091 kprobe_lookup_name(kb->name, addr); 2092 if (!addr) 2093 continue; 2094 2095 kb->start_addr = (unsigned long)addr; 2096 symbol_name = kallsyms_lookup(kb->start_addr, 2097 &size, &offset, &modname, namebuf); 2098 if (!symbol_name) 2099 kb->range = 0; 2100 else 2101 kb->range = size; 2102 } 2103 2104 if (kretprobe_blacklist_size) { 2105 /* lookup the function address from its name */ 2106 for (i = 0; kretprobe_blacklist[i].name != NULL; i++) { 2107 kprobe_lookup_name(kretprobe_blacklist[i].name, 2108 kretprobe_blacklist[i].addr); 2109 if (!kretprobe_blacklist[i].addr) 2110 printk("kretprobe: lookup failed: %s\n", 2111 kretprobe_blacklist[i].name); 2112 } 2113 } 2114 2115#if defined(CONFIG_OPTPROBES) 2116#if defined(__ARCH_WANT_KPROBES_INSN_SLOT) 2117 /* Init kprobe_optinsn_slots */ 2118 kprobe_optinsn_slots.insn_size = MAX_OPTINSN_SIZE; 2119#endif 2120 /* By default, kprobes can be optimized */ 2121 kprobes_allow_optimization = true; 2122#endif 2123 2124 /* By default, kprobes are armed */ 2125 kprobes_all_disarmed = false; 2126 2127 err = arch_init_kprobes(); 2128 if (!err) 2129 err = register_die_notifier(&kprobe_exceptions_nb); 2130 if (!err) 2131 err = register_module_notifier(&kprobe_module_nb); 2132 2133 kprobes_initialized = (err == 0); 2134 2135 if (!err) 2136 init_test_probes(); 2137 return err; 2138} 2139 2140#ifdef CONFIG_DEBUG_FS 2141static void __kprobes report_probe(struct seq_file *pi, struct kprobe *p, 2142 const char *sym, int offset, char *modname, struct kprobe *pp) 2143{ 2144 char *kprobe_type; 2145 2146 if (p->pre_handler == pre_handler_kretprobe) 2147 kprobe_type = "r"; 2148 else if (p->pre_handler == setjmp_pre_handler) 2149 kprobe_type = "j"; 2150 else 2151 kprobe_type = "k"; 2152 2153 if (sym) 2154 seq_printf(pi, "%p %s %s+0x%x %s ", 2155 p->addr, kprobe_type, sym, offset, 2156 (modname ? modname : " ")); 2157 else 2158 seq_printf(pi, "%p %s %p ", 2159 p->addr, kprobe_type, p->addr); 2160 2161 if (!pp) 2162 pp = p; 2163 seq_printf(pi, "%s%s%s%s\n", 2164 (kprobe_gone(p) ? "[GONE]" : ""), 2165 ((kprobe_disabled(p) && !kprobe_gone(p)) ? "[DISABLED]" : ""), 2166 (kprobe_optimized(pp) ? "[OPTIMIZED]" : ""), 2167 (kprobe_ftrace(pp) ? "[FTRACE]" : "")); 2168} 2169 2170static void __kprobes *kprobe_seq_start(struct seq_file *f, loff_t *pos) 2171{ 2172 return (*pos < KPROBE_TABLE_SIZE) ? pos : NULL; 2173} 2174 2175static void __kprobes *kprobe_seq_next(struct seq_file *f, void *v, loff_t *pos) 2176{ 2177 (*pos)++; 2178 if (*pos >= KPROBE_TABLE_SIZE) 2179 return NULL; 2180 return pos; 2181} 2182 2183static void __kprobes kprobe_seq_stop(struct seq_file *f, void *v) 2184{ 2185 /* Nothing to do */ 2186} 2187 2188static int __kprobes show_kprobe_addr(struct seq_file *pi, void *v) 2189{ 2190 struct hlist_head *head; 2191 struct hlist_node *node; 2192 struct kprobe *p, *kp; 2193 const char *sym = NULL; 2194 unsigned int i = *(loff_t *) v; 2195 unsigned long offset = 0; 2196 char *modname, namebuf[128]; 2197 2198 head = &kprobe_table[i]; 2199 preempt_disable(); 2200 hlist_for_each_entry_rcu(p, node, head, hlist) { 2201 sym = kallsyms_lookup((unsigned long)p->addr, NULL, 2202 &offset, &modname, namebuf); 2203 if (kprobe_aggrprobe(p)) { 2204 list_for_each_entry_rcu(kp, &p->list, list) 2205 report_probe(pi, kp, sym, offset, modname, p); 2206 } else 2207 report_probe(pi, p, sym, offset, modname, NULL); 2208 } 2209 preempt_enable(); 2210 return 0; 2211} 2212 2213static const struct seq_operations kprobes_seq_ops = { 2214 .start = kprobe_seq_start, 2215 .next = kprobe_seq_next, 2216 .stop = kprobe_seq_stop, 2217 .show = show_kprobe_addr 2218}; 2219 2220static int __kprobes kprobes_open(struct inode *inode, struct file *filp) 2221{ 2222 return seq_open(filp, &kprobes_seq_ops); 2223} 2224 2225static const struct file_operations debugfs_kprobes_operations = { 2226 .open = kprobes_open, 2227 .read = seq_read, 2228 .llseek = seq_lseek, 2229 .release = seq_release, 2230}; 2231 2232static void __kprobes arm_all_kprobes(void) 2233{ 2234 struct hlist_head *head; 2235 struct hlist_node *node; 2236 struct kprobe *p; 2237 unsigned int i; 2238 2239 mutex_lock(&kprobe_mutex); 2240 2241 /* If kprobes are armed, just return */ 2242 if (!kprobes_all_disarmed) 2243 goto already_enabled; 2244 2245 /* Arming kprobes doesn't optimize kprobe itself */ 2246 for (i = 0; i < KPROBE_TABLE_SIZE; i++) { 2247 head = &kprobe_table[i]; 2248 hlist_for_each_entry_rcu(p, node, head, hlist) 2249 if (!kprobe_disabled(p)) 2250 arm_kprobe(p); 2251 } 2252 2253 kprobes_all_disarmed = false; 2254 printk(KERN_INFO "Kprobes globally enabled\n"); 2255 2256already_enabled: 2257 mutex_unlock(&kprobe_mutex); 2258 return; 2259} 2260 2261static void __kprobes disarm_all_kprobes(void) 2262{ 2263 struct hlist_head *head; 2264 struct hlist_node *node; 2265 struct kprobe *p; 2266 unsigned int i; 2267 2268 mutex_lock(&kprobe_mutex); 2269 2270 /* If kprobes are already disarmed, just return */ 2271 if (kprobes_all_disarmed) { 2272 mutex_unlock(&kprobe_mutex); 2273 return; 2274 } 2275 2276 kprobes_all_disarmed = true; 2277 printk(KERN_INFO "Kprobes globally disabled\n"); 2278 2279 for (i = 0; i < KPROBE_TABLE_SIZE; i++) { 2280 head = &kprobe_table[i]; 2281 hlist_for_each_entry_rcu(p, node, head, hlist) { 2282 if (!arch_trampoline_kprobe(p) && !kprobe_disabled(p)) 2283 disarm_kprobe(p, false); 2284 } 2285 } 2286 mutex_unlock(&kprobe_mutex); 2287 2288 /* Wait for disarming all kprobes by optimizer */ 2289 wait_for_kprobe_optimizer(); 2290} 2291 2292/* 2293 * XXX: The debugfs bool file interface doesn't allow for callbacks 2294 * when the bool state is switched. We can reuse that facility when 2295 * available 2296 */ 2297static ssize_t read_enabled_file_bool(struct file *file, 2298 char __user *user_buf, size_t count, loff_t *ppos) 2299{ 2300 char buf[3]; 2301 2302 if (!kprobes_all_disarmed) 2303 buf[0] = '1'; 2304 else 2305 buf[0] = '0'; 2306 buf[1] = '\n'; 2307 buf[2] = 0x00; 2308 return simple_read_from_buffer(user_buf, count, ppos, buf, 2); 2309} 2310 2311static ssize_t write_enabled_file_bool(struct file *file, 2312 const char __user *user_buf, size_t count, loff_t *ppos) 2313{ 2314 char buf[32]; 2315 size_t buf_size; 2316 2317 buf_size = min(count, (sizeof(buf)-1)); 2318 if (copy_from_user(buf, user_buf, buf_size)) 2319 return -EFAULT; 2320 2321 switch (buf[0]) { 2322 case 'y': 2323 case 'Y': 2324 case '1': 2325 arm_all_kprobes(); 2326 break; 2327 case 'n': 2328 case 'N': 2329 case '0': 2330 disarm_all_kprobes(); 2331 break; 2332 } 2333 2334 return count; 2335} 2336 2337static const struct file_operations fops_kp = { 2338 .read = read_enabled_file_bool, 2339 .write = write_enabled_file_bool, 2340 .llseek = default_llseek, 2341}; 2342 2343static int __kprobes debugfs_kprobe_init(void) 2344{ 2345 struct dentry *dir, *file; 2346 unsigned int value = 1; 2347 2348 dir = debugfs_create_dir("kprobes", NULL); 2349 if (!dir) 2350 return -ENOMEM; 2351 2352 file = debugfs_create_file("list", 0444, dir, NULL, 2353 &debugfs_kprobes_operations); 2354 if (!file) { 2355 debugfs_remove(dir); 2356 return -ENOMEM; 2357 } 2358 2359 file = debugfs_create_file("enabled", 0600, dir, 2360 &value, &fops_kp); 2361 if (!file) { 2362 debugfs_remove(dir); 2363 return -ENOMEM; 2364 } 2365 2366 return 0; 2367} 2368 2369late_initcall(debugfs_kprobe_init); 2370#endif /* CONFIG_DEBUG_FS */ 2371 2372module_init(init_kprobes); 2373 2374/* defined in arch/.../kernel/kprobes.c */ 2375EXPORT_SYMBOL_GPL(jprobe_return); 2376