kprobes.c revision be8f274323c26ddc7e6fd6c44254b7abcdbe6389
1/* 2 * Kernel Probes (KProbes) 3 * kernel/kprobes.c 4 * 5 * This program is free software; you can redistribute it and/or modify 6 * it under the terms of the GNU General Public License as published by 7 * the Free Software Foundation; either version 2 of the License, or 8 * (at your option) any later version. 9 * 10 * This program is distributed in the hope that it will be useful, 11 * but WITHOUT ANY WARRANTY; without even the implied warranty of 12 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 13 * GNU General Public License for more details. 14 * 15 * You should have received a copy of the GNU General Public License 16 * along with this program; if not, write to the Free Software 17 * Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA. 18 * 19 * Copyright (C) IBM Corporation, 2002, 2004 20 * 21 * 2002-Oct Created by Vamsi Krishna S <vamsi_krishna@in.ibm.com> Kernel 22 * Probes initial implementation (includes suggestions from 23 * Rusty Russell). 24 * 2004-Aug Updated by Prasanna S Panchamukhi <prasanna@in.ibm.com> with 25 * hlists and exceptions notifier as suggested by Andi Kleen. 26 * 2004-July Suparna Bhattacharya <suparna@in.ibm.com> added jumper probes 27 * interface to access function arguments. 28 * 2004-Sep Prasanna S Panchamukhi <prasanna@in.ibm.com> Changed Kprobes 29 * exceptions notifier to be first on the priority list. 30 * 2005-May Hien Nguyen <hien@us.ibm.com>, Jim Keniston 31 * <jkenisto@us.ibm.com> and Prasanna S Panchamukhi 32 * <prasanna@in.ibm.com> added function-return probes. 33 */ 34#include <linux/kprobes.h> 35#include <linux/hash.h> 36#include <linux/init.h> 37#include <linux/slab.h> 38#include <linux/stddef.h> 39#include <linux/export.h> 40#include <linux/moduleloader.h> 41#include <linux/kallsyms.h> 42#include <linux/freezer.h> 43#include <linux/seq_file.h> 44#include <linux/debugfs.h> 45#include <linux/sysctl.h> 46#include <linux/kdebug.h> 47#include <linux/memory.h> 48#include <linux/ftrace.h> 49#include <linux/cpu.h> 50#include <linux/jump_label.h> 51 52#include <asm-generic/sections.h> 53#include <asm/cacheflush.h> 54#include <asm/errno.h> 55#include <asm/uaccess.h> 56 57#define KPROBE_HASH_BITS 6 58#define KPROBE_TABLE_SIZE (1 << KPROBE_HASH_BITS) 59 60 61/* 62 * Some oddball architectures like 64bit powerpc have function descriptors 63 * so this must be overridable. 64 */ 65#ifndef kprobe_lookup_name 66#define kprobe_lookup_name(name, addr) \ 67 addr = ((kprobe_opcode_t *)(kallsyms_lookup_name(name))) 68#endif 69 70static int kprobes_initialized; 71static struct hlist_head kprobe_table[KPROBE_TABLE_SIZE]; 72static struct hlist_head kretprobe_inst_table[KPROBE_TABLE_SIZE]; 73 74/* NOTE: change this value only with kprobe_mutex held */ 75static bool kprobes_all_disarmed; 76 77/* This protects kprobe_table and optimizing_list */ 78static DEFINE_MUTEX(kprobe_mutex); 79static DEFINE_PER_CPU(struct kprobe *, kprobe_instance) = NULL; 80static struct { 81 raw_spinlock_t lock ____cacheline_aligned_in_smp; 82} kretprobe_table_locks[KPROBE_TABLE_SIZE]; 83 84static raw_spinlock_t *kretprobe_table_lock_ptr(unsigned long hash) 85{ 86 return &(kretprobe_table_locks[hash].lock); 87} 88 89/* 90 * Normally, functions that we'd want to prohibit kprobes in, are marked 91 * __kprobes. But, there are cases where such functions already belong to 92 * a different section (__sched for preempt_schedule) 93 * 94 * For such cases, we now have a blacklist 95 */ 96static struct kprobe_blackpoint kprobe_blacklist[] = { 97 {"preempt_schedule",}, 98 {"native_get_debugreg",}, 99 {NULL} /* Terminator */ 100}; 101 102#ifdef __ARCH_WANT_KPROBES_INSN_SLOT 103/* 104 * kprobe->ainsn.insn points to the copy of the instruction to be 105 * single-stepped. x86_64, POWER4 and above have no-exec support and 106 * stepping on the instruction on a vmalloced/kmalloced/data page 107 * is a recipe for disaster 108 */ 109struct kprobe_insn_page { 110 struct list_head list; 111 kprobe_opcode_t *insns; /* Page of instruction slots */ 112 struct kprobe_insn_cache *cache; 113 int nused; 114 int ngarbage; 115 char slot_used[]; 116}; 117 118#define KPROBE_INSN_PAGE_SIZE(slots) \ 119 (offsetof(struct kprobe_insn_page, slot_used) + \ 120 (sizeof(char) * (slots))) 121 122static int slots_per_page(struct kprobe_insn_cache *c) 123{ 124 return PAGE_SIZE/(c->insn_size * sizeof(kprobe_opcode_t)); 125} 126 127enum kprobe_slot_state { 128 SLOT_CLEAN = 0, 129 SLOT_DIRTY = 1, 130 SLOT_USED = 2, 131}; 132 133static void *alloc_insn_page(void) 134{ 135 return module_alloc(PAGE_SIZE); 136} 137 138static void free_insn_page(void *page) 139{ 140 module_free(NULL, page); 141} 142 143struct kprobe_insn_cache kprobe_insn_slots = { 144 .mutex = __MUTEX_INITIALIZER(kprobe_insn_slots.mutex), 145 .alloc = alloc_insn_page, 146 .free = free_insn_page, 147 .pages = LIST_HEAD_INIT(kprobe_insn_slots.pages), 148 .insn_size = MAX_INSN_SIZE, 149 .nr_garbage = 0, 150}; 151static int __kprobes collect_garbage_slots(struct kprobe_insn_cache *c); 152 153/** 154 * __get_insn_slot() - Find a slot on an executable page for an instruction. 155 * We allocate an executable page if there's no room on existing ones. 156 */ 157kprobe_opcode_t __kprobes *__get_insn_slot(struct kprobe_insn_cache *c) 158{ 159 struct kprobe_insn_page *kip; 160 kprobe_opcode_t *slot = NULL; 161 162 mutex_lock(&c->mutex); 163 retry: 164 list_for_each_entry(kip, &c->pages, list) { 165 if (kip->nused < slots_per_page(c)) { 166 int i; 167 for (i = 0; i < slots_per_page(c); i++) { 168 if (kip->slot_used[i] == SLOT_CLEAN) { 169 kip->slot_used[i] = SLOT_USED; 170 kip->nused++; 171 slot = kip->insns + (i * c->insn_size); 172 goto out; 173 } 174 } 175 /* kip->nused is broken. Fix it. */ 176 kip->nused = slots_per_page(c); 177 WARN_ON(1); 178 } 179 } 180 181 /* If there are any garbage slots, collect it and try again. */ 182 if (c->nr_garbage && collect_garbage_slots(c) == 0) 183 goto retry; 184 185 /* All out of space. Need to allocate a new page. */ 186 kip = kmalloc(KPROBE_INSN_PAGE_SIZE(slots_per_page(c)), GFP_KERNEL); 187 if (!kip) 188 goto out; 189 190 /* 191 * Use module_alloc so this page is within +/- 2GB of where the 192 * kernel image and loaded module images reside. This is required 193 * so x86_64 can correctly handle the %rip-relative fixups. 194 */ 195 kip->insns = c->alloc(); 196 if (!kip->insns) { 197 kfree(kip); 198 goto out; 199 } 200 INIT_LIST_HEAD(&kip->list); 201 memset(kip->slot_used, SLOT_CLEAN, slots_per_page(c)); 202 kip->slot_used[0] = SLOT_USED; 203 kip->nused = 1; 204 kip->ngarbage = 0; 205 kip->cache = c; 206 list_add(&kip->list, &c->pages); 207 slot = kip->insns; 208out: 209 mutex_unlock(&c->mutex); 210 return slot; 211} 212 213/* Return 1 if all garbages are collected, otherwise 0. */ 214static int __kprobes collect_one_slot(struct kprobe_insn_page *kip, int idx) 215{ 216 kip->slot_used[idx] = SLOT_CLEAN; 217 kip->nused--; 218 if (kip->nused == 0) { 219 /* 220 * Page is no longer in use. Free it unless 221 * it's the last one. We keep the last one 222 * so as not to have to set it up again the 223 * next time somebody inserts a probe. 224 */ 225 if (!list_is_singular(&kip->list)) { 226 list_del(&kip->list); 227 kip->cache->free(kip->insns); 228 kfree(kip); 229 } 230 return 1; 231 } 232 return 0; 233} 234 235static int __kprobes collect_garbage_slots(struct kprobe_insn_cache *c) 236{ 237 struct kprobe_insn_page *kip, *next; 238 239 /* Ensure no-one is interrupted on the garbages */ 240 synchronize_sched(); 241 242 list_for_each_entry_safe(kip, next, &c->pages, list) { 243 int i; 244 if (kip->ngarbage == 0) 245 continue; 246 kip->ngarbage = 0; /* we will collect all garbages */ 247 for (i = 0; i < slots_per_page(c); i++) { 248 if (kip->slot_used[i] == SLOT_DIRTY && 249 collect_one_slot(kip, i)) 250 break; 251 } 252 } 253 c->nr_garbage = 0; 254 return 0; 255} 256 257void __kprobes __free_insn_slot(struct kprobe_insn_cache *c, 258 kprobe_opcode_t *slot, int dirty) 259{ 260 struct kprobe_insn_page *kip; 261 262 mutex_lock(&c->mutex); 263 list_for_each_entry(kip, &c->pages, list) { 264 long idx = ((long)slot - (long)kip->insns) / 265 (c->insn_size * sizeof(kprobe_opcode_t)); 266 if (idx >= 0 && idx < slots_per_page(c)) { 267 WARN_ON(kip->slot_used[idx] != SLOT_USED); 268 if (dirty) { 269 kip->slot_used[idx] = SLOT_DIRTY; 270 kip->ngarbage++; 271 if (++c->nr_garbage > slots_per_page(c)) 272 collect_garbage_slots(c); 273 } else 274 collect_one_slot(kip, idx); 275 goto out; 276 } 277 } 278 /* Could not free this slot. */ 279 WARN_ON(1); 280out: 281 mutex_unlock(&c->mutex); 282} 283 284#ifdef CONFIG_OPTPROBES 285/* For optimized_kprobe buffer */ 286struct kprobe_insn_cache kprobe_optinsn_slots = { 287 .mutex = __MUTEX_INITIALIZER(kprobe_optinsn_slots.mutex), 288 .alloc = alloc_insn_page, 289 .free = free_insn_page, 290 .pages = LIST_HEAD_INIT(kprobe_optinsn_slots.pages), 291 /* .insn_size is initialized later */ 292 .nr_garbage = 0, 293}; 294#endif 295#endif 296 297/* We have preemption disabled.. so it is safe to use __ versions */ 298static inline void set_kprobe_instance(struct kprobe *kp) 299{ 300 __this_cpu_write(kprobe_instance, kp); 301} 302 303static inline void reset_kprobe_instance(void) 304{ 305 __this_cpu_write(kprobe_instance, NULL); 306} 307 308/* 309 * This routine is called either: 310 * - under the kprobe_mutex - during kprobe_[un]register() 311 * OR 312 * - with preemption disabled - from arch/xxx/kernel/kprobes.c 313 */ 314struct kprobe __kprobes *get_kprobe(void *addr) 315{ 316 struct hlist_head *head; 317 struct kprobe *p; 318 319 head = &kprobe_table[hash_ptr(addr, KPROBE_HASH_BITS)]; 320 hlist_for_each_entry_rcu(p, head, hlist) { 321 if (p->addr == addr) 322 return p; 323 } 324 325 return NULL; 326} 327 328static int __kprobes aggr_pre_handler(struct kprobe *p, struct pt_regs *regs); 329 330/* Return true if the kprobe is an aggregator */ 331static inline int kprobe_aggrprobe(struct kprobe *p) 332{ 333 return p->pre_handler == aggr_pre_handler; 334} 335 336/* Return true(!0) if the kprobe is unused */ 337static inline int kprobe_unused(struct kprobe *p) 338{ 339 return kprobe_aggrprobe(p) && kprobe_disabled(p) && 340 list_empty(&p->list); 341} 342 343/* 344 * Keep all fields in the kprobe consistent 345 */ 346static inline void copy_kprobe(struct kprobe *ap, struct kprobe *p) 347{ 348 memcpy(&p->opcode, &ap->opcode, sizeof(kprobe_opcode_t)); 349 memcpy(&p->ainsn, &ap->ainsn, sizeof(struct arch_specific_insn)); 350} 351 352#ifdef CONFIG_OPTPROBES 353/* NOTE: change this value only with kprobe_mutex held */ 354static bool kprobes_allow_optimization; 355 356/* 357 * Call all pre_handler on the list, but ignores its return value. 358 * This must be called from arch-dep optimized caller. 359 */ 360void __kprobes opt_pre_handler(struct kprobe *p, struct pt_regs *regs) 361{ 362 struct kprobe *kp; 363 364 list_for_each_entry_rcu(kp, &p->list, list) { 365 if (kp->pre_handler && likely(!kprobe_disabled(kp))) { 366 set_kprobe_instance(kp); 367 kp->pre_handler(kp, regs); 368 } 369 reset_kprobe_instance(); 370 } 371} 372 373/* Free optimized instructions and optimized_kprobe */ 374static __kprobes void free_aggr_kprobe(struct kprobe *p) 375{ 376 struct optimized_kprobe *op; 377 378 op = container_of(p, struct optimized_kprobe, kp); 379 arch_remove_optimized_kprobe(op); 380 arch_remove_kprobe(p); 381 kfree(op); 382} 383 384/* Return true(!0) if the kprobe is ready for optimization. */ 385static inline int kprobe_optready(struct kprobe *p) 386{ 387 struct optimized_kprobe *op; 388 389 if (kprobe_aggrprobe(p)) { 390 op = container_of(p, struct optimized_kprobe, kp); 391 return arch_prepared_optinsn(&op->optinsn); 392 } 393 394 return 0; 395} 396 397/* Return true(!0) if the kprobe is disarmed. Note: p must be on hash list */ 398static inline int kprobe_disarmed(struct kprobe *p) 399{ 400 struct optimized_kprobe *op; 401 402 /* If kprobe is not aggr/opt probe, just return kprobe is disabled */ 403 if (!kprobe_aggrprobe(p)) 404 return kprobe_disabled(p); 405 406 op = container_of(p, struct optimized_kprobe, kp); 407 408 return kprobe_disabled(p) && list_empty(&op->list); 409} 410 411/* Return true(!0) if the probe is queued on (un)optimizing lists */ 412static int __kprobes kprobe_queued(struct kprobe *p) 413{ 414 struct optimized_kprobe *op; 415 416 if (kprobe_aggrprobe(p)) { 417 op = container_of(p, struct optimized_kprobe, kp); 418 if (!list_empty(&op->list)) 419 return 1; 420 } 421 return 0; 422} 423 424/* 425 * Return an optimized kprobe whose optimizing code replaces 426 * instructions including addr (exclude breakpoint). 427 */ 428static struct kprobe *__kprobes get_optimized_kprobe(unsigned long addr) 429{ 430 int i; 431 struct kprobe *p = NULL; 432 struct optimized_kprobe *op; 433 434 /* Don't check i == 0, since that is a breakpoint case. */ 435 for (i = 1; !p && i < MAX_OPTIMIZED_LENGTH; i++) 436 p = get_kprobe((void *)(addr - i)); 437 438 if (p && kprobe_optready(p)) { 439 op = container_of(p, struct optimized_kprobe, kp); 440 if (arch_within_optimized_kprobe(op, addr)) 441 return p; 442 } 443 444 return NULL; 445} 446 447/* Optimization staging list, protected by kprobe_mutex */ 448static LIST_HEAD(optimizing_list); 449static LIST_HEAD(unoptimizing_list); 450static LIST_HEAD(freeing_list); 451 452static void kprobe_optimizer(struct work_struct *work); 453static DECLARE_DELAYED_WORK(optimizing_work, kprobe_optimizer); 454#define OPTIMIZE_DELAY 5 455 456/* 457 * Optimize (replace a breakpoint with a jump) kprobes listed on 458 * optimizing_list. 459 */ 460static __kprobes void do_optimize_kprobes(void) 461{ 462 /* Optimization never be done when disarmed */ 463 if (kprobes_all_disarmed || !kprobes_allow_optimization || 464 list_empty(&optimizing_list)) 465 return; 466 467 /* 468 * The optimization/unoptimization refers online_cpus via 469 * stop_machine() and cpu-hotplug modifies online_cpus. 470 * And same time, text_mutex will be held in cpu-hotplug and here. 471 * This combination can cause a deadlock (cpu-hotplug try to lock 472 * text_mutex but stop_machine can not be done because online_cpus 473 * has been changed) 474 * To avoid this deadlock, we need to call get_online_cpus() 475 * for preventing cpu-hotplug outside of text_mutex locking. 476 */ 477 get_online_cpus(); 478 mutex_lock(&text_mutex); 479 arch_optimize_kprobes(&optimizing_list); 480 mutex_unlock(&text_mutex); 481 put_online_cpus(); 482} 483 484/* 485 * Unoptimize (replace a jump with a breakpoint and remove the breakpoint 486 * if need) kprobes listed on unoptimizing_list. 487 */ 488static __kprobes void do_unoptimize_kprobes(void) 489{ 490 struct optimized_kprobe *op, *tmp; 491 492 /* Unoptimization must be done anytime */ 493 if (list_empty(&unoptimizing_list)) 494 return; 495 496 /* Ditto to do_optimize_kprobes */ 497 get_online_cpus(); 498 mutex_lock(&text_mutex); 499 arch_unoptimize_kprobes(&unoptimizing_list, &freeing_list); 500 /* Loop free_list for disarming */ 501 list_for_each_entry_safe(op, tmp, &freeing_list, list) { 502 /* Disarm probes if marked disabled */ 503 if (kprobe_disabled(&op->kp)) 504 arch_disarm_kprobe(&op->kp); 505 if (kprobe_unused(&op->kp)) { 506 /* 507 * Remove unused probes from hash list. After waiting 508 * for synchronization, these probes are reclaimed. 509 * (reclaiming is done by do_free_cleaned_kprobes.) 510 */ 511 hlist_del_rcu(&op->kp.hlist); 512 } else 513 list_del_init(&op->list); 514 } 515 mutex_unlock(&text_mutex); 516 put_online_cpus(); 517} 518 519/* Reclaim all kprobes on the free_list */ 520static __kprobes void do_free_cleaned_kprobes(void) 521{ 522 struct optimized_kprobe *op, *tmp; 523 524 list_for_each_entry_safe(op, tmp, &freeing_list, list) { 525 BUG_ON(!kprobe_unused(&op->kp)); 526 list_del_init(&op->list); 527 free_aggr_kprobe(&op->kp); 528 } 529} 530 531/* Start optimizer after OPTIMIZE_DELAY passed */ 532static __kprobes void kick_kprobe_optimizer(void) 533{ 534 schedule_delayed_work(&optimizing_work, OPTIMIZE_DELAY); 535} 536 537/* Kprobe jump optimizer */ 538static __kprobes void kprobe_optimizer(struct work_struct *work) 539{ 540 mutex_lock(&kprobe_mutex); 541 /* Lock modules while optimizing kprobes */ 542 mutex_lock(&module_mutex); 543 544 /* 545 * Step 1: Unoptimize kprobes and collect cleaned (unused and disarmed) 546 * kprobes before waiting for quiesence period. 547 */ 548 do_unoptimize_kprobes(); 549 550 /* 551 * Step 2: Wait for quiesence period to ensure all running interrupts 552 * are done. Because optprobe may modify multiple instructions 553 * there is a chance that Nth instruction is interrupted. In that 554 * case, running interrupt can return to 2nd-Nth byte of jump 555 * instruction. This wait is for avoiding it. 556 */ 557 synchronize_sched(); 558 559 /* Step 3: Optimize kprobes after quiesence period */ 560 do_optimize_kprobes(); 561 562 /* Step 4: Free cleaned kprobes after quiesence period */ 563 do_free_cleaned_kprobes(); 564 565 mutex_unlock(&module_mutex); 566 mutex_unlock(&kprobe_mutex); 567 568 /* Step 5: Kick optimizer again if needed */ 569 if (!list_empty(&optimizing_list) || !list_empty(&unoptimizing_list)) 570 kick_kprobe_optimizer(); 571} 572 573/* Wait for completing optimization and unoptimization */ 574static __kprobes void wait_for_kprobe_optimizer(void) 575{ 576 mutex_lock(&kprobe_mutex); 577 578 while (!list_empty(&optimizing_list) || !list_empty(&unoptimizing_list)) { 579 mutex_unlock(&kprobe_mutex); 580 581 /* this will also make optimizing_work execute immmediately */ 582 flush_delayed_work(&optimizing_work); 583 /* @optimizing_work might not have been queued yet, relax */ 584 cpu_relax(); 585 586 mutex_lock(&kprobe_mutex); 587 } 588 589 mutex_unlock(&kprobe_mutex); 590} 591 592/* Optimize kprobe if p is ready to be optimized */ 593static __kprobes void optimize_kprobe(struct kprobe *p) 594{ 595 struct optimized_kprobe *op; 596 597 /* Check if the kprobe is disabled or not ready for optimization. */ 598 if (!kprobe_optready(p) || !kprobes_allow_optimization || 599 (kprobe_disabled(p) || kprobes_all_disarmed)) 600 return; 601 602 /* Both of break_handler and post_handler are not supported. */ 603 if (p->break_handler || p->post_handler) 604 return; 605 606 op = container_of(p, struct optimized_kprobe, kp); 607 608 /* Check there is no other kprobes at the optimized instructions */ 609 if (arch_check_optimized_kprobe(op) < 0) 610 return; 611 612 /* Check if it is already optimized. */ 613 if (op->kp.flags & KPROBE_FLAG_OPTIMIZED) 614 return; 615 op->kp.flags |= KPROBE_FLAG_OPTIMIZED; 616 617 if (!list_empty(&op->list)) 618 /* This is under unoptimizing. Just dequeue the probe */ 619 list_del_init(&op->list); 620 else { 621 list_add(&op->list, &optimizing_list); 622 kick_kprobe_optimizer(); 623 } 624} 625 626/* Short cut to direct unoptimizing */ 627static __kprobes void force_unoptimize_kprobe(struct optimized_kprobe *op) 628{ 629 get_online_cpus(); 630 arch_unoptimize_kprobe(op); 631 put_online_cpus(); 632 if (kprobe_disabled(&op->kp)) 633 arch_disarm_kprobe(&op->kp); 634} 635 636/* Unoptimize a kprobe if p is optimized */ 637static __kprobes void unoptimize_kprobe(struct kprobe *p, bool force) 638{ 639 struct optimized_kprobe *op; 640 641 if (!kprobe_aggrprobe(p) || kprobe_disarmed(p)) 642 return; /* This is not an optprobe nor optimized */ 643 644 op = container_of(p, struct optimized_kprobe, kp); 645 if (!kprobe_optimized(p)) { 646 /* Unoptimized or unoptimizing case */ 647 if (force && !list_empty(&op->list)) { 648 /* 649 * Only if this is unoptimizing kprobe and forced, 650 * forcibly unoptimize it. (No need to unoptimize 651 * unoptimized kprobe again :) 652 */ 653 list_del_init(&op->list); 654 force_unoptimize_kprobe(op); 655 } 656 return; 657 } 658 659 op->kp.flags &= ~KPROBE_FLAG_OPTIMIZED; 660 if (!list_empty(&op->list)) { 661 /* Dequeue from the optimization queue */ 662 list_del_init(&op->list); 663 return; 664 } 665 /* Optimized kprobe case */ 666 if (force) 667 /* Forcibly update the code: this is a special case */ 668 force_unoptimize_kprobe(op); 669 else { 670 list_add(&op->list, &unoptimizing_list); 671 kick_kprobe_optimizer(); 672 } 673} 674 675/* Cancel unoptimizing for reusing */ 676static void reuse_unused_kprobe(struct kprobe *ap) 677{ 678 struct optimized_kprobe *op; 679 680 BUG_ON(!kprobe_unused(ap)); 681 /* 682 * Unused kprobe MUST be on the way of delayed unoptimizing (means 683 * there is still a relative jump) and disabled. 684 */ 685 op = container_of(ap, struct optimized_kprobe, kp); 686 if (unlikely(list_empty(&op->list))) 687 printk(KERN_WARNING "Warning: found a stray unused " 688 "aggrprobe@%p\n", ap->addr); 689 /* Enable the probe again */ 690 ap->flags &= ~KPROBE_FLAG_DISABLED; 691 /* Optimize it again (remove from op->list) */ 692 BUG_ON(!kprobe_optready(ap)); 693 optimize_kprobe(ap); 694} 695 696/* Remove optimized instructions */ 697static void __kprobes kill_optimized_kprobe(struct kprobe *p) 698{ 699 struct optimized_kprobe *op; 700 701 op = container_of(p, struct optimized_kprobe, kp); 702 if (!list_empty(&op->list)) 703 /* Dequeue from the (un)optimization queue */ 704 list_del_init(&op->list); 705 op->kp.flags &= ~KPROBE_FLAG_OPTIMIZED; 706 707 if (kprobe_unused(p)) { 708 /* Enqueue if it is unused */ 709 list_add(&op->list, &freeing_list); 710 /* 711 * Remove unused probes from the hash list. After waiting 712 * for synchronization, this probe is reclaimed. 713 * (reclaiming is done by do_free_cleaned_kprobes().) 714 */ 715 hlist_del_rcu(&op->kp.hlist); 716 } 717 718 /* Don't touch the code, because it is already freed. */ 719 arch_remove_optimized_kprobe(op); 720} 721 722/* Try to prepare optimized instructions */ 723static __kprobes void prepare_optimized_kprobe(struct kprobe *p) 724{ 725 struct optimized_kprobe *op; 726 727 op = container_of(p, struct optimized_kprobe, kp); 728 arch_prepare_optimized_kprobe(op); 729} 730 731/* Allocate new optimized_kprobe and try to prepare optimized instructions */ 732static __kprobes struct kprobe *alloc_aggr_kprobe(struct kprobe *p) 733{ 734 struct optimized_kprobe *op; 735 736 op = kzalloc(sizeof(struct optimized_kprobe), GFP_KERNEL); 737 if (!op) 738 return NULL; 739 740 INIT_LIST_HEAD(&op->list); 741 op->kp.addr = p->addr; 742 arch_prepare_optimized_kprobe(op); 743 744 return &op->kp; 745} 746 747static void __kprobes init_aggr_kprobe(struct kprobe *ap, struct kprobe *p); 748 749/* 750 * Prepare an optimized_kprobe and optimize it 751 * NOTE: p must be a normal registered kprobe 752 */ 753static __kprobes void try_to_optimize_kprobe(struct kprobe *p) 754{ 755 struct kprobe *ap; 756 struct optimized_kprobe *op; 757 758 /* Impossible to optimize ftrace-based kprobe */ 759 if (kprobe_ftrace(p)) 760 return; 761 762 /* For preparing optimization, jump_label_text_reserved() is called */ 763 jump_label_lock(); 764 mutex_lock(&text_mutex); 765 766 ap = alloc_aggr_kprobe(p); 767 if (!ap) 768 goto out; 769 770 op = container_of(ap, struct optimized_kprobe, kp); 771 if (!arch_prepared_optinsn(&op->optinsn)) { 772 /* If failed to setup optimizing, fallback to kprobe */ 773 arch_remove_optimized_kprobe(op); 774 kfree(op); 775 goto out; 776 } 777 778 init_aggr_kprobe(ap, p); 779 optimize_kprobe(ap); /* This just kicks optimizer thread */ 780 781out: 782 mutex_unlock(&text_mutex); 783 jump_label_unlock(); 784} 785 786#ifdef CONFIG_SYSCTL 787static void __kprobes optimize_all_kprobes(void) 788{ 789 struct hlist_head *head; 790 struct kprobe *p; 791 unsigned int i; 792 793 mutex_lock(&kprobe_mutex); 794 /* If optimization is already allowed, just return */ 795 if (kprobes_allow_optimization) 796 goto out; 797 798 kprobes_allow_optimization = true; 799 for (i = 0; i < KPROBE_TABLE_SIZE; i++) { 800 head = &kprobe_table[i]; 801 hlist_for_each_entry_rcu(p, head, hlist) 802 if (!kprobe_disabled(p)) 803 optimize_kprobe(p); 804 } 805 printk(KERN_INFO "Kprobes globally optimized\n"); 806out: 807 mutex_unlock(&kprobe_mutex); 808} 809 810static void __kprobes unoptimize_all_kprobes(void) 811{ 812 struct hlist_head *head; 813 struct kprobe *p; 814 unsigned int i; 815 816 mutex_lock(&kprobe_mutex); 817 /* If optimization is already prohibited, just return */ 818 if (!kprobes_allow_optimization) { 819 mutex_unlock(&kprobe_mutex); 820 return; 821 } 822 823 kprobes_allow_optimization = false; 824 for (i = 0; i < KPROBE_TABLE_SIZE; i++) { 825 head = &kprobe_table[i]; 826 hlist_for_each_entry_rcu(p, head, hlist) { 827 if (!kprobe_disabled(p)) 828 unoptimize_kprobe(p, false); 829 } 830 } 831 mutex_unlock(&kprobe_mutex); 832 833 /* Wait for unoptimizing completion */ 834 wait_for_kprobe_optimizer(); 835 printk(KERN_INFO "Kprobes globally unoptimized\n"); 836} 837 838static DEFINE_MUTEX(kprobe_sysctl_mutex); 839int sysctl_kprobes_optimization; 840int proc_kprobes_optimization_handler(struct ctl_table *table, int write, 841 void __user *buffer, size_t *length, 842 loff_t *ppos) 843{ 844 int ret; 845 846 mutex_lock(&kprobe_sysctl_mutex); 847 sysctl_kprobes_optimization = kprobes_allow_optimization ? 1 : 0; 848 ret = proc_dointvec_minmax(table, write, buffer, length, ppos); 849 850 if (sysctl_kprobes_optimization) 851 optimize_all_kprobes(); 852 else 853 unoptimize_all_kprobes(); 854 mutex_unlock(&kprobe_sysctl_mutex); 855 856 return ret; 857} 858#endif /* CONFIG_SYSCTL */ 859 860/* Put a breakpoint for a probe. Must be called with text_mutex locked */ 861static void __kprobes __arm_kprobe(struct kprobe *p) 862{ 863 struct kprobe *_p; 864 865 /* Check collision with other optimized kprobes */ 866 _p = get_optimized_kprobe((unsigned long)p->addr); 867 if (unlikely(_p)) 868 /* Fallback to unoptimized kprobe */ 869 unoptimize_kprobe(_p, true); 870 871 arch_arm_kprobe(p); 872 optimize_kprobe(p); /* Try to optimize (add kprobe to a list) */ 873} 874 875/* Remove the breakpoint of a probe. Must be called with text_mutex locked */ 876static void __kprobes __disarm_kprobe(struct kprobe *p, bool reopt) 877{ 878 struct kprobe *_p; 879 880 unoptimize_kprobe(p, false); /* Try to unoptimize */ 881 882 if (!kprobe_queued(p)) { 883 arch_disarm_kprobe(p); 884 /* If another kprobe was blocked, optimize it. */ 885 _p = get_optimized_kprobe((unsigned long)p->addr); 886 if (unlikely(_p) && reopt) 887 optimize_kprobe(_p); 888 } 889 /* TODO: reoptimize others after unoptimized this probe */ 890} 891 892#else /* !CONFIG_OPTPROBES */ 893 894#define optimize_kprobe(p) do {} while (0) 895#define unoptimize_kprobe(p, f) do {} while (0) 896#define kill_optimized_kprobe(p) do {} while (0) 897#define prepare_optimized_kprobe(p) do {} while (0) 898#define try_to_optimize_kprobe(p) do {} while (0) 899#define __arm_kprobe(p) arch_arm_kprobe(p) 900#define __disarm_kprobe(p, o) arch_disarm_kprobe(p) 901#define kprobe_disarmed(p) kprobe_disabled(p) 902#define wait_for_kprobe_optimizer() do {} while (0) 903 904/* There should be no unused kprobes can be reused without optimization */ 905static void reuse_unused_kprobe(struct kprobe *ap) 906{ 907 printk(KERN_ERR "Error: There should be no unused kprobe here.\n"); 908 BUG_ON(kprobe_unused(ap)); 909} 910 911static __kprobes void free_aggr_kprobe(struct kprobe *p) 912{ 913 arch_remove_kprobe(p); 914 kfree(p); 915} 916 917static __kprobes struct kprobe *alloc_aggr_kprobe(struct kprobe *p) 918{ 919 return kzalloc(sizeof(struct kprobe), GFP_KERNEL); 920} 921#endif /* CONFIG_OPTPROBES */ 922 923#ifdef CONFIG_KPROBES_ON_FTRACE 924static struct ftrace_ops kprobe_ftrace_ops __read_mostly = { 925 .func = kprobe_ftrace_handler, 926 .flags = FTRACE_OPS_FL_SAVE_REGS, 927}; 928static int kprobe_ftrace_enabled; 929 930/* Must ensure p->addr is really on ftrace */ 931static int __kprobes prepare_kprobe(struct kprobe *p) 932{ 933 if (!kprobe_ftrace(p)) 934 return arch_prepare_kprobe(p); 935 936 return arch_prepare_kprobe_ftrace(p); 937} 938 939/* Caller must lock kprobe_mutex */ 940static void __kprobes arm_kprobe_ftrace(struct kprobe *p) 941{ 942 int ret; 943 944 ret = ftrace_set_filter_ip(&kprobe_ftrace_ops, 945 (unsigned long)p->addr, 0, 0); 946 WARN(ret < 0, "Failed to arm kprobe-ftrace at %p (%d)\n", p->addr, ret); 947 kprobe_ftrace_enabled++; 948 if (kprobe_ftrace_enabled == 1) { 949 ret = register_ftrace_function(&kprobe_ftrace_ops); 950 WARN(ret < 0, "Failed to init kprobe-ftrace (%d)\n", ret); 951 } 952} 953 954/* Caller must lock kprobe_mutex */ 955static void __kprobes disarm_kprobe_ftrace(struct kprobe *p) 956{ 957 int ret; 958 959 kprobe_ftrace_enabled--; 960 if (kprobe_ftrace_enabled == 0) { 961 ret = unregister_ftrace_function(&kprobe_ftrace_ops); 962 WARN(ret < 0, "Failed to init kprobe-ftrace (%d)\n", ret); 963 } 964 ret = ftrace_set_filter_ip(&kprobe_ftrace_ops, 965 (unsigned long)p->addr, 1, 0); 966 WARN(ret < 0, "Failed to disarm kprobe-ftrace at %p (%d)\n", p->addr, ret); 967} 968#else /* !CONFIG_KPROBES_ON_FTRACE */ 969#define prepare_kprobe(p) arch_prepare_kprobe(p) 970#define arm_kprobe_ftrace(p) do {} while (0) 971#define disarm_kprobe_ftrace(p) do {} while (0) 972#endif 973 974/* Arm a kprobe with text_mutex */ 975static void __kprobes arm_kprobe(struct kprobe *kp) 976{ 977 if (unlikely(kprobe_ftrace(kp))) { 978 arm_kprobe_ftrace(kp); 979 return; 980 } 981 /* 982 * Here, since __arm_kprobe() doesn't use stop_machine(), 983 * this doesn't cause deadlock on text_mutex. So, we don't 984 * need get_online_cpus(). 985 */ 986 mutex_lock(&text_mutex); 987 __arm_kprobe(kp); 988 mutex_unlock(&text_mutex); 989} 990 991/* Disarm a kprobe with text_mutex */ 992static void __kprobes disarm_kprobe(struct kprobe *kp, bool reopt) 993{ 994 if (unlikely(kprobe_ftrace(kp))) { 995 disarm_kprobe_ftrace(kp); 996 return; 997 } 998 /* Ditto */ 999 mutex_lock(&text_mutex); 1000 __disarm_kprobe(kp, reopt); 1001 mutex_unlock(&text_mutex); 1002} 1003 1004/* 1005 * Aggregate handlers for multiple kprobes support - these handlers 1006 * take care of invoking the individual kprobe handlers on p->list 1007 */ 1008static int __kprobes aggr_pre_handler(struct kprobe *p, struct pt_regs *regs) 1009{ 1010 struct kprobe *kp; 1011 1012 list_for_each_entry_rcu(kp, &p->list, list) { 1013 if (kp->pre_handler && likely(!kprobe_disabled(kp))) { 1014 set_kprobe_instance(kp); 1015 if (kp->pre_handler(kp, regs)) 1016 return 1; 1017 } 1018 reset_kprobe_instance(); 1019 } 1020 return 0; 1021} 1022 1023static void __kprobes aggr_post_handler(struct kprobe *p, struct pt_regs *regs, 1024 unsigned long flags) 1025{ 1026 struct kprobe *kp; 1027 1028 list_for_each_entry_rcu(kp, &p->list, list) { 1029 if (kp->post_handler && likely(!kprobe_disabled(kp))) { 1030 set_kprobe_instance(kp); 1031 kp->post_handler(kp, regs, flags); 1032 reset_kprobe_instance(); 1033 } 1034 } 1035} 1036 1037static int __kprobes aggr_fault_handler(struct kprobe *p, struct pt_regs *regs, 1038 int trapnr) 1039{ 1040 struct kprobe *cur = __this_cpu_read(kprobe_instance); 1041 1042 /* 1043 * if we faulted "during" the execution of a user specified 1044 * probe handler, invoke just that probe's fault handler 1045 */ 1046 if (cur && cur->fault_handler) { 1047 if (cur->fault_handler(cur, regs, trapnr)) 1048 return 1; 1049 } 1050 return 0; 1051} 1052 1053static int __kprobes aggr_break_handler(struct kprobe *p, struct pt_regs *regs) 1054{ 1055 struct kprobe *cur = __this_cpu_read(kprobe_instance); 1056 int ret = 0; 1057 1058 if (cur && cur->break_handler) { 1059 if (cur->break_handler(cur, regs)) 1060 ret = 1; 1061 } 1062 reset_kprobe_instance(); 1063 return ret; 1064} 1065 1066/* Walks the list and increments nmissed count for multiprobe case */ 1067void __kprobes kprobes_inc_nmissed_count(struct kprobe *p) 1068{ 1069 struct kprobe *kp; 1070 if (!kprobe_aggrprobe(p)) { 1071 p->nmissed++; 1072 } else { 1073 list_for_each_entry_rcu(kp, &p->list, list) 1074 kp->nmissed++; 1075 } 1076 return; 1077} 1078 1079void __kprobes recycle_rp_inst(struct kretprobe_instance *ri, 1080 struct hlist_head *head) 1081{ 1082 struct kretprobe *rp = ri->rp; 1083 1084 /* remove rp inst off the rprobe_inst_table */ 1085 hlist_del(&ri->hlist); 1086 INIT_HLIST_NODE(&ri->hlist); 1087 if (likely(rp)) { 1088 raw_spin_lock(&rp->lock); 1089 hlist_add_head(&ri->hlist, &rp->free_instances); 1090 raw_spin_unlock(&rp->lock); 1091 } else 1092 /* Unregistering */ 1093 hlist_add_head(&ri->hlist, head); 1094} 1095 1096void __kprobes kretprobe_hash_lock(struct task_struct *tsk, 1097 struct hlist_head **head, unsigned long *flags) 1098__acquires(hlist_lock) 1099{ 1100 unsigned long hash = hash_ptr(tsk, KPROBE_HASH_BITS); 1101 raw_spinlock_t *hlist_lock; 1102 1103 *head = &kretprobe_inst_table[hash]; 1104 hlist_lock = kretprobe_table_lock_ptr(hash); 1105 raw_spin_lock_irqsave(hlist_lock, *flags); 1106} 1107 1108static void __kprobes kretprobe_table_lock(unsigned long hash, 1109 unsigned long *flags) 1110__acquires(hlist_lock) 1111{ 1112 raw_spinlock_t *hlist_lock = kretprobe_table_lock_ptr(hash); 1113 raw_spin_lock_irqsave(hlist_lock, *flags); 1114} 1115 1116void __kprobes kretprobe_hash_unlock(struct task_struct *tsk, 1117 unsigned long *flags) 1118__releases(hlist_lock) 1119{ 1120 unsigned long hash = hash_ptr(tsk, KPROBE_HASH_BITS); 1121 raw_spinlock_t *hlist_lock; 1122 1123 hlist_lock = kretprobe_table_lock_ptr(hash); 1124 raw_spin_unlock_irqrestore(hlist_lock, *flags); 1125} 1126 1127static void __kprobes kretprobe_table_unlock(unsigned long hash, 1128 unsigned long *flags) 1129__releases(hlist_lock) 1130{ 1131 raw_spinlock_t *hlist_lock = kretprobe_table_lock_ptr(hash); 1132 raw_spin_unlock_irqrestore(hlist_lock, *flags); 1133} 1134 1135/* 1136 * This function is called from finish_task_switch when task tk becomes dead, 1137 * so that we can recycle any function-return probe instances associated 1138 * with this task. These left over instances represent probed functions 1139 * that have been called but will never return. 1140 */ 1141void __kprobes kprobe_flush_task(struct task_struct *tk) 1142{ 1143 struct kretprobe_instance *ri; 1144 struct hlist_head *head, empty_rp; 1145 struct hlist_node *tmp; 1146 unsigned long hash, flags = 0; 1147 1148 if (unlikely(!kprobes_initialized)) 1149 /* Early boot. kretprobe_table_locks not yet initialized. */ 1150 return; 1151 1152 INIT_HLIST_HEAD(&empty_rp); 1153 hash = hash_ptr(tk, KPROBE_HASH_BITS); 1154 head = &kretprobe_inst_table[hash]; 1155 kretprobe_table_lock(hash, &flags); 1156 hlist_for_each_entry_safe(ri, tmp, head, hlist) { 1157 if (ri->task == tk) 1158 recycle_rp_inst(ri, &empty_rp); 1159 } 1160 kretprobe_table_unlock(hash, &flags); 1161 hlist_for_each_entry_safe(ri, tmp, &empty_rp, hlist) { 1162 hlist_del(&ri->hlist); 1163 kfree(ri); 1164 } 1165} 1166 1167static inline void free_rp_inst(struct kretprobe *rp) 1168{ 1169 struct kretprobe_instance *ri; 1170 struct hlist_node *next; 1171 1172 hlist_for_each_entry_safe(ri, next, &rp->free_instances, hlist) { 1173 hlist_del(&ri->hlist); 1174 kfree(ri); 1175 } 1176} 1177 1178static void __kprobes cleanup_rp_inst(struct kretprobe *rp) 1179{ 1180 unsigned long flags, hash; 1181 struct kretprobe_instance *ri; 1182 struct hlist_node *next; 1183 struct hlist_head *head; 1184 1185 /* No race here */ 1186 for (hash = 0; hash < KPROBE_TABLE_SIZE; hash++) { 1187 kretprobe_table_lock(hash, &flags); 1188 head = &kretprobe_inst_table[hash]; 1189 hlist_for_each_entry_safe(ri, next, head, hlist) { 1190 if (ri->rp == rp) 1191 ri->rp = NULL; 1192 } 1193 kretprobe_table_unlock(hash, &flags); 1194 } 1195 free_rp_inst(rp); 1196} 1197 1198/* 1199* Add the new probe to ap->list. Fail if this is the 1200* second jprobe at the address - two jprobes can't coexist 1201*/ 1202static int __kprobes add_new_kprobe(struct kprobe *ap, struct kprobe *p) 1203{ 1204 BUG_ON(kprobe_gone(ap) || kprobe_gone(p)); 1205 1206 if (p->break_handler || p->post_handler) 1207 unoptimize_kprobe(ap, true); /* Fall back to normal kprobe */ 1208 1209 if (p->break_handler) { 1210 if (ap->break_handler) 1211 return -EEXIST; 1212 list_add_tail_rcu(&p->list, &ap->list); 1213 ap->break_handler = aggr_break_handler; 1214 } else 1215 list_add_rcu(&p->list, &ap->list); 1216 if (p->post_handler && !ap->post_handler) 1217 ap->post_handler = aggr_post_handler; 1218 1219 return 0; 1220} 1221 1222/* 1223 * Fill in the required fields of the "manager kprobe". Replace the 1224 * earlier kprobe in the hlist with the manager kprobe 1225 */ 1226static void __kprobes init_aggr_kprobe(struct kprobe *ap, struct kprobe *p) 1227{ 1228 /* Copy p's insn slot to ap */ 1229 copy_kprobe(p, ap); 1230 flush_insn_slot(ap); 1231 ap->addr = p->addr; 1232 ap->flags = p->flags & ~KPROBE_FLAG_OPTIMIZED; 1233 ap->pre_handler = aggr_pre_handler; 1234 ap->fault_handler = aggr_fault_handler; 1235 /* We don't care the kprobe which has gone. */ 1236 if (p->post_handler && !kprobe_gone(p)) 1237 ap->post_handler = aggr_post_handler; 1238 if (p->break_handler && !kprobe_gone(p)) 1239 ap->break_handler = aggr_break_handler; 1240 1241 INIT_LIST_HEAD(&ap->list); 1242 INIT_HLIST_NODE(&ap->hlist); 1243 1244 list_add_rcu(&p->list, &ap->list); 1245 hlist_replace_rcu(&p->hlist, &ap->hlist); 1246} 1247 1248/* 1249 * This is the second or subsequent kprobe at the address - handle 1250 * the intricacies 1251 */ 1252static int __kprobes register_aggr_kprobe(struct kprobe *orig_p, 1253 struct kprobe *p) 1254{ 1255 int ret = 0; 1256 struct kprobe *ap = orig_p; 1257 1258 /* For preparing optimization, jump_label_text_reserved() is called */ 1259 jump_label_lock(); 1260 /* 1261 * Get online CPUs to avoid text_mutex deadlock.with stop machine, 1262 * which is invoked by unoptimize_kprobe() in add_new_kprobe() 1263 */ 1264 get_online_cpus(); 1265 mutex_lock(&text_mutex); 1266 1267 if (!kprobe_aggrprobe(orig_p)) { 1268 /* If orig_p is not an aggr_kprobe, create new aggr_kprobe. */ 1269 ap = alloc_aggr_kprobe(orig_p); 1270 if (!ap) { 1271 ret = -ENOMEM; 1272 goto out; 1273 } 1274 init_aggr_kprobe(ap, orig_p); 1275 } else if (kprobe_unused(ap)) 1276 /* This probe is going to die. Rescue it */ 1277 reuse_unused_kprobe(ap); 1278 1279 if (kprobe_gone(ap)) { 1280 /* 1281 * Attempting to insert new probe at the same location that 1282 * had a probe in the module vaddr area which already 1283 * freed. So, the instruction slot has already been 1284 * released. We need a new slot for the new probe. 1285 */ 1286 ret = arch_prepare_kprobe(ap); 1287 if (ret) 1288 /* 1289 * Even if fail to allocate new slot, don't need to 1290 * free aggr_probe. It will be used next time, or 1291 * freed by unregister_kprobe. 1292 */ 1293 goto out; 1294 1295 /* Prepare optimized instructions if possible. */ 1296 prepare_optimized_kprobe(ap); 1297 1298 /* 1299 * Clear gone flag to prevent allocating new slot again, and 1300 * set disabled flag because it is not armed yet. 1301 */ 1302 ap->flags = (ap->flags & ~KPROBE_FLAG_GONE) 1303 | KPROBE_FLAG_DISABLED; 1304 } 1305 1306 /* Copy ap's insn slot to p */ 1307 copy_kprobe(ap, p); 1308 ret = add_new_kprobe(ap, p); 1309 1310out: 1311 mutex_unlock(&text_mutex); 1312 put_online_cpus(); 1313 jump_label_unlock(); 1314 1315 if (ret == 0 && kprobe_disabled(ap) && !kprobe_disabled(p)) { 1316 ap->flags &= ~KPROBE_FLAG_DISABLED; 1317 if (!kprobes_all_disarmed) 1318 /* Arm the breakpoint again. */ 1319 arm_kprobe(ap); 1320 } 1321 return ret; 1322} 1323 1324bool __weak arch_within_kprobe_blacklist(unsigned long addr) 1325{ 1326 /* The __kprobes marked functions and entry code must not be probed */ 1327 return addr >= (unsigned long)__kprobes_text_start && 1328 addr < (unsigned long)__kprobes_text_end; 1329} 1330 1331static int __kprobes in_kprobes_functions(unsigned long addr) 1332{ 1333 struct kprobe_blackpoint *kb; 1334 1335 if (arch_within_kprobe_blacklist(addr)) 1336 return -EINVAL; 1337 /* 1338 * If there exists a kprobe_blacklist, verify and 1339 * fail any probe registration in the prohibited area 1340 */ 1341 for (kb = kprobe_blacklist; kb->name != NULL; kb++) { 1342 if (kb->start_addr) { 1343 if (addr >= kb->start_addr && 1344 addr < (kb->start_addr + kb->range)) 1345 return -EINVAL; 1346 } 1347 } 1348 return 0; 1349} 1350 1351/* 1352 * If we have a symbol_name argument, look it up and add the offset field 1353 * to it. This way, we can specify a relative address to a symbol. 1354 * This returns encoded errors if it fails to look up symbol or invalid 1355 * combination of parameters. 1356 */ 1357static kprobe_opcode_t __kprobes *kprobe_addr(struct kprobe *p) 1358{ 1359 kprobe_opcode_t *addr = p->addr; 1360 1361 if ((p->symbol_name && p->addr) || 1362 (!p->symbol_name && !p->addr)) 1363 goto invalid; 1364 1365 if (p->symbol_name) { 1366 kprobe_lookup_name(p->symbol_name, addr); 1367 if (!addr) 1368 return ERR_PTR(-ENOENT); 1369 } 1370 1371 addr = (kprobe_opcode_t *)(((char *)addr) + p->offset); 1372 if (addr) 1373 return addr; 1374 1375invalid: 1376 return ERR_PTR(-EINVAL); 1377} 1378 1379/* Check passed kprobe is valid and return kprobe in kprobe_table. */ 1380static struct kprobe * __kprobes __get_valid_kprobe(struct kprobe *p) 1381{ 1382 struct kprobe *ap, *list_p; 1383 1384 ap = get_kprobe(p->addr); 1385 if (unlikely(!ap)) 1386 return NULL; 1387 1388 if (p != ap) { 1389 list_for_each_entry_rcu(list_p, &ap->list, list) 1390 if (list_p == p) 1391 /* kprobe p is a valid probe */ 1392 goto valid; 1393 return NULL; 1394 } 1395valid: 1396 return ap; 1397} 1398 1399/* Return error if the kprobe is being re-registered */ 1400static inline int check_kprobe_rereg(struct kprobe *p) 1401{ 1402 int ret = 0; 1403 1404 mutex_lock(&kprobe_mutex); 1405 if (__get_valid_kprobe(p)) 1406 ret = -EINVAL; 1407 mutex_unlock(&kprobe_mutex); 1408 1409 return ret; 1410} 1411 1412static __kprobes int check_kprobe_address_safe(struct kprobe *p, 1413 struct module **probed_mod) 1414{ 1415 int ret = 0; 1416 unsigned long ftrace_addr; 1417 1418 /* 1419 * If the address is located on a ftrace nop, set the 1420 * breakpoint to the following instruction. 1421 */ 1422 ftrace_addr = ftrace_location((unsigned long)p->addr); 1423 if (ftrace_addr) { 1424#ifdef CONFIG_KPROBES_ON_FTRACE 1425 /* Given address is not on the instruction boundary */ 1426 if ((unsigned long)p->addr != ftrace_addr) 1427 return -EILSEQ; 1428 p->flags |= KPROBE_FLAG_FTRACE; 1429#else /* !CONFIG_KPROBES_ON_FTRACE */ 1430 return -EINVAL; 1431#endif 1432 } 1433 1434 jump_label_lock(); 1435 preempt_disable(); 1436 1437 /* Ensure it is not in reserved area nor out of text */ 1438 if (!kernel_text_address((unsigned long) p->addr) || 1439 in_kprobes_functions((unsigned long) p->addr) || 1440 jump_label_text_reserved(p->addr, p->addr)) { 1441 ret = -EINVAL; 1442 goto out; 1443 } 1444 1445 /* Check if are we probing a module */ 1446 *probed_mod = __module_text_address((unsigned long) p->addr); 1447 if (*probed_mod) { 1448 /* 1449 * We must hold a refcount of the probed module while updating 1450 * its code to prohibit unexpected unloading. 1451 */ 1452 if (unlikely(!try_module_get(*probed_mod))) { 1453 ret = -ENOENT; 1454 goto out; 1455 } 1456 1457 /* 1458 * If the module freed .init.text, we couldn't insert 1459 * kprobes in there. 1460 */ 1461 if (within_module_init((unsigned long)p->addr, *probed_mod) && 1462 (*probed_mod)->state != MODULE_STATE_COMING) { 1463 module_put(*probed_mod); 1464 *probed_mod = NULL; 1465 ret = -ENOENT; 1466 } 1467 } 1468out: 1469 preempt_enable(); 1470 jump_label_unlock(); 1471 1472 return ret; 1473} 1474 1475int __kprobes register_kprobe(struct kprobe *p) 1476{ 1477 int ret; 1478 struct kprobe *old_p; 1479 struct module *probed_mod; 1480 kprobe_opcode_t *addr; 1481 1482 /* Adjust probe address from symbol */ 1483 addr = kprobe_addr(p); 1484 if (IS_ERR(addr)) 1485 return PTR_ERR(addr); 1486 p->addr = addr; 1487 1488 ret = check_kprobe_rereg(p); 1489 if (ret) 1490 return ret; 1491 1492 /* User can pass only KPROBE_FLAG_DISABLED to register_kprobe */ 1493 p->flags &= KPROBE_FLAG_DISABLED; 1494 p->nmissed = 0; 1495 INIT_LIST_HEAD(&p->list); 1496 1497 ret = check_kprobe_address_safe(p, &probed_mod); 1498 if (ret) 1499 return ret; 1500 1501 mutex_lock(&kprobe_mutex); 1502 1503 old_p = get_kprobe(p->addr); 1504 if (old_p) { 1505 /* Since this may unoptimize old_p, locking text_mutex. */ 1506 ret = register_aggr_kprobe(old_p, p); 1507 goto out; 1508 } 1509 1510 mutex_lock(&text_mutex); /* Avoiding text modification */ 1511 ret = prepare_kprobe(p); 1512 mutex_unlock(&text_mutex); 1513 if (ret) 1514 goto out; 1515 1516 INIT_HLIST_NODE(&p->hlist); 1517 hlist_add_head_rcu(&p->hlist, 1518 &kprobe_table[hash_ptr(p->addr, KPROBE_HASH_BITS)]); 1519 1520 if (!kprobes_all_disarmed && !kprobe_disabled(p)) 1521 arm_kprobe(p); 1522 1523 /* Try to optimize kprobe */ 1524 try_to_optimize_kprobe(p); 1525 1526out: 1527 mutex_unlock(&kprobe_mutex); 1528 1529 if (probed_mod) 1530 module_put(probed_mod); 1531 1532 return ret; 1533} 1534EXPORT_SYMBOL_GPL(register_kprobe); 1535 1536/* Check if all probes on the aggrprobe are disabled */ 1537static int __kprobes aggr_kprobe_disabled(struct kprobe *ap) 1538{ 1539 struct kprobe *kp; 1540 1541 list_for_each_entry_rcu(kp, &ap->list, list) 1542 if (!kprobe_disabled(kp)) 1543 /* 1544 * There is an active probe on the list. 1545 * We can't disable this ap. 1546 */ 1547 return 0; 1548 1549 return 1; 1550} 1551 1552/* Disable one kprobe: Make sure called under kprobe_mutex is locked */ 1553static struct kprobe *__kprobes __disable_kprobe(struct kprobe *p) 1554{ 1555 struct kprobe *orig_p; 1556 1557 /* Get an original kprobe for return */ 1558 orig_p = __get_valid_kprobe(p); 1559 if (unlikely(orig_p == NULL)) 1560 return NULL; 1561 1562 if (!kprobe_disabled(p)) { 1563 /* Disable probe if it is a child probe */ 1564 if (p != orig_p) 1565 p->flags |= KPROBE_FLAG_DISABLED; 1566 1567 /* Try to disarm and disable this/parent probe */ 1568 if (p == orig_p || aggr_kprobe_disabled(orig_p)) { 1569 disarm_kprobe(orig_p, true); 1570 orig_p->flags |= KPROBE_FLAG_DISABLED; 1571 } 1572 } 1573 1574 return orig_p; 1575} 1576 1577/* 1578 * Unregister a kprobe without a scheduler synchronization. 1579 */ 1580static int __kprobes __unregister_kprobe_top(struct kprobe *p) 1581{ 1582 struct kprobe *ap, *list_p; 1583 1584 /* Disable kprobe. This will disarm it if needed. */ 1585 ap = __disable_kprobe(p); 1586 if (ap == NULL) 1587 return -EINVAL; 1588 1589 if (ap == p) 1590 /* 1591 * This probe is an independent(and non-optimized) kprobe 1592 * (not an aggrprobe). Remove from the hash list. 1593 */ 1594 goto disarmed; 1595 1596 /* Following process expects this probe is an aggrprobe */ 1597 WARN_ON(!kprobe_aggrprobe(ap)); 1598 1599 if (list_is_singular(&ap->list) && kprobe_disarmed(ap)) 1600 /* 1601 * !disarmed could be happen if the probe is under delayed 1602 * unoptimizing. 1603 */ 1604 goto disarmed; 1605 else { 1606 /* If disabling probe has special handlers, update aggrprobe */ 1607 if (p->break_handler && !kprobe_gone(p)) 1608 ap->break_handler = NULL; 1609 if (p->post_handler && !kprobe_gone(p)) { 1610 list_for_each_entry_rcu(list_p, &ap->list, list) { 1611 if ((list_p != p) && (list_p->post_handler)) 1612 goto noclean; 1613 } 1614 ap->post_handler = NULL; 1615 } 1616noclean: 1617 /* 1618 * Remove from the aggrprobe: this path will do nothing in 1619 * __unregister_kprobe_bottom(). 1620 */ 1621 list_del_rcu(&p->list); 1622 if (!kprobe_disabled(ap) && !kprobes_all_disarmed) 1623 /* 1624 * Try to optimize this probe again, because post 1625 * handler may have been changed. 1626 */ 1627 optimize_kprobe(ap); 1628 } 1629 return 0; 1630 1631disarmed: 1632 BUG_ON(!kprobe_disarmed(ap)); 1633 hlist_del_rcu(&ap->hlist); 1634 return 0; 1635} 1636 1637static void __kprobes __unregister_kprobe_bottom(struct kprobe *p) 1638{ 1639 struct kprobe *ap; 1640 1641 if (list_empty(&p->list)) 1642 /* This is an independent kprobe */ 1643 arch_remove_kprobe(p); 1644 else if (list_is_singular(&p->list)) { 1645 /* This is the last child of an aggrprobe */ 1646 ap = list_entry(p->list.next, struct kprobe, list); 1647 list_del(&p->list); 1648 free_aggr_kprobe(ap); 1649 } 1650 /* Otherwise, do nothing. */ 1651} 1652 1653int __kprobes register_kprobes(struct kprobe **kps, int num) 1654{ 1655 int i, ret = 0; 1656 1657 if (num <= 0) 1658 return -EINVAL; 1659 for (i = 0; i < num; i++) { 1660 ret = register_kprobe(kps[i]); 1661 if (ret < 0) { 1662 if (i > 0) 1663 unregister_kprobes(kps, i); 1664 break; 1665 } 1666 } 1667 return ret; 1668} 1669EXPORT_SYMBOL_GPL(register_kprobes); 1670 1671void __kprobes unregister_kprobe(struct kprobe *p) 1672{ 1673 unregister_kprobes(&p, 1); 1674} 1675EXPORT_SYMBOL_GPL(unregister_kprobe); 1676 1677void __kprobes unregister_kprobes(struct kprobe **kps, int num) 1678{ 1679 int i; 1680 1681 if (num <= 0) 1682 return; 1683 mutex_lock(&kprobe_mutex); 1684 for (i = 0; i < num; i++) 1685 if (__unregister_kprobe_top(kps[i]) < 0) 1686 kps[i]->addr = NULL; 1687 mutex_unlock(&kprobe_mutex); 1688 1689 synchronize_sched(); 1690 for (i = 0; i < num; i++) 1691 if (kps[i]->addr) 1692 __unregister_kprobe_bottom(kps[i]); 1693} 1694EXPORT_SYMBOL_GPL(unregister_kprobes); 1695 1696static struct notifier_block kprobe_exceptions_nb = { 1697 .notifier_call = kprobe_exceptions_notify, 1698 .priority = 0x7fffffff /* we need to be notified first */ 1699}; 1700 1701unsigned long __weak arch_deref_entry_point(void *entry) 1702{ 1703 return (unsigned long)entry; 1704} 1705 1706int __kprobes register_jprobes(struct jprobe **jps, int num) 1707{ 1708 struct jprobe *jp; 1709 int ret = 0, i; 1710 1711 if (num <= 0) 1712 return -EINVAL; 1713 for (i = 0; i < num; i++) { 1714 unsigned long addr, offset; 1715 jp = jps[i]; 1716 addr = arch_deref_entry_point(jp->entry); 1717 1718 /* Verify probepoint is a function entry point */ 1719 if (kallsyms_lookup_size_offset(addr, NULL, &offset) && 1720 offset == 0) { 1721 jp->kp.pre_handler = setjmp_pre_handler; 1722 jp->kp.break_handler = longjmp_break_handler; 1723 ret = register_kprobe(&jp->kp); 1724 } else 1725 ret = -EINVAL; 1726 1727 if (ret < 0) { 1728 if (i > 0) 1729 unregister_jprobes(jps, i); 1730 break; 1731 } 1732 } 1733 return ret; 1734} 1735EXPORT_SYMBOL_GPL(register_jprobes); 1736 1737int __kprobes register_jprobe(struct jprobe *jp) 1738{ 1739 return register_jprobes(&jp, 1); 1740} 1741EXPORT_SYMBOL_GPL(register_jprobe); 1742 1743void __kprobes unregister_jprobe(struct jprobe *jp) 1744{ 1745 unregister_jprobes(&jp, 1); 1746} 1747EXPORT_SYMBOL_GPL(unregister_jprobe); 1748 1749void __kprobes unregister_jprobes(struct jprobe **jps, int num) 1750{ 1751 int i; 1752 1753 if (num <= 0) 1754 return; 1755 mutex_lock(&kprobe_mutex); 1756 for (i = 0; i < num; i++) 1757 if (__unregister_kprobe_top(&jps[i]->kp) < 0) 1758 jps[i]->kp.addr = NULL; 1759 mutex_unlock(&kprobe_mutex); 1760 1761 synchronize_sched(); 1762 for (i = 0; i < num; i++) { 1763 if (jps[i]->kp.addr) 1764 __unregister_kprobe_bottom(&jps[i]->kp); 1765 } 1766} 1767EXPORT_SYMBOL_GPL(unregister_jprobes); 1768 1769#ifdef CONFIG_KRETPROBES 1770/* 1771 * This kprobe pre_handler is registered with every kretprobe. When probe 1772 * hits it will set up the return probe. 1773 */ 1774static int __kprobes pre_handler_kretprobe(struct kprobe *p, 1775 struct pt_regs *regs) 1776{ 1777 struct kretprobe *rp = container_of(p, struct kretprobe, kp); 1778 unsigned long hash, flags = 0; 1779 struct kretprobe_instance *ri; 1780 1781 /*TODO: consider to only swap the RA after the last pre_handler fired */ 1782 hash = hash_ptr(current, KPROBE_HASH_BITS); 1783 raw_spin_lock_irqsave(&rp->lock, flags); 1784 if (!hlist_empty(&rp->free_instances)) { 1785 ri = hlist_entry(rp->free_instances.first, 1786 struct kretprobe_instance, hlist); 1787 hlist_del(&ri->hlist); 1788 raw_spin_unlock_irqrestore(&rp->lock, flags); 1789 1790 ri->rp = rp; 1791 ri->task = current; 1792 1793 if (rp->entry_handler && rp->entry_handler(ri, regs)) { 1794 raw_spin_lock_irqsave(&rp->lock, flags); 1795 hlist_add_head(&ri->hlist, &rp->free_instances); 1796 raw_spin_unlock_irqrestore(&rp->lock, flags); 1797 return 0; 1798 } 1799 1800 arch_prepare_kretprobe(ri, regs); 1801 1802 /* XXX(hch): why is there no hlist_move_head? */ 1803 INIT_HLIST_NODE(&ri->hlist); 1804 kretprobe_table_lock(hash, &flags); 1805 hlist_add_head(&ri->hlist, &kretprobe_inst_table[hash]); 1806 kretprobe_table_unlock(hash, &flags); 1807 } else { 1808 rp->nmissed++; 1809 raw_spin_unlock_irqrestore(&rp->lock, flags); 1810 } 1811 return 0; 1812} 1813 1814int __kprobes register_kretprobe(struct kretprobe *rp) 1815{ 1816 int ret = 0; 1817 struct kretprobe_instance *inst; 1818 int i; 1819 void *addr; 1820 1821 if (kretprobe_blacklist_size) { 1822 addr = kprobe_addr(&rp->kp); 1823 if (IS_ERR(addr)) 1824 return PTR_ERR(addr); 1825 1826 for (i = 0; kretprobe_blacklist[i].name != NULL; i++) { 1827 if (kretprobe_blacklist[i].addr == addr) 1828 return -EINVAL; 1829 } 1830 } 1831 1832 rp->kp.pre_handler = pre_handler_kretprobe; 1833 rp->kp.post_handler = NULL; 1834 rp->kp.fault_handler = NULL; 1835 rp->kp.break_handler = NULL; 1836 1837 /* Pre-allocate memory for max kretprobe instances */ 1838 if (rp->maxactive <= 0) { 1839#ifdef CONFIG_PREEMPT 1840 rp->maxactive = max_t(unsigned int, 10, 2*num_possible_cpus()); 1841#else 1842 rp->maxactive = num_possible_cpus(); 1843#endif 1844 } 1845 raw_spin_lock_init(&rp->lock); 1846 INIT_HLIST_HEAD(&rp->free_instances); 1847 for (i = 0; i < rp->maxactive; i++) { 1848 inst = kmalloc(sizeof(struct kretprobe_instance) + 1849 rp->data_size, GFP_KERNEL); 1850 if (inst == NULL) { 1851 free_rp_inst(rp); 1852 return -ENOMEM; 1853 } 1854 INIT_HLIST_NODE(&inst->hlist); 1855 hlist_add_head(&inst->hlist, &rp->free_instances); 1856 } 1857 1858 rp->nmissed = 0; 1859 /* Establish function entry probe point */ 1860 ret = register_kprobe(&rp->kp); 1861 if (ret != 0) 1862 free_rp_inst(rp); 1863 return ret; 1864} 1865EXPORT_SYMBOL_GPL(register_kretprobe); 1866 1867int __kprobes register_kretprobes(struct kretprobe **rps, int num) 1868{ 1869 int ret = 0, i; 1870 1871 if (num <= 0) 1872 return -EINVAL; 1873 for (i = 0; i < num; i++) { 1874 ret = register_kretprobe(rps[i]); 1875 if (ret < 0) { 1876 if (i > 0) 1877 unregister_kretprobes(rps, i); 1878 break; 1879 } 1880 } 1881 return ret; 1882} 1883EXPORT_SYMBOL_GPL(register_kretprobes); 1884 1885void __kprobes unregister_kretprobe(struct kretprobe *rp) 1886{ 1887 unregister_kretprobes(&rp, 1); 1888} 1889EXPORT_SYMBOL_GPL(unregister_kretprobe); 1890 1891void __kprobes unregister_kretprobes(struct kretprobe **rps, int num) 1892{ 1893 int i; 1894 1895 if (num <= 0) 1896 return; 1897 mutex_lock(&kprobe_mutex); 1898 for (i = 0; i < num; i++) 1899 if (__unregister_kprobe_top(&rps[i]->kp) < 0) 1900 rps[i]->kp.addr = NULL; 1901 mutex_unlock(&kprobe_mutex); 1902 1903 synchronize_sched(); 1904 for (i = 0; i < num; i++) { 1905 if (rps[i]->kp.addr) { 1906 __unregister_kprobe_bottom(&rps[i]->kp); 1907 cleanup_rp_inst(rps[i]); 1908 } 1909 } 1910} 1911EXPORT_SYMBOL_GPL(unregister_kretprobes); 1912 1913#else /* CONFIG_KRETPROBES */ 1914int __kprobes register_kretprobe(struct kretprobe *rp) 1915{ 1916 return -ENOSYS; 1917} 1918EXPORT_SYMBOL_GPL(register_kretprobe); 1919 1920int __kprobes register_kretprobes(struct kretprobe **rps, int num) 1921{ 1922 return -ENOSYS; 1923} 1924EXPORT_SYMBOL_GPL(register_kretprobes); 1925 1926void __kprobes unregister_kretprobe(struct kretprobe *rp) 1927{ 1928} 1929EXPORT_SYMBOL_GPL(unregister_kretprobe); 1930 1931void __kprobes unregister_kretprobes(struct kretprobe **rps, int num) 1932{ 1933} 1934EXPORT_SYMBOL_GPL(unregister_kretprobes); 1935 1936static int __kprobes pre_handler_kretprobe(struct kprobe *p, 1937 struct pt_regs *regs) 1938{ 1939 return 0; 1940} 1941 1942#endif /* CONFIG_KRETPROBES */ 1943 1944/* Set the kprobe gone and remove its instruction buffer. */ 1945static void __kprobes kill_kprobe(struct kprobe *p) 1946{ 1947 struct kprobe *kp; 1948 1949 p->flags |= KPROBE_FLAG_GONE; 1950 if (kprobe_aggrprobe(p)) { 1951 /* 1952 * If this is an aggr_kprobe, we have to list all the 1953 * chained probes and mark them GONE. 1954 */ 1955 list_for_each_entry_rcu(kp, &p->list, list) 1956 kp->flags |= KPROBE_FLAG_GONE; 1957 p->post_handler = NULL; 1958 p->break_handler = NULL; 1959 kill_optimized_kprobe(p); 1960 } 1961 /* 1962 * Here, we can remove insn_slot safely, because no thread calls 1963 * the original probed function (which will be freed soon) any more. 1964 */ 1965 arch_remove_kprobe(p); 1966} 1967 1968/* Disable one kprobe */ 1969int __kprobes disable_kprobe(struct kprobe *kp) 1970{ 1971 int ret = 0; 1972 1973 mutex_lock(&kprobe_mutex); 1974 1975 /* Disable this kprobe */ 1976 if (__disable_kprobe(kp) == NULL) 1977 ret = -EINVAL; 1978 1979 mutex_unlock(&kprobe_mutex); 1980 return ret; 1981} 1982EXPORT_SYMBOL_GPL(disable_kprobe); 1983 1984/* Enable one kprobe */ 1985int __kprobes enable_kprobe(struct kprobe *kp) 1986{ 1987 int ret = 0; 1988 struct kprobe *p; 1989 1990 mutex_lock(&kprobe_mutex); 1991 1992 /* Check whether specified probe is valid. */ 1993 p = __get_valid_kprobe(kp); 1994 if (unlikely(p == NULL)) { 1995 ret = -EINVAL; 1996 goto out; 1997 } 1998 1999 if (kprobe_gone(kp)) { 2000 /* This kprobe has gone, we couldn't enable it. */ 2001 ret = -EINVAL; 2002 goto out; 2003 } 2004 2005 if (p != kp) 2006 kp->flags &= ~KPROBE_FLAG_DISABLED; 2007 2008 if (!kprobes_all_disarmed && kprobe_disabled(p)) { 2009 p->flags &= ~KPROBE_FLAG_DISABLED; 2010 arm_kprobe(p); 2011 } 2012out: 2013 mutex_unlock(&kprobe_mutex); 2014 return ret; 2015} 2016EXPORT_SYMBOL_GPL(enable_kprobe); 2017 2018void __kprobes dump_kprobe(struct kprobe *kp) 2019{ 2020 printk(KERN_WARNING "Dumping kprobe:\n"); 2021 printk(KERN_WARNING "Name: %s\nAddress: %p\nOffset: %x\n", 2022 kp->symbol_name, kp->addr, kp->offset); 2023} 2024 2025/* Module notifier call back, checking kprobes on the module */ 2026static int __kprobes kprobes_module_callback(struct notifier_block *nb, 2027 unsigned long val, void *data) 2028{ 2029 struct module *mod = data; 2030 struct hlist_head *head; 2031 struct kprobe *p; 2032 unsigned int i; 2033 int checkcore = (val == MODULE_STATE_GOING); 2034 2035 if (val != MODULE_STATE_GOING && val != MODULE_STATE_LIVE) 2036 return NOTIFY_DONE; 2037 2038 /* 2039 * When MODULE_STATE_GOING was notified, both of module .text and 2040 * .init.text sections would be freed. When MODULE_STATE_LIVE was 2041 * notified, only .init.text section would be freed. We need to 2042 * disable kprobes which have been inserted in the sections. 2043 */ 2044 mutex_lock(&kprobe_mutex); 2045 for (i = 0; i < KPROBE_TABLE_SIZE; i++) { 2046 head = &kprobe_table[i]; 2047 hlist_for_each_entry_rcu(p, head, hlist) 2048 if (within_module_init((unsigned long)p->addr, mod) || 2049 (checkcore && 2050 within_module_core((unsigned long)p->addr, mod))) { 2051 /* 2052 * The vaddr this probe is installed will soon 2053 * be vfreed buy not synced to disk. Hence, 2054 * disarming the breakpoint isn't needed. 2055 */ 2056 kill_kprobe(p); 2057 } 2058 } 2059 mutex_unlock(&kprobe_mutex); 2060 return NOTIFY_DONE; 2061} 2062 2063static struct notifier_block kprobe_module_nb = { 2064 .notifier_call = kprobes_module_callback, 2065 .priority = 0 2066}; 2067 2068static int __init init_kprobes(void) 2069{ 2070 int i, err = 0; 2071 unsigned long offset = 0, size = 0; 2072 char *modname, namebuf[KSYM_NAME_LEN]; 2073 const char *symbol_name; 2074 void *addr; 2075 struct kprobe_blackpoint *kb; 2076 2077 /* FIXME allocate the probe table, currently defined statically */ 2078 /* initialize all list heads */ 2079 for (i = 0; i < KPROBE_TABLE_SIZE; i++) { 2080 INIT_HLIST_HEAD(&kprobe_table[i]); 2081 INIT_HLIST_HEAD(&kretprobe_inst_table[i]); 2082 raw_spin_lock_init(&(kretprobe_table_locks[i].lock)); 2083 } 2084 2085 /* 2086 * Lookup and populate the kprobe_blacklist. 2087 * 2088 * Unlike the kretprobe blacklist, we'll need to determine 2089 * the range of addresses that belong to the said functions, 2090 * since a kprobe need not necessarily be at the beginning 2091 * of a function. 2092 */ 2093 for (kb = kprobe_blacklist; kb->name != NULL; kb++) { 2094 kprobe_lookup_name(kb->name, addr); 2095 if (!addr) 2096 continue; 2097 2098 kb->start_addr = (unsigned long)addr; 2099 symbol_name = kallsyms_lookup(kb->start_addr, 2100 &size, &offset, &modname, namebuf); 2101 if (!symbol_name) 2102 kb->range = 0; 2103 else 2104 kb->range = size; 2105 } 2106 2107 if (kretprobe_blacklist_size) { 2108 /* lookup the function address from its name */ 2109 for (i = 0; kretprobe_blacklist[i].name != NULL; i++) { 2110 kprobe_lookup_name(kretprobe_blacklist[i].name, 2111 kretprobe_blacklist[i].addr); 2112 if (!kretprobe_blacklist[i].addr) 2113 printk("kretprobe: lookup failed: %s\n", 2114 kretprobe_blacklist[i].name); 2115 } 2116 } 2117 2118#if defined(CONFIG_OPTPROBES) 2119#if defined(__ARCH_WANT_KPROBES_INSN_SLOT) 2120 /* Init kprobe_optinsn_slots */ 2121 kprobe_optinsn_slots.insn_size = MAX_OPTINSN_SIZE; 2122#endif 2123 /* By default, kprobes can be optimized */ 2124 kprobes_allow_optimization = true; 2125#endif 2126 2127 /* By default, kprobes are armed */ 2128 kprobes_all_disarmed = false; 2129 2130 err = arch_init_kprobes(); 2131 if (!err) 2132 err = register_die_notifier(&kprobe_exceptions_nb); 2133 if (!err) 2134 err = register_module_notifier(&kprobe_module_nb); 2135 2136 kprobes_initialized = (err == 0); 2137 2138 if (!err) 2139 init_test_probes(); 2140 return err; 2141} 2142 2143#ifdef CONFIG_DEBUG_FS 2144static void __kprobes report_probe(struct seq_file *pi, struct kprobe *p, 2145 const char *sym, int offset, char *modname, struct kprobe *pp) 2146{ 2147 char *kprobe_type; 2148 2149 if (p->pre_handler == pre_handler_kretprobe) 2150 kprobe_type = "r"; 2151 else if (p->pre_handler == setjmp_pre_handler) 2152 kprobe_type = "j"; 2153 else 2154 kprobe_type = "k"; 2155 2156 if (sym) 2157 seq_printf(pi, "%p %s %s+0x%x %s ", 2158 p->addr, kprobe_type, sym, offset, 2159 (modname ? modname : " ")); 2160 else 2161 seq_printf(pi, "%p %s %p ", 2162 p->addr, kprobe_type, p->addr); 2163 2164 if (!pp) 2165 pp = p; 2166 seq_printf(pi, "%s%s%s%s\n", 2167 (kprobe_gone(p) ? "[GONE]" : ""), 2168 ((kprobe_disabled(p) && !kprobe_gone(p)) ? "[DISABLED]" : ""), 2169 (kprobe_optimized(pp) ? "[OPTIMIZED]" : ""), 2170 (kprobe_ftrace(pp) ? "[FTRACE]" : "")); 2171} 2172 2173static void __kprobes *kprobe_seq_start(struct seq_file *f, loff_t *pos) 2174{ 2175 return (*pos < KPROBE_TABLE_SIZE) ? pos : NULL; 2176} 2177 2178static void __kprobes *kprobe_seq_next(struct seq_file *f, void *v, loff_t *pos) 2179{ 2180 (*pos)++; 2181 if (*pos >= KPROBE_TABLE_SIZE) 2182 return NULL; 2183 return pos; 2184} 2185 2186static void __kprobes kprobe_seq_stop(struct seq_file *f, void *v) 2187{ 2188 /* Nothing to do */ 2189} 2190 2191static int __kprobes show_kprobe_addr(struct seq_file *pi, void *v) 2192{ 2193 struct hlist_head *head; 2194 struct kprobe *p, *kp; 2195 const char *sym = NULL; 2196 unsigned int i = *(loff_t *) v; 2197 unsigned long offset = 0; 2198 char *modname, namebuf[KSYM_NAME_LEN]; 2199 2200 head = &kprobe_table[i]; 2201 preempt_disable(); 2202 hlist_for_each_entry_rcu(p, head, hlist) { 2203 sym = kallsyms_lookup((unsigned long)p->addr, NULL, 2204 &offset, &modname, namebuf); 2205 if (kprobe_aggrprobe(p)) { 2206 list_for_each_entry_rcu(kp, &p->list, list) 2207 report_probe(pi, kp, sym, offset, modname, p); 2208 } else 2209 report_probe(pi, p, sym, offset, modname, NULL); 2210 } 2211 preempt_enable(); 2212 return 0; 2213} 2214 2215static const struct seq_operations kprobes_seq_ops = { 2216 .start = kprobe_seq_start, 2217 .next = kprobe_seq_next, 2218 .stop = kprobe_seq_stop, 2219 .show = show_kprobe_addr 2220}; 2221 2222static int __kprobes kprobes_open(struct inode *inode, struct file *filp) 2223{ 2224 return seq_open(filp, &kprobes_seq_ops); 2225} 2226 2227static const struct file_operations debugfs_kprobes_operations = { 2228 .open = kprobes_open, 2229 .read = seq_read, 2230 .llseek = seq_lseek, 2231 .release = seq_release, 2232}; 2233 2234static void __kprobes arm_all_kprobes(void) 2235{ 2236 struct hlist_head *head; 2237 struct kprobe *p; 2238 unsigned int i; 2239 2240 mutex_lock(&kprobe_mutex); 2241 2242 /* If kprobes are armed, just return */ 2243 if (!kprobes_all_disarmed) 2244 goto already_enabled; 2245 2246 /* Arming kprobes doesn't optimize kprobe itself */ 2247 for (i = 0; i < KPROBE_TABLE_SIZE; i++) { 2248 head = &kprobe_table[i]; 2249 hlist_for_each_entry_rcu(p, head, hlist) 2250 if (!kprobe_disabled(p)) 2251 arm_kprobe(p); 2252 } 2253 2254 kprobes_all_disarmed = false; 2255 printk(KERN_INFO "Kprobes globally enabled\n"); 2256 2257already_enabled: 2258 mutex_unlock(&kprobe_mutex); 2259 return; 2260} 2261 2262static void __kprobes disarm_all_kprobes(void) 2263{ 2264 struct hlist_head *head; 2265 struct kprobe *p; 2266 unsigned int i; 2267 2268 mutex_lock(&kprobe_mutex); 2269 2270 /* If kprobes are already disarmed, just return */ 2271 if (kprobes_all_disarmed) { 2272 mutex_unlock(&kprobe_mutex); 2273 return; 2274 } 2275 2276 kprobes_all_disarmed = true; 2277 printk(KERN_INFO "Kprobes globally disabled\n"); 2278 2279 for (i = 0; i < KPROBE_TABLE_SIZE; i++) { 2280 head = &kprobe_table[i]; 2281 hlist_for_each_entry_rcu(p, head, hlist) { 2282 if (!arch_trampoline_kprobe(p) && !kprobe_disabled(p)) 2283 disarm_kprobe(p, false); 2284 } 2285 } 2286 mutex_unlock(&kprobe_mutex); 2287 2288 /* Wait for disarming all kprobes by optimizer */ 2289 wait_for_kprobe_optimizer(); 2290} 2291 2292/* 2293 * XXX: The debugfs bool file interface doesn't allow for callbacks 2294 * when the bool state is switched. We can reuse that facility when 2295 * available 2296 */ 2297static ssize_t read_enabled_file_bool(struct file *file, 2298 char __user *user_buf, size_t count, loff_t *ppos) 2299{ 2300 char buf[3]; 2301 2302 if (!kprobes_all_disarmed) 2303 buf[0] = '1'; 2304 else 2305 buf[0] = '0'; 2306 buf[1] = '\n'; 2307 buf[2] = 0x00; 2308 return simple_read_from_buffer(user_buf, count, ppos, buf, 2); 2309} 2310 2311static ssize_t write_enabled_file_bool(struct file *file, 2312 const char __user *user_buf, size_t count, loff_t *ppos) 2313{ 2314 char buf[32]; 2315 size_t buf_size; 2316 2317 buf_size = min(count, (sizeof(buf)-1)); 2318 if (copy_from_user(buf, user_buf, buf_size)) 2319 return -EFAULT; 2320 2321 buf[buf_size] = '\0'; 2322 switch (buf[0]) { 2323 case 'y': 2324 case 'Y': 2325 case '1': 2326 arm_all_kprobes(); 2327 break; 2328 case 'n': 2329 case 'N': 2330 case '0': 2331 disarm_all_kprobes(); 2332 break; 2333 default: 2334 return -EINVAL; 2335 } 2336 2337 return count; 2338} 2339 2340static const struct file_operations fops_kp = { 2341 .read = read_enabled_file_bool, 2342 .write = write_enabled_file_bool, 2343 .llseek = default_llseek, 2344}; 2345 2346static int __kprobes debugfs_kprobe_init(void) 2347{ 2348 struct dentry *dir, *file; 2349 unsigned int value = 1; 2350 2351 dir = debugfs_create_dir("kprobes", NULL); 2352 if (!dir) 2353 return -ENOMEM; 2354 2355 file = debugfs_create_file("list", 0444, dir, NULL, 2356 &debugfs_kprobes_operations); 2357 if (!file) { 2358 debugfs_remove(dir); 2359 return -ENOMEM; 2360 } 2361 2362 file = debugfs_create_file("enabled", 0600, dir, 2363 &value, &fops_kp); 2364 if (!file) { 2365 debugfs_remove(dir); 2366 return -ENOMEM; 2367 } 2368 2369 return 0; 2370} 2371 2372late_initcall(debugfs_kprobe_init); 2373#endif /* CONFIG_DEBUG_FS */ 2374 2375module_init(init_kprobes); 2376 2377/* defined in arch/.../kernel/kprobes.c */ 2378EXPORT_SYMBOL_GPL(jprobe_return); 2379