181d68a96a39844853b37f20cc8282d9b65b78ef3Steven Rostedt/* 273d8b8bc4f24a97a406d09c8268ac019f4ac661eWenji Huang * trace irqs off critical timings 381d68a96a39844853b37f20cc8282d9b65b78ef3Steven Rostedt * 481d68a96a39844853b37f20cc8282d9b65b78ef3Steven Rostedt * Copyright (C) 2007-2008 Steven Rostedt <srostedt@redhat.com> 581d68a96a39844853b37f20cc8282d9b65b78ef3Steven Rostedt * Copyright (C) 2008 Ingo Molnar <mingo@redhat.com> 681d68a96a39844853b37f20cc8282d9b65b78ef3Steven Rostedt * 781d68a96a39844853b37f20cc8282d9b65b78ef3Steven Rostedt * From code in the latency_tracer, that is: 881d68a96a39844853b37f20cc8282d9b65b78ef3Steven Rostedt * 981d68a96a39844853b37f20cc8282d9b65b78ef3Steven Rostedt * Copyright (C) 2004-2006 Ingo Molnar 106d49e352ae9aed3f599041b0c0389aa924815f14Nadia Yvette Chambers * Copyright (C) 2004 Nadia Yvette Chambers 1181d68a96a39844853b37f20cc8282d9b65b78ef3Steven Rostedt */ 1281d68a96a39844853b37f20cc8282d9b65b78ef3Steven Rostedt#include <linux/kallsyms.h> 1381d68a96a39844853b37f20cc8282d9b65b78ef3Steven Rostedt#include <linux/debugfs.h> 1481d68a96a39844853b37f20cc8282d9b65b78ef3Steven Rostedt#include <linux/uaccess.h> 1581d68a96a39844853b37f20cc8282d9b65b78ef3Steven Rostedt#include <linux/module.h> 1681d68a96a39844853b37f20cc8282d9b65b78ef3Steven Rostedt#include <linux/ftrace.h> 1781d68a96a39844853b37f20cc8282d9b65b78ef3Steven Rostedt#include <linux/fs.h> 1881d68a96a39844853b37f20cc8282d9b65b78ef3Steven Rostedt 1981d68a96a39844853b37f20cc8282d9b65b78ef3Steven Rostedt#include "trace.h" 2081d68a96a39844853b37f20cc8282d9b65b78ef3Steven Rostedt 2181d68a96a39844853b37f20cc8282d9b65b78ef3Steven Rostedtstatic struct trace_array *irqsoff_trace __read_mostly; 2281d68a96a39844853b37f20cc8282d9b65b78ef3Steven Rostedtstatic int tracer_enabled __read_mostly; 2381d68a96a39844853b37f20cc8282d9b65b78ef3Steven Rostedt 246cd8a4bb2f97527a9ceb30bc77ea4e959c6a95e3Steven Rostedtstatic DEFINE_PER_CPU(int, tracing_cpu); 256cd8a4bb2f97527a9ceb30bc77ea4e959c6a95e3Steven Rostedt 265389f6fad27019f2ba78f1b332f719ec05f12a42Thomas Gleixnerstatic DEFINE_RAW_SPINLOCK(max_trace_lock); 2789b2f97819dd074297bbe3e19eaa4afcc98845adSteven Rostedt 286cd8a4bb2f97527a9ceb30bc77ea4e959c6a95e3Steven Rostedtenum { 296cd8a4bb2f97527a9ceb30bc77ea4e959c6a95e3Steven Rostedt TRACER_IRQS_OFF = (1 << 1), 306cd8a4bb2f97527a9ceb30bc77ea4e959c6a95e3Steven Rostedt TRACER_PREEMPT_OFF = (1 << 2), 316cd8a4bb2f97527a9ceb30bc77ea4e959c6a95e3Steven Rostedt}; 326cd8a4bb2f97527a9ceb30bc77ea4e959c6a95e3Steven Rostedt 336cd8a4bb2f97527a9ceb30bc77ea4e959c6a95e3Steven Rostedtstatic int trace_type __read_mostly; 346cd8a4bb2f97527a9ceb30bc77ea4e959c6a95e3Steven Rostedt 35613f04a0f51e6e68ac6fe571ab79da3c0a5eb4daSteven Rostedt (Red Hat)static int save_flags; 36328df4759c03e2c3e7429cc6cb0e180c38f32063Steven Rostedt (Red Hat)static bool function_enabled; 37e9d25fe6eaa2c720bb3ea661b660e58d54fa38bfSteven Rostedt 3862b915f1060996a8e1f69be50e3b8e9e43b710cbJiri Olsastatic void stop_irqsoff_tracer(struct trace_array *tr, int graph); 3962b915f1060996a8e1f69be50e3b8e9e43b710cbJiri Olsastatic int start_irqsoff_tracer(struct trace_array *tr, int graph); 4062b915f1060996a8e1f69be50e3b8e9e43b710cbJiri Olsa 416cd8a4bb2f97527a9ceb30bc77ea4e959c6a95e3Steven Rostedt#ifdef CONFIG_PREEMPT_TRACER 42e309b41dd65aa953f86765eeeecc941d8e1e8b8fIngo Molnarstatic inline int 436cd8a4bb2f97527a9ceb30bc77ea4e959c6a95e3Steven Rostedtpreempt_trace(void) 446cd8a4bb2f97527a9ceb30bc77ea4e959c6a95e3Steven Rostedt{ 456cd8a4bb2f97527a9ceb30bc77ea4e959c6a95e3Steven Rostedt return ((trace_type & TRACER_PREEMPT_OFF) && preempt_count()); 466cd8a4bb2f97527a9ceb30bc77ea4e959c6a95e3Steven Rostedt} 476cd8a4bb2f97527a9ceb30bc77ea4e959c6a95e3Steven Rostedt#else 486cd8a4bb2f97527a9ceb30bc77ea4e959c6a95e3Steven Rostedt# define preempt_trace() (0) 496cd8a4bb2f97527a9ceb30bc77ea4e959c6a95e3Steven Rostedt#endif 506cd8a4bb2f97527a9ceb30bc77ea4e959c6a95e3Steven Rostedt 516cd8a4bb2f97527a9ceb30bc77ea4e959c6a95e3Steven Rostedt#ifdef CONFIG_IRQSOFF_TRACER 52e309b41dd65aa953f86765eeeecc941d8e1e8b8fIngo Molnarstatic inline int 536cd8a4bb2f97527a9ceb30bc77ea4e959c6a95e3Steven Rostedtirq_trace(void) 546cd8a4bb2f97527a9ceb30bc77ea4e959c6a95e3Steven Rostedt{ 556cd8a4bb2f97527a9ceb30bc77ea4e959c6a95e3Steven Rostedt return ((trace_type & TRACER_IRQS_OFF) && 566cd8a4bb2f97527a9ceb30bc77ea4e959c6a95e3Steven Rostedt irqs_disabled()); 576cd8a4bb2f97527a9ceb30bc77ea4e959c6a95e3Steven Rostedt} 586cd8a4bb2f97527a9ceb30bc77ea4e959c6a95e3Steven Rostedt#else 596cd8a4bb2f97527a9ceb30bc77ea4e959c6a95e3Steven Rostedt# define irq_trace() (0) 606cd8a4bb2f97527a9ceb30bc77ea4e959c6a95e3Steven Rostedt#endif 616cd8a4bb2f97527a9ceb30bc77ea4e959c6a95e3Steven Rostedt 6262b915f1060996a8e1f69be50e3b8e9e43b710cbJiri Olsa#define TRACE_DISPLAY_GRAPH 1 6362b915f1060996a8e1f69be50e3b8e9e43b710cbJiri Olsa 6462b915f1060996a8e1f69be50e3b8e9e43b710cbJiri Olsastatic struct tracer_opt trace_opts[] = { 6562b915f1060996a8e1f69be50e3b8e9e43b710cbJiri Olsa#ifdef CONFIG_FUNCTION_GRAPH_TRACER 6662b915f1060996a8e1f69be50e3b8e9e43b710cbJiri Olsa /* display latency trace as call graph */ 6762b915f1060996a8e1f69be50e3b8e9e43b710cbJiri Olsa { TRACER_OPT(display-graph, TRACE_DISPLAY_GRAPH) }, 6862b915f1060996a8e1f69be50e3b8e9e43b710cbJiri Olsa#endif 6962b915f1060996a8e1f69be50e3b8e9e43b710cbJiri Olsa { } /* Empty entry */ 7062b915f1060996a8e1f69be50e3b8e9e43b710cbJiri Olsa}; 7162b915f1060996a8e1f69be50e3b8e9e43b710cbJiri Olsa 7262b915f1060996a8e1f69be50e3b8e9e43b710cbJiri Olsastatic struct tracer_flags tracer_flags = { 7362b915f1060996a8e1f69be50e3b8e9e43b710cbJiri Olsa .val = 0, 7462b915f1060996a8e1f69be50e3b8e9e43b710cbJiri Olsa .opts = trace_opts, 7562b915f1060996a8e1f69be50e3b8e9e43b710cbJiri Olsa}; 7662b915f1060996a8e1f69be50e3b8e9e43b710cbJiri Olsa 7762b915f1060996a8e1f69be50e3b8e9e43b710cbJiri Olsa#define is_graph() (tracer_flags.val & TRACE_DISPLAY_GRAPH) 7862b915f1060996a8e1f69be50e3b8e9e43b710cbJiri Olsa 7981d68a96a39844853b37f20cc8282d9b65b78ef3Steven Rostedt/* 8081d68a96a39844853b37f20cc8282d9b65b78ef3Steven Rostedt * Sequence count - we record it when starting a measurement and 8181d68a96a39844853b37f20cc8282d9b65b78ef3Steven Rostedt * skip the latency if the sequence has changed - some other section 8281d68a96a39844853b37f20cc8282d9b65b78ef3Steven Rostedt * did a maximum and could disturb our measurement with serial console 8381d68a96a39844853b37f20cc8282d9b65b78ef3Steven Rostedt * printouts, etc. Truly coinciding maximum latencies should be rare 8425985edcedea6396277003854657b5f3cb31a628Lucas De Marchi * and what happens together happens separately as well, so this doesn't 8581d68a96a39844853b37f20cc8282d9b65b78ef3Steven Rostedt * decrease the validity of the maximum found: 8681d68a96a39844853b37f20cc8282d9b65b78ef3Steven Rostedt */ 8781d68a96a39844853b37f20cc8282d9b65b78ef3Steven Rostedtstatic __cacheline_aligned_in_smp unsigned long max_sequence; 8881d68a96a39844853b37f20cc8282d9b65b78ef3Steven Rostedt 89606576ce816603d9fe1fb453a88bc6eea16ca709Steven Rostedt#ifdef CONFIG_FUNCTION_TRACER 9081d68a96a39844853b37f20cc8282d9b65b78ef3Steven Rostedt/* 915e6d2b9cfa3a6e7fe62fc0135bc1bd778f5db564Steven Rostedt * Prologue for the preempt and irqs off function tracers. 925e6d2b9cfa3a6e7fe62fc0135bc1bd778f5db564Steven Rostedt * 935e6d2b9cfa3a6e7fe62fc0135bc1bd778f5db564Steven Rostedt * Returns 1 if it is OK to continue, and data->disabled is 945e6d2b9cfa3a6e7fe62fc0135bc1bd778f5db564Steven Rostedt * incremented. 955e6d2b9cfa3a6e7fe62fc0135bc1bd778f5db564Steven Rostedt * 0 if the trace is to be ignored, and data->disabled 965e6d2b9cfa3a6e7fe62fc0135bc1bd778f5db564Steven Rostedt * is kept the same. 975e6d2b9cfa3a6e7fe62fc0135bc1bd778f5db564Steven Rostedt * 985e6d2b9cfa3a6e7fe62fc0135bc1bd778f5db564Steven Rostedt * Note, this function is also used outside this ifdef but 995e6d2b9cfa3a6e7fe62fc0135bc1bd778f5db564Steven Rostedt * inside the #ifdef of the function graph tracer below. 1005e6d2b9cfa3a6e7fe62fc0135bc1bd778f5db564Steven Rostedt * This is OK, since the function graph tracer is 1015e6d2b9cfa3a6e7fe62fc0135bc1bd778f5db564Steven Rostedt * dependent on the function tracer. 10281d68a96a39844853b37f20cc8282d9b65b78ef3Steven Rostedt */ 1035e6d2b9cfa3a6e7fe62fc0135bc1bd778f5db564Steven Rostedtstatic int func_prolog_dec(struct trace_array *tr, 1045e6d2b9cfa3a6e7fe62fc0135bc1bd778f5db564Steven Rostedt struct trace_array_cpu **data, 1055e6d2b9cfa3a6e7fe62fc0135bc1bd778f5db564Steven Rostedt unsigned long *flags) 10681d68a96a39844853b37f20cc8282d9b65b78ef3Steven Rostedt{ 10781d68a96a39844853b37f20cc8282d9b65b78ef3Steven Rostedt long disabled; 10881d68a96a39844853b37f20cc8282d9b65b78ef3Steven Rostedt int cpu; 10981d68a96a39844853b37f20cc8282d9b65b78ef3Steven Rostedt 110361943ad0ba3f16e66859e30a408915e008ba91eSteven Rostedt /* 111361943ad0ba3f16e66859e30a408915e008ba91eSteven Rostedt * Does not matter if we preempt. We test the flags 112361943ad0ba3f16e66859e30a408915e008ba91eSteven Rostedt * afterward, to see if irqs are disabled or not. 113361943ad0ba3f16e66859e30a408915e008ba91eSteven Rostedt * If we preempt and get a false positive, the flags 114361943ad0ba3f16e66859e30a408915e008ba91eSteven Rostedt * test will fail. 115361943ad0ba3f16e66859e30a408915e008ba91eSteven Rostedt */ 116361943ad0ba3f16e66859e30a408915e008ba91eSteven Rostedt cpu = raw_smp_processor_id(); 117361943ad0ba3f16e66859e30a408915e008ba91eSteven Rostedt if (likely(!per_cpu(tracing_cpu, cpu))) 1185e6d2b9cfa3a6e7fe62fc0135bc1bd778f5db564Steven Rostedt return 0; 11981d68a96a39844853b37f20cc8282d9b65b78ef3Steven Rostedt 1205e6d2b9cfa3a6e7fe62fc0135bc1bd778f5db564Steven Rostedt local_save_flags(*flags); 121361943ad0ba3f16e66859e30a408915e008ba91eSteven Rostedt /* slight chance to get a false positive on tracing_cpu */ 1225e6d2b9cfa3a6e7fe62fc0135bc1bd778f5db564Steven Rostedt if (!irqs_disabled_flags(*flags)) 1235e6d2b9cfa3a6e7fe62fc0135bc1bd778f5db564Steven Rostedt return 0; 12481d68a96a39844853b37f20cc8282d9b65b78ef3Steven Rostedt 12512883efb670c28dff57dcd7f4f995a1ffe153b2dSteven Rostedt (Red Hat) *data = per_cpu_ptr(tr->trace_buffer.data, cpu); 1265e6d2b9cfa3a6e7fe62fc0135bc1bd778f5db564Steven Rostedt disabled = atomic_inc_return(&(*data)->disabled); 12781d68a96a39844853b37f20cc8282d9b65b78ef3Steven Rostedt 12881d68a96a39844853b37f20cc8282d9b65b78ef3Steven Rostedt if (likely(disabled == 1)) 1295e6d2b9cfa3a6e7fe62fc0135bc1bd778f5db564Steven Rostedt return 1; 1305e6d2b9cfa3a6e7fe62fc0135bc1bd778f5db564Steven Rostedt 1315e6d2b9cfa3a6e7fe62fc0135bc1bd778f5db564Steven Rostedt atomic_dec(&(*data)->disabled); 1325e6d2b9cfa3a6e7fe62fc0135bc1bd778f5db564Steven Rostedt 1335e6d2b9cfa3a6e7fe62fc0135bc1bd778f5db564Steven Rostedt return 0; 1345e6d2b9cfa3a6e7fe62fc0135bc1bd778f5db564Steven Rostedt} 1355e6d2b9cfa3a6e7fe62fc0135bc1bd778f5db564Steven Rostedt 1365e6d2b9cfa3a6e7fe62fc0135bc1bd778f5db564Steven Rostedt/* 1375e6d2b9cfa3a6e7fe62fc0135bc1bd778f5db564Steven Rostedt * irqsoff uses its own tracer function to keep the overhead down: 1385e6d2b9cfa3a6e7fe62fc0135bc1bd778f5db564Steven Rostedt */ 1395e6d2b9cfa3a6e7fe62fc0135bc1bd778f5db564Steven Rostedtstatic void 1402f5f6ad9390c1ebbf738d130dbfe80b60eaa167eSteven Rostedtirqsoff_tracer_call(unsigned long ip, unsigned long parent_ip, 141a1e2e31d175a1349274eba3465d17616c6725f8cSteven Rostedt struct ftrace_ops *op, struct pt_regs *pt_regs) 1425e6d2b9cfa3a6e7fe62fc0135bc1bd778f5db564Steven Rostedt{ 1435e6d2b9cfa3a6e7fe62fc0135bc1bd778f5db564Steven Rostedt struct trace_array *tr = irqsoff_trace; 1445e6d2b9cfa3a6e7fe62fc0135bc1bd778f5db564Steven Rostedt struct trace_array_cpu *data; 1455e6d2b9cfa3a6e7fe62fc0135bc1bd778f5db564Steven Rostedt unsigned long flags; 1465e6d2b9cfa3a6e7fe62fc0135bc1bd778f5db564Steven Rostedt 1475e6d2b9cfa3a6e7fe62fc0135bc1bd778f5db564Steven Rostedt if (!func_prolog_dec(tr, &data, &flags)) 1485e6d2b9cfa3a6e7fe62fc0135bc1bd778f5db564Steven Rostedt return; 1495e6d2b9cfa3a6e7fe62fc0135bc1bd778f5db564Steven Rostedt 1505e6d2b9cfa3a6e7fe62fc0135bc1bd778f5db564Steven Rostedt trace_function(tr, ip, parent_ip, flags, preempt_count()); 15181d68a96a39844853b37f20cc8282d9b65b78ef3Steven Rostedt 15281d68a96a39844853b37f20cc8282d9b65b78ef3Steven Rostedt atomic_dec(&data->disabled); 15381d68a96a39844853b37f20cc8282d9b65b78ef3Steven Rostedt} 154606576ce816603d9fe1fb453a88bc6eea16ca709Steven Rostedt#endif /* CONFIG_FUNCTION_TRACER */ 15581d68a96a39844853b37f20cc8282d9b65b78ef3Steven Rostedt 15662b915f1060996a8e1f69be50e3b8e9e43b710cbJiri Olsa#ifdef CONFIG_FUNCTION_GRAPH_TRACER 1578c1a49aedb73fb2f15aaa32ad9e2e1c4289f45cbSteven Rostedt (Red Hat)static int 1588c1a49aedb73fb2f15aaa32ad9e2e1c4289f45cbSteven Rostedt (Red Hat)irqsoff_set_flag(struct trace_array *tr, u32 old_flags, u32 bit, int set) 15962b915f1060996a8e1f69be50e3b8e9e43b710cbJiri Olsa{ 16062b915f1060996a8e1f69be50e3b8e9e43b710cbJiri Olsa int cpu; 16162b915f1060996a8e1f69be50e3b8e9e43b710cbJiri Olsa 16262b915f1060996a8e1f69be50e3b8e9e43b710cbJiri Olsa if (!(bit & TRACE_DISPLAY_GRAPH)) 16362b915f1060996a8e1f69be50e3b8e9e43b710cbJiri Olsa return -EINVAL; 16462b915f1060996a8e1f69be50e3b8e9e43b710cbJiri Olsa 16562b915f1060996a8e1f69be50e3b8e9e43b710cbJiri Olsa if (!(is_graph() ^ set)) 16662b915f1060996a8e1f69be50e3b8e9e43b710cbJiri Olsa return 0; 16762b915f1060996a8e1f69be50e3b8e9e43b710cbJiri Olsa 16862b915f1060996a8e1f69be50e3b8e9e43b710cbJiri Olsa stop_irqsoff_tracer(irqsoff_trace, !set); 16962b915f1060996a8e1f69be50e3b8e9e43b710cbJiri Olsa 17062b915f1060996a8e1f69be50e3b8e9e43b710cbJiri Olsa for_each_possible_cpu(cpu) 17162b915f1060996a8e1f69be50e3b8e9e43b710cbJiri Olsa per_cpu(tracing_cpu, cpu) = 0; 17262b915f1060996a8e1f69be50e3b8e9e43b710cbJiri Olsa 1736d9b3fa5e7f663bbfb9d2d80d46136f75319cb28Steven Rostedt (Red Hat) tr->max_latency = 0; 17412883efb670c28dff57dcd7f4f995a1ffe153b2dSteven Rostedt (Red Hat) tracing_reset_online_cpus(&irqsoff_trace->trace_buffer); 17562b915f1060996a8e1f69be50e3b8e9e43b710cbJiri Olsa 17662b915f1060996a8e1f69be50e3b8e9e43b710cbJiri Olsa return start_irqsoff_tracer(irqsoff_trace, set); 17762b915f1060996a8e1f69be50e3b8e9e43b710cbJiri Olsa} 17862b915f1060996a8e1f69be50e3b8e9e43b710cbJiri Olsa 17962b915f1060996a8e1f69be50e3b8e9e43b710cbJiri Olsastatic int irqsoff_graph_entry(struct ftrace_graph_ent *trace) 18062b915f1060996a8e1f69be50e3b8e9e43b710cbJiri Olsa{ 18162b915f1060996a8e1f69be50e3b8e9e43b710cbJiri Olsa struct trace_array *tr = irqsoff_trace; 18262b915f1060996a8e1f69be50e3b8e9e43b710cbJiri Olsa struct trace_array_cpu *data; 18362b915f1060996a8e1f69be50e3b8e9e43b710cbJiri Olsa unsigned long flags; 18462b915f1060996a8e1f69be50e3b8e9e43b710cbJiri Olsa int ret; 18562b915f1060996a8e1f69be50e3b8e9e43b710cbJiri Olsa int pc; 18662b915f1060996a8e1f69be50e3b8e9e43b710cbJiri Olsa 1875e6d2b9cfa3a6e7fe62fc0135bc1bd778f5db564Steven Rostedt if (!func_prolog_dec(tr, &data, &flags)) 18862b915f1060996a8e1f69be50e3b8e9e43b710cbJiri Olsa return 0; 18962b915f1060996a8e1f69be50e3b8e9e43b710cbJiri Olsa 1905e6d2b9cfa3a6e7fe62fc0135bc1bd778f5db564Steven Rostedt pc = preempt_count(); 1915e6d2b9cfa3a6e7fe62fc0135bc1bd778f5db564Steven Rostedt ret = __trace_graph_entry(tr, trace, flags, pc); 19262b915f1060996a8e1f69be50e3b8e9e43b710cbJiri Olsa atomic_dec(&data->disabled); 1935e6d2b9cfa3a6e7fe62fc0135bc1bd778f5db564Steven Rostedt 19462b915f1060996a8e1f69be50e3b8e9e43b710cbJiri Olsa return ret; 19562b915f1060996a8e1f69be50e3b8e9e43b710cbJiri Olsa} 19662b915f1060996a8e1f69be50e3b8e9e43b710cbJiri Olsa 19762b915f1060996a8e1f69be50e3b8e9e43b710cbJiri Olsastatic void irqsoff_graph_return(struct ftrace_graph_ret *trace) 19862b915f1060996a8e1f69be50e3b8e9e43b710cbJiri Olsa{ 19962b915f1060996a8e1f69be50e3b8e9e43b710cbJiri Olsa struct trace_array *tr = irqsoff_trace; 20062b915f1060996a8e1f69be50e3b8e9e43b710cbJiri Olsa struct trace_array_cpu *data; 20162b915f1060996a8e1f69be50e3b8e9e43b710cbJiri Olsa unsigned long flags; 20262b915f1060996a8e1f69be50e3b8e9e43b710cbJiri Olsa int pc; 20362b915f1060996a8e1f69be50e3b8e9e43b710cbJiri Olsa 2045e6d2b9cfa3a6e7fe62fc0135bc1bd778f5db564Steven Rostedt if (!func_prolog_dec(tr, &data, &flags)) 20562b915f1060996a8e1f69be50e3b8e9e43b710cbJiri Olsa return; 20662b915f1060996a8e1f69be50e3b8e9e43b710cbJiri Olsa 2075e6d2b9cfa3a6e7fe62fc0135bc1bd778f5db564Steven Rostedt pc = preempt_count(); 2085e6d2b9cfa3a6e7fe62fc0135bc1bd778f5db564Steven Rostedt __trace_graph_return(tr, trace, flags, pc); 20962b915f1060996a8e1f69be50e3b8e9e43b710cbJiri Olsa atomic_dec(&data->disabled); 21062b915f1060996a8e1f69be50e3b8e9e43b710cbJiri Olsa} 21162b915f1060996a8e1f69be50e3b8e9e43b710cbJiri Olsa 21262b915f1060996a8e1f69be50e3b8e9e43b710cbJiri Olsastatic void irqsoff_trace_open(struct trace_iterator *iter) 21362b915f1060996a8e1f69be50e3b8e9e43b710cbJiri Olsa{ 21462b915f1060996a8e1f69be50e3b8e9e43b710cbJiri Olsa if (is_graph()) 21562b915f1060996a8e1f69be50e3b8e9e43b710cbJiri Olsa graph_trace_open(iter); 21662b915f1060996a8e1f69be50e3b8e9e43b710cbJiri Olsa 21762b915f1060996a8e1f69be50e3b8e9e43b710cbJiri Olsa} 21862b915f1060996a8e1f69be50e3b8e9e43b710cbJiri Olsa 21962b915f1060996a8e1f69be50e3b8e9e43b710cbJiri Olsastatic void irqsoff_trace_close(struct trace_iterator *iter) 22062b915f1060996a8e1f69be50e3b8e9e43b710cbJiri Olsa{ 22162b915f1060996a8e1f69be50e3b8e9e43b710cbJiri Olsa if (iter->private) 22262b915f1060996a8e1f69be50e3b8e9e43b710cbJiri Olsa graph_trace_close(iter); 22362b915f1060996a8e1f69be50e3b8e9e43b710cbJiri Olsa} 22462b915f1060996a8e1f69be50e3b8e9e43b710cbJiri Olsa 22562b915f1060996a8e1f69be50e3b8e9e43b710cbJiri Olsa#define GRAPH_TRACER_FLAGS (TRACE_GRAPH_PRINT_CPU | \ 226321e68b095addc473ca52ced9acfa59be88f76e6Jiri Olsa TRACE_GRAPH_PRINT_PROC | \ 227321e68b095addc473ca52ced9acfa59be88f76e6Jiri Olsa TRACE_GRAPH_PRINT_ABS_TIME | \ 228321e68b095addc473ca52ced9acfa59be88f76e6Jiri Olsa TRACE_GRAPH_PRINT_DURATION) 22962b915f1060996a8e1f69be50e3b8e9e43b710cbJiri Olsa 23062b915f1060996a8e1f69be50e3b8e9e43b710cbJiri Olsastatic enum print_line_t irqsoff_print_line(struct trace_iterator *iter) 23162b915f1060996a8e1f69be50e3b8e9e43b710cbJiri Olsa{ 23262b915f1060996a8e1f69be50e3b8e9e43b710cbJiri Olsa /* 23362b915f1060996a8e1f69be50e3b8e9e43b710cbJiri Olsa * In graph mode call the graph tracer output function, 23462b915f1060996a8e1f69be50e3b8e9e43b710cbJiri Olsa * otherwise go with the TRACE_FN event handler 23562b915f1060996a8e1f69be50e3b8e9e43b710cbJiri Olsa */ 23662b915f1060996a8e1f69be50e3b8e9e43b710cbJiri Olsa if (is_graph()) 2370a772620a2e21fb55a02f70fe38d4b5c3a5fbbbfJiri Olsa return print_graph_function_flags(iter, GRAPH_TRACER_FLAGS); 23862b915f1060996a8e1f69be50e3b8e9e43b710cbJiri Olsa 23962b915f1060996a8e1f69be50e3b8e9e43b710cbJiri Olsa return TRACE_TYPE_UNHANDLED; 24062b915f1060996a8e1f69be50e3b8e9e43b710cbJiri Olsa} 24162b915f1060996a8e1f69be50e3b8e9e43b710cbJiri Olsa 24262b915f1060996a8e1f69be50e3b8e9e43b710cbJiri Olsastatic void irqsoff_print_header(struct seq_file *s) 24362b915f1060996a8e1f69be50e3b8e9e43b710cbJiri Olsa{ 2440a772620a2e21fb55a02f70fe38d4b5c3a5fbbbfJiri Olsa if (is_graph()) 2450a772620a2e21fb55a02f70fe38d4b5c3a5fbbbfJiri Olsa print_graph_headers_flags(s, GRAPH_TRACER_FLAGS); 2460a772620a2e21fb55a02f70fe38d4b5c3a5fbbbfJiri Olsa else 24762b915f1060996a8e1f69be50e3b8e9e43b710cbJiri Olsa trace_default_header(s); 24862b915f1060996a8e1f69be50e3b8e9e43b710cbJiri Olsa} 24962b915f1060996a8e1f69be50e3b8e9e43b710cbJiri Olsa 25062b915f1060996a8e1f69be50e3b8e9e43b710cbJiri Olsastatic void 25162b915f1060996a8e1f69be50e3b8e9e43b710cbJiri Olsa__trace_function(struct trace_array *tr, 25262b915f1060996a8e1f69be50e3b8e9e43b710cbJiri Olsa unsigned long ip, unsigned long parent_ip, 25362b915f1060996a8e1f69be50e3b8e9e43b710cbJiri Olsa unsigned long flags, int pc) 25462b915f1060996a8e1f69be50e3b8e9e43b710cbJiri Olsa{ 2550a772620a2e21fb55a02f70fe38d4b5c3a5fbbbfJiri Olsa if (is_graph()) 2560a772620a2e21fb55a02f70fe38d4b5c3a5fbbbfJiri Olsa trace_graph_function(tr, ip, parent_ip, flags, pc); 2570a772620a2e21fb55a02f70fe38d4b5c3a5fbbbfJiri Olsa else 25862b915f1060996a8e1f69be50e3b8e9e43b710cbJiri Olsa trace_function(tr, ip, parent_ip, flags, pc); 25962b915f1060996a8e1f69be50e3b8e9e43b710cbJiri Olsa} 26062b915f1060996a8e1f69be50e3b8e9e43b710cbJiri Olsa 26162b915f1060996a8e1f69be50e3b8e9e43b710cbJiri Olsa#else 26262b915f1060996a8e1f69be50e3b8e9e43b710cbJiri Olsa#define __trace_function trace_function 26362b915f1060996a8e1f69be50e3b8e9e43b710cbJiri Olsa 2648c1a49aedb73fb2f15aaa32ad9e2e1c4289f45cbSteven Rostedt (Red Hat)static int 2658c1a49aedb73fb2f15aaa32ad9e2e1c4289f45cbSteven Rostedt (Red Hat)irqsoff_set_flag(struct trace_array *tr, u32 old_flags, u32 bit, int set) 26662b915f1060996a8e1f69be50e3b8e9e43b710cbJiri Olsa{ 26762b915f1060996a8e1f69be50e3b8e9e43b710cbJiri Olsa return -EINVAL; 26862b915f1060996a8e1f69be50e3b8e9e43b710cbJiri Olsa} 26962b915f1060996a8e1f69be50e3b8e9e43b710cbJiri Olsa 27062b915f1060996a8e1f69be50e3b8e9e43b710cbJiri Olsastatic int irqsoff_graph_entry(struct ftrace_graph_ent *trace) 27162b915f1060996a8e1f69be50e3b8e9e43b710cbJiri Olsa{ 27262b915f1060996a8e1f69be50e3b8e9e43b710cbJiri Olsa return -1; 27362b915f1060996a8e1f69be50e3b8e9e43b710cbJiri Olsa} 27462b915f1060996a8e1f69be50e3b8e9e43b710cbJiri Olsa 27562b915f1060996a8e1f69be50e3b8e9e43b710cbJiri Olsastatic enum print_line_t irqsoff_print_line(struct trace_iterator *iter) 27662b915f1060996a8e1f69be50e3b8e9e43b710cbJiri Olsa{ 27762b915f1060996a8e1f69be50e3b8e9e43b710cbJiri Olsa return TRACE_TYPE_UNHANDLED; 27862b915f1060996a8e1f69be50e3b8e9e43b710cbJiri Olsa} 27962b915f1060996a8e1f69be50e3b8e9e43b710cbJiri Olsa 28062b915f1060996a8e1f69be50e3b8e9e43b710cbJiri Olsastatic void irqsoff_graph_return(struct ftrace_graph_ret *trace) { } 28162b915f1060996a8e1f69be50e3b8e9e43b710cbJiri Olsastatic void irqsoff_trace_open(struct trace_iterator *iter) { } 28262b915f1060996a8e1f69be50e3b8e9e43b710cbJiri Olsastatic void irqsoff_trace_close(struct trace_iterator *iter) { } 2837e9a49ef542610609144d1afcd516dc3fafac4d6Jiri Olsa 2847e9a49ef542610609144d1afcd516dc3fafac4d6Jiri Olsa#ifdef CONFIG_FUNCTION_TRACER 2857e9a49ef542610609144d1afcd516dc3fafac4d6Jiri Olsastatic void irqsoff_print_header(struct seq_file *s) 2867e9a49ef542610609144d1afcd516dc3fafac4d6Jiri Olsa{ 2877e9a49ef542610609144d1afcd516dc3fafac4d6Jiri Olsa trace_default_header(s); 2887e9a49ef542610609144d1afcd516dc3fafac4d6Jiri Olsa} 2897e9a49ef542610609144d1afcd516dc3fafac4d6Jiri Olsa#else 2907e9a49ef542610609144d1afcd516dc3fafac4d6Jiri Olsastatic void irqsoff_print_header(struct seq_file *s) 2917e9a49ef542610609144d1afcd516dc3fafac4d6Jiri Olsa{ 2927e9a49ef542610609144d1afcd516dc3fafac4d6Jiri Olsa trace_latency_header(s); 2937e9a49ef542610609144d1afcd516dc3fafac4d6Jiri Olsa} 2947e9a49ef542610609144d1afcd516dc3fafac4d6Jiri Olsa#endif /* CONFIG_FUNCTION_TRACER */ 29562b915f1060996a8e1f69be50e3b8e9e43b710cbJiri Olsa#endif /* CONFIG_FUNCTION_GRAPH_TRACER */ 29662b915f1060996a8e1f69be50e3b8e9e43b710cbJiri Olsa 29781d68a96a39844853b37f20cc8282d9b65b78ef3Steven Rostedt/* 29881d68a96a39844853b37f20cc8282d9b65b78ef3Steven Rostedt * Should this new latency be reported/recorded? 29981d68a96a39844853b37f20cc8282d9b65b78ef3Steven Rostedt */ 3006d9b3fa5e7f663bbfb9d2d80d46136f75319cb28Steven Rostedt (Red Hat)static int report_latency(struct trace_array *tr, cycle_t delta) 30181d68a96a39844853b37f20cc8282d9b65b78ef3Steven Rostedt{ 30281d68a96a39844853b37f20cc8282d9b65b78ef3Steven Rostedt if (tracing_thresh) { 30381d68a96a39844853b37f20cc8282d9b65b78ef3Steven Rostedt if (delta < tracing_thresh) 30481d68a96a39844853b37f20cc8282d9b65b78ef3Steven Rostedt return 0; 30581d68a96a39844853b37f20cc8282d9b65b78ef3Steven Rostedt } else { 3066d9b3fa5e7f663bbfb9d2d80d46136f75319cb28Steven Rostedt (Red Hat) if (delta <= tr->max_latency) 30781d68a96a39844853b37f20cc8282d9b65b78ef3Steven Rostedt return 0; 30881d68a96a39844853b37f20cc8282d9b65b78ef3Steven Rostedt } 30981d68a96a39844853b37f20cc8282d9b65b78ef3Steven Rostedt return 1; 31081d68a96a39844853b37f20cc8282d9b65b78ef3Steven Rostedt} 31181d68a96a39844853b37f20cc8282d9b65b78ef3Steven Rostedt 312e309b41dd65aa953f86765eeeecc941d8e1e8b8fIngo Molnarstatic void 31381d68a96a39844853b37f20cc8282d9b65b78ef3Steven Rostedtcheck_critical_timing(struct trace_array *tr, 31481d68a96a39844853b37f20cc8282d9b65b78ef3Steven Rostedt struct trace_array_cpu *data, 31581d68a96a39844853b37f20cc8282d9b65b78ef3Steven Rostedt unsigned long parent_ip, 31681d68a96a39844853b37f20cc8282d9b65b78ef3Steven Rostedt int cpu) 31781d68a96a39844853b37f20cc8282d9b65b78ef3Steven Rostedt{ 31889b2f97819dd074297bbe3e19eaa4afcc98845adSteven Rostedt cycle_t T0, T1, delta; 31981d68a96a39844853b37f20cc8282d9b65b78ef3Steven Rostedt unsigned long flags; 32038697053fa006411224a1790e2adb8216440ab0fSteven Rostedt int pc; 32181d68a96a39844853b37f20cc8282d9b65b78ef3Steven Rostedt 32281d68a96a39844853b37f20cc8282d9b65b78ef3Steven Rostedt T0 = data->preempt_timestamp; 323750ed1a40783432d0dcb0e6c2e813a12615d7664Ingo Molnar T1 = ftrace_now(cpu); 32481d68a96a39844853b37f20cc8282d9b65b78ef3Steven Rostedt delta = T1-T0; 32581d68a96a39844853b37f20cc8282d9b65b78ef3Steven Rostedt 32681d68a96a39844853b37f20cc8282d9b65b78ef3Steven Rostedt local_save_flags(flags); 32781d68a96a39844853b37f20cc8282d9b65b78ef3Steven Rostedt 3286450c1d3213e27b0dcbf34cce7ad1ae74244c520Steven Rostedt pc = preempt_count(); 3296450c1d3213e27b0dcbf34cce7ad1ae74244c520Steven Rostedt 3306d9b3fa5e7f663bbfb9d2d80d46136f75319cb28Steven Rostedt (Red Hat) if (!report_latency(tr, delta)) 33181d68a96a39844853b37f20cc8282d9b65b78ef3Steven Rostedt goto out; 33281d68a96a39844853b37f20cc8282d9b65b78ef3Steven Rostedt 3335389f6fad27019f2ba78f1b332f719ec05f12a42Thomas Gleixner raw_spin_lock_irqsave(&max_trace_lock, flags); 33481d68a96a39844853b37f20cc8282d9b65b78ef3Steven Rostedt 33589b2f97819dd074297bbe3e19eaa4afcc98845adSteven Rostedt /* check if we are still the max latency */ 3366d9b3fa5e7f663bbfb9d2d80d46136f75319cb28Steven Rostedt (Red Hat) if (!report_latency(tr, delta)) 33789b2f97819dd074297bbe3e19eaa4afcc98845adSteven Rostedt goto out_unlock; 33889b2f97819dd074297bbe3e19eaa4afcc98845adSteven Rostedt 33962b915f1060996a8e1f69be50e3b8e9e43b710cbJiri Olsa __trace_function(tr, CALLER_ADDR0, parent_ip, flags, pc); 340cc51a0fca66658ea710db566ba17e80e3f7d4957Steven Rostedt /* Skip 5 functions to get to the irq/preempt enable function */ 341cc51a0fca66658ea710db566ba17e80e3f7d4957Steven Rostedt __trace_stack(tr, flags, 5, pc); 34281d68a96a39844853b37f20cc8282d9b65b78ef3Steven Rostedt 34381d68a96a39844853b37f20cc8282d9b65b78ef3Steven Rostedt if (data->critical_sequence != max_sequence) 34489b2f97819dd074297bbe3e19eaa4afcc98845adSteven Rostedt goto out_unlock; 34581d68a96a39844853b37f20cc8282d9b65b78ef3Steven Rostedt 34681d68a96a39844853b37f20cc8282d9b65b78ef3Steven Rostedt data->critical_end = parent_ip; 34781d68a96a39844853b37f20cc8282d9b65b78ef3Steven Rostedt 348b5130b1e7d3717d03ab1916b198bf0d49fa0a619Carsten Emde if (likely(!is_tracing_stopped())) { 3496d9b3fa5e7f663bbfb9d2d80d46136f75319cb28Steven Rostedt (Red Hat) tr->max_latency = delta; 350b5130b1e7d3717d03ab1916b198bf0d49fa0a619Carsten Emde update_max_tr_single(tr, current, cpu); 351b5130b1e7d3717d03ab1916b198bf0d49fa0a619Carsten Emde } 35281d68a96a39844853b37f20cc8282d9b65b78ef3Steven Rostedt 35381d68a96a39844853b37f20cc8282d9b65b78ef3Steven Rostedt max_sequence++; 35481d68a96a39844853b37f20cc8282d9b65b78ef3Steven Rostedt 35589b2f97819dd074297bbe3e19eaa4afcc98845adSteven Rostedtout_unlock: 3565389f6fad27019f2ba78f1b332f719ec05f12a42Thomas Gleixner raw_spin_unlock_irqrestore(&max_trace_lock, flags); 35789b2f97819dd074297bbe3e19eaa4afcc98845adSteven Rostedt 35881d68a96a39844853b37f20cc8282d9b65b78ef3Steven Rostedtout: 35981d68a96a39844853b37f20cc8282d9b65b78ef3Steven Rostedt data->critical_sequence = max_sequence; 360750ed1a40783432d0dcb0e6c2e813a12615d7664Ingo Molnar data->preempt_timestamp = ftrace_now(cpu); 36162b915f1060996a8e1f69be50e3b8e9e43b710cbJiri Olsa __trace_function(tr, CALLER_ADDR0, parent_ip, flags, pc); 36281d68a96a39844853b37f20cc8282d9b65b78ef3Steven Rostedt} 36381d68a96a39844853b37f20cc8282d9b65b78ef3Steven Rostedt 364e309b41dd65aa953f86765eeeecc941d8e1e8b8fIngo Molnarstatic inline void 36581d68a96a39844853b37f20cc8282d9b65b78ef3Steven Rostedtstart_critical_timing(unsigned long ip, unsigned long parent_ip) 36681d68a96a39844853b37f20cc8282d9b65b78ef3Steven Rostedt{ 36781d68a96a39844853b37f20cc8282d9b65b78ef3Steven Rostedt int cpu; 36881d68a96a39844853b37f20cc8282d9b65b78ef3Steven Rostedt struct trace_array *tr = irqsoff_trace; 36981d68a96a39844853b37f20cc8282d9b65b78ef3Steven Rostedt struct trace_array_cpu *data; 37081d68a96a39844853b37f20cc8282d9b65b78ef3Steven Rostedt unsigned long flags; 37181d68a96a39844853b37f20cc8282d9b65b78ef3Steven Rostedt 37210246fa35d4ffdfe472185d4cbf9c2dfd9a9f023Steven Rostedt (Red Hat) if (!tracer_enabled || !tracing_is_enabled()) 37381d68a96a39844853b37f20cc8282d9b65b78ef3Steven Rostedt return; 37481d68a96a39844853b37f20cc8282d9b65b78ef3Steven Rostedt 375c5f888cae49dfe3e86d9d1e0dab2b63ceb057be3Steven Rostedt cpu = raw_smp_processor_id(); 376c5f888cae49dfe3e86d9d1e0dab2b63ceb057be3Steven Rostedt 377c5f888cae49dfe3e86d9d1e0dab2b63ceb057be3Steven Rostedt if (per_cpu(tracing_cpu, cpu)) 3786cd8a4bb2f97527a9ceb30bc77ea4e959c6a95e3Steven Rostedt return; 3796cd8a4bb2f97527a9ceb30bc77ea4e959c6a95e3Steven Rostedt 38012883efb670c28dff57dcd7f4f995a1ffe153b2dSteven Rostedt (Red Hat) data = per_cpu_ptr(tr->trace_buffer.data, cpu); 38181d68a96a39844853b37f20cc8282d9b65b78ef3Steven Rostedt 382c5f888cae49dfe3e86d9d1e0dab2b63ceb057be3Steven Rostedt if (unlikely(!data) || atomic_read(&data->disabled)) 38381d68a96a39844853b37f20cc8282d9b65b78ef3Steven Rostedt return; 38481d68a96a39844853b37f20cc8282d9b65b78ef3Steven Rostedt 38581d68a96a39844853b37f20cc8282d9b65b78ef3Steven Rostedt atomic_inc(&data->disabled); 38681d68a96a39844853b37f20cc8282d9b65b78ef3Steven Rostedt 38781d68a96a39844853b37f20cc8282d9b65b78ef3Steven Rostedt data->critical_sequence = max_sequence; 388750ed1a40783432d0dcb0e6c2e813a12615d7664Ingo Molnar data->preempt_timestamp = ftrace_now(cpu); 3896cd8a4bb2f97527a9ceb30bc77ea4e959c6a95e3Steven Rostedt data->critical_start = parent_ip ? : ip; 39081d68a96a39844853b37f20cc8282d9b65b78ef3Steven Rostedt 39181d68a96a39844853b37f20cc8282d9b65b78ef3Steven Rostedt local_save_flags(flags); 3926cd8a4bb2f97527a9ceb30bc77ea4e959c6a95e3Steven Rostedt 39362b915f1060996a8e1f69be50e3b8e9e43b710cbJiri Olsa __trace_function(tr, ip, parent_ip, flags, preempt_count()); 39481d68a96a39844853b37f20cc8282d9b65b78ef3Steven Rostedt 395c5f888cae49dfe3e86d9d1e0dab2b63ceb057be3Steven Rostedt per_cpu(tracing_cpu, cpu) = 1; 3966cd8a4bb2f97527a9ceb30bc77ea4e959c6a95e3Steven Rostedt 39781d68a96a39844853b37f20cc8282d9b65b78ef3Steven Rostedt atomic_dec(&data->disabled); 39881d68a96a39844853b37f20cc8282d9b65b78ef3Steven Rostedt} 39981d68a96a39844853b37f20cc8282d9b65b78ef3Steven Rostedt 400e309b41dd65aa953f86765eeeecc941d8e1e8b8fIngo Molnarstatic inline void 40181d68a96a39844853b37f20cc8282d9b65b78ef3Steven Rostedtstop_critical_timing(unsigned long ip, unsigned long parent_ip) 40281d68a96a39844853b37f20cc8282d9b65b78ef3Steven Rostedt{ 40381d68a96a39844853b37f20cc8282d9b65b78ef3Steven Rostedt int cpu; 40481d68a96a39844853b37f20cc8282d9b65b78ef3Steven Rostedt struct trace_array *tr = irqsoff_trace; 40581d68a96a39844853b37f20cc8282d9b65b78ef3Steven Rostedt struct trace_array_cpu *data; 40681d68a96a39844853b37f20cc8282d9b65b78ef3Steven Rostedt unsigned long flags; 40781d68a96a39844853b37f20cc8282d9b65b78ef3Steven Rostedt 408c5f888cae49dfe3e86d9d1e0dab2b63ceb057be3Steven Rostedt cpu = raw_smp_processor_id(); 4096cd8a4bb2f97527a9ceb30bc77ea4e959c6a95e3Steven Rostedt /* Always clear the tracing cpu on stopping the trace */ 410c5f888cae49dfe3e86d9d1e0dab2b63ceb057be3Steven Rostedt if (unlikely(per_cpu(tracing_cpu, cpu))) 411c5f888cae49dfe3e86d9d1e0dab2b63ceb057be3Steven Rostedt per_cpu(tracing_cpu, cpu) = 0; 4126cd8a4bb2f97527a9ceb30bc77ea4e959c6a95e3Steven Rostedt else 4136cd8a4bb2f97527a9ceb30bc77ea4e959c6a95e3Steven Rostedt return; 4146cd8a4bb2f97527a9ceb30bc77ea4e959c6a95e3Steven Rostedt 41510246fa35d4ffdfe472185d4cbf9c2dfd9a9f023Steven Rostedt (Red Hat) if (!tracer_enabled || !tracing_is_enabled()) 41681d68a96a39844853b37f20cc8282d9b65b78ef3Steven Rostedt return; 41781d68a96a39844853b37f20cc8282d9b65b78ef3Steven Rostedt 41812883efb670c28dff57dcd7f4f995a1ffe153b2dSteven Rostedt (Red Hat) data = per_cpu_ptr(tr->trace_buffer.data, cpu); 41981d68a96a39844853b37f20cc8282d9b65b78ef3Steven Rostedt 4203928a8a2d98081d1bc3c0a84a2d70e29b90ecf1cSteven Rostedt if (unlikely(!data) || 42181d68a96a39844853b37f20cc8282d9b65b78ef3Steven Rostedt !data->critical_start || atomic_read(&data->disabled)) 42281d68a96a39844853b37f20cc8282d9b65b78ef3Steven Rostedt return; 42381d68a96a39844853b37f20cc8282d9b65b78ef3Steven Rostedt 42481d68a96a39844853b37f20cc8282d9b65b78ef3Steven Rostedt atomic_inc(&data->disabled); 425c5f888cae49dfe3e86d9d1e0dab2b63ceb057be3Steven Rostedt 42681d68a96a39844853b37f20cc8282d9b65b78ef3Steven Rostedt local_save_flags(flags); 42762b915f1060996a8e1f69be50e3b8e9e43b710cbJiri Olsa __trace_function(tr, ip, parent_ip, flags, preempt_count()); 4286cd8a4bb2f97527a9ceb30bc77ea4e959c6a95e3Steven Rostedt check_critical_timing(tr, data, parent_ip ? : ip, cpu); 42981d68a96a39844853b37f20cc8282d9b65b78ef3Steven Rostedt data->critical_start = 0; 43081d68a96a39844853b37f20cc8282d9b65b78ef3Steven Rostedt atomic_dec(&data->disabled); 43181d68a96a39844853b37f20cc8282d9b65b78ef3Steven Rostedt} 43281d68a96a39844853b37f20cc8282d9b65b78ef3Steven Rostedt 4336cd8a4bb2f97527a9ceb30bc77ea4e959c6a95e3Steven Rostedt/* start and stop critical timings used to for stoppage (in idle) */ 434e309b41dd65aa953f86765eeeecc941d8e1e8b8fIngo Molnarvoid start_critical_timings(void) 43581d68a96a39844853b37f20cc8282d9b65b78ef3Steven Rostedt{ 4366cd8a4bb2f97527a9ceb30bc77ea4e959c6a95e3Steven Rostedt if (preempt_trace() || irq_trace()) 43781d68a96a39844853b37f20cc8282d9b65b78ef3Steven Rostedt start_critical_timing(CALLER_ADDR0, CALLER_ADDR1); 43881d68a96a39844853b37f20cc8282d9b65b78ef3Steven Rostedt} 4391fe371044b21b226b96a9dd959e971b50b28c78eIngo MolnarEXPORT_SYMBOL_GPL(start_critical_timings); 44081d68a96a39844853b37f20cc8282d9b65b78ef3Steven Rostedt 441e309b41dd65aa953f86765eeeecc941d8e1e8b8fIngo Molnarvoid stop_critical_timings(void) 44281d68a96a39844853b37f20cc8282d9b65b78ef3Steven Rostedt{ 4436cd8a4bb2f97527a9ceb30bc77ea4e959c6a95e3Steven Rostedt if (preempt_trace() || irq_trace()) 44481d68a96a39844853b37f20cc8282d9b65b78ef3Steven Rostedt stop_critical_timing(CALLER_ADDR0, CALLER_ADDR1); 44581d68a96a39844853b37f20cc8282d9b65b78ef3Steven Rostedt} 4461fe371044b21b226b96a9dd959e971b50b28c78eIngo MolnarEXPORT_SYMBOL_GPL(stop_critical_timings); 44781d68a96a39844853b37f20cc8282d9b65b78ef3Steven Rostedt 4486cd8a4bb2f97527a9ceb30bc77ea4e959c6a95e3Steven Rostedt#ifdef CONFIG_IRQSOFF_TRACER 44981d68a96a39844853b37f20cc8282d9b65b78ef3Steven Rostedt#ifdef CONFIG_PROVE_LOCKING 450e309b41dd65aa953f86765eeeecc941d8e1e8b8fIngo Molnarvoid time_hardirqs_on(unsigned long a0, unsigned long a1) 45181d68a96a39844853b37f20cc8282d9b65b78ef3Steven Rostedt{ 4526cd8a4bb2f97527a9ceb30bc77ea4e959c6a95e3Steven Rostedt if (!preempt_trace() && irq_trace()) 45381d68a96a39844853b37f20cc8282d9b65b78ef3Steven Rostedt stop_critical_timing(a0, a1); 45481d68a96a39844853b37f20cc8282d9b65b78ef3Steven Rostedt} 45581d68a96a39844853b37f20cc8282d9b65b78ef3Steven Rostedt 456e309b41dd65aa953f86765eeeecc941d8e1e8b8fIngo Molnarvoid time_hardirqs_off(unsigned long a0, unsigned long a1) 45781d68a96a39844853b37f20cc8282d9b65b78ef3Steven Rostedt{ 4586cd8a4bb2f97527a9ceb30bc77ea4e959c6a95e3Steven Rostedt if (!preempt_trace() && irq_trace()) 45981d68a96a39844853b37f20cc8282d9b65b78ef3Steven Rostedt start_critical_timing(a0, a1); 46081d68a96a39844853b37f20cc8282d9b65b78ef3Steven Rostedt} 46181d68a96a39844853b37f20cc8282d9b65b78ef3Steven Rostedt 46281d68a96a39844853b37f20cc8282d9b65b78ef3Steven Rostedt#else /* !CONFIG_PROVE_LOCKING */ 46381d68a96a39844853b37f20cc8282d9b65b78ef3Steven Rostedt 46481d68a96a39844853b37f20cc8282d9b65b78ef3Steven Rostedt/* 46581d68a96a39844853b37f20cc8282d9b65b78ef3Steven Rostedt * Stubs: 46681d68a96a39844853b37f20cc8282d9b65b78ef3Steven Rostedt */ 46781d68a96a39844853b37f20cc8282d9b65b78ef3Steven Rostedt 46881d68a96a39844853b37f20cc8282d9b65b78ef3Steven Rostedtvoid trace_softirqs_on(unsigned long ip) 46981d68a96a39844853b37f20cc8282d9b65b78ef3Steven Rostedt{ 47081d68a96a39844853b37f20cc8282d9b65b78ef3Steven Rostedt} 47181d68a96a39844853b37f20cc8282d9b65b78ef3Steven Rostedt 47281d68a96a39844853b37f20cc8282d9b65b78ef3Steven Rostedtvoid trace_softirqs_off(unsigned long ip) 47381d68a96a39844853b37f20cc8282d9b65b78ef3Steven Rostedt{ 47481d68a96a39844853b37f20cc8282d9b65b78ef3Steven Rostedt} 47581d68a96a39844853b37f20cc8282d9b65b78ef3Steven Rostedt 476e309b41dd65aa953f86765eeeecc941d8e1e8b8fIngo Molnarinline void print_irqtrace_events(struct task_struct *curr) 47781d68a96a39844853b37f20cc8282d9b65b78ef3Steven Rostedt{ 47881d68a96a39844853b37f20cc8282d9b65b78ef3Steven Rostedt} 47981d68a96a39844853b37f20cc8282d9b65b78ef3Steven Rostedt 48081d68a96a39844853b37f20cc8282d9b65b78ef3Steven Rostedt/* 48181d68a96a39844853b37f20cc8282d9b65b78ef3Steven Rostedt * We are only interested in hardirq on/off events: 48281d68a96a39844853b37f20cc8282d9b65b78ef3Steven Rostedt */ 483e309b41dd65aa953f86765eeeecc941d8e1e8b8fIngo Molnarvoid trace_hardirqs_on(void) 48481d68a96a39844853b37f20cc8282d9b65b78ef3Steven Rostedt{ 4856cd8a4bb2f97527a9ceb30bc77ea4e959c6a95e3Steven Rostedt if (!preempt_trace() && irq_trace()) 48681d68a96a39844853b37f20cc8282d9b65b78ef3Steven Rostedt stop_critical_timing(CALLER_ADDR0, CALLER_ADDR1); 48781d68a96a39844853b37f20cc8282d9b65b78ef3Steven Rostedt} 48881d68a96a39844853b37f20cc8282d9b65b78ef3Steven RostedtEXPORT_SYMBOL(trace_hardirqs_on); 48981d68a96a39844853b37f20cc8282d9b65b78ef3Steven Rostedt 490e309b41dd65aa953f86765eeeecc941d8e1e8b8fIngo Molnarvoid trace_hardirqs_off(void) 49181d68a96a39844853b37f20cc8282d9b65b78ef3Steven Rostedt{ 4926cd8a4bb2f97527a9ceb30bc77ea4e959c6a95e3Steven Rostedt if (!preempt_trace() && irq_trace()) 49381d68a96a39844853b37f20cc8282d9b65b78ef3Steven Rostedt start_critical_timing(CALLER_ADDR0, CALLER_ADDR1); 49481d68a96a39844853b37f20cc8282d9b65b78ef3Steven Rostedt} 49581d68a96a39844853b37f20cc8282d9b65b78ef3Steven RostedtEXPORT_SYMBOL(trace_hardirqs_off); 49681d68a96a39844853b37f20cc8282d9b65b78ef3Steven Rostedt 497285c00adf651c9b1d6c73d5eee482d2a617a64c1Andi Kleen__visible void trace_hardirqs_on_caller(unsigned long caller_addr) 49881d68a96a39844853b37f20cc8282d9b65b78ef3Steven Rostedt{ 4996cd8a4bb2f97527a9ceb30bc77ea4e959c6a95e3Steven Rostedt if (!preempt_trace() && irq_trace()) 50081d68a96a39844853b37f20cc8282d9b65b78ef3Steven Rostedt stop_critical_timing(CALLER_ADDR0, caller_addr); 50181d68a96a39844853b37f20cc8282d9b65b78ef3Steven Rostedt} 50281d68a96a39844853b37f20cc8282d9b65b78ef3Steven RostedtEXPORT_SYMBOL(trace_hardirqs_on_caller); 50381d68a96a39844853b37f20cc8282d9b65b78ef3Steven Rostedt 504285c00adf651c9b1d6c73d5eee482d2a617a64c1Andi Kleen__visible void trace_hardirqs_off_caller(unsigned long caller_addr) 50581d68a96a39844853b37f20cc8282d9b65b78ef3Steven Rostedt{ 5066cd8a4bb2f97527a9ceb30bc77ea4e959c6a95e3Steven Rostedt if (!preempt_trace() && irq_trace()) 50781d68a96a39844853b37f20cc8282d9b65b78ef3Steven Rostedt start_critical_timing(CALLER_ADDR0, caller_addr); 50881d68a96a39844853b37f20cc8282d9b65b78ef3Steven Rostedt} 50981d68a96a39844853b37f20cc8282d9b65b78ef3Steven RostedtEXPORT_SYMBOL(trace_hardirqs_off_caller); 51081d68a96a39844853b37f20cc8282d9b65b78ef3Steven Rostedt 51181d68a96a39844853b37f20cc8282d9b65b78ef3Steven Rostedt#endif /* CONFIG_PROVE_LOCKING */ 5126cd8a4bb2f97527a9ceb30bc77ea4e959c6a95e3Steven Rostedt#endif /* CONFIG_IRQSOFF_TRACER */ 5136cd8a4bb2f97527a9ceb30bc77ea4e959c6a95e3Steven Rostedt 5146cd8a4bb2f97527a9ceb30bc77ea4e959c6a95e3Steven Rostedt#ifdef CONFIG_PREEMPT_TRACER 515e309b41dd65aa953f86765eeeecc941d8e1e8b8fIngo Molnarvoid trace_preempt_on(unsigned long a0, unsigned long a1) 5166cd8a4bb2f97527a9ceb30bc77ea4e959c6a95e3Steven Rostedt{ 517e36de1de4a5f95b7cb3e5c37d10e6bbb91833ef0Steven Rostedt if (preempt_trace() && !irq_trace()) 5181e01cb0c6ff7e9ddb6547551794c6aa82785a7cbSteven Rostedt stop_critical_timing(a0, a1); 5196cd8a4bb2f97527a9ceb30bc77ea4e959c6a95e3Steven Rostedt} 5206cd8a4bb2f97527a9ceb30bc77ea4e959c6a95e3Steven Rostedt 521e309b41dd65aa953f86765eeeecc941d8e1e8b8fIngo Molnarvoid trace_preempt_off(unsigned long a0, unsigned long a1) 5226cd8a4bb2f97527a9ceb30bc77ea4e959c6a95e3Steven Rostedt{ 523e36de1de4a5f95b7cb3e5c37d10e6bbb91833ef0Steven Rostedt if (preempt_trace() && !irq_trace()) 5241e01cb0c6ff7e9ddb6547551794c6aa82785a7cbSteven Rostedt start_critical_timing(a0, a1); 5256cd8a4bb2f97527a9ceb30bc77ea4e959c6a95e3Steven Rostedt} 5266cd8a4bb2f97527a9ceb30bc77ea4e959c6a95e3Steven Rostedt#endif /* CONFIG_PREEMPT_TRACER */ 52781d68a96a39844853b37f20cc8282d9b65b78ef3Steven Rostedt 5284104d326b670c2b66f575d2004daa28b2d1b4c8dSteven Rostedt (Red Hat)static int register_irqsoff_function(struct trace_array *tr, int graph, int set) 52981d68a96a39844853b37f20cc8282d9b65b78ef3Steven Rostedt{ 530328df4759c03e2c3e7429cc6cb0e180c38f32063Steven Rostedt (Red Hat) int ret; 53162b915f1060996a8e1f69be50e3b8e9e43b710cbJiri Olsa 532328df4759c03e2c3e7429cc6cb0e180c38f32063Steven Rostedt (Red Hat) /* 'set' is set if TRACE_ITER_FUNCTION is about to be set */ 533328df4759c03e2c3e7429cc6cb0e180c38f32063Steven Rostedt (Red Hat) if (function_enabled || (!set && !(trace_flags & TRACE_ITER_FUNCTION))) 534328df4759c03e2c3e7429cc6cb0e180c38f32063Steven Rostedt (Red Hat) return 0; 535328df4759c03e2c3e7429cc6cb0e180c38f32063Steven Rostedt (Red Hat) 536328df4759c03e2c3e7429cc6cb0e180c38f32063Steven Rostedt (Red Hat) if (graph) 53762b915f1060996a8e1f69be50e3b8e9e43b710cbJiri Olsa ret = register_ftrace_graph(&irqsoff_graph_return, 53862b915f1060996a8e1f69be50e3b8e9e43b710cbJiri Olsa &irqsoff_graph_entry); 539328df4759c03e2c3e7429cc6cb0e180c38f32063Steven Rostedt (Red Hat) else 5404104d326b670c2b66f575d2004daa28b2d1b4c8dSteven Rostedt (Red Hat) ret = register_ftrace_function(tr->ops); 541328df4759c03e2c3e7429cc6cb0e180c38f32063Steven Rostedt (Red Hat) 542328df4759c03e2c3e7429cc6cb0e180c38f32063Steven Rostedt (Red Hat) if (!ret) 543328df4759c03e2c3e7429cc6cb0e180c38f32063Steven Rostedt (Red Hat) function_enabled = true; 544328df4759c03e2c3e7429cc6cb0e180c38f32063Steven Rostedt (Red Hat) 545328df4759c03e2c3e7429cc6cb0e180c38f32063Steven Rostedt (Red Hat) return ret; 546328df4759c03e2c3e7429cc6cb0e180c38f32063Steven Rostedt (Red Hat)} 547328df4759c03e2c3e7429cc6cb0e180c38f32063Steven Rostedt (Red Hat) 5484104d326b670c2b66f575d2004daa28b2d1b4c8dSteven Rostedt (Red Hat)static void unregister_irqsoff_function(struct trace_array *tr, int graph) 549328df4759c03e2c3e7429cc6cb0e180c38f32063Steven Rostedt (Red Hat){ 550328df4759c03e2c3e7429cc6cb0e180c38f32063Steven Rostedt (Red Hat) if (!function_enabled) 551328df4759c03e2c3e7429cc6cb0e180c38f32063Steven Rostedt (Red Hat) return; 552328df4759c03e2c3e7429cc6cb0e180c38f32063Steven Rostedt (Red Hat) 553328df4759c03e2c3e7429cc6cb0e180c38f32063Steven Rostedt (Red Hat) if (graph) 554328df4759c03e2c3e7429cc6cb0e180c38f32063Steven Rostedt (Red Hat) unregister_ftrace_graph(); 555328df4759c03e2c3e7429cc6cb0e180c38f32063Steven Rostedt (Red Hat) else 5564104d326b670c2b66f575d2004daa28b2d1b4c8dSteven Rostedt (Red Hat) unregister_ftrace_function(tr->ops); 557328df4759c03e2c3e7429cc6cb0e180c38f32063Steven Rostedt (Red Hat) 558328df4759c03e2c3e7429cc6cb0e180c38f32063Steven Rostedt (Red Hat) function_enabled = false; 559328df4759c03e2c3e7429cc6cb0e180c38f32063Steven Rostedt (Red Hat)} 560328df4759c03e2c3e7429cc6cb0e180c38f32063Steven Rostedt (Red Hat) 5614104d326b670c2b66f575d2004daa28b2d1b4c8dSteven Rostedt (Red Hat)static void irqsoff_function_set(struct trace_array *tr, int set) 562328df4759c03e2c3e7429cc6cb0e180c38f32063Steven Rostedt (Red Hat){ 563328df4759c03e2c3e7429cc6cb0e180c38f32063Steven Rostedt (Red Hat) if (set) 5644104d326b670c2b66f575d2004daa28b2d1b4c8dSteven Rostedt (Red Hat) register_irqsoff_function(tr, is_graph(), 1); 565328df4759c03e2c3e7429cc6cb0e180c38f32063Steven Rostedt (Red Hat) else 5664104d326b670c2b66f575d2004daa28b2d1b4c8dSteven Rostedt (Red Hat) unregister_irqsoff_function(tr, is_graph()); 567328df4759c03e2c3e7429cc6cb0e180c38f32063Steven Rostedt (Red Hat)} 568328df4759c03e2c3e7429cc6cb0e180c38f32063Steven Rostedt (Red Hat) 569bf6065b5c7014ab30383405718c7a6b96d2cbdb2Steven Rostedt (Red Hat)static int irqsoff_flag_changed(struct trace_array *tr, u32 mask, int set) 570328df4759c03e2c3e7429cc6cb0e180c38f32063Steven Rostedt (Red Hat){ 571bf6065b5c7014ab30383405718c7a6b96d2cbdb2Steven Rostedt (Red Hat) struct tracer *tracer = tr->current_trace; 572bf6065b5c7014ab30383405718c7a6b96d2cbdb2Steven Rostedt (Red Hat) 573328df4759c03e2c3e7429cc6cb0e180c38f32063Steven Rostedt (Red Hat) if (mask & TRACE_ITER_FUNCTION) 5744104d326b670c2b66f575d2004daa28b2d1b4c8dSteven Rostedt (Red Hat) irqsoff_function_set(tr, set); 575328df4759c03e2c3e7429cc6cb0e180c38f32063Steven Rostedt (Red Hat) 576328df4759c03e2c3e7429cc6cb0e180c38f32063Steven Rostedt (Red Hat) return trace_keep_overwrite(tracer, mask, set); 577328df4759c03e2c3e7429cc6cb0e180c38f32063Steven Rostedt (Red Hat)} 578328df4759c03e2c3e7429cc6cb0e180c38f32063Steven Rostedt (Red Hat) 579328df4759c03e2c3e7429cc6cb0e180c38f32063Steven Rostedt (Red Hat)static int start_irqsoff_tracer(struct trace_array *tr, int graph) 580328df4759c03e2c3e7429cc6cb0e180c38f32063Steven Rostedt (Red Hat){ 581328df4759c03e2c3e7429cc6cb0e180c38f32063Steven Rostedt (Red Hat) int ret; 582328df4759c03e2c3e7429cc6cb0e180c38f32063Steven Rostedt (Red Hat) 5834104d326b670c2b66f575d2004daa28b2d1b4c8dSteven Rostedt (Red Hat) ret = register_irqsoff_function(tr, graph, 0); 58462b915f1060996a8e1f69be50e3b8e9e43b710cbJiri Olsa 58562b915f1060996a8e1f69be50e3b8e9e43b710cbJiri Olsa if (!ret && tracing_is_enabled()) 5869036990d462e09366f7297a2d1da6582c3e6b1d3Steven Rostedt tracer_enabled = 1; 58794523e818f283d3c69f621406f633afff46dbf82Steven Rostedt else 5889036990d462e09366f7297a2d1da6582c3e6b1d3Steven Rostedt tracer_enabled = 0; 58962b915f1060996a8e1f69be50e3b8e9e43b710cbJiri Olsa 59062b915f1060996a8e1f69be50e3b8e9e43b710cbJiri Olsa return ret; 59181d68a96a39844853b37f20cc8282d9b65b78ef3Steven Rostedt} 59281d68a96a39844853b37f20cc8282d9b65b78ef3Steven Rostedt 59362b915f1060996a8e1f69be50e3b8e9e43b710cbJiri Olsastatic void stop_irqsoff_tracer(struct trace_array *tr, int graph) 59481d68a96a39844853b37f20cc8282d9b65b78ef3Steven Rostedt{ 59581d68a96a39844853b37f20cc8282d9b65b78ef3Steven Rostedt tracer_enabled = 0; 59662b915f1060996a8e1f69be50e3b8e9e43b710cbJiri Olsa 5974104d326b670c2b66f575d2004daa28b2d1b4c8dSteven Rostedt (Red Hat) unregister_irqsoff_function(tr, graph); 59881d68a96a39844853b37f20cc8282d9b65b78ef3Steven Rostedt} 59981d68a96a39844853b37f20cc8282d9b65b78ef3Steven Rostedt 60002f2f7646fd5d0482e805d728601e8e4305ad56fSteven Rostedt (Red Hat)static bool irqsoff_busy; 60102f2f7646fd5d0482e805d728601e8e4305ad56fSteven Rostedt (Red Hat) 60202f2f7646fd5d0482e805d728601e8e4305ad56fSteven Rostedt (Red Hat)static int __irqsoff_tracer_init(struct trace_array *tr) 60381d68a96a39844853b37f20cc8282d9b65b78ef3Steven Rostedt{ 60402f2f7646fd5d0482e805d728601e8e4305ad56fSteven Rostedt (Red Hat) if (irqsoff_busy) 60502f2f7646fd5d0482e805d728601e8e4305ad56fSteven Rostedt (Red Hat) return -EBUSY; 60602f2f7646fd5d0482e805d728601e8e4305ad56fSteven Rostedt (Red Hat) 607613f04a0f51e6e68ac6fe571ab79da3c0a5eb4daSteven Rostedt (Red Hat) save_flags = trace_flags; 608613f04a0f51e6e68ac6fe571ab79da3c0a5eb4daSteven Rostedt (Red Hat) 609613f04a0f51e6e68ac6fe571ab79da3c0a5eb4daSteven Rostedt (Red Hat) /* non overwrite screws up the latency tracers */ 6102b6080f28c7cc3efc8625ab71495aae89aeb63a0Steven Rostedt set_tracer_flag(tr, TRACE_ITER_OVERWRITE, 1); 6112b6080f28c7cc3efc8625ab71495aae89aeb63a0Steven Rostedt set_tracer_flag(tr, TRACE_ITER_LATENCY_FMT, 1); 612e9d25fe6eaa2c720bb3ea661b660e58d54fa38bfSteven Rostedt 6136d9b3fa5e7f663bbfb9d2d80d46136f75319cb28Steven Rostedt (Red Hat) tr->max_latency = 0; 61481d68a96a39844853b37f20cc8282d9b65b78ef3Steven Rostedt irqsoff_trace = tr; 615c5f888cae49dfe3e86d9d1e0dab2b63ceb057be3Steven Rostedt /* make sure that the tracer is visible */ 61681d68a96a39844853b37f20cc8282d9b65b78ef3Steven Rostedt smp_wmb(); 61712883efb670c28dff57dcd7f4f995a1ffe153b2dSteven Rostedt (Red Hat) tracing_reset_online_cpus(&tr->trace_buffer); 61862b915f1060996a8e1f69be50e3b8e9e43b710cbJiri Olsa 6194104d326b670c2b66f575d2004daa28b2d1b4c8dSteven Rostedt (Red Hat) ftrace_init_array_ops(tr, irqsoff_tracer_call); 6204104d326b670c2b66f575d2004daa28b2d1b4c8dSteven Rostedt (Red Hat) 6214104d326b670c2b66f575d2004daa28b2d1b4c8dSteven Rostedt (Red Hat) /* Only toplevel instance supports graph tracing */ 6224104d326b670c2b66f575d2004daa28b2d1b4c8dSteven Rostedt (Red Hat) if (start_irqsoff_tracer(tr, (tr->flags & TRACE_ARRAY_FL_GLOBAL && 6234104d326b670c2b66f575d2004daa28b2d1b4c8dSteven Rostedt (Red Hat) is_graph()))) 62462b915f1060996a8e1f69be50e3b8e9e43b710cbJiri Olsa printk(KERN_ERR "failed to start irqsoff tracer\n"); 62502f2f7646fd5d0482e805d728601e8e4305ad56fSteven Rostedt (Red Hat) 62602f2f7646fd5d0482e805d728601e8e4305ad56fSteven Rostedt (Red Hat) irqsoff_busy = true; 62702f2f7646fd5d0482e805d728601e8e4305ad56fSteven Rostedt (Red Hat) return 0; 62881d68a96a39844853b37f20cc8282d9b65b78ef3Steven Rostedt} 62981d68a96a39844853b37f20cc8282d9b65b78ef3Steven Rostedt 63081d68a96a39844853b37f20cc8282d9b65b78ef3Steven Rostedtstatic void irqsoff_tracer_reset(struct trace_array *tr) 63181d68a96a39844853b37f20cc8282d9b65b78ef3Steven Rostedt{ 632613f04a0f51e6e68ac6fe571ab79da3c0a5eb4daSteven Rostedt (Red Hat) int lat_flag = save_flags & TRACE_ITER_LATENCY_FMT; 633613f04a0f51e6e68ac6fe571ab79da3c0a5eb4daSteven Rostedt (Red Hat) int overwrite_flag = save_flags & TRACE_ITER_OVERWRITE; 634613f04a0f51e6e68ac6fe571ab79da3c0a5eb4daSteven Rostedt (Red Hat) 63562b915f1060996a8e1f69be50e3b8e9e43b710cbJiri Olsa stop_irqsoff_tracer(tr, is_graph()); 636e9d25fe6eaa2c720bb3ea661b660e58d54fa38bfSteven Rostedt 6372b6080f28c7cc3efc8625ab71495aae89aeb63a0Steven Rostedt set_tracer_flag(tr, TRACE_ITER_LATENCY_FMT, lat_flag); 6382b6080f28c7cc3efc8625ab71495aae89aeb63a0Steven Rostedt set_tracer_flag(tr, TRACE_ITER_OVERWRITE, overwrite_flag); 6394104d326b670c2b66f575d2004daa28b2d1b4c8dSteven Rostedt (Red Hat) ftrace_reset_array_ops(tr); 64002f2f7646fd5d0482e805d728601e8e4305ad56fSteven Rostedt (Red Hat) 64102f2f7646fd5d0482e805d728601e8e4305ad56fSteven Rostedt (Red Hat) irqsoff_busy = false; 64281d68a96a39844853b37f20cc8282d9b65b78ef3Steven Rostedt} 64381d68a96a39844853b37f20cc8282d9b65b78ef3Steven Rostedt 6449036990d462e09366f7297a2d1da6582c3e6b1d3Steven Rostedtstatic void irqsoff_tracer_start(struct trace_array *tr) 6459036990d462e09366f7297a2d1da6582c3e6b1d3Steven Rostedt{ 6469036990d462e09366f7297a2d1da6582c3e6b1d3Steven Rostedt tracer_enabled = 1; 6479036990d462e09366f7297a2d1da6582c3e6b1d3Steven Rostedt} 6489036990d462e09366f7297a2d1da6582c3e6b1d3Steven Rostedt 6499036990d462e09366f7297a2d1da6582c3e6b1d3Steven Rostedtstatic void irqsoff_tracer_stop(struct trace_array *tr) 6509036990d462e09366f7297a2d1da6582c3e6b1d3Steven Rostedt{ 6519036990d462e09366f7297a2d1da6582c3e6b1d3Steven Rostedt tracer_enabled = 0; 65281d68a96a39844853b37f20cc8282d9b65b78ef3Steven Rostedt} 65381d68a96a39844853b37f20cc8282d9b65b78ef3Steven Rostedt 6546cd8a4bb2f97527a9ceb30bc77ea4e959c6a95e3Steven Rostedt#ifdef CONFIG_IRQSOFF_TRACER 6551c80025a49855b12fa09bb6db71820e3367b1369Frederic Weisbeckerstatic int irqsoff_tracer_init(struct trace_array *tr) 6566cd8a4bb2f97527a9ceb30bc77ea4e959c6a95e3Steven Rostedt{ 6576cd8a4bb2f97527a9ceb30bc77ea4e959c6a95e3Steven Rostedt trace_type = TRACER_IRQS_OFF; 6586cd8a4bb2f97527a9ceb30bc77ea4e959c6a95e3Steven Rostedt 65902f2f7646fd5d0482e805d728601e8e4305ad56fSteven Rostedt (Red Hat) return __irqsoff_tracer_init(tr); 6606cd8a4bb2f97527a9ceb30bc77ea4e959c6a95e3Steven Rostedt} 66181d68a96a39844853b37f20cc8282d9b65b78ef3Steven Rostedtstatic struct tracer irqsoff_tracer __read_mostly = 66281d68a96a39844853b37f20cc8282d9b65b78ef3Steven Rostedt{ 66381d68a96a39844853b37f20cc8282d9b65b78ef3Steven Rostedt .name = "irqsoff", 66481d68a96a39844853b37f20cc8282d9b65b78ef3Steven Rostedt .init = irqsoff_tracer_init, 66581d68a96a39844853b37f20cc8282d9b65b78ef3Steven Rostedt .reset = irqsoff_tracer_reset, 6669036990d462e09366f7297a2d1da6582c3e6b1d3Steven Rostedt .start = irqsoff_tracer_start, 6679036990d462e09366f7297a2d1da6582c3e6b1d3Steven Rostedt .stop = irqsoff_tracer_stop, 668f43c738bfa8608424610e4fc1aef4d4644e2ce11Hiraku Toyooka .print_max = true, 66962b915f1060996a8e1f69be50e3b8e9e43b710cbJiri Olsa .print_header = irqsoff_print_header, 67062b915f1060996a8e1f69be50e3b8e9e43b710cbJiri Olsa .print_line = irqsoff_print_line, 67162b915f1060996a8e1f69be50e3b8e9e43b710cbJiri Olsa .flags = &tracer_flags, 67262b915f1060996a8e1f69be50e3b8e9e43b710cbJiri Olsa .set_flag = irqsoff_set_flag, 673328df4759c03e2c3e7429cc6cb0e180c38f32063Steven Rostedt (Red Hat) .flag_changed = irqsoff_flag_changed, 67460a11774b38fef1ab90b18c5353bd1c7c4d311c8Steven Rostedt#ifdef CONFIG_FTRACE_SELFTEST 67560a11774b38fef1ab90b18c5353bd1c7c4d311c8Steven Rostedt .selftest = trace_selftest_startup_irqsoff, 67660a11774b38fef1ab90b18c5353bd1c7c4d311c8Steven Rostedt#endif 67762b915f1060996a8e1f69be50e3b8e9e43b710cbJiri Olsa .open = irqsoff_trace_open, 67862b915f1060996a8e1f69be50e3b8e9e43b710cbJiri Olsa .close = irqsoff_trace_close, 67902f2f7646fd5d0482e805d728601e8e4305ad56fSteven Rostedt (Red Hat) .allow_instances = true, 680f43c738bfa8608424610e4fc1aef4d4644e2ce11Hiraku Toyooka .use_max_tr = true, 68181d68a96a39844853b37f20cc8282d9b65b78ef3Steven Rostedt}; 6826cd8a4bb2f97527a9ceb30bc77ea4e959c6a95e3Steven Rostedt# define register_irqsoff(trace) register_tracer(&trace) 6836cd8a4bb2f97527a9ceb30bc77ea4e959c6a95e3Steven Rostedt#else 6846cd8a4bb2f97527a9ceb30bc77ea4e959c6a95e3Steven Rostedt# define register_irqsoff(trace) do { } while (0) 6856cd8a4bb2f97527a9ceb30bc77ea4e959c6a95e3Steven Rostedt#endif 6866cd8a4bb2f97527a9ceb30bc77ea4e959c6a95e3Steven Rostedt 6876cd8a4bb2f97527a9ceb30bc77ea4e959c6a95e3Steven Rostedt#ifdef CONFIG_PREEMPT_TRACER 6881c80025a49855b12fa09bb6db71820e3367b1369Frederic Weisbeckerstatic int preemptoff_tracer_init(struct trace_array *tr) 6896cd8a4bb2f97527a9ceb30bc77ea4e959c6a95e3Steven Rostedt{ 6906cd8a4bb2f97527a9ceb30bc77ea4e959c6a95e3Steven Rostedt trace_type = TRACER_PREEMPT_OFF; 6916cd8a4bb2f97527a9ceb30bc77ea4e959c6a95e3Steven Rostedt 69202f2f7646fd5d0482e805d728601e8e4305ad56fSteven Rostedt (Red Hat) return __irqsoff_tracer_init(tr); 6936cd8a4bb2f97527a9ceb30bc77ea4e959c6a95e3Steven Rostedt} 6946cd8a4bb2f97527a9ceb30bc77ea4e959c6a95e3Steven Rostedt 6956cd8a4bb2f97527a9ceb30bc77ea4e959c6a95e3Steven Rostedtstatic struct tracer preemptoff_tracer __read_mostly = 6966cd8a4bb2f97527a9ceb30bc77ea4e959c6a95e3Steven Rostedt{ 6976cd8a4bb2f97527a9ceb30bc77ea4e959c6a95e3Steven Rostedt .name = "preemptoff", 6986cd8a4bb2f97527a9ceb30bc77ea4e959c6a95e3Steven Rostedt .init = preemptoff_tracer_init, 6996cd8a4bb2f97527a9ceb30bc77ea4e959c6a95e3Steven Rostedt .reset = irqsoff_tracer_reset, 7009036990d462e09366f7297a2d1da6582c3e6b1d3Steven Rostedt .start = irqsoff_tracer_start, 7019036990d462e09366f7297a2d1da6582c3e6b1d3Steven Rostedt .stop = irqsoff_tracer_stop, 702f43c738bfa8608424610e4fc1aef4d4644e2ce11Hiraku Toyooka .print_max = true, 70362b915f1060996a8e1f69be50e3b8e9e43b710cbJiri Olsa .print_header = irqsoff_print_header, 70462b915f1060996a8e1f69be50e3b8e9e43b710cbJiri Olsa .print_line = irqsoff_print_line, 70562b915f1060996a8e1f69be50e3b8e9e43b710cbJiri Olsa .flags = &tracer_flags, 70662b915f1060996a8e1f69be50e3b8e9e43b710cbJiri Olsa .set_flag = irqsoff_set_flag, 707328df4759c03e2c3e7429cc6cb0e180c38f32063Steven Rostedt (Red Hat) .flag_changed = irqsoff_flag_changed, 70860a11774b38fef1ab90b18c5353bd1c7c4d311c8Steven Rostedt#ifdef CONFIG_FTRACE_SELFTEST 70960a11774b38fef1ab90b18c5353bd1c7c4d311c8Steven Rostedt .selftest = trace_selftest_startup_preemptoff, 71060a11774b38fef1ab90b18c5353bd1c7c4d311c8Steven Rostedt#endif 71162b915f1060996a8e1f69be50e3b8e9e43b710cbJiri Olsa .open = irqsoff_trace_open, 71262b915f1060996a8e1f69be50e3b8e9e43b710cbJiri Olsa .close = irqsoff_trace_close, 71302f2f7646fd5d0482e805d728601e8e4305ad56fSteven Rostedt (Red Hat) .allow_instances = true, 714f43c738bfa8608424610e4fc1aef4d4644e2ce11Hiraku Toyooka .use_max_tr = true, 7156cd8a4bb2f97527a9ceb30bc77ea4e959c6a95e3Steven Rostedt}; 7166cd8a4bb2f97527a9ceb30bc77ea4e959c6a95e3Steven Rostedt# define register_preemptoff(trace) register_tracer(&trace) 7176cd8a4bb2f97527a9ceb30bc77ea4e959c6a95e3Steven Rostedt#else 7186cd8a4bb2f97527a9ceb30bc77ea4e959c6a95e3Steven Rostedt# define register_preemptoff(trace) do { } while (0) 7196cd8a4bb2f97527a9ceb30bc77ea4e959c6a95e3Steven Rostedt#endif 7206cd8a4bb2f97527a9ceb30bc77ea4e959c6a95e3Steven Rostedt 7216cd8a4bb2f97527a9ceb30bc77ea4e959c6a95e3Steven Rostedt#if defined(CONFIG_IRQSOFF_TRACER) && \ 7226cd8a4bb2f97527a9ceb30bc77ea4e959c6a95e3Steven Rostedt defined(CONFIG_PREEMPT_TRACER) 7236cd8a4bb2f97527a9ceb30bc77ea4e959c6a95e3Steven Rostedt 7241c80025a49855b12fa09bb6db71820e3367b1369Frederic Weisbeckerstatic int preemptirqsoff_tracer_init(struct trace_array *tr) 7256cd8a4bb2f97527a9ceb30bc77ea4e959c6a95e3Steven Rostedt{ 7266cd8a4bb2f97527a9ceb30bc77ea4e959c6a95e3Steven Rostedt trace_type = TRACER_IRQS_OFF | TRACER_PREEMPT_OFF; 7276cd8a4bb2f97527a9ceb30bc77ea4e959c6a95e3Steven Rostedt 72802f2f7646fd5d0482e805d728601e8e4305ad56fSteven Rostedt (Red Hat) return __irqsoff_tracer_init(tr); 7296cd8a4bb2f97527a9ceb30bc77ea4e959c6a95e3Steven Rostedt} 7306cd8a4bb2f97527a9ceb30bc77ea4e959c6a95e3Steven Rostedt 7316cd8a4bb2f97527a9ceb30bc77ea4e959c6a95e3Steven Rostedtstatic struct tracer preemptirqsoff_tracer __read_mostly = 7326cd8a4bb2f97527a9ceb30bc77ea4e959c6a95e3Steven Rostedt{ 7336cd8a4bb2f97527a9ceb30bc77ea4e959c6a95e3Steven Rostedt .name = "preemptirqsoff", 7346cd8a4bb2f97527a9ceb30bc77ea4e959c6a95e3Steven Rostedt .init = preemptirqsoff_tracer_init, 7356cd8a4bb2f97527a9ceb30bc77ea4e959c6a95e3Steven Rostedt .reset = irqsoff_tracer_reset, 7369036990d462e09366f7297a2d1da6582c3e6b1d3Steven Rostedt .start = irqsoff_tracer_start, 7379036990d462e09366f7297a2d1da6582c3e6b1d3Steven Rostedt .stop = irqsoff_tracer_stop, 738f43c738bfa8608424610e4fc1aef4d4644e2ce11Hiraku Toyooka .print_max = true, 73962b915f1060996a8e1f69be50e3b8e9e43b710cbJiri Olsa .print_header = irqsoff_print_header, 74062b915f1060996a8e1f69be50e3b8e9e43b710cbJiri Olsa .print_line = irqsoff_print_line, 74162b915f1060996a8e1f69be50e3b8e9e43b710cbJiri Olsa .flags = &tracer_flags, 74262b915f1060996a8e1f69be50e3b8e9e43b710cbJiri Olsa .set_flag = irqsoff_set_flag, 743328df4759c03e2c3e7429cc6cb0e180c38f32063Steven Rostedt (Red Hat) .flag_changed = irqsoff_flag_changed, 74460a11774b38fef1ab90b18c5353bd1c7c4d311c8Steven Rostedt#ifdef CONFIG_FTRACE_SELFTEST 74560a11774b38fef1ab90b18c5353bd1c7c4d311c8Steven Rostedt .selftest = trace_selftest_startup_preemptirqsoff, 74660a11774b38fef1ab90b18c5353bd1c7c4d311c8Steven Rostedt#endif 74762b915f1060996a8e1f69be50e3b8e9e43b710cbJiri Olsa .open = irqsoff_trace_open, 74862b915f1060996a8e1f69be50e3b8e9e43b710cbJiri Olsa .close = irqsoff_trace_close, 74902f2f7646fd5d0482e805d728601e8e4305ad56fSteven Rostedt (Red Hat) .allow_instances = true, 750f43c738bfa8608424610e4fc1aef4d4644e2ce11Hiraku Toyooka .use_max_tr = true, 7516cd8a4bb2f97527a9ceb30bc77ea4e959c6a95e3Steven Rostedt}; 7526cd8a4bb2f97527a9ceb30bc77ea4e959c6a95e3Steven Rostedt 7536cd8a4bb2f97527a9ceb30bc77ea4e959c6a95e3Steven Rostedt# define register_preemptirqsoff(trace) register_tracer(&trace) 7546cd8a4bb2f97527a9ceb30bc77ea4e959c6a95e3Steven Rostedt#else 7556cd8a4bb2f97527a9ceb30bc77ea4e959c6a95e3Steven Rostedt# define register_preemptirqsoff(trace) do { } while (0) 7566cd8a4bb2f97527a9ceb30bc77ea4e959c6a95e3Steven Rostedt#endif 75781d68a96a39844853b37f20cc8282d9b65b78ef3Steven Rostedt 75881d68a96a39844853b37f20cc8282d9b65b78ef3Steven Rostedt__init static int init_irqsoff_tracer(void) 75981d68a96a39844853b37f20cc8282d9b65b78ef3Steven Rostedt{ 7606cd8a4bb2f97527a9ceb30bc77ea4e959c6a95e3Steven Rostedt register_irqsoff(irqsoff_tracer); 7616cd8a4bb2f97527a9ceb30bc77ea4e959c6a95e3Steven Rostedt register_preemptoff(preemptoff_tracer); 7626cd8a4bb2f97527a9ceb30bc77ea4e959c6a95e3Steven Rostedt register_preemptirqsoff(preemptirqsoff_tracer); 76381d68a96a39844853b37f20cc8282d9b65b78ef3Steven Rostedt 76481d68a96a39844853b37f20cc8282d9b65b78ef3Steven Rostedt return 0; 76581d68a96a39844853b37f20cc8282d9b65b78ef3Steven Rostedt} 7666f4156723c084bfc0c0f72205c541fafb8ad3dedSteven Rostedtcore_initcall(init_irqsoff_tracer); 767