blktrace.c revision 94b5eb28b41cc79d9713696e0005ae167b5afd1b
12056a782f8e7e65fd4bfd027506b4ce1c5e9ccd4Jens Axboe/*
20fe23479577124bd2687e6783e39fa0fa4c28005Jens Axboe * Copyright (C) 2006 Jens Axboe <axboe@kernel.dk>
32056a782f8e7e65fd4bfd027506b4ce1c5e9ccd4Jens Axboe *
42056a782f8e7e65fd4bfd027506b4ce1c5e9ccd4Jens Axboe * This program is free software; you can redistribute it and/or modify
52056a782f8e7e65fd4bfd027506b4ce1c5e9ccd4Jens Axboe * it under the terms of the GNU General Public License version 2 as
62056a782f8e7e65fd4bfd027506b4ce1c5e9ccd4Jens Axboe * published by the Free Software Foundation.
72056a782f8e7e65fd4bfd027506b4ce1c5e9ccd4Jens Axboe *
82056a782f8e7e65fd4bfd027506b4ce1c5e9ccd4Jens Axboe * This program is distributed in the hope that it will be useful,
92056a782f8e7e65fd4bfd027506b4ce1c5e9ccd4Jens Axboe * but WITHOUT ANY WARRANTY; without even the implied warranty of
102056a782f8e7e65fd4bfd027506b4ce1c5e9ccd4Jens Axboe * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
112056a782f8e7e65fd4bfd027506b4ce1c5e9ccd4Jens Axboe * GNU General Public License for more details.
122056a782f8e7e65fd4bfd027506b4ce1c5e9ccd4Jens Axboe *
132056a782f8e7e65fd4bfd027506b4ce1c5e9ccd4Jens Axboe * You should have received a copy of the GNU General Public License
142056a782f8e7e65fd4bfd027506b4ce1c5e9ccd4Jens Axboe * along with this program; if not, write to the Free Software
152056a782f8e7e65fd4bfd027506b4ce1c5e9ccd4Jens Axboe * Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA  02110-1301  USA
162056a782f8e7e65fd4bfd027506b4ce1c5e9ccd4Jens Axboe *
172056a782f8e7e65fd4bfd027506b4ce1c5e9ccd4Jens Axboe */
182056a782f8e7e65fd4bfd027506b4ce1c5e9ccd4Jens Axboe#include <linux/kernel.h>
192056a782f8e7e65fd4bfd027506b4ce1c5e9ccd4Jens Axboe#include <linux/blkdev.h>
202056a782f8e7e65fd4bfd027506b4ce1c5e9ccd4Jens Axboe#include <linux/blktrace_api.h>
212056a782f8e7e65fd4bfd027506b4ce1c5e9ccd4Jens Axboe#include <linux/percpu.h>
222056a782f8e7e65fd4bfd027506b4ce1c5e9ccd4Jens Axboe#include <linux/init.h>
232056a782f8e7e65fd4bfd027506b4ce1c5e9ccd4Jens Axboe#include <linux/mutex.h>
245a0e3ad6af8660be21ca98a971cd00f331318c05Tejun Heo#include <linux/slab.h>
252056a782f8e7e65fd4bfd027506b4ce1c5e9ccd4Jens Axboe#include <linux/debugfs.h>
26be1c63411addba3ad750eb4fdfc50b97bc82825eOlaf Kirch#include <linux/time.h>
27939b366977d29b5c0d53d1ea3b0b8cefb1e76202Arnaldo Carvalho de Melo#include <linux/uaccess.h>
2855782138e47d9baf2f7d3a7af9e7cf42adf72c56Li Zefan
2955782138e47d9baf2f7d3a7af9e7cf42adf72c56Li Zefan#include <trace/events/block.h>
3055782138e47d9baf2f7d3a7af9e7cf42adf72c56Li Zefan
312db270a80b8f2238e536876cfb3987af02684df8Frederic Weisbecker#include "trace_output.h"
322056a782f8e7e65fd4bfd027506b4ce1c5e9ccd4Jens Axboe
3355782138e47d9baf2f7d3a7af9e7cf42adf72c56Li Zefan#ifdef CONFIG_BLK_DEV_IO_TRACE
3455782138e47d9baf2f7d3a7af9e7cf42adf72c56Li Zefan
352056a782f8e7e65fd4bfd027506b4ce1c5e9ccd4Jens Axboestatic unsigned int blktrace_seq __read_mostly = 1;
362056a782f8e7e65fd4bfd027506b4ce1c5e9ccd4Jens Axboe
37c71a896154119f4ca9e89d6078f5f63ad60ef199Arnaldo Carvalho de Melostatic struct trace_array *blk_tr;
385006ea73f38caef6065d1136808413813271633fLi Zefanstatic bool blk_tracer_enabled __read_mostly;
39c71a896154119f4ca9e89d6078f5f63ad60ef199Arnaldo Carvalho de Melo
40c71a896154119f4ca9e89d6078f5f63ad60ef199Arnaldo Carvalho de Melo/* Select an alternative, minimalistic output than the original one */
41ef18012b248b47ec9a12c3a83ca5e99782d39c5dSteven Rostedt#define TRACE_BLK_OPT_CLASSIC	0x1
42c71a896154119f4ca9e89d6078f5f63ad60ef199Arnaldo Carvalho de Melo
43c71a896154119f4ca9e89d6078f5f63ad60ef199Arnaldo Carvalho de Melostatic struct tracer_opt blk_tracer_opts[] = {
44c71a896154119f4ca9e89d6078f5f63ad60ef199Arnaldo Carvalho de Melo	/* Default disable the minimalistic output */
45157f9c00e88529ed84bd7d581a40d411e5414cf0Arnaldo Carvalho de Melo	{ TRACER_OPT(blk_classic, TRACE_BLK_OPT_CLASSIC) },
46c71a896154119f4ca9e89d6078f5f63ad60ef199Arnaldo Carvalho de Melo	{ }
47c71a896154119f4ca9e89d6078f5f63ad60ef199Arnaldo Carvalho de Melo};
48c71a896154119f4ca9e89d6078f5f63ad60ef199Arnaldo Carvalho de Melo
49c71a896154119f4ca9e89d6078f5f63ad60ef199Arnaldo Carvalho de Melostatic struct tracer_flags blk_tracer_flags = {
50c71a896154119f4ca9e89d6078f5f63ad60ef199Arnaldo Carvalho de Melo	.val  = 0,
51c71a896154119f4ca9e89d6078f5f63ad60ef199Arnaldo Carvalho de Melo	.opts = blk_tracer_opts,
52c71a896154119f4ca9e89d6078f5f63ad60ef199Arnaldo Carvalho de Melo};
53c71a896154119f4ca9e89d6078f5f63ad60ef199Arnaldo Carvalho de Melo
545f3ea37c7716db4e894a480e0c18b24399595b6bArnaldo Carvalho de Melo/* Global reference count of probes */
555f3ea37c7716db4e894a480e0c18b24399595b6bArnaldo Carvalho de Melostatic atomic_t blk_probes_ref = ATOMIC_INIT(0);
565f3ea37c7716db4e894a480e0c18b24399595b6bArnaldo Carvalho de Melo
573c289ba7c320560ee74979a8895141c829046a2dLi Zefanstatic void blk_register_tracepoints(void);
585f3ea37c7716db4e894a480e0c18b24399595b6bArnaldo Carvalho de Melostatic void blk_unregister_tracepoints(void);
595f3ea37c7716db4e894a480e0c18b24399595b6bArnaldo Carvalho de Melo
602056a782f8e7e65fd4bfd027506b4ce1c5e9ccd4Jens Axboe/*
61be1c63411addba3ad750eb4fdfc50b97bc82825eOlaf Kirch * Send out a notify message.
62be1c63411addba3ad750eb4fdfc50b97bc82825eOlaf Kirch */
63a863055b1066be1b8e1107f71be01daeed6bde31Jens Axboestatic void trace_note(struct blk_trace *bt, pid_t pid, int action,
64a863055b1066be1b8e1107f71be01daeed6bde31Jens Axboe		       const void *data, size_t len)
65be1c63411addba3ad750eb4fdfc50b97bc82825eOlaf Kirch{
66be1c63411addba3ad750eb4fdfc50b97bc82825eOlaf Kirch	struct blk_io_trace *t;
6718cea4591a98817697017bcb056a848bae1205dfLi Zefan	struct ring_buffer_event *event = NULL;
68e77405ad80f53966524b5c31244e13fbbbecbd84Steven Rostedt	struct ring_buffer *buffer = NULL;
6918cea4591a98817697017bcb056a848bae1205dfLi Zefan	int pc = 0;
7018cea4591a98817697017bcb056a848bae1205dfLi Zefan	int cpu = smp_processor_id();
7118cea4591a98817697017bcb056a848bae1205dfLi Zefan	bool blk_tracer = blk_tracer_enabled;
7218cea4591a98817697017bcb056a848bae1205dfLi Zefan
7318cea4591a98817697017bcb056a848bae1205dfLi Zefan	if (blk_tracer) {
74e77405ad80f53966524b5c31244e13fbbbecbd84Steven Rostedt		buffer = blk_tr->buffer;
7518cea4591a98817697017bcb056a848bae1205dfLi Zefan		pc = preempt_count();
76e77405ad80f53966524b5c31244e13fbbbecbd84Steven Rostedt		event = trace_buffer_lock_reserve(buffer, TRACE_BLK,
7718cea4591a98817697017bcb056a848bae1205dfLi Zefan						  sizeof(*t) + len,
7818cea4591a98817697017bcb056a848bae1205dfLi Zefan						  0, pc);
7918cea4591a98817697017bcb056a848bae1205dfLi Zefan		if (!event)
8018cea4591a98817697017bcb056a848bae1205dfLi Zefan			return;
8118cea4591a98817697017bcb056a848bae1205dfLi Zefan		t = ring_buffer_event_data(event);
8218cea4591a98817697017bcb056a848bae1205dfLi Zefan		goto record_it;
8318cea4591a98817697017bcb056a848bae1205dfLi Zefan	}
84be1c63411addba3ad750eb4fdfc50b97bc82825eOlaf Kirch
85c71a896154119f4ca9e89d6078f5f63ad60ef199Arnaldo Carvalho de Melo	if (!bt->rchan)
86c71a896154119f4ca9e89d6078f5f63ad60ef199Arnaldo Carvalho de Melo		return;
87c71a896154119f4ca9e89d6078f5f63ad60ef199Arnaldo Carvalho de Melo
88be1c63411addba3ad750eb4fdfc50b97bc82825eOlaf Kirch	t = relay_reserve(bt->rchan, sizeof(*t) + len);
89d3d9d2a5ea9770db07aeb13a07f999aa48e8f865Jens Axboe	if (t) {
90d3d9d2a5ea9770db07aeb13a07f999aa48e8f865Jens Axboe		t->magic = BLK_IO_TRACE_MAGIC | BLK_IO_TRACE_VERSION;
912997c8c4a0b179e8b834a7f30ba4323f2c60ccf4Ingo Molnar		t->time = ktime_to_ns(ktime_get());
9218cea4591a98817697017bcb056a848bae1205dfLi Zefanrecord_it:
93d3d9d2a5ea9770db07aeb13a07f999aa48e8f865Jens Axboe		t->device = bt->dev;
94d3d9d2a5ea9770db07aeb13a07f999aa48e8f865Jens Axboe		t->action = action;
95d3d9d2a5ea9770db07aeb13a07f999aa48e8f865Jens Axboe		t->pid = pid;
96d3d9d2a5ea9770db07aeb13a07f999aa48e8f865Jens Axboe		t->cpu = cpu;
97d3d9d2a5ea9770db07aeb13a07f999aa48e8f865Jens Axboe		t->pdu_len = len;
98d3d9d2a5ea9770db07aeb13a07f999aa48e8f865Jens Axboe		memcpy((void *) t + sizeof(*t), data, len);
9918cea4591a98817697017bcb056a848bae1205dfLi Zefan
10018cea4591a98817697017bcb056a848bae1205dfLi Zefan		if (blk_tracer)
101e77405ad80f53966524b5c31244e13fbbbecbd84Steven Rostedt			trace_buffer_unlock_commit(buffer, event, 0, pc);
102d3d9d2a5ea9770db07aeb13a07f999aa48e8f865Jens Axboe	}
103be1c63411addba3ad750eb4fdfc50b97bc82825eOlaf Kirch}
104be1c63411addba3ad750eb4fdfc50b97bc82825eOlaf Kirch
105be1c63411addba3ad750eb4fdfc50b97bc82825eOlaf Kirch/*
1062056a782f8e7e65fd4bfd027506b4ce1c5e9ccd4Jens Axboe * Send out a notify for this process, if we haven't done so since a trace
1072056a782f8e7e65fd4bfd027506b4ce1c5e9ccd4Jens Axboe * started
1082056a782f8e7e65fd4bfd027506b4ce1c5e9ccd4Jens Axboe */
1092056a782f8e7e65fd4bfd027506b4ce1c5e9ccd4Jens Axboestatic void trace_note_tsk(struct blk_trace *bt, struct task_struct *tsk)
1102056a782f8e7e65fd4bfd027506b4ce1c5e9ccd4Jens Axboe{
111a863055b1066be1b8e1107f71be01daeed6bde31Jens Axboe	tsk->btrace_seq = blktrace_seq;
112a863055b1066be1b8e1107f71be01daeed6bde31Jens Axboe	trace_note(bt, tsk->pid, BLK_TN_PROCESS, tsk->comm, sizeof(tsk->comm));
113be1c63411addba3ad750eb4fdfc50b97bc82825eOlaf Kirch}
1142056a782f8e7e65fd4bfd027506b4ce1c5e9ccd4Jens Axboe
115be1c63411addba3ad750eb4fdfc50b97bc82825eOlaf Kirchstatic void trace_note_time(struct blk_trace *bt)
116be1c63411addba3ad750eb4fdfc50b97bc82825eOlaf Kirch{
117be1c63411addba3ad750eb4fdfc50b97bc82825eOlaf Kirch	struct timespec now;
118be1c63411addba3ad750eb4fdfc50b97bc82825eOlaf Kirch	unsigned long flags;
119be1c63411addba3ad750eb4fdfc50b97bc82825eOlaf Kirch	u32 words[2];
120be1c63411addba3ad750eb4fdfc50b97bc82825eOlaf Kirch
121be1c63411addba3ad750eb4fdfc50b97bc82825eOlaf Kirch	getnstimeofday(&now);
122be1c63411addba3ad750eb4fdfc50b97bc82825eOlaf Kirch	words[0] = now.tv_sec;
123be1c63411addba3ad750eb4fdfc50b97bc82825eOlaf Kirch	words[1] = now.tv_nsec;
124be1c63411addba3ad750eb4fdfc50b97bc82825eOlaf Kirch
125be1c63411addba3ad750eb4fdfc50b97bc82825eOlaf Kirch	local_irq_save(flags);
126be1c63411addba3ad750eb4fdfc50b97bc82825eOlaf Kirch	trace_note(bt, 0, BLK_TN_TIMESTAMP, words, sizeof(words));
127be1c63411addba3ad750eb4fdfc50b97bc82825eOlaf Kirch	local_irq_restore(flags);
1282056a782f8e7e65fd4bfd027506b4ce1c5e9ccd4Jens Axboe}
1292056a782f8e7e65fd4bfd027506b4ce1c5e9ccd4Jens Axboe
1309d5f09a424a67ddb959829894efb4c71cbf6d600Alan D. Brunellevoid __trace_note_message(struct blk_trace *bt, const char *fmt, ...)
1319d5f09a424a67ddb959829894efb4c71cbf6d600Alan D. Brunelle{
1329d5f09a424a67ddb959829894efb4c71cbf6d600Alan D. Brunelle	int n;
1339d5f09a424a67ddb959829894efb4c71cbf6d600Alan D. Brunelle	va_list args;
13414a73f54798f39854e521fb596da7d50b7566bbdCarl Henrik Lunde	unsigned long flags;
13564565911cdb57c2f512a9715b985b5617402cc67Jens Axboe	char *buf;
1369d5f09a424a67ddb959829894efb4c71cbf6d600Alan D. Brunelle
13718cea4591a98817697017bcb056a848bae1205dfLi Zefan	if (unlikely(bt->trace_state != Blktrace_running &&
13818cea4591a98817697017bcb056a848bae1205dfLi Zefan		     !blk_tracer_enabled))
139c71a896154119f4ca9e89d6078f5f63ad60ef199Arnaldo Carvalho de Melo		return;
140c71a896154119f4ca9e89d6078f5f63ad60ef199Arnaldo Carvalho de Melo
141490da40d82b31c0562d3f5edb37810f492ca1c34Tao Ma	/*
142490da40d82b31c0562d3f5edb37810f492ca1c34Tao Ma	 * If the BLK_TC_NOTIFY action mask isn't set, don't send any note
143490da40d82b31c0562d3f5edb37810f492ca1c34Tao Ma	 * message to the trace.
144490da40d82b31c0562d3f5edb37810f492ca1c34Tao Ma	 */
145490da40d82b31c0562d3f5edb37810f492ca1c34Tao Ma	if (!(bt->act_mask & BLK_TC_NOTIFY))
146490da40d82b31c0562d3f5edb37810f492ca1c34Tao Ma		return;
147490da40d82b31c0562d3f5edb37810f492ca1c34Tao Ma
14814a73f54798f39854e521fb596da7d50b7566bbdCarl Henrik Lunde	local_irq_save(flags);
14964565911cdb57c2f512a9715b985b5617402cc67Jens Axboe	buf = per_cpu_ptr(bt->msg_data, smp_processor_id());
1509d5f09a424a67ddb959829894efb4c71cbf6d600Alan D. Brunelle	va_start(args, fmt);
15164565911cdb57c2f512a9715b985b5617402cc67Jens Axboe	n = vscnprintf(buf, BLK_TN_MAX_MSG, fmt, args);
1529d5f09a424a67ddb959829894efb4c71cbf6d600Alan D. Brunelle	va_end(args);
1539d5f09a424a67ddb959829894efb4c71cbf6d600Alan D. Brunelle
15464565911cdb57c2f512a9715b985b5617402cc67Jens Axboe	trace_note(bt, 0, BLK_TN_MESSAGE, buf, n);
15514a73f54798f39854e521fb596da7d50b7566bbdCarl Henrik Lunde	local_irq_restore(flags);
1569d5f09a424a67ddb959829894efb4c71cbf6d600Alan D. Brunelle}
1579d5f09a424a67ddb959829894efb4c71cbf6d600Alan D. BrunelleEXPORT_SYMBOL_GPL(__trace_note_message);
1589d5f09a424a67ddb959829894efb4c71cbf6d600Alan D. Brunelle
1592056a782f8e7e65fd4bfd027506b4ce1c5e9ccd4Jens Axboestatic int act_log_check(struct blk_trace *bt, u32 what, sector_t sector,
1602056a782f8e7e65fd4bfd027506b4ce1c5e9ccd4Jens Axboe			 pid_t pid)
1612056a782f8e7e65fd4bfd027506b4ce1c5e9ccd4Jens Axboe{
1622056a782f8e7e65fd4bfd027506b4ce1c5e9ccd4Jens Axboe	if (((bt->act_mask << BLK_TC_SHIFT) & what) == 0)
1632056a782f8e7e65fd4bfd027506b4ce1c5e9ccd4Jens Axboe		return 1;
164d0deef5b14af7d5bbd0003a0a2a1a32326e20a6dShawn Du	if (sector && (sector < bt->start_lba || sector > bt->end_lba))
1652056a782f8e7e65fd4bfd027506b4ce1c5e9ccd4Jens Axboe		return 1;
1662056a782f8e7e65fd4bfd027506b4ce1c5e9ccd4Jens Axboe	if (bt->pid && pid != bt->pid)
1672056a782f8e7e65fd4bfd027506b4ce1c5e9ccd4Jens Axboe		return 1;
1682056a782f8e7e65fd4bfd027506b4ce1c5e9ccd4Jens Axboe
1692056a782f8e7e65fd4bfd027506b4ce1c5e9ccd4Jens Axboe	return 0;
1702056a782f8e7e65fd4bfd027506b4ce1c5e9ccd4Jens Axboe}
1712056a782f8e7e65fd4bfd027506b4ce1c5e9ccd4Jens Axboe
1722056a782f8e7e65fd4bfd027506b4ce1c5e9ccd4Jens Axboe/*
1732056a782f8e7e65fd4bfd027506b4ce1c5e9ccd4Jens Axboe * Data direction bit lookup
1742056a782f8e7e65fd4bfd027506b4ce1c5e9ccd4Jens Axboe */
175e4955c9986a27bb47ddeb6cd55803053f547e2e9Li Zefanstatic const u32 ddir_act[2] = { BLK_TC_ACT(BLK_TC_READ),
176e4955c9986a27bb47ddeb6cd55803053f547e2e9Li Zefan				 BLK_TC_ACT(BLK_TC_WRITE) };
1772056a782f8e7e65fd4bfd027506b4ce1c5e9ccd4Jens Axboe
1787b6d91daee5cac6402186ff224c3af39d79f4a0eChristoph Hellwig#define BLK_TC_RAHEAD		BLK_TC_AHEAD
1797b6d91daee5cac6402186ff224c3af39d79f4a0eChristoph Hellwig
18035ba8f7083e87602b695d6eaca38a6464d5b74dbDavid Woodhouse/* The ilog2() calls fall out because they're constant */
1817b6d91daee5cac6402186ff224c3af39d79f4a0eChristoph Hellwig#define MASK_TC_BIT(rw, __name) ((rw & REQ_ ## __name) << \
1827b6d91daee5cac6402186ff224c3af39d79f4a0eChristoph Hellwig	  (ilog2(BLK_TC_ ## __name) + BLK_TC_SHIFT - __REQ_ ## __name))
1832056a782f8e7e65fd4bfd027506b4ce1c5e9ccd4Jens Axboe
1842056a782f8e7e65fd4bfd027506b4ce1c5e9ccd4Jens Axboe/*
1852056a782f8e7e65fd4bfd027506b4ce1c5e9ccd4Jens Axboe * The worker for the various blk_add_trace*() types. Fills out a
1862056a782f8e7e65fd4bfd027506b4ce1c5e9ccd4Jens Axboe * blk_io_trace structure and places it in a per-cpu subbuffer.
1872056a782f8e7e65fd4bfd027506b4ce1c5e9ccd4Jens Axboe */
1885f3ea37c7716db4e894a480e0c18b24399595b6bArnaldo Carvalho de Melostatic void __blk_add_trace(struct blk_trace *bt, sector_t sector, int bytes,
1892056a782f8e7e65fd4bfd027506b4ce1c5e9ccd4Jens Axboe		     int rw, u32 what, int error, int pdu_len, void *pdu_data)
1902056a782f8e7e65fd4bfd027506b4ce1c5e9ccd4Jens Axboe{
1912056a782f8e7e65fd4bfd027506b4ce1c5e9ccd4Jens Axboe	struct task_struct *tsk = current;
192c71a896154119f4ca9e89d6078f5f63ad60ef199Arnaldo Carvalho de Melo	struct ring_buffer_event *event = NULL;
193e77405ad80f53966524b5c31244e13fbbbecbd84Steven Rostedt	struct ring_buffer *buffer = NULL;
1942056a782f8e7e65fd4bfd027506b4ce1c5e9ccd4Jens Axboe	struct blk_io_trace *t;
1950a9877514c4fed10a70720293b37213dd172ee3eArnaldo Carvalho de Melo	unsigned long flags = 0;
1962056a782f8e7e65fd4bfd027506b4ce1c5e9ccd4Jens Axboe	unsigned long *sequence;
1972056a782f8e7e65fd4bfd027506b4ce1c5e9ccd4Jens Axboe	pid_t pid;
198c71a896154119f4ca9e89d6078f5f63ad60ef199Arnaldo Carvalho de Melo	int cpu, pc = 0;
19918cea4591a98817697017bcb056a848bae1205dfLi Zefan	bool blk_tracer = blk_tracer_enabled;
2002056a782f8e7e65fd4bfd027506b4ce1c5e9ccd4Jens Axboe
20118cea4591a98817697017bcb056a848bae1205dfLi Zefan	if (unlikely(bt->trace_state != Blktrace_running && !blk_tracer))
2022056a782f8e7e65fd4bfd027506b4ce1c5e9ccd4Jens Axboe		return;
2032056a782f8e7e65fd4bfd027506b4ce1c5e9ccd4Jens Axboe
2042056a782f8e7e65fd4bfd027506b4ce1c5e9ccd4Jens Axboe	what |= ddir_act[rw & WRITE];
2057b6d91daee5cac6402186ff224c3af39d79f4a0eChristoph Hellwig	what |= MASK_TC_BIT(rw, SYNC);
2067b6d91daee5cac6402186ff224c3af39d79f4a0eChristoph Hellwig	what |= MASK_TC_BIT(rw, RAHEAD);
20735ba8f7083e87602b695d6eaca38a6464d5b74dbDavid Woodhouse	what |= MASK_TC_BIT(rw, META);
20835ba8f7083e87602b695d6eaca38a6464d5b74dbDavid Woodhouse	what |= MASK_TC_BIT(rw, DISCARD);
2092056a782f8e7e65fd4bfd027506b4ce1c5e9ccd4Jens Axboe
2102056a782f8e7e65fd4bfd027506b4ce1c5e9ccd4Jens Axboe	pid = tsk->pid;
211d0deef5b14af7d5bbd0003a0a2a1a32326e20a6dShawn Du	if (act_log_check(bt, what, sector, pid))
2122056a782f8e7e65fd4bfd027506b4ce1c5e9ccd4Jens Axboe		return;
213c71a896154119f4ca9e89d6078f5f63ad60ef199Arnaldo Carvalho de Melo	cpu = raw_smp_processor_id();
214c71a896154119f4ca9e89d6078f5f63ad60ef199Arnaldo Carvalho de Melo
21518cea4591a98817697017bcb056a848bae1205dfLi Zefan	if (blk_tracer) {
216c71a896154119f4ca9e89d6078f5f63ad60ef199Arnaldo Carvalho de Melo		tracing_record_cmdline(current);
217c71a896154119f4ca9e89d6078f5f63ad60ef199Arnaldo Carvalho de Melo
218e77405ad80f53966524b5c31244e13fbbbecbd84Steven Rostedt		buffer = blk_tr->buffer;
21951a763dd84253bab1d0a1e68e11a7753d1b702caArnaldo Carvalho de Melo		pc = preempt_count();
220e77405ad80f53966524b5c31244e13fbbbecbd84Steven Rostedt		event = trace_buffer_lock_reserve(buffer, TRACE_BLK,
22151a763dd84253bab1d0a1e68e11a7753d1b702caArnaldo Carvalho de Melo						  sizeof(*t) + pdu_len,
22251a763dd84253bab1d0a1e68e11a7753d1b702caArnaldo Carvalho de Melo						  0, pc);
223c71a896154119f4ca9e89d6078f5f63ad60ef199Arnaldo Carvalho de Melo		if (!event)
224c71a896154119f4ca9e89d6078f5f63ad60ef199Arnaldo Carvalho de Melo			return;
22551a763dd84253bab1d0a1e68e11a7753d1b702caArnaldo Carvalho de Melo		t = ring_buffer_event_data(event);
226c71a896154119f4ca9e89d6078f5f63ad60ef199Arnaldo Carvalho de Melo		goto record_it;
227c71a896154119f4ca9e89d6078f5f63ad60ef199Arnaldo Carvalho de Melo	}
2282056a782f8e7e65fd4bfd027506b4ce1c5e9ccd4Jens Axboe
2292056a782f8e7e65fd4bfd027506b4ce1c5e9ccd4Jens Axboe	/*
2302056a782f8e7e65fd4bfd027506b4ce1c5e9ccd4Jens Axboe	 * A word about the locking here - we disable interrupts to reserve
2312056a782f8e7e65fd4bfd027506b4ce1c5e9ccd4Jens Axboe	 * some space in the relay per-cpu buffer, to prevent an irq
23214a73f54798f39854e521fb596da7d50b7566bbdCarl Henrik Lunde	 * from coming in and stepping on our toes.
2332056a782f8e7e65fd4bfd027506b4ce1c5e9ccd4Jens Axboe	 */
2342056a782f8e7e65fd4bfd027506b4ce1c5e9ccd4Jens Axboe	local_irq_save(flags);
2352056a782f8e7e65fd4bfd027506b4ce1c5e9ccd4Jens Axboe
2362056a782f8e7e65fd4bfd027506b4ce1c5e9ccd4Jens Axboe	if (unlikely(tsk->btrace_seq != blktrace_seq))
2372056a782f8e7e65fd4bfd027506b4ce1c5e9ccd4Jens Axboe		trace_note_tsk(bt, tsk);
2382056a782f8e7e65fd4bfd027506b4ce1c5e9ccd4Jens Axboe
2392056a782f8e7e65fd4bfd027506b4ce1c5e9ccd4Jens Axboe	t = relay_reserve(bt->rchan, sizeof(*t) + pdu_len);
2402056a782f8e7e65fd4bfd027506b4ce1c5e9ccd4Jens Axboe	if (t) {
2412056a782f8e7e65fd4bfd027506b4ce1c5e9ccd4Jens Axboe		sequence = per_cpu_ptr(bt->sequence, cpu);
2422056a782f8e7e65fd4bfd027506b4ce1c5e9ccd4Jens Axboe
2432056a782f8e7e65fd4bfd027506b4ce1c5e9ccd4Jens Axboe		t->magic = BLK_IO_TRACE_MAGIC | BLK_IO_TRACE_VERSION;
2442056a782f8e7e65fd4bfd027506b4ce1c5e9ccd4Jens Axboe		t->sequence = ++(*sequence);
2452997c8c4a0b179e8b834a7f30ba4323f2c60ccf4Ingo Molnar		t->time = ktime_to_ns(ktime_get());
246c71a896154119f4ca9e89d6078f5f63ad60ef199Arnaldo Carvalho de Melorecord_it:
24708a06b83ff8b2779289f733348c669f31cb65d51Arnaldo Carvalho de Melo		/*
248939b366977d29b5c0d53d1ea3b0b8cefb1e76202Arnaldo Carvalho de Melo		 * These two are not needed in ftrace as they are in the
249939b366977d29b5c0d53d1ea3b0b8cefb1e76202Arnaldo Carvalho de Melo		 * generic trace_entry, filled by tracing_generic_entry_update,
250939b366977d29b5c0d53d1ea3b0b8cefb1e76202Arnaldo Carvalho de Melo		 * but for the trace_event->bin() synthesizer benefit we do it
251939b366977d29b5c0d53d1ea3b0b8cefb1e76202Arnaldo Carvalho de Melo		 * here too.
252939b366977d29b5c0d53d1ea3b0b8cefb1e76202Arnaldo Carvalho de Melo		 */
253939b366977d29b5c0d53d1ea3b0b8cefb1e76202Arnaldo Carvalho de Melo		t->cpu = cpu;
254939b366977d29b5c0d53d1ea3b0b8cefb1e76202Arnaldo Carvalho de Melo		t->pid = pid;
25508a06b83ff8b2779289f733348c669f31cb65d51Arnaldo Carvalho de Melo
2562056a782f8e7e65fd4bfd027506b4ce1c5e9ccd4Jens Axboe		t->sector = sector;
2572056a782f8e7e65fd4bfd027506b4ce1c5e9ccd4Jens Axboe		t->bytes = bytes;
2582056a782f8e7e65fd4bfd027506b4ce1c5e9ccd4Jens Axboe		t->action = what;
2592056a782f8e7e65fd4bfd027506b4ce1c5e9ccd4Jens Axboe		t->device = bt->dev;
2602056a782f8e7e65fd4bfd027506b4ce1c5e9ccd4Jens Axboe		t->error = error;
2612056a782f8e7e65fd4bfd027506b4ce1c5e9ccd4Jens Axboe		t->pdu_len = pdu_len;
2622056a782f8e7e65fd4bfd027506b4ce1c5e9ccd4Jens Axboe
2632056a782f8e7e65fd4bfd027506b4ce1c5e9ccd4Jens Axboe		if (pdu_len)
2642056a782f8e7e65fd4bfd027506b4ce1c5e9ccd4Jens Axboe			memcpy((void *) t + sizeof(*t), pdu_data, pdu_len);
265c71a896154119f4ca9e89d6078f5f63ad60ef199Arnaldo Carvalho de Melo
26618cea4591a98817697017bcb056a848bae1205dfLi Zefan		if (blk_tracer) {
267e77405ad80f53966524b5c31244e13fbbbecbd84Steven Rostedt			trace_buffer_unlock_commit(buffer, event, 0, pc);
268c71a896154119f4ca9e89d6078f5f63ad60ef199Arnaldo Carvalho de Melo			return;
269c71a896154119f4ca9e89d6078f5f63ad60ef199Arnaldo Carvalho de Melo		}
2702056a782f8e7e65fd4bfd027506b4ce1c5e9ccd4Jens Axboe	}
2712056a782f8e7e65fd4bfd027506b4ce1c5e9ccd4Jens Axboe
2722056a782f8e7e65fd4bfd027506b4ce1c5e9ccd4Jens Axboe	local_irq_restore(flags);
2732056a782f8e7e65fd4bfd027506b4ce1c5e9ccd4Jens Axboe}
2742056a782f8e7e65fd4bfd027506b4ce1c5e9ccd4Jens Axboe
2752056a782f8e7e65fd4bfd027506b4ce1c5e9ccd4Jens Axboestatic struct dentry *blk_tree_root;
27611a57153e3377ffdf8cfca2eda9a99063f66b957Jens Axboestatic DEFINE_MUTEX(blk_tree_mutex);
2772056a782f8e7e65fd4bfd027506b4ce1c5e9ccd4Jens Axboe
278ad5dd5493a55e462796e42e50a49e76df76fdb05Li Zefanstatic void blk_trace_free(struct blk_trace *bt)
2792056a782f8e7e65fd4bfd027506b4ce1c5e9ccd4Jens Axboe{
28002c62304e6af60f1963695c6bc1bbffe619aa585Alan D. Brunelle	debugfs_remove(bt->msg_file);
2812056a782f8e7e65fd4bfd027506b4ce1c5e9ccd4Jens Axboe	debugfs_remove(bt->dropped_file);
282f48fc4d32e24c0b6a18aad30305d819bcc68c049Jens Axboe	relay_close(bt->rchan);
28339cbb602b543e477df71dca84b5b2e36f8bd29fcAlan D. Brunelle	debugfs_remove(bt->dir);
2842056a782f8e7e65fd4bfd027506b4ce1c5e9ccd4Jens Axboe	free_percpu(bt->sequence);
28564565911cdb57c2f512a9715b985b5617402cc67Jens Axboe	free_percpu(bt->msg_data);
2862056a782f8e7e65fd4bfd027506b4ce1c5e9ccd4Jens Axboe	kfree(bt);
287ad5dd5493a55e462796e42e50a49e76df76fdb05Li Zefan}
288ad5dd5493a55e462796e42e50a49e76df76fdb05Li Zefan
289ad5dd5493a55e462796e42e50a49e76df76fdb05Li Zefanstatic void blk_trace_cleanup(struct blk_trace *bt)
290ad5dd5493a55e462796e42e50a49e76df76fdb05Li Zefan{
291ad5dd5493a55e462796e42e50a49e76df76fdb05Li Zefan	blk_trace_free(bt);
2925f3ea37c7716db4e894a480e0c18b24399595b6bArnaldo Carvalho de Melo	if (atomic_dec_and_test(&blk_probes_ref))
2935f3ea37c7716db4e894a480e0c18b24399595b6bArnaldo Carvalho de Melo		blk_unregister_tracepoints();
2942056a782f8e7e65fd4bfd027506b4ce1c5e9ccd4Jens Axboe}
2952056a782f8e7e65fd4bfd027506b4ce1c5e9ccd4Jens Axboe
2966da127ad0918f93ea93678dad62ce15ffed18797Christof Schmittint blk_trace_remove(struct request_queue *q)
2972056a782f8e7e65fd4bfd027506b4ce1c5e9ccd4Jens Axboe{
2982056a782f8e7e65fd4bfd027506b4ce1c5e9ccd4Jens Axboe	struct blk_trace *bt;
2992056a782f8e7e65fd4bfd027506b4ce1c5e9ccd4Jens Axboe
3002056a782f8e7e65fd4bfd027506b4ce1c5e9ccd4Jens Axboe	bt = xchg(&q->blk_trace, NULL);
3012056a782f8e7e65fd4bfd027506b4ce1c5e9ccd4Jens Axboe	if (!bt)
3022056a782f8e7e65fd4bfd027506b4ce1c5e9ccd4Jens Axboe		return -EINVAL;
3032056a782f8e7e65fd4bfd027506b4ce1c5e9ccd4Jens Axboe
3045554720482a631702146a959db22fe417740e0a6Li Zefan	if (bt->trace_state != Blktrace_running)
3052056a782f8e7e65fd4bfd027506b4ce1c5e9ccd4Jens Axboe		blk_trace_cleanup(bt);
3062056a782f8e7e65fd4bfd027506b4ce1c5e9ccd4Jens Axboe
3072056a782f8e7e65fd4bfd027506b4ce1c5e9ccd4Jens Axboe	return 0;
3082056a782f8e7e65fd4bfd027506b4ce1c5e9ccd4Jens Axboe}
3096da127ad0918f93ea93678dad62ce15ffed18797Christof SchmittEXPORT_SYMBOL_GPL(blk_trace_remove);
3102056a782f8e7e65fd4bfd027506b4ce1c5e9ccd4Jens Axboe
3112056a782f8e7e65fd4bfd027506b4ce1c5e9ccd4Jens Axboestatic int blk_dropped_open(struct inode *inode, struct file *filp)
3122056a782f8e7e65fd4bfd027506b4ce1c5e9ccd4Jens Axboe{
3138e18e2941c53416aa219708e7dcad21fb4bd6794Theodore Ts'o	filp->private_data = inode->i_private;
3142056a782f8e7e65fd4bfd027506b4ce1c5e9ccd4Jens Axboe
3152056a782f8e7e65fd4bfd027506b4ce1c5e9ccd4Jens Axboe	return 0;
3162056a782f8e7e65fd4bfd027506b4ce1c5e9ccd4Jens Axboe}
3172056a782f8e7e65fd4bfd027506b4ce1c5e9ccd4Jens Axboe
3182056a782f8e7e65fd4bfd027506b4ce1c5e9ccd4Jens Axboestatic ssize_t blk_dropped_read(struct file *filp, char __user *buffer,
3192056a782f8e7e65fd4bfd027506b4ce1c5e9ccd4Jens Axboe				size_t count, loff_t *ppos)
3202056a782f8e7e65fd4bfd027506b4ce1c5e9ccd4Jens Axboe{
3212056a782f8e7e65fd4bfd027506b4ce1c5e9ccd4Jens Axboe	struct blk_trace *bt = filp->private_data;
3222056a782f8e7e65fd4bfd027506b4ce1c5e9ccd4Jens Axboe	char buf[16];
3232056a782f8e7e65fd4bfd027506b4ce1c5e9ccd4Jens Axboe
3242056a782f8e7e65fd4bfd027506b4ce1c5e9ccd4Jens Axboe	snprintf(buf, sizeof(buf), "%u\n", atomic_read(&bt->dropped));
3252056a782f8e7e65fd4bfd027506b4ce1c5e9ccd4Jens Axboe
3262056a782f8e7e65fd4bfd027506b4ce1c5e9ccd4Jens Axboe	return simple_read_from_buffer(buffer, count, ppos, buf, strlen(buf));
3272056a782f8e7e65fd4bfd027506b4ce1c5e9ccd4Jens Axboe}
3282056a782f8e7e65fd4bfd027506b4ce1c5e9ccd4Jens Axboe
3292b8693c0617e972fc0b2fd1ebf8de97e15b656c3Arjan van de Venstatic const struct file_operations blk_dropped_fops = {
3302056a782f8e7e65fd4bfd027506b4ce1c5e9ccd4Jens Axboe	.owner =	THIS_MODULE,
3312056a782f8e7e65fd4bfd027506b4ce1c5e9ccd4Jens Axboe	.open =		blk_dropped_open,
3322056a782f8e7e65fd4bfd027506b4ce1c5e9ccd4Jens Axboe	.read =		blk_dropped_read,
3336038f373a3dc1f1c26496e60b6c40b164716f07eArnd Bergmann	.llseek =	default_llseek,
3342056a782f8e7e65fd4bfd027506b4ce1c5e9ccd4Jens Axboe};
3352056a782f8e7e65fd4bfd027506b4ce1c5e9ccd4Jens Axboe
33602c62304e6af60f1963695c6bc1bbffe619aa585Alan D. Brunellestatic int blk_msg_open(struct inode *inode, struct file *filp)
33702c62304e6af60f1963695c6bc1bbffe619aa585Alan D. Brunelle{
33802c62304e6af60f1963695c6bc1bbffe619aa585Alan D. Brunelle	filp->private_data = inode->i_private;
33902c62304e6af60f1963695c6bc1bbffe619aa585Alan D. Brunelle
34002c62304e6af60f1963695c6bc1bbffe619aa585Alan D. Brunelle	return 0;
34102c62304e6af60f1963695c6bc1bbffe619aa585Alan D. Brunelle}
34202c62304e6af60f1963695c6bc1bbffe619aa585Alan D. Brunelle
34302c62304e6af60f1963695c6bc1bbffe619aa585Alan D. Brunellestatic ssize_t blk_msg_write(struct file *filp, const char __user *buffer,
34402c62304e6af60f1963695c6bc1bbffe619aa585Alan D. Brunelle				size_t count, loff_t *ppos)
34502c62304e6af60f1963695c6bc1bbffe619aa585Alan D. Brunelle{
34602c62304e6af60f1963695c6bc1bbffe619aa585Alan D. Brunelle	char *msg;
34702c62304e6af60f1963695c6bc1bbffe619aa585Alan D. Brunelle	struct blk_trace *bt;
34802c62304e6af60f1963695c6bc1bbffe619aa585Alan D. Brunelle
3497635b03adf3d7b84da7649b81efa91e6ebf11b85Li Zefan	if (count >= BLK_TN_MAX_MSG)
35002c62304e6af60f1963695c6bc1bbffe619aa585Alan D. Brunelle		return -EINVAL;
35102c62304e6af60f1963695c6bc1bbffe619aa585Alan D. Brunelle
352a4b3ada83d06554d307dd54abdc62b2e5648264aCarl Henrik Lunde	msg = kmalloc(count + 1, GFP_KERNEL);
35302c62304e6af60f1963695c6bc1bbffe619aa585Alan D. Brunelle	if (msg == NULL)
35402c62304e6af60f1963695c6bc1bbffe619aa585Alan D. Brunelle		return -ENOMEM;
35502c62304e6af60f1963695c6bc1bbffe619aa585Alan D. Brunelle
35602c62304e6af60f1963695c6bc1bbffe619aa585Alan D. Brunelle	if (copy_from_user(msg, buffer, count)) {
35702c62304e6af60f1963695c6bc1bbffe619aa585Alan D. Brunelle		kfree(msg);
35802c62304e6af60f1963695c6bc1bbffe619aa585Alan D. Brunelle		return -EFAULT;
35902c62304e6af60f1963695c6bc1bbffe619aa585Alan D. Brunelle	}
36002c62304e6af60f1963695c6bc1bbffe619aa585Alan D. Brunelle
361a4b3ada83d06554d307dd54abdc62b2e5648264aCarl Henrik Lunde	msg[count] = '\0';
36202c62304e6af60f1963695c6bc1bbffe619aa585Alan D. Brunelle	bt = filp->private_data;
36302c62304e6af60f1963695c6bc1bbffe619aa585Alan D. Brunelle	__trace_note_message(bt, "%s", msg);
36402c62304e6af60f1963695c6bc1bbffe619aa585Alan D. Brunelle	kfree(msg);
36502c62304e6af60f1963695c6bc1bbffe619aa585Alan D. Brunelle
36602c62304e6af60f1963695c6bc1bbffe619aa585Alan D. Brunelle	return count;
36702c62304e6af60f1963695c6bc1bbffe619aa585Alan D. Brunelle}
36802c62304e6af60f1963695c6bc1bbffe619aa585Alan D. Brunelle
36902c62304e6af60f1963695c6bc1bbffe619aa585Alan D. Brunellestatic const struct file_operations blk_msg_fops = {
37002c62304e6af60f1963695c6bc1bbffe619aa585Alan D. Brunelle	.owner =	THIS_MODULE,
37102c62304e6af60f1963695c6bc1bbffe619aa585Alan D. Brunelle	.open =		blk_msg_open,
37202c62304e6af60f1963695c6bc1bbffe619aa585Alan D. Brunelle	.write =	blk_msg_write,
3736038f373a3dc1f1c26496e60b6c40b164716f07eArnd Bergmann	.llseek =	noop_llseek,
37402c62304e6af60f1963695c6bc1bbffe619aa585Alan D. Brunelle};
37502c62304e6af60f1963695c6bc1bbffe619aa585Alan D. Brunelle
3762056a782f8e7e65fd4bfd027506b4ce1c5e9ccd4Jens Axboe/*
3772056a782f8e7e65fd4bfd027506b4ce1c5e9ccd4Jens Axboe * Keep track of how many times we encountered a full subbuffer, to aid
3782056a782f8e7e65fd4bfd027506b4ce1c5e9ccd4Jens Axboe * the user space app in telling how many lost events there were.
3792056a782f8e7e65fd4bfd027506b4ce1c5e9ccd4Jens Axboe */
3802056a782f8e7e65fd4bfd027506b4ce1c5e9ccd4Jens Axboestatic int blk_subbuf_start_callback(struct rchan_buf *buf, void *subbuf,
3812056a782f8e7e65fd4bfd027506b4ce1c5e9ccd4Jens Axboe				     void *prev_subbuf, size_t prev_padding)
3822056a782f8e7e65fd4bfd027506b4ce1c5e9ccd4Jens Axboe{
3832056a782f8e7e65fd4bfd027506b4ce1c5e9ccd4Jens Axboe	struct blk_trace *bt;
3842056a782f8e7e65fd4bfd027506b4ce1c5e9ccd4Jens Axboe
3852056a782f8e7e65fd4bfd027506b4ce1c5e9ccd4Jens Axboe	if (!relay_buf_full(buf))
3862056a782f8e7e65fd4bfd027506b4ce1c5e9ccd4Jens Axboe		return 1;
3872056a782f8e7e65fd4bfd027506b4ce1c5e9ccd4Jens Axboe
3882056a782f8e7e65fd4bfd027506b4ce1c5e9ccd4Jens Axboe	bt = buf->chan->private_data;
3892056a782f8e7e65fd4bfd027506b4ce1c5e9ccd4Jens Axboe	atomic_inc(&bt->dropped);
3902056a782f8e7e65fd4bfd027506b4ce1c5e9ccd4Jens Axboe	return 0;
3912056a782f8e7e65fd4bfd027506b4ce1c5e9ccd4Jens Axboe}
3922056a782f8e7e65fd4bfd027506b4ce1c5e9ccd4Jens Axboe
3932056a782f8e7e65fd4bfd027506b4ce1c5e9ccd4Jens Axboestatic int blk_remove_buf_file_callback(struct dentry *dentry)
3942056a782f8e7e65fd4bfd027506b4ce1c5e9ccd4Jens Axboe{
3952056a782f8e7e65fd4bfd027506b4ce1c5e9ccd4Jens Axboe	debugfs_remove(dentry);
396f48fc4d32e24c0b6a18aad30305d819bcc68c049Jens Axboe
3972056a782f8e7e65fd4bfd027506b4ce1c5e9ccd4Jens Axboe	return 0;
3982056a782f8e7e65fd4bfd027506b4ce1c5e9ccd4Jens Axboe}
3992056a782f8e7e65fd4bfd027506b4ce1c5e9ccd4Jens Axboe
4002056a782f8e7e65fd4bfd027506b4ce1c5e9ccd4Jens Axboestatic struct dentry *blk_create_buf_file_callback(const char *filename,
4012056a782f8e7e65fd4bfd027506b4ce1c5e9ccd4Jens Axboe						   struct dentry *parent,
4022056a782f8e7e65fd4bfd027506b4ce1c5e9ccd4Jens Axboe						   int mode,
4032056a782f8e7e65fd4bfd027506b4ce1c5e9ccd4Jens Axboe						   struct rchan_buf *buf,
4042056a782f8e7e65fd4bfd027506b4ce1c5e9ccd4Jens Axboe						   int *is_global)
4052056a782f8e7e65fd4bfd027506b4ce1c5e9ccd4Jens Axboe{
4062056a782f8e7e65fd4bfd027506b4ce1c5e9ccd4Jens Axboe	return debugfs_create_file(filename, mode, parent, buf,
4072056a782f8e7e65fd4bfd027506b4ce1c5e9ccd4Jens Axboe					&relay_file_operations);
4082056a782f8e7e65fd4bfd027506b4ce1c5e9ccd4Jens Axboe}
4092056a782f8e7e65fd4bfd027506b4ce1c5e9ccd4Jens Axboe
4102056a782f8e7e65fd4bfd027506b4ce1c5e9ccd4Jens Axboestatic struct rchan_callbacks blk_relay_callbacks = {
4112056a782f8e7e65fd4bfd027506b4ce1c5e9ccd4Jens Axboe	.subbuf_start		= blk_subbuf_start_callback,
4122056a782f8e7e65fd4bfd027506b4ce1c5e9ccd4Jens Axboe	.create_buf_file	= blk_create_buf_file_callback,
4132056a782f8e7e65fd4bfd027506b4ce1c5e9ccd4Jens Axboe	.remove_buf_file	= blk_remove_buf_file_callback,
4142056a782f8e7e65fd4bfd027506b4ce1c5e9ccd4Jens Axboe};
4152056a782f8e7e65fd4bfd027506b4ce1c5e9ccd4Jens Axboe
4169908c30997b8a73c95f836170b9998dae9aa3f4aLi Zefanstatic void blk_trace_setup_lba(struct blk_trace *bt,
4179908c30997b8a73c95f836170b9998dae9aa3f4aLi Zefan				struct block_device *bdev)
4189908c30997b8a73c95f836170b9998dae9aa3f4aLi Zefan{
4199908c30997b8a73c95f836170b9998dae9aa3f4aLi Zefan	struct hd_struct *part = NULL;
4209908c30997b8a73c95f836170b9998dae9aa3f4aLi Zefan
4219908c30997b8a73c95f836170b9998dae9aa3f4aLi Zefan	if (bdev)
4229908c30997b8a73c95f836170b9998dae9aa3f4aLi Zefan		part = bdev->bd_part;
4239908c30997b8a73c95f836170b9998dae9aa3f4aLi Zefan
4249908c30997b8a73c95f836170b9998dae9aa3f4aLi Zefan	if (part) {
4259908c30997b8a73c95f836170b9998dae9aa3f4aLi Zefan		bt->start_lba = part->start_sect;
4269908c30997b8a73c95f836170b9998dae9aa3f4aLi Zefan		bt->end_lba = part->start_sect + part->nr_sects;
4279908c30997b8a73c95f836170b9998dae9aa3f4aLi Zefan	} else {
4289908c30997b8a73c95f836170b9998dae9aa3f4aLi Zefan		bt->start_lba = 0;
4299908c30997b8a73c95f836170b9998dae9aa3f4aLi Zefan		bt->end_lba = -1ULL;
4309908c30997b8a73c95f836170b9998dae9aa3f4aLi Zefan	}
4319908c30997b8a73c95f836170b9998dae9aa3f4aLi Zefan}
4329908c30997b8a73c95f836170b9998dae9aa3f4aLi Zefan
4332056a782f8e7e65fd4bfd027506b4ce1c5e9ccd4Jens Axboe/*
4342056a782f8e7e65fd4bfd027506b4ce1c5e9ccd4Jens Axboe * Setup everything required to start tracing
4352056a782f8e7e65fd4bfd027506b4ce1c5e9ccd4Jens Axboe */
4366da127ad0918f93ea93678dad62ce15ffed18797Christof Schmittint do_blk_trace_setup(struct request_queue *q, char *name, dev_t dev,
437d0deef5b14af7d5bbd0003a0a2a1a32326e20a6dShawn Du		       struct block_device *bdev,
438d0deef5b14af7d5bbd0003a0a2a1a32326e20a6dShawn Du		       struct blk_user_trace_setup *buts)
4392056a782f8e7e65fd4bfd027506b4ce1c5e9ccd4Jens Axboe{
4402056a782f8e7e65fd4bfd027506b4ce1c5e9ccd4Jens Axboe	struct blk_trace *old_bt, *bt = NULL;
4412056a782f8e7e65fd4bfd027506b4ce1c5e9ccd4Jens Axboe	struct dentry *dir = NULL;
4422056a782f8e7e65fd4bfd027506b4ce1c5e9ccd4Jens Axboe	int ret, i;
4432056a782f8e7e65fd4bfd027506b4ce1c5e9ccd4Jens Axboe
444171044d449611c6e5040b37210ff6aba47f33ee4Arnd Bergmann	if (!buts->buf_size || !buts->buf_nr)
4452056a782f8e7e65fd4bfd027506b4ce1c5e9ccd4Jens Axboe		return -EINVAL;
4462056a782f8e7e65fd4bfd027506b4ce1c5e9ccd4Jens Axboe
4470497b345e7d067109e0dd9bf9f4978a6847ee13bJens Axboe	strncpy(buts->name, name, BLKTRACE_BDEV_SIZE);
4480497b345e7d067109e0dd9bf9f4978a6847ee13bJens Axboe	buts->name[BLKTRACE_BDEV_SIZE - 1] = '\0';
4492056a782f8e7e65fd4bfd027506b4ce1c5e9ccd4Jens Axboe
4502056a782f8e7e65fd4bfd027506b4ce1c5e9ccd4Jens Axboe	/*
4512056a782f8e7e65fd4bfd027506b4ce1c5e9ccd4Jens Axboe	 * some device names have larger paths - convert the slashes
4522056a782f8e7e65fd4bfd027506b4ce1c5e9ccd4Jens Axboe	 * to underscores for this to work as expected
4532056a782f8e7e65fd4bfd027506b4ce1c5e9ccd4Jens Axboe	 */
454171044d449611c6e5040b37210ff6aba47f33ee4Arnd Bergmann	for (i = 0; i < strlen(buts->name); i++)
455171044d449611c6e5040b37210ff6aba47f33ee4Arnd Bergmann		if (buts->name[i] == '/')
456171044d449611c6e5040b37210ff6aba47f33ee4Arnd Bergmann			buts->name[i] = '_';
4572056a782f8e7e65fd4bfd027506b4ce1c5e9ccd4Jens Axboe
4582056a782f8e7e65fd4bfd027506b4ce1c5e9ccd4Jens Axboe	bt = kzalloc(sizeof(*bt), GFP_KERNEL);
4592056a782f8e7e65fd4bfd027506b4ce1c5e9ccd4Jens Axboe	if (!bt)
460ad5dd5493a55e462796e42e50a49e76df76fdb05Li Zefan		return -ENOMEM;
4612056a782f8e7e65fd4bfd027506b4ce1c5e9ccd4Jens Axboe
462ad5dd5493a55e462796e42e50a49e76df76fdb05Li Zefan	ret = -ENOMEM;
4632056a782f8e7e65fd4bfd027506b4ce1c5e9ccd4Jens Axboe	bt->sequence = alloc_percpu(unsigned long);
4642056a782f8e7e65fd4bfd027506b4ce1c5e9ccd4Jens Axboe	if (!bt->sequence)
4652056a782f8e7e65fd4bfd027506b4ce1c5e9ccd4Jens Axboe		goto err;
4662056a782f8e7e65fd4bfd027506b4ce1c5e9ccd4Jens Axboe
467f0ef03985130287c6c84ebe69416cf790e6cc00eIngo Molnar	bt->msg_data = __alloc_percpu(BLK_TN_MAX_MSG, __alignof__(char));
46864565911cdb57c2f512a9715b985b5617402cc67Jens Axboe	if (!bt->msg_data)
46964565911cdb57c2f512a9715b985b5617402cc67Jens Axboe		goto err;
47064565911cdb57c2f512a9715b985b5617402cc67Jens Axboe
4712056a782f8e7e65fd4bfd027506b4ce1c5e9ccd4Jens Axboe	ret = -ENOENT;
472f48fc4d32e24c0b6a18aad30305d819bcc68c049Jens Axboe
473b5230b56ee6caeb27cedb7753c0c319646383bb4Li Zefan	mutex_lock(&blk_tree_mutex);
474f48fc4d32e24c0b6a18aad30305d819bcc68c049Jens Axboe	if (!blk_tree_root) {
475f48fc4d32e24c0b6a18aad30305d819bcc68c049Jens Axboe		blk_tree_root = debugfs_create_dir("block", NULL);
476b5230b56ee6caeb27cedb7753c0c319646383bb4Li Zefan		if (!blk_tree_root) {
477b5230b56ee6caeb27cedb7753c0c319646383bb4Li Zefan			mutex_unlock(&blk_tree_mutex);
4781a17662ea033674a58bad3603531b0b5d42572f6Li Zefan			goto err;
479b5230b56ee6caeb27cedb7753c0c319646383bb4Li Zefan		}
480f48fc4d32e24c0b6a18aad30305d819bcc68c049Jens Axboe	}
481b5230b56ee6caeb27cedb7753c0c319646383bb4Li Zefan	mutex_unlock(&blk_tree_mutex);
482f48fc4d32e24c0b6a18aad30305d819bcc68c049Jens Axboe
483f48fc4d32e24c0b6a18aad30305d819bcc68c049Jens Axboe	dir = debugfs_create_dir(buts->name, blk_tree_root);
484f48fc4d32e24c0b6a18aad30305d819bcc68c049Jens Axboe
4852056a782f8e7e65fd4bfd027506b4ce1c5e9ccd4Jens Axboe	if (!dir)
4862056a782f8e7e65fd4bfd027506b4ce1c5e9ccd4Jens Axboe		goto err;
4872056a782f8e7e65fd4bfd027506b4ce1c5e9ccd4Jens Axboe
4882056a782f8e7e65fd4bfd027506b4ce1c5e9ccd4Jens Axboe	bt->dir = dir;
4896da127ad0918f93ea93678dad62ce15ffed18797Christof Schmitt	bt->dev = dev;
4902056a782f8e7e65fd4bfd027506b4ce1c5e9ccd4Jens Axboe	atomic_set(&bt->dropped, 0);
4912056a782f8e7e65fd4bfd027506b4ce1c5e9ccd4Jens Axboe
4922056a782f8e7e65fd4bfd027506b4ce1c5e9ccd4Jens Axboe	ret = -EIO;
493939b366977d29b5c0d53d1ea3b0b8cefb1e76202Arnaldo Carvalho de Melo	bt->dropped_file = debugfs_create_file("dropped", 0444, dir, bt,
494939b366977d29b5c0d53d1ea3b0b8cefb1e76202Arnaldo Carvalho de Melo					       &blk_dropped_fops);
4952056a782f8e7e65fd4bfd027506b4ce1c5e9ccd4Jens Axboe	if (!bt->dropped_file)
4962056a782f8e7e65fd4bfd027506b4ce1c5e9ccd4Jens Axboe		goto err;
4972056a782f8e7e65fd4bfd027506b4ce1c5e9ccd4Jens Axboe
49802c62304e6af60f1963695c6bc1bbffe619aa585Alan D. Brunelle	bt->msg_file = debugfs_create_file("msg", 0222, dir, bt, &blk_msg_fops);
49902c62304e6af60f1963695c6bc1bbffe619aa585Alan D. Brunelle	if (!bt->msg_file)
50002c62304e6af60f1963695c6bc1bbffe619aa585Alan D. Brunelle		goto err;
50102c62304e6af60f1963695c6bc1bbffe619aa585Alan D. Brunelle
502171044d449611c6e5040b37210ff6aba47f33ee4Arnd Bergmann	bt->rchan = relay_open("trace", dir, buts->buf_size,
503171044d449611c6e5040b37210ff6aba47f33ee4Arnd Bergmann				buts->buf_nr, &blk_relay_callbacks, bt);
5042056a782f8e7e65fd4bfd027506b4ce1c5e9ccd4Jens Axboe	if (!bt->rchan)
5052056a782f8e7e65fd4bfd027506b4ce1c5e9ccd4Jens Axboe		goto err;
5062056a782f8e7e65fd4bfd027506b4ce1c5e9ccd4Jens Axboe
507171044d449611c6e5040b37210ff6aba47f33ee4Arnd Bergmann	bt->act_mask = buts->act_mask;
5082056a782f8e7e65fd4bfd027506b4ce1c5e9ccd4Jens Axboe	if (!bt->act_mask)
5092056a782f8e7e65fd4bfd027506b4ce1c5e9ccd4Jens Axboe		bt->act_mask = (u16) -1;
5102056a782f8e7e65fd4bfd027506b4ce1c5e9ccd4Jens Axboe
5119908c30997b8a73c95f836170b9998dae9aa3f4aLi Zefan	blk_trace_setup_lba(bt, bdev);
5122056a782f8e7e65fd4bfd027506b4ce1c5e9ccd4Jens Axboe
513d0deef5b14af7d5bbd0003a0a2a1a32326e20a6dShawn Du	/* overwrite with user settings */
514d0deef5b14af7d5bbd0003a0a2a1a32326e20a6dShawn Du	if (buts->start_lba)
515d0deef5b14af7d5bbd0003a0a2a1a32326e20a6dShawn Du		bt->start_lba = buts->start_lba;
516d0deef5b14af7d5bbd0003a0a2a1a32326e20a6dShawn Du	if (buts->end_lba)
517d0deef5b14af7d5bbd0003a0a2a1a32326e20a6dShawn Du		bt->end_lba = buts->end_lba;
518d0deef5b14af7d5bbd0003a0a2a1a32326e20a6dShawn Du
519171044d449611c6e5040b37210ff6aba47f33ee4Arnd Bergmann	bt->pid = buts->pid;
5202056a782f8e7e65fd4bfd027506b4ce1c5e9ccd4Jens Axboe	bt->trace_state = Blktrace_setup;
5212056a782f8e7e65fd4bfd027506b4ce1c5e9ccd4Jens Axboe
5222056a782f8e7e65fd4bfd027506b4ce1c5e9ccd4Jens Axboe	ret = -EBUSY;
5232056a782f8e7e65fd4bfd027506b4ce1c5e9ccd4Jens Axboe	old_bt = xchg(&q->blk_trace, bt);
5242056a782f8e7e65fd4bfd027506b4ce1c5e9ccd4Jens Axboe	if (old_bt) {
5252056a782f8e7e65fd4bfd027506b4ce1c5e9ccd4Jens Axboe		(void) xchg(&q->blk_trace, old_bt);
5262056a782f8e7e65fd4bfd027506b4ce1c5e9ccd4Jens Axboe		goto err;
5272056a782f8e7e65fd4bfd027506b4ce1c5e9ccd4Jens Axboe	}
5282056a782f8e7e65fd4bfd027506b4ce1c5e9ccd4Jens Axboe
52917ba97e347bec9bbc47a0877c7a098708982129dLi Zefan	if (atomic_inc_return(&blk_probes_ref) == 1)
530cbe28296eb1ac441b35cf45804d0ae808add7dd1Li Zefan		blk_register_tracepoints();
531cbe28296eb1ac441b35cf45804d0ae808add7dd1Li Zefan
5322056a782f8e7e65fd4bfd027506b4ce1c5e9ccd4Jens Axboe	return 0;
5332056a782f8e7e65fd4bfd027506b4ce1c5e9ccd4Jens Axboeerr:
534ad5dd5493a55e462796e42e50a49e76df76fdb05Li Zefan	blk_trace_free(bt);
5352056a782f8e7e65fd4bfd027506b4ce1c5e9ccd4Jens Axboe	return ret;
5362056a782f8e7e65fd4bfd027506b4ce1c5e9ccd4Jens Axboe}
537171044d449611c6e5040b37210ff6aba47f33ee4Arnd Bergmann
5386da127ad0918f93ea93678dad62ce15ffed18797Christof Schmittint blk_trace_setup(struct request_queue *q, char *name, dev_t dev,
539d0deef5b14af7d5bbd0003a0a2a1a32326e20a6dShawn Du		    struct block_device *bdev,
5406da127ad0918f93ea93678dad62ce15ffed18797Christof Schmitt		    char __user *arg)
541171044d449611c6e5040b37210ff6aba47f33ee4Arnd Bergmann{
542171044d449611c6e5040b37210ff6aba47f33ee4Arnd Bergmann	struct blk_user_trace_setup buts;
543171044d449611c6e5040b37210ff6aba47f33ee4Arnd Bergmann	int ret;
544171044d449611c6e5040b37210ff6aba47f33ee4Arnd Bergmann
545171044d449611c6e5040b37210ff6aba47f33ee4Arnd Bergmann	ret = copy_from_user(&buts, arg, sizeof(buts));
546171044d449611c6e5040b37210ff6aba47f33ee4Arnd Bergmann	if (ret)
547171044d449611c6e5040b37210ff6aba47f33ee4Arnd Bergmann		return -EFAULT;
548171044d449611c6e5040b37210ff6aba47f33ee4Arnd Bergmann
549d0deef5b14af7d5bbd0003a0a2a1a32326e20a6dShawn Du	ret = do_blk_trace_setup(q, name, dev, bdev, &buts);
550171044d449611c6e5040b37210ff6aba47f33ee4Arnd Bergmann	if (ret)
551171044d449611c6e5040b37210ff6aba47f33ee4Arnd Bergmann		return ret;
552171044d449611c6e5040b37210ff6aba47f33ee4Arnd Bergmann
5539a8c28c8311e30ba97499447d5a11662f5aea094Dmitry Monakhov	if (copy_to_user(arg, &buts, sizeof(buts))) {
5549a8c28c8311e30ba97499447d5a11662f5aea094Dmitry Monakhov		blk_trace_remove(q);
555171044d449611c6e5040b37210ff6aba47f33ee4Arnd Bergmann		return -EFAULT;
5569a8c28c8311e30ba97499447d5a11662f5aea094Dmitry Monakhov	}
557171044d449611c6e5040b37210ff6aba47f33ee4Arnd Bergmann	return 0;
558171044d449611c6e5040b37210ff6aba47f33ee4Arnd Bergmann}
5596da127ad0918f93ea93678dad62ce15ffed18797Christof SchmittEXPORT_SYMBOL_GPL(blk_trace_setup);
5602056a782f8e7e65fd4bfd027506b4ce1c5e9ccd4Jens Axboe
56162c2a7d969f30163f733c81158254b3095b23e72Arnd Bergmann#if defined(CONFIG_COMPAT) && defined(CONFIG_X86_64)
56262c2a7d969f30163f733c81158254b3095b23e72Arnd Bergmannstatic int compat_blk_trace_setup(struct request_queue *q, char *name,
56362c2a7d969f30163f733c81158254b3095b23e72Arnd Bergmann				  dev_t dev, struct block_device *bdev,
56462c2a7d969f30163f733c81158254b3095b23e72Arnd Bergmann				  char __user *arg)
56562c2a7d969f30163f733c81158254b3095b23e72Arnd Bergmann{
56662c2a7d969f30163f733c81158254b3095b23e72Arnd Bergmann	struct blk_user_trace_setup buts;
56762c2a7d969f30163f733c81158254b3095b23e72Arnd Bergmann	struct compat_blk_user_trace_setup cbuts;
56862c2a7d969f30163f733c81158254b3095b23e72Arnd Bergmann	int ret;
56962c2a7d969f30163f733c81158254b3095b23e72Arnd Bergmann
57062c2a7d969f30163f733c81158254b3095b23e72Arnd Bergmann	if (copy_from_user(&cbuts, arg, sizeof(cbuts)))
57162c2a7d969f30163f733c81158254b3095b23e72Arnd Bergmann		return -EFAULT;
57262c2a7d969f30163f733c81158254b3095b23e72Arnd Bergmann
57362c2a7d969f30163f733c81158254b3095b23e72Arnd Bergmann	buts = (struct blk_user_trace_setup) {
57462c2a7d969f30163f733c81158254b3095b23e72Arnd Bergmann		.act_mask = cbuts.act_mask,
57562c2a7d969f30163f733c81158254b3095b23e72Arnd Bergmann		.buf_size = cbuts.buf_size,
57662c2a7d969f30163f733c81158254b3095b23e72Arnd Bergmann		.buf_nr = cbuts.buf_nr,
57762c2a7d969f30163f733c81158254b3095b23e72Arnd Bergmann		.start_lba = cbuts.start_lba,
57862c2a7d969f30163f733c81158254b3095b23e72Arnd Bergmann		.end_lba = cbuts.end_lba,
57962c2a7d969f30163f733c81158254b3095b23e72Arnd Bergmann		.pid = cbuts.pid,
58062c2a7d969f30163f733c81158254b3095b23e72Arnd Bergmann	};
58162c2a7d969f30163f733c81158254b3095b23e72Arnd Bergmann	memcpy(&buts.name, &cbuts.name, 32);
58262c2a7d969f30163f733c81158254b3095b23e72Arnd Bergmann
58362c2a7d969f30163f733c81158254b3095b23e72Arnd Bergmann	ret = do_blk_trace_setup(q, name, dev, bdev, &buts);
58462c2a7d969f30163f733c81158254b3095b23e72Arnd Bergmann	if (ret)
58562c2a7d969f30163f733c81158254b3095b23e72Arnd Bergmann		return ret;
58662c2a7d969f30163f733c81158254b3095b23e72Arnd Bergmann
58762c2a7d969f30163f733c81158254b3095b23e72Arnd Bergmann	if (copy_to_user(arg, &buts.name, 32)) {
58862c2a7d969f30163f733c81158254b3095b23e72Arnd Bergmann		blk_trace_remove(q);
58962c2a7d969f30163f733c81158254b3095b23e72Arnd Bergmann		return -EFAULT;
59062c2a7d969f30163f733c81158254b3095b23e72Arnd Bergmann	}
59162c2a7d969f30163f733c81158254b3095b23e72Arnd Bergmann
59262c2a7d969f30163f733c81158254b3095b23e72Arnd Bergmann	return 0;
59362c2a7d969f30163f733c81158254b3095b23e72Arnd Bergmann}
59462c2a7d969f30163f733c81158254b3095b23e72Arnd Bergmann#endif
59562c2a7d969f30163f733c81158254b3095b23e72Arnd Bergmann
5966da127ad0918f93ea93678dad62ce15ffed18797Christof Schmittint blk_trace_startstop(struct request_queue *q, int start)
5972056a782f8e7e65fd4bfd027506b4ce1c5e9ccd4Jens Axboe{
5982056a782f8e7e65fd4bfd027506b4ce1c5e9ccd4Jens Axboe	int ret;
599939b366977d29b5c0d53d1ea3b0b8cefb1e76202Arnaldo Carvalho de Melo	struct blk_trace *bt = q->blk_trace;
6002056a782f8e7e65fd4bfd027506b4ce1c5e9ccd4Jens Axboe
601939b366977d29b5c0d53d1ea3b0b8cefb1e76202Arnaldo Carvalho de Melo	if (bt == NULL)
6022056a782f8e7e65fd4bfd027506b4ce1c5e9ccd4Jens Axboe		return -EINVAL;
6032056a782f8e7e65fd4bfd027506b4ce1c5e9ccd4Jens Axboe
6042056a782f8e7e65fd4bfd027506b4ce1c5e9ccd4Jens Axboe	/*
6052056a782f8e7e65fd4bfd027506b4ce1c5e9ccd4Jens Axboe	 * For starting a trace, we can transition from a setup or stopped
6062056a782f8e7e65fd4bfd027506b4ce1c5e9ccd4Jens Axboe	 * trace. For stopping a trace, the state must be running
6072056a782f8e7e65fd4bfd027506b4ce1c5e9ccd4Jens Axboe	 */
6082056a782f8e7e65fd4bfd027506b4ce1c5e9ccd4Jens Axboe	ret = -EINVAL;
6092056a782f8e7e65fd4bfd027506b4ce1c5e9ccd4Jens Axboe	if (start) {
6102056a782f8e7e65fd4bfd027506b4ce1c5e9ccd4Jens Axboe		if (bt->trace_state == Blktrace_setup ||
6112056a782f8e7e65fd4bfd027506b4ce1c5e9ccd4Jens Axboe		    bt->trace_state == Blktrace_stopped) {
6122056a782f8e7e65fd4bfd027506b4ce1c5e9ccd4Jens Axboe			blktrace_seq++;
6132056a782f8e7e65fd4bfd027506b4ce1c5e9ccd4Jens Axboe			smp_mb();
6142056a782f8e7e65fd4bfd027506b4ce1c5e9ccd4Jens Axboe			bt->trace_state = Blktrace_running;
615be1c63411addba3ad750eb4fdfc50b97bc82825eOlaf Kirch
616be1c63411addba3ad750eb4fdfc50b97bc82825eOlaf Kirch			trace_note_time(bt);
6172056a782f8e7e65fd4bfd027506b4ce1c5e9ccd4Jens Axboe			ret = 0;
6182056a782f8e7e65fd4bfd027506b4ce1c5e9ccd4Jens Axboe		}
6192056a782f8e7e65fd4bfd027506b4ce1c5e9ccd4Jens Axboe	} else {
6202056a782f8e7e65fd4bfd027506b4ce1c5e9ccd4Jens Axboe		if (bt->trace_state == Blktrace_running) {
6212056a782f8e7e65fd4bfd027506b4ce1c5e9ccd4Jens Axboe			bt->trace_state = Blktrace_stopped;
6222056a782f8e7e65fd4bfd027506b4ce1c5e9ccd4Jens Axboe			relay_flush(bt->rchan);
6232056a782f8e7e65fd4bfd027506b4ce1c5e9ccd4Jens Axboe			ret = 0;
6242056a782f8e7e65fd4bfd027506b4ce1c5e9ccd4Jens Axboe		}
6252056a782f8e7e65fd4bfd027506b4ce1c5e9ccd4Jens Axboe	}
6262056a782f8e7e65fd4bfd027506b4ce1c5e9ccd4Jens Axboe
6272056a782f8e7e65fd4bfd027506b4ce1c5e9ccd4Jens Axboe	return ret;
6282056a782f8e7e65fd4bfd027506b4ce1c5e9ccd4Jens Axboe}
6296da127ad0918f93ea93678dad62ce15ffed18797Christof SchmittEXPORT_SYMBOL_GPL(blk_trace_startstop);
6302056a782f8e7e65fd4bfd027506b4ce1c5e9ccd4Jens Axboe
6312056a782f8e7e65fd4bfd027506b4ce1c5e9ccd4Jens Axboe/**
6322056a782f8e7e65fd4bfd027506b4ce1c5e9ccd4Jens Axboe * blk_trace_ioctl: - handle the ioctls associated with tracing
6332056a782f8e7e65fd4bfd027506b4ce1c5e9ccd4Jens Axboe * @bdev:	the block device
634ef18012b248b47ec9a12c3a83ca5e99782d39c5dSteven Rostedt * @cmd:	the ioctl cmd
6352056a782f8e7e65fd4bfd027506b4ce1c5e9ccd4Jens Axboe * @arg:	the argument data, if any
6362056a782f8e7e65fd4bfd027506b4ce1c5e9ccd4Jens Axboe *
6372056a782f8e7e65fd4bfd027506b4ce1c5e9ccd4Jens Axboe **/
6382056a782f8e7e65fd4bfd027506b4ce1c5e9ccd4Jens Axboeint blk_trace_ioctl(struct block_device *bdev, unsigned cmd, char __user *arg)
6392056a782f8e7e65fd4bfd027506b4ce1c5e9ccd4Jens Axboe{
640165125e1e480f9510a5ffcfbfee4e3ee38c05f23Jens Axboe	struct request_queue *q;
6412056a782f8e7e65fd4bfd027506b4ce1c5e9ccd4Jens Axboe	int ret, start = 0;
6426da127ad0918f93ea93678dad62ce15ffed18797Christof Schmitt	char b[BDEVNAME_SIZE];
6432056a782f8e7e65fd4bfd027506b4ce1c5e9ccd4Jens Axboe
6442056a782f8e7e65fd4bfd027506b4ce1c5e9ccd4Jens Axboe	q = bdev_get_queue(bdev);
6452056a782f8e7e65fd4bfd027506b4ce1c5e9ccd4Jens Axboe	if (!q)
6462056a782f8e7e65fd4bfd027506b4ce1c5e9ccd4Jens Axboe		return -ENXIO;
6472056a782f8e7e65fd4bfd027506b4ce1c5e9ccd4Jens Axboe
6482056a782f8e7e65fd4bfd027506b4ce1c5e9ccd4Jens Axboe	mutex_lock(&bdev->bd_mutex);
6492056a782f8e7e65fd4bfd027506b4ce1c5e9ccd4Jens Axboe
6502056a782f8e7e65fd4bfd027506b4ce1c5e9ccd4Jens Axboe	switch (cmd) {
6512056a782f8e7e65fd4bfd027506b4ce1c5e9ccd4Jens Axboe	case BLKTRACESETUP:
652f36f21ecca9ee688301174e5f2e0827827a7a7ffJean Delvare		bdevname(bdev, b);
653d0deef5b14af7d5bbd0003a0a2a1a32326e20a6dShawn Du		ret = blk_trace_setup(q, b, bdev->bd_dev, bdev, arg);
6542056a782f8e7e65fd4bfd027506b4ce1c5e9ccd4Jens Axboe		break;
65562c2a7d969f30163f733c81158254b3095b23e72Arnd Bergmann#if defined(CONFIG_COMPAT) && defined(CONFIG_X86_64)
65662c2a7d969f30163f733c81158254b3095b23e72Arnd Bergmann	case BLKTRACESETUP32:
65762c2a7d969f30163f733c81158254b3095b23e72Arnd Bergmann		bdevname(bdev, b);
65862c2a7d969f30163f733c81158254b3095b23e72Arnd Bergmann		ret = compat_blk_trace_setup(q, b, bdev->bd_dev, bdev, arg);
65962c2a7d969f30163f733c81158254b3095b23e72Arnd Bergmann		break;
66062c2a7d969f30163f733c81158254b3095b23e72Arnd Bergmann#endif
6612056a782f8e7e65fd4bfd027506b4ce1c5e9ccd4Jens Axboe	case BLKTRACESTART:
6622056a782f8e7e65fd4bfd027506b4ce1c5e9ccd4Jens Axboe		start = 1;
6632056a782f8e7e65fd4bfd027506b4ce1c5e9ccd4Jens Axboe	case BLKTRACESTOP:
6642056a782f8e7e65fd4bfd027506b4ce1c5e9ccd4Jens Axboe		ret = blk_trace_startstop(q, start);
6652056a782f8e7e65fd4bfd027506b4ce1c5e9ccd4Jens Axboe		break;
6662056a782f8e7e65fd4bfd027506b4ce1c5e9ccd4Jens Axboe	case BLKTRACETEARDOWN:
6672056a782f8e7e65fd4bfd027506b4ce1c5e9ccd4Jens Axboe		ret = blk_trace_remove(q);
6682056a782f8e7e65fd4bfd027506b4ce1c5e9ccd4Jens Axboe		break;
6692056a782f8e7e65fd4bfd027506b4ce1c5e9ccd4Jens Axboe	default:
6702056a782f8e7e65fd4bfd027506b4ce1c5e9ccd4Jens Axboe		ret = -ENOTTY;
6712056a782f8e7e65fd4bfd027506b4ce1c5e9ccd4Jens Axboe		break;
6722056a782f8e7e65fd4bfd027506b4ce1c5e9ccd4Jens Axboe	}
6732056a782f8e7e65fd4bfd027506b4ce1c5e9ccd4Jens Axboe
6742056a782f8e7e65fd4bfd027506b4ce1c5e9ccd4Jens Axboe	mutex_unlock(&bdev->bd_mutex);
6752056a782f8e7e65fd4bfd027506b4ce1c5e9ccd4Jens Axboe	return ret;
6762056a782f8e7e65fd4bfd027506b4ce1c5e9ccd4Jens Axboe}
6772056a782f8e7e65fd4bfd027506b4ce1c5e9ccd4Jens Axboe
6782056a782f8e7e65fd4bfd027506b4ce1c5e9ccd4Jens Axboe/**
6792056a782f8e7e65fd4bfd027506b4ce1c5e9ccd4Jens Axboe * blk_trace_shutdown: - stop and cleanup trace structures
6802056a782f8e7e65fd4bfd027506b4ce1c5e9ccd4Jens Axboe * @q:    the request queue associated with the device
6812056a782f8e7e65fd4bfd027506b4ce1c5e9ccd4Jens Axboe *
6822056a782f8e7e65fd4bfd027506b4ce1c5e9ccd4Jens Axboe **/
683165125e1e480f9510a5ffcfbfee4e3ee38c05f23Jens Axboevoid blk_trace_shutdown(struct request_queue *q)
6842056a782f8e7e65fd4bfd027506b4ce1c5e9ccd4Jens Axboe{
6856c5c934153513dc72e2d6464f39e8ef1f27c0a3eAlexey Dobriyan	if (q->blk_trace) {
6866c5c934153513dc72e2d6464f39e8ef1f27c0a3eAlexey Dobriyan		blk_trace_startstop(q, 0);
6876c5c934153513dc72e2d6464f39e8ef1f27c0a3eAlexey Dobriyan		blk_trace_remove(q);
6886c5c934153513dc72e2d6464f39e8ef1f27c0a3eAlexey Dobriyan	}
6892056a782f8e7e65fd4bfd027506b4ce1c5e9ccd4Jens Axboe}
6905f3ea37c7716db4e894a480e0c18b24399595b6bArnaldo Carvalho de Melo
6915f3ea37c7716db4e894a480e0c18b24399595b6bArnaldo Carvalho de Melo/*
6925f3ea37c7716db4e894a480e0c18b24399595b6bArnaldo Carvalho de Melo * blktrace probes
6935f3ea37c7716db4e894a480e0c18b24399595b6bArnaldo Carvalho de Melo */
6945f3ea37c7716db4e894a480e0c18b24399595b6bArnaldo Carvalho de Melo
6955f3ea37c7716db4e894a480e0c18b24399595b6bArnaldo Carvalho de Melo/**
6965f3ea37c7716db4e894a480e0c18b24399595b6bArnaldo Carvalho de Melo * blk_add_trace_rq - Add a trace for a request oriented action
6975f3ea37c7716db4e894a480e0c18b24399595b6bArnaldo Carvalho de Melo * @q:		queue the io is for
6985f3ea37c7716db4e894a480e0c18b24399595b6bArnaldo Carvalho de Melo * @rq:		the source request
6995f3ea37c7716db4e894a480e0c18b24399595b6bArnaldo Carvalho de Melo * @what:	the action
7005f3ea37c7716db4e894a480e0c18b24399595b6bArnaldo Carvalho de Melo *
7015f3ea37c7716db4e894a480e0c18b24399595b6bArnaldo Carvalho de Melo * Description:
7025f3ea37c7716db4e894a480e0c18b24399595b6bArnaldo Carvalho de Melo *     Records an action against a request. Will log the bio offset + size.
7035f3ea37c7716db4e894a480e0c18b24399595b6bArnaldo Carvalho de Melo *
7045f3ea37c7716db4e894a480e0c18b24399595b6bArnaldo Carvalho de Melo **/
7055f3ea37c7716db4e894a480e0c18b24399595b6bArnaldo Carvalho de Melostatic void blk_add_trace_rq(struct request_queue *q, struct request *rq,
706805f6b5e1cbfedfb9b3d354013e7f4b13a79270fTao Ma			     u32 what)
7075f3ea37c7716db4e894a480e0c18b24399595b6bArnaldo Carvalho de Melo{
7085f3ea37c7716db4e894a480e0c18b24399595b6bArnaldo Carvalho de Melo	struct blk_trace *bt = q->blk_trace;
7095f3ea37c7716db4e894a480e0c18b24399595b6bArnaldo Carvalho de Melo
7105f3ea37c7716db4e894a480e0c18b24399595b6bArnaldo Carvalho de Melo	if (likely(!bt))
7115f3ea37c7716db4e894a480e0c18b24399595b6bArnaldo Carvalho de Melo		return;
7125f3ea37c7716db4e894a480e0c18b24399595b6bArnaldo Carvalho de Melo
71333659ebbae262228eef4e0fe990f393d1f0ed941Christoph Hellwig	if (rq->cmd_type == REQ_TYPE_BLOCK_PC) {
7145f3ea37c7716db4e894a480e0c18b24399595b6bArnaldo Carvalho de Melo		what |= BLK_TC_ACT(BLK_TC_PC);
715805f6b5e1cbfedfb9b3d354013e7f4b13a79270fTao Ma		__blk_add_trace(bt, 0, blk_rq_bytes(rq), rq->cmd_flags,
7162e46e8b27aa57c6bd34b3102b40ee4d0144b4fabTejun Heo				what, rq->errors, rq->cmd_len, rq->cmd);
7175f3ea37c7716db4e894a480e0c18b24399595b6bArnaldo Carvalho de Melo	} else  {
7185f3ea37c7716db4e894a480e0c18b24399595b6bArnaldo Carvalho de Melo		what |= BLK_TC_ACT(BLK_TC_FS);
719805f6b5e1cbfedfb9b3d354013e7f4b13a79270fTao Ma		__blk_add_trace(bt, blk_rq_pos(rq), blk_rq_bytes(rq),
720805f6b5e1cbfedfb9b3d354013e7f4b13a79270fTao Ma				rq->cmd_flags, what, rq->errors, 0, NULL);
7215f3ea37c7716db4e894a480e0c18b24399595b6bArnaldo Carvalho de Melo	}
7225f3ea37c7716db4e894a480e0c18b24399595b6bArnaldo Carvalho de Melo}
7235f3ea37c7716db4e894a480e0c18b24399595b6bArnaldo Carvalho de Melo
72438516ab59fbc5b3bb278cf5e1fe2867c70cff32eSteven Rostedtstatic void blk_add_trace_rq_abort(void *ignore,
72538516ab59fbc5b3bb278cf5e1fe2867c70cff32eSteven Rostedt				   struct request_queue *q, struct request *rq)
7265f3ea37c7716db4e894a480e0c18b24399595b6bArnaldo Carvalho de Melo{
7275f3ea37c7716db4e894a480e0c18b24399595b6bArnaldo Carvalho de Melo	blk_add_trace_rq(q, rq, BLK_TA_ABORT);
7285f3ea37c7716db4e894a480e0c18b24399595b6bArnaldo Carvalho de Melo}
7295f3ea37c7716db4e894a480e0c18b24399595b6bArnaldo Carvalho de Melo
73038516ab59fbc5b3bb278cf5e1fe2867c70cff32eSteven Rostedtstatic void blk_add_trace_rq_insert(void *ignore,
73138516ab59fbc5b3bb278cf5e1fe2867c70cff32eSteven Rostedt				    struct request_queue *q, struct request *rq)
7325f3ea37c7716db4e894a480e0c18b24399595b6bArnaldo Carvalho de Melo{
7335f3ea37c7716db4e894a480e0c18b24399595b6bArnaldo Carvalho de Melo	blk_add_trace_rq(q, rq, BLK_TA_INSERT);
7345f3ea37c7716db4e894a480e0c18b24399595b6bArnaldo Carvalho de Melo}
7355f3ea37c7716db4e894a480e0c18b24399595b6bArnaldo Carvalho de Melo
73638516ab59fbc5b3bb278cf5e1fe2867c70cff32eSteven Rostedtstatic void blk_add_trace_rq_issue(void *ignore,
73738516ab59fbc5b3bb278cf5e1fe2867c70cff32eSteven Rostedt				   struct request_queue *q, struct request *rq)
7385f3ea37c7716db4e894a480e0c18b24399595b6bArnaldo Carvalho de Melo{
7395f3ea37c7716db4e894a480e0c18b24399595b6bArnaldo Carvalho de Melo	blk_add_trace_rq(q, rq, BLK_TA_ISSUE);
7405f3ea37c7716db4e894a480e0c18b24399595b6bArnaldo Carvalho de Melo}
7415f3ea37c7716db4e894a480e0c18b24399595b6bArnaldo Carvalho de Melo
74238516ab59fbc5b3bb278cf5e1fe2867c70cff32eSteven Rostedtstatic void blk_add_trace_rq_requeue(void *ignore,
74338516ab59fbc5b3bb278cf5e1fe2867c70cff32eSteven Rostedt				     struct request_queue *q,
744939b366977d29b5c0d53d1ea3b0b8cefb1e76202Arnaldo Carvalho de Melo				     struct request *rq)
7455f3ea37c7716db4e894a480e0c18b24399595b6bArnaldo Carvalho de Melo{
7465f3ea37c7716db4e894a480e0c18b24399595b6bArnaldo Carvalho de Melo	blk_add_trace_rq(q, rq, BLK_TA_REQUEUE);
7475f3ea37c7716db4e894a480e0c18b24399595b6bArnaldo Carvalho de Melo}
7485f3ea37c7716db4e894a480e0c18b24399595b6bArnaldo Carvalho de Melo
74938516ab59fbc5b3bb278cf5e1fe2867c70cff32eSteven Rostedtstatic void blk_add_trace_rq_complete(void *ignore,
75038516ab59fbc5b3bb278cf5e1fe2867c70cff32eSteven Rostedt				      struct request_queue *q,
751939b366977d29b5c0d53d1ea3b0b8cefb1e76202Arnaldo Carvalho de Melo				      struct request *rq)
7525f3ea37c7716db4e894a480e0c18b24399595b6bArnaldo Carvalho de Melo{
7535f3ea37c7716db4e894a480e0c18b24399595b6bArnaldo Carvalho de Melo	blk_add_trace_rq(q, rq, BLK_TA_COMPLETE);
7545f3ea37c7716db4e894a480e0c18b24399595b6bArnaldo Carvalho de Melo}
7555f3ea37c7716db4e894a480e0c18b24399595b6bArnaldo Carvalho de Melo
7565f3ea37c7716db4e894a480e0c18b24399595b6bArnaldo Carvalho de Melo/**
7575f3ea37c7716db4e894a480e0c18b24399595b6bArnaldo Carvalho de Melo * blk_add_trace_bio - Add a trace for a bio oriented action
7585f3ea37c7716db4e894a480e0c18b24399595b6bArnaldo Carvalho de Melo * @q:		queue the io is for
7595f3ea37c7716db4e894a480e0c18b24399595b6bArnaldo Carvalho de Melo * @bio:	the source bio
7605f3ea37c7716db4e894a480e0c18b24399595b6bArnaldo Carvalho de Melo * @what:	the action
761797a455d2c682476c3797dbfecf5bf84c1e3b9d3Jens Axboe * @error:	error, if any
7625f3ea37c7716db4e894a480e0c18b24399595b6bArnaldo Carvalho de Melo *
7635f3ea37c7716db4e894a480e0c18b24399595b6bArnaldo Carvalho de Melo * Description:
7645f3ea37c7716db4e894a480e0c18b24399595b6bArnaldo Carvalho de Melo *     Records an action against a bio. Will log the bio offset + size.
7655f3ea37c7716db4e894a480e0c18b24399595b6bArnaldo Carvalho de Melo *
7665f3ea37c7716db4e894a480e0c18b24399595b6bArnaldo Carvalho de Melo **/
7675f3ea37c7716db4e894a480e0c18b24399595b6bArnaldo Carvalho de Melostatic void blk_add_trace_bio(struct request_queue *q, struct bio *bio,
768797a455d2c682476c3797dbfecf5bf84c1e3b9d3Jens Axboe			      u32 what, int error)
7695f3ea37c7716db4e894a480e0c18b24399595b6bArnaldo Carvalho de Melo{
7705f3ea37c7716db4e894a480e0c18b24399595b6bArnaldo Carvalho de Melo	struct blk_trace *bt = q->blk_trace;
7715f3ea37c7716db4e894a480e0c18b24399595b6bArnaldo Carvalho de Melo
7725f3ea37c7716db4e894a480e0c18b24399595b6bArnaldo Carvalho de Melo	if (likely(!bt))
7735f3ea37c7716db4e894a480e0c18b24399595b6bArnaldo Carvalho de Melo		return;
7745f3ea37c7716db4e894a480e0c18b24399595b6bArnaldo Carvalho de Melo
775797a455d2c682476c3797dbfecf5bf84c1e3b9d3Jens Axboe	if (!error && !bio_flagged(bio, BIO_UPTODATE))
776797a455d2c682476c3797dbfecf5bf84c1e3b9d3Jens Axboe		error = EIO;
777797a455d2c682476c3797dbfecf5bf84c1e3b9d3Jens Axboe
7785f3ea37c7716db4e894a480e0c18b24399595b6bArnaldo Carvalho de Melo	__blk_add_trace(bt, bio->bi_sector, bio->bi_size, bio->bi_rw, what,
779797a455d2c682476c3797dbfecf5bf84c1e3b9d3Jens Axboe			error, 0, NULL);
7805f3ea37c7716db4e894a480e0c18b24399595b6bArnaldo Carvalho de Melo}
7815f3ea37c7716db4e894a480e0c18b24399595b6bArnaldo Carvalho de Melo
78238516ab59fbc5b3bb278cf5e1fe2867c70cff32eSteven Rostedtstatic void blk_add_trace_bio_bounce(void *ignore,
78338516ab59fbc5b3bb278cf5e1fe2867c70cff32eSteven Rostedt				     struct request_queue *q, struct bio *bio)
7845f3ea37c7716db4e894a480e0c18b24399595b6bArnaldo Carvalho de Melo{
785797a455d2c682476c3797dbfecf5bf84c1e3b9d3Jens Axboe	blk_add_trace_bio(q, bio, BLK_TA_BOUNCE, 0);
7865f3ea37c7716db4e894a480e0c18b24399595b6bArnaldo Carvalho de Melo}
7875f3ea37c7716db4e894a480e0c18b24399595b6bArnaldo Carvalho de Melo
78838516ab59fbc5b3bb278cf5e1fe2867c70cff32eSteven Rostedtstatic void blk_add_trace_bio_complete(void *ignore,
78923036f1a340beec19cc451ba9719526c4ffb3a57Mathieu Desnoyers				       struct request_queue *q, struct bio *bio,
79023036f1a340beec19cc451ba9719526c4ffb3a57Mathieu Desnoyers				       int error)
7915f3ea37c7716db4e894a480e0c18b24399595b6bArnaldo Carvalho de Melo{
792797a455d2c682476c3797dbfecf5bf84c1e3b9d3Jens Axboe	blk_add_trace_bio(q, bio, BLK_TA_COMPLETE, error);
7935f3ea37c7716db4e894a480e0c18b24399595b6bArnaldo Carvalho de Melo}
7945f3ea37c7716db4e894a480e0c18b24399595b6bArnaldo Carvalho de Melo
79538516ab59fbc5b3bb278cf5e1fe2867c70cff32eSteven Rostedtstatic void blk_add_trace_bio_backmerge(void *ignore,
79638516ab59fbc5b3bb278cf5e1fe2867c70cff32eSteven Rostedt					struct request_queue *q,
797939b366977d29b5c0d53d1ea3b0b8cefb1e76202Arnaldo Carvalho de Melo					struct bio *bio)
7985f3ea37c7716db4e894a480e0c18b24399595b6bArnaldo Carvalho de Melo{
799797a455d2c682476c3797dbfecf5bf84c1e3b9d3Jens Axboe	blk_add_trace_bio(q, bio, BLK_TA_BACKMERGE, 0);
8005f3ea37c7716db4e894a480e0c18b24399595b6bArnaldo Carvalho de Melo}
8015f3ea37c7716db4e894a480e0c18b24399595b6bArnaldo Carvalho de Melo
80238516ab59fbc5b3bb278cf5e1fe2867c70cff32eSteven Rostedtstatic void blk_add_trace_bio_frontmerge(void *ignore,
80338516ab59fbc5b3bb278cf5e1fe2867c70cff32eSteven Rostedt					 struct request_queue *q,
804939b366977d29b5c0d53d1ea3b0b8cefb1e76202Arnaldo Carvalho de Melo					 struct bio *bio)
8055f3ea37c7716db4e894a480e0c18b24399595b6bArnaldo Carvalho de Melo{
806797a455d2c682476c3797dbfecf5bf84c1e3b9d3Jens Axboe	blk_add_trace_bio(q, bio, BLK_TA_FRONTMERGE, 0);
8075f3ea37c7716db4e894a480e0c18b24399595b6bArnaldo Carvalho de Melo}
8085f3ea37c7716db4e894a480e0c18b24399595b6bArnaldo Carvalho de Melo
80938516ab59fbc5b3bb278cf5e1fe2867c70cff32eSteven Rostedtstatic void blk_add_trace_bio_queue(void *ignore,
81038516ab59fbc5b3bb278cf5e1fe2867c70cff32eSteven Rostedt				    struct request_queue *q, struct bio *bio)
8115f3ea37c7716db4e894a480e0c18b24399595b6bArnaldo Carvalho de Melo{
812797a455d2c682476c3797dbfecf5bf84c1e3b9d3Jens Axboe	blk_add_trace_bio(q, bio, BLK_TA_QUEUE, 0);
8135f3ea37c7716db4e894a480e0c18b24399595b6bArnaldo Carvalho de Melo}
8145f3ea37c7716db4e894a480e0c18b24399595b6bArnaldo Carvalho de Melo
81538516ab59fbc5b3bb278cf5e1fe2867c70cff32eSteven Rostedtstatic void blk_add_trace_getrq(void *ignore,
81638516ab59fbc5b3bb278cf5e1fe2867c70cff32eSteven Rostedt				struct request_queue *q,
817939b366977d29b5c0d53d1ea3b0b8cefb1e76202Arnaldo Carvalho de Melo				struct bio *bio, int rw)
8185f3ea37c7716db4e894a480e0c18b24399595b6bArnaldo Carvalho de Melo{
8195f3ea37c7716db4e894a480e0c18b24399595b6bArnaldo Carvalho de Melo	if (bio)
820797a455d2c682476c3797dbfecf5bf84c1e3b9d3Jens Axboe		blk_add_trace_bio(q, bio, BLK_TA_GETRQ, 0);
8215f3ea37c7716db4e894a480e0c18b24399595b6bArnaldo Carvalho de Melo	else {
8225f3ea37c7716db4e894a480e0c18b24399595b6bArnaldo Carvalho de Melo		struct blk_trace *bt = q->blk_trace;
8235f3ea37c7716db4e894a480e0c18b24399595b6bArnaldo Carvalho de Melo
8245f3ea37c7716db4e894a480e0c18b24399595b6bArnaldo Carvalho de Melo		if (bt)
8255f3ea37c7716db4e894a480e0c18b24399595b6bArnaldo Carvalho de Melo			__blk_add_trace(bt, 0, 0, rw, BLK_TA_GETRQ, 0, 0, NULL);
8265f3ea37c7716db4e894a480e0c18b24399595b6bArnaldo Carvalho de Melo	}
8275f3ea37c7716db4e894a480e0c18b24399595b6bArnaldo Carvalho de Melo}
8285f3ea37c7716db4e894a480e0c18b24399595b6bArnaldo Carvalho de Melo
8295f3ea37c7716db4e894a480e0c18b24399595b6bArnaldo Carvalho de Melo
83038516ab59fbc5b3bb278cf5e1fe2867c70cff32eSteven Rostedtstatic void blk_add_trace_sleeprq(void *ignore,
83138516ab59fbc5b3bb278cf5e1fe2867c70cff32eSteven Rostedt				  struct request_queue *q,
832939b366977d29b5c0d53d1ea3b0b8cefb1e76202Arnaldo Carvalho de Melo				  struct bio *bio, int rw)
8335f3ea37c7716db4e894a480e0c18b24399595b6bArnaldo Carvalho de Melo{
8345f3ea37c7716db4e894a480e0c18b24399595b6bArnaldo Carvalho de Melo	if (bio)
835797a455d2c682476c3797dbfecf5bf84c1e3b9d3Jens Axboe		blk_add_trace_bio(q, bio, BLK_TA_SLEEPRQ, 0);
8365f3ea37c7716db4e894a480e0c18b24399595b6bArnaldo Carvalho de Melo	else {
8375f3ea37c7716db4e894a480e0c18b24399595b6bArnaldo Carvalho de Melo		struct blk_trace *bt = q->blk_trace;
8385f3ea37c7716db4e894a480e0c18b24399595b6bArnaldo Carvalho de Melo
8395f3ea37c7716db4e894a480e0c18b24399595b6bArnaldo Carvalho de Melo		if (bt)
840939b366977d29b5c0d53d1ea3b0b8cefb1e76202Arnaldo Carvalho de Melo			__blk_add_trace(bt, 0, 0, rw, BLK_TA_SLEEPRQ,
841939b366977d29b5c0d53d1ea3b0b8cefb1e76202Arnaldo Carvalho de Melo					0, 0, NULL);
8425f3ea37c7716db4e894a480e0c18b24399595b6bArnaldo Carvalho de Melo	}
8435f3ea37c7716db4e894a480e0c18b24399595b6bArnaldo Carvalho de Melo}
8445f3ea37c7716db4e894a480e0c18b24399595b6bArnaldo Carvalho de Melo
84538516ab59fbc5b3bb278cf5e1fe2867c70cff32eSteven Rostedtstatic void blk_add_trace_plug(void *ignore, struct request_queue *q)
8465f3ea37c7716db4e894a480e0c18b24399595b6bArnaldo Carvalho de Melo{
8475f3ea37c7716db4e894a480e0c18b24399595b6bArnaldo Carvalho de Melo	struct blk_trace *bt = q->blk_trace;
8485f3ea37c7716db4e894a480e0c18b24399595b6bArnaldo Carvalho de Melo
8495f3ea37c7716db4e894a480e0c18b24399595b6bArnaldo Carvalho de Melo	if (bt)
8505f3ea37c7716db4e894a480e0c18b24399595b6bArnaldo Carvalho de Melo		__blk_add_trace(bt, 0, 0, 0, BLK_TA_PLUG, 0, 0, NULL);
8515f3ea37c7716db4e894a480e0c18b24399595b6bArnaldo Carvalho de Melo}
8525f3ea37c7716db4e894a480e0c18b24399595b6bArnaldo Carvalho de Melo
85394b5eb28b41cc79d9713696e0005ae167b5afd1bJens Axboestatic void blk_add_trace_unplug_io(void *ignore, struct request_queue *q,
85494b5eb28b41cc79d9713696e0005ae167b5afd1bJens Axboe				    unsigned int depth)
8555f3ea37c7716db4e894a480e0c18b24399595b6bArnaldo Carvalho de Melo{
8565f3ea37c7716db4e894a480e0c18b24399595b6bArnaldo Carvalho de Melo	struct blk_trace *bt = q->blk_trace;
8575f3ea37c7716db4e894a480e0c18b24399595b6bArnaldo Carvalho de Melo
8585f3ea37c7716db4e894a480e0c18b24399595b6bArnaldo Carvalho de Melo	if (bt) {
85994b5eb28b41cc79d9713696e0005ae167b5afd1bJens Axboe		__be64 rpdu = cpu_to_be64(depth);
8605f3ea37c7716db4e894a480e0c18b24399595b6bArnaldo Carvalho de Melo
8615f3ea37c7716db4e894a480e0c18b24399595b6bArnaldo Carvalho de Melo		__blk_add_trace(bt, 0, 0, 0, BLK_TA_UNPLUG_IO, 0,
8625f3ea37c7716db4e894a480e0c18b24399595b6bArnaldo Carvalho de Melo				sizeof(rpdu), &rpdu);
8635f3ea37c7716db4e894a480e0c18b24399595b6bArnaldo Carvalho de Melo	}
8645f3ea37c7716db4e894a480e0c18b24399595b6bArnaldo Carvalho de Melo}
8655f3ea37c7716db4e894a480e0c18b24399595b6bArnaldo Carvalho de Melo
86638516ab59fbc5b3bb278cf5e1fe2867c70cff32eSteven Rostedtstatic void blk_add_trace_split(void *ignore,
86738516ab59fbc5b3bb278cf5e1fe2867c70cff32eSteven Rostedt				struct request_queue *q, struct bio *bio,
8685f3ea37c7716db4e894a480e0c18b24399595b6bArnaldo Carvalho de Melo				unsigned int pdu)
8695f3ea37c7716db4e894a480e0c18b24399595b6bArnaldo Carvalho de Melo{
8705f3ea37c7716db4e894a480e0c18b24399595b6bArnaldo Carvalho de Melo	struct blk_trace *bt = q->blk_trace;
8715f3ea37c7716db4e894a480e0c18b24399595b6bArnaldo Carvalho de Melo
8725f3ea37c7716db4e894a480e0c18b24399595b6bArnaldo Carvalho de Melo	if (bt) {
8735f3ea37c7716db4e894a480e0c18b24399595b6bArnaldo Carvalho de Melo		__be64 rpdu = cpu_to_be64(pdu);
8745f3ea37c7716db4e894a480e0c18b24399595b6bArnaldo Carvalho de Melo
8755f3ea37c7716db4e894a480e0c18b24399595b6bArnaldo Carvalho de Melo		__blk_add_trace(bt, bio->bi_sector, bio->bi_size, bio->bi_rw,
8765f3ea37c7716db4e894a480e0c18b24399595b6bArnaldo Carvalho de Melo				BLK_TA_SPLIT, !bio_flagged(bio, BIO_UPTODATE),
8775f3ea37c7716db4e894a480e0c18b24399595b6bArnaldo Carvalho de Melo				sizeof(rpdu), &rpdu);
8785f3ea37c7716db4e894a480e0c18b24399595b6bArnaldo Carvalho de Melo	}
8795f3ea37c7716db4e894a480e0c18b24399595b6bArnaldo Carvalho de Melo}
8805f3ea37c7716db4e894a480e0c18b24399595b6bArnaldo Carvalho de Melo
8815f3ea37c7716db4e894a480e0c18b24399595b6bArnaldo Carvalho de Melo/**
882d07335e51df0c6dec202d315fc4f1f7e100eec4eMike Snitzer * blk_add_trace_bio_remap - Add a trace for a bio-remap operation
883546cf44a1b507c1cbb5cf42bbe6169780567f36fRandy Dunlap * @ignore:	trace callback data parameter (not used)
8845f3ea37c7716db4e894a480e0c18b24399595b6bArnaldo Carvalho de Melo * @q:		queue the io is for
8855f3ea37c7716db4e894a480e0c18b24399595b6bArnaldo Carvalho de Melo * @bio:	the source bio
8865f3ea37c7716db4e894a480e0c18b24399595b6bArnaldo Carvalho de Melo * @dev:	target device
887a42aaa3bbce85ac487ad4fad5db99e8e91b7aac1Alan D. Brunelle * @from:	source sector
8885f3ea37c7716db4e894a480e0c18b24399595b6bArnaldo Carvalho de Melo *
8895f3ea37c7716db4e894a480e0c18b24399595b6bArnaldo Carvalho de Melo * Description:
8905f3ea37c7716db4e894a480e0c18b24399595b6bArnaldo Carvalho de Melo *     Device mapper or raid target sometimes need to split a bio because
8915f3ea37c7716db4e894a480e0c18b24399595b6bArnaldo Carvalho de Melo *     it spans a stripe (or similar). Add a trace for that action.
8925f3ea37c7716db4e894a480e0c18b24399595b6bArnaldo Carvalho de Melo *
8935f3ea37c7716db4e894a480e0c18b24399595b6bArnaldo Carvalho de Melo **/
894d07335e51df0c6dec202d315fc4f1f7e100eec4eMike Snitzerstatic void blk_add_trace_bio_remap(void *ignore,
895d07335e51df0c6dec202d315fc4f1f7e100eec4eMike Snitzer				    struct request_queue *q, struct bio *bio,
896d07335e51df0c6dec202d315fc4f1f7e100eec4eMike Snitzer				    dev_t dev, sector_t from)
8975f3ea37c7716db4e894a480e0c18b24399595b6bArnaldo Carvalho de Melo{
8985f3ea37c7716db4e894a480e0c18b24399595b6bArnaldo Carvalho de Melo	struct blk_trace *bt = q->blk_trace;
8995f3ea37c7716db4e894a480e0c18b24399595b6bArnaldo Carvalho de Melo	struct blk_io_trace_remap r;
9005f3ea37c7716db4e894a480e0c18b24399595b6bArnaldo Carvalho de Melo
9015f3ea37c7716db4e894a480e0c18b24399595b6bArnaldo Carvalho de Melo	if (likely(!bt))
9025f3ea37c7716db4e894a480e0c18b24399595b6bArnaldo Carvalho de Melo		return;
9035f3ea37c7716db4e894a480e0c18b24399595b6bArnaldo Carvalho de Melo
904a42aaa3bbce85ac487ad4fad5db99e8e91b7aac1Alan D. Brunelle	r.device_from = cpu_to_be32(dev);
905a42aaa3bbce85ac487ad4fad5db99e8e91b7aac1Alan D. Brunelle	r.device_to   = cpu_to_be32(bio->bi_bdev->bd_dev);
906a42aaa3bbce85ac487ad4fad5db99e8e91b7aac1Alan D. Brunelle	r.sector_from = cpu_to_be64(from);
9075f3ea37c7716db4e894a480e0c18b24399595b6bArnaldo Carvalho de Melo
90822a7c31a9659deaddafbbcec6562d44141e84474Alan D. Brunelle	__blk_add_trace(bt, bio->bi_sector, bio->bi_size, bio->bi_rw,
90922a7c31a9659deaddafbbcec6562d44141e84474Alan D. Brunelle			BLK_TA_REMAP, !bio_flagged(bio, BIO_UPTODATE),
91022a7c31a9659deaddafbbcec6562d44141e84474Alan D. Brunelle			sizeof(r), &r);
9115f3ea37c7716db4e894a480e0c18b24399595b6bArnaldo Carvalho de Melo}
9125f3ea37c7716db4e894a480e0c18b24399595b6bArnaldo Carvalho de Melo
9135f3ea37c7716db4e894a480e0c18b24399595b6bArnaldo Carvalho de Melo/**
914b0da3f0dada78832c9da03ad2152ae76bd9a2496Jun'ichi Nomura * blk_add_trace_rq_remap - Add a trace for a request-remap operation
915546cf44a1b507c1cbb5cf42bbe6169780567f36fRandy Dunlap * @ignore:	trace callback data parameter (not used)
916b0da3f0dada78832c9da03ad2152ae76bd9a2496Jun'ichi Nomura * @q:		queue the io is for
917b0da3f0dada78832c9da03ad2152ae76bd9a2496Jun'ichi Nomura * @rq:		the source request
918b0da3f0dada78832c9da03ad2152ae76bd9a2496Jun'ichi Nomura * @dev:	target device
919b0da3f0dada78832c9da03ad2152ae76bd9a2496Jun'ichi Nomura * @from:	source sector
920b0da3f0dada78832c9da03ad2152ae76bd9a2496Jun'ichi Nomura *
921b0da3f0dada78832c9da03ad2152ae76bd9a2496Jun'ichi Nomura * Description:
922b0da3f0dada78832c9da03ad2152ae76bd9a2496Jun'ichi Nomura *     Device mapper remaps request to other devices.
923b0da3f0dada78832c9da03ad2152ae76bd9a2496Jun'ichi Nomura *     Add a trace for that action.
924b0da3f0dada78832c9da03ad2152ae76bd9a2496Jun'ichi Nomura *
925b0da3f0dada78832c9da03ad2152ae76bd9a2496Jun'ichi Nomura **/
92638516ab59fbc5b3bb278cf5e1fe2867c70cff32eSteven Rostedtstatic void blk_add_trace_rq_remap(void *ignore,
92738516ab59fbc5b3bb278cf5e1fe2867c70cff32eSteven Rostedt				   struct request_queue *q,
928b0da3f0dada78832c9da03ad2152ae76bd9a2496Jun'ichi Nomura				   struct request *rq, dev_t dev,
929b0da3f0dada78832c9da03ad2152ae76bd9a2496Jun'ichi Nomura				   sector_t from)
930b0da3f0dada78832c9da03ad2152ae76bd9a2496Jun'ichi Nomura{
931b0da3f0dada78832c9da03ad2152ae76bd9a2496Jun'ichi Nomura	struct blk_trace *bt = q->blk_trace;
932b0da3f0dada78832c9da03ad2152ae76bd9a2496Jun'ichi Nomura	struct blk_io_trace_remap r;
933b0da3f0dada78832c9da03ad2152ae76bd9a2496Jun'ichi Nomura
934b0da3f0dada78832c9da03ad2152ae76bd9a2496Jun'ichi Nomura	if (likely(!bt))
935b0da3f0dada78832c9da03ad2152ae76bd9a2496Jun'ichi Nomura		return;
936b0da3f0dada78832c9da03ad2152ae76bd9a2496Jun'ichi Nomura
937b0da3f0dada78832c9da03ad2152ae76bd9a2496Jun'ichi Nomura	r.device_from = cpu_to_be32(dev);
938b0da3f0dada78832c9da03ad2152ae76bd9a2496Jun'ichi Nomura	r.device_to   = cpu_to_be32(disk_devt(rq->rq_disk));
939b0da3f0dada78832c9da03ad2152ae76bd9a2496Jun'ichi Nomura	r.sector_from = cpu_to_be64(from);
940b0da3f0dada78832c9da03ad2152ae76bd9a2496Jun'ichi Nomura
941b0da3f0dada78832c9da03ad2152ae76bd9a2496Jun'ichi Nomura	__blk_add_trace(bt, blk_rq_pos(rq), blk_rq_bytes(rq),
942b0da3f0dada78832c9da03ad2152ae76bd9a2496Jun'ichi Nomura			rq_data_dir(rq), BLK_TA_REMAP, !!rq->errors,
943b0da3f0dada78832c9da03ad2152ae76bd9a2496Jun'ichi Nomura			sizeof(r), &r);
944b0da3f0dada78832c9da03ad2152ae76bd9a2496Jun'ichi Nomura}
945b0da3f0dada78832c9da03ad2152ae76bd9a2496Jun'ichi Nomura
946b0da3f0dada78832c9da03ad2152ae76bd9a2496Jun'ichi Nomura/**
9475f3ea37c7716db4e894a480e0c18b24399595b6bArnaldo Carvalho de Melo * blk_add_driver_data - Add binary message with driver-specific data
9485f3ea37c7716db4e894a480e0c18b24399595b6bArnaldo Carvalho de Melo * @q:		queue the io is for
9495f3ea37c7716db4e894a480e0c18b24399595b6bArnaldo Carvalho de Melo * @rq:		io request
9505f3ea37c7716db4e894a480e0c18b24399595b6bArnaldo Carvalho de Melo * @data:	driver-specific data
9515f3ea37c7716db4e894a480e0c18b24399595b6bArnaldo Carvalho de Melo * @len:	length of driver-specific data
9525f3ea37c7716db4e894a480e0c18b24399595b6bArnaldo Carvalho de Melo *
9535f3ea37c7716db4e894a480e0c18b24399595b6bArnaldo Carvalho de Melo * Description:
9545f3ea37c7716db4e894a480e0c18b24399595b6bArnaldo Carvalho de Melo *     Some drivers might want to write driver-specific data per request.
9555f3ea37c7716db4e894a480e0c18b24399595b6bArnaldo Carvalho de Melo *
9565f3ea37c7716db4e894a480e0c18b24399595b6bArnaldo Carvalho de Melo **/
9575f3ea37c7716db4e894a480e0c18b24399595b6bArnaldo Carvalho de Melovoid blk_add_driver_data(struct request_queue *q,
9585f3ea37c7716db4e894a480e0c18b24399595b6bArnaldo Carvalho de Melo			 struct request *rq,
9595f3ea37c7716db4e894a480e0c18b24399595b6bArnaldo Carvalho de Melo			 void *data, size_t len)
9605f3ea37c7716db4e894a480e0c18b24399595b6bArnaldo Carvalho de Melo{
9615f3ea37c7716db4e894a480e0c18b24399595b6bArnaldo Carvalho de Melo	struct blk_trace *bt = q->blk_trace;
9625f3ea37c7716db4e894a480e0c18b24399595b6bArnaldo Carvalho de Melo
9635f3ea37c7716db4e894a480e0c18b24399595b6bArnaldo Carvalho de Melo	if (likely(!bt))
9645f3ea37c7716db4e894a480e0c18b24399595b6bArnaldo Carvalho de Melo		return;
9655f3ea37c7716db4e894a480e0c18b24399595b6bArnaldo Carvalho de Melo
96633659ebbae262228eef4e0fe990f393d1f0ed941Christoph Hellwig	if (rq->cmd_type == REQ_TYPE_BLOCK_PC)
9672e46e8b27aa57c6bd34b3102b40ee4d0144b4fabTejun Heo		__blk_add_trace(bt, 0, blk_rq_bytes(rq), 0,
9682e46e8b27aa57c6bd34b3102b40ee4d0144b4fabTejun Heo				BLK_TA_DRV_DATA, rq->errors, len, data);
9695f3ea37c7716db4e894a480e0c18b24399595b6bArnaldo Carvalho de Melo	else
9702e46e8b27aa57c6bd34b3102b40ee4d0144b4fabTejun Heo		__blk_add_trace(bt, blk_rq_pos(rq), blk_rq_bytes(rq), 0,
9712e46e8b27aa57c6bd34b3102b40ee4d0144b4fabTejun Heo				BLK_TA_DRV_DATA, rq->errors, len, data);
9725f3ea37c7716db4e894a480e0c18b24399595b6bArnaldo Carvalho de Melo}
9735f3ea37c7716db4e894a480e0c18b24399595b6bArnaldo Carvalho de MeloEXPORT_SYMBOL_GPL(blk_add_driver_data);
9745f3ea37c7716db4e894a480e0c18b24399595b6bArnaldo Carvalho de Melo
9753c289ba7c320560ee74979a8895141c829046a2dLi Zefanstatic void blk_register_tracepoints(void)
9765f3ea37c7716db4e894a480e0c18b24399595b6bArnaldo Carvalho de Melo{
9775f3ea37c7716db4e894a480e0c18b24399595b6bArnaldo Carvalho de Melo	int ret;
9785f3ea37c7716db4e894a480e0c18b24399595b6bArnaldo Carvalho de Melo
97938516ab59fbc5b3bb278cf5e1fe2867c70cff32eSteven Rostedt	ret = register_trace_block_rq_abort(blk_add_trace_rq_abort, NULL);
9805f3ea37c7716db4e894a480e0c18b24399595b6bArnaldo Carvalho de Melo	WARN_ON(ret);
98138516ab59fbc5b3bb278cf5e1fe2867c70cff32eSteven Rostedt	ret = register_trace_block_rq_insert(blk_add_trace_rq_insert, NULL);
9825f3ea37c7716db4e894a480e0c18b24399595b6bArnaldo Carvalho de Melo	WARN_ON(ret);
98338516ab59fbc5b3bb278cf5e1fe2867c70cff32eSteven Rostedt	ret = register_trace_block_rq_issue(blk_add_trace_rq_issue, NULL);
9845f3ea37c7716db4e894a480e0c18b24399595b6bArnaldo Carvalho de Melo	WARN_ON(ret);
98538516ab59fbc5b3bb278cf5e1fe2867c70cff32eSteven Rostedt	ret = register_trace_block_rq_requeue(blk_add_trace_rq_requeue, NULL);
9865f3ea37c7716db4e894a480e0c18b24399595b6bArnaldo Carvalho de Melo	WARN_ON(ret);
98738516ab59fbc5b3bb278cf5e1fe2867c70cff32eSteven Rostedt	ret = register_trace_block_rq_complete(blk_add_trace_rq_complete, NULL);
9885f3ea37c7716db4e894a480e0c18b24399595b6bArnaldo Carvalho de Melo	WARN_ON(ret);
98938516ab59fbc5b3bb278cf5e1fe2867c70cff32eSteven Rostedt	ret = register_trace_block_bio_bounce(blk_add_trace_bio_bounce, NULL);
9905f3ea37c7716db4e894a480e0c18b24399595b6bArnaldo Carvalho de Melo	WARN_ON(ret);
99138516ab59fbc5b3bb278cf5e1fe2867c70cff32eSteven Rostedt	ret = register_trace_block_bio_complete(blk_add_trace_bio_complete, NULL);
9925f3ea37c7716db4e894a480e0c18b24399595b6bArnaldo Carvalho de Melo	WARN_ON(ret);
99338516ab59fbc5b3bb278cf5e1fe2867c70cff32eSteven Rostedt	ret = register_trace_block_bio_backmerge(blk_add_trace_bio_backmerge, NULL);
9945f3ea37c7716db4e894a480e0c18b24399595b6bArnaldo Carvalho de Melo	WARN_ON(ret);
99538516ab59fbc5b3bb278cf5e1fe2867c70cff32eSteven Rostedt	ret = register_trace_block_bio_frontmerge(blk_add_trace_bio_frontmerge, NULL);
9965f3ea37c7716db4e894a480e0c18b24399595b6bArnaldo Carvalho de Melo	WARN_ON(ret);
99738516ab59fbc5b3bb278cf5e1fe2867c70cff32eSteven Rostedt	ret = register_trace_block_bio_queue(blk_add_trace_bio_queue, NULL);
9985f3ea37c7716db4e894a480e0c18b24399595b6bArnaldo Carvalho de Melo	WARN_ON(ret);
99938516ab59fbc5b3bb278cf5e1fe2867c70cff32eSteven Rostedt	ret = register_trace_block_getrq(blk_add_trace_getrq, NULL);
10005f3ea37c7716db4e894a480e0c18b24399595b6bArnaldo Carvalho de Melo	WARN_ON(ret);
100138516ab59fbc5b3bb278cf5e1fe2867c70cff32eSteven Rostedt	ret = register_trace_block_sleeprq(blk_add_trace_sleeprq, NULL);
10025f3ea37c7716db4e894a480e0c18b24399595b6bArnaldo Carvalho de Melo	WARN_ON(ret);
100338516ab59fbc5b3bb278cf5e1fe2867c70cff32eSteven Rostedt	ret = register_trace_block_plug(blk_add_trace_plug, NULL);
10045f3ea37c7716db4e894a480e0c18b24399595b6bArnaldo Carvalho de Melo	WARN_ON(ret);
100538516ab59fbc5b3bb278cf5e1fe2867c70cff32eSteven Rostedt	ret = register_trace_block_unplug_io(blk_add_trace_unplug_io, NULL);
10065f3ea37c7716db4e894a480e0c18b24399595b6bArnaldo Carvalho de Melo	WARN_ON(ret);
100738516ab59fbc5b3bb278cf5e1fe2867c70cff32eSteven Rostedt	ret = register_trace_block_split(blk_add_trace_split, NULL);
10085f3ea37c7716db4e894a480e0c18b24399595b6bArnaldo Carvalho de Melo	WARN_ON(ret);
1009d07335e51df0c6dec202d315fc4f1f7e100eec4eMike Snitzer	ret = register_trace_block_bio_remap(blk_add_trace_bio_remap, NULL);
10105f3ea37c7716db4e894a480e0c18b24399595b6bArnaldo Carvalho de Melo	WARN_ON(ret);
101138516ab59fbc5b3bb278cf5e1fe2867c70cff32eSteven Rostedt	ret = register_trace_block_rq_remap(blk_add_trace_rq_remap, NULL);
1012b0da3f0dada78832c9da03ad2152ae76bd9a2496Jun'ichi Nomura	WARN_ON(ret);
10135f3ea37c7716db4e894a480e0c18b24399595b6bArnaldo Carvalho de Melo}
10145f3ea37c7716db4e894a480e0c18b24399595b6bArnaldo Carvalho de Melo
10155f3ea37c7716db4e894a480e0c18b24399595b6bArnaldo Carvalho de Melostatic void blk_unregister_tracepoints(void)
10165f3ea37c7716db4e894a480e0c18b24399595b6bArnaldo Carvalho de Melo{
101738516ab59fbc5b3bb278cf5e1fe2867c70cff32eSteven Rostedt	unregister_trace_block_rq_remap(blk_add_trace_rq_remap, NULL);
1018d07335e51df0c6dec202d315fc4f1f7e100eec4eMike Snitzer	unregister_trace_block_bio_remap(blk_add_trace_bio_remap, NULL);
101938516ab59fbc5b3bb278cf5e1fe2867c70cff32eSteven Rostedt	unregister_trace_block_split(blk_add_trace_split, NULL);
102038516ab59fbc5b3bb278cf5e1fe2867c70cff32eSteven Rostedt	unregister_trace_block_unplug_io(blk_add_trace_unplug_io, NULL);
102138516ab59fbc5b3bb278cf5e1fe2867c70cff32eSteven Rostedt	unregister_trace_block_plug(blk_add_trace_plug, NULL);
102238516ab59fbc5b3bb278cf5e1fe2867c70cff32eSteven Rostedt	unregister_trace_block_sleeprq(blk_add_trace_sleeprq, NULL);
102338516ab59fbc5b3bb278cf5e1fe2867c70cff32eSteven Rostedt	unregister_trace_block_getrq(blk_add_trace_getrq, NULL);
102438516ab59fbc5b3bb278cf5e1fe2867c70cff32eSteven Rostedt	unregister_trace_block_bio_queue(blk_add_trace_bio_queue, NULL);
102538516ab59fbc5b3bb278cf5e1fe2867c70cff32eSteven Rostedt	unregister_trace_block_bio_frontmerge(blk_add_trace_bio_frontmerge, NULL);
102638516ab59fbc5b3bb278cf5e1fe2867c70cff32eSteven Rostedt	unregister_trace_block_bio_backmerge(blk_add_trace_bio_backmerge, NULL);
102738516ab59fbc5b3bb278cf5e1fe2867c70cff32eSteven Rostedt	unregister_trace_block_bio_complete(blk_add_trace_bio_complete, NULL);
102838516ab59fbc5b3bb278cf5e1fe2867c70cff32eSteven Rostedt	unregister_trace_block_bio_bounce(blk_add_trace_bio_bounce, NULL);
102938516ab59fbc5b3bb278cf5e1fe2867c70cff32eSteven Rostedt	unregister_trace_block_rq_complete(blk_add_trace_rq_complete, NULL);
103038516ab59fbc5b3bb278cf5e1fe2867c70cff32eSteven Rostedt	unregister_trace_block_rq_requeue(blk_add_trace_rq_requeue, NULL);
103138516ab59fbc5b3bb278cf5e1fe2867c70cff32eSteven Rostedt	unregister_trace_block_rq_issue(blk_add_trace_rq_issue, NULL);
103238516ab59fbc5b3bb278cf5e1fe2867c70cff32eSteven Rostedt	unregister_trace_block_rq_insert(blk_add_trace_rq_insert, NULL);
103338516ab59fbc5b3bb278cf5e1fe2867c70cff32eSteven Rostedt	unregister_trace_block_rq_abort(blk_add_trace_rq_abort, NULL);
10345f3ea37c7716db4e894a480e0c18b24399595b6bArnaldo Carvalho de Melo
10355f3ea37c7716db4e894a480e0c18b24399595b6bArnaldo Carvalho de Melo	tracepoint_synchronize_unregister();
10365f3ea37c7716db4e894a480e0c18b24399595b6bArnaldo Carvalho de Melo}
1037c71a896154119f4ca9e89d6078f5f63ad60ef199Arnaldo Carvalho de Melo
1038c71a896154119f4ca9e89d6078f5f63ad60ef199Arnaldo Carvalho de Melo/*
1039c71a896154119f4ca9e89d6078f5f63ad60ef199Arnaldo Carvalho de Melo * struct blk_io_tracer formatting routines
1040c71a896154119f4ca9e89d6078f5f63ad60ef199Arnaldo Carvalho de Melo */
1041c71a896154119f4ca9e89d6078f5f63ad60ef199Arnaldo Carvalho de Melo
1042c71a896154119f4ca9e89d6078f5f63ad60ef199Arnaldo Carvalho de Melostatic void fill_rwbs(char *rwbs, const struct blk_io_trace *t)
1043c71a896154119f4ca9e89d6078f5f63ad60ef199Arnaldo Carvalho de Melo{
1044157f9c00e88529ed84bd7d581a40d411e5414cf0Arnaldo Carvalho de Melo	int i = 0;
104565796348e09880e12b97267d39b8857c758440a6Li Zefan	int tc = t->action >> BLK_TC_SHIFT;
1046157f9c00e88529ed84bd7d581a40d411e5414cf0Arnaldo Carvalho de Melo
104718cea4591a98817697017bcb056a848bae1205dfLi Zefan	if (t->action == BLK_TN_MESSAGE) {
104818cea4591a98817697017bcb056a848bae1205dfLi Zefan		rwbs[i++] = 'N';
104918cea4591a98817697017bcb056a848bae1205dfLi Zefan		goto out;
105018cea4591a98817697017bcb056a848bae1205dfLi Zefan	}
105118cea4591a98817697017bcb056a848bae1205dfLi Zefan
105265796348e09880e12b97267d39b8857c758440a6Li Zefan	if (tc & BLK_TC_DISCARD)
1053157f9c00e88529ed84bd7d581a40d411e5414cf0Arnaldo Carvalho de Melo		rwbs[i++] = 'D';
105465796348e09880e12b97267d39b8857c758440a6Li Zefan	else if (tc & BLK_TC_WRITE)
1055157f9c00e88529ed84bd7d581a40d411e5414cf0Arnaldo Carvalho de Melo		rwbs[i++] = 'W';
1056157f9c00e88529ed84bd7d581a40d411e5414cf0Arnaldo Carvalho de Melo	else if (t->bytes)
1057157f9c00e88529ed84bd7d581a40d411e5414cf0Arnaldo Carvalho de Melo		rwbs[i++] = 'R';
1058157f9c00e88529ed84bd7d581a40d411e5414cf0Arnaldo Carvalho de Melo	else
1059157f9c00e88529ed84bd7d581a40d411e5414cf0Arnaldo Carvalho de Melo		rwbs[i++] = 'N';
1060157f9c00e88529ed84bd7d581a40d411e5414cf0Arnaldo Carvalho de Melo
106165796348e09880e12b97267d39b8857c758440a6Li Zefan	if (tc & BLK_TC_AHEAD)
1062157f9c00e88529ed84bd7d581a40d411e5414cf0Arnaldo Carvalho de Melo		rwbs[i++] = 'A';
106365796348e09880e12b97267d39b8857c758440a6Li Zefan	if (tc & BLK_TC_BARRIER)
1064157f9c00e88529ed84bd7d581a40d411e5414cf0Arnaldo Carvalho de Melo		rwbs[i++] = 'B';
106565796348e09880e12b97267d39b8857c758440a6Li Zefan	if (tc & BLK_TC_SYNC)
1066157f9c00e88529ed84bd7d581a40d411e5414cf0Arnaldo Carvalho de Melo		rwbs[i++] = 'S';
106765796348e09880e12b97267d39b8857c758440a6Li Zefan	if (tc & BLK_TC_META)
1068157f9c00e88529ed84bd7d581a40d411e5414cf0Arnaldo Carvalho de Melo		rwbs[i++] = 'M';
106918cea4591a98817697017bcb056a848bae1205dfLi Zefanout:
1070157f9c00e88529ed84bd7d581a40d411e5414cf0Arnaldo Carvalho de Melo	rwbs[i] = '\0';
1071c71a896154119f4ca9e89d6078f5f63ad60ef199Arnaldo Carvalho de Melo}
1072c71a896154119f4ca9e89d6078f5f63ad60ef199Arnaldo Carvalho de Melo
1073c71a896154119f4ca9e89d6078f5f63ad60ef199Arnaldo Carvalho de Melostatic inline
1074c71a896154119f4ca9e89d6078f5f63ad60ef199Arnaldo Carvalho de Meloconst struct blk_io_trace *te_blk_io_trace(const struct trace_entry *ent)
1075c71a896154119f4ca9e89d6078f5f63ad60ef199Arnaldo Carvalho de Melo{
1076c71a896154119f4ca9e89d6078f5f63ad60ef199Arnaldo Carvalho de Melo	return (const struct blk_io_trace *)ent;
1077c71a896154119f4ca9e89d6078f5f63ad60ef199Arnaldo Carvalho de Melo}
1078c71a896154119f4ca9e89d6078f5f63ad60ef199Arnaldo Carvalho de Melo
1079c71a896154119f4ca9e89d6078f5f63ad60ef199Arnaldo Carvalho de Melostatic inline const void *pdu_start(const struct trace_entry *ent)
1080c71a896154119f4ca9e89d6078f5f63ad60ef199Arnaldo Carvalho de Melo{
1081c71a896154119f4ca9e89d6078f5f63ad60ef199Arnaldo Carvalho de Melo	return te_blk_io_trace(ent) + 1;
1082c71a896154119f4ca9e89d6078f5f63ad60ef199Arnaldo Carvalho de Melo}
1083c71a896154119f4ca9e89d6078f5f63ad60ef199Arnaldo Carvalho de Melo
108466de7792c02693b49671afe58c771fde3b092fc7Li Zefanstatic inline u32 t_action(const struct trace_entry *ent)
108566de7792c02693b49671afe58c771fde3b092fc7Li Zefan{
108666de7792c02693b49671afe58c771fde3b092fc7Li Zefan	return te_blk_io_trace(ent)->action;
108766de7792c02693b49671afe58c771fde3b092fc7Li Zefan}
108866de7792c02693b49671afe58c771fde3b092fc7Li Zefan
108966de7792c02693b49671afe58c771fde3b092fc7Li Zefanstatic inline u32 t_bytes(const struct trace_entry *ent)
109066de7792c02693b49671afe58c771fde3b092fc7Li Zefan{
109166de7792c02693b49671afe58c771fde3b092fc7Li Zefan	return te_blk_io_trace(ent)->bytes;
109266de7792c02693b49671afe58c771fde3b092fc7Li Zefan}
109366de7792c02693b49671afe58c771fde3b092fc7Li Zefan
1094c71a896154119f4ca9e89d6078f5f63ad60ef199Arnaldo Carvalho de Melostatic inline u32 t_sec(const struct trace_entry *ent)
1095c71a896154119f4ca9e89d6078f5f63ad60ef199Arnaldo Carvalho de Melo{
1096c71a896154119f4ca9e89d6078f5f63ad60ef199Arnaldo Carvalho de Melo	return te_blk_io_trace(ent)->bytes >> 9;
1097c71a896154119f4ca9e89d6078f5f63ad60ef199Arnaldo Carvalho de Melo}
1098c71a896154119f4ca9e89d6078f5f63ad60ef199Arnaldo Carvalho de Melo
1099c71a896154119f4ca9e89d6078f5f63ad60ef199Arnaldo Carvalho de Melostatic inline unsigned long long t_sector(const struct trace_entry *ent)
1100c71a896154119f4ca9e89d6078f5f63ad60ef199Arnaldo Carvalho de Melo{
1101c71a896154119f4ca9e89d6078f5f63ad60ef199Arnaldo Carvalho de Melo	return te_blk_io_trace(ent)->sector;
1102c71a896154119f4ca9e89d6078f5f63ad60ef199Arnaldo Carvalho de Melo}
1103c71a896154119f4ca9e89d6078f5f63ad60ef199Arnaldo Carvalho de Melo
1104c71a896154119f4ca9e89d6078f5f63ad60ef199Arnaldo Carvalho de Melostatic inline __u16 t_error(const struct trace_entry *ent)
1105c71a896154119f4ca9e89d6078f5f63ad60ef199Arnaldo Carvalho de Melo{
1106e0dc81bec0927fa0c8aabc521793161909eef7a5Li Zefan	return te_blk_io_trace(ent)->error;
1107c71a896154119f4ca9e89d6078f5f63ad60ef199Arnaldo Carvalho de Melo}
1108c71a896154119f4ca9e89d6078f5f63ad60ef199Arnaldo Carvalho de Melo
1109c71a896154119f4ca9e89d6078f5f63ad60ef199Arnaldo Carvalho de Melostatic __u64 get_pdu_int(const struct trace_entry *ent)
1110c71a896154119f4ca9e89d6078f5f63ad60ef199Arnaldo Carvalho de Melo{
1111c71a896154119f4ca9e89d6078f5f63ad60ef199Arnaldo Carvalho de Melo	const __u64 *val = pdu_start(ent);
1112c71a896154119f4ca9e89d6078f5f63ad60ef199Arnaldo Carvalho de Melo	return be64_to_cpu(*val);
1113c71a896154119f4ca9e89d6078f5f63ad60ef199Arnaldo Carvalho de Melo}
1114c71a896154119f4ca9e89d6078f5f63ad60ef199Arnaldo Carvalho de Melo
1115c71a896154119f4ca9e89d6078f5f63ad60ef199Arnaldo Carvalho de Melostatic void get_pdu_remap(const struct trace_entry *ent,
1116c71a896154119f4ca9e89d6078f5f63ad60ef199Arnaldo Carvalho de Melo			  struct blk_io_trace_remap *r)
1117c71a896154119f4ca9e89d6078f5f63ad60ef199Arnaldo Carvalho de Melo{
1118c71a896154119f4ca9e89d6078f5f63ad60ef199Arnaldo Carvalho de Melo	const struct blk_io_trace_remap *__r = pdu_start(ent);
1119a42aaa3bbce85ac487ad4fad5db99e8e91b7aac1Alan D. Brunelle	__u64 sector_from = __r->sector_from;
1120c71a896154119f4ca9e89d6078f5f63ad60ef199Arnaldo Carvalho de Melo
1121c71a896154119f4ca9e89d6078f5f63ad60ef199Arnaldo Carvalho de Melo	r->device_from = be32_to_cpu(__r->device_from);
1122a42aaa3bbce85ac487ad4fad5db99e8e91b7aac1Alan D. Brunelle	r->device_to   = be32_to_cpu(__r->device_to);
1123a42aaa3bbce85ac487ad4fad5db99e8e91b7aac1Alan D. Brunelle	r->sector_from = be64_to_cpu(sector_from);
1124c71a896154119f4ca9e89d6078f5f63ad60ef199Arnaldo Carvalho de Melo}
1125c71a896154119f4ca9e89d6078f5f63ad60ef199Arnaldo Carvalho de Melo
1126b6a4b0c3ad4c09c1d37b1040ac8e3ebd1016e10bLi Zefantypedef int (blk_log_action_t) (struct trace_iterator *iter, const char *act);
1127b6a4b0c3ad4c09c1d37b1040ac8e3ebd1016e10bLi Zefan
1128b6a4b0c3ad4c09c1d37b1040ac8e3ebd1016e10bLi Zefanstatic int blk_log_action_classic(struct trace_iterator *iter, const char *act)
1129c71a896154119f4ca9e89d6078f5f63ad60ef199Arnaldo Carvalho de Melo{
1130c71a896154119f4ca9e89d6078f5f63ad60ef199Arnaldo Carvalho de Melo	char rwbs[6];
113135ac51bfe4c293b67ce9f85082ba0b9bc6123c40Li Zefan	unsigned long long ts  = iter->ts;
113235ac51bfe4c293b67ce9f85082ba0b9bc6123c40Li Zefan	unsigned long nsec_rem = do_div(ts, NSEC_PER_SEC);
1133c71a896154119f4ca9e89d6078f5f63ad60ef199Arnaldo Carvalho de Melo	unsigned secs	       = (unsigned long)ts;
1134b6a4b0c3ad4c09c1d37b1040ac8e3ebd1016e10bLi Zefan	const struct blk_io_trace *t = te_blk_io_trace(iter->ent);
1135c71a896154119f4ca9e89d6078f5f63ad60ef199Arnaldo Carvalho de Melo
1136c71a896154119f4ca9e89d6078f5f63ad60ef199Arnaldo Carvalho de Melo	fill_rwbs(rwbs, t);
1137c71a896154119f4ca9e89d6078f5f63ad60ef199Arnaldo Carvalho de Melo
1138c71a896154119f4ca9e89d6078f5f63ad60ef199Arnaldo Carvalho de Melo	return trace_seq_printf(&iter->seq,
113935ac51bfe4c293b67ce9f85082ba0b9bc6123c40Li Zefan				"%3d,%-3d %2d %5d.%09lu %5u %2s %3s ",
1140c71a896154119f4ca9e89d6078f5f63ad60ef199Arnaldo Carvalho de Melo				MAJOR(t->device), MINOR(t->device), iter->cpu,
1141b6a4b0c3ad4c09c1d37b1040ac8e3ebd1016e10bLi Zefan				secs, nsec_rem, iter->ent->pid, act, rwbs);
1142c71a896154119f4ca9e89d6078f5f63ad60ef199Arnaldo Carvalho de Melo}
1143c71a896154119f4ca9e89d6078f5f63ad60ef199Arnaldo Carvalho de Melo
1144b6a4b0c3ad4c09c1d37b1040ac8e3ebd1016e10bLi Zefanstatic int blk_log_action(struct trace_iterator *iter, const char *act)
1145c71a896154119f4ca9e89d6078f5f63ad60ef199Arnaldo Carvalho de Melo{
1146c71a896154119f4ca9e89d6078f5f63ad60ef199Arnaldo Carvalho de Melo	char rwbs[6];
1147b6a4b0c3ad4c09c1d37b1040ac8e3ebd1016e10bLi Zefan	const struct blk_io_trace *t = te_blk_io_trace(iter->ent);
1148b6a4b0c3ad4c09c1d37b1040ac8e3ebd1016e10bLi Zefan
1149c71a896154119f4ca9e89d6078f5f63ad60ef199Arnaldo Carvalho de Melo	fill_rwbs(rwbs, t);
1150b6a4b0c3ad4c09c1d37b1040ac8e3ebd1016e10bLi Zefan	return trace_seq_printf(&iter->seq, "%3d,%-3d %2s %3s ",
1151c71a896154119f4ca9e89d6078f5f63ad60ef199Arnaldo Carvalho de Melo				MAJOR(t->device), MINOR(t->device), act, rwbs);
1152c71a896154119f4ca9e89d6078f5f63ad60ef199Arnaldo Carvalho de Melo}
1153c71a896154119f4ca9e89d6078f5f63ad60ef199Arnaldo Carvalho de Melo
115466de7792c02693b49671afe58c771fde3b092fc7Li Zefanstatic int blk_log_dump_pdu(struct trace_seq *s, const struct trace_entry *ent)
115566de7792c02693b49671afe58c771fde3b092fc7Li Zefan{
1156049862579333cc6cd9e6edfd6987cd0addfd8c59Li Zefan	const unsigned char *pdu_buf;
115766de7792c02693b49671afe58c771fde3b092fc7Li Zefan	int pdu_len;
115866de7792c02693b49671afe58c771fde3b092fc7Li Zefan	int i, end, ret;
115966de7792c02693b49671afe58c771fde3b092fc7Li Zefan
116066de7792c02693b49671afe58c771fde3b092fc7Li Zefan	pdu_buf = pdu_start(ent);
116166de7792c02693b49671afe58c771fde3b092fc7Li Zefan	pdu_len = te_blk_io_trace(ent)->pdu_len;
116266de7792c02693b49671afe58c771fde3b092fc7Li Zefan
116366de7792c02693b49671afe58c771fde3b092fc7Li Zefan	if (!pdu_len)
116466de7792c02693b49671afe58c771fde3b092fc7Li Zefan		return 1;
116566de7792c02693b49671afe58c771fde3b092fc7Li Zefan
116666de7792c02693b49671afe58c771fde3b092fc7Li Zefan	/* find the last zero that needs to be printed */
116766de7792c02693b49671afe58c771fde3b092fc7Li Zefan	for (end = pdu_len - 1; end >= 0; end--)
116866de7792c02693b49671afe58c771fde3b092fc7Li Zefan		if (pdu_buf[end])
116966de7792c02693b49671afe58c771fde3b092fc7Li Zefan			break;
117066de7792c02693b49671afe58c771fde3b092fc7Li Zefan	end++;
117166de7792c02693b49671afe58c771fde3b092fc7Li Zefan
117266de7792c02693b49671afe58c771fde3b092fc7Li Zefan	if (!trace_seq_putc(s, '('))
117366de7792c02693b49671afe58c771fde3b092fc7Li Zefan		return 0;
117466de7792c02693b49671afe58c771fde3b092fc7Li Zefan
117566de7792c02693b49671afe58c771fde3b092fc7Li Zefan	for (i = 0; i < pdu_len; i++) {
117666de7792c02693b49671afe58c771fde3b092fc7Li Zefan
117766de7792c02693b49671afe58c771fde3b092fc7Li Zefan		ret = trace_seq_printf(s, "%s%02x",
117866de7792c02693b49671afe58c771fde3b092fc7Li Zefan				       i == 0 ? "" : " ", pdu_buf[i]);
117966de7792c02693b49671afe58c771fde3b092fc7Li Zefan		if (!ret)
118066de7792c02693b49671afe58c771fde3b092fc7Li Zefan			return ret;
118166de7792c02693b49671afe58c771fde3b092fc7Li Zefan
118266de7792c02693b49671afe58c771fde3b092fc7Li Zefan		/*
118366de7792c02693b49671afe58c771fde3b092fc7Li Zefan		 * stop when the rest is just zeroes and indicate so
118466de7792c02693b49671afe58c771fde3b092fc7Li Zefan		 * with a ".." appended
118566de7792c02693b49671afe58c771fde3b092fc7Li Zefan		 */
118666de7792c02693b49671afe58c771fde3b092fc7Li Zefan		if (i == end && end != pdu_len - 1)
118766de7792c02693b49671afe58c771fde3b092fc7Li Zefan			return trace_seq_puts(s, " ..) ");
118866de7792c02693b49671afe58c771fde3b092fc7Li Zefan	}
118966de7792c02693b49671afe58c771fde3b092fc7Li Zefan
119066de7792c02693b49671afe58c771fde3b092fc7Li Zefan	return trace_seq_puts(s, ") ");
119166de7792c02693b49671afe58c771fde3b092fc7Li Zefan}
119266de7792c02693b49671afe58c771fde3b092fc7Li Zefan
1193c71a896154119f4ca9e89d6078f5f63ad60ef199Arnaldo Carvalho de Melostatic int blk_log_generic(struct trace_seq *s, const struct trace_entry *ent)
1194c71a896154119f4ca9e89d6078f5f63ad60ef199Arnaldo Carvalho de Melo{
11954ca530852346be239b7c19e7bec5d2b78855bebeSteven Rostedt	char cmd[TASK_COMM_LEN];
11964ca530852346be239b7c19e7bec5d2b78855bebeSteven Rostedt
11974ca530852346be239b7c19e7bec5d2b78855bebeSteven Rostedt	trace_find_cmdline(ent->pid, cmd);
1198c71a896154119f4ca9e89d6078f5f63ad60ef199Arnaldo Carvalho de Melo
119966de7792c02693b49671afe58c771fde3b092fc7Li Zefan	if (t_action(ent) & BLK_TC_ACT(BLK_TC_PC)) {
120066de7792c02693b49671afe58c771fde3b092fc7Li Zefan		int ret;
120166de7792c02693b49671afe58c771fde3b092fc7Li Zefan
120266de7792c02693b49671afe58c771fde3b092fc7Li Zefan		ret = trace_seq_printf(s, "%u ", t_bytes(ent));
120366de7792c02693b49671afe58c771fde3b092fc7Li Zefan		if (!ret)
120466de7792c02693b49671afe58c771fde3b092fc7Li Zefan			return 0;
120566de7792c02693b49671afe58c771fde3b092fc7Li Zefan		ret = blk_log_dump_pdu(s, ent);
120666de7792c02693b49671afe58c771fde3b092fc7Li Zefan		if (!ret)
120766de7792c02693b49671afe58c771fde3b092fc7Li Zefan			return 0;
120866de7792c02693b49671afe58c771fde3b092fc7Li Zefan		return trace_seq_printf(s, "[%s]\n", cmd);
120966de7792c02693b49671afe58c771fde3b092fc7Li Zefan	} else {
121066de7792c02693b49671afe58c771fde3b092fc7Li Zefan		if (t_sec(ent))
121166de7792c02693b49671afe58c771fde3b092fc7Li Zefan			return trace_seq_printf(s, "%llu + %u [%s]\n",
121266de7792c02693b49671afe58c771fde3b092fc7Li Zefan						t_sector(ent), t_sec(ent), cmd);
121366de7792c02693b49671afe58c771fde3b092fc7Li Zefan		return trace_seq_printf(s, "[%s]\n", cmd);
121466de7792c02693b49671afe58c771fde3b092fc7Li Zefan	}
1215c71a896154119f4ca9e89d6078f5f63ad60ef199Arnaldo Carvalho de Melo}
1216c71a896154119f4ca9e89d6078f5f63ad60ef199Arnaldo Carvalho de Melo
1217157f9c00e88529ed84bd7d581a40d411e5414cf0Arnaldo Carvalho de Melostatic int blk_log_with_error(struct trace_seq *s,
1218157f9c00e88529ed84bd7d581a40d411e5414cf0Arnaldo Carvalho de Melo			      const struct trace_entry *ent)
1219c71a896154119f4ca9e89d6078f5f63ad60ef199Arnaldo Carvalho de Melo{
122066de7792c02693b49671afe58c771fde3b092fc7Li Zefan	if (t_action(ent) & BLK_TC_ACT(BLK_TC_PC)) {
122166de7792c02693b49671afe58c771fde3b092fc7Li Zefan		int ret;
122266de7792c02693b49671afe58c771fde3b092fc7Li Zefan
122366de7792c02693b49671afe58c771fde3b092fc7Li Zefan		ret = blk_log_dump_pdu(s, ent);
122466de7792c02693b49671afe58c771fde3b092fc7Li Zefan		if (ret)
122566de7792c02693b49671afe58c771fde3b092fc7Li Zefan			return trace_seq_printf(s, "[%d]\n", t_error(ent));
122666de7792c02693b49671afe58c771fde3b092fc7Li Zefan		return 0;
122766de7792c02693b49671afe58c771fde3b092fc7Li Zefan	} else {
122866de7792c02693b49671afe58c771fde3b092fc7Li Zefan		if (t_sec(ent))
122966de7792c02693b49671afe58c771fde3b092fc7Li Zefan			return trace_seq_printf(s, "%llu + %u [%d]\n",
123066de7792c02693b49671afe58c771fde3b092fc7Li Zefan						t_sector(ent),
123166de7792c02693b49671afe58c771fde3b092fc7Li Zefan						t_sec(ent), t_error(ent));
123266de7792c02693b49671afe58c771fde3b092fc7Li Zefan		return trace_seq_printf(s, "%llu [%d]\n",
123366de7792c02693b49671afe58c771fde3b092fc7Li Zefan					t_sector(ent), t_error(ent));
123466de7792c02693b49671afe58c771fde3b092fc7Li Zefan	}
1235c71a896154119f4ca9e89d6078f5f63ad60ef199Arnaldo Carvalho de Melo}
1236c71a896154119f4ca9e89d6078f5f63ad60ef199Arnaldo Carvalho de Melo
1237c71a896154119f4ca9e89d6078f5f63ad60ef199Arnaldo Carvalho de Melostatic int blk_log_remap(struct trace_seq *s, const struct trace_entry *ent)
1238c71a896154119f4ca9e89d6078f5f63ad60ef199Arnaldo Carvalho de Melo{
1239a42aaa3bbce85ac487ad4fad5db99e8e91b7aac1Alan D. Brunelle	struct blk_io_trace_remap r = { .device_from = 0, };
1240c71a896154119f4ca9e89d6078f5f63ad60ef199Arnaldo Carvalho de Melo
1241c71a896154119f4ca9e89d6078f5f63ad60ef199Arnaldo Carvalho de Melo	get_pdu_remap(ent, &r);
1242c71a896154119f4ca9e89d6078f5f63ad60ef199Arnaldo Carvalho de Melo	return trace_seq_printf(s, "%llu + %u <- (%d,%d) %llu\n",
1243a42aaa3bbce85ac487ad4fad5db99e8e91b7aac1Alan D. Brunelle				t_sector(ent), t_sec(ent),
1244a42aaa3bbce85ac487ad4fad5db99e8e91b7aac1Alan D. Brunelle				MAJOR(r.device_from), MINOR(r.device_from),
1245a42aaa3bbce85ac487ad4fad5db99e8e91b7aac1Alan D. Brunelle				(unsigned long long)r.sector_from);
1246c71a896154119f4ca9e89d6078f5f63ad60ef199Arnaldo Carvalho de Melo}
1247c71a896154119f4ca9e89d6078f5f63ad60ef199Arnaldo Carvalho de Melo
1248c71a896154119f4ca9e89d6078f5f63ad60ef199Arnaldo Carvalho de Melostatic int blk_log_plug(struct trace_seq *s, const struct trace_entry *ent)
1249c71a896154119f4ca9e89d6078f5f63ad60ef199Arnaldo Carvalho de Melo{
12504ca530852346be239b7c19e7bec5d2b78855bebeSteven Rostedt	char cmd[TASK_COMM_LEN];
12514ca530852346be239b7c19e7bec5d2b78855bebeSteven Rostedt
12524ca530852346be239b7c19e7bec5d2b78855bebeSteven Rostedt	trace_find_cmdline(ent->pid, cmd);
12534ca530852346be239b7c19e7bec5d2b78855bebeSteven Rostedt
12544ca530852346be239b7c19e7bec5d2b78855bebeSteven Rostedt	return trace_seq_printf(s, "[%s]\n", cmd);
1255c71a896154119f4ca9e89d6078f5f63ad60ef199Arnaldo Carvalho de Melo}
1256c71a896154119f4ca9e89d6078f5f63ad60ef199Arnaldo Carvalho de Melo
1257c71a896154119f4ca9e89d6078f5f63ad60ef199Arnaldo Carvalho de Melostatic int blk_log_unplug(struct trace_seq *s, const struct trace_entry *ent)
1258c71a896154119f4ca9e89d6078f5f63ad60ef199Arnaldo Carvalho de Melo{
12594ca530852346be239b7c19e7bec5d2b78855bebeSteven Rostedt	char cmd[TASK_COMM_LEN];
12604ca530852346be239b7c19e7bec5d2b78855bebeSteven Rostedt
12614ca530852346be239b7c19e7bec5d2b78855bebeSteven Rostedt	trace_find_cmdline(ent->pid, cmd);
12624ca530852346be239b7c19e7bec5d2b78855bebeSteven Rostedt
12634ca530852346be239b7c19e7bec5d2b78855bebeSteven Rostedt	return trace_seq_printf(s, "[%s] %llu\n", cmd, get_pdu_int(ent));
1264c71a896154119f4ca9e89d6078f5f63ad60ef199Arnaldo Carvalho de Melo}
1265c71a896154119f4ca9e89d6078f5f63ad60ef199Arnaldo Carvalho de Melo
1266c71a896154119f4ca9e89d6078f5f63ad60ef199Arnaldo Carvalho de Melostatic int blk_log_split(struct trace_seq *s, const struct trace_entry *ent)
1267c71a896154119f4ca9e89d6078f5f63ad60ef199Arnaldo Carvalho de Melo{
12684ca530852346be239b7c19e7bec5d2b78855bebeSteven Rostedt	char cmd[TASK_COMM_LEN];
12694ca530852346be239b7c19e7bec5d2b78855bebeSteven Rostedt
12704ca530852346be239b7c19e7bec5d2b78855bebeSteven Rostedt	trace_find_cmdline(ent->pid, cmd);
12714ca530852346be239b7c19e7bec5d2b78855bebeSteven Rostedt
1272c71a896154119f4ca9e89d6078f5f63ad60ef199Arnaldo Carvalho de Melo	return trace_seq_printf(s, "%llu / %llu [%s]\n", t_sector(ent),
12734ca530852346be239b7c19e7bec5d2b78855bebeSteven Rostedt				get_pdu_int(ent), cmd);
1274c71a896154119f4ca9e89d6078f5f63ad60ef199Arnaldo Carvalho de Melo}
1275c71a896154119f4ca9e89d6078f5f63ad60ef199Arnaldo Carvalho de Melo
127618cea4591a98817697017bcb056a848bae1205dfLi Zefanstatic int blk_log_msg(struct trace_seq *s, const struct trace_entry *ent)
127718cea4591a98817697017bcb056a848bae1205dfLi Zefan{
127818cea4591a98817697017bcb056a848bae1205dfLi Zefan	int ret;
127918cea4591a98817697017bcb056a848bae1205dfLi Zefan	const struct blk_io_trace *t = te_blk_io_trace(ent);
128018cea4591a98817697017bcb056a848bae1205dfLi Zefan
128118cea4591a98817697017bcb056a848bae1205dfLi Zefan	ret = trace_seq_putmem(s, t + 1, t->pdu_len);
128218cea4591a98817697017bcb056a848bae1205dfLi Zefan	if (ret)
128318cea4591a98817697017bcb056a848bae1205dfLi Zefan		return trace_seq_putc(s, '\n');
128418cea4591a98817697017bcb056a848bae1205dfLi Zefan	return ret;
128518cea4591a98817697017bcb056a848bae1205dfLi Zefan}
128618cea4591a98817697017bcb056a848bae1205dfLi Zefan
1287c71a896154119f4ca9e89d6078f5f63ad60ef199Arnaldo Carvalho de Melo/*
1288c71a896154119f4ca9e89d6078f5f63ad60ef199Arnaldo Carvalho de Melo * struct tracer operations
1289c71a896154119f4ca9e89d6078f5f63ad60ef199Arnaldo Carvalho de Melo */
1290c71a896154119f4ca9e89d6078f5f63ad60ef199Arnaldo Carvalho de Melo
1291c71a896154119f4ca9e89d6078f5f63ad60ef199Arnaldo Carvalho de Melostatic void blk_tracer_print_header(struct seq_file *m)
1292c71a896154119f4ca9e89d6078f5f63ad60ef199Arnaldo Carvalho de Melo{
1293c71a896154119f4ca9e89d6078f5f63ad60ef199Arnaldo Carvalho de Melo	if (!(blk_tracer_flags.val & TRACE_BLK_OPT_CLASSIC))
1294c71a896154119f4ca9e89d6078f5f63ad60ef199Arnaldo Carvalho de Melo		return;
1295c71a896154119f4ca9e89d6078f5f63ad60ef199Arnaldo Carvalho de Melo	seq_puts(m, "# DEV   CPU TIMESTAMP     PID ACT FLG\n"
1296c71a896154119f4ca9e89d6078f5f63ad60ef199Arnaldo Carvalho de Melo		    "#  |     |     |           |   |   |\n");
1297c71a896154119f4ca9e89d6078f5f63ad60ef199Arnaldo Carvalho de Melo}
1298c71a896154119f4ca9e89d6078f5f63ad60ef199Arnaldo Carvalho de Melo
1299c71a896154119f4ca9e89d6078f5f63ad60ef199Arnaldo Carvalho de Melostatic void blk_tracer_start(struct trace_array *tr)
1300c71a896154119f4ca9e89d6078f5f63ad60ef199Arnaldo Carvalho de Melo{
1301ad5dd5493a55e462796e42e50a49e76df76fdb05Li Zefan	blk_tracer_enabled = true;
1302c71a896154119f4ca9e89d6078f5f63ad60ef199Arnaldo Carvalho de Melo}
1303c71a896154119f4ca9e89d6078f5f63ad60ef199Arnaldo Carvalho de Melo
1304c71a896154119f4ca9e89d6078f5f63ad60ef199Arnaldo Carvalho de Melostatic int blk_tracer_init(struct trace_array *tr)
1305c71a896154119f4ca9e89d6078f5f63ad60ef199Arnaldo Carvalho de Melo{
1306c71a896154119f4ca9e89d6078f5f63ad60ef199Arnaldo Carvalho de Melo	blk_tr = tr;
1307c71a896154119f4ca9e89d6078f5f63ad60ef199Arnaldo Carvalho de Melo	blk_tracer_start(tr);
1308c71a896154119f4ca9e89d6078f5f63ad60ef199Arnaldo Carvalho de Melo	return 0;
1309c71a896154119f4ca9e89d6078f5f63ad60ef199Arnaldo Carvalho de Melo}
1310c71a896154119f4ca9e89d6078f5f63ad60ef199Arnaldo Carvalho de Melo
1311c71a896154119f4ca9e89d6078f5f63ad60ef199Arnaldo Carvalho de Melostatic void blk_tracer_stop(struct trace_array *tr)
1312c71a896154119f4ca9e89d6078f5f63ad60ef199Arnaldo Carvalho de Melo{
1313ad5dd5493a55e462796e42e50a49e76df76fdb05Li Zefan	blk_tracer_enabled = false;
1314c71a896154119f4ca9e89d6078f5f63ad60ef199Arnaldo Carvalho de Melo}
1315c71a896154119f4ca9e89d6078f5f63ad60ef199Arnaldo Carvalho de Melo
1316c71a896154119f4ca9e89d6078f5f63ad60ef199Arnaldo Carvalho de Melostatic void blk_tracer_reset(struct trace_array *tr)
1317c71a896154119f4ca9e89d6078f5f63ad60ef199Arnaldo Carvalho de Melo{
1318c71a896154119f4ca9e89d6078f5f63ad60ef199Arnaldo Carvalho de Melo	blk_tracer_stop(tr);
1319c71a896154119f4ca9e89d6078f5f63ad60ef199Arnaldo Carvalho de Melo}
1320c71a896154119f4ca9e89d6078f5f63ad60ef199Arnaldo Carvalho de Melo
1321e4955c9986a27bb47ddeb6cd55803053f547e2e9Li Zefanstatic const struct {
1322c71a896154119f4ca9e89d6078f5f63ad60ef199Arnaldo Carvalho de Melo	const char *act[2];
1323ef18012b248b47ec9a12c3a83ca5e99782d39c5dSteven Rostedt	int	   (*print)(struct trace_seq *s, const struct trace_entry *ent);
1324e4955c9986a27bb47ddeb6cd55803053f547e2e9Li Zefan} what2act[] = {
1325ef18012b248b47ec9a12c3a83ca5e99782d39c5dSteven Rostedt	[__BLK_TA_QUEUE]	= {{  "Q", "queue" },	   blk_log_generic },
1326c71a896154119f4ca9e89d6078f5f63ad60ef199Arnaldo Carvalho de Melo	[__BLK_TA_BACKMERGE]	= {{  "M", "backmerge" },  blk_log_generic },
1327c71a896154119f4ca9e89d6078f5f63ad60ef199Arnaldo Carvalho de Melo	[__BLK_TA_FRONTMERGE]	= {{  "F", "frontmerge" }, blk_log_generic },
1328c71a896154119f4ca9e89d6078f5f63ad60ef199Arnaldo Carvalho de Melo	[__BLK_TA_GETRQ]	= {{  "G", "getrq" },	   blk_log_generic },
1329c71a896154119f4ca9e89d6078f5f63ad60ef199Arnaldo Carvalho de Melo	[__BLK_TA_SLEEPRQ]	= {{  "S", "sleeprq" },	   blk_log_generic },
1330c71a896154119f4ca9e89d6078f5f63ad60ef199Arnaldo Carvalho de Melo	[__BLK_TA_REQUEUE]	= {{  "R", "requeue" },	   blk_log_with_error },
1331c71a896154119f4ca9e89d6078f5f63ad60ef199Arnaldo Carvalho de Melo	[__BLK_TA_ISSUE]	= {{  "D", "issue" },	   blk_log_generic },
1332c71a896154119f4ca9e89d6078f5f63ad60ef199Arnaldo Carvalho de Melo	[__BLK_TA_COMPLETE]	= {{  "C", "complete" },   blk_log_with_error },
1333c71a896154119f4ca9e89d6078f5f63ad60ef199Arnaldo Carvalho de Melo	[__BLK_TA_PLUG]		= {{  "P", "plug" },	   blk_log_plug },
1334c71a896154119f4ca9e89d6078f5f63ad60ef199Arnaldo Carvalho de Melo	[__BLK_TA_UNPLUG_IO]	= {{  "U", "unplug_io" },  blk_log_unplug },
1335c71a896154119f4ca9e89d6078f5f63ad60ef199Arnaldo Carvalho de Melo	[__BLK_TA_INSERT]	= {{  "I", "insert" },	   blk_log_generic },
1336c71a896154119f4ca9e89d6078f5f63ad60ef199Arnaldo Carvalho de Melo	[__BLK_TA_SPLIT]	= {{  "X", "split" },	   blk_log_split },
1337c71a896154119f4ca9e89d6078f5f63ad60ef199Arnaldo Carvalho de Melo	[__BLK_TA_BOUNCE]	= {{  "B", "bounce" },	   blk_log_generic },
1338c71a896154119f4ca9e89d6078f5f63ad60ef199Arnaldo Carvalho de Melo	[__BLK_TA_REMAP]	= {{  "A", "remap" },	   blk_log_remap },
1339c71a896154119f4ca9e89d6078f5f63ad60ef199Arnaldo Carvalho de Melo};
1340c71a896154119f4ca9e89d6078f5f63ad60ef199Arnaldo Carvalho de Melo
1341b6a4b0c3ad4c09c1d37b1040ac8e3ebd1016e10bLi Zefanstatic enum print_line_t print_one_line(struct trace_iterator *iter,
1342b6a4b0c3ad4c09c1d37b1040ac8e3ebd1016e10bLi Zefan					bool classic)
1343c71a896154119f4ca9e89d6078f5f63ad60ef199Arnaldo Carvalho de Melo{
13442c9b238eb325895d3312dad64e2685783575e474Arnaldo Carvalho de Melo	struct trace_seq *s = &iter->seq;
1345b6a4b0c3ad4c09c1d37b1040ac8e3ebd1016e10bLi Zefan	const struct blk_io_trace *t;
1346b6a4b0c3ad4c09c1d37b1040ac8e3ebd1016e10bLi Zefan	u16 what;
1347c71a896154119f4ca9e89d6078f5f63ad60ef199Arnaldo Carvalho de Melo	int ret;
1348b6a4b0c3ad4c09c1d37b1040ac8e3ebd1016e10bLi Zefan	bool long_act;
1349b6a4b0c3ad4c09c1d37b1040ac8e3ebd1016e10bLi Zefan	blk_log_action_t *log_action;
1350c71a896154119f4ca9e89d6078f5f63ad60ef199Arnaldo Carvalho de Melo
1351b6a4b0c3ad4c09c1d37b1040ac8e3ebd1016e10bLi Zefan	t	   = te_blk_io_trace(iter->ent);
1352b6a4b0c3ad4c09c1d37b1040ac8e3ebd1016e10bLi Zefan	what	   = t->action & ((1 << BLK_TC_SHIFT) - 1);
1353b6a4b0c3ad4c09c1d37b1040ac8e3ebd1016e10bLi Zefan	long_act   = !!(trace_flags & TRACE_ITER_VERBOSE);
1354b6a4b0c3ad4c09c1d37b1040ac8e3ebd1016e10bLi Zefan	log_action = classic ? &blk_log_action_classic : &blk_log_action;
135508a06b83ff8b2779289f733348c669f31cb65d51Arnaldo Carvalho de Melo
135618cea4591a98817697017bcb056a848bae1205dfLi Zefan	if (t->action == BLK_TN_MESSAGE) {
135718cea4591a98817697017bcb056a848bae1205dfLi Zefan		ret = log_action(iter, long_act ? "message" : "m");
135818cea4591a98817697017bcb056a848bae1205dfLi Zefan		if (ret)
135918cea4591a98817697017bcb056a848bae1205dfLi Zefan			ret = blk_log_msg(s, iter->ent);
136018cea4591a98817697017bcb056a848bae1205dfLi Zefan		goto out;
136118cea4591a98817697017bcb056a848bae1205dfLi Zefan	}
136218cea4591a98817697017bcb056a848bae1205dfLi Zefan
1363eb08f8eb0673d9c1e62b69ad1b41593e73c40467Li Zefan	if (unlikely(what == 0 || what >= ARRAY_SIZE(what2act)))
1364b78825d608f30a47e3154ab6872a03f0de0c9d45Li Zefan		ret = trace_seq_printf(s, "Unknown action %x\n", what);
1365c71a896154119f4ca9e89d6078f5f63ad60ef199Arnaldo Carvalho de Melo	else {
1366b6a4b0c3ad4c09c1d37b1040ac8e3ebd1016e10bLi Zefan		ret = log_action(iter, what2act[what].act[long_act]);
1367c71a896154119f4ca9e89d6078f5f63ad60ef199Arnaldo Carvalho de Melo		if (ret)
13682c9b238eb325895d3312dad64e2685783575e474Arnaldo Carvalho de Melo			ret = what2act[what].print(s, iter->ent);
1369c71a896154119f4ca9e89d6078f5f63ad60ef199Arnaldo Carvalho de Melo	}
137018cea4591a98817697017bcb056a848bae1205dfLi Zefanout:
1371c71a896154119f4ca9e89d6078f5f63ad60ef199Arnaldo Carvalho de Melo	return ret ? TRACE_TYPE_HANDLED : TRACE_TYPE_PARTIAL_LINE;
1372c71a896154119f4ca9e89d6078f5f63ad60ef199Arnaldo Carvalho de Melo}
1373c71a896154119f4ca9e89d6078f5f63ad60ef199Arnaldo Carvalho de Melo
1374b6a4b0c3ad4c09c1d37b1040ac8e3ebd1016e10bLi Zefanstatic enum print_line_t blk_trace_event_print(struct trace_iterator *iter,
1375a9a5776380208a3e48a92d0c763ee1a3b486fb73Steven Rostedt					       int flags, struct trace_event *event)
1376b6a4b0c3ad4c09c1d37b1040ac8e3ebd1016e10bLi Zefan{
1377b6a4b0c3ad4c09c1d37b1040ac8e3ebd1016e10bLi Zefan	return print_one_line(iter, false);
1378b6a4b0c3ad4c09c1d37b1040ac8e3ebd1016e10bLi Zefan}
1379b6a4b0c3ad4c09c1d37b1040ac8e3ebd1016e10bLi Zefan
138008a06b83ff8b2779289f733348c669f31cb65d51Arnaldo Carvalho de Melostatic int blk_trace_synthesize_old_trace(struct trace_iterator *iter)
138108a06b83ff8b2779289f733348c669f31cb65d51Arnaldo Carvalho de Melo{
138208a06b83ff8b2779289f733348c669f31cb65d51Arnaldo Carvalho de Melo	struct trace_seq *s = &iter->seq;
138308a06b83ff8b2779289f733348c669f31cb65d51Arnaldo Carvalho de Melo	struct blk_io_trace *t = (struct blk_io_trace *)iter->ent;
138408a06b83ff8b2779289f733348c669f31cb65d51Arnaldo Carvalho de Melo	const int offset = offsetof(struct blk_io_trace, sector);
138508a06b83ff8b2779289f733348c669f31cb65d51Arnaldo Carvalho de Melo	struct blk_io_trace old = {
138608a06b83ff8b2779289f733348c669f31cb65d51Arnaldo Carvalho de Melo		.magic	  = BLK_IO_TRACE_MAGIC | BLK_IO_TRACE_VERSION,
13876c051ce0307526adec32a847f0daa1af2124f0a9Li Zefan		.time     = iter->ts,
138808a06b83ff8b2779289f733348c669f31cb65d51Arnaldo Carvalho de Melo	};
138908a06b83ff8b2779289f733348c669f31cb65d51Arnaldo Carvalho de Melo
139008a06b83ff8b2779289f733348c669f31cb65d51Arnaldo Carvalho de Melo	if (!trace_seq_putmem(s, &old, offset))
139108a06b83ff8b2779289f733348c669f31cb65d51Arnaldo Carvalho de Melo		return 0;
139208a06b83ff8b2779289f733348c669f31cb65d51Arnaldo Carvalho de Melo	return trace_seq_putmem(s, &t->sector,
139308a06b83ff8b2779289f733348c669f31cb65d51Arnaldo Carvalho de Melo				sizeof(old) - offset + t->pdu_len);
139408a06b83ff8b2779289f733348c669f31cb65d51Arnaldo Carvalho de Melo}
139508a06b83ff8b2779289f733348c669f31cb65d51Arnaldo Carvalho de Melo
1396ae7462b4f1fe1f36b5d562dbd5202a2eba01f072Arnaldo Carvalho de Melostatic enum print_line_t
1397a9a5776380208a3e48a92d0c763ee1a3b486fb73Steven Rostedtblk_trace_event_print_binary(struct trace_iterator *iter, int flags,
1398a9a5776380208a3e48a92d0c763ee1a3b486fb73Steven Rostedt			     struct trace_event *event)
139908a06b83ff8b2779289f733348c669f31cb65d51Arnaldo Carvalho de Melo{
140008a06b83ff8b2779289f733348c669f31cb65d51Arnaldo Carvalho de Melo	return blk_trace_synthesize_old_trace(iter) ?
140108a06b83ff8b2779289f733348c669f31cb65d51Arnaldo Carvalho de Melo			TRACE_TYPE_HANDLED : TRACE_TYPE_PARTIAL_LINE;
140208a06b83ff8b2779289f733348c669f31cb65d51Arnaldo Carvalho de Melo}
140308a06b83ff8b2779289f733348c669f31cb65d51Arnaldo Carvalho de Melo
1404c71a896154119f4ca9e89d6078f5f63ad60ef199Arnaldo Carvalho de Melostatic enum print_line_t blk_tracer_print_line(struct trace_iterator *iter)
1405c71a896154119f4ca9e89d6078f5f63ad60ef199Arnaldo Carvalho de Melo{
1406c71a896154119f4ca9e89d6078f5f63ad60ef199Arnaldo Carvalho de Melo	if (!(blk_tracer_flags.val & TRACE_BLK_OPT_CLASSIC))
1407c71a896154119f4ca9e89d6078f5f63ad60ef199Arnaldo Carvalho de Melo		return TRACE_TYPE_UNHANDLED;
1408c71a896154119f4ca9e89d6078f5f63ad60ef199Arnaldo Carvalho de Melo
1409b6a4b0c3ad4c09c1d37b1040ac8e3ebd1016e10bLi Zefan	return print_one_line(iter, true);
1410c71a896154119f4ca9e89d6078f5f63ad60ef199Arnaldo Carvalho de Melo}
1411c71a896154119f4ca9e89d6078f5f63ad60ef199Arnaldo Carvalho de Melo
1412f3948f8857ef5de239f28a61dddb1554a0ae4c2cLi Zefanstatic int blk_tracer_set_flag(u32 old_flags, u32 bit, int set)
1413f3948f8857ef5de239f28a61dddb1554a0ae4c2cLi Zefan{
1414f3948f8857ef5de239f28a61dddb1554a0ae4c2cLi Zefan	/* don't output context-info for blk_classic output */
1415f3948f8857ef5de239f28a61dddb1554a0ae4c2cLi Zefan	if (bit == TRACE_BLK_OPT_CLASSIC) {
1416f3948f8857ef5de239f28a61dddb1554a0ae4c2cLi Zefan		if (set)
1417f3948f8857ef5de239f28a61dddb1554a0ae4c2cLi Zefan			trace_flags &= ~TRACE_ITER_CONTEXT_INFO;
1418f3948f8857ef5de239f28a61dddb1554a0ae4c2cLi Zefan		else
1419f3948f8857ef5de239f28a61dddb1554a0ae4c2cLi Zefan			trace_flags |= TRACE_ITER_CONTEXT_INFO;
1420f3948f8857ef5de239f28a61dddb1554a0ae4c2cLi Zefan	}
1421f3948f8857ef5de239f28a61dddb1554a0ae4c2cLi Zefan	return 0;
1422f3948f8857ef5de239f28a61dddb1554a0ae4c2cLi Zefan}
1423f3948f8857ef5de239f28a61dddb1554a0ae4c2cLi Zefan
1424c71a896154119f4ca9e89d6078f5f63ad60ef199Arnaldo Carvalho de Melostatic struct tracer blk_tracer __read_mostly = {
1425c71a896154119f4ca9e89d6078f5f63ad60ef199Arnaldo Carvalho de Melo	.name		= "blk",
1426c71a896154119f4ca9e89d6078f5f63ad60ef199Arnaldo Carvalho de Melo	.init		= blk_tracer_init,
1427c71a896154119f4ca9e89d6078f5f63ad60ef199Arnaldo Carvalho de Melo	.reset		= blk_tracer_reset,
1428c71a896154119f4ca9e89d6078f5f63ad60ef199Arnaldo Carvalho de Melo	.start		= blk_tracer_start,
1429c71a896154119f4ca9e89d6078f5f63ad60ef199Arnaldo Carvalho de Melo	.stop		= blk_tracer_stop,
1430c71a896154119f4ca9e89d6078f5f63ad60ef199Arnaldo Carvalho de Melo	.print_header	= blk_tracer_print_header,
1431c71a896154119f4ca9e89d6078f5f63ad60ef199Arnaldo Carvalho de Melo	.print_line	= blk_tracer_print_line,
1432c71a896154119f4ca9e89d6078f5f63ad60ef199Arnaldo Carvalho de Melo	.flags		= &blk_tracer_flags,
1433f3948f8857ef5de239f28a61dddb1554a0ae4c2cLi Zefan	.set_flag	= blk_tracer_set_flag,
1434c71a896154119f4ca9e89d6078f5f63ad60ef199Arnaldo Carvalho de Melo};
1435c71a896154119f4ca9e89d6078f5f63ad60ef199Arnaldo Carvalho de Melo
1436a9a5776380208a3e48a92d0c763ee1a3b486fb73Steven Rostedtstatic struct trace_event_functions trace_blk_event_funcs = {
1437c71a896154119f4ca9e89d6078f5f63ad60ef199Arnaldo Carvalho de Melo	.trace		= blk_trace_event_print,
143808a06b83ff8b2779289f733348c669f31cb65d51Arnaldo Carvalho de Melo	.binary		= blk_trace_event_print_binary,
1439c71a896154119f4ca9e89d6078f5f63ad60ef199Arnaldo Carvalho de Melo};
1440c71a896154119f4ca9e89d6078f5f63ad60ef199Arnaldo Carvalho de Melo
1441a9a5776380208a3e48a92d0c763ee1a3b486fb73Steven Rostedtstatic struct trace_event trace_blk_event = {
1442a9a5776380208a3e48a92d0c763ee1a3b486fb73Steven Rostedt	.type		= TRACE_BLK,
1443a9a5776380208a3e48a92d0c763ee1a3b486fb73Steven Rostedt	.funcs		= &trace_blk_event_funcs,
1444a9a5776380208a3e48a92d0c763ee1a3b486fb73Steven Rostedt};
1445a9a5776380208a3e48a92d0c763ee1a3b486fb73Steven Rostedt
1446c71a896154119f4ca9e89d6078f5f63ad60ef199Arnaldo Carvalho de Melostatic int __init init_blk_tracer(void)
1447c71a896154119f4ca9e89d6078f5f63ad60ef199Arnaldo Carvalho de Melo{
1448c71a896154119f4ca9e89d6078f5f63ad60ef199Arnaldo Carvalho de Melo	if (!register_ftrace_event(&trace_blk_event)) {
1449c71a896154119f4ca9e89d6078f5f63ad60ef199Arnaldo Carvalho de Melo		pr_warning("Warning: could not register block events\n");
1450c71a896154119f4ca9e89d6078f5f63ad60ef199Arnaldo Carvalho de Melo		return 1;
1451c71a896154119f4ca9e89d6078f5f63ad60ef199Arnaldo Carvalho de Melo	}
1452c71a896154119f4ca9e89d6078f5f63ad60ef199Arnaldo Carvalho de Melo
1453c71a896154119f4ca9e89d6078f5f63ad60ef199Arnaldo Carvalho de Melo	if (register_tracer(&blk_tracer) != 0) {
1454c71a896154119f4ca9e89d6078f5f63ad60ef199Arnaldo Carvalho de Melo		pr_warning("Warning: could not register the block tracer\n");
1455c71a896154119f4ca9e89d6078f5f63ad60ef199Arnaldo Carvalho de Melo		unregister_ftrace_event(&trace_blk_event);
1456c71a896154119f4ca9e89d6078f5f63ad60ef199Arnaldo Carvalho de Melo		return 1;
1457c71a896154119f4ca9e89d6078f5f63ad60ef199Arnaldo Carvalho de Melo	}
1458c71a896154119f4ca9e89d6078f5f63ad60ef199Arnaldo Carvalho de Melo
1459c71a896154119f4ca9e89d6078f5f63ad60ef199Arnaldo Carvalho de Melo	return 0;
1460c71a896154119f4ca9e89d6078f5f63ad60ef199Arnaldo Carvalho de Melo}
1461c71a896154119f4ca9e89d6078f5f63ad60ef199Arnaldo Carvalho de Melo
1462c71a896154119f4ca9e89d6078f5f63ad60ef199Arnaldo Carvalho de Melodevice_initcall(init_blk_tracer);
1463c71a896154119f4ca9e89d6078f5f63ad60ef199Arnaldo Carvalho de Melo
1464c71a896154119f4ca9e89d6078f5f63ad60ef199Arnaldo Carvalho de Melostatic int blk_trace_remove_queue(struct request_queue *q)
1465c71a896154119f4ca9e89d6078f5f63ad60ef199Arnaldo Carvalho de Melo{
1466c71a896154119f4ca9e89d6078f5f63ad60ef199Arnaldo Carvalho de Melo	struct blk_trace *bt;
1467c71a896154119f4ca9e89d6078f5f63ad60ef199Arnaldo Carvalho de Melo
1468c71a896154119f4ca9e89d6078f5f63ad60ef199Arnaldo Carvalho de Melo	bt = xchg(&q->blk_trace, NULL);
1469c71a896154119f4ca9e89d6078f5f63ad60ef199Arnaldo Carvalho de Melo	if (bt == NULL)
1470c71a896154119f4ca9e89d6078f5f63ad60ef199Arnaldo Carvalho de Melo		return -EINVAL;
1471c71a896154119f4ca9e89d6078f5f63ad60ef199Arnaldo Carvalho de Melo
147217ba97e347bec9bbc47a0877c7a098708982129dLi Zefan	if (atomic_dec_and_test(&blk_probes_ref))
147317ba97e347bec9bbc47a0877c7a098708982129dLi Zefan		blk_unregister_tracepoints();
147417ba97e347bec9bbc47a0877c7a098708982129dLi Zefan
1475ad5dd5493a55e462796e42e50a49e76df76fdb05Li Zefan	blk_trace_free(bt);
1476c71a896154119f4ca9e89d6078f5f63ad60ef199Arnaldo Carvalho de Melo	return 0;
1477c71a896154119f4ca9e89d6078f5f63ad60ef199Arnaldo Carvalho de Melo}
1478c71a896154119f4ca9e89d6078f5f63ad60ef199Arnaldo Carvalho de Melo
1479c71a896154119f4ca9e89d6078f5f63ad60ef199Arnaldo Carvalho de Melo/*
1480c71a896154119f4ca9e89d6078f5f63ad60ef199Arnaldo Carvalho de Melo * Setup everything required to start tracing
1481c71a896154119f4ca9e89d6078f5f63ad60ef199Arnaldo Carvalho de Melo */
14829908c30997b8a73c95f836170b9998dae9aa3f4aLi Zefanstatic int blk_trace_setup_queue(struct request_queue *q,
14839908c30997b8a73c95f836170b9998dae9aa3f4aLi Zefan				 struct block_device *bdev)
1484c71a896154119f4ca9e89d6078f5f63ad60ef199Arnaldo Carvalho de Melo{
1485c71a896154119f4ca9e89d6078f5f63ad60ef199Arnaldo Carvalho de Melo	struct blk_trace *old_bt, *bt = NULL;
148618cea4591a98817697017bcb056a848bae1205dfLi Zefan	int ret = -ENOMEM;
1487c71a896154119f4ca9e89d6078f5f63ad60ef199Arnaldo Carvalho de Melo
1488c71a896154119f4ca9e89d6078f5f63ad60ef199Arnaldo Carvalho de Melo	bt = kzalloc(sizeof(*bt), GFP_KERNEL);
1489c71a896154119f4ca9e89d6078f5f63ad60ef199Arnaldo Carvalho de Melo	if (!bt)
149015152e448b693fa41de40f1e40ffbe717a3aab88Li Zefan		return -ENOMEM;
1491c71a896154119f4ca9e89d6078f5f63ad60ef199Arnaldo Carvalho de Melo
149218cea4591a98817697017bcb056a848bae1205dfLi Zefan	bt->msg_data = __alloc_percpu(BLK_TN_MAX_MSG, __alignof__(char));
149318cea4591a98817697017bcb056a848bae1205dfLi Zefan	if (!bt->msg_data)
149418cea4591a98817697017bcb056a848bae1205dfLi Zefan		goto free_bt;
149518cea4591a98817697017bcb056a848bae1205dfLi Zefan
14969908c30997b8a73c95f836170b9998dae9aa3f4aLi Zefan	bt->dev = bdev->bd_dev;
1497c71a896154119f4ca9e89d6078f5f63ad60ef199Arnaldo Carvalho de Melo	bt->act_mask = (u16)-1;
14989908c30997b8a73c95f836170b9998dae9aa3f4aLi Zefan
14999908c30997b8a73c95f836170b9998dae9aa3f4aLi Zefan	blk_trace_setup_lba(bt, bdev);
1500c71a896154119f4ca9e89d6078f5f63ad60ef199Arnaldo Carvalho de Melo
1501c71a896154119f4ca9e89d6078f5f63ad60ef199Arnaldo Carvalho de Melo	old_bt = xchg(&q->blk_trace, bt);
1502c71a896154119f4ca9e89d6078f5f63ad60ef199Arnaldo Carvalho de Melo	if (old_bt != NULL) {
1503c71a896154119f4ca9e89d6078f5f63ad60ef199Arnaldo Carvalho de Melo		(void)xchg(&q->blk_trace, old_bt);
150418cea4591a98817697017bcb056a848bae1205dfLi Zefan		ret = -EBUSY;
150518cea4591a98817697017bcb056a848bae1205dfLi Zefan		goto free_bt;
1506c71a896154119f4ca9e89d6078f5f63ad60ef199Arnaldo Carvalho de Melo	}
150715152e448b693fa41de40f1e40ffbe717a3aab88Li Zefan
150817ba97e347bec9bbc47a0877c7a098708982129dLi Zefan	if (atomic_inc_return(&blk_probes_ref) == 1)
150917ba97e347bec9bbc47a0877c7a098708982129dLi Zefan		blk_register_tracepoints();
1510c71a896154119f4ca9e89d6078f5f63ad60ef199Arnaldo Carvalho de Melo	return 0;
151118cea4591a98817697017bcb056a848bae1205dfLi Zefan
151218cea4591a98817697017bcb056a848bae1205dfLi Zefanfree_bt:
151318cea4591a98817697017bcb056a848bae1205dfLi Zefan	blk_trace_free(bt);
151418cea4591a98817697017bcb056a848bae1205dfLi Zefan	return ret;
1515c71a896154119f4ca9e89d6078f5f63ad60ef199Arnaldo Carvalho de Melo}
1516c71a896154119f4ca9e89d6078f5f63ad60ef199Arnaldo Carvalho de Melo
1517c71a896154119f4ca9e89d6078f5f63ad60ef199Arnaldo Carvalho de Melo/*
1518c71a896154119f4ca9e89d6078f5f63ad60ef199Arnaldo Carvalho de Melo * sysfs interface to enable and configure tracing
1519c71a896154119f4ca9e89d6078f5f63ad60ef199Arnaldo Carvalho de Melo */
1520c71a896154119f4ca9e89d6078f5f63ad60ef199Arnaldo Carvalho de Melo
1521c71a896154119f4ca9e89d6078f5f63ad60ef199Arnaldo Carvalho de Melostatic ssize_t sysfs_blk_trace_attr_show(struct device *dev,
1522c71a896154119f4ca9e89d6078f5f63ad60ef199Arnaldo Carvalho de Melo					 struct device_attribute *attr,
1523c71a896154119f4ca9e89d6078f5f63ad60ef199Arnaldo Carvalho de Melo					 char *buf);
1524c71a896154119f4ca9e89d6078f5f63ad60ef199Arnaldo Carvalho de Melostatic ssize_t sysfs_blk_trace_attr_store(struct device *dev,
1525c71a896154119f4ca9e89d6078f5f63ad60ef199Arnaldo Carvalho de Melo					  struct device_attribute *attr,
1526c71a896154119f4ca9e89d6078f5f63ad60ef199Arnaldo Carvalho de Melo					  const char *buf, size_t count);
1527c71a896154119f4ca9e89d6078f5f63ad60ef199Arnaldo Carvalho de Melo#define BLK_TRACE_DEVICE_ATTR(_name) \
1528c71a896154119f4ca9e89d6078f5f63ad60ef199Arnaldo Carvalho de Melo	DEVICE_ATTR(_name, S_IRUGO | S_IWUSR, \
1529c71a896154119f4ca9e89d6078f5f63ad60ef199Arnaldo Carvalho de Melo		    sysfs_blk_trace_attr_show, \
1530c71a896154119f4ca9e89d6078f5f63ad60ef199Arnaldo Carvalho de Melo		    sysfs_blk_trace_attr_store)
1531c71a896154119f4ca9e89d6078f5f63ad60ef199Arnaldo Carvalho de Melo
1532cd649b8bb830d65c57c3c8b98d57b5402256d8bdLi Zefanstatic BLK_TRACE_DEVICE_ATTR(enable);
1533c71a896154119f4ca9e89d6078f5f63ad60ef199Arnaldo Carvalho de Melostatic BLK_TRACE_DEVICE_ATTR(act_mask);
1534c71a896154119f4ca9e89d6078f5f63ad60ef199Arnaldo Carvalho de Melostatic BLK_TRACE_DEVICE_ATTR(pid);
1535c71a896154119f4ca9e89d6078f5f63ad60ef199Arnaldo Carvalho de Melostatic BLK_TRACE_DEVICE_ATTR(start_lba);
1536c71a896154119f4ca9e89d6078f5f63ad60ef199Arnaldo Carvalho de Melostatic BLK_TRACE_DEVICE_ATTR(end_lba);
1537c71a896154119f4ca9e89d6078f5f63ad60ef199Arnaldo Carvalho de Melo
1538c71a896154119f4ca9e89d6078f5f63ad60ef199Arnaldo Carvalho de Melostatic struct attribute *blk_trace_attrs[] = {
1539c71a896154119f4ca9e89d6078f5f63ad60ef199Arnaldo Carvalho de Melo	&dev_attr_enable.attr,
1540c71a896154119f4ca9e89d6078f5f63ad60ef199Arnaldo Carvalho de Melo	&dev_attr_act_mask.attr,
1541c71a896154119f4ca9e89d6078f5f63ad60ef199Arnaldo Carvalho de Melo	&dev_attr_pid.attr,
1542c71a896154119f4ca9e89d6078f5f63ad60ef199Arnaldo Carvalho de Melo	&dev_attr_start_lba.attr,
1543c71a896154119f4ca9e89d6078f5f63ad60ef199Arnaldo Carvalho de Melo	&dev_attr_end_lba.attr,
1544c71a896154119f4ca9e89d6078f5f63ad60ef199Arnaldo Carvalho de Melo	NULL
1545c71a896154119f4ca9e89d6078f5f63ad60ef199Arnaldo Carvalho de Melo};
1546c71a896154119f4ca9e89d6078f5f63ad60ef199Arnaldo Carvalho de Melo
1547c71a896154119f4ca9e89d6078f5f63ad60ef199Arnaldo Carvalho de Melostruct attribute_group blk_trace_attr_group = {
1548c71a896154119f4ca9e89d6078f5f63ad60ef199Arnaldo Carvalho de Melo	.name  = "trace",
1549c71a896154119f4ca9e89d6078f5f63ad60ef199Arnaldo Carvalho de Melo	.attrs = blk_trace_attrs,
1550c71a896154119f4ca9e89d6078f5f63ad60ef199Arnaldo Carvalho de Melo};
1551c71a896154119f4ca9e89d6078f5f63ad60ef199Arnaldo Carvalho de Melo
1552093419971e03362a00f499960557c119982ea09fLi Zefanstatic const struct {
1553093419971e03362a00f499960557c119982ea09fLi Zefan	int mask;
1554093419971e03362a00f499960557c119982ea09fLi Zefan	const char *str;
1555093419971e03362a00f499960557c119982ea09fLi Zefan} mask_maps[] = {
1556093419971e03362a00f499960557c119982ea09fLi Zefan	{ BLK_TC_READ,		"read"		},
1557093419971e03362a00f499960557c119982ea09fLi Zefan	{ BLK_TC_WRITE,		"write"		},
1558093419971e03362a00f499960557c119982ea09fLi Zefan	{ BLK_TC_BARRIER,	"barrier"	},
1559093419971e03362a00f499960557c119982ea09fLi Zefan	{ BLK_TC_SYNC,		"sync"		},
1560093419971e03362a00f499960557c119982ea09fLi Zefan	{ BLK_TC_QUEUE,		"queue"		},
1561093419971e03362a00f499960557c119982ea09fLi Zefan	{ BLK_TC_REQUEUE,	"requeue"	},
1562093419971e03362a00f499960557c119982ea09fLi Zefan	{ BLK_TC_ISSUE,		"issue"		},
1563093419971e03362a00f499960557c119982ea09fLi Zefan	{ BLK_TC_COMPLETE,	"complete"	},
1564093419971e03362a00f499960557c119982ea09fLi Zefan	{ BLK_TC_FS,		"fs"		},
1565093419971e03362a00f499960557c119982ea09fLi Zefan	{ BLK_TC_PC,		"pc"		},
1566093419971e03362a00f499960557c119982ea09fLi Zefan	{ BLK_TC_AHEAD,		"ahead"		},
1567093419971e03362a00f499960557c119982ea09fLi Zefan	{ BLK_TC_META,		"meta"		},
1568093419971e03362a00f499960557c119982ea09fLi Zefan	{ BLK_TC_DISCARD,	"discard"	},
1569093419971e03362a00f499960557c119982ea09fLi Zefan	{ BLK_TC_DRV_DATA,	"drv_data"	},
1570093419971e03362a00f499960557c119982ea09fLi Zefan};
1571093419971e03362a00f499960557c119982ea09fLi Zefan
1572093419971e03362a00f499960557c119982ea09fLi Zefanstatic int blk_trace_str2mask(const char *str)
1573c71a896154119f4ca9e89d6078f5f63ad60ef199Arnaldo Carvalho de Melo{
1574093419971e03362a00f499960557c119982ea09fLi Zefan	int i;
1575c71a896154119f4ca9e89d6078f5f63ad60ef199Arnaldo Carvalho de Melo	int mask = 0;
15769eb85125ce218a8b8d9a7c982510388e227adbecLi Zefan	char *buf, *s, *token;
1577c71a896154119f4ca9e89d6078f5f63ad60ef199Arnaldo Carvalho de Melo
15789eb85125ce218a8b8d9a7c982510388e227adbecLi Zefan	buf = kstrdup(str, GFP_KERNEL);
15799eb85125ce218a8b8d9a7c982510388e227adbecLi Zefan	if (buf == NULL)
1580c71a896154119f4ca9e89d6078f5f63ad60ef199Arnaldo Carvalho de Melo		return -ENOMEM;
15819eb85125ce218a8b8d9a7c982510388e227adbecLi Zefan	s = strstrip(buf);
1582c71a896154119f4ca9e89d6078f5f63ad60ef199Arnaldo Carvalho de Melo
1583c71a896154119f4ca9e89d6078f5f63ad60ef199Arnaldo Carvalho de Melo	while (1) {
1584093419971e03362a00f499960557c119982ea09fLi Zefan		token = strsep(&s, ",");
1585093419971e03362a00f499960557c119982ea09fLi Zefan		if (token == NULL)
1586c71a896154119f4ca9e89d6078f5f63ad60ef199Arnaldo Carvalho de Melo			break;
1587c71a896154119f4ca9e89d6078f5f63ad60ef199Arnaldo Carvalho de Melo
1588093419971e03362a00f499960557c119982ea09fLi Zefan		if (*token == '\0')
1589093419971e03362a00f499960557c119982ea09fLi Zefan			continue;
1590093419971e03362a00f499960557c119982ea09fLi Zefan
1591093419971e03362a00f499960557c119982ea09fLi Zefan		for (i = 0; i < ARRAY_SIZE(mask_maps); i++) {
1592093419971e03362a00f499960557c119982ea09fLi Zefan			if (strcasecmp(token, mask_maps[i].str) == 0) {
1593093419971e03362a00f499960557c119982ea09fLi Zefan				mask |= mask_maps[i].mask;
1594093419971e03362a00f499960557c119982ea09fLi Zefan				break;
1595093419971e03362a00f499960557c119982ea09fLi Zefan			}
1596093419971e03362a00f499960557c119982ea09fLi Zefan		}
1597093419971e03362a00f499960557c119982ea09fLi Zefan		if (i == ARRAY_SIZE(mask_maps)) {
1598093419971e03362a00f499960557c119982ea09fLi Zefan			mask = -EINVAL;
1599093419971e03362a00f499960557c119982ea09fLi Zefan			break;
1600093419971e03362a00f499960557c119982ea09fLi Zefan		}
1601c71a896154119f4ca9e89d6078f5f63ad60ef199Arnaldo Carvalho de Melo	}
16029eb85125ce218a8b8d9a7c982510388e227adbecLi Zefan	kfree(buf);
1603c71a896154119f4ca9e89d6078f5f63ad60ef199Arnaldo Carvalho de Melo
1604c71a896154119f4ca9e89d6078f5f63ad60ef199Arnaldo Carvalho de Melo	return mask;
1605c71a896154119f4ca9e89d6078f5f63ad60ef199Arnaldo Carvalho de Melo}
1606c71a896154119f4ca9e89d6078f5f63ad60ef199Arnaldo Carvalho de Melo
1607093419971e03362a00f499960557c119982ea09fLi Zefanstatic ssize_t blk_trace_mask2str(char *buf, int mask)
1608093419971e03362a00f499960557c119982ea09fLi Zefan{
1609093419971e03362a00f499960557c119982ea09fLi Zefan	int i;
1610093419971e03362a00f499960557c119982ea09fLi Zefan	char *p = buf;
1611093419971e03362a00f499960557c119982ea09fLi Zefan
1612093419971e03362a00f499960557c119982ea09fLi Zefan	for (i = 0; i < ARRAY_SIZE(mask_maps); i++) {
1613093419971e03362a00f499960557c119982ea09fLi Zefan		if (mask & mask_maps[i].mask) {
1614093419971e03362a00f499960557c119982ea09fLi Zefan			p += sprintf(p, "%s%s",
1615093419971e03362a00f499960557c119982ea09fLi Zefan				    (p == buf) ? "" : ",", mask_maps[i].str);
1616093419971e03362a00f499960557c119982ea09fLi Zefan		}
1617093419971e03362a00f499960557c119982ea09fLi Zefan	}
1618093419971e03362a00f499960557c119982ea09fLi Zefan	*p++ = '\n';
1619093419971e03362a00f499960557c119982ea09fLi Zefan
1620093419971e03362a00f499960557c119982ea09fLi Zefan	return p - buf;
1621093419971e03362a00f499960557c119982ea09fLi Zefan}
1622093419971e03362a00f499960557c119982ea09fLi Zefan
1623b125130b22d67f249beba10b71a254558b5279d0Li Zefanstatic struct request_queue *blk_trace_get_queue(struct block_device *bdev)
1624b125130b22d67f249beba10b71a254558b5279d0Li Zefan{
1625b125130b22d67f249beba10b71a254558b5279d0Li Zefan	if (bdev->bd_disk == NULL)
1626b125130b22d67f249beba10b71a254558b5279d0Li Zefan		return NULL;
1627b125130b22d67f249beba10b71a254558b5279d0Li Zefan
1628b125130b22d67f249beba10b71a254558b5279d0Li Zefan	return bdev_get_queue(bdev);
1629b125130b22d67f249beba10b71a254558b5279d0Li Zefan}
1630b125130b22d67f249beba10b71a254558b5279d0Li Zefan
1631c71a896154119f4ca9e89d6078f5f63ad60ef199Arnaldo Carvalho de Melostatic ssize_t sysfs_blk_trace_attr_show(struct device *dev,
1632c71a896154119f4ca9e89d6078f5f63ad60ef199Arnaldo Carvalho de Melo					 struct device_attribute *attr,
1633c71a896154119f4ca9e89d6078f5f63ad60ef199Arnaldo Carvalho de Melo					 char *buf)
1634c71a896154119f4ca9e89d6078f5f63ad60ef199Arnaldo Carvalho de Melo{
1635c71a896154119f4ca9e89d6078f5f63ad60ef199Arnaldo Carvalho de Melo	struct hd_struct *p = dev_to_part(dev);
1636c71a896154119f4ca9e89d6078f5f63ad60ef199Arnaldo Carvalho de Melo	struct request_queue *q;
1637c71a896154119f4ca9e89d6078f5f63ad60ef199Arnaldo Carvalho de Melo	struct block_device *bdev;
1638c71a896154119f4ca9e89d6078f5f63ad60ef199Arnaldo Carvalho de Melo	ssize_t ret = -ENXIO;
1639c71a896154119f4ca9e89d6078f5f63ad60ef199Arnaldo Carvalho de Melo
1640c71a896154119f4ca9e89d6078f5f63ad60ef199Arnaldo Carvalho de Melo	bdev = bdget(part_devt(p));
1641c71a896154119f4ca9e89d6078f5f63ad60ef199Arnaldo Carvalho de Melo	if (bdev == NULL)
164201b284f9b6d51cc3f3bcf3b49f16d2601d3ca22dArnd Bergmann		goto out;
1643c71a896154119f4ca9e89d6078f5f63ad60ef199Arnaldo Carvalho de Melo
1644b125130b22d67f249beba10b71a254558b5279d0Li Zefan	q = blk_trace_get_queue(bdev);
1645c71a896154119f4ca9e89d6078f5f63ad60ef199Arnaldo Carvalho de Melo	if (q == NULL)
1646c71a896154119f4ca9e89d6078f5f63ad60ef199Arnaldo Carvalho de Melo		goto out_bdput;
1647b125130b22d67f249beba10b71a254558b5279d0Li Zefan
1648c71a896154119f4ca9e89d6078f5f63ad60ef199Arnaldo Carvalho de Melo	mutex_lock(&bdev->bd_mutex);
1649cd649b8bb830d65c57c3c8b98d57b5402256d8bdLi Zefan
1650cd649b8bb830d65c57c3c8b98d57b5402256d8bdLi Zefan	if (attr == &dev_attr_enable) {
1651cd649b8bb830d65c57c3c8b98d57b5402256d8bdLi Zefan		ret = sprintf(buf, "%u\n", !!q->blk_trace);
1652cd649b8bb830d65c57c3c8b98d57b5402256d8bdLi Zefan		goto out_unlock_bdev;
1653cd649b8bb830d65c57c3c8b98d57b5402256d8bdLi Zefan	}
1654cd649b8bb830d65c57c3c8b98d57b5402256d8bdLi Zefan
1655c71a896154119f4ca9e89d6078f5f63ad60ef199Arnaldo Carvalho de Melo	if (q->blk_trace == NULL)
1656c71a896154119f4ca9e89d6078f5f63ad60ef199Arnaldo Carvalho de Melo		ret = sprintf(buf, "disabled\n");
1657c71a896154119f4ca9e89d6078f5f63ad60ef199Arnaldo Carvalho de Melo	else if (attr == &dev_attr_act_mask)
1658093419971e03362a00f499960557c119982ea09fLi Zefan		ret = blk_trace_mask2str(buf, q->blk_trace->act_mask);
1659c71a896154119f4ca9e89d6078f5f63ad60ef199Arnaldo Carvalho de Melo	else if (attr == &dev_attr_pid)
1660c71a896154119f4ca9e89d6078f5f63ad60ef199Arnaldo Carvalho de Melo		ret = sprintf(buf, "%u\n", q->blk_trace->pid);
1661c71a896154119f4ca9e89d6078f5f63ad60ef199Arnaldo Carvalho de Melo	else if (attr == &dev_attr_start_lba)
1662c71a896154119f4ca9e89d6078f5f63ad60ef199Arnaldo Carvalho de Melo		ret = sprintf(buf, "%llu\n", q->blk_trace->start_lba);
1663c71a896154119f4ca9e89d6078f5f63ad60ef199Arnaldo Carvalho de Melo	else if (attr == &dev_attr_end_lba)
1664c71a896154119f4ca9e89d6078f5f63ad60ef199Arnaldo Carvalho de Melo		ret = sprintf(buf, "%llu\n", q->blk_trace->end_lba);
1665cd649b8bb830d65c57c3c8b98d57b5402256d8bdLi Zefan
1666cd649b8bb830d65c57c3c8b98d57b5402256d8bdLi Zefanout_unlock_bdev:
1667c71a896154119f4ca9e89d6078f5f63ad60ef199Arnaldo Carvalho de Melo	mutex_unlock(&bdev->bd_mutex);
1668c71a896154119f4ca9e89d6078f5f63ad60ef199Arnaldo Carvalho de Meloout_bdput:
1669c71a896154119f4ca9e89d6078f5f63ad60ef199Arnaldo Carvalho de Melo	bdput(bdev);
167001b284f9b6d51cc3f3bcf3b49f16d2601d3ca22dArnd Bergmannout:
1671c71a896154119f4ca9e89d6078f5f63ad60ef199Arnaldo Carvalho de Melo	return ret;
1672c71a896154119f4ca9e89d6078f5f63ad60ef199Arnaldo Carvalho de Melo}
1673c71a896154119f4ca9e89d6078f5f63ad60ef199Arnaldo Carvalho de Melo
1674c71a896154119f4ca9e89d6078f5f63ad60ef199Arnaldo Carvalho de Melostatic ssize_t sysfs_blk_trace_attr_store(struct device *dev,
1675c71a896154119f4ca9e89d6078f5f63ad60ef199Arnaldo Carvalho de Melo					  struct device_attribute *attr,
1676c71a896154119f4ca9e89d6078f5f63ad60ef199Arnaldo Carvalho de Melo					  const char *buf, size_t count)
1677c71a896154119f4ca9e89d6078f5f63ad60ef199Arnaldo Carvalho de Melo{
1678c71a896154119f4ca9e89d6078f5f63ad60ef199Arnaldo Carvalho de Melo	struct block_device *bdev;
1679c71a896154119f4ca9e89d6078f5f63ad60ef199Arnaldo Carvalho de Melo	struct request_queue *q;
1680c71a896154119f4ca9e89d6078f5f63ad60ef199Arnaldo Carvalho de Melo	struct hd_struct *p;
1681c71a896154119f4ca9e89d6078f5f63ad60ef199Arnaldo Carvalho de Melo	u64 value;
1682093419971e03362a00f499960557c119982ea09fLi Zefan	ssize_t ret = -EINVAL;
1683c71a896154119f4ca9e89d6078f5f63ad60ef199Arnaldo Carvalho de Melo
1684c71a896154119f4ca9e89d6078f5f63ad60ef199Arnaldo Carvalho de Melo	if (count == 0)
1685c71a896154119f4ca9e89d6078f5f63ad60ef199Arnaldo Carvalho de Melo		goto out;
1686c71a896154119f4ca9e89d6078f5f63ad60ef199Arnaldo Carvalho de Melo
1687c71a896154119f4ca9e89d6078f5f63ad60ef199Arnaldo Carvalho de Melo	if (attr == &dev_attr_act_mask) {
1688c71a896154119f4ca9e89d6078f5f63ad60ef199Arnaldo Carvalho de Melo		if (sscanf(buf, "%llx", &value) != 1) {
1689c71a896154119f4ca9e89d6078f5f63ad60ef199Arnaldo Carvalho de Melo			/* Assume it is a list of trace category names */
1690093419971e03362a00f499960557c119982ea09fLi Zefan			ret = blk_trace_str2mask(buf);
1691093419971e03362a00f499960557c119982ea09fLi Zefan			if (ret < 0)
1692c71a896154119f4ca9e89d6078f5f63ad60ef199Arnaldo Carvalho de Melo				goto out;
1693093419971e03362a00f499960557c119982ea09fLi Zefan			value = ret;
1694c71a896154119f4ca9e89d6078f5f63ad60ef199Arnaldo Carvalho de Melo		}
1695c71a896154119f4ca9e89d6078f5f63ad60ef199Arnaldo Carvalho de Melo	} else if (sscanf(buf, "%llu", &value) != 1)
1696c71a896154119f4ca9e89d6078f5f63ad60ef199Arnaldo Carvalho de Melo		goto out;
1697c71a896154119f4ca9e89d6078f5f63ad60ef199Arnaldo Carvalho de Melo
1698093419971e03362a00f499960557c119982ea09fLi Zefan	ret = -ENXIO;
1699093419971e03362a00f499960557c119982ea09fLi Zefan
1700c71a896154119f4ca9e89d6078f5f63ad60ef199Arnaldo Carvalho de Melo	p = dev_to_part(dev);
1701c71a896154119f4ca9e89d6078f5f63ad60ef199Arnaldo Carvalho de Melo	bdev = bdget(part_devt(p));
1702c71a896154119f4ca9e89d6078f5f63ad60ef199Arnaldo Carvalho de Melo	if (bdev == NULL)
170301b284f9b6d51cc3f3bcf3b49f16d2601d3ca22dArnd Bergmann		goto out;
1704c71a896154119f4ca9e89d6078f5f63ad60ef199Arnaldo Carvalho de Melo
1705b125130b22d67f249beba10b71a254558b5279d0Li Zefan	q = blk_trace_get_queue(bdev);
1706c71a896154119f4ca9e89d6078f5f63ad60ef199Arnaldo Carvalho de Melo	if (q == NULL)
1707c71a896154119f4ca9e89d6078f5f63ad60ef199Arnaldo Carvalho de Melo		goto out_bdput;
1708c71a896154119f4ca9e89d6078f5f63ad60ef199Arnaldo Carvalho de Melo
1709c71a896154119f4ca9e89d6078f5f63ad60ef199Arnaldo Carvalho de Melo	mutex_lock(&bdev->bd_mutex);
1710cd649b8bb830d65c57c3c8b98d57b5402256d8bdLi Zefan
1711cd649b8bb830d65c57c3c8b98d57b5402256d8bdLi Zefan	if (attr == &dev_attr_enable) {
1712cd649b8bb830d65c57c3c8b98d57b5402256d8bdLi Zefan		if (value)
17139908c30997b8a73c95f836170b9998dae9aa3f4aLi Zefan			ret = blk_trace_setup_queue(q, bdev);
1714cd649b8bb830d65c57c3c8b98d57b5402256d8bdLi Zefan		else
1715cd649b8bb830d65c57c3c8b98d57b5402256d8bdLi Zefan			ret = blk_trace_remove_queue(q);
1716cd649b8bb830d65c57c3c8b98d57b5402256d8bdLi Zefan		goto out_unlock_bdev;
1717cd649b8bb830d65c57c3c8b98d57b5402256d8bdLi Zefan	}
1718cd649b8bb830d65c57c3c8b98d57b5402256d8bdLi Zefan
1719c71a896154119f4ca9e89d6078f5f63ad60ef199Arnaldo Carvalho de Melo	ret = 0;
1720c71a896154119f4ca9e89d6078f5f63ad60ef199Arnaldo Carvalho de Melo	if (q->blk_trace == NULL)
17219908c30997b8a73c95f836170b9998dae9aa3f4aLi Zefan		ret = blk_trace_setup_queue(q, bdev);
1722c71a896154119f4ca9e89d6078f5f63ad60ef199Arnaldo Carvalho de Melo
1723c71a896154119f4ca9e89d6078f5f63ad60ef199Arnaldo Carvalho de Melo	if (ret == 0) {
1724c71a896154119f4ca9e89d6078f5f63ad60ef199Arnaldo Carvalho de Melo		if (attr == &dev_attr_act_mask)
1725c71a896154119f4ca9e89d6078f5f63ad60ef199Arnaldo Carvalho de Melo			q->blk_trace->act_mask = value;
1726c71a896154119f4ca9e89d6078f5f63ad60ef199Arnaldo Carvalho de Melo		else if (attr == &dev_attr_pid)
1727c71a896154119f4ca9e89d6078f5f63ad60ef199Arnaldo Carvalho de Melo			q->blk_trace->pid = value;
1728c71a896154119f4ca9e89d6078f5f63ad60ef199Arnaldo Carvalho de Melo		else if (attr == &dev_attr_start_lba)
1729c71a896154119f4ca9e89d6078f5f63ad60ef199Arnaldo Carvalho de Melo			q->blk_trace->start_lba = value;
1730c71a896154119f4ca9e89d6078f5f63ad60ef199Arnaldo Carvalho de Melo		else if (attr == &dev_attr_end_lba)
1731c71a896154119f4ca9e89d6078f5f63ad60ef199Arnaldo Carvalho de Melo			q->blk_trace->end_lba = value;
1732c71a896154119f4ca9e89d6078f5f63ad60ef199Arnaldo Carvalho de Melo	}
1733cd649b8bb830d65c57c3c8b98d57b5402256d8bdLi Zefan
1734cd649b8bb830d65c57c3c8b98d57b5402256d8bdLi Zefanout_unlock_bdev:
1735c71a896154119f4ca9e89d6078f5f63ad60ef199Arnaldo Carvalho de Melo	mutex_unlock(&bdev->bd_mutex);
1736c71a896154119f4ca9e89d6078f5f63ad60ef199Arnaldo Carvalho de Meloout_bdput:
1737c71a896154119f4ca9e89d6078f5f63ad60ef199Arnaldo Carvalho de Melo	bdput(bdev);
1738c71a896154119f4ca9e89d6078f5f63ad60ef199Arnaldo Carvalho de Meloout:
1739cd649b8bb830d65c57c3c8b98d57b5402256d8bdLi Zefan	return ret ? ret : count;
1740c71a896154119f4ca9e89d6078f5f63ad60ef199Arnaldo Carvalho de Melo}
1741cd649b8bb830d65c57c3c8b98d57b5402256d8bdLi Zefan
17421d54ad6da9192fed5dd3b60224d9f2dfea0dcd82Li Zefanint blk_trace_init_sysfs(struct device *dev)
17431d54ad6da9192fed5dd3b60224d9f2dfea0dcd82Li Zefan{
17441d54ad6da9192fed5dd3b60224d9f2dfea0dcd82Li Zefan	return sysfs_create_group(&dev->kobj, &blk_trace_attr_group);
17451d54ad6da9192fed5dd3b60224d9f2dfea0dcd82Li Zefan}
17461d54ad6da9192fed5dd3b60224d9f2dfea0dcd82Li Zefan
174748c0d4d4c04dd520c55e0fd756fa4e7c83de3d13Zdenek Kabelacvoid blk_trace_remove_sysfs(struct device *dev)
174848c0d4d4c04dd520c55e0fd756fa4e7c83de3d13Zdenek Kabelac{
174948c0d4d4c04dd520c55e0fd756fa4e7c83de3d13Zdenek Kabelac	sysfs_remove_group(&dev->kobj, &blk_trace_attr_group);
175048c0d4d4c04dd520c55e0fd756fa4e7c83de3d13Zdenek Kabelac}
175148c0d4d4c04dd520c55e0fd756fa4e7c83de3d13Zdenek Kabelac
175255782138e47d9baf2f7d3a7af9e7cf42adf72c56Li Zefan#endif /* CONFIG_BLK_DEV_IO_TRACE */
175355782138e47d9baf2f7d3a7af9e7cf42adf72c56Li Zefan
175455782138e47d9baf2f7d3a7af9e7cf42adf72c56Li Zefan#ifdef CONFIG_EVENT_TRACING
175555782138e47d9baf2f7d3a7af9e7cf42adf72c56Li Zefan
175655782138e47d9baf2f7d3a7af9e7cf42adf72c56Li Zefanvoid blk_dump_cmd(char *buf, struct request *rq)
175755782138e47d9baf2f7d3a7af9e7cf42adf72c56Li Zefan{
175855782138e47d9baf2f7d3a7af9e7cf42adf72c56Li Zefan	int i, end;
175955782138e47d9baf2f7d3a7af9e7cf42adf72c56Li Zefan	int len = rq->cmd_len;
176055782138e47d9baf2f7d3a7af9e7cf42adf72c56Li Zefan	unsigned char *cmd = rq->cmd;
176155782138e47d9baf2f7d3a7af9e7cf42adf72c56Li Zefan
176233659ebbae262228eef4e0fe990f393d1f0ed941Christoph Hellwig	if (rq->cmd_type != REQ_TYPE_BLOCK_PC) {
176355782138e47d9baf2f7d3a7af9e7cf42adf72c56Li Zefan		buf[0] = '\0';
176455782138e47d9baf2f7d3a7af9e7cf42adf72c56Li Zefan		return;
176555782138e47d9baf2f7d3a7af9e7cf42adf72c56Li Zefan	}
176655782138e47d9baf2f7d3a7af9e7cf42adf72c56Li Zefan
176755782138e47d9baf2f7d3a7af9e7cf42adf72c56Li Zefan	for (end = len - 1; end >= 0; end--)
176855782138e47d9baf2f7d3a7af9e7cf42adf72c56Li Zefan		if (cmd[end])
176955782138e47d9baf2f7d3a7af9e7cf42adf72c56Li Zefan			break;
177055782138e47d9baf2f7d3a7af9e7cf42adf72c56Li Zefan	end++;
177155782138e47d9baf2f7d3a7af9e7cf42adf72c56Li Zefan
177255782138e47d9baf2f7d3a7af9e7cf42adf72c56Li Zefan	for (i = 0; i < len; i++) {
177355782138e47d9baf2f7d3a7af9e7cf42adf72c56Li Zefan		buf += sprintf(buf, "%s%02x", i == 0 ? "" : " ", cmd[i]);
177455782138e47d9baf2f7d3a7af9e7cf42adf72c56Li Zefan		if (i == end && end != len - 1) {
177555782138e47d9baf2f7d3a7af9e7cf42adf72c56Li Zefan			sprintf(buf, " ..");
177655782138e47d9baf2f7d3a7af9e7cf42adf72c56Li Zefan			break;
177755782138e47d9baf2f7d3a7af9e7cf42adf72c56Li Zefan		}
177855782138e47d9baf2f7d3a7af9e7cf42adf72c56Li Zefan	}
177955782138e47d9baf2f7d3a7af9e7cf42adf72c56Li Zefan}
178055782138e47d9baf2f7d3a7af9e7cf42adf72c56Li Zefan
178155782138e47d9baf2f7d3a7af9e7cf42adf72c56Li Zefanvoid blk_fill_rwbs(char *rwbs, u32 rw, int bytes)
178255782138e47d9baf2f7d3a7af9e7cf42adf72c56Li Zefan{
178355782138e47d9baf2f7d3a7af9e7cf42adf72c56Li Zefan	int i = 0;
178455782138e47d9baf2f7d3a7af9e7cf42adf72c56Li Zefan
178555782138e47d9baf2f7d3a7af9e7cf42adf72c56Li Zefan	if (rw & WRITE)
178655782138e47d9baf2f7d3a7af9e7cf42adf72c56Li Zefan		rwbs[i++] = 'W';
17877b6d91daee5cac6402186ff224c3af39d79f4a0eChristoph Hellwig	else if (rw & REQ_DISCARD)
178855782138e47d9baf2f7d3a7af9e7cf42adf72c56Li Zefan		rwbs[i++] = 'D';
178955782138e47d9baf2f7d3a7af9e7cf42adf72c56Li Zefan	else if (bytes)
179055782138e47d9baf2f7d3a7af9e7cf42adf72c56Li Zefan		rwbs[i++] = 'R';
179155782138e47d9baf2f7d3a7af9e7cf42adf72c56Li Zefan	else
179255782138e47d9baf2f7d3a7af9e7cf42adf72c56Li Zefan		rwbs[i++] = 'N';
179355782138e47d9baf2f7d3a7af9e7cf42adf72c56Li Zefan
17947b6d91daee5cac6402186ff224c3af39d79f4a0eChristoph Hellwig	if (rw & REQ_RAHEAD)
179555782138e47d9baf2f7d3a7af9e7cf42adf72c56Li Zefan		rwbs[i++] = 'A';
17967b6d91daee5cac6402186ff224c3af39d79f4a0eChristoph Hellwig	if (rw & REQ_SYNC)
179755782138e47d9baf2f7d3a7af9e7cf42adf72c56Li Zefan		rwbs[i++] = 'S';
17987b6d91daee5cac6402186ff224c3af39d79f4a0eChristoph Hellwig	if (rw & REQ_META)
179955782138e47d9baf2f7d3a7af9e7cf42adf72c56Li Zefan		rwbs[i++] = 'M';
18008d57a98ccd0b4489003473979da8f5a1363ba7a3Adrian Hunter	if (rw & REQ_SECURE)
18018d57a98ccd0b4489003473979da8f5a1363ba7a3Adrian Hunter		rwbs[i++] = 'E';
180255782138e47d9baf2f7d3a7af9e7cf42adf72c56Li Zefan
180355782138e47d9baf2f7d3a7af9e7cf42adf72c56Li Zefan	rwbs[i] = '\0';
180455782138e47d9baf2f7d3a7af9e7cf42adf72c56Li Zefan}
180555782138e47d9baf2f7d3a7af9e7cf42adf72c56Li Zefan
180655782138e47d9baf2f7d3a7af9e7cf42adf72c56Li Zefan#endif /* CONFIG_EVENT_TRACING */
180755782138e47d9baf2f7d3a7af9e7cf42adf72c56Li Zefan
1808