1
2/******************************************************************************
3 *
4 * Copyright (C) 2007-2009 Steven Rostedt <srostedt@redhat.com>
5 *
6 * ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
7 *
8 * This program is free software; you can redistribute it and/or modify
9 * it under the terms of the GNU General Public License as published by
10 * the Free Software Foundation; version 2 of the License (not later!)
11 *
12 * This program is distributed in the hope that it will be useful,
13 * but WITHOUT ANY WARRANTY; without even the implied warranty of
14 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
15 * GNU General Public License for more details.
16 *
17 * You should have received a copy of the GNU General Public License
18 * along with this program; if not, write to the Free Software
19 * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
20 *
21 * NAME
22 *      rt-migrate-test.c
23 *
24 * DESCRIPTION
25 *	This test makes sure that all the high prio tasks that are in the
26 *	running state are actually running on a CPU if it can.
27 ** Steps:
28 *	- Creates N+1 threads with lower real time priorities.
29 *	  Where N is the number of CPUs in the system.
30 *	- If the thread is high priority, and if a CPU is available, the
31 *	  thread runs on that CPU.
32 *	- The thread records the start time and the number of ticks in the run
33 *	  interval.
34 *	- The output indicates if lower prio task is quicker than higher
35 *	  priority task.
36 *
37 * USAGE:
38 *	Use run_auto.sh in the current directory to build and run the test.
39 *
40 * AUTHOR
41 *      Steven Rostedt <srostedt@redhat.com>
42 *
43 * HISTORY
44 *      30 July, 2009: Initial version by Steven Rostedt
45 *      11 Aug, 2009: Converted the coding style to the one used by the realtime
46 *		    testcases by Kiran Prakash
47 *
48 */
49#ifndef _GNU_SOURCE
50#define _GNU_SOURCE
51#endif
52#include <stdio.h>
53#include <stdlib.h>
54#include <string.h>
55#include <getopt.h>
56#include <stdarg.h>
57#include <unistd.h>
58#include <ctype.h>
59#include <time.h>
60#include <sys/types.h>
61#include <sys/stat.h>
62#include <fcntl.h>
63#include <signal.h>
64#include <sys/time.h>
65#include <linux/unistd.h>
66#include <sys/syscall.h>
67#include <errno.h>
68#include <sched.h>
69#include <pthread.h>
70#include <librttest.h>
71#include <libstats.h>
72
73#define gettid() syscall(__NR_gettid)
74
75#define VERSION_STRING "V 0.4LTP"
76
77int nr_tasks;
78int lfd;
79
80int numcpus;
81static int mark_fd = -1;
82static __thread char buff[BUFSIZ + 1];
83
84static void setup_ftrace_marker(void)
85{
86	struct stat st;
87	char *files[] = {
88		"/sys/kernel/debug/tracing/trace_marker",
89		"/debug/tracing/trace_marker",
90		"/debugfs/tracing/trace_marker",
91	};
92	int ret;
93	int i;
94
95	for (i = 0; i < (sizeof(files) / sizeof(char *)); i++) {
96		ret = stat(files[i], &st);
97		if (ret >= 0)
98			goto found;
99	}
100	/* todo, check mounts system */
101	return;
102found:
103	mark_fd = open(files[i], O_WRONLY);
104}
105
106static void ftrace_write(const char *fmt, ...)
107{
108	va_list ap;
109	int n;
110
111	if (mark_fd < 0)
112		return;
113
114	va_start(ap, fmt);
115	n = vsnprintf(buff, BUFSIZ, fmt, ap);
116	va_end(ap);
117
118	/*
119	 * This doesn't return any valid vs invalid exit codes, so printing out
120	 * a perror to warn the end-user of an issue is sufficient.
121	 */
122	if (write(mark_fd, buff, n) < 0) {
123		perror("write");
124	}
125}
126
127#define INTERVAL 100ULL * NS_PER_MS
128#define RUN_INTERVAL 20ULL * NS_PER_MS
129#define NR_RUNS 50
130#define PRIO_START 2
131/* 1 millisec off */
132#define MAX_ERR  1000 * NS_PER_US
133
134#define PROGRESS_CHARS 70
135
136static unsigned long long interval = INTERVAL;
137static unsigned long long run_interval = RUN_INTERVAL;
138static unsigned long long max_err = MAX_ERR;
139static int nr_runs = NR_RUNS;
140static int prio_start = PRIO_START;
141static int check = 1;
142static int stop;
143
144static unsigned long long now;
145
146static int done;
147static int loop;
148
149static pthread_barrier_t start_barrier;
150static pthread_barrier_t end_barrier;
151stats_container_t *intervals;
152stats_container_t *intervals_length;
153stats_container_t *intervals_loops;
154static long *thread_pids;
155
156static void print_progress_bar(int percent)
157{
158	int i;
159	int p;
160
161	if (percent > 100)
162		percent = 100;
163
164	/* Use stderr, so we don't capture it */
165	putc('\r', stderr);
166	putc('|', stderr);
167	for (i = 0; i < PROGRESS_CHARS; i++)
168		putc(' ', stderr);
169	putc('|', stderr);
170	putc('\r', stderr);
171	putc('|', stderr);
172
173	p = PROGRESS_CHARS * percent / 100;
174
175	for (i = 0; i < p; i++)
176		putc('-', stderr);
177
178	fflush(stderr);
179}
180
181static void usage()
182{
183	rt_help();
184	printf("Usage:\n"
185	       "-a priority Priority of the threads"
186	       "-r time     Run time (ms) to busy loop the threads (20)\n"
187	       "-t time     Sleep time (ms) between intervals (100)\n"
188	       "-e time     Max allowed error (microsecs)\n"
189	       "-l loops    Number of iterations to run (50)\n");
190}
191
192/*
193int rt_init(const char *options, int (*parse_arg)(int option, char *value),
194	    int argc, char *argv[]);
195 */
196static int parse_args(int c, char *v)
197{
198	int handled = 1;
199	switch (c) {
200	case 'a':
201		prio_start = atoi(v);
202		break;
203	case 'r':
204		run_interval = atoi(v);
205		break;
206	case 't':
207		interval = atoi(v);
208		break;
209	case 'l':
210		nr_runs = atoi(v);
211		break;
212	case 'e':
213		max_err = atoi(v) * NS_PER_US;
214		break;
215	case '?':
216	case 'h':
217		usage();
218		handled = 0;
219	}
220	return handled;
221}
222
223static void record_time(int id, unsigned long long time, unsigned long l)
224{
225	unsigned long long ltime;
226	stats_record_t rec;
227	if (loop >= nr_runs)
228		return;
229	time -= now;
230	ltime = rt_gettime() / NS_PER_US;
231	ltime -= now;
232	rec.x = loop;
233	rec.y = time;
234	stats_container_append(&intervals[id], rec);
235	rec.x = loop;
236	rec.y = ltime;
237	stats_container_append(&intervals_length[id], rec);
238	rec.x = loop;
239	rec.y = l;
240	stats_container_append(&intervals_loops[id], rec);
241}
242
243static void print_results(void)
244{
245	int i;
246	int t;
247	unsigned long long tasks_max[nr_tasks];
248	unsigned long long tasks_min[nr_tasks];
249	unsigned long long tasks_avg[nr_tasks];
250
251	memset(tasks_max, 0, sizeof(tasks_max[0]) * nr_tasks);
252	memset(tasks_min, 0xff, sizeof(tasks_min[0]) * nr_tasks);
253	memset(tasks_avg, 0, sizeof(tasks_avg[0]) * nr_tasks);
254
255	printf("Iter: ");
256	for (t = 0; t < nr_tasks; t++)
257		printf("%6d  ", t);
258	printf("\n");
259
260	for (t = 0; t < nr_tasks; t++) {
261		tasks_max[t] = stats_max(&intervals[t]);
262		tasks_min[t] = stats_min(&intervals[t]);
263		tasks_avg[t] = stats_avg(&intervals[t]);
264	}
265	for (i = 0; i < nr_runs; i++) {
266		printf("%4d:   ", i);
267		for (t = 0; t < nr_tasks; t++)
268			printf("%6ld  ", intervals[t].records[i].y);
269
270		printf("\n");
271		printf(" len:   ");
272		for (t = 0; t < nr_tasks; t++)
273			printf("%6ld  ", intervals_length[t].records[i].y);
274
275		printf("\n");
276		printf(" loops: ");
277		for (t = 0; t < nr_tasks; t++)
278			printf("%6ld  ", intervals_loops[t].records[i].y);
279
280		printf("\n");
281		printf("\n");
282	}
283
284	printf("Parent pid: %d\n", getpid());
285
286	for (t = 0; t < nr_tasks; t++) {
287		printf(" Task %d (prio %d) (pid %ld):\n", t, t + prio_start,
288		       thread_pids[t]);
289		printf("   Max: %lld us\n", tasks_max[t]);
290		printf("   Min: %lld us\n", tasks_min[t]);
291		printf("   Tot: %lld us\n", tasks_avg[t] * nr_runs);
292		printf("   Avg: %lld us\n", tasks_avg[t]);
293		printf("\n");
294	}
295
296	printf(" Result: %s\n", (check < 0) ? "FAIL" : "PASS");
297}
298
299static unsigned long busy_loop(unsigned long long start_time)
300{
301	unsigned long long time;
302	unsigned long l = 0;
303
304	do {
305		l++;
306		time = rt_gettime();
307	} while ((time - start_time) < RUN_INTERVAL);
308
309	return l;
310}
311
312void *start_task(void *data)
313{
314	struct thread *thr = (struct thread *)data;
315	long id = (long)thr->arg;
316	thread_pids[id] = gettid();
317	unsigned long long start_time;
318	int ret;
319	int high = 0;
320	cpu_set_t cpumask;
321	cpu_set_t save_cpumask;
322	int cpu = 0;
323	unsigned long l;
324	long pid;
325
326	ret = sched_getaffinity(0, sizeof(save_cpumask), &save_cpumask);
327	if (ret < 0)
328		debug(DBG_ERR, "sched_getaffinity failed: %s\n", strerror(ret));
329
330	pid = gettid();
331
332	/* Check if we are the highest prio task */
333	if (id == nr_tasks - 1)
334		high = 1;
335
336	while (!done) {
337		if (high) {
338			/* rotate around the CPUS */
339			if (!CPU_ISSET(cpu, &save_cpumask))
340				cpu = 0;
341			CPU_ZERO(&cpumask);
342			CPU_SET(cpu, &cpumask);
343			cpu++;
344			sched_setaffinity(0, sizeof(cpumask), &cpumask);
345		}
346		pthread_barrier_wait(&start_barrier);
347		start_time = rt_gettime();
348		ftrace_write("Thread %d: started %lld diff %lld\n",
349			     pid, start_time, start_time - now);
350		l = busy_loop(start_time);
351		record_time(id, start_time / NS_PER_US, l);
352		pthread_barrier_wait(&end_barrier);
353	}
354
355	return (void *)pid;
356}
357
358static int check_times(int l)
359{
360	int i;
361	unsigned long long last;
362	unsigned long long last_loops;
363	unsigned long long last_length;
364
365	for (i = 0; i < nr_tasks; i++) {
366		if (i && last < intervals[i].records[l].y &&
367		    ((intervals[i].records[l].y - last) > max_err)) {
368			/*
369			 * May be a false positive.
370			 * Make sure that we did more loops
371			 * our start is before the end
372			 * and the end should be tested.
373			 */
374			if (intervals_loops[i].records[l].y < last_loops ||
375			    intervals[i].records[l].y > last_length ||
376			    (intervals_length[i].records[l].y > last_length &&
377			     intervals_length[i].records[l].y - last_length >
378			     max_err)) {
379				check = -1;
380				return 1;
381			}
382		}
383		last = intervals[i].records[l].y;
384		last_loops = intervals_loops[i].records[l].y;
385		last_length = intervals_length[i].records[l].y;
386	}
387	return 0;
388}
389
390static void stop_log(int sig)
391{
392	stop = 1;
393}
394
395int main(int argc, char **argv)
396{
397	pthread_t *threads;
398	long i;
399	int ret;
400	struct timespec intv;
401	struct sched_param param;
402
403	rt_init("a:r:t:e:l:h:", parse_args, argc, argv);
404	signal(SIGINT, stop_log);
405
406	if (argc >= (optind + 1))
407		nr_tasks = atoi(argv[optind]);
408	else {
409		numcpus = sysconf(_SC_NPROCESSORS_ONLN);
410		nr_tasks = numcpus + 1;
411	}
412
413	intervals = malloc(sizeof(stats_container_t) * nr_tasks);
414	if (!intervals)
415		debug(DBG_ERR, "malloc failed\n");
416	memset(intervals, 0, sizeof(stats_container_t) * nr_tasks);
417
418	intervals_length = malloc(sizeof(stats_container_t) * nr_tasks);
419	if (!intervals_length)
420		debug(DBG_ERR, "malloc failed\n");
421	memset(intervals_length, 0, sizeof(stats_container_t) * nr_tasks);
422
423	if (!intervals_loops)
424		debug(DBG_ERR, "malloc failed\n");
425	intervals_loops = malloc(sizeof(stats_container_t) * nr_tasks);
426	memset(intervals_loops, 0, sizeof(stats_container_t) * nr_tasks);
427
428	threads = malloc(sizeof(*threads) * nr_tasks);
429	if (!threads)
430		debug(DBG_ERR, "malloc failed\n");
431	memset(threads, 0, sizeof(*threads) * nr_tasks);
432
433	ret = pthread_barrier_init(&start_barrier, NULL, nr_tasks + 1);
434	ret = pthread_barrier_init(&end_barrier, NULL, nr_tasks + 1);
435	if (ret < 0)
436		debug(DBG_ERR, "pthread_barrier_init failed: %s\n",
437		      strerror(ret));
438
439	for (i = 0; i < nr_tasks; i++) {
440		stats_container_init(&intervals[i], nr_runs);
441		stats_container_init(&intervals_length[i], nr_runs);
442		stats_container_init(&intervals_loops[i], nr_runs);
443	}
444
445	thread_pids = malloc(sizeof(long) * nr_tasks);
446	if (!thread_pids)
447		debug(DBG_ERR, "malloc thread_pids failed\n");
448
449	for (i = 0; i < nr_tasks; i++) {
450		threads[i] = create_fifo_thread(start_task, (void *)i,
451						prio_start + i);
452	}
453
454	/*
455	 * Progress bar uses stderr to let users see it when
456	 * redirecting output. So we convert stderr to use line
457	 * buffering so the progress bar doesn't flicker.
458	 */
459	setlinebuf(stderr);
460
461	/* up our prio above all tasks */
462	memset(&param, 0, sizeof(param));
463	param.sched_priority = nr_tasks + prio_start;
464	if (sched_setscheduler(0, SCHED_FIFO, &param))
465		debug(DBG_WARN, "Warning, can't set priority of"
466		      "main thread !\n");
467	intv.tv_sec = INTERVAL / NS_PER_SEC;
468	intv.tv_nsec = INTERVAL % (1 * NS_PER_SEC);
469
470	print_progress_bar(0);
471
472	setup_ftrace_marker();
473
474	for (loop = 0; loop < nr_runs; loop++) {
475		unsigned long long end;
476
477		now = rt_gettime() / NS_PER_US;
478
479		ftrace_write("Loop %d now=%lld\n", loop, now);
480
481		pthread_barrier_wait(&start_barrier);
482
483		ftrace_write("All running!!!\n");
484
485		rt_nanosleep(intv.tv_nsec);
486		print_progress_bar((loop * 100) / nr_runs);
487
488		end = rt_gettime() / NS_PER_US;
489		ftrace_write("Loop %d end now=%lld diff=%lld\n",
490			     loop, end, end - now);
491		ret = pthread_barrier_wait(&end_barrier);
492
493		if (stop || (check && check_times(loop))) {
494			loop++;
495			nr_runs = loop;
496			break;
497		}
498	}
499	putc('\n', stderr);
500
501	pthread_barrier_wait(&start_barrier);
502	done = 1;
503	pthread_barrier_wait(&end_barrier);
504
505	join_threads();
506	print_results();
507
508	if (stop) {
509		/*
510		 * We use this test in bash while loops
511		 * So if we hit Ctrl-C then let the while
512		 * loop know to break.
513		 */
514		if (check < 0)
515			exit(-1);
516		else
517			exit(1);
518	}
519
520	if (check < 0)
521		exit(-1);
522	else
523		exit(0);
524}
525