dmatest.c revision a9e554957de406d6adc581731f571b8a1503f6b0
14a776f0aa922a552460192c07b56f4fe9cd82632Haavard Skinnemoen/* 24a776f0aa922a552460192c07b56f4fe9cd82632Haavard Skinnemoen * DMA Engine test module 34a776f0aa922a552460192c07b56f4fe9cd82632Haavard Skinnemoen * 44a776f0aa922a552460192c07b56f4fe9cd82632Haavard Skinnemoen * Copyright (C) 2007 Atmel Corporation 5851b7e16a07dfda6178d4e35fea9a9e3eb8954aeAndy Shevchenko * Copyright (C) 2013 Intel Corporation 64a776f0aa922a552460192c07b56f4fe9cd82632Haavard Skinnemoen * 74a776f0aa922a552460192c07b56f4fe9cd82632Haavard Skinnemoen * This program is free software; you can redistribute it and/or modify 84a776f0aa922a552460192c07b56f4fe9cd82632Haavard Skinnemoen * it under the terms of the GNU General Public License version 2 as 94a776f0aa922a552460192c07b56f4fe9cd82632Haavard Skinnemoen * published by the Free Software Foundation. 104a776f0aa922a552460192c07b56f4fe9cd82632Haavard Skinnemoen */ 11872f05c6e9a37e9358fd58eb54deee7337863496Dan Williams#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt 12872f05c6e9a37e9358fd58eb54deee7337863496Dan Williams 134a776f0aa922a552460192c07b56f4fe9cd82632Haavard Skinnemoen#include <linux/delay.h> 14b7f080cfe223b3b7424872639d153695615a9255Alexey Dobriyan#include <linux/dma-mapping.h> 154a776f0aa922a552460192c07b56f4fe9cd82632Haavard Skinnemoen#include <linux/dmaengine.h> 16981ed70d8e4faf3689dbf3c48868a31d5b004d7aGuennadi Liakhovetski#include <linux/freezer.h> 174a776f0aa922a552460192c07b56f4fe9cd82632Haavard Skinnemoen#include <linux/init.h> 184a776f0aa922a552460192c07b56f4fe9cd82632Haavard Skinnemoen#include <linux/kthread.h> 194a776f0aa922a552460192c07b56f4fe9cd82632Haavard Skinnemoen#include <linux/module.h> 204a776f0aa922a552460192c07b56f4fe9cd82632Haavard Skinnemoen#include <linux/moduleparam.h> 214a776f0aa922a552460192c07b56f4fe9cd82632Haavard Skinnemoen#include <linux/random.h> 225a0e3ad6af8660be21ca98a971cd00f331318c05Tejun Heo#include <linux/slab.h> 234a776f0aa922a552460192c07b56f4fe9cd82632Haavard Skinnemoen#include <linux/wait.h> 244a776f0aa922a552460192c07b56f4fe9cd82632Haavard Skinnemoen 254a776f0aa922a552460192c07b56f4fe9cd82632Haavard Skinnemoenstatic unsigned int test_buf_size = 16384; 26a6c268d033b1f363e0d76c0483a0f99266542820Andy Shevchenkomodule_param(test_buf_size, uint, S_IRUGO | S_IWUSR); 274a776f0aa922a552460192c07b56f4fe9cd82632Haavard SkinnemoenMODULE_PARM_DESC(test_buf_size, "Size of the memcpy test buffer"); 284a776f0aa922a552460192c07b56f4fe9cd82632Haavard Skinnemoen 2906190d8415219d9eef7d8f04b52a109e34575a76Kay Sieversstatic char test_channel[20]; 30a6c268d033b1f363e0d76c0483a0f99266542820Andy Shevchenkomodule_param_string(channel, test_channel, sizeof(test_channel), 31a6c268d033b1f363e0d76c0483a0f99266542820Andy Shevchenko S_IRUGO | S_IWUSR); 324a776f0aa922a552460192c07b56f4fe9cd82632Haavard SkinnemoenMODULE_PARM_DESC(channel, "Bus ID of the channel to test (default: any)"); 334a776f0aa922a552460192c07b56f4fe9cd82632Haavard Skinnemoen 3406190d8415219d9eef7d8f04b52a109e34575a76Kay Sieversstatic char test_device[20]; 35a6c268d033b1f363e0d76c0483a0f99266542820Andy Shevchenkomodule_param_string(device, test_device, sizeof(test_device), 36a6c268d033b1f363e0d76c0483a0f99266542820Andy Shevchenko S_IRUGO | S_IWUSR); 374a776f0aa922a552460192c07b56f4fe9cd82632Haavard SkinnemoenMODULE_PARM_DESC(device, "Bus ID of the DMA Engine to test (default: any)"); 384a776f0aa922a552460192c07b56f4fe9cd82632Haavard Skinnemoen 394a776f0aa922a552460192c07b56f4fe9cd82632Haavard Skinnemoenstatic unsigned int threads_per_chan = 1; 40a6c268d033b1f363e0d76c0483a0f99266542820Andy Shevchenkomodule_param(threads_per_chan, uint, S_IRUGO | S_IWUSR); 414a776f0aa922a552460192c07b56f4fe9cd82632Haavard SkinnemoenMODULE_PARM_DESC(threads_per_chan, 424a776f0aa922a552460192c07b56f4fe9cd82632Haavard Skinnemoen "Number of threads to start per channel (default: 1)"); 434a776f0aa922a552460192c07b56f4fe9cd82632Haavard Skinnemoen 444a776f0aa922a552460192c07b56f4fe9cd82632Haavard Skinnemoenstatic unsigned int max_channels; 45a6c268d033b1f363e0d76c0483a0f99266542820Andy Shevchenkomodule_param(max_channels, uint, S_IRUGO | S_IWUSR); 4633df8ca068123457db56c316946a3c0e4ef787d6Dan WilliamsMODULE_PARM_DESC(max_channels, 474a776f0aa922a552460192c07b56f4fe9cd82632Haavard Skinnemoen "Maximum number of channels to use (default: all)"); 484a776f0aa922a552460192c07b56f4fe9cd82632Haavard Skinnemoen 490a2ff57d6fba92842272889b4bca447344cd9d36Nicolas Ferrestatic unsigned int iterations; 50a6c268d033b1f363e0d76c0483a0f99266542820Andy Shevchenkomodule_param(iterations, uint, S_IRUGO | S_IWUSR); 510a2ff57d6fba92842272889b4bca447344cd9d36Nicolas FerreMODULE_PARM_DESC(iterations, 520a2ff57d6fba92842272889b4bca447344cd9d36Nicolas Ferre "Iterations before stopping test (default: infinite)"); 530a2ff57d6fba92842272889b4bca447344cd9d36Nicolas Ferre 54b54d5cb9156868fb4f27ccd46a3afb0bf3ef8e0cDan Williamsstatic unsigned int xor_sources = 3; 55a6c268d033b1f363e0d76c0483a0f99266542820Andy Shevchenkomodule_param(xor_sources, uint, S_IRUGO | S_IWUSR); 56b54d5cb9156868fb4f27ccd46a3afb0bf3ef8e0cDan WilliamsMODULE_PARM_DESC(xor_sources, 57b54d5cb9156868fb4f27ccd46a3afb0bf3ef8e0cDan Williams "Number of xor source buffers (default: 3)"); 58b54d5cb9156868fb4f27ccd46a3afb0bf3ef8e0cDan Williams 5958691d64c44ae41ddf098ecb31e9a994026e3cffDan Williamsstatic unsigned int pq_sources = 3; 60a6c268d033b1f363e0d76c0483a0f99266542820Andy Shevchenkomodule_param(pq_sources, uint, S_IRUGO | S_IWUSR); 6158691d64c44ae41ddf098ecb31e9a994026e3cffDan WilliamsMODULE_PARM_DESC(pq_sources, 6258691d64c44ae41ddf098ecb31e9a994026e3cffDan Williams "Number of p+q source buffers (default: 3)"); 6358691d64c44ae41ddf098ecb31e9a994026e3cffDan Williams 64d42efe6bfb4eed8314c8ce3547f21954a4140399Viresh Kumarstatic int timeout = 3000; 65a6c268d033b1f363e0d76c0483a0f99266542820Andy Shevchenkomodule_param(timeout, uint, S_IRUGO | S_IWUSR); 6685ee7a1d39d75d23d21f3871f6dc9b87d572747aJoe PerchesMODULE_PARM_DESC(timeout, "Transfer Timeout in msec (default: 3000), " 6785ee7a1d39d75d23d21f3871f6dc9b87d572747aJoe Perches "Pass -1 for infinite timeout"); 68d42efe6bfb4eed8314c8ce3547f21954a4140399Viresh Kumar 69e03e93a976d0f0da63f02fd3384c4b99cac8d715Andy Shevchenko/** 7015b8a8ea1a87313f1b46ea878c65942fd52147edAndy Shevchenko * struct dmatest_params - test parameters. 71e03e93a976d0f0da63f02fd3384c4b99cac8d715Andy Shevchenko * @buf_size: size of the memcpy test buffer 72e03e93a976d0f0da63f02fd3384c4b99cac8d715Andy Shevchenko * @channel: bus ID of the channel to test 73e03e93a976d0f0da63f02fd3384c4b99cac8d715Andy Shevchenko * @device: bus ID of the DMA Engine to test 74e03e93a976d0f0da63f02fd3384c4b99cac8d715Andy Shevchenko * @threads_per_chan: number of threads to start per channel 75e03e93a976d0f0da63f02fd3384c4b99cac8d715Andy Shevchenko * @max_channels: maximum number of channels to use 76e03e93a976d0f0da63f02fd3384c4b99cac8d715Andy Shevchenko * @iterations: iterations before stopping test 77e03e93a976d0f0da63f02fd3384c4b99cac8d715Andy Shevchenko * @xor_sources: number of xor source buffers 78e03e93a976d0f0da63f02fd3384c4b99cac8d715Andy Shevchenko * @pq_sources: number of p+q source buffers 79e03e93a976d0f0da63f02fd3384c4b99cac8d715Andy Shevchenko * @timeout: transfer timeout in msec, -1 for infinite timeout 80e03e93a976d0f0da63f02fd3384c4b99cac8d715Andy Shevchenko */ 8115b8a8ea1a87313f1b46ea878c65942fd52147edAndy Shevchenkostruct dmatest_params { 82e03e93a976d0f0da63f02fd3384c4b99cac8d715Andy Shevchenko unsigned int buf_size; 83e03e93a976d0f0da63f02fd3384c4b99cac8d715Andy Shevchenko char channel[20]; 84e03e93a976d0f0da63f02fd3384c4b99cac8d715Andy Shevchenko char device[20]; 85e03e93a976d0f0da63f02fd3384c4b99cac8d715Andy Shevchenko unsigned int threads_per_chan; 86e03e93a976d0f0da63f02fd3384c4b99cac8d715Andy Shevchenko unsigned int max_channels; 87e03e93a976d0f0da63f02fd3384c4b99cac8d715Andy Shevchenko unsigned int iterations; 88e03e93a976d0f0da63f02fd3384c4b99cac8d715Andy Shevchenko unsigned int xor_sources; 89e03e93a976d0f0da63f02fd3384c4b99cac8d715Andy Shevchenko unsigned int pq_sources; 90e03e93a976d0f0da63f02fd3384c4b99cac8d715Andy Shevchenko int timeout; 9115b8a8ea1a87313f1b46ea878c65942fd52147edAndy Shevchenko}; 9215b8a8ea1a87313f1b46ea878c65942fd52147edAndy Shevchenko 9315b8a8ea1a87313f1b46ea878c65942fd52147edAndy Shevchenko/** 9415b8a8ea1a87313f1b46ea878c65942fd52147edAndy Shevchenko * struct dmatest_info - test information. 9515b8a8ea1a87313f1b46ea878c65942fd52147edAndy Shevchenko * @params: test parameters 96851b7e16a07dfda6178d4e35fea9a9e3eb8954aeAndy Shevchenko * @lock: access protection to the fields of this structure 9715b8a8ea1a87313f1b46ea878c65942fd52147edAndy Shevchenko */ 98a310d037b8d06755c62bb4878c00d19490af5550Dan Williamsstatic struct dmatest_info { 9915b8a8ea1a87313f1b46ea878c65942fd52147edAndy Shevchenko /* Test parameters */ 10015b8a8ea1a87313f1b46ea878c65942fd52147edAndy Shevchenko struct dmatest_params params; 101838cc704ce5c8ab2a6d64d1324e37e040fcae3d8Andy Shevchenko 102838cc704ce5c8ab2a6d64d1324e37e040fcae3d8Andy Shevchenko /* Internal state */ 103838cc704ce5c8ab2a6d64d1324e37e040fcae3d8Andy Shevchenko struct list_head channels; 104838cc704ce5c8ab2a6d64d1324e37e040fcae3d8Andy Shevchenko unsigned int nr_channels; 105851b7e16a07dfda6178d4e35fea9a9e3eb8954aeAndy Shevchenko struct mutex lock; 106a310d037b8d06755c62bb4878c00d19490af5550Dan Williams bool did_init; 107a310d037b8d06755c62bb4878c00d19490af5550Dan Williams} test_info = { 108a310d037b8d06755c62bb4878c00d19490af5550Dan Williams .channels = LIST_HEAD_INIT(test_info.channels), 109a310d037b8d06755c62bb4878c00d19490af5550Dan Williams .lock = __MUTEX_INITIALIZER(test_info.lock), 110a310d037b8d06755c62bb4878c00d19490af5550Dan Williams}; 111851b7e16a07dfda6178d4e35fea9a9e3eb8954aeAndy Shevchenko 112a310d037b8d06755c62bb4878c00d19490af5550Dan Williamsstatic int dmatest_run_set(const char *val, const struct kernel_param *kp); 113a310d037b8d06755c62bb4878c00d19490af5550Dan Williamsstatic int dmatest_run_get(char *val, const struct kernel_param *kp); 114a310d037b8d06755c62bb4878c00d19490af5550Dan Williamsstatic struct kernel_param_ops run_ops = { 115a310d037b8d06755c62bb4878c00d19490af5550Dan Williams .set = dmatest_run_set, 116a310d037b8d06755c62bb4878c00d19490af5550Dan Williams .get = dmatest_run_get, 117e03e93a976d0f0da63f02fd3384c4b99cac8d715Andy Shevchenko}; 118a310d037b8d06755c62bb4878c00d19490af5550Dan Williamsstatic bool dmatest_run; 119a310d037b8d06755c62bb4878c00d19490af5550Dan Williamsmodule_param_cb(run, &run_ops, &dmatest_run, S_IRUGO | S_IWUSR); 120a310d037b8d06755c62bb4878c00d19490af5550Dan WilliamsMODULE_PARM_DESC(run, "Run the test (default: false)"); 121e03e93a976d0f0da63f02fd3384c4b99cac8d715Andy Shevchenko 122a310d037b8d06755c62bb4878c00d19490af5550Dan Williams/* Maximum amount of mismatched bytes in buffer to print */ 123a310d037b8d06755c62bb4878c00d19490af5550Dan Williams#define MAX_ERROR_COUNT 32 124a310d037b8d06755c62bb4878c00d19490af5550Dan Williams 125a310d037b8d06755c62bb4878c00d19490af5550Dan Williams/* 126a310d037b8d06755c62bb4878c00d19490af5550Dan Williams * Initialization patterns. All bytes in the source buffer has bit 7 127a310d037b8d06755c62bb4878c00d19490af5550Dan Williams * set, all bytes in the destination buffer has bit 7 cleared. 128a310d037b8d06755c62bb4878c00d19490af5550Dan Williams * 129a310d037b8d06755c62bb4878c00d19490af5550Dan Williams * Bit 6 is set for all bytes which are to be copied by the DMA 130a310d037b8d06755c62bb4878c00d19490af5550Dan Williams * engine. Bit 5 is set for all bytes which are to be overwritten by 131a310d037b8d06755c62bb4878c00d19490af5550Dan Williams * the DMA engine. 132a310d037b8d06755c62bb4878c00d19490af5550Dan Williams * 133a310d037b8d06755c62bb4878c00d19490af5550Dan Williams * The remaining bits are the inverse of a counter which increments by 134a310d037b8d06755c62bb4878c00d19490af5550Dan Williams * one for each byte address. 135a310d037b8d06755c62bb4878c00d19490af5550Dan Williams */ 136a310d037b8d06755c62bb4878c00d19490af5550Dan Williams#define PATTERN_SRC 0x80 137a310d037b8d06755c62bb4878c00d19490af5550Dan Williams#define PATTERN_DST 0x00 138a310d037b8d06755c62bb4878c00d19490af5550Dan Williams#define PATTERN_COPY 0x40 139a310d037b8d06755c62bb4878c00d19490af5550Dan Williams#define PATTERN_OVERWRITE 0x20 140a310d037b8d06755c62bb4878c00d19490af5550Dan Williams#define PATTERN_COUNT_MASK 0x1f 141a310d037b8d06755c62bb4878c00d19490af5550Dan Williams 142a310d037b8d06755c62bb4878c00d19490af5550Dan Williamsstruct dmatest_thread { 143a310d037b8d06755c62bb4878c00d19490af5550Dan Williams struct list_head node; 144a310d037b8d06755c62bb4878c00d19490af5550Dan Williams struct dmatest_info *info; 145a310d037b8d06755c62bb4878c00d19490af5550Dan Williams struct task_struct *task; 146a310d037b8d06755c62bb4878c00d19490af5550Dan Williams struct dma_chan *chan; 147a310d037b8d06755c62bb4878c00d19490af5550Dan Williams u8 **srcs; 148a310d037b8d06755c62bb4878c00d19490af5550Dan Williams u8 **dsts; 149a310d037b8d06755c62bb4878c00d19490af5550Dan Williams enum dma_transaction_type type; 150a310d037b8d06755c62bb4878c00d19490af5550Dan Williams bool done; 151a310d037b8d06755c62bb4878c00d19490af5550Dan Williams}; 152a310d037b8d06755c62bb4878c00d19490af5550Dan Williams 153a310d037b8d06755c62bb4878c00d19490af5550Dan Williamsstruct dmatest_chan { 154a310d037b8d06755c62bb4878c00d19490af5550Dan Williams struct list_head node; 155a310d037b8d06755c62bb4878c00d19490af5550Dan Williams struct dma_chan *chan; 156a310d037b8d06755c62bb4878c00d19490af5550Dan Williams struct list_head threads; 157a310d037b8d06755c62bb4878c00d19490af5550Dan Williams}; 158e03e93a976d0f0da63f02fd3384c4b99cac8d715Andy Shevchenko 15915b8a8ea1a87313f1b46ea878c65942fd52147edAndy Shevchenkostatic bool dmatest_match_channel(struct dmatest_params *params, 160e03e93a976d0f0da63f02fd3384c4b99cac8d715Andy Shevchenko struct dma_chan *chan) 1614a776f0aa922a552460192c07b56f4fe9cd82632Haavard Skinnemoen{ 16215b8a8ea1a87313f1b46ea878c65942fd52147edAndy Shevchenko if (params->channel[0] == '\0') 1634a776f0aa922a552460192c07b56f4fe9cd82632Haavard Skinnemoen return true; 16415b8a8ea1a87313f1b46ea878c65942fd52147edAndy Shevchenko return strcmp(dma_chan_name(chan), params->channel) == 0; 1654a776f0aa922a552460192c07b56f4fe9cd82632Haavard Skinnemoen} 1664a776f0aa922a552460192c07b56f4fe9cd82632Haavard Skinnemoen 16715b8a8ea1a87313f1b46ea878c65942fd52147edAndy Shevchenkostatic bool dmatest_match_device(struct dmatest_params *params, 168e03e93a976d0f0da63f02fd3384c4b99cac8d715Andy Shevchenko struct dma_device *device) 1694a776f0aa922a552460192c07b56f4fe9cd82632Haavard Skinnemoen{ 17015b8a8ea1a87313f1b46ea878c65942fd52147edAndy Shevchenko if (params->device[0] == '\0') 1714a776f0aa922a552460192c07b56f4fe9cd82632Haavard Skinnemoen return true; 17215b8a8ea1a87313f1b46ea878c65942fd52147edAndy Shevchenko return strcmp(dev_name(device->dev), params->device) == 0; 1734a776f0aa922a552460192c07b56f4fe9cd82632Haavard Skinnemoen} 1744a776f0aa922a552460192c07b56f4fe9cd82632Haavard Skinnemoen 1754a776f0aa922a552460192c07b56f4fe9cd82632Haavard Skinnemoenstatic unsigned long dmatest_random(void) 1764a776f0aa922a552460192c07b56f4fe9cd82632Haavard Skinnemoen{ 1774a776f0aa922a552460192c07b56f4fe9cd82632Haavard Skinnemoen unsigned long buf; 1784a776f0aa922a552460192c07b56f4fe9cd82632Haavard Skinnemoen 1794a776f0aa922a552460192c07b56f4fe9cd82632Haavard Skinnemoen get_random_bytes(&buf, sizeof(buf)); 1804a776f0aa922a552460192c07b56f4fe9cd82632Haavard Skinnemoen return buf; 1814a776f0aa922a552460192c07b56f4fe9cd82632Haavard Skinnemoen} 1824a776f0aa922a552460192c07b56f4fe9cd82632Haavard Skinnemoen 183e03e93a976d0f0da63f02fd3384c4b99cac8d715Andy Shevchenkostatic void dmatest_init_srcs(u8 **bufs, unsigned int start, unsigned int len, 184e03e93a976d0f0da63f02fd3384c4b99cac8d715Andy Shevchenko unsigned int buf_size) 1854a776f0aa922a552460192c07b56f4fe9cd82632Haavard Skinnemoen{ 1864a776f0aa922a552460192c07b56f4fe9cd82632Haavard Skinnemoen unsigned int i; 187b54d5cb9156868fb4f27ccd46a3afb0bf3ef8e0cDan Williams u8 *buf; 188b54d5cb9156868fb4f27ccd46a3afb0bf3ef8e0cDan Williams 189b54d5cb9156868fb4f27ccd46a3afb0bf3ef8e0cDan Williams for (; (buf = *bufs); bufs++) { 190b54d5cb9156868fb4f27ccd46a3afb0bf3ef8e0cDan Williams for (i = 0; i < start; i++) 191b54d5cb9156868fb4f27ccd46a3afb0bf3ef8e0cDan Williams buf[i] = PATTERN_SRC | (~i & PATTERN_COUNT_MASK); 192b54d5cb9156868fb4f27ccd46a3afb0bf3ef8e0cDan Williams for ( ; i < start + len; i++) 193b54d5cb9156868fb4f27ccd46a3afb0bf3ef8e0cDan Williams buf[i] = PATTERN_SRC | PATTERN_COPY 194c019894efc9c9ba5939948caa78c133b1ec8ae63Joe Perches | (~i & PATTERN_COUNT_MASK); 195e03e93a976d0f0da63f02fd3384c4b99cac8d715Andy Shevchenko for ( ; i < buf_size; i++) 196b54d5cb9156868fb4f27ccd46a3afb0bf3ef8e0cDan Williams buf[i] = PATTERN_SRC | (~i & PATTERN_COUNT_MASK); 197b54d5cb9156868fb4f27ccd46a3afb0bf3ef8e0cDan Williams buf++; 198b54d5cb9156868fb4f27ccd46a3afb0bf3ef8e0cDan Williams } 1994a776f0aa922a552460192c07b56f4fe9cd82632Haavard Skinnemoen} 2004a776f0aa922a552460192c07b56f4fe9cd82632Haavard Skinnemoen 201e03e93a976d0f0da63f02fd3384c4b99cac8d715Andy Shevchenkostatic void dmatest_init_dsts(u8 **bufs, unsigned int start, unsigned int len, 202e03e93a976d0f0da63f02fd3384c4b99cac8d715Andy Shevchenko unsigned int buf_size) 2034a776f0aa922a552460192c07b56f4fe9cd82632Haavard Skinnemoen{ 2044a776f0aa922a552460192c07b56f4fe9cd82632Haavard Skinnemoen unsigned int i; 205b54d5cb9156868fb4f27ccd46a3afb0bf3ef8e0cDan Williams u8 *buf; 206b54d5cb9156868fb4f27ccd46a3afb0bf3ef8e0cDan Williams 207b54d5cb9156868fb4f27ccd46a3afb0bf3ef8e0cDan Williams for (; (buf = *bufs); bufs++) { 208b54d5cb9156868fb4f27ccd46a3afb0bf3ef8e0cDan Williams for (i = 0; i < start; i++) 209b54d5cb9156868fb4f27ccd46a3afb0bf3ef8e0cDan Williams buf[i] = PATTERN_DST | (~i & PATTERN_COUNT_MASK); 210b54d5cb9156868fb4f27ccd46a3afb0bf3ef8e0cDan Williams for ( ; i < start + len; i++) 211b54d5cb9156868fb4f27ccd46a3afb0bf3ef8e0cDan Williams buf[i] = PATTERN_DST | PATTERN_OVERWRITE 212b54d5cb9156868fb4f27ccd46a3afb0bf3ef8e0cDan Williams | (~i & PATTERN_COUNT_MASK); 213e03e93a976d0f0da63f02fd3384c4b99cac8d715Andy Shevchenko for ( ; i < buf_size; i++) 214b54d5cb9156868fb4f27ccd46a3afb0bf3ef8e0cDan Williams buf[i] = PATTERN_DST | (~i & PATTERN_COUNT_MASK); 215b54d5cb9156868fb4f27ccd46a3afb0bf3ef8e0cDan Williams } 2164a776f0aa922a552460192c07b56f4fe9cd82632Haavard Skinnemoen} 2174a776f0aa922a552460192c07b56f4fe9cd82632Haavard Skinnemoen 2187b61017822cdff9c18ae70005cf52d84e8dafe5dDan Williamsstatic void dmatest_mismatch(u8 actual, u8 pattern, unsigned int index, 2197b61017822cdff9c18ae70005cf52d84e8dafe5dDan Williams unsigned int counter, bool is_srcbuf) 2207b61017822cdff9c18ae70005cf52d84e8dafe5dDan Williams{ 2217b61017822cdff9c18ae70005cf52d84e8dafe5dDan Williams u8 diff = actual ^ pattern; 2227b61017822cdff9c18ae70005cf52d84e8dafe5dDan Williams u8 expected = pattern | (~counter & PATTERN_COUNT_MASK); 2237b61017822cdff9c18ae70005cf52d84e8dafe5dDan Williams const char *thread_name = current->comm; 2247b61017822cdff9c18ae70005cf52d84e8dafe5dDan Williams 2257b61017822cdff9c18ae70005cf52d84e8dafe5dDan Williams if (is_srcbuf) 2267b61017822cdff9c18ae70005cf52d84e8dafe5dDan Williams pr_warn("%s: srcbuf[0x%x] overwritten! Expected %02x, got %02x\n", 2277b61017822cdff9c18ae70005cf52d84e8dafe5dDan Williams thread_name, index, expected, actual); 2287b61017822cdff9c18ae70005cf52d84e8dafe5dDan Williams else if ((pattern & PATTERN_COPY) 2297b61017822cdff9c18ae70005cf52d84e8dafe5dDan Williams && (diff & (PATTERN_COPY | PATTERN_OVERWRITE))) 2307b61017822cdff9c18ae70005cf52d84e8dafe5dDan Williams pr_warn("%s: dstbuf[0x%x] not copied! Expected %02x, got %02x\n", 2317b61017822cdff9c18ae70005cf52d84e8dafe5dDan Williams thread_name, index, expected, actual); 2327b61017822cdff9c18ae70005cf52d84e8dafe5dDan Williams else if (diff & PATTERN_SRC) 2337b61017822cdff9c18ae70005cf52d84e8dafe5dDan Williams pr_warn("%s: dstbuf[0x%x] was copied! Expected %02x, got %02x\n", 2347b61017822cdff9c18ae70005cf52d84e8dafe5dDan Williams thread_name, index, expected, actual); 2357b61017822cdff9c18ae70005cf52d84e8dafe5dDan Williams else 2367b61017822cdff9c18ae70005cf52d84e8dafe5dDan Williams pr_warn("%s: dstbuf[0x%x] mismatch! Expected %02x, got %02x\n", 2377b61017822cdff9c18ae70005cf52d84e8dafe5dDan Williams thread_name, index, expected, actual); 2387b61017822cdff9c18ae70005cf52d84e8dafe5dDan Williams} 2397b61017822cdff9c18ae70005cf52d84e8dafe5dDan Williams 2407b61017822cdff9c18ae70005cf52d84e8dafe5dDan Williamsstatic unsigned int dmatest_verify(u8 **bufs, unsigned int start, 2417b61017822cdff9c18ae70005cf52d84e8dafe5dDan Williams unsigned int end, unsigned int counter, u8 pattern, 2427b61017822cdff9c18ae70005cf52d84e8dafe5dDan Williams bool is_srcbuf) 2434a776f0aa922a552460192c07b56f4fe9cd82632Haavard Skinnemoen{ 2444a776f0aa922a552460192c07b56f4fe9cd82632Haavard Skinnemoen unsigned int i; 2454a776f0aa922a552460192c07b56f4fe9cd82632Haavard Skinnemoen unsigned int error_count = 0; 2464a776f0aa922a552460192c07b56f4fe9cd82632Haavard Skinnemoen u8 actual; 247b54d5cb9156868fb4f27ccd46a3afb0bf3ef8e0cDan Williams u8 expected; 248b54d5cb9156868fb4f27ccd46a3afb0bf3ef8e0cDan Williams u8 *buf; 249b54d5cb9156868fb4f27ccd46a3afb0bf3ef8e0cDan Williams unsigned int counter_orig = counter; 250b54d5cb9156868fb4f27ccd46a3afb0bf3ef8e0cDan Williams 251b54d5cb9156868fb4f27ccd46a3afb0bf3ef8e0cDan Williams for (; (buf = *bufs); bufs++) { 252b54d5cb9156868fb4f27ccd46a3afb0bf3ef8e0cDan Williams counter = counter_orig; 253b54d5cb9156868fb4f27ccd46a3afb0bf3ef8e0cDan Williams for (i = start; i < end; i++) { 254b54d5cb9156868fb4f27ccd46a3afb0bf3ef8e0cDan Williams actual = buf[i]; 255b54d5cb9156868fb4f27ccd46a3afb0bf3ef8e0cDan Williams expected = pattern | (~counter & PATTERN_COUNT_MASK); 256b54d5cb9156868fb4f27ccd46a3afb0bf3ef8e0cDan Williams if (actual != expected) { 2577b61017822cdff9c18ae70005cf52d84e8dafe5dDan Williams if (error_count < MAX_ERROR_COUNT) 2587b61017822cdff9c18ae70005cf52d84e8dafe5dDan Williams dmatest_mismatch(actual, pattern, i, 2597b61017822cdff9c18ae70005cf52d84e8dafe5dDan Williams counter, is_srcbuf); 260b54d5cb9156868fb4f27ccd46a3afb0bf3ef8e0cDan Williams error_count++; 261b54d5cb9156868fb4f27ccd46a3afb0bf3ef8e0cDan Williams } 262b54d5cb9156868fb4f27ccd46a3afb0bf3ef8e0cDan Williams counter++; 2634a776f0aa922a552460192c07b56f4fe9cd82632Haavard Skinnemoen } 2644a776f0aa922a552460192c07b56f4fe9cd82632Haavard Skinnemoen } 2654a776f0aa922a552460192c07b56f4fe9cd82632Haavard Skinnemoen 26674b5c07a515b2986c9bdfe649213b8e358d32ad2Andy Shevchenko if (error_count > MAX_ERROR_COUNT) 2677b61017822cdff9c18ae70005cf52d84e8dafe5dDan Williams pr_warn("%s: %u errors suppressed\n", 26874b5c07a515b2986c9bdfe649213b8e358d32ad2Andy Shevchenko current->comm, error_count - MAX_ERROR_COUNT); 2694a776f0aa922a552460192c07b56f4fe9cd82632Haavard Skinnemoen 2704a776f0aa922a552460192c07b56f4fe9cd82632Haavard Skinnemoen return error_count; 2714a776f0aa922a552460192c07b56f4fe9cd82632Haavard Skinnemoen} 2724a776f0aa922a552460192c07b56f4fe9cd82632Haavard Skinnemoen 273adfa543e7314b36ac55a40019977de6e47946dd7Tejun Heo/* poor man's completion - we want to use wait_event_freezable() on it */ 274adfa543e7314b36ac55a40019977de6e47946dd7Tejun Heostruct dmatest_done { 275adfa543e7314b36ac55a40019977de6e47946dd7Tejun Heo bool done; 276adfa543e7314b36ac55a40019977de6e47946dd7Tejun Heo wait_queue_head_t *wait; 277adfa543e7314b36ac55a40019977de6e47946dd7Tejun Heo}; 278adfa543e7314b36ac55a40019977de6e47946dd7Tejun Heo 279adfa543e7314b36ac55a40019977de6e47946dd7Tejun Heostatic void dmatest_callback(void *arg) 280e44e0aa3cfa97cddff01704751a4b25151830c72Dan Williams{ 281adfa543e7314b36ac55a40019977de6e47946dd7Tejun Heo struct dmatest_done *done = arg; 282adfa543e7314b36ac55a40019977de6e47946dd7Tejun Heo 283adfa543e7314b36ac55a40019977de6e47946dd7Tejun Heo done->done = true; 284adfa543e7314b36ac55a40019977de6e47946dd7Tejun Heo wake_up_all(done->wait); 285e44e0aa3cfa97cddff01704751a4b25151830c72Dan Williams} 286e44e0aa3cfa97cddff01704751a4b25151830c72Dan Williams 287632fd28326c0cc7be9c51ea0d76d8bec39a695e9Andy Shevchenkostatic inline void unmap_src(struct device *dev, dma_addr_t *addr, size_t len, 288632fd28326c0cc7be9c51ea0d76d8bec39a695e9Andy Shevchenko unsigned int count) 289632fd28326c0cc7be9c51ea0d76d8bec39a695e9Andy Shevchenko{ 290632fd28326c0cc7be9c51ea0d76d8bec39a695e9Andy Shevchenko while (count--) 291632fd28326c0cc7be9c51ea0d76d8bec39a695e9Andy Shevchenko dma_unmap_single(dev, addr[count], len, DMA_TO_DEVICE); 292632fd28326c0cc7be9c51ea0d76d8bec39a695e9Andy Shevchenko} 293632fd28326c0cc7be9c51ea0d76d8bec39a695e9Andy Shevchenko 294632fd28326c0cc7be9c51ea0d76d8bec39a695e9Andy Shevchenkostatic inline void unmap_dst(struct device *dev, dma_addr_t *addr, size_t len, 295632fd28326c0cc7be9c51ea0d76d8bec39a695e9Andy Shevchenko unsigned int count) 296632fd28326c0cc7be9c51ea0d76d8bec39a695e9Andy Shevchenko{ 297632fd28326c0cc7be9c51ea0d76d8bec39a695e9Andy Shevchenko while (count--) 298632fd28326c0cc7be9c51ea0d76d8bec39a695e9Andy Shevchenko dma_unmap_single(dev, addr[count], len, DMA_BIDIRECTIONAL); 299632fd28326c0cc7be9c51ea0d76d8bec39a695e9Andy Shevchenko} 300632fd28326c0cc7be9c51ea0d76d8bec39a695e9Andy Shevchenko 3018be9e32b310cd8c4302991c8ff6692689c7d9d76Akinobu Mitastatic unsigned int min_odd(unsigned int x, unsigned int y) 3028be9e32b310cd8c4302991c8ff6692689c7d9d76Akinobu Mita{ 3038be9e32b310cd8c4302991c8ff6692689c7d9d76Akinobu Mita unsigned int val = min(x, y); 3048be9e32b310cd8c4302991c8ff6692689c7d9d76Akinobu Mita 3058be9e32b310cd8c4302991c8ff6692689c7d9d76Akinobu Mita return val % 2 ? val : val - 1; 3068be9e32b310cd8c4302991c8ff6692689c7d9d76Akinobu Mita} 3078be9e32b310cd8c4302991c8ff6692689c7d9d76Akinobu Mita 308872f05c6e9a37e9358fd58eb54deee7337863496Dan Williamsstatic void result(const char *err, unsigned int n, unsigned int src_off, 309872f05c6e9a37e9358fd58eb54deee7337863496Dan Williams unsigned int dst_off, unsigned int len, unsigned long data) 31095019c8c5af947f64e4a62e08a4a275bc36148eeAndy Shevchenko{ 311872f05c6e9a37e9358fd58eb54deee7337863496Dan Williams pr_info("%s: result #%u: '%s' with src_off=0x%x ""dst_off=0x%x len=0x%x (%lu)", 312872f05c6e9a37e9358fd58eb54deee7337863496Dan Williams current->comm, n, err, src_off, dst_off, len, data); 31395019c8c5af947f64e4a62e08a4a275bc36148eeAndy Shevchenko} 31495019c8c5af947f64e4a62e08a4a275bc36148eeAndy Shevchenko 315872f05c6e9a37e9358fd58eb54deee7337863496Dan Williamsstatic void dbg_result(const char *err, unsigned int n, unsigned int src_off, 316872f05c6e9a37e9358fd58eb54deee7337863496Dan Williams unsigned int dst_off, unsigned int len, 317872f05c6e9a37e9358fd58eb54deee7337863496Dan Williams unsigned long data) 31895019c8c5af947f64e4a62e08a4a275bc36148eeAndy Shevchenko{ 319872f05c6e9a37e9358fd58eb54deee7337863496Dan Williams pr_debug("%s: result #%u: '%s' with src_off=0x%x ""dst_off=0x%x len=0x%x (%lu)", 320872f05c6e9a37e9358fd58eb54deee7337863496Dan Williams current->comm, n, err, src_off, dst_off, len, data); 32195019c8c5af947f64e4a62e08a4a275bc36148eeAndy Shevchenko} 32295019c8c5af947f64e4a62e08a4a275bc36148eeAndy Shevchenko 3234a776f0aa922a552460192c07b56f4fe9cd82632Haavard Skinnemoen/* 3244a776f0aa922a552460192c07b56f4fe9cd82632Haavard Skinnemoen * This function repeatedly tests DMA transfers of various lengths and 325b54d5cb9156868fb4f27ccd46a3afb0bf3ef8e0cDan Williams * offsets for a given operation type until it is told to exit by 326b54d5cb9156868fb4f27ccd46a3afb0bf3ef8e0cDan Williams * kthread_stop(). There may be multiple threads running this function 327b54d5cb9156868fb4f27ccd46a3afb0bf3ef8e0cDan Williams * in parallel for a single channel, and there may be multiple channels 328b54d5cb9156868fb4f27ccd46a3afb0bf3ef8e0cDan Williams * being tested in parallel. 3294a776f0aa922a552460192c07b56f4fe9cd82632Haavard Skinnemoen * 3304a776f0aa922a552460192c07b56f4fe9cd82632Haavard Skinnemoen * Before each test, the source and destination buffer is initialized 3314a776f0aa922a552460192c07b56f4fe9cd82632Haavard Skinnemoen * with a known pattern. This pattern is different depending on 3324a776f0aa922a552460192c07b56f4fe9cd82632Haavard Skinnemoen * whether it's in an area which is supposed to be copied or 3334a776f0aa922a552460192c07b56f4fe9cd82632Haavard Skinnemoen * overwritten, and different in the source and destination buffers. 3344a776f0aa922a552460192c07b56f4fe9cd82632Haavard Skinnemoen * So if the DMA engine doesn't copy exactly what we tell it to copy, 3354a776f0aa922a552460192c07b56f4fe9cd82632Haavard Skinnemoen * we'll notice. 3364a776f0aa922a552460192c07b56f4fe9cd82632Haavard Skinnemoen */ 3374a776f0aa922a552460192c07b56f4fe9cd82632Haavard Skinnemoenstatic int dmatest_func(void *data) 3384a776f0aa922a552460192c07b56f4fe9cd82632Haavard Skinnemoen{ 339adfa543e7314b36ac55a40019977de6e47946dd7Tejun Heo DECLARE_WAIT_QUEUE_HEAD_ONSTACK(done_wait); 3404a776f0aa922a552460192c07b56f4fe9cd82632Haavard Skinnemoen struct dmatest_thread *thread = data; 341adfa543e7314b36ac55a40019977de6e47946dd7Tejun Heo struct dmatest_done done = { .wait = &done_wait }; 342e03e93a976d0f0da63f02fd3384c4b99cac8d715Andy Shevchenko struct dmatest_info *info; 34315b8a8ea1a87313f1b46ea878c65942fd52147edAndy Shevchenko struct dmatest_params *params; 3444a776f0aa922a552460192c07b56f4fe9cd82632Haavard Skinnemoen struct dma_chan *chan; 3458be9e32b310cd8c4302991c8ff6692689c7d9d76Akinobu Mita struct dma_device *dev; 3464a776f0aa922a552460192c07b56f4fe9cd82632Haavard Skinnemoen unsigned int src_off, dst_off, len; 3474a776f0aa922a552460192c07b56f4fe9cd82632Haavard Skinnemoen unsigned int error_count; 3484a776f0aa922a552460192c07b56f4fe9cd82632Haavard Skinnemoen unsigned int failed_tests = 0; 3494a776f0aa922a552460192c07b56f4fe9cd82632Haavard Skinnemoen unsigned int total_tests = 0; 3504a776f0aa922a552460192c07b56f4fe9cd82632Haavard Skinnemoen dma_cookie_t cookie; 3514a776f0aa922a552460192c07b56f4fe9cd82632Haavard Skinnemoen enum dma_status status; 352b54d5cb9156868fb4f27ccd46a3afb0bf3ef8e0cDan Williams enum dma_ctrl_flags flags; 353945b5af3cedcdfed6d2d940e53cd19933bb57386Andy Shevchenko u8 *pq_coefs = NULL; 3544a776f0aa922a552460192c07b56f4fe9cd82632Haavard Skinnemoen int ret; 355b54d5cb9156868fb4f27ccd46a3afb0bf3ef8e0cDan Williams int src_cnt; 356b54d5cb9156868fb4f27ccd46a3afb0bf3ef8e0cDan Williams int dst_cnt; 357b54d5cb9156868fb4f27ccd46a3afb0bf3ef8e0cDan Williams int i; 3584a776f0aa922a552460192c07b56f4fe9cd82632Haavard Skinnemoen 359adfa543e7314b36ac55a40019977de6e47946dd7Tejun Heo set_freezable(); 3604a776f0aa922a552460192c07b56f4fe9cd82632Haavard Skinnemoen 3614a776f0aa922a552460192c07b56f4fe9cd82632Haavard Skinnemoen ret = -ENOMEM; 3624a776f0aa922a552460192c07b56f4fe9cd82632Haavard Skinnemoen 3634a776f0aa922a552460192c07b56f4fe9cd82632Haavard Skinnemoen smp_rmb(); 364e03e93a976d0f0da63f02fd3384c4b99cac8d715Andy Shevchenko info = thread->info; 36515b8a8ea1a87313f1b46ea878c65942fd52147edAndy Shevchenko params = &info->params; 3664a776f0aa922a552460192c07b56f4fe9cd82632Haavard Skinnemoen chan = thread->chan; 3678be9e32b310cd8c4302991c8ff6692689c7d9d76Akinobu Mita dev = chan->device; 368b54d5cb9156868fb4f27ccd46a3afb0bf3ef8e0cDan Williams if (thread->type == DMA_MEMCPY) 369b54d5cb9156868fb4f27ccd46a3afb0bf3ef8e0cDan Williams src_cnt = dst_cnt = 1; 370b54d5cb9156868fb4f27ccd46a3afb0bf3ef8e0cDan Williams else if (thread->type == DMA_XOR) { 3718be9e32b310cd8c4302991c8ff6692689c7d9d76Akinobu Mita /* force odd to ensure dst = src */ 37215b8a8ea1a87313f1b46ea878c65942fd52147edAndy Shevchenko src_cnt = min_odd(params->xor_sources | 1, dev->max_xor); 373b54d5cb9156868fb4f27ccd46a3afb0bf3ef8e0cDan Williams dst_cnt = 1; 37458691d64c44ae41ddf098ecb31e9a994026e3cffDan Williams } else if (thread->type == DMA_PQ) { 3758be9e32b310cd8c4302991c8ff6692689c7d9d76Akinobu Mita /* force odd to ensure dst = src */ 37615b8a8ea1a87313f1b46ea878c65942fd52147edAndy Shevchenko src_cnt = min_odd(params->pq_sources | 1, dma_maxpq(dev, 0)); 37758691d64c44ae41ddf098ecb31e9a994026e3cffDan Williams dst_cnt = 2; 378945b5af3cedcdfed6d2d940e53cd19933bb57386Andy Shevchenko 37915b8a8ea1a87313f1b46ea878c65942fd52147edAndy Shevchenko pq_coefs = kmalloc(params->pq_sources+1, GFP_KERNEL); 380945b5af3cedcdfed6d2d940e53cd19933bb57386Andy Shevchenko if (!pq_coefs) 381945b5af3cedcdfed6d2d940e53cd19933bb57386Andy Shevchenko goto err_thread_type; 382945b5af3cedcdfed6d2d940e53cd19933bb57386Andy Shevchenko 38394de648d72c8bc833590523f22386d4babbea988Anatolij Gustschin for (i = 0; i < src_cnt; i++) 38458691d64c44ae41ddf098ecb31e9a994026e3cffDan Williams pq_coefs[i] = 1; 385b54d5cb9156868fb4f27ccd46a3afb0bf3ef8e0cDan Williams } else 386945b5af3cedcdfed6d2d940e53cd19933bb57386Andy Shevchenko goto err_thread_type; 387b54d5cb9156868fb4f27ccd46a3afb0bf3ef8e0cDan Williams 388b54d5cb9156868fb4f27ccd46a3afb0bf3ef8e0cDan Williams thread->srcs = kcalloc(src_cnt+1, sizeof(u8 *), GFP_KERNEL); 389b54d5cb9156868fb4f27ccd46a3afb0bf3ef8e0cDan Williams if (!thread->srcs) 390b54d5cb9156868fb4f27ccd46a3afb0bf3ef8e0cDan Williams goto err_srcs; 391b54d5cb9156868fb4f27ccd46a3afb0bf3ef8e0cDan Williams for (i = 0; i < src_cnt; i++) { 39215b8a8ea1a87313f1b46ea878c65942fd52147edAndy Shevchenko thread->srcs[i] = kmalloc(params->buf_size, GFP_KERNEL); 393b54d5cb9156868fb4f27ccd46a3afb0bf3ef8e0cDan Williams if (!thread->srcs[i]) 394b54d5cb9156868fb4f27ccd46a3afb0bf3ef8e0cDan Williams goto err_srcbuf; 395b54d5cb9156868fb4f27ccd46a3afb0bf3ef8e0cDan Williams } 396b54d5cb9156868fb4f27ccd46a3afb0bf3ef8e0cDan Williams thread->srcs[i] = NULL; 397b54d5cb9156868fb4f27ccd46a3afb0bf3ef8e0cDan Williams 398b54d5cb9156868fb4f27ccd46a3afb0bf3ef8e0cDan Williams thread->dsts = kcalloc(dst_cnt+1, sizeof(u8 *), GFP_KERNEL); 399b54d5cb9156868fb4f27ccd46a3afb0bf3ef8e0cDan Williams if (!thread->dsts) 400b54d5cb9156868fb4f27ccd46a3afb0bf3ef8e0cDan Williams goto err_dsts; 401b54d5cb9156868fb4f27ccd46a3afb0bf3ef8e0cDan Williams for (i = 0; i < dst_cnt; i++) { 40215b8a8ea1a87313f1b46ea878c65942fd52147edAndy Shevchenko thread->dsts[i] = kmalloc(params->buf_size, GFP_KERNEL); 403b54d5cb9156868fb4f27ccd46a3afb0bf3ef8e0cDan Williams if (!thread->dsts[i]) 404b54d5cb9156868fb4f27ccd46a3afb0bf3ef8e0cDan Williams goto err_dstbuf; 405b54d5cb9156868fb4f27ccd46a3afb0bf3ef8e0cDan Williams } 406b54d5cb9156868fb4f27ccd46a3afb0bf3ef8e0cDan Williams thread->dsts[i] = NULL; 407b54d5cb9156868fb4f27ccd46a3afb0bf3ef8e0cDan Williams 408e44e0aa3cfa97cddff01704751a4b25151830c72Dan Williams set_user_nice(current, 10); 409e44e0aa3cfa97cddff01704751a4b25151830c72Dan Williams 410b203bd3f6b9c3db3b1979c2ff79bb2b9be8f03a3Ira Snyder /* 411d1cab34c039584ebe76b04d2f2109e0d87d344e1Bartlomiej Zolnierkiewicz * src and dst buffers are freed by ourselves below 412b203bd3f6b9c3db3b1979c2ff79bb2b9be8f03a3Ira Snyder */ 4130776ae7b89782124ddd72eafe0b1e0fdcdabe32eBartlomiej Zolnierkiewicz flags = DMA_CTRL_ACK | DMA_PREP_INTERRUPT; 4144a776f0aa922a552460192c07b56f4fe9cd82632Haavard Skinnemoen 4150a2ff57d6fba92842272889b4bca447344cd9d36Nicolas Ferre while (!kthread_should_stop() 41615b8a8ea1a87313f1b46ea878c65942fd52147edAndy Shevchenko && !(params->iterations && total_tests >= params->iterations)) { 417b54d5cb9156868fb4f27ccd46a3afb0bf3ef8e0cDan Williams struct dma_async_tx_descriptor *tx = NULL; 418b54d5cb9156868fb4f27ccd46a3afb0bf3ef8e0cDan Williams dma_addr_t dma_srcs[src_cnt]; 419b54d5cb9156868fb4f27ccd46a3afb0bf3ef8e0cDan Williams dma_addr_t dma_dsts[dst_cnt]; 42083544ae9f3991bfc7d5e0fe9a3008cd05a8d57b7Dan Williams u8 align = 0; 421d86be86e9aab221089d72399072511f13fe2a771Atsushi Nemoto 4224a776f0aa922a552460192c07b56f4fe9cd82632Haavard Skinnemoen total_tests++; 4234a776f0aa922a552460192c07b56f4fe9cd82632Haavard Skinnemoen 42483544ae9f3991bfc7d5e0fe9a3008cd05a8d57b7Dan Williams /* honor alignment restrictions */ 42583544ae9f3991bfc7d5e0fe9a3008cd05a8d57b7Dan Williams if (thread->type == DMA_MEMCPY) 42683544ae9f3991bfc7d5e0fe9a3008cd05a8d57b7Dan Williams align = dev->copy_align; 42783544ae9f3991bfc7d5e0fe9a3008cd05a8d57b7Dan Williams else if (thread->type == DMA_XOR) 42883544ae9f3991bfc7d5e0fe9a3008cd05a8d57b7Dan Williams align = dev->xor_align; 42983544ae9f3991bfc7d5e0fe9a3008cd05a8d57b7Dan Williams else if (thread->type == DMA_PQ) 43083544ae9f3991bfc7d5e0fe9a3008cd05a8d57b7Dan Williams align = dev->pq_align; 43183544ae9f3991bfc7d5e0fe9a3008cd05a8d57b7Dan Williams 43215b8a8ea1a87313f1b46ea878c65942fd52147edAndy Shevchenko if (1 << align > params->buf_size) { 433cfe4f2751ef1a5390b56c5d263f90b6ff138ba31Guennadi Liakhovetski pr_err("%u-byte buffer too small for %d-byte alignment\n", 43415b8a8ea1a87313f1b46ea878c65942fd52147edAndy Shevchenko params->buf_size, 1 << align); 435cfe4f2751ef1a5390b56c5d263f90b6ff138ba31Guennadi Liakhovetski break; 436cfe4f2751ef1a5390b56c5d263f90b6ff138ba31Guennadi Liakhovetski } 437cfe4f2751ef1a5390b56c5d263f90b6ff138ba31Guennadi Liakhovetski 43815b8a8ea1a87313f1b46ea878c65942fd52147edAndy Shevchenko len = dmatest_random() % params->buf_size + 1; 43983544ae9f3991bfc7d5e0fe9a3008cd05a8d57b7Dan Williams len = (len >> align) << align; 440cfe4f2751ef1a5390b56c5d263f90b6ff138ba31Guennadi Liakhovetski if (!len) 441cfe4f2751ef1a5390b56c5d263f90b6ff138ba31Guennadi Liakhovetski len = 1 << align; 44215b8a8ea1a87313f1b46ea878c65942fd52147edAndy Shevchenko src_off = dmatest_random() % (params->buf_size - len + 1); 44315b8a8ea1a87313f1b46ea878c65942fd52147edAndy Shevchenko dst_off = dmatest_random() % (params->buf_size - len + 1); 444cfe4f2751ef1a5390b56c5d263f90b6ff138ba31Guennadi Liakhovetski 44583544ae9f3991bfc7d5e0fe9a3008cd05a8d57b7Dan Williams src_off = (src_off >> align) << align; 44683544ae9f3991bfc7d5e0fe9a3008cd05a8d57b7Dan Williams dst_off = (dst_off >> align) << align; 44783544ae9f3991bfc7d5e0fe9a3008cd05a8d57b7Dan Williams 44815b8a8ea1a87313f1b46ea878c65942fd52147edAndy Shevchenko dmatest_init_srcs(thread->srcs, src_off, len, params->buf_size); 44915b8a8ea1a87313f1b46ea878c65942fd52147edAndy Shevchenko dmatest_init_dsts(thread->dsts, dst_off, len, params->buf_size); 4504a776f0aa922a552460192c07b56f4fe9cd82632Haavard Skinnemoen 451b54d5cb9156868fb4f27ccd46a3afb0bf3ef8e0cDan Williams for (i = 0; i < src_cnt; i++) { 452b54d5cb9156868fb4f27ccd46a3afb0bf3ef8e0cDan Williams u8 *buf = thread->srcs[i] + src_off; 453b54d5cb9156868fb4f27ccd46a3afb0bf3ef8e0cDan Williams 454b54d5cb9156868fb4f27ccd46a3afb0bf3ef8e0cDan Williams dma_srcs[i] = dma_map_single(dev->dev, buf, len, 455b54d5cb9156868fb4f27ccd46a3afb0bf3ef8e0cDan Williams DMA_TO_DEVICE); 456afde3be121efcc658e26f8cc71ead04af96d38f9Andy Shevchenko ret = dma_mapping_error(dev->dev, dma_srcs[i]); 457afde3be121efcc658e26f8cc71ead04af96d38f9Andy Shevchenko if (ret) { 458afde3be121efcc658e26f8cc71ead04af96d38f9Andy Shevchenko unmap_src(dev->dev, dma_srcs, len, i); 459872f05c6e9a37e9358fd58eb54deee7337863496Dan Williams result("src mapping error", total_tests, 460872f05c6e9a37e9358fd58eb54deee7337863496Dan Williams src_off, dst_off, len, ret); 461afde3be121efcc658e26f8cc71ead04af96d38f9Andy Shevchenko failed_tests++; 462afde3be121efcc658e26f8cc71ead04af96d38f9Andy Shevchenko continue; 463afde3be121efcc658e26f8cc71ead04af96d38f9Andy Shevchenko } 464b54d5cb9156868fb4f27ccd46a3afb0bf3ef8e0cDan Williams } 465d86be86e9aab221089d72399072511f13fe2a771Atsushi Nemoto /* map with DMA_BIDIRECTIONAL to force writeback/invalidate */ 466b54d5cb9156868fb4f27ccd46a3afb0bf3ef8e0cDan Williams for (i = 0; i < dst_cnt; i++) { 467b54d5cb9156868fb4f27ccd46a3afb0bf3ef8e0cDan Williams dma_dsts[i] = dma_map_single(dev->dev, thread->dsts[i], 46815b8a8ea1a87313f1b46ea878c65942fd52147edAndy Shevchenko params->buf_size, 469b54d5cb9156868fb4f27ccd46a3afb0bf3ef8e0cDan Williams DMA_BIDIRECTIONAL); 470afde3be121efcc658e26f8cc71ead04af96d38f9Andy Shevchenko ret = dma_mapping_error(dev->dev, dma_dsts[i]); 471afde3be121efcc658e26f8cc71ead04af96d38f9Andy Shevchenko if (ret) { 472afde3be121efcc658e26f8cc71ead04af96d38f9Andy Shevchenko unmap_src(dev->dev, dma_srcs, len, src_cnt); 47315b8a8ea1a87313f1b46ea878c65942fd52147edAndy Shevchenko unmap_dst(dev->dev, dma_dsts, params->buf_size, 47415b8a8ea1a87313f1b46ea878c65942fd52147edAndy Shevchenko i); 475872f05c6e9a37e9358fd58eb54deee7337863496Dan Williams result("dst mapping error", total_tests, 476872f05c6e9a37e9358fd58eb54deee7337863496Dan Williams src_off, dst_off, len, ret); 477afde3be121efcc658e26f8cc71ead04af96d38f9Andy Shevchenko failed_tests++; 478afde3be121efcc658e26f8cc71ead04af96d38f9Andy Shevchenko continue; 479afde3be121efcc658e26f8cc71ead04af96d38f9Andy Shevchenko } 480b54d5cb9156868fb4f27ccd46a3afb0bf3ef8e0cDan Williams } 481b54d5cb9156868fb4f27ccd46a3afb0bf3ef8e0cDan Williams 482b54d5cb9156868fb4f27ccd46a3afb0bf3ef8e0cDan Williams if (thread->type == DMA_MEMCPY) 483b54d5cb9156868fb4f27ccd46a3afb0bf3ef8e0cDan Williams tx = dev->device_prep_dma_memcpy(chan, 484b54d5cb9156868fb4f27ccd46a3afb0bf3ef8e0cDan Williams dma_dsts[0] + dst_off, 485b54d5cb9156868fb4f27ccd46a3afb0bf3ef8e0cDan Williams dma_srcs[0], len, 486b54d5cb9156868fb4f27ccd46a3afb0bf3ef8e0cDan Williams flags); 487b54d5cb9156868fb4f27ccd46a3afb0bf3ef8e0cDan Williams else if (thread->type == DMA_XOR) 488b54d5cb9156868fb4f27ccd46a3afb0bf3ef8e0cDan Williams tx = dev->device_prep_dma_xor(chan, 489b54d5cb9156868fb4f27ccd46a3afb0bf3ef8e0cDan Williams dma_dsts[0] + dst_off, 49067b9124f734b22b30d9adf18c39fe795e2901070Dan Williams dma_srcs, src_cnt, 491b54d5cb9156868fb4f27ccd46a3afb0bf3ef8e0cDan Williams len, flags); 49258691d64c44ae41ddf098ecb31e9a994026e3cffDan Williams else if (thread->type == DMA_PQ) { 49358691d64c44ae41ddf098ecb31e9a994026e3cffDan Williams dma_addr_t dma_pq[dst_cnt]; 49458691d64c44ae41ddf098ecb31e9a994026e3cffDan Williams 49558691d64c44ae41ddf098ecb31e9a994026e3cffDan Williams for (i = 0; i < dst_cnt; i++) 49658691d64c44ae41ddf098ecb31e9a994026e3cffDan Williams dma_pq[i] = dma_dsts[i] + dst_off; 49758691d64c44ae41ddf098ecb31e9a994026e3cffDan Williams tx = dev->device_prep_dma_pq(chan, dma_pq, dma_srcs, 49894de648d72c8bc833590523f22386d4babbea988Anatolij Gustschin src_cnt, pq_coefs, 49958691d64c44ae41ddf098ecb31e9a994026e3cffDan Williams len, flags); 50058691d64c44ae41ddf098ecb31e9a994026e3cffDan Williams } 501d86be86e9aab221089d72399072511f13fe2a771Atsushi Nemoto 502d86be86e9aab221089d72399072511f13fe2a771Atsushi Nemoto if (!tx) { 503632fd28326c0cc7be9c51ea0d76d8bec39a695e9Andy Shevchenko unmap_src(dev->dev, dma_srcs, len, src_cnt); 50415b8a8ea1a87313f1b46ea878c65942fd52147edAndy Shevchenko unmap_dst(dev->dev, dma_dsts, params->buf_size, 50515b8a8ea1a87313f1b46ea878c65942fd52147edAndy Shevchenko dst_cnt); 506872f05c6e9a37e9358fd58eb54deee7337863496Dan Williams result("prep error", total_tests, src_off, 507872f05c6e9a37e9358fd58eb54deee7337863496Dan Williams dst_off, len, ret); 508d86be86e9aab221089d72399072511f13fe2a771Atsushi Nemoto msleep(100); 509d86be86e9aab221089d72399072511f13fe2a771Atsushi Nemoto failed_tests++; 510d86be86e9aab221089d72399072511f13fe2a771Atsushi Nemoto continue; 511d86be86e9aab221089d72399072511f13fe2a771Atsushi Nemoto } 512e44e0aa3cfa97cddff01704751a4b25151830c72Dan Williams 513adfa543e7314b36ac55a40019977de6e47946dd7Tejun Heo done.done = false; 514e44e0aa3cfa97cddff01704751a4b25151830c72Dan Williams tx->callback = dmatest_callback; 515adfa543e7314b36ac55a40019977de6e47946dd7Tejun Heo tx->callback_param = &done; 516d86be86e9aab221089d72399072511f13fe2a771Atsushi Nemoto cookie = tx->tx_submit(tx); 517d86be86e9aab221089d72399072511f13fe2a771Atsushi Nemoto 5184a776f0aa922a552460192c07b56f4fe9cd82632Haavard Skinnemoen if (dma_submit_error(cookie)) { 519872f05c6e9a37e9358fd58eb54deee7337863496Dan Williams result("submit error", total_tests, src_off, 520872f05c6e9a37e9358fd58eb54deee7337863496Dan Williams dst_off, len, ret); 5214a776f0aa922a552460192c07b56f4fe9cd82632Haavard Skinnemoen msleep(100); 5224a776f0aa922a552460192c07b56f4fe9cd82632Haavard Skinnemoen failed_tests++; 5234a776f0aa922a552460192c07b56f4fe9cd82632Haavard Skinnemoen continue; 5244a776f0aa922a552460192c07b56f4fe9cd82632Haavard Skinnemoen } 525b54d5cb9156868fb4f27ccd46a3afb0bf3ef8e0cDan Williams dma_async_issue_pending(chan); 5264a776f0aa922a552460192c07b56f4fe9cd82632Haavard Skinnemoen 527bcc567e3115055a9cc256183d72864f01286be22Andy Shevchenko wait_event_freezable_timeout(done_wait, done.done, 52815b8a8ea1a87313f1b46ea878c65942fd52147edAndy Shevchenko msecs_to_jiffies(params->timeout)); 529981ed70d8e4faf3689dbf3c48868a31d5b004d7aGuennadi Liakhovetski 530e44e0aa3cfa97cddff01704751a4b25151830c72Dan Williams status = dma_async_is_tx_complete(chan, cookie, NULL, NULL); 5314a776f0aa922a552460192c07b56f4fe9cd82632Haavard Skinnemoen 532adfa543e7314b36ac55a40019977de6e47946dd7Tejun Heo if (!done.done) { 533adfa543e7314b36ac55a40019977de6e47946dd7Tejun Heo /* 534adfa543e7314b36ac55a40019977de6e47946dd7Tejun Heo * We're leaving the timed out dma operation with 535adfa543e7314b36ac55a40019977de6e47946dd7Tejun Heo * dangling pointer to done_wait. To make this 536adfa543e7314b36ac55a40019977de6e47946dd7Tejun Heo * correct, we'll need to allocate wait_done for 537adfa543e7314b36ac55a40019977de6e47946dd7Tejun Heo * each test iteration and perform "who's gonna 538adfa543e7314b36ac55a40019977de6e47946dd7Tejun Heo * free it this time?" dancing. For now, just 539adfa543e7314b36ac55a40019977de6e47946dd7Tejun Heo * leave it dangling. 540adfa543e7314b36ac55a40019977de6e47946dd7Tejun Heo */ 541872f05c6e9a37e9358fd58eb54deee7337863496Dan Williams result("test timed out", total_tests, src_off, dst_off, 542872f05c6e9a37e9358fd58eb54deee7337863496Dan Williams len, 0); 543e44e0aa3cfa97cddff01704751a4b25151830c72Dan Williams failed_tests++; 544e44e0aa3cfa97cddff01704751a4b25151830c72Dan Williams continue; 545e44e0aa3cfa97cddff01704751a4b25151830c72Dan Williams } else if (status != DMA_SUCCESS) { 546872f05c6e9a37e9358fd58eb54deee7337863496Dan Williams result(status == DMA_ERROR ? 547872f05c6e9a37e9358fd58eb54deee7337863496Dan Williams "completion error status" : 548872f05c6e9a37e9358fd58eb54deee7337863496Dan Williams "completion busy status", total_tests, src_off, 549872f05c6e9a37e9358fd58eb54deee7337863496Dan Williams dst_off, len, ret); 5504a776f0aa922a552460192c07b56f4fe9cd82632Haavard Skinnemoen failed_tests++; 5514a776f0aa922a552460192c07b56f4fe9cd82632Haavard Skinnemoen continue; 5524a776f0aa922a552460192c07b56f4fe9cd82632Haavard Skinnemoen } 553e44e0aa3cfa97cddff01704751a4b25151830c72Dan Williams 554d1cab34c039584ebe76b04d2f2109e0d87d344e1Bartlomiej Zolnierkiewicz /* Unmap by myself */ 555d1cab34c039584ebe76b04d2f2109e0d87d344e1Bartlomiej Zolnierkiewicz unmap_src(dev->dev, dma_srcs, len, src_cnt); 55615b8a8ea1a87313f1b46ea878c65942fd52147edAndy Shevchenko unmap_dst(dev->dev, dma_dsts, params->buf_size, dst_cnt); 5574a776f0aa922a552460192c07b56f4fe9cd82632Haavard Skinnemoen 5584a776f0aa922a552460192c07b56f4fe9cd82632Haavard Skinnemoen error_count = 0; 5594a776f0aa922a552460192c07b56f4fe9cd82632Haavard Skinnemoen 560872f05c6e9a37e9358fd58eb54deee7337863496Dan Williams pr_debug("%s: verifying source buffer...\n", current->comm); 5617b61017822cdff9c18ae70005cf52d84e8dafe5dDan Williams error_count += dmatest_verify(thread->srcs, 0, src_off, 5624a776f0aa922a552460192c07b56f4fe9cd82632Haavard Skinnemoen 0, PATTERN_SRC, true); 5637b61017822cdff9c18ae70005cf52d84e8dafe5dDan Williams error_count += dmatest_verify(thread->srcs, src_off, 5647b61017822cdff9c18ae70005cf52d84e8dafe5dDan Williams src_off + len, src_off, 5657b61017822cdff9c18ae70005cf52d84e8dafe5dDan Williams PATTERN_SRC | PATTERN_COPY, true); 5667b61017822cdff9c18ae70005cf52d84e8dafe5dDan Williams error_count += dmatest_verify(thread->srcs, src_off + len, 5677b61017822cdff9c18ae70005cf52d84e8dafe5dDan Williams params->buf_size, src_off + len, 5687b61017822cdff9c18ae70005cf52d84e8dafe5dDan Williams PATTERN_SRC, true); 5697b61017822cdff9c18ae70005cf52d84e8dafe5dDan Williams 570872f05c6e9a37e9358fd58eb54deee7337863496Dan Williams pr_debug("%s: verifying dest buffer...\n", current->comm); 5717b61017822cdff9c18ae70005cf52d84e8dafe5dDan Williams error_count += dmatest_verify(thread->dsts, 0, dst_off, 5724a776f0aa922a552460192c07b56f4fe9cd82632Haavard Skinnemoen 0, PATTERN_DST, false); 5737b61017822cdff9c18ae70005cf52d84e8dafe5dDan Williams error_count += dmatest_verify(thread->dsts, dst_off, 5747b61017822cdff9c18ae70005cf52d84e8dafe5dDan Williams dst_off + len, src_off, 5757b61017822cdff9c18ae70005cf52d84e8dafe5dDan Williams PATTERN_SRC | PATTERN_COPY, false); 5767b61017822cdff9c18ae70005cf52d84e8dafe5dDan Williams error_count += dmatest_verify(thread->dsts, dst_off + len, 5777b61017822cdff9c18ae70005cf52d84e8dafe5dDan Williams params->buf_size, dst_off + len, 5787b61017822cdff9c18ae70005cf52d84e8dafe5dDan Williams PATTERN_DST, false); 5794a776f0aa922a552460192c07b56f4fe9cd82632Haavard Skinnemoen 5804a776f0aa922a552460192c07b56f4fe9cd82632Haavard Skinnemoen if (error_count) { 581872f05c6e9a37e9358fd58eb54deee7337863496Dan Williams result("data error", total_tests, src_off, dst_off, 582872f05c6e9a37e9358fd58eb54deee7337863496Dan Williams len, error_count); 5834a776f0aa922a552460192c07b56f4fe9cd82632Haavard Skinnemoen failed_tests++; 5844a776f0aa922a552460192c07b56f4fe9cd82632Haavard Skinnemoen } else { 585872f05c6e9a37e9358fd58eb54deee7337863496Dan Williams dbg_result("test passed", total_tests, src_off, dst_off, 586872f05c6e9a37e9358fd58eb54deee7337863496Dan Williams len, 0); 5874a776f0aa922a552460192c07b56f4fe9cd82632Haavard Skinnemoen } 5884a776f0aa922a552460192c07b56f4fe9cd82632Haavard Skinnemoen } 5894a776f0aa922a552460192c07b56f4fe9cd82632Haavard Skinnemoen 5904a776f0aa922a552460192c07b56f4fe9cd82632Haavard Skinnemoen ret = 0; 591b54d5cb9156868fb4f27ccd46a3afb0bf3ef8e0cDan Williams for (i = 0; thread->dsts[i]; i++) 592b54d5cb9156868fb4f27ccd46a3afb0bf3ef8e0cDan Williams kfree(thread->dsts[i]); 5934a776f0aa922a552460192c07b56f4fe9cd82632Haavard Skinnemoenerr_dstbuf: 594b54d5cb9156868fb4f27ccd46a3afb0bf3ef8e0cDan Williams kfree(thread->dsts); 595b54d5cb9156868fb4f27ccd46a3afb0bf3ef8e0cDan Williamserr_dsts: 596b54d5cb9156868fb4f27ccd46a3afb0bf3ef8e0cDan Williams for (i = 0; thread->srcs[i]; i++) 597b54d5cb9156868fb4f27ccd46a3afb0bf3ef8e0cDan Williams kfree(thread->srcs[i]); 5984a776f0aa922a552460192c07b56f4fe9cd82632Haavard Skinnemoenerr_srcbuf: 599b54d5cb9156868fb4f27ccd46a3afb0bf3ef8e0cDan Williams kfree(thread->srcs); 600b54d5cb9156868fb4f27ccd46a3afb0bf3ef8e0cDan Williamserr_srcs: 601945b5af3cedcdfed6d2d940e53cd19933bb57386Andy Shevchenko kfree(pq_coefs); 602945b5af3cedcdfed6d2d940e53cd19933bb57386Andy Shevchenkoerr_thread_type: 603872f05c6e9a37e9358fd58eb54deee7337863496Dan Williams pr_info("%s: terminating after %u tests, %u failures (status %d)\n", 604872f05c6e9a37e9358fd58eb54deee7337863496Dan Williams current->comm, total_tests, failed_tests, ret); 6050a2ff57d6fba92842272889b4bca447344cd9d36Nicolas Ferre 6069704efaa52ab18eb3504c4e0bc421c1d01b7981aViresh Kumar /* terminate all transfers on specified channels */ 6075e034f7b659be9d94e64aaaa985ab530dd847fdbShiraz Hashim if (ret) 6085e034f7b659be9d94e64aaaa985ab530dd847fdbShiraz Hashim dmaengine_terminate_all(chan); 6095e034f7b659be9d94e64aaaa985ab530dd847fdbShiraz Hashim 6103e5ccd866fdf3a1e1d4d2c08c81f861ad6798d32Andy Shevchenko thread->done = true; 6113e5ccd866fdf3a1e1d4d2c08c81f861ad6798d32Andy Shevchenko 61215b8a8ea1a87313f1b46ea878c65942fd52147edAndy Shevchenko if (params->iterations > 0) 6130a2ff57d6fba92842272889b4bca447344cd9d36Nicolas Ferre while (!kthread_should_stop()) { 614b953df7c70740cd7593072ebec77a8f658505630Yong Zhang DECLARE_WAIT_QUEUE_HEAD_ONSTACK(wait_dmatest_exit); 6150a2ff57d6fba92842272889b4bca447344cd9d36Nicolas Ferre interruptible_sleep_on(&wait_dmatest_exit); 6160a2ff57d6fba92842272889b4bca447344cd9d36Nicolas Ferre } 6170a2ff57d6fba92842272889b4bca447344cd9d36Nicolas Ferre 6184a776f0aa922a552460192c07b56f4fe9cd82632Haavard Skinnemoen return ret; 6194a776f0aa922a552460192c07b56f4fe9cd82632Haavard Skinnemoen} 6204a776f0aa922a552460192c07b56f4fe9cd82632Haavard Skinnemoen 6214a776f0aa922a552460192c07b56f4fe9cd82632Haavard Skinnemoenstatic void dmatest_cleanup_channel(struct dmatest_chan *dtc) 6224a776f0aa922a552460192c07b56f4fe9cd82632Haavard Skinnemoen{ 6234a776f0aa922a552460192c07b56f4fe9cd82632Haavard Skinnemoen struct dmatest_thread *thread; 6244a776f0aa922a552460192c07b56f4fe9cd82632Haavard Skinnemoen struct dmatest_thread *_thread; 6254a776f0aa922a552460192c07b56f4fe9cd82632Haavard Skinnemoen int ret; 6264a776f0aa922a552460192c07b56f4fe9cd82632Haavard Skinnemoen 6274a776f0aa922a552460192c07b56f4fe9cd82632Haavard Skinnemoen list_for_each_entry_safe(thread, _thread, &dtc->threads, node) { 6284a776f0aa922a552460192c07b56f4fe9cd82632Haavard Skinnemoen ret = kthread_stop(thread->task); 6290adff800662f52d0ffc3e420db231769cb3fff13Dan Williams pr_debug("thread %s exited with status %d\n", 6300adff800662f52d0ffc3e420db231769cb3fff13Dan Williams thread->task->comm, ret); 6314a776f0aa922a552460192c07b56f4fe9cd82632Haavard Skinnemoen list_del(&thread->node); 6324a776f0aa922a552460192c07b56f4fe9cd82632Haavard Skinnemoen kfree(thread); 6334a776f0aa922a552460192c07b56f4fe9cd82632Haavard Skinnemoen } 6349704efaa52ab18eb3504c4e0bc421c1d01b7981aViresh Kumar 6359704efaa52ab18eb3504c4e0bc421c1d01b7981aViresh Kumar /* terminate all transfers on specified channels */ 636944ea4dd38b8575e30a5699633c81945bff1864dJon Mason dmaengine_terminate_all(dtc->chan); 6379704efaa52ab18eb3504c4e0bc421c1d01b7981aViresh Kumar 6384a776f0aa922a552460192c07b56f4fe9cd82632Haavard Skinnemoen kfree(dtc); 6394a776f0aa922a552460192c07b56f4fe9cd82632Haavard Skinnemoen} 6404a776f0aa922a552460192c07b56f4fe9cd82632Haavard Skinnemoen 641e03e93a976d0f0da63f02fd3384c4b99cac8d715Andy Shevchenkostatic int dmatest_add_threads(struct dmatest_info *info, 642e03e93a976d0f0da63f02fd3384c4b99cac8d715Andy Shevchenko struct dmatest_chan *dtc, enum dma_transaction_type type) 6434a776f0aa922a552460192c07b56f4fe9cd82632Haavard Skinnemoen{ 64415b8a8ea1a87313f1b46ea878c65942fd52147edAndy Shevchenko struct dmatest_params *params = &info->params; 645b54d5cb9156868fb4f27ccd46a3afb0bf3ef8e0cDan Williams struct dmatest_thread *thread; 646b54d5cb9156868fb4f27ccd46a3afb0bf3ef8e0cDan Williams struct dma_chan *chan = dtc->chan; 647b54d5cb9156868fb4f27ccd46a3afb0bf3ef8e0cDan Williams char *op; 648b54d5cb9156868fb4f27ccd46a3afb0bf3ef8e0cDan Williams unsigned int i; 6494a776f0aa922a552460192c07b56f4fe9cd82632Haavard Skinnemoen 650b54d5cb9156868fb4f27ccd46a3afb0bf3ef8e0cDan Williams if (type == DMA_MEMCPY) 651b54d5cb9156868fb4f27ccd46a3afb0bf3ef8e0cDan Williams op = "copy"; 652b54d5cb9156868fb4f27ccd46a3afb0bf3ef8e0cDan Williams else if (type == DMA_XOR) 653b54d5cb9156868fb4f27ccd46a3afb0bf3ef8e0cDan Williams op = "xor"; 65458691d64c44ae41ddf098ecb31e9a994026e3cffDan Williams else if (type == DMA_PQ) 65558691d64c44ae41ddf098ecb31e9a994026e3cffDan Williams op = "pq"; 656b54d5cb9156868fb4f27ccd46a3afb0bf3ef8e0cDan Williams else 657b54d5cb9156868fb4f27ccd46a3afb0bf3ef8e0cDan Williams return -EINVAL; 6584a776f0aa922a552460192c07b56f4fe9cd82632Haavard Skinnemoen 65915b8a8ea1a87313f1b46ea878c65942fd52147edAndy Shevchenko for (i = 0; i < params->threads_per_chan; i++) { 6604a776f0aa922a552460192c07b56f4fe9cd82632Haavard Skinnemoen thread = kzalloc(sizeof(struct dmatest_thread), GFP_KERNEL); 6614a776f0aa922a552460192c07b56f4fe9cd82632Haavard Skinnemoen if (!thread) { 6620adff800662f52d0ffc3e420db231769cb3fff13Dan Williams pr_warn("No memory for %s-%s%u\n", 6630adff800662f52d0ffc3e420db231769cb3fff13Dan Williams dma_chan_name(chan), op, i); 6644a776f0aa922a552460192c07b56f4fe9cd82632Haavard Skinnemoen break; 6654a776f0aa922a552460192c07b56f4fe9cd82632Haavard Skinnemoen } 666e03e93a976d0f0da63f02fd3384c4b99cac8d715Andy Shevchenko thread->info = info; 6674a776f0aa922a552460192c07b56f4fe9cd82632Haavard Skinnemoen thread->chan = dtc->chan; 668b54d5cb9156868fb4f27ccd46a3afb0bf3ef8e0cDan Williams thread->type = type; 6694a776f0aa922a552460192c07b56f4fe9cd82632Haavard Skinnemoen smp_wmb(); 670b54d5cb9156868fb4f27ccd46a3afb0bf3ef8e0cDan Williams thread->task = kthread_run(dmatest_func, thread, "%s-%s%u", 671b54d5cb9156868fb4f27ccd46a3afb0bf3ef8e0cDan Williams dma_chan_name(chan), op, i); 6724a776f0aa922a552460192c07b56f4fe9cd82632Haavard Skinnemoen if (IS_ERR(thread->task)) { 6730adff800662f52d0ffc3e420db231769cb3fff13Dan Williams pr_warn("Failed to run thread %s-%s%u\n", 6740adff800662f52d0ffc3e420db231769cb3fff13Dan Williams dma_chan_name(chan), op, i); 6754a776f0aa922a552460192c07b56f4fe9cd82632Haavard Skinnemoen kfree(thread); 6764a776f0aa922a552460192c07b56f4fe9cd82632Haavard Skinnemoen break; 6774a776f0aa922a552460192c07b56f4fe9cd82632Haavard Skinnemoen } 6784a776f0aa922a552460192c07b56f4fe9cd82632Haavard Skinnemoen 6794a776f0aa922a552460192c07b56f4fe9cd82632Haavard Skinnemoen /* srcbuf and dstbuf are allocated by the thread itself */ 6804a776f0aa922a552460192c07b56f4fe9cd82632Haavard Skinnemoen 6814a776f0aa922a552460192c07b56f4fe9cd82632Haavard Skinnemoen list_add_tail(&thread->node, &dtc->threads); 6824a776f0aa922a552460192c07b56f4fe9cd82632Haavard Skinnemoen } 6834a776f0aa922a552460192c07b56f4fe9cd82632Haavard Skinnemoen 684b54d5cb9156868fb4f27ccd46a3afb0bf3ef8e0cDan Williams return i; 685b54d5cb9156868fb4f27ccd46a3afb0bf3ef8e0cDan Williams} 686b54d5cb9156868fb4f27ccd46a3afb0bf3ef8e0cDan Williams 687e03e93a976d0f0da63f02fd3384c4b99cac8d715Andy Shevchenkostatic int dmatest_add_channel(struct dmatest_info *info, 688e03e93a976d0f0da63f02fd3384c4b99cac8d715Andy Shevchenko struct dma_chan *chan) 689b54d5cb9156868fb4f27ccd46a3afb0bf3ef8e0cDan Williams{ 690b54d5cb9156868fb4f27ccd46a3afb0bf3ef8e0cDan Williams struct dmatest_chan *dtc; 691b54d5cb9156868fb4f27ccd46a3afb0bf3ef8e0cDan Williams struct dma_device *dma_dev = chan->device; 692b54d5cb9156868fb4f27ccd46a3afb0bf3ef8e0cDan Williams unsigned int thread_count = 0; 693b9033e682e86f3c6a66763f9b6a3935c5c64e145Kulikov Vasiliy int cnt; 694b54d5cb9156868fb4f27ccd46a3afb0bf3ef8e0cDan Williams 695b54d5cb9156868fb4f27ccd46a3afb0bf3ef8e0cDan Williams dtc = kmalloc(sizeof(struct dmatest_chan), GFP_KERNEL); 696b54d5cb9156868fb4f27ccd46a3afb0bf3ef8e0cDan Williams if (!dtc) { 6970adff800662f52d0ffc3e420db231769cb3fff13Dan Williams pr_warn("No memory for %s\n", dma_chan_name(chan)); 698b54d5cb9156868fb4f27ccd46a3afb0bf3ef8e0cDan Williams return -ENOMEM; 699b54d5cb9156868fb4f27ccd46a3afb0bf3ef8e0cDan Williams } 700b54d5cb9156868fb4f27ccd46a3afb0bf3ef8e0cDan Williams 701b54d5cb9156868fb4f27ccd46a3afb0bf3ef8e0cDan Williams dtc->chan = chan; 702b54d5cb9156868fb4f27ccd46a3afb0bf3ef8e0cDan Williams INIT_LIST_HEAD(&dtc->threads); 703b54d5cb9156868fb4f27ccd46a3afb0bf3ef8e0cDan Williams 704b54d5cb9156868fb4f27ccd46a3afb0bf3ef8e0cDan Williams if (dma_has_cap(DMA_MEMCPY, dma_dev->cap_mask)) { 705e03e93a976d0f0da63f02fd3384c4b99cac8d715Andy Shevchenko cnt = dmatest_add_threads(info, dtc, DMA_MEMCPY); 706f1aef8b6e6abf32a3a269542f95a19e2cb319f6cNicolas Ferre thread_count += cnt > 0 ? cnt : 0; 707b54d5cb9156868fb4f27ccd46a3afb0bf3ef8e0cDan Williams } 708b54d5cb9156868fb4f27ccd46a3afb0bf3ef8e0cDan Williams if (dma_has_cap(DMA_XOR, dma_dev->cap_mask)) { 709e03e93a976d0f0da63f02fd3384c4b99cac8d715Andy Shevchenko cnt = dmatest_add_threads(info, dtc, DMA_XOR); 710f1aef8b6e6abf32a3a269542f95a19e2cb319f6cNicolas Ferre thread_count += cnt > 0 ? cnt : 0; 711b54d5cb9156868fb4f27ccd46a3afb0bf3ef8e0cDan Williams } 71258691d64c44ae41ddf098ecb31e9a994026e3cffDan Williams if (dma_has_cap(DMA_PQ, dma_dev->cap_mask)) { 713e03e93a976d0f0da63f02fd3384c4b99cac8d715Andy Shevchenko cnt = dmatest_add_threads(info, dtc, DMA_PQ); 714d07a74a546981a09ba490936645fbf0d1340b96cDr. David Alan Gilbert thread_count += cnt > 0 ? cnt : 0; 71558691d64c44ae41ddf098ecb31e9a994026e3cffDan Williams } 716b54d5cb9156868fb4f27ccd46a3afb0bf3ef8e0cDan Williams 7170adff800662f52d0ffc3e420db231769cb3fff13Dan Williams pr_info("Started %u threads using %s\n", 718b54d5cb9156868fb4f27ccd46a3afb0bf3ef8e0cDan Williams thread_count, dma_chan_name(chan)); 7194a776f0aa922a552460192c07b56f4fe9cd82632Haavard Skinnemoen 720838cc704ce5c8ab2a6d64d1324e37e040fcae3d8Andy Shevchenko list_add_tail(&dtc->node, &info->channels); 721838cc704ce5c8ab2a6d64d1324e37e040fcae3d8Andy Shevchenko info->nr_channels++; 7224a776f0aa922a552460192c07b56f4fe9cd82632Haavard Skinnemoen 72333df8ca068123457db56c316946a3c0e4ef787d6Dan Williams return 0; 7244a776f0aa922a552460192c07b56f4fe9cd82632Haavard Skinnemoen} 7254a776f0aa922a552460192c07b56f4fe9cd82632Haavard Skinnemoen 7267dd602510128d7a64b11ff3b7d4f30ac8e3946ceDan Williamsstatic bool filter(struct dma_chan *chan, void *param) 7274a776f0aa922a552460192c07b56f4fe9cd82632Haavard Skinnemoen{ 72815b8a8ea1a87313f1b46ea878c65942fd52147edAndy Shevchenko struct dmatest_params *params = param; 729e03e93a976d0f0da63f02fd3384c4b99cac8d715Andy Shevchenko 73015b8a8ea1a87313f1b46ea878c65942fd52147edAndy Shevchenko if (!dmatest_match_channel(params, chan) || 73115b8a8ea1a87313f1b46ea878c65942fd52147edAndy Shevchenko !dmatest_match_device(params, chan->device)) 7327dd602510128d7a64b11ff3b7d4f30ac8e3946ceDan Williams return false; 73333df8ca068123457db56c316946a3c0e4ef787d6Dan Williams else 7347dd602510128d7a64b11ff3b7d4f30ac8e3946ceDan Williams return true; 7354a776f0aa922a552460192c07b56f4fe9cd82632Haavard Skinnemoen} 7364a776f0aa922a552460192c07b56f4fe9cd82632Haavard Skinnemoen 737a9e554957de406d6adc581731f571b8a1503f6b0Dan Williamsstatic void request_channels(struct dmatest_info *info, 738a9e554957de406d6adc581731f571b8a1503f6b0Dan Williams enum dma_transaction_type type) 7394a776f0aa922a552460192c07b56f4fe9cd82632Haavard Skinnemoen{ 74033df8ca068123457db56c316946a3c0e4ef787d6Dan Williams dma_cap_mask_t mask; 741a310d037b8d06755c62bb4878c00d19490af5550Dan Williams 74233df8ca068123457db56c316946a3c0e4ef787d6Dan Williams dma_cap_zero(mask); 743a9e554957de406d6adc581731f571b8a1503f6b0Dan Williams dma_cap_set(type, mask); 74433df8ca068123457db56c316946a3c0e4ef787d6Dan Williams for (;;) { 745a9e554957de406d6adc581731f571b8a1503f6b0Dan Williams struct dmatest_params *params = &info->params; 746a9e554957de406d6adc581731f571b8a1503f6b0Dan Williams struct dma_chan *chan; 747a9e554957de406d6adc581731f571b8a1503f6b0Dan Williams 74815b8a8ea1a87313f1b46ea878c65942fd52147edAndy Shevchenko chan = dma_request_channel(mask, filter, params); 74933df8ca068123457db56c316946a3c0e4ef787d6Dan Williams if (chan) { 750a9e554957de406d6adc581731f571b8a1503f6b0Dan Williams if (dmatest_add_channel(info, chan)) { 75133df8ca068123457db56c316946a3c0e4ef787d6Dan Williams dma_release_channel(chan); 75233df8ca068123457db56c316946a3c0e4ef787d6Dan Williams break; /* add_channel failed, punt */ 75333df8ca068123457db56c316946a3c0e4ef787d6Dan Williams } 75433df8ca068123457db56c316946a3c0e4ef787d6Dan Williams } else 75533df8ca068123457db56c316946a3c0e4ef787d6Dan Williams break; /* no more channels available */ 75615b8a8ea1a87313f1b46ea878c65942fd52147edAndy Shevchenko if (params->max_channels && 75715b8a8ea1a87313f1b46ea878c65942fd52147edAndy Shevchenko info->nr_channels >= params->max_channels) 75833df8ca068123457db56c316946a3c0e4ef787d6Dan Williams break; /* we have all we need */ 75933df8ca068123457db56c316946a3c0e4ef787d6Dan Williams } 7604a776f0aa922a552460192c07b56f4fe9cd82632Haavard Skinnemoen} 7614a776f0aa922a552460192c07b56f4fe9cd82632Haavard Skinnemoen 762a9e554957de406d6adc581731f571b8a1503f6b0Dan Williamsstatic void run_threaded_test(struct dmatest_info *info) 763a9e554957de406d6adc581731f571b8a1503f6b0Dan Williams{ 764a9e554957de406d6adc581731f571b8a1503f6b0Dan Williams struct dmatest_params *params = &info->params; 765a9e554957de406d6adc581731f571b8a1503f6b0Dan Williams 766a9e554957de406d6adc581731f571b8a1503f6b0Dan Williams /* Copy test parameters */ 767a9e554957de406d6adc581731f571b8a1503f6b0Dan Williams params->buf_size = test_buf_size; 768a9e554957de406d6adc581731f571b8a1503f6b0Dan Williams strlcpy(params->channel, strim(test_channel), sizeof(params->channel)); 769a9e554957de406d6adc581731f571b8a1503f6b0Dan Williams strlcpy(params->device, strim(test_device), sizeof(params->device)); 770a9e554957de406d6adc581731f571b8a1503f6b0Dan Williams params->threads_per_chan = threads_per_chan; 771a9e554957de406d6adc581731f571b8a1503f6b0Dan Williams params->max_channels = max_channels; 772a9e554957de406d6adc581731f571b8a1503f6b0Dan Williams params->iterations = iterations; 773a9e554957de406d6adc581731f571b8a1503f6b0Dan Williams params->xor_sources = xor_sources; 774a9e554957de406d6adc581731f571b8a1503f6b0Dan Williams params->pq_sources = pq_sources; 775a9e554957de406d6adc581731f571b8a1503f6b0Dan Williams params->timeout = timeout; 776a9e554957de406d6adc581731f571b8a1503f6b0Dan Williams 777a9e554957de406d6adc581731f571b8a1503f6b0Dan Williams request_channels(info, DMA_MEMCPY); 778a9e554957de406d6adc581731f571b8a1503f6b0Dan Williams request_channels(info, DMA_XOR); 779a9e554957de406d6adc581731f571b8a1503f6b0Dan Williams request_channels(info, DMA_PQ); 780a9e554957de406d6adc581731f571b8a1503f6b0Dan Williams} 781851b7e16a07dfda6178d4e35fea9a9e3eb8954aeAndy Shevchenko 782a310d037b8d06755c62bb4878c00d19490af5550Dan Williamsstatic void stop_threaded_test(struct dmatest_info *info) 7834a776f0aa922a552460192c07b56f4fe9cd82632Haavard Skinnemoen{ 78433df8ca068123457db56c316946a3c0e4ef787d6Dan Williams struct dmatest_chan *dtc, *_dtc; 7857cbd4877e5b167b56a3d6033b926a9f925186e12Dan Williams struct dma_chan *chan; 78633df8ca068123457db56c316946a3c0e4ef787d6Dan Williams 787838cc704ce5c8ab2a6d64d1324e37e040fcae3d8Andy Shevchenko list_for_each_entry_safe(dtc, _dtc, &info->channels, node) { 78833df8ca068123457db56c316946a3c0e4ef787d6Dan Williams list_del(&dtc->node); 7897cbd4877e5b167b56a3d6033b926a9f925186e12Dan Williams chan = dtc->chan; 79033df8ca068123457db56c316946a3c0e4ef787d6Dan Williams dmatest_cleanup_channel(dtc); 7910adff800662f52d0ffc3e420db231769cb3fff13Dan Williams pr_debug("dropped channel %s\n", dma_chan_name(chan)); 7927cbd4877e5b167b56a3d6033b926a9f925186e12Dan Williams dma_release_channel(chan); 79333df8ca068123457db56c316946a3c0e4ef787d6Dan Williams } 794838cc704ce5c8ab2a6d64d1324e37e040fcae3d8Andy Shevchenko 795838cc704ce5c8ab2a6d64d1324e37e040fcae3d8Andy Shevchenko info->nr_channels = 0; 7964a776f0aa922a552460192c07b56f4fe9cd82632Haavard Skinnemoen} 797e03e93a976d0f0da63f02fd3384c4b99cac8d715Andy Shevchenko 798a9e554957de406d6adc581731f571b8a1503f6b0Dan Williamsstatic void restart_threaded_test(struct dmatest_info *info, bool run) 799851b7e16a07dfda6178d4e35fea9a9e3eb8954aeAndy Shevchenko{ 800a310d037b8d06755c62bb4878c00d19490af5550Dan Williams /* we might be called early to set run=, defer running until all 801a310d037b8d06755c62bb4878c00d19490af5550Dan Williams * parameters have been evaluated 802a310d037b8d06755c62bb4878c00d19490af5550Dan Williams */ 803a310d037b8d06755c62bb4878c00d19490af5550Dan Williams if (!info->did_init) 804a9e554957de406d6adc581731f571b8a1503f6b0Dan Williams return; 805851b7e16a07dfda6178d4e35fea9a9e3eb8954aeAndy Shevchenko 806a310d037b8d06755c62bb4878c00d19490af5550Dan Williams /* Stop any running test first */ 807a310d037b8d06755c62bb4878c00d19490af5550Dan Williams stop_threaded_test(info); 808851b7e16a07dfda6178d4e35fea9a9e3eb8954aeAndy Shevchenko 809851b7e16a07dfda6178d4e35fea9a9e3eb8954aeAndy Shevchenko /* Run test with new parameters */ 810a9e554957de406d6adc581731f571b8a1503f6b0Dan Williams run_threaded_test(info); 811bcc567e3115055a9cc256183d72864f01286be22Andy Shevchenko} 812bcc567e3115055a9cc256183d72864f01286be22Andy Shevchenko 813a310d037b8d06755c62bb4878c00d19490af5550Dan Williamsstatic bool is_threaded_test_run(struct dmatest_info *info) 814bcc567e3115055a9cc256183d72864f01286be22Andy Shevchenko{ 815bcc567e3115055a9cc256183d72864f01286be22Andy Shevchenko struct dmatest_chan *dtc; 816bcc567e3115055a9cc256183d72864f01286be22Andy Shevchenko 817bcc567e3115055a9cc256183d72864f01286be22Andy Shevchenko list_for_each_entry(dtc, &info->channels, node) { 818bcc567e3115055a9cc256183d72864f01286be22Andy Shevchenko struct dmatest_thread *thread; 819bcc567e3115055a9cc256183d72864f01286be22Andy Shevchenko 820bcc567e3115055a9cc256183d72864f01286be22Andy Shevchenko list_for_each_entry(thread, &dtc->threads, node) { 821bcc567e3115055a9cc256183d72864f01286be22Andy Shevchenko if (!thread->done) 822bcc567e3115055a9cc256183d72864f01286be22Andy Shevchenko return true; 823bcc567e3115055a9cc256183d72864f01286be22Andy Shevchenko } 824851b7e16a07dfda6178d4e35fea9a9e3eb8954aeAndy Shevchenko } 825851b7e16a07dfda6178d4e35fea9a9e3eb8954aeAndy Shevchenko 826bcc567e3115055a9cc256183d72864f01286be22Andy Shevchenko return false; 827851b7e16a07dfda6178d4e35fea9a9e3eb8954aeAndy Shevchenko} 828851b7e16a07dfda6178d4e35fea9a9e3eb8954aeAndy Shevchenko 829a310d037b8d06755c62bb4878c00d19490af5550Dan Williamsstatic int dmatest_run_get(char *val, const struct kernel_param *kp) 830851b7e16a07dfda6178d4e35fea9a9e3eb8954aeAndy Shevchenko{ 831a310d037b8d06755c62bb4878c00d19490af5550Dan Williams struct dmatest_info *info = &test_info; 832851b7e16a07dfda6178d4e35fea9a9e3eb8954aeAndy Shevchenko 833851b7e16a07dfda6178d4e35fea9a9e3eb8954aeAndy Shevchenko mutex_lock(&info->lock); 834a310d037b8d06755c62bb4878c00d19490af5550Dan Williams if (is_threaded_test_run(info)) { 835a310d037b8d06755c62bb4878c00d19490af5550Dan Williams dmatest_run = true; 8363e5ccd866fdf3a1e1d4d2c08c81f861ad6798d32Andy Shevchenko } else { 837a310d037b8d06755c62bb4878c00d19490af5550Dan Williams stop_threaded_test(info); 838a310d037b8d06755c62bb4878c00d19490af5550Dan Williams dmatest_run = false; 8393e5ccd866fdf3a1e1d4d2c08c81f861ad6798d32Andy Shevchenko } 840851b7e16a07dfda6178d4e35fea9a9e3eb8954aeAndy Shevchenko mutex_unlock(&info->lock); 841a310d037b8d06755c62bb4878c00d19490af5550Dan Williams 842a310d037b8d06755c62bb4878c00d19490af5550Dan Williams return param_get_bool(val, kp); 843851b7e16a07dfda6178d4e35fea9a9e3eb8954aeAndy Shevchenko} 844851b7e16a07dfda6178d4e35fea9a9e3eb8954aeAndy Shevchenko 845a310d037b8d06755c62bb4878c00d19490af5550Dan Williamsstatic int dmatest_run_set(const char *val, const struct kernel_param *kp) 846851b7e16a07dfda6178d4e35fea9a9e3eb8954aeAndy Shevchenko{ 847a310d037b8d06755c62bb4878c00d19490af5550Dan Williams struct dmatest_info *info = &test_info; 848a310d037b8d06755c62bb4878c00d19490af5550Dan Williams int ret; 849bcc567e3115055a9cc256183d72864f01286be22Andy Shevchenko 850a310d037b8d06755c62bb4878c00d19490af5550Dan Williams mutex_lock(&info->lock); 851a310d037b8d06755c62bb4878c00d19490af5550Dan Williams ret = param_set_bool(val, kp); 852a310d037b8d06755c62bb4878c00d19490af5550Dan Williams if (ret) { 853851b7e16a07dfda6178d4e35fea9a9e3eb8954aeAndy Shevchenko mutex_unlock(&info->lock); 854a310d037b8d06755c62bb4878c00d19490af5550Dan Williams return ret; 855851b7e16a07dfda6178d4e35fea9a9e3eb8954aeAndy Shevchenko } 856851b7e16a07dfda6178d4e35fea9a9e3eb8954aeAndy Shevchenko 857a310d037b8d06755c62bb4878c00d19490af5550Dan Williams if (is_threaded_test_run(info)) 858a310d037b8d06755c62bb4878c00d19490af5550Dan Williams ret = -EBUSY; 859a310d037b8d06755c62bb4878c00d19490af5550Dan Williams else if (dmatest_run) 860a9e554957de406d6adc581731f571b8a1503f6b0Dan Williams restart_threaded_test(info, dmatest_run); 861851b7e16a07dfda6178d4e35fea9a9e3eb8954aeAndy Shevchenko 862a310d037b8d06755c62bb4878c00d19490af5550Dan Williams mutex_unlock(&info->lock); 863851b7e16a07dfda6178d4e35fea9a9e3eb8954aeAndy Shevchenko 864a310d037b8d06755c62bb4878c00d19490af5550Dan Williams return ret; 865851b7e16a07dfda6178d4e35fea9a9e3eb8954aeAndy Shevchenko} 866851b7e16a07dfda6178d4e35fea9a9e3eb8954aeAndy Shevchenko 867e03e93a976d0f0da63f02fd3384c4b99cac8d715Andy Shevchenkostatic int __init dmatest_init(void) 868e03e93a976d0f0da63f02fd3384c4b99cac8d715Andy Shevchenko{ 869e03e93a976d0f0da63f02fd3384c4b99cac8d715Andy Shevchenko struct dmatest_info *info = &test_info; 870e03e93a976d0f0da63f02fd3384c4b99cac8d715Andy Shevchenko 871a310d037b8d06755c62bb4878c00d19490af5550Dan Williams if (dmatest_run) { 872a310d037b8d06755c62bb4878c00d19490af5550Dan Williams mutex_lock(&info->lock); 873a9e554957de406d6adc581731f571b8a1503f6b0Dan Williams run_threaded_test(info); 874a310d037b8d06755c62bb4878c00d19490af5550Dan Williams mutex_unlock(&info->lock); 875a310d037b8d06755c62bb4878c00d19490af5550Dan Williams } 876838cc704ce5c8ab2a6d64d1324e37e040fcae3d8Andy Shevchenko 877a310d037b8d06755c62bb4878c00d19490af5550Dan Williams /* module parameters are stable, inittime tests are started, 878a310d037b8d06755c62bb4878c00d19490af5550Dan Williams * let userspace take over 'run' control 879a310d037b8d06755c62bb4878c00d19490af5550Dan Williams */ 880a310d037b8d06755c62bb4878c00d19490af5550Dan Williams info->did_init = true; 881851b7e16a07dfda6178d4e35fea9a9e3eb8954aeAndy Shevchenko 882a9e554957de406d6adc581731f571b8a1503f6b0Dan Williams return 0; 883e03e93a976d0f0da63f02fd3384c4b99cac8d715Andy Shevchenko} 884e03e93a976d0f0da63f02fd3384c4b99cac8d715Andy Shevchenko/* when compiled-in wait for drivers to load first */ 885e03e93a976d0f0da63f02fd3384c4b99cac8d715Andy Shevchenkolate_initcall(dmatest_init); 886e03e93a976d0f0da63f02fd3384c4b99cac8d715Andy Shevchenko 887e03e93a976d0f0da63f02fd3384c4b99cac8d715Andy Shevchenkostatic void __exit dmatest_exit(void) 888e03e93a976d0f0da63f02fd3384c4b99cac8d715Andy Shevchenko{ 889e03e93a976d0f0da63f02fd3384c4b99cac8d715Andy Shevchenko struct dmatest_info *info = &test_info; 890e03e93a976d0f0da63f02fd3384c4b99cac8d715Andy Shevchenko 891a310d037b8d06755c62bb4878c00d19490af5550Dan Williams mutex_lock(&info->lock); 892e03e93a976d0f0da63f02fd3384c4b99cac8d715Andy Shevchenko stop_threaded_test(info); 893a310d037b8d06755c62bb4878c00d19490af5550Dan Williams mutex_unlock(&info->lock); 894e03e93a976d0f0da63f02fd3384c4b99cac8d715Andy Shevchenko} 8954a776f0aa922a552460192c07b56f4fe9cd82632Haavard Skinnemoenmodule_exit(dmatest_exit); 8964a776f0aa922a552460192c07b56f4fe9cd82632Haavard Skinnemoen 897e05503ef1186ad33dfe56794407891eb1dd93ef6Jean DelvareMODULE_AUTHOR("Haavard Skinnemoen (Atmel)"); 8984a776f0aa922a552460192c07b56f4fe9cd82632Haavard SkinnemoenMODULE_LICENSE("GPL v2"); 899