svc_rdma.c revision ef7fbf59e6f780a0fa03536839e3c42e9ce40ad1 
1/*
2 * Copyright (c) 2005-2006 Network Appliance, Inc. All rights reserved.
3 *
4 * This software is available to you under a choice of one of two
5 * licenses.  You may choose to be licensed under the terms of the GNU
6 * General Public License (GPL) Version 2, available from the file
7 * COPYING in the main directory of this source tree, or the BSD-type
8 * license below:
9 *
10 * Redistribution and use in source and binary forms, with or without
11 * modification, are permitted provided that the following conditions
12 * are met:
13 *
14 *      Redistributions of source code must retain the above copyright
15 *      notice, this list of conditions and the following disclaimer.
16 *
17 *      Redistributions in binary form must reproduce the above
18 *      copyright notice, this list of conditions and the following
19 *      disclaimer in the documentation and/or other materials provided
20 *      with the distribution.
21 *
22 *      Neither the name of the Network Appliance, Inc. nor the names of
23 *      its contributors may be used to endorse or promote products
24 *      derived from this software without specific prior written
25 *      permission.
26 *
27 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
28 * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
29 * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
30 * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
31 * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
32 * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
33 * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
34 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
35 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
36 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
37 * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
38 *
39 * Author: Tom Tucker <tom@opengridcomputing.com>
40 */
41#include <linux/module.h>
42#include <linux/init.h>
43#include <linux/fs.h>
44#include <linux/sysctl.h>
45#include <linux/sunrpc/clnt.h>
46#include <linux/sunrpc/sched.h>
47#include <linux/sunrpc/svc_rdma.h>
48
49#define RPCDBG_FACILITY	RPCDBG_SVCXPRT
50
51/* RPC/RDMA parameters */
52unsigned int svcrdma_ord = RPCRDMA_ORD;
53static unsigned int min_ord = 1;
54static unsigned int max_ord = 4096;
55unsigned int svcrdma_max_requests = RPCRDMA_MAX_REQUESTS;
56static unsigned int min_max_requests = 4;
57static unsigned int max_max_requests = 16384;
58unsigned int svcrdma_max_req_size = RPCRDMA_MAX_REQ_SIZE;
59static unsigned int min_max_inline = 4096;
60static unsigned int max_max_inline = 65536;
61
62atomic_t rdma_stat_recv;
63atomic_t rdma_stat_read;
64atomic_t rdma_stat_write;
65atomic_t rdma_stat_sq_starve;
66atomic_t rdma_stat_rq_starve;
67atomic_t rdma_stat_rq_poll;
68atomic_t rdma_stat_rq_prod;
69atomic_t rdma_stat_sq_poll;
70atomic_t rdma_stat_sq_prod;
71
72/*
73 * This function implements reading and resetting an atomic_t stat
74 * variable through read/write to a proc file. Any write to the file
75 * resets the associated statistic to zero. Any read returns it's
76 * current value.
77 */
78static int read_reset_stat(ctl_table *table, int write,
79			   struct file *filp, void __user *buffer, size_t *lenp,
80			   loff_t *ppos)
81{
82	atomic_t *stat = (atomic_t *)table->data;
83
84	if (!stat)
85		return -EINVAL;
86
87	if (write)
88		atomic_set(stat, 0);
89	else {
90		char str_buf[32];
91		char *data;
92		int len = snprintf(str_buf, 32, "%d\n", atomic_read(stat));
93		if (len >= 32)
94			return -EFAULT;
95		len = strlen(str_buf);
96		if (*ppos > len) {
97			*lenp = 0;
98			return 0;
99		}
100		data = &str_buf[*ppos];
101		len -= *ppos;
102		if (len > *lenp)
103			len = *lenp;
104		if (len && copy_to_user(buffer, str_buf, len))
105			return -EFAULT;
106		*lenp = len;
107		*ppos += len;
108	}
109	return 0;
110}
111
112static struct ctl_table_header *svcrdma_table_header;
113static ctl_table svcrdma_parm_table[] = {
114	{
115		.procname	= "max_requests",
116		.data		= &svcrdma_max_requests,
117		.maxlen		= sizeof(unsigned int),
118		.mode		= 0644,
119		.proc_handler	= &proc_dointvec_minmax,
120		.strategy	= &sysctl_intvec,
121		.extra1		= &min_max_requests,
122		.extra2		= &max_max_requests
123	},
124	{
125		.procname	= "max_req_size",
126		.data		= &svcrdma_max_req_size,
127		.maxlen		= sizeof(unsigned int),
128		.mode		= 0644,
129		.proc_handler	= &proc_dointvec_minmax,
130		.strategy	= &sysctl_intvec,
131		.extra1		= &min_max_inline,
132		.extra2		= &max_max_inline
133	},
134	{
135		.procname	= "max_outbound_read_requests",
136		.data		= &svcrdma_ord,
137		.maxlen		= sizeof(unsigned int),
138		.mode		= 0644,
139		.proc_handler	= &proc_dointvec_minmax,
140		.strategy	= &sysctl_intvec,
141		.extra1		= &min_ord,
142		.extra2		= &max_ord,
143	},
144
145	{
146		.procname	= "rdma_stat_read",
147		.data		= &rdma_stat_read,
148		.maxlen		= sizeof(atomic_t),
149		.mode		= 0644,
150		.proc_handler	= &read_reset_stat,
151	},
152	{
153		.procname	= "rdma_stat_recv",
154		.data		= &rdma_stat_recv,
155		.maxlen		= sizeof(atomic_t),
156		.mode		= 0644,
157		.proc_handler	= &read_reset_stat,
158	},
159	{
160		.procname	= "rdma_stat_write",
161		.data		= &rdma_stat_write,
162		.maxlen		= sizeof(atomic_t),
163		.mode		= 0644,
164		.proc_handler	= &read_reset_stat,
165	},
166	{
167		.procname	= "rdma_stat_sq_starve",
168		.data		= &rdma_stat_sq_starve,
169		.maxlen		= sizeof(atomic_t),
170		.mode		= 0644,
171		.proc_handler	= &read_reset_stat,
172	},
173	{
174		.procname	= "rdma_stat_rq_starve",
175		.data		= &rdma_stat_rq_starve,
176		.maxlen		= sizeof(atomic_t),
177		.mode		= 0644,
178		.proc_handler	= &read_reset_stat,
179	},
180	{
181		.procname	= "rdma_stat_rq_poll",
182		.data		= &rdma_stat_rq_poll,
183		.maxlen		= sizeof(atomic_t),
184		.mode		= 0644,
185		.proc_handler	= &read_reset_stat,
186	},
187	{
188		.procname	= "rdma_stat_rq_prod",
189		.data		= &rdma_stat_rq_prod,
190		.maxlen		= sizeof(atomic_t),
191		.mode		= 0644,
192		.proc_handler	= &read_reset_stat,
193	},
194	{
195		.procname	= "rdma_stat_sq_poll",
196		.data		= &rdma_stat_sq_poll,
197		.maxlen		= sizeof(atomic_t),
198		.mode		= 0644,
199		.proc_handler	= &read_reset_stat,
200	},
201	{
202		.procname	= "rdma_stat_sq_prod",
203		.data		= &rdma_stat_sq_prod,
204		.maxlen		= sizeof(atomic_t),
205		.mode		= 0644,
206		.proc_handler	= &read_reset_stat,
207	},
208	{
209		.ctl_name = 0,
210	},
211};
212
213static ctl_table svcrdma_table[] = {
214	{
215		.procname	= "svc_rdma",
216		.mode		= 0555,
217		.child		= svcrdma_parm_table
218	},
219	{
220		.ctl_name = 0,
221	},
222};
223
224static ctl_table svcrdma_root_table[] = {
225	{
226		.ctl_name	= CTL_SUNRPC,
227		.procname	= "sunrpc",
228		.mode		= 0555,
229		.child		= svcrdma_table
230	},
231	{
232		.ctl_name = 0,
233	},
234};
235
236void svc_rdma_cleanup(void)
237{
238	dprintk("SVCRDMA Module Removed, deregister RPC RDMA transport\n");
239	if (svcrdma_table_header) {
240		unregister_sysctl_table(svcrdma_table_header);
241		svcrdma_table_header = NULL;
242	}
243	svc_unreg_xprt_class(&svc_rdma_class);
244}
245
246int svc_rdma_init(void)
247{
248	dprintk("SVCRDMA Module Init, register RPC RDMA transport\n");
249	dprintk("\tsvcrdma_ord      : %d\n", svcrdma_ord);
250	dprintk("\tmax_requests     : %d\n", svcrdma_max_requests);
251	dprintk("\tsq_depth         : %d\n",
252		svcrdma_max_requests * RPCRDMA_SQ_DEPTH_MULT);
253	dprintk("\tmax_inline       : %d\n", svcrdma_max_req_size);
254	if (!svcrdma_table_header)
255		svcrdma_table_header =
256			register_sysctl_table(svcrdma_root_table);
257
258	/* Register RDMA with the SVC transport switch */
259	svc_reg_xprt_class(&svc_rdma_class);
260	return 0;
261}
262MODULE_AUTHOR("Tom Tucker <tom@opengridcomputing.com>");
263MODULE_DESCRIPTION("SVC RDMA Transport");
264MODULE_LICENSE("Dual BSD/GPL");
265module_init(svc_rdma_init);
266module_exit(svc_rdma_cleanup);
267