11da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds/*
21da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds *  linux/net/sunrpc/xprt.c
31da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds *
41da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds *  This is a generic RPC call interface supporting congestion avoidance,
51da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds *  and asynchronous calls.
61da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds *
71da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds *  The interface works like this:
81da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds *
91da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds *  -	When a process places a call, it allocates a request slot if
101da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds *	one is available. Otherwise, it sleeps on the backlog queue
111da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds *	(xprt_reserve).
121da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds *  -	Next, the caller puts together the RPC message, stuffs it into
1355aa4f58aa43dc9a51fb80010630d94b96053a2eChuck Lever *	the request struct, and calls xprt_transmit().
1455aa4f58aa43dc9a51fb80010630d94b96053a2eChuck Lever *  -	xprt_transmit sends the message and installs the caller on the
1555ae1aabfb108106dd095de2578ceef1c755a8b8Ricardo Labiaga *	transport's wait list. At the same time, if a reply is expected,
1655ae1aabfb108106dd095de2578ceef1c755a8b8Ricardo Labiaga *	it installs a timer that is run after the packet's timeout has
1755ae1aabfb108106dd095de2578ceef1c755a8b8Ricardo Labiaga *	expired.
181da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds *  -	When a packet arrives, the data_ready handler walks the list of
1955aa4f58aa43dc9a51fb80010630d94b96053a2eChuck Lever *	pending requests for that transport. If a matching XID is found, the
201da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds *	caller is woken up, and the timer removed.
211da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds *  -	When no reply arrives within the timeout interval, the timer is
221da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds *	fired by the kernel and runs xprt_timer(). It either adjusts the
231da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds *	timeout values (minor timeout) or wakes up the caller with a status
241da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds *	of -ETIMEDOUT.
251da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds *  -	When the caller receives a notification from RPC that a reply arrived,
261da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds *	it should release the RPC slot, and process the reply.
271da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds *	If the call timed out, it may choose to retry the operation by
281da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds *	adjusting the initial timeout value, and simply calling rpc_call
291da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds *	again.
301da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds *
311da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds *  Support for async RPC is done through a set of RPC-specific scheduling
321da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds *  primitives that `transparently' work for processes as well as async
331da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds *  tasks that rely on callbacks.
341da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds *
351da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds *  Copyright (C) 1995-1997, Olaf Kirch <okir@monad.swb.de>
3655aa4f58aa43dc9a51fb80010630d94b96053a2eChuck Lever *
3755aa4f58aa43dc9a51fb80010630d94b96053a2eChuck Lever *  Transport switch API copyright (C) 2005, Chuck Lever <cel@netapp.com>
381da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds */
391da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds
40a246b0105bbd9a70a698f69baae2042996f2a0e9Chuck Lever#include <linux/module.h>
41a246b0105bbd9a70a698f69baae2042996f2a0e9Chuck Lever
421da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds#include <linux/types.h>
43a246b0105bbd9a70a698f69baae2042996f2a0e9Chuck Lever#include <linux/interrupt.h>
441da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds#include <linux/workqueue.h>
45bf3fcf89552f24657bcfb6a9d73cd167ebb496c6Chuck Lever#include <linux/net.h>
46ff8399709e41bf72b4cb145612a0f9a9f7283c83Chuck Lever#include <linux/ktime.h>
471da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds
48a246b0105bbd9a70a698f69baae2042996f2a0e9Chuck Lever#include <linux/sunrpc/clnt.h>
4911c556b3d8d481829ab5f9933a25d29b00913b5aChuck Lever#include <linux/sunrpc/metrics.h>
50c9acb42ef1904d15d0fb315061cefbe638f67f3aTrond Myklebust#include <linux/sunrpc/bc_xprt.h>
511da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds
5255ae1aabfb108106dd095de2578ceef1c755a8b8Ricardo Labiaga#include "sunrpc.h"
5355ae1aabfb108106dd095de2578ceef1c755a8b8Ricardo Labiaga
541da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds/*
551da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds * Local variables
561da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds */
571da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds
581da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds#ifdef RPC_DEBUG
591da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds# define RPCDBG_FACILITY	RPCDBG_XPRT
601da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds#endif
611da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds
621da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds/*
631da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds * Local functions
641da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds */
6521de0a955f3af29fa1100d96f66e6adade89e77aTrond Myklebuststatic void	 xprt_init(struct rpc_xprt *xprt, struct net *net);
661da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvaldsstatic void	xprt_request_init(struct rpc_task *, struct rpc_xprt *);
671da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvaldsstatic void	xprt_connect_status(struct rpc_task *task);
681da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvaldsstatic int      __xprt_get_cong(struct rpc_xprt *, struct rpc_task *);
694e0038b6b246e4145fc4a53dca61a556d17bc52cTrond Myklebuststatic void	 xprt_destroy(struct rpc_xprt *xprt);
701da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds
715ba03e82b3dac41bb1c5ca29060aa44b5e44b486Jiri Slabystatic DEFINE_SPINLOCK(xprt_list_lock);
7281c098af3da7981902e9f8163aeccc2467c4ba6d\"Talpey, Thomas\static LIST_HEAD(xprt_list);
7381c098af3da7981902e9f8163aeccc2467c4ba6d\"Talpey, Thomas\
7412a804698b29d040b7cdd92e8a44b0e75164dae9Chuck Lever/**
7581c098af3da7981902e9f8163aeccc2467c4ba6d\"Talpey, Thomas\ * xprt_register_transport - register a transport implementation
7681c098af3da7981902e9f8163aeccc2467c4ba6d\"Talpey, Thomas\ * @transport: transport to register
7781c098af3da7981902e9f8163aeccc2467c4ba6d\"Talpey, Thomas\ *
7881c098af3da7981902e9f8163aeccc2467c4ba6d\"Talpey, Thomas\ * If a transport implementation is loaded as a kernel module, it can
7981c098af3da7981902e9f8163aeccc2467c4ba6d\"Talpey, Thomas\ * call this interface to make itself known to the RPC client.
8081c098af3da7981902e9f8163aeccc2467c4ba6d\"Talpey, Thomas\ *
8181c098af3da7981902e9f8163aeccc2467c4ba6d\"Talpey, Thomas\ * Returns:
8281c098af3da7981902e9f8163aeccc2467c4ba6d\"Talpey, Thomas\ * 0:		transport successfully registered
8381c098af3da7981902e9f8163aeccc2467c4ba6d\"Talpey, Thomas\ * -EEXIST:	transport already registered
8481c098af3da7981902e9f8163aeccc2467c4ba6d\"Talpey, Thomas\ * -EINVAL:	transport module being unloaded
8581c098af3da7981902e9f8163aeccc2467c4ba6d\"Talpey, Thomas\ */
8681c098af3da7981902e9f8163aeccc2467c4ba6d\"Talpey, Thomas\int xprt_register_transport(struct xprt_class *transport)
8781c098af3da7981902e9f8163aeccc2467c4ba6d\"Talpey, Thomas\{
8881c098af3da7981902e9f8163aeccc2467c4ba6d\"Talpey, Thomas\	struct xprt_class *t;
8981c098af3da7981902e9f8163aeccc2467c4ba6d\"Talpey, Thomas\	int result;
9081c098af3da7981902e9f8163aeccc2467c4ba6d\"Talpey, Thomas\
9181c098af3da7981902e9f8163aeccc2467c4ba6d\"Talpey, Thomas\	result = -EEXIST;
9281c098af3da7981902e9f8163aeccc2467c4ba6d\"Talpey, Thomas\	spin_lock(&xprt_list_lock);
9381c098af3da7981902e9f8163aeccc2467c4ba6d\"Talpey, Thomas\	list_for_each_entry(t, &xprt_list, list) {
9481c098af3da7981902e9f8163aeccc2467c4ba6d\"Talpey, Thomas\		/* don't register the same transport class twice */
954fa016eb248cac875541fa199af550a8aefa0e90\"Talpey, Thomas\		if (t->ident == transport->ident)
9681c098af3da7981902e9f8163aeccc2467c4ba6d\"Talpey, Thomas\			goto out;
9781c098af3da7981902e9f8163aeccc2467c4ba6d\"Talpey, Thomas\	}
9881c098af3da7981902e9f8163aeccc2467c4ba6d\"Talpey, Thomas\
99c9f6cde6e26ef98ee9c4b6288b126ac9c580d88bDenis V. Lunev	list_add_tail(&transport->list, &xprt_list);
100c9f6cde6e26ef98ee9c4b6288b126ac9c580d88bDenis V. Lunev	printk(KERN_INFO "RPC: Registered %s transport module.\n",
101c9f6cde6e26ef98ee9c4b6288b126ac9c580d88bDenis V. Lunev	       transport->name);
102c9f6cde6e26ef98ee9c4b6288b126ac9c580d88bDenis V. Lunev	result = 0;
10381c098af3da7981902e9f8163aeccc2467c4ba6d\"Talpey, Thomas\
10481c098af3da7981902e9f8163aeccc2467c4ba6d\"Talpey, Thomas\out:
10581c098af3da7981902e9f8163aeccc2467c4ba6d\"Talpey, Thomas\	spin_unlock(&xprt_list_lock);
10681c098af3da7981902e9f8163aeccc2467c4ba6d\"Talpey, Thomas\	return result;
10781c098af3da7981902e9f8163aeccc2467c4ba6d\"Talpey, Thomas\}
10881c098af3da7981902e9f8163aeccc2467c4ba6d\"Talpey, Thomas\EXPORT_SYMBOL_GPL(xprt_register_transport);
10981c098af3da7981902e9f8163aeccc2467c4ba6d\"Talpey, Thomas\
11081c098af3da7981902e9f8163aeccc2467c4ba6d\"Talpey, Thomas\/**
11181c098af3da7981902e9f8163aeccc2467c4ba6d\"Talpey, Thomas\ * xprt_unregister_transport - unregister a transport implementation
11265b6e42cdc5b6a1ce2ada31cc294d7e60b22bb43Randy Dunlap * @transport: transport to unregister
11381c098af3da7981902e9f8163aeccc2467c4ba6d\"Talpey, Thomas\ *
11481c098af3da7981902e9f8163aeccc2467c4ba6d\"Talpey, Thomas\ * Returns:
11581c098af3da7981902e9f8163aeccc2467c4ba6d\"Talpey, Thomas\ * 0:		transport successfully unregistered
11681c098af3da7981902e9f8163aeccc2467c4ba6d\"Talpey, Thomas\ * -ENOENT:	transport never registered
11781c098af3da7981902e9f8163aeccc2467c4ba6d\"Talpey, Thomas\ */
11881c098af3da7981902e9f8163aeccc2467c4ba6d\"Talpey, Thomas\int xprt_unregister_transport(struct xprt_class *transport)
11981c098af3da7981902e9f8163aeccc2467c4ba6d\"Talpey, Thomas\{
12081c098af3da7981902e9f8163aeccc2467c4ba6d\"Talpey, Thomas\	struct xprt_class *t;
12181c098af3da7981902e9f8163aeccc2467c4ba6d\"Talpey, Thomas\	int result;
12281c098af3da7981902e9f8163aeccc2467c4ba6d\"Talpey, Thomas\
12381c098af3da7981902e9f8163aeccc2467c4ba6d\"Talpey, Thomas\	result = 0;
12481c098af3da7981902e9f8163aeccc2467c4ba6d\"Talpey, Thomas\	spin_lock(&xprt_list_lock);
12581c098af3da7981902e9f8163aeccc2467c4ba6d\"Talpey, Thomas\	list_for_each_entry(t, &xprt_list, list) {
12681c098af3da7981902e9f8163aeccc2467c4ba6d\"Talpey, Thomas\		if (t == transport) {
12781c098af3da7981902e9f8163aeccc2467c4ba6d\"Talpey, Thomas\			printk(KERN_INFO
12881c098af3da7981902e9f8163aeccc2467c4ba6d\"Talpey, Thomas\				"RPC: Unregistered %s transport module.\n",
12981c098af3da7981902e9f8163aeccc2467c4ba6d\"Talpey, Thomas\				transport->name);
13081c098af3da7981902e9f8163aeccc2467c4ba6d\"Talpey, Thomas\			list_del_init(&transport->list);
13181c098af3da7981902e9f8163aeccc2467c4ba6d\"Talpey, Thomas\			goto out;
13281c098af3da7981902e9f8163aeccc2467c4ba6d\"Talpey, Thomas\		}
13381c098af3da7981902e9f8163aeccc2467c4ba6d\"Talpey, Thomas\	}
13481c098af3da7981902e9f8163aeccc2467c4ba6d\"Talpey, Thomas\	result = -ENOENT;
13581c098af3da7981902e9f8163aeccc2467c4ba6d\"Talpey, Thomas\
13681c098af3da7981902e9f8163aeccc2467c4ba6d\"Talpey, Thomas\out:
13781c098af3da7981902e9f8163aeccc2467c4ba6d\"Talpey, Thomas\	spin_unlock(&xprt_list_lock);
13881c098af3da7981902e9f8163aeccc2467c4ba6d\"Talpey, Thomas\	return result;
13981c098af3da7981902e9f8163aeccc2467c4ba6d\"Talpey, Thomas\}
14081c098af3da7981902e9f8163aeccc2467c4ba6d\"Talpey, Thomas\EXPORT_SYMBOL_GPL(xprt_unregister_transport);
14181c098af3da7981902e9f8163aeccc2467c4ba6d\"Talpey, Thomas\
14281c098af3da7981902e9f8163aeccc2467c4ba6d\"Talpey, Thomas\/**
143441e3e242903f9b190d5764bed73edb58f977413Tom Talpey * xprt_load_transport - load a transport implementation
144441e3e242903f9b190d5764bed73edb58f977413Tom Talpey * @transport_name: transport to load
145441e3e242903f9b190d5764bed73edb58f977413Tom Talpey *
146441e3e242903f9b190d5764bed73edb58f977413Tom Talpey * Returns:
147441e3e242903f9b190d5764bed73edb58f977413Tom Talpey * 0:		transport successfully loaded
148441e3e242903f9b190d5764bed73edb58f977413Tom Talpey * -ENOENT:	transport module not available
149441e3e242903f9b190d5764bed73edb58f977413Tom Talpey */
150441e3e242903f9b190d5764bed73edb58f977413Tom Talpeyint xprt_load_transport(const char *transport_name)
151441e3e242903f9b190d5764bed73edb58f977413Tom Talpey{
152441e3e242903f9b190d5764bed73edb58f977413Tom Talpey	struct xprt_class *t;
153441e3e242903f9b190d5764bed73edb58f977413Tom Talpey	int result;
154441e3e242903f9b190d5764bed73edb58f977413Tom Talpey
155441e3e242903f9b190d5764bed73edb58f977413Tom Talpey	result = 0;
156441e3e242903f9b190d5764bed73edb58f977413Tom Talpey	spin_lock(&xprt_list_lock);
157441e3e242903f9b190d5764bed73edb58f977413Tom Talpey	list_for_each_entry(t, &xprt_list, list) {
158441e3e242903f9b190d5764bed73edb58f977413Tom Talpey		if (strcmp(t->name, transport_name) == 0) {
159441e3e242903f9b190d5764bed73edb58f977413Tom Talpey			spin_unlock(&xprt_list_lock);
160441e3e242903f9b190d5764bed73edb58f977413Tom Talpey			goto out;
161441e3e242903f9b190d5764bed73edb58f977413Tom Talpey		}
162441e3e242903f9b190d5764bed73edb58f977413Tom Talpey	}
163441e3e242903f9b190d5764bed73edb58f977413Tom Talpey	spin_unlock(&xprt_list_lock);
164ef7ffe8f06895312aeb08a5f8a1d4c90e34335eaAlex Riesen	result = request_module("xprt%s", transport_name);
165441e3e242903f9b190d5764bed73edb58f977413Tom Talpeyout:
166441e3e242903f9b190d5764bed73edb58f977413Tom Talpey	return result;
167441e3e242903f9b190d5764bed73edb58f977413Tom Talpey}
168441e3e242903f9b190d5764bed73edb58f977413Tom TalpeyEXPORT_SYMBOL_GPL(xprt_load_transport);
169441e3e242903f9b190d5764bed73edb58f977413Tom Talpey
170441e3e242903f9b190d5764bed73edb58f977413Tom Talpey/**
17112a804698b29d040b7cdd92e8a44b0e75164dae9Chuck Lever * xprt_reserve_xprt - serialize write access to transports
17212a804698b29d040b7cdd92e8a44b0e75164dae9Chuck Lever * @task: task that is requesting access to the transport
173177c27bf05d0ea508e65afdbe4b6998c81e46af5Randy Dunlap * @xprt: pointer to the target transport
17412a804698b29d040b7cdd92e8a44b0e75164dae9Chuck Lever *
17512a804698b29d040b7cdd92e8a44b0e75164dae9Chuck Lever * This prevents mixing the payload of separate requests, and prevents
17612a804698b29d040b7cdd92e8a44b0e75164dae9Chuck Lever * transport connects from colliding with writes.  No congestion control
17712a804698b29d040b7cdd92e8a44b0e75164dae9Chuck Lever * is provided.
17812a804698b29d040b7cdd92e8a44b0e75164dae9Chuck Lever */
17943cedbf0e8dfb9c5610eb7985d5f21263e313802Trond Myklebustint xprt_reserve_xprt(struct rpc_xprt *xprt, struct rpc_task *task)
18012a804698b29d040b7cdd92e8a44b0e75164dae9Chuck Lever{
18112a804698b29d040b7cdd92e8a44b0e75164dae9Chuck Lever	struct rpc_rqst *req = task->tk_rqstp;
18234006cee28f7344f9557a4be3816c7891b1bbab1Trond Myklebust	int priority;
18312a804698b29d040b7cdd92e8a44b0e75164dae9Chuck Lever
18412a804698b29d040b7cdd92e8a44b0e75164dae9Chuck Lever	if (test_and_set_bit(XPRT_LOCKED, &xprt->state)) {
18512a804698b29d040b7cdd92e8a44b0e75164dae9Chuck Lever		if (task == xprt->snd_task)
18612a804698b29d040b7cdd92e8a44b0e75164dae9Chuck Lever			return 1;
18712a804698b29d040b7cdd92e8a44b0e75164dae9Chuck Lever		goto out_sleep;
18812a804698b29d040b7cdd92e8a44b0e75164dae9Chuck Lever	}
18912a804698b29d040b7cdd92e8a44b0e75164dae9Chuck Lever	xprt->snd_task = task;
19092551948174d079b12541437f51cbe3e17d9dd24Trond Myklebust	if (req != NULL)
19143cedbf0e8dfb9c5610eb7985d5f21263e313802Trond Myklebust		req->rq_ntrans++;
1924d4a76f3309edc671918a767b336492fbc80a16dj
19312a804698b29d040b7cdd92e8a44b0e75164dae9Chuck Lever	return 1;
19412a804698b29d040b7cdd92e8a44b0e75164dae9Chuck Lever
19512a804698b29d040b7cdd92e8a44b0e75164dae9Chuck Leverout_sleep:
19646121cf7d85869bfe9588bac7ccf55aa0bc7f278Chuck Lever	dprintk("RPC: %5u failed to lock transport %p\n",
19712a804698b29d040b7cdd92e8a44b0e75164dae9Chuck Lever			task->tk_pid, xprt);
19812a804698b29d040b7cdd92e8a44b0e75164dae9Chuck Lever	task->tk_timeout = 0;
19912a804698b29d040b7cdd92e8a44b0e75164dae9Chuck Lever	task->tk_status = -EAGAIN;
20034006cee28f7344f9557a4be3816c7891b1bbab1Trond Myklebust	if (req == NULL)
20134006cee28f7344f9557a4be3816c7891b1bbab1Trond Myklebust		priority = RPC_PRIORITY_LOW;
20234006cee28f7344f9557a4be3816c7891b1bbab1Trond Myklebust	else if (!req->rq_ntrans)
20334006cee28f7344f9557a4be3816c7891b1bbab1Trond Myklebust		priority = RPC_PRIORITY_NORMAL;
20412a804698b29d040b7cdd92e8a44b0e75164dae9Chuck Lever	else
20534006cee28f7344f9557a4be3816c7891b1bbab1Trond Myklebust		priority = RPC_PRIORITY_HIGH;
20634006cee28f7344f9557a4be3816c7891b1bbab1Trond Myklebust	rpc_sleep_on_priority(&xprt->sending, task, NULL, priority);
20712a804698b29d040b7cdd92e8a44b0e75164dae9Chuck Lever	return 0;
20812a804698b29d040b7cdd92e8a44b0e75164dae9Chuck Lever}
2091244480976d357447aeddd3f44977586bfa0462b\"Talpey, Thomas\EXPORT_SYMBOL_GPL(xprt_reserve_xprt);
21012a804698b29d040b7cdd92e8a44b0e75164dae9Chuck Lever
211632e3bdc5006334cea894d078660b691685e1075Trond Myklebuststatic void xprt_clear_locked(struct rpc_xprt *xprt)
212632e3bdc5006334cea894d078660b691685e1075Trond Myklebust{
213632e3bdc5006334cea894d078660b691685e1075Trond Myklebust	xprt->snd_task = NULL;
214d19751e7b9bd8a01d00372325439589886674f79Trond Myklebust	if (!test_bit(XPRT_CLOSE_WAIT, &xprt->state)) {
2154e857c58efeb99393cba5a5d0d8ec7117183137cPeter Zijlstra		smp_mb__before_atomic();
216632e3bdc5006334cea894d078660b691685e1075Trond Myklebust		clear_bit(XPRT_LOCKED, &xprt->state);
2174e857c58efeb99393cba5a5d0d8ec7117183137cPeter Zijlstra		smp_mb__after_atomic();
218632e3bdc5006334cea894d078660b691685e1075Trond Myklebust	} else
219c1384c9c4c184543375b52a0997d06cd98145164Trond Myklebust		queue_work(rpciod_workqueue, &xprt->task_cleanup);
220632e3bdc5006334cea894d078660b691685e1075Trond Myklebust}
221632e3bdc5006334cea894d078660b691685e1075Trond Myklebust
2221da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds/*
22312a804698b29d040b7cdd92e8a44b0e75164dae9Chuck Lever * xprt_reserve_xprt_cong - serialize write access to transports
22412a804698b29d040b7cdd92e8a44b0e75164dae9Chuck Lever * @task: task that is requesting access to the transport
22512a804698b29d040b7cdd92e8a44b0e75164dae9Chuck Lever *
22612a804698b29d040b7cdd92e8a44b0e75164dae9Chuck Lever * Same as xprt_reserve_xprt, but Van Jacobson congestion control is
22712a804698b29d040b7cdd92e8a44b0e75164dae9Chuck Lever * integrated into the decision of whether a request is allowed to be
22812a804698b29d040b7cdd92e8a44b0e75164dae9Chuck Lever * woken up and given access to the transport.
2291da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds */
23043cedbf0e8dfb9c5610eb7985d5f21263e313802Trond Myklebustint xprt_reserve_xprt_cong(struct rpc_xprt *xprt, struct rpc_task *task)
2311da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds{
2321da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds	struct rpc_rqst *req = task->tk_rqstp;
23334006cee28f7344f9557a4be3816c7891b1bbab1Trond Myklebust	int priority;
2341da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds
2352226feb6bcd0e5e117a9be3ea3dd3ffc14f3e41eChuck Lever	if (test_and_set_bit(XPRT_LOCKED, &xprt->state)) {
2361da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds		if (task == xprt->snd_task)
2371da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds			return 1;
2381da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds		goto out_sleep;
2391da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds	}
24043cedbf0e8dfb9c5610eb7985d5f21263e313802Trond Myklebust	if (req == NULL) {
24143cedbf0e8dfb9c5610eb7985d5f21263e313802Trond Myklebust		xprt->snd_task = task;
24243cedbf0e8dfb9c5610eb7985d5f21263e313802Trond Myklebust		return 1;
24343cedbf0e8dfb9c5610eb7985d5f21263e313802Trond Myklebust	}
24412a804698b29d040b7cdd92e8a44b0e75164dae9Chuck Lever	if (__xprt_get_cong(xprt, task)) {
2451da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds		xprt->snd_task = task;
24643cedbf0e8dfb9c5610eb7985d5f21263e313802Trond Myklebust		req->rq_ntrans++;
2471da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds		return 1;
2481da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds	}
249632e3bdc5006334cea894d078660b691685e1075Trond Myklebust	xprt_clear_locked(xprt);
2501da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvaldsout_sleep:
25146121cf7d85869bfe9588bac7ccf55aa0bc7f278Chuck Lever	dprintk("RPC: %5u failed to lock transport %p\n", task->tk_pid, xprt);
2521da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds	task->tk_timeout = 0;
2531da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds	task->tk_status = -EAGAIN;
25434006cee28f7344f9557a4be3816c7891b1bbab1Trond Myklebust	if (req == NULL)
25534006cee28f7344f9557a4be3816c7891b1bbab1Trond Myklebust		priority = RPC_PRIORITY_LOW;
25634006cee28f7344f9557a4be3816c7891b1bbab1Trond Myklebust	else if (!req->rq_ntrans)
25734006cee28f7344f9557a4be3816c7891b1bbab1Trond Myklebust		priority = RPC_PRIORITY_NORMAL;
2581da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds	else
25934006cee28f7344f9557a4be3816c7891b1bbab1Trond Myklebust		priority = RPC_PRIORITY_HIGH;
26034006cee28f7344f9557a4be3816c7891b1bbab1Trond Myklebust	rpc_sleep_on_priority(&xprt->sending, task, NULL, priority);
2611da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds	return 0;
2621da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds}
2631244480976d357447aeddd3f44977586bfa0462b\"Talpey, Thomas\EXPORT_SYMBOL_GPL(xprt_reserve_xprt_cong);
2641da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds
26512a804698b29d040b7cdd92e8a44b0e75164dae9Chuck Leverstatic inline int xprt_lock_write(struct rpc_xprt *xprt, struct rpc_task *task)
2661da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds{
2671da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds	int retval;
2681da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds
2694a0f8c04f2ece949d54a0c4fd7490259cf23a58aChuck Lever	spin_lock_bh(&xprt->transport_lock);
27043cedbf0e8dfb9c5610eb7985d5f21263e313802Trond Myklebust	retval = xprt->ops->reserve_xprt(xprt, task);
2714a0f8c04f2ece949d54a0c4fd7490259cf23a58aChuck Lever	spin_unlock_bh(&xprt->transport_lock);
2721da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds	return retval;
2731da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds}
2741da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds
275961a828df64979d2a9faeeeee043391670a193b9Trond Myklebuststatic bool __xprt_lock_write_func(struct rpc_task *task, void *data)
2761da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds{
277961a828df64979d2a9faeeeee043391670a193b9Trond Myklebust	struct rpc_xprt *xprt = data;
27849e9a89086b3cae784a4868ca852863e4f4ea3feChuck Lever	struct rpc_rqst *req;
27949e9a89086b3cae784a4868ca852863e4f4ea3feChuck Lever
28049e9a89086b3cae784a4868ca852863e4f4ea3feChuck Lever	req = task->tk_rqstp;
28149e9a89086b3cae784a4868ca852863e4f4ea3feChuck Lever	xprt->snd_task = task;
28292551948174d079b12541437f51cbe3e17d9dd24Trond Myklebust	if (req)
28349e9a89086b3cae784a4868ca852863e4f4ea3feChuck Lever		req->rq_ntrans++;
284961a828df64979d2a9faeeeee043391670a193b9Trond Myklebust	return true;
285961a828df64979d2a9faeeeee043391670a193b9Trond Myklebust}
28649e9a89086b3cae784a4868ca852863e4f4ea3feChuck Lever
287961a828df64979d2a9faeeeee043391670a193b9Trond Myklebuststatic void __xprt_lock_write_next(struct rpc_xprt *xprt)
288961a828df64979d2a9faeeeee043391670a193b9Trond Myklebust{
289961a828df64979d2a9faeeeee043391670a193b9Trond Myklebust	if (test_and_set_bit(XPRT_LOCKED, &xprt->state))
290961a828df64979d2a9faeeeee043391670a193b9Trond Myklebust		return;
291961a828df64979d2a9faeeeee043391670a193b9Trond Myklebust
292961a828df64979d2a9faeeeee043391670a193b9Trond Myklebust	if (rpc_wake_up_first(&xprt->sending, __xprt_lock_write_func, xprt))
293961a828df64979d2a9faeeeee043391670a193b9Trond Myklebust		return;
294632e3bdc5006334cea894d078660b691685e1075Trond Myklebust	xprt_clear_locked(xprt);
29549e9a89086b3cae784a4868ca852863e4f4ea3feChuck Lever}
29649e9a89086b3cae784a4868ca852863e4f4ea3feChuck Lever
297961a828df64979d2a9faeeeee043391670a193b9Trond Myklebuststatic bool __xprt_lock_write_cong_func(struct rpc_task *task, void *data)
29849e9a89086b3cae784a4868ca852863e4f4ea3feChuck Lever{
299961a828df64979d2a9faeeeee043391670a193b9Trond Myklebust	struct rpc_xprt *xprt = data;
30043cedbf0e8dfb9c5610eb7985d5f21263e313802Trond Myklebust	struct rpc_rqst *req;
3011da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds
30243cedbf0e8dfb9c5610eb7985d5f21263e313802Trond Myklebust	req = task->tk_rqstp;
30343cedbf0e8dfb9c5610eb7985d5f21263e313802Trond Myklebust	if (req == NULL) {
30443cedbf0e8dfb9c5610eb7985d5f21263e313802Trond Myklebust		xprt->snd_task = task;
305961a828df64979d2a9faeeeee043391670a193b9Trond Myklebust		return true;
30643cedbf0e8dfb9c5610eb7985d5f21263e313802Trond Myklebust	}
30749e9a89086b3cae784a4868ca852863e4f4ea3feChuck Lever	if (__xprt_get_cong(xprt, task)) {
3081da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds		xprt->snd_task = task;
30943cedbf0e8dfb9c5610eb7985d5f21263e313802Trond Myklebust		req->rq_ntrans++;
310961a828df64979d2a9faeeeee043391670a193b9Trond Myklebust		return true;
3111da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds	}
312961a828df64979d2a9faeeeee043391670a193b9Trond Myklebust	return false;
313961a828df64979d2a9faeeeee043391670a193b9Trond Myklebust}
314961a828df64979d2a9faeeeee043391670a193b9Trond Myklebust
315961a828df64979d2a9faeeeee043391670a193b9Trond Myklebuststatic void __xprt_lock_write_next_cong(struct rpc_xprt *xprt)
316961a828df64979d2a9faeeeee043391670a193b9Trond Myklebust{
317961a828df64979d2a9faeeeee043391670a193b9Trond Myklebust	if (test_and_set_bit(XPRT_LOCKED, &xprt->state))
318961a828df64979d2a9faeeeee043391670a193b9Trond Myklebust		return;
319961a828df64979d2a9faeeeee043391670a193b9Trond Myklebust	if (RPCXPRT_CONGESTED(xprt))
320961a828df64979d2a9faeeeee043391670a193b9Trond Myklebust		goto out_unlock;
321961a828df64979d2a9faeeeee043391670a193b9Trond Myklebust	if (rpc_wake_up_first(&xprt->sending, __xprt_lock_write_cong_func, xprt))
322961a828df64979d2a9faeeeee043391670a193b9Trond Myklebust		return;
3231da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvaldsout_unlock:
324632e3bdc5006334cea894d078660b691685e1075Trond Myklebust	xprt_clear_locked(xprt);
3251da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds}
3261da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds
32749e9a89086b3cae784a4868ca852863e4f4ea3feChuck Lever/**
32849e9a89086b3cae784a4868ca852863e4f4ea3feChuck Lever * xprt_release_xprt - allow other requests to use a transport
32949e9a89086b3cae784a4868ca852863e4f4ea3feChuck Lever * @xprt: transport with other tasks potentially waiting
33049e9a89086b3cae784a4868ca852863e4f4ea3feChuck Lever * @task: task that is releasing access to the transport
33149e9a89086b3cae784a4868ca852863e4f4ea3feChuck Lever *
33249e9a89086b3cae784a4868ca852863e4f4ea3feChuck Lever * Note that "task" can be NULL.  No congestion control is provided.
3331da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds */
33449e9a89086b3cae784a4868ca852863e4f4ea3feChuck Levervoid xprt_release_xprt(struct rpc_xprt *xprt, struct rpc_task *task)
3351da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds{
3361da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds	if (xprt->snd_task == task) {
337ee071eff0f1afafa9917254a6e4ee19d28085f1dTrond Myklebust		if (task != NULL) {
338ee071eff0f1afafa9917254a6e4ee19d28085f1dTrond Myklebust			struct rpc_rqst *req = task->tk_rqstp;
339ee071eff0f1afafa9917254a6e4ee19d28085f1dTrond Myklebust			if (req != NULL)
340ee071eff0f1afafa9917254a6e4ee19d28085f1dTrond Myklebust				req->rq_bytes_sent = 0;
341ee071eff0f1afafa9917254a6e4ee19d28085f1dTrond Myklebust		}
342632e3bdc5006334cea894d078660b691685e1075Trond Myklebust		xprt_clear_locked(xprt);
3431da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds		__xprt_lock_write_next(xprt);
3441da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds	}
3451da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds}
3461244480976d357447aeddd3f44977586bfa0462b\"Talpey, Thomas\EXPORT_SYMBOL_GPL(xprt_release_xprt);
3471da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds
34849e9a89086b3cae784a4868ca852863e4f4ea3feChuck Lever/**
34949e9a89086b3cae784a4868ca852863e4f4ea3feChuck Lever * xprt_release_xprt_cong - allow other requests to use a transport
35049e9a89086b3cae784a4868ca852863e4f4ea3feChuck Lever * @xprt: transport with other tasks potentially waiting
35149e9a89086b3cae784a4868ca852863e4f4ea3feChuck Lever * @task: task that is releasing access to the transport
35249e9a89086b3cae784a4868ca852863e4f4ea3feChuck Lever *
35349e9a89086b3cae784a4868ca852863e4f4ea3feChuck Lever * Note that "task" can be NULL.  Another task is awoken to use the
35449e9a89086b3cae784a4868ca852863e4f4ea3feChuck Lever * transport if the transport's congestion window allows it.
35549e9a89086b3cae784a4868ca852863e4f4ea3feChuck Lever */
35649e9a89086b3cae784a4868ca852863e4f4ea3feChuck Levervoid xprt_release_xprt_cong(struct rpc_xprt *xprt, struct rpc_task *task)
35749e9a89086b3cae784a4868ca852863e4f4ea3feChuck Lever{
35849e9a89086b3cae784a4868ca852863e4f4ea3feChuck Lever	if (xprt->snd_task == task) {
359ee071eff0f1afafa9917254a6e4ee19d28085f1dTrond Myklebust		if (task != NULL) {
360ee071eff0f1afafa9917254a6e4ee19d28085f1dTrond Myklebust			struct rpc_rqst *req = task->tk_rqstp;
361ee071eff0f1afafa9917254a6e4ee19d28085f1dTrond Myklebust			if (req != NULL)
362ee071eff0f1afafa9917254a6e4ee19d28085f1dTrond Myklebust				req->rq_bytes_sent = 0;
363ee071eff0f1afafa9917254a6e4ee19d28085f1dTrond Myklebust		}
364632e3bdc5006334cea894d078660b691685e1075Trond Myklebust		xprt_clear_locked(xprt);
36549e9a89086b3cae784a4868ca852863e4f4ea3feChuck Lever		__xprt_lock_write_next_cong(xprt);
36649e9a89086b3cae784a4868ca852863e4f4ea3feChuck Lever	}
36749e9a89086b3cae784a4868ca852863e4f4ea3feChuck Lever}
3681244480976d357447aeddd3f44977586bfa0462b\"Talpey, Thomas\EXPORT_SYMBOL_GPL(xprt_release_xprt_cong);
36949e9a89086b3cae784a4868ca852863e4f4ea3feChuck Lever
37049e9a89086b3cae784a4868ca852863e4f4ea3feChuck Leverstatic inline void xprt_release_write(struct rpc_xprt *xprt, struct rpc_task *task)
3711da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds{
3724a0f8c04f2ece949d54a0c4fd7490259cf23a58aChuck Lever	spin_lock_bh(&xprt->transport_lock);
37349e9a89086b3cae784a4868ca852863e4f4ea3feChuck Lever	xprt->ops->release_xprt(xprt, task);
3744a0f8c04f2ece949d54a0c4fd7490259cf23a58aChuck Lever	spin_unlock_bh(&xprt->transport_lock);
3751da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds}
3761da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds
3771da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds/*
3781da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds * Van Jacobson congestion avoidance. Check if the congestion window
3791da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds * overflowed. Put the task to sleep if this is the case.
3801da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds */
3811da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvaldsstatic int
3821da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds__xprt_get_cong(struct rpc_xprt *xprt, struct rpc_task *task)
3831da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds{
3841da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds	struct rpc_rqst *req = task->tk_rqstp;
3851da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds
3861da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds	if (req->rq_cong)
3871da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds		return 1;
38846121cf7d85869bfe9588bac7ccf55aa0bc7f278Chuck Lever	dprintk("RPC: %5u xprt_cwnd_limited cong = %lu cwnd = %lu\n",
3891da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds			task->tk_pid, xprt->cong, xprt->cwnd);
3901da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds	if (RPCXPRT_CONGESTED(xprt))
3911da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds		return 0;
3921da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds	req->rq_cong = 1;
3931da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds	xprt->cong += RPC_CWNDSCALE;
3941da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds	return 1;
3951da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds}
3961da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds
3971da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds/*
3981da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds * Adjust the congestion window, and wake up the next task
3991da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds * that has been sleeping due to congestion
4001da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds */
4011da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvaldsstatic void
4021da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds__xprt_put_cong(struct rpc_xprt *xprt, struct rpc_rqst *req)
4031da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds{
4041da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds	if (!req->rq_cong)
4051da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds		return;
4061da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds	req->rq_cong = 0;
4071da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds	xprt->cong -= RPC_CWNDSCALE;
40849e9a89086b3cae784a4868ca852863e4f4ea3feChuck Lever	__xprt_lock_write_next_cong(xprt);
4091da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds}
4101da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds
41146c0ee8bc4ad3743de05e8b8b20201df44dcb6d3Chuck Lever/**
412a58dd398f5db4f73d5c581069fd70a4304cc4f0aChuck Lever * xprt_release_rqst_cong - housekeeping when request is complete
413a58dd398f5db4f73d5c581069fd70a4304cc4f0aChuck Lever * @task: RPC request that recently completed
414a58dd398f5db4f73d5c581069fd70a4304cc4f0aChuck Lever *
415a58dd398f5db4f73d5c581069fd70a4304cc4f0aChuck Lever * Useful for transports that require congestion control.
416a58dd398f5db4f73d5c581069fd70a4304cc4f0aChuck Lever */
417a58dd398f5db4f73d5c581069fd70a4304cc4f0aChuck Levervoid xprt_release_rqst_cong(struct rpc_task *task)
418a58dd398f5db4f73d5c581069fd70a4304cc4f0aChuck Lever{
419a4f0835c604f80f945ab3e72ffd00547145c4b2bTrond Myklebust	struct rpc_rqst *req = task->tk_rqstp;
420a4f0835c604f80f945ab3e72ffd00547145c4b2bTrond Myklebust
421a4f0835c604f80f945ab3e72ffd00547145c4b2bTrond Myklebust	__xprt_put_cong(req->rq_xprt, req);
422a58dd398f5db4f73d5c581069fd70a4304cc4f0aChuck Lever}
4231244480976d357447aeddd3f44977586bfa0462b\"Talpey, Thomas\EXPORT_SYMBOL_GPL(xprt_release_rqst_cong);
424a58dd398f5db4f73d5c581069fd70a4304cc4f0aChuck Lever
425a58dd398f5db4f73d5c581069fd70a4304cc4f0aChuck Lever/**
42646c0ee8bc4ad3743de05e8b8b20201df44dcb6d3Chuck Lever * xprt_adjust_cwnd - adjust transport congestion window
4276a24dfb645dbcb05b34d08b991d082bdaa3ff072Trond Myklebust * @xprt: pointer to xprt
42846c0ee8bc4ad3743de05e8b8b20201df44dcb6d3Chuck Lever * @task: recently completed RPC request used to adjust window
42946c0ee8bc4ad3743de05e8b8b20201df44dcb6d3Chuck Lever * @result: result code of completed RPC request
43046c0ee8bc4ad3743de05e8b8b20201df44dcb6d3Chuck Lever *
4314f4cf5ad6fc1b16dc8dc9d750bb80b35eba5e98dChuck Lever * The transport code maintains an estimate on the maximum number of out-
4324f4cf5ad6fc1b16dc8dc9d750bb80b35eba5e98dChuck Lever * standing RPC requests, using a smoothed version of the congestion
4334f4cf5ad6fc1b16dc8dc9d750bb80b35eba5e98dChuck Lever * avoidance implemented in 44BSD. This is basically the Van Jacobson
4344f4cf5ad6fc1b16dc8dc9d750bb80b35eba5e98dChuck Lever * congestion algorithm: If a retransmit occurs, the congestion window is
4354f4cf5ad6fc1b16dc8dc9d750bb80b35eba5e98dChuck Lever * halved; otherwise, it is incremented by 1/cwnd when
4364f4cf5ad6fc1b16dc8dc9d750bb80b35eba5e98dChuck Lever *
4374f4cf5ad6fc1b16dc8dc9d750bb80b35eba5e98dChuck Lever *	-	a reply is received and
4384f4cf5ad6fc1b16dc8dc9d750bb80b35eba5e98dChuck Lever *	-	a full number of requests are outstanding and
4394f4cf5ad6fc1b16dc8dc9d750bb80b35eba5e98dChuck Lever *	-	the congestion window hasn't been updated recently.
4401da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds */
4416a24dfb645dbcb05b34d08b991d082bdaa3ff072Trond Myklebustvoid xprt_adjust_cwnd(struct rpc_xprt *xprt, struct rpc_task *task, int result)
4421da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds{
44346c0ee8bc4ad3743de05e8b8b20201df44dcb6d3Chuck Lever	struct rpc_rqst *req = task->tk_rqstp;
44446c0ee8bc4ad3743de05e8b8b20201df44dcb6d3Chuck Lever	unsigned long cwnd = xprt->cwnd;
4451da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds
4461da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds	if (result >= 0 && cwnd <= xprt->cong) {
4471da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds		/* The (cwnd >> 1) term makes sure
4481da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds		 * the result gets rounded properly. */
4491da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds		cwnd += (RPC_CWNDSCALE * RPC_CWNDSCALE + (cwnd >> 1)) / cwnd;
4501da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds		if (cwnd > RPC_MAXCWND(xprt))
4511da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds			cwnd = RPC_MAXCWND(xprt);
45249e9a89086b3cae784a4868ca852863e4f4ea3feChuck Lever		__xprt_lock_write_next_cong(xprt);
4531da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds	} else if (result == -ETIMEDOUT) {
4541da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds		cwnd >>= 1;
4551da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds		if (cwnd < RPC_CWNDSCALE)
4561da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds			cwnd = RPC_CWNDSCALE;
4571da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds	}
45846121cf7d85869bfe9588bac7ccf55aa0bc7f278Chuck Lever	dprintk("RPC:       cong %ld, cwnd was %ld, now %ld\n",
4591da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds			xprt->cong, xprt->cwnd, cwnd);
4601da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds	xprt->cwnd = cwnd;
46146c0ee8bc4ad3743de05e8b8b20201df44dcb6d3Chuck Lever	__xprt_put_cong(xprt, req);
4621da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds}
4631244480976d357447aeddd3f44977586bfa0462b\"Talpey, Thomas\EXPORT_SYMBOL_GPL(xprt_adjust_cwnd);
4641da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds
46544fbac2288dfed6f1963ac00bf922c3bcd779cd1Chuck Lever/**
46644fbac2288dfed6f1963ac00bf922c3bcd779cd1Chuck Lever * xprt_wake_pending_tasks - wake all tasks on a transport's pending queue
46744fbac2288dfed6f1963ac00bf922c3bcd779cd1Chuck Lever * @xprt: transport with waiting tasks
46844fbac2288dfed6f1963ac00bf922c3bcd779cd1Chuck Lever * @status: result code to plant in each task before waking it
46944fbac2288dfed6f1963ac00bf922c3bcd779cd1Chuck Lever *
47044fbac2288dfed6f1963ac00bf922c3bcd779cd1Chuck Lever */
47144fbac2288dfed6f1963ac00bf922c3bcd779cd1Chuck Levervoid xprt_wake_pending_tasks(struct rpc_xprt *xprt, int status)
47244fbac2288dfed6f1963ac00bf922c3bcd779cd1Chuck Lever{
47344fbac2288dfed6f1963ac00bf922c3bcd779cd1Chuck Lever	if (status < 0)
47444fbac2288dfed6f1963ac00bf922c3bcd779cd1Chuck Lever		rpc_wake_up_status(&xprt->pending, status);
47544fbac2288dfed6f1963ac00bf922c3bcd779cd1Chuck Lever	else
47644fbac2288dfed6f1963ac00bf922c3bcd779cd1Chuck Lever		rpc_wake_up(&xprt->pending);
47744fbac2288dfed6f1963ac00bf922c3bcd779cd1Chuck Lever}
4781244480976d357447aeddd3f44977586bfa0462b\"Talpey, Thomas\EXPORT_SYMBOL_GPL(xprt_wake_pending_tasks);
47944fbac2288dfed6f1963ac00bf922c3bcd779cd1Chuck Lever
480c7b2cae8a634015b72941ba2fc6c4bc9b8d3a129Chuck Lever/**
481c7b2cae8a634015b72941ba2fc6c4bc9b8d3a129Chuck Lever * xprt_wait_for_buffer_space - wait for transport output buffer to clear
482c7b2cae8a634015b72941ba2fc6c4bc9b8d3a129Chuck Lever * @task: task to be put to sleep
4830b80ae4201e5128e16e5161825f5cd377a5d1feeRandy Dunlap * @action: function pointer to be executed after wait
484a9a6b52ee1baa865283a91eb8d443ee91adfca56Trond Myklebust *
485a9a6b52ee1baa865283a91eb8d443ee91adfca56Trond Myklebust * Note that we only set the timer for the case of RPC_IS_SOFT(), since
486a9a6b52ee1baa865283a91eb8d443ee91adfca56Trond Myklebust * we don't in general want to force a socket disconnection due to
487a9a6b52ee1baa865283a91eb8d443ee91adfca56Trond Myklebust * an incomplete RPC call transmission.
488c7b2cae8a634015b72941ba2fc6c4bc9b8d3a129Chuck Lever */
489b6ddf64ffe9d59577a9176856bb6fe69a539f573Trond Myklebustvoid xprt_wait_for_buffer_space(struct rpc_task *task, rpc_action action)
490c7b2cae8a634015b72941ba2fc6c4bc9b8d3a129Chuck Lever{
491c7b2cae8a634015b72941ba2fc6c4bc9b8d3a129Chuck Lever	struct rpc_rqst *req = task->tk_rqstp;
492c7b2cae8a634015b72941ba2fc6c4bc9b8d3a129Chuck Lever	struct rpc_xprt *xprt = req->rq_xprt;
493c7b2cae8a634015b72941ba2fc6c4bc9b8d3a129Chuck Lever
494a9a6b52ee1baa865283a91eb8d443ee91adfca56Trond Myklebust	task->tk_timeout = RPC_IS_SOFT(task) ? req->rq_timeout : 0;
495b6ddf64ffe9d59577a9176856bb6fe69a539f573Trond Myklebust	rpc_sleep_on(&xprt->pending, task, action);
496c7b2cae8a634015b72941ba2fc6c4bc9b8d3a129Chuck Lever}
4971244480976d357447aeddd3f44977586bfa0462b\"Talpey, Thomas\EXPORT_SYMBOL_GPL(xprt_wait_for_buffer_space);
498c7b2cae8a634015b72941ba2fc6c4bc9b8d3a129Chuck Lever
499c7b2cae8a634015b72941ba2fc6c4bc9b8d3a129Chuck Lever/**
500c7b2cae8a634015b72941ba2fc6c4bc9b8d3a129Chuck Lever * xprt_write_space - wake the task waiting for transport output buffer space
501c7b2cae8a634015b72941ba2fc6c4bc9b8d3a129Chuck Lever * @xprt: transport with waiting tasks
502c7b2cae8a634015b72941ba2fc6c4bc9b8d3a129Chuck Lever *
503c7b2cae8a634015b72941ba2fc6c4bc9b8d3a129Chuck Lever * Can be called in a soft IRQ context, so xprt_write_space never sleeps.
504c7b2cae8a634015b72941ba2fc6c4bc9b8d3a129Chuck Lever */
505c7b2cae8a634015b72941ba2fc6c4bc9b8d3a129Chuck Levervoid xprt_write_space(struct rpc_xprt *xprt)
506c7b2cae8a634015b72941ba2fc6c4bc9b8d3a129Chuck Lever{
507c7b2cae8a634015b72941ba2fc6c4bc9b8d3a129Chuck Lever	spin_lock_bh(&xprt->transport_lock);
508c7b2cae8a634015b72941ba2fc6c4bc9b8d3a129Chuck Lever	if (xprt->snd_task) {
50946121cf7d85869bfe9588bac7ccf55aa0bc7f278Chuck Lever		dprintk("RPC:       write space: waking waiting task on "
51046121cf7d85869bfe9588bac7ccf55aa0bc7f278Chuck Lever				"xprt %p\n", xprt);
511fda1393938035559b417dd5b26b9cc293a7aee00Trond Myklebust		rpc_wake_up_queued_task(&xprt->pending, xprt->snd_task);
512c7b2cae8a634015b72941ba2fc6c4bc9b8d3a129Chuck Lever	}
513c7b2cae8a634015b72941ba2fc6c4bc9b8d3a129Chuck Lever	spin_unlock_bh(&xprt->transport_lock);
514c7b2cae8a634015b72941ba2fc6c4bc9b8d3a129Chuck Lever}
5151244480976d357447aeddd3f44977586bfa0462b\"Talpey, Thomas\EXPORT_SYMBOL_GPL(xprt_write_space);
516c7b2cae8a634015b72941ba2fc6c4bc9b8d3a129Chuck Lever
517fe3aca290f17ae4978bd73d02aa4029f1c9c024cChuck Lever/**
518fe3aca290f17ae4978bd73d02aa4029f1c9c024cChuck Lever * xprt_set_retrans_timeout_def - set a request's retransmit timeout
519fe3aca290f17ae4978bd73d02aa4029f1c9c024cChuck Lever * @task: task whose timeout is to be set
520fe3aca290f17ae4978bd73d02aa4029f1c9c024cChuck Lever *
521fe3aca290f17ae4978bd73d02aa4029f1c9c024cChuck Lever * Set a request's retransmit timeout based on the transport's
522fe3aca290f17ae4978bd73d02aa4029f1c9c024cChuck Lever * default timeout parameters.  Used by transports that don't adjust
523fe3aca290f17ae4978bd73d02aa4029f1c9c024cChuck Lever * the retransmit timeout based on round-trip time estimation.
524fe3aca290f17ae4978bd73d02aa4029f1c9c024cChuck Lever */
525fe3aca290f17ae4978bd73d02aa4029f1c9c024cChuck Levervoid xprt_set_retrans_timeout_def(struct rpc_task *task)
526fe3aca290f17ae4978bd73d02aa4029f1c9c024cChuck Lever{
527fe3aca290f17ae4978bd73d02aa4029f1c9c024cChuck Lever	task->tk_timeout = task->tk_rqstp->rq_timeout;
528fe3aca290f17ae4978bd73d02aa4029f1c9c024cChuck Lever}
5291244480976d357447aeddd3f44977586bfa0462b\"Talpey, Thomas\EXPORT_SYMBOL_GPL(xprt_set_retrans_timeout_def);
530fe3aca290f17ae4978bd73d02aa4029f1c9c024cChuck Lever
5312c53040f018b6c36a46eec75b9b937aaa5f78e6dBen Hutchings/**
532fe3aca290f17ae4978bd73d02aa4029f1c9c024cChuck Lever * xprt_set_retrans_timeout_rtt - set a request's retransmit timeout
533fe3aca290f17ae4978bd73d02aa4029f1c9c024cChuck Lever * @task: task whose timeout is to be set
534cca5172a7ec10dfdb0b787cd8e9d5b0b8f179793YOSHIFUJI Hideaki *
535fe3aca290f17ae4978bd73d02aa4029f1c9c024cChuck Lever * Set a request's retransmit timeout using the RTT estimator.
536fe3aca290f17ae4978bd73d02aa4029f1c9c024cChuck Lever */
537fe3aca290f17ae4978bd73d02aa4029f1c9c024cChuck Levervoid xprt_set_retrans_timeout_rtt(struct rpc_task *task)
538fe3aca290f17ae4978bd73d02aa4029f1c9c024cChuck Lever{
539fe3aca290f17ae4978bd73d02aa4029f1c9c024cChuck Lever	int timer = task->tk_msg.rpc_proc->p_timer;
540ba7392bb37cb12781890f45d7ddee1618e33a036Trond Myklebust	struct rpc_clnt *clnt = task->tk_client;
541ba7392bb37cb12781890f45d7ddee1618e33a036Trond Myklebust	struct rpc_rtt *rtt = clnt->cl_rtt;
542fe3aca290f17ae4978bd73d02aa4029f1c9c024cChuck Lever	struct rpc_rqst *req = task->tk_rqstp;
543ba7392bb37cb12781890f45d7ddee1618e33a036Trond Myklebust	unsigned long max_timeout = clnt->cl_timeout->to_maxval;
544fe3aca290f17ae4978bd73d02aa4029f1c9c024cChuck Lever
545fe3aca290f17ae4978bd73d02aa4029f1c9c024cChuck Lever	task->tk_timeout = rpc_calc_rto(rtt, timer);
546fe3aca290f17ae4978bd73d02aa4029f1c9c024cChuck Lever	task->tk_timeout <<= rpc_ntimeo(rtt, timer) + req->rq_retries;
547fe3aca290f17ae4978bd73d02aa4029f1c9c024cChuck Lever	if (task->tk_timeout > max_timeout || task->tk_timeout == 0)
548fe3aca290f17ae4978bd73d02aa4029f1c9c024cChuck Lever		task->tk_timeout = max_timeout;
549fe3aca290f17ae4978bd73d02aa4029f1c9c024cChuck Lever}
5501244480976d357447aeddd3f44977586bfa0462b\"Talpey, Thomas\EXPORT_SYMBOL_GPL(xprt_set_retrans_timeout_rtt);
551fe3aca290f17ae4978bd73d02aa4029f1c9c024cChuck Lever
5521da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvaldsstatic void xprt_reset_majortimeo(struct rpc_rqst *req)
5531da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds{
554ba7392bb37cb12781890f45d7ddee1618e33a036Trond Myklebust	const struct rpc_timeout *to = req->rq_task->tk_client->cl_timeout;
5551da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds
5561da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds	req->rq_majortimeo = req->rq_timeout;
5571da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds	if (to->to_exponential)
5581da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds		req->rq_majortimeo <<= to->to_retries;
5591da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds	else
5601da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds		req->rq_majortimeo += to->to_increment * to->to_retries;
5611da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds	if (req->rq_majortimeo > to->to_maxval || req->rq_majortimeo == 0)
5621da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds		req->rq_majortimeo = to->to_maxval;
5631da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds	req->rq_majortimeo += jiffies;
5641da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds}
5651da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds
5669903cd1c27a1f30e8efea75e125be3b2002f7cb9Chuck Lever/**
5679903cd1c27a1f30e8efea75e125be3b2002f7cb9Chuck Lever * xprt_adjust_timeout - adjust timeout values for next retransmit
5689903cd1c27a1f30e8efea75e125be3b2002f7cb9Chuck Lever * @req: RPC request containing parameters to use for the adjustment
5699903cd1c27a1f30e8efea75e125be3b2002f7cb9Chuck Lever *
5701da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds */
5711da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvaldsint xprt_adjust_timeout(struct rpc_rqst *req)
5721da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds{
5731da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds	struct rpc_xprt *xprt = req->rq_xprt;
574ba7392bb37cb12781890f45d7ddee1618e33a036Trond Myklebust	const struct rpc_timeout *to = req->rq_task->tk_client->cl_timeout;
5751da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds	int status = 0;
5761da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds
5771da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds	if (time_before(jiffies, req->rq_majortimeo)) {
5781da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds		if (to->to_exponential)
5791da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds			req->rq_timeout <<= 1;
5801da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds		else
5811da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds			req->rq_timeout += to->to_increment;
5821da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds		if (to->to_maxval && req->rq_timeout >= to->to_maxval)
5831da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds			req->rq_timeout = to->to_maxval;
5841da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds		req->rq_retries++;
5851da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds	} else {
5861da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds		req->rq_timeout = to->to_initval;
5871da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds		req->rq_retries = 0;
5881da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds		xprt_reset_majortimeo(req);
5891da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds		/* Reset the RTT counters == "slow start" */
5904a0f8c04f2ece949d54a0c4fd7490259cf23a58aChuck Lever		spin_lock_bh(&xprt->transport_lock);
5911da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds		rpc_init_rtt(req->rq_task->tk_client->cl_rtt, to->to_initval);
5924a0f8c04f2ece949d54a0c4fd7490259cf23a58aChuck Lever		spin_unlock_bh(&xprt->transport_lock);
5931da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds		status = -ETIMEDOUT;
5941da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds	}
5951da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds
5961da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds	if (req->rq_timeout == 0) {
5971da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds		printk(KERN_WARNING "xprt_adjust_timeout: rq_timeout = 0!\n");
5981da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds		req->rq_timeout = 5 * HZ;
5991da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds	}
6001da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds	return status;
6011da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds}
6021da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds
60365f27f38446e1976cc98fd3004b110fedcddd189David Howellsstatic void xprt_autoclose(struct work_struct *work)
6041da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds{
60565f27f38446e1976cc98fd3004b110fedcddd189David Howells	struct rpc_xprt *xprt =
60665f27f38446e1976cc98fd3004b110fedcddd189David Howells		container_of(work, struct rpc_xprt, task_cleanup);
6071da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds
608a246b0105bbd9a70a698f69baae2042996f2a0e9Chuck Lever	xprt->ops->close(xprt);
60966af1e558538137080615e7ad6d1f2f80862de01Trond Myklebust	clear_bit(XPRT_CLOSE_WAIT, &xprt->state);
6101da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds	xprt_release_write(xprt, NULL);
6111da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds}
6121da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds
6139903cd1c27a1f30e8efea75e125be3b2002f7cb9Chuck Lever/**
61462da3b24880bccd4ffc32cf8d9a7e23fab475bddTrond Myklebust * xprt_disconnect_done - mark a transport as disconnected
6159903cd1c27a1f30e8efea75e125be3b2002f7cb9Chuck Lever * @xprt: transport to flag for disconnect
6169903cd1c27a1f30e8efea75e125be3b2002f7cb9Chuck Lever *
6171da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds */
61862da3b24880bccd4ffc32cf8d9a7e23fab475bddTrond Myklebustvoid xprt_disconnect_done(struct rpc_xprt *xprt)
6191da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds{
62046121cf7d85869bfe9588bac7ccf55aa0bc7f278Chuck Lever	dprintk("RPC:       disconnected transport %p\n", xprt);
6214a0f8c04f2ece949d54a0c4fd7490259cf23a58aChuck Lever	spin_lock_bh(&xprt->transport_lock);
6221da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds	xprt_clear_connected(xprt);
6232a4919919a97911b0aa4b9f5ac1eab90ba87652bTrond Myklebust	xprt_wake_pending_tasks(xprt, -EAGAIN);
6244a0f8c04f2ece949d54a0c4fd7490259cf23a58aChuck Lever	spin_unlock_bh(&xprt->transport_lock);
6251da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds}
62662da3b24880bccd4ffc32cf8d9a7e23fab475bddTrond MyklebustEXPORT_SYMBOL_GPL(xprt_disconnect_done);
6271da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds
62866af1e558538137080615e7ad6d1f2f80862de01Trond Myklebust/**
62966af1e558538137080615e7ad6d1f2f80862de01Trond Myklebust * xprt_force_disconnect - force a transport to disconnect
63066af1e558538137080615e7ad6d1f2f80862de01Trond Myklebust * @xprt: transport to disconnect
63166af1e558538137080615e7ad6d1f2f80862de01Trond Myklebust *
63266af1e558538137080615e7ad6d1f2f80862de01Trond Myklebust */
63366af1e558538137080615e7ad6d1f2f80862de01Trond Myklebustvoid xprt_force_disconnect(struct rpc_xprt *xprt)
63466af1e558538137080615e7ad6d1f2f80862de01Trond Myklebust{
63566af1e558538137080615e7ad6d1f2f80862de01Trond Myklebust	/* Don't race with the test_bit() in xprt_clear_locked() */
63666af1e558538137080615e7ad6d1f2f80862de01Trond Myklebust	spin_lock_bh(&xprt->transport_lock);
63766af1e558538137080615e7ad6d1f2f80862de01Trond Myklebust	set_bit(XPRT_CLOSE_WAIT, &xprt->state);
63866af1e558538137080615e7ad6d1f2f80862de01Trond Myklebust	/* Try to schedule an autoclose RPC call */
63966af1e558538137080615e7ad6d1f2f80862de01Trond Myklebust	if (test_and_set_bit(XPRT_LOCKED, &xprt->state) == 0)
64066af1e558538137080615e7ad6d1f2f80862de01Trond Myklebust		queue_work(rpciod_workqueue, &xprt->task_cleanup);
6412a4919919a97911b0aa4b9f5ac1eab90ba87652bTrond Myklebust	xprt_wake_pending_tasks(xprt, -EAGAIN);
64266af1e558538137080615e7ad6d1f2f80862de01Trond Myklebust	spin_unlock_bh(&xprt->transport_lock);
64366af1e558538137080615e7ad6d1f2f80862de01Trond Myklebust}
64466af1e558538137080615e7ad6d1f2f80862de01Trond Myklebust
6457c1d71cf56feebfb5b98219b9d11dfc3a2feca62Trond Myklebust/**
6467c1d71cf56feebfb5b98219b9d11dfc3a2feca62Trond Myklebust * xprt_conditional_disconnect - force a transport to disconnect
6477c1d71cf56feebfb5b98219b9d11dfc3a2feca62Trond Myklebust * @xprt: transport to disconnect
6487c1d71cf56feebfb5b98219b9d11dfc3a2feca62Trond Myklebust * @cookie: 'connection cookie'
6497c1d71cf56feebfb5b98219b9d11dfc3a2feca62Trond Myklebust *
6507c1d71cf56feebfb5b98219b9d11dfc3a2feca62Trond Myklebust * This attempts to break the connection if and only if 'cookie' matches
6517c1d71cf56feebfb5b98219b9d11dfc3a2feca62Trond Myklebust * the current transport 'connection cookie'. It ensures that we don't
6527c1d71cf56feebfb5b98219b9d11dfc3a2feca62Trond Myklebust * try to break the connection more than once when we need to retransmit
6537c1d71cf56feebfb5b98219b9d11dfc3a2feca62Trond Myklebust * a batch of RPC requests.
6547c1d71cf56feebfb5b98219b9d11dfc3a2feca62Trond Myklebust *
6557c1d71cf56feebfb5b98219b9d11dfc3a2feca62Trond Myklebust */
6567c1d71cf56feebfb5b98219b9d11dfc3a2feca62Trond Myklebustvoid xprt_conditional_disconnect(struct rpc_xprt *xprt, unsigned int cookie)
6577c1d71cf56feebfb5b98219b9d11dfc3a2feca62Trond Myklebust{
6587c1d71cf56feebfb5b98219b9d11dfc3a2feca62Trond Myklebust	/* Don't race with the test_bit() in xprt_clear_locked() */
6597c1d71cf56feebfb5b98219b9d11dfc3a2feca62Trond Myklebust	spin_lock_bh(&xprt->transport_lock);
6607c1d71cf56feebfb5b98219b9d11dfc3a2feca62Trond Myklebust	if (cookie != xprt->connect_cookie)
6617c1d71cf56feebfb5b98219b9d11dfc3a2feca62Trond Myklebust		goto out;
6627c1d71cf56feebfb5b98219b9d11dfc3a2feca62Trond Myklebust	if (test_bit(XPRT_CLOSING, &xprt->state) || !xprt_connected(xprt))
6637c1d71cf56feebfb5b98219b9d11dfc3a2feca62Trond Myklebust		goto out;
6647c1d71cf56feebfb5b98219b9d11dfc3a2feca62Trond Myklebust	set_bit(XPRT_CLOSE_WAIT, &xprt->state);
6657c1d71cf56feebfb5b98219b9d11dfc3a2feca62Trond Myklebust	/* Try to schedule an autoclose RPC call */
6667c1d71cf56feebfb5b98219b9d11dfc3a2feca62Trond Myklebust	if (test_and_set_bit(XPRT_LOCKED, &xprt->state) == 0)
6677c1d71cf56feebfb5b98219b9d11dfc3a2feca62Trond Myklebust		queue_work(rpciod_workqueue, &xprt->task_cleanup);
6682a4919919a97911b0aa4b9f5ac1eab90ba87652bTrond Myklebust	xprt_wake_pending_tasks(xprt, -EAGAIN);
6697c1d71cf56feebfb5b98219b9d11dfc3a2feca62Trond Myklebustout:
6707c1d71cf56feebfb5b98219b9d11dfc3a2feca62Trond Myklebust	spin_unlock_bh(&xprt->transport_lock);
6717c1d71cf56feebfb5b98219b9d11dfc3a2feca62Trond Myklebust}
6727c1d71cf56feebfb5b98219b9d11dfc3a2feca62Trond Myklebust
6731da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvaldsstatic void
6741da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvaldsxprt_init_autodisconnect(unsigned long data)
6751da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds{
6761da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds	struct rpc_xprt *xprt = (struct rpc_xprt *)data;
6771da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds
6784a0f8c04f2ece949d54a0c4fd7490259cf23a58aChuck Lever	spin_lock(&xprt->transport_lock);
679d19751e7b9bd8a01d00372325439589886674f79Trond Myklebust	if (!list_empty(&xprt->recv))
6801da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds		goto out_abort;
6812226feb6bcd0e5e117a9be3ea3dd3ffc14f3e41eChuck Lever	if (test_and_set_bit(XPRT_LOCKED, &xprt->state))
6821da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds		goto out_abort;
6834a0f8c04f2ece949d54a0c4fd7490259cf23a58aChuck Lever	spin_unlock(&xprt->transport_lock);
684f75e6745aa3084124ae1434fd7629853bdaf6798Trond Myklebust	set_bit(XPRT_CONNECTION_CLOSE, &xprt->state);
685f75e6745aa3084124ae1434fd7629853bdaf6798Trond Myklebust	queue_work(rpciod_workqueue, &xprt->task_cleanup);
6861da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds	return;
6871da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvaldsout_abort:
6884a0f8c04f2ece949d54a0c4fd7490259cf23a58aChuck Lever	spin_unlock(&xprt->transport_lock);
6891da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds}
6901da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds
6919903cd1c27a1f30e8efea75e125be3b2002f7cb9Chuck Lever/**
6929903cd1c27a1f30e8efea75e125be3b2002f7cb9Chuck Lever * xprt_connect - schedule a transport connect operation
6939903cd1c27a1f30e8efea75e125be3b2002f7cb9Chuck Lever * @task: RPC task that is requesting the connect
6941da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds *
6951da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds */
6961da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvaldsvoid xprt_connect(struct rpc_task *task)
6971da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds{
698ad2368d6f5ec6467b9503176e9fb878daf999629Trond Myklebust	struct rpc_xprt	*xprt = task->tk_rqstp->rq_xprt;
6991da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds
70046121cf7d85869bfe9588bac7ccf55aa0bc7f278Chuck Lever	dprintk("RPC: %5u xprt_connect xprt %p %s connected\n", task->tk_pid,
7011da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds			xprt, (xprt_connected(xprt) ? "is" : "is not"));
7021da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds
703ec739ef03dc926d05051c8c5838971445504470aChuck Lever	if (!xprt_bound(xprt)) {
70401d37c428ae080563c0a3bb8bdfa88c65a6891d3Trond Myklebust		task->tk_status = -EAGAIN;
7051da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds		return;
7061da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds	}
7071da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds	if (!xprt_lock_write(xprt, task))
7081da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds		return;
709feb8ca37cc3d83c07fd042509ef1e176cfeb2cfaTrond Myklebust
710feb8ca37cc3d83c07fd042509ef1e176cfeb2cfaTrond Myklebust	if (test_and_clear_bit(XPRT_CLOSE_WAIT, &xprt->state))
711feb8ca37cc3d83c07fd042509ef1e176cfeb2cfaTrond Myklebust		xprt->ops->close(xprt);
712feb8ca37cc3d83c07fd042509ef1e176cfeb2cfaTrond Myklebust
7131da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds	if (xprt_connected(xprt))
714a246b0105bbd9a70a698f69baae2042996f2a0e9Chuck Lever		xprt_release_write(xprt, task);
715a246b0105bbd9a70a698f69baae2042996f2a0e9Chuck Lever	else {
71687e3c0553fcbea79bf9f17fc5694484ecf3ae5e8Dan Carpenter		task->tk_rqstp->rq_bytes_sent = 0;
717a8ce4a8f37fef0a09a1e920c2e09f67a80426c7eTrond Myklebust		task->tk_timeout = task->tk_rqstp->rq_timeout;
7185d00837b90340af9106dcd93af75fd664c8eb87fTrond Myklebust		rpc_sleep_on(&xprt->pending, task, xprt_connect_status);
7190b9e79431377df452348e78262dd5a3dc359eeefTrond Myklebust
7200b9e79431377df452348e78262dd5a3dc359eeefTrond Myklebust		if (test_bit(XPRT_CLOSING, &xprt->state))
7210b9e79431377df452348e78262dd5a3dc359eeefTrond Myklebust			return;
7220b9e79431377df452348e78262dd5a3dc359eeefTrond Myklebust		if (xprt_test_and_set_connecting(xprt))
7230b9e79431377df452348e78262dd5a3dc359eeefTrond Myklebust			return;
724262ca07de4d7f1bff20361c1353bb14b3607afb2Chuck Lever		xprt->stat.connect_start = jiffies;
7251b092092bf0e2e8b7af1c2a03f615b4e60b05d47Trond Myklebust		xprt->ops->connect(xprt, task);
7261da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds	}
7271da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds}
7281da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds
7299903cd1c27a1f30e8efea75e125be3b2002f7cb9Chuck Leverstatic void xprt_connect_status(struct rpc_task *task)
7301da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds{
731ad2368d6f5ec6467b9503176e9fb878daf999629Trond Myklebust	struct rpc_xprt	*xprt = task->tk_rqstp->rq_xprt;
7321da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds
733cd983ef81b9d79573848dabf81277c7314220257Chuck Lever	if (task->tk_status == 0) {
734262ca07de4d7f1bff20361c1353bb14b3607afb2Chuck Lever		xprt->stat.connect_count++;
735262ca07de4d7f1bff20361c1353bb14b3607afb2Chuck Lever		xprt->stat.connect_time += (long)jiffies - xprt->stat.connect_start;
73646121cf7d85869bfe9588bac7ccf55aa0bc7f278Chuck Lever		dprintk("RPC: %5u xprt_connect_status: connection established\n",
7371da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds				task->tk_pid);
7381da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds		return;
7391da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds	}
7401da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds
7411da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds	switch (task->tk_status) {
7420fe8d04e8c3a1eb49089793e38b60a17cee564e3Trond Myklebust	case -ECONNREFUSED:
7430fe8d04e8c3a1eb49089793e38b60a17cee564e3Trond Myklebust	case -ECONNRESET:
7440fe8d04e8c3a1eb49089793e38b60a17cee564e3Trond Myklebust	case -ECONNABORTED:
7450fe8d04e8c3a1eb49089793e38b60a17cee564e3Trond Myklebust	case -ENETUNREACH:
7460fe8d04e8c3a1eb49089793e38b60a17cee564e3Trond Myklebust	case -EHOSTUNREACH:
7472fc193cf924ea6eb74f6a0cf73b94b2e62938ae5Trond Myklebust	case -EPIPE:
7482a4919919a97911b0aa4b9f5ac1eab90ba87652bTrond Myklebust	case -EAGAIN:
7492a4919919a97911b0aa4b9f5ac1eab90ba87652bTrond Myklebust		dprintk("RPC: %5u xprt_connect_status: retrying\n", task->tk_pid);
75023475d66bd8600e0c5353f86c1b74f68df27bdb5Chuck Lever		break;
7511da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds	case -ETIMEDOUT:
75246121cf7d85869bfe9588bac7ccf55aa0bc7f278Chuck Lever		dprintk("RPC: %5u xprt_connect_status: connect attempt timed "
75346121cf7d85869bfe9588bac7ccf55aa0bc7f278Chuck Lever				"out\n", task->tk_pid);
7541da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds		break;
7551da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds	default:
75646121cf7d85869bfe9588bac7ccf55aa0bc7f278Chuck Lever		dprintk("RPC: %5u xprt_connect_status: error %d connecting to "
75746121cf7d85869bfe9588bac7ccf55aa0bc7f278Chuck Lever				"server %s\n", task->tk_pid, -task->tk_status,
7584e0038b6b246e4145fc4a53dca61a556d17bc52cTrond Myklebust				xprt->servername);
75923475d66bd8600e0c5353f86c1b74f68df27bdb5Chuck Lever		xprt_release_write(xprt, task);
76023475d66bd8600e0c5353f86c1b74f68df27bdb5Chuck Lever		task->tk_status = -EIO;
7611da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds	}
7621da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds}
7631da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds
7649903cd1c27a1f30e8efea75e125be3b2002f7cb9Chuck Lever/**
7659903cd1c27a1f30e8efea75e125be3b2002f7cb9Chuck Lever * xprt_lookup_rqst - find an RPC request corresponding to an XID
7669903cd1c27a1f30e8efea75e125be3b2002f7cb9Chuck Lever * @xprt: transport on which the original request was transmitted
7679903cd1c27a1f30e8efea75e125be3b2002f7cb9Chuck Lever * @xid: RPC XID of incoming reply
7689903cd1c27a1f30e8efea75e125be3b2002f7cb9Chuck Lever *
7691da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds */
770d8ed029d6000ba2e2908d9286409e4833c091b4cAlexey Dobriyanstruct rpc_rqst *xprt_lookup_rqst(struct rpc_xprt *xprt, __be32 xid)
7711da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds{
7728f3a6de313391b6910aa7db185eb9f3e930a51cfPavel Emelyanov	struct rpc_rqst *entry;
7731da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds
7748f3a6de313391b6910aa7db185eb9f3e930a51cfPavel Emelyanov	list_for_each_entry(entry, &xprt->recv, rq_list)
775262ca07de4d7f1bff20361c1353bb14b3607afb2Chuck Lever		if (entry->rq_xid == xid)
776262ca07de4d7f1bff20361c1353bb14b3607afb2Chuck Lever			return entry;
77746121cf7d85869bfe9588bac7ccf55aa0bc7f278Chuck Lever
77846121cf7d85869bfe9588bac7ccf55aa0bc7f278Chuck Lever	dprintk("RPC:       xprt_lookup_rqst did not find xid %08x\n",
77946121cf7d85869bfe9588bac7ccf55aa0bc7f278Chuck Lever			ntohl(xid));
780262ca07de4d7f1bff20361c1353bb14b3607afb2Chuck Lever	xprt->stat.bad_xids++;
781262ca07de4d7f1bff20361c1353bb14b3607afb2Chuck Lever	return NULL;
7821da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds}
7831244480976d357447aeddd3f44977586bfa0462b\"Talpey, Thomas\EXPORT_SYMBOL_GPL(xprt_lookup_rqst);
7841da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds
785bbc72cea58f671665b6362be0d4e391813ac0eeeChuck Leverstatic void xprt_update_rtt(struct rpc_task *task)
7861570c1e41eabf6b7031f3e4322a2cf1cbe319feeChuck Lever{
7871570c1e41eabf6b7031f3e4322a2cf1cbe319feeChuck Lever	struct rpc_rqst *req = task->tk_rqstp;
7881570c1e41eabf6b7031f3e4322a2cf1cbe319feeChuck Lever	struct rpc_rtt *rtt = task->tk_client->cl_rtt;
78995c961747284a6b83a5e2d81240e214b0fa3464dEric Dumazet	unsigned int timer = task->tk_msg.rpc_proc->p_timer;
790d60dbb20a74c2cfa142be0a34dac3c6547ea086cTrond Myklebust	long m = usecs_to_jiffies(ktime_to_us(req->rq_rtt));
7911570c1e41eabf6b7031f3e4322a2cf1cbe319feeChuck Lever
7921570c1e41eabf6b7031f3e4322a2cf1cbe319feeChuck Lever	if (timer) {
7931570c1e41eabf6b7031f3e4322a2cf1cbe319feeChuck Lever		if (req->rq_ntrans == 1)
794ff8399709e41bf72b4cb145612a0f9a9f7283c83Chuck Lever			rpc_update_rtt(rtt, timer, m);
7951570c1e41eabf6b7031f3e4322a2cf1cbe319feeChuck Lever		rpc_set_timeo(rtt, timer, req->rq_ntrans - 1);
7961570c1e41eabf6b7031f3e4322a2cf1cbe319feeChuck Lever	}
7971570c1e41eabf6b7031f3e4322a2cf1cbe319feeChuck Lever}
7981570c1e41eabf6b7031f3e4322a2cf1cbe319feeChuck Lever
7991570c1e41eabf6b7031f3e4322a2cf1cbe319feeChuck Lever/**
8009903cd1c27a1f30e8efea75e125be3b2002f7cb9Chuck Lever * xprt_complete_rqst - called when reply processing is complete
8011570c1e41eabf6b7031f3e4322a2cf1cbe319feeChuck Lever * @task: RPC request that recently completed
8029903cd1c27a1f30e8efea75e125be3b2002f7cb9Chuck Lever * @copied: actual number of bytes received from the transport
8039903cd1c27a1f30e8efea75e125be3b2002f7cb9Chuck Lever *
8041570c1e41eabf6b7031f3e4322a2cf1cbe319feeChuck Lever * Caller holds transport lock.
8051da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds */
8061570c1e41eabf6b7031f3e4322a2cf1cbe319feeChuck Levervoid xprt_complete_rqst(struct rpc_task *task, int copied)
8071da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds{
8081570c1e41eabf6b7031f3e4322a2cf1cbe319feeChuck Lever	struct rpc_rqst *req = task->tk_rqstp;
809fda1393938035559b417dd5b26b9cc293a7aee00Trond Myklebust	struct rpc_xprt *xprt = req->rq_xprt;
8101da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds
8111570c1e41eabf6b7031f3e4322a2cf1cbe319feeChuck Lever	dprintk("RPC: %5u xid %08x complete (%d bytes received)\n",
8121570c1e41eabf6b7031f3e4322a2cf1cbe319feeChuck Lever			task->tk_pid, ntohl(req->rq_xid), copied);
8131da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds
814fda1393938035559b417dd5b26b9cc293a7aee00Trond Myklebust	xprt->stat.recvs++;
815d60dbb20a74c2cfa142be0a34dac3c6547ea086cTrond Myklebust	req->rq_rtt = ktime_sub(ktime_get(), req->rq_xtime);
816bbc72cea58f671665b6362be0d4e391813ac0eeeChuck Lever	if (xprt->ops->timer != NULL)
817bbc72cea58f671665b6362be0d4e391813ac0eeeChuck Lever		xprt_update_rtt(task);
818ef759a2e54ed434b2f72b52a14edecd6d4eadf74Chuck Lever
8191da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds	list_del_init(&req->rq_list);
8201e799b673c6b82b336ab13c48b5651d511ca3000Trond Myklebust	req->rq_private_buf.len = copied;
821dd2b63d049480979016b959abc2d141cdddb1389Ricardo Labiaga	/* Ensure all writes are done before we update */
822dd2b63d049480979016b959abc2d141cdddb1389Ricardo Labiaga	/* req->rq_reply_bytes_recvd */
82343ac3f2961b8616da26114ec6dc76ac2a61f76adTrond Myklebust	smp_wmb();
824dd2b63d049480979016b959abc2d141cdddb1389Ricardo Labiaga	req->rq_reply_bytes_recvd = copied;
825fda1393938035559b417dd5b26b9cc293a7aee00Trond Myklebust	rpc_wake_up_queued_task(&xprt->pending, task);
8261da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds}
8271244480976d357447aeddd3f44977586bfa0462b\"Talpey, Thomas\EXPORT_SYMBOL_GPL(xprt_complete_rqst);
8281da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds
82946c0ee8bc4ad3743de05e8b8b20201df44dcb6d3Chuck Leverstatic void xprt_timer(struct rpc_task *task)
8301da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds{
83146c0ee8bc4ad3743de05e8b8b20201df44dcb6d3Chuck Lever	struct rpc_rqst *req = task->tk_rqstp;
8321da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds	struct rpc_xprt *xprt = req->rq_xprt;
8331da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds
8345d00837b90340af9106dcd93af75fd664c8eb87fTrond Myklebust	if (task->tk_status != -ETIMEDOUT)
8355d00837b90340af9106dcd93af75fd664c8eb87fTrond Myklebust		return;
83646121cf7d85869bfe9588bac7ccf55aa0bc7f278Chuck Lever	dprintk("RPC: %5u xprt_timer\n", task->tk_pid);
8371da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds
8385d00837b90340af9106dcd93af75fd664c8eb87fTrond Myklebust	spin_lock_bh(&xprt->transport_lock);
839dd2b63d049480979016b959abc2d141cdddb1389Ricardo Labiaga	if (!req->rq_reply_bytes_recvd) {
84046c0ee8bc4ad3743de05e8b8b20201df44dcb6d3Chuck Lever		if (xprt->ops->timer)
8416a24dfb645dbcb05b34d08b991d082bdaa3ff072Trond Myklebust			xprt->ops->timer(xprt, task);
8425d00837b90340af9106dcd93af75fd664c8eb87fTrond Myklebust	} else
8435d00837b90340af9106dcd93af75fd664c8eb87fTrond Myklebust		task->tk_status = 0;
8445d00837b90340af9106dcd93af75fd664c8eb87fTrond Myklebust	spin_unlock_bh(&xprt->transport_lock);
8451da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds}
8461da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds
8474cfc7e6019caa3e97d2a81c48c8d575d7b38d751Rahul Iyerstatic inline int xprt_has_timer(struct rpc_xprt *xprt)
8484cfc7e6019caa3e97d2a81c48c8d575d7b38d751Rahul Iyer{
8494cfc7e6019caa3e97d2a81c48c8d575d7b38d751Rahul Iyer	return xprt->idle_timeout != 0;
8504cfc7e6019caa3e97d2a81c48c8d575d7b38d751Rahul Iyer}
8514cfc7e6019caa3e97d2a81c48c8d575d7b38d751Rahul Iyer
8529903cd1c27a1f30e8efea75e125be3b2002f7cb9Chuck Lever/**
8539903cd1c27a1f30e8efea75e125be3b2002f7cb9Chuck Lever * xprt_prepare_transmit - reserve the transport before sending a request
8549903cd1c27a1f30e8efea75e125be3b2002f7cb9Chuck Lever * @task: RPC task about to send a request
8559903cd1c27a1f30e8efea75e125be3b2002f7cb9Chuck Lever *
8561da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds */
85790051ea774613ffc6b8aad3dc665c8505d6205a8Trond Myklebustbool xprt_prepare_transmit(struct rpc_task *task)
8581da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds{
8591da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds	struct rpc_rqst	*req = task->tk_rqstp;
8601da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds	struct rpc_xprt	*xprt = req->rq_xprt;
86190051ea774613ffc6b8aad3dc665c8505d6205a8Trond Myklebust	bool ret = false;
8621da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds
86346121cf7d85869bfe9588bac7ccf55aa0bc7f278Chuck Lever	dprintk("RPC: %5u xprt_prepare_transmit\n", task->tk_pid);
8641da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds
8654a0f8c04f2ece949d54a0c4fd7490259cf23a58aChuck Lever	spin_lock_bh(&xprt->transport_lock);
8668a19a0b6cb2e2216afd68ef2047f30260cc8a220Trond Myklebust	if (!req->rq_bytes_sent) {
8678a19a0b6cb2e2216afd68ef2047f30260cc8a220Trond Myklebust		if (req->rq_reply_bytes_recvd) {
8688a19a0b6cb2e2216afd68ef2047f30260cc8a220Trond Myklebust			task->tk_status = req->rq_reply_bytes_recvd;
8698a19a0b6cb2e2216afd68ef2047f30260cc8a220Trond Myklebust			goto out_unlock;
8708a19a0b6cb2e2216afd68ef2047f30260cc8a220Trond Myklebust		}
8718a19a0b6cb2e2216afd68ef2047f30260cc8a220Trond Myklebust		if ((task->tk_flags & RPC_TASK_NO_RETRANS_TIMEOUT)
8728a19a0b6cb2e2216afd68ef2047f30260cc8a220Trond Myklebust		    && xprt_connected(xprt)
8738a19a0b6cb2e2216afd68ef2047f30260cc8a220Trond Myklebust		    && req->rq_connect_cookie == xprt->connect_cookie) {
8748a19a0b6cb2e2216afd68ef2047f30260cc8a220Trond Myklebust			xprt->ops->set_retrans_timeout(task);
8758a19a0b6cb2e2216afd68ef2047f30260cc8a220Trond Myklebust			rpc_sleep_on(&xprt->pending, task, xprt_timer);
8768a19a0b6cb2e2216afd68ef2047f30260cc8a220Trond Myklebust			goto out_unlock;
8778a19a0b6cb2e2216afd68ef2047f30260cc8a220Trond Myklebust		}
8781da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds	}
87990051ea774613ffc6b8aad3dc665c8505d6205a8Trond Myklebust	if (!xprt->ops->reserve_xprt(xprt, task)) {
88090051ea774613ffc6b8aad3dc665c8505d6205a8Trond Myklebust		task->tk_status = -EAGAIN;
88190051ea774613ffc6b8aad3dc665c8505d6205a8Trond Myklebust		goto out_unlock;
88290051ea774613ffc6b8aad3dc665c8505d6205a8Trond Myklebust	}
88390051ea774613ffc6b8aad3dc665c8505d6205a8Trond Myklebust	ret = true;
8841da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvaldsout_unlock:
8854a0f8c04f2ece949d54a0c4fd7490259cf23a58aChuck Lever	spin_unlock_bh(&xprt->transport_lock);
88690051ea774613ffc6b8aad3dc665c8505d6205a8Trond Myklebust	return ret;
8871da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds}
8881da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds
889e0ab53deaa91293a7958d63d5a2cf4c5645ad6f0Trond Myklebustvoid xprt_end_transmit(struct rpc_task *task)
8905e5ce5be6f0161d2a069a4f8a1154fe639c5c02fTrond Myklebust{
891343952fa5aac888934ffc203abed26a823400eb6Rahul Iyer	xprt_release_write(task->tk_rqstp->rq_xprt, task);
8925e5ce5be6f0161d2a069a4f8a1154fe639c5c02fTrond Myklebust}
8935e5ce5be6f0161d2a069a4f8a1154fe639c5c02fTrond Myklebust
8949903cd1c27a1f30e8efea75e125be3b2002f7cb9Chuck Lever/**
8959903cd1c27a1f30e8efea75e125be3b2002f7cb9Chuck Lever * xprt_transmit - send an RPC request on a transport
8969903cd1c27a1f30e8efea75e125be3b2002f7cb9Chuck Lever * @task: controlling RPC task
8979903cd1c27a1f30e8efea75e125be3b2002f7cb9Chuck Lever *
8989903cd1c27a1f30e8efea75e125be3b2002f7cb9Chuck Lever * We have to copy the iovec because sendmsg fiddles with its contents.
8999903cd1c27a1f30e8efea75e125be3b2002f7cb9Chuck Lever */
9009903cd1c27a1f30e8efea75e125be3b2002f7cb9Chuck Levervoid xprt_transmit(struct rpc_task *task)
9011da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds{
9021da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds	struct rpc_rqst	*req = task->tk_rqstp;
9031da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds	struct rpc_xprt	*xprt = req->rq_xprt;
90415a4520621824a3c2eb2de2d1f3984bc1663d3c8Andy Adamson	int status, numreqs;
9051da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds
90646121cf7d85869bfe9588bac7ccf55aa0bc7f278Chuck Lever	dprintk("RPC: %5u xprt_transmit(%u)\n", task->tk_pid, req->rq_slen);
9071da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds
908dd2b63d049480979016b959abc2d141cdddb1389Ricardo Labiaga	if (!req->rq_reply_bytes_recvd) {
90955ae1aabfb108106dd095de2578ceef1c755a8b8Ricardo Labiaga		if (list_empty(&req->rq_list) && rpc_reply_expected(task)) {
91055ae1aabfb108106dd095de2578ceef1c755a8b8Ricardo Labiaga			/*
91155ae1aabfb108106dd095de2578ceef1c755a8b8Ricardo Labiaga			 * Add to the list only if we're expecting a reply
91255ae1aabfb108106dd095de2578ceef1c755a8b8Ricardo Labiaga			 */
9134a0f8c04f2ece949d54a0c4fd7490259cf23a58aChuck Lever			spin_lock_bh(&xprt->transport_lock);
9141da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds			/* Update the softirq receive buffer */
9151da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds			memcpy(&req->rq_private_buf, &req->rq_rcv_buf,
9161da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds					sizeof(req->rq_private_buf));
9171da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds			/* Add request to the receive list */
9181da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds			list_add_tail(&req->rq_list, &xprt->recv);
9194a0f8c04f2ece949d54a0c4fd7490259cf23a58aChuck Lever			spin_unlock_bh(&xprt->transport_lock);
9201da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds			xprt_reset_majortimeo(req);
9210f9dc2b16884bb5957d010ed8e9114e771a05916Trond Myklebust			/* Turn off autodisconnect */
9220f9dc2b16884bb5957d010ed8e9114e771a05916Trond Myklebust			del_singleshot_timer_sync(&xprt->timer);
9231da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds		}
9241da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds	} else if (!req->rq_bytes_sent)
9251da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds		return;
9261da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds
927ff8399709e41bf72b4cb145612a0f9a9f7283c83Chuck Lever	req->rq_xtime = ktime_get();
928a246b0105bbd9a70a698f69baae2042996f2a0e9Chuck Lever	status = xprt->ops->send_request(task);
929c8485e4d634f6df155040293928707f127f0d06dTrond Myklebust	if (status != 0) {
930c8485e4d634f6df155040293928707f127f0d06dTrond Myklebust		task->tk_status = status;
931c8485e4d634f6df155040293928707f127f0d06dTrond Myklebust		return;
932c8485e4d634f6df155040293928707f127f0d06dTrond Myklebust	}
933262ca07de4d7f1bff20361c1353bb14b3607afb2Chuck Lever
934c8485e4d634f6df155040293928707f127f0d06dTrond Myklebust	dprintk("RPC: %5u xmit complete\n", task->tk_pid);
935468f86134ee515234afe5c5b3f39f266c50e61a5Bryan Schumaker	task->tk_flags |= RPC_TASK_SENT;
936c8485e4d634f6df155040293928707f127f0d06dTrond Myklebust	spin_lock_bh(&xprt->transport_lock);
937262ca07de4d7f1bff20361c1353bb14b3607afb2Chuck Lever
938c8485e4d634f6df155040293928707f127f0d06dTrond Myklebust	xprt->ops->set_retrans_timeout(task);
939262ca07de4d7f1bff20361c1353bb14b3607afb2Chuck Lever
94015a4520621824a3c2eb2de2d1f3984bc1663d3c8Andy Adamson	numreqs = atomic_read(&xprt->num_reqs);
94115a4520621824a3c2eb2de2d1f3984bc1663d3c8Andy Adamson	if (numreqs > xprt->stat.max_slots)
94215a4520621824a3c2eb2de2d1f3984bc1663d3c8Andy Adamson		xprt->stat.max_slots = numreqs;
943c8485e4d634f6df155040293928707f127f0d06dTrond Myklebust	xprt->stat.sends++;
944c8485e4d634f6df155040293928707f127f0d06dTrond Myklebust	xprt->stat.req_u += xprt->stat.sends - xprt->stat.recvs;
945c8485e4d634f6df155040293928707f127f0d06dTrond Myklebust	xprt->stat.bklog_u += xprt->backlog.qlen;
94615a4520621824a3c2eb2de2d1f3984bc1663d3c8Andy Adamson	xprt->stat.sending_u += xprt->sending.qlen;
94715a4520621824a3c2eb2de2d1f3984bc1663d3c8Andy Adamson	xprt->stat.pending_u += xprt->pending.qlen;
9481da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds
949c8485e4d634f6df155040293928707f127f0d06dTrond Myklebust	/* Don't race with disconnect */
950c8485e4d634f6df155040293928707f127f0d06dTrond Myklebust	if (!xprt_connected(xprt))
951c8485e4d634f6df155040293928707f127f0d06dTrond Myklebust		task->tk_status = -ENOTCONN;
9520a6605213040dd2fb479f0d1a9a87a1d7fa70904Trond Myklebust	else {
95355ae1aabfb108106dd095de2578ceef1c755a8b8Ricardo Labiaga		/*
95455ae1aabfb108106dd095de2578ceef1c755a8b8Ricardo Labiaga		 * Sleep on the pending queue since
95555ae1aabfb108106dd095de2578ceef1c755a8b8Ricardo Labiaga		 * we're expecting a reply.
95655ae1aabfb108106dd095de2578ceef1c755a8b8Ricardo Labiaga		 */
9570a6605213040dd2fb479f0d1a9a87a1d7fa70904Trond Myklebust		if (!req->rq_reply_bytes_recvd && rpc_reply_expected(task))
9580a6605213040dd2fb479f0d1a9a87a1d7fa70904Trond Myklebust			rpc_sleep_on(&xprt->pending, task, xprt_timer);
9590a6605213040dd2fb479f0d1a9a87a1d7fa70904Trond Myklebust		req->rq_connect_cookie = xprt->connect_cookie;
96055ae1aabfb108106dd095de2578ceef1c755a8b8Ricardo Labiaga	}
961c8485e4d634f6df155040293928707f127f0d06dTrond Myklebust	spin_unlock_bh(&xprt->transport_lock);
9621da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds}
9631da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds
964ba60eb25ff6be6f8e60488cdfd454e5c612bce60Trond Myklebuststatic void xprt_add_backlog(struct rpc_xprt *xprt, struct rpc_task *task)
965ba60eb25ff6be6f8e60488cdfd454e5c612bce60Trond Myklebust{
966ba60eb25ff6be6f8e60488cdfd454e5c612bce60Trond Myklebust	set_bit(XPRT_CONGESTED, &xprt->state);
967ba60eb25ff6be6f8e60488cdfd454e5c612bce60Trond Myklebust	rpc_sleep_on(&xprt->backlog, task, NULL);
968ba60eb25ff6be6f8e60488cdfd454e5c612bce60Trond Myklebust}
969ba60eb25ff6be6f8e60488cdfd454e5c612bce60Trond Myklebust
970ba60eb25ff6be6f8e60488cdfd454e5c612bce60Trond Myklebuststatic void xprt_wake_up_backlog(struct rpc_xprt *xprt)
971ba60eb25ff6be6f8e60488cdfd454e5c612bce60Trond Myklebust{
972ba60eb25ff6be6f8e60488cdfd454e5c612bce60Trond Myklebust	if (rpc_wake_up_next(&xprt->backlog) == NULL)
973ba60eb25ff6be6f8e60488cdfd454e5c612bce60Trond Myklebust		clear_bit(XPRT_CONGESTED, &xprt->state);
974ba60eb25ff6be6f8e60488cdfd454e5c612bce60Trond Myklebust}
975ba60eb25ff6be6f8e60488cdfd454e5c612bce60Trond Myklebust
976ba60eb25ff6be6f8e60488cdfd454e5c612bce60Trond Myklebuststatic bool xprt_throttle_congested(struct rpc_xprt *xprt, struct rpc_task *task)
977ba60eb25ff6be6f8e60488cdfd454e5c612bce60Trond Myklebust{
978ba60eb25ff6be6f8e60488cdfd454e5c612bce60Trond Myklebust	bool ret = false;
979ba60eb25ff6be6f8e60488cdfd454e5c612bce60Trond Myklebust
980ba60eb25ff6be6f8e60488cdfd454e5c612bce60Trond Myklebust	if (!test_bit(XPRT_CONGESTED, &xprt->state))
981ba60eb25ff6be6f8e60488cdfd454e5c612bce60Trond Myklebust		goto out;
982ba60eb25ff6be6f8e60488cdfd454e5c612bce60Trond Myklebust	spin_lock(&xprt->reserve_lock);
983ba60eb25ff6be6f8e60488cdfd454e5c612bce60Trond Myklebust	if (test_bit(XPRT_CONGESTED, &xprt->state)) {
984ba60eb25ff6be6f8e60488cdfd454e5c612bce60Trond Myklebust		rpc_sleep_on(&xprt->backlog, task, NULL);
985ba60eb25ff6be6f8e60488cdfd454e5c612bce60Trond Myklebust		ret = true;
986ba60eb25ff6be6f8e60488cdfd454e5c612bce60Trond Myklebust	}
987ba60eb25ff6be6f8e60488cdfd454e5c612bce60Trond Myklebust	spin_unlock(&xprt->reserve_lock);
988ba60eb25ff6be6f8e60488cdfd454e5c612bce60Trond Myklebustout:
989ba60eb25ff6be6f8e60488cdfd454e5c612bce60Trond Myklebust	return ret;
990ba60eb25ff6be6f8e60488cdfd454e5c612bce60Trond Myklebust}
991ba60eb25ff6be6f8e60488cdfd454e5c612bce60Trond Myklebust
992d9ba131d8f58c0d2ff5029e7002ab43f913b36f9Trond Myklebuststatic struct rpc_rqst *xprt_dynamic_alloc_slot(struct rpc_xprt *xprt, gfp_t gfp_flags)
993d9ba131d8f58c0d2ff5029e7002ab43f913b36f9Trond Myklebust{
994d9ba131d8f58c0d2ff5029e7002ab43f913b36f9Trond Myklebust	struct rpc_rqst *req = ERR_PTR(-EAGAIN);
995d9ba131d8f58c0d2ff5029e7002ab43f913b36f9Trond Myklebust
996d9ba131d8f58c0d2ff5029e7002ab43f913b36f9Trond Myklebust	if (!atomic_add_unless(&xprt->num_reqs, 1, xprt->max_reqs))
997d9ba131d8f58c0d2ff5029e7002ab43f913b36f9Trond Myklebust		goto out;
998d9ba131d8f58c0d2ff5029e7002ab43f913b36f9Trond Myklebust	req = kzalloc(sizeof(struct rpc_rqst), gfp_flags);
999d9ba131d8f58c0d2ff5029e7002ab43f913b36f9Trond Myklebust	if (req != NULL)
1000d9ba131d8f58c0d2ff5029e7002ab43f913b36f9Trond Myklebust		goto out;
1001d9ba131d8f58c0d2ff5029e7002ab43f913b36f9Trond Myklebust	atomic_dec(&xprt->num_reqs);
1002d9ba131d8f58c0d2ff5029e7002ab43f913b36f9Trond Myklebust	req = ERR_PTR(-ENOMEM);
1003d9ba131d8f58c0d2ff5029e7002ab43f913b36f9Trond Myklebustout:
1004d9ba131d8f58c0d2ff5029e7002ab43f913b36f9Trond Myklebust	return req;
1005d9ba131d8f58c0d2ff5029e7002ab43f913b36f9Trond Myklebust}
1006d9ba131d8f58c0d2ff5029e7002ab43f913b36f9Trond Myklebust
1007d9ba131d8f58c0d2ff5029e7002ab43f913b36f9Trond Myklebuststatic bool xprt_dynamic_free_slot(struct rpc_xprt *xprt, struct rpc_rqst *req)
1008d9ba131d8f58c0d2ff5029e7002ab43f913b36f9Trond Myklebust{
1009d9ba131d8f58c0d2ff5029e7002ab43f913b36f9Trond Myklebust	if (atomic_add_unless(&xprt->num_reqs, -1, xprt->min_reqs)) {
1010d9ba131d8f58c0d2ff5029e7002ab43f913b36f9Trond Myklebust		kfree(req);
1011d9ba131d8f58c0d2ff5029e7002ab43f913b36f9Trond Myklebust		return true;
1012d9ba131d8f58c0d2ff5029e7002ab43f913b36f9Trond Myklebust	}
1013d9ba131d8f58c0d2ff5029e7002ab43f913b36f9Trond Myklebust	return false;
1014d9ba131d8f58c0d2ff5029e7002ab43f913b36f9Trond Myklebust}
1015d9ba131d8f58c0d2ff5029e7002ab43f913b36f9Trond Myklebust
1016f39c1bfb5a03e2d255451bff05be0d7255298fa4Trond Myklebustvoid xprt_alloc_slot(struct rpc_xprt *xprt, struct rpc_task *task)
10171da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds{
1018d9ba131d8f58c0d2ff5029e7002ab43f913b36f9Trond Myklebust	struct rpc_rqst *req;
10191da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds
1020f39c1bfb5a03e2d255451bff05be0d7255298fa4Trond Myklebust	spin_lock(&xprt->reserve_lock);
10211da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds	if (!list_empty(&xprt->free)) {
1022d9ba131d8f58c0d2ff5029e7002ab43f913b36f9Trond Myklebust		req = list_entry(xprt->free.next, struct rpc_rqst, rq_list);
1023d9ba131d8f58c0d2ff5029e7002ab43f913b36f9Trond Myklebust		list_del(&req->rq_list);
1024d9ba131d8f58c0d2ff5029e7002ab43f913b36f9Trond Myklebust		goto out_init_req;
1025d9ba131d8f58c0d2ff5029e7002ab43f913b36f9Trond Myklebust	}
10266b34309936ed5c85cbe5868655814065f42c2f38Jeff Layton	req = xprt_dynamic_alloc_slot(xprt, GFP_NOWAIT|__GFP_NOWARN);
1027d9ba131d8f58c0d2ff5029e7002ab43f913b36f9Trond Myklebust	if (!IS_ERR(req))
1028d9ba131d8f58c0d2ff5029e7002ab43f913b36f9Trond Myklebust		goto out_init_req;
1029d9ba131d8f58c0d2ff5029e7002ab43f913b36f9Trond Myklebust	switch (PTR_ERR(req)) {
1030d9ba131d8f58c0d2ff5029e7002ab43f913b36f9Trond Myklebust	case -ENOMEM:
1031d9ba131d8f58c0d2ff5029e7002ab43f913b36f9Trond Myklebust		dprintk("RPC:       dynamic allocation of request slot "
1032d9ba131d8f58c0d2ff5029e7002ab43f913b36f9Trond Myklebust				"failed! Retrying\n");
10331afeaf5c29aa07db25760d2fbed5c08a3aec3498Trond Myklebust		task->tk_status = -ENOMEM;
1034d9ba131d8f58c0d2ff5029e7002ab43f913b36f9Trond Myklebust		break;
1035d9ba131d8f58c0d2ff5029e7002ab43f913b36f9Trond Myklebust	case -EAGAIN:
1036ba60eb25ff6be6f8e60488cdfd454e5c612bce60Trond Myklebust		xprt_add_backlog(xprt, task);
1037d9ba131d8f58c0d2ff5029e7002ab43f913b36f9Trond Myklebust		dprintk("RPC:       waiting for request slot\n");
10381afeaf5c29aa07db25760d2fbed5c08a3aec3498Trond Myklebust	default:
10391afeaf5c29aa07db25760d2fbed5c08a3aec3498Trond Myklebust		task->tk_status = -EAGAIN;
10401da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds	}
1041f39c1bfb5a03e2d255451bff05be0d7255298fa4Trond Myklebust	spin_unlock(&xprt->reserve_lock);
1042d9ba131d8f58c0d2ff5029e7002ab43f913b36f9Trond Myklebust	return;
1043d9ba131d8f58c0d2ff5029e7002ab43f913b36f9Trond Myklebustout_init_req:
1044d9ba131d8f58c0d2ff5029e7002ab43f913b36f9Trond Myklebust	task->tk_status = 0;
1045d9ba131d8f58c0d2ff5029e7002ab43f913b36f9Trond Myklebust	task->tk_rqstp = req;
1046d9ba131d8f58c0d2ff5029e7002ab43f913b36f9Trond Myklebust	xprt_request_init(task, xprt);
1047f39c1bfb5a03e2d255451bff05be0d7255298fa4Trond Myklebust	spin_unlock(&xprt->reserve_lock);
1048f39c1bfb5a03e2d255451bff05be0d7255298fa4Trond Myklebust}
1049f39c1bfb5a03e2d255451bff05be0d7255298fa4Trond MyklebustEXPORT_SYMBOL_GPL(xprt_alloc_slot);
1050f39c1bfb5a03e2d255451bff05be0d7255298fa4Trond Myklebust
1051f39c1bfb5a03e2d255451bff05be0d7255298fa4Trond Myklebustvoid xprt_lock_and_alloc_slot(struct rpc_xprt *xprt, struct rpc_task *task)
1052f39c1bfb5a03e2d255451bff05be0d7255298fa4Trond Myklebust{
1053f39c1bfb5a03e2d255451bff05be0d7255298fa4Trond Myklebust	/* Note: grabbing the xprt_lock_write() ensures that we throttle
1054f39c1bfb5a03e2d255451bff05be0d7255298fa4Trond Myklebust	 * new slot allocation if the transport is congested (i.e. when
1055f39c1bfb5a03e2d255451bff05be0d7255298fa4Trond Myklebust	 * reconnecting a stream transport or when out of socket write
1056f39c1bfb5a03e2d255451bff05be0d7255298fa4Trond Myklebust	 * buffer space).
1057f39c1bfb5a03e2d255451bff05be0d7255298fa4Trond Myklebust	 */
1058f39c1bfb5a03e2d255451bff05be0d7255298fa4Trond Myklebust	if (xprt_lock_write(xprt, task)) {
1059f39c1bfb5a03e2d255451bff05be0d7255298fa4Trond Myklebust		xprt_alloc_slot(xprt, task);
1060f39c1bfb5a03e2d255451bff05be0d7255298fa4Trond Myklebust		xprt_release_write(xprt, task);
1061f39c1bfb5a03e2d255451bff05be0d7255298fa4Trond Myklebust	}
10621da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds}
1063f39c1bfb5a03e2d255451bff05be0d7255298fa4Trond MyklebustEXPORT_SYMBOL_GPL(xprt_lock_and_alloc_slot);
10641da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds
1065ee5ebe851ed60206f150d3f189416f9c63245b66Trond Myklebuststatic void xprt_free_slot(struct rpc_xprt *xprt, struct rpc_rqst *req)
1066ee5ebe851ed60206f150d3f189416f9c63245b66Trond Myklebust{
1067ee5ebe851ed60206f150d3f189416f9c63245b66Trond Myklebust	spin_lock(&xprt->reserve_lock);
1068c25573b5134294c0be82bfaecc6d08136835b271Trond Myklebust	if (!xprt_dynamic_free_slot(xprt, req)) {
1069c25573b5134294c0be82bfaecc6d08136835b271Trond Myklebust		memset(req, 0, sizeof(*req));	/* mark unused */
1070c25573b5134294c0be82bfaecc6d08136835b271Trond Myklebust		list_add(&req->rq_list, &xprt->free);
1071c25573b5134294c0be82bfaecc6d08136835b271Trond Myklebust	}
1072ba60eb25ff6be6f8e60488cdfd454e5c612bce60Trond Myklebust	xprt_wake_up_backlog(xprt);
1073ee5ebe851ed60206f150d3f189416f9c63245b66Trond Myklebust	spin_unlock(&xprt->reserve_lock);
1074ee5ebe851ed60206f150d3f189416f9c63245b66Trond Myklebust}
1075ee5ebe851ed60206f150d3f189416f9c63245b66Trond Myklebust
107621de0a955f3af29fa1100d96f66e6adade89e77aTrond Myklebuststatic void xprt_free_all_slots(struct rpc_xprt *xprt)
107721de0a955f3af29fa1100d96f66e6adade89e77aTrond Myklebust{
107821de0a955f3af29fa1100d96f66e6adade89e77aTrond Myklebust	struct rpc_rqst *req;
107921de0a955f3af29fa1100d96f66e6adade89e77aTrond Myklebust	while (!list_empty(&xprt->free)) {
108021de0a955f3af29fa1100d96f66e6adade89e77aTrond Myklebust		req = list_first_entry(&xprt->free, struct rpc_rqst, rq_list);
108121de0a955f3af29fa1100d96f66e6adade89e77aTrond Myklebust		list_del(&req->rq_list);
108221de0a955f3af29fa1100d96f66e6adade89e77aTrond Myklebust		kfree(req);
108321de0a955f3af29fa1100d96f66e6adade89e77aTrond Myklebust	}
108421de0a955f3af29fa1100d96f66e6adade89e77aTrond Myklebust}
108521de0a955f3af29fa1100d96f66e6adade89e77aTrond Myklebust
1086d9ba131d8f58c0d2ff5029e7002ab43f913b36f9Trond Myklebuststruct rpc_xprt *xprt_alloc(struct net *net, size_t size,
1087d9ba131d8f58c0d2ff5029e7002ab43f913b36f9Trond Myklebust		unsigned int num_prealloc,
1088d9ba131d8f58c0d2ff5029e7002ab43f913b36f9Trond Myklebust		unsigned int max_alloc)
1089bd1722d4316e42a12fe6337ebe34d7e1e2c088b2Pavel Emelyanov{
1090bd1722d4316e42a12fe6337ebe34d7e1e2c088b2Pavel Emelyanov	struct rpc_xprt *xprt;
109121de0a955f3af29fa1100d96f66e6adade89e77aTrond Myklebust	struct rpc_rqst *req;
109221de0a955f3af29fa1100d96f66e6adade89e77aTrond Myklebust	int i;
1093bd1722d4316e42a12fe6337ebe34d7e1e2c088b2Pavel Emelyanov
1094bd1722d4316e42a12fe6337ebe34d7e1e2c088b2Pavel Emelyanov	xprt = kzalloc(size, GFP_KERNEL);
1095bd1722d4316e42a12fe6337ebe34d7e1e2c088b2Pavel Emelyanov	if (xprt == NULL)
1096bd1722d4316e42a12fe6337ebe34d7e1e2c088b2Pavel Emelyanov		goto out;
1097bd1722d4316e42a12fe6337ebe34d7e1e2c088b2Pavel Emelyanov
109821de0a955f3af29fa1100d96f66e6adade89e77aTrond Myklebust	xprt_init(xprt, net);
109921de0a955f3af29fa1100d96f66e6adade89e77aTrond Myklebust
110021de0a955f3af29fa1100d96f66e6adade89e77aTrond Myklebust	for (i = 0; i < num_prealloc; i++) {
110121de0a955f3af29fa1100d96f66e6adade89e77aTrond Myklebust		req = kzalloc(sizeof(struct rpc_rqst), GFP_KERNEL);
110221de0a955f3af29fa1100d96f66e6adade89e77aTrond Myklebust		if (!req)
11038313164c36473193c8034de643dc32f35a22bf59wangweidong			goto out_free;
110421de0a955f3af29fa1100d96f66e6adade89e77aTrond Myklebust		list_add(&req->rq_list, &xprt->free);
110521de0a955f3af29fa1100d96f66e6adade89e77aTrond Myklebust	}
1106d9ba131d8f58c0d2ff5029e7002ab43f913b36f9Trond Myklebust	if (max_alloc > num_prealloc)
1107d9ba131d8f58c0d2ff5029e7002ab43f913b36f9Trond Myklebust		xprt->max_reqs = max_alloc;
1108d9ba131d8f58c0d2ff5029e7002ab43f913b36f9Trond Myklebust	else
1109d9ba131d8f58c0d2ff5029e7002ab43f913b36f9Trond Myklebust		xprt->max_reqs = num_prealloc;
1110d9ba131d8f58c0d2ff5029e7002ab43f913b36f9Trond Myklebust	xprt->min_reqs = num_prealloc;
1111d9ba131d8f58c0d2ff5029e7002ab43f913b36f9Trond Myklebust	atomic_set(&xprt->num_reqs, num_prealloc);
1112bd1722d4316e42a12fe6337ebe34d7e1e2c088b2Pavel Emelyanov
1113bd1722d4316e42a12fe6337ebe34d7e1e2c088b2Pavel Emelyanov	return xprt;
1114bd1722d4316e42a12fe6337ebe34d7e1e2c088b2Pavel Emelyanov
1115bd1722d4316e42a12fe6337ebe34d7e1e2c088b2Pavel Emelyanovout_free:
111621de0a955f3af29fa1100d96f66e6adade89e77aTrond Myklebust	xprt_free(xprt);
1117bd1722d4316e42a12fe6337ebe34d7e1e2c088b2Pavel Emelyanovout:
1118bd1722d4316e42a12fe6337ebe34d7e1e2c088b2Pavel Emelyanov	return NULL;
1119bd1722d4316e42a12fe6337ebe34d7e1e2c088b2Pavel Emelyanov}
1120bd1722d4316e42a12fe6337ebe34d7e1e2c088b2Pavel EmelyanovEXPORT_SYMBOL_GPL(xprt_alloc);
1121bd1722d4316e42a12fe6337ebe34d7e1e2c088b2Pavel Emelyanov
1122e204e621b4160c802315bc2d0fa335337c0d62e8Pavel Emelyanovvoid xprt_free(struct rpc_xprt *xprt)
1123e204e621b4160c802315bc2d0fa335337c0d62e8Pavel Emelyanov{
112437aa2133731d9231eb834f700119f0d3f1ed2664Pavel Emelyanov	put_net(xprt->xprt_net);
112521de0a955f3af29fa1100d96f66e6adade89e77aTrond Myklebust	xprt_free_all_slots(xprt);
1126e204e621b4160c802315bc2d0fa335337c0d62e8Pavel Emelyanov	kfree(xprt);
1127e204e621b4160c802315bc2d0fa335337c0d62e8Pavel Emelyanov}
1128e204e621b4160c802315bc2d0fa335337c0d62e8Pavel EmelyanovEXPORT_SYMBOL_GPL(xprt_free);
1129e204e621b4160c802315bc2d0fa335337c0d62e8Pavel Emelyanov
11309903cd1c27a1f30e8efea75e125be3b2002f7cb9Chuck Lever/**
11319903cd1c27a1f30e8efea75e125be3b2002f7cb9Chuck Lever * xprt_reserve - allocate an RPC request slot
11329903cd1c27a1f30e8efea75e125be3b2002f7cb9Chuck Lever * @task: RPC task requesting a slot allocation
11339903cd1c27a1f30e8efea75e125be3b2002f7cb9Chuck Lever *
1134ba60eb25ff6be6f8e60488cdfd454e5c612bce60Trond Myklebust * If the transport is marked as being congested, or if no more
1135ba60eb25ff6be6f8e60488cdfd454e5c612bce60Trond Myklebust * slots are available, place the task on the transport's
11369903cd1c27a1f30e8efea75e125be3b2002f7cb9Chuck Lever * backlog queue.
11379903cd1c27a1f30e8efea75e125be3b2002f7cb9Chuck Lever */
11389903cd1c27a1f30e8efea75e125be3b2002f7cb9Chuck Levervoid xprt_reserve(struct rpc_task *task)
11391da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds{
114045bc0dce9879505d6fd9ff68dcd0359fb260dfd7Trond Myklebust	struct rpc_xprt	*xprt;
11411da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds
114243cedbf0e8dfb9c5610eb7985d5f21263e313802Trond Myklebust	task->tk_status = 0;
114343cedbf0e8dfb9c5610eb7985d5f21263e313802Trond Myklebust	if (task->tk_rqstp != NULL)
114443cedbf0e8dfb9c5610eb7985d5f21263e313802Trond Myklebust		return;
114543cedbf0e8dfb9c5610eb7985d5f21263e313802Trond Myklebust
114643cedbf0e8dfb9c5610eb7985d5f21263e313802Trond Myklebust	task->tk_timeout = 0;
114743cedbf0e8dfb9c5610eb7985d5f21263e313802Trond Myklebust	task->tk_status = -EAGAIN;
114845bc0dce9879505d6fd9ff68dcd0359fb260dfd7Trond Myklebust	rcu_read_lock();
114945bc0dce9879505d6fd9ff68dcd0359fb260dfd7Trond Myklebust	xprt = rcu_dereference(task->tk_client->cl_xprt);
1150ba60eb25ff6be6f8e60488cdfd454e5c612bce60Trond Myklebust	if (!xprt_throttle_congested(xprt, task))
1151ba60eb25ff6be6f8e60488cdfd454e5c612bce60Trond Myklebust		xprt->ops->alloc_slot(xprt, task);
1152ba60eb25ff6be6f8e60488cdfd454e5c612bce60Trond Myklebust	rcu_read_unlock();
1153ba60eb25ff6be6f8e60488cdfd454e5c612bce60Trond Myklebust}
1154ba60eb25ff6be6f8e60488cdfd454e5c612bce60Trond Myklebust
1155ba60eb25ff6be6f8e60488cdfd454e5c612bce60Trond Myklebust/**
1156ba60eb25ff6be6f8e60488cdfd454e5c612bce60Trond Myklebust * xprt_retry_reserve - allocate an RPC request slot
1157ba60eb25ff6be6f8e60488cdfd454e5c612bce60Trond Myklebust * @task: RPC task requesting a slot allocation
1158ba60eb25ff6be6f8e60488cdfd454e5c612bce60Trond Myklebust *
1159ba60eb25ff6be6f8e60488cdfd454e5c612bce60Trond Myklebust * If no more slots are available, place the task on the transport's
1160ba60eb25ff6be6f8e60488cdfd454e5c612bce60Trond Myklebust * backlog queue.
1161ba60eb25ff6be6f8e60488cdfd454e5c612bce60Trond Myklebust * Note that the only difference with xprt_reserve is that we now
1162ba60eb25ff6be6f8e60488cdfd454e5c612bce60Trond Myklebust * ignore the value of the XPRT_CONGESTED flag.
1163ba60eb25ff6be6f8e60488cdfd454e5c612bce60Trond Myklebust */
1164ba60eb25ff6be6f8e60488cdfd454e5c612bce60Trond Myklebustvoid xprt_retry_reserve(struct rpc_task *task)
1165ba60eb25ff6be6f8e60488cdfd454e5c612bce60Trond Myklebust{
1166ba60eb25ff6be6f8e60488cdfd454e5c612bce60Trond Myklebust	struct rpc_xprt	*xprt;
1167ba60eb25ff6be6f8e60488cdfd454e5c612bce60Trond Myklebust
1168ba60eb25ff6be6f8e60488cdfd454e5c612bce60Trond Myklebust	task->tk_status = 0;
1169ba60eb25ff6be6f8e60488cdfd454e5c612bce60Trond Myklebust	if (task->tk_rqstp != NULL)
1170ba60eb25ff6be6f8e60488cdfd454e5c612bce60Trond Myklebust		return;
1171ba60eb25ff6be6f8e60488cdfd454e5c612bce60Trond Myklebust
1172ba60eb25ff6be6f8e60488cdfd454e5c612bce60Trond Myklebust	task->tk_timeout = 0;
1173ba60eb25ff6be6f8e60488cdfd454e5c612bce60Trond Myklebust	task->tk_status = -EAGAIN;
1174ba60eb25ff6be6f8e60488cdfd454e5c612bce60Trond Myklebust	rcu_read_lock();
1175ba60eb25ff6be6f8e60488cdfd454e5c612bce60Trond Myklebust	xprt = rcu_dereference(task->tk_client->cl_xprt);
1176f39c1bfb5a03e2d255451bff05be0d7255298fa4Trond Myklebust	xprt->ops->alloc_slot(xprt, task);
117745bc0dce9879505d6fd9ff68dcd0359fb260dfd7Trond Myklebust	rcu_read_unlock();
11781da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds}
11791da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds
1180d8ed029d6000ba2e2908d9286409e4833c091b4cAlexey Dobriyanstatic inline __be32 xprt_alloc_xid(struct rpc_xprt *xprt)
11811da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds{
11820eae88f31ca2b88911ce843452054139e028771fEric Dumazet	return (__force __be32)xprt->xid++;
11831da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds}
11841da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds
11851da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvaldsstatic inline void xprt_init_xid(struct rpc_xprt *xprt)
11861da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds{
118763862b5bef7349dd1137e4c70702c67d77565785Aruna-Hewapathirane	xprt->xid = prandom_u32();
11881da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds}
11891da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds
11909903cd1c27a1f30e8efea75e125be3b2002f7cb9Chuck Leverstatic void xprt_request_init(struct rpc_task *task, struct rpc_xprt *xprt)
11911da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds{
11921da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds	struct rpc_rqst	*req = task->tk_rqstp;
11931da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds
1194d9ba131d8f58c0d2ff5029e7002ab43f913b36f9Trond Myklebust	INIT_LIST_HEAD(&req->rq_list);
1195ba7392bb37cb12781890f45d7ddee1618e33a036Trond Myklebust	req->rq_timeout = task->tk_client->cl_timeout->to_initval;
11961da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds	req->rq_task	= task;
11971da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds	req->rq_xprt    = xprt;
119802107148349f31eee7c0fb06fd7a880df73dbd20Chuck Lever	req->rq_buffer  = NULL;
11991da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds	req->rq_xid     = xprt_alloc_xid(xprt);
12000a6605213040dd2fb479f0d1a9a87a1d7fa70904Trond Myklebust	req->rq_connect_cookie = xprt->connect_cookie - 1;
120192551948174d079b12541437f51cbe3e17d9dd24Trond Myklebust	req->rq_bytes_sent = 0;
120292551948174d079b12541437f51cbe3e17d9dd24Trond Myklebust	req->rq_snd_buf.len = 0;
120392551948174d079b12541437f51cbe3e17d9dd24Trond Myklebust	req->rq_snd_buf.buflen = 0;
120492551948174d079b12541437f51cbe3e17d9dd24Trond Myklebust	req->rq_rcv_buf.len = 0;
120592551948174d079b12541437f51cbe3e17d9dd24Trond Myklebust	req->rq_rcv_buf.buflen = 0;
1206ead5e1c26fdcd969cf40c49cb0589d56879d240dJ. Bruce Fields	req->rq_release_snd_buf = NULL;
1207da45828e2835057045150b318c4fbe9bb91f18ddTrond Myklebust	xprt_reset_majortimeo(req);
120846121cf7d85869bfe9588bac7ccf55aa0bc7f278Chuck Lever	dprintk("RPC: %5u reserved req %p xid %08x\n", task->tk_pid,
12091da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds			req, ntohl(req->rq_xid));
12101da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds}
12111da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds
12129903cd1c27a1f30e8efea75e125be3b2002f7cb9Chuck Lever/**
12139903cd1c27a1f30e8efea75e125be3b2002f7cb9Chuck Lever * xprt_release - release an RPC request slot
12149903cd1c27a1f30e8efea75e125be3b2002f7cb9Chuck Lever * @task: task which is finished with the slot
12159903cd1c27a1f30e8efea75e125be3b2002f7cb9Chuck Lever *
12161da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds */
12179903cd1c27a1f30e8efea75e125be3b2002f7cb9Chuck Levervoid xprt_release(struct rpc_task *task)
12181da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds{
121955ae1aabfb108106dd095de2578ceef1c755a8b8Ricardo Labiaga	struct rpc_xprt	*xprt;
122087ed50036b866db2ec2ba16b2a7aec4a2b0b7c39Trond Myklebust	struct rpc_rqst	*req = task->tk_rqstp;
12211da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds
122287ed50036b866db2ec2ba16b2a7aec4a2b0b7c39Trond Myklebust	if (req == NULL) {
122387ed50036b866db2ec2ba16b2a7aec4a2b0b7c39Trond Myklebust		if (task->tk_client) {
122487ed50036b866db2ec2ba16b2a7aec4a2b0b7c39Trond Myklebust			rcu_read_lock();
122587ed50036b866db2ec2ba16b2a7aec4a2b0b7c39Trond Myklebust			xprt = rcu_dereference(task->tk_client->cl_xprt);
122687ed50036b866db2ec2ba16b2a7aec4a2b0b7c39Trond Myklebust			if (xprt->snd_task == task)
122787ed50036b866db2ec2ba16b2a7aec4a2b0b7c39Trond Myklebust				xprt_release_write(xprt, task);
122887ed50036b866db2ec2ba16b2a7aec4a2b0b7c39Trond Myklebust			rcu_read_unlock();
122987ed50036b866db2ec2ba16b2a7aec4a2b0b7c39Trond Myklebust		}
12301da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds		return;
123187ed50036b866db2ec2ba16b2a7aec4a2b0b7c39Trond Myklebust	}
123255ae1aabfb108106dd095de2578ceef1c755a8b8Ricardo Labiaga
123355ae1aabfb108106dd095de2578ceef1c755a8b8Ricardo Labiaga	xprt = req->rq_xprt;
12340a702195234eb77c4097148285cccf7f095de9cfWeston Andros Adamson	if (task->tk_ops->rpc_count_stats != NULL)
12350a702195234eb77c4097148285cccf7f095de9cfWeston Andros Adamson		task->tk_ops->rpc_count_stats(task, task->tk_calldata);
12360a702195234eb77c4097148285cccf7f095de9cfWeston Andros Adamson	else if (task->tk_client)
12370a702195234eb77c4097148285cccf7f095de9cfWeston Andros Adamson		rpc_count_iostats(task, task->tk_client->cl_metrics);
12384a0f8c04f2ece949d54a0c4fd7490259cf23a58aChuck Lever	spin_lock_bh(&xprt->transport_lock);
123949e9a89086b3cae784a4868ca852863e4f4ea3feChuck Lever	xprt->ops->release_xprt(xprt, task);
1240a58dd398f5db4f73d5c581069fd70a4304cc4f0aChuck Lever	if (xprt->ops->release_request)
1241a58dd398f5db4f73d5c581069fd70a4304cc4f0aChuck Lever		xprt->ops->release_request(task);
12421da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds	if (!list_empty(&req->rq_list))
12431da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds		list_del(&req->rq_list);
12441da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds	xprt->last_used = jiffies;
12454cfc7e6019caa3e97d2a81c48c8d575d7b38d751Rahul Iyer	if (list_empty(&xprt->recv) && xprt_has_timer(xprt))
1246a246b0105bbd9a70a698f69baae2042996f2a0e9Chuck Lever		mod_timer(&xprt->timer,
124703bf4b707eee06706c9db343dd5c905b7ee47ed2Chuck Lever				xprt->last_used + xprt->idle_timeout);
12484a0f8c04f2ece949d54a0c4fd7490259cf23a58aChuck Lever	spin_unlock_bh(&xprt->transport_lock);
1249ee5ebe851ed60206f150d3f189416f9c63245b66Trond Myklebust	if (req->rq_buffer)
125055ae1aabfb108106dd095de2578ceef1c755a8b8Ricardo Labiaga		xprt->ops->buf_free(req->rq_buffer);
1251a17c2153d2e271b0cbacae9bed83b0eaa41db7e1Trond Myklebust	if (req->rq_cred != NULL)
1252a17c2153d2e271b0cbacae9bed83b0eaa41db7e1Trond Myklebust		put_rpccred(req->rq_cred);
12531da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds	task->tk_rqstp = NULL;
1254ead5e1c26fdcd969cf40c49cb0589d56879d240dJ. Bruce Fields	if (req->rq_release_snd_buf)
1255ead5e1c26fdcd969cf40c49cb0589d56879d240dJ. Bruce Fields		req->rq_release_snd_buf(req);
125655ae1aabfb108106dd095de2578ceef1c755a8b8Ricardo Labiaga
125746121cf7d85869bfe9588bac7ccf55aa0bc7f278Chuck Lever	dprintk("RPC: %5u release request %p\n", task->tk_pid, req);
1258ee5ebe851ed60206f150d3f189416f9c63245b66Trond Myklebust	if (likely(!bc_prealloc(req)))
1259ee5ebe851ed60206f150d3f189416f9c63245b66Trond Myklebust		xprt_free_slot(xprt, req);
1260ee5ebe851ed60206f150d3f189416f9c63245b66Trond Myklebust	else
1261c9acb42ef1904d15d0fb315061cefbe638f67f3aTrond Myklebust		xprt_free_bc_request(req);
12621da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds}
12631da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds
126421de0a955f3af29fa1100d96f66e6adade89e77aTrond Myklebuststatic void xprt_init(struct rpc_xprt *xprt, struct net *net)
1265c2866763b4029411d166040306691773c12d4cafChuck Lever{
126621de0a955f3af29fa1100d96f66e6adade89e77aTrond Myklebust	atomic_set(&xprt->count, 1);
1267c2866763b4029411d166040306691773c12d4cafChuck Lever
1268c2866763b4029411d166040306691773c12d4cafChuck Lever	spin_lock_init(&xprt->transport_lock);
1269c2866763b4029411d166040306691773c12d4cafChuck Lever	spin_lock_init(&xprt->reserve_lock);
1270c2866763b4029411d166040306691773c12d4cafChuck Lever
1271c2866763b4029411d166040306691773c12d4cafChuck Lever	INIT_LIST_HEAD(&xprt->free);
1272c2866763b4029411d166040306691773c12d4cafChuck Lever	INIT_LIST_HEAD(&xprt->recv);
12739e00abc3c20904fd6a5d888bb7023925799ec8a5Trond Myklebust#if defined(CONFIG_SUNRPC_BACKCHANNEL)
1274f9acac1a4710ce88871f1ae323fc91c1cb6e9d52Ricardo Labiaga	spin_lock_init(&xprt->bc_pa_lock);
1275f9acac1a4710ce88871f1ae323fc91c1cb6e9d52Ricardo Labiaga	INIT_LIST_HEAD(&xprt->bc_pa_list);
12769e00abc3c20904fd6a5d888bb7023925799ec8a5Trond Myklebust#endif /* CONFIG_SUNRPC_BACKCHANNEL */
1277f9acac1a4710ce88871f1ae323fc91c1cb6e9d52Ricardo Labiaga
1278c2866763b4029411d166040306691773c12d4cafChuck Lever	xprt->last_used = jiffies;
1279c2866763b4029411d166040306691773c12d4cafChuck Lever	xprt->cwnd = RPC_INITCWND;
1280a509050bd3b8e0aa269c2241aa10d74ca7701e2fChuck Lever	xprt->bind_index = 0;
1281c2866763b4029411d166040306691773c12d4cafChuck Lever
1282c2866763b4029411d166040306691773c12d4cafChuck Lever	rpc_init_wait_queue(&xprt->binding, "xprt_binding");
1283c2866763b4029411d166040306691773c12d4cafChuck Lever	rpc_init_wait_queue(&xprt->pending, "xprt_pending");
128434006cee28f7344f9557a4be3816c7891b1bbab1Trond Myklebust	rpc_init_priority_wait_queue(&xprt->sending, "xprt_sending");
1285c2866763b4029411d166040306691773c12d4cafChuck Lever	rpc_init_priority_wait_queue(&xprt->backlog, "xprt_backlog");
1286c2866763b4029411d166040306691773c12d4cafChuck Lever
1287c2866763b4029411d166040306691773c12d4cafChuck Lever	xprt_init_xid(xprt);
1288c2866763b4029411d166040306691773c12d4cafChuck Lever
128921de0a955f3af29fa1100d96f66e6adade89e77aTrond Myklebust	xprt->xprt_net = get_net(net);
12908d9266ffe4332afc5ac9de401ef6f825b3798585Trond Myklebust}
12918d9266ffe4332afc5ac9de401ef6f825b3798585Trond Myklebust
12928d9266ffe4332afc5ac9de401ef6f825b3798585Trond Myklebust/**
12938d9266ffe4332afc5ac9de401ef6f825b3798585Trond Myklebust * xprt_create_transport - create an RPC transport
12948d9266ffe4332afc5ac9de401ef6f825b3798585Trond Myklebust * @args: rpc transport creation arguments
12958d9266ffe4332afc5ac9de401ef6f825b3798585Trond Myklebust *
12968d9266ffe4332afc5ac9de401ef6f825b3798585Trond Myklebust */
12978d9266ffe4332afc5ac9de401ef6f825b3798585Trond Myklebuststruct rpc_xprt *xprt_create_transport(struct xprt_create *args)
12988d9266ffe4332afc5ac9de401ef6f825b3798585Trond Myklebust{
12998d9266ffe4332afc5ac9de401ef6f825b3798585Trond Myklebust	struct rpc_xprt	*xprt;
13008d9266ffe4332afc5ac9de401ef6f825b3798585Trond Myklebust	struct xprt_class *t;
13018d9266ffe4332afc5ac9de401ef6f825b3798585Trond Myklebust
13028d9266ffe4332afc5ac9de401ef6f825b3798585Trond Myklebust	spin_lock(&xprt_list_lock);
13038d9266ffe4332afc5ac9de401ef6f825b3798585Trond Myklebust	list_for_each_entry(t, &xprt_list, list) {
13048d9266ffe4332afc5ac9de401ef6f825b3798585Trond Myklebust		if (t->ident == args->ident) {
13058d9266ffe4332afc5ac9de401ef6f825b3798585Trond Myklebust			spin_unlock(&xprt_list_lock);
13068d9266ffe4332afc5ac9de401ef6f825b3798585Trond Myklebust			goto found;
13078d9266ffe4332afc5ac9de401ef6f825b3798585Trond Myklebust		}
13088d9266ffe4332afc5ac9de401ef6f825b3798585Trond Myklebust	}
13098d9266ffe4332afc5ac9de401ef6f825b3798585Trond Myklebust	spin_unlock(&xprt_list_lock);
13103c45ddf823d679a820adddd53b52c6699c9a05acChuck Lever	dprintk("RPC: transport (%d) not supported\n", args->ident);
13118d9266ffe4332afc5ac9de401ef6f825b3798585Trond Myklebust	return ERR_PTR(-EIO);
13128d9266ffe4332afc5ac9de401ef6f825b3798585Trond Myklebust
13138d9266ffe4332afc5ac9de401ef6f825b3798585Trond Myklebustfound:
13148d9266ffe4332afc5ac9de401ef6f825b3798585Trond Myklebust	xprt = t->setup(args);
13158d9266ffe4332afc5ac9de401ef6f825b3798585Trond Myklebust	if (IS_ERR(xprt)) {
13168d9266ffe4332afc5ac9de401ef6f825b3798585Trond Myklebust		dprintk("RPC:       xprt_create_transport: failed, %ld\n",
13178d9266ffe4332afc5ac9de401ef6f825b3798585Trond Myklebust				-PTR_ERR(xprt));
131821de0a955f3af29fa1100d96f66e6adade89e77aTrond Myklebust		goto out;
13198d9266ffe4332afc5ac9de401ef6f825b3798585Trond Myklebust	}
132033d90ac0581ce81d1ebfc51918a2757e41a6011cJ. Bruce Fields	if (args->flags & XPRT_CREATE_NO_IDLE_TIMEOUT)
132133d90ac0581ce81d1ebfc51918a2757e41a6011cJ. Bruce Fields		xprt->idle_timeout = 0;
132221de0a955f3af29fa1100d96f66e6adade89e77aTrond Myklebust	INIT_WORK(&xprt->task_cleanup, xprt_autoclose);
132321de0a955f3af29fa1100d96f66e6adade89e77aTrond Myklebust	if (xprt_has_timer(xprt))
132421de0a955f3af29fa1100d96f66e6adade89e77aTrond Myklebust		setup_timer(&xprt->timer, xprt_init_autodisconnect,
132521de0a955f3af29fa1100d96f66e6adade89e77aTrond Myklebust			    (unsigned long)xprt);
132621de0a955f3af29fa1100d96f66e6adade89e77aTrond Myklebust	else
132721de0a955f3af29fa1100d96f66e6adade89e77aTrond Myklebust		init_timer(&xprt->timer);
13284e0038b6b246e4145fc4a53dca61a556d17bc52cTrond Myklebust
13294e0038b6b246e4145fc4a53dca61a556d17bc52cTrond Myklebust	if (strlen(args->servername) > RPC_MAXNETNAMELEN) {
13304e0038b6b246e4145fc4a53dca61a556d17bc52cTrond Myklebust		xprt_destroy(xprt);
13314e0038b6b246e4145fc4a53dca61a556d17bc52cTrond Myklebust		return ERR_PTR(-EINVAL);
13324e0038b6b246e4145fc4a53dca61a556d17bc52cTrond Myklebust	}
13334e0038b6b246e4145fc4a53dca61a556d17bc52cTrond Myklebust	xprt->servername = kstrdup(args->servername, GFP_KERNEL);
13344e0038b6b246e4145fc4a53dca61a556d17bc52cTrond Myklebust	if (xprt->servername == NULL) {
13354e0038b6b246e4145fc4a53dca61a556d17bc52cTrond Myklebust		xprt_destroy(xprt);
13364e0038b6b246e4145fc4a53dca61a556d17bc52cTrond Myklebust		return ERR_PTR(-ENOMEM);
13374e0038b6b246e4145fc4a53dca61a556d17bc52cTrond Myklebust	}
13384e0038b6b246e4145fc4a53dca61a556d17bc52cTrond Myklebust
133946121cf7d85869bfe9588bac7ccf55aa0bc7f278Chuck Lever	dprintk("RPC:       created transport %p with %u slots\n", xprt,
1340c2866763b4029411d166040306691773c12d4cafChuck Lever			xprt->max_reqs);
134121de0a955f3af29fa1100d96f66e6adade89e77aTrond Myklebustout:
1342c2866763b4029411d166040306691773c12d4cafChuck Lever	return xprt;
1343c2866763b4029411d166040306691773c12d4cafChuck Lever}
1344c2866763b4029411d166040306691773c12d4cafChuck Lever
13459903cd1c27a1f30e8efea75e125be3b2002f7cb9Chuck Lever/**
13469903cd1c27a1f30e8efea75e125be3b2002f7cb9Chuck Lever * xprt_destroy - destroy an RPC transport, killing off all requests.
1347a8de240a9074b72b156d9e6d53f00076e6cd5f03Trond Myklebust * @xprt: transport to destroy
13489903cd1c27a1f30e8efea75e125be3b2002f7cb9Chuck Lever *
13491da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds */
1350a8de240a9074b72b156d9e6d53f00076e6cd5f03Trond Myklebuststatic void xprt_destroy(struct rpc_xprt *xprt)
13511da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds{
135246121cf7d85869bfe9588bac7ccf55aa0bc7f278Chuck Lever	dprintk("RPC:       destroying transport %p\n", xprt);
13530065db328533c390fbfb0fe0c46bcf9a278fb99eTrond Myklebust	del_timer_sync(&xprt->timer);
1354c8541ecdd5692bcfbcb5305cab9a873288d29175Chuck Lever
1355f6a1cc89309f0ae847a9b6fe418d1c4215e5bc55Trond Myklebust	rpc_destroy_wait_queue(&xprt->binding);
1356f6a1cc89309f0ae847a9b6fe418d1c4215e5bc55Trond Myklebust	rpc_destroy_wait_queue(&xprt->pending);
1357f6a1cc89309f0ae847a9b6fe418d1c4215e5bc55Trond Myklebust	rpc_destroy_wait_queue(&xprt->sending);
1358f6a1cc89309f0ae847a9b6fe418d1c4215e5bc55Trond Myklebust	rpc_destroy_wait_queue(&xprt->backlog);
1359c3ae62ae08bb0db3639d8c579e4ff0967d908199J. Bruce Fields	cancel_work_sync(&xprt->task_cleanup);
13604e0038b6b246e4145fc4a53dca61a556d17bc52cTrond Myklebust	kfree(xprt->servername);
1361c8541ecdd5692bcfbcb5305cab9a873288d29175Chuck Lever	/*
1362c8541ecdd5692bcfbcb5305cab9a873288d29175Chuck Lever	 * Tear down transport state and free the rpc_xprt
1363c8541ecdd5692bcfbcb5305cab9a873288d29175Chuck Lever	 */
1364a246b0105bbd9a70a698f69baae2042996f2a0e9Chuck Lever	xprt->ops->destroy(xprt);
13656b6ca86b77b62b798cf9ca2599036420abce7796Trond Myklebust}
13661da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds
13676b6ca86b77b62b798cf9ca2599036420abce7796Trond Myklebust/**
13686b6ca86b77b62b798cf9ca2599036420abce7796Trond Myklebust * xprt_put - release a reference to an RPC transport.
13696b6ca86b77b62b798cf9ca2599036420abce7796Trond Myklebust * @xprt: pointer to the transport
13706b6ca86b77b62b798cf9ca2599036420abce7796Trond Myklebust *
13716b6ca86b77b62b798cf9ca2599036420abce7796Trond Myklebust */
13726b6ca86b77b62b798cf9ca2599036420abce7796Trond Myklebustvoid xprt_put(struct rpc_xprt *xprt)
13736b6ca86b77b62b798cf9ca2599036420abce7796Trond Myklebust{
1374a8de240a9074b72b156d9e6d53f00076e6cd5f03Trond Myklebust	if (atomic_dec_and_test(&xprt->count))
1375a8de240a9074b72b156d9e6d53f00076e6cd5f03Trond Myklebust		xprt_destroy(xprt);
13766b6ca86b77b62b798cf9ca2599036420abce7796Trond Myklebust}
1377