1/* RxRPC packet transmission
2 *
3 * Copyright (C) 2007 Red Hat, Inc. All Rights Reserved.
4 * Written by David Howells (dhowells@redhat.com)
5 *
6 * This program is free software; you can redistribute it and/or
7 * modify it under the terms of the GNU General Public License
8 * as published by the Free Software Foundation; either version
9 * 2 of the License, or (at your option) any later version.
10 */
11
12#include <linux/net.h>
13#include <linux/gfp.h>
14#include <linux/skbuff.h>
15#include <linux/circ_buf.h>
16#include <linux/export.h>
17#include <net/sock.h>
18#include <net/af_rxrpc.h>
19#include "ar-internal.h"
20
21/*
22 * Time till packet resend (in jiffies).
23 */
24unsigned rxrpc_resend_timeout = 4 * HZ;
25
26static int rxrpc_send_data(struct kiocb *iocb,
27			   struct rxrpc_sock *rx,
28			   struct rxrpc_call *call,
29			   struct msghdr *msg, size_t len);
30
31/*
32 * extract control messages from the sendmsg() control buffer
33 */
34static int rxrpc_sendmsg_cmsg(struct rxrpc_sock *rx, struct msghdr *msg,
35			      unsigned long *user_call_ID,
36			      enum rxrpc_command *command,
37			      u32 *abort_code,
38			      bool server)
39{
40	struct cmsghdr *cmsg;
41	int len;
42
43	*command = RXRPC_CMD_SEND_DATA;
44
45	if (msg->msg_controllen == 0)
46		return -EINVAL;
47
48	for (cmsg = CMSG_FIRSTHDR(msg); cmsg; cmsg = CMSG_NXTHDR(msg, cmsg)) {
49		if (!CMSG_OK(msg, cmsg))
50			return -EINVAL;
51
52		len = cmsg->cmsg_len - CMSG_ALIGN(sizeof(struct cmsghdr));
53		_debug("CMSG %d, %d, %d",
54		       cmsg->cmsg_level, cmsg->cmsg_type, len);
55
56		if (cmsg->cmsg_level != SOL_RXRPC)
57			continue;
58
59		switch (cmsg->cmsg_type) {
60		case RXRPC_USER_CALL_ID:
61			if (msg->msg_flags & MSG_CMSG_COMPAT) {
62				if (len != sizeof(u32))
63					return -EINVAL;
64				*user_call_ID = *(u32 *) CMSG_DATA(cmsg);
65			} else {
66				if (len != sizeof(unsigned long))
67					return -EINVAL;
68				*user_call_ID = *(unsigned long *)
69					CMSG_DATA(cmsg);
70			}
71			_debug("User Call ID %lx", *user_call_ID);
72			break;
73
74		case RXRPC_ABORT:
75			if (*command != RXRPC_CMD_SEND_DATA)
76				return -EINVAL;
77			*command = RXRPC_CMD_SEND_ABORT;
78			if (len != sizeof(*abort_code))
79				return -EINVAL;
80			*abort_code = *(unsigned int *) CMSG_DATA(cmsg);
81			_debug("Abort %x", *abort_code);
82			if (*abort_code == 0)
83				return -EINVAL;
84			break;
85
86		case RXRPC_ACCEPT:
87			if (*command != RXRPC_CMD_SEND_DATA)
88				return -EINVAL;
89			*command = RXRPC_CMD_ACCEPT;
90			if (len != 0)
91				return -EINVAL;
92			if (!server)
93				return -EISCONN;
94			break;
95
96		default:
97			return -EINVAL;
98		}
99	}
100
101	_leave(" = 0");
102	return 0;
103}
104
105/*
106 * abort a call, sending an ABORT packet to the peer
107 */
108static void rxrpc_send_abort(struct rxrpc_call *call, u32 abort_code)
109{
110	write_lock_bh(&call->state_lock);
111
112	if (call->state <= RXRPC_CALL_COMPLETE) {
113		call->state = RXRPC_CALL_LOCALLY_ABORTED;
114		call->abort_code = abort_code;
115		set_bit(RXRPC_CALL_ABORT, &call->events);
116		del_timer_sync(&call->resend_timer);
117		del_timer_sync(&call->ack_timer);
118		clear_bit(RXRPC_CALL_RESEND_TIMER, &call->events);
119		clear_bit(RXRPC_CALL_ACK, &call->events);
120		clear_bit(RXRPC_CALL_RUN_RTIMER, &call->flags);
121		rxrpc_queue_call(call);
122	}
123
124	write_unlock_bh(&call->state_lock);
125}
126
127/*
128 * send a message forming part of a client call through an RxRPC socket
129 * - caller holds the socket locked
130 * - the socket may be either a client socket or a server socket
131 */
132int rxrpc_client_sendmsg(struct kiocb *iocb, struct rxrpc_sock *rx,
133			 struct rxrpc_transport *trans, struct msghdr *msg,
134			 size_t len)
135{
136	struct rxrpc_conn_bundle *bundle;
137	enum rxrpc_command cmd;
138	struct rxrpc_call *call;
139	unsigned long user_call_ID = 0;
140	struct key *key;
141	__be16 service_id;
142	u32 abort_code = 0;
143	int ret;
144
145	_enter("");
146
147	ASSERT(trans != NULL);
148
149	ret = rxrpc_sendmsg_cmsg(rx, msg, &user_call_ID, &cmd, &abort_code,
150				 false);
151	if (ret < 0)
152		return ret;
153
154	bundle = NULL;
155	if (trans) {
156		service_id = rx->service_id;
157		if (msg->msg_name) {
158			DECLARE_SOCKADDR(struct sockaddr_rxrpc *, srx,
159					 msg->msg_name);
160			service_id = htons(srx->srx_service);
161		}
162		key = rx->key;
163		if (key && !rx->key->payload.data)
164			key = NULL;
165		bundle = rxrpc_get_bundle(rx, trans, key, service_id,
166					  GFP_KERNEL);
167		if (IS_ERR(bundle))
168			return PTR_ERR(bundle);
169	}
170
171	call = rxrpc_get_client_call(rx, trans, bundle, user_call_ID,
172				     abort_code == 0, GFP_KERNEL);
173	if (trans)
174		rxrpc_put_bundle(trans, bundle);
175	if (IS_ERR(call)) {
176		_leave(" = %ld", PTR_ERR(call));
177		return PTR_ERR(call);
178	}
179
180	_debug("CALL %d USR %lx ST %d on CONN %p",
181	       call->debug_id, call->user_call_ID, call->state, call->conn);
182
183	if (call->state >= RXRPC_CALL_COMPLETE) {
184		/* it's too late for this call */
185		ret = -ESHUTDOWN;
186	} else if (cmd == RXRPC_CMD_SEND_ABORT) {
187		rxrpc_send_abort(call, abort_code);
188	} else if (cmd != RXRPC_CMD_SEND_DATA) {
189		ret = -EINVAL;
190	} else if (call->state != RXRPC_CALL_CLIENT_SEND_REQUEST) {
191		/* request phase complete for this client call */
192		ret = -EPROTO;
193	} else {
194		ret = rxrpc_send_data(iocb, rx, call, msg, len);
195	}
196
197	rxrpc_put_call(call);
198	_leave(" = %d", ret);
199	return ret;
200}
201
202/**
203 * rxrpc_kernel_send_data - Allow a kernel service to send data on a call
204 * @call: The call to send data through
205 * @msg: The data to send
206 * @len: The amount of data to send
207 *
208 * Allow a kernel service to send data on a call.  The call must be in an state
209 * appropriate to sending data.  No control data should be supplied in @msg,
210 * nor should an address be supplied.  MSG_MORE should be flagged if there's
211 * more data to come, otherwise this data will end the transmission phase.
212 */
213int rxrpc_kernel_send_data(struct rxrpc_call *call, struct msghdr *msg,
214			   size_t len)
215{
216	int ret;
217
218	_enter("{%d,%s},", call->debug_id, rxrpc_call_states[call->state]);
219
220	ASSERTCMP(msg->msg_name, ==, NULL);
221	ASSERTCMP(msg->msg_control, ==, NULL);
222
223	lock_sock(&call->socket->sk);
224
225	_debug("CALL %d USR %lx ST %d on CONN %p",
226	       call->debug_id, call->user_call_ID, call->state, call->conn);
227
228	if (call->state >= RXRPC_CALL_COMPLETE) {
229		ret = -ESHUTDOWN; /* it's too late for this call */
230	} else if (call->state != RXRPC_CALL_CLIENT_SEND_REQUEST &&
231		   call->state != RXRPC_CALL_SERVER_ACK_REQUEST &&
232		   call->state != RXRPC_CALL_SERVER_SEND_REPLY) {
233		ret = -EPROTO; /* request phase complete for this client call */
234	} else {
235		mm_segment_t oldfs = get_fs();
236		set_fs(KERNEL_DS);
237		ret = rxrpc_send_data(NULL, call->socket, call, msg, len);
238		set_fs(oldfs);
239	}
240
241	release_sock(&call->socket->sk);
242	_leave(" = %d", ret);
243	return ret;
244}
245
246EXPORT_SYMBOL(rxrpc_kernel_send_data);
247
248/**
249 * rxrpc_kernel_abort_call - Allow a kernel service to abort a call
250 * @call: The call to be aborted
251 * @abort_code: The abort code to stick into the ABORT packet
252 *
253 * Allow a kernel service to abort a call, if it's still in an abortable state.
254 */
255void rxrpc_kernel_abort_call(struct rxrpc_call *call, u32 abort_code)
256{
257	_enter("{%d},%d", call->debug_id, abort_code);
258
259	lock_sock(&call->socket->sk);
260
261	_debug("CALL %d USR %lx ST %d on CONN %p",
262	       call->debug_id, call->user_call_ID, call->state, call->conn);
263
264	if (call->state < RXRPC_CALL_COMPLETE)
265		rxrpc_send_abort(call, abort_code);
266
267	release_sock(&call->socket->sk);
268	_leave("");
269}
270
271EXPORT_SYMBOL(rxrpc_kernel_abort_call);
272
273/*
274 * send a message through a server socket
275 * - caller holds the socket locked
276 */
277int rxrpc_server_sendmsg(struct kiocb *iocb, struct rxrpc_sock *rx,
278			 struct msghdr *msg, size_t len)
279{
280	enum rxrpc_command cmd;
281	struct rxrpc_call *call;
282	unsigned long user_call_ID = 0;
283	u32 abort_code = 0;
284	int ret;
285
286	_enter("");
287
288	ret = rxrpc_sendmsg_cmsg(rx, msg, &user_call_ID, &cmd, &abort_code,
289				 true);
290	if (ret < 0)
291		return ret;
292
293	if (cmd == RXRPC_CMD_ACCEPT) {
294		call = rxrpc_accept_call(rx, user_call_ID);
295		if (IS_ERR(call))
296			return PTR_ERR(call);
297		rxrpc_put_call(call);
298		return 0;
299	}
300
301	call = rxrpc_find_server_call(rx, user_call_ID);
302	if (!call)
303		return -EBADSLT;
304	if (call->state >= RXRPC_CALL_COMPLETE) {
305		ret = -ESHUTDOWN;
306		goto out;
307	}
308
309	switch (cmd) {
310	case RXRPC_CMD_SEND_DATA:
311		if (call->state != RXRPC_CALL_CLIENT_SEND_REQUEST &&
312		    call->state != RXRPC_CALL_SERVER_ACK_REQUEST &&
313		    call->state != RXRPC_CALL_SERVER_SEND_REPLY) {
314			/* Tx phase not yet begun for this call */
315			ret = -EPROTO;
316			break;
317		}
318
319		ret = rxrpc_send_data(iocb, rx, call, msg, len);
320		break;
321
322	case RXRPC_CMD_SEND_ABORT:
323		rxrpc_send_abort(call, abort_code);
324		break;
325	default:
326		BUG();
327	}
328
329	out:
330	rxrpc_put_call(call);
331	_leave(" = %d", ret);
332	return ret;
333}
334
335/*
336 * send a packet through the transport endpoint
337 */
338int rxrpc_send_packet(struct rxrpc_transport *trans, struct sk_buff *skb)
339{
340	struct kvec iov[1];
341	struct msghdr msg;
342	int ret, opt;
343
344	_enter(",{%d}", skb->len);
345
346	iov[0].iov_base = skb->head;
347	iov[0].iov_len = skb->len;
348
349	msg.msg_name = &trans->peer->srx.transport.sin;
350	msg.msg_namelen = sizeof(trans->peer->srx.transport.sin);
351	msg.msg_control = NULL;
352	msg.msg_controllen = 0;
353	msg.msg_flags = 0;
354
355	/* send the packet with the don't fragment bit set if we currently
356	 * think it's small enough */
357	if (skb->len - sizeof(struct rxrpc_header) < trans->peer->maxdata) {
358		down_read(&trans->local->defrag_sem);
359		/* send the packet by UDP
360		 * - returns -EMSGSIZE if UDP would have to fragment the packet
361		 *   to go out of the interface
362		 *   - in which case, we'll have processed the ICMP error
363		 *     message and update the peer record
364		 */
365		ret = kernel_sendmsg(trans->local->socket, &msg, iov, 1,
366				     iov[0].iov_len);
367
368		up_read(&trans->local->defrag_sem);
369		if (ret == -EMSGSIZE)
370			goto send_fragmentable;
371
372		_leave(" = %d [%u]", ret, trans->peer->maxdata);
373		return ret;
374	}
375
376send_fragmentable:
377	/* attempt to send this message with fragmentation enabled */
378	_debug("send fragment");
379
380	down_write(&trans->local->defrag_sem);
381	opt = IP_PMTUDISC_DONT;
382	ret = kernel_setsockopt(trans->local->socket, SOL_IP, IP_MTU_DISCOVER,
383				(char *) &opt, sizeof(opt));
384	if (ret == 0) {
385		ret = kernel_sendmsg(trans->local->socket, &msg, iov, 1,
386				     iov[0].iov_len);
387
388		opt = IP_PMTUDISC_DO;
389		kernel_setsockopt(trans->local->socket, SOL_IP,
390				  IP_MTU_DISCOVER, (char *) &opt, sizeof(opt));
391	}
392
393	up_write(&trans->local->defrag_sem);
394	_leave(" = %d [frag %u]", ret, trans->peer->maxdata);
395	return ret;
396}
397
398/*
399 * wait for space to appear in the transmit/ACK window
400 * - caller holds the socket locked
401 */
402static int rxrpc_wait_for_tx_window(struct rxrpc_sock *rx,
403				    struct rxrpc_call *call,
404				    long *timeo)
405{
406	DECLARE_WAITQUEUE(myself, current);
407	int ret;
408
409	_enter(",{%d},%ld",
410	       CIRC_SPACE(call->acks_head, call->acks_tail, call->acks_winsz),
411	       *timeo);
412
413	add_wait_queue(&call->tx_waitq, &myself);
414
415	for (;;) {
416		set_current_state(TASK_INTERRUPTIBLE);
417		ret = 0;
418		if (CIRC_SPACE(call->acks_head, call->acks_tail,
419			       call->acks_winsz) > 0)
420			break;
421		if (signal_pending(current)) {
422			ret = sock_intr_errno(*timeo);
423			break;
424		}
425
426		release_sock(&rx->sk);
427		*timeo = schedule_timeout(*timeo);
428		lock_sock(&rx->sk);
429	}
430
431	remove_wait_queue(&call->tx_waitq, &myself);
432	set_current_state(TASK_RUNNING);
433	_leave(" = %d", ret);
434	return ret;
435}
436
437/*
438 * attempt to schedule an instant Tx resend
439 */
440static inline void rxrpc_instant_resend(struct rxrpc_call *call)
441{
442	read_lock_bh(&call->state_lock);
443	if (try_to_del_timer_sync(&call->resend_timer) >= 0) {
444		clear_bit(RXRPC_CALL_RUN_RTIMER, &call->flags);
445		if (call->state < RXRPC_CALL_COMPLETE &&
446		    !test_and_set_bit(RXRPC_CALL_RESEND_TIMER, &call->events))
447			rxrpc_queue_call(call);
448	}
449	read_unlock_bh(&call->state_lock);
450}
451
452/*
453 * queue a packet for transmission, set the resend timer and attempt
454 * to send the packet immediately
455 */
456static void rxrpc_queue_packet(struct rxrpc_call *call, struct sk_buff *skb,
457			       bool last)
458{
459	struct rxrpc_skb_priv *sp = rxrpc_skb(skb);
460	int ret;
461
462	_net("queue skb %p [%d]", skb, call->acks_head);
463
464	ASSERT(call->acks_window != NULL);
465	call->acks_window[call->acks_head] = (unsigned long) skb;
466	smp_wmb();
467	call->acks_head = (call->acks_head + 1) & (call->acks_winsz - 1);
468
469	if (last || call->state == RXRPC_CALL_SERVER_ACK_REQUEST) {
470		_debug("________awaiting reply/ACK__________");
471		write_lock_bh(&call->state_lock);
472		switch (call->state) {
473		case RXRPC_CALL_CLIENT_SEND_REQUEST:
474			call->state = RXRPC_CALL_CLIENT_AWAIT_REPLY;
475			break;
476		case RXRPC_CALL_SERVER_ACK_REQUEST:
477			call->state = RXRPC_CALL_SERVER_SEND_REPLY;
478			if (!last)
479				break;
480		case RXRPC_CALL_SERVER_SEND_REPLY:
481			call->state = RXRPC_CALL_SERVER_AWAIT_ACK;
482			break;
483		default:
484			break;
485		}
486		write_unlock_bh(&call->state_lock);
487	}
488
489	_proto("Tx DATA %%%u { #%u }",
490	       ntohl(sp->hdr.serial), ntohl(sp->hdr.seq));
491
492	sp->need_resend = false;
493	sp->resend_at = jiffies + rxrpc_resend_timeout;
494	if (!test_and_set_bit(RXRPC_CALL_RUN_RTIMER, &call->flags)) {
495		_debug("run timer");
496		call->resend_timer.expires = sp->resend_at;
497		add_timer(&call->resend_timer);
498	}
499
500	/* attempt to cancel the rx-ACK timer, deferring reply transmission if
501	 * we're ACK'ing the request phase of an incoming call */
502	ret = -EAGAIN;
503	if (try_to_del_timer_sync(&call->ack_timer) >= 0) {
504		/* the packet may be freed by rxrpc_process_call() before this
505		 * returns */
506		ret = rxrpc_send_packet(call->conn->trans, skb);
507		_net("sent skb %p", skb);
508	} else {
509		_debug("failed to delete ACK timer");
510	}
511
512	if (ret < 0) {
513		_debug("need instant resend %d", ret);
514		sp->need_resend = true;
515		rxrpc_instant_resend(call);
516	}
517
518	_leave("");
519}
520
521/*
522 * send data through a socket
523 * - must be called in process context
524 * - caller holds the socket locked
525 */
526static int rxrpc_send_data(struct kiocb *iocb,
527			   struct rxrpc_sock *rx,
528			   struct rxrpc_call *call,
529			   struct msghdr *msg, size_t len)
530{
531	struct rxrpc_skb_priv *sp;
532	unsigned char __user *from;
533	struct sk_buff *skb;
534	struct iovec *iov;
535	struct sock *sk = &rx->sk;
536	long timeo;
537	bool more;
538	int ret, ioc, segment, copied;
539
540	_enter(",,,{%zu},%zu", msg->msg_iovlen, len);
541
542	timeo = sock_sndtimeo(sk, msg->msg_flags & MSG_DONTWAIT);
543
544	/* this should be in poll */
545	clear_bit(SOCK_ASYNC_NOSPACE, &sk->sk_socket->flags);
546
547	if (sk->sk_err || (sk->sk_shutdown & SEND_SHUTDOWN))
548		return -EPIPE;
549
550	iov = msg->msg_iov;
551	ioc = msg->msg_iovlen - 1;
552	from = iov->iov_base;
553	segment = iov->iov_len;
554	iov++;
555	more = msg->msg_flags & MSG_MORE;
556
557	skb = call->tx_pending;
558	call->tx_pending = NULL;
559
560	copied = 0;
561	do {
562		int copy;
563
564		if (segment > len)
565			segment = len;
566
567		_debug("SEGMENT %d @%p", segment, from);
568
569		if (!skb) {
570			size_t size, chunk, max, space;
571
572			_debug("alloc");
573
574			if (CIRC_SPACE(call->acks_head, call->acks_tail,
575				       call->acks_winsz) <= 0) {
576				ret = -EAGAIN;
577				if (msg->msg_flags & MSG_DONTWAIT)
578					goto maybe_error;
579				ret = rxrpc_wait_for_tx_window(rx, call,
580							       &timeo);
581				if (ret < 0)
582					goto maybe_error;
583			}
584
585			max = call->conn->trans->peer->maxdata;
586			max -= call->conn->security_size;
587			max &= ~(call->conn->size_align - 1UL);
588
589			chunk = max;
590			if (chunk > len && !more)
591				chunk = len;
592
593			space = chunk + call->conn->size_align;
594			space &= ~(call->conn->size_align - 1UL);
595
596			size = space + call->conn->header_size;
597
598			_debug("SIZE: %zu/%zu/%zu", chunk, space, size);
599
600			/* create a buffer that we can retain until it's ACK'd */
601			skb = sock_alloc_send_skb(
602				sk, size, msg->msg_flags & MSG_DONTWAIT, &ret);
603			if (!skb)
604				goto maybe_error;
605
606			rxrpc_new_skb(skb);
607
608			_debug("ALLOC SEND %p", skb);
609
610			ASSERTCMP(skb->mark, ==, 0);
611
612			_debug("HS: %u", call->conn->header_size);
613			skb_reserve(skb, call->conn->header_size);
614			skb->len += call->conn->header_size;
615
616			sp = rxrpc_skb(skb);
617			sp->remain = chunk;
618			if (sp->remain > skb_tailroom(skb))
619				sp->remain = skb_tailroom(skb);
620
621			_net("skb: hr %d, tr %d, hl %d, rm %d",
622			       skb_headroom(skb),
623			       skb_tailroom(skb),
624			       skb_headlen(skb),
625			       sp->remain);
626
627			skb->ip_summed = CHECKSUM_UNNECESSARY;
628		}
629
630		_debug("append");
631		sp = rxrpc_skb(skb);
632
633		/* append next segment of data to the current buffer */
634		copy = skb_tailroom(skb);
635		ASSERTCMP(copy, >, 0);
636		if (copy > segment)
637			copy = segment;
638		if (copy > sp->remain)
639			copy = sp->remain;
640
641		_debug("add");
642		ret = skb_add_data(skb, from, copy);
643		_debug("added");
644		if (ret < 0)
645			goto efault;
646		sp->remain -= copy;
647		skb->mark += copy;
648		copied += copy;
649
650		len -= copy;
651		segment -= copy;
652		from += copy;
653		while (segment == 0 && ioc > 0) {
654			from = iov->iov_base;
655			segment = iov->iov_len;
656			iov++;
657			ioc--;
658		}
659		if (len == 0) {
660			segment = 0;
661			ioc = 0;
662		}
663
664		/* check for the far side aborting the call or a network error
665		 * occurring */
666		if (call->state > RXRPC_CALL_COMPLETE)
667			goto call_aborted;
668
669		/* add the packet to the send queue if it's now full */
670		if (sp->remain <= 0 || (segment == 0 && !more)) {
671			struct rxrpc_connection *conn = call->conn;
672			uint32_t seq;
673			size_t pad;
674
675			/* pad out if we're using security */
676			if (conn->security) {
677				pad = conn->security_size + skb->mark;
678				pad = conn->size_align - pad;
679				pad &= conn->size_align - 1;
680				_debug("pad %zu", pad);
681				if (pad)
682					memset(skb_put(skb, pad), 0, pad);
683			}
684
685			seq = atomic_inc_return(&call->sequence);
686
687			sp->hdr.epoch = conn->epoch;
688			sp->hdr.cid = call->cid;
689			sp->hdr.callNumber = call->call_id;
690			sp->hdr.seq = htonl(seq);
691			sp->hdr.serial =
692				htonl(atomic_inc_return(&conn->serial));
693			sp->hdr.type = RXRPC_PACKET_TYPE_DATA;
694			sp->hdr.userStatus = 0;
695			sp->hdr.securityIndex = conn->security_ix;
696			sp->hdr._rsvd = 0;
697			sp->hdr.serviceId = conn->service_id;
698
699			sp->hdr.flags = conn->out_clientflag;
700			if (len == 0 && !more)
701				sp->hdr.flags |= RXRPC_LAST_PACKET;
702			else if (CIRC_SPACE(call->acks_head, call->acks_tail,
703					    call->acks_winsz) > 1)
704				sp->hdr.flags |= RXRPC_MORE_PACKETS;
705			if (more && seq & 1)
706				sp->hdr.flags |= RXRPC_REQUEST_ACK;
707
708			ret = rxrpc_secure_packet(
709				call, skb, skb->mark,
710				skb->head + sizeof(struct rxrpc_header));
711			if (ret < 0)
712				goto out;
713
714			memcpy(skb->head, &sp->hdr,
715			       sizeof(struct rxrpc_header));
716			rxrpc_queue_packet(call, skb, segment == 0 && !more);
717			skb = NULL;
718		}
719
720	} while (segment > 0);
721
722success:
723	ret = copied;
724out:
725	call->tx_pending = skb;
726	_leave(" = %d", ret);
727	return ret;
728
729call_aborted:
730	rxrpc_free_skb(skb);
731	if (call->state == RXRPC_CALL_NETWORK_ERROR)
732		ret = call->conn->trans->peer->net_error;
733	else
734		ret = -ECONNABORTED;
735	_leave(" = %d", ret);
736	return ret;
737
738maybe_error:
739	if (copied)
740		goto success;
741	goto out;
742
743efault:
744	ret = -EFAULT;
745	goto out;
746}
747