datagram.c revision c3bdeb5c7cc073ccf5ff9624642022a8613a956e
11da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds/* 21da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds * SUCS NET3: 31da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds * 41da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds * Generic datagram handling routines. These are generic for all 51da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds * protocols. Possibly a generic IP version on top of these would 61da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds * make sense. Not tonight however 8-). 71da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds * This is used because UDP, RAW, PACKET, DDP, IPX, AX.25 and 81da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds * NetROM layer all have identical poll code and mostly 91da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds * identical recvmsg() code. So we share it here. The poll was 101da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds * shared before but buried in udp.c so I moved it. 111da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds * 12113aa838ec3a235d883f8357d31d90e16c47fc89Alan Cox * Authors: Alan Cox <alan@lxorguk.ukuu.org.uk>. (datagram_poll() from old 131da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds * udp.c code) 141da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds * 151da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds * Fixes: 161da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds * Alan Cox : NULL return from skb_peek_copy() 171da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds * understood 181da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds * Alan Cox : Rewrote skb_read_datagram to avoid the 191da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds * skb_peek_copy stuff. 201da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds * Alan Cox : Added support for SOCK_SEQPACKET. 211da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds * IPX can no longer use the SO_TYPE hack 221da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds * but AX.25 now works right, and SPX is 231da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds * feasible. 241da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds * Alan Cox : Fixed write poll of non IP protocol 251da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds * crash. 261da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds * Florian La Roche: Changed for my new skbuff handling. 271da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds * Darryl Miles : Fixed non-blocking SOCK_SEQPACKET. 281da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds * Linus Torvalds : BSD semantic fixes. 291da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds * Alan Cox : Datagram iovec handling 301da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds * Darryl Miles : Fixed non-blocking SOCK_STREAM. 311da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds * Alan Cox : POSIXisms 321da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds * Pete Wyckoff : Unconnected accept() fix. 331da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds * 341da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds */ 351da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 361da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds#include <linux/module.h> 371da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds#include <linux/types.h> 381da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds#include <linux/kernel.h> 391da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds#include <asm/uaccess.h> 401da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds#include <linux/mm.h> 411da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds#include <linux/interrupt.h> 421da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds#include <linux/errno.h> 431da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds#include <linux/sched.h> 441da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds#include <linux/inet.h> 451da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds#include <linux/netdevice.h> 461da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds#include <linux/rtnetlink.h> 471da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds#include <linux/poll.h> 481da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds#include <linux/highmem.h> 493305b80c214c642b89cd5c21af83bc91ec13f8bdHerbert Xu#include <linux/spinlock.h> 505a0e3ad6af8660be21ca98a971cd00f331318c05Tejun Heo#include <linux/slab.h> 511da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 521da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds#include <net/protocol.h> 531da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds#include <linux/skbuff.h> 541da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 55c752f0739f09b803aed191c4765a3b6650a08653Arnaldo Carvalho de Melo#include <net/checksum.h> 56c752f0739f09b803aed191c4765a3b6650a08653Arnaldo Carvalho de Melo#include <net/sock.h> 57c752f0739f09b803aed191c4765a3b6650a08653Arnaldo Carvalho de Melo#include <net/tcp_states.h> 58e9b3cc1b3779fe10a80de4c3e7404bd308d0eae3Neil Horman#include <trace/events/skb.h> 59076bb0c82a44fbe46fe2c8527a5b5b64b69f679dEliezer Tamir#include <net/busy_poll.h> 601da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 611da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds/* 621da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds * Is a socket 'connection oriented' ? 631da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds */ 641da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvaldsstatic inline int connection_based(struct sock *sk) 651da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds{ 661da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds return sk->sk_type == SOCK_SEQPACKET || sk->sk_type == SOCK_STREAM; 671da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds} 681da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 6995c961747284a6b83a5e2d81240e214b0fa3464dEric Dumazetstatic int receiver_wake_function(wait_queue_t *wait, unsigned int mode, int sync, 70bf368e4e70cd4e0f880923c44e95a4273d725ab4Eric Dumazet void *key) 71bf368e4e70cd4e0f880923c44e95a4273d725ab4Eric Dumazet{ 72bf368e4e70cd4e0f880923c44e95a4273d725ab4Eric Dumazet unsigned long bits = (unsigned long)key; 73bf368e4e70cd4e0f880923c44e95a4273d725ab4Eric Dumazet 74bf368e4e70cd4e0f880923c44e95a4273d725ab4Eric Dumazet /* 75bf368e4e70cd4e0f880923c44e95a4273d725ab4Eric Dumazet * Avoid a wakeup if event not interesting for us 76bf368e4e70cd4e0f880923c44e95a4273d725ab4Eric Dumazet */ 77bf368e4e70cd4e0f880923c44e95a4273d725ab4Eric Dumazet if (bits && !(bits & (POLLIN | POLLERR))) 78bf368e4e70cd4e0f880923c44e95a4273d725ab4Eric Dumazet return 0; 79bf368e4e70cd4e0f880923c44e95a4273d725ab4Eric Dumazet return autoremove_wake_function(wait, mode, sync, key); 80bf368e4e70cd4e0f880923c44e95a4273d725ab4Eric Dumazet} 811da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds/* 8239cc86130bc045d87f525ce7742da308ff757cecBenjamin Poirier * Wait for the last received packet to be different from skb 831da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds */ 8439cc86130bc045d87f525ce7742da308ff757cecBenjamin Poirierstatic int wait_for_more_packets(struct sock *sk, int *err, long *timeo_p, 8539cc86130bc045d87f525ce7742da308ff757cecBenjamin Poirier const struct sk_buff *skb) 861da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds{ 871da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds int error; 88bf368e4e70cd4e0f880923c44e95a4273d725ab4Eric Dumazet DEFINE_WAIT_FUNC(wait, receiver_wake_function); 891da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 90aa395145165cb06a0d0885221bbe0ce4a564391dEric Dumazet prepare_to_wait_exclusive(sk_sleep(sk), &wait, TASK_INTERRUPTIBLE); 911da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 921da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds /* Socket errors? */ 931da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds error = sock_error(sk); 941da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds if (error) 951da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds goto out_err; 961da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 9739cc86130bc045d87f525ce7742da308ff757cecBenjamin Poirier if (sk->sk_receive_queue.prev != skb) 981da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds goto out; 991da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 1001da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds /* Socket shut down? */ 1011da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds if (sk->sk_shutdown & RCV_SHUTDOWN) 1021da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds goto out_noerr; 1031da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 1041da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds /* Sequenced packets can come disconnected. 1051da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds * If so we report the problem 1061da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds */ 1071da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds error = -ENOTCONN; 1081da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds if (connection_based(sk) && 1091da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds !(sk->sk_state == TCP_ESTABLISHED || sk->sk_state == TCP_LISTEN)) 1101da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds goto out_err; 1111da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 1121da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds /* handle signals */ 1131da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds if (signal_pending(current)) 1141da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds goto interrupted; 1151da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 1161da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds error = 0; 1171da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds *timeo_p = schedule_timeout(*timeo_p); 1181da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvaldsout: 119aa395145165cb06a0d0885221bbe0ce4a564391dEric Dumazet finish_wait(sk_sleep(sk), &wait); 1201da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds return error; 1211da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvaldsinterrupted: 1221da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds error = sock_intr_errno(*timeo_p); 1231da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvaldsout_err: 1241da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds *err = error; 1251da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds goto out; 1261da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvaldsout_noerr: 1271da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds *err = 0; 1281da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds error = 1; 1291da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds goto out; 1301da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds} 1311da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 1321da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds/** 133a59322be07c964e916d15be3df473fb7ba20c41eHerbert Xu * __skb_recv_datagram - Receive a datagram skbuff 1344dc3b16ba18c0f967ad100c52fa65b01a4f76ff0Pavel Pisa * @sk: socket 1354dc3b16ba18c0f967ad100c52fa65b01a4f76ff0Pavel Pisa * @flags: MSG_ flags 13639cc86130bc045d87f525ce7742da308ff757cecBenjamin Poirier * @peeked: returns non-zero if this packet has been seen before 1373f518bf745cbd6007d8069100fb9cb09e960c872Pavel Emelyanov * @off: an offset in bytes to peek skb from. Returns an offset 1383f518bf745cbd6007d8069100fb9cb09e960c872Pavel Emelyanov * within an skb where data actually starts 1394dc3b16ba18c0f967ad100c52fa65b01a4f76ff0Pavel Pisa * @err: error code returned 1401da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds * 1411da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds * Get a datagram skbuff, understands the peeking, nonblocking wakeups 1421da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds * and possible races. This replaces identical code in packet, raw and 1431da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds * udp, as well as the IPX AX.25 and Appletalk. It also finally fixes 1441da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds * the long standing peek and read race for datagram sockets. If you 1451da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds * alter this routine remember it must be re-entrant. 1461da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds * 1471da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds * This function will lock the socket if a skb is returned, so the caller 1481da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds * needs to unlock the socket in that case (usually by calling 1491da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds * skb_free_datagram) 1501da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds * 1511da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds * * It does not lock socket since today. This function is 1521da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds * * free of race conditions. This measure should/can improve 1531da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds * * significantly datagram socket latencies at high loads, 1541da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds * * when data copying to user space takes lots of time. 1551da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds * * (BTW I've just killed the last cli() in IP/IPv6/core/netlink/packet 1561da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds * * 8) Great win.) 1571da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds * * --ANK (980729) 1581da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds * 1591da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds * The order of the tests when we find no data waiting are specified 1601da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds * quite explicitly by POSIX 1003.1g, don't change them without having 1611da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds * the standard around please. 1621da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds */ 16395c961747284a6b83a5e2d81240e214b0fa3464dEric Dumazetstruct sk_buff *__skb_recv_datagram(struct sock *sk, unsigned int flags, 1643f518bf745cbd6007d8069100fb9cb09e960c872Pavel Emelyanov int *peeked, int *off, int *err) 1651da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds{ 16639cc86130bc045d87f525ce7742da308ff757cecBenjamin Poirier struct sk_buff *skb, *last; 1671da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds long timeo; 1681da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds /* 1691da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds * Caller is allowed not to check sk->sk_err before skb_recv_datagram() 1701da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds */ 1711da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds int error = sock_error(sk); 1721da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 1731da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds if (error) 1741da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds goto no_packet; 1751da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 176a59322be07c964e916d15be3df473fb7ba20c41eHerbert Xu timeo = sock_rcvtimeo(sk, flags & MSG_DONTWAIT); 1771da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 1781da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds do { 1791da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds /* Again only user level code calls this function, so nothing 1801da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds * interrupt level will suddenly eat the receive_queue. 1811da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds * 1821da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds * Look at current nfs client by the way... 1838917a3c0b7d1557548f50bfe3f0e18e0354e38f6David Shwatrz * However, this function was correct in any case. 8) 1841da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds */ 185a59322be07c964e916d15be3df473fb7ba20c41eHerbert Xu unsigned long cpu_flags; 1864934b0329f7150dcb5f90506860e2db32274c755Pavel Emelyanov struct sk_buff_head *queue = &sk->sk_receive_queue; 18739cc86130bc045d87f525ce7742da308ff757cecBenjamin Poirier int _off = *off; 188a59322be07c964e916d15be3df473fb7ba20c41eHerbert Xu 18939cc86130bc045d87f525ce7742da308ff757cecBenjamin Poirier last = (struct sk_buff *)queue; 1904934b0329f7150dcb5f90506860e2db32274c755Pavel Emelyanov spin_lock_irqsave(&queue->lock, cpu_flags); 1913f518bf745cbd6007d8069100fb9cb09e960c872Pavel Emelyanov skb_queue_walk(queue, skb) { 19239cc86130bc045d87f525ce7742da308ff757cecBenjamin Poirier last = skb; 193a59322be07c964e916d15be3df473fb7ba20c41eHerbert Xu *peeked = skb->peeked; 194a59322be07c964e916d15be3df473fb7ba20c41eHerbert Xu if (flags & MSG_PEEK) { 19539cc86130bc045d87f525ce7742da308ff757cecBenjamin Poirier if (_off >= skb->len && (skb->len || _off || 196add05ad4e9f5c4efee9b98535db5efa32b0d0492Benjamin Poirier skb->peeked)) { 19739cc86130bc045d87f525ce7742da308ff757cecBenjamin Poirier _off -= skb->len; 1983f518bf745cbd6007d8069100fb9cb09e960c872Pavel Emelyanov continue; 1993f518bf745cbd6007d8069100fb9cb09e960c872Pavel Emelyanov } 200a59322be07c964e916d15be3df473fb7ba20c41eHerbert Xu skb->peeked = 1; 2011da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds atomic_inc(&skb->users); 202a59322be07c964e916d15be3df473fb7ba20c41eHerbert Xu } else 2034934b0329f7150dcb5f90506860e2db32274c755Pavel Emelyanov __skb_unlink(skb, queue); 2041da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 2053f518bf745cbd6007d8069100fb9cb09e960c872Pavel Emelyanov spin_unlock_irqrestore(&queue->lock, cpu_flags); 20639cc86130bc045d87f525ce7742da308ff757cecBenjamin Poirier *off = _off; 2071da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds return skb; 2083f518bf745cbd6007d8069100fb9cb09e960c872Pavel Emelyanov } 2093f518bf745cbd6007d8069100fb9cb09e960c872Pavel Emelyanov spin_unlock_irqrestore(&queue->lock, cpu_flags); 2101da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 211cbf55001b2ddb814329735641be5d29b08c82b08Eliezer Tamir if (sk_can_busy_loop(sk) && 212cbf55001b2ddb814329735641be5d29b08c82b08Eliezer Tamir sk_busy_loop(sk, flags & MSG_DONTWAIT)) 213a5b50476f77a8fcc8055c955720d05a7c2d9c532Eliezer Tamir continue; 214a5b50476f77a8fcc8055c955720d05a7c2d9c532Eliezer Tamir 2151da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds /* User doesn't want to wait */ 2161da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds error = -EAGAIN; 2171da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds if (!timeo) 2181da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds goto no_packet; 2191da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 22039cc86130bc045d87f525ce7742da308ff757cecBenjamin Poirier } while (!wait_for_more_packets(sk, err, &timeo, last)); 2211da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 2221da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds return NULL; 2231da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 2241da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvaldsno_packet: 2251da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds *err = error; 2261da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds return NULL; 2271da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds} 228a59322be07c964e916d15be3df473fb7ba20c41eHerbert XuEXPORT_SYMBOL(__skb_recv_datagram); 229a59322be07c964e916d15be3df473fb7ba20c41eHerbert Xu 23095c961747284a6b83a5e2d81240e214b0fa3464dEric Dumazetstruct sk_buff *skb_recv_datagram(struct sock *sk, unsigned int flags, 231a59322be07c964e916d15be3df473fb7ba20c41eHerbert Xu int noblock, int *err) 232a59322be07c964e916d15be3df473fb7ba20c41eHerbert Xu{ 2333f518bf745cbd6007d8069100fb9cb09e960c872Pavel Emelyanov int peeked, off = 0; 234a59322be07c964e916d15be3df473fb7ba20c41eHerbert Xu 235a59322be07c964e916d15be3df473fb7ba20c41eHerbert Xu return __skb_recv_datagram(sk, flags | (noblock ? MSG_DONTWAIT : 0), 2363f518bf745cbd6007d8069100fb9cb09e960c872Pavel Emelyanov &peeked, &off, err); 237a59322be07c964e916d15be3df473fb7ba20c41eHerbert Xu} 2389e34a5b51684bc90ac827ec4ba339f3892632eacEric DumazetEXPORT_SYMBOL(skb_recv_datagram); 2391da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 2401da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvaldsvoid skb_free_datagram(struct sock *sk, struct sk_buff *skb) 2411da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds{ 242ead2ceb0ec9f85cff19c43b5cdb2f8a054484431Neil Horman consume_skb(skb); 243270acefafeb74ce2fe93d35b75733870bf1e11e7Eric Dumazet sk_mem_reclaim_partial(sk); 2441da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds} 2459d410c796067686b1e032d54ce475b7055537138Eric DumazetEXPORT_SYMBOL(skb_free_datagram); 2469d410c796067686b1e032d54ce475b7055537138Eric Dumazet 2479d410c796067686b1e032d54ce475b7055537138Eric Dumazetvoid skb_free_datagram_locked(struct sock *sk, struct sk_buff *skb) 2489d410c796067686b1e032d54ce475b7055537138Eric Dumazet{ 2498a74ad60a546b13bd1096b2a61a7a5c6fd9ae17cEric Dumazet bool slow; 2508a74ad60a546b13bd1096b2a61a7a5c6fd9ae17cEric Dumazet 25193bb64eac10aad3dae6178d7da94765f207d121fEric Dumazet if (likely(atomic_read(&skb->users) == 1)) 25293bb64eac10aad3dae6178d7da94765f207d121fEric Dumazet smp_rmb(); 25393bb64eac10aad3dae6178d7da94765f207d121fEric Dumazet else if (likely(!atomic_dec_and_test(&skb->users))) 25493bb64eac10aad3dae6178d7da94765f207d121fEric Dumazet return; 25593bb64eac10aad3dae6178d7da94765f207d121fEric Dumazet 2568a74ad60a546b13bd1096b2a61a7a5c6fd9ae17cEric Dumazet slow = lock_sock_fast(sk); 2574b0b72f7dd617b13abd1b04c947e15873e011a24Eric Dumazet skb_orphan(skb); 2584b0b72f7dd617b13abd1b04c947e15873e011a24Eric Dumazet sk_mem_reclaim_partial(sk); 2598a74ad60a546b13bd1096b2a61a7a5c6fd9ae17cEric Dumazet unlock_sock_fast(sk, slow); 2604b0b72f7dd617b13abd1b04c947e15873e011a24Eric Dumazet 26193bb64eac10aad3dae6178d7da94765f207d121fEric Dumazet /* skb is now orphaned, can be freed outside of locked section */ 26293bb64eac10aad3dae6178d7da94765f207d121fEric Dumazet __kfree_skb(skb); 2639d410c796067686b1e032d54ce475b7055537138Eric Dumazet} 2649d410c796067686b1e032d54ce475b7055537138Eric DumazetEXPORT_SYMBOL(skb_free_datagram_locked); 2651da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 2661da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds/** 2673305b80c214c642b89cd5c21af83bc91ec13f8bdHerbert Xu * skb_kill_datagram - Free a datagram skbuff forcibly 2683305b80c214c642b89cd5c21af83bc91ec13f8bdHerbert Xu * @sk: socket 2693305b80c214c642b89cd5c21af83bc91ec13f8bdHerbert Xu * @skb: datagram skbuff 2703305b80c214c642b89cd5c21af83bc91ec13f8bdHerbert Xu * @flags: MSG_ flags 2713305b80c214c642b89cd5c21af83bc91ec13f8bdHerbert Xu * 2723305b80c214c642b89cd5c21af83bc91ec13f8bdHerbert Xu * This function frees a datagram skbuff that was received by 2733305b80c214c642b89cd5c21af83bc91ec13f8bdHerbert Xu * skb_recv_datagram. The flags argument must match the one 2743305b80c214c642b89cd5c21af83bc91ec13f8bdHerbert Xu * used for skb_recv_datagram. 2753305b80c214c642b89cd5c21af83bc91ec13f8bdHerbert Xu * 2763305b80c214c642b89cd5c21af83bc91ec13f8bdHerbert Xu * If the MSG_PEEK flag is set, and the packet is still on the 2773305b80c214c642b89cd5c21af83bc91ec13f8bdHerbert Xu * receive queue of the socket, it will be taken off the queue 2783305b80c214c642b89cd5c21af83bc91ec13f8bdHerbert Xu * before it is freed. 2793305b80c214c642b89cd5c21af83bc91ec13f8bdHerbert Xu * 2803305b80c214c642b89cd5c21af83bc91ec13f8bdHerbert Xu * This function currently only disables BH when acquiring the 2813305b80c214c642b89cd5c21af83bc91ec13f8bdHerbert Xu * sk_receive_queue lock. Therefore it must not be used in a 2823305b80c214c642b89cd5c21af83bc91ec13f8bdHerbert Xu * context where that lock is acquired in an IRQ context. 28327ab2568649d5ba6c5a20212079b7c4f6da4ca0dHerbert Xu * 28427ab2568649d5ba6c5a20212079b7c4f6da4ca0dHerbert Xu * It returns 0 if the packet was removed by us. 2853305b80c214c642b89cd5c21af83bc91ec13f8bdHerbert Xu */ 2863305b80c214c642b89cd5c21af83bc91ec13f8bdHerbert Xu 28727ab2568649d5ba6c5a20212079b7c4f6da4ca0dHerbert Xuint skb_kill_datagram(struct sock *sk, struct sk_buff *skb, unsigned int flags) 2883305b80c214c642b89cd5c21af83bc91ec13f8bdHerbert Xu{ 28927ab2568649d5ba6c5a20212079b7c4f6da4ca0dHerbert Xu int err = 0; 29027ab2568649d5ba6c5a20212079b7c4f6da4ca0dHerbert Xu 2913305b80c214c642b89cd5c21af83bc91ec13f8bdHerbert Xu if (flags & MSG_PEEK) { 29227ab2568649d5ba6c5a20212079b7c4f6da4ca0dHerbert Xu err = -ENOENT; 2933305b80c214c642b89cd5c21af83bc91ec13f8bdHerbert Xu spin_lock_bh(&sk->sk_receive_queue.lock); 2943305b80c214c642b89cd5c21af83bc91ec13f8bdHerbert Xu if (skb == skb_peek(&sk->sk_receive_queue)) { 2953305b80c214c642b89cd5c21af83bc91ec13f8bdHerbert Xu __skb_unlink(skb, &sk->sk_receive_queue); 2963305b80c214c642b89cd5c21af83bc91ec13f8bdHerbert Xu atomic_dec(&skb->users); 29727ab2568649d5ba6c5a20212079b7c4f6da4ca0dHerbert Xu err = 0; 2983305b80c214c642b89cd5c21af83bc91ec13f8bdHerbert Xu } 2993305b80c214c642b89cd5c21af83bc91ec13f8bdHerbert Xu spin_unlock_bh(&sk->sk_receive_queue.lock); 3003305b80c214c642b89cd5c21af83bc91ec13f8bdHerbert Xu } 3013305b80c214c642b89cd5c21af83bc91ec13f8bdHerbert Xu 30261de71c67caec39df0a854a1ef5be0c6be385c2aJohn Dykstra kfree_skb(skb); 3038edf19c2fe028563fc6ea9cb1995b8ee4172d4b6Eric Dumazet atomic_inc(&sk->sk_drops); 30461de71c67caec39df0a854a1ef5be0c6be385c2aJohn Dykstra sk_mem_reclaim_partial(sk); 30561de71c67caec39df0a854a1ef5be0c6be385c2aJohn Dykstra 30627ab2568649d5ba6c5a20212079b7c4f6da4ca0dHerbert Xu return err; 3073305b80c214c642b89cd5c21af83bc91ec13f8bdHerbert Xu} 3083305b80c214c642b89cd5c21af83bc91ec13f8bdHerbert XuEXPORT_SYMBOL(skb_kill_datagram); 3093305b80c214c642b89cd5c21af83bc91ec13f8bdHerbert Xu 3103305b80c214c642b89cd5c21af83bc91ec13f8bdHerbert Xu/** 3111da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds * skb_copy_datagram_iovec - Copy a datagram to an iovec. 3124dc3b16ba18c0f967ad100c52fa65b01a4f76ff0Pavel Pisa * @skb: buffer to copy 3134dc3b16ba18c0f967ad100c52fa65b01a4f76ff0Pavel Pisa * @offset: offset in the buffer to start copying from 31467be2dd1bace0ec7ce2dbc1bba3f8df3d7be597eMartin Waitz * @to: io vector to copy to 3154dc3b16ba18c0f967ad100c52fa65b01a4f76ff0Pavel Pisa * @len: amount of data to copy from buffer to iovec 3161da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds * 3171da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds * Note: the iovec is modified during the copy. 3181da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds */ 3191da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvaldsint skb_copy_datagram_iovec(const struct sk_buff *skb, int offset, 3201da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds struct iovec *to, int len) 3211da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds{ 3221a028e50729b85d0a038fad13daf0ee201a37454David S. Miller int start = skb_headlen(skb); 3231a028e50729b85d0a038fad13daf0ee201a37454David S. Miller int i, copy = start - offset; 3245b1a002ade68173f21b2126a778278df72202ba6David S. Miller struct sk_buff *frag_iter; 325c75d721c761ad0f2d8725c40af9e4f376efefd24Herbert Xu 326e9b3cc1b3779fe10a80de4c3e7404bd308d0eae3Neil Horman trace_skb_copy_datagram_iovec(skb, len); 327e9b3cc1b3779fe10a80de4c3e7404bd308d0eae3Neil Horman 328b4d9eda028e8becbb5057b554e63eea12e496a88David S. Miller /* Copy header. */ 329b4d9eda028e8becbb5057b554e63eea12e496a88David S. Miller if (copy > 0) { 330b4d9eda028e8becbb5057b554e63eea12e496a88David S. Miller if (copy > len) 331b4d9eda028e8becbb5057b554e63eea12e496a88David S. Miller copy = len; 332b4d9eda028e8becbb5057b554e63eea12e496a88David S. Miller if (memcpy_toiovec(to, skb->data + offset, copy)) 333b4d9eda028e8becbb5057b554e63eea12e496a88David S. Miller goto fault; 334b4d9eda028e8becbb5057b554e63eea12e496a88David S. Miller if ((len -= copy) == 0) 335b4d9eda028e8becbb5057b554e63eea12e496a88David S. Miller return 0; 336b4d9eda028e8becbb5057b554e63eea12e496a88David S. Miller offset += copy; 337b4d9eda028e8becbb5057b554e63eea12e496a88David S. Miller } 338c75d721c761ad0f2d8725c40af9e4f376efefd24Herbert Xu 339b4d9eda028e8becbb5057b554e63eea12e496a88David S. Miller /* Copy paged appendix. Hmm... why does this look so complicated? */ 340b4d9eda028e8becbb5057b554e63eea12e496a88David S. Miller for (i = 0; i < skb_shinfo(skb)->nr_frags; i++) { 3411a028e50729b85d0a038fad13daf0ee201a37454David S. Miller int end; 3429e903e085262ffbf1fc44a17ac06058aca03524aEric Dumazet const skb_frag_t *frag = &skb_shinfo(skb)->frags[i]; 3431da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 344547b792cac0a038b9dbf958d3c120df3740b5572Ilpo Järvinen WARN_ON(start > offset + len); 3451a028e50729b85d0a038fad13daf0ee201a37454David S. Miller 3469e903e085262ffbf1fc44a17ac06058aca03524aEric Dumazet end = start + skb_frag_size(frag); 347b4d9eda028e8becbb5057b554e63eea12e496a88David S. Miller if ((copy = end - offset) > 0) { 348b4d9eda028e8becbb5057b554e63eea12e496a88David S. Miller int err; 349b4d9eda028e8becbb5057b554e63eea12e496a88David S. Miller u8 *vaddr; 350ea2ab69379a941c6f8884e290fdd28c93936a778Ian Campbell struct page *page = skb_frag_page(frag); 3511da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 3521da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds if (copy > len) 3531da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds copy = len; 354b4d9eda028e8becbb5057b554e63eea12e496a88David S. Miller vaddr = kmap(page); 3551a028e50729b85d0a038fad13daf0ee201a37454David S. Miller err = memcpy_toiovec(to, vaddr + frag->page_offset + 3561a028e50729b85d0a038fad13daf0ee201a37454David S. Miller offset - start, copy); 357b4d9eda028e8becbb5057b554e63eea12e496a88David S. Miller kunmap(page); 3581da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds if (err) 3591da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds goto fault; 3601da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds if (!(len -= copy)) 3611da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds return 0; 3621da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds offset += copy; 3631da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds } 3641a028e50729b85d0a038fad13daf0ee201a37454David S. Miller start = end; 3651da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds } 366b4d9eda028e8becbb5057b554e63eea12e496a88David S. Miller 3675b1a002ade68173f21b2126a778278df72202ba6David S. Miller skb_walk_frags(skb, frag_iter) { 3685b1a002ade68173f21b2126a778278df72202ba6David S. Miller int end; 3695b1a002ade68173f21b2126a778278df72202ba6David S. Miller 3705b1a002ade68173f21b2126a778278df72202ba6David S. Miller WARN_ON(start > offset + len); 3715b1a002ade68173f21b2126a778278df72202ba6David S. Miller 3725b1a002ade68173f21b2126a778278df72202ba6David S. Miller end = start + frag_iter->len; 3735b1a002ade68173f21b2126a778278df72202ba6David S. Miller if ((copy = end - offset) > 0) { 3745b1a002ade68173f21b2126a778278df72202ba6David S. Miller if (copy > len) 3755b1a002ade68173f21b2126a778278df72202ba6David S. Miller copy = len; 3765b1a002ade68173f21b2126a778278df72202ba6David S. Miller if (skb_copy_datagram_iovec(frag_iter, 3775b1a002ade68173f21b2126a778278df72202ba6David S. Miller offset - start, 3785b1a002ade68173f21b2126a778278df72202ba6David S. Miller to, copy)) 3795b1a002ade68173f21b2126a778278df72202ba6David S. Miller goto fault; 3805b1a002ade68173f21b2126a778278df72202ba6David S. Miller if ((len -= copy) == 0) 3815b1a002ade68173f21b2126a778278df72202ba6David S. Miller return 0; 3825b1a002ade68173f21b2126a778278df72202ba6David S. Miller offset += copy; 383b4d9eda028e8becbb5057b554e63eea12e496a88David S. Miller } 3845b1a002ade68173f21b2126a778278df72202ba6David S. Miller start = end; 3851da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds } 386b4d9eda028e8becbb5057b554e63eea12e496a88David S. Miller if (!len) 387b4d9eda028e8becbb5057b554e63eea12e496a88David S. Miller return 0; 388b4d9eda028e8becbb5057b554e63eea12e496a88David S. Miller 3891da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvaldsfault: 3901da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds return -EFAULT; 3911da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds} 3929e34a5b51684bc90ac827ec4ba339f3892632eacEric DumazetEXPORT_SYMBOL(skb_copy_datagram_iovec); 3931da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 394db543c1f973cd1d557cc32ceee76737c1e4d2898Rusty Russell/** 3950a1ec07a67bd8b0033dace237249654d015efa21Michael S. Tsirkin * skb_copy_datagram_const_iovec - Copy a datagram to an iovec. 3960a1ec07a67bd8b0033dace237249654d015efa21Michael S. Tsirkin * @skb: buffer to copy 3970a1ec07a67bd8b0033dace237249654d015efa21Michael S. Tsirkin * @offset: offset in the buffer to start copying from 3980a1ec07a67bd8b0033dace237249654d015efa21Michael S. Tsirkin * @to: io vector to copy to 3990a1ec07a67bd8b0033dace237249654d015efa21Michael S. Tsirkin * @to_offset: offset in the io vector to start copying to 4000a1ec07a67bd8b0033dace237249654d015efa21Michael S. Tsirkin * @len: amount of data to copy from buffer to iovec 4010a1ec07a67bd8b0033dace237249654d015efa21Michael S. Tsirkin * 4020a1ec07a67bd8b0033dace237249654d015efa21Michael S. Tsirkin * Returns 0 or -EFAULT. 4030a1ec07a67bd8b0033dace237249654d015efa21Michael S. Tsirkin * Note: the iovec is not modified during the copy. 4040a1ec07a67bd8b0033dace237249654d015efa21Michael S. Tsirkin */ 4050a1ec07a67bd8b0033dace237249654d015efa21Michael S. Tsirkinint skb_copy_datagram_const_iovec(const struct sk_buff *skb, int offset, 4060a1ec07a67bd8b0033dace237249654d015efa21Michael S. Tsirkin const struct iovec *to, int to_offset, 4070a1ec07a67bd8b0033dace237249654d015efa21Michael S. Tsirkin int len) 4080a1ec07a67bd8b0033dace237249654d015efa21Michael S. Tsirkin{ 4090a1ec07a67bd8b0033dace237249654d015efa21Michael S. Tsirkin int start = skb_headlen(skb); 4100a1ec07a67bd8b0033dace237249654d015efa21Michael S. Tsirkin int i, copy = start - offset; 4115b1a002ade68173f21b2126a778278df72202ba6David S. Miller struct sk_buff *frag_iter; 4120a1ec07a67bd8b0033dace237249654d015efa21Michael S. Tsirkin 4130a1ec07a67bd8b0033dace237249654d015efa21Michael S. Tsirkin /* Copy header. */ 4140a1ec07a67bd8b0033dace237249654d015efa21Michael S. Tsirkin if (copy > 0) { 4150a1ec07a67bd8b0033dace237249654d015efa21Michael S. Tsirkin if (copy > len) 4160a1ec07a67bd8b0033dace237249654d015efa21Michael S. Tsirkin copy = len; 4170a1ec07a67bd8b0033dace237249654d015efa21Michael S. Tsirkin if (memcpy_toiovecend(to, skb->data + offset, to_offset, copy)) 4180a1ec07a67bd8b0033dace237249654d015efa21Michael S. Tsirkin goto fault; 4190a1ec07a67bd8b0033dace237249654d015efa21Michael S. Tsirkin if ((len -= copy) == 0) 4200a1ec07a67bd8b0033dace237249654d015efa21Michael S. Tsirkin return 0; 4210a1ec07a67bd8b0033dace237249654d015efa21Michael S. Tsirkin offset += copy; 4220a1ec07a67bd8b0033dace237249654d015efa21Michael S. Tsirkin to_offset += copy; 4230a1ec07a67bd8b0033dace237249654d015efa21Michael S. Tsirkin } 4240a1ec07a67bd8b0033dace237249654d015efa21Michael S. Tsirkin 4250a1ec07a67bd8b0033dace237249654d015efa21Michael S. Tsirkin /* Copy paged appendix. Hmm... why does this look so complicated? */ 4260a1ec07a67bd8b0033dace237249654d015efa21Michael S. Tsirkin for (i = 0; i < skb_shinfo(skb)->nr_frags; i++) { 4270a1ec07a67bd8b0033dace237249654d015efa21Michael S. Tsirkin int end; 4289e903e085262ffbf1fc44a17ac06058aca03524aEric Dumazet const skb_frag_t *frag = &skb_shinfo(skb)->frags[i]; 4290a1ec07a67bd8b0033dace237249654d015efa21Michael S. Tsirkin 4300a1ec07a67bd8b0033dace237249654d015efa21Michael S. Tsirkin WARN_ON(start > offset + len); 4310a1ec07a67bd8b0033dace237249654d015efa21Michael S. Tsirkin 4329e903e085262ffbf1fc44a17ac06058aca03524aEric Dumazet end = start + skb_frag_size(frag); 4330a1ec07a67bd8b0033dace237249654d015efa21Michael S. Tsirkin if ((copy = end - offset) > 0) { 4340a1ec07a67bd8b0033dace237249654d015efa21Michael S. Tsirkin int err; 4350a1ec07a67bd8b0033dace237249654d015efa21Michael S. Tsirkin u8 *vaddr; 436ea2ab69379a941c6f8884e290fdd28c93936a778Ian Campbell struct page *page = skb_frag_page(frag); 4370a1ec07a67bd8b0033dace237249654d015efa21Michael S. Tsirkin 4380a1ec07a67bd8b0033dace237249654d015efa21Michael S. Tsirkin if (copy > len) 4390a1ec07a67bd8b0033dace237249654d015efa21Michael S. Tsirkin copy = len; 4400a1ec07a67bd8b0033dace237249654d015efa21Michael S. Tsirkin vaddr = kmap(page); 4410a1ec07a67bd8b0033dace237249654d015efa21Michael S. Tsirkin err = memcpy_toiovecend(to, vaddr + frag->page_offset + 4420a1ec07a67bd8b0033dace237249654d015efa21Michael S. Tsirkin offset - start, to_offset, copy); 4430a1ec07a67bd8b0033dace237249654d015efa21Michael S. Tsirkin kunmap(page); 4440a1ec07a67bd8b0033dace237249654d015efa21Michael S. Tsirkin if (err) 4450a1ec07a67bd8b0033dace237249654d015efa21Michael S. Tsirkin goto fault; 4460a1ec07a67bd8b0033dace237249654d015efa21Michael S. Tsirkin if (!(len -= copy)) 4470a1ec07a67bd8b0033dace237249654d015efa21Michael S. Tsirkin return 0; 4480a1ec07a67bd8b0033dace237249654d015efa21Michael S. Tsirkin offset += copy; 4490a1ec07a67bd8b0033dace237249654d015efa21Michael S. Tsirkin to_offset += copy; 4500a1ec07a67bd8b0033dace237249654d015efa21Michael S. Tsirkin } 4510a1ec07a67bd8b0033dace237249654d015efa21Michael S. Tsirkin start = end; 4520a1ec07a67bd8b0033dace237249654d015efa21Michael S. Tsirkin } 4530a1ec07a67bd8b0033dace237249654d015efa21Michael S. Tsirkin 4545b1a002ade68173f21b2126a778278df72202ba6David S. Miller skb_walk_frags(skb, frag_iter) { 4555b1a002ade68173f21b2126a778278df72202ba6David S. Miller int end; 4565b1a002ade68173f21b2126a778278df72202ba6David S. Miller 4575b1a002ade68173f21b2126a778278df72202ba6David S. Miller WARN_ON(start > offset + len); 4585b1a002ade68173f21b2126a778278df72202ba6David S. Miller 4595b1a002ade68173f21b2126a778278df72202ba6David S. Miller end = start + frag_iter->len; 4605b1a002ade68173f21b2126a778278df72202ba6David S. Miller if ((copy = end - offset) > 0) { 4615b1a002ade68173f21b2126a778278df72202ba6David S. Miller if (copy > len) 4625b1a002ade68173f21b2126a778278df72202ba6David S. Miller copy = len; 4635b1a002ade68173f21b2126a778278df72202ba6David S. Miller if (skb_copy_datagram_const_iovec(frag_iter, 4645b1a002ade68173f21b2126a778278df72202ba6David S. Miller offset - start, 4655b1a002ade68173f21b2126a778278df72202ba6David S. Miller to, to_offset, 4665b1a002ade68173f21b2126a778278df72202ba6David S. Miller copy)) 4675b1a002ade68173f21b2126a778278df72202ba6David S. Miller goto fault; 4685b1a002ade68173f21b2126a778278df72202ba6David S. Miller if ((len -= copy) == 0) 4695b1a002ade68173f21b2126a778278df72202ba6David S. Miller return 0; 4705b1a002ade68173f21b2126a778278df72202ba6David S. Miller offset += copy; 4715b1a002ade68173f21b2126a778278df72202ba6David S. Miller to_offset += copy; 4720a1ec07a67bd8b0033dace237249654d015efa21Michael S. Tsirkin } 4735b1a002ade68173f21b2126a778278df72202ba6David S. Miller start = end; 4740a1ec07a67bd8b0033dace237249654d015efa21Michael S. Tsirkin } 4750a1ec07a67bd8b0033dace237249654d015efa21Michael S. Tsirkin if (!len) 4760a1ec07a67bd8b0033dace237249654d015efa21Michael S. Tsirkin return 0; 4770a1ec07a67bd8b0033dace237249654d015efa21Michael S. Tsirkin 4780a1ec07a67bd8b0033dace237249654d015efa21Michael S. Tsirkinfault: 4790a1ec07a67bd8b0033dace237249654d015efa21Michael S. Tsirkin return -EFAULT; 4800a1ec07a67bd8b0033dace237249654d015efa21Michael S. Tsirkin} 4810a1ec07a67bd8b0033dace237249654d015efa21Michael S. TsirkinEXPORT_SYMBOL(skb_copy_datagram_const_iovec); 4820a1ec07a67bd8b0033dace237249654d015efa21Michael S. Tsirkin 4830a1ec07a67bd8b0033dace237249654d015efa21Michael S. Tsirkin/** 484db543c1f973cd1d557cc32ceee76737c1e4d2898Rusty Russell * skb_copy_datagram_from_iovec - Copy a datagram from an iovec. 485db543c1f973cd1d557cc32ceee76737c1e4d2898Rusty Russell * @skb: buffer to copy 486db543c1f973cd1d557cc32ceee76737c1e4d2898Rusty Russell * @offset: offset in the buffer to start copying to 487db543c1f973cd1d557cc32ceee76737c1e4d2898Rusty Russell * @from: io vector to copy to 4886f26c9a7555e5bcca3560919db9b852015077daeMichael S. Tsirkin * @from_offset: offset in the io vector to start copying from 489db543c1f973cd1d557cc32ceee76737c1e4d2898Rusty Russell * @len: amount of data to copy to buffer from iovec 490db543c1f973cd1d557cc32ceee76737c1e4d2898Rusty Russell * 491db543c1f973cd1d557cc32ceee76737c1e4d2898Rusty Russell * Returns 0 or -EFAULT. 4926f26c9a7555e5bcca3560919db9b852015077daeMichael S. Tsirkin * Note: the iovec is not modified during the copy. 493db543c1f973cd1d557cc32ceee76737c1e4d2898Rusty Russell */ 494db543c1f973cd1d557cc32ceee76737c1e4d2898Rusty Russellint skb_copy_datagram_from_iovec(struct sk_buff *skb, int offset, 4956f26c9a7555e5bcca3560919db9b852015077daeMichael S. Tsirkin const struct iovec *from, int from_offset, 4966f26c9a7555e5bcca3560919db9b852015077daeMichael S. Tsirkin int len) 497db543c1f973cd1d557cc32ceee76737c1e4d2898Rusty Russell{ 498db543c1f973cd1d557cc32ceee76737c1e4d2898Rusty Russell int start = skb_headlen(skb); 499db543c1f973cd1d557cc32ceee76737c1e4d2898Rusty Russell int i, copy = start - offset; 5005b1a002ade68173f21b2126a778278df72202ba6David S. Miller struct sk_buff *frag_iter; 501db543c1f973cd1d557cc32ceee76737c1e4d2898Rusty Russell 502db543c1f973cd1d557cc32ceee76737c1e4d2898Rusty Russell /* Copy header. */ 503db543c1f973cd1d557cc32ceee76737c1e4d2898Rusty Russell if (copy > 0) { 504db543c1f973cd1d557cc32ceee76737c1e4d2898Rusty Russell if (copy > len) 505db543c1f973cd1d557cc32ceee76737c1e4d2898Rusty Russell copy = len; 506d2d27bfd11659675fdd1c20b1c7f8f59873cad60Sridhar Samudrala if (memcpy_fromiovecend(skb->data + offset, from, from_offset, 507d2d27bfd11659675fdd1c20b1c7f8f59873cad60Sridhar Samudrala copy)) 508db543c1f973cd1d557cc32ceee76737c1e4d2898Rusty Russell goto fault; 509db543c1f973cd1d557cc32ceee76737c1e4d2898Rusty Russell if ((len -= copy) == 0) 510db543c1f973cd1d557cc32ceee76737c1e4d2898Rusty Russell return 0; 511db543c1f973cd1d557cc32ceee76737c1e4d2898Rusty Russell offset += copy; 5126f26c9a7555e5bcca3560919db9b852015077daeMichael S. Tsirkin from_offset += copy; 513db543c1f973cd1d557cc32ceee76737c1e4d2898Rusty Russell } 514db543c1f973cd1d557cc32ceee76737c1e4d2898Rusty Russell 515db543c1f973cd1d557cc32ceee76737c1e4d2898Rusty Russell /* Copy paged appendix. Hmm... why does this look so complicated? */ 516db543c1f973cd1d557cc32ceee76737c1e4d2898Rusty Russell for (i = 0; i < skb_shinfo(skb)->nr_frags; i++) { 517db543c1f973cd1d557cc32ceee76737c1e4d2898Rusty Russell int end; 5189e903e085262ffbf1fc44a17ac06058aca03524aEric Dumazet const skb_frag_t *frag = &skb_shinfo(skb)->frags[i]; 519db543c1f973cd1d557cc32ceee76737c1e4d2898Rusty Russell 520db543c1f973cd1d557cc32ceee76737c1e4d2898Rusty Russell WARN_ON(start > offset + len); 521db543c1f973cd1d557cc32ceee76737c1e4d2898Rusty Russell 5229e903e085262ffbf1fc44a17ac06058aca03524aEric Dumazet end = start + skb_frag_size(frag); 523db543c1f973cd1d557cc32ceee76737c1e4d2898Rusty Russell if ((copy = end - offset) > 0) { 524db543c1f973cd1d557cc32ceee76737c1e4d2898Rusty Russell int err; 525db543c1f973cd1d557cc32ceee76737c1e4d2898Rusty Russell u8 *vaddr; 526ea2ab69379a941c6f8884e290fdd28c93936a778Ian Campbell struct page *page = skb_frag_page(frag); 527db543c1f973cd1d557cc32ceee76737c1e4d2898Rusty Russell 528db543c1f973cd1d557cc32ceee76737c1e4d2898Rusty Russell if (copy > len) 529db543c1f973cd1d557cc32ceee76737c1e4d2898Rusty Russell copy = len; 530db543c1f973cd1d557cc32ceee76737c1e4d2898Rusty Russell vaddr = kmap(page); 5316f26c9a7555e5bcca3560919db9b852015077daeMichael S. Tsirkin err = memcpy_fromiovecend(vaddr + frag->page_offset + 5326f26c9a7555e5bcca3560919db9b852015077daeMichael S. Tsirkin offset - start, 5336f26c9a7555e5bcca3560919db9b852015077daeMichael S. Tsirkin from, from_offset, copy); 534db543c1f973cd1d557cc32ceee76737c1e4d2898Rusty Russell kunmap(page); 535db543c1f973cd1d557cc32ceee76737c1e4d2898Rusty Russell if (err) 536db543c1f973cd1d557cc32ceee76737c1e4d2898Rusty Russell goto fault; 537db543c1f973cd1d557cc32ceee76737c1e4d2898Rusty Russell 538db543c1f973cd1d557cc32ceee76737c1e4d2898Rusty Russell if (!(len -= copy)) 539db543c1f973cd1d557cc32ceee76737c1e4d2898Rusty Russell return 0; 540db543c1f973cd1d557cc32ceee76737c1e4d2898Rusty Russell offset += copy; 5416f26c9a7555e5bcca3560919db9b852015077daeMichael S. Tsirkin from_offset += copy; 542db543c1f973cd1d557cc32ceee76737c1e4d2898Rusty Russell } 543db543c1f973cd1d557cc32ceee76737c1e4d2898Rusty Russell start = end; 544db543c1f973cd1d557cc32ceee76737c1e4d2898Rusty Russell } 545db543c1f973cd1d557cc32ceee76737c1e4d2898Rusty Russell 5465b1a002ade68173f21b2126a778278df72202ba6David S. Miller skb_walk_frags(skb, frag_iter) { 5475b1a002ade68173f21b2126a778278df72202ba6David S. Miller int end; 5485b1a002ade68173f21b2126a778278df72202ba6David S. Miller 5495b1a002ade68173f21b2126a778278df72202ba6David S. Miller WARN_ON(start > offset + len); 5505b1a002ade68173f21b2126a778278df72202ba6David S. Miller 5515b1a002ade68173f21b2126a778278df72202ba6David S. Miller end = start + frag_iter->len; 5525b1a002ade68173f21b2126a778278df72202ba6David S. Miller if ((copy = end - offset) > 0) { 5535b1a002ade68173f21b2126a778278df72202ba6David S. Miller if (copy > len) 5545b1a002ade68173f21b2126a778278df72202ba6David S. Miller copy = len; 5555b1a002ade68173f21b2126a778278df72202ba6David S. Miller if (skb_copy_datagram_from_iovec(frag_iter, 5565b1a002ade68173f21b2126a778278df72202ba6David S. Miller offset - start, 5575b1a002ade68173f21b2126a778278df72202ba6David S. Miller from, 5585b1a002ade68173f21b2126a778278df72202ba6David S. Miller from_offset, 5595b1a002ade68173f21b2126a778278df72202ba6David S. Miller copy)) 5605b1a002ade68173f21b2126a778278df72202ba6David S. Miller goto fault; 5615b1a002ade68173f21b2126a778278df72202ba6David S. Miller if ((len -= copy) == 0) 5625b1a002ade68173f21b2126a778278df72202ba6David S. Miller return 0; 5635b1a002ade68173f21b2126a778278df72202ba6David S. Miller offset += copy; 5645b1a002ade68173f21b2126a778278df72202ba6David S. Miller from_offset += copy; 565db543c1f973cd1d557cc32ceee76737c1e4d2898Rusty Russell } 5665b1a002ade68173f21b2126a778278df72202ba6David S. Miller start = end; 567db543c1f973cd1d557cc32ceee76737c1e4d2898Rusty Russell } 568db543c1f973cd1d557cc32ceee76737c1e4d2898Rusty Russell if (!len) 569db543c1f973cd1d557cc32ceee76737c1e4d2898Rusty Russell return 0; 570db543c1f973cd1d557cc32ceee76737c1e4d2898Rusty Russell 571db543c1f973cd1d557cc32ceee76737c1e4d2898Rusty Russellfault: 572db543c1f973cd1d557cc32ceee76737c1e4d2898Rusty Russell return -EFAULT; 573db543c1f973cd1d557cc32ceee76737c1e4d2898Rusty Russell} 574db543c1f973cd1d557cc32ceee76737c1e4d2898Rusty RussellEXPORT_SYMBOL(skb_copy_datagram_from_iovec); 575db543c1f973cd1d557cc32ceee76737c1e4d2898Rusty Russell 576c3bdeb5c7cc073ccf5ff9624642022a8613a956eJason Wang/** 577c3bdeb5c7cc073ccf5ff9624642022a8613a956eJason Wang * zerocopy_sg_from_iovec - Build a zerocopy datagram from an iovec 578c3bdeb5c7cc073ccf5ff9624642022a8613a956eJason Wang * @skb: buffer to copy 579c3bdeb5c7cc073ccf5ff9624642022a8613a956eJason Wang * @from: io vector to copy to 580c3bdeb5c7cc073ccf5ff9624642022a8613a956eJason Wang * @offset: offset in the io vector to start copying from 581c3bdeb5c7cc073ccf5ff9624642022a8613a956eJason Wang * @count: amount of vectors to copy to buffer from 582c3bdeb5c7cc073ccf5ff9624642022a8613a956eJason Wang * 583c3bdeb5c7cc073ccf5ff9624642022a8613a956eJason Wang * The function will first copy up to headlen, and then pin the userspace 584c3bdeb5c7cc073ccf5ff9624642022a8613a956eJason Wang * pages and build frags through them. 585c3bdeb5c7cc073ccf5ff9624642022a8613a956eJason Wang * 586c3bdeb5c7cc073ccf5ff9624642022a8613a956eJason Wang * Returns 0, -EFAULT or -EMSGSIZE. 587c3bdeb5c7cc073ccf5ff9624642022a8613a956eJason Wang * Note: the iovec is not modified during the copy 588c3bdeb5c7cc073ccf5ff9624642022a8613a956eJason Wang */ 589c3bdeb5c7cc073ccf5ff9624642022a8613a956eJason Wangint zerocopy_sg_from_iovec(struct sk_buff *skb, const struct iovec *from, 590c3bdeb5c7cc073ccf5ff9624642022a8613a956eJason Wang int offset, size_t count) 591c3bdeb5c7cc073ccf5ff9624642022a8613a956eJason Wang{ 592c3bdeb5c7cc073ccf5ff9624642022a8613a956eJason Wang int len = iov_length(from, count) - offset; 593c3bdeb5c7cc073ccf5ff9624642022a8613a956eJason Wang int copy = skb_headlen(skb); 594c3bdeb5c7cc073ccf5ff9624642022a8613a956eJason Wang int size, offset1 = 0; 595c3bdeb5c7cc073ccf5ff9624642022a8613a956eJason Wang int i = 0; 596c3bdeb5c7cc073ccf5ff9624642022a8613a956eJason Wang 597c3bdeb5c7cc073ccf5ff9624642022a8613a956eJason Wang /* Skip over from offset */ 598c3bdeb5c7cc073ccf5ff9624642022a8613a956eJason Wang while (count && (offset >= from->iov_len)) { 599c3bdeb5c7cc073ccf5ff9624642022a8613a956eJason Wang offset -= from->iov_len; 600c3bdeb5c7cc073ccf5ff9624642022a8613a956eJason Wang ++from; 601c3bdeb5c7cc073ccf5ff9624642022a8613a956eJason Wang --count; 602c3bdeb5c7cc073ccf5ff9624642022a8613a956eJason Wang } 603c3bdeb5c7cc073ccf5ff9624642022a8613a956eJason Wang 604c3bdeb5c7cc073ccf5ff9624642022a8613a956eJason Wang /* copy up to skb headlen */ 605c3bdeb5c7cc073ccf5ff9624642022a8613a956eJason Wang while (count && (copy > 0)) { 606c3bdeb5c7cc073ccf5ff9624642022a8613a956eJason Wang size = min_t(unsigned int, copy, from->iov_len - offset); 607c3bdeb5c7cc073ccf5ff9624642022a8613a956eJason Wang if (copy_from_user(skb->data + offset1, from->iov_base + offset, 608c3bdeb5c7cc073ccf5ff9624642022a8613a956eJason Wang size)) 609c3bdeb5c7cc073ccf5ff9624642022a8613a956eJason Wang return -EFAULT; 610c3bdeb5c7cc073ccf5ff9624642022a8613a956eJason Wang if (copy > size) { 611c3bdeb5c7cc073ccf5ff9624642022a8613a956eJason Wang ++from; 612c3bdeb5c7cc073ccf5ff9624642022a8613a956eJason Wang --count; 613c3bdeb5c7cc073ccf5ff9624642022a8613a956eJason Wang offset = 0; 614c3bdeb5c7cc073ccf5ff9624642022a8613a956eJason Wang } else 615c3bdeb5c7cc073ccf5ff9624642022a8613a956eJason Wang offset += size; 616c3bdeb5c7cc073ccf5ff9624642022a8613a956eJason Wang copy -= size; 617c3bdeb5c7cc073ccf5ff9624642022a8613a956eJason Wang offset1 += size; 618c3bdeb5c7cc073ccf5ff9624642022a8613a956eJason Wang } 619c3bdeb5c7cc073ccf5ff9624642022a8613a956eJason Wang 620c3bdeb5c7cc073ccf5ff9624642022a8613a956eJason Wang if (len == offset1) 621c3bdeb5c7cc073ccf5ff9624642022a8613a956eJason Wang return 0; 622c3bdeb5c7cc073ccf5ff9624642022a8613a956eJason Wang 623c3bdeb5c7cc073ccf5ff9624642022a8613a956eJason Wang while (count--) { 624c3bdeb5c7cc073ccf5ff9624642022a8613a956eJason Wang struct page *page[MAX_SKB_FRAGS]; 625c3bdeb5c7cc073ccf5ff9624642022a8613a956eJason Wang int num_pages; 626c3bdeb5c7cc073ccf5ff9624642022a8613a956eJason Wang unsigned long base; 627c3bdeb5c7cc073ccf5ff9624642022a8613a956eJason Wang unsigned long truesize; 628c3bdeb5c7cc073ccf5ff9624642022a8613a956eJason Wang 629c3bdeb5c7cc073ccf5ff9624642022a8613a956eJason Wang len = from->iov_len - offset; 630c3bdeb5c7cc073ccf5ff9624642022a8613a956eJason Wang if (!len) { 631c3bdeb5c7cc073ccf5ff9624642022a8613a956eJason Wang offset = 0; 632c3bdeb5c7cc073ccf5ff9624642022a8613a956eJason Wang ++from; 633c3bdeb5c7cc073ccf5ff9624642022a8613a956eJason Wang continue; 634c3bdeb5c7cc073ccf5ff9624642022a8613a956eJason Wang } 635c3bdeb5c7cc073ccf5ff9624642022a8613a956eJason Wang base = (unsigned long)from->iov_base + offset; 636c3bdeb5c7cc073ccf5ff9624642022a8613a956eJason Wang size = ((base & ~PAGE_MASK) + len + ~PAGE_MASK) >> PAGE_SHIFT; 637c3bdeb5c7cc073ccf5ff9624642022a8613a956eJason Wang if (i + size > MAX_SKB_FRAGS) 638c3bdeb5c7cc073ccf5ff9624642022a8613a956eJason Wang return -EMSGSIZE; 639c3bdeb5c7cc073ccf5ff9624642022a8613a956eJason Wang num_pages = get_user_pages_fast(base, size, 0, &page[i]); 640c3bdeb5c7cc073ccf5ff9624642022a8613a956eJason Wang if (num_pages != size) { 641c3bdeb5c7cc073ccf5ff9624642022a8613a956eJason Wang int j; 642c3bdeb5c7cc073ccf5ff9624642022a8613a956eJason Wang 643c3bdeb5c7cc073ccf5ff9624642022a8613a956eJason Wang for (j = 0; j < num_pages; j++) 644c3bdeb5c7cc073ccf5ff9624642022a8613a956eJason Wang put_page(page[i + j]); 645c3bdeb5c7cc073ccf5ff9624642022a8613a956eJason Wang return -EFAULT; 646c3bdeb5c7cc073ccf5ff9624642022a8613a956eJason Wang } 647c3bdeb5c7cc073ccf5ff9624642022a8613a956eJason Wang truesize = size * PAGE_SIZE; 648c3bdeb5c7cc073ccf5ff9624642022a8613a956eJason Wang skb->data_len += len; 649c3bdeb5c7cc073ccf5ff9624642022a8613a956eJason Wang skb->len += len; 650c3bdeb5c7cc073ccf5ff9624642022a8613a956eJason Wang skb->truesize += truesize; 651c3bdeb5c7cc073ccf5ff9624642022a8613a956eJason Wang atomic_add(truesize, &skb->sk->sk_wmem_alloc); 652c3bdeb5c7cc073ccf5ff9624642022a8613a956eJason Wang while (len) { 653c3bdeb5c7cc073ccf5ff9624642022a8613a956eJason Wang int off = base & ~PAGE_MASK; 654c3bdeb5c7cc073ccf5ff9624642022a8613a956eJason Wang int size = min_t(int, len, PAGE_SIZE - off); 655c3bdeb5c7cc073ccf5ff9624642022a8613a956eJason Wang __skb_fill_page_desc(skb, i, page[i], off, size); 656c3bdeb5c7cc073ccf5ff9624642022a8613a956eJason Wang skb_shinfo(skb)->nr_frags++; 657c3bdeb5c7cc073ccf5ff9624642022a8613a956eJason Wang /* increase sk_wmem_alloc */ 658c3bdeb5c7cc073ccf5ff9624642022a8613a956eJason Wang base += size; 659c3bdeb5c7cc073ccf5ff9624642022a8613a956eJason Wang len -= size; 660c3bdeb5c7cc073ccf5ff9624642022a8613a956eJason Wang i++; 661c3bdeb5c7cc073ccf5ff9624642022a8613a956eJason Wang } 662c3bdeb5c7cc073ccf5ff9624642022a8613a956eJason Wang offset = 0; 663c3bdeb5c7cc073ccf5ff9624642022a8613a956eJason Wang ++from; 664c3bdeb5c7cc073ccf5ff9624642022a8613a956eJason Wang } 665c3bdeb5c7cc073ccf5ff9624642022a8613a956eJason Wang return 0; 666c3bdeb5c7cc073ccf5ff9624642022a8613a956eJason Wang} 667c3bdeb5c7cc073ccf5ff9624642022a8613a956eJason WangEXPORT_SYMBOL(zerocopy_sg_from_iovec); 668c3bdeb5c7cc073ccf5ff9624642022a8613a956eJason Wang 6691da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvaldsstatic int skb_copy_and_csum_datagram(const struct sk_buff *skb, int offset, 6701da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds u8 __user *to, int len, 6715084205faf45384fff25c4cf77dd5c96279283adAl Viro __wsum *csump) 6721da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds{ 6731a028e50729b85d0a038fad13daf0ee201a37454David S. Miller int start = skb_headlen(skb); 6741a028e50729b85d0a038fad13daf0ee201a37454David S. Miller int i, copy = start - offset; 6755b1a002ade68173f21b2126a778278df72202ba6David S. Miller struct sk_buff *frag_iter; 6765b1a002ade68173f21b2126a778278df72202ba6David S. Miller int pos = 0; 6771da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 6781da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds /* Copy header. */ 6791da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds if (copy > 0) { 6801da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds int err = 0; 6811da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds if (copy > len) 6821da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds copy = len; 6831da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds *csump = csum_and_copy_to_user(skb->data + offset, to, copy, 6841da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds *csump, &err); 6851da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds if (err) 6861da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds goto fault; 6871da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds if ((len -= copy) == 0) 6881da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds return 0; 6891da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds offset += copy; 6901da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds to += copy; 6911da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds pos = copy; 6921da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds } 6931da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 6941da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds for (i = 0; i < skb_shinfo(skb)->nr_frags; i++) { 6951a028e50729b85d0a038fad13daf0ee201a37454David S. Miller int end; 6969e903e085262ffbf1fc44a17ac06058aca03524aEric Dumazet const skb_frag_t *frag = &skb_shinfo(skb)->frags[i]; 6971da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 698547b792cac0a038b9dbf958d3c120df3740b5572Ilpo Järvinen WARN_ON(start > offset + len); 6991a028e50729b85d0a038fad13daf0ee201a37454David S. Miller 7009e903e085262ffbf1fc44a17ac06058aca03524aEric Dumazet end = start + skb_frag_size(frag); 7011da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds if ((copy = end - offset) > 0) { 7025084205faf45384fff25c4cf77dd5c96279283adAl Viro __wsum csum2; 7031da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds int err = 0; 7041da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds u8 *vaddr; 705ea2ab69379a941c6f8884e290fdd28c93936a778Ian Campbell struct page *page = skb_frag_page(frag); 7061da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 7071da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds if (copy > len) 7081da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds copy = len; 7091da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds vaddr = kmap(page); 7101da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds csum2 = csum_and_copy_to_user(vaddr + 7111a028e50729b85d0a038fad13daf0ee201a37454David S. Miller frag->page_offset + 7121a028e50729b85d0a038fad13daf0ee201a37454David S. Miller offset - start, 7131da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds to, copy, 0, &err); 7141da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds kunmap(page); 7151da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds if (err) 7161da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds goto fault; 7171da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds *csump = csum_block_add(*csump, csum2, pos); 7181da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds if (!(len -= copy)) 7191da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds return 0; 7201da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds offset += copy; 7211da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds to += copy; 7221da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds pos += copy; 7231da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds } 7241a028e50729b85d0a038fad13daf0ee201a37454David S. Miller start = end; 7251da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds } 7261da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 7275b1a002ade68173f21b2126a778278df72202ba6David S. Miller skb_walk_frags(skb, frag_iter) { 7285b1a002ade68173f21b2126a778278df72202ba6David S. Miller int end; 7295b1a002ade68173f21b2126a778278df72202ba6David S. Miller 7305b1a002ade68173f21b2126a778278df72202ba6David S. Miller WARN_ON(start > offset + len); 7315b1a002ade68173f21b2126a778278df72202ba6David S. Miller 7325b1a002ade68173f21b2126a778278df72202ba6David S. Miller end = start + frag_iter->len; 7335b1a002ade68173f21b2126a778278df72202ba6David S. Miller if ((copy = end - offset) > 0) { 7345b1a002ade68173f21b2126a778278df72202ba6David S. Miller __wsum csum2 = 0; 7355b1a002ade68173f21b2126a778278df72202ba6David S. Miller if (copy > len) 7365b1a002ade68173f21b2126a778278df72202ba6David S. Miller copy = len; 7375b1a002ade68173f21b2126a778278df72202ba6David S. Miller if (skb_copy_and_csum_datagram(frag_iter, 7385b1a002ade68173f21b2126a778278df72202ba6David S. Miller offset - start, 7395b1a002ade68173f21b2126a778278df72202ba6David S. Miller to, copy, 7405b1a002ade68173f21b2126a778278df72202ba6David S. Miller &csum2)) 7415b1a002ade68173f21b2126a778278df72202ba6David S. Miller goto fault; 7425b1a002ade68173f21b2126a778278df72202ba6David S. Miller *csump = csum_block_add(*csump, csum2, pos); 7435b1a002ade68173f21b2126a778278df72202ba6David S. Miller if ((len -= copy) == 0) 7445b1a002ade68173f21b2126a778278df72202ba6David S. Miller return 0; 7455b1a002ade68173f21b2126a778278df72202ba6David S. Miller offset += copy; 7465b1a002ade68173f21b2126a778278df72202ba6David S. Miller to += copy; 7475b1a002ade68173f21b2126a778278df72202ba6David S. Miller pos += copy; 7481da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds } 7495b1a002ade68173f21b2126a778278df72202ba6David S. Miller start = end; 7501da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds } 7511da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds if (!len) 7521da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds return 0; 7531da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 7541da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvaldsfault: 7551da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds return -EFAULT; 7561da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds} 7571da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 758759e5d006462d53fb708daa8284b4ad909415da1Herbert Xu__sum16 __skb_checksum_complete_head(struct sk_buff *skb, int len) 759fb286bb2990a107009dbf25f6ffebeb7df77f9beHerbert Xu{ 760d3bc23e7ee9db8023dff5a86bb3b0069ed018789Al Viro __sum16 sum; 761fb286bb2990a107009dbf25f6ffebeb7df77f9beHerbert Xu 762759e5d006462d53fb708daa8284b4ad909415da1Herbert Xu sum = csum_fold(skb_checksum(skb, 0, len, skb->csum)); 763fb286bb2990a107009dbf25f6ffebeb7df77f9beHerbert Xu if (likely(!sum)) { 76484fa7933a33f806bbbaae6775e87459b1ec584c0Patrick McHardy if (unlikely(skb->ip_summed == CHECKSUM_COMPLETE)) 765fb286bb2990a107009dbf25f6ffebeb7df77f9beHerbert Xu netdev_rx_csum_fault(skb->dev); 766fb286bb2990a107009dbf25f6ffebeb7df77f9beHerbert Xu skb->ip_summed = CHECKSUM_UNNECESSARY; 767fb286bb2990a107009dbf25f6ffebeb7df77f9beHerbert Xu } 768fb286bb2990a107009dbf25f6ffebeb7df77f9beHerbert Xu return sum; 769fb286bb2990a107009dbf25f6ffebeb7df77f9beHerbert Xu} 770759e5d006462d53fb708daa8284b4ad909415da1Herbert XuEXPORT_SYMBOL(__skb_checksum_complete_head); 771759e5d006462d53fb708daa8284b4ad909415da1Herbert Xu 772759e5d006462d53fb708daa8284b4ad909415da1Herbert Xu__sum16 __skb_checksum_complete(struct sk_buff *skb) 773759e5d006462d53fb708daa8284b4ad909415da1Herbert Xu{ 774759e5d006462d53fb708daa8284b4ad909415da1Herbert Xu return __skb_checksum_complete_head(skb, skb->len); 775759e5d006462d53fb708daa8284b4ad909415da1Herbert Xu} 776fb286bb2990a107009dbf25f6ffebeb7df77f9beHerbert XuEXPORT_SYMBOL(__skb_checksum_complete); 777fb286bb2990a107009dbf25f6ffebeb7df77f9beHerbert Xu 7781da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds/** 7791da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds * skb_copy_and_csum_datagram_iovec - Copy and checkum skb to user iovec. 7804dc3b16ba18c0f967ad100c52fa65b01a4f76ff0Pavel Pisa * @skb: skbuff 7814dc3b16ba18c0f967ad100c52fa65b01a4f76ff0Pavel Pisa * @hlen: hardware length 78267be2dd1bace0ec7ce2dbc1bba3f8df3d7be597eMartin Waitz * @iov: io vector 7834ec93edb14fe5fdee9fae6335f2cbba204627eacYOSHIFUJI Hideaki * 7841da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds * Caller _must_ check that skb will fit to this iovec. 7851da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds * 7861da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds * Returns: 0 - success. 7871da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds * -EINVAL - checksum failure. 7881da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds * -EFAULT - fault during copy. Beware, in this case iovec 7891da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds * can be modified! 7901da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds */ 791fb286bb2990a107009dbf25f6ffebeb7df77f9beHerbert Xuint skb_copy_and_csum_datagram_iovec(struct sk_buff *skb, 7921da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds int hlen, struct iovec *iov) 7931da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds{ 794d3bc23e7ee9db8023dff5a86bb3b0069ed018789Al Viro __wsum csum; 7951da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds int chunk = skb->len - hlen; 7961da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 797ef8aef55ce61fd0e2af798695f7386ac756ae1e7Herbert Xu if (!chunk) 798ef8aef55ce61fd0e2af798695f7386ac756ae1e7Herbert Xu return 0; 799ef8aef55ce61fd0e2af798695f7386ac756ae1e7Herbert Xu 8001da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds /* Skip filled elements. 8011da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds * Pretty silly, look at memcpy_toiovec, though 8) 8021da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds */ 8031da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds while (!iov->iov_len) 8041da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds iov++; 8051da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 8061da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds if (iov->iov_len < chunk) { 807fb286bb2990a107009dbf25f6ffebeb7df77f9beHerbert Xu if (__skb_checksum_complete(skb)) 8081da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds goto csum_error; 8091da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds if (skb_copy_datagram_iovec(skb, hlen, iov, chunk)) 8101da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds goto fault; 8111da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds } else { 8121da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds csum = csum_partial(skb->data, hlen, skb->csum); 8131da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds if (skb_copy_and_csum_datagram(skb, hlen, iov->iov_base, 8141da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds chunk, &csum)) 8151da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds goto fault; 816d3bc23e7ee9db8023dff5a86bb3b0069ed018789Al Viro if (csum_fold(csum)) 8171da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds goto csum_error; 81884fa7933a33f806bbbaae6775e87459b1ec584c0Patrick McHardy if (unlikely(skb->ip_summed == CHECKSUM_COMPLETE)) 819fb286bb2990a107009dbf25f6ffebeb7df77f9beHerbert Xu netdev_rx_csum_fault(skb->dev); 8201da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds iov->iov_len -= chunk; 8211da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds iov->iov_base += chunk; 8221da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds } 8231da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds return 0; 8241da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvaldscsum_error: 8251da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds return -EINVAL; 8261da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvaldsfault: 8271da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds return -EFAULT; 8281da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds} 8299e34a5b51684bc90ac827ec4ba339f3892632eacEric DumazetEXPORT_SYMBOL(skb_copy_and_csum_datagram_iovec); 8301da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 8311da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds/** 8321da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds * datagram_poll - generic datagram poll 8334dc3b16ba18c0f967ad100c52fa65b01a4f76ff0Pavel Pisa * @file: file struct 8344dc3b16ba18c0f967ad100c52fa65b01a4f76ff0Pavel Pisa * @sock: socket 8354dc3b16ba18c0f967ad100c52fa65b01a4f76ff0Pavel Pisa * @wait: poll table 8361da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds * 8371da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds * Datagram poll: Again totally generic. This also handles 8381da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds * sequenced packet sockets providing the socket receive queue 8391da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds * is only ever holding data ready to receive. 8401da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds * 8411da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds * Note: when you _don't_ use this routine for this protocol, 8421da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds * and you use a different write policy from sock_writeable() 8431da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds * then please supply your own write_space callback. 8441da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds */ 8451da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvaldsunsigned int datagram_poll(struct file *file, struct socket *sock, 8461da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds poll_table *wait) 8471da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds{ 8481da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds struct sock *sk = sock->sk; 8491da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds unsigned int mask; 8501da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 851aa395145165cb06a0d0885221bbe0ce4a564391dEric Dumazet sock_poll_wait(file, sk_sleep(sk), wait); 8521da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds mask = 0; 8531da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 8541da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds /* exceptional events? */ 8551da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds if (sk->sk_err || !skb_queue_empty(&sk->sk_error_queue)) 8567d4c04fc170087119727119074e72445f2bb192bKeller, Jacob E mask |= POLLERR | 8578facd5fb73c6e960555e5913743dfbb6c3d984a5Jacob Keller (sock_flag(sk, SOCK_SELECT_ERR_QUEUE) ? POLLPRI : 0); 8587d4c04fc170087119727119074e72445f2bb192bKeller, Jacob E 859f348d70a324e15afc701a494f32ec468abb7d1ebDavide Libenzi if (sk->sk_shutdown & RCV_SHUTDOWN) 860db40980fcdb560d7992b0511df16cdd3f7e381f3Eric Dumazet mask |= POLLRDHUP | POLLIN | POLLRDNORM; 8611da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds if (sk->sk_shutdown == SHUTDOWN_MASK) 8621da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds mask |= POLLHUP; 8631da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 8641da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds /* readable? */ 865db40980fcdb560d7992b0511df16cdd3f7e381f3Eric Dumazet if (!skb_queue_empty(&sk->sk_receive_queue)) 8661da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds mask |= POLLIN | POLLRDNORM; 8671da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 8681da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds /* Connection-based need to check for termination and startup */ 8691da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds if (connection_based(sk)) { 8701da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds if (sk->sk_state == TCP_CLOSE) 8711da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds mask |= POLLHUP; 8721da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds /* connection hasn't started yet? */ 8731da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds if (sk->sk_state == TCP_SYN_SENT) 8741da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds return mask; 8751da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds } 8761da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 8771da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds /* writable? */ 8781da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds if (sock_writeable(sk)) 8791da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds mask |= POLLOUT | POLLWRNORM | POLLWRBAND; 8801da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds else 8811da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds set_bit(SOCK_ASYNC_NOSPACE, &sk->sk_socket->flags); 8821da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 8831da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds return mask; 8841da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds} 8851da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus TorvaldsEXPORT_SYMBOL(datagram_poll); 886