11da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds/* 21da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds * SUCS NET3: 31da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds * 41da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds * Generic datagram handling routines. These are generic for all 51da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds * protocols. Possibly a generic IP version on top of these would 61da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds * make sense. Not tonight however 8-). 71da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds * This is used because UDP, RAW, PACKET, DDP, IPX, AX.25 and 81da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds * NetROM layer all have identical poll code and mostly 91da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds * identical recvmsg() code. So we share it here. The poll was 101da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds * shared before but buried in udp.c so I moved it. 111da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds * 12113aa838ec3a235d883f8357d31d90e16c47fc89Alan Cox * Authors: Alan Cox <alan@lxorguk.ukuu.org.uk>. (datagram_poll() from old 131da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds * udp.c code) 141da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds * 151da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds * Fixes: 161da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds * Alan Cox : NULL return from skb_peek_copy() 171da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds * understood 181da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds * Alan Cox : Rewrote skb_read_datagram to avoid the 191da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds * skb_peek_copy stuff. 201da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds * Alan Cox : Added support for SOCK_SEQPACKET. 211da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds * IPX can no longer use the SO_TYPE hack 221da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds * but AX.25 now works right, and SPX is 231da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds * feasible. 241da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds * Alan Cox : Fixed write poll of non IP protocol 251da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds * crash. 261da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds * Florian La Roche: Changed for my new skbuff handling. 271da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds * Darryl Miles : Fixed non-blocking SOCK_SEQPACKET. 281da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds * Linus Torvalds : BSD semantic fixes. 291da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds * Alan Cox : Datagram iovec handling 301da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds * Darryl Miles : Fixed non-blocking SOCK_STREAM. 311da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds * Alan Cox : POSIXisms 321da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds * Pete Wyckoff : Unconnected accept() fix. 331da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds * 341da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds */ 351da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 361da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds#include <linux/module.h> 371da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds#include <linux/types.h> 381da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds#include <linux/kernel.h> 391da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds#include <asm/uaccess.h> 401da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds#include <linux/mm.h> 411da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds#include <linux/interrupt.h> 421da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds#include <linux/errno.h> 431da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds#include <linux/sched.h> 441da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds#include <linux/inet.h> 451da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds#include <linux/netdevice.h> 461da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds#include <linux/rtnetlink.h> 471da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds#include <linux/poll.h> 481da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds#include <linux/highmem.h> 493305b80c214c642b89cd5c21af83bc91ec13f8bdHerbert Xu#include <linux/spinlock.h> 505a0e3ad6af8660be21ca98a971cd00f331318c05Tejun Heo#include <linux/slab.h> 510433547aa7443cefc89d9b533169bdc50f1206b8Jason Wang#include <linux/pagemap.h> 521da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 531da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds#include <net/protocol.h> 541da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds#include <linux/skbuff.h> 551da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 56c752f0739f09b803aed191c4765a3b6650a08653Arnaldo Carvalho de Melo#include <net/checksum.h> 57c752f0739f09b803aed191c4765a3b6650a08653Arnaldo Carvalho de Melo#include <net/sock.h> 58c752f0739f09b803aed191c4765a3b6650a08653Arnaldo Carvalho de Melo#include <net/tcp_states.h> 59e9b3cc1b3779fe10a80de4c3e7404bd308d0eae3Neil Horman#include <trace/events/skb.h> 60076bb0c82a44fbe46fe2c8527a5b5b64b69f679dEliezer Tamir#include <net/busy_poll.h> 611da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 621da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds/* 631da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds * Is a socket 'connection oriented' ? 641da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds */ 651da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvaldsstatic inline int connection_based(struct sock *sk) 661da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds{ 671da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds return sk->sk_type == SOCK_SEQPACKET || sk->sk_type == SOCK_STREAM; 681da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds} 691da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 7095c961747284a6b83a5e2d81240e214b0fa3464dEric Dumazetstatic int receiver_wake_function(wait_queue_t *wait, unsigned int mode, int sync, 71bf368e4e70cd4e0f880923c44e95a4273d725ab4Eric Dumazet void *key) 72bf368e4e70cd4e0f880923c44e95a4273d725ab4Eric Dumazet{ 73bf368e4e70cd4e0f880923c44e95a4273d725ab4Eric Dumazet unsigned long bits = (unsigned long)key; 74bf368e4e70cd4e0f880923c44e95a4273d725ab4Eric Dumazet 75bf368e4e70cd4e0f880923c44e95a4273d725ab4Eric Dumazet /* 76bf368e4e70cd4e0f880923c44e95a4273d725ab4Eric Dumazet * Avoid a wakeup if event not interesting for us 77bf368e4e70cd4e0f880923c44e95a4273d725ab4Eric Dumazet */ 78bf368e4e70cd4e0f880923c44e95a4273d725ab4Eric Dumazet if (bits && !(bits & (POLLIN | POLLERR))) 79bf368e4e70cd4e0f880923c44e95a4273d725ab4Eric Dumazet return 0; 80bf368e4e70cd4e0f880923c44e95a4273d725ab4Eric Dumazet return autoremove_wake_function(wait, mode, sync, key); 81bf368e4e70cd4e0f880923c44e95a4273d725ab4Eric Dumazet} 821da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds/* 8339cc86130bc045d87f525ce7742da308ff757cecBenjamin Poirier * Wait for the last received packet to be different from skb 841da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds */ 8539cc86130bc045d87f525ce7742da308ff757cecBenjamin Poirierstatic int wait_for_more_packets(struct sock *sk, int *err, long *timeo_p, 8639cc86130bc045d87f525ce7742da308ff757cecBenjamin Poirier const struct sk_buff *skb) 871da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds{ 881da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds int error; 89bf368e4e70cd4e0f880923c44e95a4273d725ab4Eric Dumazet DEFINE_WAIT_FUNC(wait, receiver_wake_function); 901da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 91aa395145165cb06a0d0885221bbe0ce4a564391dEric Dumazet prepare_to_wait_exclusive(sk_sleep(sk), &wait, TASK_INTERRUPTIBLE); 921da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 931da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds /* Socket errors? */ 941da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds error = sock_error(sk); 951da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds if (error) 961da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds goto out_err; 971da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 9839cc86130bc045d87f525ce7742da308ff757cecBenjamin Poirier if (sk->sk_receive_queue.prev != skb) 991da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds goto out; 1001da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 1011da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds /* Socket shut down? */ 1021da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds if (sk->sk_shutdown & RCV_SHUTDOWN) 1031da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds goto out_noerr; 1041da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 1051da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds /* Sequenced packets can come disconnected. 1061da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds * If so we report the problem 1071da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds */ 1081da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds error = -ENOTCONN; 1091da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds if (connection_based(sk) && 1101da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds !(sk->sk_state == TCP_ESTABLISHED || sk->sk_state == TCP_LISTEN)) 1111da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds goto out_err; 1121da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 1131da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds /* handle signals */ 1141da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds if (signal_pending(current)) 1151da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds goto interrupted; 1161da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 1171da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds error = 0; 1181da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds *timeo_p = schedule_timeout(*timeo_p); 1191da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvaldsout: 120aa395145165cb06a0d0885221bbe0ce4a564391dEric Dumazet finish_wait(sk_sleep(sk), &wait); 1211da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds return error; 1221da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvaldsinterrupted: 1231da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds error = sock_intr_errno(*timeo_p); 1241da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvaldsout_err: 1251da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds *err = error; 1261da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds goto out; 1271da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvaldsout_noerr: 1281da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds *err = 0; 1291da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds error = 1; 1301da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds goto out; 1311da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds} 1321da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 1331da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds/** 134a59322be07c964e916d15be3df473fb7ba20c41eHerbert Xu * __skb_recv_datagram - Receive a datagram skbuff 1354dc3b16ba18c0f967ad100c52fa65b01a4f76ff0Pavel Pisa * @sk: socket 1364dc3b16ba18c0f967ad100c52fa65b01a4f76ff0Pavel Pisa * @flags: MSG_ flags 13739cc86130bc045d87f525ce7742da308ff757cecBenjamin Poirier * @peeked: returns non-zero if this packet has been seen before 1383f518bf745cbd6007d8069100fb9cb09e960c872Pavel Emelyanov * @off: an offset in bytes to peek skb from. Returns an offset 1393f518bf745cbd6007d8069100fb9cb09e960c872Pavel Emelyanov * within an skb where data actually starts 1404dc3b16ba18c0f967ad100c52fa65b01a4f76ff0Pavel Pisa * @err: error code returned 1411da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds * 1421da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds * Get a datagram skbuff, understands the peeking, nonblocking wakeups 1431da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds * and possible races. This replaces identical code in packet, raw and 1441da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds * udp, as well as the IPX AX.25 and Appletalk. It also finally fixes 1451da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds * the long standing peek and read race for datagram sockets. If you 1461da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds * alter this routine remember it must be re-entrant. 1471da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds * 1481da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds * This function will lock the socket if a skb is returned, so the caller 1491da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds * needs to unlock the socket in that case (usually by calling 1501da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds * skb_free_datagram) 1511da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds * 1521da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds * * It does not lock socket since today. This function is 1531da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds * * free of race conditions. This measure should/can improve 1541da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds * * significantly datagram socket latencies at high loads, 1551da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds * * when data copying to user space takes lots of time. 1561da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds * * (BTW I've just killed the last cli() in IP/IPv6/core/netlink/packet 1571da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds * * 8) Great win.) 1581da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds * * --ANK (980729) 1591da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds * 1601da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds * The order of the tests when we find no data waiting are specified 1611da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds * quite explicitly by POSIX 1003.1g, don't change them without having 1621da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds * the standard around please. 1631da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds */ 16495c961747284a6b83a5e2d81240e214b0fa3464dEric Dumazetstruct sk_buff *__skb_recv_datagram(struct sock *sk, unsigned int flags, 1653f518bf745cbd6007d8069100fb9cb09e960c872Pavel Emelyanov int *peeked, int *off, int *err) 1661da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds{ 16739cc86130bc045d87f525ce7742da308ff757cecBenjamin Poirier struct sk_buff *skb, *last; 1681da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds long timeo; 1691da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds /* 1701da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds * Caller is allowed not to check sk->sk_err before skb_recv_datagram() 1711da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds */ 1721da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds int error = sock_error(sk); 1731da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 1741da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds if (error) 1751da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds goto no_packet; 1761da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 177a59322be07c964e916d15be3df473fb7ba20c41eHerbert Xu timeo = sock_rcvtimeo(sk, flags & MSG_DONTWAIT); 1781da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 1791da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds do { 1801da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds /* Again only user level code calls this function, so nothing 1811da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds * interrupt level will suddenly eat the receive_queue. 1821da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds * 1831da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds * Look at current nfs client by the way... 1848917a3c0b7d1557548f50bfe3f0e18e0354e38f6David Shwatrz * However, this function was correct in any case. 8) 1851da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds */ 186a59322be07c964e916d15be3df473fb7ba20c41eHerbert Xu unsigned long cpu_flags; 1874934b0329f7150dcb5f90506860e2db32274c755Pavel Emelyanov struct sk_buff_head *queue = &sk->sk_receive_queue; 18839cc86130bc045d87f525ce7742da308ff757cecBenjamin Poirier int _off = *off; 189a59322be07c964e916d15be3df473fb7ba20c41eHerbert Xu 19039cc86130bc045d87f525ce7742da308ff757cecBenjamin Poirier last = (struct sk_buff *)queue; 1914934b0329f7150dcb5f90506860e2db32274c755Pavel Emelyanov spin_lock_irqsave(&queue->lock, cpu_flags); 1923f518bf745cbd6007d8069100fb9cb09e960c872Pavel Emelyanov skb_queue_walk(queue, skb) { 19339cc86130bc045d87f525ce7742da308ff757cecBenjamin Poirier last = skb; 194a59322be07c964e916d15be3df473fb7ba20c41eHerbert Xu *peeked = skb->peeked; 195a59322be07c964e916d15be3df473fb7ba20c41eHerbert Xu if (flags & MSG_PEEK) { 19639cc86130bc045d87f525ce7742da308ff757cecBenjamin Poirier if (_off >= skb->len && (skb->len || _off || 197add05ad4e9f5c4efee9b98535db5efa32b0d0492Benjamin Poirier skb->peeked)) { 19839cc86130bc045d87f525ce7742da308ff757cecBenjamin Poirier _off -= skb->len; 1993f518bf745cbd6007d8069100fb9cb09e960c872Pavel Emelyanov continue; 2003f518bf745cbd6007d8069100fb9cb09e960c872Pavel Emelyanov } 201a59322be07c964e916d15be3df473fb7ba20c41eHerbert Xu skb->peeked = 1; 2021da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds atomic_inc(&skb->users); 203a59322be07c964e916d15be3df473fb7ba20c41eHerbert Xu } else 2044934b0329f7150dcb5f90506860e2db32274c755Pavel Emelyanov __skb_unlink(skb, queue); 2051da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 2063f518bf745cbd6007d8069100fb9cb09e960c872Pavel Emelyanov spin_unlock_irqrestore(&queue->lock, cpu_flags); 20739cc86130bc045d87f525ce7742da308ff757cecBenjamin Poirier *off = _off; 2081da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds return skb; 2093f518bf745cbd6007d8069100fb9cb09e960c872Pavel Emelyanov } 2103f518bf745cbd6007d8069100fb9cb09e960c872Pavel Emelyanov spin_unlock_irqrestore(&queue->lock, cpu_flags); 2111da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 212cbf55001b2ddb814329735641be5d29b08c82b08Eliezer Tamir if (sk_can_busy_loop(sk) && 213cbf55001b2ddb814329735641be5d29b08c82b08Eliezer Tamir sk_busy_loop(sk, flags & MSG_DONTWAIT)) 214a5b50476f77a8fcc8055c955720d05a7c2d9c532Eliezer Tamir continue; 215a5b50476f77a8fcc8055c955720d05a7c2d9c532Eliezer Tamir 2161da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds /* User doesn't want to wait */ 2171da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds error = -EAGAIN; 2181da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds if (!timeo) 2191da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds goto no_packet; 2201da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 22139cc86130bc045d87f525ce7742da308ff757cecBenjamin Poirier } while (!wait_for_more_packets(sk, err, &timeo, last)); 2221da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 2231da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds return NULL; 2241da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 2251da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvaldsno_packet: 2261da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds *err = error; 2271da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds return NULL; 2281da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds} 229a59322be07c964e916d15be3df473fb7ba20c41eHerbert XuEXPORT_SYMBOL(__skb_recv_datagram); 230a59322be07c964e916d15be3df473fb7ba20c41eHerbert Xu 23195c961747284a6b83a5e2d81240e214b0fa3464dEric Dumazetstruct sk_buff *skb_recv_datagram(struct sock *sk, unsigned int flags, 232a59322be07c964e916d15be3df473fb7ba20c41eHerbert Xu int noblock, int *err) 233a59322be07c964e916d15be3df473fb7ba20c41eHerbert Xu{ 2343f518bf745cbd6007d8069100fb9cb09e960c872Pavel Emelyanov int peeked, off = 0; 235a59322be07c964e916d15be3df473fb7ba20c41eHerbert Xu 236a59322be07c964e916d15be3df473fb7ba20c41eHerbert Xu return __skb_recv_datagram(sk, flags | (noblock ? MSG_DONTWAIT : 0), 2373f518bf745cbd6007d8069100fb9cb09e960c872Pavel Emelyanov &peeked, &off, err); 238a59322be07c964e916d15be3df473fb7ba20c41eHerbert Xu} 2399e34a5b51684bc90ac827ec4ba339f3892632eacEric DumazetEXPORT_SYMBOL(skb_recv_datagram); 2401da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 2411da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvaldsvoid skb_free_datagram(struct sock *sk, struct sk_buff *skb) 2421da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds{ 243ead2ceb0ec9f85cff19c43b5cdb2f8a054484431Neil Horman consume_skb(skb); 244270acefafeb74ce2fe93d35b75733870bf1e11e7Eric Dumazet sk_mem_reclaim_partial(sk); 2451da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds} 2469d410c796067686b1e032d54ce475b7055537138Eric DumazetEXPORT_SYMBOL(skb_free_datagram); 2479d410c796067686b1e032d54ce475b7055537138Eric Dumazet 2489d410c796067686b1e032d54ce475b7055537138Eric Dumazetvoid skb_free_datagram_locked(struct sock *sk, struct sk_buff *skb) 2499d410c796067686b1e032d54ce475b7055537138Eric Dumazet{ 2508a74ad60a546b13bd1096b2a61a7a5c6fd9ae17cEric Dumazet bool slow; 2518a74ad60a546b13bd1096b2a61a7a5c6fd9ae17cEric Dumazet 25293bb64eac10aad3dae6178d7da94765f207d121fEric Dumazet if (likely(atomic_read(&skb->users) == 1)) 25393bb64eac10aad3dae6178d7da94765f207d121fEric Dumazet smp_rmb(); 25493bb64eac10aad3dae6178d7da94765f207d121fEric Dumazet else if (likely(!atomic_dec_and_test(&skb->users))) 25593bb64eac10aad3dae6178d7da94765f207d121fEric Dumazet return; 25693bb64eac10aad3dae6178d7da94765f207d121fEric Dumazet 2578a74ad60a546b13bd1096b2a61a7a5c6fd9ae17cEric Dumazet slow = lock_sock_fast(sk); 2584b0b72f7dd617b13abd1b04c947e15873e011a24Eric Dumazet skb_orphan(skb); 2594b0b72f7dd617b13abd1b04c947e15873e011a24Eric Dumazet sk_mem_reclaim_partial(sk); 2608a74ad60a546b13bd1096b2a61a7a5c6fd9ae17cEric Dumazet unlock_sock_fast(sk, slow); 2614b0b72f7dd617b13abd1b04c947e15873e011a24Eric Dumazet 26293bb64eac10aad3dae6178d7da94765f207d121fEric Dumazet /* skb is now orphaned, can be freed outside of locked section */ 26393bb64eac10aad3dae6178d7da94765f207d121fEric Dumazet __kfree_skb(skb); 2649d410c796067686b1e032d54ce475b7055537138Eric Dumazet} 2659d410c796067686b1e032d54ce475b7055537138Eric DumazetEXPORT_SYMBOL(skb_free_datagram_locked); 2661da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 2671da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds/** 2683305b80c214c642b89cd5c21af83bc91ec13f8bdHerbert Xu * skb_kill_datagram - Free a datagram skbuff forcibly 2693305b80c214c642b89cd5c21af83bc91ec13f8bdHerbert Xu * @sk: socket 2703305b80c214c642b89cd5c21af83bc91ec13f8bdHerbert Xu * @skb: datagram skbuff 2713305b80c214c642b89cd5c21af83bc91ec13f8bdHerbert Xu * @flags: MSG_ flags 2723305b80c214c642b89cd5c21af83bc91ec13f8bdHerbert Xu * 2733305b80c214c642b89cd5c21af83bc91ec13f8bdHerbert Xu * This function frees a datagram skbuff that was received by 2743305b80c214c642b89cd5c21af83bc91ec13f8bdHerbert Xu * skb_recv_datagram. The flags argument must match the one 2753305b80c214c642b89cd5c21af83bc91ec13f8bdHerbert Xu * used for skb_recv_datagram. 2763305b80c214c642b89cd5c21af83bc91ec13f8bdHerbert Xu * 2773305b80c214c642b89cd5c21af83bc91ec13f8bdHerbert Xu * If the MSG_PEEK flag is set, and the packet is still on the 2783305b80c214c642b89cd5c21af83bc91ec13f8bdHerbert Xu * receive queue of the socket, it will be taken off the queue 2793305b80c214c642b89cd5c21af83bc91ec13f8bdHerbert Xu * before it is freed. 2803305b80c214c642b89cd5c21af83bc91ec13f8bdHerbert Xu * 2813305b80c214c642b89cd5c21af83bc91ec13f8bdHerbert Xu * This function currently only disables BH when acquiring the 2823305b80c214c642b89cd5c21af83bc91ec13f8bdHerbert Xu * sk_receive_queue lock. Therefore it must not be used in a 2833305b80c214c642b89cd5c21af83bc91ec13f8bdHerbert Xu * context where that lock is acquired in an IRQ context. 28427ab2568649d5ba6c5a20212079b7c4f6da4ca0dHerbert Xu * 28527ab2568649d5ba6c5a20212079b7c4f6da4ca0dHerbert Xu * It returns 0 if the packet was removed by us. 2863305b80c214c642b89cd5c21af83bc91ec13f8bdHerbert Xu */ 2873305b80c214c642b89cd5c21af83bc91ec13f8bdHerbert Xu 28827ab2568649d5ba6c5a20212079b7c4f6da4ca0dHerbert Xuint skb_kill_datagram(struct sock *sk, struct sk_buff *skb, unsigned int flags) 2893305b80c214c642b89cd5c21af83bc91ec13f8bdHerbert Xu{ 29027ab2568649d5ba6c5a20212079b7c4f6da4ca0dHerbert Xu int err = 0; 29127ab2568649d5ba6c5a20212079b7c4f6da4ca0dHerbert Xu 2923305b80c214c642b89cd5c21af83bc91ec13f8bdHerbert Xu if (flags & MSG_PEEK) { 29327ab2568649d5ba6c5a20212079b7c4f6da4ca0dHerbert Xu err = -ENOENT; 2943305b80c214c642b89cd5c21af83bc91ec13f8bdHerbert Xu spin_lock_bh(&sk->sk_receive_queue.lock); 2953305b80c214c642b89cd5c21af83bc91ec13f8bdHerbert Xu if (skb == skb_peek(&sk->sk_receive_queue)) { 2963305b80c214c642b89cd5c21af83bc91ec13f8bdHerbert Xu __skb_unlink(skb, &sk->sk_receive_queue); 2973305b80c214c642b89cd5c21af83bc91ec13f8bdHerbert Xu atomic_dec(&skb->users); 29827ab2568649d5ba6c5a20212079b7c4f6da4ca0dHerbert Xu err = 0; 2993305b80c214c642b89cd5c21af83bc91ec13f8bdHerbert Xu } 3003305b80c214c642b89cd5c21af83bc91ec13f8bdHerbert Xu spin_unlock_bh(&sk->sk_receive_queue.lock); 3013305b80c214c642b89cd5c21af83bc91ec13f8bdHerbert Xu } 3023305b80c214c642b89cd5c21af83bc91ec13f8bdHerbert Xu 30361de71c67caec39df0a854a1ef5be0c6be385c2aJohn Dykstra kfree_skb(skb); 3048edf19c2fe028563fc6ea9cb1995b8ee4172d4b6Eric Dumazet atomic_inc(&sk->sk_drops); 30561de71c67caec39df0a854a1ef5be0c6be385c2aJohn Dykstra sk_mem_reclaim_partial(sk); 30661de71c67caec39df0a854a1ef5be0c6be385c2aJohn Dykstra 30727ab2568649d5ba6c5a20212079b7c4f6da4ca0dHerbert Xu return err; 3083305b80c214c642b89cd5c21af83bc91ec13f8bdHerbert Xu} 3093305b80c214c642b89cd5c21af83bc91ec13f8bdHerbert XuEXPORT_SYMBOL(skb_kill_datagram); 3103305b80c214c642b89cd5c21af83bc91ec13f8bdHerbert Xu 3113305b80c214c642b89cd5c21af83bc91ec13f8bdHerbert Xu/** 3121da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds * skb_copy_datagram_iovec - Copy a datagram to an iovec. 3134dc3b16ba18c0f967ad100c52fa65b01a4f76ff0Pavel Pisa * @skb: buffer to copy 3144dc3b16ba18c0f967ad100c52fa65b01a4f76ff0Pavel Pisa * @offset: offset in the buffer to start copying from 31567be2dd1bace0ec7ce2dbc1bba3f8df3d7be597eMartin Waitz * @to: io vector to copy to 3164dc3b16ba18c0f967ad100c52fa65b01a4f76ff0Pavel Pisa * @len: amount of data to copy from buffer to iovec 3171da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds * 3181da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds * Note: the iovec is modified during the copy. 3191da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds */ 3201da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvaldsint skb_copy_datagram_iovec(const struct sk_buff *skb, int offset, 3211da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds struct iovec *to, int len) 3221da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds{ 3231a028e50729b85d0a038fad13daf0ee201a37454David S. Miller int start = skb_headlen(skb); 3241a028e50729b85d0a038fad13daf0ee201a37454David S. Miller int i, copy = start - offset; 3255b1a002ade68173f21b2126a778278df72202ba6David S. Miller struct sk_buff *frag_iter; 326c75d721c761ad0f2d8725c40af9e4f376efefd24Herbert Xu 327e9b3cc1b3779fe10a80de4c3e7404bd308d0eae3Neil Horman trace_skb_copy_datagram_iovec(skb, len); 328e9b3cc1b3779fe10a80de4c3e7404bd308d0eae3Neil Horman 329b4d9eda028e8becbb5057b554e63eea12e496a88David S. Miller /* Copy header. */ 330b4d9eda028e8becbb5057b554e63eea12e496a88David S. Miller if (copy > 0) { 331b4d9eda028e8becbb5057b554e63eea12e496a88David S. Miller if (copy > len) 332b4d9eda028e8becbb5057b554e63eea12e496a88David S. Miller copy = len; 333b4d9eda028e8becbb5057b554e63eea12e496a88David S. Miller if (memcpy_toiovec(to, skb->data + offset, copy)) 334b4d9eda028e8becbb5057b554e63eea12e496a88David S. Miller goto fault; 335b4d9eda028e8becbb5057b554e63eea12e496a88David S. Miller if ((len -= copy) == 0) 336b4d9eda028e8becbb5057b554e63eea12e496a88David S. Miller return 0; 337b4d9eda028e8becbb5057b554e63eea12e496a88David S. Miller offset += copy; 338b4d9eda028e8becbb5057b554e63eea12e496a88David S. Miller } 339c75d721c761ad0f2d8725c40af9e4f376efefd24Herbert Xu 340b4d9eda028e8becbb5057b554e63eea12e496a88David S. Miller /* Copy paged appendix. Hmm... why does this look so complicated? */ 341b4d9eda028e8becbb5057b554e63eea12e496a88David S. Miller for (i = 0; i < skb_shinfo(skb)->nr_frags; i++) { 3421a028e50729b85d0a038fad13daf0ee201a37454David S. Miller int end; 3439e903e085262ffbf1fc44a17ac06058aca03524aEric Dumazet const skb_frag_t *frag = &skb_shinfo(skb)->frags[i]; 3441da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 345547b792cac0a038b9dbf958d3c120df3740b5572Ilpo Järvinen WARN_ON(start > offset + len); 3461a028e50729b85d0a038fad13daf0ee201a37454David S. Miller 3479e903e085262ffbf1fc44a17ac06058aca03524aEric Dumazet end = start + skb_frag_size(frag); 348b4d9eda028e8becbb5057b554e63eea12e496a88David S. Miller if ((copy = end - offset) > 0) { 349b4d9eda028e8becbb5057b554e63eea12e496a88David S. Miller int err; 350b4d9eda028e8becbb5057b554e63eea12e496a88David S. Miller u8 *vaddr; 351ea2ab69379a941c6f8884e290fdd28c93936a778Ian Campbell struct page *page = skb_frag_page(frag); 3521da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 3531da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds if (copy > len) 3541da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds copy = len; 355b4d9eda028e8becbb5057b554e63eea12e496a88David S. Miller vaddr = kmap(page); 3561a028e50729b85d0a038fad13daf0ee201a37454David S. Miller err = memcpy_toiovec(to, vaddr + frag->page_offset + 3571a028e50729b85d0a038fad13daf0ee201a37454David S. Miller offset - start, copy); 358b4d9eda028e8becbb5057b554e63eea12e496a88David S. Miller kunmap(page); 3591da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds if (err) 3601da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds goto fault; 3611da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds if (!(len -= copy)) 3621da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds return 0; 3631da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds offset += copy; 3641da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds } 3651a028e50729b85d0a038fad13daf0ee201a37454David S. Miller start = end; 3661da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds } 367b4d9eda028e8becbb5057b554e63eea12e496a88David S. Miller 3685b1a002ade68173f21b2126a778278df72202ba6David S. Miller skb_walk_frags(skb, frag_iter) { 3695b1a002ade68173f21b2126a778278df72202ba6David S. Miller int end; 3705b1a002ade68173f21b2126a778278df72202ba6David S. Miller 3715b1a002ade68173f21b2126a778278df72202ba6David S. Miller WARN_ON(start > offset + len); 3725b1a002ade68173f21b2126a778278df72202ba6David S. Miller 3735b1a002ade68173f21b2126a778278df72202ba6David S. Miller end = start + frag_iter->len; 3745b1a002ade68173f21b2126a778278df72202ba6David S. Miller if ((copy = end - offset) > 0) { 3755b1a002ade68173f21b2126a778278df72202ba6David S. Miller if (copy > len) 3765b1a002ade68173f21b2126a778278df72202ba6David S. Miller copy = len; 3775b1a002ade68173f21b2126a778278df72202ba6David S. Miller if (skb_copy_datagram_iovec(frag_iter, 3785b1a002ade68173f21b2126a778278df72202ba6David S. Miller offset - start, 3795b1a002ade68173f21b2126a778278df72202ba6David S. Miller to, copy)) 3805b1a002ade68173f21b2126a778278df72202ba6David S. Miller goto fault; 3815b1a002ade68173f21b2126a778278df72202ba6David S. Miller if ((len -= copy) == 0) 3825b1a002ade68173f21b2126a778278df72202ba6David S. Miller return 0; 3835b1a002ade68173f21b2126a778278df72202ba6David S. Miller offset += copy; 384b4d9eda028e8becbb5057b554e63eea12e496a88David S. Miller } 3855b1a002ade68173f21b2126a778278df72202ba6David S. Miller start = end; 3861da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds } 387b4d9eda028e8becbb5057b554e63eea12e496a88David S. Miller if (!len) 388b4d9eda028e8becbb5057b554e63eea12e496a88David S. Miller return 0; 389b4d9eda028e8becbb5057b554e63eea12e496a88David S. Miller 3901da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvaldsfault: 3911da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds return -EFAULT; 3921da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds} 3939e34a5b51684bc90ac827ec4ba339f3892632eacEric DumazetEXPORT_SYMBOL(skb_copy_datagram_iovec); 3941da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 395db543c1f973cd1d557cc32ceee76737c1e4d2898Rusty Russell/** 3960a1ec07a67bd8b0033dace237249654d015efa21Michael S. Tsirkin * skb_copy_datagram_const_iovec - Copy a datagram to an iovec. 3970a1ec07a67bd8b0033dace237249654d015efa21Michael S. Tsirkin * @skb: buffer to copy 3980a1ec07a67bd8b0033dace237249654d015efa21Michael S. Tsirkin * @offset: offset in the buffer to start copying from 3990a1ec07a67bd8b0033dace237249654d015efa21Michael S. Tsirkin * @to: io vector to copy to 4000a1ec07a67bd8b0033dace237249654d015efa21Michael S. Tsirkin * @to_offset: offset in the io vector to start copying to 4010a1ec07a67bd8b0033dace237249654d015efa21Michael S. Tsirkin * @len: amount of data to copy from buffer to iovec 4020a1ec07a67bd8b0033dace237249654d015efa21Michael S. Tsirkin * 4030a1ec07a67bd8b0033dace237249654d015efa21Michael S. Tsirkin * Returns 0 or -EFAULT. 4040a1ec07a67bd8b0033dace237249654d015efa21Michael S. Tsirkin * Note: the iovec is not modified during the copy. 4050a1ec07a67bd8b0033dace237249654d015efa21Michael S. Tsirkin */ 4060a1ec07a67bd8b0033dace237249654d015efa21Michael S. Tsirkinint skb_copy_datagram_const_iovec(const struct sk_buff *skb, int offset, 4070a1ec07a67bd8b0033dace237249654d015efa21Michael S. Tsirkin const struct iovec *to, int to_offset, 4080a1ec07a67bd8b0033dace237249654d015efa21Michael S. Tsirkin int len) 4090a1ec07a67bd8b0033dace237249654d015efa21Michael S. Tsirkin{ 4100a1ec07a67bd8b0033dace237249654d015efa21Michael S. Tsirkin int start = skb_headlen(skb); 4110a1ec07a67bd8b0033dace237249654d015efa21Michael S. Tsirkin int i, copy = start - offset; 4125b1a002ade68173f21b2126a778278df72202ba6David S. Miller struct sk_buff *frag_iter; 4130a1ec07a67bd8b0033dace237249654d015efa21Michael S. Tsirkin 4140a1ec07a67bd8b0033dace237249654d015efa21Michael S. Tsirkin /* Copy header. */ 4150a1ec07a67bd8b0033dace237249654d015efa21Michael S. Tsirkin if (copy > 0) { 4160a1ec07a67bd8b0033dace237249654d015efa21Michael S. Tsirkin if (copy > len) 4170a1ec07a67bd8b0033dace237249654d015efa21Michael S. Tsirkin copy = len; 4180a1ec07a67bd8b0033dace237249654d015efa21Michael S. Tsirkin if (memcpy_toiovecend(to, skb->data + offset, to_offset, copy)) 4190a1ec07a67bd8b0033dace237249654d015efa21Michael S. Tsirkin goto fault; 4200a1ec07a67bd8b0033dace237249654d015efa21Michael S. Tsirkin if ((len -= copy) == 0) 4210a1ec07a67bd8b0033dace237249654d015efa21Michael S. Tsirkin return 0; 4220a1ec07a67bd8b0033dace237249654d015efa21Michael S. Tsirkin offset += copy; 4230a1ec07a67bd8b0033dace237249654d015efa21Michael S. Tsirkin to_offset += copy; 4240a1ec07a67bd8b0033dace237249654d015efa21Michael S. Tsirkin } 4250a1ec07a67bd8b0033dace237249654d015efa21Michael S. Tsirkin 4260a1ec07a67bd8b0033dace237249654d015efa21Michael S. Tsirkin /* Copy paged appendix. Hmm... why does this look so complicated? */ 4270a1ec07a67bd8b0033dace237249654d015efa21Michael S. Tsirkin for (i = 0; i < skb_shinfo(skb)->nr_frags; i++) { 4280a1ec07a67bd8b0033dace237249654d015efa21Michael S. Tsirkin int end; 4299e903e085262ffbf1fc44a17ac06058aca03524aEric Dumazet const skb_frag_t *frag = &skb_shinfo(skb)->frags[i]; 4300a1ec07a67bd8b0033dace237249654d015efa21Michael S. Tsirkin 4310a1ec07a67bd8b0033dace237249654d015efa21Michael S. Tsirkin WARN_ON(start > offset + len); 4320a1ec07a67bd8b0033dace237249654d015efa21Michael S. Tsirkin 4339e903e085262ffbf1fc44a17ac06058aca03524aEric Dumazet end = start + skb_frag_size(frag); 4340a1ec07a67bd8b0033dace237249654d015efa21Michael S. Tsirkin if ((copy = end - offset) > 0) { 4350a1ec07a67bd8b0033dace237249654d015efa21Michael S. Tsirkin int err; 4360a1ec07a67bd8b0033dace237249654d015efa21Michael S. Tsirkin u8 *vaddr; 437ea2ab69379a941c6f8884e290fdd28c93936a778Ian Campbell struct page *page = skb_frag_page(frag); 4380a1ec07a67bd8b0033dace237249654d015efa21Michael S. Tsirkin 4390a1ec07a67bd8b0033dace237249654d015efa21Michael S. Tsirkin if (copy > len) 4400a1ec07a67bd8b0033dace237249654d015efa21Michael S. Tsirkin copy = len; 4410a1ec07a67bd8b0033dace237249654d015efa21Michael S. Tsirkin vaddr = kmap(page); 4420a1ec07a67bd8b0033dace237249654d015efa21Michael S. Tsirkin err = memcpy_toiovecend(to, vaddr + frag->page_offset + 4430a1ec07a67bd8b0033dace237249654d015efa21Michael S. Tsirkin offset - start, to_offset, copy); 4440a1ec07a67bd8b0033dace237249654d015efa21Michael S. Tsirkin kunmap(page); 4450a1ec07a67bd8b0033dace237249654d015efa21Michael S. Tsirkin if (err) 4460a1ec07a67bd8b0033dace237249654d015efa21Michael S. Tsirkin goto fault; 4470a1ec07a67bd8b0033dace237249654d015efa21Michael S. Tsirkin if (!(len -= copy)) 4480a1ec07a67bd8b0033dace237249654d015efa21Michael S. Tsirkin return 0; 4490a1ec07a67bd8b0033dace237249654d015efa21Michael S. Tsirkin offset += copy; 4500a1ec07a67bd8b0033dace237249654d015efa21Michael S. Tsirkin to_offset += copy; 4510a1ec07a67bd8b0033dace237249654d015efa21Michael S. Tsirkin } 4520a1ec07a67bd8b0033dace237249654d015efa21Michael S. Tsirkin start = end; 4530a1ec07a67bd8b0033dace237249654d015efa21Michael S. Tsirkin } 4540a1ec07a67bd8b0033dace237249654d015efa21Michael S. Tsirkin 4555b1a002ade68173f21b2126a778278df72202ba6David S. Miller skb_walk_frags(skb, frag_iter) { 4565b1a002ade68173f21b2126a778278df72202ba6David S. Miller int end; 4575b1a002ade68173f21b2126a778278df72202ba6David S. Miller 4585b1a002ade68173f21b2126a778278df72202ba6David S. Miller WARN_ON(start > offset + len); 4595b1a002ade68173f21b2126a778278df72202ba6David S. Miller 4605b1a002ade68173f21b2126a778278df72202ba6David S. Miller end = start + frag_iter->len; 4615b1a002ade68173f21b2126a778278df72202ba6David S. Miller if ((copy = end - offset) > 0) { 4625b1a002ade68173f21b2126a778278df72202ba6David S. Miller if (copy > len) 4635b1a002ade68173f21b2126a778278df72202ba6David S. Miller copy = len; 4645b1a002ade68173f21b2126a778278df72202ba6David S. Miller if (skb_copy_datagram_const_iovec(frag_iter, 4655b1a002ade68173f21b2126a778278df72202ba6David S. Miller offset - start, 4665b1a002ade68173f21b2126a778278df72202ba6David S. Miller to, to_offset, 4675b1a002ade68173f21b2126a778278df72202ba6David S. Miller copy)) 4685b1a002ade68173f21b2126a778278df72202ba6David S. Miller goto fault; 4695b1a002ade68173f21b2126a778278df72202ba6David S. Miller if ((len -= copy) == 0) 4705b1a002ade68173f21b2126a778278df72202ba6David S. Miller return 0; 4715b1a002ade68173f21b2126a778278df72202ba6David S. Miller offset += copy; 4725b1a002ade68173f21b2126a778278df72202ba6David S. Miller to_offset += copy; 4730a1ec07a67bd8b0033dace237249654d015efa21Michael S. Tsirkin } 4745b1a002ade68173f21b2126a778278df72202ba6David S. Miller start = end; 4750a1ec07a67bd8b0033dace237249654d015efa21Michael S. Tsirkin } 4760a1ec07a67bd8b0033dace237249654d015efa21Michael S. Tsirkin if (!len) 4770a1ec07a67bd8b0033dace237249654d015efa21Michael S. Tsirkin return 0; 4780a1ec07a67bd8b0033dace237249654d015efa21Michael S. Tsirkin 4790a1ec07a67bd8b0033dace237249654d015efa21Michael S. Tsirkinfault: 4800a1ec07a67bd8b0033dace237249654d015efa21Michael S. Tsirkin return -EFAULT; 4810a1ec07a67bd8b0033dace237249654d015efa21Michael S. Tsirkin} 4820a1ec07a67bd8b0033dace237249654d015efa21Michael S. TsirkinEXPORT_SYMBOL(skb_copy_datagram_const_iovec); 4830a1ec07a67bd8b0033dace237249654d015efa21Michael S. Tsirkin 4840a1ec07a67bd8b0033dace237249654d015efa21Michael S. Tsirkin/** 485db543c1f973cd1d557cc32ceee76737c1e4d2898Rusty Russell * skb_copy_datagram_from_iovec - Copy a datagram from an iovec. 486db543c1f973cd1d557cc32ceee76737c1e4d2898Rusty Russell * @skb: buffer to copy 487db543c1f973cd1d557cc32ceee76737c1e4d2898Rusty Russell * @offset: offset in the buffer to start copying to 488db543c1f973cd1d557cc32ceee76737c1e4d2898Rusty Russell * @from: io vector to copy to 4896f26c9a7555e5bcca3560919db9b852015077daeMichael S. Tsirkin * @from_offset: offset in the io vector to start copying from 490db543c1f973cd1d557cc32ceee76737c1e4d2898Rusty Russell * @len: amount of data to copy to buffer from iovec 491db543c1f973cd1d557cc32ceee76737c1e4d2898Rusty Russell * 492db543c1f973cd1d557cc32ceee76737c1e4d2898Rusty Russell * Returns 0 or -EFAULT. 4936f26c9a7555e5bcca3560919db9b852015077daeMichael S. Tsirkin * Note: the iovec is not modified during the copy. 494db543c1f973cd1d557cc32ceee76737c1e4d2898Rusty Russell */ 495db543c1f973cd1d557cc32ceee76737c1e4d2898Rusty Russellint skb_copy_datagram_from_iovec(struct sk_buff *skb, int offset, 4966f26c9a7555e5bcca3560919db9b852015077daeMichael S. Tsirkin const struct iovec *from, int from_offset, 4976f26c9a7555e5bcca3560919db9b852015077daeMichael S. Tsirkin int len) 498db543c1f973cd1d557cc32ceee76737c1e4d2898Rusty Russell{ 499db543c1f973cd1d557cc32ceee76737c1e4d2898Rusty Russell int start = skb_headlen(skb); 500db543c1f973cd1d557cc32ceee76737c1e4d2898Rusty Russell int i, copy = start - offset; 5015b1a002ade68173f21b2126a778278df72202ba6David S. Miller struct sk_buff *frag_iter; 502db543c1f973cd1d557cc32ceee76737c1e4d2898Rusty Russell 503db543c1f973cd1d557cc32ceee76737c1e4d2898Rusty Russell /* Copy header. */ 504db543c1f973cd1d557cc32ceee76737c1e4d2898Rusty Russell if (copy > 0) { 505db543c1f973cd1d557cc32ceee76737c1e4d2898Rusty Russell if (copy > len) 506db543c1f973cd1d557cc32ceee76737c1e4d2898Rusty Russell copy = len; 507d2d27bfd11659675fdd1c20b1c7f8f59873cad60Sridhar Samudrala if (memcpy_fromiovecend(skb->data + offset, from, from_offset, 508d2d27bfd11659675fdd1c20b1c7f8f59873cad60Sridhar Samudrala copy)) 509db543c1f973cd1d557cc32ceee76737c1e4d2898Rusty Russell goto fault; 510db543c1f973cd1d557cc32ceee76737c1e4d2898Rusty Russell if ((len -= copy) == 0) 511db543c1f973cd1d557cc32ceee76737c1e4d2898Rusty Russell return 0; 512db543c1f973cd1d557cc32ceee76737c1e4d2898Rusty Russell offset += copy; 5136f26c9a7555e5bcca3560919db9b852015077daeMichael S. Tsirkin from_offset += copy; 514db543c1f973cd1d557cc32ceee76737c1e4d2898Rusty Russell } 515db543c1f973cd1d557cc32ceee76737c1e4d2898Rusty Russell 516db543c1f973cd1d557cc32ceee76737c1e4d2898Rusty Russell /* Copy paged appendix. Hmm... why does this look so complicated? */ 517db543c1f973cd1d557cc32ceee76737c1e4d2898Rusty Russell for (i = 0; i < skb_shinfo(skb)->nr_frags; i++) { 518db543c1f973cd1d557cc32ceee76737c1e4d2898Rusty Russell int end; 5199e903e085262ffbf1fc44a17ac06058aca03524aEric Dumazet const skb_frag_t *frag = &skb_shinfo(skb)->frags[i]; 520db543c1f973cd1d557cc32ceee76737c1e4d2898Rusty Russell 521db543c1f973cd1d557cc32ceee76737c1e4d2898Rusty Russell WARN_ON(start > offset + len); 522db543c1f973cd1d557cc32ceee76737c1e4d2898Rusty Russell 5239e903e085262ffbf1fc44a17ac06058aca03524aEric Dumazet end = start + skb_frag_size(frag); 524db543c1f973cd1d557cc32ceee76737c1e4d2898Rusty Russell if ((copy = end - offset) > 0) { 525db543c1f973cd1d557cc32ceee76737c1e4d2898Rusty Russell int err; 526db543c1f973cd1d557cc32ceee76737c1e4d2898Rusty Russell u8 *vaddr; 527ea2ab69379a941c6f8884e290fdd28c93936a778Ian Campbell struct page *page = skb_frag_page(frag); 528db543c1f973cd1d557cc32ceee76737c1e4d2898Rusty Russell 529db543c1f973cd1d557cc32ceee76737c1e4d2898Rusty Russell if (copy > len) 530db543c1f973cd1d557cc32ceee76737c1e4d2898Rusty Russell copy = len; 531db543c1f973cd1d557cc32ceee76737c1e4d2898Rusty Russell vaddr = kmap(page); 5326f26c9a7555e5bcca3560919db9b852015077daeMichael S. Tsirkin err = memcpy_fromiovecend(vaddr + frag->page_offset + 5336f26c9a7555e5bcca3560919db9b852015077daeMichael S. Tsirkin offset - start, 5346f26c9a7555e5bcca3560919db9b852015077daeMichael S. Tsirkin from, from_offset, copy); 535db543c1f973cd1d557cc32ceee76737c1e4d2898Rusty Russell kunmap(page); 536db543c1f973cd1d557cc32ceee76737c1e4d2898Rusty Russell if (err) 537db543c1f973cd1d557cc32ceee76737c1e4d2898Rusty Russell goto fault; 538db543c1f973cd1d557cc32ceee76737c1e4d2898Rusty Russell 539db543c1f973cd1d557cc32ceee76737c1e4d2898Rusty Russell if (!(len -= copy)) 540db543c1f973cd1d557cc32ceee76737c1e4d2898Rusty Russell return 0; 541db543c1f973cd1d557cc32ceee76737c1e4d2898Rusty Russell offset += copy; 5426f26c9a7555e5bcca3560919db9b852015077daeMichael S. Tsirkin from_offset += copy; 543db543c1f973cd1d557cc32ceee76737c1e4d2898Rusty Russell } 544db543c1f973cd1d557cc32ceee76737c1e4d2898Rusty Russell start = end; 545db543c1f973cd1d557cc32ceee76737c1e4d2898Rusty Russell } 546db543c1f973cd1d557cc32ceee76737c1e4d2898Rusty Russell 5475b1a002ade68173f21b2126a778278df72202ba6David S. Miller skb_walk_frags(skb, frag_iter) { 5485b1a002ade68173f21b2126a778278df72202ba6David S. Miller int end; 5495b1a002ade68173f21b2126a778278df72202ba6David S. Miller 5505b1a002ade68173f21b2126a778278df72202ba6David S. Miller WARN_ON(start > offset + len); 5515b1a002ade68173f21b2126a778278df72202ba6David S. Miller 5525b1a002ade68173f21b2126a778278df72202ba6David S. Miller end = start + frag_iter->len; 5535b1a002ade68173f21b2126a778278df72202ba6David S. Miller if ((copy = end - offset) > 0) { 5545b1a002ade68173f21b2126a778278df72202ba6David S. Miller if (copy > len) 5555b1a002ade68173f21b2126a778278df72202ba6David S. Miller copy = len; 5565b1a002ade68173f21b2126a778278df72202ba6David S. Miller if (skb_copy_datagram_from_iovec(frag_iter, 5575b1a002ade68173f21b2126a778278df72202ba6David S. Miller offset - start, 5585b1a002ade68173f21b2126a778278df72202ba6David S. Miller from, 5595b1a002ade68173f21b2126a778278df72202ba6David S. Miller from_offset, 5605b1a002ade68173f21b2126a778278df72202ba6David S. Miller copy)) 5615b1a002ade68173f21b2126a778278df72202ba6David S. Miller goto fault; 5625b1a002ade68173f21b2126a778278df72202ba6David S. Miller if ((len -= copy) == 0) 5635b1a002ade68173f21b2126a778278df72202ba6David S. Miller return 0; 5645b1a002ade68173f21b2126a778278df72202ba6David S. Miller offset += copy; 5655b1a002ade68173f21b2126a778278df72202ba6David S. Miller from_offset += copy; 566db543c1f973cd1d557cc32ceee76737c1e4d2898Rusty Russell } 5675b1a002ade68173f21b2126a778278df72202ba6David S. Miller start = end; 568db543c1f973cd1d557cc32ceee76737c1e4d2898Rusty Russell } 569db543c1f973cd1d557cc32ceee76737c1e4d2898Rusty Russell if (!len) 570db543c1f973cd1d557cc32ceee76737c1e4d2898Rusty Russell return 0; 571db543c1f973cd1d557cc32ceee76737c1e4d2898Rusty Russell 572db543c1f973cd1d557cc32ceee76737c1e4d2898Rusty Russellfault: 573db543c1f973cd1d557cc32ceee76737c1e4d2898Rusty Russell return -EFAULT; 574db543c1f973cd1d557cc32ceee76737c1e4d2898Rusty Russell} 575db543c1f973cd1d557cc32ceee76737c1e4d2898Rusty RussellEXPORT_SYMBOL(skb_copy_datagram_from_iovec); 576db543c1f973cd1d557cc32ceee76737c1e4d2898Rusty Russell 577c3bdeb5c7cc073ccf5ff9624642022a8613a956eJason Wang/** 578c3bdeb5c7cc073ccf5ff9624642022a8613a956eJason Wang * zerocopy_sg_from_iovec - Build a zerocopy datagram from an iovec 579c3bdeb5c7cc073ccf5ff9624642022a8613a956eJason Wang * @skb: buffer to copy 580c4e819d16c0f46fbdd3706adbd990b3b292d726cZhi Yong Wu * @from: io vector to copy from 581c3bdeb5c7cc073ccf5ff9624642022a8613a956eJason Wang * @offset: offset in the io vector to start copying from 582c3bdeb5c7cc073ccf5ff9624642022a8613a956eJason Wang * @count: amount of vectors to copy to buffer from 583c3bdeb5c7cc073ccf5ff9624642022a8613a956eJason Wang * 584c3bdeb5c7cc073ccf5ff9624642022a8613a956eJason Wang * The function will first copy up to headlen, and then pin the userspace 585c3bdeb5c7cc073ccf5ff9624642022a8613a956eJason Wang * pages and build frags through them. 586c3bdeb5c7cc073ccf5ff9624642022a8613a956eJason Wang * 587c3bdeb5c7cc073ccf5ff9624642022a8613a956eJason Wang * Returns 0, -EFAULT or -EMSGSIZE. 588c3bdeb5c7cc073ccf5ff9624642022a8613a956eJason Wang * Note: the iovec is not modified during the copy 589c3bdeb5c7cc073ccf5ff9624642022a8613a956eJason Wang */ 590c3bdeb5c7cc073ccf5ff9624642022a8613a956eJason Wangint zerocopy_sg_from_iovec(struct sk_buff *skb, const struct iovec *from, 591c3bdeb5c7cc073ccf5ff9624642022a8613a956eJason Wang int offset, size_t count) 592c3bdeb5c7cc073ccf5ff9624642022a8613a956eJason Wang{ 593c3bdeb5c7cc073ccf5ff9624642022a8613a956eJason Wang int len = iov_length(from, count) - offset; 5943d9953a2ef2182d56e268742259b11dedb8e281dJason Wang int copy = min_t(int, skb_headlen(skb), len); 5953d9953a2ef2182d56e268742259b11dedb8e281dJason Wang int size; 596c3bdeb5c7cc073ccf5ff9624642022a8613a956eJason Wang int i = 0; 597c3bdeb5c7cc073ccf5ff9624642022a8613a956eJason Wang 598c3bdeb5c7cc073ccf5ff9624642022a8613a956eJason Wang /* copy up to skb headlen */ 5993d9953a2ef2182d56e268742259b11dedb8e281dJason Wang if (skb_copy_datagram_from_iovec(skb, 0, from, offset, copy)) 6003d9953a2ef2182d56e268742259b11dedb8e281dJason Wang return -EFAULT; 601c3bdeb5c7cc073ccf5ff9624642022a8613a956eJason Wang 6023d9953a2ef2182d56e268742259b11dedb8e281dJason Wang if (len == copy) 603c3bdeb5c7cc073ccf5ff9624642022a8613a956eJason Wang return 0; 604c3bdeb5c7cc073ccf5ff9624642022a8613a956eJason Wang 6053d9953a2ef2182d56e268742259b11dedb8e281dJason Wang offset += copy; 606c3bdeb5c7cc073ccf5ff9624642022a8613a956eJason Wang while (count--) { 607c3bdeb5c7cc073ccf5ff9624642022a8613a956eJason Wang struct page *page[MAX_SKB_FRAGS]; 608c3bdeb5c7cc073ccf5ff9624642022a8613a956eJason Wang int num_pages; 609c3bdeb5c7cc073ccf5ff9624642022a8613a956eJason Wang unsigned long base; 610c3bdeb5c7cc073ccf5ff9624642022a8613a956eJason Wang unsigned long truesize; 611c3bdeb5c7cc073ccf5ff9624642022a8613a956eJason Wang 6123d9953a2ef2182d56e268742259b11dedb8e281dJason Wang /* Skip over from offset and copied */ 6133d9953a2ef2182d56e268742259b11dedb8e281dJason Wang if (offset >= from->iov_len) { 6143d9953a2ef2182d56e268742259b11dedb8e281dJason Wang offset -= from->iov_len; 615c3bdeb5c7cc073ccf5ff9624642022a8613a956eJason Wang ++from; 616c3bdeb5c7cc073ccf5ff9624642022a8613a956eJason Wang continue; 617c3bdeb5c7cc073ccf5ff9624642022a8613a956eJason Wang } 6183d9953a2ef2182d56e268742259b11dedb8e281dJason Wang len = from->iov_len - offset; 619c3bdeb5c7cc073ccf5ff9624642022a8613a956eJason Wang base = (unsigned long)from->iov_base + offset; 620c3bdeb5c7cc073ccf5ff9624642022a8613a956eJason Wang size = ((base & ~PAGE_MASK) + len + ~PAGE_MASK) >> PAGE_SHIFT; 621c3bdeb5c7cc073ccf5ff9624642022a8613a956eJason Wang if (i + size > MAX_SKB_FRAGS) 622c3bdeb5c7cc073ccf5ff9624642022a8613a956eJason Wang return -EMSGSIZE; 623c3bdeb5c7cc073ccf5ff9624642022a8613a956eJason Wang num_pages = get_user_pages_fast(base, size, 0, &page[i]); 624c3bdeb5c7cc073ccf5ff9624642022a8613a956eJason Wang if (num_pages != size) { 6250433547aa7443cefc89d9b533169bdc50f1206b8Jason Wang release_pages(&page[i], num_pages, 0); 626c3bdeb5c7cc073ccf5ff9624642022a8613a956eJason Wang return -EFAULT; 627c3bdeb5c7cc073ccf5ff9624642022a8613a956eJason Wang } 628c3bdeb5c7cc073ccf5ff9624642022a8613a956eJason Wang truesize = size * PAGE_SIZE; 629c3bdeb5c7cc073ccf5ff9624642022a8613a956eJason Wang skb->data_len += len; 630c3bdeb5c7cc073ccf5ff9624642022a8613a956eJason Wang skb->len += len; 631c3bdeb5c7cc073ccf5ff9624642022a8613a956eJason Wang skb->truesize += truesize; 632c3bdeb5c7cc073ccf5ff9624642022a8613a956eJason Wang atomic_add(truesize, &skb->sk->sk_wmem_alloc); 633c3bdeb5c7cc073ccf5ff9624642022a8613a956eJason Wang while (len) { 634c3bdeb5c7cc073ccf5ff9624642022a8613a956eJason Wang int off = base & ~PAGE_MASK; 635c3bdeb5c7cc073ccf5ff9624642022a8613a956eJason Wang int size = min_t(int, len, PAGE_SIZE - off); 6360a57ec62dfa4b639d86aba60c236904def1d0356Jason Wang skb_fill_page_desc(skb, i, page[i], off, size); 637c3bdeb5c7cc073ccf5ff9624642022a8613a956eJason Wang base += size; 638c3bdeb5c7cc073ccf5ff9624642022a8613a956eJason Wang len -= size; 639c3bdeb5c7cc073ccf5ff9624642022a8613a956eJason Wang i++; 640c3bdeb5c7cc073ccf5ff9624642022a8613a956eJason Wang } 641c3bdeb5c7cc073ccf5ff9624642022a8613a956eJason Wang offset = 0; 642c3bdeb5c7cc073ccf5ff9624642022a8613a956eJason Wang ++from; 643c3bdeb5c7cc073ccf5ff9624642022a8613a956eJason Wang } 644c3bdeb5c7cc073ccf5ff9624642022a8613a956eJason Wang return 0; 645c3bdeb5c7cc073ccf5ff9624642022a8613a956eJason Wang} 646c3bdeb5c7cc073ccf5ff9624642022a8613a956eJason WangEXPORT_SYMBOL(zerocopy_sg_from_iovec); 647c3bdeb5c7cc073ccf5ff9624642022a8613a956eJason Wang 6481da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvaldsstatic int skb_copy_and_csum_datagram(const struct sk_buff *skb, int offset, 6491da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds u8 __user *to, int len, 6505084205faf45384fff25c4cf77dd5c96279283adAl Viro __wsum *csump) 6511da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds{ 6521a028e50729b85d0a038fad13daf0ee201a37454David S. Miller int start = skb_headlen(skb); 6531a028e50729b85d0a038fad13daf0ee201a37454David S. Miller int i, copy = start - offset; 6545b1a002ade68173f21b2126a778278df72202ba6David S. Miller struct sk_buff *frag_iter; 6555b1a002ade68173f21b2126a778278df72202ba6David S. Miller int pos = 0; 6561da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 6571da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds /* Copy header. */ 6581da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds if (copy > 0) { 6591da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds int err = 0; 6601da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds if (copy > len) 6611da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds copy = len; 6621da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds *csump = csum_and_copy_to_user(skb->data + offset, to, copy, 6631da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds *csump, &err); 6641da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds if (err) 6651da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds goto fault; 6661da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds if ((len -= copy) == 0) 6671da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds return 0; 6681da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds offset += copy; 6691da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds to += copy; 6701da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds pos = copy; 6711da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds } 6721da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 6731da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds for (i = 0; i < skb_shinfo(skb)->nr_frags; i++) { 6741a028e50729b85d0a038fad13daf0ee201a37454David S. Miller int end; 6759e903e085262ffbf1fc44a17ac06058aca03524aEric Dumazet const skb_frag_t *frag = &skb_shinfo(skb)->frags[i]; 6761da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 677547b792cac0a038b9dbf958d3c120df3740b5572Ilpo Järvinen WARN_ON(start > offset + len); 6781a028e50729b85d0a038fad13daf0ee201a37454David S. Miller 6799e903e085262ffbf1fc44a17ac06058aca03524aEric Dumazet end = start + skb_frag_size(frag); 6801da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds if ((copy = end - offset) > 0) { 6815084205faf45384fff25c4cf77dd5c96279283adAl Viro __wsum csum2; 6821da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds int err = 0; 6831da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds u8 *vaddr; 684ea2ab69379a941c6f8884e290fdd28c93936a778Ian Campbell struct page *page = skb_frag_page(frag); 6851da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 6861da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds if (copy > len) 6871da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds copy = len; 6881da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds vaddr = kmap(page); 6891da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds csum2 = csum_and_copy_to_user(vaddr + 6901a028e50729b85d0a038fad13daf0ee201a37454David S. Miller frag->page_offset + 6911a028e50729b85d0a038fad13daf0ee201a37454David S. Miller offset - start, 6921da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds to, copy, 0, &err); 6931da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds kunmap(page); 6941da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds if (err) 6951da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds goto fault; 6961da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds *csump = csum_block_add(*csump, csum2, pos); 6971da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds if (!(len -= copy)) 6981da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds return 0; 6991da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds offset += copy; 7001da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds to += copy; 7011da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds pos += copy; 7021da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds } 7031a028e50729b85d0a038fad13daf0ee201a37454David S. Miller start = end; 7041da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds } 7051da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 7065b1a002ade68173f21b2126a778278df72202ba6David S. Miller skb_walk_frags(skb, frag_iter) { 7075b1a002ade68173f21b2126a778278df72202ba6David S. Miller int end; 7085b1a002ade68173f21b2126a778278df72202ba6David S. Miller 7095b1a002ade68173f21b2126a778278df72202ba6David S. Miller WARN_ON(start > offset + len); 7105b1a002ade68173f21b2126a778278df72202ba6David S. Miller 7115b1a002ade68173f21b2126a778278df72202ba6David S. Miller end = start + frag_iter->len; 7125b1a002ade68173f21b2126a778278df72202ba6David S. Miller if ((copy = end - offset) > 0) { 7135b1a002ade68173f21b2126a778278df72202ba6David S. Miller __wsum csum2 = 0; 7145b1a002ade68173f21b2126a778278df72202ba6David S. Miller if (copy > len) 7155b1a002ade68173f21b2126a778278df72202ba6David S. Miller copy = len; 7165b1a002ade68173f21b2126a778278df72202ba6David S. Miller if (skb_copy_and_csum_datagram(frag_iter, 7175b1a002ade68173f21b2126a778278df72202ba6David S. Miller offset - start, 7185b1a002ade68173f21b2126a778278df72202ba6David S. Miller to, copy, 7195b1a002ade68173f21b2126a778278df72202ba6David S. Miller &csum2)) 7205b1a002ade68173f21b2126a778278df72202ba6David S. Miller goto fault; 7215b1a002ade68173f21b2126a778278df72202ba6David S. Miller *csump = csum_block_add(*csump, csum2, pos); 7225b1a002ade68173f21b2126a778278df72202ba6David S. Miller if ((len -= copy) == 0) 7235b1a002ade68173f21b2126a778278df72202ba6David S. Miller return 0; 7245b1a002ade68173f21b2126a778278df72202ba6David S. Miller offset += copy; 7255b1a002ade68173f21b2126a778278df72202ba6David S. Miller to += copy; 7265b1a002ade68173f21b2126a778278df72202ba6David S. Miller pos += copy; 7271da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds } 7285b1a002ade68173f21b2126a778278df72202ba6David S. Miller start = end; 7291da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds } 7301da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds if (!len) 7311da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds return 0; 7321da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 7331da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvaldsfault: 7341da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds return -EFAULT; 7351da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds} 7361da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 737759e5d006462d53fb708daa8284b4ad909415da1Herbert Xu__sum16 __skb_checksum_complete_head(struct sk_buff *skb, int len) 738fb286bb2990a107009dbf25f6ffebeb7df77f9beHerbert Xu{ 739d3bc23e7ee9db8023dff5a86bb3b0069ed018789Al Viro __sum16 sum; 740fb286bb2990a107009dbf25f6ffebeb7df77f9beHerbert Xu 741759e5d006462d53fb708daa8284b4ad909415da1Herbert Xu sum = csum_fold(skb_checksum(skb, 0, len, skb->csum)); 74246fb51eb96cafb2c148b7b5119adb5e31a2bf3c4Tom Herbert if (likely(!sum)) { 74346fb51eb96cafb2c148b7b5119adb5e31a2bf3c4Tom Herbert if (unlikely(skb->ip_summed == CHECKSUM_COMPLETE) && 74446fb51eb96cafb2c148b7b5119adb5e31a2bf3c4Tom Herbert !skb->csum_complete_sw) 74546fb51eb96cafb2c148b7b5119adb5e31a2bf3c4Tom Herbert netdev_rx_csum_fault(skb->dev); 74646fb51eb96cafb2c148b7b5119adb5e31a2bf3c4Tom Herbert } 74746fb51eb96cafb2c148b7b5119adb5e31a2bf3c4Tom Herbert skb->csum_valid = !sum; 748fb286bb2990a107009dbf25f6ffebeb7df77f9beHerbert Xu return sum; 749fb286bb2990a107009dbf25f6ffebeb7df77f9beHerbert Xu} 750759e5d006462d53fb708daa8284b4ad909415da1Herbert XuEXPORT_SYMBOL(__skb_checksum_complete_head); 751759e5d006462d53fb708daa8284b4ad909415da1Herbert Xu 752759e5d006462d53fb708daa8284b4ad909415da1Herbert Xu__sum16 __skb_checksum_complete(struct sk_buff *skb) 753759e5d006462d53fb708daa8284b4ad909415da1Herbert Xu{ 75446fb51eb96cafb2c148b7b5119adb5e31a2bf3c4Tom Herbert __wsum csum; 75546fb51eb96cafb2c148b7b5119adb5e31a2bf3c4Tom Herbert __sum16 sum; 75646fb51eb96cafb2c148b7b5119adb5e31a2bf3c4Tom Herbert 75746fb51eb96cafb2c148b7b5119adb5e31a2bf3c4Tom Herbert csum = skb_checksum(skb, 0, skb->len, 0); 75846fb51eb96cafb2c148b7b5119adb5e31a2bf3c4Tom Herbert 75946fb51eb96cafb2c148b7b5119adb5e31a2bf3c4Tom Herbert /* skb->csum holds pseudo checksum */ 76046fb51eb96cafb2c148b7b5119adb5e31a2bf3c4Tom Herbert sum = csum_fold(csum_add(skb->csum, csum)); 76146fb51eb96cafb2c148b7b5119adb5e31a2bf3c4Tom Herbert if (likely(!sum)) { 76246fb51eb96cafb2c148b7b5119adb5e31a2bf3c4Tom Herbert if (unlikely(skb->ip_summed == CHECKSUM_COMPLETE) && 76346fb51eb96cafb2c148b7b5119adb5e31a2bf3c4Tom Herbert !skb->csum_complete_sw) 76446fb51eb96cafb2c148b7b5119adb5e31a2bf3c4Tom Herbert netdev_rx_csum_fault(skb->dev); 76546fb51eb96cafb2c148b7b5119adb5e31a2bf3c4Tom Herbert } 76646fb51eb96cafb2c148b7b5119adb5e31a2bf3c4Tom Herbert 76746fb51eb96cafb2c148b7b5119adb5e31a2bf3c4Tom Herbert /* Save full packet checksum */ 76846fb51eb96cafb2c148b7b5119adb5e31a2bf3c4Tom Herbert skb->csum = csum; 76946fb51eb96cafb2c148b7b5119adb5e31a2bf3c4Tom Herbert skb->ip_summed = CHECKSUM_COMPLETE; 77046fb51eb96cafb2c148b7b5119adb5e31a2bf3c4Tom Herbert skb->csum_complete_sw = 1; 77146fb51eb96cafb2c148b7b5119adb5e31a2bf3c4Tom Herbert skb->csum_valid = !sum; 77246fb51eb96cafb2c148b7b5119adb5e31a2bf3c4Tom Herbert 77346fb51eb96cafb2c148b7b5119adb5e31a2bf3c4Tom Herbert return sum; 774759e5d006462d53fb708daa8284b4ad909415da1Herbert Xu} 775fb286bb2990a107009dbf25f6ffebeb7df77f9beHerbert XuEXPORT_SYMBOL(__skb_checksum_complete); 776fb286bb2990a107009dbf25f6ffebeb7df77f9beHerbert Xu 7771da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds/** 778e793c0f70e9bdf4a2e71c151a1a3cf85c4db92adMasanari Iida * skb_copy_and_csum_datagram_iovec - Copy and checksum skb to user iovec. 7794dc3b16ba18c0f967ad100c52fa65b01a4f76ff0Pavel Pisa * @skb: skbuff 7804dc3b16ba18c0f967ad100c52fa65b01a4f76ff0Pavel Pisa * @hlen: hardware length 78167be2dd1bace0ec7ce2dbc1bba3f8df3d7be597eMartin Waitz * @iov: io vector 7824ec93edb14fe5fdee9fae6335f2cbba204627eacYOSHIFUJI Hideaki * 7831da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds * Caller _must_ check that skb will fit to this iovec. 7841da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds * 7851da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds * Returns: 0 - success. 7861da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds * -EINVAL - checksum failure. 7871da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds * -EFAULT - fault during copy. Beware, in this case iovec 7881da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds * can be modified! 7891da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds */ 790fb286bb2990a107009dbf25f6ffebeb7df77f9beHerbert Xuint skb_copy_and_csum_datagram_iovec(struct sk_buff *skb, 7911da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds int hlen, struct iovec *iov) 7921da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds{ 793d3bc23e7ee9db8023dff5a86bb3b0069ed018789Al Viro __wsum csum; 7941da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds int chunk = skb->len - hlen; 7951da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 796ef8aef55ce61fd0e2af798695f7386ac756ae1e7Herbert Xu if (!chunk) 797ef8aef55ce61fd0e2af798695f7386ac756ae1e7Herbert Xu return 0; 798ef8aef55ce61fd0e2af798695f7386ac756ae1e7Herbert Xu 7991da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds /* Skip filled elements. 8001da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds * Pretty silly, look at memcpy_toiovec, though 8) 8011da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds */ 8021da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds while (!iov->iov_len) 8031da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds iov++; 8041da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 8051da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds if (iov->iov_len < chunk) { 806fb286bb2990a107009dbf25f6ffebeb7df77f9beHerbert Xu if (__skb_checksum_complete(skb)) 8071da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds goto csum_error; 8081da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds if (skb_copy_datagram_iovec(skb, hlen, iov, chunk)) 8091da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds goto fault; 8101da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds } else { 8111da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds csum = csum_partial(skb->data, hlen, skb->csum); 8121da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds if (skb_copy_and_csum_datagram(skb, hlen, iov->iov_base, 8131da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds chunk, &csum)) 8141da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds goto fault; 815d3bc23e7ee9db8023dff5a86bb3b0069ed018789Al Viro if (csum_fold(csum)) 8161da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds goto csum_error; 81784fa7933a33f806bbbaae6775e87459b1ec584c0Patrick McHardy if (unlikely(skb->ip_summed == CHECKSUM_COMPLETE)) 818fb286bb2990a107009dbf25f6ffebeb7df77f9beHerbert Xu netdev_rx_csum_fault(skb->dev); 8191da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds iov->iov_len -= chunk; 8201da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds iov->iov_base += chunk; 8211da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds } 8221da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds return 0; 8231da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvaldscsum_error: 8241da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds return -EINVAL; 8251da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvaldsfault: 8261da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds return -EFAULT; 8271da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds} 8289e34a5b51684bc90ac827ec4ba339f3892632eacEric DumazetEXPORT_SYMBOL(skb_copy_and_csum_datagram_iovec); 8291da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 8301da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds/** 8311da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds * datagram_poll - generic datagram poll 8324dc3b16ba18c0f967ad100c52fa65b01a4f76ff0Pavel Pisa * @file: file struct 8334dc3b16ba18c0f967ad100c52fa65b01a4f76ff0Pavel Pisa * @sock: socket 8344dc3b16ba18c0f967ad100c52fa65b01a4f76ff0Pavel Pisa * @wait: poll table 8351da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds * 8361da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds * Datagram poll: Again totally generic. This also handles 8371da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds * sequenced packet sockets providing the socket receive queue 8381da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds * is only ever holding data ready to receive. 8391da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds * 8401da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds * Note: when you _don't_ use this routine for this protocol, 8411da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds * and you use a different write policy from sock_writeable() 8421da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds * then please supply your own write_space callback. 8431da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds */ 8441da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvaldsunsigned int datagram_poll(struct file *file, struct socket *sock, 8451da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds poll_table *wait) 8461da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds{ 8471da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds struct sock *sk = sock->sk; 8481da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds unsigned int mask; 8491da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 850aa395145165cb06a0d0885221bbe0ce4a564391dEric Dumazet sock_poll_wait(file, sk_sleep(sk), wait); 8511da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds mask = 0; 8521da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 8531da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds /* exceptional events? */ 8541da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds if (sk->sk_err || !skb_queue_empty(&sk->sk_error_queue)) 8557d4c04fc170087119727119074e72445f2bb192bKeller, Jacob E mask |= POLLERR | 8568facd5fb73c6e960555e5913743dfbb6c3d984a5Jacob Keller (sock_flag(sk, SOCK_SELECT_ERR_QUEUE) ? POLLPRI : 0); 8577d4c04fc170087119727119074e72445f2bb192bKeller, Jacob E 858f348d70a324e15afc701a494f32ec468abb7d1ebDavide Libenzi if (sk->sk_shutdown & RCV_SHUTDOWN) 859db40980fcdb560d7992b0511df16cdd3f7e381f3Eric Dumazet mask |= POLLRDHUP | POLLIN | POLLRDNORM; 8601da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds if (sk->sk_shutdown == SHUTDOWN_MASK) 8611da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds mask |= POLLHUP; 8621da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 8631da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds /* readable? */ 864db40980fcdb560d7992b0511df16cdd3f7e381f3Eric Dumazet if (!skb_queue_empty(&sk->sk_receive_queue)) 8651da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds mask |= POLLIN | POLLRDNORM; 8661da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 8671da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds /* Connection-based need to check for termination and startup */ 8681da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds if (connection_based(sk)) { 8691da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds if (sk->sk_state == TCP_CLOSE) 8701da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds mask |= POLLHUP; 8711da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds /* connection hasn't started yet? */ 8721da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds if (sk->sk_state == TCP_SYN_SENT) 8731da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds return mask; 8741da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds } 8751da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 8761da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds /* writable? */ 8771da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds if (sock_writeable(sk)) 8781da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds mask |= POLLOUT | POLLWRNORM | POLLWRBAND; 8791da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds else 8801da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds set_bit(SOCK_ASYNC_NOSPACE, &sk->sk_socket->flags); 8811da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 8821da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds return mask; 8831da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds} 8841da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus TorvaldsEXPORT_SYMBOL(datagram_poll); 885