1/* 2 * Routines having to do with the 'struct sk_buff' memory handlers. 3 * 4 * Authors: Alan Cox <alan@lxorguk.ukuu.org.uk> 5 * Florian La Roche <rzsfl@rz.uni-sb.de> 6 * 7 * Fixes: 8 * Alan Cox : Fixed the worst of the load 9 * balancer bugs. 10 * Dave Platt : Interrupt stacking fix. 11 * Richard Kooijman : Timestamp fixes. 12 * Alan Cox : Changed buffer format. 13 * Alan Cox : destructor hook for AF_UNIX etc. 14 * Linus Torvalds : Better skb_clone. 15 * Alan Cox : Added skb_copy. 16 * Alan Cox : Added all the changed routines Linus 17 * only put in the headers 18 * Ray VanTassle : Fixed --skb->lock in free 19 * Alan Cox : skb_copy copy arp field 20 * Andi Kleen : slabified it. 21 * Robert Olsson : Removed skb_head_pool 22 * 23 * NOTE: 24 * The __skb_ routines should be called with interrupts 25 * disabled, or you better be *real* sure that the operation is atomic 26 * with respect to whatever list is being frobbed (e.g. via lock_sock() 27 * or via disabling bottom half handlers, etc). 28 * 29 * This program is free software; you can redistribute it and/or 30 * modify it under the terms of the GNU General Public License 31 * as published by the Free Software Foundation; either version 32 * 2 of the License, or (at your option) any later version. 33 */ 34 35/* 36 * The functions in this file will not compile correctly with gcc 2.4.x 37 */ 38 39#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt 40 41#include <linux/module.h> 42#include <linux/types.h> 43#include <linux/kernel.h> 44#include <linux/kmemcheck.h> 45#include <linux/mm.h> 46#include <linux/interrupt.h> 47#include <linux/in.h> 48#include <linux/inet.h> 49#include <linux/slab.h> 50#include <linux/tcp.h> 51#include <linux/udp.h> 52#include <linux/netdevice.h> 53#ifdef CONFIG_NET_CLS_ACT 54#include <net/pkt_sched.h> 55#endif 56#include <linux/string.h> 57#include <linux/skbuff.h> 58#include <linux/splice.h> 59#include <linux/cache.h> 60#include <linux/rtnetlink.h> 61#include <linux/init.h> 62#include <linux/scatterlist.h> 63#include <linux/errqueue.h> 64#include <linux/prefetch.h> 65#include <linux/if_vlan.h> 66 67#include <net/protocol.h> 68#include <net/dst.h> 69#include <net/sock.h> 70#include <net/checksum.h> 71#include <net/ip6_checksum.h> 72#include <net/xfrm.h> 73 74#include <asm/uaccess.h> 75#include <trace/events/skb.h> 76#include <linux/highmem.h> 77 78struct kmem_cache *skbuff_head_cache __read_mostly; 79static struct kmem_cache *skbuff_fclone_cache __read_mostly; 80 81/** 82 * skb_panic - private function for out-of-line support 83 * @skb: buffer 84 * @sz: size 85 * @addr: address 86 * @msg: skb_over_panic or skb_under_panic 87 * 88 * Out-of-line support for skb_put() and skb_push(). 89 * Called via the wrapper skb_over_panic() or skb_under_panic(). 90 * Keep out of line to prevent kernel bloat. 91 * __builtin_return_address is not used because it is not always reliable. 92 */ 93static void skb_panic(struct sk_buff *skb, unsigned int sz, void *addr, 94 const char msg[]) 95{ 96 pr_emerg("%s: text:%p len:%d put:%d head:%p data:%p tail:%#lx end:%#lx dev:%s\n", 97 msg, addr, skb->len, sz, skb->head, skb->data, 98 (unsigned long)skb->tail, (unsigned long)skb->end, 99 skb->dev ? skb->dev->name : "<NULL>"); 100 BUG(); 101} 102 103static void skb_over_panic(struct sk_buff *skb, unsigned int sz, void *addr) 104{ 105 skb_panic(skb, sz, addr, __func__); 106} 107 108static void skb_under_panic(struct sk_buff *skb, unsigned int sz, void *addr) 109{ 110 skb_panic(skb, sz, addr, __func__); 111} 112 113/* 114 * kmalloc_reserve is a wrapper around kmalloc_node_track_caller that tells 115 * the caller if emergency pfmemalloc reserves are being used. If it is and 116 * the socket is later found to be SOCK_MEMALLOC then PFMEMALLOC reserves 117 * may be used. Otherwise, the packet data may be discarded until enough 118 * memory is free 119 */ 120#define kmalloc_reserve(size, gfp, node, pfmemalloc) \ 121 __kmalloc_reserve(size, gfp, node, _RET_IP_, pfmemalloc) 122 123static void *__kmalloc_reserve(size_t size, gfp_t flags, int node, 124 unsigned long ip, bool *pfmemalloc) 125{ 126 void *obj; 127 bool ret_pfmemalloc = false; 128 129 /* 130 * Try a regular allocation, when that fails and we're not entitled 131 * to the reserves, fail. 132 */ 133 obj = kmalloc_node_track_caller(size, 134 flags | __GFP_NOMEMALLOC | __GFP_NOWARN, 135 node); 136 if (obj || !(gfp_pfmemalloc_allowed(flags))) 137 goto out; 138 139 /* Try again but now we are using pfmemalloc reserves */ 140 ret_pfmemalloc = true; 141 obj = kmalloc_node_track_caller(size, flags, node); 142 143out: 144 if (pfmemalloc) 145 *pfmemalloc = ret_pfmemalloc; 146 147 return obj; 148} 149 150/* Allocate a new skbuff. We do this ourselves so we can fill in a few 151 * 'private' fields and also do memory statistics to find all the 152 * [BEEP] leaks. 153 * 154 */ 155 156struct sk_buff *__alloc_skb_head(gfp_t gfp_mask, int node) 157{ 158 struct sk_buff *skb; 159 160 /* Get the HEAD */ 161 skb = kmem_cache_alloc_node(skbuff_head_cache, 162 gfp_mask & ~__GFP_DMA, node); 163 if (!skb) 164 goto out; 165 166 /* 167 * Only clear those fields we need to clear, not those that we will 168 * actually initialise below. Hence, don't put any more fields after 169 * the tail pointer in struct sk_buff! 170 */ 171 memset(skb, 0, offsetof(struct sk_buff, tail)); 172 skb->head = NULL; 173 skb->truesize = sizeof(struct sk_buff); 174 atomic_set(&skb->users, 1); 175 176 skb->mac_header = (typeof(skb->mac_header))~0U; 177out: 178 return skb; 179} 180 181/** 182 * __alloc_skb - allocate a network buffer 183 * @size: size to allocate 184 * @gfp_mask: allocation mask 185 * @flags: If SKB_ALLOC_FCLONE is set, allocate from fclone cache 186 * instead of head cache and allocate a cloned (child) skb. 187 * If SKB_ALLOC_RX is set, __GFP_MEMALLOC will be used for 188 * allocations in case the data is required for writeback 189 * @node: numa node to allocate memory on 190 * 191 * Allocate a new &sk_buff. The returned buffer has no headroom and a 192 * tail room of at least size bytes. The object has a reference count 193 * of one. The return is the buffer. On a failure the return is %NULL. 194 * 195 * Buffers may only be allocated from interrupts using a @gfp_mask of 196 * %GFP_ATOMIC. 197 */ 198struct sk_buff *__alloc_skb(unsigned int size, gfp_t gfp_mask, 199 int flags, int node) 200{ 201 struct kmem_cache *cache; 202 struct skb_shared_info *shinfo; 203 struct sk_buff *skb; 204 u8 *data; 205 bool pfmemalloc; 206 207 cache = (flags & SKB_ALLOC_FCLONE) 208 ? skbuff_fclone_cache : skbuff_head_cache; 209 210 if (sk_memalloc_socks() && (flags & SKB_ALLOC_RX)) 211 gfp_mask |= __GFP_MEMALLOC; 212 213 /* Get the HEAD */ 214 skb = kmem_cache_alloc_node(cache, gfp_mask & ~__GFP_DMA, node); 215 if (!skb) 216 goto out; 217 prefetchw(skb); 218 219 /* We do our best to align skb_shared_info on a separate cache 220 * line. It usually works because kmalloc(X > SMP_CACHE_BYTES) gives 221 * aligned memory blocks, unless SLUB/SLAB debug is enabled. 222 * Both skb->head and skb_shared_info are cache line aligned. 223 */ 224 size = SKB_DATA_ALIGN(size); 225 size += SKB_DATA_ALIGN(sizeof(struct skb_shared_info)); 226 data = kmalloc_reserve(size, gfp_mask, node, &pfmemalloc); 227 if (!data) 228 goto nodata; 229 /* kmalloc(size) might give us more room than requested. 230 * Put skb_shared_info exactly at the end of allocated zone, 231 * to allow max possible filling before reallocation. 232 */ 233 size = SKB_WITH_OVERHEAD(ksize(data)); 234 prefetchw(data + size); 235 236 /* 237 * Only clear those fields we need to clear, not those that we will 238 * actually initialise below. Hence, don't put any more fields after 239 * the tail pointer in struct sk_buff! 240 */ 241 memset(skb, 0, offsetof(struct sk_buff, tail)); 242 /* Account for allocated memory : skb + skb->head */ 243 skb->truesize = SKB_TRUESIZE(size); 244 skb->pfmemalloc = pfmemalloc; 245 atomic_set(&skb->users, 1); 246 skb->head = data; 247 skb->data = data; 248 skb_reset_tail_pointer(skb); 249 skb->end = skb->tail + size; 250 skb->mac_header = (typeof(skb->mac_header))~0U; 251 skb->transport_header = (typeof(skb->transport_header))~0U; 252 253 /* make sure we initialize shinfo sequentially */ 254 shinfo = skb_shinfo(skb); 255 memset(shinfo, 0, offsetof(struct skb_shared_info, dataref)); 256 atomic_set(&shinfo->dataref, 1); 257 kmemcheck_annotate_variable(shinfo->destructor_arg); 258 259 if (flags & SKB_ALLOC_FCLONE) { 260 struct sk_buff_fclones *fclones; 261 262 fclones = container_of(skb, struct sk_buff_fclones, skb1); 263 264 kmemcheck_annotate_bitfield(&fclones->skb2, flags1); 265 skb->fclone = SKB_FCLONE_ORIG; 266 atomic_set(&fclones->fclone_ref, 1); 267 268 fclones->skb2.fclone = SKB_FCLONE_FREE; 269 fclones->skb2.pfmemalloc = pfmemalloc; 270 } 271out: 272 return skb; 273nodata: 274 kmem_cache_free(cache, skb); 275 skb = NULL; 276 goto out; 277} 278EXPORT_SYMBOL(__alloc_skb); 279 280/** 281 * build_skb - build a network buffer 282 * @data: data buffer provided by caller 283 * @frag_size: size of fragment, or 0 if head was kmalloced 284 * 285 * Allocate a new &sk_buff. Caller provides space holding head and 286 * skb_shared_info. @data must have been allocated by kmalloc() only if 287 * @frag_size is 0, otherwise data should come from the page allocator. 288 * The return is the new skb buffer. 289 * On a failure the return is %NULL, and @data is not freed. 290 * Notes : 291 * Before IO, driver allocates only data buffer where NIC put incoming frame 292 * Driver should add room at head (NET_SKB_PAD) and 293 * MUST add room at tail (SKB_DATA_ALIGN(skb_shared_info)) 294 * After IO, driver calls build_skb(), to allocate sk_buff and populate it 295 * before giving packet to stack. 296 * RX rings only contains data buffers, not full skbs. 297 */ 298struct sk_buff *build_skb(void *data, unsigned int frag_size) 299{ 300 struct skb_shared_info *shinfo; 301 struct sk_buff *skb; 302 unsigned int size = frag_size ? : ksize(data); 303 304 skb = kmem_cache_alloc(skbuff_head_cache, GFP_ATOMIC); 305 if (!skb) 306 return NULL; 307 308 size -= SKB_DATA_ALIGN(sizeof(struct skb_shared_info)); 309 310 memset(skb, 0, offsetof(struct sk_buff, tail)); 311 skb->truesize = SKB_TRUESIZE(size); 312 skb->head_frag = frag_size != 0; 313 atomic_set(&skb->users, 1); 314 skb->head = data; 315 skb->data = data; 316 skb_reset_tail_pointer(skb); 317 skb->end = skb->tail + size; 318 skb->mac_header = (typeof(skb->mac_header))~0U; 319 skb->transport_header = (typeof(skb->transport_header))~0U; 320 321 /* make sure we initialize shinfo sequentially */ 322 shinfo = skb_shinfo(skb); 323 memset(shinfo, 0, offsetof(struct skb_shared_info, dataref)); 324 atomic_set(&shinfo->dataref, 1); 325 kmemcheck_annotate_variable(shinfo->destructor_arg); 326 327 return skb; 328} 329EXPORT_SYMBOL(build_skb); 330 331struct netdev_alloc_cache { 332 struct page_frag frag; 333 /* we maintain a pagecount bias, so that we dont dirty cache line 334 * containing page->_count every time we allocate a fragment. 335 */ 336 unsigned int pagecnt_bias; 337}; 338static DEFINE_PER_CPU(struct netdev_alloc_cache, netdev_alloc_cache); 339 340static void *__netdev_alloc_frag(unsigned int fragsz, gfp_t gfp_mask) 341{ 342 struct netdev_alloc_cache *nc; 343 void *data = NULL; 344 int order; 345 unsigned long flags; 346 347 local_irq_save(flags); 348 nc = this_cpu_ptr(&netdev_alloc_cache); 349 if (unlikely(!nc->frag.page)) { 350refill: 351 for (order = NETDEV_FRAG_PAGE_MAX_ORDER; ;) { 352 gfp_t gfp = gfp_mask; 353 354 if (order) 355 gfp |= __GFP_COMP | __GFP_NOWARN; 356 nc->frag.page = alloc_pages(gfp, order); 357 if (likely(nc->frag.page)) 358 break; 359 if (--order < 0) 360 goto end; 361 } 362 nc->frag.size = PAGE_SIZE << order; 363 /* Even if we own the page, we do not use atomic_set(). 364 * This would break get_page_unless_zero() users. 365 */ 366 atomic_add(NETDEV_PAGECNT_MAX_BIAS - 1, 367 &nc->frag.page->_count); 368 nc->pagecnt_bias = NETDEV_PAGECNT_MAX_BIAS; 369 nc->frag.offset = 0; 370 } 371 372 if (nc->frag.offset + fragsz > nc->frag.size) { 373 if (atomic_read(&nc->frag.page->_count) != nc->pagecnt_bias) { 374 if (!atomic_sub_and_test(nc->pagecnt_bias, 375 &nc->frag.page->_count)) 376 goto refill; 377 /* OK, page count is 0, we can safely set it */ 378 atomic_set(&nc->frag.page->_count, 379 NETDEV_PAGECNT_MAX_BIAS); 380 } else { 381 atomic_add(NETDEV_PAGECNT_MAX_BIAS - nc->pagecnt_bias, 382 &nc->frag.page->_count); 383 } 384 nc->pagecnt_bias = NETDEV_PAGECNT_MAX_BIAS; 385 nc->frag.offset = 0; 386 } 387 388 data = page_address(nc->frag.page) + nc->frag.offset; 389 nc->frag.offset += fragsz; 390 nc->pagecnt_bias--; 391end: 392 local_irq_restore(flags); 393 return data; 394} 395 396/** 397 * netdev_alloc_frag - allocate a page fragment 398 * @fragsz: fragment size 399 * 400 * Allocates a frag from a page for receive buffer. 401 * Uses GFP_ATOMIC allocations. 402 */ 403void *netdev_alloc_frag(unsigned int fragsz) 404{ 405 return __netdev_alloc_frag(fragsz, GFP_ATOMIC | __GFP_COLD); 406} 407EXPORT_SYMBOL(netdev_alloc_frag); 408 409/** 410 * __netdev_alloc_skb - allocate an skbuff for rx on a specific device 411 * @dev: network device to receive on 412 * @length: length to allocate 413 * @gfp_mask: get_free_pages mask, passed to alloc_skb 414 * 415 * Allocate a new &sk_buff and assign it a usage count of one. The 416 * buffer has unspecified headroom built in. Users should allocate 417 * the headroom they think they need without accounting for the 418 * built in space. The built in space is used for optimisations. 419 * 420 * %NULL is returned if there is no free memory. 421 */ 422struct sk_buff *__netdev_alloc_skb(struct net_device *dev, 423 unsigned int length, gfp_t gfp_mask) 424{ 425 struct sk_buff *skb = NULL; 426 unsigned int fragsz = SKB_DATA_ALIGN(length + NET_SKB_PAD) + 427 SKB_DATA_ALIGN(sizeof(struct skb_shared_info)); 428 429 if (fragsz <= PAGE_SIZE && !(gfp_mask & (__GFP_WAIT | GFP_DMA))) { 430 void *data; 431 432 if (sk_memalloc_socks()) 433 gfp_mask |= __GFP_MEMALLOC; 434 435 data = __netdev_alloc_frag(fragsz, gfp_mask); 436 437 if (likely(data)) { 438 skb = build_skb(data, fragsz); 439 if (unlikely(!skb)) 440 put_page(virt_to_head_page(data)); 441 } 442 } else { 443 skb = __alloc_skb(length + NET_SKB_PAD, gfp_mask, 444 SKB_ALLOC_RX, NUMA_NO_NODE); 445 } 446 if (likely(skb)) { 447 skb_reserve(skb, NET_SKB_PAD); 448 skb->dev = dev; 449 } 450 return skb; 451} 452EXPORT_SYMBOL(__netdev_alloc_skb); 453 454void skb_add_rx_frag(struct sk_buff *skb, int i, struct page *page, int off, 455 int size, unsigned int truesize) 456{ 457 skb_fill_page_desc(skb, i, page, off, size); 458 skb->len += size; 459 skb->data_len += size; 460 skb->truesize += truesize; 461} 462EXPORT_SYMBOL(skb_add_rx_frag); 463 464void skb_coalesce_rx_frag(struct sk_buff *skb, int i, int size, 465 unsigned int truesize) 466{ 467 skb_frag_t *frag = &skb_shinfo(skb)->frags[i]; 468 469 skb_frag_size_add(frag, size); 470 skb->len += size; 471 skb->data_len += size; 472 skb->truesize += truesize; 473} 474EXPORT_SYMBOL(skb_coalesce_rx_frag); 475 476static void skb_drop_list(struct sk_buff **listp) 477{ 478 kfree_skb_list(*listp); 479 *listp = NULL; 480} 481 482static inline void skb_drop_fraglist(struct sk_buff *skb) 483{ 484 skb_drop_list(&skb_shinfo(skb)->frag_list); 485} 486 487static void skb_clone_fraglist(struct sk_buff *skb) 488{ 489 struct sk_buff *list; 490 491 skb_walk_frags(skb, list) 492 skb_get(list); 493} 494 495static void skb_free_head(struct sk_buff *skb) 496{ 497 if (skb->head_frag) 498 put_page(virt_to_head_page(skb->head)); 499 else 500 kfree(skb->head); 501} 502 503static void skb_release_data(struct sk_buff *skb) 504{ 505 struct skb_shared_info *shinfo = skb_shinfo(skb); 506 int i; 507 508 if (skb->cloned && 509 atomic_sub_return(skb->nohdr ? (1 << SKB_DATAREF_SHIFT) + 1 : 1, 510 &shinfo->dataref)) 511 return; 512 513 for (i = 0; i < shinfo->nr_frags; i++) 514 __skb_frag_unref(&shinfo->frags[i]); 515 516 /* 517 * If skb buf is from userspace, we need to notify the caller 518 * the lower device DMA has done; 519 */ 520 if (shinfo->tx_flags & SKBTX_DEV_ZEROCOPY) { 521 struct ubuf_info *uarg; 522 523 uarg = shinfo->destructor_arg; 524 if (uarg->callback) 525 uarg->callback(uarg, true); 526 } 527 528 if (shinfo->frag_list) 529 kfree_skb_list(shinfo->frag_list); 530 531 skb_free_head(skb); 532} 533 534/* 535 * Free an skbuff by memory without cleaning the state. 536 */ 537static void kfree_skbmem(struct sk_buff *skb) 538{ 539 struct sk_buff_fclones *fclones; 540 541 switch (skb->fclone) { 542 case SKB_FCLONE_UNAVAILABLE: 543 kmem_cache_free(skbuff_head_cache, skb); 544 break; 545 546 case SKB_FCLONE_ORIG: 547 fclones = container_of(skb, struct sk_buff_fclones, skb1); 548 if (atomic_dec_and_test(&fclones->fclone_ref)) 549 kmem_cache_free(skbuff_fclone_cache, fclones); 550 break; 551 552 case SKB_FCLONE_CLONE: 553 fclones = container_of(skb, struct sk_buff_fclones, skb2); 554 555 /* The clone portion is available for 556 * fast-cloning again. 557 */ 558 skb->fclone = SKB_FCLONE_FREE; 559 560 if (atomic_dec_and_test(&fclones->fclone_ref)) 561 kmem_cache_free(skbuff_fclone_cache, fclones); 562 break; 563 } 564} 565 566static void skb_release_head_state(struct sk_buff *skb) 567{ 568 skb_dst_drop(skb); 569#ifdef CONFIG_XFRM 570 secpath_put(skb->sp); 571#endif 572 if (skb->destructor) { 573 WARN_ON(in_irq()); 574 skb->destructor(skb); 575 } 576#if IS_ENABLED(CONFIG_NF_CONNTRACK) 577 nf_conntrack_put(skb->nfct); 578#endif 579#if IS_ENABLED(CONFIG_BRIDGE_NETFILTER) 580 nf_bridge_put(skb->nf_bridge); 581#endif 582/* XXX: IS this still necessary? - JHS */ 583#ifdef CONFIG_NET_SCHED 584 skb->tc_index = 0; 585#ifdef CONFIG_NET_CLS_ACT 586 skb->tc_verd = 0; 587#endif 588#endif 589} 590 591/* Free everything but the sk_buff shell. */ 592static void skb_release_all(struct sk_buff *skb) 593{ 594 skb_release_head_state(skb); 595 if (likely(skb->head)) 596 skb_release_data(skb); 597} 598 599/** 600 * __kfree_skb - private function 601 * @skb: buffer 602 * 603 * Free an sk_buff. Release anything attached to the buffer. 604 * Clean the state. This is an internal helper function. Users should 605 * always call kfree_skb 606 */ 607 608void __kfree_skb(struct sk_buff *skb) 609{ 610 skb_release_all(skb); 611 kfree_skbmem(skb); 612} 613EXPORT_SYMBOL(__kfree_skb); 614 615/** 616 * kfree_skb - free an sk_buff 617 * @skb: buffer to free 618 * 619 * Drop a reference to the buffer and free it if the usage count has 620 * hit zero. 621 */ 622void kfree_skb(struct sk_buff *skb) 623{ 624 if (unlikely(!skb)) 625 return; 626 if (likely(atomic_read(&skb->users) == 1)) 627 smp_rmb(); 628 else if (likely(!atomic_dec_and_test(&skb->users))) 629 return; 630 trace_kfree_skb(skb, __builtin_return_address(0)); 631 __kfree_skb(skb); 632} 633EXPORT_SYMBOL(kfree_skb); 634 635void kfree_skb_list(struct sk_buff *segs) 636{ 637 while (segs) { 638 struct sk_buff *next = segs->next; 639 640 kfree_skb(segs); 641 segs = next; 642 } 643} 644EXPORT_SYMBOL(kfree_skb_list); 645 646/** 647 * skb_tx_error - report an sk_buff xmit error 648 * @skb: buffer that triggered an error 649 * 650 * Report xmit error if a device callback is tracking this skb. 651 * skb must be freed afterwards. 652 */ 653void skb_tx_error(struct sk_buff *skb) 654{ 655 if (skb_shinfo(skb)->tx_flags & SKBTX_DEV_ZEROCOPY) { 656 struct ubuf_info *uarg; 657 658 uarg = skb_shinfo(skb)->destructor_arg; 659 if (uarg->callback) 660 uarg->callback(uarg, false); 661 skb_shinfo(skb)->tx_flags &= ~SKBTX_DEV_ZEROCOPY; 662 } 663} 664EXPORT_SYMBOL(skb_tx_error); 665 666/** 667 * consume_skb - free an skbuff 668 * @skb: buffer to free 669 * 670 * Drop a ref to the buffer and free it if the usage count has hit zero 671 * Functions identically to kfree_skb, but kfree_skb assumes that the frame 672 * is being dropped after a failure and notes that 673 */ 674void consume_skb(struct sk_buff *skb) 675{ 676 if (unlikely(!skb)) 677 return; 678 if (likely(atomic_read(&skb->users) == 1)) 679 smp_rmb(); 680 else if (likely(!atomic_dec_and_test(&skb->users))) 681 return; 682 trace_consume_skb(skb); 683 __kfree_skb(skb); 684} 685EXPORT_SYMBOL(consume_skb); 686 687/* Make sure a field is enclosed inside headers_start/headers_end section */ 688#define CHECK_SKB_FIELD(field) \ 689 BUILD_BUG_ON(offsetof(struct sk_buff, field) < \ 690 offsetof(struct sk_buff, headers_start)); \ 691 BUILD_BUG_ON(offsetof(struct sk_buff, field) > \ 692 offsetof(struct sk_buff, headers_end)); \ 693 694static void __copy_skb_header(struct sk_buff *new, const struct sk_buff *old) 695{ 696 new->tstamp = old->tstamp; 697 /* We do not copy old->sk */ 698 new->dev = old->dev; 699 memcpy(new->cb, old->cb, sizeof(old->cb)); 700 skb_dst_copy(new, old); 701#ifdef CONFIG_XFRM 702 new->sp = secpath_get(old->sp); 703#endif 704 __nf_copy(new, old, false); 705 706 /* Note : this field could be in headers_start/headers_end section 707 * It is not yet because we do not want to have a 16 bit hole 708 */ 709 new->queue_mapping = old->queue_mapping; 710 711 memcpy(&new->headers_start, &old->headers_start, 712 offsetof(struct sk_buff, headers_end) - 713 offsetof(struct sk_buff, headers_start)); 714 CHECK_SKB_FIELD(protocol); 715 CHECK_SKB_FIELD(csum); 716 CHECK_SKB_FIELD(hash); 717 CHECK_SKB_FIELD(priority); 718 CHECK_SKB_FIELD(skb_iif); 719 CHECK_SKB_FIELD(vlan_proto); 720 CHECK_SKB_FIELD(vlan_tci); 721 CHECK_SKB_FIELD(transport_header); 722 CHECK_SKB_FIELD(network_header); 723 CHECK_SKB_FIELD(mac_header); 724 CHECK_SKB_FIELD(inner_protocol); 725 CHECK_SKB_FIELD(inner_transport_header); 726 CHECK_SKB_FIELD(inner_network_header); 727 CHECK_SKB_FIELD(inner_mac_header); 728 CHECK_SKB_FIELD(mark); 729#ifdef CONFIG_NETWORK_SECMARK 730 CHECK_SKB_FIELD(secmark); 731#endif 732#ifdef CONFIG_NET_RX_BUSY_POLL 733 CHECK_SKB_FIELD(napi_id); 734#endif 735#ifdef CONFIG_NET_SCHED 736 CHECK_SKB_FIELD(tc_index); 737#ifdef CONFIG_NET_CLS_ACT 738 CHECK_SKB_FIELD(tc_verd); 739#endif 740#endif 741 742} 743 744/* 745 * You should not add any new code to this function. Add it to 746 * __copy_skb_header above instead. 747 */ 748static struct sk_buff *__skb_clone(struct sk_buff *n, struct sk_buff *skb) 749{ 750#define C(x) n->x = skb->x 751 752 n->next = n->prev = NULL; 753 n->sk = NULL; 754 __copy_skb_header(n, skb); 755 756 C(len); 757 C(data_len); 758 C(mac_len); 759 n->hdr_len = skb->nohdr ? skb_headroom(skb) : skb->hdr_len; 760 n->cloned = 1; 761 n->nohdr = 0; 762 n->destructor = NULL; 763 C(tail); 764 C(end); 765 C(head); 766 C(head_frag); 767 C(data); 768 C(truesize); 769 atomic_set(&n->users, 1); 770 771 atomic_inc(&(skb_shinfo(skb)->dataref)); 772 skb->cloned = 1; 773 774 return n; 775#undef C 776} 777 778/** 779 * skb_morph - morph one skb into another 780 * @dst: the skb to receive the contents 781 * @src: the skb to supply the contents 782 * 783 * This is identical to skb_clone except that the target skb is 784 * supplied by the user. 785 * 786 * The target skb is returned upon exit. 787 */ 788struct sk_buff *skb_morph(struct sk_buff *dst, struct sk_buff *src) 789{ 790 skb_release_all(dst); 791 return __skb_clone(dst, src); 792} 793EXPORT_SYMBOL_GPL(skb_morph); 794 795/** 796 * skb_copy_ubufs - copy userspace skb frags buffers to kernel 797 * @skb: the skb to modify 798 * @gfp_mask: allocation priority 799 * 800 * This must be called on SKBTX_DEV_ZEROCOPY skb. 801 * It will copy all frags into kernel and drop the reference 802 * to userspace pages. 803 * 804 * If this function is called from an interrupt gfp_mask() must be 805 * %GFP_ATOMIC. 806 * 807 * Returns 0 on success or a negative error code on failure 808 * to allocate kernel memory to copy to. 809 */ 810int skb_copy_ubufs(struct sk_buff *skb, gfp_t gfp_mask) 811{ 812 int i; 813 int num_frags = skb_shinfo(skb)->nr_frags; 814 struct page *page, *head = NULL; 815 struct ubuf_info *uarg = skb_shinfo(skb)->destructor_arg; 816 817 for (i = 0; i < num_frags; i++) { 818 u8 *vaddr; 819 skb_frag_t *f = &skb_shinfo(skb)->frags[i]; 820 821 page = alloc_page(gfp_mask); 822 if (!page) { 823 while (head) { 824 struct page *next = (struct page *)page_private(head); 825 put_page(head); 826 head = next; 827 } 828 return -ENOMEM; 829 } 830 vaddr = kmap_atomic(skb_frag_page(f)); 831 memcpy(page_address(page), 832 vaddr + f->page_offset, skb_frag_size(f)); 833 kunmap_atomic(vaddr); 834 set_page_private(page, (unsigned long)head); 835 head = page; 836 } 837 838 /* skb frags release userspace buffers */ 839 for (i = 0; i < num_frags; i++) 840 skb_frag_unref(skb, i); 841 842 uarg->callback(uarg, false); 843 844 /* skb frags point to kernel buffers */ 845 for (i = num_frags - 1; i >= 0; i--) { 846 __skb_fill_page_desc(skb, i, head, 0, 847 skb_shinfo(skb)->frags[i].size); 848 head = (struct page *)page_private(head); 849 } 850 851 skb_shinfo(skb)->tx_flags &= ~SKBTX_DEV_ZEROCOPY; 852 return 0; 853} 854EXPORT_SYMBOL_GPL(skb_copy_ubufs); 855 856/** 857 * skb_clone - duplicate an sk_buff 858 * @skb: buffer to clone 859 * @gfp_mask: allocation priority 860 * 861 * Duplicate an &sk_buff. The new one is not owned by a socket. Both 862 * copies share the same packet data but not structure. The new 863 * buffer has a reference count of 1. If the allocation fails the 864 * function returns %NULL otherwise the new buffer is returned. 865 * 866 * If this function is called from an interrupt gfp_mask() must be 867 * %GFP_ATOMIC. 868 */ 869 870struct sk_buff *skb_clone(struct sk_buff *skb, gfp_t gfp_mask) 871{ 872 struct sk_buff_fclones *fclones = container_of(skb, 873 struct sk_buff_fclones, 874 skb1); 875 struct sk_buff *n = &fclones->skb2; 876 877 if (skb_orphan_frags(skb, gfp_mask)) 878 return NULL; 879 880 if (skb->fclone == SKB_FCLONE_ORIG && 881 n->fclone == SKB_FCLONE_FREE) { 882 n->fclone = SKB_FCLONE_CLONE; 883 atomic_inc(&fclones->fclone_ref); 884 } else { 885 if (skb_pfmemalloc(skb)) 886 gfp_mask |= __GFP_MEMALLOC; 887 888 n = kmem_cache_alloc(skbuff_head_cache, gfp_mask); 889 if (!n) 890 return NULL; 891 892 kmemcheck_annotate_bitfield(n, flags1); 893 n->fclone = SKB_FCLONE_UNAVAILABLE; 894 } 895 896 return __skb_clone(n, skb); 897} 898EXPORT_SYMBOL(skb_clone); 899 900static void skb_headers_offset_update(struct sk_buff *skb, int off) 901{ 902 /* Only adjust this if it actually is csum_start rather than csum */ 903 if (skb->ip_summed == CHECKSUM_PARTIAL) 904 skb->csum_start += off; 905 /* {transport,network,mac}_header and tail are relative to skb->head */ 906 skb->transport_header += off; 907 skb->network_header += off; 908 if (skb_mac_header_was_set(skb)) 909 skb->mac_header += off; 910 skb->inner_transport_header += off; 911 skb->inner_network_header += off; 912 skb->inner_mac_header += off; 913} 914 915static void copy_skb_header(struct sk_buff *new, const struct sk_buff *old) 916{ 917 __copy_skb_header(new, old); 918 919 skb_shinfo(new)->gso_size = skb_shinfo(old)->gso_size; 920 skb_shinfo(new)->gso_segs = skb_shinfo(old)->gso_segs; 921 skb_shinfo(new)->gso_type = skb_shinfo(old)->gso_type; 922} 923 924static inline int skb_alloc_rx_flag(const struct sk_buff *skb) 925{ 926 if (skb_pfmemalloc(skb)) 927 return SKB_ALLOC_RX; 928 return 0; 929} 930 931/** 932 * skb_copy - create private copy of an sk_buff 933 * @skb: buffer to copy 934 * @gfp_mask: allocation priority 935 * 936 * Make a copy of both an &sk_buff and its data. This is used when the 937 * caller wishes to modify the data and needs a private copy of the 938 * data to alter. Returns %NULL on failure or the pointer to the buffer 939 * on success. The returned buffer has a reference count of 1. 940 * 941 * As by-product this function converts non-linear &sk_buff to linear 942 * one, so that &sk_buff becomes completely private and caller is allowed 943 * to modify all the data of returned buffer. This means that this 944 * function is not recommended for use in circumstances when only 945 * header is going to be modified. Use pskb_copy() instead. 946 */ 947 948struct sk_buff *skb_copy(const struct sk_buff *skb, gfp_t gfp_mask) 949{ 950 int headerlen = skb_headroom(skb); 951 unsigned int size = skb_end_offset(skb) + skb->data_len; 952 struct sk_buff *n = __alloc_skb(size, gfp_mask, 953 skb_alloc_rx_flag(skb), NUMA_NO_NODE); 954 955 if (!n) 956 return NULL; 957 958 /* Set the data pointer */ 959 skb_reserve(n, headerlen); 960 /* Set the tail pointer and length */ 961 skb_put(n, skb->len); 962 963 if (skb_copy_bits(skb, -headerlen, n->head, headerlen + skb->len)) 964 BUG(); 965 966 copy_skb_header(n, skb); 967 return n; 968} 969EXPORT_SYMBOL(skb_copy); 970 971/** 972 * __pskb_copy_fclone - create copy of an sk_buff with private head. 973 * @skb: buffer to copy 974 * @headroom: headroom of new skb 975 * @gfp_mask: allocation priority 976 * @fclone: if true allocate the copy of the skb from the fclone 977 * cache instead of the head cache; it is recommended to set this 978 * to true for the cases where the copy will likely be cloned 979 * 980 * Make a copy of both an &sk_buff and part of its data, located 981 * in header. Fragmented data remain shared. This is used when 982 * the caller wishes to modify only header of &sk_buff and needs 983 * private copy of the header to alter. Returns %NULL on failure 984 * or the pointer to the buffer on success. 985 * The returned buffer has a reference count of 1. 986 */ 987 988struct sk_buff *__pskb_copy_fclone(struct sk_buff *skb, int headroom, 989 gfp_t gfp_mask, bool fclone) 990{ 991 unsigned int size = skb_headlen(skb) + headroom; 992 int flags = skb_alloc_rx_flag(skb) | (fclone ? SKB_ALLOC_FCLONE : 0); 993 struct sk_buff *n = __alloc_skb(size, gfp_mask, flags, NUMA_NO_NODE); 994 995 if (!n) 996 goto out; 997 998 /* Set the data pointer */ 999 skb_reserve(n, headroom); 1000 /* Set the tail pointer and length */ 1001 skb_put(n, skb_headlen(skb)); 1002 /* Copy the bytes */ 1003 skb_copy_from_linear_data(skb, n->data, n->len); 1004 1005 n->truesize += skb->data_len; 1006 n->data_len = skb->data_len; 1007 n->len = skb->len; 1008 1009 if (skb_shinfo(skb)->nr_frags) { 1010 int i; 1011 1012 if (skb_orphan_frags(skb, gfp_mask)) { 1013 kfree_skb(n); 1014 n = NULL; 1015 goto out; 1016 } 1017 for (i = 0; i < skb_shinfo(skb)->nr_frags; i++) { 1018 skb_shinfo(n)->frags[i] = skb_shinfo(skb)->frags[i]; 1019 skb_frag_ref(skb, i); 1020 } 1021 skb_shinfo(n)->nr_frags = i; 1022 } 1023 1024 if (skb_has_frag_list(skb)) { 1025 skb_shinfo(n)->frag_list = skb_shinfo(skb)->frag_list; 1026 skb_clone_fraglist(n); 1027 } 1028 1029 copy_skb_header(n, skb); 1030out: 1031 return n; 1032} 1033EXPORT_SYMBOL(__pskb_copy_fclone); 1034 1035/** 1036 * pskb_expand_head - reallocate header of &sk_buff 1037 * @skb: buffer to reallocate 1038 * @nhead: room to add at head 1039 * @ntail: room to add at tail 1040 * @gfp_mask: allocation priority 1041 * 1042 * Expands (or creates identical copy, if @nhead and @ntail are zero) 1043 * header of @skb. &sk_buff itself is not changed. &sk_buff MUST have 1044 * reference count of 1. Returns zero in the case of success or error, 1045 * if expansion failed. In the last case, &sk_buff is not changed. 1046 * 1047 * All the pointers pointing into skb header may change and must be 1048 * reloaded after call to this function. 1049 */ 1050 1051int pskb_expand_head(struct sk_buff *skb, int nhead, int ntail, 1052 gfp_t gfp_mask) 1053{ 1054 int i; 1055 u8 *data; 1056 int size = nhead + skb_end_offset(skb) + ntail; 1057 long off; 1058 1059 BUG_ON(nhead < 0); 1060 1061 if (skb_shared(skb)) 1062 BUG(); 1063 1064 size = SKB_DATA_ALIGN(size); 1065 1066 if (skb_pfmemalloc(skb)) 1067 gfp_mask |= __GFP_MEMALLOC; 1068 data = kmalloc_reserve(size + SKB_DATA_ALIGN(sizeof(struct skb_shared_info)), 1069 gfp_mask, NUMA_NO_NODE, NULL); 1070 if (!data) 1071 goto nodata; 1072 size = SKB_WITH_OVERHEAD(ksize(data)); 1073 1074 /* Copy only real data... and, alas, header. This should be 1075 * optimized for the cases when header is void. 1076 */ 1077 memcpy(data + nhead, skb->head, skb_tail_pointer(skb) - skb->head); 1078 1079 memcpy((struct skb_shared_info *)(data + size), 1080 skb_shinfo(skb), 1081 offsetof(struct skb_shared_info, frags[skb_shinfo(skb)->nr_frags])); 1082 1083 /* 1084 * if shinfo is shared we must drop the old head gracefully, but if it 1085 * is not we can just drop the old head and let the existing refcount 1086 * be since all we did is relocate the values 1087 */ 1088 if (skb_cloned(skb)) { 1089 /* copy this zero copy skb frags */ 1090 if (skb_orphan_frags(skb, gfp_mask)) 1091 goto nofrags; 1092 for (i = 0; i < skb_shinfo(skb)->nr_frags; i++) 1093 skb_frag_ref(skb, i); 1094 1095 if (skb_has_frag_list(skb)) 1096 skb_clone_fraglist(skb); 1097 1098 skb_release_data(skb); 1099 } else { 1100 skb_free_head(skb); 1101 } 1102 off = (data + nhead) - skb->head; 1103 1104 skb->head = data; 1105 skb->head_frag = 0; 1106 skb->data += off; 1107#ifdef NET_SKBUFF_DATA_USES_OFFSET 1108 skb->end = size; 1109 off = nhead; 1110#else 1111 skb->end = skb->head + size; 1112#endif 1113 skb->tail += off; 1114 skb_headers_offset_update(skb, nhead); 1115 skb->cloned = 0; 1116 skb->hdr_len = 0; 1117 skb->nohdr = 0; 1118 atomic_set(&skb_shinfo(skb)->dataref, 1); 1119 return 0; 1120 1121nofrags: 1122 kfree(data); 1123nodata: 1124 return -ENOMEM; 1125} 1126EXPORT_SYMBOL(pskb_expand_head); 1127 1128/* Make private copy of skb with writable head and some headroom */ 1129 1130struct sk_buff *skb_realloc_headroom(struct sk_buff *skb, unsigned int headroom) 1131{ 1132 struct sk_buff *skb2; 1133 int delta = headroom - skb_headroom(skb); 1134 1135 if (delta <= 0) 1136 skb2 = pskb_copy(skb, GFP_ATOMIC); 1137 else { 1138 skb2 = skb_clone(skb, GFP_ATOMIC); 1139 if (skb2 && pskb_expand_head(skb2, SKB_DATA_ALIGN(delta), 0, 1140 GFP_ATOMIC)) { 1141 kfree_skb(skb2); 1142 skb2 = NULL; 1143 } 1144 } 1145 return skb2; 1146} 1147EXPORT_SYMBOL(skb_realloc_headroom); 1148 1149/** 1150 * skb_copy_expand - copy and expand sk_buff 1151 * @skb: buffer to copy 1152 * @newheadroom: new free bytes at head 1153 * @newtailroom: new free bytes at tail 1154 * @gfp_mask: allocation priority 1155 * 1156 * Make a copy of both an &sk_buff and its data and while doing so 1157 * allocate additional space. 1158 * 1159 * This is used when the caller wishes to modify the data and needs a 1160 * private copy of the data to alter as well as more space for new fields. 1161 * Returns %NULL on failure or the pointer to the buffer 1162 * on success. The returned buffer has a reference count of 1. 1163 * 1164 * You must pass %GFP_ATOMIC as the allocation priority if this function 1165 * is called from an interrupt. 1166 */ 1167struct sk_buff *skb_copy_expand(const struct sk_buff *skb, 1168 int newheadroom, int newtailroom, 1169 gfp_t gfp_mask) 1170{ 1171 /* 1172 * Allocate the copy buffer 1173 */ 1174 struct sk_buff *n = __alloc_skb(newheadroom + skb->len + newtailroom, 1175 gfp_mask, skb_alloc_rx_flag(skb), 1176 NUMA_NO_NODE); 1177 int oldheadroom = skb_headroom(skb); 1178 int head_copy_len, head_copy_off; 1179 1180 if (!n) 1181 return NULL; 1182 1183 skb_reserve(n, newheadroom); 1184 1185 /* Set the tail pointer and length */ 1186 skb_put(n, skb->len); 1187 1188 head_copy_len = oldheadroom; 1189 head_copy_off = 0; 1190 if (newheadroom <= head_copy_len) 1191 head_copy_len = newheadroom; 1192 else 1193 head_copy_off = newheadroom - head_copy_len; 1194 1195 /* Copy the linear header and data. */ 1196 if (skb_copy_bits(skb, -head_copy_len, n->head + head_copy_off, 1197 skb->len + head_copy_len)) 1198 BUG(); 1199 1200 copy_skb_header(n, skb); 1201 1202 skb_headers_offset_update(n, newheadroom - oldheadroom); 1203 1204 return n; 1205} 1206EXPORT_SYMBOL(skb_copy_expand); 1207 1208/** 1209 * skb_pad - zero pad the tail of an skb 1210 * @skb: buffer to pad 1211 * @pad: space to pad 1212 * 1213 * Ensure that a buffer is followed by a padding area that is zero 1214 * filled. Used by network drivers which may DMA or transfer data 1215 * beyond the buffer end onto the wire. 1216 * 1217 * May return error in out of memory cases. The skb is freed on error. 1218 */ 1219 1220int skb_pad(struct sk_buff *skb, int pad) 1221{ 1222 int err; 1223 int ntail; 1224 1225 /* If the skbuff is non linear tailroom is always zero.. */ 1226 if (!skb_cloned(skb) && skb_tailroom(skb) >= pad) { 1227 memset(skb->data+skb->len, 0, pad); 1228 return 0; 1229 } 1230 1231 ntail = skb->data_len + pad - (skb->end - skb->tail); 1232 if (likely(skb_cloned(skb) || ntail > 0)) { 1233 err = pskb_expand_head(skb, 0, ntail, GFP_ATOMIC); 1234 if (unlikely(err)) 1235 goto free_skb; 1236 } 1237 1238 /* FIXME: The use of this function with non-linear skb's really needs 1239 * to be audited. 1240 */ 1241 err = skb_linearize(skb); 1242 if (unlikely(err)) 1243 goto free_skb; 1244 1245 memset(skb->data + skb->len, 0, pad); 1246 return 0; 1247 1248free_skb: 1249 kfree_skb(skb); 1250 return err; 1251} 1252EXPORT_SYMBOL(skb_pad); 1253 1254/** 1255 * pskb_put - add data to the tail of a potentially fragmented buffer 1256 * @skb: start of the buffer to use 1257 * @tail: tail fragment of the buffer to use 1258 * @len: amount of data to add 1259 * 1260 * This function extends the used data area of the potentially 1261 * fragmented buffer. @tail must be the last fragment of @skb -- or 1262 * @skb itself. If this would exceed the total buffer size the kernel 1263 * will panic. A pointer to the first byte of the extra data is 1264 * returned. 1265 */ 1266 1267unsigned char *pskb_put(struct sk_buff *skb, struct sk_buff *tail, int len) 1268{ 1269 if (tail != skb) { 1270 skb->data_len += len; 1271 skb->len += len; 1272 } 1273 return skb_put(tail, len); 1274} 1275EXPORT_SYMBOL_GPL(pskb_put); 1276 1277/** 1278 * skb_put - add data to a buffer 1279 * @skb: buffer to use 1280 * @len: amount of data to add 1281 * 1282 * This function extends the used data area of the buffer. If this would 1283 * exceed the total buffer size the kernel will panic. A pointer to the 1284 * first byte of the extra data is returned. 1285 */ 1286unsigned char *skb_put(struct sk_buff *skb, unsigned int len) 1287{ 1288 unsigned char *tmp = skb_tail_pointer(skb); 1289 SKB_LINEAR_ASSERT(skb); 1290 skb->tail += len; 1291 skb->len += len; 1292 if (unlikely(skb->tail > skb->end)) 1293 skb_over_panic(skb, len, __builtin_return_address(0)); 1294 return tmp; 1295} 1296EXPORT_SYMBOL(skb_put); 1297 1298/** 1299 * skb_push - add data to the start of a buffer 1300 * @skb: buffer to use 1301 * @len: amount of data to add 1302 * 1303 * This function extends the used data area of the buffer at the buffer 1304 * start. If this would exceed the total buffer headroom the kernel will 1305 * panic. A pointer to the first byte of the extra data is returned. 1306 */ 1307unsigned char *skb_push(struct sk_buff *skb, unsigned int len) 1308{ 1309 skb->data -= len; 1310 skb->len += len; 1311 if (unlikely(skb->data<skb->head)) 1312 skb_under_panic(skb, len, __builtin_return_address(0)); 1313 return skb->data; 1314} 1315EXPORT_SYMBOL(skb_push); 1316 1317/** 1318 * skb_pull - remove data from the start of a buffer 1319 * @skb: buffer to use 1320 * @len: amount of data to remove 1321 * 1322 * This function removes data from the start of a buffer, returning 1323 * the memory to the headroom. A pointer to the next data in the buffer 1324 * is returned. Once the data has been pulled future pushes will overwrite 1325 * the old data. 1326 */ 1327unsigned char *skb_pull(struct sk_buff *skb, unsigned int len) 1328{ 1329 return skb_pull_inline(skb, len); 1330} 1331EXPORT_SYMBOL(skb_pull); 1332 1333/** 1334 * skb_trim - remove end from a buffer 1335 * @skb: buffer to alter 1336 * @len: new length 1337 * 1338 * Cut the length of a buffer down by removing data from the tail. If 1339 * the buffer is already under the length specified it is not modified. 1340 * The skb must be linear. 1341 */ 1342void skb_trim(struct sk_buff *skb, unsigned int len) 1343{ 1344 if (skb->len > len) 1345 __skb_trim(skb, len); 1346} 1347EXPORT_SYMBOL(skb_trim); 1348 1349/* Trims skb to length len. It can change skb pointers. 1350 */ 1351 1352int ___pskb_trim(struct sk_buff *skb, unsigned int len) 1353{ 1354 struct sk_buff **fragp; 1355 struct sk_buff *frag; 1356 int offset = skb_headlen(skb); 1357 int nfrags = skb_shinfo(skb)->nr_frags; 1358 int i; 1359 int err; 1360 1361 if (skb_cloned(skb) && 1362 unlikely((err = pskb_expand_head(skb, 0, 0, GFP_ATOMIC)))) 1363 return err; 1364 1365 i = 0; 1366 if (offset >= len) 1367 goto drop_pages; 1368 1369 for (; i < nfrags; i++) { 1370 int end = offset + skb_frag_size(&skb_shinfo(skb)->frags[i]); 1371 1372 if (end < len) { 1373 offset = end; 1374 continue; 1375 } 1376 1377 skb_frag_size_set(&skb_shinfo(skb)->frags[i++], len - offset); 1378 1379drop_pages: 1380 skb_shinfo(skb)->nr_frags = i; 1381 1382 for (; i < nfrags; i++) 1383 skb_frag_unref(skb, i); 1384 1385 if (skb_has_frag_list(skb)) 1386 skb_drop_fraglist(skb); 1387 goto done; 1388 } 1389 1390 for (fragp = &skb_shinfo(skb)->frag_list; (frag = *fragp); 1391 fragp = &frag->next) { 1392 int end = offset + frag->len; 1393 1394 if (skb_shared(frag)) { 1395 struct sk_buff *nfrag; 1396 1397 nfrag = skb_clone(frag, GFP_ATOMIC); 1398 if (unlikely(!nfrag)) 1399 return -ENOMEM; 1400 1401 nfrag->next = frag->next; 1402 consume_skb(frag); 1403 frag = nfrag; 1404 *fragp = frag; 1405 } 1406 1407 if (end < len) { 1408 offset = end; 1409 continue; 1410 } 1411 1412 if (end > len && 1413 unlikely((err = pskb_trim(frag, len - offset)))) 1414 return err; 1415 1416 if (frag->next) 1417 skb_drop_list(&frag->next); 1418 break; 1419 } 1420 1421done: 1422 if (len > skb_headlen(skb)) { 1423 skb->data_len -= skb->len - len; 1424 skb->len = len; 1425 } else { 1426 skb->len = len; 1427 skb->data_len = 0; 1428 skb_set_tail_pointer(skb, len); 1429 } 1430 1431 return 0; 1432} 1433EXPORT_SYMBOL(___pskb_trim); 1434 1435/** 1436 * __pskb_pull_tail - advance tail of skb header 1437 * @skb: buffer to reallocate 1438 * @delta: number of bytes to advance tail 1439 * 1440 * The function makes a sense only on a fragmented &sk_buff, 1441 * it expands header moving its tail forward and copying necessary 1442 * data from fragmented part. 1443 * 1444 * &sk_buff MUST have reference count of 1. 1445 * 1446 * Returns %NULL (and &sk_buff does not change) if pull failed 1447 * or value of new tail of skb in the case of success. 1448 * 1449 * All the pointers pointing into skb header may change and must be 1450 * reloaded after call to this function. 1451 */ 1452 1453/* Moves tail of skb head forward, copying data from fragmented part, 1454 * when it is necessary. 1455 * 1. It may fail due to malloc failure. 1456 * 2. It may change skb pointers. 1457 * 1458 * It is pretty complicated. Luckily, it is called only in exceptional cases. 1459 */ 1460unsigned char *__pskb_pull_tail(struct sk_buff *skb, int delta) 1461{ 1462 /* If skb has not enough free space at tail, get new one 1463 * plus 128 bytes for future expansions. If we have enough 1464 * room at tail, reallocate without expansion only if skb is cloned. 1465 */ 1466 int i, k, eat = (skb->tail + delta) - skb->end; 1467 1468 if (eat > 0 || skb_cloned(skb)) { 1469 if (pskb_expand_head(skb, 0, eat > 0 ? eat + 128 : 0, 1470 GFP_ATOMIC)) 1471 return NULL; 1472 } 1473 1474 if (skb_copy_bits(skb, skb_headlen(skb), skb_tail_pointer(skb), delta)) 1475 BUG(); 1476 1477 /* Optimization: no fragments, no reasons to preestimate 1478 * size of pulled pages. Superb. 1479 */ 1480 if (!skb_has_frag_list(skb)) 1481 goto pull_pages; 1482 1483 /* Estimate size of pulled pages. */ 1484 eat = delta; 1485 for (i = 0; i < skb_shinfo(skb)->nr_frags; i++) { 1486 int size = skb_frag_size(&skb_shinfo(skb)->frags[i]); 1487 1488 if (size >= eat) 1489 goto pull_pages; 1490 eat -= size; 1491 } 1492 1493 /* If we need update frag list, we are in troubles. 1494 * Certainly, it possible to add an offset to skb data, 1495 * but taking into account that pulling is expected to 1496 * be very rare operation, it is worth to fight against 1497 * further bloating skb head and crucify ourselves here instead. 1498 * Pure masohism, indeed. 8)8) 1499 */ 1500 if (eat) { 1501 struct sk_buff *list = skb_shinfo(skb)->frag_list; 1502 struct sk_buff *clone = NULL; 1503 struct sk_buff *insp = NULL; 1504 1505 do { 1506 BUG_ON(!list); 1507 1508 if (list->len <= eat) { 1509 /* Eaten as whole. */ 1510 eat -= list->len; 1511 list = list->next; 1512 insp = list; 1513 } else { 1514 /* Eaten partially. */ 1515 1516 if (skb_shared(list)) { 1517 /* Sucks! We need to fork list. :-( */ 1518 clone = skb_clone(list, GFP_ATOMIC); 1519 if (!clone) 1520 return NULL; 1521 insp = list->next; 1522 list = clone; 1523 } else { 1524 /* This may be pulled without 1525 * problems. */ 1526 insp = list; 1527 } 1528 if (!pskb_pull(list, eat)) { 1529 kfree_skb(clone); 1530 return NULL; 1531 } 1532 break; 1533 } 1534 } while (eat); 1535 1536 /* Free pulled out fragments. */ 1537 while ((list = skb_shinfo(skb)->frag_list) != insp) { 1538 skb_shinfo(skb)->frag_list = list->next; 1539 kfree_skb(list); 1540 } 1541 /* And insert new clone at head. */ 1542 if (clone) { 1543 clone->next = list; 1544 skb_shinfo(skb)->frag_list = clone; 1545 } 1546 } 1547 /* Success! Now we may commit changes to skb data. */ 1548 1549pull_pages: 1550 eat = delta; 1551 k = 0; 1552 for (i = 0; i < skb_shinfo(skb)->nr_frags; i++) { 1553 int size = skb_frag_size(&skb_shinfo(skb)->frags[i]); 1554 1555 if (size <= eat) { 1556 skb_frag_unref(skb, i); 1557 eat -= size; 1558 } else { 1559 skb_shinfo(skb)->frags[k] = skb_shinfo(skb)->frags[i]; 1560 if (eat) { 1561 skb_shinfo(skb)->frags[k].page_offset += eat; 1562 skb_frag_size_sub(&skb_shinfo(skb)->frags[k], eat); 1563 eat = 0; 1564 } 1565 k++; 1566 } 1567 } 1568 skb_shinfo(skb)->nr_frags = k; 1569 1570 skb->tail += delta; 1571 skb->data_len -= delta; 1572 1573 return skb_tail_pointer(skb); 1574} 1575EXPORT_SYMBOL(__pskb_pull_tail); 1576 1577/** 1578 * skb_copy_bits - copy bits from skb to kernel buffer 1579 * @skb: source skb 1580 * @offset: offset in source 1581 * @to: destination buffer 1582 * @len: number of bytes to copy 1583 * 1584 * Copy the specified number of bytes from the source skb to the 1585 * destination buffer. 1586 * 1587 * CAUTION ! : 1588 * If its prototype is ever changed, 1589 * check arch/{*}/net/{*}.S files, 1590 * since it is called from BPF assembly code. 1591 */ 1592int skb_copy_bits(const struct sk_buff *skb, int offset, void *to, int len) 1593{ 1594 int start = skb_headlen(skb); 1595 struct sk_buff *frag_iter; 1596 int i, copy; 1597 1598 if (offset > (int)skb->len - len) 1599 goto fault; 1600 1601 /* Copy header. */ 1602 if ((copy = start - offset) > 0) { 1603 if (copy > len) 1604 copy = len; 1605 skb_copy_from_linear_data_offset(skb, offset, to, copy); 1606 if ((len -= copy) == 0) 1607 return 0; 1608 offset += copy; 1609 to += copy; 1610 } 1611 1612 for (i = 0; i < skb_shinfo(skb)->nr_frags; i++) { 1613 int end; 1614 skb_frag_t *f = &skb_shinfo(skb)->frags[i]; 1615 1616 WARN_ON(start > offset + len); 1617 1618 end = start + skb_frag_size(f); 1619 if ((copy = end - offset) > 0) { 1620 u8 *vaddr; 1621 1622 if (copy > len) 1623 copy = len; 1624 1625 vaddr = kmap_atomic(skb_frag_page(f)); 1626 memcpy(to, 1627 vaddr + f->page_offset + offset - start, 1628 copy); 1629 kunmap_atomic(vaddr); 1630 1631 if ((len -= copy) == 0) 1632 return 0; 1633 offset += copy; 1634 to += copy; 1635 } 1636 start = end; 1637 } 1638 1639 skb_walk_frags(skb, frag_iter) { 1640 int end; 1641 1642 WARN_ON(start > offset + len); 1643 1644 end = start + frag_iter->len; 1645 if ((copy = end - offset) > 0) { 1646 if (copy > len) 1647 copy = len; 1648 if (skb_copy_bits(frag_iter, offset - start, to, copy)) 1649 goto fault; 1650 if ((len -= copy) == 0) 1651 return 0; 1652 offset += copy; 1653 to += copy; 1654 } 1655 start = end; 1656 } 1657 1658 if (!len) 1659 return 0; 1660 1661fault: 1662 return -EFAULT; 1663} 1664EXPORT_SYMBOL(skb_copy_bits); 1665 1666/* 1667 * Callback from splice_to_pipe(), if we need to release some pages 1668 * at the end of the spd in case we error'ed out in filling the pipe. 1669 */ 1670static void sock_spd_release(struct splice_pipe_desc *spd, unsigned int i) 1671{ 1672 put_page(spd->pages[i]); 1673} 1674 1675static struct page *linear_to_page(struct page *page, unsigned int *len, 1676 unsigned int *offset, 1677 struct sock *sk) 1678{ 1679 struct page_frag *pfrag = sk_page_frag(sk); 1680 1681 if (!sk_page_frag_refill(sk, pfrag)) 1682 return NULL; 1683 1684 *len = min_t(unsigned int, *len, pfrag->size - pfrag->offset); 1685 1686 memcpy(page_address(pfrag->page) + pfrag->offset, 1687 page_address(page) + *offset, *len); 1688 *offset = pfrag->offset; 1689 pfrag->offset += *len; 1690 1691 return pfrag->page; 1692} 1693 1694static bool spd_can_coalesce(const struct splice_pipe_desc *spd, 1695 struct page *page, 1696 unsigned int offset) 1697{ 1698 return spd->nr_pages && 1699 spd->pages[spd->nr_pages - 1] == page && 1700 (spd->partial[spd->nr_pages - 1].offset + 1701 spd->partial[spd->nr_pages - 1].len == offset); 1702} 1703 1704/* 1705 * Fill page/offset/length into spd, if it can hold more pages. 1706 */ 1707static bool spd_fill_page(struct splice_pipe_desc *spd, 1708 struct pipe_inode_info *pipe, struct page *page, 1709 unsigned int *len, unsigned int offset, 1710 bool linear, 1711 struct sock *sk) 1712{ 1713 if (unlikely(spd->nr_pages == MAX_SKB_FRAGS)) 1714 return true; 1715 1716 if (linear) { 1717 page = linear_to_page(page, len, &offset, sk); 1718 if (!page) 1719 return true; 1720 } 1721 if (spd_can_coalesce(spd, page, offset)) { 1722 spd->partial[spd->nr_pages - 1].len += *len; 1723 return false; 1724 } 1725 get_page(page); 1726 spd->pages[spd->nr_pages] = page; 1727 spd->partial[spd->nr_pages].len = *len; 1728 spd->partial[spd->nr_pages].offset = offset; 1729 spd->nr_pages++; 1730 1731 return false; 1732} 1733 1734static bool __splice_segment(struct page *page, unsigned int poff, 1735 unsigned int plen, unsigned int *off, 1736 unsigned int *len, 1737 struct splice_pipe_desc *spd, bool linear, 1738 struct sock *sk, 1739 struct pipe_inode_info *pipe) 1740{ 1741 if (!*len) 1742 return true; 1743 1744 /* skip this segment if already processed */ 1745 if (*off >= plen) { 1746 *off -= plen; 1747 return false; 1748 } 1749 1750 /* ignore any bits we already processed */ 1751 poff += *off; 1752 plen -= *off; 1753 *off = 0; 1754 1755 do { 1756 unsigned int flen = min(*len, plen); 1757 1758 if (spd_fill_page(spd, pipe, page, &flen, poff, 1759 linear, sk)) 1760 return true; 1761 poff += flen; 1762 plen -= flen; 1763 *len -= flen; 1764 } while (*len && plen); 1765 1766 return false; 1767} 1768 1769/* 1770 * Map linear and fragment data from the skb to spd. It reports true if the 1771 * pipe is full or if we already spliced the requested length. 1772 */ 1773static bool __skb_splice_bits(struct sk_buff *skb, struct pipe_inode_info *pipe, 1774 unsigned int *offset, unsigned int *len, 1775 struct splice_pipe_desc *spd, struct sock *sk) 1776{ 1777 int seg; 1778 1779 /* map the linear part : 1780 * If skb->head_frag is set, this 'linear' part is backed by a 1781 * fragment, and if the head is not shared with any clones then 1782 * we can avoid a copy since we own the head portion of this page. 1783 */ 1784 if (__splice_segment(virt_to_page(skb->data), 1785 (unsigned long) skb->data & (PAGE_SIZE - 1), 1786 skb_headlen(skb), 1787 offset, len, spd, 1788 skb_head_is_locked(skb), 1789 sk, pipe)) 1790 return true; 1791 1792 /* 1793 * then map the fragments 1794 */ 1795 for (seg = 0; seg < skb_shinfo(skb)->nr_frags; seg++) { 1796 const skb_frag_t *f = &skb_shinfo(skb)->frags[seg]; 1797 1798 if (__splice_segment(skb_frag_page(f), 1799 f->page_offset, skb_frag_size(f), 1800 offset, len, spd, false, sk, pipe)) 1801 return true; 1802 } 1803 1804 return false; 1805} 1806 1807/* 1808 * Map data from the skb to a pipe. Should handle both the linear part, 1809 * the fragments, and the frag list. It does NOT handle frag lists within 1810 * the frag list, if such a thing exists. We'd probably need to recurse to 1811 * handle that cleanly. 1812 */ 1813int skb_splice_bits(struct sk_buff *skb, unsigned int offset, 1814 struct pipe_inode_info *pipe, unsigned int tlen, 1815 unsigned int flags) 1816{ 1817 struct partial_page partial[MAX_SKB_FRAGS]; 1818 struct page *pages[MAX_SKB_FRAGS]; 1819 struct splice_pipe_desc spd = { 1820 .pages = pages, 1821 .partial = partial, 1822 .nr_pages_max = MAX_SKB_FRAGS, 1823 .flags = flags, 1824 .ops = &nosteal_pipe_buf_ops, 1825 .spd_release = sock_spd_release, 1826 }; 1827 struct sk_buff *frag_iter; 1828 struct sock *sk = skb->sk; 1829 int ret = 0; 1830 1831 /* 1832 * __skb_splice_bits() only fails if the output has no room left, 1833 * so no point in going over the frag_list for the error case. 1834 */ 1835 if (__skb_splice_bits(skb, pipe, &offset, &tlen, &spd, sk)) 1836 goto done; 1837 else if (!tlen) 1838 goto done; 1839 1840 /* 1841 * now see if we have a frag_list to map 1842 */ 1843 skb_walk_frags(skb, frag_iter) { 1844 if (!tlen) 1845 break; 1846 if (__skb_splice_bits(frag_iter, pipe, &offset, &tlen, &spd, sk)) 1847 break; 1848 } 1849 1850done: 1851 if (spd.nr_pages) { 1852 /* 1853 * Drop the socket lock, otherwise we have reverse 1854 * locking dependencies between sk_lock and i_mutex 1855 * here as compared to sendfile(). We enter here 1856 * with the socket lock held, and splice_to_pipe() will 1857 * grab the pipe inode lock. For sendfile() emulation, 1858 * we call into ->sendpage() with the i_mutex lock held 1859 * and networking will grab the socket lock. 1860 */ 1861 release_sock(sk); 1862 ret = splice_to_pipe(pipe, &spd); 1863 lock_sock(sk); 1864 } 1865 1866 return ret; 1867} 1868 1869/** 1870 * skb_store_bits - store bits from kernel buffer to skb 1871 * @skb: destination buffer 1872 * @offset: offset in destination 1873 * @from: source buffer 1874 * @len: number of bytes to copy 1875 * 1876 * Copy the specified number of bytes from the source buffer to the 1877 * destination skb. This function handles all the messy bits of 1878 * traversing fragment lists and such. 1879 */ 1880 1881int skb_store_bits(struct sk_buff *skb, int offset, const void *from, int len) 1882{ 1883 int start = skb_headlen(skb); 1884 struct sk_buff *frag_iter; 1885 int i, copy; 1886 1887 if (offset > (int)skb->len - len) 1888 goto fault; 1889 1890 if ((copy = start - offset) > 0) { 1891 if (copy > len) 1892 copy = len; 1893 skb_copy_to_linear_data_offset(skb, offset, from, copy); 1894 if ((len -= copy) == 0) 1895 return 0; 1896 offset += copy; 1897 from += copy; 1898 } 1899 1900 for (i = 0; i < skb_shinfo(skb)->nr_frags; i++) { 1901 skb_frag_t *frag = &skb_shinfo(skb)->frags[i]; 1902 int end; 1903 1904 WARN_ON(start > offset + len); 1905 1906 end = start + skb_frag_size(frag); 1907 if ((copy = end - offset) > 0) { 1908 u8 *vaddr; 1909 1910 if (copy > len) 1911 copy = len; 1912 1913 vaddr = kmap_atomic(skb_frag_page(frag)); 1914 memcpy(vaddr + frag->page_offset + offset - start, 1915 from, copy); 1916 kunmap_atomic(vaddr); 1917 1918 if ((len -= copy) == 0) 1919 return 0; 1920 offset += copy; 1921 from += copy; 1922 } 1923 start = end; 1924 } 1925 1926 skb_walk_frags(skb, frag_iter) { 1927 int end; 1928 1929 WARN_ON(start > offset + len); 1930 1931 end = start + frag_iter->len; 1932 if ((copy = end - offset) > 0) { 1933 if (copy > len) 1934 copy = len; 1935 if (skb_store_bits(frag_iter, offset - start, 1936 from, copy)) 1937 goto fault; 1938 if ((len -= copy) == 0) 1939 return 0; 1940 offset += copy; 1941 from += copy; 1942 } 1943 start = end; 1944 } 1945 if (!len) 1946 return 0; 1947 1948fault: 1949 return -EFAULT; 1950} 1951EXPORT_SYMBOL(skb_store_bits); 1952 1953/* Checksum skb data. */ 1954__wsum __skb_checksum(const struct sk_buff *skb, int offset, int len, 1955 __wsum csum, const struct skb_checksum_ops *ops) 1956{ 1957 int start = skb_headlen(skb); 1958 int i, copy = start - offset; 1959 struct sk_buff *frag_iter; 1960 int pos = 0; 1961 1962 /* Checksum header. */ 1963 if (copy > 0) { 1964 if (copy > len) 1965 copy = len; 1966 csum = ops->update(skb->data + offset, copy, csum); 1967 if ((len -= copy) == 0) 1968 return csum; 1969 offset += copy; 1970 pos = copy; 1971 } 1972 1973 for (i = 0; i < skb_shinfo(skb)->nr_frags; i++) { 1974 int end; 1975 skb_frag_t *frag = &skb_shinfo(skb)->frags[i]; 1976 1977 WARN_ON(start > offset + len); 1978 1979 end = start + skb_frag_size(frag); 1980 if ((copy = end - offset) > 0) { 1981 __wsum csum2; 1982 u8 *vaddr; 1983 1984 if (copy > len) 1985 copy = len; 1986 vaddr = kmap_atomic(skb_frag_page(frag)); 1987 csum2 = ops->update(vaddr + frag->page_offset + 1988 offset - start, copy, 0); 1989 kunmap_atomic(vaddr); 1990 csum = ops->combine(csum, csum2, pos, copy); 1991 if (!(len -= copy)) 1992 return csum; 1993 offset += copy; 1994 pos += copy; 1995 } 1996 start = end; 1997 } 1998 1999 skb_walk_frags(skb, frag_iter) { 2000 int end; 2001 2002 WARN_ON(start > offset + len); 2003 2004 end = start + frag_iter->len; 2005 if ((copy = end - offset) > 0) { 2006 __wsum csum2; 2007 if (copy > len) 2008 copy = len; 2009 csum2 = __skb_checksum(frag_iter, offset - start, 2010 copy, 0, ops); 2011 csum = ops->combine(csum, csum2, pos, copy); 2012 if ((len -= copy) == 0) 2013 return csum; 2014 offset += copy; 2015 pos += copy; 2016 } 2017 start = end; 2018 } 2019 BUG_ON(len); 2020 2021 return csum; 2022} 2023EXPORT_SYMBOL(__skb_checksum); 2024 2025__wsum skb_checksum(const struct sk_buff *skb, int offset, 2026 int len, __wsum csum) 2027{ 2028 const struct skb_checksum_ops ops = { 2029 .update = csum_partial_ext, 2030 .combine = csum_block_add_ext, 2031 }; 2032 2033 return __skb_checksum(skb, offset, len, csum, &ops); 2034} 2035EXPORT_SYMBOL(skb_checksum); 2036 2037/* Both of above in one bottle. */ 2038 2039__wsum skb_copy_and_csum_bits(const struct sk_buff *skb, int offset, 2040 u8 *to, int len, __wsum csum) 2041{ 2042 int start = skb_headlen(skb); 2043 int i, copy = start - offset; 2044 struct sk_buff *frag_iter; 2045 int pos = 0; 2046 2047 /* Copy header. */ 2048 if (copy > 0) { 2049 if (copy > len) 2050 copy = len; 2051 csum = csum_partial_copy_nocheck(skb->data + offset, to, 2052 copy, csum); 2053 if ((len -= copy) == 0) 2054 return csum; 2055 offset += copy; 2056 to += copy; 2057 pos = copy; 2058 } 2059 2060 for (i = 0; i < skb_shinfo(skb)->nr_frags; i++) { 2061 int end; 2062 2063 WARN_ON(start > offset + len); 2064 2065 end = start + skb_frag_size(&skb_shinfo(skb)->frags[i]); 2066 if ((copy = end - offset) > 0) { 2067 __wsum csum2; 2068 u8 *vaddr; 2069 skb_frag_t *frag = &skb_shinfo(skb)->frags[i]; 2070 2071 if (copy > len) 2072 copy = len; 2073 vaddr = kmap_atomic(skb_frag_page(frag)); 2074 csum2 = csum_partial_copy_nocheck(vaddr + 2075 frag->page_offset + 2076 offset - start, to, 2077 copy, 0); 2078 kunmap_atomic(vaddr); 2079 csum = csum_block_add(csum, csum2, pos); 2080 if (!(len -= copy)) 2081 return csum; 2082 offset += copy; 2083 to += copy; 2084 pos += copy; 2085 } 2086 start = end; 2087 } 2088 2089 skb_walk_frags(skb, frag_iter) { 2090 __wsum csum2; 2091 int end; 2092 2093 WARN_ON(start > offset + len); 2094 2095 end = start + frag_iter->len; 2096 if ((copy = end - offset) > 0) { 2097 if (copy > len) 2098 copy = len; 2099 csum2 = skb_copy_and_csum_bits(frag_iter, 2100 offset - start, 2101 to, copy, 0); 2102 csum = csum_block_add(csum, csum2, pos); 2103 if ((len -= copy) == 0) 2104 return csum; 2105 offset += copy; 2106 to += copy; 2107 pos += copy; 2108 } 2109 start = end; 2110 } 2111 BUG_ON(len); 2112 return csum; 2113} 2114EXPORT_SYMBOL(skb_copy_and_csum_bits); 2115 2116 /** 2117 * skb_zerocopy_headlen - Calculate headroom needed for skb_zerocopy() 2118 * @from: source buffer 2119 * 2120 * Calculates the amount of linear headroom needed in the 'to' skb passed 2121 * into skb_zerocopy(). 2122 */ 2123unsigned int 2124skb_zerocopy_headlen(const struct sk_buff *from) 2125{ 2126 unsigned int hlen = 0; 2127 2128 if (!from->head_frag || 2129 skb_headlen(from) < L1_CACHE_BYTES || 2130 skb_shinfo(from)->nr_frags >= MAX_SKB_FRAGS) 2131 hlen = skb_headlen(from); 2132 2133 if (skb_has_frag_list(from)) 2134 hlen = from->len; 2135 2136 return hlen; 2137} 2138EXPORT_SYMBOL_GPL(skb_zerocopy_headlen); 2139 2140/** 2141 * skb_zerocopy - Zero copy skb to skb 2142 * @to: destination buffer 2143 * @from: source buffer 2144 * @len: number of bytes to copy from source buffer 2145 * @hlen: size of linear headroom in destination buffer 2146 * 2147 * Copies up to `len` bytes from `from` to `to` by creating references 2148 * to the frags in the source buffer. 2149 * 2150 * The `hlen` as calculated by skb_zerocopy_headlen() specifies the 2151 * headroom in the `to` buffer. 2152 * 2153 * Return value: 2154 * 0: everything is OK 2155 * -ENOMEM: couldn't orphan frags of @from due to lack of memory 2156 * -EFAULT: skb_copy_bits() found some problem with skb geometry 2157 */ 2158int 2159skb_zerocopy(struct sk_buff *to, struct sk_buff *from, int len, int hlen) 2160{ 2161 int i, j = 0; 2162 int plen = 0; /* length of skb->head fragment */ 2163 int ret; 2164 struct page *page; 2165 unsigned int offset; 2166 2167 BUG_ON(!from->head_frag && !hlen); 2168 2169 /* dont bother with small payloads */ 2170 if (len <= skb_tailroom(to)) 2171 return skb_copy_bits(from, 0, skb_put(to, len), len); 2172 2173 if (hlen) { 2174 ret = skb_copy_bits(from, 0, skb_put(to, hlen), hlen); 2175 if (unlikely(ret)) 2176 return ret; 2177 len -= hlen; 2178 } else { 2179 plen = min_t(int, skb_headlen(from), len); 2180 if (plen) { 2181 page = virt_to_head_page(from->head); 2182 offset = from->data - (unsigned char *)page_address(page); 2183 __skb_fill_page_desc(to, 0, page, offset, plen); 2184 get_page(page); 2185 j = 1; 2186 len -= plen; 2187 } 2188 } 2189 2190 to->truesize += len + plen; 2191 to->len += len + plen; 2192 to->data_len += len + plen; 2193 2194 if (unlikely(skb_orphan_frags(from, GFP_ATOMIC))) { 2195 skb_tx_error(from); 2196 return -ENOMEM; 2197 } 2198 2199 for (i = 0; i < skb_shinfo(from)->nr_frags; i++) { 2200 if (!len) 2201 break; 2202 skb_shinfo(to)->frags[j] = skb_shinfo(from)->frags[i]; 2203 skb_shinfo(to)->frags[j].size = min_t(int, skb_shinfo(to)->frags[j].size, len); 2204 len -= skb_shinfo(to)->frags[j].size; 2205 skb_frag_ref(to, j); 2206 j++; 2207 } 2208 skb_shinfo(to)->nr_frags = j; 2209 2210 return 0; 2211} 2212EXPORT_SYMBOL_GPL(skb_zerocopy); 2213 2214void skb_copy_and_csum_dev(const struct sk_buff *skb, u8 *to) 2215{ 2216 __wsum csum; 2217 long csstart; 2218 2219 if (skb->ip_summed == CHECKSUM_PARTIAL) 2220 csstart = skb_checksum_start_offset(skb); 2221 else 2222 csstart = skb_headlen(skb); 2223 2224 BUG_ON(csstart > skb_headlen(skb)); 2225 2226 skb_copy_from_linear_data(skb, to, csstart); 2227 2228 csum = 0; 2229 if (csstart != skb->len) 2230 csum = skb_copy_and_csum_bits(skb, csstart, to + csstart, 2231 skb->len - csstart, 0); 2232 2233 if (skb->ip_summed == CHECKSUM_PARTIAL) { 2234 long csstuff = csstart + skb->csum_offset; 2235 2236 *((__sum16 *)(to + csstuff)) = csum_fold(csum); 2237 } 2238} 2239EXPORT_SYMBOL(skb_copy_and_csum_dev); 2240 2241/** 2242 * skb_dequeue - remove from the head of the queue 2243 * @list: list to dequeue from 2244 * 2245 * Remove the head of the list. The list lock is taken so the function 2246 * may be used safely with other locking list functions. The head item is 2247 * returned or %NULL if the list is empty. 2248 */ 2249 2250struct sk_buff *skb_dequeue(struct sk_buff_head *list) 2251{ 2252 unsigned long flags; 2253 struct sk_buff *result; 2254 2255 spin_lock_irqsave(&list->lock, flags); 2256 result = __skb_dequeue(list); 2257 spin_unlock_irqrestore(&list->lock, flags); 2258 return result; 2259} 2260EXPORT_SYMBOL(skb_dequeue); 2261 2262/** 2263 * skb_dequeue_tail - remove from the tail of the queue 2264 * @list: list to dequeue from 2265 * 2266 * Remove the tail of the list. The list lock is taken so the function 2267 * may be used safely with other locking list functions. The tail item is 2268 * returned or %NULL if the list is empty. 2269 */ 2270struct sk_buff *skb_dequeue_tail(struct sk_buff_head *list) 2271{ 2272 unsigned long flags; 2273 struct sk_buff *result; 2274 2275 spin_lock_irqsave(&list->lock, flags); 2276 result = __skb_dequeue_tail(list); 2277 spin_unlock_irqrestore(&list->lock, flags); 2278 return result; 2279} 2280EXPORT_SYMBOL(skb_dequeue_tail); 2281 2282/** 2283 * skb_queue_purge - empty a list 2284 * @list: list to empty 2285 * 2286 * Delete all buffers on an &sk_buff list. Each buffer is removed from 2287 * the list and one reference dropped. This function takes the list 2288 * lock and is atomic with respect to other list locking functions. 2289 */ 2290void skb_queue_purge(struct sk_buff_head *list) 2291{ 2292 struct sk_buff *skb; 2293 while ((skb = skb_dequeue(list)) != NULL) 2294 kfree_skb(skb); 2295} 2296EXPORT_SYMBOL(skb_queue_purge); 2297 2298/** 2299 * skb_queue_head - queue a buffer at the list head 2300 * @list: list to use 2301 * @newsk: buffer to queue 2302 * 2303 * Queue a buffer at the start of the list. This function takes the 2304 * list lock and can be used safely with other locking &sk_buff functions 2305 * safely. 2306 * 2307 * A buffer cannot be placed on two lists at the same time. 2308 */ 2309void skb_queue_head(struct sk_buff_head *list, struct sk_buff *newsk) 2310{ 2311 unsigned long flags; 2312 2313 spin_lock_irqsave(&list->lock, flags); 2314 __skb_queue_head(list, newsk); 2315 spin_unlock_irqrestore(&list->lock, flags); 2316} 2317EXPORT_SYMBOL(skb_queue_head); 2318 2319/** 2320 * skb_queue_tail - queue a buffer at the list tail 2321 * @list: list to use 2322 * @newsk: buffer to queue 2323 * 2324 * Queue a buffer at the tail of the list. This function takes the 2325 * list lock and can be used safely with other locking &sk_buff functions 2326 * safely. 2327 * 2328 * A buffer cannot be placed on two lists at the same time. 2329 */ 2330void skb_queue_tail(struct sk_buff_head *list, struct sk_buff *newsk) 2331{ 2332 unsigned long flags; 2333 2334 spin_lock_irqsave(&list->lock, flags); 2335 __skb_queue_tail(list, newsk); 2336 spin_unlock_irqrestore(&list->lock, flags); 2337} 2338EXPORT_SYMBOL(skb_queue_tail); 2339 2340/** 2341 * skb_unlink - remove a buffer from a list 2342 * @skb: buffer to remove 2343 * @list: list to use 2344 * 2345 * Remove a packet from a list. The list locks are taken and this 2346 * function is atomic with respect to other list locked calls 2347 * 2348 * You must know what list the SKB is on. 2349 */ 2350void skb_unlink(struct sk_buff *skb, struct sk_buff_head *list) 2351{ 2352 unsigned long flags; 2353 2354 spin_lock_irqsave(&list->lock, flags); 2355 __skb_unlink(skb, list); 2356 spin_unlock_irqrestore(&list->lock, flags); 2357} 2358EXPORT_SYMBOL(skb_unlink); 2359 2360/** 2361 * skb_append - append a buffer 2362 * @old: buffer to insert after 2363 * @newsk: buffer to insert 2364 * @list: list to use 2365 * 2366 * Place a packet after a given packet in a list. The list locks are taken 2367 * and this function is atomic with respect to other list locked calls. 2368 * A buffer cannot be placed on two lists at the same time. 2369 */ 2370void skb_append(struct sk_buff *old, struct sk_buff *newsk, struct sk_buff_head *list) 2371{ 2372 unsigned long flags; 2373 2374 spin_lock_irqsave(&list->lock, flags); 2375 __skb_queue_after(list, old, newsk); 2376 spin_unlock_irqrestore(&list->lock, flags); 2377} 2378EXPORT_SYMBOL(skb_append); 2379 2380/** 2381 * skb_insert - insert a buffer 2382 * @old: buffer to insert before 2383 * @newsk: buffer to insert 2384 * @list: list to use 2385 * 2386 * Place a packet before a given packet in a list. The list locks are 2387 * taken and this function is atomic with respect to other list locked 2388 * calls. 2389 * 2390 * A buffer cannot be placed on two lists at the same time. 2391 */ 2392void skb_insert(struct sk_buff *old, struct sk_buff *newsk, struct sk_buff_head *list) 2393{ 2394 unsigned long flags; 2395 2396 spin_lock_irqsave(&list->lock, flags); 2397 __skb_insert(newsk, old->prev, old, list); 2398 spin_unlock_irqrestore(&list->lock, flags); 2399} 2400EXPORT_SYMBOL(skb_insert); 2401 2402static inline void skb_split_inside_header(struct sk_buff *skb, 2403 struct sk_buff* skb1, 2404 const u32 len, const int pos) 2405{ 2406 int i; 2407 2408 skb_copy_from_linear_data_offset(skb, len, skb_put(skb1, pos - len), 2409 pos - len); 2410 /* And move data appendix as is. */ 2411 for (i = 0; i < skb_shinfo(skb)->nr_frags; i++) 2412 skb_shinfo(skb1)->frags[i] = skb_shinfo(skb)->frags[i]; 2413 2414 skb_shinfo(skb1)->nr_frags = skb_shinfo(skb)->nr_frags; 2415 skb_shinfo(skb)->nr_frags = 0; 2416 skb1->data_len = skb->data_len; 2417 skb1->len += skb1->data_len; 2418 skb->data_len = 0; 2419 skb->len = len; 2420 skb_set_tail_pointer(skb, len); 2421} 2422 2423static inline void skb_split_no_header(struct sk_buff *skb, 2424 struct sk_buff* skb1, 2425 const u32 len, int pos) 2426{ 2427 int i, k = 0; 2428 const int nfrags = skb_shinfo(skb)->nr_frags; 2429 2430 skb_shinfo(skb)->nr_frags = 0; 2431 skb1->len = skb1->data_len = skb->len - len; 2432 skb->len = len; 2433 skb->data_len = len - pos; 2434 2435 for (i = 0; i < nfrags; i++) { 2436 int size = skb_frag_size(&skb_shinfo(skb)->frags[i]); 2437 2438 if (pos + size > len) { 2439 skb_shinfo(skb1)->frags[k] = skb_shinfo(skb)->frags[i]; 2440 2441 if (pos < len) { 2442 /* Split frag. 2443 * We have two variants in this case: 2444 * 1. Move all the frag to the second 2445 * part, if it is possible. F.e. 2446 * this approach is mandatory for TUX, 2447 * where splitting is expensive. 2448 * 2. Split is accurately. We make this. 2449 */ 2450 skb_frag_ref(skb, i); 2451 skb_shinfo(skb1)->frags[0].page_offset += len - pos; 2452 skb_frag_size_sub(&skb_shinfo(skb1)->frags[0], len - pos); 2453 skb_frag_size_set(&skb_shinfo(skb)->frags[i], len - pos); 2454 skb_shinfo(skb)->nr_frags++; 2455 } 2456 k++; 2457 } else 2458 skb_shinfo(skb)->nr_frags++; 2459 pos += size; 2460 } 2461 skb_shinfo(skb1)->nr_frags = k; 2462} 2463 2464/** 2465 * skb_split - Split fragmented skb to two parts at length len. 2466 * @skb: the buffer to split 2467 * @skb1: the buffer to receive the second part 2468 * @len: new length for skb 2469 */ 2470void skb_split(struct sk_buff *skb, struct sk_buff *skb1, const u32 len) 2471{ 2472 int pos = skb_headlen(skb); 2473 2474 skb_shinfo(skb1)->tx_flags = skb_shinfo(skb)->tx_flags & SKBTX_SHARED_FRAG; 2475 if (len < pos) /* Split line is inside header. */ 2476 skb_split_inside_header(skb, skb1, len, pos); 2477 else /* Second chunk has no header, nothing to copy. */ 2478 skb_split_no_header(skb, skb1, len, pos); 2479} 2480EXPORT_SYMBOL(skb_split); 2481 2482/* Shifting from/to a cloned skb is a no-go. 2483 * 2484 * Caller cannot keep skb_shinfo related pointers past calling here! 2485 */ 2486static int skb_prepare_for_shift(struct sk_buff *skb) 2487{ 2488 return skb_cloned(skb) && pskb_expand_head(skb, 0, 0, GFP_ATOMIC); 2489} 2490 2491/** 2492 * skb_shift - Shifts paged data partially from skb to another 2493 * @tgt: buffer into which tail data gets added 2494 * @skb: buffer from which the paged data comes from 2495 * @shiftlen: shift up to this many bytes 2496 * 2497 * Attempts to shift up to shiftlen worth of bytes, which may be less than 2498 * the length of the skb, from skb to tgt. Returns number bytes shifted. 2499 * It's up to caller to free skb if everything was shifted. 2500 * 2501 * If @tgt runs out of frags, the whole operation is aborted. 2502 * 2503 * Skb cannot include anything else but paged data while tgt is allowed 2504 * to have non-paged data as well. 2505 * 2506 * TODO: full sized shift could be optimized but that would need 2507 * specialized skb free'er to handle frags without up-to-date nr_frags. 2508 */ 2509int skb_shift(struct sk_buff *tgt, struct sk_buff *skb, int shiftlen) 2510{ 2511 int from, to, merge, todo; 2512 struct skb_frag_struct *fragfrom, *fragto; 2513 2514 BUG_ON(shiftlen > skb->len); 2515 BUG_ON(skb_headlen(skb)); /* Would corrupt stream */ 2516 2517 todo = shiftlen; 2518 from = 0; 2519 to = skb_shinfo(tgt)->nr_frags; 2520 fragfrom = &skb_shinfo(skb)->frags[from]; 2521 2522 /* Actual merge is delayed until the point when we know we can 2523 * commit all, so that we don't have to undo partial changes 2524 */ 2525 if (!to || 2526 !skb_can_coalesce(tgt, to, skb_frag_page(fragfrom), 2527 fragfrom->page_offset)) { 2528 merge = -1; 2529 } else { 2530 merge = to - 1; 2531 2532 todo -= skb_frag_size(fragfrom); 2533 if (todo < 0) { 2534 if (skb_prepare_for_shift(skb) || 2535 skb_prepare_for_shift(tgt)) 2536 return 0; 2537 2538 /* All previous frag pointers might be stale! */ 2539 fragfrom = &skb_shinfo(skb)->frags[from]; 2540 fragto = &skb_shinfo(tgt)->frags[merge]; 2541 2542 skb_frag_size_add(fragto, shiftlen); 2543 skb_frag_size_sub(fragfrom, shiftlen); 2544 fragfrom->page_offset += shiftlen; 2545 2546 goto onlymerged; 2547 } 2548 2549 from++; 2550 } 2551 2552 /* Skip full, not-fitting skb to avoid expensive operations */ 2553 if ((shiftlen == skb->len) && 2554 (skb_shinfo(skb)->nr_frags - from) > (MAX_SKB_FRAGS - to)) 2555 return 0; 2556 2557 if (skb_prepare_for_shift(skb) || skb_prepare_for_shift(tgt)) 2558 return 0; 2559 2560 while ((todo > 0) && (from < skb_shinfo(skb)->nr_frags)) { 2561 if (to == MAX_SKB_FRAGS) 2562 return 0; 2563 2564 fragfrom = &skb_shinfo(skb)->frags[from]; 2565 fragto = &skb_shinfo(tgt)->frags[to]; 2566 2567 if (todo >= skb_frag_size(fragfrom)) { 2568 *fragto = *fragfrom; 2569 todo -= skb_frag_size(fragfrom); 2570 from++; 2571 to++; 2572 2573 } else { 2574 __skb_frag_ref(fragfrom); 2575 fragto->page = fragfrom->page; 2576 fragto->page_offset = fragfrom->page_offset; 2577 skb_frag_size_set(fragto, todo); 2578 2579 fragfrom->page_offset += todo; 2580 skb_frag_size_sub(fragfrom, todo); 2581 todo = 0; 2582 2583 to++; 2584 break; 2585 } 2586 } 2587 2588 /* Ready to "commit" this state change to tgt */ 2589 skb_shinfo(tgt)->nr_frags = to; 2590 2591 if (merge >= 0) { 2592 fragfrom = &skb_shinfo(skb)->frags[0]; 2593 fragto = &skb_shinfo(tgt)->frags[merge]; 2594 2595 skb_frag_size_add(fragto, skb_frag_size(fragfrom)); 2596 __skb_frag_unref(fragfrom); 2597 } 2598 2599 /* Reposition in the original skb */ 2600 to = 0; 2601 while (from < skb_shinfo(skb)->nr_frags) 2602 skb_shinfo(skb)->frags[to++] = skb_shinfo(skb)->frags[from++]; 2603 skb_shinfo(skb)->nr_frags = to; 2604 2605 BUG_ON(todo > 0 && !skb_shinfo(skb)->nr_frags); 2606 2607onlymerged: 2608 /* Most likely the tgt won't ever need its checksum anymore, skb on 2609 * the other hand might need it if it needs to be resent 2610 */ 2611 tgt->ip_summed = CHECKSUM_PARTIAL; 2612 skb->ip_summed = CHECKSUM_PARTIAL; 2613 2614 /* Yak, is it really working this way? Some helper please? */ 2615 skb->len -= shiftlen; 2616 skb->data_len -= shiftlen; 2617 skb->truesize -= shiftlen; 2618 tgt->len += shiftlen; 2619 tgt->data_len += shiftlen; 2620 tgt->truesize += shiftlen; 2621 2622 return shiftlen; 2623} 2624 2625/** 2626 * skb_prepare_seq_read - Prepare a sequential read of skb data 2627 * @skb: the buffer to read 2628 * @from: lower offset of data to be read 2629 * @to: upper offset of data to be read 2630 * @st: state variable 2631 * 2632 * Initializes the specified state variable. Must be called before 2633 * invoking skb_seq_read() for the first time. 2634 */ 2635void skb_prepare_seq_read(struct sk_buff *skb, unsigned int from, 2636 unsigned int to, struct skb_seq_state *st) 2637{ 2638 st->lower_offset = from; 2639 st->upper_offset = to; 2640 st->root_skb = st->cur_skb = skb; 2641 st->frag_idx = st->stepped_offset = 0; 2642 st->frag_data = NULL; 2643} 2644EXPORT_SYMBOL(skb_prepare_seq_read); 2645 2646/** 2647 * skb_seq_read - Sequentially read skb data 2648 * @consumed: number of bytes consumed by the caller so far 2649 * @data: destination pointer for data to be returned 2650 * @st: state variable 2651 * 2652 * Reads a block of skb data at @consumed relative to the 2653 * lower offset specified to skb_prepare_seq_read(). Assigns 2654 * the head of the data block to @data and returns the length 2655 * of the block or 0 if the end of the skb data or the upper 2656 * offset has been reached. 2657 * 2658 * The caller is not required to consume all of the data 2659 * returned, i.e. @consumed is typically set to the number 2660 * of bytes already consumed and the next call to 2661 * skb_seq_read() will return the remaining part of the block. 2662 * 2663 * Note 1: The size of each block of data returned can be arbitrary, 2664 * this limitation is the cost for zerocopy sequential 2665 * reads of potentially non linear data. 2666 * 2667 * Note 2: Fragment lists within fragments are not implemented 2668 * at the moment, state->root_skb could be replaced with 2669 * a stack for this purpose. 2670 */ 2671unsigned int skb_seq_read(unsigned int consumed, const u8 **data, 2672 struct skb_seq_state *st) 2673{ 2674 unsigned int block_limit, abs_offset = consumed + st->lower_offset; 2675 skb_frag_t *frag; 2676 2677 if (unlikely(abs_offset >= st->upper_offset)) { 2678 if (st->frag_data) { 2679 kunmap_atomic(st->frag_data); 2680 st->frag_data = NULL; 2681 } 2682 return 0; 2683 } 2684 2685next_skb: 2686 block_limit = skb_headlen(st->cur_skb) + st->stepped_offset; 2687 2688 if (abs_offset < block_limit && !st->frag_data) { 2689 *data = st->cur_skb->data + (abs_offset - st->stepped_offset); 2690 return block_limit - abs_offset; 2691 } 2692 2693 if (st->frag_idx == 0 && !st->frag_data) 2694 st->stepped_offset += skb_headlen(st->cur_skb); 2695 2696 while (st->frag_idx < skb_shinfo(st->cur_skb)->nr_frags) { 2697 frag = &skb_shinfo(st->cur_skb)->frags[st->frag_idx]; 2698 block_limit = skb_frag_size(frag) + st->stepped_offset; 2699 2700 if (abs_offset < block_limit) { 2701 if (!st->frag_data) 2702 st->frag_data = kmap_atomic(skb_frag_page(frag)); 2703 2704 *data = (u8 *) st->frag_data + frag->page_offset + 2705 (abs_offset - st->stepped_offset); 2706 2707 return block_limit - abs_offset; 2708 } 2709 2710 if (st->frag_data) { 2711 kunmap_atomic(st->frag_data); 2712 st->frag_data = NULL; 2713 } 2714 2715 st->frag_idx++; 2716 st->stepped_offset += skb_frag_size(frag); 2717 } 2718 2719 if (st->frag_data) { 2720 kunmap_atomic(st->frag_data); 2721 st->frag_data = NULL; 2722 } 2723 2724 if (st->root_skb == st->cur_skb && skb_has_frag_list(st->root_skb)) { 2725 st->cur_skb = skb_shinfo(st->root_skb)->frag_list; 2726 st->frag_idx = 0; 2727 goto next_skb; 2728 } else if (st->cur_skb->next) { 2729 st->cur_skb = st->cur_skb->next; 2730 st->frag_idx = 0; 2731 goto next_skb; 2732 } 2733 2734 return 0; 2735} 2736EXPORT_SYMBOL(skb_seq_read); 2737 2738/** 2739 * skb_abort_seq_read - Abort a sequential read of skb data 2740 * @st: state variable 2741 * 2742 * Must be called if skb_seq_read() was not called until it 2743 * returned 0. 2744 */ 2745void skb_abort_seq_read(struct skb_seq_state *st) 2746{ 2747 if (st->frag_data) 2748 kunmap_atomic(st->frag_data); 2749} 2750EXPORT_SYMBOL(skb_abort_seq_read); 2751 2752#define TS_SKB_CB(state) ((struct skb_seq_state *) &((state)->cb)) 2753 2754static unsigned int skb_ts_get_next_block(unsigned int offset, const u8 **text, 2755 struct ts_config *conf, 2756 struct ts_state *state) 2757{ 2758 return skb_seq_read(offset, text, TS_SKB_CB(state)); 2759} 2760 2761static void skb_ts_finish(struct ts_config *conf, struct ts_state *state) 2762{ 2763 skb_abort_seq_read(TS_SKB_CB(state)); 2764} 2765 2766/** 2767 * skb_find_text - Find a text pattern in skb data 2768 * @skb: the buffer to look in 2769 * @from: search offset 2770 * @to: search limit 2771 * @config: textsearch configuration 2772 * @state: uninitialized textsearch state variable 2773 * 2774 * Finds a pattern in the skb data according to the specified 2775 * textsearch configuration. Use textsearch_next() to retrieve 2776 * subsequent occurrences of the pattern. Returns the offset 2777 * to the first occurrence or UINT_MAX if no match was found. 2778 */ 2779unsigned int skb_find_text(struct sk_buff *skb, unsigned int from, 2780 unsigned int to, struct ts_config *config, 2781 struct ts_state *state) 2782{ 2783 unsigned int ret; 2784 2785 config->get_next_block = skb_ts_get_next_block; 2786 config->finish = skb_ts_finish; 2787 2788 skb_prepare_seq_read(skb, from, to, TS_SKB_CB(state)); 2789 2790 ret = textsearch_find(config, state); 2791 return (ret <= to - from ? ret : UINT_MAX); 2792} 2793EXPORT_SYMBOL(skb_find_text); 2794 2795/** 2796 * skb_append_datato_frags - append the user data to a skb 2797 * @sk: sock structure 2798 * @skb: skb structure to be appended with user data. 2799 * @getfrag: call back function to be used for getting the user data 2800 * @from: pointer to user message iov 2801 * @length: length of the iov message 2802 * 2803 * Description: This procedure append the user data in the fragment part 2804 * of the skb if any page alloc fails user this procedure returns -ENOMEM 2805 */ 2806int skb_append_datato_frags(struct sock *sk, struct sk_buff *skb, 2807 int (*getfrag)(void *from, char *to, int offset, 2808 int len, int odd, struct sk_buff *skb), 2809 void *from, int length) 2810{ 2811 int frg_cnt = skb_shinfo(skb)->nr_frags; 2812 int copy; 2813 int offset = 0; 2814 int ret; 2815 struct page_frag *pfrag = ¤t->task_frag; 2816 2817 do { 2818 /* Return error if we don't have space for new frag */ 2819 if (frg_cnt >= MAX_SKB_FRAGS) 2820 return -EMSGSIZE; 2821 2822 if (!sk_page_frag_refill(sk, pfrag)) 2823 return -ENOMEM; 2824 2825 /* copy the user data to page */ 2826 copy = min_t(int, length, pfrag->size - pfrag->offset); 2827 2828 ret = getfrag(from, page_address(pfrag->page) + pfrag->offset, 2829 offset, copy, 0, skb); 2830 if (ret < 0) 2831 return -EFAULT; 2832 2833 /* copy was successful so update the size parameters */ 2834 skb_fill_page_desc(skb, frg_cnt, pfrag->page, pfrag->offset, 2835 copy); 2836 frg_cnt++; 2837 pfrag->offset += copy; 2838 get_page(pfrag->page); 2839 2840 skb->truesize += copy; 2841 atomic_add(copy, &sk->sk_wmem_alloc); 2842 skb->len += copy; 2843 skb->data_len += copy; 2844 offset += copy; 2845 length -= copy; 2846 2847 } while (length > 0); 2848 2849 return 0; 2850} 2851EXPORT_SYMBOL(skb_append_datato_frags); 2852 2853/** 2854 * skb_pull_rcsum - pull skb and update receive checksum 2855 * @skb: buffer to update 2856 * @len: length of data pulled 2857 * 2858 * This function performs an skb_pull on the packet and updates 2859 * the CHECKSUM_COMPLETE checksum. It should be used on 2860 * receive path processing instead of skb_pull unless you know 2861 * that the checksum difference is zero (e.g., a valid IP header) 2862 * or you are setting ip_summed to CHECKSUM_NONE. 2863 */ 2864unsigned char *skb_pull_rcsum(struct sk_buff *skb, unsigned int len) 2865{ 2866 BUG_ON(len > skb->len); 2867 skb->len -= len; 2868 BUG_ON(skb->len < skb->data_len); 2869 skb_postpull_rcsum(skb, skb->data, len); 2870 return skb->data += len; 2871} 2872EXPORT_SYMBOL_GPL(skb_pull_rcsum); 2873 2874/** 2875 * skb_segment - Perform protocol segmentation on skb. 2876 * @head_skb: buffer to segment 2877 * @features: features for the output path (see dev->features) 2878 * 2879 * This function performs segmentation on the given skb. It returns 2880 * a pointer to the first in a list of new skbs for the segments. 2881 * In case of error it returns ERR_PTR(err). 2882 */ 2883struct sk_buff *skb_segment(struct sk_buff *head_skb, 2884 netdev_features_t features) 2885{ 2886 struct sk_buff *segs = NULL; 2887 struct sk_buff *tail = NULL; 2888 struct sk_buff *list_skb = skb_shinfo(head_skb)->frag_list; 2889 skb_frag_t *frag = skb_shinfo(head_skb)->frags; 2890 unsigned int mss = skb_shinfo(head_skb)->gso_size; 2891 unsigned int doffset = head_skb->data - skb_mac_header(head_skb); 2892 struct sk_buff *frag_skb = head_skb; 2893 unsigned int offset = doffset; 2894 unsigned int tnl_hlen = skb_tnl_header_len(head_skb); 2895 unsigned int headroom; 2896 unsigned int len; 2897 __be16 proto; 2898 bool csum; 2899 int sg = !!(features & NETIF_F_SG); 2900 int nfrags = skb_shinfo(head_skb)->nr_frags; 2901 int err = -ENOMEM; 2902 int i = 0; 2903 int pos; 2904 int dummy; 2905 2906 __skb_push(head_skb, doffset); 2907 proto = skb_network_protocol(head_skb, &dummy); 2908 if (unlikely(!proto)) 2909 return ERR_PTR(-EINVAL); 2910 2911 csum = !head_skb->encap_hdr_csum && 2912 !!can_checksum_protocol(features, proto); 2913 2914 headroom = skb_headroom(head_skb); 2915 pos = skb_headlen(head_skb); 2916 2917 do { 2918 struct sk_buff *nskb; 2919 skb_frag_t *nskb_frag; 2920 int hsize; 2921 int size; 2922 2923 len = head_skb->len - offset; 2924 if (len > mss) 2925 len = mss; 2926 2927 hsize = skb_headlen(head_skb) - offset; 2928 if (hsize < 0) 2929 hsize = 0; 2930 if (hsize > len || !sg) 2931 hsize = len; 2932 2933 if (!hsize && i >= nfrags && skb_headlen(list_skb) && 2934 (skb_headlen(list_skb) == len || sg)) { 2935 BUG_ON(skb_headlen(list_skb) > len); 2936 2937 i = 0; 2938 nfrags = skb_shinfo(list_skb)->nr_frags; 2939 frag = skb_shinfo(list_skb)->frags; 2940 frag_skb = list_skb; 2941 pos += skb_headlen(list_skb); 2942 2943 while (pos < offset + len) { 2944 BUG_ON(i >= nfrags); 2945 2946 size = skb_frag_size(frag); 2947 if (pos + size > offset + len) 2948 break; 2949 2950 i++; 2951 pos += size; 2952 frag++; 2953 } 2954 2955 nskb = skb_clone(list_skb, GFP_ATOMIC); 2956 list_skb = list_skb->next; 2957 2958 if (unlikely(!nskb)) 2959 goto err; 2960 2961 if (unlikely(pskb_trim(nskb, len))) { 2962 kfree_skb(nskb); 2963 goto err; 2964 } 2965 2966 hsize = skb_end_offset(nskb); 2967 if (skb_cow_head(nskb, doffset + headroom)) { 2968 kfree_skb(nskb); 2969 goto err; 2970 } 2971 2972 nskb->truesize += skb_end_offset(nskb) - hsize; 2973 skb_release_head_state(nskb); 2974 __skb_push(nskb, doffset); 2975 } else { 2976 nskb = __alloc_skb(hsize + doffset + headroom, 2977 GFP_ATOMIC, skb_alloc_rx_flag(head_skb), 2978 NUMA_NO_NODE); 2979 2980 if (unlikely(!nskb)) 2981 goto err; 2982 2983 skb_reserve(nskb, headroom); 2984 __skb_put(nskb, doffset); 2985 } 2986 2987 if (segs) 2988 tail->next = nskb; 2989 else 2990 segs = nskb; 2991 tail = nskb; 2992 2993 __copy_skb_header(nskb, head_skb); 2994 2995 skb_headers_offset_update(nskb, skb_headroom(nskb) - headroom); 2996 skb_reset_mac_len(nskb); 2997 2998 skb_copy_from_linear_data_offset(head_skb, -tnl_hlen, 2999 nskb->data - tnl_hlen, 3000 doffset + tnl_hlen); 3001 3002 if (nskb->len == len + doffset) 3003 goto perform_csum_check; 3004 3005 if (!sg) { 3006 nskb->ip_summed = CHECKSUM_NONE; 3007 nskb->csum = skb_copy_and_csum_bits(head_skb, offset, 3008 skb_put(nskb, len), 3009 len, 0); 3010 SKB_GSO_CB(nskb)->csum_start = 3011 skb_headroom(nskb) + doffset; 3012 continue; 3013 } 3014 3015 nskb_frag = skb_shinfo(nskb)->frags; 3016 3017 skb_copy_from_linear_data_offset(head_skb, offset, 3018 skb_put(nskb, hsize), hsize); 3019 3020 skb_shinfo(nskb)->tx_flags = skb_shinfo(head_skb)->tx_flags & 3021 SKBTX_SHARED_FRAG; 3022 3023 while (pos < offset + len) { 3024 if (i >= nfrags) { 3025 BUG_ON(skb_headlen(list_skb)); 3026 3027 i = 0; 3028 nfrags = skb_shinfo(list_skb)->nr_frags; 3029 frag = skb_shinfo(list_skb)->frags; 3030 frag_skb = list_skb; 3031 3032 BUG_ON(!nfrags); 3033 3034 list_skb = list_skb->next; 3035 } 3036 3037 if (unlikely(skb_shinfo(nskb)->nr_frags >= 3038 MAX_SKB_FRAGS)) { 3039 net_warn_ratelimited( 3040 "skb_segment: too many frags: %u %u\n", 3041 pos, mss); 3042 goto err; 3043 } 3044 3045 if (unlikely(skb_orphan_frags(frag_skb, GFP_ATOMIC))) 3046 goto err; 3047 3048 *nskb_frag = *frag; 3049 __skb_frag_ref(nskb_frag); 3050 size = skb_frag_size(nskb_frag); 3051 3052 if (pos < offset) { 3053 nskb_frag->page_offset += offset - pos; 3054 skb_frag_size_sub(nskb_frag, offset - pos); 3055 } 3056 3057 skb_shinfo(nskb)->nr_frags++; 3058 3059 if (pos + size <= offset + len) { 3060 i++; 3061 frag++; 3062 pos += size; 3063 } else { 3064 skb_frag_size_sub(nskb_frag, pos + size - (offset + len)); 3065 goto skip_fraglist; 3066 } 3067 3068 nskb_frag++; 3069 } 3070 3071skip_fraglist: 3072 nskb->data_len = len - hsize; 3073 nskb->len += nskb->data_len; 3074 nskb->truesize += nskb->data_len; 3075 3076perform_csum_check: 3077 if (!csum) { 3078 nskb->csum = skb_checksum(nskb, doffset, 3079 nskb->len - doffset, 0); 3080 nskb->ip_summed = CHECKSUM_NONE; 3081 SKB_GSO_CB(nskb)->csum_start = 3082 skb_headroom(nskb) + doffset; 3083 } 3084 } while ((offset += len) < head_skb->len); 3085 3086 /* Some callers want to get the end of the list. 3087 * Put it in segs->prev to avoid walking the list. 3088 * (see validate_xmit_skb_list() for example) 3089 */ 3090 segs->prev = tail; 3091 return segs; 3092 3093err: 3094 kfree_skb_list(segs); 3095 return ERR_PTR(err); 3096} 3097EXPORT_SYMBOL_GPL(skb_segment); 3098 3099int skb_gro_receive(struct sk_buff **head, struct sk_buff *skb) 3100{ 3101 struct skb_shared_info *pinfo, *skbinfo = skb_shinfo(skb); 3102 unsigned int offset = skb_gro_offset(skb); 3103 unsigned int headlen = skb_headlen(skb); 3104 struct sk_buff *nskb, *lp, *p = *head; 3105 unsigned int len = skb_gro_len(skb); 3106 unsigned int delta_truesize; 3107 unsigned int headroom; 3108 3109 if (unlikely(p->len + len >= 65536)) 3110 return -E2BIG; 3111 3112 lp = NAPI_GRO_CB(p)->last; 3113 pinfo = skb_shinfo(lp); 3114 3115 if (headlen <= offset) { 3116 skb_frag_t *frag; 3117 skb_frag_t *frag2; 3118 int i = skbinfo->nr_frags; 3119 int nr_frags = pinfo->nr_frags + i; 3120 3121 if (nr_frags > MAX_SKB_FRAGS) 3122 goto merge; 3123 3124 offset -= headlen; 3125 pinfo->nr_frags = nr_frags; 3126 skbinfo->nr_frags = 0; 3127 3128 frag = pinfo->frags + nr_frags; 3129 frag2 = skbinfo->frags + i; 3130 do { 3131 *--frag = *--frag2; 3132 } while (--i); 3133 3134 frag->page_offset += offset; 3135 skb_frag_size_sub(frag, offset); 3136 3137 /* all fragments truesize : remove (head size + sk_buff) */ 3138 delta_truesize = skb->truesize - 3139 SKB_TRUESIZE(skb_end_offset(skb)); 3140 3141 skb->truesize -= skb->data_len; 3142 skb->len -= skb->data_len; 3143 skb->data_len = 0; 3144 3145 NAPI_GRO_CB(skb)->free = NAPI_GRO_FREE; 3146 goto done; 3147 } else if (skb->head_frag) { 3148 int nr_frags = pinfo->nr_frags; 3149 skb_frag_t *frag = pinfo->frags + nr_frags; 3150 struct page *page = virt_to_head_page(skb->head); 3151 unsigned int first_size = headlen - offset; 3152 unsigned int first_offset; 3153 3154 if (nr_frags + 1 + skbinfo->nr_frags > MAX_SKB_FRAGS) 3155 goto merge; 3156 3157 first_offset = skb->data - 3158 (unsigned char *)page_address(page) + 3159 offset; 3160 3161 pinfo->nr_frags = nr_frags + 1 + skbinfo->nr_frags; 3162 3163 frag->page.p = page; 3164 frag->page_offset = first_offset; 3165 skb_frag_size_set(frag, first_size); 3166 3167 memcpy(frag + 1, skbinfo->frags, sizeof(*frag) * skbinfo->nr_frags); 3168 /* We dont need to clear skbinfo->nr_frags here */ 3169 3170 delta_truesize = skb->truesize - SKB_DATA_ALIGN(sizeof(struct sk_buff)); 3171 NAPI_GRO_CB(skb)->free = NAPI_GRO_FREE_STOLEN_HEAD; 3172 goto done; 3173 } 3174 /* switch back to head shinfo */ 3175 pinfo = skb_shinfo(p); 3176 3177 if (pinfo->frag_list) 3178 goto merge; 3179 if (skb_gro_len(p) != pinfo->gso_size) 3180 return -E2BIG; 3181 3182 headroom = skb_headroom(p); 3183 nskb = alloc_skb(headroom + skb_gro_offset(p), GFP_ATOMIC); 3184 if (unlikely(!nskb)) 3185 return -ENOMEM; 3186 3187 __copy_skb_header(nskb, p); 3188 nskb->mac_len = p->mac_len; 3189 3190 skb_reserve(nskb, headroom); 3191 __skb_put(nskb, skb_gro_offset(p)); 3192 3193 skb_set_mac_header(nskb, skb_mac_header(p) - p->data); 3194 skb_set_network_header(nskb, skb_network_offset(p)); 3195 skb_set_transport_header(nskb, skb_transport_offset(p)); 3196 3197 __skb_pull(p, skb_gro_offset(p)); 3198 memcpy(skb_mac_header(nskb), skb_mac_header(p), 3199 p->data - skb_mac_header(p)); 3200 3201 skb_shinfo(nskb)->frag_list = p; 3202 skb_shinfo(nskb)->gso_size = pinfo->gso_size; 3203 pinfo->gso_size = 0; 3204 __skb_header_release(p); 3205 NAPI_GRO_CB(nskb)->last = p; 3206 3207 nskb->data_len += p->len; 3208 nskb->truesize += p->truesize; 3209 nskb->len += p->len; 3210 3211 *head = nskb; 3212 nskb->next = p->next; 3213 p->next = NULL; 3214 3215 p = nskb; 3216 3217merge: 3218 delta_truesize = skb->truesize; 3219 if (offset > headlen) { 3220 unsigned int eat = offset - headlen; 3221 3222 skbinfo->frags[0].page_offset += eat; 3223 skb_frag_size_sub(&skbinfo->frags[0], eat); 3224 skb->data_len -= eat; 3225 skb->len -= eat; 3226 offset = headlen; 3227 } 3228 3229 __skb_pull(skb, offset); 3230 3231 if (NAPI_GRO_CB(p)->last == p) 3232 skb_shinfo(p)->frag_list = skb; 3233 else 3234 NAPI_GRO_CB(p)->last->next = skb; 3235 NAPI_GRO_CB(p)->last = skb; 3236 __skb_header_release(skb); 3237 lp = p; 3238 3239done: 3240 NAPI_GRO_CB(p)->count++; 3241 p->data_len += len; 3242 p->truesize += delta_truesize; 3243 p->len += len; 3244 if (lp != p) { 3245 lp->data_len += len; 3246 lp->truesize += delta_truesize; 3247 lp->len += len; 3248 } 3249 NAPI_GRO_CB(skb)->same_flow = 1; 3250 return 0; 3251} 3252 3253void __init skb_init(void) 3254{ 3255 skbuff_head_cache = kmem_cache_create("skbuff_head_cache", 3256 sizeof(struct sk_buff), 3257 0, 3258 SLAB_HWCACHE_ALIGN|SLAB_PANIC, 3259 NULL); 3260 skbuff_fclone_cache = kmem_cache_create("skbuff_fclone_cache", 3261 sizeof(struct sk_buff_fclones), 3262 0, 3263 SLAB_HWCACHE_ALIGN|SLAB_PANIC, 3264 NULL); 3265} 3266 3267/** 3268 * skb_to_sgvec - Fill a scatter-gather list from a socket buffer 3269 * @skb: Socket buffer containing the buffers to be mapped 3270 * @sg: The scatter-gather list to map into 3271 * @offset: The offset into the buffer's contents to start mapping 3272 * @len: Length of buffer space to be mapped 3273 * 3274 * Fill the specified scatter-gather list with mappings/pointers into a 3275 * region of the buffer space attached to a socket buffer. 3276 */ 3277static int 3278__skb_to_sgvec(struct sk_buff *skb, struct scatterlist *sg, int offset, int len) 3279{ 3280 int start = skb_headlen(skb); 3281 int i, copy = start - offset; 3282 struct sk_buff *frag_iter; 3283 int elt = 0; 3284 3285 if (copy > 0) { 3286 if (copy > len) 3287 copy = len; 3288 sg_set_buf(sg, skb->data + offset, copy); 3289 elt++; 3290 if ((len -= copy) == 0) 3291 return elt; 3292 offset += copy; 3293 } 3294 3295 for (i = 0; i < skb_shinfo(skb)->nr_frags; i++) { 3296 int end; 3297 3298 WARN_ON(start > offset + len); 3299 3300 end = start + skb_frag_size(&skb_shinfo(skb)->frags[i]); 3301 if ((copy = end - offset) > 0) { 3302 skb_frag_t *frag = &skb_shinfo(skb)->frags[i]; 3303 3304 if (copy > len) 3305 copy = len; 3306 sg_set_page(&sg[elt], skb_frag_page(frag), copy, 3307 frag->page_offset+offset-start); 3308 elt++; 3309 if (!(len -= copy)) 3310 return elt; 3311 offset += copy; 3312 } 3313 start = end; 3314 } 3315 3316 skb_walk_frags(skb, frag_iter) { 3317 int end; 3318 3319 WARN_ON(start > offset + len); 3320 3321 end = start + frag_iter->len; 3322 if ((copy = end - offset) > 0) { 3323 if (copy > len) 3324 copy = len; 3325 elt += __skb_to_sgvec(frag_iter, sg+elt, offset - start, 3326 copy); 3327 if ((len -= copy) == 0) 3328 return elt; 3329 offset += copy; 3330 } 3331 start = end; 3332 } 3333 BUG_ON(len); 3334 return elt; 3335} 3336 3337/* As compared with skb_to_sgvec, skb_to_sgvec_nomark only map skb to given 3338 * sglist without mark the sg which contain last skb data as the end. 3339 * So the caller can mannipulate sg list as will when padding new data after 3340 * the first call without calling sg_unmark_end to expend sg list. 3341 * 3342 * Scenario to use skb_to_sgvec_nomark: 3343 * 1. sg_init_table 3344 * 2. skb_to_sgvec_nomark(payload1) 3345 * 3. skb_to_sgvec_nomark(payload2) 3346 * 3347 * This is equivalent to: 3348 * 1. sg_init_table 3349 * 2. skb_to_sgvec(payload1) 3350 * 3. sg_unmark_end 3351 * 4. skb_to_sgvec(payload2) 3352 * 3353 * When mapping mutilple payload conditionally, skb_to_sgvec_nomark 3354 * is more preferable. 3355 */ 3356int skb_to_sgvec_nomark(struct sk_buff *skb, struct scatterlist *sg, 3357 int offset, int len) 3358{ 3359 return __skb_to_sgvec(skb, sg, offset, len); 3360} 3361EXPORT_SYMBOL_GPL(skb_to_sgvec_nomark); 3362 3363int skb_to_sgvec(struct sk_buff *skb, struct scatterlist *sg, int offset, int len) 3364{ 3365 int nsg = __skb_to_sgvec(skb, sg, offset, len); 3366 3367 sg_mark_end(&sg[nsg - 1]); 3368 3369 return nsg; 3370} 3371EXPORT_SYMBOL_GPL(skb_to_sgvec); 3372 3373/** 3374 * skb_cow_data - Check that a socket buffer's data buffers are writable 3375 * @skb: The socket buffer to check. 3376 * @tailbits: Amount of trailing space to be added 3377 * @trailer: Returned pointer to the skb where the @tailbits space begins 3378 * 3379 * Make sure that the data buffers attached to a socket buffer are 3380 * writable. If they are not, private copies are made of the data buffers 3381 * and the socket buffer is set to use these instead. 3382 * 3383 * If @tailbits is given, make sure that there is space to write @tailbits 3384 * bytes of data beyond current end of socket buffer. @trailer will be 3385 * set to point to the skb in which this space begins. 3386 * 3387 * The number of scatterlist elements required to completely map the 3388 * COW'd and extended socket buffer will be returned. 3389 */ 3390int skb_cow_data(struct sk_buff *skb, int tailbits, struct sk_buff **trailer) 3391{ 3392 int copyflag; 3393 int elt; 3394 struct sk_buff *skb1, **skb_p; 3395 3396 /* If skb is cloned or its head is paged, reallocate 3397 * head pulling out all the pages (pages are considered not writable 3398 * at the moment even if they are anonymous). 3399 */ 3400 if ((skb_cloned(skb) || skb_shinfo(skb)->nr_frags) && 3401 __pskb_pull_tail(skb, skb_pagelen(skb)-skb_headlen(skb)) == NULL) 3402 return -ENOMEM; 3403 3404 /* Easy case. Most of packets will go this way. */ 3405 if (!skb_has_frag_list(skb)) { 3406 /* A little of trouble, not enough of space for trailer. 3407 * This should not happen, when stack is tuned to generate 3408 * good frames. OK, on miss we reallocate and reserve even more 3409 * space, 128 bytes is fair. */ 3410 3411 if (skb_tailroom(skb) < tailbits && 3412 pskb_expand_head(skb, 0, tailbits-skb_tailroom(skb)+128, GFP_ATOMIC)) 3413 return -ENOMEM; 3414 3415 /* Voila! */ 3416 *trailer = skb; 3417 return 1; 3418 } 3419 3420 /* Misery. We are in troubles, going to mincer fragments... */ 3421 3422 elt = 1; 3423 skb_p = &skb_shinfo(skb)->frag_list; 3424 copyflag = 0; 3425 3426 while ((skb1 = *skb_p) != NULL) { 3427 int ntail = 0; 3428 3429 /* The fragment is partially pulled by someone, 3430 * this can happen on input. Copy it and everything 3431 * after it. */ 3432 3433 if (skb_shared(skb1)) 3434 copyflag = 1; 3435 3436 /* If the skb is the last, worry about trailer. */ 3437 3438 if (skb1->next == NULL && tailbits) { 3439 if (skb_shinfo(skb1)->nr_frags || 3440 skb_has_frag_list(skb1) || 3441 skb_tailroom(skb1) < tailbits) 3442 ntail = tailbits + 128; 3443 } 3444 3445 if (copyflag || 3446 skb_cloned(skb1) || 3447 ntail || 3448 skb_shinfo(skb1)->nr_frags || 3449 skb_has_frag_list(skb1)) { 3450 struct sk_buff *skb2; 3451 3452 /* Fuck, we are miserable poor guys... */ 3453 if (ntail == 0) 3454 skb2 = skb_copy(skb1, GFP_ATOMIC); 3455 else 3456 skb2 = skb_copy_expand(skb1, 3457 skb_headroom(skb1), 3458 ntail, 3459 GFP_ATOMIC); 3460 if (unlikely(skb2 == NULL)) 3461 return -ENOMEM; 3462 3463 if (skb1->sk) 3464 skb_set_owner_w(skb2, skb1->sk); 3465 3466 /* Looking around. Are we still alive? 3467 * OK, link new skb, drop old one */ 3468 3469 skb2->next = skb1->next; 3470 *skb_p = skb2; 3471 kfree_skb(skb1); 3472 skb1 = skb2; 3473 } 3474 elt++; 3475 *trailer = skb1; 3476 skb_p = &skb1->next; 3477 } 3478 3479 return elt; 3480} 3481EXPORT_SYMBOL_GPL(skb_cow_data); 3482 3483static void sock_rmem_free(struct sk_buff *skb) 3484{ 3485 struct sock *sk = skb->sk; 3486 3487 atomic_sub(skb->truesize, &sk->sk_rmem_alloc); 3488} 3489 3490/* 3491 * Note: We dont mem charge error packets (no sk_forward_alloc changes) 3492 */ 3493int sock_queue_err_skb(struct sock *sk, struct sk_buff *skb) 3494{ 3495 if (atomic_read(&sk->sk_rmem_alloc) + skb->truesize >= 3496 (unsigned int)sk->sk_rcvbuf) 3497 return -ENOMEM; 3498 3499 skb_orphan(skb); 3500 skb->sk = sk; 3501 skb->destructor = sock_rmem_free; 3502 atomic_add(skb->truesize, &sk->sk_rmem_alloc); 3503 3504 /* before exiting rcu section, make sure dst is refcounted */ 3505 skb_dst_force(skb); 3506 3507 skb_queue_tail(&sk->sk_error_queue, skb); 3508 if (!sock_flag(sk, SOCK_DEAD)) 3509 sk->sk_data_ready(sk); 3510 return 0; 3511} 3512EXPORT_SYMBOL(sock_queue_err_skb); 3513 3514struct sk_buff *sock_dequeue_err_skb(struct sock *sk) 3515{ 3516 struct sk_buff_head *q = &sk->sk_error_queue; 3517 struct sk_buff *skb, *skb_next; 3518 int err = 0; 3519 3520 spin_lock_bh(&q->lock); 3521 skb = __skb_dequeue(q); 3522 if (skb && (skb_next = skb_peek(q))) 3523 err = SKB_EXT_ERR(skb_next)->ee.ee_errno; 3524 spin_unlock_bh(&q->lock); 3525 3526 sk->sk_err = err; 3527 if (err) 3528 sk->sk_error_report(sk); 3529 3530 return skb; 3531} 3532EXPORT_SYMBOL(sock_dequeue_err_skb); 3533 3534/** 3535 * skb_clone_sk - create clone of skb, and take reference to socket 3536 * @skb: the skb to clone 3537 * 3538 * This function creates a clone of a buffer that holds a reference on 3539 * sk_refcnt. Buffers created via this function are meant to be 3540 * returned using sock_queue_err_skb, or free via kfree_skb. 3541 * 3542 * When passing buffers allocated with this function to sock_queue_err_skb 3543 * it is necessary to wrap the call with sock_hold/sock_put in order to 3544 * prevent the socket from being released prior to being enqueued on 3545 * the sk_error_queue. 3546 */ 3547struct sk_buff *skb_clone_sk(struct sk_buff *skb) 3548{ 3549 struct sock *sk = skb->sk; 3550 struct sk_buff *clone; 3551 3552 if (!sk || !atomic_inc_not_zero(&sk->sk_refcnt)) 3553 return NULL; 3554 3555 clone = skb_clone(skb, GFP_ATOMIC); 3556 if (!clone) { 3557 sock_put(sk); 3558 return NULL; 3559 } 3560 3561 clone->sk = sk; 3562 clone->destructor = sock_efree; 3563 3564 return clone; 3565} 3566EXPORT_SYMBOL(skb_clone_sk); 3567 3568static void __skb_complete_tx_timestamp(struct sk_buff *skb, 3569 struct sock *sk, 3570 int tstype) 3571{ 3572 struct sock_exterr_skb *serr; 3573 int err; 3574 3575 serr = SKB_EXT_ERR(skb); 3576 memset(serr, 0, sizeof(*serr)); 3577 serr->ee.ee_errno = ENOMSG; 3578 serr->ee.ee_origin = SO_EE_ORIGIN_TIMESTAMPING; 3579 serr->ee.ee_info = tstype; 3580 if (sk->sk_tsflags & SOF_TIMESTAMPING_OPT_ID) { 3581 serr->ee.ee_data = skb_shinfo(skb)->tskey; 3582 if (sk->sk_protocol == IPPROTO_TCP) 3583 serr->ee.ee_data -= sk->sk_tskey; 3584 } 3585 3586 err = sock_queue_err_skb(sk, skb); 3587 3588 if (err) 3589 kfree_skb(skb); 3590} 3591 3592void skb_complete_tx_timestamp(struct sk_buff *skb, 3593 struct skb_shared_hwtstamps *hwtstamps) 3594{ 3595 struct sock *sk = skb->sk; 3596 3597 /* take a reference to prevent skb_orphan() from freeing the socket */ 3598 sock_hold(sk); 3599 3600 *skb_hwtstamps(skb) = *hwtstamps; 3601 __skb_complete_tx_timestamp(skb, sk, SCM_TSTAMP_SND); 3602 3603 sock_put(sk); 3604} 3605EXPORT_SYMBOL_GPL(skb_complete_tx_timestamp); 3606 3607void __skb_tstamp_tx(struct sk_buff *orig_skb, 3608 struct skb_shared_hwtstamps *hwtstamps, 3609 struct sock *sk, int tstype) 3610{ 3611 struct sk_buff *skb; 3612 3613 if (!sk) 3614 return; 3615 3616 if (hwtstamps) 3617 *skb_hwtstamps(orig_skb) = *hwtstamps; 3618 else 3619 orig_skb->tstamp = ktime_get_real(); 3620 3621 skb = skb_clone(orig_skb, GFP_ATOMIC); 3622 if (!skb) 3623 return; 3624 3625 __skb_complete_tx_timestamp(skb, sk, tstype); 3626} 3627EXPORT_SYMBOL_GPL(__skb_tstamp_tx); 3628 3629void skb_tstamp_tx(struct sk_buff *orig_skb, 3630 struct skb_shared_hwtstamps *hwtstamps) 3631{ 3632 return __skb_tstamp_tx(orig_skb, hwtstamps, orig_skb->sk, 3633 SCM_TSTAMP_SND); 3634} 3635EXPORT_SYMBOL_GPL(skb_tstamp_tx); 3636 3637void skb_complete_wifi_ack(struct sk_buff *skb, bool acked) 3638{ 3639 struct sock *sk = skb->sk; 3640 struct sock_exterr_skb *serr; 3641 int err; 3642 3643 skb->wifi_acked_valid = 1; 3644 skb->wifi_acked = acked; 3645 3646 serr = SKB_EXT_ERR(skb); 3647 memset(serr, 0, sizeof(*serr)); 3648 serr->ee.ee_errno = ENOMSG; 3649 serr->ee.ee_origin = SO_EE_ORIGIN_TXSTATUS; 3650 3651 /* take a reference to prevent skb_orphan() from freeing the socket */ 3652 sock_hold(sk); 3653 3654 err = sock_queue_err_skb(sk, skb); 3655 if (err) 3656 kfree_skb(skb); 3657 3658 sock_put(sk); 3659} 3660EXPORT_SYMBOL_GPL(skb_complete_wifi_ack); 3661 3662 3663/** 3664 * skb_partial_csum_set - set up and verify partial csum values for packet 3665 * @skb: the skb to set 3666 * @start: the number of bytes after skb->data to start checksumming. 3667 * @off: the offset from start to place the checksum. 3668 * 3669 * For untrusted partially-checksummed packets, we need to make sure the values 3670 * for skb->csum_start and skb->csum_offset are valid so we don't oops. 3671 * 3672 * This function checks and sets those values and skb->ip_summed: if this 3673 * returns false you should drop the packet. 3674 */ 3675bool skb_partial_csum_set(struct sk_buff *skb, u16 start, u16 off) 3676{ 3677 if (unlikely(start > skb_headlen(skb)) || 3678 unlikely((int)start + off > skb_headlen(skb) - 2)) { 3679 net_warn_ratelimited("bad partial csum: csum=%u/%u len=%u\n", 3680 start, off, skb_headlen(skb)); 3681 return false; 3682 } 3683 skb->ip_summed = CHECKSUM_PARTIAL; 3684 skb->csum_start = skb_headroom(skb) + start; 3685 skb->csum_offset = off; 3686 skb_set_transport_header(skb, start); 3687 return true; 3688} 3689EXPORT_SYMBOL_GPL(skb_partial_csum_set); 3690 3691static int skb_maybe_pull_tail(struct sk_buff *skb, unsigned int len, 3692 unsigned int max) 3693{ 3694 if (skb_headlen(skb) >= len) 3695 return 0; 3696 3697 /* If we need to pullup then pullup to the max, so we 3698 * won't need to do it again. 3699 */ 3700 if (max > skb->len) 3701 max = skb->len; 3702 3703 if (__pskb_pull_tail(skb, max - skb_headlen(skb)) == NULL) 3704 return -ENOMEM; 3705 3706 if (skb_headlen(skb) < len) 3707 return -EPROTO; 3708 3709 return 0; 3710} 3711 3712#define MAX_TCP_HDR_LEN (15 * 4) 3713 3714static __sum16 *skb_checksum_setup_ip(struct sk_buff *skb, 3715 typeof(IPPROTO_IP) proto, 3716 unsigned int off) 3717{ 3718 switch (proto) { 3719 int err; 3720 3721 case IPPROTO_TCP: 3722 err = skb_maybe_pull_tail(skb, off + sizeof(struct tcphdr), 3723 off + MAX_TCP_HDR_LEN); 3724 if (!err && !skb_partial_csum_set(skb, off, 3725 offsetof(struct tcphdr, 3726 check))) 3727 err = -EPROTO; 3728 return err ? ERR_PTR(err) : &tcp_hdr(skb)->check; 3729 3730 case IPPROTO_UDP: 3731 err = skb_maybe_pull_tail(skb, off + sizeof(struct udphdr), 3732 off + sizeof(struct udphdr)); 3733 if (!err && !skb_partial_csum_set(skb, off, 3734 offsetof(struct udphdr, 3735 check))) 3736 err = -EPROTO; 3737 return err ? ERR_PTR(err) : &udp_hdr(skb)->check; 3738 } 3739 3740 return ERR_PTR(-EPROTO); 3741} 3742 3743/* This value should be large enough to cover a tagged ethernet header plus 3744 * maximally sized IP and TCP or UDP headers. 3745 */ 3746#define MAX_IP_HDR_LEN 128 3747 3748static int skb_checksum_setup_ipv4(struct sk_buff *skb, bool recalculate) 3749{ 3750 unsigned int off; 3751 bool fragment; 3752 __sum16 *csum; 3753 int err; 3754 3755 fragment = false; 3756 3757 err = skb_maybe_pull_tail(skb, 3758 sizeof(struct iphdr), 3759 MAX_IP_HDR_LEN); 3760 if (err < 0) 3761 goto out; 3762 3763 if (ip_hdr(skb)->frag_off & htons(IP_OFFSET | IP_MF)) 3764 fragment = true; 3765 3766 off = ip_hdrlen(skb); 3767 3768 err = -EPROTO; 3769 3770 if (fragment) 3771 goto out; 3772 3773 csum = skb_checksum_setup_ip(skb, ip_hdr(skb)->protocol, off); 3774 if (IS_ERR(csum)) 3775 return PTR_ERR(csum); 3776 3777 if (recalculate) 3778 *csum = ~csum_tcpudp_magic(ip_hdr(skb)->saddr, 3779 ip_hdr(skb)->daddr, 3780 skb->len - off, 3781 ip_hdr(skb)->protocol, 0); 3782 err = 0; 3783 3784out: 3785 return err; 3786} 3787 3788/* This value should be large enough to cover a tagged ethernet header plus 3789 * an IPv6 header, all options, and a maximal TCP or UDP header. 3790 */ 3791#define MAX_IPV6_HDR_LEN 256 3792 3793#define OPT_HDR(type, skb, off) \ 3794 (type *)(skb_network_header(skb) + (off)) 3795 3796static int skb_checksum_setup_ipv6(struct sk_buff *skb, bool recalculate) 3797{ 3798 int err; 3799 u8 nexthdr; 3800 unsigned int off; 3801 unsigned int len; 3802 bool fragment; 3803 bool done; 3804 __sum16 *csum; 3805 3806 fragment = false; 3807 done = false; 3808 3809 off = sizeof(struct ipv6hdr); 3810 3811 err = skb_maybe_pull_tail(skb, off, MAX_IPV6_HDR_LEN); 3812 if (err < 0) 3813 goto out; 3814 3815 nexthdr = ipv6_hdr(skb)->nexthdr; 3816 3817 len = sizeof(struct ipv6hdr) + ntohs(ipv6_hdr(skb)->payload_len); 3818 while (off <= len && !done) { 3819 switch (nexthdr) { 3820 case IPPROTO_DSTOPTS: 3821 case IPPROTO_HOPOPTS: 3822 case IPPROTO_ROUTING: { 3823 struct ipv6_opt_hdr *hp; 3824 3825 err = skb_maybe_pull_tail(skb, 3826 off + 3827 sizeof(struct ipv6_opt_hdr), 3828 MAX_IPV6_HDR_LEN); 3829 if (err < 0) 3830 goto out; 3831 3832 hp = OPT_HDR(struct ipv6_opt_hdr, skb, off); 3833 nexthdr = hp->nexthdr; 3834 off += ipv6_optlen(hp); 3835 break; 3836 } 3837 case IPPROTO_AH: { 3838 struct ip_auth_hdr *hp; 3839 3840 err = skb_maybe_pull_tail(skb, 3841 off + 3842 sizeof(struct ip_auth_hdr), 3843 MAX_IPV6_HDR_LEN); 3844 if (err < 0) 3845 goto out; 3846 3847 hp = OPT_HDR(struct ip_auth_hdr, skb, off); 3848 nexthdr = hp->nexthdr; 3849 off += ipv6_authlen(hp); 3850 break; 3851 } 3852 case IPPROTO_FRAGMENT: { 3853 struct frag_hdr *hp; 3854 3855 err = skb_maybe_pull_tail(skb, 3856 off + 3857 sizeof(struct frag_hdr), 3858 MAX_IPV6_HDR_LEN); 3859 if (err < 0) 3860 goto out; 3861 3862 hp = OPT_HDR(struct frag_hdr, skb, off); 3863 3864 if (hp->frag_off & htons(IP6_OFFSET | IP6_MF)) 3865 fragment = true; 3866 3867 nexthdr = hp->nexthdr; 3868 off += sizeof(struct frag_hdr); 3869 break; 3870 } 3871 default: 3872 done = true; 3873 break; 3874 } 3875 } 3876 3877 err = -EPROTO; 3878 3879 if (!done || fragment) 3880 goto out; 3881 3882 csum = skb_checksum_setup_ip(skb, nexthdr, off); 3883 if (IS_ERR(csum)) 3884 return PTR_ERR(csum); 3885 3886 if (recalculate) 3887 *csum = ~csum_ipv6_magic(&ipv6_hdr(skb)->saddr, 3888 &ipv6_hdr(skb)->daddr, 3889 skb->len - off, nexthdr, 0); 3890 err = 0; 3891 3892out: 3893 return err; 3894} 3895 3896/** 3897 * skb_checksum_setup - set up partial checksum offset 3898 * @skb: the skb to set up 3899 * @recalculate: if true the pseudo-header checksum will be recalculated 3900 */ 3901int skb_checksum_setup(struct sk_buff *skb, bool recalculate) 3902{ 3903 int err; 3904 3905 switch (skb->protocol) { 3906 case htons(ETH_P_IP): 3907 err = skb_checksum_setup_ipv4(skb, recalculate); 3908 break; 3909 3910 case htons(ETH_P_IPV6): 3911 err = skb_checksum_setup_ipv6(skb, recalculate); 3912 break; 3913 3914 default: 3915 err = -EPROTO; 3916 break; 3917 } 3918 3919 return err; 3920} 3921EXPORT_SYMBOL(skb_checksum_setup); 3922 3923void __skb_warn_lro_forwarding(const struct sk_buff *skb) 3924{ 3925 net_warn_ratelimited("%s: received packets cannot be forwarded while LRO is enabled\n", 3926 skb->dev->name); 3927} 3928EXPORT_SYMBOL(__skb_warn_lro_forwarding); 3929 3930void kfree_skb_partial(struct sk_buff *skb, bool head_stolen) 3931{ 3932 if (head_stolen) { 3933 skb_release_head_state(skb); 3934 kmem_cache_free(skbuff_head_cache, skb); 3935 } else { 3936 __kfree_skb(skb); 3937 } 3938} 3939EXPORT_SYMBOL(kfree_skb_partial); 3940 3941/** 3942 * skb_try_coalesce - try to merge skb to prior one 3943 * @to: prior buffer 3944 * @from: buffer to add 3945 * @fragstolen: pointer to boolean 3946 * @delta_truesize: how much more was allocated than was requested 3947 */ 3948bool skb_try_coalesce(struct sk_buff *to, struct sk_buff *from, 3949 bool *fragstolen, int *delta_truesize) 3950{ 3951 int i, delta, len = from->len; 3952 3953 *fragstolen = false; 3954 3955 if (skb_cloned(to)) 3956 return false; 3957 3958 if (len <= skb_tailroom(to)) { 3959 if (len) 3960 BUG_ON(skb_copy_bits(from, 0, skb_put(to, len), len)); 3961 *delta_truesize = 0; 3962 return true; 3963 } 3964 3965 if (skb_has_frag_list(to) || skb_has_frag_list(from)) 3966 return false; 3967 3968 if (skb_headlen(from) != 0) { 3969 struct page *page; 3970 unsigned int offset; 3971 3972 if (skb_shinfo(to)->nr_frags + 3973 skb_shinfo(from)->nr_frags >= MAX_SKB_FRAGS) 3974 return false; 3975 3976 if (skb_head_is_locked(from)) 3977 return false; 3978 3979 delta = from->truesize - SKB_DATA_ALIGN(sizeof(struct sk_buff)); 3980 3981 page = virt_to_head_page(from->head); 3982 offset = from->data - (unsigned char *)page_address(page); 3983 3984 skb_fill_page_desc(to, skb_shinfo(to)->nr_frags, 3985 page, offset, skb_headlen(from)); 3986 *fragstolen = true; 3987 } else { 3988 if (skb_shinfo(to)->nr_frags + 3989 skb_shinfo(from)->nr_frags > MAX_SKB_FRAGS) 3990 return false; 3991 3992 delta = from->truesize - SKB_TRUESIZE(skb_end_offset(from)); 3993 } 3994 3995 WARN_ON_ONCE(delta < len); 3996 3997 memcpy(skb_shinfo(to)->frags + skb_shinfo(to)->nr_frags, 3998 skb_shinfo(from)->frags, 3999 skb_shinfo(from)->nr_frags * sizeof(skb_frag_t)); 4000 skb_shinfo(to)->nr_frags += skb_shinfo(from)->nr_frags; 4001 4002 if (!skb_cloned(from)) 4003 skb_shinfo(from)->nr_frags = 0; 4004 4005 /* if the skb is not cloned this does nothing 4006 * since we set nr_frags to 0. 4007 */ 4008 for (i = 0; i < skb_shinfo(from)->nr_frags; i++) 4009 skb_frag_ref(from, i); 4010 4011 to->truesize += delta; 4012 to->len += len; 4013 to->data_len += len; 4014 4015 *delta_truesize = delta; 4016 return true; 4017} 4018EXPORT_SYMBOL(skb_try_coalesce); 4019 4020/** 4021 * skb_scrub_packet - scrub an skb 4022 * 4023 * @skb: buffer to clean 4024 * @xnet: packet is crossing netns 4025 * 4026 * skb_scrub_packet can be used after encapsulating or decapsulting a packet 4027 * into/from a tunnel. Some information have to be cleared during these 4028 * operations. 4029 * skb_scrub_packet can also be used to clean a skb before injecting it in 4030 * another namespace (@xnet == true). We have to clear all information in the 4031 * skb that could impact namespace isolation. 4032 */ 4033void skb_scrub_packet(struct sk_buff *skb, bool xnet) 4034{ 4035 if (xnet) 4036 skb_orphan(skb); 4037 skb->tstamp.tv64 = 0; 4038 skb->pkt_type = PACKET_HOST; 4039 skb->skb_iif = 0; 4040 skb->ignore_df = 0; 4041 skb_dst_drop(skb); 4042 skb->mark = 0; 4043 secpath_reset(skb); 4044 nf_reset(skb); 4045 nf_reset_trace(skb); 4046} 4047EXPORT_SYMBOL_GPL(skb_scrub_packet); 4048 4049/** 4050 * skb_gso_transport_seglen - Return length of individual segments of a gso packet 4051 * 4052 * @skb: GSO skb 4053 * 4054 * skb_gso_transport_seglen is used to determine the real size of the 4055 * individual segments, including Layer4 headers (TCP/UDP). 4056 * 4057 * The MAC/L2 or network (IP, IPv6) headers are not accounted for. 4058 */ 4059unsigned int skb_gso_transport_seglen(const struct sk_buff *skb) 4060{ 4061 const struct skb_shared_info *shinfo = skb_shinfo(skb); 4062 unsigned int thlen = 0; 4063 4064 if (skb->encapsulation) { 4065 thlen = skb_inner_transport_header(skb) - 4066 skb_transport_header(skb); 4067 4068 if (likely(shinfo->gso_type & (SKB_GSO_TCPV4 | SKB_GSO_TCPV6))) 4069 thlen += inner_tcp_hdrlen(skb); 4070 } else if (likely(shinfo->gso_type & (SKB_GSO_TCPV4 | SKB_GSO_TCPV6))) { 4071 thlen = tcp_hdrlen(skb); 4072 } 4073 /* UFO sets gso_size to the size of the fragmentation 4074 * payload, i.e. the size of the L4 (UDP) header is already 4075 * accounted for. 4076 */ 4077 return thlen + shinfo->gso_size; 4078} 4079EXPORT_SYMBOL_GPL(skb_gso_transport_seglen); 4080 4081static struct sk_buff *skb_reorder_vlan_header(struct sk_buff *skb) 4082{ 4083 if (skb_cow(skb, skb_headroom(skb)) < 0) { 4084 kfree_skb(skb); 4085 return NULL; 4086 } 4087 4088 memmove(skb->data - ETH_HLEN, skb->data - VLAN_ETH_HLEN, 2 * ETH_ALEN); 4089 skb->mac_header += VLAN_HLEN; 4090 return skb; 4091} 4092 4093struct sk_buff *skb_vlan_untag(struct sk_buff *skb) 4094{ 4095 struct vlan_hdr *vhdr; 4096 u16 vlan_tci; 4097 4098 if (unlikely(vlan_tx_tag_present(skb))) { 4099 /* vlan_tci is already set-up so leave this for another time */ 4100 return skb; 4101 } 4102 4103 skb = skb_share_check(skb, GFP_ATOMIC); 4104 if (unlikely(!skb)) 4105 goto err_free; 4106 4107 if (unlikely(!pskb_may_pull(skb, VLAN_HLEN))) 4108 goto err_free; 4109 4110 vhdr = (struct vlan_hdr *)skb->data; 4111 vlan_tci = ntohs(vhdr->h_vlan_TCI); 4112 __vlan_hwaccel_put_tag(skb, skb->protocol, vlan_tci); 4113 4114 skb_pull_rcsum(skb, VLAN_HLEN); 4115 vlan_set_encap_proto(skb, vhdr); 4116 4117 skb = skb_reorder_vlan_header(skb); 4118 if (unlikely(!skb)) 4119 goto err_free; 4120 4121 skb_reset_network_header(skb); 4122 skb_reset_transport_header(skb); 4123 skb_reset_mac_len(skb); 4124 4125 return skb; 4126 4127err_free: 4128 kfree_skb(skb); 4129 return NULL; 4130} 4131EXPORT_SYMBOL(skb_vlan_untag); 4132 4133/** 4134 * alloc_skb_with_frags - allocate skb with page frags 4135 * 4136 * @header_len: size of linear part 4137 * @data_len: needed length in frags 4138 * @max_page_order: max page order desired. 4139 * @errcode: pointer to error code if any 4140 * @gfp_mask: allocation mask 4141 * 4142 * This can be used to allocate a paged skb, given a maximal order for frags. 4143 */ 4144struct sk_buff *alloc_skb_with_frags(unsigned long header_len, 4145 unsigned long data_len, 4146 int max_page_order, 4147 int *errcode, 4148 gfp_t gfp_mask) 4149{ 4150 int npages = (data_len + (PAGE_SIZE - 1)) >> PAGE_SHIFT; 4151 unsigned long chunk; 4152 struct sk_buff *skb; 4153 struct page *page; 4154 gfp_t gfp_head; 4155 int i; 4156 4157 *errcode = -EMSGSIZE; 4158 /* Note this test could be relaxed, if we succeed to allocate 4159 * high order pages... 4160 */ 4161 if (npages > MAX_SKB_FRAGS) 4162 return NULL; 4163 4164 gfp_head = gfp_mask; 4165 if (gfp_head & __GFP_WAIT) 4166 gfp_head |= __GFP_REPEAT; 4167 4168 *errcode = -ENOBUFS; 4169 skb = alloc_skb(header_len, gfp_head); 4170 if (!skb) 4171 return NULL; 4172 4173 skb->truesize += npages << PAGE_SHIFT; 4174 4175 for (i = 0; npages > 0; i++) { 4176 int order = max_page_order; 4177 4178 while (order) { 4179 if (npages >= 1 << order) { 4180 page = alloc_pages(gfp_mask | 4181 __GFP_COMP | 4182 __GFP_NOWARN | 4183 __GFP_NORETRY, 4184 order); 4185 if (page) 4186 goto fill_page; 4187 /* Do not retry other high order allocations */ 4188 order = 1; 4189 max_page_order = 0; 4190 } 4191 order--; 4192 } 4193 page = alloc_page(gfp_mask); 4194 if (!page) 4195 goto failure; 4196fill_page: 4197 chunk = min_t(unsigned long, data_len, 4198 PAGE_SIZE << order); 4199 skb_fill_page_desc(skb, i, page, 0, chunk); 4200 data_len -= chunk; 4201 npages -= 1 << order; 4202 } 4203 return skb; 4204 4205failure: 4206 kfree_skb(skb); 4207 return NULL; 4208} 4209EXPORT_SYMBOL(alloc_skb_with_frags); 4210