10744dd00c1b1be99a25b62b1b48df440e82e57e0Eric Dumazet#include <linux/skbuff.h> 2c452ed70771cea3af73d21a5914989137fbd28b8Jesper Dangaard Brouer#include <linux/export.h> 30744dd00c1b1be99a25b62b1b48df440e82e57e0Eric Dumazet#include <linux/ip.h> 40744dd00c1b1be99a25b62b1b48df440e82e57e0Eric Dumazet#include <linux/ipv6.h> 50744dd00c1b1be99a25b62b1b48df440e82e57e0Eric Dumazet#include <linux/if_vlan.h> 60744dd00c1b1be99a25b62b1b48df440e82e57e0Eric Dumazet#include <net/ip.h> 7ddbe503203855939946430e39bae58de11b70b69Eric Dumazet#include <net/ipv6.h> 8f77668dc25b27270fe589031b22c432c3462b1d8Daniel Borkmann#include <linux/igmp.h> 9f77668dc25b27270fe589031b22c432c3462b1d8Daniel Borkmann#include <linux/icmp.h> 10f77668dc25b27270fe589031b22c432c3462b1d8Daniel Borkmann#include <linux/sctp.h> 11f77668dc25b27270fe589031b22c432c3462b1d8Daniel Borkmann#include <linux/dccp.h> 120744dd00c1b1be99a25b62b1b48df440e82e57e0Eric Dumazet#include <linux/if_tunnel.h> 130744dd00c1b1be99a25b62b1b48df440e82e57e0Eric Dumazet#include <linux/if_pppox.h> 140744dd00c1b1be99a25b62b1b48df440e82e57e0Eric Dumazet#include <linux/ppp_defs.h> 150744dd00c1b1be99a25b62b1b48df440e82e57e0Eric Dumazet#include <net/flow_keys.h> 1656193d1bce2b2759cb4bdcc00cd05544894a0c90Alexander Duyck#include <scsi/fc/fc_fcoe.h> 170744dd00c1b1be99a25b62b1b48df440e82e57e0Eric Dumazet 184d77d2b567ec66a443792d99e96ac760991d80d0Eric Dumazet/* copy saddr & daddr, possibly using 64bit load/store 194d77d2b567ec66a443792d99e96ac760991d80d0Eric Dumazet * Equivalent to : flow->src = iph->saddr; 204d77d2b567ec66a443792d99e96ac760991d80d0Eric Dumazet * flow->dst = iph->daddr; 214d77d2b567ec66a443792d99e96ac760991d80d0Eric Dumazet */ 224d77d2b567ec66a443792d99e96ac760991d80d0Eric Dumazetstatic void iph_to_flow_copy_addrs(struct flow_keys *flow, const struct iphdr *iph) 234d77d2b567ec66a443792d99e96ac760991d80d0Eric Dumazet{ 244d77d2b567ec66a443792d99e96ac760991d80d0Eric Dumazet BUILD_BUG_ON(offsetof(typeof(*flow), dst) != 254d77d2b567ec66a443792d99e96ac760991d80d0Eric Dumazet offsetof(typeof(*flow), src) + sizeof(flow->src)); 264d77d2b567ec66a443792d99e96ac760991d80d0Eric Dumazet memcpy(&flow->src, &iph->saddr, sizeof(flow->src) + sizeof(flow->dst)); 274d77d2b567ec66a443792d99e96ac760991d80d0Eric Dumazet} 280744dd00c1b1be99a25b62b1b48df440e82e57e0Eric Dumazet 29357afe9c46c951c34769e39cabdf8d1637e2eeccNikolay Aleksandrov/** 306451b3f59ab39162d1fbb5a5d0c8f46c0d9e1231WANG Cong * __skb_flow_get_ports - extract the upper layer ports and return them 316451b3f59ab39162d1fbb5a5d0c8f46c0d9e1231WANG Cong * @skb: sk_buff to extract the ports from 32357afe9c46c951c34769e39cabdf8d1637e2eeccNikolay Aleksandrov * @thoff: transport header offset 33357afe9c46c951c34769e39cabdf8d1637e2eeccNikolay Aleksandrov * @ip_proto: protocol for which to get port offset 346451b3f59ab39162d1fbb5a5d0c8f46c0d9e1231WANG Cong * @data: raw buffer pointer to the packet, if NULL use skb->data 356451b3f59ab39162d1fbb5a5d0c8f46c0d9e1231WANG Cong * @hlen: packet header length, if @data is NULL use skb_headlen(skb) 36357afe9c46c951c34769e39cabdf8d1637e2eeccNikolay Aleksandrov * 37357afe9c46c951c34769e39cabdf8d1637e2eeccNikolay Aleksandrov * The function will try to retrieve the ports at offset thoff + poff where poff 38357afe9c46c951c34769e39cabdf8d1637e2eeccNikolay Aleksandrov * is the protocol port offset returned from proto_ports_offset 39357afe9c46c951c34769e39cabdf8d1637e2eeccNikolay Aleksandrov */ 40690e36e726d00d2528bc569809048adf61550d80David S. Miller__be32 __skb_flow_get_ports(const struct sk_buff *skb, int thoff, u8 ip_proto, 41690e36e726d00d2528bc569809048adf61550d80David S. Miller void *data, int hlen) 42357afe9c46c951c34769e39cabdf8d1637e2eeccNikolay Aleksandrov{ 43357afe9c46c951c34769e39cabdf8d1637e2eeccNikolay Aleksandrov int poff = proto_ports_offset(ip_proto); 44357afe9c46c951c34769e39cabdf8d1637e2eeccNikolay Aleksandrov 45690e36e726d00d2528bc569809048adf61550d80David S. Miller if (!data) { 46690e36e726d00d2528bc569809048adf61550d80David S. Miller data = skb->data; 47690e36e726d00d2528bc569809048adf61550d80David S. Miller hlen = skb_headlen(skb); 48690e36e726d00d2528bc569809048adf61550d80David S. Miller } 49690e36e726d00d2528bc569809048adf61550d80David S. Miller 50357afe9c46c951c34769e39cabdf8d1637e2eeccNikolay Aleksandrov if (poff >= 0) { 51357afe9c46c951c34769e39cabdf8d1637e2eeccNikolay Aleksandrov __be32 *ports, _ports; 52357afe9c46c951c34769e39cabdf8d1637e2eeccNikolay Aleksandrov 53690e36e726d00d2528bc569809048adf61550d80David S. Miller ports = __skb_header_pointer(skb, thoff + poff, 54690e36e726d00d2528bc569809048adf61550d80David S. Miller sizeof(_ports), data, hlen, &_ports); 55357afe9c46c951c34769e39cabdf8d1637e2eeccNikolay Aleksandrov if (ports) 56357afe9c46c951c34769e39cabdf8d1637e2eeccNikolay Aleksandrov return *ports; 57357afe9c46c951c34769e39cabdf8d1637e2eeccNikolay Aleksandrov } 58357afe9c46c951c34769e39cabdf8d1637e2eeccNikolay Aleksandrov 59357afe9c46c951c34769e39cabdf8d1637e2eeccNikolay Aleksandrov return 0; 60357afe9c46c951c34769e39cabdf8d1637e2eeccNikolay Aleksandrov} 61690e36e726d00d2528bc569809048adf61550d80David S. MillerEXPORT_SYMBOL(__skb_flow_get_ports); 62357afe9c46c951c34769e39cabdf8d1637e2eeccNikolay Aleksandrov 63453a940ea725d692282f9e66475cec0d1b1e12f2WANG Cong/** 64453a940ea725d692282f9e66475cec0d1b1e12f2WANG Cong * __skb_flow_dissect - extract the flow_keys struct and return it 65453a940ea725d692282f9e66475cec0d1b1e12f2WANG Cong * @skb: sk_buff to extract the flow from, can be NULL if the rest are specified 66453a940ea725d692282f9e66475cec0d1b1e12f2WANG Cong * @data: raw buffer pointer to the packet, if NULL use skb->data 67453a940ea725d692282f9e66475cec0d1b1e12f2WANG Cong * @proto: protocol for which to get the flow, if @data is NULL use skb->protocol 68453a940ea725d692282f9e66475cec0d1b1e12f2WANG Cong * @nhoff: network header offset, if @data is NULL use skb_network_offset(skb) 69453a940ea725d692282f9e66475cec0d1b1e12f2WANG Cong * @hlen: packet header length, if @data is NULL use skb_headlen(skb) 70453a940ea725d692282f9e66475cec0d1b1e12f2WANG Cong * 71453a940ea725d692282f9e66475cec0d1b1e12f2WANG Cong * The function will try to retrieve the struct flow_keys from either the skbuff 72453a940ea725d692282f9e66475cec0d1b1e12f2WANG Cong * or a raw buffer specified by the rest parameters 73453a940ea725d692282f9e66475cec0d1b1e12f2WANG Cong */ 74453a940ea725d692282f9e66475cec0d1b1e12f2WANG Congbool __skb_flow_dissect(const struct sk_buff *skb, struct flow_keys *flow, 75453a940ea725d692282f9e66475cec0d1b1e12f2WANG Cong void *data, __be16 proto, int nhoff, int hlen) 760744dd00c1b1be99a25b62b1b48df440e82e57e0Eric Dumazet{ 770744dd00c1b1be99a25b62b1b48df440e82e57e0Eric Dumazet u8 ip_proto; 780744dd00c1b1be99a25b62b1b48df440e82e57e0Eric Dumazet 79690e36e726d00d2528bc569809048adf61550d80David S. Miller if (!data) { 80690e36e726d00d2528bc569809048adf61550d80David S. Miller data = skb->data; 81453a940ea725d692282f9e66475cec0d1b1e12f2WANG Cong proto = skb->protocol; 82453a940ea725d692282f9e66475cec0d1b1e12f2WANG Cong nhoff = skb_network_offset(skb); 83690e36e726d00d2528bc569809048adf61550d80David S. Miller hlen = skb_headlen(skb); 84690e36e726d00d2528bc569809048adf61550d80David S. Miller } 85690e36e726d00d2528bc569809048adf61550d80David S. Miller 860744dd00c1b1be99a25b62b1b48df440e82e57e0Eric Dumazet memset(flow, 0, sizeof(*flow)); 870744dd00c1b1be99a25b62b1b48df440e82e57e0Eric Dumazet 880744dd00c1b1be99a25b62b1b48df440e82e57e0Eric Dumazetagain: 890744dd00c1b1be99a25b62b1b48df440e82e57e0Eric Dumazet switch (proto) { 902b8837aeaaa0bb6b4b3be1b3afd1cc088f68a362Joe Perches case htons(ETH_P_IP): { 910744dd00c1b1be99a25b62b1b48df440e82e57e0Eric Dumazet const struct iphdr *iph; 920744dd00c1b1be99a25b62b1b48df440e82e57e0Eric Dumazet struct iphdr _iph; 930744dd00c1b1be99a25b62b1b48df440e82e57e0Eric Dumazetip: 94690e36e726d00d2528bc569809048adf61550d80David S. Miller iph = __skb_header_pointer(skb, nhoff, sizeof(_iph), data, hlen, &_iph); 956f092343855a71e03b8d209815d8c45bf3a27fcdJason Wang if (!iph || iph->ihl < 5) 960744dd00c1b1be99a25b62b1b48df440e82e57e0Eric Dumazet return false; 973797d3e8462efdaadb64164ca540626b55fe8336Eric Dumazet nhoff += iph->ihl * 4; 980744dd00c1b1be99a25b62b1b48df440e82e57e0Eric Dumazet 993797d3e8462efdaadb64164ca540626b55fe8336Eric Dumazet ip_proto = iph->protocol; 1000744dd00c1b1be99a25b62b1b48df440e82e57e0Eric Dumazet if (ip_is_fragment(iph)) 1010744dd00c1b1be99a25b62b1b48df440e82e57e0Eric Dumazet ip_proto = 0; 1023797d3e8462efdaadb64164ca540626b55fe8336Eric Dumazet 1035af7fb6e3e92c2797ee30d66138cf6aa6b29240dAlexander Duyck /* skip the address processing if skb is NULL. The assumption 1045af7fb6e3e92c2797ee30d66138cf6aa6b29240dAlexander Duyck * here is that if there is no skb we are not looking for flow 1055af7fb6e3e92c2797ee30d66138cf6aa6b29240dAlexander Duyck * info but lengths and protocols. 1065af7fb6e3e92c2797ee30d66138cf6aa6b29240dAlexander Duyck */ 1075af7fb6e3e92c2797ee30d66138cf6aa6b29240dAlexander Duyck if (!skb) 1085af7fb6e3e92c2797ee30d66138cf6aa6b29240dAlexander Duyck break; 1095af7fb6e3e92c2797ee30d66138cf6aa6b29240dAlexander Duyck 1104d77d2b567ec66a443792d99e96ac760991d80d0Eric Dumazet iph_to_flow_copy_addrs(flow, iph); 1110744dd00c1b1be99a25b62b1b48df440e82e57e0Eric Dumazet break; 1120744dd00c1b1be99a25b62b1b48df440e82e57e0Eric Dumazet } 1132b8837aeaaa0bb6b4b3be1b3afd1cc088f68a362Joe Perches case htons(ETH_P_IPV6): { 1140744dd00c1b1be99a25b62b1b48df440e82e57e0Eric Dumazet const struct ipv6hdr *iph; 1150744dd00c1b1be99a25b62b1b48df440e82e57e0Eric Dumazet struct ipv6hdr _iph; 11619469a873bafd4e65daef3597db2bd724c1b03c9Tom Herbert __be32 flow_label; 11719469a873bafd4e65daef3597db2bd724c1b03c9Tom Herbert 1180744dd00c1b1be99a25b62b1b48df440e82e57e0Eric Dumazetipv6: 119690e36e726d00d2528bc569809048adf61550d80David S. Miller iph = __skb_header_pointer(skb, nhoff, sizeof(_iph), data, hlen, &_iph); 1200744dd00c1b1be99a25b62b1b48df440e82e57e0Eric Dumazet if (!iph) 1210744dd00c1b1be99a25b62b1b48df440e82e57e0Eric Dumazet return false; 1220744dd00c1b1be99a25b62b1b48df440e82e57e0Eric Dumazet 1230744dd00c1b1be99a25b62b1b48df440e82e57e0Eric Dumazet ip_proto = iph->nexthdr; 1240744dd00c1b1be99a25b62b1b48df440e82e57e0Eric Dumazet nhoff += sizeof(struct ipv6hdr); 12519469a873bafd4e65daef3597db2bd724c1b03c9Tom Herbert 1265af7fb6e3e92c2797ee30d66138cf6aa6b29240dAlexander Duyck /* see comment above in IPv4 section */ 12756193d1bce2b2759cb4bdcc00cd05544894a0c90Alexander Duyck if (!skb) 12856193d1bce2b2759cb4bdcc00cd05544894a0c90Alexander Duyck break; 12956193d1bce2b2759cb4bdcc00cd05544894a0c90Alexander Duyck 1305af7fb6e3e92c2797ee30d66138cf6aa6b29240dAlexander Duyck flow->src = (__force __be32)ipv6_addr_hash(&iph->saddr); 1315af7fb6e3e92c2797ee30d66138cf6aa6b29240dAlexander Duyck flow->dst = (__force __be32)ipv6_addr_hash(&iph->daddr); 1325af7fb6e3e92c2797ee30d66138cf6aa6b29240dAlexander Duyck 13319469a873bafd4e65daef3597db2bd724c1b03c9Tom Herbert flow_label = ip6_flowlabel(iph); 13419469a873bafd4e65daef3597db2bd724c1b03c9Tom Herbert if (flow_label) { 13519469a873bafd4e65daef3597db2bd724c1b03c9Tom Herbert /* Awesome, IPv6 packet has a flow label so we can 13619469a873bafd4e65daef3597db2bd724c1b03c9Tom Herbert * use that to represent the ports without any 13719469a873bafd4e65daef3597db2bd724c1b03c9Tom Herbert * further dissection. 13819469a873bafd4e65daef3597db2bd724c1b03c9Tom Herbert */ 13919469a873bafd4e65daef3597db2bd724c1b03c9Tom Herbert flow->n_proto = proto; 14019469a873bafd4e65daef3597db2bd724c1b03c9Tom Herbert flow->ip_proto = ip_proto; 14119469a873bafd4e65daef3597db2bd724c1b03c9Tom Herbert flow->ports = flow_label; 14219469a873bafd4e65daef3597db2bd724c1b03c9Tom Herbert flow->thoff = (u16)nhoff; 14319469a873bafd4e65daef3597db2bd724c1b03c9Tom Herbert 14419469a873bafd4e65daef3597db2bd724c1b03c9Tom Herbert return true; 14519469a873bafd4e65daef3597db2bd724c1b03c9Tom Herbert } 14619469a873bafd4e65daef3597db2bd724c1b03c9Tom Herbert 1470744dd00c1b1be99a25b62b1b48df440e82e57e0Eric Dumazet break; 1480744dd00c1b1be99a25b62b1b48df440e82e57e0Eric Dumazet } 1492b8837aeaaa0bb6b4b3be1b3afd1cc088f68a362Joe Perches case htons(ETH_P_8021AD): 1502b8837aeaaa0bb6b4b3be1b3afd1cc088f68a362Joe Perches case htons(ETH_P_8021Q): { 1510744dd00c1b1be99a25b62b1b48df440e82e57e0Eric Dumazet const struct vlan_hdr *vlan; 1520744dd00c1b1be99a25b62b1b48df440e82e57e0Eric Dumazet struct vlan_hdr _vlan; 1530744dd00c1b1be99a25b62b1b48df440e82e57e0Eric Dumazet 154690e36e726d00d2528bc569809048adf61550d80David S. Miller vlan = __skb_header_pointer(skb, nhoff, sizeof(_vlan), data, hlen, &_vlan); 1550744dd00c1b1be99a25b62b1b48df440e82e57e0Eric Dumazet if (!vlan) 1560744dd00c1b1be99a25b62b1b48df440e82e57e0Eric Dumazet return false; 1570744dd00c1b1be99a25b62b1b48df440e82e57e0Eric Dumazet 1580744dd00c1b1be99a25b62b1b48df440e82e57e0Eric Dumazet proto = vlan->h_vlan_encapsulated_proto; 1590744dd00c1b1be99a25b62b1b48df440e82e57e0Eric Dumazet nhoff += sizeof(*vlan); 1600744dd00c1b1be99a25b62b1b48df440e82e57e0Eric Dumazet goto again; 1610744dd00c1b1be99a25b62b1b48df440e82e57e0Eric Dumazet } 1622b8837aeaaa0bb6b4b3be1b3afd1cc088f68a362Joe Perches case htons(ETH_P_PPP_SES): { 1630744dd00c1b1be99a25b62b1b48df440e82e57e0Eric Dumazet struct { 1640744dd00c1b1be99a25b62b1b48df440e82e57e0Eric Dumazet struct pppoe_hdr hdr; 1650744dd00c1b1be99a25b62b1b48df440e82e57e0Eric Dumazet __be16 proto; 1660744dd00c1b1be99a25b62b1b48df440e82e57e0Eric Dumazet } *hdr, _hdr; 167690e36e726d00d2528bc569809048adf61550d80David S. Miller hdr = __skb_header_pointer(skb, nhoff, sizeof(_hdr), data, hlen, &_hdr); 1680744dd00c1b1be99a25b62b1b48df440e82e57e0Eric Dumazet if (!hdr) 1690744dd00c1b1be99a25b62b1b48df440e82e57e0Eric Dumazet return false; 1700744dd00c1b1be99a25b62b1b48df440e82e57e0Eric Dumazet proto = hdr->proto; 1710744dd00c1b1be99a25b62b1b48df440e82e57e0Eric Dumazet nhoff += PPPOE_SES_HLEN; 1720744dd00c1b1be99a25b62b1b48df440e82e57e0Eric Dumazet switch (proto) { 1732b8837aeaaa0bb6b4b3be1b3afd1cc088f68a362Joe Perches case htons(PPP_IP): 1740744dd00c1b1be99a25b62b1b48df440e82e57e0Eric Dumazet goto ip; 1752b8837aeaaa0bb6b4b3be1b3afd1cc088f68a362Joe Perches case htons(PPP_IPV6): 1760744dd00c1b1be99a25b62b1b48df440e82e57e0Eric Dumazet goto ipv6; 1770744dd00c1b1be99a25b62b1b48df440e82e57e0Eric Dumazet default: 1780744dd00c1b1be99a25b62b1b48df440e82e57e0Eric Dumazet return false; 1790744dd00c1b1be99a25b62b1b48df440e82e57e0Eric Dumazet } 1800744dd00c1b1be99a25b62b1b48df440e82e57e0Eric Dumazet } 18156193d1bce2b2759cb4bdcc00cd05544894a0c90Alexander Duyck case htons(ETH_P_FCOE): 18256193d1bce2b2759cb4bdcc00cd05544894a0c90Alexander Duyck flow->thoff = (u16)(nhoff + FCOE_HEADER_LEN); 18356193d1bce2b2759cb4bdcc00cd05544894a0c90Alexander Duyck /* fall through */ 1840744dd00c1b1be99a25b62b1b48df440e82e57e0Eric Dumazet default: 1850744dd00c1b1be99a25b62b1b48df440e82e57e0Eric Dumazet return false; 1860744dd00c1b1be99a25b62b1b48df440e82e57e0Eric Dumazet } 1870744dd00c1b1be99a25b62b1b48df440e82e57e0Eric Dumazet 1880744dd00c1b1be99a25b62b1b48df440e82e57e0Eric Dumazet switch (ip_proto) { 1890744dd00c1b1be99a25b62b1b48df440e82e57e0Eric Dumazet case IPPROTO_GRE: { 1900744dd00c1b1be99a25b62b1b48df440e82e57e0Eric Dumazet struct gre_hdr { 1910744dd00c1b1be99a25b62b1b48df440e82e57e0Eric Dumazet __be16 flags; 1920744dd00c1b1be99a25b62b1b48df440e82e57e0Eric Dumazet __be16 proto; 1930744dd00c1b1be99a25b62b1b48df440e82e57e0Eric Dumazet } *hdr, _hdr; 1940744dd00c1b1be99a25b62b1b48df440e82e57e0Eric Dumazet 195690e36e726d00d2528bc569809048adf61550d80David S. Miller hdr = __skb_header_pointer(skb, nhoff, sizeof(_hdr), data, hlen, &_hdr); 1960744dd00c1b1be99a25b62b1b48df440e82e57e0Eric Dumazet if (!hdr) 1970744dd00c1b1be99a25b62b1b48df440e82e57e0Eric Dumazet return false; 1980744dd00c1b1be99a25b62b1b48df440e82e57e0Eric Dumazet /* 1990744dd00c1b1be99a25b62b1b48df440e82e57e0Eric Dumazet * Only look inside GRE if version zero and no 2000744dd00c1b1be99a25b62b1b48df440e82e57e0Eric Dumazet * routing 2010744dd00c1b1be99a25b62b1b48df440e82e57e0Eric Dumazet */ 2020744dd00c1b1be99a25b62b1b48df440e82e57e0Eric Dumazet if (!(hdr->flags & (GRE_VERSION|GRE_ROUTING))) { 2030744dd00c1b1be99a25b62b1b48df440e82e57e0Eric Dumazet proto = hdr->proto; 2040744dd00c1b1be99a25b62b1b48df440e82e57e0Eric Dumazet nhoff += 4; 2050744dd00c1b1be99a25b62b1b48df440e82e57e0Eric Dumazet if (hdr->flags & GRE_CSUM) 2060744dd00c1b1be99a25b62b1b48df440e82e57e0Eric Dumazet nhoff += 4; 2070744dd00c1b1be99a25b62b1b48df440e82e57e0Eric Dumazet if (hdr->flags & GRE_KEY) 2080744dd00c1b1be99a25b62b1b48df440e82e57e0Eric Dumazet nhoff += 4; 2090744dd00c1b1be99a25b62b1b48df440e82e57e0Eric Dumazet if (hdr->flags & GRE_SEQ) 2100744dd00c1b1be99a25b62b1b48df440e82e57e0Eric Dumazet nhoff += 4; 211e1733de2243609073534cf56afb146a62af3c3d8Michael Dalton if (proto == htons(ETH_P_TEB)) { 212e1733de2243609073534cf56afb146a62af3c3d8Michael Dalton const struct ethhdr *eth; 213e1733de2243609073534cf56afb146a62af3c3d8Michael Dalton struct ethhdr _eth; 214e1733de2243609073534cf56afb146a62af3c3d8Michael Dalton 215690e36e726d00d2528bc569809048adf61550d80David S. Miller eth = __skb_header_pointer(skb, nhoff, 216690e36e726d00d2528bc569809048adf61550d80David S. Miller sizeof(_eth), 217690e36e726d00d2528bc569809048adf61550d80David S. Miller data, hlen, &_eth); 218e1733de2243609073534cf56afb146a62af3c3d8Michael Dalton if (!eth) 219e1733de2243609073534cf56afb146a62af3c3d8Michael Dalton return false; 220e1733de2243609073534cf56afb146a62af3c3d8Michael Dalton proto = eth->h_proto; 221e1733de2243609073534cf56afb146a62af3c3d8Michael Dalton nhoff += sizeof(*eth); 222e1733de2243609073534cf56afb146a62af3c3d8Michael Dalton } 2230744dd00c1b1be99a25b62b1b48df440e82e57e0Eric Dumazet goto again; 2240744dd00c1b1be99a25b62b1b48df440e82e57e0Eric Dumazet } 2250744dd00c1b1be99a25b62b1b48df440e82e57e0Eric Dumazet break; 2260744dd00c1b1be99a25b62b1b48df440e82e57e0Eric Dumazet } 2270744dd00c1b1be99a25b62b1b48df440e82e57e0Eric Dumazet case IPPROTO_IPIP: 228fca418955148e4f4555d7ce911e9eee3e7970a7fTom Herbert proto = htons(ETH_P_IP); 229fca418955148e4f4555d7ce911e9eee3e7970a7fTom Herbert goto ip; 230b438f940d3541f478c6b37106ed095f1be7959efTom Herbert case IPPROTO_IPV6: 231b438f940d3541f478c6b37106ed095f1be7959efTom Herbert proto = htons(ETH_P_IPV6); 232b438f940d3541f478c6b37106ed095f1be7959efTom Herbert goto ipv6; 2330744dd00c1b1be99a25b62b1b48df440e82e57e0Eric Dumazet default: 2340744dd00c1b1be99a25b62b1b48df440e82e57e0Eric Dumazet break; 2350744dd00c1b1be99a25b62b1b48df440e82e57e0Eric Dumazet } 2360744dd00c1b1be99a25b62b1b48df440e82e57e0Eric Dumazet 237e0f31d8498676fda36289603a054d0d490aa2679Govindarajulu Varadarajan flow->n_proto = proto; 2380744dd00c1b1be99a25b62b1b48df440e82e57e0Eric Dumazet flow->ip_proto = ip_proto; 2398ed781668dd49b608f1e67a22e3b445fd0c2cd6fDaniel Borkmann flow->thoff = (u16) nhoff; 2408ed781668dd49b608f1e67a22e3b445fd0c2cd6fDaniel Borkmann 2415af7fb6e3e92c2797ee30d66138cf6aa6b29240dAlexander Duyck /* unless skb is set we don't need to record port info */ 2425af7fb6e3e92c2797ee30d66138cf6aa6b29240dAlexander Duyck if (skb) 2435af7fb6e3e92c2797ee30d66138cf6aa6b29240dAlexander Duyck flow->ports = __skb_flow_get_ports(skb, nhoff, ip_proto, 2445af7fb6e3e92c2797ee30d66138cf6aa6b29240dAlexander Duyck data, hlen); 2455af7fb6e3e92c2797ee30d66138cf6aa6b29240dAlexander Duyck 2460744dd00c1b1be99a25b62b1b48df440e82e57e0Eric Dumazet return true; 2470744dd00c1b1be99a25b62b1b48df440e82e57e0Eric Dumazet} 248690e36e726d00d2528bc569809048adf61550d80David S. MillerEXPORT_SYMBOL(__skb_flow_dissect); 249441d9d327f1e770f5aa76fd91735851ac6e1e236Cong Wang 250441d9d327f1e770f5aa76fd91735851ac6e1e236Cong Wangstatic u32 hashrnd __read_mostly; 25166415cf8a1b99d101317f5aa08574b1ec8832672Hannes Frederic Sowastatic __always_inline void __flow_hash_secret_init(void) 25266415cf8a1b99d101317f5aa08574b1ec8832672Hannes Frederic Sowa{ 25366415cf8a1b99d101317f5aa08574b1ec8832672Hannes Frederic Sowa net_get_random_once(&hashrnd, sizeof(hashrnd)); 25466415cf8a1b99d101317f5aa08574b1ec8832672Hannes Frederic Sowa} 25566415cf8a1b99d101317f5aa08574b1ec8832672Hannes Frederic Sowa 25666415cf8a1b99d101317f5aa08574b1ec8832672Hannes Frederic Sowastatic __always_inline u32 __flow_hash_3words(u32 a, u32 b, u32 c) 25766415cf8a1b99d101317f5aa08574b1ec8832672Hannes Frederic Sowa{ 25866415cf8a1b99d101317f5aa08574b1ec8832672Hannes Frederic Sowa __flow_hash_secret_init(); 25966415cf8a1b99d101317f5aa08574b1ec8832672Hannes Frederic Sowa return jhash_3words(a, b, c, hashrnd); 26066415cf8a1b99d101317f5aa08574b1ec8832672Hannes Frederic Sowa} 26166415cf8a1b99d101317f5aa08574b1ec8832672Hannes Frederic Sowa 2625ed20a68cd6ca4adc0aa2d240913d604a2eb3e25Tom Herbertstatic inline u32 __flow_hash_from_keys(struct flow_keys *keys) 2635ed20a68cd6ca4adc0aa2d240913d604a2eb3e25Tom Herbert{ 2645ed20a68cd6ca4adc0aa2d240913d604a2eb3e25Tom Herbert u32 hash; 2655ed20a68cd6ca4adc0aa2d240913d604a2eb3e25Tom Herbert 2665ed20a68cd6ca4adc0aa2d240913d604a2eb3e25Tom Herbert /* get a consistent hash (same value on both flow directions) */ 2675ed20a68cd6ca4adc0aa2d240913d604a2eb3e25Tom Herbert if (((__force u32)keys->dst < (__force u32)keys->src) || 2685ed20a68cd6ca4adc0aa2d240913d604a2eb3e25Tom Herbert (((__force u32)keys->dst == (__force u32)keys->src) && 2695ed20a68cd6ca4adc0aa2d240913d604a2eb3e25Tom Herbert ((__force u16)keys->port16[1] < (__force u16)keys->port16[0]))) { 2705ed20a68cd6ca4adc0aa2d240913d604a2eb3e25Tom Herbert swap(keys->dst, keys->src); 2715ed20a68cd6ca4adc0aa2d240913d604a2eb3e25Tom Herbert swap(keys->port16[0], keys->port16[1]); 2725ed20a68cd6ca4adc0aa2d240913d604a2eb3e25Tom Herbert } 2735ed20a68cd6ca4adc0aa2d240913d604a2eb3e25Tom Herbert 2745ed20a68cd6ca4adc0aa2d240913d604a2eb3e25Tom Herbert hash = __flow_hash_3words((__force u32)keys->dst, 2755ed20a68cd6ca4adc0aa2d240913d604a2eb3e25Tom Herbert (__force u32)keys->src, 2765ed20a68cd6ca4adc0aa2d240913d604a2eb3e25Tom Herbert (__force u32)keys->ports); 2775ed20a68cd6ca4adc0aa2d240913d604a2eb3e25Tom Herbert if (!hash) 2785ed20a68cd6ca4adc0aa2d240913d604a2eb3e25Tom Herbert hash = 1; 2795ed20a68cd6ca4adc0aa2d240913d604a2eb3e25Tom Herbert 2805ed20a68cd6ca4adc0aa2d240913d604a2eb3e25Tom Herbert return hash; 2815ed20a68cd6ca4adc0aa2d240913d604a2eb3e25Tom Herbert} 2825ed20a68cd6ca4adc0aa2d240913d604a2eb3e25Tom Herbert 2835ed20a68cd6ca4adc0aa2d240913d604a2eb3e25Tom Herbertu32 flow_hash_from_keys(struct flow_keys *keys) 2845ed20a68cd6ca4adc0aa2d240913d604a2eb3e25Tom Herbert{ 2855ed20a68cd6ca4adc0aa2d240913d604a2eb3e25Tom Herbert return __flow_hash_from_keys(keys); 2865ed20a68cd6ca4adc0aa2d240913d604a2eb3e25Tom Herbert} 2875ed20a68cd6ca4adc0aa2d240913d604a2eb3e25Tom HerbertEXPORT_SYMBOL(flow_hash_from_keys); 2885ed20a68cd6ca4adc0aa2d240913d604a2eb3e25Tom Herbert 289441d9d327f1e770f5aa76fd91735851ac6e1e236Cong Wang/* 2903958afa1b272eb07109fd31549e69193b4d7c364Tom Herbert * __skb_get_hash: calculate a flow hash based on src/dst addresses 29161b905da33ae25edb6b9d2a5de21e34c3a77efe3Tom Herbert * and src/dst port numbers. Sets hash in skb to non-zero hash value 29261b905da33ae25edb6b9d2a5de21e34c3a77efe3Tom Herbert * on success, zero indicates no valid hash. Also, sets l4_hash in skb 293441d9d327f1e770f5aa76fd91735851ac6e1e236Cong Wang * if hash is a canonical 4-tuple hash over transport ports. 294441d9d327f1e770f5aa76fd91735851ac6e1e236Cong Wang */ 2953958afa1b272eb07109fd31549e69193b4d7c364Tom Herbertvoid __skb_get_hash(struct sk_buff *skb) 296441d9d327f1e770f5aa76fd91735851ac6e1e236Cong Wang{ 297441d9d327f1e770f5aa76fd91735851ac6e1e236Cong Wang struct flow_keys keys; 298441d9d327f1e770f5aa76fd91735851ac6e1e236Cong Wang 299441d9d327f1e770f5aa76fd91735851ac6e1e236Cong Wang if (!skb_flow_dissect(skb, &keys)) 300441d9d327f1e770f5aa76fd91735851ac6e1e236Cong Wang return; 301441d9d327f1e770f5aa76fd91735851ac6e1e236Cong Wang 302441d9d327f1e770f5aa76fd91735851ac6e1e236Cong Wang if (keys.ports) 30361b905da33ae25edb6b9d2a5de21e34c3a77efe3Tom Herbert skb->l4_hash = 1; 304441d9d327f1e770f5aa76fd91735851ac6e1e236Cong Wang 305a3b18ddb9cc1056eea24e3edc1828cfb3fd0726fTom Herbert skb->sw_hash = 1; 306a3b18ddb9cc1056eea24e3edc1828cfb3fd0726fTom Herbert 3075ed20a68cd6ca4adc0aa2d240913d604a2eb3e25Tom Herbert skb->hash = __flow_hash_from_keys(&keys); 308441d9d327f1e770f5aa76fd91735851ac6e1e236Cong Wang} 3093958afa1b272eb07109fd31549e69193b4d7c364Tom HerbertEXPORT_SYMBOL(__skb_get_hash); 310441d9d327f1e770f5aa76fd91735851ac6e1e236Cong Wang 311441d9d327f1e770f5aa76fd91735851ac6e1e236Cong Wang/* 312441d9d327f1e770f5aa76fd91735851ac6e1e236Cong Wang * Returns a Tx hash based on the given packet descriptor a Tx queues' number 313441d9d327f1e770f5aa76fd91735851ac6e1e236Cong Wang * to be used as a distribution range. 314441d9d327f1e770f5aa76fd91735851ac6e1e236Cong Wang */ 3150e001614e849b68cff94cda8db8b550569d3dba6Tom Herbertu16 __skb_tx_hash(const struct net_device *dev, struct sk_buff *skb, 316441d9d327f1e770f5aa76fd91735851ac6e1e236Cong Wang unsigned int num_tx_queues) 317441d9d327f1e770f5aa76fd91735851ac6e1e236Cong Wang{ 318441d9d327f1e770f5aa76fd91735851ac6e1e236Cong Wang u32 hash; 319441d9d327f1e770f5aa76fd91735851ac6e1e236Cong Wang u16 qoffset = 0; 320441d9d327f1e770f5aa76fd91735851ac6e1e236Cong Wang u16 qcount = num_tx_queues; 321441d9d327f1e770f5aa76fd91735851ac6e1e236Cong Wang 322441d9d327f1e770f5aa76fd91735851ac6e1e236Cong Wang if (skb_rx_queue_recorded(skb)) { 323441d9d327f1e770f5aa76fd91735851ac6e1e236Cong Wang hash = skb_get_rx_queue(skb); 324441d9d327f1e770f5aa76fd91735851ac6e1e236Cong Wang while (unlikely(hash >= num_tx_queues)) 325441d9d327f1e770f5aa76fd91735851ac6e1e236Cong Wang hash -= num_tx_queues; 326441d9d327f1e770f5aa76fd91735851ac6e1e236Cong Wang return hash; 327441d9d327f1e770f5aa76fd91735851ac6e1e236Cong Wang } 328441d9d327f1e770f5aa76fd91735851ac6e1e236Cong Wang 329441d9d327f1e770f5aa76fd91735851ac6e1e236Cong Wang if (dev->num_tc) { 330441d9d327f1e770f5aa76fd91735851ac6e1e236Cong Wang u8 tc = netdev_get_prio_tc_map(dev, skb->priority); 331441d9d327f1e770f5aa76fd91735851ac6e1e236Cong Wang qoffset = dev->tc_to_txq[tc].offset; 332441d9d327f1e770f5aa76fd91735851ac6e1e236Cong Wang qcount = dev->tc_to_txq[tc].count; 333441d9d327f1e770f5aa76fd91735851ac6e1e236Cong Wang } 334441d9d327f1e770f5aa76fd91735851ac6e1e236Cong Wang 3358fc54f68919298ff9689d980efb495707ef43f30Daniel Borkmann return (u16) reciprocal_scale(skb_get_hash(skb), qcount) + qoffset; 336441d9d327f1e770f5aa76fd91735851ac6e1e236Cong Wang} 337441d9d327f1e770f5aa76fd91735851ac6e1e236Cong WangEXPORT_SYMBOL(__skb_tx_hash); 338441d9d327f1e770f5aa76fd91735851ac6e1e236Cong Wang 33956193d1bce2b2759cb4bdcc00cd05544894a0c90Alexander Duycku32 __skb_get_poff(const struct sk_buff *skb, void *data, 34056193d1bce2b2759cb4bdcc00cd05544894a0c90Alexander Duyck const struct flow_keys *keys, int hlen) 341f77668dc25b27270fe589031b22c432c3462b1d8Daniel Borkmann{ 34256193d1bce2b2759cb4bdcc00cd05544894a0c90Alexander Duyck u32 poff = keys->thoff; 343f77668dc25b27270fe589031b22c432c3462b1d8Daniel Borkmann 34456193d1bce2b2759cb4bdcc00cd05544894a0c90Alexander Duyck switch (keys->ip_proto) { 345f77668dc25b27270fe589031b22c432c3462b1d8Daniel Borkmann case IPPROTO_TCP: { 3465af7fb6e3e92c2797ee30d66138cf6aa6b29240dAlexander Duyck /* access doff as u8 to avoid unaligned access */ 3475af7fb6e3e92c2797ee30d66138cf6aa6b29240dAlexander Duyck const u8 *doff; 3485af7fb6e3e92c2797ee30d66138cf6aa6b29240dAlexander Duyck u8 _doff; 349f77668dc25b27270fe589031b22c432c3462b1d8Daniel Borkmann 3505af7fb6e3e92c2797ee30d66138cf6aa6b29240dAlexander Duyck doff = __skb_header_pointer(skb, poff + 12, sizeof(_doff), 3515af7fb6e3e92c2797ee30d66138cf6aa6b29240dAlexander Duyck data, hlen, &_doff); 3525af7fb6e3e92c2797ee30d66138cf6aa6b29240dAlexander Duyck if (!doff) 353f77668dc25b27270fe589031b22c432c3462b1d8Daniel Borkmann return poff; 354f77668dc25b27270fe589031b22c432c3462b1d8Daniel Borkmann 3555af7fb6e3e92c2797ee30d66138cf6aa6b29240dAlexander Duyck poff += max_t(u32, sizeof(struct tcphdr), (*doff & 0xF0) >> 2); 356f77668dc25b27270fe589031b22c432c3462b1d8Daniel Borkmann break; 357f77668dc25b27270fe589031b22c432c3462b1d8Daniel Borkmann } 358f77668dc25b27270fe589031b22c432c3462b1d8Daniel Borkmann case IPPROTO_UDP: 359f77668dc25b27270fe589031b22c432c3462b1d8Daniel Borkmann case IPPROTO_UDPLITE: 360f77668dc25b27270fe589031b22c432c3462b1d8Daniel Borkmann poff += sizeof(struct udphdr); 361f77668dc25b27270fe589031b22c432c3462b1d8Daniel Borkmann break; 362f77668dc25b27270fe589031b22c432c3462b1d8Daniel Borkmann /* For the rest, we do not really care about header 363f77668dc25b27270fe589031b22c432c3462b1d8Daniel Borkmann * extensions at this point for now. 364f77668dc25b27270fe589031b22c432c3462b1d8Daniel Borkmann */ 365f77668dc25b27270fe589031b22c432c3462b1d8Daniel Borkmann case IPPROTO_ICMP: 366f77668dc25b27270fe589031b22c432c3462b1d8Daniel Borkmann poff += sizeof(struct icmphdr); 367f77668dc25b27270fe589031b22c432c3462b1d8Daniel Borkmann break; 368f77668dc25b27270fe589031b22c432c3462b1d8Daniel Borkmann case IPPROTO_ICMPV6: 369f77668dc25b27270fe589031b22c432c3462b1d8Daniel Borkmann poff += sizeof(struct icmp6hdr); 370f77668dc25b27270fe589031b22c432c3462b1d8Daniel Borkmann break; 371f77668dc25b27270fe589031b22c432c3462b1d8Daniel Borkmann case IPPROTO_IGMP: 372f77668dc25b27270fe589031b22c432c3462b1d8Daniel Borkmann poff += sizeof(struct igmphdr); 373f77668dc25b27270fe589031b22c432c3462b1d8Daniel Borkmann break; 374f77668dc25b27270fe589031b22c432c3462b1d8Daniel Borkmann case IPPROTO_DCCP: 375f77668dc25b27270fe589031b22c432c3462b1d8Daniel Borkmann poff += sizeof(struct dccp_hdr); 376f77668dc25b27270fe589031b22c432c3462b1d8Daniel Borkmann break; 377f77668dc25b27270fe589031b22c432c3462b1d8Daniel Borkmann case IPPROTO_SCTP: 378f77668dc25b27270fe589031b22c432c3462b1d8Daniel Borkmann poff += sizeof(struct sctphdr); 379f77668dc25b27270fe589031b22c432c3462b1d8Daniel Borkmann break; 380f77668dc25b27270fe589031b22c432c3462b1d8Daniel Borkmann } 381f77668dc25b27270fe589031b22c432c3462b1d8Daniel Borkmann 382f77668dc25b27270fe589031b22c432c3462b1d8Daniel Borkmann return poff; 383f77668dc25b27270fe589031b22c432c3462b1d8Daniel Borkmann} 384f77668dc25b27270fe589031b22c432c3462b1d8Daniel Borkmann 38556193d1bce2b2759cb4bdcc00cd05544894a0c90Alexander Duyck/* skb_get_poff() returns the offset to the payload as far as it could 38656193d1bce2b2759cb4bdcc00cd05544894a0c90Alexander Duyck * be dissected. The main user is currently BPF, so that we can dynamically 38756193d1bce2b2759cb4bdcc00cd05544894a0c90Alexander Duyck * truncate packets without needing to push actual payload to the user 38856193d1bce2b2759cb4bdcc00cd05544894a0c90Alexander Duyck * space and can analyze headers only, instead. 38956193d1bce2b2759cb4bdcc00cd05544894a0c90Alexander Duyck */ 39056193d1bce2b2759cb4bdcc00cd05544894a0c90Alexander Duycku32 skb_get_poff(const struct sk_buff *skb) 39156193d1bce2b2759cb4bdcc00cd05544894a0c90Alexander Duyck{ 39256193d1bce2b2759cb4bdcc00cd05544894a0c90Alexander Duyck struct flow_keys keys; 39356193d1bce2b2759cb4bdcc00cd05544894a0c90Alexander Duyck 39456193d1bce2b2759cb4bdcc00cd05544894a0c90Alexander Duyck if (!skb_flow_dissect(skb, &keys)) 39556193d1bce2b2759cb4bdcc00cd05544894a0c90Alexander Duyck return 0; 39656193d1bce2b2759cb4bdcc00cd05544894a0c90Alexander Duyck 39756193d1bce2b2759cb4bdcc00cd05544894a0c90Alexander Duyck return __skb_get_poff(skb, skb->data, &keys, skb_headlen(skb)); 39856193d1bce2b2759cb4bdcc00cd05544894a0c90Alexander Duyck} 39956193d1bce2b2759cb4bdcc00cd05544894a0c90Alexander Duyck 400441d9d327f1e770f5aa76fd91735851ac6e1e236Cong Wangstatic inline int get_xps_queue(struct net_device *dev, struct sk_buff *skb) 401441d9d327f1e770f5aa76fd91735851ac6e1e236Cong Wang{ 402441d9d327f1e770f5aa76fd91735851ac6e1e236Cong Wang#ifdef CONFIG_XPS 403441d9d327f1e770f5aa76fd91735851ac6e1e236Cong Wang struct xps_dev_maps *dev_maps; 404441d9d327f1e770f5aa76fd91735851ac6e1e236Cong Wang struct xps_map *map; 405441d9d327f1e770f5aa76fd91735851ac6e1e236Cong Wang int queue_index = -1; 406441d9d327f1e770f5aa76fd91735851ac6e1e236Cong Wang 407441d9d327f1e770f5aa76fd91735851ac6e1e236Cong Wang rcu_read_lock(); 408441d9d327f1e770f5aa76fd91735851ac6e1e236Cong Wang dev_maps = rcu_dereference(dev->xps_maps); 409441d9d327f1e770f5aa76fd91735851ac6e1e236Cong Wang if (dev_maps) { 410441d9d327f1e770f5aa76fd91735851ac6e1e236Cong Wang map = rcu_dereference( 411441d9d327f1e770f5aa76fd91735851ac6e1e236Cong Wang dev_maps->cpu_map[raw_smp_processor_id()]); 412441d9d327f1e770f5aa76fd91735851ac6e1e236Cong Wang if (map) { 413441d9d327f1e770f5aa76fd91735851ac6e1e236Cong Wang if (map->len == 1) 414441d9d327f1e770f5aa76fd91735851ac6e1e236Cong Wang queue_index = map->queues[0]; 4150e001614e849b68cff94cda8db8b550569d3dba6Tom Herbert else 4168fc54f68919298ff9689d980efb495707ef43f30Daniel Borkmann queue_index = map->queues[reciprocal_scale(skb_get_hash(skb), 4178fc54f68919298ff9689d980efb495707ef43f30Daniel Borkmann map->len)]; 418441d9d327f1e770f5aa76fd91735851ac6e1e236Cong Wang if (unlikely(queue_index >= dev->real_num_tx_queues)) 419441d9d327f1e770f5aa76fd91735851ac6e1e236Cong Wang queue_index = -1; 420441d9d327f1e770f5aa76fd91735851ac6e1e236Cong Wang } 421441d9d327f1e770f5aa76fd91735851ac6e1e236Cong Wang } 422441d9d327f1e770f5aa76fd91735851ac6e1e236Cong Wang rcu_read_unlock(); 423441d9d327f1e770f5aa76fd91735851ac6e1e236Cong Wang 424441d9d327f1e770f5aa76fd91735851ac6e1e236Cong Wang return queue_index; 425441d9d327f1e770f5aa76fd91735851ac6e1e236Cong Wang#else 426441d9d327f1e770f5aa76fd91735851ac6e1e236Cong Wang return -1; 427441d9d327f1e770f5aa76fd91735851ac6e1e236Cong Wang#endif 428441d9d327f1e770f5aa76fd91735851ac6e1e236Cong Wang} 429441d9d327f1e770f5aa76fd91735851ac6e1e236Cong Wang 43099932d4fc03a13bb3e94938fe25458fabc8f2fc3Daniel Borkmannstatic u16 __netdev_pick_tx(struct net_device *dev, struct sk_buff *skb) 431441d9d327f1e770f5aa76fd91735851ac6e1e236Cong Wang{ 432441d9d327f1e770f5aa76fd91735851ac6e1e236Cong Wang struct sock *sk = skb->sk; 433441d9d327f1e770f5aa76fd91735851ac6e1e236Cong Wang int queue_index = sk_tx_queue_get(sk); 434441d9d327f1e770f5aa76fd91735851ac6e1e236Cong Wang 435441d9d327f1e770f5aa76fd91735851ac6e1e236Cong Wang if (queue_index < 0 || skb->ooo_okay || 436441d9d327f1e770f5aa76fd91735851ac6e1e236Cong Wang queue_index >= dev->real_num_tx_queues) { 437441d9d327f1e770f5aa76fd91735851ac6e1e236Cong Wang int new_index = get_xps_queue(dev, skb); 438441d9d327f1e770f5aa76fd91735851ac6e1e236Cong Wang if (new_index < 0) 439441d9d327f1e770f5aa76fd91735851ac6e1e236Cong Wang new_index = skb_tx_hash(dev, skb); 440441d9d327f1e770f5aa76fd91735851ac6e1e236Cong Wang 441702821f4ea6f68db18aa1de7d8ed62c6ba586a64Eric Dumazet if (queue_index != new_index && sk && 442702821f4ea6f68db18aa1de7d8ed62c6ba586a64Eric Dumazet rcu_access_pointer(sk->sk_dst_cache)) 44350d1784ee4683f073c0362ee360bfae7a3333d6cEric Dumazet sk_tx_queue_set(sk, new_index); 444441d9d327f1e770f5aa76fd91735851ac6e1e236Cong Wang 445441d9d327f1e770f5aa76fd91735851ac6e1e236Cong Wang queue_index = new_index; 446441d9d327f1e770f5aa76fd91735851ac6e1e236Cong Wang } 447441d9d327f1e770f5aa76fd91735851ac6e1e236Cong Wang 448441d9d327f1e770f5aa76fd91735851ac6e1e236Cong Wang return queue_index; 449441d9d327f1e770f5aa76fd91735851ac6e1e236Cong Wang} 450441d9d327f1e770f5aa76fd91735851ac6e1e236Cong Wang 451441d9d327f1e770f5aa76fd91735851ac6e1e236Cong Wangstruct netdev_queue *netdev_pick_tx(struct net_device *dev, 452f663dd9aaf9ed124f25f0f8452edf238f087ad50Jason Wang struct sk_buff *skb, 453f663dd9aaf9ed124f25f0f8452edf238f087ad50Jason Wang void *accel_priv) 454441d9d327f1e770f5aa76fd91735851ac6e1e236Cong Wang{ 455441d9d327f1e770f5aa76fd91735851ac6e1e236Cong Wang int queue_index = 0; 456441d9d327f1e770f5aa76fd91735851ac6e1e236Cong Wang 457441d9d327f1e770f5aa76fd91735851ac6e1e236Cong Wang if (dev->real_num_tx_queues != 1) { 458441d9d327f1e770f5aa76fd91735851ac6e1e236Cong Wang const struct net_device_ops *ops = dev->netdev_ops; 459441d9d327f1e770f5aa76fd91735851ac6e1e236Cong Wang if (ops->ndo_select_queue) 46099932d4fc03a13bb3e94938fe25458fabc8f2fc3Daniel Borkmann queue_index = ops->ndo_select_queue(dev, skb, accel_priv, 46199932d4fc03a13bb3e94938fe25458fabc8f2fc3Daniel Borkmann __netdev_pick_tx); 462441d9d327f1e770f5aa76fd91735851ac6e1e236Cong Wang else 463441d9d327f1e770f5aa76fd91735851ac6e1e236Cong Wang queue_index = __netdev_pick_tx(dev, skb); 464f663dd9aaf9ed124f25f0f8452edf238f087ad50Jason Wang 465f663dd9aaf9ed124f25f0f8452edf238f087ad50Jason Wang if (!accel_priv) 466b9507bdaf40e91fea2b1c0c1ee7dc627c8ee6fd6Daniel Borkmann queue_index = netdev_cap_txqueue(dev, queue_index); 467441d9d327f1e770f5aa76fd91735851ac6e1e236Cong Wang } 468441d9d327f1e770f5aa76fd91735851ac6e1e236Cong Wang 469441d9d327f1e770f5aa76fd91735851ac6e1e236Cong Wang skb_set_queue_mapping(skb, queue_index); 470441d9d327f1e770f5aa76fd91735851ac6e1e236Cong Wang return netdev_get_tx_queue(dev, queue_index); 471441d9d327f1e770f5aa76fd91735851ac6e1e236Cong Wang} 472