flow_dissector.c revision e0f31d8498676fda36289603a054d0d490aa2679
10744dd00c1b1be99a25b62b1b48df440e82e57e0Eric Dumazet#include <linux/skbuff.h> 2c452ed70771cea3af73d21a5914989137fbd28b8Jesper Dangaard Brouer#include <linux/export.h> 30744dd00c1b1be99a25b62b1b48df440e82e57e0Eric Dumazet#include <linux/ip.h> 40744dd00c1b1be99a25b62b1b48df440e82e57e0Eric Dumazet#include <linux/ipv6.h> 50744dd00c1b1be99a25b62b1b48df440e82e57e0Eric Dumazet#include <linux/if_vlan.h> 60744dd00c1b1be99a25b62b1b48df440e82e57e0Eric Dumazet#include <net/ip.h> 7ddbe503203855939946430e39bae58de11b70b69Eric Dumazet#include <net/ipv6.h> 8f77668dc25b27270fe589031b22c432c3462b1d8Daniel Borkmann#include <linux/igmp.h> 9f77668dc25b27270fe589031b22c432c3462b1d8Daniel Borkmann#include <linux/icmp.h> 10f77668dc25b27270fe589031b22c432c3462b1d8Daniel Borkmann#include <linux/sctp.h> 11f77668dc25b27270fe589031b22c432c3462b1d8Daniel Borkmann#include <linux/dccp.h> 120744dd00c1b1be99a25b62b1b48df440e82e57e0Eric Dumazet#include <linux/if_tunnel.h> 130744dd00c1b1be99a25b62b1b48df440e82e57e0Eric Dumazet#include <linux/if_pppox.h> 140744dd00c1b1be99a25b62b1b48df440e82e57e0Eric Dumazet#include <linux/ppp_defs.h> 150744dd00c1b1be99a25b62b1b48df440e82e57e0Eric Dumazet#include <net/flow_keys.h> 160744dd00c1b1be99a25b62b1b48df440e82e57e0Eric Dumazet 174d77d2b567ec66a443792d99e96ac760991d80d0Eric Dumazet/* copy saddr & daddr, possibly using 64bit load/store 184d77d2b567ec66a443792d99e96ac760991d80d0Eric Dumazet * Equivalent to : flow->src = iph->saddr; 194d77d2b567ec66a443792d99e96ac760991d80d0Eric Dumazet * flow->dst = iph->daddr; 204d77d2b567ec66a443792d99e96ac760991d80d0Eric Dumazet */ 214d77d2b567ec66a443792d99e96ac760991d80d0Eric Dumazetstatic void iph_to_flow_copy_addrs(struct flow_keys *flow, const struct iphdr *iph) 224d77d2b567ec66a443792d99e96ac760991d80d0Eric Dumazet{ 234d77d2b567ec66a443792d99e96ac760991d80d0Eric Dumazet BUILD_BUG_ON(offsetof(typeof(*flow), dst) != 244d77d2b567ec66a443792d99e96ac760991d80d0Eric Dumazet offsetof(typeof(*flow), src) + sizeof(flow->src)); 254d77d2b567ec66a443792d99e96ac760991d80d0Eric Dumazet memcpy(&flow->src, &iph->saddr, sizeof(flow->src) + sizeof(flow->dst)); 264d77d2b567ec66a443792d99e96ac760991d80d0Eric Dumazet} 270744dd00c1b1be99a25b62b1b48df440e82e57e0Eric Dumazet 28357afe9c46c951c34769e39cabdf8d1637e2eeccNikolay Aleksandrov/** 29357afe9c46c951c34769e39cabdf8d1637e2eeccNikolay Aleksandrov * skb_flow_get_ports - extract the upper layer ports and return them 30357afe9c46c951c34769e39cabdf8d1637e2eeccNikolay Aleksandrov * @skb: buffer to extract the ports from 31357afe9c46c951c34769e39cabdf8d1637e2eeccNikolay Aleksandrov * @thoff: transport header offset 32357afe9c46c951c34769e39cabdf8d1637e2eeccNikolay Aleksandrov * @ip_proto: protocol for which to get port offset 33357afe9c46c951c34769e39cabdf8d1637e2eeccNikolay Aleksandrov * 34357afe9c46c951c34769e39cabdf8d1637e2eeccNikolay Aleksandrov * The function will try to retrieve the ports at offset thoff + poff where poff 35357afe9c46c951c34769e39cabdf8d1637e2eeccNikolay Aleksandrov * is the protocol port offset returned from proto_ports_offset 36357afe9c46c951c34769e39cabdf8d1637e2eeccNikolay Aleksandrov */ 37357afe9c46c951c34769e39cabdf8d1637e2eeccNikolay Aleksandrov__be32 skb_flow_get_ports(const struct sk_buff *skb, int thoff, u8 ip_proto) 38357afe9c46c951c34769e39cabdf8d1637e2eeccNikolay Aleksandrov{ 39357afe9c46c951c34769e39cabdf8d1637e2eeccNikolay Aleksandrov int poff = proto_ports_offset(ip_proto); 40357afe9c46c951c34769e39cabdf8d1637e2eeccNikolay Aleksandrov 41357afe9c46c951c34769e39cabdf8d1637e2eeccNikolay Aleksandrov if (poff >= 0) { 42357afe9c46c951c34769e39cabdf8d1637e2eeccNikolay Aleksandrov __be32 *ports, _ports; 43357afe9c46c951c34769e39cabdf8d1637e2eeccNikolay Aleksandrov 44357afe9c46c951c34769e39cabdf8d1637e2eeccNikolay Aleksandrov ports = skb_header_pointer(skb, thoff + poff, 45357afe9c46c951c34769e39cabdf8d1637e2eeccNikolay Aleksandrov sizeof(_ports), &_ports); 46357afe9c46c951c34769e39cabdf8d1637e2eeccNikolay Aleksandrov if (ports) 47357afe9c46c951c34769e39cabdf8d1637e2eeccNikolay Aleksandrov return *ports; 48357afe9c46c951c34769e39cabdf8d1637e2eeccNikolay Aleksandrov } 49357afe9c46c951c34769e39cabdf8d1637e2eeccNikolay Aleksandrov 50357afe9c46c951c34769e39cabdf8d1637e2eeccNikolay Aleksandrov return 0; 51357afe9c46c951c34769e39cabdf8d1637e2eeccNikolay Aleksandrov} 52357afe9c46c951c34769e39cabdf8d1637e2eeccNikolay AleksandrovEXPORT_SYMBOL(skb_flow_get_ports); 53357afe9c46c951c34769e39cabdf8d1637e2eeccNikolay Aleksandrov 540744dd00c1b1be99a25b62b1b48df440e82e57e0Eric Dumazetbool skb_flow_dissect(const struct sk_buff *skb, struct flow_keys *flow) 550744dd00c1b1be99a25b62b1b48df440e82e57e0Eric Dumazet{ 56357afe9c46c951c34769e39cabdf8d1637e2eeccNikolay Aleksandrov int nhoff = skb_network_offset(skb); 570744dd00c1b1be99a25b62b1b48df440e82e57e0Eric Dumazet u8 ip_proto; 580744dd00c1b1be99a25b62b1b48df440e82e57e0Eric Dumazet __be16 proto = skb->protocol; 590744dd00c1b1be99a25b62b1b48df440e82e57e0Eric Dumazet 600744dd00c1b1be99a25b62b1b48df440e82e57e0Eric Dumazet memset(flow, 0, sizeof(*flow)); 610744dd00c1b1be99a25b62b1b48df440e82e57e0Eric Dumazet 620744dd00c1b1be99a25b62b1b48df440e82e57e0Eric Dumazetagain: 630744dd00c1b1be99a25b62b1b48df440e82e57e0Eric Dumazet switch (proto) { 642b8837aeaaa0bb6b4b3be1b3afd1cc088f68a362Joe Perches case htons(ETH_P_IP): { 650744dd00c1b1be99a25b62b1b48df440e82e57e0Eric Dumazet const struct iphdr *iph; 660744dd00c1b1be99a25b62b1b48df440e82e57e0Eric Dumazet struct iphdr _iph; 670744dd00c1b1be99a25b62b1b48df440e82e57e0Eric Dumazetip: 680744dd00c1b1be99a25b62b1b48df440e82e57e0Eric Dumazet iph = skb_header_pointer(skb, nhoff, sizeof(_iph), &_iph); 696f092343855a71e03b8d209815d8c45bf3a27fcdJason Wang if (!iph || iph->ihl < 5) 700744dd00c1b1be99a25b62b1b48df440e82e57e0Eric Dumazet return false; 713797d3e8462efdaadb64164ca540626b55fe8336Eric Dumazet nhoff += iph->ihl * 4; 720744dd00c1b1be99a25b62b1b48df440e82e57e0Eric Dumazet 733797d3e8462efdaadb64164ca540626b55fe8336Eric Dumazet ip_proto = iph->protocol; 740744dd00c1b1be99a25b62b1b48df440e82e57e0Eric Dumazet if (ip_is_fragment(iph)) 750744dd00c1b1be99a25b62b1b48df440e82e57e0Eric Dumazet ip_proto = 0; 763797d3e8462efdaadb64164ca540626b55fe8336Eric Dumazet 774d77d2b567ec66a443792d99e96ac760991d80d0Eric Dumazet iph_to_flow_copy_addrs(flow, iph); 780744dd00c1b1be99a25b62b1b48df440e82e57e0Eric Dumazet break; 790744dd00c1b1be99a25b62b1b48df440e82e57e0Eric Dumazet } 802b8837aeaaa0bb6b4b3be1b3afd1cc088f68a362Joe Perches case htons(ETH_P_IPV6): { 810744dd00c1b1be99a25b62b1b48df440e82e57e0Eric Dumazet const struct ipv6hdr *iph; 820744dd00c1b1be99a25b62b1b48df440e82e57e0Eric Dumazet struct ipv6hdr _iph; 830744dd00c1b1be99a25b62b1b48df440e82e57e0Eric Dumazetipv6: 840744dd00c1b1be99a25b62b1b48df440e82e57e0Eric Dumazet iph = skb_header_pointer(skb, nhoff, sizeof(_iph), &_iph); 850744dd00c1b1be99a25b62b1b48df440e82e57e0Eric Dumazet if (!iph) 860744dd00c1b1be99a25b62b1b48df440e82e57e0Eric Dumazet return false; 870744dd00c1b1be99a25b62b1b48df440e82e57e0Eric Dumazet 880744dd00c1b1be99a25b62b1b48df440e82e57e0Eric Dumazet ip_proto = iph->nexthdr; 89ddbe503203855939946430e39bae58de11b70b69Eric Dumazet flow->src = (__force __be32)ipv6_addr_hash(&iph->saddr); 90ddbe503203855939946430e39bae58de11b70b69Eric Dumazet flow->dst = (__force __be32)ipv6_addr_hash(&iph->daddr); 910744dd00c1b1be99a25b62b1b48df440e82e57e0Eric Dumazet nhoff += sizeof(struct ipv6hdr); 920744dd00c1b1be99a25b62b1b48df440e82e57e0Eric Dumazet break; 930744dd00c1b1be99a25b62b1b48df440e82e57e0Eric Dumazet } 942b8837aeaaa0bb6b4b3be1b3afd1cc088f68a362Joe Perches case htons(ETH_P_8021AD): 952b8837aeaaa0bb6b4b3be1b3afd1cc088f68a362Joe Perches case htons(ETH_P_8021Q): { 960744dd00c1b1be99a25b62b1b48df440e82e57e0Eric Dumazet const struct vlan_hdr *vlan; 970744dd00c1b1be99a25b62b1b48df440e82e57e0Eric Dumazet struct vlan_hdr _vlan; 980744dd00c1b1be99a25b62b1b48df440e82e57e0Eric Dumazet 990744dd00c1b1be99a25b62b1b48df440e82e57e0Eric Dumazet vlan = skb_header_pointer(skb, nhoff, sizeof(_vlan), &_vlan); 1000744dd00c1b1be99a25b62b1b48df440e82e57e0Eric Dumazet if (!vlan) 1010744dd00c1b1be99a25b62b1b48df440e82e57e0Eric Dumazet return false; 1020744dd00c1b1be99a25b62b1b48df440e82e57e0Eric Dumazet 1030744dd00c1b1be99a25b62b1b48df440e82e57e0Eric Dumazet proto = vlan->h_vlan_encapsulated_proto; 1040744dd00c1b1be99a25b62b1b48df440e82e57e0Eric Dumazet nhoff += sizeof(*vlan); 1050744dd00c1b1be99a25b62b1b48df440e82e57e0Eric Dumazet goto again; 1060744dd00c1b1be99a25b62b1b48df440e82e57e0Eric Dumazet } 1072b8837aeaaa0bb6b4b3be1b3afd1cc088f68a362Joe Perches case htons(ETH_P_PPP_SES): { 1080744dd00c1b1be99a25b62b1b48df440e82e57e0Eric Dumazet struct { 1090744dd00c1b1be99a25b62b1b48df440e82e57e0Eric Dumazet struct pppoe_hdr hdr; 1100744dd00c1b1be99a25b62b1b48df440e82e57e0Eric Dumazet __be16 proto; 1110744dd00c1b1be99a25b62b1b48df440e82e57e0Eric Dumazet } *hdr, _hdr; 1120744dd00c1b1be99a25b62b1b48df440e82e57e0Eric Dumazet hdr = skb_header_pointer(skb, nhoff, sizeof(_hdr), &_hdr); 1130744dd00c1b1be99a25b62b1b48df440e82e57e0Eric Dumazet if (!hdr) 1140744dd00c1b1be99a25b62b1b48df440e82e57e0Eric Dumazet return false; 1150744dd00c1b1be99a25b62b1b48df440e82e57e0Eric Dumazet proto = hdr->proto; 1160744dd00c1b1be99a25b62b1b48df440e82e57e0Eric Dumazet nhoff += PPPOE_SES_HLEN; 1170744dd00c1b1be99a25b62b1b48df440e82e57e0Eric Dumazet switch (proto) { 1182b8837aeaaa0bb6b4b3be1b3afd1cc088f68a362Joe Perches case htons(PPP_IP): 1190744dd00c1b1be99a25b62b1b48df440e82e57e0Eric Dumazet goto ip; 1202b8837aeaaa0bb6b4b3be1b3afd1cc088f68a362Joe Perches case htons(PPP_IPV6): 1210744dd00c1b1be99a25b62b1b48df440e82e57e0Eric Dumazet goto ipv6; 1220744dd00c1b1be99a25b62b1b48df440e82e57e0Eric Dumazet default: 1230744dd00c1b1be99a25b62b1b48df440e82e57e0Eric Dumazet return false; 1240744dd00c1b1be99a25b62b1b48df440e82e57e0Eric Dumazet } 1250744dd00c1b1be99a25b62b1b48df440e82e57e0Eric Dumazet } 1260744dd00c1b1be99a25b62b1b48df440e82e57e0Eric Dumazet default: 1270744dd00c1b1be99a25b62b1b48df440e82e57e0Eric Dumazet return false; 1280744dd00c1b1be99a25b62b1b48df440e82e57e0Eric Dumazet } 1290744dd00c1b1be99a25b62b1b48df440e82e57e0Eric Dumazet 1300744dd00c1b1be99a25b62b1b48df440e82e57e0Eric Dumazet switch (ip_proto) { 1310744dd00c1b1be99a25b62b1b48df440e82e57e0Eric Dumazet case IPPROTO_GRE: { 1320744dd00c1b1be99a25b62b1b48df440e82e57e0Eric Dumazet struct gre_hdr { 1330744dd00c1b1be99a25b62b1b48df440e82e57e0Eric Dumazet __be16 flags; 1340744dd00c1b1be99a25b62b1b48df440e82e57e0Eric Dumazet __be16 proto; 1350744dd00c1b1be99a25b62b1b48df440e82e57e0Eric Dumazet } *hdr, _hdr; 1360744dd00c1b1be99a25b62b1b48df440e82e57e0Eric Dumazet 1370744dd00c1b1be99a25b62b1b48df440e82e57e0Eric Dumazet hdr = skb_header_pointer(skb, nhoff, sizeof(_hdr), &_hdr); 1380744dd00c1b1be99a25b62b1b48df440e82e57e0Eric Dumazet if (!hdr) 1390744dd00c1b1be99a25b62b1b48df440e82e57e0Eric Dumazet return false; 1400744dd00c1b1be99a25b62b1b48df440e82e57e0Eric Dumazet /* 1410744dd00c1b1be99a25b62b1b48df440e82e57e0Eric Dumazet * Only look inside GRE if version zero and no 1420744dd00c1b1be99a25b62b1b48df440e82e57e0Eric Dumazet * routing 1430744dd00c1b1be99a25b62b1b48df440e82e57e0Eric Dumazet */ 1440744dd00c1b1be99a25b62b1b48df440e82e57e0Eric Dumazet if (!(hdr->flags & (GRE_VERSION|GRE_ROUTING))) { 1450744dd00c1b1be99a25b62b1b48df440e82e57e0Eric Dumazet proto = hdr->proto; 1460744dd00c1b1be99a25b62b1b48df440e82e57e0Eric Dumazet nhoff += 4; 1470744dd00c1b1be99a25b62b1b48df440e82e57e0Eric Dumazet if (hdr->flags & GRE_CSUM) 1480744dd00c1b1be99a25b62b1b48df440e82e57e0Eric Dumazet nhoff += 4; 1490744dd00c1b1be99a25b62b1b48df440e82e57e0Eric Dumazet if (hdr->flags & GRE_KEY) 1500744dd00c1b1be99a25b62b1b48df440e82e57e0Eric Dumazet nhoff += 4; 1510744dd00c1b1be99a25b62b1b48df440e82e57e0Eric Dumazet if (hdr->flags & GRE_SEQ) 1520744dd00c1b1be99a25b62b1b48df440e82e57e0Eric Dumazet nhoff += 4; 153e1733de2243609073534cf56afb146a62af3c3d8Michael Dalton if (proto == htons(ETH_P_TEB)) { 154e1733de2243609073534cf56afb146a62af3c3d8Michael Dalton const struct ethhdr *eth; 155e1733de2243609073534cf56afb146a62af3c3d8Michael Dalton struct ethhdr _eth; 156e1733de2243609073534cf56afb146a62af3c3d8Michael Dalton 157e1733de2243609073534cf56afb146a62af3c3d8Michael Dalton eth = skb_header_pointer(skb, nhoff, 158e1733de2243609073534cf56afb146a62af3c3d8Michael Dalton sizeof(_eth), &_eth); 159e1733de2243609073534cf56afb146a62af3c3d8Michael Dalton if (!eth) 160e1733de2243609073534cf56afb146a62af3c3d8Michael Dalton return false; 161e1733de2243609073534cf56afb146a62af3c3d8Michael Dalton proto = eth->h_proto; 162e1733de2243609073534cf56afb146a62af3c3d8Michael Dalton nhoff += sizeof(*eth); 163e1733de2243609073534cf56afb146a62af3c3d8Michael Dalton } 1640744dd00c1b1be99a25b62b1b48df440e82e57e0Eric Dumazet goto again; 1650744dd00c1b1be99a25b62b1b48df440e82e57e0Eric Dumazet } 1660744dd00c1b1be99a25b62b1b48df440e82e57e0Eric Dumazet break; 1670744dd00c1b1be99a25b62b1b48df440e82e57e0Eric Dumazet } 1680744dd00c1b1be99a25b62b1b48df440e82e57e0Eric Dumazet case IPPROTO_IPIP: 169fca418955148e4f4555d7ce911e9eee3e7970a7fTom Herbert proto = htons(ETH_P_IP); 170fca418955148e4f4555d7ce911e9eee3e7970a7fTom Herbert goto ip; 171b438f940d3541f478c6b37106ed095f1be7959efTom Herbert case IPPROTO_IPV6: 172b438f940d3541f478c6b37106ed095f1be7959efTom Herbert proto = htons(ETH_P_IPV6); 173b438f940d3541f478c6b37106ed095f1be7959efTom Herbert goto ipv6; 1740744dd00c1b1be99a25b62b1b48df440e82e57e0Eric Dumazet default: 1750744dd00c1b1be99a25b62b1b48df440e82e57e0Eric Dumazet break; 1760744dd00c1b1be99a25b62b1b48df440e82e57e0Eric Dumazet } 1770744dd00c1b1be99a25b62b1b48df440e82e57e0Eric Dumazet 178e0f31d8498676fda36289603a054d0d490aa2679Govindarajulu Varadarajan flow->n_proto = proto; 1790744dd00c1b1be99a25b62b1b48df440e82e57e0Eric Dumazet flow->ip_proto = ip_proto; 180357afe9c46c951c34769e39cabdf8d1637e2eeccNikolay Aleksandrov flow->ports = skb_flow_get_ports(skb, nhoff, ip_proto); 1818ed781668dd49b608f1e67a22e3b445fd0c2cd6fDaniel Borkmann flow->thoff = (u16) nhoff; 1828ed781668dd49b608f1e67a22e3b445fd0c2cd6fDaniel Borkmann 1830744dd00c1b1be99a25b62b1b48df440e82e57e0Eric Dumazet return true; 1840744dd00c1b1be99a25b62b1b48df440e82e57e0Eric Dumazet} 1850744dd00c1b1be99a25b62b1b48df440e82e57e0Eric DumazetEXPORT_SYMBOL(skb_flow_dissect); 186441d9d327f1e770f5aa76fd91735851ac6e1e236Cong Wang 187441d9d327f1e770f5aa76fd91735851ac6e1e236Cong Wangstatic u32 hashrnd __read_mostly; 18866415cf8a1b99d101317f5aa08574b1ec8832672Hannes Frederic Sowastatic __always_inline void __flow_hash_secret_init(void) 18966415cf8a1b99d101317f5aa08574b1ec8832672Hannes Frederic Sowa{ 19066415cf8a1b99d101317f5aa08574b1ec8832672Hannes Frederic Sowa net_get_random_once(&hashrnd, sizeof(hashrnd)); 19166415cf8a1b99d101317f5aa08574b1ec8832672Hannes Frederic Sowa} 19266415cf8a1b99d101317f5aa08574b1ec8832672Hannes Frederic Sowa 19366415cf8a1b99d101317f5aa08574b1ec8832672Hannes Frederic Sowastatic __always_inline u32 __flow_hash_3words(u32 a, u32 b, u32 c) 19466415cf8a1b99d101317f5aa08574b1ec8832672Hannes Frederic Sowa{ 19566415cf8a1b99d101317f5aa08574b1ec8832672Hannes Frederic Sowa __flow_hash_secret_init(); 19666415cf8a1b99d101317f5aa08574b1ec8832672Hannes Frederic Sowa return jhash_3words(a, b, c, hashrnd); 19766415cf8a1b99d101317f5aa08574b1ec8832672Hannes Frederic Sowa} 19866415cf8a1b99d101317f5aa08574b1ec8832672Hannes Frederic Sowa 19966415cf8a1b99d101317f5aa08574b1ec8832672Hannes Frederic Sowastatic __always_inline u32 __flow_hash_1word(u32 a) 20066415cf8a1b99d101317f5aa08574b1ec8832672Hannes Frederic Sowa{ 20166415cf8a1b99d101317f5aa08574b1ec8832672Hannes Frederic Sowa __flow_hash_secret_init(); 20266415cf8a1b99d101317f5aa08574b1ec8832672Hannes Frederic Sowa return jhash_1word(a, hashrnd); 20366415cf8a1b99d101317f5aa08574b1ec8832672Hannes Frederic Sowa} 204441d9d327f1e770f5aa76fd91735851ac6e1e236Cong Wang 205441d9d327f1e770f5aa76fd91735851ac6e1e236Cong Wang/* 2063958afa1b272eb07109fd31549e69193b4d7c364Tom Herbert * __skb_get_hash: calculate a flow hash based on src/dst addresses 20761b905da33ae25edb6b9d2a5de21e34c3a77efe3Tom Herbert * and src/dst port numbers. Sets hash in skb to non-zero hash value 20861b905da33ae25edb6b9d2a5de21e34c3a77efe3Tom Herbert * on success, zero indicates no valid hash. Also, sets l4_hash in skb 209441d9d327f1e770f5aa76fd91735851ac6e1e236Cong Wang * if hash is a canonical 4-tuple hash over transport ports. 210441d9d327f1e770f5aa76fd91735851ac6e1e236Cong Wang */ 2113958afa1b272eb07109fd31549e69193b4d7c364Tom Herbertvoid __skb_get_hash(struct sk_buff *skb) 212441d9d327f1e770f5aa76fd91735851ac6e1e236Cong Wang{ 213441d9d327f1e770f5aa76fd91735851ac6e1e236Cong Wang struct flow_keys keys; 214441d9d327f1e770f5aa76fd91735851ac6e1e236Cong Wang u32 hash; 215441d9d327f1e770f5aa76fd91735851ac6e1e236Cong Wang 216441d9d327f1e770f5aa76fd91735851ac6e1e236Cong Wang if (!skb_flow_dissect(skb, &keys)) 217441d9d327f1e770f5aa76fd91735851ac6e1e236Cong Wang return; 218441d9d327f1e770f5aa76fd91735851ac6e1e236Cong Wang 219441d9d327f1e770f5aa76fd91735851ac6e1e236Cong Wang if (keys.ports) 22061b905da33ae25edb6b9d2a5de21e34c3a77efe3Tom Herbert skb->l4_hash = 1; 221441d9d327f1e770f5aa76fd91735851ac6e1e236Cong Wang 222441d9d327f1e770f5aa76fd91735851ac6e1e236Cong Wang /* get a consistent hash (same value on both flow directions) */ 223441d9d327f1e770f5aa76fd91735851ac6e1e236Cong Wang if (((__force u32)keys.dst < (__force u32)keys.src) || 224441d9d327f1e770f5aa76fd91735851ac6e1e236Cong Wang (((__force u32)keys.dst == (__force u32)keys.src) && 225441d9d327f1e770f5aa76fd91735851ac6e1e236Cong Wang ((__force u16)keys.port16[1] < (__force u16)keys.port16[0]))) { 226441d9d327f1e770f5aa76fd91735851ac6e1e236Cong Wang swap(keys.dst, keys.src); 227441d9d327f1e770f5aa76fd91735851ac6e1e236Cong Wang swap(keys.port16[0], keys.port16[1]); 228441d9d327f1e770f5aa76fd91735851ac6e1e236Cong Wang } 229441d9d327f1e770f5aa76fd91735851ac6e1e236Cong Wang 23066415cf8a1b99d101317f5aa08574b1ec8832672Hannes Frederic Sowa hash = __flow_hash_3words((__force u32)keys.dst, 23166415cf8a1b99d101317f5aa08574b1ec8832672Hannes Frederic Sowa (__force u32)keys.src, 23266415cf8a1b99d101317f5aa08574b1ec8832672Hannes Frederic Sowa (__force u32)keys.ports); 233441d9d327f1e770f5aa76fd91735851ac6e1e236Cong Wang if (!hash) 234441d9d327f1e770f5aa76fd91735851ac6e1e236Cong Wang hash = 1; 235441d9d327f1e770f5aa76fd91735851ac6e1e236Cong Wang 23661b905da33ae25edb6b9d2a5de21e34c3a77efe3Tom Herbert skb->hash = hash; 237441d9d327f1e770f5aa76fd91735851ac6e1e236Cong Wang} 2383958afa1b272eb07109fd31549e69193b4d7c364Tom HerbertEXPORT_SYMBOL(__skb_get_hash); 239441d9d327f1e770f5aa76fd91735851ac6e1e236Cong Wang 240441d9d327f1e770f5aa76fd91735851ac6e1e236Cong Wang/* 241441d9d327f1e770f5aa76fd91735851ac6e1e236Cong Wang * Returns a Tx hash based on the given packet descriptor a Tx queues' number 242441d9d327f1e770f5aa76fd91735851ac6e1e236Cong Wang * to be used as a distribution range. 243441d9d327f1e770f5aa76fd91735851ac6e1e236Cong Wang */ 244441d9d327f1e770f5aa76fd91735851ac6e1e236Cong Wangu16 __skb_tx_hash(const struct net_device *dev, const struct sk_buff *skb, 245441d9d327f1e770f5aa76fd91735851ac6e1e236Cong Wang unsigned int num_tx_queues) 246441d9d327f1e770f5aa76fd91735851ac6e1e236Cong Wang{ 247441d9d327f1e770f5aa76fd91735851ac6e1e236Cong Wang u32 hash; 248441d9d327f1e770f5aa76fd91735851ac6e1e236Cong Wang u16 qoffset = 0; 249441d9d327f1e770f5aa76fd91735851ac6e1e236Cong Wang u16 qcount = num_tx_queues; 250441d9d327f1e770f5aa76fd91735851ac6e1e236Cong Wang 251441d9d327f1e770f5aa76fd91735851ac6e1e236Cong Wang if (skb_rx_queue_recorded(skb)) { 252441d9d327f1e770f5aa76fd91735851ac6e1e236Cong Wang hash = skb_get_rx_queue(skb); 253441d9d327f1e770f5aa76fd91735851ac6e1e236Cong Wang while (unlikely(hash >= num_tx_queues)) 254441d9d327f1e770f5aa76fd91735851ac6e1e236Cong Wang hash -= num_tx_queues; 255441d9d327f1e770f5aa76fd91735851ac6e1e236Cong Wang return hash; 256441d9d327f1e770f5aa76fd91735851ac6e1e236Cong Wang } 257441d9d327f1e770f5aa76fd91735851ac6e1e236Cong Wang 258441d9d327f1e770f5aa76fd91735851ac6e1e236Cong Wang if (dev->num_tc) { 259441d9d327f1e770f5aa76fd91735851ac6e1e236Cong Wang u8 tc = netdev_get_prio_tc_map(dev, skb->priority); 260441d9d327f1e770f5aa76fd91735851ac6e1e236Cong Wang qoffset = dev->tc_to_txq[tc].offset; 261441d9d327f1e770f5aa76fd91735851ac6e1e236Cong Wang qcount = dev->tc_to_txq[tc].count; 262441d9d327f1e770f5aa76fd91735851ac6e1e236Cong Wang } 263441d9d327f1e770f5aa76fd91735851ac6e1e236Cong Wang 264441d9d327f1e770f5aa76fd91735851ac6e1e236Cong Wang if (skb->sk && skb->sk->sk_hash) 265441d9d327f1e770f5aa76fd91735851ac6e1e236Cong Wang hash = skb->sk->sk_hash; 266441d9d327f1e770f5aa76fd91735851ac6e1e236Cong Wang else 267441d9d327f1e770f5aa76fd91735851ac6e1e236Cong Wang hash = (__force u16) skb->protocol; 26866415cf8a1b99d101317f5aa08574b1ec8832672Hannes Frederic Sowa hash = __flow_hash_1word(hash); 269441d9d327f1e770f5aa76fd91735851ac6e1e236Cong Wang 270441d9d327f1e770f5aa76fd91735851ac6e1e236Cong Wang return (u16) (((u64) hash * qcount) >> 32) + qoffset; 271441d9d327f1e770f5aa76fd91735851ac6e1e236Cong Wang} 272441d9d327f1e770f5aa76fd91735851ac6e1e236Cong WangEXPORT_SYMBOL(__skb_tx_hash); 273441d9d327f1e770f5aa76fd91735851ac6e1e236Cong Wang 274f77668dc25b27270fe589031b22c432c3462b1d8Daniel Borkmann/* __skb_get_poff() returns the offset to the payload as far as it could 275f77668dc25b27270fe589031b22c432c3462b1d8Daniel Borkmann * be dissected. The main user is currently BPF, so that we can dynamically 276f77668dc25b27270fe589031b22c432c3462b1d8Daniel Borkmann * truncate packets without needing to push actual payload to the user 277f77668dc25b27270fe589031b22c432c3462b1d8Daniel Borkmann * space and can analyze headers only, instead. 278f77668dc25b27270fe589031b22c432c3462b1d8Daniel Borkmann */ 279f77668dc25b27270fe589031b22c432c3462b1d8Daniel Borkmannu32 __skb_get_poff(const struct sk_buff *skb) 280f77668dc25b27270fe589031b22c432c3462b1d8Daniel Borkmann{ 281f77668dc25b27270fe589031b22c432c3462b1d8Daniel Borkmann struct flow_keys keys; 282f77668dc25b27270fe589031b22c432c3462b1d8Daniel Borkmann u32 poff = 0; 283f77668dc25b27270fe589031b22c432c3462b1d8Daniel Borkmann 284f77668dc25b27270fe589031b22c432c3462b1d8Daniel Borkmann if (!skb_flow_dissect(skb, &keys)) 285f77668dc25b27270fe589031b22c432c3462b1d8Daniel Borkmann return 0; 286f77668dc25b27270fe589031b22c432c3462b1d8Daniel Borkmann 287f77668dc25b27270fe589031b22c432c3462b1d8Daniel Borkmann poff += keys.thoff; 288f77668dc25b27270fe589031b22c432c3462b1d8Daniel Borkmann switch (keys.ip_proto) { 289f77668dc25b27270fe589031b22c432c3462b1d8Daniel Borkmann case IPPROTO_TCP: { 290f77668dc25b27270fe589031b22c432c3462b1d8Daniel Borkmann const struct tcphdr *tcph; 291f77668dc25b27270fe589031b22c432c3462b1d8Daniel Borkmann struct tcphdr _tcph; 292f77668dc25b27270fe589031b22c432c3462b1d8Daniel Borkmann 293f77668dc25b27270fe589031b22c432c3462b1d8Daniel Borkmann tcph = skb_header_pointer(skb, poff, sizeof(_tcph), &_tcph); 294f77668dc25b27270fe589031b22c432c3462b1d8Daniel Borkmann if (!tcph) 295f77668dc25b27270fe589031b22c432c3462b1d8Daniel Borkmann return poff; 296f77668dc25b27270fe589031b22c432c3462b1d8Daniel Borkmann 297f77668dc25b27270fe589031b22c432c3462b1d8Daniel Borkmann poff += max_t(u32, sizeof(struct tcphdr), tcph->doff * 4); 298f77668dc25b27270fe589031b22c432c3462b1d8Daniel Borkmann break; 299f77668dc25b27270fe589031b22c432c3462b1d8Daniel Borkmann } 300f77668dc25b27270fe589031b22c432c3462b1d8Daniel Borkmann case IPPROTO_UDP: 301f77668dc25b27270fe589031b22c432c3462b1d8Daniel Borkmann case IPPROTO_UDPLITE: 302f77668dc25b27270fe589031b22c432c3462b1d8Daniel Borkmann poff += sizeof(struct udphdr); 303f77668dc25b27270fe589031b22c432c3462b1d8Daniel Borkmann break; 304f77668dc25b27270fe589031b22c432c3462b1d8Daniel Borkmann /* For the rest, we do not really care about header 305f77668dc25b27270fe589031b22c432c3462b1d8Daniel Borkmann * extensions at this point for now. 306f77668dc25b27270fe589031b22c432c3462b1d8Daniel Borkmann */ 307f77668dc25b27270fe589031b22c432c3462b1d8Daniel Borkmann case IPPROTO_ICMP: 308f77668dc25b27270fe589031b22c432c3462b1d8Daniel Borkmann poff += sizeof(struct icmphdr); 309f77668dc25b27270fe589031b22c432c3462b1d8Daniel Borkmann break; 310f77668dc25b27270fe589031b22c432c3462b1d8Daniel Borkmann case IPPROTO_ICMPV6: 311f77668dc25b27270fe589031b22c432c3462b1d8Daniel Borkmann poff += sizeof(struct icmp6hdr); 312f77668dc25b27270fe589031b22c432c3462b1d8Daniel Borkmann break; 313f77668dc25b27270fe589031b22c432c3462b1d8Daniel Borkmann case IPPROTO_IGMP: 314f77668dc25b27270fe589031b22c432c3462b1d8Daniel Borkmann poff += sizeof(struct igmphdr); 315f77668dc25b27270fe589031b22c432c3462b1d8Daniel Borkmann break; 316f77668dc25b27270fe589031b22c432c3462b1d8Daniel Borkmann case IPPROTO_DCCP: 317f77668dc25b27270fe589031b22c432c3462b1d8Daniel Borkmann poff += sizeof(struct dccp_hdr); 318f77668dc25b27270fe589031b22c432c3462b1d8Daniel Borkmann break; 319f77668dc25b27270fe589031b22c432c3462b1d8Daniel Borkmann case IPPROTO_SCTP: 320f77668dc25b27270fe589031b22c432c3462b1d8Daniel Borkmann poff += sizeof(struct sctphdr); 321f77668dc25b27270fe589031b22c432c3462b1d8Daniel Borkmann break; 322f77668dc25b27270fe589031b22c432c3462b1d8Daniel Borkmann } 323f77668dc25b27270fe589031b22c432c3462b1d8Daniel Borkmann 324f77668dc25b27270fe589031b22c432c3462b1d8Daniel Borkmann return poff; 325f77668dc25b27270fe589031b22c432c3462b1d8Daniel Borkmann} 326f77668dc25b27270fe589031b22c432c3462b1d8Daniel Borkmann 327441d9d327f1e770f5aa76fd91735851ac6e1e236Cong Wangstatic inline int get_xps_queue(struct net_device *dev, struct sk_buff *skb) 328441d9d327f1e770f5aa76fd91735851ac6e1e236Cong Wang{ 329441d9d327f1e770f5aa76fd91735851ac6e1e236Cong Wang#ifdef CONFIG_XPS 330441d9d327f1e770f5aa76fd91735851ac6e1e236Cong Wang struct xps_dev_maps *dev_maps; 331441d9d327f1e770f5aa76fd91735851ac6e1e236Cong Wang struct xps_map *map; 332441d9d327f1e770f5aa76fd91735851ac6e1e236Cong Wang int queue_index = -1; 333441d9d327f1e770f5aa76fd91735851ac6e1e236Cong Wang 334441d9d327f1e770f5aa76fd91735851ac6e1e236Cong Wang rcu_read_lock(); 335441d9d327f1e770f5aa76fd91735851ac6e1e236Cong Wang dev_maps = rcu_dereference(dev->xps_maps); 336441d9d327f1e770f5aa76fd91735851ac6e1e236Cong Wang if (dev_maps) { 337441d9d327f1e770f5aa76fd91735851ac6e1e236Cong Wang map = rcu_dereference( 338441d9d327f1e770f5aa76fd91735851ac6e1e236Cong Wang dev_maps->cpu_map[raw_smp_processor_id()]); 339441d9d327f1e770f5aa76fd91735851ac6e1e236Cong Wang if (map) { 340441d9d327f1e770f5aa76fd91735851ac6e1e236Cong Wang if (map->len == 1) 341441d9d327f1e770f5aa76fd91735851ac6e1e236Cong Wang queue_index = map->queues[0]; 342441d9d327f1e770f5aa76fd91735851ac6e1e236Cong Wang else { 343441d9d327f1e770f5aa76fd91735851ac6e1e236Cong Wang u32 hash; 344441d9d327f1e770f5aa76fd91735851ac6e1e236Cong Wang if (skb->sk && skb->sk->sk_hash) 345441d9d327f1e770f5aa76fd91735851ac6e1e236Cong Wang hash = skb->sk->sk_hash; 346441d9d327f1e770f5aa76fd91735851ac6e1e236Cong Wang else 347441d9d327f1e770f5aa76fd91735851ac6e1e236Cong Wang hash = (__force u16) skb->protocol ^ 34861b905da33ae25edb6b9d2a5de21e34c3a77efe3Tom Herbert skb->hash; 34966415cf8a1b99d101317f5aa08574b1ec8832672Hannes Frederic Sowa hash = __flow_hash_1word(hash); 350441d9d327f1e770f5aa76fd91735851ac6e1e236Cong Wang queue_index = map->queues[ 351441d9d327f1e770f5aa76fd91735851ac6e1e236Cong Wang ((u64)hash * map->len) >> 32]; 352441d9d327f1e770f5aa76fd91735851ac6e1e236Cong Wang } 353441d9d327f1e770f5aa76fd91735851ac6e1e236Cong Wang if (unlikely(queue_index >= dev->real_num_tx_queues)) 354441d9d327f1e770f5aa76fd91735851ac6e1e236Cong Wang queue_index = -1; 355441d9d327f1e770f5aa76fd91735851ac6e1e236Cong Wang } 356441d9d327f1e770f5aa76fd91735851ac6e1e236Cong Wang } 357441d9d327f1e770f5aa76fd91735851ac6e1e236Cong Wang rcu_read_unlock(); 358441d9d327f1e770f5aa76fd91735851ac6e1e236Cong Wang 359441d9d327f1e770f5aa76fd91735851ac6e1e236Cong Wang return queue_index; 360441d9d327f1e770f5aa76fd91735851ac6e1e236Cong Wang#else 361441d9d327f1e770f5aa76fd91735851ac6e1e236Cong Wang return -1; 362441d9d327f1e770f5aa76fd91735851ac6e1e236Cong Wang#endif 363441d9d327f1e770f5aa76fd91735851ac6e1e236Cong Wang} 364441d9d327f1e770f5aa76fd91735851ac6e1e236Cong Wang 36599932d4fc03a13bb3e94938fe25458fabc8f2fc3Daniel Borkmannstatic u16 __netdev_pick_tx(struct net_device *dev, struct sk_buff *skb) 366441d9d327f1e770f5aa76fd91735851ac6e1e236Cong Wang{ 367441d9d327f1e770f5aa76fd91735851ac6e1e236Cong Wang struct sock *sk = skb->sk; 368441d9d327f1e770f5aa76fd91735851ac6e1e236Cong Wang int queue_index = sk_tx_queue_get(sk); 369441d9d327f1e770f5aa76fd91735851ac6e1e236Cong Wang 370441d9d327f1e770f5aa76fd91735851ac6e1e236Cong Wang if (queue_index < 0 || skb->ooo_okay || 371441d9d327f1e770f5aa76fd91735851ac6e1e236Cong Wang queue_index >= dev->real_num_tx_queues) { 372441d9d327f1e770f5aa76fd91735851ac6e1e236Cong Wang int new_index = get_xps_queue(dev, skb); 373441d9d327f1e770f5aa76fd91735851ac6e1e236Cong Wang if (new_index < 0) 374441d9d327f1e770f5aa76fd91735851ac6e1e236Cong Wang new_index = skb_tx_hash(dev, skb); 375441d9d327f1e770f5aa76fd91735851ac6e1e236Cong Wang 376702821f4ea6f68db18aa1de7d8ed62c6ba586a64Eric Dumazet if (queue_index != new_index && sk && 377702821f4ea6f68db18aa1de7d8ed62c6ba586a64Eric Dumazet rcu_access_pointer(sk->sk_dst_cache)) 37850d1784ee4683f073c0362ee360bfae7a3333d6cEric Dumazet sk_tx_queue_set(sk, new_index); 379441d9d327f1e770f5aa76fd91735851ac6e1e236Cong Wang 380441d9d327f1e770f5aa76fd91735851ac6e1e236Cong Wang queue_index = new_index; 381441d9d327f1e770f5aa76fd91735851ac6e1e236Cong Wang } 382441d9d327f1e770f5aa76fd91735851ac6e1e236Cong Wang 383441d9d327f1e770f5aa76fd91735851ac6e1e236Cong Wang return queue_index; 384441d9d327f1e770f5aa76fd91735851ac6e1e236Cong Wang} 385441d9d327f1e770f5aa76fd91735851ac6e1e236Cong Wang 386441d9d327f1e770f5aa76fd91735851ac6e1e236Cong Wangstruct netdev_queue *netdev_pick_tx(struct net_device *dev, 387f663dd9aaf9ed124f25f0f8452edf238f087ad50Jason Wang struct sk_buff *skb, 388f663dd9aaf9ed124f25f0f8452edf238f087ad50Jason Wang void *accel_priv) 389441d9d327f1e770f5aa76fd91735851ac6e1e236Cong Wang{ 390441d9d327f1e770f5aa76fd91735851ac6e1e236Cong Wang int queue_index = 0; 391441d9d327f1e770f5aa76fd91735851ac6e1e236Cong Wang 392441d9d327f1e770f5aa76fd91735851ac6e1e236Cong Wang if (dev->real_num_tx_queues != 1) { 393441d9d327f1e770f5aa76fd91735851ac6e1e236Cong Wang const struct net_device_ops *ops = dev->netdev_ops; 394441d9d327f1e770f5aa76fd91735851ac6e1e236Cong Wang if (ops->ndo_select_queue) 39599932d4fc03a13bb3e94938fe25458fabc8f2fc3Daniel Borkmann queue_index = ops->ndo_select_queue(dev, skb, accel_priv, 39699932d4fc03a13bb3e94938fe25458fabc8f2fc3Daniel Borkmann __netdev_pick_tx); 397441d9d327f1e770f5aa76fd91735851ac6e1e236Cong Wang else 398441d9d327f1e770f5aa76fd91735851ac6e1e236Cong Wang queue_index = __netdev_pick_tx(dev, skb); 399f663dd9aaf9ed124f25f0f8452edf238f087ad50Jason Wang 400f663dd9aaf9ed124f25f0f8452edf238f087ad50Jason Wang if (!accel_priv) 401b9507bdaf40e91fea2b1c0c1ee7dc627c8ee6fd6Daniel Borkmann queue_index = netdev_cap_txqueue(dev, queue_index); 402441d9d327f1e770f5aa76fd91735851ac6e1e236Cong Wang } 403441d9d327f1e770f5aa76fd91735851ac6e1e236Cong Wang 404441d9d327f1e770f5aa76fd91735851ac6e1e236Cong Wang skb_set_queue_mapping(skb, queue_index); 405441d9d327f1e770f5aa76fd91735851ac6e1e236Cong Wang return netdev_get_tx_queue(dev, queue_index); 406441d9d327f1e770f5aa76fd91735851ac6e1e236Cong Wang} 407