151c5d0c4b169bf762f09e0d5b283a7f0b2a45739David S. Miller#include <linux/rcupdate.h> 251c5d0c4b169bf762f09e0d5b283a7f0b2a45739David S. Miller#include <linux/spinlock.h> 351c5d0c4b169bf762f09e0d5b283a7f0b2a45739David S. Miller#include <linux/jiffies.h> 4ab92bb2f679d66c7e12a6b1c0cdd76fe308f6546David S. Miller#include <linux/module.h> 54aabd8ef8c43677cfee3e1e36c5a79edddb41942David S. Miller#include <linux/cache.h> 651c5d0c4b169bf762f09e0d5b283a7f0b2a45739David S. Miller#include <linux/slab.h> 751c5d0c4b169bf762f09e0d5b283a7f0b2a45739David S. Miller#include <linux/init.h> 84aabd8ef8c43677cfee3e1e36c5a79edddb41942David S. Miller#include <linux/tcp.h> 95815d5e7aae3cc9c5e85af83094d4d6498c3f4fcEric Dumazet#include <linux/hash.h> 10d23ff701643a4a725e2c7a8ba2d567d39daa29eaJulian Anastasov#include <linux/tcp_metrics.h> 11976a702ac9eeacea09e588456ab165dc06f9ee83Eric Dumazet#include <linux/vmalloc.h> 124aabd8ef8c43677cfee3e1e36c5a79edddb41942David S. Miller 134aabd8ef8c43677cfee3e1e36c5a79edddb41942David S. Miller#include <net/inet_connection_sock.h> 1451c5d0c4b169bf762f09e0d5b283a7f0b2a45739David S. Miller#include <net/net_namespace.h> 15ab92bb2f679d66c7e12a6b1c0cdd76fe308f6546David S. Miller#include <net/request_sock.h> 1651c5d0c4b169bf762f09e0d5b283a7f0b2a45739David S. Miller#include <net/inetpeer.h> 174aabd8ef8c43677cfee3e1e36c5a79edddb41942David S. Miller#include <net/sock.h> 1851c5d0c4b169bf762f09e0d5b283a7f0b2a45739David S. Miller#include <net/ipv6.h> 194aabd8ef8c43677cfee3e1e36c5a79edddb41942David S. Miller#include <net/dst.h> 204aabd8ef8c43677cfee3e1e36c5a79edddb41942David S. Miller#include <net/tcp.h> 21d23ff701643a4a725e2c7a8ba2d567d39daa29eaJulian Anastasov#include <net/genetlink.h> 224aabd8ef8c43677cfee3e1e36c5a79edddb41942David S. Miller 234aabd8ef8c43677cfee3e1e36c5a79edddb41942David S. Millerint sysctl_tcp_nometrics_save __read_mostly; 244aabd8ef8c43677cfee3e1e36c5a79edddb41942David S. Miller 2541804420586ab41049a14ab7ef04eaa2280b8647David S. Millerstatic struct tcp_metrics_block *__tcp_get_metrics(const struct inetpeer_addr *saddr, 2641804420586ab41049a14ab7ef04eaa2280b8647David S. Miller const struct inetpeer_addr *daddr, 2777f99ad16a07aa062c2d30fae57b1fee456f6ef6Christoph Paasch struct net *net, unsigned int hash); 2877f99ad16a07aa062c2d30fae57b1fee456f6ef6Christoph Paasch 291fe4c481ba637660793217769695c146a037bd54Yuchung Chengstruct tcp_fastopen_metrics { 301fe4c481ba637660793217769695c146a037bd54Yuchung Cheng u16 mss; 31aab4874355679c70f93993cf3b3fd74643b9ac33Yuchung Cheng u16 syn_loss:10; /* Recurring Fast Open SYN losses */ 32aab4874355679c70f93993cf3b3fd74643b9ac33Yuchung Cheng unsigned long last_syn_loss; /* Last Fast Open SYN loss */ 331fe4c481ba637660793217769695c146a037bd54Yuchung Cheng struct tcp_fastopen_cookie cookie; 341fe4c481ba637660793217769695c146a037bd54Yuchung Cheng}; 351fe4c481ba637660793217769695c146a037bd54Yuchung Cheng 36740b0f1841f6e39085b711d41db9ffb07198682bEric Dumazet/* TCP_METRIC_MAX includes 2 extra fields for userspace compatibility 37740b0f1841f6e39085b711d41db9ffb07198682bEric Dumazet * Kernel only stores RTT and RTTVAR in usec resolution 38740b0f1841f6e39085b711d41db9ffb07198682bEric Dumazet */ 39740b0f1841f6e39085b711d41db9ffb07198682bEric Dumazet#define TCP_METRIC_MAX_KERNEL (TCP_METRIC_MAX - 2) 40740b0f1841f6e39085b711d41db9ffb07198682bEric Dumazet 4151c5d0c4b169bf762f09e0d5b283a7f0b2a45739David S. Millerstruct tcp_metrics_block { 4251c5d0c4b169bf762f09e0d5b283a7f0b2a45739David S. Miller struct tcp_metrics_block __rcu *tcpm_next; 43a544302820db12660b15de185b9e67c781a6b74eChristoph Paasch struct inetpeer_addr tcpm_saddr; 44324fd55a19827b7191cc6ab73865e30c0e6e6423Christoph Paasch struct inetpeer_addr tcpm_daddr; 4551c5d0c4b169bf762f09e0d5b283a7f0b2a45739David S. Miller unsigned long tcpm_stamp; 4681166dd6fa8eb780b2132d32fbc77eb6ac04e44eDavid S. Miller u32 tcpm_ts; 4781166dd6fa8eb780b2132d32fbc77eb6ac04e44eDavid S. Miller u32 tcpm_ts_stamp; 4851c5d0c4b169bf762f09e0d5b283a7f0b2a45739David S. Miller u32 tcpm_lock; 49740b0f1841f6e39085b711d41db9ffb07198682bEric Dumazet u32 tcpm_vals[TCP_METRIC_MAX_KERNEL + 1]; 501fe4c481ba637660793217769695c146a037bd54Yuchung Cheng struct tcp_fastopen_metrics tcpm_fastopen; 51d23ff701643a4a725e2c7a8ba2d567d39daa29eaJulian Anastasov 52d23ff701643a4a725e2c7a8ba2d567d39daa29eaJulian Anastasov struct rcu_head rcu_head; 5351c5d0c4b169bf762f09e0d5b283a7f0b2a45739David S. Miller}; 5451c5d0c4b169bf762f09e0d5b283a7f0b2a45739David S. Miller 5551c5d0c4b169bf762f09e0d5b283a7f0b2a45739David S. Millerstatic bool tcp_metric_locked(struct tcp_metrics_block *tm, 5651c5d0c4b169bf762f09e0d5b283a7f0b2a45739David S. Miller enum tcp_metric_index idx) 5751c5d0c4b169bf762f09e0d5b283a7f0b2a45739David S. Miller{ 5851c5d0c4b169bf762f09e0d5b283a7f0b2a45739David S. Miller return tm->tcpm_lock & (1 << idx); 5951c5d0c4b169bf762f09e0d5b283a7f0b2a45739David S. Miller} 6051c5d0c4b169bf762f09e0d5b283a7f0b2a45739David S. Miller 6151c5d0c4b169bf762f09e0d5b283a7f0b2a45739David S. Millerstatic u32 tcp_metric_get(struct tcp_metrics_block *tm, 6251c5d0c4b169bf762f09e0d5b283a7f0b2a45739David S. Miller enum tcp_metric_index idx) 6351c5d0c4b169bf762f09e0d5b283a7f0b2a45739David S. Miller{ 6451c5d0c4b169bf762f09e0d5b283a7f0b2a45739David S. Miller return tm->tcpm_vals[idx]; 6551c5d0c4b169bf762f09e0d5b283a7f0b2a45739David S. Miller} 6651c5d0c4b169bf762f09e0d5b283a7f0b2a45739David S. Miller 6751c5d0c4b169bf762f09e0d5b283a7f0b2a45739David S. Millerstatic void tcp_metric_set(struct tcp_metrics_block *tm, 6851c5d0c4b169bf762f09e0d5b283a7f0b2a45739David S. Miller enum tcp_metric_index idx, 6951c5d0c4b169bf762f09e0d5b283a7f0b2a45739David S. Miller u32 val) 7051c5d0c4b169bf762f09e0d5b283a7f0b2a45739David S. Miller{ 7151c5d0c4b169bf762f09e0d5b283a7f0b2a45739David S. Miller tm->tcpm_vals[idx] = val; 7251c5d0c4b169bf762f09e0d5b283a7f0b2a45739David S. Miller} 7351c5d0c4b169bf762f09e0d5b283a7f0b2a45739David S. Miller 7451c5d0c4b169bf762f09e0d5b283a7f0b2a45739David S. Millerstatic bool addr_same(const struct inetpeer_addr *a, 7551c5d0c4b169bf762f09e0d5b283a7f0b2a45739David S. Miller const struct inetpeer_addr *b) 7651c5d0c4b169bf762f09e0d5b283a7f0b2a45739David S. Miller{ 7751c5d0c4b169bf762f09e0d5b283a7f0b2a45739David S. Miller const struct in6_addr *a6, *b6; 7851c5d0c4b169bf762f09e0d5b283a7f0b2a45739David S. Miller 7951c5d0c4b169bf762f09e0d5b283a7f0b2a45739David S. Miller if (a->family != b->family) 8051c5d0c4b169bf762f09e0d5b283a7f0b2a45739David S. Miller return false; 8151c5d0c4b169bf762f09e0d5b283a7f0b2a45739David S. Miller if (a->family == AF_INET) 8251c5d0c4b169bf762f09e0d5b283a7f0b2a45739David S. Miller return a->addr.a4 == b->addr.a4; 8351c5d0c4b169bf762f09e0d5b283a7f0b2a45739David S. Miller 8451c5d0c4b169bf762f09e0d5b283a7f0b2a45739David S. Miller a6 = (const struct in6_addr *) &a->addr.a6[0]; 8551c5d0c4b169bf762f09e0d5b283a7f0b2a45739David S. Miller b6 = (const struct in6_addr *) &b->addr.a6[0]; 8651c5d0c4b169bf762f09e0d5b283a7f0b2a45739David S. Miller 8751c5d0c4b169bf762f09e0d5b283a7f0b2a45739David S. Miller return ipv6_addr_equal(a6, b6); 8851c5d0c4b169bf762f09e0d5b283a7f0b2a45739David S. Miller} 8951c5d0c4b169bf762f09e0d5b283a7f0b2a45739David S. Miller 9051c5d0c4b169bf762f09e0d5b283a7f0b2a45739David S. Millerstruct tcpm_hash_bucket { 9151c5d0c4b169bf762f09e0d5b283a7f0b2a45739David S. Miller struct tcp_metrics_block __rcu *chain; 9251c5d0c4b169bf762f09e0d5b283a7f0b2a45739David S. Miller}; 9351c5d0c4b169bf762f09e0d5b283a7f0b2a45739David S. Miller 9451c5d0c4b169bf762f09e0d5b283a7f0b2a45739David S. Millerstatic DEFINE_SPINLOCK(tcp_metrics_lock); 9551c5d0c4b169bf762f09e0d5b283a7f0b2a45739David S. Miller 96740b0f1841f6e39085b711d41db9ffb07198682bEric Dumazetstatic void tcpm_suck_dst(struct tcp_metrics_block *tm, 97740b0f1841f6e39085b711d41db9ffb07198682bEric Dumazet const struct dst_entry *dst, 98efeaa5550e4bfd335396415958fe3615530e5d5cEric Dumazet bool fastopen_clear) 9951c5d0c4b169bf762f09e0d5b283a7f0b2a45739David S. Miller{ 100740b0f1841f6e39085b711d41db9ffb07198682bEric Dumazet u32 msval; 10151c5d0c4b169bf762f09e0d5b283a7f0b2a45739David S. Miller u32 val; 10251c5d0c4b169bf762f09e0d5b283a7f0b2a45739David S. Miller 1039a0a9502cbf19d31f7387e3066f3d1a491bef616Julian Anastasov tm->tcpm_stamp = jiffies; 1049a0a9502cbf19d31f7387e3066f3d1a491bef616Julian Anastasov 10551c5d0c4b169bf762f09e0d5b283a7f0b2a45739David S. Miller val = 0; 10651c5d0c4b169bf762f09e0d5b283a7f0b2a45739David S. Miller if (dst_metric_locked(dst, RTAX_RTT)) 10751c5d0c4b169bf762f09e0d5b283a7f0b2a45739David S. Miller val |= 1 << TCP_METRIC_RTT; 10851c5d0c4b169bf762f09e0d5b283a7f0b2a45739David S. Miller if (dst_metric_locked(dst, RTAX_RTTVAR)) 10951c5d0c4b169bf762f09e0d5b283a7f0b2a45739David S. Miller val |= 1 << TCP_METRIC_RTTVAR; 11051c5d0c4b169bf762f09e0d5b283a7f0b2a45739David S. Miller if (dst_metric_locked(dst, RTAX_SSTHRESH)) 11151c5d0c4b169bf762f09e0d5b283a7f0b2a45739David S. Miller val |= 1 << TCP_METRIC_SSTHRESH; 11251c5d0c4b169bf762f09e0d5b283a7f0b2a45739David S. Miller if (dst_metric_locked(dst, RTAX_CWND)) 11351c5d0c4b169bf762f09e0d5b283a7f0b2a45739David S. Miller val |= 1 << TCP_METRIC_CWND; 11451c5d0c4b169bf762f09e0d5b283a7f0b2a45739David S. Miller if (dst_metric_locked(dst, RTAX_REORDERING)) 11551c5d0c4b169bf762f09e0d5b283a7f0b2a45739David S. Miller val |= 1 << TCP_METRIC_REORDERING; 11651c5d0c4b169bf762f09e0d5b283a7f0b2a45739David S. Miller tm->tcpm_lock = val; 11751c5d0c4b169bf762f09e0d5b283a7f0b2a45739David S. Miller 118740b0f1841f6e39085b711d41db9ffb07198682bEric Dumazet msval = dst_metric_raw(dst, RTAX_RTT); 119740b0f1841f6e39085b711d41db9ffb07198682bEric Dumazet tm->tcpm_vals[TCP_METRIC_RTT] = msval * USEC_PER_MSEC; 120740b0f1841f6e39085b711d41db9ffb07198682bEric Dumazet 121740b0f1841f6e39085b711d41db9ffb07198682bEric Dumazet msval = dst_metric_raw(dst, RTAX_RTTVAR); 122740b0f1841f6e39085b711d41db9ffb07198682bEric Dumazet tm->tcpm_vals[TCP_METRIC_RTTVAR] = msval * USEC_PER_MSEC; 12351c5d0c4b169bf762f09e0d5b283a7f0b2a45739David S. Miller tm->tcpm_vals[TCP_METRIC_SSTHRESH] = dst_metric_raw(dst, RTAX_SSTHRESH); 12451c5d0c4b169bf762f09e0d5b283a7f0b2a45739David S. Miller tm->tcpm_vals[TCP_METRIC_CWND] = dst_metric_raw(dst, RTAX_CWND); 12551c5d0c4b169bf762f09e0d5b283a7f0b2a45739David S. Miller tm->tcpm_vals[TCP_METRIC_REORDERING] = dst_metric_raw(dst, RTAX_REORDERING); 12681166dd6fa8eb780b2132d32fbc77eb6ac04e44eDavid S. Miller tm->tcpm_ts = 0; 12781166dd6fa8eb780b2132d32fbc77eb6ac04e44eDavid S. Miller tm->tcpm_ts_stamp = 0; 128efeaa5550e4bfd335396415958fe3615530e5d5cEric Dumazet if (fastopen_clear) { 129efeaa5550e4bfd335396415958fe3615530e5d5cEric Dumazet tm->tcpm_fastopen.mss = 0; 130efeaa5550e4bfd335396415958fe3615530e5d5cEric Dumazet tm->tcpm_fastopen.syn_loss = 0; 131efeaa5550e4bfd335396415958fe3615530e5d5cEric Dumazet tm->tcpm_fastopen.cookie.len = 0; 132efeaa5550e4bfd335396415958fe3615530e5d5cEric Dumazet } 13351c5d0c4b169bf762f09e0d5b283a7f0b2a45739David S. Miller} 13451c5d0c4b169bf762f09e0d5b283a7f0b2a45739David S. Miller 13577f99ad16a07aa062c2d30fae57b1fee456f6ef6Christoph Paasch#define TCP_METRICS_TIMEOUT (60 * 60 * HZ) 13677f99ad16a07aa062c2d30fae57b1fee456f6ef6Christoph Paasch 13777f99ad16a07aa062c2d30fae57b1fee456f6ef6Christoph Paaschstatic void tcpm_check_stamp(struct tcp_metrics_block *tm, struct dst_entry *dst) 13877f99ad16a07aa062c2d30fae57b1fee456f6ef6Christoph Paasch{ 13977f99ad16a07aa062c2d30fae57b1fee456f6ef6Christoph Paasch if (tm && unlikely(time_after(jiffies, tm->tcpm_stamp + TCP_METRICS_TIMEOUT))) 14077f99ad16a07aa062c2d30fae57b1fee456f6ef6Christoph Paasch tcpm_suck_dst(tm, dst, false); 14177f99ad16a07aa062c2d30fae57b1fee456f6ef6Christoph Paasch} 14277f99ad16a07aa062c2d30fae57b1fee456f6ef6Christoph Paasch 14377f99ad16a07aa062c2d30fae57b1fee456f6ef6Christoph Paasch#define TCP_METRICS_RECLAIM_DEPTH 5 14477f99ad16a07aa062c2d30fae57b1fee456f6ef6Christoph Paasch#define TCP_METRICS_RECLAIM_PTR (struct tcp_metrics_block *) 0x1UL 14577f99ad16a07aa062c2d30fae57b1fee456f6ef6Christoph Paasch 14651c5d0c4b169bf762f09e0d5b283a7f0b2a45739David S. Millerstatic struct tcp_metrics_block *tcpm_new(struct dst_entry *dst, 147a544302820db12660b15de185b9e67c781a6b74eChristoph Paasch struct inetpeer_addr *saddr, 148324fd55a19827b7191cc6ab73865e30c0e6e6423Christoph Paasch struct inetpeer_addr *daddr, 14977f99ad16a07aa062c2d30fae57b1fee456f6ef6Christoph Paasch unsigned int hash) 15051c5d0c4b169bf762f09e0d5b283a7f0b2a45739David S. Miller{ 15151c5d0c4b169bf762f09e0d5b283a7f0b2a45739David S. Miller struct tcp_metrics_block *tm; 15251c5d0c4b169bf762f09e0d5b283a7f0b2a45739David S. Miller struct net *net; 15377f99ad16a07aa062c2d30fae57b1fee456f6ef6Christoph Paasch bool reclaim = false; 15451c5d0c4b169bf762f09e0d5b283a7f0b2a45739David S. Miller 15551c5d0c4b169bf762f09e0d5b283a7f0b2a45739David S. Miller spin_lock_bh(&tcp_metrics_lock); 15651c5d0c4b169bf762f09e0d5b283a7f0b2a45739David S. Miller net = dev_net(dst->dev); 15777f99ad16a07aa062c2d30fae57b1fee456f6ef6Christoph Paasch 15877f99ad16a07aa062c2d30fae57b1fee456f6ef6Christoph Paasch /* While waiting for the spin-lock the cache might have been populated 15977f99ad16a07aa062c2d30fae57b1fee456f6ef6Christoph Paasch * with this entry and so we have to check again. 16077f99ad16a07aa062c2d30fae57b1fee456f6ef6Christoph Paasch */ 16141804420586ab41049a14ab7ef04eaa2280b8647David S. Miller tm = __tcp_get_metrics(saddr, daddr, net, hash); 16277f99ad16a07aa062c2d30fae57b1fee456f6ef6Christoph Paasch if (tm == TCP_METRICS_RECLAIM_PTR) { 16377f99ad16a07aa062c2d30fae57b1fee456f6ef6Christoph Paasch reclaim = true; 16477f99ad16a07aa062c2d30fae57b1fee456f6ef6Christoph Paasch tm = NULL; 16577f99ad16a07aa062c2d30fae57b1fee456f6ef6Christoph Paasch } 16677f99ad16a07aa062c2d30fae57b1fee456f6ef6Christoph Paasch if (tm) { 16777f99ad16a07aa062c2d30fae57b1fee456f6ef6Christoph Paasch tcpm_check_stamp(tm, dst); 16877f99ad16a07aa062c2d30fae57b1fee456f6ef6Christoph Paasch goto out_unlock; 16977f99ad16a07aa062c2d30fae57b1fee456f6ef6Christoph Paasch } 17077f99ad16a07aa062c2d30fae57b1fee456f6ef6Christoph Paasch 17151c5d0c4b169bf762f09e0d5b283a7f0b2a45739David S. Miller if (unlikely(reclaim)) { 17251c5d0c4b169bf762f09e0d5b283a7f0b2a45739David S. Miller struct tcp_metrics_block *oldest; 17351c5d0c4b169bf762f09e0d5b283a7f0b2a45739David S. Miller 17451c5d0c4b169bf762f09e0d5b283a7f0b2a45739David S. Miller oldest = rcu_dereference(net->ipv4.tcp_metrics_hash[hash].chain); 17551c5d0c4b169bf762f09e0d5b283a7f0b2a45739David S. Miller for (tm = rcu_dereference(oldest->tcpm_next); tm; 17651c5d0c4b169bf762f09e0d5b283a7f0b2a45739David S. Miller tm = rcu_dereference(tm->tcpm_next)) { 17751c5d0c4b169bf762f09e0d5b283a7f0b2a45739David S. Miller if (time_before(tm->tcpm_stamp, oldest->tcpm_stamp)) 17851c5d0c4b169bf762f09e0d5b283a7f0b2a45739David S. Miller oldest = tm; 17951c5d0c4b169bf762f09e0d5b283a7f0b2a45739David S. Miller } 18051c5d0c4b169bf762f09e0d5b283a7f0b2a45739David S. Miller tm = oldest; 18151c5d0c4b169bf762f09e0d5b283a7f0b2a45739David S. Miller } else { 18251c5d0c4b169bf762f09e0d5b283a7f0b2a45739David S. Miller tm = kmalloc(sizeof(*tm), GFP_ATOMIC); 18351c5d0c4b169bf762f09e0d5b283a7f0b2a45739David S. Miller if (!tm) 18451c5d0c4b169bf762f09e0d5b283a7f0b2a45739David S. Miller goto out_unlock; 18551c5d0c4b169bf762f09e0d5b283a7f0b2a45739David S. Miller } 186a544302820db12660b15de185b9e67c781a6b74eChristoph Paasch tm->tcpm_saddr = *saddr; 187324fd55a19827b7191cc6ab73865e30c0e6e6423Christoph Paasch tm->tcpm_daddr = *daddr; 18851c5d0c4b169bf762f09e0d5b283a7f0b2a45739David S. Miller 189efeaa5550e4bfd335396415958fe3615530e5d5cEric Dumazet tcpm_suck_dst(tm, dst, true); 19051c5d0c4b169bf762f09e0d5b283a7f0b2a45739David S. Miller 19151c5d0c4b169bf762f09e0d5b283a7f0b2a45739David S. Miller if (likely(!reclaim)) { 19251c5d0c4b169bf762f09e0d5b283a7f0b2a45739David S. Miller tm->tcpm_next = net->ipv4.tcp_metrics_hash[hash].chain; 19351c5d0c4b169bf762f09e0d5b283a7f0b2a45739David S. Miller rcu_assign_pointer(net->ipv4.tcp_metrics_hash[hash].chain, tm); 19451c5d0c4b169bf762f09e0d5b283a7f0b2a45739David S. Miller } 19551c5d0c4b169bf762f09e0d5b283a7f0b2a45739David S. Miller 19651c5d0c4b169bf762f09e0d5b283a7f0b2a45739David S. Millerout_unlock: 19751c5d0c4b169bf762f09e0d5b283a7f0b2a45739David S. Miller spin_unlock_bh(&tcp_metrics_lock); 19851c5d0c4b169bf762f09e0d5b283a7f0b2a45739David S. Miller return tm; 19951c5d0c4b169bf762f09e0d5b283a7f0b2a45739David S. Miller} 20051c5d0c4b169bf762f09e0d5b283a7f0b2a45739David S. Miller 20151c5d0c4b169bf762f09e0d5b283a7f0b2a45739David S. Millerstatic struct tcp_metrics_block *tcp_get_encode(struct tcp_metrics_block *tm, int depth) 20251c5d0c4b169bf762f09e0d5b283a7f0b2a45739David S. Miller{ 20351c5d0c4b169bf762f09e0d5b283a7f0b2a45739David S. Miller if (tm) 20451c5d0c4b169bf762f09e0d5b283a7f0b2a45739David S. Miller return tm; 20551c5d0c4b169bf762f09e0d5b283a7f0b2a45739David S. Miller if (depth > TCP_METRICS_RECLAIM_DEPTH) 20651c5d0c4b169bf762f09e0d5b283a7f0b2a45739David S. Miller return TCP_METRICS_RECLAIM_PTR; 20751c5d0c4b169bf762f09e0d5b283a7f0b2a45739David S. Miller return NULL; 20851c5d0c4b169bf762f09e0d5b283a7f0b2a45739David S. Miller} 20951c5d0c4b169bf762f09e0d5b283a7f0b2a45739David S. Miller 210a544302820db12660b15de185b9e67c781a6b74eChristoph Paaschstatic struct tcp_metrics_block *__tcp_get_metrics(const struct inetpeer_addr *saddr, 211a544302820db12660b15de185b9e67c781a6b74eChristoph Paasch const struct inetpeer_addr *daddr, 21251c5d0c4b169bf762f09e0d5b283a7f0b2a45739David S. Miller struct net *net, unsigned int hash) 21351c5d0c4b169bf762f09e0d5b283a7f0b2a45739David S. Miller{ 21451c5d0c4b169bf762f09e0d5b283a7f0b2a45739David S. Miller struct tcp_metrics_block *tm; 21551c5d0c4b169bf762f09e0d5b283a7f0b2a45739David S. Miller int depth = 0; 21651c5d0c4b169bf762f09e0d5b283a7f0b2a45739David S. Miller 21751c5d0c4b169bf762f09e0d5b283a7f0b2a45739David S. Miller for (tm = rcu_dereference(net->ipv4.tcp_metrics_hash[hash].chain); tm; 21851c5d0c4b169bf762f09e0d5b283a7f0b2a45739David S. Miller tm = rcu_dereference(tm->tcpm_next)) { 219a544302820db12660b15de185b9e67c781a6b74eChristoph Paasch if (addr_same(&tm->tcpm_saddr, saddr) && 220a544302820db12660b15de185b9e67c781a6b74eChristoph Paasch addr_same(&tm->tcpm_daddr, daddr)) 22151c5d0c4b169bf762f09e0d5b283a7f0b2a45739David S. Miller break; 22251c5d0c4b169bf762f09e0d5b283a7f0b2a45739David S. Miller depth++; 22351c5d0c4b169bf762f09e0d5b283a7f0b2a45739David S. Miller } 22451c5d0c4b169bf762f09e0d5b283a7f0b2a45739David S. Miller return tcp_get_encode(tm, depth); 22551c5d0c4b169bf762f09e0d5b283a7f0b2a45739David S. Miller} 22651c5d0c4b169bf762f09e0d5b283a7f0b2a45739David S. Miller 22751c5d0c4b169bf762f09e0d5b283a7f0b2a45739David S. Millerstatic struct tcp_metrics_block *__tcp_get_metrics_req(struct request_sock *req, 22851c5d0c4b169bf762f09e0d5b283a7f0b2a45739David S. Miller struct dst_entry *dst) 22951c5d0c4b169bf762f09e0d5b283a7f0b2a45739David S. Miller{ 23051c5d0c4b169bf762f09e0d5b283a7f0b2a45739David S. Miller struct tcp_metrics_block *tm; 231a544302820db12660b15de185b9e67c781a6b74eChristoph Paasch struct inetpeer_addr saddr, daddr; 23251c5d0c4b169bf762f09e0d5b283a7f0b2a45739David S. Miller unsigned int hash; 23351c5d0c4b169bf762f09e0d5b283a7f0b2a45739David S. Miller struct net *net; 23451c5d0c4b169bf762f09e0d5b283a7f0b2a45739David S. Miller 235a544302820db12660b15de185b9e67c781a6b74eChristoph Paasch saddr.family = req->rsk_ops->family; 236324fd55a19827b7191cc6ab73865e30c0e6e6423Christoph Paasch daddr.family = req->rsk_ops->family; 237324fd55a19827b7191cc6ab73865e30c0e6e6423Christoph Paasch switch (daddr.family) { 23851c5d0c4b169bf762f09e0d5b283a7f0b2a45739David S. Miller case AF_INET: 239a544302820db12660b15de185b9e67c781a6b74eChristoph Paasch saddr.addr.a4 = inet_rsk(req)->ir_loc_addr; 240324fd55a19827b7191cc6ab73865e30c0e6e6423Christoph Paasch daddr.addr.a4 = inet_rsk(req)->ir_rmt_addr; 241324fd55a19827b7191cc6ab73865e30c0e6e6423Christoph Paasch hash = (__force unsigned int) daddr.addr.a4; 24251c5d0c4b169bf762f09e0d5b283a7f0b2a45739David S. Miller break; 243634fb979e8f3a70f04c1f2f519d0cd1142eb5c1aEric Dumazet#if IS_ENABLED(CONFIG_IPV6) 24451c5d0c4b169bf762f09e0d5b283a7f0b2a45739David S. Miller case AF_INET6: 245a544302820db12660b15de185b9e67c781a6b74eChristoph Paasch *(struct in6_addr *)saddr.addr.a6 = inet_rsk(req)->ir_v6_loc_addr; 246324fd55a19827b7191cc6ab73865e30c0e6e6423Christoph Paasch *(struct in6_addr *)daddr.addr.a6 = inet_rsk(req)->ir_v6_rmt_addr; 247634fb979e8f3a70f04c1f2f519d0cd1142eb5c1aEric Dumazet hash = ipv6_addr_hash(&inet_rsk(req)->ir_v6_rmt_addr); 24851c5d0c4b169bf762f09e0d5b283a7f0b2a45739David S. Miller break; 249634fb979e8f3a70f04c1f2f519d0cd1142eb5c1aEric Dumazet#endif 25051c5d0c4b169bf762f09e0d5b283a7f0b2a45739David S. Miller default: 25151c5d0c4b169bf762f09e0d5b283a7f0b2a45739David S. Miller return NULL; 25251c5d0c4b169bf762f09e0d5b283a7f0b2a45739David S. Miller } 25351c5d0c4b169bf762f09e0d5b283a7f0b2a45739David S. Miller 25451c5d0c4b169bf762f09e0d5b283a7f0b2a45739David S. Miller net = dev_net(dst->dev); 2555815d5e7aae3cc9c5e85af83094d4d6498c3f4fcEric Dumazet hash = hash_32(hash, net->ipv4.tcp_metrics_hash_log); 25651c5d0c4b169bf762f09e0d5b283a7f0b2a45739David S. Miller 25751c5d0c4b169bf762f09e0d5b283a7f0b2a45739David S. Miller for (tm = rcu_dereference(net->ipv4.tcp_metrics_hash[hash].chain); tm; 25851c5d0c4b169bf762f09e0d5b283a7f0b2a45739David S. Miller tm = rcu_dereference(tm->tcpm_next)) { 259a544302820db12660b15de185b9e67c781a6b74eChristoph Paasch if (addr_same(&tm->tcpm_saddr, &saddr) && 260a544302820db12660b15de185b9e67c781a6b74eChristoph Paasch addr_same(&tm->tcpm_daddr, &daddr)) 26151c5d0c4b169bf762f09e0d5b283a7f0b2a45739David S. Miller break; 26251c5d0c4b169bf762f09e0d5b283a7f0b2a45739David S. Miller } 26351c5d0c4b169bf762f09e0d5b283a7f0b2a45739David S. Miller tcpm_check_stamp(tm, dst); 26451c5d0c4b169bf762f09e0d5b283a7f0b2a45739David S. Miller return tm; 26551c5d0c4b169bf762f09e0d5b283a7f0b2a45739David S. Miller} 26651c5d0c4b169bf762f09e0d5b283a7f0b2a45739David S. Miller 26781166dd6fa8eb780b2132d32fbc77eb6ac04e44eDavid S. Millerstatic struct tcp_metrics_block *__tcp_get_metrics_tw(struct inet_timewait_sock *tw) 26881166dd6fa8eb780b2132d32fbc77eb6ac04e44eDavid S. Miller{ 26981166dd6fa8eb780b2132d32fbc77eb6ac04e44eDavid S. Miller struct tcp_metrics_block *tm; 270a544302820db12660b15de185b9e67c781a6b74eChristoph Paasch struct inetpeer_addr saddr, daddr; 27181166dd6fa8eb780b2132d32fbc77eb6ac04e44eDavid S. Miller unsigned int hash; 27281166dd6fa8eb780b2132d32fbc77eb6ac04e44eDavid S. Miller struct net *net; 27381166dd6fa8eb780b2132d32fbc77eb6ac04e44eDavid S. Miller 2743ad88cf70af79e6f19c4f89dd85453ba4fdf425eChristoph Paasch if (tw->tw_family == AF_INET) { 2753ad88cf70af79e6f19c4f89dd85453ba4fdf425eChristoph Paasch saddr.family = AF_INET; 276a544302820db12660b15de185b9e67c781a6b74eChristoph Paasch saddr.addr.a4 = tw->tw_rcv_saddr; 2773ad88cf70af79e6f19c4f89dd85453ba4fdf425eChristoph Paasch daddr.family = AF_INET; 278324fd55a19827b7191cc6ab73865e30c0e6e6423Christoph Paasch daddr.addr.a4 = tw->tw_daddr; 279324fd55a19827b7191cc6ab73865e30c0e6e6423Christoph Paasch hash = (__force unsigned int) daddr.addr.a4; 2803ad88cf70af79e6f19c4f89dd85453ba4fdf425eChristoph Paasch } 281c2bb06db59eaf92eb5ca9c6faed590597c6ceccbEric Dumazet#if IS_ENABLED(CONFIG_IPV6) 2823ad88cf70af79e6f19c4f89dd85453ba4fdf425eChristoph Paasch else if (tw->tw_family == AF_INET6) { 2833ad88cf70af79e6f19c4f89dd85453ba4fdf425eChristoph Paasch if (ipv6_addr_v4mapped(&tw->tw_v6_daddr)) { 2843ad88cf70af79e6f19c4f89dd85453ba4fdf425eChristoph Paasch saddr.family = AF_INET; 2853ad88cf70af79e6f19c4f89dd85453ba4fdf425eChristoph Paasch saddr.addr.a4 = tw->tw_rcv_saddr; 2863ad88cf70af79e6f19c4f89dd85453ba4fdf425eChristoph Paasch daddr.family = AF_INET; 2873ad88cf70af79e6f19c4f89dd85453ba4fdf425eChristoph Paasch daddr.addr.a4 = tw->tw_daddr; 2883ad88cf70af79e6f19c4f89dd85453ba4fdf425eChristoph Paasch hash = (__force unsigned int) daddr.addr.a4; 2893ad88cf70af79e6f19c4f89dd85453ba4fdf425eChristoph Paasch } else { 2903ad88cf70af79e6f19c4f89dd85453ba4fdf425eChristoph Paasch saddr.family = AF_INET6; 2913ad88cf70af79e6f19c4f89dd85453ba4fdf425eChristoph Paasch *(struct in6_addr *)saddr.addr.a6 = tw->tw_v6_rcv_saddr; 2923ad88cf70af79e6f19c4f89dd85453ba4fdf425eChristoph Paasch daddr.family = AF_INET6; 2933ad88cf70af79e6f19c4f89dd85453ba4fdf425eChristoph Paasch *(struct in6_addr *)daddr.addr.a6 = tw->tw_v6_daddr; 2943ad88cf70af79e6f19c4f89dd85453ba4fdf425eChristoph Paasch hash = ipv6_addr_hash(&tw->tw_v6_daddr); 2953ad88cf70af79e6f19c4f89dd85453ba4fdf425eChristoph Paasch } 2963ad88cf70af79e6f19c4f89dd85453ba4fdf425eChristoph Paasch } 297c2bb06db59eaf92eb5ca9c6faed590597c6ceccbEric Dumazet#endif 2983ad88cf70af79e6f19c4f89dd85453ba4fdf425eChristoph Paasch else 29981166dd6fa8eb780b2132d32fbc77eb6ac04e44eDavid S. Miller return NULL; 30081166dd6fa8eb780b2132d32fbc77eb6ac04e44eDavid S. Miller 30181166dd6fa8eb780b2132d32fbc77eb6ac04e44eDavid S. Miller net = twsk_net(tw); 3025815d5e7aae3cc9c5e85af83094d4d6498c3f4fcEric Dumazet hash = hash_32(hash, net->ipv4.tcp_metrics_hash_log); 30381166dd6fa8eb780b2132d32fbc77eb6ac04e44eDavid S. Miller 30481166dd6fa8eb780b2132d32fbc77eb6ac04e44eDavid S. Miller for (tm = rcu_dereference(net->ipv4.tcp_metrics_hash[hash].chain); tm; 30581166dd6fa8eb780b2132d32fbc77eb6ac04e44eDavid S. Miller tm = rcu_dereference(tm->tcpm_next)) { 306a544302820db12660b15de185b9e67c781a6b74eChristoph Paasch if (addr_same(&tm->tcpm_saddr, &saddr) && 307a544302820db12660b15de185b9e67c781a6b74eChristoph Paasch addr_same(&tm->tcpm_daddr, &daddr)) 30881166dd6fa8eb780b2132d32fbc77eb6ac04e44eDavid S. Miller break; 30981166dd6fa8eb780b2132d32fbc77eb6ac04e44eDavid S. Miller } 31081166dd6fa8eb780b2132d32fbc77eb6ac04e44eDavid S. Miller return tm; 31181166dd6fa8eb780b2132d32fbc77eb6ac04e44eDavid S. Miller} 31281166dd6fa8eb780b2132d32fbc77eb6ac04e44eDavid S. Miller 31351c5d0c4b169bf762f09e0d5b283a7f0b2a45739David S. Millerstatic struct tcp_metrics_block *tcp_get_metrics(struct sock *sk, 31451c5d0c4b169bf762f09e0d5b283a7f0b2a45739David S. Miller struct dst_entry *dst, 31551c5d0c4b169bf762f09e0d5b283a7f0b2a45739David S. Miller bool create) 31651c5d0c4b169bf762f09e0d5b283a7f0b2a45739David S. Miller{ 31751c5d0c4b169bf762f09e0d5b283a7f0b2a45739David S. Miller struct tcp_metrics_block *tm; 318a544302820db12660b15de185b9e67c781a6b74eChristoph Paasch struct inetpeer_addr saddr, daddr; 31951c5d0c4b169bf762f09e0d5b283a7f0b2a45739David S. Miller unsigned int hash; 32051c5d0c4b169bf762f09e0d5b283a7f0b2a45739David S. Miller struct net *net; 32151c5d0c4b169bf762f09e0d5b283a7f0b2a45739David S. Miller 3223ad88cf70af79e6f19c4f89dd85453ba4fdf425eChristoph Paasch if (sk->sk_family == AF_INET) { 3233ad88cf70af79e6f19c4f89dd85453ba4fdf425eChristoph Paasch saddr.family = AF_INET; 324a544302820db12660b15de185b9e67c781a6b74eChristoph Paasch saddr.addr.a4 = inet_sk(sk)->inet_saddr; 3253ad88cf70af79e6f19c4f89dd85453ba4fdf425eChristoph Paasch daddr.family = AF_INET; 326324fd55a19827b7191cc6ab73865e30c0e6e6423Christoph Paasch daddr.addr.a4 = inet_sk(sk)->inet_daddr; 327324fd55a19827b7191cc6ab73865e30c0e6e6423Christoph Paasch hash = (__force unsigned int) daddr.addr.a4; 3283ad88cf70af79e6f19c4f89dd85453ba4fdf425eChristoph Paasch } 329c2bb06db59eaf92eb5ca9c6faed590597c6ceccbEric Dumazet#if IS_ENABLED(CONFIG_IPV6) 3303ad88cf70af79e6f19c4f89dd85453ba4fdf425eChristoph Paasch else if (sk->sk_family == AF_INET6) { 3313ad88cf70af79e6f19c4f89dd85453ba4fdf425eChristoph Paasch if (ipv6_addr_v4mapped(&sk->sk_v6_daddr)) { 3323ad88cf70af79e6f19c4f89dd85453ba4fdf425eChristoph Paasch saddr.family = AF_INET; 3333ad88cf70af79e6f19c4f89dd85453ba4fdf425eChristoph Paasch saddr.addr.a4 = inet_sk(sk)->inet_saddr; 3343ad88cf70af79e6f19c4f89dd85453ba4fdf425eChristoph Paasch daddr.family = AF_INET; 3353ad88cf70af79e6f19c4f89dd85453ba4fdf425eChristoph Paasch daddr.addr.a4 = inet_sk(sk)->inet_daddr; 3363ad88cf70af79e6f19c4f89dd85453ba4fdf425eChristoph Paasch hash = (__force unsigned int) daddr.addr.a4; 3373ad88cf70af79e6f19c4f89dd85453ba4fdf425eChristoph Paasch } else { 3383ad88cf70af79e6f19c4f89dd85453ba4fdf425eChristoph Paasch saddr.family = AF_INET6; 3393ad88cf70af79e6f19c4f89dd85453ba4fdf425eChristoph Paasch *(struct in6_addr *)saddr.addr.a6 = sk->sk_v6_rcv_saddr; 3403ad88cf70af79e6f19c4f89dd85453ba4fdf425eChristoph Paasch daddr.family = AF_INET6; 3413ad88cf70af79e6f19c4f89dd85453ba4fdf425eChristoph Paasch *(struct in6_addr *)daddr.addr.a6 = sk->sk_v6_daddr; 3423ad88cf70af79e6f19c4f89dd85453ba4fdf425eChristoph Paasch hash = ipv6_addr_hash(&sk->sk_v6_daddr); 3433ad88cf70af79e6f19c4f89dd85453ba4fdf425eChristoph Paasch } 3443ad88cf70af79e6f19c4f89dd85453ba4fdf425eChristoph Paasch } 345c2bb06db59eaf92eb5ca9c6faed590597c6ceccbEric Dumazet#endif 3463ad88cf70af79e6f19c4f89dd85453ba4fdf425eChristoph Paasch else 34751c5d0c4b169bf762f09e0d5b283a7f0b2a45739David S. Miller return NULL; 34851c5d0c4b169bf762f09e0d5b283a7f0b2a45739David S. Miller 34951c5d0c4b169bf762f09e0d5b283a7f0b2a45739David S. Miller net = dev_net(dst->dev); 3505815d5e7aae3cc9c5e85af83094d4d6498c3f4fcEric Dumazet hash = hash_32(hash, net->ipv4.tcp_metrics_hash_log); 35151c5d0c4b169bf762f09e0d5b283a7f0b2a45739David S. Miller 352a544302820db12660b15de185b9e67c781a6b74eChristoph Paasch tm = __tcp_get_metrics(&saddr, &daddr, net, hash); 35377f99ad16a07aa062c2d30fae57b1fee456f6ef6Christoph Paasch if (tm == TCP_METRICS_RECLAIM_PTR) 35451c5d0c4b169bf762f09e0d5b283a7f0b2a45739David S. Miller tm = NULL; 35551c5d0c4b169bf762f09e0d5b283a7f0b2a45739David S. Miller if (!tm && create) 35641804420586ab41049a14ab7ef04eaa2280b8647David S. Miller tm = tcpm_new(dst, &saddr, &daddr, hash); 35751c5d0c4b169bf762f09e0d5b283a7f0b2a45739David S. Miller else 35851c5d0c4b169bf762f09e0d5b283a7f0b2a45739David S. Miller tcpm_check_stamp(tm, dst); 35951c5d0c4b169bf762f09e0d5b283a7f0b2a45739David S. Miller 36051c5d0c4b169bf762f09e0d5b283a7f0b2a45739David S. Miller return tm; 36151c5d0c4b169bf762f09e0d5b283a7f0b2a45739David S. Miller} 36251c5d0c4b169bf762f09e0d5b283a7f0b2a45739David S. Miller 3634aabd8ef8c43677cfee3e1e36c5a79edddb41942David S. Miller/* Save metrics learned by this TCP session. This function is called 3644aabd8ef8c43677cfee3e1e36c5a79edddb41942David S. Miller * only, when TCP finishes successfully i.e. when it enters TIME-WAIT 3654aabd8ef8c43677cfee3e1e36c5a79edddb41942David S. Miller * or goes from LAST-ACK to CLOSE. 3664aabd8ef8c43677cfee3e1e36c5a79edddb41942David S. Miller */ 3674aabd8ef8c43677cfee3e1e36c5a79edddb41942David S. Millervoid tcp_update_metrics(struct sock *sk) 3684aabd8ef8c43677cfee3e1e36c5a79edddb41942David S. Miller{ 36951c5d0c4b169bf762f09e0d5b283a7f0b2a45739David S. Miller const struct inet_connection_sock *icsk = inet_csk(sk); 3704aabd8ef8c43677cfee3e1e36c5a79edddb41942David S. Miller struct dst_entry *dst = __sk_dst_get(sk); 37151c5d0c4b169bf762f09e0d5b283a7f0b2a45739David S. Miller struct tcp_sock *tp = tcp_sk(sk); 37251c5d0c4b169bf762f09e0d5b283a7f0b2a45739David S. Miller struct tcp_metrics_block *tm; 37351c5d0c4b169bf762f09e0d5b283a7f0b2a45739David S. Miller unsigned long rtt; 37451c5d0c4b169bf762f09e0d5b283a7f0b2a45739David S. Miller u32 val; 37551c5d0c4b169bf762f09e0d5b283a7f0b2a45739David S. Miller int m; 3764aabd8ef8c43677cfee3e1e36c5a79edddb41942David S. Miller 37751c5d0c4b169bf762f09e0d5b283a7f0b2a45739David S. Miller if (sysctl_tcp_nometrics_save || !dst) 3784aabd8ef8c43677cfee3e1e36c5a79edddb41942David S. Miller return; 3794aabd8ef8c43677cfee3e1e36c5a79edddb41942David S. Miller 38051c5d0c4b169bf762f09e0d5b283a7f0b2a45739David S. Miller if (dst->flags & DST_HOST) 3814aabd8ef8c43677cfee3e1e36c5a79edddb41942David S. Miller dst_confirm(dst); 3824aabd8ef8c43677cfee3e1e36c5a79edddb41942David S. Miller 38351c5d0c4b169bf762f09e0d5b283a7f0b2a45739David S. Miller rcu_read_lock(); 384740b0f1841f6e39085b711d41db9ffb07198682bEric Dumazet if (icsk->icsk_backoff || !tp->srtt_us) { 38551c5d0c4b169bf762f09e0d5b283a7f0b2a45739David S. Miller /* This session failed to estimate rtt. Why? 38651c5d0c4b169bf762f09e0d5b283a7f0b2a45739David S. Miller * Probably, no packets returned in time. Reset our 38751c5d0c4b169bf762f09e0d5b283a7f0b2a45739David S. Miller * results. 38851c5d0c4b169bf762f09e0d5b283a7f0b2a45739David S. Miller */ 38951c5d0c4b169bf762f09e0d5b283a7f0b2a45739David S. Miller tm = tcp_get_metrics(sk, dst, false); 39051c5d0c4b169bf762f09e0d5b283a7f0b2a45739David S. Miller if (tm && !tcp_metric_locked(tm, TCP_METRIC_RTT)) 39151c5d0c4b169bf762f09e0d5b283a7f0b2a45739David S. Miller tcp_metric_set(tm, TCP_METRIC_RTT, 0); 39251c5d0c4b169bf762f09e0d5b283a7f0b2a45739David S. Miller goto out_unlock; 39351c5d0c4b169bf762f09e0d5b283a7f0b2a45739David S. Miller } else 39451c5d0c4b169bf762f09e0d5b283a7f0b2a45739David S. Miller tm = tcp_get_metrics(sk, dst, true); 3954aabd8ef8c43677cfee3e1e36c5a79edddb41942David S. Miller 39651c5d0c4b169bf762f09e0d5b283a7f0b2a45739David S. Miller if (!tm) 39751c5d0c4b169bf762f09e0d5b283a7f0b2a45739David S. Miller goto out_unlock; 3984aabd8ef8c43677cfee3e1e36c5a79edddb41942David S. Miller 399740b0f1841f6e39085b711d41db9ffb07198682bEric Dumazet rtt = tcp_metric_get(tm, TCP_METRIC_RTT); 400740b0f1841f6e39085b711d41db9ffb07198682bEric Dumazet m = rtt - tp->srtt_us; 4014aabd8ef8c43677cfee3e1e36c5a79edddb41942David S. Miller 40251c5d0c4b169bf762f09e0d5b283a7f0b2a45739David S. Miller /* If newly calculated rtt larger than stored one, store new 40351c5d0c4b169bf762f09e0d5b283a7f0b2a45739David S. Miller * one. Otherwise, use EWMA. Remember, rtt overestimation is 40451c5d0c4b169bf762f09e0d5b283a7f0b2a45739David S. Miller * always better than underestimation. 40551c5d0c4b169bf762f09e0d5b283a7f0b2a45739David S. Miller */ 40651c5d0c4b169bf762f09e0d5b283a7f0b2a45739David S. Miller if (!tcp_metric_locked(tm, TCP_METRIC_RTT)) { 40751c5d0c4b169bf762f09e0d5b283a7f0b2a45739David S. Miller if (m <= 0) 408740b0f1841f6e39085b711d41db9ffb07198682bEric Dumazet rtt = tp->srtt_us; 40951c5d0c4b169bf762f09e0d5b283a7f0b2a45739David S. Miller else 41051c5d0c4b169bf762f09e0d5b283a7f0b2a45739David S. Miller rtt -= (m >> 3); 411740b0f1841f6e39085b711d41db9ffb07198682bEric Dumazet tcp_metric_set(tm, TCP_METRIC_RTT, rtt); 41251c5d0c4b169bf762f09e0d5b283a7f0b2a45739David S. Miller } 4134aabd8ef8c43677cfee3e1e36c5a79edddb41942David S. Miller 41451c5d0c4b169bf762f09e0d5b283a7f0b2a45739David S. Miller if (!tcp_metric_locked(tm, TCP_METRIC_RTTVAR)) { 41551c5d0c4b169bf762f09e0d5b283a7f0b2a45739David S. Miller unsigned long var; 4164aabd8ef8c43677cfee3e1e36c5a79edddb41942David S. Miller 41751c5d0c4b169bf762f09e0d5b283a7f0b2a45739David S. Miller if (m < 0) 41851c5d0c4b169bf762f09e0d5b283a7f0b2a45739David S. Miller m = -m; 4194aabd8ef8c43677cfee3e1e36c5a79edddb41942David S. Miller 42051c5d0c4b169bf762f09e0d5b283a7f0b2a45739David S. Miller /* Scale deviation to rttvar fixed point */ 42151c5d0c4b169bf762f09e0d5b283a7f0b2a45739David S. Miller m >>= 1; 422740b0f1841f6e39085b711d41db9ffb07198682bEric Dumazet if (m < tp->mdev_us) 423740b0f1841f6e39085b711d41db9ffb07198682bEric Dumazet m = tp->mdev_us; 4244aabd8ef8c43677cfee3e1e36c5a79edddb41942David S. Miller 425740b0f1841f6e39085b711d41db9ffb07198682bEric Dumazet var = tcp_metric_get(tm, TCP_METRIC_RTTVAR); 42651c5d0c4b169bf762f09e0d5b283a7f0b2a45739David S. Miller if (m >= var) 42751c5d0c4b169bf762f09e0d5b283a7f0b2a45739David S. Miller var = m; 42851c5d0c4b169bf762f09e0d5b283a7f0b2a45739David S. Miller else 42951c5d0c4b169bf762f09e0d5b283a7f0b2a45739David S. Miller var -= (var - m) >> 2; 4304aabd8ef8c43677cfee3e1e36c5a79edddb41942David S. Miller 431740b0f1841f6e39085b711d41db9ffb07198682bEric Dumazet tcp_metric_set(tm, TCP_METRIC_RTTVAR, var); 43251c5d0c4b169bf762f09e0d5b283a7f0b2a45739David S. Miller } 43351c5d0c4b169bf762f09e0d5b283a7f0b2a45739David S. Miller 43451c5d0c4b169bf762f09e0d5b283a7f0b2a45739David S. Miller if (tcp_in_initial_slowstart(tp)) { 43551c5d0c4b169bf762f09e0d5b283a7f0b2a45739David S. Miller /* Slow start still did not finish. */ 43651c5d0c4b169bf762f09e0d5b283a7f0b2a45739David S. Miller if (!tcp_metric_locked(tm, TCP_METRIC_SSTHRESH)) { 43751c5d0c4b169bf762f09e0d5b283a7f0b2a45739David S. Miller val = tcp_metric_get(tm, TCP_METRIC_SSTHRESH); 43851c5d0c4b169bf762f09e0d5b283a7f0b2a45739David S. Miller if (val && (tp->snd_cwnd >> 1) > val) 43951c5d0c4b169bf762f09e0d5b283a7f0b2a45739David S. Miller tcp_metric_set(tm, TCP_METRIC_SSTHRESH, 44051c5d0c4b169bf762f09e0d5b283a7f0b2a45739David S. Miller tp->snd_cwnd >> 1); 44151c5d0c4b169bf762f09e0d5b283a7f0b2a45739David S. Miller } 44251c5d0c4b169bf762f09e0d5b283a7f0b2a45739David S. Miller if (!tcp_metric_locked(tm, TCP_METRIC_CWND)) { 44351c5d0c4b169bf762f09e0d5b283a7f0b2a45739David S. Miller val = tcp_metric_get(tm, TCP_METRIC_CWND); 44451c5d0c4b169bf762f09e0d5b283a7f0b2a45739David S. Miller if (tp->snd_cwnd > val) 44551c5d0c4b169bf762f09e0d5b283a7f0b2a45739David S. Miller tcp_metric_set(tm, TCP_METRIC_CWND, 44651c5d0c4b169bf762f09e0d5b283a7f0b2a45739David S. Miller tp->snd_cwnd); 44751c5d0c4b169bf762f09e0d5b283a7f0b2a45739David S. Miller } 44851c5d0c4b169bf762f09e0d5b283a7f0b2a45739David S. Miller } else if (tp->snd_cwnd > tp->snd_ssthresh && 44951c5d0c4b169bf762f09e0d5b283a7f0b2a45739David S. Miller icsk->icsk_ca_state == TCP_CA_Open) { 45051c5d0c4b169bf762f09e0d5b283a7f0b2a45739David S. Miller /* Cong. avoidance phase, cwnd is reliable. */ 45151c5d0c4b169bf762f09e0d5b283a7f0b2a45739David S. Miller if (!tcp_metric_locked(tm, TCP_METRIC_SSTHRESH)) 45251c5d0c4b169bf762f09e0d5b283a7f0b2a45739David S. Miller tcp_metric_set(tm, TCP_METRIC_SSTHRESH, 45351c5d0c4b169bf762f09e0d5b283a7f0b2a45739David S. Miller max(tp->snd_cwnd >> 1, tp->snd_ssthresh)); 45451c5d0c4b169bf762f09e0d5b283a7f0b2a45739David S. Miller if (!tcp_metric_locked(tm, TCP_METRIC_CWND)) { 45551c5d0c4b169bf762f09e0d5b283a7f0b2a45739David S. Miller val = tcp_metric_get(tm, TCP_METRIC_CWND); 4562100844ca9d7055d5cddce2f8ed13af94c01f85bAlexander Duyck tcp_metric_set(tm, TCP_METRIC_CWND, (val + tp->snd_cwnd) >> 1); 45751c5d0c4b169bf762f09e0d5b283a7f0b2a45739David S. Miller } 45851c5d0c4b169bf762f09e0d5b283a7f0b2a45739David S. Miller } else { 45951c5d0c4b169bf762f09e0d5b283a7f0b2a45739David S. Miller /* Else slow start did not finish, cwnd is non-sense, 46051c5d0c4b169bf762f09e0d5b283a7f0b2a45739David S. Miller * ssthresh may be also invalid. 46151c5d0c4b169bf762f09e0d5b283a7f0b2a45739David S. Miller */ 46251c5d0c4b169bf762f09e0d5b283a7f0b2a45739David S. Miller if (!tcp_metric_locked(tm, TCP_METRIC_CWND)) { 46351c5d0c4b169bf762f09e0d5b283a7f0b2a45739David S. Miller val = tcp_metric_get(tm, TCP_METRIC_CWND); 46451c5d0c4b169bf762f09e0d5b283a7f0b2a45739David S. Miller tcp_metric_set(tm, TCP_METRIC_CWND, 46551c5d0c4b169bf762f09e0d5b283a7f0b2a45739David S. Miller (val + tp->snd_ssthresh) >> 1); 46651c5d0c4b169bf762f09e0d5b283a7f0b2a45739David S. Miller } 46751c5d0c4b169bf762f09e0d5b283a7f0b2a45739David S. Miller if (!tcp_metric_locked(tm, TCP_METRIC_SSTHRESH)) { 46851c5d0c4b169bf762f09e0d5b283a7f0b2a45739David S. Miller val = tcp_metric_get(tm, TCP_METRIC_SSTHRESH); 46951c5d0c4b169bf762f09e0d5b283a7f0b2a45739David S. Miller if (val && tp->snd_ssthresh > val) 47051c5d0c4b169bf762f09e0d5b283a7f0b2a45739David S. Miller tcp_metric_set(tm, TCP_METRIC_SSTHRESH, 47151c5d0c4b169bf762f09e0d5b283a7f0b2a45739David S. Miller tp->snd_ssthresh); 47251c5d0c4b169bf762f09e0d5b283a7f0b2a45739David S. Miller } 47351c5d0c4b169bf762f09e0d5b283a7f0b2a45739David S. Miller if (!tcp_metric_locked(tm, TCP_METRIC_REORDERING)) { 47451c5d0c4b169bf762f09e0d5b283a7f0b2a45739David S. Miller val = tcp_metric_get(tm, TCP_METRIC_REORDERING); 47551c5d0c4b169bf762f09e0d5b283a7f0b2a45739David S. Miller if (val < tp->reordering && 4764aabd8ef8c43677cfee3e1e36c5a79edddb41942David S. Miller tp->reordering != sysctl_tcp_reordering) 47751c5d0c4b169bf762f09e0d5b283a7f0b2a45739David S. Miller tcp_metric_set(tm, TCP_METRIC_REORDERING, 47851c5d0c4b169bf762f09e0d5b283a7f0b2a45739David S. Miller tp->reordering); 4794aabd8ef8c43677cfee3e1e36c5a79edddb41942David S. Miller } 4804aabd8ef8c43677cfee3e1e36c5a79edddb41942David S. Miller } 48151c5d0c4b169bf762f09e0d5b283a7f0b2a45739David S. Miller tm->tcpm_stamp = jiffies; 48251c5d0c4b169bf762f09e0d5b283a7f0b2a45739David S. Millerout_unlock: 48351c5d0c4b169bf762f09e0d5b283a7f0b2a45739David S. Miller rcu_read_unlock(); 4844aabd8ef8c43677cfee3e1e36c5a79edddb41942David S. Miller} 4854aabd8ef8c43677cfee3e1e36c5a79edddb41942David S. Miller 4864aabd8ef8c43677cfee3e1e36c5a79edddb41942David S. Miller/* Initialize metrics on socket. */ 4874aabd8ef8c43677cfee3e1e36c5a79edddb41942David S. Miller 4884aabd8ef8c43677cfee3e1e36c5a79edddb41942David S. Millervoid tcp_init_metrics(struct sock *sk) 4894aabd8ef8c43677cfee3e1e36c5a79edddb41942David S. Miller{ 4904aabd8ef8c43677cfee3e1e36c5a79edddb41942David S. Miller struct dst_entry *dst = __sk_dst_get(sk); 49151c5d0c4b169bf762f09e0d5b283a7f0b2a45739David S. Miller struct tcp_sock *tp = tcp_sk(sk); 49251c5d0c4b169bf762f09e0d5b283a7f0b2a45739David S. Miller struct tcp_metrics_block *tm; 4931b7fdd2ab5852717a4fc7d79847759c67065d7e9Yuchung Cheng u32 val, crtt = 0; /* cached RTT scaled by 8 */ 4944aabd8ef8c43677cfee3e1e36c5a79edddb41942David S. Miller 4954aabd8ef8c43677cfee3e1e36c5a79edddb41942David S. Miller if (dst == NULL) 4964aabd8ef8c43677cfee3e1e36c5a79edddb41942David S. Miller goto reset; 4974aabd8ef8c43677cfee3e1e36c5a79edddb41942David S. Miller 4984aabd8ef8c43677cfee3e1e36c5a79edddb41942David S. Miller dst_confirm(dst); 4994aabd8ef8c43677cfee3e1e36c5a79edddb41942David S. Miller 50051c5d0c4b169bf762f09e0d5b283a7f0b2a45739David S. Miller rcu_read_lock(); 50151c5d0c4b169bf762f09e0d5b283a7f0b2a45739David S. Miller tm = tcp_get_metrics(sk, dst, true); 50251c5d0c4b169bf762f09e0d5b283a7f0b2a45739David S. Miller if (!tm) { 50351c5d0c4b169bf762f09e0d5b283a7f0b2a45739David S. Miller rcu_read_unlock(); 50451c5d0c4b169bf762f09e0d5b283a7f0b2a45739David S. Miller goto reset; 50551c5d0c4b169bf762f09e0d5b283a7f0b2a45739David S. Miller } 50651c5d0c4b169bf762f09e0d5b283a7f0b2a45739David S. Miller 50751c5d0c4b169bf762f09e0d5b283a7f0b2a45739David S. Miller if (tcp_metric_locked(tm, TCP_METRIC_CWND)) 50851c5d0c4b169bf762f09e0d5b283a7f0b2a45739David S. Miller tp->snd_cwnd_clamp = tcp_metric_get(tm, TCP_METRIC_CWND); 50951c5d0c4b169bf762f09e0d5b283a7f0b2a45739David S. Miller 51051c5d0c4b169bf762f09e0d5b283a7f0b2a45739David S. Miller val = tcp_metric_get(tm, TCP_METRIC_SSTHRESH); 51151c5d0c4b169bf762f09e0d5b283a7f0b2a45739David S. Miller if (val) { 51251c5d0c4b169bf762f09e0d5b283a7f0b2a45739David S. Miller tp->snd_ssthresh = val; 5134aabd8ef8c43677cfee3e1e36c5a79edddb41942David S. Miller if (tp->snd_ssthresh > tp->snd_cwnd_clamp) 5144aabd8ef8c43677cfee3e1e36c5a79edddb41942David S. Miller tp->snd_ssthresh = tp->snd_cwnd_clamp; 5154aabd8ef8c43677cfee3e1e36c5a79edddb41942David S. Miller } else { 5164aabd8ef8c43677cfee3e1e36c5a79edddb41942David S. Miller /* ssthresh may have been reduced unnecessarily during. 5174aabd8ef8c43677cfee3e1e36c5a79edddb41942David S. Miller * 3WHS. Restore it back to its initial default. 5184aabd8ef8c43677cfee3e1e36c5a79edddb41942David S. Miller */ 5194aabd8ef8c43677cfee3e1e36c5a79edddb41942David S. Miller tp->snd_ssthresh = TCP_INFINITE_SSTHRESH; 5204aabd8ef8c43677cfee3e1e36c5a79edddb41942David S. Miller } 52151c5d0c4b169bf762f09e0d5b283a7f0b2a45739David S. Miller val = tcp_metric_get(tm, TCP_METRIC_REORDERING); 52251c5d0c4b169bf762f09e0d5b283a7f0b2a45739David S. Miller if (val && tp->reordering != val) { 5234aabd8ef8c43677cfee3e1e36c5a79edddb41942David S. Miller tcp_disable_fack(tp); 5244aabd8ef8c43677cfee3e1e36c5a79edddb41942David S. Miller tcp_disable_early_retrans(tp); 52551c5d0c4b169bf762f09e0d5b283a7f0b2a45739David S. Miller tp->reordering = val; 5264aabd8ef8c43677cfee3e1e36c5a79edddb41942David S. Miller } 5274aabd8ef8c43677cfee3e1e36c5a79edddb41942David S. Miller 528740b0f1841f6e39085b711d41db9ffb07198682bEric Dumazet crtt = tcp_metric_get(tm, TCP_METRIC_RTT); 52951c5d0c4b169bf762f09e0d5b283a7f0b2a45739David S. Miller rcu_read_unlock(); 5304aabd8ef8c43677cfee3e1e36c5a79edddb41942David S. Millerreset: 53152f20e655d9f6f7f937a1cdacf219d9df3ab6166Yuchung Cheng /* The initial RTT measurement from the SYN/SYN-ACK is not ideal 53252f20e655d9f6f7f937a1cdacf219d9df3ab6166Yuchung Cheng * to seed the RTO for later data packets because SYN packets are 53352f20e655d9f6f7f937a1cdacf219d9df3ab6166Yuchung Cheng * small. Use the per-dst cached values to seed the RTO but keep 53452f20e655d9f6f7f937a1cdacf219d9df3ab6166Yuchung Cheng * the RTT estimator variables intact (e.g., srtt, mdev, rttvar). 53552f20e655d9f6f7f937a1cdacf219d9df3ab6166Yuchung Cheng * Later the RTO will be updated immediately upon obtaining the first 53652f20e655d9f6f7f937a1cdacf219d9df3ab6166Yuchung Cheng * data RTT sample (tcp_rtt_estimator()). Hence the cached RTT only 53752f20e655d9f6f7f937a1cdacf219d9df3ab6166Yuchung Cheng * influences the first RTO but not later RTT estimation. 53852f20e655d9f6f7f937a1cdacf219d9df3ab6166Yuchung Cheng * 53952f20e655d9f6f7f937a1cdacf219d9df3ab6166Yuchung Cheng * But if RTT is not available from the SYN (due to retransmits or 54052f20e655d9f6f7f937a1cdacf219d9df3ab6166Yuchung Cheng * syn cookies) or the cache, force a conservative 3secs timeout. 54152f20e655d9f6f7f937a1cdacf219d9df3ab6166Yuchung Cheng * 54252f20e655d9f6f7f937a1cdacf219d9df3ab6166Yuchung Cheng * A bit of theory. RTT is time passed after "normal" sized packet 54352f20e655d9f6f7f937a1cdacf219d9df3ab6166Yuchung Cheng * is sent until it is ACKed. In normal circumstances sending small 54452f20e655d9f6f7f937a1cdacf219d9df3ab6166Yuchung Cheng * packets force peer to delay ACKs and calculation is correct too. 54552f20e655d9f6f7f937a1cdacf219d9df3ab6166Yuchung Cheng * The algorithm is adaptive and, provided we follow specs, it 54652f20e655d9f6f7f937a1cdacf219d9df3ab6166Yuchung Cheng * NEVER underestimate RTT. BUT! If peer tries to make some clever 54752f20e655d9f6f7f937a1cdacf219d9df3ab6166Yuchung Cheng * tricks sort of "quick acks" for time long enough to decrease RTT 54852f20e655d9f6f7f937a1cdacf219d9df3ab6166Yuchung Cheng * to low value, and then abruptly stops to do it and starts to delay 54952f20e655d9f6f7f937a1cdacf219d9df3ab6166Yuchung Cheng * ACKs, wait for troubles. 55052f20e655d9f6f7f937a1cdacf219d9df3ab6166Yuchung Cheng */ 551740b0f1841f6e39085b711d41db9ffb07198682bEric Dumazet if (crtt > tp->srtt_us) { 552269aa759b474570fa642452742741525cfc226a9Neal Cardwell /* Set RTO like tcp_rtt_estimator(), but from cached RTT. */ 553740b0f1841f6e39085b711d41db9ffb07198682bEric Dumazet crtt /= 8 * USEC_PER_MSEC; 554269aa759b474570fa642452742741525cfc226a9Neal Cardwell inet_csk(sk)->icsk_rto = crtt + max(2 * crtt, tcp_rto_min(sk)); 555740b0f1841f6e39085b711d41db9ffb07198682bEric Dumazet } else if (tp->srtt_us == 0) { 5564aabd8ef8c43677cfee3e1e36c5a79edddb41942David S. Miller /* RFC6298: 5.7 We've failed to get a valid RTT sample from 5574aabd8ef8c43677cfee3e1e36c5a79edddb41942David S. Miller * 3WHS. This is most likely due to retransmission, 5584aabd8ef8c43677cfee3e1e36c5a79edddb41942David S. Miller * including spurious one. Reset the RTO back to 3secs 5594aabd8ef8c43677cfee3e1e36c5a79edddb41942David S. Miller * from the more aggressive 1sec to avoid more spurious 5604aabd8ef8c43677cfee3e1e36c5a79edddb41942David S. Miller * retransmission. 5614aabd8ef8c43677cfee3e1e36c5a79edddb41942David S. Miller */ 562740b0f1841f6e39085b711d41db9ffb07198682bEric Dumazet tp->rttvar_us = jiffies_to_usecs(TCP_TIMEOUT_FALLBACK); 563740b0f1841f6e39085b711d41db9ffb07198682bEric Dumazet tp->mdev_us = tp->mdev_max_us = tp->rttvar_us; 564740b0f1841f6e39085b711d41db9ffb07198682bEric Dumazet 5654aabd8ef8c43677cfee3e1e36c5a79edddb41942David S. Miller inet_csk(sk)->icsk_rto = TCP_TIMEOUT_FALLBACK; 5664aabd8ef8c43677cfee3e1e36c5a79edddb41942David S. Miller } 5674aabd8ef8c43677cfee3e1e36c5a79edddb41942David S. Miller /* Cut cwnd down to 1 per RFC5681 if SYN or SYN-ACK has been 5684aabd8ef8c43677cfee3e1e36c5a79edddb41942David S. Miller * retransmitted. In light of RFC6298 more aggressive 1sec 5694aabd8ef8c43677cfee3e1e36c5a79edddb41942David S. Miller * initRTO, we only reset cwnd when more than 1 SYN/SYN-ACK 5704aabd8ef8c43677cfee3e1e36c5a79edddb41942David S. Miller * retransmission has occurred. 5714aabd8ef8c43677cfee3e1e36c5a79edddb41942David S. Miller */ 5724aabd8ef8c43677cfee3e1e36c5a79edddb41942David S. Miller if (tp->total_retrans > 1) 5734aabd8ef8c43677cfee3e1e36c5a79edddb41942David S. Miller tp->snd_cwnd = 1; 5744aabd8ef8c43677cfee3e1e36c5a79edddb41942David S. Miller else 5754aabd8ef8c43677cfee3e1e36c5a79edddb41942David S. Miller tp->snd_cwnd = tcp_init_cwnd(tp, dst); 5764aabd8ef8c43677cfee3e1e36c5a79edddb41942David S. Miller tp->snd_cwnd_stamp = tcp_time_stamp; 5774aabd8ef8c43677cfee3e1e36c5a79edddb41942David S. Miller} 578ab92bb2f679d66c7e12a6b1c0cdd76fe308f6546David S. Miller 579a26552afe89438eefbe097512b3187f2c7e929fdHannes Frederic Sowabool tcp_peer_is_proven(struct request_sock *req, struct dst_entry *dst, 580a26552afe89438eefbe097512b3187f2c7e929fdHannes Frederic Sowa bool paws_check, bool timestamps) 581ab92bb2f679d66c7e12a6b1c0cdd76fe308f6546David S. Miller{ 58251c5d0c4b169bf762f09e0d5b283a7f0b2a45739David S. Miller struct tcp_metrics_block *tm; 58351c5d0c4b169bf762f09e0d5b283a7f0b2a45739David S. Miller bool ret; 58451c5d0c4b169bf762f09e0d5b283a7f0b2a45739David S. Miller 585ab92bb2f679d66c7e12a6b1c0cdd76fe308f6546David S. Miller if (!dst) 586ab92bb2f679d66c7e12a6b1c0cdd76fe308f6546David S. Miller return false; 58751c5d0c4b169bf762f09e0d5b283a7f0b2a45739David S. Miller 58851c5d0c4b169bf762f09e0d5b283a7f0b2a45739David S. Miller rcu_read_lock(); 58951c5d0c4b169bf762f09e0d5b283a7f0b2a45739David S. Miller tm = __tcp_get_metrics_req(req, dst); 59081166dd6fa8eb780b2132d32fbc77eb6ac04e44eDavid S. Miller if (paws_check) { 59181166dd6fa8eb780b2132d32fbc77eb6ac04e44eDavid S. Miller if (tm && 59281166dd6fa8eb780b2132d32fbc77eb6ac04e44eDavid S. Miller (u32)get_seconds() - tm->tcpm_ts_stamp < TCP_PAWS_MSL && 593a26552afe89438eefbe097512b3187f2c7e929fdHannes Frederic Sowa ((s32)(tm->tcpm_ts - req->ts_recent) > TCP_PAWS_WINDOW || 594a26552afe89438eefbe097512b3187f2c7e929fdHannes Frederic Sowa !timestamps)) 59581166dd6fa8eb780b2132d32fbc77eb6ac04e44eDavid S. Miller ret = false; 59681166dd6fa8eb780b2132d32fbc77eb6ac04e44eDavid S. Miller else 59781166dd6fa8eb780b2132d32fbc77eb6ac04e44eDavid S. Miller ret = true; 59881166dd6fa8eb780b2132d32fbc77eb6ac04e44eDavid S. Miller } else { 59981166dd6fa8eb780b2132d32fbc77eb6ac04e44eDavid S. Miller if (tm && tcp_metric_get(tm, TCP_METRIC_RTT) && tm->tcpm_ts_stamp) 60081166dd6fa8eb780b2132d32fbc77eb6ac04e44eDavid S. Miller ret = true; 60181166dd6fa8eb780b2132d32fbc77eb6ac04e44eDavid S. Miller else 60281166dd6fa8eb780b2132d32fbc77eb6ac04e44eDavid S. Miller ret = false; 60381166dd6fa8eb780b2132d32fbc77eb6ac04e44eDavid S. Miller } 60451c5d0c4b169bf762f09e0d5b283a7f0b2a45739David S. Miller rcu_read_unlock(); 60551c5d0c4b169bf762f09e0d5b283a7f0b2a45739David S. Miller 60651c5d0c4b169bf762f09e0d5b283a7f0b2a45739David S. Miller return ret; 607ab92bb2f679d66c7e12a6b1c0cdd76fe308f6546David S. Miller} 608ab92bb2f679d66c7e12a6b1c0cdd76fe308f6546David S. MillerEXPORT_SYMBOL_GPL(tcp_peer_is_proven); 60951c5d0c4b169bf762f09e0d5b283a7f0b2a45739David S. Miller 61081166dd6fa8eb780b2132d32fbc77eb6ac04e44eDavid S. Millervoid tcp_fetch_timewait_stamp(struct sock *sk, struct dst_entry *dst) 61181166dd6fa8eb780b2132d32fbc77eb6ac04e44eDavid S. Miller{ 61281166dd6fa8eb780b2132d32fbc77eb6ac04e44eDavid S. Miller struct tcp_metrics_block *tm; 61381166dd6fa8eb780b2132d32fbc77eb6ac04e44eDavid S. Miller 61481166dd6fa8eb780b2132d32fbc77eb6ac04e44eDavid S. Miller rcu_read_lock(); 61581166dd6fa8eb780b2132d32fbc77eb6ac04e44eDavid S. Miller tm = tcp_get_metrics(sk, dst, true); 61681166dd6fa8eb780b2132d32fbc77eb6ac04e44eDavid S. Miller if (tm) { 61781166dd6fa8eb780b2132d32fbc77eb6ac04e44eDavid S. Miller struct tcp_sock *tp = tcp_sk(sk); 61881166dd6fa8eb780b2132d32fbc77eb6ac04e44eDavid S. Miller 61981166dd6fa8eb780b2132d32fbc77eb6ac04e44eDavid S. Miller if ((u32)get_seconds() - tm->tcpm_ts_stamp <= TCP_PAWS_MSL) { 62081166dd6fa8eb780b2132d32fbc77eb6ac04e44eDavid S. Miller tp->rx_opt.ts_recent_stamp = tm->tcpm_ts_stamp; 62181166dd6fa8eb780b2132d32fbc77eb6ac04e44eDavid S. Miller tp->rx_opt.ts_recent = tm->tcpm_ts; 62281166dd6fa8eb780b2132d32fbc77eb6ac04e44eDavid S. Miller } 62381166dd6fa8eb780b2132d32fbc77eb6ac04e44eDavid S. Miller } 62481166dd6fa8eb780b2132d32fbc77eb6ac04e44eDavid S. Miller rcu_read_unlock(); 62581166dd6fa8eb780b2132d32fbc77eb6ac04e44eDavid S. Miller} 62681166dd6fa8eb780b2132d32fbc77eb6ac04e44eDavid S. MillerEXPORT_SYMBOL_GPL(tcp_fetch_timewait_stamp); 62781166dd6fa8eb780b2132d32fbc77eb6ac04e44eDavid S. Miller 62881166dd6fa8eb780b2132d32fbc77eb6ac04e44eDavid S. Miller/* VJ's idea. Save last timestamp seen from this destination and hold 62981166dd6fa8eb780b2132d32fbc77eb6ac04e44eDavid S. Miller * it at least for normal timewait interval to use for duplicate 63081166dd6fa8eb780b2132d32fbc77eb6ac04e44eDavid S. Miller * segment detection in subsequent connections, before they enter 63181166dd6fa8eb780b2132d32fbc77eb6ac04e44eDavid S. Miller * synchronized state. 63281166dd6fa8eb780b2132d32fbc77eb6ac04e44eDavid S. Miller */ 63381166dd6fa8eb780b2132d32fbc77eb6ac04e44eDavid S. Millerbool tcp_remember_stamp(struct sock *sk) 63481166dd6fa8eb780b2132d32fbc77eb6ac04e44eDavid S. Miller{ 63581166dd6fa8eb780b2132d32fbc77eb6ac04e44eDavid S. Miller struct dst_entry *dst = __sk_dst_get(sk); 63681166dd6fa8eb780b2132d32fbc77eb6ac04e44eDavid S. Miller bool ret = false; 63781166dd6fa8eb780b2132d32fbc77eb6ac04e44eDavid S. Miller 63881166dd6fa8eb780b2132d32fbc77eb6ac04e44eDavid S. Miller if (dst) { 63981166dd6fa8eb780b2132d32fbc77eb6ac04e44eDavid S. Miller struct tcp_metrics_block *tm; 64081166dd6fa8eb780b2132d32fbc77eb6ac04e44eDavid S. Miller 64181166dd6fa8eb780b2132d32fbc77eb6ac04e44eDavid S. Miller rcu_read_lock(); 64281166dd6fa8eb780b2132d32fbc77eb6ac04e44eDavid S. Miller tm = tcp_get_metrics(sk, dst, true); 64381166dd6fa8eb780b2132d32fbc77eb6ac04e44eDavid S. Miller if (tm) { 64481166dd6fa8eb780b2132d32fbc77eb6ac04e44eDavid S. Miller struct tcp_sock *tp = tcp_sk(sk); 64581166dd6fa8eb780b2132d32fbc77eb6ac04e44eDavid S. Miller 64681166dd6fa8eb780b2132d32fbc77eb6ac04e44eDavid S. Miller if ((s32)(tm->tcpm_ts - tp->rx_opt.ts_recent) <= 0 || 64781166dd6fa8eb780b2132d32fbc77eb6ac04e44eDavid S. Miller ((u32)get_seconds() - tm->tcpm_ts_stamp > TCP_PAWS_MSL && 64881166dd6fa8eb780b2132d32fbc77eb6ac04e44eDavid S. Miller tm->tcpm_ts_stamp <= (u32)tp->rx_opt.ts_recent_stamp)) { 64981166dd6fa8eb780b2132d32fbc77eb6ac04e44eDavid S. Miller tm->tcpm_ts_stamp = (u32)tp->rx_opt.ts_recent_stamp; 65081166dd6fa8eb780b2132d32fbc77eb6ac04e44eDavid S. Miller tm->tcpm_ts = tp->rx_opt.ts_recent; 65181166dd6fa8eb780b2132d32fbc77eb6ac04e44eDavid S. Miller } 65281166dd6fa8eb780b2132d32fbc77eb6ac04e44eDavid S. Miller ret = true; 65381166dd6fa8eb780b2132d32fbc77eb6ac04e44eDavid S. Miller } 65481166dd6fa8eb780b2132d32fbc77eb6ac04e44eDavid S. Miller rcu_read_unlock(); 65581166dd6fa8eb780b2132d32fbc77eb6ac04e44eDavid S. Miller } 65681166dd6fa8eb780b2132d32fbc77eb6ac04e44eDavid S. Miller return ret; 65781166dd6fa8eb780b2132d32fbc77eb6ac04e44eDavid S. Miller} 65881166dd6fa8eb780b2132d32fbc77eb6ac04e44eDavid S. Miller 65981166dd6fa8eb780b2132d32fbc77eb6ac04e44eDavid S. Millerbool tcp_tw_remember_stamp(struct inet_timewait_sock *tw) 66081166dd6fa8eb780b2132d32fbc77eb6ac04e44eDavid S. Miller{ 66181166dd6fa8eb780b2132d32fbc77eb6ac04e44eDavid S. Miller struct tcp_metrics_block *tm; 66281166dd6fa8eb780b2132d32fbc77eb6ac04e44eDavid S. Miller bool ret = false; 66381166dd6fa8eb780b2132d32fbc77eb6ac04e44eDavid S. Miller 66481166dd6fa8eb780b2132d32fbc77eb6ac04e44eDavid S. Miller rcu_read_lock(); 66581166dd6fa8eb780b2132d32fbc77eb6ac04e44eDavid S. Miller tm = __tcp_get_metrics_tw(tw); 6669a0a9502cbf19d31f7387e3066f3d1a491bef616Julian Anastasov if (tm) { 66781166dd6fa8eb780b2132d32fbc77eb6ac04e44eDavid S. Miller const struct tcp_timewait_sock *tcptw; 66881166dd6fa8eb780b2132d32fbc77eb6ac04e44eDavid S. Miller struct sock *sk = (struct sock *) tw; 66981166dd6fa8eb780b2132d32fbc77eb6ac04e44eDavid S. Miller 67081166dd6fa8eb780b2132d32fbc77eb6ac04e44eDavid S. Miller tcptw = tcp_twsk(sk); 67181166dd6fa8eb780b2132d32fbc77eb6ac04e44eDavid S. Miller if ((s32)(tm->tcpm_ts - tcptw->tw_ts_recent) <= 0 || 67281166dd6fa8eb780b2132d32fbc77eb6ac04e44eDavid S. Miller ((u32)get_seconds() - tm->tcpm_ts_stamp > TCP_PAWS_MSL && 67381166dd6fa8eb780b2132d32fbc77eb6ac04e44eDavid S. Miller tm->tcpm_ts_stamp <= (u32)tcptw->tw_ts_recent_stamp)) { 67481166dd6fa8eb780b2132d32fbc77eb6ac04e44eDavid S. Miller tm->tcpm_ts_stamp = (u32)tcptw->tw_ts_recent_stamp; 67581166dd6fa8eb780b2132d32fbc77eb6ac04e44eDavid S. Miller tm->tcpm_ts = tcptw->tw_ts_recent; 67681166dd6fa8eb780b2132d32fbc77eb6ac04e44eDavid S. Miller } 67781166dd6fa8eb780b2132d32fbc77eb6ac04e44eDavid S. Miller ret = true; 67881166dd6fa8eb780b2132d32fbc77eb6ac04e44eDavid S. Miller } 67981166dd6fa8eb780b2132d32fbc77eb6ac04e44eDavid S. Miller rcu_read_unlock(); 68081166dd6fa8eb780b2132d32fbc77eb6ac04e44eDavid S. Miller 68181166dd6fa8eb780b2132d32fbc77eb6ac04e44eDavid S. Miller return ret; 68281166dd6fa8eb780b2132d32fbc77eb6ac04e44eDavid S. Miller} 68381166dd6fa8eb780b2132d32fbc77eb6ac04e44eDavid S. Miller 6841fe4c481ba637660793217769695c146a037bd54Yuchung Chengstatic DEFINE_SEQLOCK(fastopen_seqlock); 6851fe4c481ba637660793217769695c146a037bd54Yuchung Cheng 6861fe4c481ba637660793217769695c146a037bd54Yuchung Chengvoid tcp_fastopen_cache_get(struct sock *sk, u16 *mss, 687aab4874355679c70f93993cf3b3fd74643b9ac33Yuchung Cheng struct tcp_fastopen_cookie *cookie, 688aab4874355679c70f93993cf3b3fd74643b9ac33Yuchung Cheng int *syn_loss, unsigned long *last_syn_loss) 6891fe4c481ba637660793217769695c146a037bd54Yuchung Cheng{ 6901fe4c481ba637660793217769695c146a037bd54Yuchung Cheng struct tcp_metrics_block *tm; 6911fe4c481ba637660793217769695c146a037bd54Yuchung Cheng 6921fe4c481ba637660793217769695c146a037bd54Yuchung Cheng rcu_read_lock(); 6931fe4c481ba637660793217769695c146a037bd54Yuchung Cheng tm = tcp_get_metrics(sk, __sk_dst_get(sk), false); 6941fe4c481ba637660793217769695c146a037bd54Yuchung Cheng if (tm) { 6951fe4c481ba637660793217769695c146a037bd54Yuchung Cheng struct tcp_fastopen_metrics *tfom = &tm->tcpm_fastopen; 6961fe4c481ba637660793217769695c146a037bd54Yuchung Cheng unsigned int seq; 6971fe4c481ba637660793217769695c146a037bd54Yuchung Cheng 6981fe4c481ba637660793217769695c146a037bd54Yuchung Cheng do { 6991fe4c481ba637660793217769695c146a037bd54Yuchung Cheng seq = read_seqbegin(&fastopen_seqlock); 7001fe4c481ba637660793217769695c146a037bd54Yuchung Cheng if (tfom->mss) 7011fe4c481ba637660793217769695c146a037bd54Yuchung Cheng *mss = tfom->mss; 7021fe4c481ba637660793217769695c146a037bd54Yuchung Cheng *cookie = tfom->cookie; 703aab4874355679c70f93993cf3b3fd74643b9ac33Yuchung Cheng *syn_loss = tfom->syn_loss; 704aab4874355679c70f93993cf3b3fd74643b9ac33Yuchung Cheng *last_syn_loss = *syn_loss ? tfom->last_syn_loss : 0; 7051fe4c481ba637660793217769695c146a037bd54Yuchung Cheng } while (read_seqretry(&fastopen_seqlock, seq)); 7061fe4c481ba637660793217769695c146a037bd54Yuchung Cheng } 7071fe4c481ba637660793217769695c146a037bd54Yuchung Cheng rcu_read_unlock(); 7081fe4c481ba637660793217769695c146a037bd54Yuchung Cheng} 7091fe4c481ba637660793217769695c146a037bd54Yuchung Cheng 7101fe4c481ba637660793217769695c146a037bd54Yuchung Chengvoid tcp_fastopen_cache_set(struct sock *sk, u16 mss, 711aab4874355679c70f93993cf3b3fd74643b9ac33Yuchung Cheng struct tcp_fastopen_cookie *cookie, bool syn_lost) 7121fe4c481ba637660793217769695c146a037bd54Yuchung Cheng{ 713dccf76ca6b626c0c4a4e09bb221adee3270ab0efEric Dumazet struct dst_entry *dst = __sk_dst_get(sk); 7141fe4c481ba637660793217769695c146a037bd54Yuchung Cheng struct tcp_metrics_block *tm; 7151fe4c481ba637660793217769695c146a037bd54Yuchung Cheng 716dccf76ca6b626c0c4a4e09bb221adee3270ab0efEric Dumazet if (!dst) 717dccf76ca6b626c0c4a4e09bb221adee3270ab0efEric Dumazet return; 7181fe4c481ba637660793217769695c146a037bd54Yuchung Cheng rcu_read_lock(); 719dccf76ca6b626c0c4a4e09bb221adee3270ab0efEric Dumazet tm = tcp_get_metrics(sk, dst, true); 7201fe4c481ba637660793217769695c146a037bd54Yuchung Cheng if (tm) { 7211fe4c481ba637660793217769695c146a037bd54Yuchung Cheng struct tcp_fastopen_metrics *tfom = &tm->tcpm_fastopen; 7221fe4c481ba637660793217769695c146a037bd54Yuchung Cheng 7231fe4c481ba637660793217769695c146a037bd54Yuchung Cheng write_seqlock_bh(&fastopen_seqlock); 724c968601d174739cb1e7100c95e0eb3d2f7e91bc9Yuchung Cheng if (mss) 725c968601d174739cb1e7100c95e0eb3d2f7e91bc9Yuchung Cheng tfom->mss = mss; 726c968601d174739cb1e7100c95e0eb3d2f7e91bc9Yuchung Cheng if (cookie && cookie->len > 0) 7271fe4c481ba637660793217769695c146a037bd54Yuchung Cheng tfom->cookie = *cookie; 728aab4874355679c70f93993cf3b3fd74643b9ac33Yuchung Cheng if (syn_lost) { 729aab4874355679c70f93993cf3b3fd74643b9ac33Yuchung Cheng ++tfom->syn_loss; 730aab4874355679c70f93993cf3b3fd74643b9ac33Yuchung Cheng tfom->last_syn_loss = jiffies; 731aab4874355679c70f93993cf3b3fd74643b9ac33Yuchung Cheng } else 732aab4874355679c70f93993cf3b3fd74643b9ac33Yuchung Cheng tfom->syn_loss = 0; 7331fe4c481ba637660793217769695c146a037bd54Yuchung Cheng write_sequnlock_bh(&fastopen_seqlock); 7341fe4c481ba637660793217769695c146a037bd54Yuchung Cheng } 7351fe4c481ba637660793217769695c146a037bd54Yuchung Cheng rcu_read_unlock(); 7361fe4c481ba637660793217769695c146a037bd54Yuchung Cheng} 7371fe4c481ba637660793217769695c146a037bd54Yuchung Cheng 738d23ff701643a4a725e2c7a8ba2d567d39daa29eaJulian Anastasovstatic struct genl_family tcp_metrics_nl_family = { 739d23ff701643a4a725e2c7a8ba2d567d39daa29eaJulian Anastasov .id = GENL_ID_GENERATE, 740d23ff701643a4a725e2c7a8ba2d567d39daa29eaJulian Anastasov .hdrsize = 0, 741d23ff701643a4a725e2c7a8ba2d567d39daa29eaJulian Anastasov .name = TCP_METRICS_GENL_NAME, 742d23ff701643a4a725e2c7a8ba2d567d39daa29eaJulian Anastasov .version = TCP_METRICS_GENL_VERSION, 743d23ff701643a4a725e2c7a8ba2d567d39daa29eaJulian Anastasov .maxattr = TCP_METRICS_ATTR_MAX, 744d23ff701643a4a725e2c7a8ba2d567d39daa29eaJulian Anastasov .netnsok = true, 745d23ff701643a4a725e2c7a8ba2d567d39daa29eaJulian Anastasov}; 746d23ff701643a4a725e2c7a8ba2d567d39daa29eaJulian Anastasov 747d23ff701643a4a725e2c7a8ba2d567d39daa29eaJulian Anastasovstatic struct nla_policy tcp_metrics_nl_policy[TCP_METRICS_ATTR_MAX + 1] = { 748d23ff701643a4a725e2c7a8ba2d567d39daa29eaJulian Anastasov [TCP_METRICS_ATTR_ADDR_IPV4] = { .type = NLA_U32, }, 749d23ff701643a4a725e2c7a8ba2d567d39daa29eaJulian Anastasov [TCP_METRICS_ATTR_ADDR_IPV6] = { .type = NLA_BINARY, 750d23ff701643a4a725e2c7a8ba2d567d39daa29eaJulian Anastasov .len = sizeof(struct in6_addr), }, 751d23ff701643a4a725e2c7a8ba2d567d39daa29eaJulian Anastasov /* Following attributes are not received for GET/DEL, 752d23ff701643a4a725e2c7a8ba2d567d39daa29eaJulian Anastasov * we keep them for reference 753d23ff701643a4a725e2c7a8ba2d567d39daa29eaJulian Anastasov */ 754d23ff701643a4a725e2c7a8ba2d567d39daa29eaJulian Anastasov#if 0 755d23ff701643a4a725e2c7a8ba2d567d39daa29eaJulian Anastasov [TCP_METRICS_ATTR_AGE] = { .type = NLA_MSECS, }, 756d23ff701643a4a725e2c7a8ba2d567d39daa29eaJulian Anastasov [TCP_METRICS_ATTR_TW_TSVAL] = { .type = NLA_U32, }, 757d23ff701643a4a725e2c7a8ba2d567d39daa29eaJulian Anastasov [TCP_METRICS_ATTR_TW_TS_STAMP] = { .type = NLA_S32, }, 758d23ff701643a4a725e2c7a8ba2d567d39daa29eaJulian Anastasov [TCP_METRICS_ATTR_VALS] = { .type = NLA_NESTED, }, 759d23ff701643a4a725e2c7a8ba2d567d39daa29eaJulian Anastasov [TCP_METRICS_ATTR_FOPEN_MSS] = { .type = NLA_U16, }, 760d23ff701643a4a725e2c7a8ba2d567d39daa29eaJulian Anastasov [TCP_METRICS_ATTR_FOPEN_SYN_DROPS] = { .type = NLA_U16, }, 761d23ff701643a4a725e2c7a8ba2d567d39daa29eaJulian Anastasov [TCP_METRICS_ATTR_FOPEN_SYN_DROP_TS] = { .type = NLA_MSECS, }, 762d23ff701643a4a725e2c7a8ba2d567d39daa29eaJulian Anastasov [TCP_METRICS_ATTR_FOPEN_COOKIE] = { .type = NLA_BINARY, 763d23ff701643a4a725e2c7a8ba2d567d39daa29eaJulian Anastasov .len = TCP_FASTOPEN_COOKIE_MAX, }, 764d23ff701643a4a725e2c7a8ba2d567d39daa29eaJulian Anastasov#endif 765d23ff701643a4a725e2c7a8ba2d567d39daa29eaJulian Anastasov}; 766d23ff701643a4a725e2c7a8ba2d567d39daa29eaJulian Anastasov 767d23ff701643a4a725e2c7a8ba2d567d39daa29eaJulian Anastasov/* Add attributes, caller cancels its header on failure */ 768d23ff701643a4a725e2c7a8ba2d567d39daa29eaJulian Anastasovstatic int tcp_metrics_fill_info(struct sk_buff *msg, 769d23ff701643a4a725e2c7a8ba2d567d39daa29eaJulian Anastasov struct tcp_metrics_block *tm) 770d23ff701643a4a725e2c7a8ba2d567d39daa29eaJulian Anastasov{ 771d23ff701643a4a725e2c7a8ba2d567d39daa29eaJulian Anastasov struct nlattr *nest; 772d23ff701643a4a725e2c7a8ba2d567d39daa29eaJulian Anastasov int i; 773d23ff701643a4a725e2c7a8ba2d567d39daa29eaJulian Anastasov 774324fd55a19827b7191cc6ab73865e30c0e6e6423Christoph Paasch switch (tm->tcpm_daddr.family) { 775d23ff701643a4a725e2c7a8ba2d567d39daa29eaJulian Anastasov case AF_INET: 776d23ff701643a4a725e2c7a8ba2d567d39daa29eaJulian Anastasov if (nla_put_be32(msg, TCP_METRICS_ATTR_ADDR_IPV4, 777324fd55a19827b7191cc6ab73865e30c0e6e6423Christoph Paasch tm->tcpm_daddr.addr.a4) < 0) 778d23ff701643a4a725e2c7a8ba2d567d39daa29eaJulian Anastasov goto nla_put_failure; 7798a59359cb80f448923a7bc9f555d477e74547d7aChristoph Paasch if (nla_put_be32(msg, TCP_METRICS_ATTR_SADDR_IPV4, 7808a59359cb80f448923a7bc9f555d477e74547d7aChristoph Paasch tm->tcpm_saddr.addr.a4) < 0) 7818a59359cb80f448923a7bc9f555d477e74547d7aChristoph Paasch goto nla_put_failure; 782d23ff701643a4a725e2c7a8ba2d567d39daa29eaJulian Anastasov break; 783d23ff701643a4a725e2c7a8ba2d567d39daa29eaJulian Anastasov case AF_INET6: 784d23ff701643a4a725e2c7a8ba2d567d39daa29eaJulian Anastasov if (nla_put(msg, TCP_METRICS_ATTR_ADDR_IPV6, 16, 785324fd55a19827b7191cc6ab73865e30c0e6e6423Christoph Paasch tm->tcpm_daddr.addr.a6) < 0) 786d23ff701643a4a725e2c7a8ba2d567d39daa29eaJulian Anastasov goto nla_put_failure; 7878a59359cb80f448923a7bc9f555d477e74547d7aChristoph Paasch if (nla_put(msg, TCP_METRICS_ATTR_SADDR_IPV6, 16, 7888a59359cb80f448923a7bc9f555d477e74547d7aChristoph Paasch tm->tcpm_saddr.addr.a6) < 0) 7898a59359cb80f448923a7bc9f555d477e74547d7aChristoph Paasch goto nla_put_failure; 790d23ff701643a4a725e2c7a8ba2d567d39daa29eaJulian Anastasov break; 791d23ff701643a4a725e2c7a8ba2d567d39daa29eaJulian Anastasov default: 792d23ff701643a4a725e2c7a8ba2d567d39daa29eaJulian Anastasov return -EAFNOSUPPORT; 793d23ff701643a4a725e2c7a8ba2d567d39daa29eaJulian Anastasov } 794d23ff701643a4a725e2c7a8ba2d567d39daa29eaJulian Anastasov 795d23ff701643a4a725e2c7a8ba2d567d39daa29eaJulian Anastasov if (nla_put_msecs(msg, TCP_METRICS_ATTR_AGE, 796d23ff701643a4a725e2c7a8ba2d567d39daa29eaJulian Anastasov jiffies - tm->tcpm_stamp) < 0) 797d23ff701643a4a725e2c7a8ba2d567d39daa29eaJulian Anastasov goto nla_put_failure; 798d23ff701643a4a725e2c7a8ba2d567d39daa29eaJulian Anastasov if (tm->tcpm_ts_stamp) { 799d23ff701643a4a725e2c7a8ba2d567d39daa29eaJulian Anastasov if (nla_put_s32(msg, TCP_METRICS_ATTR_TW_TS_STAMP, 800d23ff701643a4a725e2c7a8ba2d567d39daa29eaJulian Anastasov (s32) (get_seconds() - tm->tcpm_ts_stamp)) < 0) 801d23ff701643a4a725e2c7a8ba2d567d39daa29eaJulian Anastasov goto nla_put_failure; 802d23ff701643a4a725e2c7a8ba2d567d39daa29eaJulian Anastasov if (nla_put_u32(msg, TCP_METRICS_ATTR_TW_TSVAL, 803d23ff701643a4a725e2c7a8ba2d567d39daa29eaJulian Anastasov tm->tcpm_ts) < 0) 804d23ff701643a4a725e2c7a8ba2d567d39daa29eaJulian Anastasov goto nla_put_failure; 805d23ff701643a4a725e2c7a8ba2d567d39daa29eaJulian Anastasov } 806d23ff701643a4a725e2c7a8ba2d567d39daa29eaJulian Anastasov 807d23ff701643a4a725e2c7a8ba2d567d39daa29eaJulian Anastasov { 808d23ff701643a4a725e2c7a8ba2d567d39daa29eaJulian Anastasov int n = 0; 809d23ff701643a4a725e2c7a8ba2d567d39daa29eaJulian Anastasov 810d23ff701643a4a725e2c7a8ba2d567d39daa29eaJulian Anastasov nest = nla_nest_start(msg, TCP_METRICS_ATTR_VALS); 811d23ff701643a4a725e2c7a8ba2d567d39daa29eaJulian Anastasov if (!nest) 812d23ff701643a4a725e2c7a8ba2d567d39daa29eaJulian Anastasov goto nla_put_failure; 813740b0f1841f6e39085b711d41db9ffb07198682bEric Dumazet for (i = 0; i < TCP_METRIC_MAX_KERNEL + 1; i++) { 814740b0f1841f6e39085b711d41db9ffb07198682bEric Dumazet u32 val = tm->tcpm_vals[i]; 815740b0f1841f6e39085b711d41db9ffb07198682bEric Dumazet 816740b0f1841f6e39085b711d41db9ffb07198682bEric Dumazet if (!val) 817d23ff701643a4a725e2c7a8ba2d567d39daa29eaJulian Anastasov continue; 818740b0f1841f6e39085b711d41db9ffb07198682bEric Dumazet if (i == TCP_METRIC_RTT) { 819740b0f1841f6e39085b711d41db9ffb07198682bEric Dumazet if (nla_put_u32(msg, TCP_METRIC_RTT_US + 1, 820740b0f1841f6e39085b711d41db9ffb07198682bEric Dumazet val) < 0) 821740b0f1841f6e39085b711d41db9ffb07198682bEric Dumazet goto nla_put_failure; 822740b0f1841f6e39085b711d41db9ffb07198682bEric Dumazet n++; 823740b0f1841f6e39085b711d41db9ffb07198682bEric Dumazet val = max(val / 1000, 1U); 824740b0f1841f6e39085b711d41db9ffb07198682bEric Dumazet } 825740b0f1841f6e39085b711d41db9ffb07198682bEric Dumazet if (i == TCP_METRIC_RTTVAR) { 826740b0f1841f6e39085b711d41db9ffb07198682bEric Dumazet if (nla_put_u32(msg, TCP_METRIC_RTTVAR_US + 1, 827740b0f1841f6e39085b711d41db9ffb07198682bEric Dumazet val) < 0) 828740b0f1841f6e39085b711d41db9ffb07198682bEric Dumazet goto nla_put_failure; 829740b0f1841f6e39085b711d41db9ffb07198682bEric Dumazet n++; 830740b0f1841f6e39085b711d41db9ffb07198682bEric Dumazet val = max(val / 1000, 1U); 831740b0f1841f6e39085b711d41db9ffb07198682bEric Dumazet } 832740b0f1841f6e39085b711d41db9ffb07198682bEric Dumazet if (nla_put_u32(msg, i + 1, val) < 0) 833d23ff701643a4a725e2c7a8ba2d567d39daa29eaJulian Anastasov goto nla_put_failure; 834d23ff701643a4a725e2c7a8ba2d567d39daa29eaJulian Anastasov n++; 835d23ff701643a4a725e2c7a8ba2d567d39daa29eaJulian Anastasov } 836d23ff701643a4a725e2c7a8ba2d567d39daa29eaJulian Anastasov if (n) 837d23ff701643a4a725e2c7a8ba2d567d39daa29eaJulian Anastasov nla_nest_end(msg, nest); 838d23ff701643a4a725e2c7a8ba2d567d39daa29eaJulian Anastasov else 839d23ff701643a4a725e2c7a8ba2d567d39daa29eaJulian Anastasov nla_nest_cancel(msg, nest); 840d23ff701643a4a725e2c7a8ba2d567d39daa29eaJulian Anastasov } 841d23ff701643a4a725e2c7a8ba2d567d39daa29eaJulian Anastasov 842d23ff701643a4a725e2c7a8ba2d567d39daa29eaJulian Anastasov { 843d23ff701643a4a725e2c7a8ba2d567d39daa29eaJulian Anastasov struct tcp_fastopen_metrics tfom_copy[1], *tfom; 844d23ff701643a4a725e2c7a8ba2d567d39daa29eaJulian Anastasov unsigned int seq; 845d23ff701643a4a725e2c7a8ba2d567d39daa29eaJulian Anastasov 846d23ff701643a4a725e2c7a8ba2d567d39daa29eaJulian Anastasov do { 847d23ff701643a4a725e2c7a8ba2d567d39daa29eaJulian Anastasov seq = read_seqbegin(&fastopen_seqlock); 848d23ff701643a4a725e2c7a8ba2d567d39daa29eaJulian Anastasov tfom_copy[0] = tm->tcpm_fastopen; 849d23ff701643a4a725e2c7a8ba2d567d39daa29eaJulian Anastasov } while (read_seqretry(&fastopen_seqlock, seq)); 850d23ff701643a4a725e2c7a8ba2d567d39daa29eaJulian Anastasov 851d23ff701643a4a725e2c7a8ba2d567d39daa29eaJulian Anastasov tfom = tfom_copy; 852d23ff701643a4a725e2c7a8ba2d567d39daa29eaJulian Anastasov if (tfom->mss && 853d23ff701643a4a725e2c7a8ba2d567d39daa29eaJulian Anastasov nla_put_u16(msg, TCP_METRICS_ATTR_FOPEN_MSS, 854d23ff701643a4a725e2c7a8ba2d567d39daa29eaJulian Anastasov tfom->mss) < 0) 855d23ff701643a4a725e2c7a8ba2d567d39daa29eaJulian Anastasov goto nla_put_failure; 856d23ff701643a4a725e2c7a8ba2d567d39daa29eaJulian Anastasov if (tfom->syn_loss && 857d23ff701643a4a725e2c7a8ba2d567d39daa29eaJulian Anastasov (nla_put_u16(msg, TCP_METRICS_ATTR_FOPEN_SYN_DROPS, 858d23ff701643a4a725e2c7a8ba2d567d39daa29eaJulian Anastasov tfom->syn_loss) < 0 || 859d23ff701643a4a725e2c7a8ba2d567d39daa29eaJulian Anastasov nla_put_msecs(msg, TCP_METRICS_ATTR_FOPEN_SYN_DROP_TS, 860d23ff701643a4a725e2c7a8ba2d567d39daa29eaJulian Anastasov jiffies - tfom->last_syn_loss) < 0)) 861d23ff701643a4a725e2c7a8ba2d567d39daa29eaJulian Anastasov goto nla_put_failure; 862d23ff701643a4a725e2c7a8ba2d567d39daa29eaJulian Anastasov if (tfom->cookie.len > 0 && 863d23ff701643a4a725e2c7a8ba2d567d39daa29eaJulian Anastasov nla_put(msg, TCP_METRICS_ATTR_FOPEN_COOKIE, 864d23ff701643a4a725e2c7a8ba2d567d39daa29eaJulian Anastasov tfom->cookie.len, tfom->cookie.val) < 0) 865d23ff701643a4a725e2c7a8ba2d567d39daa29eaJulian Anastasov goto nla_put_failure; 866d23ff701643a4a725e2c7a8ba2d567d39daa29eaJulian Anastasov } 867d23ff701643a4a725e2c7a8ba2d567d39daa29eaJulian Anastasov 868d23ff701643a4a725e2c7a8ba2d567d39daa29eaJulian Anastasov return 0; 869d23ff701643a4a725e2c7a8ba2d567d39daa29eaJulian Anastasov 870d23ff701643a4a725e2c7a8ba2d567d39daa29eaJulian Anastasovnla_put_failure: 871d23ff701643a4a725e2c7a8ba2d567d39daa29eaJulian Anastasov return -EMSGSIZE; 872d23ff701643a4a725e2c7a8ba2d567d39daa29eaJulian Anastasov} 873d23ff701643a4a725e2c7a8ba2d567d39daa29eaJulian Anastasov 874d23ff701643a4a725e2c7a8ba2d567d39daa29eaJulian Anastasovstatic int tcp_metrics_dump_info(struct sk_buff *skb, 875d23ff701643a4a725e2c7a8ba2d567d39daa29eaJulian Anastasov struct netlink_callback *cb, 876d23ff701643a4a725e2c7a8ba2d567d39daa29eaJulian Anastasov struct tcp_metrics_block *tm) 877d23ff701643a4a725e2c7a8ba2d567d39daa29eaJulian Anastasov{ 878d23ff701643a4a725e2c7a8ba2d567d39daa29eaJulian Anastasov void *hdr; 879d23ff701643a4a725e2c7a8ba2d567d39daa29eaJulian Anastasov 88015e473046cb6e5d18a4d0057e61d76315230382bEric W. Biederman hdr = genlmsg_put(skb, NETLINK_CB(cb->skb).portid, cb->nlh->nlmsg_seq, 881d23ff701643a4a725e2c7a8ba2d567d39daa29eaJulian Anastasov &tcp_metrics_nl_family, NLM_F_MULTI, 882d23ff701643a4a725e2c7a8ba2d567d39daa29eaJulian Anastasov TCP_METRICS_CMD_GET); 883d23ff701643a4a725e2c7a8ba2d567d39daa29eaJulian Anastasov if (!hdr) 884d23ff701643a4a725e2c7a8ba2d567d39daa29eaJulian Anastasov return -EMSGSIZE; 885d23ff701643a4a725e2c7a8ba2d567d39daa29eaJulian Anastasov 886d23ff701643a4a725e2c7a8ba2d567d39daa29eaJulian Anastasov if (tcp_metrics_fill_info(skb, tm) < 0) 887d23ff701643a4a725e2c7a8ba2d567d39daa29eaJulian Anastasov goto nla_put_failure; 888d23ff701643a4a725e2c7a8ba2d567d39daa29eaJulian Anastasov 889d23ff701643a4a725e2c7a8ba2d567d39daa29eaJulian Anastasov return genlmsg_end(skb, hdr); 890d23ff701643a4a725e2c7a8ba2d567d39daa29eaJulian Anastasov 891d23ff701643a4a725e2c7a8ba2d567d39daa29eaJulian Anastasovnla_put_failure: 892d23ff701643a4a725e2c7a8ba2d567d39daa29eaJulian Anastasov genlmsg_cancel(skb, hdr); 893d23ff701643a4a725e2c7a8ba2d567d39daa29eaJulian Anastasov return -EMSGSIZE; 894d23ff701643a4a725e2c7a8ba2d567d39daa29eaJulian Anastasov} 895d23ff701643a4a725e2c7a8ba2d567d39daa29eaJulian Anastasov 896d23ff701643a4a725e2c7a8ba2d567d39daa29eaJulian Anastasovstatic int tcp_metrics_nl_dump(struct sk_buff *skb, 897d23ff701643a4a725e2c7a8ba2d567d39daa29eaJulian Anastasov struct netlink_callback *cb) 898d23ff701643a4a725e2c7a8ba2d567d39daa29eaJulian Anastasov{ 899d23ff701643a4a725e2c7a8ba2d567d39daa29eaJulian Anastasov struct net *net = sock_net(skb->sk); 900d23ff701643a4a725e2c7a8ba2d567d39daa29eaJulian Anastasov unsigned int max_rows = 1U << net->ipv4.tcp_metrics_hash_log; 901d23ff701643a4a725e2c7a8ba2d567d39daa29eaJulian Anastasov unsigned int row, s_row = cb->args[0]; 902d23ff701643a4a725e2c7a8ba2d567d39daa29eaJulian Anastasov int s_col = cb->args[1], col = s_col; 903d23ff701643a4a725e2c7a8ba2d567d39daa29eaJulian Anastasov 904d23ff701643a4a725e2c7a8ba2d567d39daa29eaJulian Anastasov for (row = s_row; row < max_rows; row++, s_col = 0) { 905d23ff701643a4a725e2c7a8ba2d567d39daa29eaJulian Anastasov struct tcp_metrics_block *tm; 906d23ff701643a4a725e2c7a8ba2d567d39daa29eaJulian Anastasov struct tcpm_hash_bucket *hb = net->ipv4.tcp_metrics_hash + row; 907d23ff701643a4a725e2c7a8ba2d567d39daa29eaJulian Anastasov 908d23ff701643a4a725e2c7a8ba2d567d39daa29eaJulian Anastasov rcu_read_lock(); 909d23ff701643a4a725e2c7a8ba2d567d39daa29eaJulian Anastasov for (col = 0, tm = rcu_dereference(hb->chain); tm; 910d23ff701643a4a725e2c7a8ba2d567d39daa29eaJulian Anastasov tm = rcu_dereference(tm->tcpm_next), col++) { 911d23ff701643a4a725e2c7a8ba2d567d39daa29eaJulian Anastasov if (col < s_col) 912d23ff701643a4a725e2c7a8ba2d567d39daa29eaJulian Anastasov continue; 913d23ff701643a4a725e2c7a8ba2d567d39daa29eaJulian Anastasov if (tcp_metrics_dump_info(skb, cb, tm) < 0) { 914d23ff701643a4a725e2c7a8ba2d567d39daa29eaJulian Anastasov rcu_read_unlock(); 915d23ff701643a4a725e2c7a8ba2d567d39daa29eaJulian Anastasov goto done; 916d23ff701643a4a725e2c7a8ba2d567d39daa29eaJulian Anastasov } 917d23ff701643a4a725e2c7a8ba2d567d39daa29eaJulian Anastasov } 918d23ff701643a4a725e2c7a8ba2d567d39daa29eaJulian Anastasov rcu_read_unlock(); 919d23ff701643a4a725e2c7a8ba2d567d39daa29eaJulian Anastasov } 920d23ff701643a4a725e2c7a8ba2d567d39daa29eaJulian Anastasov 921d23ff701643a4a725e2c7a8ba2d567d39daa29eaJulian Anastasovdone: 922d23ff701643a4a725e2c7a8ba2d567d39daa29eaJulian Anastasov cb->args[0] = row; 923d23ff701643a4a725e2c7a8ba2d567d39daa29eaJulian Anastasov cb->args[1] = col; 924d23ff701643a4a725e2c7a8ba2d567d39daa29eaJulian Anastasov return skb->len; 925d23ff701643a4a725e2c7a8ba2d567d39daa29eaJulian Anastasov} 926d23ff701643a4a725e2c7a8ba2d567d39daa29eaJulian Anastasov 9273e7013ddf55af7bc191792b8aea0c2b94fb0fef5Christoph Paaschstatic int __parse_nl_addr(struct genl_info *info, struct inetpeer_addr *addr, 9283e7013ddf55af7bc191792b8aea0c2b94fb0fef5Christoph Paasch unsigned int *hash, int optional, int v4, int v6) 929d23ff701643a4a725e2c7a8ba2d567d39daa29eaJulian Anastasov{ 930d23ff701643a4a725e2c7a8ba2d567d39daa29eaJulian Anastasov struct nlattr *a; 931d23ff701643a4a725e2c7a8ba2d567d39daa29eaJulian Anastasov 9323e7013ddf55af7bc191792b8aea0c2b94fb0fef5Christoph Paasch a = info->attrs[v4]; 933d23ff701643a4a725e2c7a8ba2d567d39daa29eaJulian Anastasov if (a) { 934d23ff701643a4a725e2c7a8ba2d567d39daa29eaJulian Anastasov addr->family = AF_INET; 935d23ff701643a4a725e2c7a8ba2d567d39daa29eaJulian Anastasov addr->addr.a4 = nla_get_be32(a); 9363e7013ddf55af7bc191792b8aea0c2b94fb0fef5Christoph Paasch if (hash) 9373e7013ddf55af7bc191792b8aea0c2b94fb0fef5Christoph Paasch *hash = (__force unsigned int) addr->addr.a4; 938d23ff701643a4a725e2c7a8ba2d567d39daa29eaJulian Anastasov return 0; 939d23ff701643a4a725e2c7a8ba2d567d39daa29eaJulian Anastasov } 9403e7013ddf55af7bc191792b8aea0c2b94fb0fef5Christoph Paasch a = info->attrs[v6]; 941d23ff701643a4a725e2c7a8ba2d567d39daa29eaJulian Anastasov if (a) { 9422c42a3fb30845867bfcaf0747ff50c1375884ff2Julian Anastasov if (nla_len(a) != sizeof(struct in6_addr)) 943d23ff701643a4a725e2c7a8ba2d567d39daa29eaJulian Anastasov return -EINVAL; 944d23ff701643a4a725e2c7a8ba2d567d39daa29eaJulian Anastasov addr->family = AF_INET6; 945d23ff701643a4a725e2c7a8ba2d567d39daa29eaJulian Anastasov memcpy(addr->addr.a6, nla_data(a), sizeof(addr->addr.a6)); 9463e7013ddf55af7bc191792b8aea0c2b94fb0fef5Christoph Paasch if (hash) 9473e7013ddf55af7bc191792b8aea0c2b94fb0fef5Christoph Paasch *hash = ipv6_addr_hash((struct in6_addr *) addr->addr.a6); 948d23ff701643a4a725e2c7a8ba2d567d39daa29eaJulian Anastasov return 0; 949d23ff701643a4a725e2c7a8ba2d567d39daa29eaJulian Anastasov } 950d23ff701643a4a725e2c7a8ba2d567d39daa29eaJulian Anastasov return optional ? 1 : -EAFNOSUPPORT; 951d23ff701643a4a725e2c7a8ba2d567d39daa29eaJulian Anastasov} 952d23ff701643a4a725e2c7a8ba2d567d39daa29eaJulian Anastasov 9533e7013ddf55af7bc191792b8aea0c2b94fb0fef5Christoph Paaschstatic int parse_nl_addr(struct genl_info *info, struct inetpeer_addr *addr, 9543e7013ddf55af7bc191792b8aea0c2b94fb0fef5Christoph Paasch unsigned int *hash, int optional) 9553e7013ddf55af7bc191792b8aea0c2b94fb0fef5Christoph Paasch{ 9563e7013ddf55af7bc191792b8aea0c2b94fb0fef5Christoph Paasch return __parse_nl_addr(info, addr, hash, optional, 9573e7013ddf55af7bc191792b8aea0c2b94fb0fef5Christoph Paasch TCP_METRICS_ATTR_ADDR_IPV4, 9583e7013ddf55af7bc191792b8aea0c2b94fb0fef5Christoph Paasch TCP_METRICS_ATTR_ADDR_IPV6); 9593e7013ddf55af7bc191792b8aea0c2b94fb0fef5Christoph Paasch} 9603e7013ddf55af7bc191792b8aea0c2b94fb0fef5Christoph Paasch 9613e7013ddf55af7bc191792b8aea0c2b94fb0fef5Christoph Paaschstatic int parse_nl_saddr(struct genl_info *info, struct inetpeer_addr *addr) 9623e7013ddf55af7bc191792b8aea0c2b94fb0fef5Christoph Paasch{ 9633e7013ddf55af7bc191792b8aea0c2b94fb0fef5Christoph Paasch return __parse_nl_addr(info, addr, NULL, 0, 9643e7013ddf55af7bc191792b8aea0c2b94fb0fef5Christoph Paasch TCP_METRICS_ATTR_SADDR_IPV4, 9653e7013ddf55af7bc191792b8aea0c2b94fb0fef5Christoph Paasch TCP_METRICS_ATTR_SADDR_IPV6); 9663e7013ddf55af7bc191792b8aea0c2b94fb0fef5Christoph Paasch} 9673e7013ddf55af7bc191792b8aea0c2b94fb0fef5Christoph Paasch 968d23ff701643a4a725e2c7a8ba2d567d39daa29eaJulian Anastasovstatic int tcp_metrics_nl_cmd_get(struct sk_buff *skb, struct genl_info *info) 969d23ff701643a4a725e2c7a8ba2d567d39daa29eaJulian Anastasov{ 970d23ff701643a4a725e2c7a8ba2d567d39daa29eaJulian Anastasov struct tcp_metrics_block *tm; 9713e7013ddf55af7bc191792b8aea0c2b94fb0fef5Christoph Paasch struct inetpeer_addr saddr, daddr; 972d23ff701643a4a725e2c7a8ba2d567d39daa29eaJulian Anastasov unsigned int hash; 973d23ff701643a4a725e2c7a8ba2d567d39daa29eaJulian Anastasov struct sk_buff *msg; 974d23ff701643a4a725e2c7a8ba2d567d39daa29eaJulian Anastasov struct net *net = genl_info_net(info); 975d23ff701643a4a725e2c7a8ba2d567d39daa29eaJulian Anastasov void *reply; 976d23ff701643a4a725e2c7a8ba2d567d39daa29eaJulian Anastasov int ret; 9773e7013ddf55af7bc191792b8aea0c2b94fb0fef5Christoph Paasch bool src = true; 978d23ff701643a4a725e2c7a8ba2d567d39daa29eaJulian Anastasov 979324fd55a19827b7191cc6ab73865e30c0e6e6423Christoph Paasch ret = parse_nl_addr(info, &daddr, &hash, 0); 980d23ff701643a4a725e2c7a8ba2d567d39daa29eaJulian Anastasov if (ret < 0) 981d23ff701643a4a725e2c7a8ba2d567d39daa29eaJulian Anastasov return ret; 982d23ff701643a4a725e2c7a8ba2d567d39daa29eaJulian Anastasov 9833e7013ddf55af7bc191792b8aea0c2b94fb0fef5Christoph Paasch ret = parse_nl_saddr(info, &saddr); 9843e7013ddf55af7bc191792b8aea0c2b94fb0fef5Christoph Paasch if (ret < 0) 9853e7013ddf55af7bc191792b8aea0c2b94fb0fef5Christoph Paasch src = false; 9863e7013ddf55af7bc191792b8aea0c2b94fb0fef5Christoph Paasch 987d23ff701643a4a725e2c7a8ba2d567d39daa29eaJulian Anastasov msg = nlmsg_new(NLMSG_DEFAULT_SIZE, GFP_KERNEL); 988d23ff701643a4a725e2c7a8ba2d567d39daa29eaJulian Anastasov if (!msg) 989d23ff701643a4a725e2c7a8ba2d567d39daa29eaJulian Anastasov return -ENOMEM; 990d23ff701643a4a725e2c7a8ba2d567d39daa29eaJulian Anastasov 991d23ff701643a4a725e2c7a8ba2d567d39daa29eaJulian Anastasov reply = genlmsg_put_reply(msg, info, &tcp_metrics_nl_family, 0, 992d23ff701643a4a725e2c7a8ba2d567d39daa29eaJulian Anastasov info->genlhdr->cmd); 993d23ff701643a4a725e2c7a8ba2d567d39daa29eaJulian Anastasov if (!reply) 994d23ff701643a4a725e2c7a8ba2d567d39daa29eaJulian Anastasov goto nla_put_failure; 995d23ff701643a4a725e2c7a8ba2d567d39daa29eaJulian Anastasov 996d23ff701643a4a725e2c7a8ba2d567d39daa29eaJulian Anastasov hash = hash_32(hash, net->ipv4.tcp_metrics_hash_log); 997d23ff701643a4a725e2c7a8ba2d567d39daa29eaJulian Anastasov ret = -ESRCH; 998d23ff701643a4a725e2c7a8ba2d567d39daa29eaJulian Anastasov rcu_read_lock(); 999d23ff701643a4a725e2c7a8ba2d567d39daa29eaJulian Anastasov for (tm = rcu_dereference(net->ipv4.tcp_metrics_hash[hash].chain); tm; 1000d23ff701643a4a725e2c7a8ba2d567d39daa29eaJulian Anastasov tm = rcu_dereference(tm->tcpm_next)) { 10013e7013ddf55af7bc191792b8aea0c2b94fb0fef5Christoph Paasch if (addr_same(&tm->tcpm_daddr, &daddr) && 10023e7013ddf55af7bc191792b8aea0c2b94fb0fef5Christoph Paasch (!src || addr_same(&tm->tcpm_saddr, &saddr))) { 1003d23ff701643a4a725e2c7a8ba2d567d39daa29eaJulian Anastasov ret = tcp_metrics_fill_info(msg, tm); 1004d23ff701643a4a725e2c7a8ba2d567d39daa29eaJulian Anastasov break; 1005d23ff701643a4a725e2c7a8ba2d567d39daa29eaJulian Anastasov } 1006d23ff701643a4a725e2c7a8ba2d567d39daa29eaJulian Anastasov } 1007d23ff701643a4a725e2c7a8ba2d567d39daa29eaJulian Anastasov rcu_read_unlock(); 1008d23ff701643a4a725e2c7a8ba2d567d39daa29eaJulian Anastasov if (ret < 0) 1009d23ff701643a4a725e2c7a8ba2d567d39daa29eaJulian Anastasov goto out_free; 1010d23ff701643a4a725e2c7a8ba2d567d39daa29eaJulian Anastasov 1011d23ff701643a4a725e2c7a8ba2d567d39daa29eaJulian Anastasov genlmsg_end(msg, reply); 1012d23ff701643a4a725e2c7a8ba2d567d39daa29eaJulian Anastasov return genlmsg_reply(msg, info); 1013d23ff701643a4a725e2c7a8ba2d567d39daa29eaJulian Anastasov 1014d23ff701643a4a725e2c7a8ba2d567d39daa29eaJulian Anastasovnla_put_failure: 1015d23ff701643a4a725e2c7a8ba2d567d39daa29eaJulian Anastasov ret = -EMSGSIZE; 1016d23ff701643a4a725e2c7a8ba2d567d39daa29eaJulian Anastasov 1017d23ff701643a4a725e2c7a8ba2d567d39daa29eaJulian Anastasovout_free: 1018d23ff701643a4a725e2c7a8ba2d567d39daa29eaJulian Anastasov nlmsg_free(msg); 1019d23ff701643a4a725e2c7a8ba2d567d39daa29eaJulian Anastasov return ret; 1020d23ff701643a4a725e2c7a8ba2d567d39daa29eaJulian Anastasov} 1021d23ff701643a4a725e2c7a8ba2d567d39daa29eaJulian Anastasov 1022d23ff701643a4a725e2c7a8ba2d567d39daa29eaJulian Anastasov#define deref_locked_genl(p) \ 1023d23ff701643a4a725e2c7a8ba2d567d39daa29eaJulian Anastasov rcu_dereference_protected(p, lockdep_genl_is_held() && \ 1024d23ff701643a4a725e2c7a8ba2d567d39daa29eaJulian Anastasov lockdep_is_held(&tcp_metrics_lock)) 1025d23ff701643a4a725e2c7a8ba2d567d39daa29eaJulian Anastasov 1026d23ff701643a4a725e2c7a8ba2d567d39daa29eaJulian Anastasov#define deref_genl(p) rcu_dereference_protected(p, lockdep_genl_is_held()) 1027d23ff701643a4a725e2c7a8ba2d567d39daa29eaJulian Anastasov 1028d23ff701643a4a725e2c7a8ba2d567d39daa29eaJulian Anastasovstatic int tcp_metrics_flush_all(struct net *net) 1029d23ff701643a4a725e2c7a8ba2d567d39daa29eaJulian Anastasov{ 1030d23ff701643a4a725e2c7a8ba2d567d39daa29eaJulian Anastasov unsigned int max_rows = 1U << net->ipv4.tcp_metrics_hash_log; 1031d23ff701643a4a725e2c7a8ba2d567d39daa29eaJulian Anastasov struct tcpm_hash_bucket *hb = net->ipv4.tcp_metrics_hash; 1032d23ff701643a4a725e2c7a8ba2d567d39daa29eaJulian Anastasov struct tcp_metrics_block *tm; 1033d23ff701643a4a725e2c7a8ba2d567d39daa29eaJulian Anastasov unsigned int row; 1034d23ff701643a4a725e2c7a8ba2d567d39daa29eaJulian Anastasov 1035d23ff701643a4a725e2c7a8ba2d567d39daa29eaJulian Anastasov for (row = 0; row < max_rows; row++, hb++) { 1036d23ff701643a4a725e2c7a8ba2d567d39daa29eaJulian Anastasov spin_lock_bh(&tcp_metrics_lock); 1037d23ff701643a4a725e2c7a8ba2d567d39daa29eaJulian Anastasov tm = deref_locked_genl(hb->chain); 1038d23ff701643a4a725e2c7a8ba2d567d39daa29eaJulian Anastasov if (tm) 1039d23ff701643a4a725e2c7a8ba2d567d39daa29eaJulian Anastasov hb->chain = NULL; 1040d23ff701643a4a725e2c7a8ba2d567d39daa29eaJulian Anastasov spin_unlock_bh(&tcp_metrics_lock); 1041d23ff701643a4a725e2c7a8ba2d567d39daa29eaJulian Anastasov while (tm) { 1042d23ff701643a4a725e2c7a8ba2d567d39daa29eaJulian Anastasov struct tcp_metrics_block *next; 1043d23ff701643a4a725e2c7a8ba2d567d39daa29eaJulian Anastasov 1044d23ff701643a4a725e2c7a8ba2d567d39daa29eaJulian Anastasov next = deref_genl(tm->tcpm_next); 1045d23ff701643a4a725e2c7a8ba2d567d39daa29eaJulian Anastasov kfree_rcu(tm, rcu_head); 1046d23ff701643a4a725e2c7a8ba2d567d39daa29eaJulian Anastasov tm = next; 1047d23ff701643a4a725e2c7a8ba2d567d39daa29eaJulian Anastasov } 1048d23ff701643a4a725e2c7a8ba2d567d39daa29eaJulian Anastasov } 1049d23ff701643a4a725e2c7a8ba2d567d39daa29eaJulian Anastasov return 0; 1050d23ff701643a4a725e2c7a8ba2d567d39daa29eaJulian Anastasov} 1051d23ff701643a4a725e2c7a8ba2d567d39daa29eaJulian Anastasov 1052d23ff701643a4a725e2c7a8ba2d567d39daa29eaJulian Anastasovstatic int tcp_metrics_nl_cmd_del(struct sk_buff *skb, struct genl_info *info) 1053d23ff701643a4a725e2c7a8ba2d567d39daa29eaJulian Anastasov{ 1054d23ff701643a4a725e2c7a8ba2d567d39daa29eaJulian Anastasov struct tcpm_hash_bucket *hb; 105500ca9c5b2b11d44eaf20a4b647efc999734323ecChristoph Paasch struct tcp_metrics_block *tm; 1056d23ff701643a4a725e2c7a8ba2d567d39daa29eaJulian Anastasov struct tcp_metrics_block __rcu **pp; 10573e7013ddf55af7bc191792b8aea0c2b94fb0fef5Christoph Paasch struct inetpeer_addr saddr, daddr; 1058d23ff701643a4a725e2c7a8ba2d567d39daa29eaJulian Anastasov unsigned int hash; 1059d23ff701643a4a725e2c7a8ba2d567d39daa29eaJulian Anastasov struct net *net = genl_info_net(info); 1060d23ff701643a4a725e2c7a8ba2d567d39daa29eaJulian Anastasov int ret; 106100ca9c5b2b11d44eaf20a4b647efc999734323ecChristoph Paasch bool src = true, found = false; 1062d23ff701643a4a725e2c7a8ba2d567d39daa29eaJulian Anastasov 1063324fd55a19827b7191cc6ab73865e30c0e6e6423Christoph Paasch ret = parse_nl_addr(info, &daddr, &hash, 1); 1064d23ff701643a4a725e2c7a8ba2d567d39daa29eaJulian Anastasov if (ret < 0) 1065d23ff701643a4a725e2c7a8ba2d567d39daa29eaJulian Anastasov return ret; 1066d23ff701643a4a725e2c7a8ba2d567d39daa29eaJulian Anastasov if (ret > 0) 1067d23ff701643a4a725e2c7a8ba2d567d39daa29eaJulian Anastasov return tcp_metrics_flush_all(net); 10683e7013ddf55af7bc191792b8aea0c2b94fb0fef5Christoph Paasch ret = parse_nl_saddr(info, &saddr); 10693e7013ddf55af7bc191792b8aea0c2b94fb0fef5Christoph Paasch if (ret < 0) 10703e7013ddf55af7bc191792b8aea0c2b94fb0fef5Christoph Paasch src = false; 1071d23ff701643a4a725e2c7a8ba2d567d39daa29eaJulian Anastasov 1072d23ff701643a4a725e2c7a8ba2d567d39daa29eaJulian Anastasov hash = hash_32(hash, net->ipv4.tcp_metrics_hash_log); 1073d23ff701643a4a725e2c7a8ba2d567d39daa29eaJulian Anastasov hb = net->ipv4.tcp_metrics_hash + hash; 1074d23ff701643a4a725e2c7a8ba2d567d39daa29eaJulian Anastasov pp = &hb->chain; 1075d23ff701643a4a725e2c7a8ba2d567d39daa29eaJulian Anastasov spin_lock_bh(&tcp_metrics_lock); 1076bbf852b96ebdc6d1be7a67143824523280bbcf44Christoph Paasch for (tm = deref_locked_genl(*pp); tm; tm = deref_locked_genl(*pp)) { 10773e7013ddf55af7bc191792b8aea0c2b94fb0fef5Christoph Paasch if (addr_same(&tm->tcpm_daddr, &daddr) && 10783e7013ddf55af7bc191792b8aea0c2b94fb0fef5Christoph Paasch (!src || addr_same(&tm->tcpm_saddr, &saddr))) { 1079d23ff701643a4a725e2c7a8ba2d567d39daa29eaJulian Anastasov *pp = tm->tcpm_next; 108000ca9c5b2b11d44eaf20a4b647efc999734323ecChristoph Paasch kfree_rcu(tm, rcu_head); 108100ca9c5b2b11d44eaf20a4b647efc999734323ecChristoph Paasch found = true; 1082bbf852b96ebdc6d1be7a67143824523280bbcf44Christoph Paasch } else { 1083bbf852b96ebdc6d1be7a67143824523280bbcf44Christoph Paasch pp = &tm->tcpm_next; 1084d23ff701643a4a725e2c7a8ba2d567d39daa29eaJulian Anastasov } 1085d23ff701643a4a725e2c7a8ba2d567d39daa29eaJulian Anastasov } 1086d23ff701643a4a725e2c7a8ba2d567d39daa29eaJulian Anastasov spin_unlock_bh(&tcp_metrics_lock); 108700ca9c5b2b11d44eaf20a4b647efc999734323ecChristoph Paasch if (!found) 1088d23ff701643a4a725e2c7a8ba2d567d39daa29eaJulian Anastasov return -ESRCH; 1089d23ff701643a4a725e2c7a8ba2d567d39daa29eaJulian Anastasov return 0; 1090d23ff701643a4a725e2c7a8ba2d567d39daa29eaJulian Anastasov} 1091d23ff701643a4a725e2c7a8ba2d567d39daa29eaJulian Anastasov 10924534de8305b3f1460a527a0cda0e3dc2224c6f0cJohannes Bergstatic const struct genl_ops tcp_metrics_nl_ops[] = { 1093d23ff701643a4a725e2c7a8ba2d567d39daa29eaJulian Anastasov { 1094d23ff701643a4a725e2c7a8ba2d567d39daa29eaJulian Anastasov .cmd = TCP_METRICS_CMD_GET, 1095d23ff701643a4a725e2c7a8ba2d567d39daa29eaJulian Anastasov .doit = tcp_metrics_nl_cmd_get, 1096d23ff701643a4a725e2c7a8ba2d567d39daa29eaJulian Anastasov .dumpit = tcp_metrics_nl_dump, 1097d23ff701643a4a725e2c7a8ba2d567d39daa29eaJulian Anastasov .policy = tcp_metrics_nl_policy, 1098d23ff701643a4a725e2c7a8ba2d567d39daa29eaJulian Anastasov }, 1099d23ff701643a4a725e2c7a8ba2d567d39daa29eaJulian Anastasov { 1100d23ff701643a4a725e2c7a8ba2d567d39daa29eaJulian Anastasov .cmd = TCP_METRICS_CMD_DEL, 1101d23ff701643a4a725e2c7a8ba2d567d39daa29eaJulian Anastasov .doit = tcp_metrics_nl_cmd_del, 1102d23ff701643a4a725e2c7a8ba2d567d39daa29eaJulian Anastasov .policy = tcp_metrics_nl_policy, 1103d23ff701643a4a725e2c7a8ba2d567d39daa29eaJulian Anastasov .flags = GENL_ADMIN_PERM, 1104d23ff701643a4a725e2c7a8ba2d567d39daa29eaJulian Anastasov }, 1105d23ff701643a4a725e2c7a8ba2d567d39daa29eaJulian Anastasov}; 1106d23ff701643a4a725e2c7a8ba2d567d39daa29eaJulian Anastasov 11075815d5e7aae3cc9c5e85af83094d4d6498c3f4fcEric Dumazetstatic unsigned int tcpmhash_entries; 110851c5d0c4b169bf762f09e0d5b283a7f0b2a45739David S. Millerstatic int __init set_tcpmhash_entries(char *str) 110951c5d0c4b169bf762f09e0d5b283a7f0b2a45739David S. Miller{ 111051c5d0c4b169bf762f09e0d5b283a7f0b2a45739David S. Miller ssize_t ret; 111151c5d0c4b169bf762f09e0d5b283a7f0b2a45739David S. Miller 111251c5d0c4b169bf762f09e0d5b283a7f0b2a45739David S. Miller if (!str) 111351c5d0c4b169bf762f09e0d5b283a7f0b2a45739David S. Miller return 0; 111451c5d0c4b169bf762f09e0d5b283a7f0b2a45739David S. Miller 11155815d5e7aae3cc9c5e85af83094d4d6498c3f4fcEric Dumazet ret = kstrtouint(str, 0, &tcpmhash_entries); 111651c5d0c4b169bf762f09e0d5b283a7f0b2a45739David S. Miller if (ret) 111751c5d0c4b169bf762f09e0d5b283a7f0b2a45739David S. Miller return 0; 111851c5d0c4b169bf762f09e0d5b283a7f0b2a45739David S. Miller 111951c5d0c4b169bf762f09e0d5b283a7f0b2a45739David S. Miller return 1; 112051c5d0c4b169bf762f09e0d5b283a7f0b2a45739David S. Miller} 112151c5d0c4b169bf762f09e0d5b283a7f0b2a45739David S. Miller__setup("tcpmhash_entries=", set_tcpmhash_entries); 112251c5d0c4b169bf762f09e0d5b283a7f0b2a45739David S. Miller 112351c5d0c4b169bf762f09e0d5b283a7f0b2a45739David S. Millerstatic int __net_init tcp_net_metrics_init(struct net *net) 112451c5d0c4b169bf762f09e0d5b283a7f0b2a45739David S. Miller{ 11255815d5e7aae3cc9c5e85af83094d4d6498c3f4fcEric Dumazet size_t size; 11265815d5e7aae3cc9c5e85af83094d4d6498c3f4fcEric Dumazet unsigned int slots; 112751c5d0c4b169bf762f09e0d5b283a7f0b2a45739David S. Miller 112851c5d0c4b169bf762f09e0d5b283a7f0b2a45739David S. Miller slots = tcpmhash_entries; 112951c5d0c4b169bf762f09e0d5b283a7f0b2a45739David S. Miller if (!slots) { 113051c5d0c4b169bf762f09e0d5b283a7f0b2a45739David S. Miller if (totalram_pages >= 128 * 1024) 113151c5d0c4b169bf762f09e0d5b283a7f0b2a45739David S. Miller slots = 16 * 1024; 113251c5d0c4b169bf762f09e0d5b283a7f0b2a45739David S. Miller else 113351c5d0c4b169bf762f09e0d5b283a7f0b2a45739David S. Miller slots = 8 * 1024; 113451c5d0c4b169bf762f09e0d5b283a7f0b2a45739David S. Miller } 113551c5d0c4b169bf762f09e0d5b283a7f0b2a45739David S. Miller 11365815d5e7aae3cc9c5e85af83094d4d6498c3f4fcEric Dumazet net->ipv4.tcp_metrics_hash_log = order_base_2(slots); 11375815d5e7aae3cc9c5e85af83094d4d6498c3f4fcEric Dumazet size = sizeof(struct tcpm_hash_bucket) << net->ipv4.tcp_metrics_hash_log; 113851c5d0c4b169bf762f09e0d5b283a7f0b2a45739David S. Miller 1139976a702ac9eeacea09e588456ab165dc06f9ee83Eric Dumazet net->ipv4.tcp_metrics_hash = kzalloc(size, GFP_KERNEL | __GFP_NOWARN); 1140976a702ac9eeacea09e588456ab165dc06f9ee83Eric Dumazet if (!net->ipv4.tcp_metrics_hash) 1141976a702ac9eeacea09e588456ab165dc06f9ee83Eric Dumazet net->ipv4.tcp_metrics_hash = vzalloc(size); 1142976a702ac9eeacea09e588456ab165dc06f9ee83Eric Dumazet 114351c5d0c4b169bf762f09e0d5b283a7f0b2a45739David S. Miller if (!net->ipv4.tcp_metrics_hash) 114451c5d0c4b169bf762f09e0d5b283a7f0b2a45739David S. Miller return -ENOMEM; 114551c5d0c4b169bf762f09e0d5b283a7f0b2a45739David S. Miller 114651c5d0c4b169bf762f09e0d5b283a7f0b2a45739David S. Miller return 0; 114751c5d0c4b169bf762f09e0d5b283a7f0b2a45739David S. Miller} 114851c5d0c4b169bf762f09e0d5b283a7f0b2a45739David S. Miller 114951c5d0c4b169bf762f09e0d5b283a7f0b2a45739David S. Millerstatic void __net_exit tcp_net_metrics_exit(struct net *net) 115051c5d0c4b169bf762f09e0d5b283a7f0b2a45739David S. Miller{ 115136471012e2ae28ca3178f84d4687a2d88a36593eEric Dumazet unsigned int i; 115236471012e2ae28ca3178f84d4687a2d88a36593eEric Dumazet 115336471012e2ae28ca3178f84d4687a2d88a36593eEric Dumazet for (i = 0; i < (1U << net->ipv4.tcp_metrics_hash_log) ; i++) { 115436471012e2ae28ca3178f84d4687a2d88a36593eEric Dumazet struct tcp_metrics_block *tm, *next; 115536471012e2ae28ca3178f84d4687a2d88a36593eEric Dumazet 115636471012e2ae28ca3178f84d4687a2d88a36593eEric Dumazet tm = rcu_dereference_protected(net->ipv4.tcp_metrics_hash[i].chain, 1); 115736471012e2ae28ca3178f84d4687a2d88a36593eEric Dumazet while (tm) { 115836471012e2ae28ca3178f84d4687a2d88a36593eEric Dumazet next = rcu_dereference_protected(tm->tcpm_next, 1); 115936471012e2ae28ca3178f84d4687a2d88a36593eEric Dumazet kfree(tm); 116036471012e2ae28ca3178f84d4687a2d88a36593eEric Dumazet tm = next; 116136471012e2ae28ca3178f84d4687a2d88a36593eEric Dumazet } 116236471012e2ae28ca3178f84d4687a2d88a36593eEric Dumazet } 11634cb28970a23ff209199b0a4358d68efe82c8f493WANG Cong kvfree(net->ipv4.tcp_metrics_hash); 116451c5d0c4b169bf762f09e0d5b283a7f0b2a45739David S. Miller} 116551c5d0c4b169bf762f09e0d5b283a7f0b2a45739David S. Miller 116651c5d0c4b169bf762f09e0d5b283a7f0b2a45739David S. Millerstatic __net_initdata struct pernet_operations tcp_net_metrics_ops = { 116751c5d0c4b169bf762f09e0d5b283a7f0b2a45739David S. Miller .init = tcp_net_metrics_init, 116851c5d0c4b169bf762f09e0d5b283a7f0b2a45739David S. Miller .exit = tcp_net_metrics_exit, 116951c5d0c4b169bf762f09e0d5b283a7f0b2a45739David S. Miller}; 117051c5d0c4b169bf762f09e0d5b283a7f0b2a45739David S. Miller 117151c5d0c4b169bf762f09e0d5b283a7f0b2a45739David S. Millervoid __init tcp_metrics_init(void) 117251c5d0c4b169bf762f09e0d5b283a7f0b2a45739David S. Miller{ 1173d23ff701643a4a725e2c7a8ba2d567d39daa29eaJulian Anastasov int ret; 1174d23ff701643a4a725e2c7a8ba2d567d39daa29eaJulian Anastasov 1175d23ff701643a4a725e2c7a8ba2d567d39daa29eaJulian Anastasov ret = register_pernet_subsys(&tcp_net_metrics_ops); 1176d23ff701643a4a725e2c7a8ba2d567d39daa29eaJulian Anastasov if (ret < 0) 1177d23ff701643a4a725e2c7a8ba2d567d39daa29eaJulian Anastasov goto cleanup; 1178d23ff701643a4a725e2c7a8ba2d567d39daa29eaJulian Anastasov ret = genl_register_family_with_ops(&tcp_metrics_nl_family, 1179c53ed7423619b4e8108914a9f31b426dd58ad591Johannes Berg tcp_metrics_nl_ops); 1180d23ff701643a4a725e2c7a8ba2d567d39daa29eaJulian Anastasov if (ret < 0) 1181d23ff701643a4a725e2c7a8ba2d567d39daa29eaJulian Anastasov goto cleanup_subsys; 1182d23ff701643a4a725e2c7a8ba2d567d39daa29eaJulian Anastasov return; 1183d23ff701643a4a725e2c7a8ba2d567d39daa29eaJulian Anastasov 1184d23ff701643a4a725e2c7a8ba2d567d39daa29eaJulian Anastasovcleanup_subsys: 1185d23ff701643a4a725e2c7a8ba2d567d39daa29eaJulian Anastasov unregister_pernet_subsys(&tcp_net_metrics_ops); 1186d23ff701643a4a725e2c7a8ba2d567d39daa29eaJulian Anastasov 1187d23ff701643a4a725e2c7a8ba2d567d39daa29eaJulian Anastasovcleanup: 1188d23ff701643a4a725e2c7a8ba2d567d39daa29eaJulian Anastasov return; 118951c5d0c4b169bf762f09e0d5b283a7f0b2a45739David S. Miller} 1190