nf_conntrack_core.c revision 15f585bd76b6bd2974b23c9e69ff038a0826a0be
1/* Connection state tracking for netfilter.  This is separated from,
2   but required by, the NAT layer; it can also be used by an iptables
3   extension. */
4
5/* (C) 1999-2001 Paul `Rusty' Russell
6 * (C) 2002-2006 Netfilter Core Team <coreteam@netfilter.org>
7 * (C) 2003,2004 USAGI/WIDE Project <http://www.linux-ipv6.org>
8 *
9 * This program is free software; you can redistribute it and/or modify
10 * it under the terms of the GNU General Public License version 2 as
11 * published by the Free Software Foundation.
12 */
13
14#include <linux/types.h>
15#include <linux/netfilter.h>
16#include <linux/module.h>
17#include <linux/sched.h>
18#include <linux/skbuff.h>
19#include <linux/proc_fs.h>
20#include <linux/vmalloc.h>
21#include <linux/stddef.h>
22#include <linux/slab.h>
23#include <linux/random.h>
24#include <linux/jhash.h>
25#include <linux/err.h>
26#include <linux/percpu.h>
27#include <linux/moduleparam.h>
28#include <linux/notifier.h>
29#include <linux/kernel.h>
30#include <linux/netdevice.h>
31#include <linux/socket.h>
32#include <linux/mm.h>
33#include <linux/nsproxy.h>
34#include <linux/rculist_nulls.h>
35
36#include <net/netfilter/nf_conntrack.h>
37#include <net/netfilter/nf_conntrack_l3proto.h>
38#include <net/netfilter/nf_conntrack_l4proto.h>
39#include <net/netfilter/nf_conntrack_expect.h>
40#include <net/netfilter/nf_conntrack_helper.h>
41#include <net/netfilter/nf_conntrack_core.h>
42#include <net/netfilter/nf_conntrack_extend.h>
43#include <net/netfilter/nf_conntrack_acct.h>
44#include <net/netfilter/nf_conntrack_ecache.h>
45#include <net/netfilter/nf_conntrack_zones.h>
46#include <net/netfilter/nf_conntrack_timestamp.h>
47#include <net/netfilter/nf_conntrack_timeout.h>
48#include <net/netfilter/nf_nat.h>
49#include <net/netfilter/nf_nat_core.h>
50
51#define NF_CONNTRACK_VERSION	"0.5.0"
52
53int (*nfnetlink_parse_nat_setup_hook)(struct nf_conn *ct,
54				      enum nf_nat_manip_type manip,
55				      const struct nlattr *attr) __read_mostly;
56EXPORT_SYMBOL_GPL(nfnetlink_parse_nat_setup_hook);
57
58DEFINE_SPINLOCK(nf_conntrack_lock);
59EXPORT_SYMBOL_GPL(nf_conntrack_lock);
60
61unsigned int nf_conntrack_htable_size __read_mostly;
62EXPORT_SYMBOL_GPL(nf_conntrack_htable_size);
63
64unsigned int nf_conntrack_max __read_mostly;
65EXPORT_SYMBOL_GPL(nf_conntrack_max);
66
67DEFINE_PER_CPU(struct nf_conn, nf_conntrack_untracked);
68EXPORT_PER_CPU_SYMBOL(nf_conntrack_untracked);
69
70unsigned int nf_conntrack_hash_rnd __read_mostly;
71EXPORT_SYMBOL_GPL(nf_conntrack_hash_rnd);
72
73static u32 hash_conntrack_raw(const struct nf_conntrack_tuple *tuple, u16 zone)
74{
75	unsigned int n;
76
77	/* The direction must be ignored, so we hash everything up to the
78	 * destination ports (which is a multiple of 4) and treat the last
79	 * three bytes manually.
80	 */
81	n = (sizeof(tuple->src) + sizeof(tuple->dst.u3)) / sizeof(u32);
82	return jhash2((u32 *)tuple, n, zone ^ nf_conntrack_hash_rnd ^
83		      (((__force __u16)tuple->dst.u.all << 16) |
84		      tuple->dst.protonum));
85}
86
87static u32 __hash_bucket(u32 hash, unsigned int size)
88{
89	return ((u64)hash * size) >> 32;
90}
91
92static u32 hash_bucket(u32 hash, const struct net *net)
93{
94	return __hash_bucket(hash, net->ct.htable_size);
95}
96
97static u_int32_t __hash_conntrack(const struct nf_conntrack_tuple *tuple,
98				  u16 zone, unsigned int size)
99{
100	return __hash_bucket(hash_conntrack_raw(tuple, zone), size);
101}
102
103static inline u_int32_t hash_conntrack(const struct net *net, u16 zone,
104				       const struct nf_conntrack_tuple *tuple)
105{
106	return __hash_conntrack(tuple, zone, net->ct.htable_size);
107}
108
109bool
110nf_ct_get_tuple(const struct sk_buff *skb,
111		unsigned int nhoff,
112		unsigned int dataoff,
113		u_int16_t l3num,
114		u_int8_t protonum,
115		struct nf_conntrack_tuple *tuple,
116		const struct nf_conntrack_l3proto *l3proto,
117		const struct nf_conntrack_l4proto *l4proto)
118{
119	memset(tuple, 0, sizeof(*tuple));
120
121	tuple->src.l3num = l3num;
122	if (l3proto->pkt_to_tuple(skb, nhoff, tuple) == 0)
123		return false;
124
125	tuple->dst.protonum = protonum;
126	tuple->dst.dir = IP_CT_DIR_ORIGINAL;
127
128	return l4proto->pkt_to_tuple(skb, dataoff, tuple);
129}
130EXPORT_SYMBOL_GPL(nf_ct_get_tuple);
131
132bool nf_ct_get_tuplepr(const struct sk_buff *skb, unsigned int nhoff,
133		       u_int16_t l3num, struct nf_conntrack_tuple *tuple)
134{
135	struct nf_conntrack_l3proto *l3proto;
136	struct nf_conntrack_l4proto *l4proto;
137	unsigned int protoff;
138	u_int8_t protonum;
139	int ret;
140
141	rcu_read_lock();
142
143	l3proto = __nf_ct_l3proto_find(l3num);
144	ret = l3proto->get_l4proto(skb, nhoff, &protoff, &protonum);
145	if (ret != NF_ACCEPT) {
146		rcu_read_unlock();
147		return false;
148	}
149
150	l4proto = __nf_ct_l4proto_find(l3num, protonum);
151
152	ret = nf_ct_get_tuple(skb, nhoff, protoff, l3num, protonum, tuple,
153			      l3proto, l4proto);
154
155	rcu_read_unlock();
156	return ret;
157}
158EXPORT_SYMBOL_GPL(nf_ct_get_tuplepr);
159
160bool
161nf_ct_invert_tuple(struct nf_conntrack_tuple *inverse,
162		   const struct nf_conntrack_tuple *orig,
163		   const struct nf_conntrack_l3proto *l3proto,
164		   const struct nf_conntrack_l4proto *l4proto)
165{
166	memset(inverse, 0, sizeof(*inverse));
167
168	inverse->src.l3num = orig->src.l3num;
169	if (l3proto->invert_tuple(inverse, orig) == 0)
170		return false;
171
172	inverse->dst.dir = !orig->dst.dir;
173
174	inverse->dst.protonum = orig->dst.protonum;
175	return l4proto->invert_tuple(inverse, orig);
176}
177EXPORT_SYMBOL_GPL(nf_ct_invert_tuple);
178
179static void
180clean_from_lists(struct nf_conn *ct)
181{
182	pr_debug("clean_from_lists(%p)\n", ct);
183	hlist_nulls_del_rcu(&ct->tuplehash[IP_CT_DIR_ORIGINAL].hnnode);
184	hlist_nulls_del_rcu(&ct->tuplehash[IP_CT_DIR_REPLY].hnnode);
185
186	/* Destroy all pending expectations */
187	nf_ct_remove_expectations(ct);
188}
189
190static void
191destroy_conntrack(struct nf_conntrack *nfct)
192{
193	struct nf_conn *ct = (struct nf_conn *)nfct;
194	struct net *net = nf_ct_net(ct);
195	struct nf_conntrack_l4proto *l4proto;
196
197	pr_debug("destroy_conntrack(%p)\n", ct);
198	NF_CT_ASSERT(atomic_read(&nfct->use) == 0);
199	NF_CT_ASSERT(!timer_pending(&ct->timeout));
200
201	/* To make sure we don't get any weird locking issues here:
202	 * destroy_conntrack() MUST NOT be called with a write lock
203	 * to nf_conntrack_lock!!! -HW */
204	rcu_read_lock();
205	l4proto = __nf_ct_l4proto_find(nf_ct_l3num(ct), nf_ct_protonum(ct));
206	if (l4proto && l4proto->destroy)
207		l4proto->destroy(ct);
208
209	rcu_read_unlock();
210
211	spin_lock_bh(&nf_conntrack_lock);
212	/* Expectations will have been removed in clean_from_lists,
213	 * except TFTP can create an expectation on the first packet,
214	 * before connection is in the list, so we need to clean here,
215	 * too. */
216	nf_ct_remove_expectations(ct);
217
218	/* We overload first tuple to link into unconfirmed list. */
219	if (!nf_ct_is_confirmed(ct)) {
220		BUG_ON(hlist_nulls_unhashed(&ct->tuplehash[IP_CT_DIR_ORIGINAL].hnnode));
221		hlist_nulls_del_rcu(&ct->tuplehash[IP_CT_DIR_ORIGINAL].hnnode);
222	}
223
224	NF_CT_STAT_INC(net, delete);
225	spin_unlock_bh(&nf_conntrack_lock);
226
227	if (ct->master)
228		nf_ct_put(ct->master);
229
230	pr_debug("destroy_conntrack: returning ct=%p to slab\n", ct);
231	nf_conntrack_free(ct);
232}
233
234void nf_ct_delete_from_lists(struct nf_conn *ct)
235{
236	struct net *net = nf_ct_net(ct);
237
238	nf_ct_helper_destroy(ct);
239	spin_lock_bh(&nf_conntrack_lock);
240	/* Inside lock so preempt is disabled on module removal path.
241	 * Otherwise we can get spurious warnings. */
242	NF_CT_STAT_INC(net, delete_list);
243	clean_from_lists(ct);
244	spin_unlock_bh(&nf_conntrack_lock);
245}
246EXPORT_SYMBOL_GPL(nf_ct_delete_from_lists);
247
248static void death_by_event(unsigned long ul_conntrack)
249{
250	struct nf_conn *ct = (void *)ul_conntrack;
251	struct net *net = nf_ct_net(ct);
252
253	if (nf_conntrack_event(IPCT_DESTROY, ct) < 0) {
254		/* bad luck, let's retry again */
255		ct->timeout.expires = jiffies +
256			(random32() % net->ct.sysctl_events_retry_timeout);
257		add_timer(&ct->timeout);
258		return;
259	}
260	/* we've got the event delivered, now it's dying */
261	set_bit(IPS_DYING_BIT, &ct->status);
262	spin_lock(&nf_conntrack_lock);
263	hlist_nulls_del(&ct->tuplehash[IP_CT_DIR_ORIGINAL].hnnode);
264	spin_unlock(&nf_conntrack_lock);
265	nf_ct_put(ct);
266}
267
268void nf_ct_insert_dying_list(struct nf_conn *ct)
269{
270	struct net *net = nf_ct_net(ct);
271
272	/* add this conntrack to the dying list */
273	spin_lock_bh(&nf_conntrack_lock);
274	hlist_nulls_add_head(&ct->tuplehash[IP_CT_DIR_ORIGINAL].hnnode,
275			     &net->ct.dying);
276	spin_unlock_bh(&nf_conntrack_lock);
277	/* set a new timer to retry event delivery */
278	setup_timer(&ct->timeout, death_by_event, (unsigned long)ct);
279	ct->timeout.expires = jiffies +
280		(random32() % net->ct.sysctl_events_retry_timeout);
281	add_timer(&ct->timeout);
282}
283EXPORT_SYMBOL_GPL(nf_ct_insert_dying_list);
284
285static void death_by_timeout(unsigned long ul_conntrack)
286{
287	struct nf_conn *ct = (void *)ul_conntrack;
288	struct nf_conn_tstamp *tstamp;
289
290	tstamp = nf_conn_tstamp_find(ct);
291	if (tstamp && tstamp->stop == 0)
292		tstamp->stop = ktime_to_ns(ktime_get_real());
293
294	if (!test_bit(IPS_DYING_BIT, &ct->status) &&
295	    unlikely(nf_conntrack_event(IPCT_DESTROY, ct) < 0)) {
296		/* destroy event was not delivered */
297		nf_ct_delete_from_lists(ct);
298		nf_ct_insert_dying_list(ct);
299		return;
300	}
301	set_bit(IPS_DYING_BIT, &ct->status);
302	nf_ct_delete_from_lists(ct);
303	nf_ct_put(ct);
304}
305
306/*
307 * Warning :
308 * - Caller must take a reference on returned object
309 *   and recheck nf_ct_tuple_equal(tuple, &h->tuple)
310 * OR
311 * - Caller must lock nf_conntrack_lock before calling this function
312 */
313static struct nf_conntrack_tuple_hash *
314____nf_conntrack_find(struct net *net, u16 zone,
315		      const struct nf_conntrack_tuple *tuple, u32 hash)
316{
317	struct nf_conntrack_tuple_hash *h;
318	struct hlist_nulls_node *n;
319	unsigned int bucket = hash_bucket(hash, net);
320
321	/* Disable BHs the entire time since we normally need to disable them
322	 * at least once for the stats anyway.
323	 */
324	local_bh_disable();
325begin:
326	hlist_nulls_for_each_entry_rcu(h, n, &net->ct.hash[bucket], hnnode) {
327		if (nf_ct_tuple_equal(tuple, &h->tuple) &&
328		    nf_ct_zone(nf_ct_tuplehash_to_ctrack(h)) == zone) {
329			NF_CT_STAT_INC(net, found);
330			local_bh_enable();
331			return h;
332		}
333		NF_CT_STAT_INC(net, searched);
334	}
335	/*
336	 * if the nulls value we got at the end of this lookup is
337	 * not the expected one, we must restart lookup.
338	 * We probably met an item that was moved to another chain.
339	 */
340	if (get_nulls_value(n) != bucket) {
341		NF_CT_STAT_INC(net, search_restart);
342		goto begin;
343	}
344	local_bh_enable();
345
346	return NULL;
347}
348
349struct nf_conntrack_tuple_hash *
350__nf_conntrack_find(struct net *net, u16 zone,
351		    const struct nf_conntrack_tuple *tuple)
352{
353	return ____nf_conntrack_find(net, zone, tuple,
354				     hash_conntrack_raw(tuple, zone));
355}
356EXPORT_SYMBOL_GPL(__nf_conntrack_find);
357
358/* Find a connection corresponding to a tuple. */
359static struct nf_conntrack_tuple_hash *
360__nf_conntrack_find_get(struct net *net, u16 zone,
361			const struct nf_conntrack_tuple *tuple, u32 hash)
362{
363	struct nf_conntrack_tuple_hash *h;
364	struct nf_conn *ct;
365
366	rcu_read_lock();
367begin:
368	h = ____nf_conntrack_find(net, zone, tuple, hash);
369	if (h) {
370		ct = nf_ct_tuplehash_to_ctrack(h);
371		if (unlikely(nf_ct_is_dying(ct) ||
372			     !atomic_inc_not_zero(&ct->ct_general.use)))
373			h = NULL;
374		else {
375			if (unlikely(!nf_ct_tuple_equal(tuple, &h->tuple) ||
376				     nf_ct_zone(ct) != zone)) {
377				nf_ct_put(ct);
378				goto begin;
379			}
380		}
381	}
382	rcu_read_unlock();
383
384	return h;
385}
386
387struct nf_conntrack_tuple_hash *
388nf_conntrack_find_get(struct net *net, u16 zone,
389		      const struct nf_conntrack_tuple *tuple)
390{
391	return __nf_conntrack_find_get(net, zone, tuple,
392				       hash_conntrack_raw(tuple, zone));
393}
394EXPORT_SYMBOL_GPL(nf_conntrack_find_get);
395
396static void __nf_conntrack_hash_insert(struct nf_conn *ct,
397				       unsigned int hash,
398				       unsigned int repl_hash)
399{
400	struct net *net = nf_ct_net(ct);
401
402	hlist_nulls_add_head_rcu(&ct->tuplehash[IP_CT_DIR_ORIGINAL].hnnode,
403			   &net->ct.hash[hash]);
404	hlist_nulls_add_head_rcu(&ct->tuplehash[IP_CT_DIR_REPLY].hnnode,
405			   &net->ct.hash[repl_hash]);
406}
407
408int
409nf_conntrack_hash_check_insert(struct nf_conn *ct)
410{
411	struct net *net = nf_ct_net(ct);
412	unsigned int hash, repl_hash;
413	struct nf_conntrack_tuple_hash *h;
414	struct hlist_nulls_node *n;
415	u16 zone;
416
417	zone = nf_ct_zone(ct);
418	hash = hash_conntrack(net, zone,
419			      &ct->tuplehash[IP_CT_DIR_ORIGINAL].tuple);
420	repl_hash = hash_conntrack(net, zone,
421				   &ct->tuplehash[IP_CT_DIR_REPLY].tuple);
422
423	spin_lock_bh(&nf_conntrack_lock);
424
425	/* See if there's one in the list already, including reverse */
426	hlist_nulls_for_each_entry(h, n, &net->ct.hash[hash], hnnode)
427		if (nf_ct_tuple_equal(&ct->tuplehash[IP_CT_DIR_ORIGINAL].tuple,
428				      &h->tuple) &&
429		    zone == nf_ct_zone(nf_ct_tuplehash_to_ctrack(h)))
430			goto out;
431	hlist_nulls_for_each_entry(h, n, &net->ct.hash[repl_hash], hnnode)
432		if (nf_ct_tuple_equal(&ct->tuplehash[IP_CT_DIR_REPLY].tuple,
433				      &h->tuple) &&
434		    zone == nf_ct_zone(nf_ct_tuplehash_to_ctrack(h)))
435			goto out;
436
437	add_timer(&ct->timeout);
438	nf_conntrack_get(&ct->ct_general);
439	__nf_conntrack_hash_insert(ct, hash, repl_hash);
440	NF_CT_STAT_INC(net, insert);
441	spin_unlock_bh(&nf_conntrack_lock);
442
443	return 0;
444
445out:
446	NF_CT_STAT_INC(net, insert_failed);
447	spin_unlock_bh(&nf_conntrack_lock);
448	return -EEXIST;
449}
450EXPORT_SYMBOL_GPL(nf_conntrack_hash_check_insert);
451
452/* Confirm a connection given skb; places it in hash table */
453int
454__nf_conntrack_confirm(struct sk_buff *skb)
455{
456	unsigned int hash, repl_hash;
457	struct nf_conntrack_tuple_hash *h;
458	struct nf_conn *ct;
459	struct nf_conn_help *help;
460	struct nf_conn_tstamp *tstamp;
461	struct hlist_nulls_node *n;
462	enum ip_conntrack_info ctinfo;
463	struct net *net;
464	u16 zone;
465
466	ct = nf_ct_get(skb, &ctinfo);
467	net = nf_ct_net(ct);
468
469	/* ipt_REJECT uses nf_conntrack_attach to attach related
470	   ICMP/TCP RST packets in other direction.  Actual packet
471	   which created connection will be IP_CT_NEW or for an
472	   expected connection, IP_CT_RELATED. */
473	if (CTINFO2DIR(ctinfo) != IP_CT_DIR_ORIGINAL)
474		return NF_ACCEPT;
475
476	zone = nf_ct_zone(ct);
477	/* reuse the hash saved before */
478	hash = *(unsigned long *)&ct->tuplehash[IP_CT_DIR_REPLY].hnnode.pprev;
479	hash = hash_bucket(hash, net);
480	repl_hash = hash_conntrack(net, zone,
481				   &ct->tuplehash[IP_CT_DIR_REPLY].tuple);
482
483	/* We're not in hash table, and we refuse to set up related
484	   connections for unconfirmed conns.  But packet copies and
485	   REJECT will give spurious warnings here. */
486	/* NF_CT_ASSERT(atomic_read(&ct->ct_general.use) == 1); */
487
488	/* No external references means no one else could have
489	   confirmed us. */
490	NF_CT_ASSERT(!nf_ct_is_confirmed(ct));
491	pr_debug("Confirming conntrack %p\n", ct);
492
493	spin_lock_bh(&nf_conntrack_lock);
494
495	/* We have to check the DYING flag inside the lock to prevent
496	   a race against nf_ct_get_next_corpse() possibly called from
497	   user context, else we insert an already 'dead' hash, blocking
498	   further use of that particular connection -JM */
499
500	if (unlikely(nf_ct_is_dying(ct))) {
501		spin_unlock_bh(&nf_conntrack_lock);
502		return NF_ACCEPT;
503	}
504
505	/* See if there's one in the list already, including reverse:
506	   NAT could have grabbed it without realizing, since we're
507	   not in the hash.  If there is, we lost race. */
508	hlist_nulls_for_each_entry(h, n, &net->ct.hash[hash], hnnode)
509		if (nf_ct_tuple_equal(&ct->tuplehash[IP_CT_DIR_ORIGINAL].tuple,
510				      &h->tuple) &&
511		    zone == nf_ct_zone(nf_ct_tuplehash_to_ctrack(h)))
512			goto out;
513	hlist_nulls_for_each_entry(h, n, &net->ct.hash[repl_hash], hnnode)
514		if (nf_ct_tuple_equal(&ct->tuplehash[IP_CT_DIR_REPLY].tuple,
515				      &h->tuple) &&
516		    zone == nf_ct_zone(nf_ct_tuplehash_to_ctrack(h)))
517			goto out;
518
519	/* Remove from unconfirmed list */
520	hlist_nulls_del_rcu(&ct->tuplehash[IP_CT_DIR_ORIGINAL].hnnode);
521
522	/* Timer relative to confirmation time, not original
523	   setting time, otherwise we'd get timer wrap in
524	   weird delay cases. */
525	ct->timeout.expires += jiffies;
526	add_timer(&ct->timeout);
527	atomic_inc(&ct->ct_general.use);
528	ct->status |= IPS_CONFIRMED;
529
530	/* set conntrack timestamp, if enabled. */
531	tstamp = nf_conn_tstamp_find(ct);
532	if (tstamp) {
533		if (skb->tstamp.tv64 == 0)
534			__net_timestamp((struct sk_buff *)skb);
535
536		tstamp->start = ktime_to_ns(skb->tstamp);
537	}
538	/* Since the lookup is lockless, hash insertion must be done after
539	 * starting the timer and setting the CONFIRMED bit. The RCU barriers
540	 * guarantee that no other CPU can find the conntrack before the above
541	 * stores are visible.
542	 */
543	__nf_conntrack_hash_insert(ct, hash, repl_hash);
544	NF_CT_STAT_INC(net, insert);
545	spin_unlock_bh(&nf_conntrack_lock);
546
547	help = nfct_help(ct);
548	if (help && help->helper)
549		nf_conntrack_event_cache(IPCT_HELPER, ct);
550
551	nf_conntrack_event_cache(master_ct(ct) ?
552				 IPCT_RELATED : IPCT_NEW, ct);
553	return NF_ACCEPT;
554
555out:
556	NF_CT_STAT_INC(net, insert_failed);
557	spin_unlock_bh(&nf_conntrack_lock);
558	return NF_DROP;
559}
560EXPORT_SYMBOL_GPL(__nf_conntrack_confirm);
561
562/* Returns true if a connection correspondings to the tuple (required
563   for NAT). */
564int
565nf_conntrack_tuple_taken(const struct nf_conntrack_tuple *tuple,
566			 const struct nf_conn *ignored_conntrack)
567{
568	struct net *net = nf_ct_net(ignored_conntrack);
569	struct nf_conntrack_tuple_hash *h;
570	struct hlist_nulls_node *n;
571	struct nf_conn *ct;
572	u16 zone = nf_ct_zone(ignored_conntrack);
573	unsigned int hash = hash_conntrack(net, zone, tuple);
574
575	/* Disable BHs the entire time since we need to disable them at
576	 * least once for the stats anyway.
577	 */
578	rcu_read_lock_bh();
579	hlist_nulls_for_each_entry_rcu(h, n, &net->ct.hash[hash], hnnode) {
580		ct = nf_ct_tuplehash_to_ctrack(h);
581		if (ct != ignored_conntrack &&
582		    nf_ct_tuple_equal(tuple, &h->tuple) &&
583		    nf_ct_zone(ct) == zone) {
584			NF_CT_STAT_INC(net, found);
585			rcu_read_unlock_bh();
586			return 1;
587		}
588		NF_CT_STAT_INC(net, searched);
589	}
590	rcu_read_unlock_bh();
591
592	return 0;
593}
594EXPORT_SYMBOL_GPL(nf_conntrack_tuple_taken);
595
596#define NF_CT_EVICTION_RANGE	8
597
598/* There's a small race here where we may free a just-assured
599   connection.  Too bad: we're in trouble anyway. */
600static noinline int early_drop(struct net *net, unsigned int hash)
601{
602	/* Use oldest entry, which is roughly LRU */
603	struct nf_conntrack_tuple_hash *h;
604	struct nf_conn *ct = NULL, *tmp;
605	struct hlist_nulls_node *n;
606	unsigned int i, cnt = 0;
607	int dropped = 0;
608
609	rcu_read_lock();
610	for (i = 0; i < net->ct.htable_size; i++) {
611		hlist_nulls_for_each_entry_rcu(h, n, &net->ct.hash[hash],
612					 hnnode) {
613			tmp = nf_ct_tuplehash_to_ctrack(h);
614			if (!test_bit(IPS_ASSURED_BIT, &tmp->status))
615				ct = tmp;
616			cnt++;
617		}
618
619		if (ct != NULL) {
620			if (likely(!nf_ct_is_dying(ct) &&
621				   atomic_inc_not_zero(&ct->ct_general.use)))
622				break;
623			else
624				ct = NULL;
625		}
626
627		if (cnt >= NF_CT_EVICTION_RANGE)
628			break;
629
630		hash = (hash + 1) % net->ct.htable_size;
631	}
632	rcu_read_unlock();
633
634	if (!ct)
635		return dropped;
636
637	if (del_timer(&ct->timeout)) {
638		death_by_timeout((unsigned long)ct);
639		/* Check if we indeed killed this entry. Reliable event
640		   delivery may have inserted it into the dying list. */
641		if (test_bit(IPS_DYING_BIT, &ct->status)) {
642			dropped = 1;
643			NF_CT_STAT_INC_ATOMIC(net, early_drop);
644		}
645	}
646	nf_ct_put(ct);
647	return dropped;
648}
649
650void init_nf_conntrack_hash_rnd(void)
651{
652	unsigned int rand;
653
654	/*
655	 * Why not initialize nf_conntrack_rnd in a "init()" function ?
656	 * Because there isn't enough entropy when system initializing,
657	 * and we initialize it as late as possible.
658	 */
659	do {
660		get_random_bytes(&rand, sizeof(rand));
661	} while (!rand);
662	cmpxchg(&nf_conntrack_hash_rnd, 0, rand);
663}
664
665static struct nf_conn *
666__nf_conntrack_alloc(struct net *net, u16 zone,
667		     const struct nf_conntrack_tuple *orig,
668		     const struct nf_conntrack_tuple *repl,
669		     gfp_t gfp, u32 hash)
670{
671	struct nf_conn *ct;
672
673	if (unlikely(!nf_conntrack_hash_rnd)) {
674		init_nf_conntrack_hash_rnd();
675		/* recompute the hash as nf_conntrack_hash_rnd is initialized */
676		hash = hash_conntrack_raw(orig, zone);
677	}
678
679	/* We don't want any race condition at early drop stage */
680	atomic_inc(&net->ct.count);
681
682	if (nf_conntrack_max &&
683	    unlikely(atomic_read(&net->ct.count) > nf_conntrack_max)) {
684		if (!early_drop(net, hash_bucket(hash, net))) {
685			atomic_dec(&net->ct.count);
686			net_warn_ratelimited("nf_conntrack: table full, dropping packet\n");
687			return ERR_PTR(-ENOMEM);
688		}
689	}
690
691	/*
692	 * Do not use kmem_cache_zalloc(), as this cache uses
693	 * SLAB_DESTROY_BY_RCU.
694	 */
695	ct = kmem_cache_alloc(net->ct.nf_conntrack_cachep, gfp);
696	if (ct == NULL) {
697		atomic_dec(&net->ct.count);
698		return ERR_PTR(-ENOMEM);
699	}
700	/*
701	 * Let ct->tuplehash[IP_CT_DIR_ORIGINAL].hnnode.next
702	 * and ct->tuplehash[IP_CT_DIR_REPLY].hnnode.next unchanged.
703	 */
704	memset(&ct->tuplehash[IP_CT_DIR_MAX], 0,
705	       offsetof(struct nf_conn, proto) -
706	       offsetof(struct nf_conn, tuplehash[IP_CT_DIR_MAX]));
707	spin_lock_init(&ct->lock);
708	ct->tuplehash[IP_CT_DIR_ORIGINAL].tuple = *orig;
709	ct->tuplehash[IP_CT_DIR_ORIGINAL].hnnode.pprev = NULL;
710	ct->tuplehash[IP_CT_DIR_REPLY].tuple = *repl;
711	/* save hash for reusing when confirming */
712	*(unsigned long *)(&ct->tuplehash[IP_CT_DIR_REPLY].hnnode.pprev) = hash;
713	/* Don't set timer yet: wait for confirmation */
714	setup_timer(&ct->timeout, death_by_timeout, (unsigned long)ct);
715	write_pnet(&ct->ct_net, net);
716#ifdef CONFIG_NF_CONNTRACK_ZONES
717	if (zone) {
718		struct nf_conntrack_zone *nf_ct_zone;
719
720		nf_ct_zone = nf_ct_ext_add(ct, NF_CT_EXT_ZONE, GFP_ATOMIC);
721		if (!nf_ct_zone)
722			goto out_free;
723		nf_ct_zone->id = zone;
724	}
725#endif
726	/*
727	 * changes to lookup keys must be done before setting refcnt to 1
728	 */
729	smp_wmb();
730	atomic_set(&ct->ct_general.use, 1);
731	return ct;
732
733#ifdef CONFIG_NF_CONNTRACK_ZONES
734out_free:
735	atomic_dec(&net->ct.count);
736	kmem_cache_free(net->ct.nf_conntrack_cachep, ct);
737	return ERR_PTR(-ENOMEM);
738#endif
739}
740
741struct nf_conn *nf_conntrack_alloc(struct net *net, u16 zone,
742				   const struct nf_conntrack_tuple *orig,
743				   const struct nf_conntrack_tuple *repl,
744				   gfp_t gfp)
745{
746	return __nf_conntrack_alloc(net, zone, orig, repl, gfp, 0);
747}
748EXPORT_SYMBOL_GPL(nf_conntrack_alloc);
749
750void nf_conntrack_free(struct nf_conn *ct)
751{
752	struct net *net = nf_ct_net(ct);
753
754	nf_ct_ext_destroy(ct);
755	atomic_dec(&net->ct.count);
756	nf_ct_ext_free(ct);
757	kmem_cache_free(net->ct.nf_conntrack_cachep, ct);
758}
759EXPORT_SYMBOL_GPL(nf_conntrack_free);
760
761/* Allocate a new conntrack: we return -ENOMEM if classification
762   failed due to stress.  Otherwise it really is unclassifiable. */
763static struct nf_conntrack_tuple_hash *
764init_conntrack(struct net *net, struct nf_conn *tmpl,
765	       const struct nf_conntrack_tuple *tuple,
766	       struct nf_conntrack_l3proto *l3proto,
767	       struct nf_conntrack_l4proto *l4proto,
768	       struct sk_buff *skb,
769	       unsigned int dataoff, u32 hash)
770{
771	struct nf_conn *ct;
772	struct nf_conn_help *help;
773	struct nf_conntrack_tuple repl_tuple;
774	struct nf_conntrack_ecache *ecache;
775	struct nf_conntrack_expect *exp;
776	u16 zone = tmpl ? nf_ct_zone(tmpl) : NF_CT_DEFAULT_ZONE;
777	struct nf_conn_timeout *timeout_ext;
778	unsigned int *timeouts;
779
780	if (!nf_ct_invert_tuple(&repl_tuple, tuple, l3proto, l4proto)) {
781		pr_debug("Can't invert tuple.\n");
782		return NULL;
783	}
784
785	ct = __nf_conntrack_alloc(net, zone, tuple, &repl_tuple, GFP_ATOMIC,
786				  hash);
787	if (IS_ERR(ct))
788		return (struct nf_conntrack_tuple_hash *)ct;
789
790	timeout_ext = tmpl ? nf_ct_timeout_find(tmpl) : NULL;
791	if (timeout_ext)
792		timeouts = NF_CT_TIMEOUT_EXT_DATA(timeout_ext);
793	else
794		timeouts = l4proto->get_timeouts(net);
795
796	if (!l4proto->new(ct, skb, dataoff, timeouts)) {
797		nf_conntrack_free(ct);
798		pr_debug("init conntrack: can't track with proto module\n");
799		return NULL;
800	}
801
802	if (timeout_ext)
803		nf_ct_timeout_ext_add(ct, timeout_ext->timeout, GFP_ATOMIC);
804
805	nf_ct_acct_ext_add(ct, GFP_ATOMIC);
806	nf_ct_tstamp_ext_add(ct, GFP_ATOMIC);
807
808	ecache = tmpl ? nf_ct_ecache_find(tmpl) : NULL;
809	nf_ct_ecache_ext_add(ct, ecache ? ecache->ctmask : 0,
810				 ecache ? ecache->expmask : 0,
811			     GFP_ATOMIC);
812
813	spin_lock_bh(&nf_conntrack_lock);
814	exp = nf_ct_find_expectation(net, zone, tuple);
815	if (exp) {
816		pr_debug("conntrack: expectation arrives ct=%p exp=%p\n",
817			 ct, exp);
818		/* Welcome, Mr. Bond.  We've been expecting you... */
819		__set_bit(IPS_EXPECTED_BIT, &ct->status);
820		ct->master = exp->master;
821		if (exp->helper) {
822			help = nf_ct_helper_ext_add(ct, GFP_ATOMIC);
823			if (help)
824				rcu_assign_pointer(help->helper, exp->helper);
825		}
826
827#ifdef CONFIG_NF_CONNTRACK_MARK
828		ct->mark = exp->master->mark;
829#endif
830#ifdef CONFIG_NF_CONNTRACK_SECMARK
831		ct->secmark = exp->master->secmark;
832#endif
833		nf_conntrack_get(&ct->master->ct_general);
834		NF_CT_STAT_INC(net, expect_new);
835	} else {
836		__nf_ct_try_assign_helper(ct, tmpl, GFP_ATOMIC);
837		NF_CT_STAT_INC(net, new);
838	}
839
840	/* Overload tuple linked list to put us in unconfirmed list. */
841	hlist_nulls_add_head_rcu(&ct->tuplehash[IP_CT_DIR_ORIGINAL].hnnode,
842		       &net->ct.unconfirmed);
843
844	spin_unlock_bh(&nf_conntrack_lock);
845
846	if (exp) {
847		if (exp->expectfn)
848			exp->expectfn(ct, exp);
849		nf_ct_expect_put(exp);
850	}
851
852	return &ct->tuplehash[IP_CT_DIR_ORIGINAL];
853}
854
855/* On success, returns conntrack ptr, sets skb->nfct and ctinfo */
856static inline struct nf_conn *
857resolve_normal_ct(struct net *net, struct nf_conn *tmpl,
858		  struct sk_buff *skb,
859		  unsigned int dataoff,
860		  u_int16_t l3num,
861		  u_int8_t protonum,
862		  struct nf_conntrack_l3proto *l3proto,
863		  struct nf_conntrack_l4proto *l4proto,
864		  int *set_reply,
865		  enum ip_conntrack_info *ctinfo)
866{
867	struct nf_conntrack_tuple tuple;
868	struct nf_conntrack_tuple_hash *h;
869	struct nf_conn *ct;
870	u16 zone = tmpl ? nf_ct_zone(tmpl) : NF_CT_DEFAULT_ZONE;
871	u32 hash;
872
873	if (!nf_ct_get_tuple(skb, skb_network_offset(skb),
874			     dataoff, l3num, protonum, &tuple, l3proto,
875			     l4proto)) {
876		pr_debug("resolve_normal_ct: Can't get tuple\n");
877		return NULL;
878	}
879
880	/* look for tuple match */
881	hash = hash_conntrack_raw(&tuple, zone);
882	h = __nf_conntrack_find_get(net, zone, &tuple, hash);
883	if (!h) {
884		h = init_conntrack(net, tmpl, &tuple, l3proto, l4proto,
885				   skb, dataoff, hash);
886		if (!h)
887			return NULL;
888		if (IS_ERR(h))
889			return (void *)h;
890	}
891	ct = nf_ct_tuplehash_to_ctrack(h);
892
893	/* It exists; we have (non-exclusive) reference. */
894	if (NF_CT_DIRECTION(h) == IP_CT_DIR_REPLY) {
895		*ctinfo = IP_CT_ESTABLISHED_REPLY;
896		/* Please set reply bit if this packet OK */
897		*set_reply = 1;
898	} else {
899		/* Once we've had two way comms, always ESTABLISHED. */
900		if (test_bit(IPS_SEEN_REPLY_BIT, &ct->status)) {
901			pr_debug("nf_conntrack_in: normal packet for %p\n", ct);
902			*ctinfo = IP_CT_ESTABLISHED;
903		} else if (test_bit(IPS_EXPECTED_BIT, &ct->status)) {
904			pr_debug("nf_conntrack_in: related packet for %p\n",
905				 ct);
906			*ctinfo = IP_CT_RELATED;
907		} else {
908			pr_debug("nf_conntrack_in: new packet for %p\n", ct);
909			*ctinfo = IP_CT_NEW;
910		}
911		*set_reply = 0;
912	}
913	skb->nfct = &ct->ct_general;
914	skb->nfctinfo = *ctinfo;
915	return ct;
916}
917
918unsigned int
919nf_conntrack_in(struct net *net, u_int8_t pf, unsigned int hooknum,
920		struct sk_buff *skb)
921{
922	struct nf_conn *ct, *tmpl = NULL;
923	enum ip_conntrack_info ctinfo;
924	struct nf_conntrack_l3proto *l3proto;
925	struct nf_conntrack_l4proto *l4proto;
926	struct nf_conn_timeout *timeout_ext;
927	unsigned int *timeouts;
928	unsigned int dataoff;
929	u_int8_t protonum;
930	int set_reply = 0;
931	int ret;
932
933	if (skb->nfct) {
934		/* Previously seen (loopback or untracked)?  Ignore. */
935		tmpl = (struct nf_conn *)skb->nfct;
936		if (!nf_ct_is_template(tmpl)) {
937			NF_CT_STAT_INC_ATOMIC(net, ignore);
938			return NF_ACCEPT;
939		}
940		skb->nfct = NULL;
941	}
942
943	/* rcu_read_lock()ed by nf_hook_slow */
944	l3proto = __nf_ct_l3proto_find(pf);
945	ret = l3proto->get_l4proto(skb, skb_network_offset(skb),
946				   &dataoff, &protonum);
947	if (ret <= 0) {
948		pr_debug("not prepared to track yet or error occurred\n");
949		NF_CT_STAT_INC_ATOMIC(net, error);
950		NF_CT_STAT_INC_ATOMIC(net, invalid);
951		ret = -ret;
952		goto out;
953	}
954
955	l4proto = __nf_ct_l4proto_find(pf, protonum);
956
957	/* It may be an special packet, error, unclean...
958	 * inverse of the return code tells to the netfilter
959	 * core what to do with the packet. */
960	if (l4proto->error != NULL) {
961		ret = l4proto->error(net, tmpl, skb, dataoff, &ctinfo,
962				     pf, hooknum);
963		if (ret <= 0) {
964			NF_CT_STAT_INC_ATOMIC(net, error);
965			NF_CT_STAT_INC_ATOMIC(net, invalid);
966			ret = -ret;
967			goto out;
968		}
969		/* ICMP[v6] protocol trackers may assign one conntrack. */
970		if (skb->nfct)
971			goto out;
972	}
973
974	ct = resolve_normal_ct(net, tmpl, skb, dataoff, pf, protonum,
975			       l3proto, l4proto, &set_reply, &ctinfo);
976	if (!ct) {
977		/* Not valid part of a connection */
978		NF_CT_STAT_INC_ATOMIC(net, invalid);
979		ret = NF_ACCEPT;
980		goto out;
981	}
982
983	if (IS_ERR(ct)) {
984		/* Too stressed to deal. */
985		NF_CT_STAT_INC_ATOMIC(net, drop);
986		ret = NF_DROP;
987		goto out;
988	}
989
990	NF_CT_ASSERT(skb->nfct);
991
992	/* Decide what timeout policy we want to apply to this flow. */
993	timeout_ext = nf_ct_timeout_find(ct);
994	if (timeout_ext)
995		timeouts = NF_CT_TIMEOUT_EXT_DATA(timeout_ext);
996	else
997		timeouts = l4proto->get_timeouts(net);
998
999	ret = l4proto->packet(ct, skb, dataoff, ctinfo, pf, hooknum, timeouts);
1000	if (ret <= 0) {
1001		/* Invalid: inverse of the return code tells
1002		 * the netfilter core what to do */
1003		pr_debug("nf_conntrack_in: Can't track with proto module\n");
1004		nf_conntrack_put(skb->nfct);
1005		skb->nfct = NULL;
1006		NF_CT_STAT_INC_ATOMIC(net, invalid);
1007		if (ret == -NF_DROP)
1008			NF_CT_STAT_INC_ATOMIC(net, drop);
1009		ret = -ret;
1010		goto out;
1011	}
1012
1013	if (set_reply && !test_and_set_bit(IPS_SEEN_REPLY_BIT, &ct->status))
1014		nf_conntrack_event_cache(IPCT_REPLY, ct);
1015out:
1016	if (tmpl) {
1017		/* Special case: we have to repeat this hook, assign the
1018		 * template again to this packet. We assume that this packet
1019		 * has no conntrack assigned. This is used by nf_ct_tcp. */
1020		if (ret == NF_REPEAT)
1021			skb->nfct = (struct nf_conntrack *)tmpl;
1022		else
1023			nf_ct_put(tmpl);
1024	}
1025
1026	return ret;
1027}
1028EXPORT_SYMBOL_GPL(nf_conntrack_in);
1029
1030bool nf_ct_invert_tuplepr(struct nf_conntrack_tuple *inverse,
1031			  const struct nf_conntrack_tuple *orig)
1032{
1033	bool ret;
1034
1035	rcu_read_lock();
1036	ret = nf_ct_invert_tuple(inverse, orig,
1037				 __nf_ct_l3proto_find(orig->src.l3num),
1038				 __nf_ct_l4proto_find(orig->src.l3num,
1039						      orig->dst.protonum));
1040	rcu_read_unlock();
1041	return ret;
1042}
1043EXPORT_SYMBOL_GPL(nf_ct_invert_tuplepr);
1044
1045/* Alter reply tuple (maybe alter helper).  This is for NAT, and is
1046   implicitly racy: see __nf_conntrack_confirm */
1047void nf_conntrack_alter_reply(struct nf_conn *ct,
1048			      const struct nf_conntrack_tuple *newreply)
1049{
1050	struct nf_conn_help *help = nfct_help(ct);
1051
1052	/* Should be unconfirmed, so not in hash table yet */
1053	NF_CT_ASSERT(!nf_ct_is_confirmed(ct));
1054
1055	pr_debug("Altering reply tuple of %p to ", ct);
1056	nf_ct_dump_tuple(newreply);
1057
1058	ct->tuplehash[IP_CT_DIR_REPLY].tuple = *newreply;
1059	if (ct->master || (help && !hlist_empty(&help->expectations)))
1060		return;
1061
1062	rcu_read_lock();
1063	__nf_ct_try_assign_helper(ct, NULL, GFP_ATOMIC);
1064	rcu_read_unlock();
1065}
1066EXPORT_SYMBOL_GPL(nf_conntrack_alter_reply);
1067
1068/* Refresh conntrack for this many jiffies and do accounting if do_acct is 1 */
1069void __nf_ct_refresh_acct(struct nf_conn *ct,
1070			  enum ip_conntrack_info ctinfo,
1071			  const struct sk_buff *skb,
1072			  unsigned long extra_jiffies,
1073			  int do_acct)
1074{
1075	NF_CT_ASSERT(ct->timeout.data == (unsigned long)ct);
1076	NF_CT_ASSERT(skb);
1077
1078	/* Only update if this is not a fixed timeout */
1079	if (test_bit(IPS_FIXED_TIMEOUT_BIT, &ct->status))
1080		goto acct;
1081
1082	/* If not in hash table, timer will not be active yet */
1083	if (!nf_ct_is_confirmed(ct)) {
1084		ct->timeout.expires = extra_jiffies;
1085	} else {
1086		unsigned long newtime = jiffies + extra_jiffies;
1087
1088		/* Only update the timeout if the new timeout is at least
1089		   HZ jiffies from the old timeout. Need del_timer for race
1090		   avoidance (may already be dying). */
1091		if (newtime - ct->timeout.expires >= HZ)
1092			mod_timer_pending(&ct->timeout, newtime);
1093	}
1094
1095acct:
1096	if (do_acct) {
1097		struct nf_conn_counter *acct;
1098
1099		acct = nf_conn_acct_find(ct);
1100		if (acct) {
1101			atomic64_inc(&acct[CTINFO2DIR(ctinfo)].packets);
1102			atomic64_add(skb->len, &acct[CTINFO2DIR(ctinfo)].bytes);
1103		}
1104	}
1105}
1106EXPORT_SYMBOL_GPL(__nf_ct_refresh_acct);
1107
1108bool __nf_ct_kill_acct(struct nf_conn *ct,
1109		       enum ip_conntrack_info ctinfo,
1110		       const struct sk_buff *skb,
1111		       int do_acct)
1112{
1113	if (do_acct) {
1114		struct nf_conn_counter *acct;
1115
1116		acct = nf_conn_acct_find(ct);
1117		if (acct) {
1118			atomic64_inc(&acct[CTINFO2DIR(ctinfo)].packets);
1119			atomic64_add(skb->len - skb_network_offset(skb),
1120				     &acct[CTINFO2DIR(ctinfo)].bytes);
1121		}
1122	}
1123
1124	if (del_timer(&ct->timeout)) {
1125		ct->timeout.function((unsigned long)ct);
1126		return true;
1127	}
1128	return false;
1129}
1130EXPORT_SYMBOL_GPL(__nf_ct_kill_acct);
1131
1132#ifdef CONFIG_NF_CONNTRACK_ZONES
1133static struct nf_ct_ext_type nf_ct_zone_extend __read_mostly = {
1134	.len	= sizeof(struct nf_conntrack_zone),
1135	.align	= __alignof__(struct nf_conntrack_zone),
1136	.id	= NF_CT_EXT_ZONE,
1137};
1138#endif
1139
1140#if IS_ENABLED(CONFIG_NF_CT_NETLINK)
1141
1142#include <linux/netfilter/nfnetlink.h>
1143#include <linux/netfilter/nfnetlink_conntrack.h>
1144#include <linux/mutex.h>
1145
1146/* Generic function for tcp/udp/sctp/dccp and alike. This needs to be
1147 * in ip_conntrack_core, since we don't want the protocols to autoload
1148 * or depend on ctnetlink */
1149int nf_ct_port_tuple_to_nlattr(struct sk_buff *skb,
1150			       const struct nf_conntrack_tuple *tuple)
1151{
1152	if (nla_put_be16(skb, CTA_PROTO_SRC_PORT, tuple->src.u.tcp.port) ||
1153	    nla_put_be16(skb, CTA_PROTO_DST_PORT, tuple->dst.u.tcp.port))
1154		goto nla_put_failure;
1155	return 0;
1156
1157nla_put_failure:
1158	return -1;
1159}
1160EXPORT_SYMBOL_GPL(nf_ct_port_tuple_to_nlattr);
1161
1162const struct nla_policy nf_ct_port_nla_policy[CTA_PROTO_MAX+1] = {
1163	[CTA_PROTO_SRC_PORT]  = { .type = NLA_U16 },
1164	[CTA_PROTO_DST_PORT]  = { .type = NLA_U16 },
1165};
1166EXPORT_SYMBOL_GPL(nf_ct_port_nla_policy);
1167
1168int nf_ct_port_nlattr_to_tuple(struct nlattr *tb[],
1169			       struct nf_conntrack_tuple *t)
1170{
1171	if (!tb[CTA_PROTO_SRC_PORT] || !tb[CTA_PROTO_DST_PORT])
1172		return -EINVAL;
1173
1174	t->src.u.tcp.port = nla_get_be16(tb[CTA_PROTO_SRC_PORT]);
1175	t->dst.u.tcp.port = nla_get_be16(tb[CTA_PROTO_DST_PORT]);
1176
1177	return 0;
1178}
1179EXPORT_SYMBOL_GPL(nf_ct_port_nlattr_to_tuple);
1180
1181int nf_ct_port_nlattr_tuple_size(void)
1182{
1183	return nla_policy_len(nf_ct_port_nla_policy, CTA_PROTO_MAX + 1);
1184}
1185EXPORT_SYMBOL_GPL(nf_ct_port_nlattr_tuple_size);
1186#endif
1187
1188/* Used by ipt_REJECT and ip6t_REJECT. */
1189static void nf_conntrack_attach(struct sk_buff *nskb, struct sk_buff *skb)
1190{
1191	struct nf_conn *ct;
1192	enum ip_conntrack_info ctinfo;
1193
1194	/* This ICMP is in reverse direction to the packet which caused it */
1195	ct = nf_ct_get(skb, &ctinfo);
1196	if (CTINFO2DIR(ctinfo) == IP_CT_DIR_ORIGINAL)
1197		ctinfo = IP_CT_RELATED_REPLY;
1198	else
1199		ctinfo = IP_CT_RELATED;
1200
1201	/* Attach to new skbuff, and increment count */
1202	nskb->nfct = &ct->ct_general;
1203	nskb->nfctinfo = ctinfo;
1204	nf_conntrack_get(nskb->nfct);
1205}
1206
1207/* Bring out ya dead! */
1208static struct nf_conn *
1209get_next_corpse(struct net *net, int (*iter)(struct nf_conn *i, void *data),
1210		void *data, unsigned int *bucket)
1211{
1212	struct nf_conntrack_tuple_hash *h;
1213	struct nf_conn *ct;
1214	struct hlist_nulls_node *n;
1215
1216	spin_lock_bh(&nf_conntrack_lock);
1217	for (; *bucket < net->ct.htable_size; (*bucket)++) {
1218		hlist_nulls_for_each_entry(h, n, &net->ct.hash[*bucket], hnnode) {
1219			ct = nf_ct_tuplehash_to_ctrack(h);
1220			if (iter(ct, data))
1221				goto found;
1222		}
1223	}
1224	hlist_nulls_for_each_entry(h, n, &net->ct.unconfirmed, hnnode) {
1225		ct = nf_ct_tuplehash_to_ctrack(h);
1226		if (iter(ct, data))
1227			set_bit(IPS_DYING_BIT, &ct->status);
1228	}
1229	spin_unlock_bh(&nf_conntrack_lock);
1230	return NULL;
1231found:
1232	atomic_inc(&ct->ct_general.use);
1233	spin_unlock_bh(&nf_conntrack_lock);
1234	return ct;
1235}
1236
1237void nf_ct_iterate_cleanup(struct net *net,
1238			   int (*iter)(struct nf_conn *i, void *data),
1239			   void *data)
1240{
1241	struct nf_conn *ct;
1242	unsigned int bucket = 0;
1243
1244	while ((ct = get_next_corpse(net, iter, data, &bucket)) != NULL) {
1245		/* Time to push up daises... */
1246		if (del_timer(&ct->timeout))
1247			death_by_timeout((unsigned long)ct);
1248		/* ... else the timer will get him soon. */
1249
1250		nf_ct_put(ct);
1251	}
1252}
1253EXPORT_SYMBOL_GPL(nf_ct_iterate_cleanup);
1254
1255struct __nf_ct_flush_report {
1256	u32 pid;
1257	int report;
1258};
1259
1260static int kill_report(struct nf_conn *i, void *data)
1261{
1262	struct __nf_ct_flush_report *fr = (struct __nf_ct_flush_report *)data;
1263	struct nf_conn_tstamp *tstamp;
1264
1265	tstamp = nf_conn_tstamp_find(i);
1266	if (tstamp && tstamp->stop == 0)
1267		tstamp->stop = ktime_to_ns(ktime_get_real());
1268
1269	/* If we fail to deliver the event, death_by_timeout() will retry */
1270	if (nf_conntrack_event_report(IPCT_DESTROY, i,
1271				      fr->pid, fr->report) < 0)
1272		return 1;
1273
1274	/* Avoid the delivery of the destroy event in death_by_timeout(). */
1275	set_bit(IPS_DYING_BIT, &i->status);
1276	return 1;
1277}
1278
1279static int kill_all(struct nf_conn *i, void *data)
1280{
1281	return 1;
1282}
1283
1284void nf_ct_free_hashtable(void *hash, unsigned int size)
1285{
1286	if (is_vmalloc_addr(hash))
1287		vfree(hash);
1288	else
1289		free_pages((unsigned long)hash,
1290			   get_order(sizeof(struct hlist_head) * size));
1291}
1292EXPORT_SYMBOL_GPL(nf_ct_free_hashtable);
1293
1294void nf_conntrack_flush_report(struct net *net, u32 pid, int report)
1295{
1296	struct __nf_ct_flush_report fr = {
1297		.pid 	= pid,
1298		.report = report,
1299	};
1300	nf_ct_iterate_cleanup(net, kill_report, &fr);
1301}
1302EXPORT_SYMBOL_GPL(nf_conntrack_flush_report);
1303
1304static void nf_ct_release_dying_list(struct net *net)
1305{
1306	struct nf_conntrack_tuple_hash *h;
1307	struct nf_conn *ct;
1308	struct hlist_nulls_node *n;
1309
1310	spin_lock_bh(&nf_conntrack_lock);
1311	hlist_nulls_for_each_entry(h, n, &net->ct.dying, hnnode) {
1312		ct = nf_ct_tuplehash_to_ctrack(h);
1313		/* never fails to remove them, no listeners at this point */
1314		nf_ct_kill(ct);
1315	}
1316	spin_unlock_bh(&nf_conntrack_lock);
1317}
1318
1319static int untrack_refs(void)
1320{
1321	int cnt = 0, cpu;
1322
1323	for_each_possible_cpu(cpu) {
1324		struct nf_conn *ct = &per_cpu(nf_conntrack_untracked, cpu);
1325
1326		cnt += atomic_read(&ct->ct_general.use) - 1;
1327	}
1328	return cnt;
1329}
1330
1331static void nf_conntrack_cleanup_init_net(void)
1332{
1333	while (untrack_refs() > 0)
1334		schedule();
1335
1336#ifdef CONFIG_NF_CONNTRACK_ZONES
1337	nf_ct_extend_unregister(&nf_ct_zone_extend);
1338#endif
1339}
1340
1341static void nf_conntrack_cleanup_net(struct net *net)
1342{
1343 i_see_dead_people:
1344	nf_ct_iterate_cleanup(net, kill_all, NULL);
1345	nf_ct_release_dying_list(net);
1346	if (atomic_read(&net->ct.count) != 0) {
1347		schedule();
1348		goto i_see_dead_people;
1349	}
1350
1351	nf_ct_free_hashtable(net->ct.hash, net->ct.htable_size);
1352	nf_conntrack_helper_fini(net);
1353	nf_conntrack_timeout_fini(net);
1354	nf_conntrack_ecache_fini(net);
1355	nf_conntrack_tstamp_fini(net);
1356	nf_conntrack_acct_fini(net);
1357	nf_conntrack_expect_fini(net);
1358	kmem_cache_destroy(net->ct.nf_conntrack_cachep);
1359	kfree(net->ct.slabname);
1360	free_percpu(net->ct.stat);
1361}
1362
1363/* Mishearing the voices in his head, our hero wonders how he's
1364   supposed to kill the mall. */
1365void nf_conntrack_cleanup(struct net *net)
1366{
1367	if (net_eq(net, &init_net))
1368		RCU_INIT_POINTER(ip_ct_attach, NULL);
1369
1370	/* This makes sure all current packets have passed through
1371	   netfilter framework.  Roll on, two-stage module
1372	   delete... */
1373	synchronize_net();
1374	nf_conntrack_proto_fini(net);
1375	nf_conntrack_cleanup_net(net);
1376
1377	if (net_eq(net, &init_net)) {
1378		RCU_INIT_POINTER(nf_ct_destroy, NULL);
1379		nf_conntrack_cleanup_init_net();
1380	}
1381}
1382
1383void *nf_ct_alloc_hashtable(unsigned int *sizep, int nulls)
1384{
1385	struct hlist_nulls_head *hash;
1386	unsigned int nr_slots, i;
1387	size_t sz;
1388
1389	BUILD_BUG_ON(sizeof(struct hlist_nulls_head) != sizeof(struct hlist_head));
1390	nr_slots = *sizep = roundup(*sizep, PAGE_SIZE / sizeof(struct hlist_nulls_head));
1391	sz = nr_slots * sizeof(struct hlist_nulls_head);
1392	hash = (void *)__get_free_pages(GFP_KERNEL | __GFP_NOWARN | __GFP_ZERO,
1393					get_order(sz));
1394	if (!hash) {
1395		printk(KERN_WARNING "nf_conntrack: falling back to vmalloc.\n");
1396		hash = vzalloc(sz);
1397	}
1398
1399	if (hash && nulls)
1400		for (i = 0; i < nr_slots; i++)
1401			INIT_HLIST_NULLS_HEAD(&hash[i], i);
1402
1403	return hash;
1404}
1405EXPORT_SYMBOL_GPL(nf_ct_alloc_hashtable);
1406
1407int nf_conntrack_set_hashsize(const char *val, struct kernel_param *kp)
1408{
1409	int i, bucket;
1410	unsigned int hashsize, old_size;
1411	struct hlist_nulls_head *hash, *old_hash;
1412	struct nf_conntrack_tuple_hash *h;
1413	struct nf_conn *ct;
1414
1415	if (current->nsproxy->net_ns != &init_net)
1416		return -EOPNOTSUPP;
1417
1418	/* On boot, we can set this without any fancy locking. */
1419	if (!nf_conntrack_htable_size)
1420		return param_set_uint(val, kp);
1421
1422	hashsize = simple_strtoul(val, NULL, 0);
1423	if (!hashsize)
1424		return -EINVAL;
1425
1426	hash = nf_ct_alloc_hashtable(&hashsize, 1);
1427	if (!hash)
1428		return -ENOMEM;
1429
1430	/* Lookups in the old hash might happen in parallel, which means we
1431	 * might get false negatives during connection lookup. New connections
1432	 * created because of a false negative won't make it into the hash
1433	 * though since that required taking the lock.
1434	 */
1435	spin_lock_bh(&nf_conntrack_lock);
1436	for (i = 0; i < init_net.ct.htable_size; i++) {
1437		while (!hlist_nulls_empty(&init_net.ct.hash[i])) {
1438			h = hlist_nulls_entry(init_net.ct.hash[i].first,
1439					struct nf_conntrack_tuple_hash, hnnode);
1440			ct = nf_ct_tuplehash_to_ctrack(h);
1441			hlist_nulls_del_rcu(&h->hnnode);
1442			bucket = __hash_conntrack(&h->tuple, nf_ct_zone(ct),
1443						  hashsize);
1444			hlist_nulls_add_head_rcu(&h->hnnode, &hash[bucket]);
1445		}
1446	}
1447	old_size = init_net.ct.htable_size;
1448	old_hash = init_net.ct.hash;
1449
1450	init_net.ct.htable_size = nf_conntrack_htable_size = hashsize;
1451	init_net.ct.hash = hash;
1452	spin_unlock_bh(&nf_conntrack_lock);
1453
1454	nf_ct_free_hashtable(old_hash, old_size);
1455	return 0;
1456}
1457EXPORT_SYMBOL_GPL(nf_conntrack_set_hashsize);
1458
1459module_param_call(hashsize, nf_conntrack_set_hashsize, param_get_uint,
1460		  &nf_conntrack_htable_size, 0600);
1461
1462void nf_ct_untracked_status_or(unsigned long bits)
1463{
1464	int cpu;
1465
1466	for_each_possible_cpu(cpu)
1467		per_cpu(nf_conntrack_untracked, cpu).status |= bits;
1468}
1469EXPORT_SYMBOL_GPL(nf_ct_untracked_status_or);
1470
1471static int nf_conntrack_init_init_net(void)
1472{
1473	int max_factor = 8;
1474	int ret, cpu;
1475
1476	/* Idea from tcp.c: use 1/16384 of memory.  On i386: 32MB
1477	 * machine has 512 buckets. >= 1GB machines have 16384 buckets. */
1478	if (!nf_conntrack_htable_size) {
1479		nf_conntrack_htable_size
1480			= (((totalram_pages << PAGE_SHIFT) / 16384)
1481			   / sizeof(struct hlist_head));
1482		if (totalram_pages > (1024 * 1024 * 1024 / PAGE_SIZE))
1483			nf_conntrack_htable_size = 16384;
1484		if (nf_conntrack_htable_size < 32)
1485			nf_conntrack_htable_size = 32;
1486
1487		/* Use a max. factor of four by default to get the same max as
1488		 * with the old struct list_heads. When a table size is given
1489		 * we use the old value of 8 to avoid reducing the max.
1490		 * entries. */
1491		max_factor = 4;
1492	}
1493	nf_conntrack_max = max_factor * nf_conntrack_htable_size;
1494
1495	printk(KERN_INFO "nf_conntrack version %s (%u buckets, %d max)\n",
1496	       NF_CONNTRACK_VERSION, nf_conntrack_htable_size,
1497	       nf_conntrack_max);
1498#ifdef CONFIG_NF_CONNTRACK_ZONES
1499	ret = nf_ct_extend_register(&nf_ct_zone_extend);
1500	if (ret < 0)
1501		goto err_extend;
1502#endif
1503	/* Set up fake conntrack: to never be deleted, not in any hashes */
1504	for_each_possible_cpu(cpu) {
1505		struct nf_conn *ct = &per_cpu(nf_conntrack_untracked, cpu);
1506		write_pnet(&ct->ct_net, &init_net);
1507		atomic_set(&ct->ct_general.use, 1);
1508	}
1509	/*  - and look it like as a confirmed connection */
1510	nf_ct_untracked_status_or(IPS_CONFIRMED | IPS_UNTRACKED);
1511	return 0;
1512
1513#ifdef CONFIG_NF_CONNTRACK_ZONES
1514err_extend:
1515#endif
1516	return ret;
1517}
1518
1519/*
1520 * We need to use special "null" values, not used in hash table
1521 */
1522#define UNCONFIRMED_NULLS_VAL	((1<<30)+0)
1523#define DYING_NULLS_VAL		((1<<30)+1)
1524
1525static int nf_conntrack_init_net(struct net *net)
1526{
1527	int ret;
1528
1529	atomic_set(&net->ct.count, 0);
1530	INIT_HLIST_NULLS_HEAD(&net->ct.unconfirmed, UNCONFIRMED_NULLS_VAL);
1531	INIT_HLIST_NULLS_HEAD(&net->ct.dying, DYING_NULLS_VAL);
1532	net->ct.stat = alloc_percpu(struct ip_conntrack_stat);
1533	if (!net->ct.stat) {
1534		ret = -ENOMEM;
1535		goto err_stat;
1536	}
1537
1538	net->ct.slabname = kasprintf(GFP_KERNEL, "nf_conntrack_%p", net);
1539	if (!net->ct.slabname) {
1540		ret = -ENOMEM;
1541		goto err_slabname;
1542	}
1543
1544	net->ct.nf_conntrack_cachep = kmem_cache_create(net->ct.slabname,
1545							sizeof(struct nf_conn), 0,
1546							SLAB_DESTROY_BY_RCU, NULL);
1547	if (!net->ct.nf_conntrack_cachep) {
1548		printk(KERN_ERR "Unable to create nf_conn slab cache\n");
1549		ret = -ENOMEM;
1550		goto err_cache;
1551	}
1552
1553	net->ct.htable_size = nf_conntrack_htable_size;
1554	net->ct.hash = nf_ct_alloc_hashtable(&net->ct.htable_size, 1);
1555	if (!net->ct.hash) {
1556		ret = -ENOMEM;
1557		printk(KERN_ERR "Unable to create nf_conntrack_hash\n");
1558		goto err_hash;
1559	}
1560	ret = nf_conntrack_expect_init(net);
1561	if (ret < 0)
1562		goto err_expect;
1563	ret = nf_conntrack_acct_init(net);
1564	if (ret < 0)
1565		goto err_acct;
1566	ret = nf_conntrack_tstamp_init(net);
1567	if (ret < 0)
1568		goto err_tstamp;
1569	ret = nf_conntrack_ecache_init(net);
1570	if (ret < 0)
1571		goto err_ecache;
1572	ret = nf_conntrack_timeout_init(net);
1573	if (ret < 0)
1574		goto err_timeout;
1575	ret = nf_conntrack_helper_init(net);
1576	if (ret < 0)
1577		goto err_helper;
1578	return 0;
1579err_helper:
1580	nf_conntrack_timeout_fini(net);
1581err_timeout:
1582	nf_conntrack_ecache_fini(net);
1583err_ecache:
1584	nf_conntrack_tstamp_fini(net);
1585err_tstamp:
1586	nf_conntrack_acct_fini(net);
1587err_acct:
1588	nf_conntrack_expect_fini(net);
1589err_expect:
1590	nf_ct_free_hashtable(net->ct.hash, net->ct.htable_size);
1591err_hash:
1592	kmem_cache_destroy(net->ct.nf_conntrack_cachep);
1593err_cache:
1594	kfree(net->ct.slabname);
1595err_slabname:
1596	free_percpu(net->ct.stat);
1597err_stat:
1598	return ret;
1599}
1600
1601s16 (*nf_ct_nat_offset)(const struct nf_conn *ct,
1602			enum ip_conntrack_dir dir,
1603			u32 seq);
1604EXPORT_SYMBOL_GPL(nf_ct_nat_offset);
1605
1606int nf_conntrack_init(struct net *net)
1607{
1608	int ret;
1609
1610	if (net_eq(net, &init_net)) {
1611		ret = nf_conntrack_init_init_net();
1612		if (ret < 0)
1613			goto out_init_net;
1614	}
1615	ret = nf_conntrack_proto_init(net);
1616	if (ret < 0)
1617		goto out_proto;
1618	ret = nf_conntrack_init_net(net);
1619	if (ret < 0)
1620		goto out_net;
1621
1622	if (net_eq(net, &init_net)) {
1623		/* For use by REJECT target */
1624		RCU_INIT_POINTER(ip_ct_attach, nf_conntrack_attach);
1625		RCU_INIT_POINTER(nf_ct_destroy, destroy_conntrack);
1626
1627		/* Howto get NAT offsets */
1628		RCU_INIT_POINTER(nf_ct_nat_offset, NULL);
1629	}
1630	return 0;
1631
1632out_net:
1633	nf_conntrack_proto_fini(net);
1634out_proto:
1635	if (net_eq(net, &init_net))
1636		nf_conntrack_cleanup_init_net();
1637out_init_net:
1638	return ret;
1639}
1640