11da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds/* flow.c: Generic flow cache. 21da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds * 31da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds * Copyright (C) 2003 Alexey N. Kuznetsov (kuznet@ms2.inr.ac.ru) 41da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds * Copyright (C) 2003 David S. Miller (davem@redhat.com) 51da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds */ 61da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 71da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds#include <linux/kernel.h> 81da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds#include <linux/module.h> 91da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds#include <linux/list.h> 101da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds#include <linux/jhash.h> 111da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds#include <linux/interrupt.h> 121da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds#include <linux/mm.h> 131da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds#include <linux/random.h> 141da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds#include <linux/init.h> 151da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds#include <linux/slab.h> 161da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds#include <linux/smp.h> 171da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds#include <linux/completion.h> 181da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds#include <linux/percpu.h> 191da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds#include <linux/bitops.h> 201da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds#include <linux/notifier.h> 211da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds#include <linux/cpu.h> 221da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds#include <linux/cpumask.h> 234a3e2f711a00a1feb72ae12fdc749da10179d185Arjan van de Ven#include <linux/mutex.h> 241da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds#include <net/flow.h> 2560063497a95e716c9a689af3be2687d261f115b4Arun Sharma#include <linux/atomic.h> 26df71837d5024e2524cd51c93621e558aa7dd9f3fTrent Jaeger#include <linux/security.h> 27ca925cf1534ebcec332c08719a7dee6ee1782ce4Fan Du#include <net/net_namespace.h> 281da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 291da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvaldsstruct flow_cache_entry { 308e4795605d1e1b39113818ad7c147b8a867a1f6aTimo Teräs union { 318e4795605d1e1b39113818ad7c147b8a867a1f6aTimo Teräs struct hlist_node hlist; 328e4795605d1e1b39113818ad7c147b8a867a1f6aTimo Teräs struct list_head gc_list; 338e4795605d1e1b39113818ad7c147b8a867a1f6aTimo Teräs } u; 340542b69e2c57fc9668ce6a03155bea6e1f557901dpward struct net *net; 35fe1a5f031e76bd8761a7803d75b95ee96e84a574Timo Teräs u16 family; 36fe1a5f031e76bd8761a7803d75b95ee96e84a574Timo Teräs u8 dir; 37fe1a5f031e76bd8761a7803d75b95ee96e84a574Timo Teräs u32 genid; 38fe1a5f031e76bd8761a7803d75b95ee96e84a574Timo Teräs struct flowi key; 39fe1a5f031e76bd8761a7803d75b95ee96e84a574Timo Teräs struct flow_cache_object *object; 401da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds}; 411da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 421da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvaldsstruct flow_flush_info { 43fe1a5f031e76bd8761a7803d75b95ee96e84a574Timo Teräs struct flow_cache *cache; 44d7997fe1f4584da12e9c29fb682c18e9bdc13b73Timo Teräs atomic_t cpuleft; 45d7997fe1f4584da12e9c29fb682c18e9bdc13b73Timo Teräs struct completion completion; 461da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds}; 471da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 48d32d9bb85c65f52bed99a0149b47e9f6578c44c5Eric Dumazetstatic struct kmem_cache *flow_cachep __read_mostly; 49d32d9bb85c65f52bed99a0149b47e9f6578c44c5Eric Dumazet 50d7997fe1f4584da12e9c29fb682c18e9bdc13b73Timo Teräs#define flow_cache_hash_size(cache) (1 << (cache)->hash_shift) 51d7997fe1f4584da12e9c29fb682c18e9bdc13b73Timo Teräs#define FLOW_HASH_RND_PERIOD (10 * 60 * HZ) 521da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 531da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvaldsstatic void flow_cache_new_hashrnd(unsigned long arg) 541da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds{ 55d7997fe1f4584da12e9c29fb682c18e9bdc13b73Timo Teräs struct flow_cache *fc = (void *) arg; 561da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds int i; 571da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 586f912042256c12b0927438122594f5379b364f5dKAMEZAWA Hiroyuki for_each_possible_cpu(i) 59d7997fe1f4584da12e9c29fb682c18e9bdc13b73Timo Teräs per_cpu_ptr(fc->percpu, i)->hash_rnd_recalc = 1; 601da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 61d7997fe1f4584da12e9c29fb682c18e9bdc13b73Timo Teräs fc->rnd_timer.expires = jiffies + FLOW_HASH_RND_PERIOD; 62d7997fe1f4584da12e9c29fb682c18e9bdc13b73Timo Teräs add_timer(&fc->rnd_timer); 631da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds} 641da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 65ca925cf1534ebcec332c08719a7dee6ee1782ce4Fan Dustatic int flow_entry_valid(struct flow_cache_entry *fle, 66ca925cf1534ebcec332c08719a7dee6ee1782ce4Fan Du struct netns_xfrm *xfrm) 67fe1a5f031e76bd8761a7803d75b95ee96e84a574Timo Teräs{ 68ca925cf1534ebcec332c08719a7dee6ee1782ce4Fan Du if (atomic_read(&xfrm->flow_cache_genid) != fle->genid) 69fe1a5f031e76bd8761a7803d75b95ee96e84a574Timo Teräs return 0; 70fe1a5f031e76bd8761a7803d75b95ee96e84a574Timo Teräs if (fle->object && !fle->object->ops->check(fle->object)) 71fe1a5f031e76bd8761a7803d75b95ee96e84a574Timo Teräs return 0; 72fe1a5f031e76bd8761a7803d75b95ee96e84a574Timo Teräs return 1; 73fe1a5f031e76bd8761a7803d75b95ee96e84a574Timo Teräs} 74fe1a5f031e76bd8761a7803d75b95ee96e84a574Timo Teräs 75ca925cf1534ebcec332c08719a7dee6ee1782ce4Fan Dustatic void flow_entry_kill(struct flow_cache_entry *fle, 76ca925cf1534ebcec332c08719a7dee6ee1782ce4Fan Du struct netns_xfrm *xfrm) 77134b0fc544ba062498451611cb6f3e4454221b3dJames Morris{ 78134b0fc544ba062498451611cb6f3e4454221b3dJames Morris if (fle->object) 79fe1a5f031e76bd8761a7803d75b95ee96e84a574Timo Teräs fle->object->ops->delete(fle->object); 80d32d9bb85c65f52bed99a0149b47e9f6578c44c5Eric Dumazet kmem_cache_free(flow_cachep, fle); 818e4795605d1e1b39113818ad7c147b8a867a1f6aTimo Teräs} 828e4795605d1e1b39113818ad7c147b8a867a1f6aTimo Teräs 838e4795605d1e1b39113818ad7c147b8a867a1f6aTimo Terässtatic void flow_cache_gc_task(struct work_struct *work) 848e4795605d1e1b39113818ad7c147b8a867a1f6aTimo Teräs{ 858e4795605d1e1b39113818ad7c147b8a867a1f6aTimo Teräs struct list_head gc_list; 868e4795605d1e1b39113818ad7c147b8a867a1f6aTimo Teräs struct flow_cache_entry *fce, *n; 87ca925cf1534ebcec332c08719a7dee6ee1782ce4Fan Du struct netns_xfrm *xfrm = container_of(work, struct netns_xfrm, 88ca925cf1534ebcec332c08719a7dee6ee1782ce4Fan Du flow_cache_gc_work); 898e4795605d1e1b39113818ad7c147b8a867a1f6aTimo Teräs 908e4795605d1e1b39113818ad7c147b8a867a1f6aTimo Teräs INIT_LIST_HEAD(&gc_list); 91ca925cf1534ebcec332c08719a7dee6ee1782ce4Fan Du spin_lock_bh(&xfrm->flow_cache_gc_lock); 92ca925cf1534ebcec332c08719a7dee6ee1782ce4Fan Du list_splice_tail_init(&xfrm->flow_cache_gc_list, &gc_list); 93ca925cf1534ebcec332c08719a7dee6ee1782ce4Fan Du spin_unlock_bh(&xfrm->flow_cache_gc_lock); 948e4795605d1e1b39113818ad7c147b8a867a1f6aTimo Teräs 958e4795605d1e1b39113818ad7c147b8a867a1f6aTimo Teräs list_for_each_entry_safe(fce, n, &gc_list, u.gc_list) 96ca925cf1534ebcec332c08719a7dee6ee1782ce4Fan Du flow_entry_kill(fce, xfrm); 978e4795605d1e1b39113818ad7c147b8a867a1f6aTimo Teräs} 988e4795605d1e1b39113818ad7c147b8a867a1f6aTimo Teräs 998e4795605d1e1b39113818ad7c147b8a867a1f6aTimo Terässtatic void flow_cache_queue_garbage(struct flow_cache_percpu *fcp, 100ca925cf1534ebcec332c08719a7dee6ee1782ce4Fan Du int deleted, struct list_head *gc_list, 101ca925cf1534ebcec332c08719a7dee6ee1782ce4Fan Du struct netns_xfrm *xfrm) 1028e4795605d1e1b39113818ad7c147b8a867a1f6aTimo Teräs{ 1038e4795605d1e1b39113818ad7c147b8a867a1f6aTimo Teräs if (deleted) { 1048e4795605d1e1b39113818ad7c147b8a867a1f6aTimo Teräs fcp->hash_count -= deleted; 105ca925cf1534ebcec332c08719a7dee6ee1782ce4Fan Du spin_lock_bh(&xfrm->flow_cache_gc_lock); 106ca925cf1534ebcec332c08719a7dee6ee1782ce4Fan Du list_splice_tail(gc_list, &xfrm->flow_cache_gc_list); 107ca925cf1534ebcec332c08719a7dee6ee1782ce4Fan Du spin_unlock_bh(&xfrm->flow_cache_gc_lock); 108ca925cf1534ebcec332c08719a7dee6ee1782ce4Fan Du schedule_work(&xfrm->flow_cache_gc_work); 1098e4795605d1e1b39113818ad7c147b8a867a1f6aTimo Teräs } 110134b0fc544ba062498451611cb6f3e4454221b3dJames Morris} 111134b0fc544ba062498451611cb6f3e4454221b3dJames Morris 112d7997fe1f4584da12e9c29fb682c18e9bdc13b73Timo Terässtatic void __flow_cache_shrink(struct flow_cache *fc, 113d7997fe1f4584da12e9c29fb682c18e9bdc13b73Timo Teräs struct flow_cache_percpu *fcp, 114d7997fe1f4584da12e9c29fb682c18e9bdc13b73Timo Teräs int shrink_to) 1151da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds{ 1168e4795605d1e1b39113818ad7c147b8a867a1f6aTimo Teräs struct flow_cache_entry *fle; 117b67bfe0d42cac56c512dd5da4b1b347a23f4b70aSasha Levin struct hlist_node *tmp; 1188e4795605d1e1b39113818ad7c147b8a867a1f6aTimo Teräs LIST_HEAD(gc_list); 1198e4795605d1e1b39113818ad7c147b8a867a1f6aTimo Teräs int i, deleted = 0; 120ca925cf1534ebcec332c08719a7dee6ee1782ce4Fan Du struct netns_xfrm *xfrm = container_of(fc, struct netns_xfrm, 121ca925cf1534ebcec332c08719a7dee6ee1782ce4Fan Du flow_cache_global); 1221da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 123d7997fe1f4584da12e9c29fb682c18e9bdc13b73Timo Teräs for (i = 0; i < flow_cache_hash_size(fc); i++) { 124fe1a5f031e76bd8761a7803d75b95ee96e84a574Timo Teräs int saved = 0; 1251da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 126b67bfe0d42cac56c512dd5da4b1b347a23f4b70aSasha Levin hlist_for_each_entry_safe(fle, tmp, 1278e4795605d1e1b39113818ad7c147b8a867a1f6aTimo Teräs &fcp->hash_table[i], u.hlist) { 128fe1a5f031e76bd8761a7803d75b95ee96e84a574Timo Teräs if (saved < shrink_to && 129ca925cf1534ebcec332c08719a7dee6ee1782ce4Fan Du flow_entry_valid(fle, xfrm)) { 130fe1a5f031e76bd8761a7803d75b95ee96e84a574Timo Teräs saved++; 131fe1a5f031e76bd8761a7803d75b95ee96e84a574Timo Teräs } else { 1328e4795605d1e1b39113818ad7c147b8a867a1f6aTimo Teräs deleted++; 1338e4795605d1e1b39113818ad7c147b8a867a1f6aTimo Teräs hlist_del(&fle->u.hlist); 1348e4795605d1e1b39113818ad7c147b8a867a1f6aTimo Teräs list_add_tail(&fle->u.gc_list, &gc_list); 135fe1a5f031e76bd8761a7803d75b95ee96e84a574Timo Teräs } 1361da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds } 1371da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds } 1388e4795605d1e1b39113818ad7c147b8a867a1f6aTimo Teräs 139ca925cf1534ebcec332c08719a7dee6ee1782ce4Fan Du flow_cache_queue_garbage(fcp, deleted, &gc_list, xfrm); 1401da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds} 1411da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 142d7997fe1f4584da12e9c29fb682c18e9bdc13b73Timo Terässtatic void flow_cache_shrink(struct flow_cache *fc, 143d7997fe1f4584da12e9c29fb682c18e9bdc13b73Timo Teräs struct flow_cache_percpu *fcp) 1441da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds{ 145d7997fe1f4584da12e9c29fb682c18e9bdc13b73Timo Teräs int shrink_to = fc->low_watermark / flow_cache_hash_size(fc); 1461da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 147d7997fe1f4584da12e9c29fb682c18e9bdc13b73Timo Teräs __flow_cache_shrink(fc, fcp, shrink_to); 1481da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds} 1491da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 150d7997fe1f4584da12e9c29fb682c18e9bdc13b73Timo Terässtatic void flow_new_hash_rnd(struct flow_cache *fc, 151d7997fe1f4584da12e9c29fb682c18e9bdc13b73Timo Teräs struct flow_cache_percpu *fcp) 1521da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds{ 153d7997fe1f4584da12e9c29fb682c18e9bdc13b73Timo Teräs get_random_bytes(&fcp->hash_rnd, sizeof(u32)); 154d7997fe1f4584da12e9c29fb682c18e9bdc13b73Timo Teräs fcp->hash_rnd_recalc = 0; 155d7997fe1f4584da12e9c29fb682c18e9bdc13b73Timo Teräs __flow_cache_shrink(fc, fcp, 0); 1561da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds} 1571da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 158d7997fe1f4584da12e9c29fb682c18e9bdc13b73Timo Terässtatic u32 flow_hash_code(struct flow_cache *fc, 159d7997fe1f4584da12e9c29fb682c18e9bdc13b73Timo Teräs struct flow_cache_percpu *fcp, 160aa1c366e4febc7f5c2b84958a2dd7cd70e28f9d0dpward const struct flowi *key, 161aa1c366e4febc7f5c2b84958a2dd7cd70e28f9d0dpward size_t keysize) 1621da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds{ 163dee9f4bceb5fd9dbfcc1567148fccdbf16d6a38aDavid S. Miller const u32 *k = (const u32 *) key; 164aa1c366e4febc7f5c2b84958a2dd7cd70e28f9d0dpward const u32 length = keysize * sizeof(flow_compare_t) / sizeof(u32); 1651da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 166aa1c366e4febc7f5c2b84958a2dd7cd70e28f9d0dpward return jhash2(k, length, fcp->hash_rnd) 167a02cec2155fbea457eca8881870fd2de1a4c4c76Eric Dumazet & (flow_cache_hash_size(fc) - 1); 1681da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds} 1691da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 1701da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds/* I hear what you're saying, use memcmp. But memcmp cannot make 171aa1c366e4febc7f5c2b84958a2dd7cd70e28f9d0dpward * important assumptions that we can here, such as alignment. 1721da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds */ 173aa1c366e4febc7f5c2b84958a2dd7cd70e28f9d0dpwardstatic int flow_key_compare(const struct flowi *key1, const struct flowi *key2, 174aa1c366e4febc7f5c2b84958a2dd7cd70e28f9d0dpward size_t keysize) 1751da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds{ 176dee9f4bceb5fd9dbfcc1567148fccdbf16d6a38aDavid S. Miller const flow_compare_t *k1, *k1_lim, *k2; 1771da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 178dee9f4bceb5fd9dbfcc1567148fccdbf16d6a38aDavid S. Miller k1 = (const flow_compare_t *) key1; 179aa1c366e4febc7f5c2b84958a2dd7cd70e28f9d0dpward k1_lim = k1 + keysize; 1801da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 181dee9f4bceb5fd9dbfcc1567148fccdbf16d6a38aDavid S. Miller k2 = (const flow_compare_t *) key2; 1821da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 1831da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds do { 1841da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds if (*k1++ != *k2++) 1851da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds return 1; 1861da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds } while (k1 < k1_lim); 1871da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 1881da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds return 0; 1891da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds} 1901da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 191fe1a5f031e76bd8761a7803d75b95ee96e84a574Timo Terässtruct flow_cache_object * 192dee9f4bceb5fd9dbfcc1567148fccdbf16d6a38aDavid S. Millerflow_cache_lookup(struct net *net, const struct flowi *key, u16 family, u8 dir, 193fe1a5f031e76bd8761a7803d75b95ee96e84a574Timo Teräs flow_resolve_t resolver, void *ctx) 1941da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds{ 195ca925cf1534ebcec332c08719a7dee6ee1782ce4Fan Du struct flow_cache *fc = &net->xfrm.flow_cache_global; 196d7997fe1f4584da12e9c29fb682c18e9bdc13b73Timo Teräs struct flow_cache_percpu *fcp; 1978e4795605d1e1b39113818ad7c147b8a867a1f6aTimo Teräs struct flow_cache_entry *fle, *tfle; 198fe1a5f031e76bd8761a7803d75b95ee96e84a574Timo Teräs struct flow_cache_object *flo; 199aa1c366e4febc7f5c2b84958a2dd7cd70e28f9d0dpward size_t keysize; 2001da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds unsigned int hash; 2011da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 2021da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds local_bh_disable(); 2037a9b2d59507d85569b8a456688ef40cf2ac73e48Eric Dumazet fcp = this_cpu_ptr(fc->percpu); 2041da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 2051da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds fle = NULL; 206fe1a5f031e76bd8761a7803d75b95ee96e84a574Timo Teräs flo = NULL; 207aa1c366e4febc7f5c2b84958a2dd7cd70e28f9d0dpward 208aa1c366e4febc7f5c2b84958a2dd7cd70e28f9d0dpward keysize = flow_key_size(family); 209aa1c366e4febc7f5c2b84958a2dd7cd70e28f9d0dpward if (!keysize) 210aa1c366e4febc7f5c2b84958a2dd7cd70e28f9d0dpward goto nocache; 211aa1c366e4febc7f5c2b84958a2dd7cd70e28f9d0dpward 2121da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds /* Packet really early in init? Making flow_cache_init a 2131da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds * pre-smp initcall would solve this. --RR */ 214d7997fe1f4584da12e9c29fb682c18e9bdc13b73Timo Teräs if (!fcp->hash_table) 2151da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds goto nocache; 2161da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 217d7997fe1f4584da12e9c29fb682c18e9bdc13b73Timo Teräs if (fcp->hash_rnd_recalc) 218d7997fe1f4584da12e9c29fb682c18e9bdc13b73Timo Teräs flow_new_hash_rnd(fc, fcp); 2191da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 220aa1c366e4febc7f5c2b84958a2dd7cd70e28f9d0dpward hash = flow_hash_code(fc, fcp, key, keysize); 221b67bfe0d42cac56c512dd5da4b1b347a23f4b70aSasha Levin hlist_for_each_entry(tfle, &fcp->hash_table[hash], u.hlist) { 2220542b69e2c57fc9668ce6a03155bea6e1f557901dpward if (tfle->net == net && 2230542b69e2c57fc9668ce6a03155bea6e1f557901dpward tfle->family == family && 2248e4795605d1e1b39113818ad7c147b8a867a1f6aTimo Teräs tfle->dir == dir && 225aa1c366e4febc7f5c2b84958a2dd7cd70e28f9d0dpward flow_key_compare(key, &tfle->key, keysize) == 0) { 2268e4795605d1e1b39113818ad7c147b8a867a1f6aTimo Teräs fle = tfle; 2271da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds break; 2288e4795605d1e1b39113818ad7c147b8a867a1f6aTimo Teräs } 2291da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds } 2301da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 231fe1a5f031e76bd8761a7803d75b95ee96e84a574Timo Teräs if (unlikely(!fle)) { 232d7997fe1f4584da12e9c29fb682c18e9bdc13b73Timo Teräs if (fcp->hash_count > fc->high_watermark) 233d7997fe1f4584da12e9c29fb682c18e9bdc13b73Timo Teräs flow_cache_shrink(fc, fcp); 2341da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 235d32d9bb85c65f52bed99a0149b47e9f6578c44c5Eric Dumazet fle = kmem_cache_alloc(flow_cachep, GFP_ATOMIC); 2361da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds if (fle) { 2370542b69e2c57fc9668ce6a03155bea6e1f557901dpward fle->net = net; 2381da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds fle->family = family; 2391da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds fle->dir = dir; 240aa1c366e4febc7f5c2b84958a2dd7cd70e28f9d0dpward memcpy(&fle->key, key, keysize * sizeof(flow_compare_t)); 2411da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds fle->object = NULL; 2428e4795605d1e1b39113818ad7c147b8a867a1f6aTimo Teräs hlist_add_head(&fle->u.hlist, &fcp->hash_table[hash]); 243d7997fe1f4584da12e9c29fb682c18e9bdc13b73Timo Teräs fcp->hash_count++; 2441da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds } 245ca925cf1534ebcec332c08719a7dee6ee1782ce4Fan Du } else if (likely(fle->genid == atomic_read(&net->xfrm.flow_cache_genid))) { 246fe1a5f031e76bd8761a7803d75b95ee96e84a574Timo Teräs flo = fle->object; 247fe1a5f031e76bd8761a7803d75b95ee96e84a574Timo Teräs if (!flo) 248fe1a5f031e76bd8761a7803d75b95ee96e84a574Timo Teräs goto ret_object; 249fe1a5f031e76bd8761a7803d75b95ee96e84a574Timo Teräs flo = flo->ops->get(flo); 250fe1a5f031e76bd8761a7803d75b95ee96e84a574Timo Teräs if (flo) 251fe1a5f031e76bd8761a7803d75b95ee96e84a574Timo Teräs goto ret_object; 252fe1a5f031e76bd8761a7803d75b95ee96e84a574Timo Teräs } else if (fle->object) { 253fe1a5f031e76bd8761a7803d75b95ee96e84a574Timo Teräs flo = fle->object; 254fe1a5f031e76bd8761a7803d75b95ee96e84a574Timo Teräs flo->ops->delete(flo); 255fe1a5f031e76bd8761a7803d75b95ee96e84a574Timo Teräs fle->object = NULL; 2561da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds } 2571da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 2581da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvaldsnocache: 259fe1a5f031e76bd8761a7803d75b95ee96e84a574Timo Teräs flo = NULL; 260fe1a5f031e76bd8761a7803d75b95ee96e84a574Timo Teräs if (fle) { 261fe1a5f031e76bd8761a7803d75b95ee96e84a574Timo Teräs flo = fle->object; 262fe1a5f031e76bd8761a7803d75b95ee96e84a574Timo Teräs fle->object = NULL; 263fe1a5f031e76bd8761a7803d75b95ee96e84a574Timo Teräs } 264fe1a5f031e76bd8761a7803d75b95ee96e84a574Timo Teräs flo = resolver(net, key, family, dir, flo, ctx); 265fe1a5f031e76bd8761a7803d75b95ee96e84a574Timo Teräs if (fle) { 266ca925cf1534ebcec332c08719a7dee6ee1782ce4Fan Du fle->genid = atomic_read(&net->xfrm.flow_cache_genid); 267fe1a5f031e76bd8761a7803d75b95ee96e84a574Timo Teräs if (!IS_ERR(flo)) 268fe1a5f031e76bd8761a7803d75b95ee96e84a574Timo Teräs fle->object = flo; 269fe1a5f031e76bd8761a7803d75b95ee96e84a574Timo Teräs else 270fe1a5f031e76bd8761a7803d75b95ee96e84a574Timo Teräs fle->genid--; 271fe1a5f031e76bd8761a7803d75b95ee96e84a574Timo Teräs } else { 2728fbcec241df21d1ba2aba09974ea9017832b69b0YOSHIFUJI Hideaki / 吉藤英明 if (!IS_ERR_OR_NULL(flo)) 273fe1a5f031e76bd8761a7803d75b95ee96e84a574Timo Teräs flo->ops->delete(flo); 2741da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds } 275fe1a5f031e76bd8761a7803d75b95ee96e84a574Timo Teräsret_object: 276fe1a5f031e76bd8761a7803d75b95ee96e84a574Timo Teräs local_bh_enable(); 277fe1a5f031e76bd8761a7803d75b95ee96e84a574Timo Teräs return flo; 2781da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds} 2799e34a5b51684bc90ac827ec4ba339f3892632eacEric DumazetEXPORT_SYMBOL(flow_cache_lookup); 2801da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 2811da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvaldsstatic void flow_cache_flush_tasklet(unsigned long data) 2821da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds{ 2831da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds struct flow_flush_info *info = (void *)data; 284d7997fe1f4584da12e9c29fb682c18e9bdc13b73Timo Teräs struct flow_cache *fc = info->cache; 285d7997fe1f4584da12e9c29fb682c18e9bdc13b73Timo Teräs struct flow_cache_percpu *fcp; 2868e4795605d1e1b39113818ad7c147b8a867a1f6aTimo Teräs struct flow_cache_entry *fle; 287b67bfe0d42cac56c512dd5da4b1b347a23f4b70aSasha Levin struct hlist_node *tmp; 2888e4795605d1e1b39113818ad7c147b8a867a1f6aTimo Teräs LIST_HEAD(gc_list); 2898e4795605d1e1b39113818ad7c147b8a867a1f6aTimo Teräs int i, deleted = 0; 290ca925cf1534ebcec332c08719a7dee6ee1782ce4Fan Du struct netns_xfrm *xfrm = container_of(fc, struct netns_xfrm, 291ca925cf1534ebcec332c08719a7dee6ee1782ce4Fan Du flow_cache_global); 2921da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 2937a9b2d59507d85569b8a456688ef40cf2ac73e48Eric Dumazet fcp = this_cpu_ptr(fc->percpu); 294d7997fe1f4584da12e9c29fb682c18e9bdc13b73Timo Teräs for (i = 0; i < flow_cache_hash_size(fc); i++) { 295b67bfe0d42cac56c512dd5da4b1b347a23f4b70aSasha Levin hlist_for_each_entry_safe(fle, tmp, 2968e4795605d1e1b39113818ad7c147b8a867a1f6aTimo Teräs &fcp->hash_table[i], u.hlist) { 297ca925cf1534ebcec332c08719a7dee6ee1782ce4Fan Du if (flow_entry_valid(fle, xfrm)) 2981da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds continue; 2991da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 3008e4795605d1e1b39113818ad7c147b8a867a1f6aTimo Teräs deleted++; 3018e4795605d1e1b39113818ad7c147b8a867a1f6aTimo Teräs hlist_del(&fle->u.hlist); 3028e4795605d1e1b39113818ad7c147b8a867a1f6aTimo Teräs list_add_tail(&fle->u.gc_list, &gc_list); 3031da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds } 3041da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds } 3051da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 306ca925cf1534ebcec332c08719a7dee6ee1782ce4Fan Du flow_cache_queue_garbage(fcp, deleted, &gc_list, xfrm); 3078e4795605d1e1b39113818ad7c147b8a867a1f6aTimo Teräs 3081da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds if (atomic_dec_and_test(&info->cpuleft)) 3091da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds complete(&info->completion); 3101da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds} 3111da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 3128fdc929f5727d999d11ba3763b92f6eeacc096f9Chris Metcalf/* 3138fdc929f5727d999d11ba3763b92f6eeacc096f9Chris Metcalf * Return whether a cpu needs flushing. Conservatively, we assume 3148fdc929f5727d999d11ba3763b92f6eeacc096f9Chris Metcalf * the presence of any entries means the core may require flushing, 3158fdc929f5727d999d11ba3763b92f6eeacc096f9Chris Metcalf * since the flow_cache_ops.check() function may assume it's running 3168fdc929f5727d999d11ba3763b92f6eeacc096f9Chris Metcalf * on the same core as the per-cpu cache component. 3178fdc929f5727d999d11ba3763b92f6eeacc096f9Chris Metcalf */ 3188fdc929f5727d999d11ba3763b92f6eeacc096f9Chris Metcalfstatic int flow_cache_percpu_empty(struct flow_cache *fc, int cpu) 3198fdc929f5727d999d11ba3763b92f6eeacc096f9Chris Metcalf{ 3208fdc929f5727d999d11ba3763b92f6eeacc096f9Chris Metcalf struct flow_cache_percpu *fcp; 3218fdc929f5727d999d11ba3763b92f6eeacc096f9Chris Metcalf int i; 3228fdc929f5727d999d11ba3763b92f6eeacc096f9Chris Metcalf 323278150321a3f5af85803e1214807ca5cfbace0e1Li RongQing fcp = per_cpu_ptr(fc->percpu, cpu); 3248fdc929f5727d999d11ba3763b92f6eeacc096f9Chris Metcalf for (i = 0; i < flow_cache_hash_size(fc); i++) 3258fdc929f5727d999d11ba3763b92f6eeacc096f9Chris Metcalf if (!hlist_empty(&fcp->hash_table[i])) 3268fdc929f5727d999d11ba3763b92f6eeacc096f9Chris Metcalf return 0; 3278fdc929f5727d999d11ba3763b92f6eeacc096f9Chris Metcalf return 1; 3288fdc929f5727d999d11ba3763b92f6eeacc096f9Chris Metcalf} 3298fdc929f5727d999d11ba3763b92f6eeacc096f9Chris Metcalf 3301da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvaldsstatic void flow_cache_flush_per_cpu(void *data) 3311da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds{ 3321da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds struct flow_flush_info *info = data; 3331da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds struct tasklet_struct *tasklet; 3341da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 33550eab0503a7579ada512e4968738b7c9737cf36eLi RongQing tasklet = &this_cpu_ptr(info->cache->percpu)->flush_tasklet; 3361da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds tasklet->data = (unsigned long)info; 3371da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds tasklet_schedule(tasklet); 3381da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds} 3391da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 340ca925cf1534ebcec332c08719a7dee6ee1782ce4Fan Duvoid flow_cache_flush(struct net *net) 3411da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds{ 3421da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds struct flow_flush_info info; 3438fdc929f5727d999d11ba3763b92f6eeacc096f9Chris Metcalf cpumask_var_t mask; 3448fdc929f5727d999d11ba3763b92f6eeacc096f9Chris Metcalf int i, self; 3458fdc929f5727d999d11ba3763b92f6eeacc096f9Chris Metcalf 3468fdc929f5727d999d11ba3763b92f6eeacc096f9Chris Metcalf /* Track which cpus need flushing to avoid disturbing all cores. */ 3478fdc929f5727d999d11ba3763b92f6eeacc096f9Chris Metcalf if (!alloc_cpumask_var(&mask, GFP_KERNEL)) 3488fdc929f5727d999d11ba3763b92f6eeacc096f9Chris Metcalf return; 3498fdc929f5727d999d11ba3763b92f6eeacc096f9Chris Metcalf cpumask_clear(mask); 3501da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 3511da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds /* Don't want cpus going down or up during this. */ 35286ef5c9a8edd78e6bf92879f32329d89b2d55b5aGautham R Shenoy get_online_cpus(); 353ca925cf1534ebcec332c08719a7dee6ee1782ce4Fan Du mutex_lock(&net->xfrm.flow_flush_sem); 354ca925cf1534ebcec332c08719a7dee6ee1782ce4Fan Du info.cache = &net->xfrm.flow_cache_global; 3558fdc929f5727d999d11ba3763b92f6eeacc096f9Chris Metcalf for_each_online_cpu(i) 3568fdc929f5727d999d11ba3763b92f6eeacc096f9Chris Metcalf if (!flow_cache_percpu_empty(info.cache, i)) 3578fdc929f5727d999d11ba3763b92f6eeacc096f9Chris Metcalf cpumask_set_cpu(i, mask); 3588fdc929f5727d999d11ba3763b92f6eeacc096f9Chris Metcalf atomic_set(&info.cpuleft, cpumask_weight(mask)); 3598fdc929f5727d999d11ba3763b92f6eeacc096f9Chris Metcalf if (atomic_read(&info.cpuleft) == 0) 3608fdc929f5727d999d11ba3763b92f6eeacc096f9Chris Metcalf goto done; 3618fdc929f5727d999d11ba3763b92f6eeacc096f9Chris Metcalf 3621da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds init_completion(&info.completion); 3631da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 3641da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds local_bh_disable(); 3658fdc929f5727d999d11ba3763b92f6eeacc096f9Chris Metcalf self = cpumask_test_and_clear_cpu(smp_processor_id(), mask); 3668fdc929f5727d999d11ba3763b92f6eeacc096f9Chris Metcalf on_each_cpu_mask(mask, flow_cache_flush_per_cpu, &info, 0); 3678fdc929f5727d999d11ba3763b92f6eeacc096f9Chris Metcalf if (self) 3688fdc929f5727d999d11ba3763b92f6eeacc096f9Chris Metcalf flow_cache_flush_tasklet((unsigned long)&info); 3691da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds local_bh_enable(); 3701da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 3711da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds wait_for_completion(&info.completion); 3728fdc929f5727d999d11ba3763b92f6eeacc096f9Chris Metcalf 3738fdc929f5727d999d11ba3763b92f6eeacc096f9Chris Metcalfdone: 374ca925cf1534ebcec332c08719a7dee6ee1782ce4Fan Du mutex_unlock(&net->xfrm.flow_flush_sem); 37586ef5c9a8edd78e6bf92879f32329d89b2d55b5aGautham R Shenoy put_online_cpus(); 3768fdc929f5727d999d11ba3763b92f6eeacc096f9Chris Metcalf free_cpumask_var(mask); 3771da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds} 3781da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 379c0ed1c14a72ca9ebacd51fb94a8aca488b0d361eSteffen Klassertstatic void flow_cache_flush_task(struct work_struct *work) 380c0ed1c14a72ca9ebacd51fb94a8aca488b0d361eSteffen Klassert{ 381ca925cf1534ebcec332c08719a7dee6ee1782ce4Fan Du struct netns_xfrm *xfrm = container_of(work, struct netns_xfrm, 382ca925cf1534ebcec332c08719a7dee6ee1782ce4Fan Du flow_cache_gc_work); 383ca925cf1534ebcec332c08719a7dee6ee1782ce4Fan Du struct net *net = container_of(xfrm, struct net, xfrm); 384c0ed1c14a72ca9ebacd51fb94a8aca488b0d361eSteffen Klassert 385ca925cf1534ebcec332c08719a7dee6ee1782ce4Fan Du flow_cache_flush(net); 386ca925cf1534ebcec332c08719a7dee6ee1782ce4Fan Du} 387c0ed1c14a72ca9ebacd51fb94a8aca488b0d361eSteffen Klassert 388ca925cf1534ebcec332c08719a7dee6ee1782ce4Fan Duvoid flow_cache_flush_deferred(struct net *net) 389c0ed1c14a72ca9ebacd51fb94a8aca488b0d361eSteffen Klassert{ 390ca925cf1534ebcec332c08719a7dee6ee1782ce4Fan Du schedule_work(&net->xfrm.flow_cache_flush_work); 391c0ed1c14a72ca9ebacd51fb94a8aca488b0d361eSteffen Klassert} 392c0ed1c14a72ca9ebacd51fb94a8aca488b0d361eSteffen Klassert 393013dbb325be702d777118d5e4ffefff3dad2b153Paul Gortmakerstatic int flow_cache_cpu_prepare(struct flow_cache *fc, int cpu) 3941da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds{ 39583b6b1f5d13414d0cb5c4f0a567a6aec0af073bdEric Dumazet struct flow_cache_percpu *fcp = per_cpu_ptr(fc->percpu, cpu); 39683b6b1f5d13414d0cb5c4f0a567a6aec0af073bdEric Dumazet size_t sz = sizeof(struct hlist_head) * flow_cache_hash_size(fc); 397d7997fe1f4584da12e9c29fb682c18e9bdc13b73Timo Teräs 39883b6b1f5d13414d0cb5c4f0a567a6aec0af073bdEric Dumazet if (!fcp->hash_table) { 39983b6b1f5d13414d0cb5c4f0a567a6aec0af073bdEric Dumazet fcp->hash_table = kzalloc_node(sz, GFP_KERNEL, cpu_to_node(cpu)); 40083b6b1f5d13414d0cb5c4f0a567a6aec0af073bdEric Dumazet if (!fcp->hash_table) { 40183b6b1f5d13414d0cb5c4f0a567a6aec0af073bdEric Dumazet pr_err("NET: failed to allocate flow cache sz %zu\n", sz); 40283b6b1f5d13414d0cb5c4f0a567a6aec0af073bdEric Dumazet return -ENOMEM; 40383b6b1f5d13414d0cb5c4f0a567a6aec0af073bdEric Dumazet } 40483b6b1f5d13414d0cb5c4f0a567a6aec0af073bdEric Dumazet fcp->hash_rnd_recalc = 1; 40583b6b1f5d13414d0cb5c4f0a567a6aec0af073bdEric Dumazet fcp->hash_count = 0; 40683b6b1f5d13414d0cb5c4f0a567a6aec0af073bdEric Dumazet tasklet_init(&fcp->flush_tasklet, flow_cache_flush_tasklet, 0); 40783b6b1f5d13414d0cb5c4f0a567a6aec0af073bdEric Dumazet } 40883b6b1f5d13414d0cb5c4f0a567a6aec0af073bdEric Dumazet return 0; 4091da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds} 4101da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 411013dbb325be702d777118d5e4ffefff3dad2b153Paul Gortmakerstatic int flow_cache_cpu(struct notifier_block *nfb, 4121da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds unsigned long action, 4131da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds void *hcpu) 4141da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds{ 415ca925cf1534ebcec332c08719a7dee6ee1782ce4Fan Du struct flow_cache *fc = container_of(nfb, struct flow_cache, 416ca925cf1534ebcec332c08719a7dee6ee1782ce4Fan Du hotcpu_notifier); 41783b6b1f5d13414d0cb5c4f0a567a6aec0af073bdEric Dumazet int res, cpu = (unsigned long) hcpu; 418d7997fe1f4584da12e9c29fb682c18e9bdc13b73Timo Teräs struct flow_cache_percpu *fcp = per_cpu_ptr(fc->percpu, cpu); 419d7997fe1f4584da12e9c29fb682c18e9bdc13b73Timo Teräs 42083b6b1f5d13414d0cb5c4f0a567a6aec0af073bdEric Dumazet switch (action) { 42183b6b1f5d13414d0cb5c4f0a567a6aec0af073bdEric Dumazet case CPU_UP_PREPARE: 42283b6b1f5d13414d0cb5c4f0a567a6aec0af073bdEric Dumazet case CPU_UP_PREPARE_FROZEN: 42383b6b1f5d13414d0cb5c4f0a567a6aec0af073bdEric Dumazet res = flow_cache_cpu_prepare(fc, cpu); 42483b6b1f5d13414d0cb5c4f0a567a6aec0af073bdEric Dumazet if (res) 42583b6b1f5d13414d0cb5c4f0a567a6aec0af073bdEric Dumazet return notifier_from_errno(res); 42683b6b1f5d13414d0cb5c4f0a567a6aec0af073bdEric Dumazet break; 42783b6b1f5d13414d0cb5c4f0a567a6aec0af073bdEric Dumazet case CPU_DEAD: 42883b6b1f5d13414d0cb5c4f0a567a6aec0af073bdEric Dumazet case CPU_DEAD_FROZEN: 429d7997fe1f4584da12e9c29fb682c18e9bdc13b73Timo Teräs __flow_cache_shrink(fc, fcp, 0); 43083b6b1f5d13414d0cb5c4f0a567a6aec0af073bdEric Dumazet break; 43183b6b1f5d13414d0cb5c4f0a567a6aec0af073bdEric Dumazet } 4321da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds return NOTIFY_OK; 4331da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds} 4341da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 435ca925cf1534ebcec332c08719a7dee6ee1782ce4Fan Duint flow_cache_init(struct net *net) 4361da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds{ 4371da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds int i; 438ca925cf1534ebcec332c08719a7dee6ee1782ce4Fan Du struct flow_cache *fc = &net->xfrm.flow_cache_global; 439ca925cf1534ebcec332c08719a7dee6ee1782ce4Fan Du 440d32d9bb85c65f52bed99a0149b47e9f6578c44c5Eric Dumazet if (!flow_cachep) 441d32d9bb85c65f52bed99a0149b47e9f6578c44c5Eric Dumazet flow_cachep = kmem_cache_create("flow_cache", 442d32d9bb85c65f52bed99a0149b47e9f6578c44c5Eric Dumazet sizeof(struct flow_cache_entry), 443d32d9bb85c65f52bed99a0149b47e9f6578c44c5Eric Dumazet 0, SLAB_PANIC, NULL); 444ca925cf1534ebcec332c08719a7dee6ee1782ce4Fan Du spin_lock_init(&net->xfrm.flow_cache_gc_lock); 445ca925cf1534ebcec332c08719a7dee6ee1782ce4Fan Du INIT_LIST_HEAD(&net->xfrm.flow_cache_gc_list); 446ca925cf1534ebcec332c08719a7dee6ee1782ce4Fan Du INIT_WORK(&net->xfrm.flow_cache_gc_work, flow_cache_gc_task); 447ca925cf1534ebcec332c08719a7dee6ee1782ce4Fan Du INIT_WORK(&net->xfrm.flow_cache_flush_work, flow_cache_flush_task); 448ca925cf1534ebcec332c08719a7dee6ee1782ce4Fan Du mutex_init(&net->xfrm.flow_flush_sem); 4491da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 450d7997fe1f4584da12e9c29fb682c18e9bdc13b73Timo Teräs fc->hash_shift = 10; 451d7997fe1f4584da12e9c29fb682c18e9bdc13b73Timo Teräs fc->low_watermark = 2 * flow_cache_hash_size(fc); 452d7997fe1f4584da12e9c29fb682c18e9bdc13b73Timo Teräs fc->high_watermark = 4 * flow_cache_hash_size(fc); 453d7997fe1f4584da12e9c29fb682c18e9bdc13b73Timo Teräs 454d7997fe1f4584da12e9c29fb682c18e9bdc13b73Timo Teräs fc->percpu = alloc_percpu(struct flow_cache_percpu); 45583b6b1f5d13414d0cb5c4f0a567a6aec0af073bdEric Dumazet if (!fc->percpu) 45683b6b1f5d13414d0cb5c4f0a567a6aec0af073bdEric Dumazet return -ENOMEM; 4571da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 458e30a293e8ad7e6048d6d88bcc114094f964bd67bSrivatsa S. Bhat cpu_notifier_register_begin(); 459e30a293e8ad7e6048d6d88bcc114094f964bd67bSrivatsa S. Bhat 46083b6b1f5d13414d0cb5c4f0a567a6aec0af073bdEric Dumazet for_each_online_cpu(i) { 46183b6b1f5d13414d0cb5c4f0a567a6aec0af073bdEric Dumazet if (flow_cache_cpu_prepare(fc, i)) 4626ccc3abdc97e07e6895323fdab89f84e58b40ecehuajun li goto err; 46383b6b1f5d13414d0cb5c4f0a567a6aec0af073bdEric Dumazet } 464d7997fe1f4584da12e9c29fb682c18e9bdc13b73Timo Teräs fc->hotcpu_notifier = (struct notifier_block){ 465d7997fe1f4584da12e9c29fb682c18e9bdc13b73Timo Teräs .notifier_call = flow_cache_cpu, 466d7997fe1f4584da12e9c29fb682c18e9bdc13b73Timo Teräs }; 467e30a293e8ad7e6048d6d88bcc114094f964bd67bSrivatsa S. Bhat __register_hotcpu_notifier(&fc->hotcpu_notifier); 468e30a293e8ad7e6048d6d88bcc114094f964bd67bSrivatsa S. Bhat 469e30a293e8ad7e6048d6d88bcc114094f964bd67bSrivatsa S. Bhat cpu_notifier_register_done(); 4701da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 47183b6b1f5d13414d0cb5c4f0a567a6aec0af073bdEric Dumazet setup_timer(&fc->rnd_timer, flow_cache_new_hashrnd, 47283b6b1f5d13414d0cb5c4f0a567a6aec0af073bdEric Dumazet (unsigned long) fc); 47383b6b1f5d13414d0cb5c4f0a567a6aec0af073bdEric Dumazet fc->rnd_timer.expires = jiffies + FLOW_HASH_RND_PERIOD; 47483b6b1f5d13414d0cb5c4f0a567a6aec0af073bdEric Dumazet add_timer(&fc->rnd_timer); 47583b6b1f5d13414d0cb5c4f0a567a6aec0af073bdEric Dumazet 4761da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds return 0; 4776ccc3abdc97e07e6895323fdab89f84e58b40ecehuajun li 4786ccc3abdc97e07e6895323fdab89f84e58b40ecehuajun lierr: 4796ccc3abdc97e07e6895323fdab89f84e58b40ecehuajun li for_each_possible_cpu(i) { 4806ccc3abdc97e07e6895323fdab89f84e58b40ecehuajun li struct flow_cache_percpu *fcp = per_cpu_ptr(fc->percpu, i); 4816ccc3abdc97e07e6895323fdab89f84e58b40ecehuajun li kfree(fcp->hash_table); 4826ccc3abdc97e07e6895323fdab89f84e58b40ecehuajun li fcp->hash_table = NULL; 4836ccc3abdc97e07e6895323fdab89f84e58b40ecehuajun li } 4846ccc3abdc97e07e6895323fdab89f84e58b40ecehuajun li 485e30a293e8ad7e6048d6d88bcc114094f964bd67bSrivatsa S. Bhat cpu_notifier_register_done(); 486e30a293e8ad7e6048d6d88bcc114094f964bd67bSrivatsa S. Bhat 4876ccc3abdc97e07e6895323fdab89f84e58b40ecehuajun li free_percpu(fc->percpu); 4886ccc3abdc97e07e6895323fdab89f84e58b40ecehuajun li fc->percpu = NULL; 4896ccc3abdc97e07e6895323fdab89f84e58b40ecehuajun li 4906ccc3abdc97e07e6895323fdab89f84e58b40ecehuajun li return -ENOMEM; 4911da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds} 492ca925cf1534ebcec332c08719a7dee6ee1782ce4Fan DuEXPORT_SYMBOL(flow_cache_init); 4934a93f5095a628d812b0b30c16d7bacea1efd783cSteffen Klassert 4944a93f5095a628d812b0b30c16d7bacea1efd783cSteffen Klassertvoid flow_cache_fini(struct net *net) 4954a93f5095a628d812b0b30c16d7bacea1efd783cSteffen Klassert{ 4964a93f5095a628d812b0b30c16d7bacea1efd783cSteffen Klassert int i; 4974a93f5095a628d812b0b30c16d7bacea1efd783cSteffen Klassert struct flow_cache *fc = &net->xfrm.flow_cache_global; 4984a93f5095a628d812b0b30c16d7bacea1efd783cSteffen Klassert 4994a93f5095a628d812b0b30c16d7bacea1efd783cSteffen Klassert del_timer_sync(&fc->rnd_timer); 5004a93f5095a628d812b0b30c16d7bacea1efd783cSteffen Klassert unregister_hotcpu_notifier(&fc->hotcpu_notifier); 5014a93f5095a628d812b0b30c16d7bacea1efd783cSteffen Klassert 5024a93f5095a628d812b0b30c16d7bacea1efd783cSteffen Klassert for_each_possible_cpu(i) { 5034a93f5095a628d812b0b30c16d7bacea1efd783cSteffen Klassert struct flow_cache_percpu *fcp = per_cpu_ptr(fc->percpu, i); 5044a93f5095a628d812b0b30c16d7bacea1efd783cSteffen Klassert kfree(fcp->hash_table); 5054a93f5095a628d812b0b30c16d7bacea1efd783cSteffen Klassert fcp->hash_table = NULL; 5064a93f5095a628d812b0b30c16d7bacea1efd783cSteffen Klassert } 5074a93f5095a628d812b0b30c16d7bacea1efd783cSteffen Klassert 5084a93f5095a628d812b0b30c16d7bacea1efd783cSteffen Klassert free_percpu(fc->percpu); 5094a93f5095a628d812b0b30c16d7bacea1efd783cSteffen Klassert fc->percpu = NULL; 5104a93f5095a628d812b0b30c16d7bacea1efd783cSteffen Klassert} 5114a93f5095a628d812b0b30c16d7bacea1efd783cSteffen KlassertEXPORT_SYMBOL(flow_cache_fini); 512