flow.c revision 8fbcec241df21d1ba2aba09974ea9017832b69b0
11da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds/* flow.c: Generic flow cache. 21da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds * 31da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds * Copyright (C) 2003 Alexey N. Kuznetsov (kuznet@ms2.inr.ac.ru) 41da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds * Copyright (C) 2003 David S. Miller (davem@redhat.com) 51da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds */ 61da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 71da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds#include <linux/kernel.h> 81da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds#include <linux/module.h> 91da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds#include <linux/list.h> 101da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds#include <linux/jhash.h> 111da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds#include <linux/interrupt.h> 121da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds#include <linux/mm.h> 131da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds#include <linux/random.h> 141da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds#include <linux/init.h> 151da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds#include <linux/slab.h> 161da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds#include <linux/smp.h> 171da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds#include <linux/completion.h> 181da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds#include <linux/percpu.h> 191da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds#include <linux/bitops.h> 201da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds#include <linux/notifier.h> 211da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds#include <linux/cpu.h> 221da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds#include <linux/cpumask.h> 234a3e2f711a00a1feb72ae12fdc749da10179d185Arjan van de Ven#include <linux/mutex.h> 241da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds#include <net/flow.h> 2560063497a95e716c9a689af3be2687d261f115b4Arun Sharma#include <linux/atomic.h> 26df71837d5024e2524cd51c93621e558aa7dd9f3fTrent Jaeger#include <linux/security.h> 271da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 281da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvaldsstruct flow_cache_entry { 298e4795605d1e1b39113818ad7c147b8a867a1f6aTimo Teräs union { 308e4795605d1e1b39113818ad7c147b8a867a1f6aTimo Teräs struct hlist_node hlist; 318e4795605d1e1b39113818ad7c147b8a867a1f6aTimo Teräs struct list_head gc_list; 328e4795605d1e1b39113818ad7c147b8a867a1f6aTimo Teräs } u; 330542b69e2c57fc9668ce6a03155bea6e1f557901dpward struct net *net; 34fe1a5f031e76bd8761a7803d75b95ee96e84a574Timo Teräs u16 family; 35fe1a5f031e76bd8761a7803d75b95ee96e84a574Timo Teräs u8 dir; 36fe1a5f031e76bd8761a7803d75b95ee96e84a574Timo Teräs u32 genid; 37fe1a5f031e76bd8761a7803d75b95ee96e84a574Timo Teräs struct flowi key; 38fe1a5f031e76bd8761a7803d75b95ee96e84a574Timo Teräs struct flow_cache_object *object; 391da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds}; 401da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 41d7997fe1f4584da12e9c29fb682c18e9bdc13b73Timo Terässtruct flow_cache_percpu { 428e4795605d1e1b39113818ad7c147b8a867a1f6aTimo Teräs struct hlist_head *hash_table; 43d7997fe1f4584da12e9c29fb682c18e9bdc13b73Timo Teräs int hash_count; 44d7997fe1f4584da12e9c29fb682c18e9bdc13b73Timo Teräs u32 hash_rnd; 45d7997fe1f4584da12e9c29fb682c18e9bdc13b73Timo Teräs int hash_rnd_recalc; 46d7997fe1f4584da12e9c29fb682c18e9bdc13b73Timo Teräs struct tasklet_struct flush_tasklet; 475f58a5c8725b48f3e32851f9748527c8d1ff71b2Eric Dumazet}; 481da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 491da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvaldsstruct flow_flush_info { 50fe1a5f031e76bd8761a7803d75b95ee96e84a574Timo Teräs struct flow_cache *cache; 51d7997fe1f4584da12e9c29fb682c18e9bdc13b73Timo Teräs atomic_t cpuleft; 52d7997fe1f4584da12e9c29fb682c18e9bdc13b73Timo Teräs struct completion completion; 531da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds}; 541da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 55d7997fe1f4584da12e9c29fb682c18e9bdc13b73Timo Terässtruct flow_cache { 56d7997fe1f4584da12e9c29fb682c18e9bdc13b73Timo Teräs u32 hash_shift; 5783b6b1f5d13414d0cb5c4f0a567a6aec0af073bdEric Dumazet struct flow_cache_percpu __percpu *percpu; 58d7997fe1f4584da12e9c29fb682c18e9bdc13b73Timo Teräs struct notifier_block hotcpu_notifier; 59d7997fe1f4584da12e9c29fb682c18e9bdc13b73Timo Teräs int low_watermark; 60d7997fe1f4584da12e9c29fb682c18e9bdc13b73Timo Teräs int high_watermark; 61d7997fe1f4584da12e9c29fb682c18e9bdc13b73Timo Teräs struct timer_list rnd_timer; 62d7997fe1f4584da12e9c29fb682c18e9bdc13b73Timo Teräs}; 63d7997fe1f4584da12e9c29fb682c18e9bdc13b73Timo Teräs 64d7997fe1f4584da12e9c29fb682c18e9bdc13b73Timo Teräsatomic_t flow_cache_genid = ATOMIC_INIT(0); 659e34a5b51684bc90ac827ec4ba339f3892632eacEric DumazetEXPORT_SYMBOL(flow_cache_genid); 66d7997fe1f4584da12e9c29fb682c18e9bdc13b73Timo Terässtatic struct flow_cache flow_cache_global; 6783b6b1f5d13414d0cb5c4f0a567a6aec0af073bdEric Dumazetstatic struct kmem_cache *flow_cachep __read_mostly; 68d7997fe1f4584da12e9c29fb682c18e9bdc13b73Timo Teräs 698e4795605d1e1b39113818ad7c147b8a867a1f6aTimo Terässtatic DEFINE_SPINLOCK(flow_cache_gc_lock); 708e4795605d1e1b39113818ad7c147b8a867a1f6aTimo Terässtatic LIST_HEAD(flow_cache_gc_list); 718e4795605d1e1b39113818ad7c147b8a867a1f6aTimo Teräs 72d7997fe1f4584da12e9c29fb682c18e9bdc13b73Timo Teräs#define flow_cache_hash_size(cache) (1 << (cache)->hash_shift) 73d7997fe1f4584da12e9c29fb682c18e9bdc13b73Timo Teräs#define FLOW_HASH_RND_PERIOD (10 * 60 * HZ) 741da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 751da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvaldsstatic void flow_cache_new_hashrnd(unsigned long arg) 761da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds{ 77d7997fe1f4584da12e9c29fb682c18e9bdc13b73Timo Teräs struct flow_cache *fc = (void *) arg; 781da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds int i; 791da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 806f912042256c12b0927438122594f5379b364f5dKAMEZAWA Hiroyuki for_each_possible_cpu(i) 81d7997fe1f4584da12e9c29fb682c18e9bdc13b73Timo Teräs per_cpu_ptr(fc->percpu, i)->hash_rnd_recalc = 1; 821da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 83d7997fe1f4584da12e9c29fb682c18e9bdc13b73Timo Teräs fc->rnd_timer.expires = jiffies + FLOW_HASH_RND_PERIOD; 84d7997fe1f4584da12e9c29fb682c18e9bdc13b73Timo Teräs add_timer(&fc->rnd_timer); 851da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds} 861da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 87fe1a5f031e76bd8761a7803d75b95ee96e84a574Timo Terässtatic int flow_entry_valid(struct flow_cache_entry *fle) 88fe1a5f031e76bd8761a7803d75b95ee96e84a574Timo Teräs{ 89fe1a5f031e76bd8761a7803d75b95ee96e84a574Timo Teräs if (atomic_read(&flow_cache_genid) != fle->genid) 90fe1a5f031e76bd8761a7803d75b95ee96e84a574Timo Teräs return 0; 91fe1a5f031e76bd8761a7803d75b95ee96e84a574Timo Teräs if (fle->object && !fle->object->ops->check(fle->object)) 92fe1a5f031e76bd8761a7803d75b95ee96e84a574Timo Teräs return 0; 93fe1a5f031e76bd8761a7803d75b95ee96e84a574Timo Teräs return 1; 94fe1a5f031e76bd8761a7803d75b95ee96e84a574Timo Teräs} 95fe1a5f031e76bd8761a7803d75b95ee96e84a574Timo Teräs 968e4795605d1e1b39113818ad7c147b8a867a1f6aTimo Terässtatic void flow_entry_kill(struct flow_cache_entry *fle) 97134b0fc544ba062498451611cb6f3e4454221b3dJames Morris{ 98134b0fc544ba062498451611cb6f3e4454221b3dJames Morris if (fle->object) 99fe1a5f031e76bd8761a7803d75b95ee96e84a574Timo Teräs fle->object->ops->delete(fle->object); 100134b0fc544ba062498451611cb6f3e4454221b3dJames Morris kmem_cache_free(flow_cachep, fle); 1018e4795605d1e1b39113818ad7c147b8a867a1f6aTimo Teräs} 1028e4795605d1e1b39113818ad7c147b8a867a1f6aTimo Teräs 1038e4795605d1e1b39113818ad7c147b8a867a1f6aTimo Terässtatic void flow_cache_gc_task(struct work_struct *work) 1048e4795605d1e1b39113818ad7c147b8a867a1f6aTimo Teräs{ 1058e4795605d1e1b39113818ad7c147b8a867a1f6aTimo Teräs struct list_head gc_list; 1068e4795605d1e1b39113818ad7c147b8a867a1f6aTimo Teräs struct flow_cache_entry *fce, *n; 1078e4795605d1e1b39113818ad7c147b8a867a1f6aTimo Teräs 1088e4795605d1e1b39113818ad7c147b8a867a1f6aTimo Teräs INIT_LIST_HEAD(&gc_list); 1098e4795605d1e1b39113818ad7c147b8a867a1f6aTimo Teräs spin_lock_bh(&flow_cache_gc_lock); 1108e4795605d1e1b39113818ad7c147b8a867a1f6aTimo Teräs list_splice_tail_init(&flow_cache_gc_list, &gc_list); 1118e4795605d1e1b39113818ad7c147b8a867a1f6aTimo Teräs spin_unlock_bh(&flow_cache_gc_lock); 1128e4795605d1e1b39113818ad7c147b8a867a1f6aTimo Teräs 1138e4795605d1e1b39113818ad7c147b8a867a1f6aTimo Teräs list_for_each_entry_safe(fce, n, &gc_list, u.gc_list) 1148e4795605d1e1b39113818ad7c147b8a867a1f6aTimo Teräs flow_entry_kill(fce); 1158e4795605d1e1b39113818ad7c147b8a867a1f6aTimo Teräs} 1168e4795605d1e1b39113818ad7c147b8a867a1f6aTimo Terässtatic DECLARE_WORK(flow_cache_gc_work, flow_cache_gc_task); 1178e4795605d1e1b39113818ad7c147b8a867a1f6aTimo Teräs 1188e4795605d1e1b39113818ad7c147b8a867a1f6aTimo Terässtatic void flow_cache_queue_garbage(struct flow_cache_percpu *fcp, 1198e4795605d1e1b39113818ad7c147b8a867a1f6aTimo Teräs int deleted, struct list_head *gc_list) 1208e4795605d1e1b39113818ad7c147b8a867a1f6aTimo Teräs{ 1218e4795605d1e1b39113818ad7c147b8a867a1f6aTimo Teräs if (deleted) { 1228e4795605d1e1b39113818ad7c147b8a867a1f6aTimo Teräs fcp->hash_count -= deleted; 1238e4795605d1e1b39113818ad7c147b8a867a1f6aTimo Teräs spin_lock_bh(&flow_cache_gc_lock); 1248e4795605d1e1b39113818ad7c147b8a867a1f6aTimo Teräs list_splice_tail(gc_list, &flow_cache_gc_list); 1258e4795605d1e1b39113818ad7c147b8a867a1f6aTimo Teräs spin_unlock_bh(&flow_cache_gc_lock); 1268e4795605d1e1b39113818ad7c147b8a867a1f6aTimo Teräs schedule_work(&flow_cache_gc_work); 1278e4795605d1e1b39113818ad7c147b8a867a1f6aTimo Teräs } 128134b0fc544ba062498451611cb6f3e4454221b3dJames Morris} 129134b0fc544ba062498451611cb6f3e4454221b3dJames Morris 130d7997fe1f4584da12e9c29fb682c18e9bdc13b73Timo Terässtatic void __flow_cache_shrink(struct flow_cache *fc, 131d7997fe1f4584da12e9c29fb682c18e9bdc13b73Timo Teräs struct flow_cache_percpu *fcp, 132d7997fe1f4584da12e9c29fb682c18e9bdc13b73Timo Teräs int shrink_to) 1331da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds{ 1348e4795605d1e1b39113818ad7c147b8a867a1f6aTimo Teräs struct flow_cache_entry *fle; 1358e4795605d1e1b39113818ad7c147b8a867a1f6aTimo Teräs struct hlist_node *entry, *tmp; 1368e4795605d1e1b39113818ad7c147b8a867a1f6aTimo Teräs LIST_HEAD(gc_list); 1378e4795605d1e1b39113818ad7c147b8a867a1f6aTimo Teräs int i, deleted = 0; 1381da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 139d7997fe1f4584da12e9c29fb682c18e9bdc13b73Timo Teräs for (i = 0; i < flow_cache_hash_size(fc); i++) { 140fe1a5f031e76bd8761a7803d75b95ee96e84a574Timo Teräs int saved = 0; 1411da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 1428e4795605d1e1b39113818ad7c147b8a867a1f6aTimo Teräs hlist_for_each_entry_safe(fle, entry, tmp, 1438e4795605d1e1b39113818ad7c147b8a867a1f6aTimo Teräs &fcp->hash_table[i], u.hlist) { 144fe1a5f031e76bd8761a7803d75b95ee96e84a574Timo Teräs if (saved < shrink_to && 145fe1a5f031e76bd8761a7803d75b95ee96e84a574Timo Teräs flow_entry_valid(fle)) { 146fe1a5f031e76bd8761a7803d75b95ee96e84a574Timo Teräs saved++; 147fe1a5f031e76bd8761a7803d75b95ee96e84a574Timo Teräs } else { 1488e4795605d1e1b39113818ad7c147b8a867a1f6aTimo Teräs deleted++; 1498e4795605d1e1b39113818ad7c147b8a867a1f6aTimo Teräs hlist_del(&fle->u.hlist); 1508e4795605d1e1b39113818ad7c147b8a867a1f6aTimo Teräs list_add_tail(&fle->u.gc_list, &gc_list); 151fe1a5f031e76bd8761a7803d75b95ee96e84a574Timo Teräs } 1521da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds } 1531da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds } 1548e4795605d1e1b39113818ad7c147b8a867a1f6aTimo Teräs 1558e4795605d1e1b39113818ad7c147b8a867a1f6aTimo Teräs flow_cache_queue_garbage(fcp, deleted, &gc_list); 1561da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds} 1571da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 158d7997fe1f4584da12e9c29fb682c18e9bdc13b73Timo Terässtatic void flow_cache_shrink(struct flow_cache *fc, 159d7997fe1f4584da12e9c29fb682c18e9bdc13b73Timo Teräs struct flow_cache_percpu *fcp) 1601da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds{ 161d7997fe1f4584da12e9c29fb682c18e9bdc13b73Timo Teräs int shrink_to = fc->low_watermark / flow_cache_hash_size(fc); 1621da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 163d7997fe1f4584da12e9c29fb682c18e9bdc13b73Timo Teräs __flow_cache_shrink(fc, fcp, shrink_to); 1641da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds} 1651da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 166d7997fe1f4584da12e9c29fb682c18e9bdc13b73Timo Terässtatic void flow_new_hash_rnd(struct flow_cache *fc, 167d7997fe1f4584da12e9c29fb682c18e9bdc13b73Timo Teräs struct flow_cache_percpu *fcp) 1681da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds{ 169d7997fe1f4584da12e9c29fb682c18e9bdc13b73Timo Teräs get_random_bytes(&fcp->hash_rnd, sizeof(u32)); 170d7997fe1f4584da12e9c29fb682c18e9bdc13b73Timo Teräs fcp->hash_rnd_recalc = 0; 171d7997fe1f4584da12e9c29fb682c18e9bdc13b73Timo Teräs __flow_cache_shrink(fc, fcp, 0); 1721da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds} 1731da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 174d7997fe1f4584da12e9c29fb682c18e9bdc13b73Timo Terässtatic u32 flow_hash_code(struct flow_cache *fc, 175d7997fe1f4584da12e9c29fb682c18e9bdc13b73Timo Teräs struct flow_cache_percpu *fcp, 176aa1c366e4febc7f5c2b84958a2dd7cd70e28f9d0dpward const struct flowi *key, 177aa1c366e4febc7f5c2b84958a2dd7cd70e28f9d0dpward size_t keysize) 1781da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds{ 179dee9f4bceb5fd9dbfcc1567148fccdbf16d6a38aDavid S. Miller const u32 *k = (const u32 *) key; 180aa1c366e4febc7f5c2b84958a2dd7cd70e28f9d0dpward const u32 length = keysize * sizeof(flow_compare_t) / sizeof(u32); 1811da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 182aa1c366e4febc7f5c2b84958a2dd7cd70e28f9d0dpward return jhash2(k, length, fcp->hash_rnd) 183a02cec2155fbea457eca8881870fd2de1a4c4c76Eric Dumazet & (flow_cache_hash_size(fc) - 1); 1841da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds} 1851da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 1861da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds/* I hear what you're saying, use memcmp. But memcmp cannot make 187aa1c366e4febc7f5c2b84958a2dd7cd70e28f9d0dpward * important assumptions that we can here, such as alignment. 1881da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds */ 189aa1c366e4febc7f5c2b84958a2dd7cd70e28f9d0dpwardstatic int flow_key_compare(const struct flowi *key1, const struct flowi *key2, 190aa1c366e4febc7f5c2b84958a2dd7cd70e28f9d0dpward size_t keysize) 1911da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds{ 192dee9f4bceb5fd9dbfcc1567148fccdbf16d6a38aDavid S. Miller const flow_compare_t *k1, *k1_lim, *k2; 1931da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 194dee9f4bceb5fd9dbfcc1567148fccdbf16d6a38aDavid S. Miller k1 = (const flow_compare_t *) key1; 195aa1c366e4febc7f5c2b84958a2dd7cd70e28f9d0dpward k1_lim = k1 + keysize; 1961da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 197dee9f4bceb5fd9dbfcc1567148fccdbf16d6a38aDavid S. Miller k2 = (const flow_compare_t *) key2; 1981da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 1991da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds do { 2001da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds if (*k1++ != *k2++) 2011da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds return 1; 2021da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds } while (k1 < k1_lim); 2031da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 2041da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds return 0; 2051da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds} 2061da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 207fe1a5f031e76bd8761a7803d75b95ee96e84a574Timo Terässtruct flow_cache_object * 208dee9f4bceb5fd9dbfcc1567148fccdbf16d6a38aDavid S. Millerflow_cache_lookup(struct net *net, const struct flowi *key, u16 family, u8 dir, 209fe1a5f031e76bd8761a7803d75b95ee96e84a574Timo Teräs flow_resolve_t resolver, void *ctx) 2101da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds{ 211d7997fe1f4584da12e9c29fb682c18e9bdc13b73Timo Teräs struct flow_cache *fc = &flow_cache_global; 212d7997fe1f4584da12e9c29fb682c18e9bdc13b73Timo Teräs struct flow_cache_percpu *fcp; 2138e4795605d1e1b39113818ad7c147b8a867a1f6aTimo Teräs struct flow_cache_entry *fle, *tfle; 2148e4795605d1e1b39113818ad7c147b8a867a1f6aTimo Teräs struct hlist_node *entry; 215fe1a5f031e76bd8761a7803d75b95ee96e84a574Timo Teräs struct flow_cache_object *flo; 216aa1c366e4febc7f5c2b84958a2dd7cd70e28f9d0dpward size_t keysize; 2171da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds unsigned int hash; 2181da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 2191da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds local_bh_disable(); 2207a9b2d59507d85569b8a456688ef40cf2ac73e48Eric Dumazet fcp = this_cpu_ptr(fc->percpu); 2211da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 2221da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds fle = NULL; 223fe1a5f031e76bd8761a7803d75b95ee96e84a574Timo Teräs flo = NULL; 224aa1c366e4febc7f5c2b84958a2dd7cd70e28f9d0dpward 225aa1c366e4febc7f5c2b84958a2dd7cd70e28f9d0dpward keysize = flow_key_size(family); 226aa1c366e4febc7f5c2b84958a2dd7cd70e28f9d0dpward if (!keysize) 227aa1c366e4febc7f5c2b84958a2dd7cd70e28f9d0dpward goto nocache; 228aa1c366e4febc7f5c2b84958a2dd7cd70e28f9d0dpward 2291da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds /* Packet really early in init? Making flow_cache_init a 2301da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds * pre-smp initcall would solve this. --RR */ 231d7997fe1f4584da12e9c29fb682c18e9bdc13b73Timo Teräs if (!fcp->hash_table) 2321da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds goto nocache; 2331da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 234d7997fe1f4584da12e9c29fb682c18e9bdc13b73Timo Teräs if (fcp->hash_rnd_recalc) 235d7997fe1f4584da12e9c29fb682c18e9bdc13b73Timo Teräs flow_new_hash_rnd(fc, fcp); 2361da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 237aa1c366e4febc7f5c2b84958a2dd7cd70e28f9d0dpward hash = flow_hash_code(fc, fcp, key, keysize); 2388e4795605d1e1b39113818ad7c147b8a867a1f6aTimo Teräs hlist_for_each_entry(tfle, entry, &fcp->hash_table[hash], u.hlist) { 2390542b69e2c57fc9668ce6a03155bea6e1f557901dpward if (tfle->net == net && 2400542b69e2c57fc9668ce6a03155bea6e1f557901dpward tfle->family == family && 2418e4795605d1e1b39113818ad7c147b8a867a1f6aTimo Teräs tfle->dir == dir && 242aa1c366e4febc7f5c2b84958a2dd7cd70e28f9d0dpward flow_key_compare(key, &tfle->key, keysize) == 0) { 2438e4795605d1e1b39113818ad7c147b8a867a1f6aTimo Teräs fle = tfle; 2441da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds break; 2458e4795605d1e1b39113818ad7c147b8a867a1f6aTimo Teräs } 2461da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds } 2471da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 248fe1a5f031e76bd8761a7803d75b95ee96e84a574Timo Teräs if (unlikely(!fle)) { 249d7997fe1f4584da12e9c29fb682c18e9bdc13b73Timo Teräs if (fcp->hash_count > fc->high_watermark) 250d7997fe1f4584da12e9c29fb682c18e9bdc13b73Timo Teräs flow_cache_shrink(fc, fcp); 2511da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 25254e6ecb23951b195d02433a741c7f7cb0b796c78Christoph Lameter fle = kmem_cache_alloc(flow_cachep, GFP_ATOMIC); 2531da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds if (fle) { 2540542b69e2c57fc9668ce6a03155bea6e1f557901dpward fle->net = net; 2551da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds fle->family = family; 2561da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds fle->dir = dir; 257aa1c366e4febc7f5c2b84958a2dd7cd70e28f9d0dpward memcpy(&fle->key, key, keysize * sizeof(flow_compare_t)); 2581da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds fle->object = NULL; 2598e4795605d1e1b39113818ad7c147b8a867a1f6aTimo Teräs hlist_add_head(&fle->u.hlist, &fcp->hash_table[hash]); 260d7997fe1f4584da12e9c29fb682c18e9bdc13b73Timo Teräs fcp->hash_count++; 2611da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds } 262fe1a5f031e76bd8761a7803d75b95ee96e84a574Timo Teräs } else if (likely(fle->genid == atomic_read(&flow_cache_genid))) { 263fe1a5f031e76bd8761a7803d75b95ee96e84a574Timo Teräs flo = fle->object; 264fe1a5f031e76bd8761a7803d75b95ee96e84a574Timo Teräs if (!flo) 265fe1a5f031e76bd8761a7803d75b95ee96e84a574Timo Teräs goto ret_object; 266fe1a5f031e76bd8761a7803d75b95ee96e84a574Timo Teräs flo = flo->ops->get(flo); 267fe1a5f031e76bd8761a7803d75b95ee96e84a574Timo Teräs if (flo) 268fe1a5f031e76bd8761a7803d75b95ee96e84a574Timo Teräs goto ret_object; 269fe1a5f031e76bd8761a7803d75b95ee96e84a574Timo Teräs } else if (fle->object) { 270fe1a5f031e76bd8761a7803d75b95ee96e84a574Timo Teräs flo = fle->object; 271fe1a5f031e76bd8761a7803d75b95ee96e84a574Timo Teräs flo->ops->delete(flo); 272fe1a5f031e76bd8761a7803d75b95ee96e84a574Timo Teräs fle->object = NULL; 2731da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds } 2741da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 2751da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvaldsnocache: 276fe1a5f031e76bd8761a7803d75b95ee96e84a574Timo Teräs flo = NULL; 277fe1a5f031e76bd8761a7803d75b95ee96e84a574Timo Teräs if (fle) { 278fe1a5f031e76bd8761a7803d75b95ee96e84a574Timo Teräs flo = fle->object; 279fe1a5f031e76bd8761a7803d75b95ee96e84a574Timo Teräs fle->object = NULL; 280fe1a5f031e76bd8761a7803d75b95ee96e84a574Timo Teräs } 281fe1a5f031e76bd8761a7803d75b95ee96e84a574Timo Teräs flo = resolver(net, key, family, dir, flo, ctx); 282fe1a5f031e76bd8761a7803d75b95ee96e84a574Timo Teräs if (fle) { 283fe1a5f031e76bd8761a7803d75b95ee96e84a574Timo Teräs fle->genid = atomic_read(&flow_cache_genid); 284fe1a5f031e76bd8761a7803d75b95ee96e84a574Timo Teräs if (!IS_ERR(flo)) 285fe1a5f031e76bd8761a7803d75b95ee96e84a574Timo Teräs fle->object = flo; 286fe1a5f031e76bd8761a7803d75b95ee96e84a574Timo Teräs else 287fe1a5f031e76bd8761a7803d75b95ee96e84a574Timo Teräs fle->genid--; 288fe1a5f031e76bd8761a7803d75b95ee96e84a574Timo Teräs } else { 2898fbcec241df21d1ba2aba09974ea9017832b69b0YOSHIFUJI Hideaki / 吉藤英明 if (!IS_ERR_OR_NULL(flo)) 290fe1a5f031e76bd8761a7803d75b95ee96e84a574Timo Teräs flo->ops->delete(flo); 2911da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds } 292fe1a5f031e76bd8761a7803d75b95ee96e84a574Timo Teräsret_object: 293fe1a5f031e76bd8761a7803d75b95ee96e84a574Timo Teräs local_bh_enable(); 294fe1a5f031e76bd8761a7803d75b95ee96e84a574Timo Teräs return flo; 2951da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds} 2969e34a5b51684bc90ac827ec4ba339f3892632eacEric DumazetEXPORT_SYMBOL(flow_cache_lookup); 2971da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 2981da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvaldsstatic void flow_cache_flush_tasklet(unsigned long data) 2991da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds{ 3001da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds struct flow_flush_info *info = (void *)data; 301d7997fe1f4584da12e9c29fb682c18e9bdc13b73Timo Teräs struct flow_cache *fc = info->cache; 302d7997fe1f4584da12e9c29fb682c18e9bdc13b73Timo Teräs struct flow_cache_percpu *fcp; 3038e4795605d1e1b39113818ad7c147b8a867a1f6aTimo Teräs struct flow_cache_entry *fle; 3048e4795605d1e1b39113818ad7c147b8a867a1f6aTimo Teräs struct hlist_node *entry, *tmp; 3058e4795605d1e1b39113818ad7c147b8a867a1f6aTimo Teräs LIST_HEAD(gc_list); 3068e4795605d1e1b39113818ad7c147b8a867a1f6aTimo Teräs int i, deleted = 0; 3071da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 3087a9b2d59507d85569b8a456688ef40cf2ac73e48Eric Dumazet fcp = this_cpu_ptr(fc->percpu); 309d7997fe1f4584da12e9c29fb682c18e9bdc13b73Timo Teräs for (i = 0; i < flow_cache_hash_size(fc); i++) { 3108e4795605d1e1b39113818ad7c147b8a867a1f6aTimo Teräs hlist_for_each_entry_safe(fle, entry, tmp, 3118e4795605d1e1b39113818ad7c147b8a867a1f6aTimo Teräs &fcp->hash_table[i], u.hlist) { 312fe1a5f031e76bd8761a7803d75b95ee96e84a574Timo Teräs if (flow_entry_valid(fle)) 3131da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds continue; 3141da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 3158e4795605d1e1b39113818ad7c147b8a867a1f6aTimo Teräs deleted++; 3168e4795605d1e1b39113818ad7c147b8a867a1f6aTimo Teräs hlist_del(&fle->u.hlist); 3178e4795605d1e1b39113818ad7c147b8a867a1f6aTimo Teräs list_add_tail(&fle->u.gc_list, &gc_list); 3181da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds } 3191da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds } 3201da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 3218e4795605d1e1b39113818ad7c147b8a867a1f6aTimo Teräs flow_cache_queue_garbage(fcp, deleted, &gc_list); 3228e4795605d1e1b39113818ad7c147b8a867a1f6aTimo Teräs 3231da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds if (atomic_dec_and_test(&info->cpuleft)) 3241da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds complete(&info->completion); 3251da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds} 3261da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 3271da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvaldsstatic void flow_cache_flush_per_cpu(void *data) 3281da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds{ 3291da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds struct flow_flush_info *info = data; 3301da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds struct tasklet_struct *tasklet; 3311da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 3321f743b07652f11100bee004e261b9931632beac1Shan Wei tasklet = this_cpu_ptr(&info->cache->percpu->flush_tasklet); 3331da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds tasklet->data = (unsigned long)info; 3341da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds tasklet_schedule(tasklet); 3351da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds} 3361da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 3371da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvaldsvoid flow_cache_flush(void) 3381da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds{ 3391da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds struct flow_flush_info info; 3404a3e2f711a00a1feb72ae12fdc749da10179d185Arjan van de Ven static DEFINE_MUTEX(flow_flush_sem); 3411da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 3421da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds /* Don't want cpus going down or up during this. */ 34386ef5c9a8edd78e6bf92879f32329d89b2d55b5aGautham R Shenoy get_online_cpus(); 3444a3e2f711a00a1feb72ae12fdc749da10179d185Arjan van de Ven mutex_lock(&flow_flush_sem); 345d7997fe1f4584da12e9c29fb682c18e9bdc13b73Timo Teräs info.cache = &flow_cache_global; 3461da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds atomic_set(&info.cpuleft, num_online_cpus()); 3471da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds init_completion(&info.completion); 3481da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 3491da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds local_bh_disable(); 3508691e5a8f691cc2a4fda0651e8d307aaba0e7d68Jens Axboe smp_call_function(flow_cache_flush_per_cpu, &info, 0); 3511da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds flow_cache_flush_tasklet((unsigned long)&info); 3521da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds local_bh_enable(); 3531da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 3541da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds wait_for_completion(&info.completion); 3554a3e2f711a00a1feb72ae12fdc749da10179d185Arjan van de Ven mutex_unlock(&flow_flush_sem); 35686ef5c9a8edd78e6bf92879f32329d89b2d55b5aGautham R Shenoy put_online_cpus(); 3571da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds} 3581da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 359c0ed1c14a72ca9ebacd51fb94a8aca488b0d361eSteffen Klassertstatic void flow_cache_flush_task(struct work_struct *work) 360c0ed1c14a72ca9ebacd51fb94a8aca488b0d361eSteffen Klassert{ 361c0ed1c14a72ca9ebacd51fb94a8aca488b0d361eSteffen Klassert flow_cache_flush(); 362c0ed1c14a72ca9ebacd51fb94a8aca488b0d361eSteffen Klassert} 363c0ed1c14a72ca9ebacd51fb94a8aca488b0d361eSteffen Klassert 364c0ed1c14a72ca9ebacd51fb94a8aca488b0d361eSteffen Klassertstatic DECLARE_WORK(flow_cache_flush_work, flow_cache_flush_task); 365c0ed1c14a72ca9ebacd51fb94a8aca488b0d361eSteffen Klassert 366c0ed1c14a72ca9ebacd51fb94a8aca488b0d361eSteffen Klassertvoid flow_cache_flush_deferred(void) 367c0ed1c14a72ca9ebacd51fb94a8aca488b0d361eSteffen Klassert{ 368c0ed1c14a72ca9ebacd51fb94a8aca488b0d361eSteffen Klassert schedule_work(&flow_cache_flush_work); 369c0ed1c14a72ca9ebacd51fb94a8aca488b0d361eSteffen Klassert} 370c0ed1c14a72ca9ebacd51fb94a8aca488b0d361eSteffen Klassert 37183b6b1f5d13414d0cb5c4f0a567a6aec0af073bdEric Dumazetstatic int __cpuinit flow_cache_cpu_prepare(struct flow_cache *fc, int cpu) 3721da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds{ 37383b6b1f5d13414d0cb5c4f0a567a6aec0af073bdEric Dumazet struct flow_cache_percpu *fcp = per_cpu_ptr(fc->percpu, cpu); 37483b6b1f5d13414d0cb5c4f0a567a6aec0af073bdEric Dumazet size_t sz = sizeof(struct hlist_head) * flow_cache_hash_size(fc); 375d7997fe1f4584da12e9c29fb682c18e9bdc13b73Timo Teräs 37683b6b1f5d13414d0cb5c4f0a567a6aec0af073bdEric Dumazet if (!fcp->hash_table) { 37783b6b1f5d13414d0cb5c4f0a567a6aec0af073bdEric Dumazet fcp->hash_table = kzalloc_node(sz, GFP_KERNEL, cpu_to_node(cpu)); 37883b6b1f5d13414d0cb5c4f0a567a6aec0af073bdEric Dumazet if (!fcp->hash_table) { 37983b6b1f5d13414d0cb5c4f0a567a6aec0af073bdEric Dumazet pr_err("NET: failed to allocate flow cache sz %zu\n", sz); 38083b6b1f5d13414d0cb5c4f0a567a6aec0af073bdEric Dumazet return -ENOMEM; 38183b6b1f5d13414d0cb5c4f0a567a6aec0af073bdEric Dumazet } 38283b6b1f5d13414d0cb5c4f0a567a6aec0af073bdEric Dumazet fcp->hash_rnd_recalc = 1; 38383b6b1f5d13414d0cb5c4f0a567a6aec0af073bdEric Dumazet fcp->hash_count = 0; 38483b6b1f5d13414d0cb5c4f0a567a6aec0af073bdEric Dumazet tasklet_init(&fcp->flush_tasklet, flow_cache_flush_tasklet, 0); 38583b6b1f5d13414d0cb5c4f0a567a6aec0af073bdEric Dumazet } 38683b6b1f5d13414d0cb5c4f0a567a6aec0af073bdEric Dumazet return 0; 3871da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds} 3881da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 38983b6b1f5d13414d0cb5c4f0a567a6aec0af073bdEric Dumazetstatic int __cpuinit flow_cache_cpu(struct notifier_block *nfb, 3901da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds unsigned long action, 3911da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds void *hcpu) 3921da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds{ 393d7997fe1f4584da12e9c29fb682c18e9bdc13b73Timo Teräs struct flow_cache *fc = container_of(nfb, struct flow_cache, hotcpu_notifier); 39483b6b1f5d13414d0cb5c4f0a567a6aec0af073bdEric Dumazet int res, cpu = (unsigned long) hcpu; 395d7997fe1f4584da12e9c29fb682c18e9bdc13b73Timo Teräs struct flow_cache_percpu *fcp = per_cpu_ptr(fc->percpu, cpu); 396d7997fe1f4584da12e9c29fb682c18e9bdc13b73Timo Teräs 39783b6b1f5d13414d0cb5c4f0a567a6aec0af073bdEric Dumazet switch (action) { 39883b6b1f5d13414d0cb5c4f0a567a6aec0af073bdEric Dumazet case CPU_UP_PREPARE: 39983b6b1f5d13414d0cb5c4f0a567a6aec0af073bdEric Dumazet case CPU_UP_PREPARE_FROZEN: 40083b6b1f5d13414d0cb5c4f0a567a6aec0af073bdEric Dumazet res = flow_cache_cpu_prepare(fc, cpu); 40183b6b1f5d13414d0cb5c4f0a567a6aec0af073bdEric Dumazet if (res) 40283b6b1f5d13414d0cb5c4f0a567a6aec0af073bdEric Dumazet return notifier_from_errno(res); 40383b6b1f5d13414d0cb5c4f0a567a6aec0af073bdEric Dumazet break; 40483b6b1f5d13414d0cb5c4f0a567a6aec0af073bdEric Dumazet case CPU_DEAD: 40583b6b1f5d13414d0cb5c4f0a567a6aec0af073bdEric Dumazet case CPU_DEAD_FROZEN: 406d7997fe1f4584da12e9c29fb682c18e9bdc13b73Timo Teräs __flow_cache_shrink(fc, fcp, 0); 40783b6b1f5d13414d0cb5c4f0a567a6aec0af073bdEric Dumazet break; 40883b6b1f5d13414d0cb5c4f0a567a6aec0af073bdEric Dumazet } 4091da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds return NOTIFY_OK; 4101da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds} 4111da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 41283b6b1f5d13414d0cb5c4f0a567a6aec0af073bdEric Dumazetstatic int __init flow_cache_init(struct flow_cache *fc) 4131da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds{ 4141da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds int i; 4151da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 416d7997fe1f4584da12e9c29fb682c18e9bdc13b73Timo Teräs fc->hash_shift = 10; 417d7997fe1f4584da12e9c29fb682c18e9bdc13b73Timo Teräs fc->low_watermark = 2 * flow_cache_hash_size(fc); 418d7997fe1f4584da12e9c29fb682c18e9bdc13b73Timo Teräs fc->high_watermark = 4 * flow_cache_hash_size(fc); 419d7997fe1f4584da12e9c29fb682c18e9bdc13b73Timo Teräs 420d7997fe1f4584da12e9c29fb682c18e9bdc13b73Timo Teräs fc->percpu = alloc_percpu(struct flow_cache_percpu); 42183b6b1f5d13414d0cb5c4f0a567a6aec0af073bdEric Dumazet if (!fc->percpu) 42283b6b1f5d13414d0cb5c4f0a567a6aec0af073bdEric Dumazet return -ENOMEM; 4231da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 42483b6b1f5d13414d0cb5c4f0a567a6aec0af073bdEric Dumazet for_each_online_cpu(i) { 42583b6b1f5d13414d0cb5c4f0a567a6aec0af073bdEric Dumazet if (flow_cache_cpu_prepare(fc, i)) 4266ccc3abdc97e07e6895323fdab89f84e58b40ecehuajun li goto err; 42783b6b1f5d13414d0cb5c4f0a567a6aec0af073bdEric Dumazet } 428d7997fe1f4584da12e9c29fb682c18e9bdc13b73Timo Teräs fc->hotcpu_notifier = (struct notifier_block){ 429d7997fe1f4584da12e9c29fb682c18e9bdc13b73Timo Teräs .notifier_call = flow_cache_cpu, 430d7997fe1f4584da12e9c29fb682c18e9bdc13b73Timo Teräs }; 431d7997fe1f4584da12e9c29fb682c18e9bdc13b73Timo Teräs register_hotcpu_notifier(&fc->hotcpu_notifier); 4321da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 43383b6b1f5d13414d0cb5c4f0a567a6aec0af073bdEric Dumazet setup_timer(&fc->rnd_timer, flow_cache_new_hashrnd, 43483b6b1f5d13414d0cb5c4f0a567a6aec0af073bdEric Dumazet (unsigned long) fc); 43583b6b1f5d13414d0cb5c4f0a567a6aec0af073bdEric Dumazet fc->rnd_timer.expires = jiffies + FLOW_HASH_RND_PERIOD; 43683b6b1f5d13414d0cb5c4f0a567a6aec0af073bdEric Dumazet add_timer(&fc->rnd_timer); 43783b6b1f5d13414d0cb5c4f0a567a6aec0af073bdEric Dumazet 4381da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds return 0; 4396ccc3abdc97e07e6895323fdab89f84e58b40ecehuajun li 4406ccc3abdc97e07e6895323fdab89f84e58b40ecehuajun lierr: 4416ccc3abdc97e07e6895323fdab89f84e58b40ecehuajun li for_each_possible_cpu(i) { 4426ccc3abdc97e07e6895323fdab89f84e58b40ecehuajun li struct flow_cache_percpu *fcp = per_cpu_ptr(fc->percpu, i); 4436ccc3abdc97e07e6895323fdab89f84e58b40ecehuajun li kfree(fcp->hash_table); 4446ccc3abdc97e07e6895323fdab89f84e58b40ecehuajun li fcp->hash_table = NULL; 4456ccc3abdc97e07e6895323fdab89f84e58b40ecehuajun li } 4466ccc3abdc97e07e6895323fdab89f84e58b40ecehuajun li 4476ccc3abdc97e07e6895323fdab89f84e58b40ecehuajun li free_percpu(fc->percpu); 4486ccc3abdc97e07e6895323fdab89f84e58b40ecehuajun li fc->percpu = NULL; 4496ccc3abdc97e07e6895323fdab89f84e58b40ecehuajun li 4506ccc3abdc97e07e6895323fdab89f84e58b40ecehuajun li return -ENOMEM; 4511da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds} 4521da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 453d7997fe1f4584da12e9c29fb682c18e9bdc13b73Timo Terässtatic int __init flow_cache_init_global(void) 454d7997fe1f4584da12e9c29fb682c18e9bdc13b73Timo Teräs{ 455d7997fe1f4584da12e9c29fb682c18e9bdc13b73Timo Teräs flow_cachep = kmem_cache_create("flow_cache", 456d7997fe1f4584da12e9c29fb682c18e9bdc13b73Timo Teräs sizeof(struct flow_cache_entry), 457d7997fe1f4584da12e9c29fb682c18e9bdc13b73Timo Teräs 0, SLAB_PANIC, NULL); 458d7997fe1f4584da12e9c29fb682c18e9bdc13b73Timo Teräs 459d7997fe1f4584da12e9c29fb682c18e9bdc13b73Timo Teräs return flow_cache_init(&flow_cache_global); 460d7997fe1f4584da12e9c29fb682c18e9bdc13b73Timo Teräs} 461d7997fe1f4584da12e9c29fb682c18e9bdc13b73Timo Teräs 462d7997fe1f4584da12e9c29fb682c18e9bdc13b73Timo Teräsmodule_init(flow_cache_init_global); 463