flow.c revision 134b0fc544ba062498451611cb6f3e4454221b3d
11da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds/* flow.c: Generic flow cache. 21da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds * 31da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds * Copyright (C) 2003 Alexey N. Kuznetsov (kuznet@ms2.inr.ac.ru) 41da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds * Copyright (C) 2003 David S. Miller (davem@redhat.com) 51da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds */ 61da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 71da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds#include <linux/kernel.h> 81da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds#include <linux/module.h> 91da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds#include <linux/list.h> 101da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds#include <linux/jhash.h> 111da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds#include <linux/interrupt.h> 121da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds#include <linux/mm.h> 131da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds#include <linux/random.h> 141da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds#include <linux/init.h> 151da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds#include <linux/slab.h> 161da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds#include <linux/smp.h> 171da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds#include <linux/completion.h> 181da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds#include <linux/percpu.h> 191da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds#include <linux/bitops.h> 201da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds#include <linux/notifier.h> 211da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds#include <linux/cpu.h> 221da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds#include <linux/cpumask.h> 234a3e2f711a00a1feb72ae12fdc749da10179d185Arjan van de Ven#include <linux/mutex.h> 241da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds#include <net/flow.h> 251da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds#include <asm/atomic.h> 261da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds#include <asm/semaphore.h> 27df71837d5024e2524cd51c93621e558aa7dd9f3fTrent Jaeger#include <linux/security.h> 281da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 291da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvaldsstruct flow_cache_entry { 301da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds struct flow_cache_entry *next; 311da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds u16 family; 321da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds u8 dir; 331da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds struct flowi key; 341da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds u32 genid; 351da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds void *object; 361da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds atomic_t *object_ref; 371da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds}; 381da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 391da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvaldsatomic_t flow_cache_genid = ATOMIC_INIT(0); 401da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 411da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvaldsstatic u32 flow_hash_shift; 421da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds#define flow_hash_size (1 << flow_hash_shift) 431da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvaldsstatic DEFINE_PER_CPU(struct flow_cache_entry **, flow_tables) = { NULL }; 441da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 451da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds#define flow_table(cpu) (per_cpu(flow_tables, cpu)) 461da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 47ba89966c1984513f4f2cc0a6c182266be44ddd03Eric Dumazetstatic kmem_cache_t *flow_cachep __read_mostly; 481da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 491da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvaldsstatic int flow_lwm, flow_hwm; 501da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 511da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvaldsstruct flow_percpu_info { 521da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds int hash_rnd_recalc; 531da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds u32 hash_rnd; 541da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds int count; 551da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds} ____cacheline_aligned; 561da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvaldsstatic DEFINE_PER_CPU(struct flow_percpu_info, flow_hash_info) = { 0 }; 571da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 581da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds#define flow_hash_rnd_recalc(cpu) \ 591da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds (per_cpu(flow_hash_info, cpu).hash_rnd_recalc) 601da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds#define flow_hash_rnd(cpu) \ 611da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds (per_cpu(flow_hash_info, cpu).hash_rnd) 621da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds#define flow_count(cpu) \ 631da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds (per_cpu(flow_hash_info, cpu).count) 641da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 651da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvaldsstatic struct timer_list flow_hash_rnd_timer; 661da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 671da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds#define FLOW_HASH_RND_PERIOD (10 * 60 * HZ) 681da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 691da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvaldsstruct flow_flush_info { 701da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds atomic_t cpuleft; 711da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds struct completion completion; 721da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds}; 731da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvaldsstatic DEFINE_PER_CPU(struct tasklet_struct, flow_flush_tasklets) = { NULL }; 741da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 751da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds#define flow_flush_tasklet(cpu) (&per_cpu(flow_flush_tasklets, cpu)) 761da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 771da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvaldsstatic void flow_cache_new_hashrnd(unsigned long arg) 781da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds{ 791da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds int i; 801da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 816f912042256c12b0927438122594f5379b364f5dKAMEZAWA Hiroyuki for_each_possible_cpu(i) 821da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds flow_hash_rnd_recalc(i) = 1; 831da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 841da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds flow_hash_rnd_timer.expires = jiffies + FLOW_HASH_RND_PERIOD; 851da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds add_timer(&flow_hash_rnd_timer); 861da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds} 871da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 88134b0fc544ba062498451611cb6f3e4454221b3dJames Morrisstatic void flow_entry_kill(int cpu, struct flow_cache_entry *fle) 89134b0fc544ba062498451611cb6f3e4454221b3dJames Morris{ 90134b0fc544ba062498451611cb6f3e4454221b3dJames Morris if (fle->object) 91134b0fc544ba062498451611cb6f3e4454221b3dJames Morris atomic_dec(fle->object_ref); 92134b0fc544ba062498451611cb6f3e4454221b3dJames Morris kmem_cache_free(flow_cachep, fle); 93134b0fc544ba062498451611cb6f3e4454221b3dJames Morris flow_count(cpu)--; 94134b0fc544ba062498451611cb6f3e4454221b3dJames Morris} 95134b0fc544ba062498451611cb6f3e4454221b3dJames Morris 961da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvaldsstatic void __flow_cache_shrink(int cpu, int shrink_to) 971da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds{ 981da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds struct flow_cache_entry *fle, **flp; 991da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds int i; 1001da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 1011da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds for (i = 0; i < flow_hash_size; i++) { 1021da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds int k = 0; 1031da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 1041da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds flp = &flow_table(cpu)[i]; 1051da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds while ((fle = *flp) != NULL && k < shrink_to) { 1061da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds k++; 1071da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds flp = &fle->next; 1081da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds } 1091da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds while ((fle = *flp) != NULL) { 1101da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds *flp = fle->next; 111134b0fc544ba062498451611cb6f3e4454221b3dJames Morris flow_entry_kill(cpu, fle); 1121da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds } 1131da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds } 1141da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds} 1151da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 1161da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvaldsstatic void flow_cache_shrink(int cpu) 1171da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds{ 1181da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds int shrink_to = flow_lwm / flow_hash_size; 1191da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 1201da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds __flow_cache_shrink(cpu, shrink_to); 1211da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds} 1221da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 1231da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvaldsstatic void flow_new_hash_rnd(int cpu) 1241da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds{ 1251da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds get_random_bytes(&flow_hash_rnd(cpu), sizeof(u32)); 1261da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds flow_hash_rnd_recalc(cpu) = 0; 1271da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 1281da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds __flow_cache_shrink(cpu, 0); 1291da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds} 1301da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 1311da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvaldsstatic u32 flow_hash_code(struct flowi *key, int cpu) 1321da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds{ 1331da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds u32 *k = (u32 *) key; 1341da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 1351da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds return (jhash2(k, (sizeof(*key) / sizeof(u32)), flow_hash_rnd(cpu)) & 1361da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds (flow_hash_size - 1)); 1371da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds} 1381da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 1391da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds#if (BITS_PER_LONG == 64) 1401da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvaldstypedef u64 flow_compare_t; 1411da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds#else 1421da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvaldstypedef u32 flow_compare_t; 1431da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds#endif 1441da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 1451da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvaldsextern void flowi_is_missized(void); 1461da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 1471da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds/* I hear what you're saying, use memcmp. But memcmp cannot make 1481da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds * important assumptions that we can here, such as alignment and 1491da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds * constant size. 1501da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds */ 1511da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvaldsstatic int flow_key_compare(struct flowi *key1, struct flowi *key2) 1521da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds{ 1531da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds flow_compare_t *k1, *k1_lim, *k2; 1541da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds const int n_elem = sizeof(struct flowi) / sizeof(flow_compare_t); 1551da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 1561da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds if (sizeof(struct flowi) % sizeof(flow_compare_t)) 1571da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds flowi_is_missized(); 1581da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 1591da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds k1 = (flow_compare_t *) key1; 1601da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds k1_lim = k1 + n_elem; 1611da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 1621da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds k2 = (flow_compare_t *) key2; 1631da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 1641da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds do { 1651da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds if (*k1++ != *k2++) 1661da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds return 1; 1671da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds } while (k1 < k1_lim); 1681da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 1691da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds return 0; 1701da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds} 1711da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 172e0d1caa7b0d5f02e4f34aa09c695d04251310c6cVenkat Yekkiralavoid *flow_cache_lookup(struct flowi *key, u16 family, u8 dir, 1731da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds flow_resolve_t resolver) 1741da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds{ 1751da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds struct flow_cache_entry *fle, **head; 1761da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds unsigned int hash; 1771da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds int cpu; 1781da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 1791da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds local_bh_disable(); 1801da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds cpu = smp_processor_id(); 1811da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 1821da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds fle = NULL; 1831da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds /* Packet really early in init? Making flow_cache_init a 1841da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds * pre-smp initcall would solve this. --RR */ 1851da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds if (!flow_table(cpu)) 1861da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds goto nocache; 1871da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 1881da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds if (flow_hash_rnd_recalc(cpu)) 1891da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds flow_new_hash_rnd(cpu); 1901da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds hash = flow_hash_code(key, cpu); 1911da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 1921da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds head = &flow_table(cpu)[hash]; 1931da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds for (fle = *head; fle; fle = fle->next) { 1941da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds if (fle->family == family && 1951da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds fle->dir == dir && 1961da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds flow_key_compare(key, &fle->key) == 0) { 1971da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds if (fle->genid == atomic_read(&flow_cache_genid)) { 1981da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds void *ret = fle->object; 1991da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 2001da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds if (ret) 2011da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds atomic_inc(fle->object_ref); 2021da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds local_bh_enable(); 2031da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 2041da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds return ret; 2051da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds } 2061da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds break; 2071da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds } 2081da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds } 2091da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 2101da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds if (!fle) { 2111da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds if (flow_count(cpu) > flow_hwm) 2121da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds flow_cache_shrink(cpu); 2131da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 2141da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds fle = kmem_cache_alloc(flow_cachep, SLAB_ATOMIC); 2151da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds if (fle) { 2161da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds fle->next = *head; 2171da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds *head = fle; 2181da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds fle->family = family; 2191da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds fle->dir = dir; 2201da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds memcpy(&fle->key, key, sizeof(*key)); 2211da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds fle->object = NULL; 2221da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds flow_count(cpu)++; 2231da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds } 2241da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds } 2251da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 2261da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvaldsnocache: 2271da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds { 228134b0fc544ba062498451611cb6f3e4454221b3dJames Morris int err; 2291da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds void *obj; 2301da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds atomic_t *obj_ref; 2311da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 232134b0fc544ba062498451611cb6f3e4454221b3dJames Morris err = resolver(key, family, dir, &obj, &obj_ref); 2331da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 2341da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds if (fle) { 235134b0fc544ba062498451611cb6f3e4454221b3dJames Morris if (err) { 236134b0fc544ba062498451611cb6f3e4454221b3dJames Morris /* Force security policy check on next lookup */ 237134b0fc544ba062498451611cb6f3e4454221b3dJames Morris *head = fle->next; 238134b0fc544ba062498451611cb6f3e4454221b3dJames Morris flow_entry_kill(cpu, fle); 239134b0fc544ba062498451611cb6f3e4454221b3dJames Morris } else { 240134b0fc544ba062498451611cb6f3e4454221b3dJames Morris fle->genid = atomic_read(&flow_cache_genid); 241134b0fc544ba062498451611cb6f3e4454221b3dJames Morris 242134b0fc544ba062498451611cb6f3e4454221b3dJames Morris if (fle->object) 243134b0fc544ba062498451611cb6f3e4454221b3dJames Morris atomic_dec(fle->object_ref); 244134b0fc544ba062498451611cb6f3e4454221b3dJames Morris 245134b0fc544ba062498451611cb6f3e4454221b3dJames Morris fle->object = obj; 246134b0fc544ba062498451611cb6f3e4454221b3dJames Morris fle->object_ref = obj_ref; 247134b0fc544ba062498451611cb6f3e4454221b3dJames Morris if (obj) 248134b0fc544ba062498451611cb6f3e4454221b3dJames Morris atomic_inc(fle->object_ref); 249134b0fc544ba062498451611cb6f3e4454221b3dJames Morris } 2501da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds } 2511da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds local_bh_enable(); 2521da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 253134b0fc544ba062498451611cb6f3e4454221b3dJames Morris if (err) 254134b0fc544ba062498451611cb6f3e4454221b3dJames Morris obj = ERR_PTR(err); 2551da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds return obj; 2561da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds } 2571da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds} 2581da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 2591da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvaldsstatic void flow_cache_flush_tasklet(unsigned long data) 2601da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds{ 2611da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds struct flow_flush_info *info = (void *)data; 2621da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds int i; 2631da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds int cpu; 2641da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 2651da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds cpu = smp_processor_id(); 2661da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds for (i = 0; i < flow_hash_size; i++) { 2671da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds struct flow_cache_entry *fle; 2681da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 2691da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds fle = flow_table(cpu)[i]; 2701da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds for (; fle; fle = fle->next) { 2711da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds unsigned genid = atomic_read(&flow_cache_genid); 2721da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 2731da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds if (!fle->object || fle->genid == genid) 2741da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds continue; 2751da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 2761da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds fle->object = NULL; 2771da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds atomic_dec(fle->object_ref); 2781da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds } 2791da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds } 2801da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 2811da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds if (atomic_dec_and_test(&info->cpuleft)) 2821da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds complete(&info->completion); 2831da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds} 2841da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 2851da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvaldsstatic void flow_cache_flush_per_cpu(void *) __attribute__((__unused__)); 2861da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvaldsstatic void flow_cache_flush_per_cpu(void *data) 2871da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds{ 2881da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds struct flow_flush_info *info = data; 2891da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds int cpu; 2901da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds struct tasklet_struct *tasklet; 2911da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 2921da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds cpu = smp_processor_id(); 2931da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 2941da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds tasklet = flow_flush_tasklet(cpu); 2951da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds tasklet->data = (unsigned long)info; 2961da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds tasklet_schedule(tasklet); 2971da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds} 2981da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 2991da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvaldsvoid flow_cache_flush(void) 3001da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds{ 3011da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds struct flow_flush_info info; 3024a3e2f711a00a1feb72ae12fdc749da10179d185Arjan van de Ven static DEFINE_MUTEX(flow_flush_sem); 3031da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 3041da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds /* Don't want cpus going down or up during this. */ 3051da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds lock_cpu_hotplug(); 3064a3e2f711a00a1feb72ae12fdc749da10179d185Arjan van de Ven mutex_lock(&flow_flush_sem); 3071da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds atomic_set(&info.cpuleft, num_online_cpus()); 3081da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds init_completion(&info.completion); 3091da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 3101da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds local_bh_disable(); 3111da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds smp_call_function(flow_cache_flush_per_cpu, &info, 1, 0); 3121da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds flow_cache_flush_tasklet((unsigned long)&info); 3131da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds local_bh_enable(); 3141da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 3151da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds wait_for_completion(&info.completion); 3164a3e2f711a00a1feb72ae12fdc749da10179d185Arjan van de Ven mutex_unlock(&flow_flush_sem); 3171da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds unlock_cpu_hotplug(); 3181da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds} 3191da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 3201da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvaldsstatic void __devinit flow_cache_cpu_prepare(int cpu) 3211da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds{ 3221da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds struct tasklet_struct *tasklet; 3231da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds unsigned long order; 3241da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 3251da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds for (order = 0; 3261da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds (PAGE_SIZE << order) < 3271da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds (sizeof(struct flow_cache_entry *)*flow_hash_size); 3281da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds order++) 3291da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds /* NOTHING */; 3301da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 3311da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds flow_table(cpu) = (struct flow_cache_entry **) 33277d04bd957ddca9d48a664e28b40f33993f4550eAndrew Morton __get_free_pages(GFP_KERNEL|__GFP_ZERO, order); 3331da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds if (!flow_table(cpu)) 3341da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds panic("NET: failed to allocate flow cache order %lu\n", order); 3351da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 3361da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds flow_hash_rnd_recalc(cpu) = 1; 3371da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds flow_count(cpu) = 0; 3381da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 3391da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds tasklet = flow_flush_tasklet(cpu); 3401da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds tasklet_init(tasklet, flow_cache_flush_tasklet, 0); 3411da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds} 3421da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 3431da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds#ifdef CONFIG_HOTPLUG_CPU 3441da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvaldsstatic int flow_cache_cpu(struct notifier_block *nfb, 3451da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds unsigned long action, 3461da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds void *hcpu) 3471da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds{ 3481da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds if (action == CPU_DEAD) 3491da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds __flow_cache_shrink((unsigned long)hcpu, 0); 3501da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds return NOTIFY_OK; 3511da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds} 3521da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds#endif /* CONFIG_HOTPLUG_CPU */ 3531da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 3541da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvaldsstatic int __init flow_cache_init(void) 3551da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds{ 3561da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds int i; 3571da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 3581da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds flow_cachep = kmem_cache_create("flow_cache", 3591da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds sizeof(struct flow_cache_entry), 360e5d679f33900c71d1a76ba07c5b04055abd34480Alexey Dobriyan 0, SLAB_HWCACHE_ALIGN|SLAB_PANIC, 3611da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds NULL, NULL); 3621da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds flow_hash_shift = 10; 3631da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds flow_lwm = 2 * flow_hash_size; 3641da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds flow_hwm = 4 * flow_hash_size; 3651da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 3661da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds init_timer(&flow_hash_rnd_timer); 3671da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds flow_hash_rnd_timer.function = flow_cache_new_hashrnd; 3681da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds flow_hash_rnd_timer.expires = jiffies + FLOW_HASH_RND_PERIOD; 3691da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds add_timer(&flow_hash_rnd_timer); 3701da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 3716f912042256c12b0927438122594f5379b364f5dKAMEZAWA Hiroyuki for_each_possible_cpu(i) 3721da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds flow_cache_cpu_prepare(i); 3731da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 3741da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds hotcpu_notifier(flow_cache_cpu, 0); 3751da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds return 0; 3761da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds} 3771da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 3781da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvaldsmodule_init(flow_cache_init); 3791da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 3801da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus TorvaldsEXPORT_SYMBOL(flow_cache_genid); 3811da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus TorvaldsEXPORT_SYMBOL(flow_cache_lookup); 382