12787b04b6c5e7607510e8248b38b0aeacb5505f6Pavel Emelyanov#ifndef __PACKET_INTERNAL_H__ 22787b04b6c5e7607510e8248b38b0aeacb5505f6Pavel Emelyanov#define __PACKET_INTERNAL_H__ 32787b04b6c5e7607510e8248b38b0aeacb5505f6Pavel Emelyanov 42787b04b6c5e7607510e8248b38b0aeacb5505f6Pavel Emelyanovstruct packet_mclist { 52787b04b6c5e7607510e8248b38b0aeacb5505f6Pavel Emelyanov struct packet_mclist *next; 62787b04b6c5e7607510e8248b38b0aeacb5505f6Pavel Emelyanov int ifindex; 72787b04b6c5e7607510e8248b38b0aeacb5505f6Pavel Emelyanov int count; 82787b04b6c5e7607510e8248b38b0aeacb5505f6Pavel Emelyanov unsigned short type; 92787b04b6c5e7607510e8248b38b0aeacb5505f6Pavel Emelyanov unsigned short alen; 102787b04b6c5e7607510e8248b38b0aeacb5505f6Pavel Emelyanov unsigned char addr[MAX_ADDR_LEN]; 112787b04b6c5e7607510e8248b38b0aeacb5505f6Pavel Emelyanov}; 122787b04b6c5e7607510e8248b38b0aeacb5505f6Pavel Emelyanov 132787b04b6c5e7607510e8248b38b0aeacb5505f6Pavel Emelyanov/* kbdq - kernel block descriptor queue */ 142787b04b6c5e7607510e8248b38b0aeacb5505f6Pavel Emelyanovstruct tpacket_kbdq_core { 152787b04b6c5e7607510e8248b38b0aeacb5505f6Pavel Emelyanov struct pgv *pkbdq; 162787b04b6c5e7607510e8248b38b0aeacb5505f6Pavel Emelyanov unsigned int feature_req_word; 172787b04b6c5e7607510e8248b38b0aeacb5505f6Pavel Emelyanov unsigned int hdrlen; 182787b04b6c5e7607510e8248b38b0aeacb5505f6Pavel Emelyanov unsigned char reset_pending_on_curr_blk; 192787b04b6c5e7607510e8248b38b0aeacb5505f6Pavel Emelyanov unsigned char delete_blk_timer; 202787b04b6c5e7607510e8248b38b0aeacb5505f6Pavel Emelyanov unsigned short kactive_blk_num; 212787b04b6c5e7607510e8248b38b0aeacb5505f6Pavel Emelyanov unsigned short blk_sizeof_priv; 222787b04b6c5e7607510e8248b38b0aeacb5505f6Pavel Emelyanov 232787b04b6c5e7607510e8248b38b0aeacb5505f6Pavel Emelyanov /* last_kactive_blk_num: 242787b04b6c5e7607510e8248b38b0aeacb5505f6Pavel Emelyanov * trick to see if user-space has caught up 252787b04b6c5e7607510e8248b38b0aeacb5505f6Pavel Emelyanov * in order to avoid refreshing timer when every single pkt arrives. 262787b04b6c5e7607510e8248b38b0aeacb5505f6Pavel Emelyanov */ 272787b04b6c5e7607510e8248b38b0aeacb5505f6Pavel Emelyanov unsigned short last_kactive_blk_num; 282787b04b6c5e7607510e8248b38b0aeacb5505f6Pavel Emelyanov 292787b04b6c5e7607510e8248b38b0aeacb5505f6Pavel Emelyanov char *pkblk_start; 302787b04b6c5e7607510e8248b38b0aeacb5505f6Pavel Emelyanov char *pkblk_end; 312787b04b6c5e7607510e8248b38b0aeacb5505f6Pavel Emelyanov int kblk_size; 32dc808110bb62b64a448696ecac3938902c92e1abEric Dumazet unsigned int max_frame_len; 332787b04b6c5e7607510e8248b38b0aeacb5505f6Pavel Emelyanov unsigned int knum_blocks; 342787b04b6c5e7607510e8248b38b0aeacb5505f6Pavel Emelyanov uint64_t knxt_seq_num; 352787b04b6c5e7607510e8248b38b0aeacb5505f6Pavel Emelyanov char *prev; 362787b04b6c5e7607510e8248b38b0aeacb5505f6Pavel Emelyanov char *nxt_offset; 372787b04b6c5e7607510e8248b38b0aeacb5505f6Pavel Emelyanov struct sk_buff *skb; 382787b04b6c5e7607510e8248b38b0aeacb5505f6Pavel Emelyanov 392787b04b6c5e7607510e8248b38b0aeacb5505f6Pavel Emelyanov atomic_t blk_fill_in_prog; 402787b04b6c5e7607510e8248b38b0aeacb5505f6Pavel Emelyanov 412787b04b6c5e7607510e8248b38b0aeacb5505f6Pavel Emelyanov /* Default is set to 8ms */ 422787b04b6c5e7607510e8248b38b0aeacb5505f6Pavel Emelyanov#define DEFAULT_PRB_RETIRE_TOV (8) 432787b04b6c5e7607510e8248b38b0aeacb5505f6Pavel Emelyanov 442787b04b6c5e7607510e8248b38b0aeacb5505f6Pavel Emelyanov unsigned short retire_blk_tov; 452787b04b6c5e7607510e8248b38b0aeacb5505f6Pavel Emelyanov unsigned short version; 462787b04b6c5e7607510e8248b38b0aeacb5505f6Pavel Emelyanov unsigned long tov_in_jiffies; 472787b04b6c5e7607510e8248b38b0aeacb5505f6Pavel Emelyanov 482787b04b6c5e7607510e8248b38b0aeacb5505f6Pavel Emelyanov /* timer to retire an outstanding block */ 492787b04b6c5e7607510e8248b38b0aeacb5505f6Pavel Emelyanov struct timer_list retire_blk_timer; 502787b04b6c5e7607510e8248b38b0aeacb5505f6Pavel Emelyanov}; 512787b04b6c5e7607510e8248b38b0aeacb5505f6Pavel Emelyanov 522787b04b6c5e7607510e8248b38b0aeacb5505f6Pavel Emelyanovstruct pgv { 532787b04b6c5e7607510e8248b38b0aeacb5505f6Pavel Emelyanov char *buffer; 542787b04b6c5e7607510e8248b38b0aeacb5505f6Pavel Emelyanov}; 552787b04b6c5e7607510e8248b38b0aeacb5505f6Pavel Emelyanov 562787b04b6c5e7607510e8248b38b0aeacb5505f6Pavel Emelyanovstruct packet_ring_buffer { 572787b04b6c5e7607510e8248b38b0aeacb5505f6Pavel Emelyanov struct pgv *pg_vec; 580578edc5604e3e192980f406f9e1040aa6434ca4Daniel Borkmann 592787b04b6c5e7607510e8248b38b0aeacb5505f6Pavel Emelyanov unsigned int head; 602787b04b6c5e7607510e8248b38b0aeacb5505f6Pavel Emelyanov unsigned int frames_per_block; 612787b04b6c5e7607510e8248b38b0aeacb5505f6Pavel Emelyanov unsigned int frame_size; 622787b04b6c5e7607510e8248b38b0aeacb5505f6Pavel Emelyanov unsigned int frame_max; 632787b04b6c5e7607510e8248b38b0aeacb5505f6Pavel Emelyanov 642787b04b6c5e7607510e8248b38b0aeacb5505f6Pavel Emelyanov unsigned int pg_vec_order; 652787b04b6c5e7607510e8248b38b0aeacb5505f6Pavel Emelyanov unsigned int pg_vec_pages; 662787b04b6c5e7607510e8248b38b0aeacb5505f6Pavel Emelyanov unsigned int pg_vec_len; 672787b04b6c5e7607510e8248b38b0aeacb5505f6Pavel Emelyanov 68b013840810c221f2b0cf641d01531526052dc1fbDaniel Borkmann unsigned int __percpu *pending_refcnt; 690578edc5604e3e192980f406f9e1040aa6434ca4Daniel Borkmann 700578edc5604e3e192980f406f9e1040aa6434ca4Daniel Borkmann struct tpacket_kbdq_core prb_bdqc; 712787b04b6c5e7607510e8248b38b0aeacb5505f6Pavel Emelyanov}; 722787b04b6c5e7607510e8248b38b0aeacb5505f6Pavel Emelyanov 73fff3321d75b1a18231876a1aceb36eacbbf6221ePavel Emelyanovextern struct mutex fanout_mutex; 74fff3321d75b1a18231876a1aceb36eacbbf6221ePavel Emelyanov#define PACKET_FANOUT_MAX 256 75fff3321d75b1a18231876a1aceb36eacbbf6221ePavel Emelyanov 76fff3321d75b1a18231876a1aceb36eacbbf6221ePavel Emelyanovstruct packet_fanout { 77fff3321d75b1a18231876a1aceb36eacbbf6221ePavel Emelyanov#ifdef CONFIG_NET_NS 78fff3321d75b1a18231876a1aceb36eacbbf6221ePavel Emelyanov struct net *net; 79fff3321d75b1a18231876a1aceb36eacbbf6221ePavel Emelyanov#endif 80fff3321d75b1a18231876a1aceb36eacbbf6221ePavel Emelyanov unsigned int num_members; 81fff3321d75b1a18231876a1aceb36eacbbf6221ePavel Emelyanov u16 id; 82fff3321d75b1a18231876a1aceb36eacbbf6221ePavel Emelyanov u8 type; 8377f65ebdca506870d99bfabe52bde222511022ecWillem de Bruijn u8 flags; 84fff3321d75b1a18231876a1aceb36eacbbf6221ePavel Emelyanov atomic_t rr_cur; 85fff3321d75b1a18231876a1aceb36eacbbf6221ePavel Emelyanov struct list_head list; 86fff3321d75b1a18231876a1aceb36eacbbf6221ePavel Emelyanov struct sock *arr[PACKET_FANOUT_MAX]; 8777f65ebdca506870d99bfabe52bde222511022ecWillem de Bruijn int next[PACKET_FANOUT_MAX]; 88fff3321d75b1a18231876a1aceb36eacbbf6221ePavel Emelyanov spinlock_t lock; 89fff3321d75b1a18231876a1aceb36eacbbf6221ePavel Emelyanov atomic_t sk_ref; 90fff3321d75b1a18231876a1aceb36eacbbf6221ePavel Emelyanov struct packet_type prot_hook ____cacheline_aligned_in_smp; 91fff3321d75b1a18231876a1aceb36eacbbf6221ePavel Emelyanov}; 92fff3321d75b1a18231876a1aceb36eacbbf6221ePavel Emelyanov 932787b04b6c5e7607510e8248b38b0aeacb5505f6Pavel Emelyanovstruct packet_sock { 942787b04b6c5e7607510e8248b38b0aeacb5505f6Pavel Emelyanov /* struct sock has to be the first member of packet_sock */ 952787b04b6c5e7607510e8248b38b0aeacb5505f6Pavel Emelyanov struct sock sk; 962787b04b6c5e7607510e8248b38b0aeacb5505f6Pavel Emelyanov struct packet_fanout *fanout; 97ee80fbf301adac644d0c9465194a7ec87bcd4a07Daniel Borkmann union tpacket_stats_u stats; 982787b04b6c5e7607510e8248b38b0aeacb5505f6Pavel Emelyanov struct packet_ring_buffer rx_ring; 992787b04b6c5e7607510e8248b38b0aeacb5505f6Pavel Emelyanov struct packet_ring_buffer tx_ring; 1002787b04b6c5e7607510e8248b38b0aeacb5505f6Pavel Emelyanov int copy_thresh; 1012787b04b6c5e7607510e8248b38b0aeacb5505f6Pavel Emelyanov spinlock_t bind_lock; 1022787b04b6c5e7607510e8248b38b0aeacb5505f6Pavel Emelyanov struct mutex pg_vec_lock; 1032787b04b6c5e7607510e8248b38b0aeacb5505f6Pavel Emelyanov unsigned int running:1, /* prot_hook is attached*/ 1042787b04b6c5e7607510e8248b38b0aeacb5505f6Pavel Emelyanov auxdata:1, 1052787b04b6c5e7607510e8248b38b0aeacb5505f6Pavel Emelyanov origdev:1, 1062787b04b6c5e7607510e8248b38b0aeacb5505f6Pavel Emelyanov has_vnet_hdr:1; 1072787b04b6c5e7607510e8248b38b0aeacb5505f6Pavel Emelyanov int ifindex; /* bound device */ 1082787b04b6c5e7607510e8248b38b0aeacb5505f6Pavel Emelyanov __be16 num; 1092787b04b6c5e7607510e8248b38b0aeacb5505f6Pavel Emelyanov struct packet_mclist *mclist; 1102787b04b6c5e7607510e8248b38b0aeacb5505f6Pavel Emelyanov atomic_t mapped; 1112787b04b6c5e7607510e8248b38b0aeacb5505f6Pavel Emelyanov enum tpacket_versions tp_version; 1122787b04b6c5e7607510e8248b38b0aeacb5505f6Pavel Emelyanov unsigned int tp_hdrlen; 1132787b04b6c5e7607510e8248b38b0aeacb5505f6Pavel Emelyanov unsigned int tp_reserve; 1142787b04b6c5e7607510e8248b38b0aeacb5505f6Pavel Emelyanov unsigned int tp_loss:1; 1155920cd3a41f1aefc30e9ce86384fc2fe9f5fe0c0Paul Chavent unsigned int tp_tx_has_off:1; 1162787b04b6c5e7607510e8248b38b0aeacb5505f6Pavel Emelyanov unsigned int tp_tstamp; 117e40526cb20b5ee53419452e1f03d97092f144418Daniel Borkmann struct net_device __rcu *cached_dev; 118d346a3fae3ff1d99f5d0c819bf86edf9094a26a1Daniel Borkmann int (*xmit)(struct sk_buff *skb); 1192787b04b6c5e7607510e8248b38b0aeacb5505f6Pavel Emelyanov struct packet_type prot_hook ____cacheline_aligned_in_smp; 1202787b04b6c5e7607510e8248b38b0aeacb5505f6Pavel Emelyanov}; 1212787b04b6c5e7607510e8248b38b0aeacb5505f6Pavel Emelyanov 1222787b04b6c5e7607510e8248b38b0aeacb5505f6Pavel Emelyanovstatic struct packet_sock *pkt_sk(struct sock *sk) 1232787b04b6c5e7607510e8248b38b0aeacb5505f6Pavel Emelyanov{ 1242787b04b6c5e7607510e8248b38b0aeacb5505f6Pavel Emelyanov return (struct packet_sock *)sk; 1252787b04b6c5e7607510e8248b38b0aeacb5505f6Pavel Emelyanov} 1262787b04b6c5e7607510e8248b38b0aeacb5505f6Pavel Emelyanov 1272787b04b6c5e7607510e8248b38b0aeacb5505f6Pavel Emelyanov#endif 128