1f5bffecda951b59d0d3cdd616d68952abc52bc40Alexei Starovoitov/* 2f5bffecda951b59d0d3cdd616d68952abc52bc40Alexei Starovoitov * Linux Socket Filter - Kernel level socket filtering 3f5bffecda951b59d0d3cdd616d68952abc52bc40Alexei Starovoitov * 4f5bffecda951b59d0d3cdd616d68952abc52bc40Alexei Starovoitov * Based on the design of the Berkeley Packet Filter. The new 5f5bffecda951b59d0d3cdd616d68952abc52bc40Alexei Starovoitov * internal format has been designed by PLUMgrid: 6f5bffecda951b59d0d3cdd616d68952abc52bc40Alexei Starovoitov * 7f5bffecda951b59d0d3cdd616d68952abc52bc40Alexei Starovoitov * Copyright (c) 2011 - 2014 PLUMgrid, http://plumgrid.com 8f5bffecda951b59d0d3cdd616d68952abc52bc40Alexei Starovoitov * 9f5bffecda951b59d0d3cdd616d68952abc52bc40Alexei Starovoitov * Authors: 10f5bffecda951b59d0d3cdd616d68952abc52bc40Alexei Starovoitov * 11f5bffecda951b59d0d3cdd616d68952abc52bc40Alexei Starovoitov * Jay Schulist <jschlst@samba.org> 12f5bffecda951b59d0d3cdd616d68952abc52bc40Alexei Starovoitov * Alexei Starovoitov <ast@plumgrid.com> 13f5bffecda951b59d0d3cdd616d68952abc52bc40Alexei Starovoitov * Daniel Borkmann <dborkman@redhat.com> 14f5bffecda951b59d0d3cdd616d68952abc52bc40Alexei Starovoitov * 15f5bffecda951b59d0d3cdd616d68952abc52bc40Alexei Starovoitov * This program is free software; you can redistribute it and/or 16f5bffecda951b59d0d3cdd616d68952abc52bc40Alexei Starovoitov * modify it under the terms of the GNU General Public License 17f5bffecda951b59d0d3cdd616d68952abc52bc40Alexei Starovoitov * as published by the Free Software Foundation; either version 18f5bffecda951b59d0d3cdd616d68952abc52bc40Alexei Starovoitov * 2 of the License, or (at your option) any later version. 19f5bffecda951b59d0d3cdd616d68952abc52bc40Alexei Starovoitov * 20f5bffecda951b59d0d3cdd616d68952abc52bc40Alexei Starovoitov * Andi Kleen - Fix a few bad bugs and races. 214df95ff488eb796aab9566652c250330179def17Alexei Starovoitov * Kris Katterjohn - Added many additional checks in bpf_check_classic() 22f5bffecda951b59d0d3cdd616d68952abc52bc40Alexei Starovoitov */ 23738cbe72adc5c8f2016c4c68aa5162631d4f27e1Daniel Borkmann 24f5bffecda951b59d0d3cdd616d68952abc52bc40Alexei Starovoitov#include <linux/filter.h> 25f5bffecda951b59d0d3cdd616d68952abc52bc40Alexei Starovoitov#include <linux/skbuff.h> 2660a3b2253c413cf601783b070507d7dd6620c954Daniel Borkmann#include <linux/vmalloc.h> 27738cbe72adc5c8f2016c4c68aa5162631d4f27e1Daniel Borkmann#include <linux/random.h> 28738cbe72adc5c8f2016c4c68aa5162631d4f27e1Daniel Borkmann#include <linux/moduleloader.h> 29f5bffecda951b59d0d3cdd616d68952abc52bc40Alexei Starovoitov#include <asm/unaligned.h> 3009756af46893c18839062976c3252e93a1beeba7Alexei Starovoitov#include <linux/bpf.h> 31f5bffecda951b59d0d3cdd616d68952abc52bc40Alexei Starovoitov 32f5bffecda951b59d0d3cdd616d68952abc52bc40Alexei Starovoitov/* Registers */ 33f5bffecda951b59d0d3cdd616d68952abc52bc40Alexei Starovoitov#define BPF_R0 regs[BPF_REG_0] 34f5bffecda951b59d0d3cdd616d68952abc52bc40Alexei Starovoitov#define BPF_R1 regs[BPF_REG_1] 35f5bffecda951b59d0d3cdd616d68952abc52bc40Alexei Starovoitov#define BPF_R2 regs[BPF_REG_2] 36f5bffecda951b59d0d3cdd616d68952abc52bc40Alexei Starovoitov#define BPF_R3 regs[BPF_REG_3] 37f5bffecda951b59d0d3cdd616d68952abc52bc40Alexei Starovoitov#define BPF_R4 regs[BPF_REG_4] 38f5bffecda951b59d0d3cdd616d68952abc52bc40Alexei Starovoitov#define BPF_R5 regs[BPF_REG_5] 39f5bffecda951b59d0d3cdd616d68952abc52bc40Alexei Starovoitov#define BPF_R6 regs[BPF_REG_6] 40f5bffecda951b59d0d3cdd616d68952abc52bc40Alexei Starovoitov#define BPF_R7 regs[BPF_REG_7] 41f5bffecda951b59d0d3cdd616d68952abc52bc40Alexei Starovoitov#define BPF_R8 regs[BPF_REG_8] 42f5bffecda951b59d0d3cdd616d68952abc52bc40Alexei Starovoitov#define BPF_R9 regs[BPF_REG_9] 43f5bffecda951b59d0d3cdd616d68952abc52bc40Alexei Starovoitov#define BPF_R10 regs[BPF_REG_10] 44f5bffecda951b59d0d3cdd616d68952abc52bc40Alexei Starovoitov 45f5bffecda951b59d0d3cdd616d68952abc52bc40Alexei Starovoitov/* Named registers */ 46f5bffecda951b59d0d3cdd616d68952abc52bc40Alexei Starovoitov#define DST regs[insn->dst_reg] 47f5bffecda951b59d0d3cdd616d68952abc52bc40Alexei Starovoitov#define SRC regs[insn->src_reg] 48f5bffecda951b59d0d3cdd616d68952abc52bc40Alexei Starovoitov#define FP regs[BPF_REG_FP] 49f5bffecda951b59d0d3cdd616d68952abc52bc40Alexei Starovoitov#define ARG1 regs[BPF_REG_ARG1] 50f5bffecda951b59d0d3cdd616d68952abc52bc40Alexei Starovoitov#define CTX regs[BPF_REG_CTX] 51f5bffecda951b59d0d3cdd616d68952abc52bc40Alexei Starovoitov#define IMM insn->imm 52f5bffecda951b59d0d3cdd616d68952abc52bc40Alexei Starovoitov 53f5bffecda951b59d0d3cdd616d68952abc52bc40Alexei Starovoitov/* No hurry in this branch 54f5bffecda951b59d0d3cdd616d68952abc52bc40Alexei Starovoitov * 55f5bffecda951b59d0d3cdd616d68952abc52bc40Alexei Starovoitov * Exported for the bpf jit load helper. 56f5bffecda951b59d0d3cdd616d68952abc52bc40Alexei Starovoitov */ 57f5bffecda951b59d0d3cdd616d68952abc52bc40Alexei Starovoitovvoid *bpf_internal_load_pointer_neg_helper(const struct sk_buff *skb, int k, unsigned int size) 58f5bffecda951b59d0d3cdd616d68952abc52bc40Alexei Starovoitov{ 59f5bffecda951b59d0d3cdd616d68952abc52bc40Alexei Starovoitov u8 *ptr = NULL; 60f5bffecda951b59d0d3cdd616d68952abc52bc40Alexei Starovoitov 61f5bffecda951b59d0d3cdd616d68952abc52bc40Alexei Starovoitov if (k >= SKF_NET_OFF) 62f5bffecda951b59d0d3cdd616d68952abc52bc40Alexei Starovoitov ptr = skb_network_header(skb) + k - SKF_NET_OFF; 63f5bffecda951b59d0d3cdd616d68952abc52bc40Alexei Starovoitov else if (k >= SKF_LL_OFF) 64f5bffecda951b59d0d3cdd616d68952abc52bc40Alexei Starovoitov ptr = skb_mac_header(skb) + k - SKF_LL_OFF; 65f5bffecda951b59d0d3cdd616d68952abc52bc40Alexei Starovoitov if (ptr >= skb->head && ptr + size <= skb_tail_pointer(skb)) 66f5bffecda951b59d0d3cdd616d68952abc52bc40Alexei Starovoitov return ptr; 67f5bffecda951b59d0d3cdd616d68952abc52bc40Alexei Starovoitov 68f5bffecda951b59d0d3cdd616d68952abc52bc40Alexei Starovoitov return NULL; 69f5bffecda951b59d0d3cdd616d68952abc52bc40Alexei Starovoitov} 70f5bffecda951b59d0d3cdd616d68952abc52bc40Alexei Starovoitov 7160a3b2253c413cf601783b070507d7dd6620c954Daniel Borkmannstruct bpf_prog *bpf_prog_alloc(unsigned int size, gfp_t gfp_extra_flags) 7260a3b2253c413cf601783b070507d7dd6620c954Daniel Borkmann{ 7360a3b2253c413cf601783b070507d7dd6620c954Daniel Borkmann gfp_t gfp_flags = GFP_KERNEL | __GFP_HIGHMEM | __GFP_ZERO | 7460a3b2253c413cf601783b070507d7dd6620c954Daniel Borkmann gfp_extra_flags; 7509756af46893c18839062976c3252e93a1beeba7Alexei Starovoitov struct bpf_prog_aux *aux; 7660a3b2253c413cf601783b070507d7dd6620c954Daniel Borkmann struct bpf_prog *fp; 7760a3b2253c413cf601783b070507d7dd6620c954Daniel Borkmann 7860a3b2253c413cf601783b070507d7dd6620c954Daniel Borkmann size = round_up(size, PAGE_SIZE); 7960a3b2253c413cf601783b070507d7dd6620c954Daniel Borkmann fp = __vmalloc(size, gfp_flags, PAGE_KERNEL); 8060a3b2253c413cf601783b070507d7dd6620c954Daniel Borkmann if (fp == NULL) 8160a3b2253c413cf601783b070507d7dd6620c954Daniel Borkmann return NULL; 8260a3b2253c413cf601783b070507d7dd6620c954Daniel Borkmann 8309756af46893c18839062976c3252e93a1beeba7Alexei Starovoitov aux = kzalloc(sizeof(*aux), GFP_KERNEL | gfp_extra_flags); 8409756af46893c18839062976c3252e93a1beeba7Alexei Starovoitov if (aux == NULL) { 8560a3b2253c413cf601783b070507d7dd6620c954Daniel Borkmann vfree(fp); 8660a3b2253c413cf601783b070507d7dd6620c954Daniel Borkmann return NULL; 8760a3b2253c413cf601783b070507d7dd6620c954Daniel Borkmann } 8860a3b2253c413cf601783b070507d7dd6620c954Daniel Borkmann 8960a3b2253c413cf601783b070507d7dd6620c954Daniel Borkmann fp->pages = size / PAGE_SIZE; 9009756af46893c18839062976c3252e93a1beeba7Alexei Starovoitov fp->aux = aux; 9160a3b2253c413cf601783b070507d7dd6620c954Daniel Borkmann 9260a3b2253c413cf601783b070507d7dd6620c954Daniel Borkmann return fp; 9360a3b2253c413cf601783b070507d7dd6620c954Daniel Borkmann} 9460a3b2253c413cf601783b070507d7dd6620c954Daniel BorkmannEXPORT_SYMBOL_GPL(bpf_prog_alloc); 9560a3b2253c413cf601783b070507d7dd6620c954Daniel Borkmann 9660a3b2253c413cf601783b070507d7dd6620c954Daniel Borkmannstruct bpf_prog *bpf_prog_realloc(struct bpf_prog *fp_old, unsigned int size, 9760a3b2253c413cf601783b070507d7dd6620c954Daniel Borkmann gfp_t gfp_extra_flags) 9860a3b2253c413cf601783b070507d7dd6620c954Daniel Borkmann{ 9960a3b2253c413cf601783b070507d7dd6620c954Daniel Borkmann gfp_t gfp_flags = GFP_KERNEL | __GFP_HIGHMEM | __GFP_ZERO | 10060a3b2253c413cf601783b070507d7dd6620c954Daniel Borkmann gfp_extra_flags; 10160a3b2253c413cf601783b070507d7dd6620c954Daniel Borkmann struct bpf_prog *fp; 10260a3b2253c413cf601783b070507d7dd6620c954Daniel Borkmann 10360a3b2253c413cf601783b070507d7dd6620c954Daniel Borkmann BUG_ON(fp_old == NULL); 10460a3b2253c413cf601783b070507d7dd6620c954Daniel Borkmann 10560a3b2253c413cf601783b070507d7dd6620c954Daniel Borkmann size = round_up(size, PAGE_SIZE); 10660a3b2253c413cf601783b070507d7dd6620c954Daniel Borkmann if (size <= fp_old->pages * PAGE_SIZE) 10760a3b2253c413cf601783b070507d7dd6620c954Daniel Borkmann return fp_old; 10860a3b2253c413cf601783b070507d7dd6620c954Daniel Borkmann 10960a3b2253c413cf601783b070507d7dd6620c954Daniel Borkmann fp = __vmalloc(size, gfp_flags, PAGE_KERNEL); 11060a3b2253c413cf601783b070507d7dd6620c954Daniel Borkmann if (fp != NULL) { 11160a3b2253c413cf601783b070507d7dd6620c954Daniel Borkmann memcpy(fp, fp_old, fp_old->pages * PAGE_SIZE); 11260a3b2253c413cf601783b070507d7dd6620c954Daniel Borkmann fp->pages = size / PAGE_SIZE; 11360a3b2253c413cf601783b070507d7dd6620c954Daniel Borkmann 11409756af46893c18839062976c3252e93a1beeba7Alexei Starovoitov /* We keep fp->aux from fp_old around in the new 11560a3b2253c413cf601783b070507d7dd6620c954Daniel Borkmann * reallocated structure. 11660a3b2253c413cf601783b070507d7dd6620c954Daniel Borkmann */ 11709756af46893c18839062976c3252e93a1beeba7Alexei Starovoitov fp_old->aux = NULL; 11860a3b2253c413cf601783b070507d7dd6620c954Daniel Borkmann __bpf_prog_free(fp_old); 11960a3b2253c413cf601783b070507d7dd6620c954Daniel Borkmann } 12060a3b2253c413cf601783b070507d7dd6620c954Daniel Borkmann 12160a3b2253c413cf601783b070507d7dd6620c954Daniel Borkmann return fp; 12260a3b2253c413cf601783b070507d7dd6620c954Daniel Borkmann} 12360a3b2253c413cf601783b070507d7dd6620c954Daniel BorkmannEXPORT_SYMBOL_GPL(bpf_prog_realloc); 12460a3b2253c413cf601783b070507d7dd6620c954Daniel Borkmann 12560a3b2253c413cf601783b070507d7dd6620c954Daniel Borkmannvoid __bpf_prog_free(struct bpf_prog *fp) 12660a3b2253c413cf601783b070507d7dd6620c954Daniel Borkmann{ 12709756af46893c18839062976c3252e93a1beeba7Alexei Starovoitov kfree(fp->aux); 12860a3b2253c413cf601783b070507d7dd6620c954Daniel Borkmann vfree(fp); 12960a3b2253c413cf601783b070507d7dd6620c954Daniel Borkmann} 13060a3b2253c413cf601783b070507d7dd6620c954Daniel BorkmannEXPORT_SYMBOL_GPL(__bpf_prog_free); 13160a3b2253c413cf601783b070507d7dd6620c954Daniel Borkmann 132b954d83421d51d822c42e5ab7b65069b25ad3005Daniel Borkmann#ifdef CONFIG_BPF_JIT 133738cbe72adc5c8f2016c4c68aa5162631d4f27e1Daniel Borkmannstruct bpf_binary_header * 134738cbe72adc5c8f2016c4c68aa5162631d4f27e1Daniel Borkmannbpf_jit_binary_alloc(unsigned int proglen, u8 **image_ptr, 135738cbe72adc5c8f2016c4c68aa5162631d4f27e1Daniel Borkmann unsigned int alignment, 136738cbe72adc5c8f2016c4c68aa5162631d4f27e1Daniel Borkmann bpf_jit_fill_hole_t bpf_fill_ill_insns) 137738cbe72adc5c8f2016c4c68aa5162631d4f27e1Daniel Borkmann{ 138738cbe72adc5c8f2016c4c68aa5162631d4f27e1Daniel Borkmann struct bpf_binary_header *hdr; 139738cbe72adc5c8f2016c4c68aa5162631d4f27e1Daniel Borkmann unsigned int size, hole, start; 140738cbe72adc5c8f2016c4c68aa5162631d4f27e1Daniel Borkmann 141738cbe72adc5c8f2016c4c68aa5162631d4f27e1Daniel Borkmann /* Most of BPF filters are really small, but if some of them 142738cbe72adc5c8f2016c4c68aa5162631d4f27e1Daniel Borkmann * fill a page, allow at least 128 extra bytes to insert a 143738cbe72adc5c8f2016c4c68aa5162631d4f27e1Daniel Borkmann * random section of illegal instructions. 144738cbe72adc5c8f2016c4c68aa5162631d4f27e1Daniel Borkmann */ 145738cbe72adc5c8f2016c4c68aa5162631d4f27e1Daniel Borkmann size = round_up(proglen + sizeof(*hdr) + 128, PAGE_SIZE); 146738cbe72adc5c8f2016c4c68aa5162631d4f27e1Daniel Borkmann hdr = module_alloc(size); 147738cbe72adc5c8f2016c4c68aa5162631d4f27e1Daniel Borkmann if (hdr == NULL) 148738cbe72adc5c8f2016c4c68aa5162631d4f27e1Daniel Borkmann return NULL; 149738cbe72adc5c8f2016c4c68aa5162631d4f27e1Daniel Borkmann 150738cbe72adc5c8f2016c4c68aa5162631d4f27e1Daniel Borkmann /* Fill space with illegal/arch-dep instructions. */ 151738cbe72adc5c8f2016c4c68aa5162631d4f27e1Daniel Borkmann bpf_fill_ill_insns(hdr, size); 152738cbe72adc5c8f2016c4c68aa5162631d4f27e1Daniel Borkmann 153738cbe72adc5c8f2016c4c68aa5162631d4f27e1Daniel Borkmann hdr->pages = size / PAGE_SIZE; 154738cbe72adc5c8f2016c4c68aa5162631d4f27e1Daniel Borkmann hole = min_t(unsigned int, size - (proglen + sizeof(*hdr)), 155738cbe72adc5c8f2016c4c68aa5162631d4f27e1Daniel Borkmann PAGE_SIZE - sizeof(*hdr)); 156738cbe72adc5c8f2016c4c68aa5162631d4f27e1Daniel Borkmann start = (prandom_u32() % hole) & ~(alignment - 1); 157738cbe72adc5c8f2016c4c68aa5162631d4f27e1Daniel Borkmann 158738cbe72adc5c8f2016c4c68aa5162631d4f27e1Daniel Borkmann /* Leave a random number of instructions before BPF code. */ 159738cbe72adc5c8f2016c4c68aa5162631d4f27e1Daniel Borkmann *image_ptr = &hdr->image[start]; 160738cbe72adc5c8f2016c4c68aa5162631d4f27e1Daniel Borkmann 161738cbe72adc5c8f2016c4c68aa5162631d4f27e1Daniel Borkmann return hdr; 162738cbe72adc5c8f2016c4c68aa5162631d4f27e1Daniel Borkmann} 163738cbe72adc5c8f2016c4c68aa5162631d4f27e1Daniel Borkmann 164738cbe72adc5c8f2016c4c68aa5162631d4f27e1Daniel Borkmannvoid bpf_jit_binary_free(struct bpf_binary_header *hdr) 165738cbe72adc5c8f2016c4c68aa5162631d4f27e1Daniel Borkmann{ 166738cbe72adc5c8f2016c4c68aa5162631d4f27e1Daniel Borkmann module_free(NULL, hdr); 167738cbe72adc5c8f2016c4c68aa5162631d4f27e1Daniel Borkmann} 168b954d83421d51d822c42e5ab7b65069b25ad3005Daniel Borkmann#endif /* CONFIG_BPF_JIT */ 169738cbe72adc5c8f2016c4c68aa5162631d4f27e1Daniel Borkmann 170f5bffecda951b59d0d3cdd616d68952abc52bc40Alexei Starovoitov/* Base function for offset calculation. Needs to go into .text section, 171f5bffecda951b59d0d3cdd616d68952abc52bc40Alexei Starovoitov * therefore keeping it non-static as well; will also be used by JITs 172f5bffecda951b59d0d3cdd616d68952abc52bc40Alexei Starovoitov * anyway later on, so do not let the compiler omit it. 173f5bffecda951b59d0d3cdd616d68952abc52bc40Alexei Starovoitov */ 174f5bffecda951b59d0d3cdd616d68952abc52bc40Alexei Starovoitovnoinline u64 __bpf_call_base(u64 r1, u64 r2, u64 r3, u64 r4, u64 r5) 175f5bffecda951b59d0d3cdd616d68952abc52bc40Alexei Starovoitov{ 176f5bffecda951b59d0d3cdd616d68952abc52bc40Alexei Starovoitov return 0; 177f5bffecda951b59d0d3cdd616d68952abc52bc40Alexei Starovoitov} 178f5bffecda951b59d0d3cdd616d68952abc52bc40Alexei Starovoitov 179f5bffecda951b59d0d3cdd616d68952abc52bc40Alexei Starovoitov/** 1807ae457c1e5b45a1b826fad9d62b32191d2bdcfdbAlexei Starovoitov * __bpf_prog_run - run eBPF program on a given context 1817ae457c1e5b45a1b826fad9d62b32191d2bdcfdbAlexei Starovoitov * @ctx: is the data we are operating on 1827ae457c1e5b45a1b826fad9d62b32191d2bdcfdbAlexei Starovoitov * @insn: is the array of eBPF instructions 183f5bffecda951b59d0d3cdd616d68952abc52bc40Alexei Starovoitov * 1847ae457c1e5b45a1b826fad9d62b32191d2bdcfdbAlexei Starovoitov * Decode and execute eBPF instructions. 185f5bffecda951b59d0d3cdd616d68952abc52bc40Alexei Starovoitov */ 1867ae457c1e5b45a1b826fad9d62b32191d2bdcfdbAlexei Starovoitovstatic unsigned int __bpf_prog_run(void *ctx, const struct bpf_insn *insn) 187f5bffecda951b59d0d3cdd616d68952abc52bc40Alexei Starovoitov{ 188f5bffecda951b59d0d3cdd616d68952abc52bc40Alexei Starovoitov u64 stack[MAX_BPF_STACK / sizeof(u64)]; 189f5bffecda951b59d0d3cdd616d68952abc52bc40Alexei Starovoitov u64 regs[MAX_BPF_REG], tmp; 190f5bffecda951b59d0d3cdd616d68952abc52bc40Alexei Starovoitov static const void *jumptable[256] = { 191f5bffecda951b59d0d3cdd616d68952abc52bc40Alexei Starovoitov [0 ... 255] = &&default_label, 192f5bffecda951b59d0d3cdd616d68952abc52bc40Alexei Starovoitov /* Now overwrite non-defaults ... */ 193f5bffecda951b59d0d3cdd616d68952abc52bc40Alexei Starovoitov /* 32 bit ALU operations */ 194f5bffecda951b59d0d3cdd616d68952abc52bc40Alexei Starovoitov [BPF_ALU | BPF_ADD | BPF_X] = &&ALU_ADD_X, 195f5bffecda951b59d0d3cdd616d68952abc52bc40Alexei Starovoitov [BPF_ALU | BPF_ADD | BPF_K] = &&ALU_ADD_K, 196f5bffecda951b59d0d3cdd616d68952abc52bc40Alexei Starovoitov [BPF_ALU | BPF_SUB | BPF_X] = &&ALU_SUB_X, 197f5bffecda951b59d0d3cdd616d68952abc52bc40Alexei Starovoitov [BPF_ALU | BPF_SUB | BPF_K] = &&ALU_SUB_K, 198f5bffecda951b59d0d3cdd616d68952abc52bc40Alexei Starovoitov [BPF_ALU | BPF_AND | BPF_X] = &&ALU_AND_X, 199f5bffecda951b59d0d3cdd616d68952abc52bc40Alexei Starovoitov [BPF_ALU | BPF_AND | BPF_K] = &&ALU_AND_K, 200f5bffecda951b59d0d3cdd616d68952abc52bc40Alexei Starovoitov [BPF_ALU | BPF_OR | BPF_X] = &&ALU_OR_X, 201f5bffecda951b59d0d3cdd616d68952abc52bc40Alexei Starovoitov [BPF_ALU | BPF_OR | BPF_K] = &&ALU_OR_K, 202f5bffecda951b59d0d3cdd616d68952abc52bc40Alexei Starovoitov [BPF_ALU | BPF_LSH | BPF_X] = &&ALU_LSH_X, 203f5bffecda951b59d0d3cdd616d68952abc52bc40Alexei Starovoitov [BPF_ALU | BPF_LSH | BPF_K] = &&ALU_LSH_K, 204f5bffecda951b59d0d3cdd616d68952abc52bc40Alexei Starovoitov [BPF_ALU | BPF_RSH | BPF_X] = &&ALU_RSH_X, 205f5bffecda951b59d0d3cdd616d68952abc52bc40Alexei Starovoitov [BPF_ALU | BPF_RSH | BPF_K] = &&ALU_RSH_K, 206f5bffecda951b59d0d3cdd616d68952abc52bc40Alexei Starovoitov [BPF_ALU | BPF_XOR | BPF_X] = &&ALU_XOR_X, 207f5bffecda951b59d0d3cdd616d68952abc52bc40Alexei Starovoitov [BPF_ALU | BPF_XOR | BPF_K] = &&ALU_XOR_K, 208f5bffecda951b59d0d3cdd616d68952abc52bc40Alexei Starovoitov [BPF_ALU | BPF_MUL | BPF_X] = &&ALU_MUL_X, 209f5bffecda951b59d0d3cdd616d68952abc52bc40Alexei Starovoitov [BPF_ALU | BPF_MUL | BPF_K] = &&ALU_MUL_K, 210f5bffecda951b59d0d3cdd616d68952abc52bc40Alexei Starovoitov [BPF_ALU | BPF_MOV | BPF_X] = &&ALU_MOV_X, 211f5bffecda951b59d0d3cdd616d68952abc52bc40Alexei Starovoitov [BPF_ALU | BPF_MOV | BPF_K] = &&ALU_MOV_K, 212f5bffecda951b59d0d3cdd616d68952abc52bc40Alexei Starovoitov [BPF_ALU | BPF_DIV | BPF_X] = &&ALU_DIV_X, 213f5bffecda951b59d0d3cdd616d68952abc52bc40Alexei Starovoitov [BPF_ALU | BPF_DIV | BPF_K] = &&ALU_DIV_K, 214f5bffecda951b59d0d3cdd616d68952abc52bc40Alexei Starovoitov [BPF_ALU | BPF_MOD | BPF_X] = &&ALU_MOD_X, 215f5bffecda951b59d0d3cdd616d68952abc52bc40Alexei Starovoitov [BPF_ALU | BPF_MOD | BPF_K] = &&ALU_MOD_K, 216f5bffecda951b59d0d3cdd616d68952abc52bc40Alexei Starovoitov [BPF_ALU | BPF_NEG] = &&ALU_NEG, 217f5bffecda951b59d0d3cdd616d68952abc52bc40Alexei Starovoitov [BPF_ALU | BPF_END | BPF_TO_BE] = &&ALU_END_TO_BE, 218f5bffecda951b59d0d3cdd616d68952abc52bc40Alexei Starovoitov [BPF_ALU | BPF_END | BPF_TO_LE] = &&ALU_END_TO_LE, 219f5bffecda951b59d0d3cdd616d68952abc52bc40Alexei Starovoitov /* 64 bit ALU operations */ 220f5bffecda951b59d0d3cdd616d68952abc52bc40Alexei Starovoitov [BPF_ALU64 | BPF_ADD | BPF_X] = &&ALU64_ADD_X, 221f5bffecda951b59d0d3cdd616d68952abc52bc40Alexei Starovoitov [BPF_ALU64 | BPF_ADD | BPF_K] = &&ALU64_ADD_K, 222f5bffecda951b59d0d3cdd616d68952abc52bc40Alexei Starovoitov [BPF_ALU64 | BPF_SUB | BPF_X] = &&ALU64_SUB_X, 223f5bffecda951b59d0d3cdd616d68952abc52bc40Alexei Starovoitov [BPF_ALU64 | BPF_SUB | BPF_K] = &&ALU64_SUB_K, 224f5bffecda951b59d0d3cdd616d68952abc52bc40Alexei Starovoitov [BPF_ALU64 | BPF_AND | BPF_X] = &&ALU64_AND_X, 225f5bffecda951b59d0d3cdd616d68952abc52bc40Alexei Starovoitov [BPF_ALU64 | BPF_AND | BPF_K] = &&ALU64_AND_K, 226f5bffecda951b59d0d3cdd616d68952abc52bc40Alexei Starovoitov [BPF_ALU64 | BPF_OR | BPF_X] = &&ALU64_OR_X, 227f5bffecda951b59d0d3cdd616d68952abc52bc40Alexei Starovoitov [BPF_ALU64 | BPF_OR | BPF_K] = &&ALU64_OR_K, 228f5bffecda951b59d0d3cdd616d68952abc52bc40Alexei Starovoitov [BPF_ALU64 | BPF_LSH | BPF_X] = &&ALU64_LSH_X, 229f5bffecda951b59d0d3cdd616d68952abc52bc40Alexei Starovoitov [BPF_ALU64 | BPF_LSH | BPF_K] = &&ALU64_LSH_K, 230f5bffecda951b59d0d3cdd616d68952abc52bc40Alexei Starovoitov [BPF_ALU64 | BPF_RSH | BPF_X] = &&ALU64_RSH_X, 231f5bffecda951b59d0d3cdd616d68952abc52bc40Alexei Starovoitov [BPF_ALU64 | BPF_RSH | BPF_K] = &&ALU64_RSH_K, 232f5bffecda951b59d0d3cdd616d68952abc52bc40Alexei Starovoitov [BPF_ALU64 | BPF_XOR | BPF_X] = &&ALU64_XOR_X, 233f5bffecda951b59d0d3cdd616d68952abc52bc40Alexei Starovoitov [BPF_ALU64 | BPF_XOR | BPF_K] = &&ALU64_XOR_K, 234f5bffecda951b59d0d3cdd616d68952abc52bc40Alexei Starovoitov [BPF_ALU64 | BPF_MUL | BPF_X] = &&ALU64_MUL_X, 235f5bffecda951b59d0d3cdd616d68952abc52bc40Alexei Starovoitov [BPF_ALU64 | BPF_MUL | BPF_K] = &&ALU64_MUL_K, 236f5bffecda951b59d0d3cdd616d68952abc52bc40Alexei Starovoitov [BPF_ALU64 | BPF_MOV | BPF_X] = &&ALU64_MOV_X, 237f5bffecda951b59d0d3cdd616d68952abc52bc40Alexei Starovoitov [BPF_ALU64 | BPF_MOV | BPF_K] = &&ALU64_MOV_K, 238f5bffecda951b59d0d3cdd616d68952abc52bc40Alexei Starovoitov [BPF_ALU64 | BPF_ARSH | BPF_X] = &&ALU64_ARSH_X, 239f5bffecda951b59d0d3cdd616d68952abc52bc40Alexei Starovoitov [BPF_ALU64 | BPF_ARSH | BPF_K] = &&ALU64_ARSH_K, 240f5bffecda951b59d0d3cdd616d68952abc52bc40Alexei Starovoitov [BPF_ALU64 | BPF_DIV | BPF_X] = &&ALU64_DIV_X, 241f5bffecda951b59d0d3cdd616d68952abc52bc40Alexei Starovoitov [BPF_ALU64 | BPF_DIV | BPF_K] = &&ALU64_DIV_K, 242f5bffecda951b59d0d3cdd616d68952abc52bc40Alexei Starovoitov [BPF_ALU64 | BPF_MOD | BPF_X] = &&ALU64_MOD_X, 243f5bffecda951b59d0d3cdd616d68952abc52bc40Alexei Starovoitov [BPF_ALU64 | BPF_MOD | BPF_K] = &&ALU64_MOD_K, 244f5bffecda951b59d0d3cdd616d68952abc52bc40Alexei Starovoitov [BPF_ALU64 | BPF_NEG] = &&ALU64_NEG, 245f5bffecda951b59d0d3cdd616d68952abc52bc40Alexei Starovoitov /* Call instruction */ 246f5bffecda951b59d0d3cdd616d68952abc52bc40Alexei Starovoitov [BPF_JMP | BPF_CALL] = &&JMP_CALL, 247f5bffecda951b59d0d3cdd616d68952abc52bc40Alexei Starovoitov /* Jumps */ 248f5bffecda951b59d0d3cdd616d68952abc52bc40Alexei Starovoitov [BPF_JMP | BPF_JA] = &&JMP_JA, 249f5bffecda951b59d0d3cdd616d68952abc52bc40Alexei Starovoitov [BPF_JMP | BPF_JEQ | BPF_X] = &&JMP_JEQ_X, 250f5bffecda951b59d0d3cdd616d68952abc52bc40Alexei Starovoitov [BPF_JMP | BPF_JEQ | BPF_K] = &&JMP_JEQ_K, 251f5bffecda951b59d0d3cdd616d68952abc52bc40Alexei Starovoitov [BPF_JMP | BPF_JNE | BPF_X] = &&JMP_JNE_X, 252f5bffecda951b59d0d3cdd616d68952abc52bc40Alexei Starovoitov [BPF_JMP | BPF_JNE | BPF_K] = &&JMP_JNE_K, 253f5bffecda951b59d0d3cdd616d68952abc52bc40Alexei Starovoitov [BPF_JMP | BPF_JGT | BPF_X] = &&JMP_JGT_X, 254f5bffecda951b59d0d3cdd616d68952abc52bc40Alexei Starovoitov [BPF_JMP | BPF_JGT | BPF_K] = &&JMP_JGT_K, 255f5bffecda951b59d0d3cdd616d68952abc52bc40Alexei Starovoitov [BPF_JMP | BPF_JGE | BPF_X] = &&JMP_JGE_X, 256f5bffecda951b59d0d3cdd616d68952abc52bc40Alexei Starovoitov [BPF_JMP | BPF_JGE | BPF_K] = &&JMP_JGE_K, 257f5bffecda951b59d0d3cdd616d68952abc52bc40Alexei Starovoitov [BPF_JMP | BPF_JSGT | BPF_X] = &&JMP_JSGT_X, 258f5bffecda951b59d0d3cdd616d68952abc52bc40Alexei Starovoitov [BPF_JMP | BPF_JSGT | BPF_K] = &&JMP_JSGT_K, 259f5bffecda951b59d0d3cdd616d68952abc52bc40Alexei Starovoitov [BPF_JMP | BPF_JSGE | BPF_X] = &&JMP_JSGE_X, 260f5bffecda951b59d0d3cdd616d68952abc52bc40Alexei Starovoitov [BPF_JMP | BPF_JSGE | BPF_K] = &&JMP_JSGE_K, 261f5bffecda951b59d0d3cdd616d68952abc52bc40Alexei Starovoitov [BPF_JMP | BPF_JSET | BPF_X] = &&JMP_JSET_X, 262f5bffecda951b59d0d3cdd616d68952abc52bc40Alexei Starovoitov [BPF_JMP | BPF_JSET | BPF_K] = &&JMP_JSET_K, 263f5bffecda951b59d0d3cdd616d68952abc52bc40Alexei Starovoitov /* Program return */ 264f5bffecda951b59d0d3cdd616d68952abc52bc40Alexei Starovoitov [BPF_JMP | BPF_EXIT] = &&JMP_EXIT, 265f5bffecda951b59d0d3cdd616d68952abc52bc40Alexei Starovoitov /* Store instructions */ 266f5bffecda951b59d0d3cdd616d68952abc52bc40Alexei Starovoitov [BPF_STX | BPF_MEM | BPF_B] = &&STX_MEM_B, 267f5bffecda951b59d0d3cdd616d68952abc52bc40Alexei Starovoitov [BPF_STX | BPF_MEM | BPF_H] = &&STX_MEM_H, 268f5bffecda951b59d0d3cdd616d68952abc52bc40Alexei Starovoitov [BPF_STX | BPF_MEM | BPF_W] = &&STX_MEM_W, 269f5bffecda951b59d0d3cdd616d68952abc52bc40Alexei Starovoitov [BPF_STX | BPF_MEM | BPF_DW] = &&STX_MEM_DW, 270f5bffecda951b59d0d3cdd616d68952abc52bc40Alexei Starovoitov [BPF_STX | BPF_XADD | BPF_W] = &&STX_XADD_W, 271f5bffecda951b59d0d3cdd616d68952abc52bc40Alexei Starovoitov [BPF_STX | BPF_XADD | BPF_DW] = &&STX_XADD_DW, 272f5bffecda951b59d0d3cdd616d68952abc52bc40Alexei Starovoitov [BPF_ST | BPF_MEM | BPF_B] = &&ST_MEM_B, 273f5bffecda951b59d0d3cdd616d68952abc52bc40Alexei Starovoitov [BPF_ST | BPF_MEM | BPF_H] = &&ST_MEM_H, 274f5bffecda951b59d0d3cdd616d68952abc52bc40Alexei Starovoitov [BPF_ST | BPF_MEM | BPF_W] = &&ST_MEM_W, 275f5bffecda951b59d0d3cdd616d68952abc52bc40Alexei Starovoitov [BPF_ST | BPF_MEM | BPF_DW] = &&ST_MEM_DW, 276f5bffecda951b59d0d3cdd616d68952abc52bc40Alexei Starovoitov /* Load instructions */ 277f5bffecda951b59d0d3cdd616d68952abc52bc40Alexei Starovoitov [BPF_LDX | BPF_MEM | BPF_B] = &&LDX_MEM_B, 278f5bffecda951b59d0d3cdd616d68952abc52bc40Alexei Starovoitov [BPF_LDX | BPF_MEM | BPF_H] = &&LDX_MEM_H, 279f5bffecda951b59d0d3cdd616d68952abc52bc40Alexei Starovoitov [BPF_LDX | BPF_MEM | BPF_W] = &&LDX_MEM_W, 280f5bffecda951b59d0d3cdd616d68952abc52bc40Alexei Starovoitov [BPF_LDX | BPF_MEM | BPF_DW] = &&LDX_MEM_DW, 281f5bffecda951b59d0d3cdd616d68952abc52bc40Alexei Starovoitov [BPF_LD | BPF_ABS | BPF_W] = &&LD_ABS_W, 282f5bffecda951b59d0d3cdd616d68952abc52bc40Alexei Starovoitov [BPF_LD | BPF_ABS | BPF_H] = &&LD_ABS_H, 283f5bffecda951b59d0d3cdd616d68952abc52bc40Alexei Starovoitov [BPF_LD | BPF_ABS | BPF_B] = &&LD_ABS_B, 284f5bffecda951b59d0d3cdd616d68952abc52bc40Alexei Starovoitov [BPF_LD | BPF_IND | BPF_W] = &&LD_IND_W, 285f5bffecda951b59d0d3cdd616d68952abc52bc40Alexei Starovoitov [BPF_LD | BPF_IND | BPF_H] = &&LD_IND_H, 286f5bffecda951b59d0d3cdd616d68952abc52bc40Alexei Starovoitov [BPF_LD | BPF_IND | BPF_B] = &&LD_IND_B, 28702ab695bb37ee9ad515df0d0790d5977505dd04aAlexei Starovoitov [BPF_LD | BPF_IMM | BPF_DW] = &&LD_IMM_DW, 288f5bffecda951b59d0d3cdd616d68952abc52bc40Alexei Starovoitov }; 289f5bffecda951b59d0d3cdd616d68952abc52bc40Alexei Starovoitov void *ptr; 290f5bffecda951b59d0d3cdd616d68952abc52bc40Alexei Starovoitov int off; 291f5bffecda951b59d0d3cdd616d68952abc52bc40Alexei Starovoitov 292f5bffecda951b59d0d3cdd616d68952abc52bc40Alexei Starovoitov#define CONT ({ insn++; goto select_insn; }) 293f5bffecda951b59d0d3cdd616d68952abc52bc40Alexei Starovoitov#define CONT_JMP ({ insn++; goto select_insn; }) 294f5bffecda951b59d0d3cdd616d68952abc52bc40Alexei Starovoitov 295f5bffecda951b59d0d3cdd616d68952abc52bc40Alexei Starovoitov FP = (u64) (unsigned long) &stack[ARRAY_SIZE(stack)]; 296f5bffecda951b59d0d3cdd616d68952abc52bc40Alexei Starovoitov ARG1 = (u64) (unsigned long) ctx; 297f5bffecda951b59d0d3cdd616d68952abc52bc40Alexei Starovoitov 298f5bffecda951b59d0d3cdd616d68952abc52bc40Alexei Starovoitov /* Registers used in classic BPF programs need to be reset first. */ 299f5bffecda951b59d0d3cdd616d68952abc52bc40Alexei Starovoitov regs[BPF_REG_A] = 0; 300f5bffecda951b59d0d3cdd616d68952abc52bc40Alexei Starovoitov regs[BPF_REG_X] = 0; 301f5bffecda951b59d0d3cdd616d68952abc52bc40Alexei Starovoitov 302f5bffecda951b59d0d3cdd616d68952abc52bc40Alexei Starovoitovselect_insn: 303f5bffecda951b59d0d3cdd616d68952abc52bc40Alexei Starovoitov goto *jumptable[insn->code]; 304f5bffecda951b59d0d3cdd616d68952abc52bc40Alexei Starovoitov 305f5bffecda951b59d0d3cdd616d68952abc52bc40Alexei Starovoitov /* ALU */ 306f5bffecda951b59d0d3cdd616d68952abc52bc40Alexei Starovoitov#define ALU(OPCODE, OP) \ 307f5bffecda951b59d0d3cdd616d68952abc52bc40Alexei Starovoitov ALU64_##OPCODE##_X: \ 308f5bffecda951b59d0d3cdd616d68952abc52bc40Alexei Starovoitov DST = DST OP SRC; \ 309f5bffecda951b59d0d3cdd616d68952abc52bc40Alexei Starovoitov CONT; \ 310f5bffecda951b59d0d3cdd616d68952abc52bc40Alexei Starovoitov ALU_##OPCODE##_X: \ 311f5bffecda951b59d0d3cdd616d68952abc52bc40Alexei Starovoitov DST = (u32) DST OP (u32) SRC; \ 312f5bffecda951b59d0d3cdd616d68952abc52bc40Alexei Starovoitov CONT; \ 313f5bffecda951b59d0d3cdd616d68952abc52bc40Alexei Starovoitov ALU64_##OPCODE##_K: \ 314f5bffecda951b59d0d3cdd616d68952abc52bc40Alexei Starovoitov DST = DST OP IMM; \ 315f5bffecda951b59d0d3cdd616d68952abc52bc40Alexei Starovoitov CONT; \ 316f5bffecda951b59d0d3cdd616d68952abc52bc40Alexei Starovoitov ALU_##OPCODE##_K: \ 317f5bffecda951b59d0d3cdd616d68952abc52bc40Alexei Starovoitov DST = (u32) DST OP (u32) IMM; \ 318f5bffecda951b59d0d3cdd616d68952abc52bc40Alexei Starovoitov CONT; 319f5bffecda951b59d0d3cdd616d68952abc52bc40Alexei Starovoitov 320f5bffecda951b59d0d3cdd616d68952abc52bc40Alexei Starovoitov ALU(ADD, +) 321f5bffecda951b59d0d3cdd616d68952abc52bc40Alexei Starovoitov ALU(SUB, -) 322f5bffecda951b59d0d3cdd616d68952abc52bc40Alexei Starovoitov ALU(AND, &) 323f5bffecda951b59d0d3cdd616d68952abc52bc40Alexei Starovoitov ALU(OR, |) 324f5bffecda951b59d0d3cdd616d68952abc52bc40Alexei Starovoitov ALU(LSH, <<) 325f5bffecda951b59d0d3cdd616d68952abc52bc40Alexei Starovoitov ALU(RSH, >>) 326f5bffecda951b59d0d3cdd616d68952abc52bc40Alexei Starovoitov ALU(XOR, ^) 327f5bffecda951b59d0d3cdd616d68952abc52bc40Alexei Starovoitov ALU(MUL, *) 328f5bffecda951b59d0d3cdd616d68952abc52bc40Alexei Starovoitov#undef ALU 329f5bffecda951b59d0d3cdd616d68952abc52bc40Alexei Starovoitov ALU_NEG: 330f5bffecda951b59d0d3cdd616d68952abc52bc40Alexei Starovoitov DST = (u32) -DST; 331f5bffecda951b59d0d3cdd616d68952abc52bc40Alexei Starovoitov CONT; 332f5bffecda951b59d0d3cdd616d68952abc52bc40Alexei Starovoitov ALU64_NEG: 333f5bffecda951b59d0d3cdd616d68952abc52bc40Alexei Starovoitov DST = -DST; 334f5bffecda951b59d0d3cdd616d68952abc52bc40Alexei Starovoitov CONT; 335f5bffecda951b59d0d3cdd616d68952abc52bc40Alexei Starovoitov ALU_MOV_X: 336f5bffecda951b59d0d3cdd616d68952abc52bc40Alexei Starovoitov DST = (u32) SRC; 337f5bffecda951b59d0d3cdd616d68952abc52bc40Alexei Starovoitov CONT; 338f5bffecda951b59d0d3cdd616d68952abc52bc40Alexei Starovoitov ALU_MOV_K: 339f5bffecda951b59d0d3cdd616d68952abc52bc40Alexei Starovoitov DST = (u32) IMM; 340f5bffecda951b59d0d3cdd616d68952abc52bc40Alexei Starovoitov CONT; 341f5bffecda951b59d0d3cdd616d68952abc52bc40Alexei Starovoitov ALU64_MOV_X: 342f5bffecda951b59d0d3cdd616d68952abc52bc40Alexei Starovoitov DST = SRC; 343f5bffecda951b59d0d3cdd616d68952abc52bc40Alexei Starovoitov CONT; 344f5bffecda951b59d0d3cdd616d68952abc52bc40Alexei Starovoitov ALU64_MOV_K: 345f5bffecda951b59d0d3cdd616d68952abc52bc40Alexei Starovoitov DST = IMM; 346f5bffecda951b59d0d3cdd616d68952abc52bc40Alexei Starovoitov CONT; 34702ab695bb37ee9ad515df0d0790d5977505dd04aAlexei Starovoitov LD_IMM_DW: 34802ab695bb37ee9ad515df0d0790d5977505dd04aAlexei Starovoitov DST = (u64) (u32) insn[0].imm | ((u64) (u32) insn[1].imm) << 32; 34902ab695bb37ee9ad515df0d0790d5977505dd04aAlexei Starovoitov insn++; 35002ab695bb37ee9ad515df0d0790d5977505dd04aAlexei Starovoitov CONT; 351f5bffecda951b59d0d3cdd616d68952abc52bc40Alexei Starovoitov ALU64_ARSH_X: 352f5bffecda951b59d0d3cdd616d68952abc52bc40Alexei Starovoitov (*(s64 *) &DST) >>= SRC; 353f5bffecda951b59d0d3cdd616d68952abc52bc40Alexei Starovoitov CONT; 354f5bffecda951b59d0d3cdd616d68952abc52bc40Alexei Starovoitov ALU64_ARSH_K: 355f5bffecda951b59d0d3cdd616d68952abc52bc40Alexei Starovoitov (*(s64 *) &DST) >>= IMM; 356f5bffecda951b59d0d3cdd616d68952abc52bc40Alexei Starovoitov CONT; 357f5bffecda951b59d0d3cdd616d68952abc52bc40Alexei Starovoitov ALU64_MOD_X: 358f5bffecda951b59d0d3cdd616d68952abc52bc40Alexei Starovoitov if (unlikely(SRC == 0)) 359f5bffecda951b59d0d3cdd616d68952abc52bc40Alexei Starovoitov return 0; 360f5bffecda951b59d0d3cdd616d68952abc52bc40Alexei Starovoitov tmp = DST; 361f5bffecda951b59d0d3cdd616d68952abc52bc40Alexei Starovoitov DST = do_div(tmp, SRC); 362f5bffecda951b59d0d3cdd616d68952abc52bc40Alexei Starovoitov CONT; 363f5bffecda951b59d0d3cdd616d68952abc52bc40Alexei Starovoitov ALU_MOD_X: 364f5bffecda951b59d0d3cdd616d68952abc52bc40Alexei Starovoitov if (unlikely(SRC == 0)) 365f5bffecda951b59d0d3cdd616d68952abc52bc40Alexei Starovoitov return 0; 366f5bffecda951b59d0d3cdd616d68952abc52bc40Alexei Starovoitov tmp = (u32) DST; 367f5bffecda951b59d0d3cdd616d68952abc52bc40Alexei Starovoitov DST = do_div(tmp, (u32) SRC); 368f5bffecda951b59d0d3cdd616d68952abc52bc40Alexei Starovoitov CONT; 369f5bffecda951b59d0d3cdd616d68952abc52bc40Alexei Starovoitov ALU64_MOD_K: 370f5bffecda951b59d0d3cdd616d68952abc52bc40Alexei Starovoitov tmp = DST; 371f5bffecda951b59d0d3cdd616d68952abc52bc40Alexei Starovoitov DST = do_div(tmp, IMM); 372f5bffecda951b59d0d3cdd616d68952abc52bc40Alexei Starovoitov CONT; 373f5bffecda951b59d0d3cdd616d68952abc52bc40Alexei Starovoitov ALU_MOD_K: 374f5bffecda951b59d0d3cdd616d68952abc52bc40Alexei Starovoitov tmp = (u32) DST; 375f5bffecda951b59d0d3cdd616d68952abc52bc40Alexei Starovoitov DST = do_div(tmp, (u32) IMM); 376f5bffecda951b59d0d3cdd616d68952abc52bc40Alexei Starovoitov CONT; 377f5bffecda951b59d0d3cdd616d68952abc52bc40Alexei Starovoitov ALU64_DIV_X: 378f5bffecda951b59d0d3cdd616d68952abc52bc40Alexei Starovoitov if (unlikely(SRC == 0)) 379f5bffecda951b59d0d3cdd616d68952abc52bc40Alexei Starovoitov return 0; 380f5bffecda951b59d0d3cdd616d68952abc52bc40Alexei Starovoitov do_div(DST, SRC); 381f5bffecda951b59d0d3cdd616d68952abc52bc40Alexei Starovoitov CONT; 382f5bffecda951b59d0d3cdd616d68952abc52bc40Alexei Starovoitov ALU_DIV_X: 383f5bffecda951b59d0d3cdd616d68952abc52bc40Alexei Starovoitov if (unlikely(SRC == 0)) 384f5bffecda951b59d0d3cdd616d68952abc52bc40Alexei Starovoitov return 0; 385f5bffecda951b59d0d3cdd616d68952abc52bc40Alexei Starovoitov tmp = (u32) DST; 386f5bffecda951b59d0d3cdd616d68952abc52bc40Alexei Starovoitov do_div(tmp, (u32) SRC); 387f5bffecda951b59d0d3cdd616d68952abc52bc40Alexei Starovoitov DST = (u32) tmp; 388f5bffecda951b59d0d3cdd616d68952abc52bc40Alexei Starovoitov CONT; 389f5bffecda951b59d0d3cdd616d68952abc52bc40Alexei Starovoitov ALU64_DIV_K: 390f5bffecda951b59d0d3cdd616d68952abc52bc40Alexei Starovoitov do_div(DST, IMM); 391f5bffecda951b59d0d3cdd616d68952abc52bc40Alexei Starovoitov CONT; 392f5bffecda951b59d0d3cdd616d68952abc52bc40Alexei Starovoitov ALU_DIV_K: 393f5bffecda951b59d0d3cdd616d68952abc52bc40Alexei Starovoitov tmp = (u32) DST; 394f5bffecda951b59d0d3cdd616d68952abc52bc40Alexei Starovoitov do_div(tmp, (u32) IMM); 395f5bffecda951b59d0d3cdd616d68952abc52bc40Alexei Starovoitov DST = (u32) tmp; 396f5bffecda951b59d0d3cdd616d68952abc52bc40Alexei Starovoitov CONT; 397f5bffecda951b59d0d3cdd616d68952abc52bc40Alexei Starovoitov ALU_END_TO_BE: 398f5bffecda951b59d0d3cdd616d68952abc52bc40Alexei Starovoitov switch (IMM) { 399f5bffecda951b59d0d3cdd616d68952abc52bc40Alexei Starovoitov case 16: 400f5bffecda951b59d0d3cdd616d68952abc52bc40Alexei Starovoitov DST = (__force u16) cpu_to_be16(DST); 401f5bffecda951b59d0d3cdd616d68952abc52bc40Alexei Starovoitov break; 402f5bffecda951b59d0d3cdd616d68952abc52bc40Alexei Starovoitov case 32: 403f5bffecda951b59d0d3cdd616d68952abc52bc40Alexei Starovoitov DST = (__force u32) cpu_to_be32(DST); 404f5bffecda951b59d0d3cdd616d68952abc52bc40Alexei Starovoitov break; 405f5bffecda951b59d0d3cdd616d68952abc52bc40Alexei Starovoitov case 64: 406f5bffecda951b59d0d3cdd616d68952abc52bc40Alexei Starovoitov DST = (__force u64) cpu_to_be64(DST); 407f5bffecda951b59d0d3cdd616d68952abc52bc40Alexei Starovoitov break; 408f5bffecda951b59d0d3cdd616d68952abc52bc40Alexei Starovoitov } 409f5bffecda951b59d0d3cdd616d68952abc52bc40Alexei Starovoitov CONT; 410f5bffecda951b59d0d3cdd616d68952abc52bc40Alexei Starovoitov ALU_END_TO_LE: 411f5bffecda951b59d0d3cdd616d68952abc52bc40Alexei Starovoitov switch (IMM) { 412f5bffecda951b59d0d3cdd616d68952abc52bc40Alexei Starovoitov case 16: 413f5bffecda951b59d0d3cdd616d68952abc52bc40Alexei Starovoitov DST = (__force u16) cpu_to_le16(DST); 414f5bffecda951b59d0d3cdd616d68952abc52bc40Alexei Starovoitov break; 415f5bffecda951b59d0d3cdd616d68952abc52bc40Alexei Starovoitov case 32: 416f5bffecda951b59d0d3cdd616d68952abc52bc40Alexei Starovoitov DST = (__force u32) cpu_to_le32(DST); 417f5bffecda951b59d0d3cdd616d68952abc52bc40Alexei Starovoitov break; 418f5bffecda951b59d0d3cdd616d68952abc52bc40Alexei Starovoitov case 64: 419f5bffecda951b59d0d3cdd616d68952abc52bc40Alexei Starovoitov DST = (__force u64) cpu_to_le64(DST); 420f5bffecda951b59d0d3cdd616d68952abc52bc40Alexei Starovoitov break; 421f5bffecda951b59d0d3cdd616d68952abc52bc40Alexei Starovoitov } 422f5bffecda951b59d0d3cdd616d68952abc52bc40Alexei Starovoitov CONT; 423f5bffecda951b59d0d3cdd616d68952abc52bc40Alexei Starovoitov 424f5bffecda951b59d0d3cdd616d68952abc52bc40Alexei Starovoitov /* CALL */ 425f5bffecda951b59d0d3cdd616d68952abc52bc40Alexei Starovoitov JMP_CALL: 426f5bffecda951b59d0d3cdd616d68952abc52bc40Alexei Starovoitov /* Function call scratches BPF_R1-BPF_R5 registers, 427f5bffecda951b59d0d3cdd616d68952abc52bc40Alexei Starovoitov * preserves BPF_R6-BPF_R9, and stores return value 428f5bffecda951b59d0d3cdd616d68952abc52bc40Alexei Starovoitov * into BPF_R0. 429f5bffecda951b59d0d3cdd616d68952abc52bc40Alexei Starovoitov */ 430f5bffecda951b59d0d3cdd616d68952abc52bc40Alexei Starovoitov BPF_R0 = (__bpf_call_base + insn->imm)(BPF_R1, BPF_R2, BPF_R3, 431f5bffecda951b59d0d3cdd616d68952abc52bc40Alexei Starovoitov BPF_R4, BPF_R5); 432f5bffecda951b59d0d3cdd616d68952abc52bc40Alexei Starovoitov CONT; 433f5bffecda951b59d0d3cdd616d68952abc52bc40Alexei Starovoitov 434f5bffecda951b59d0d3cdd616d68952abc52bc40Alexei Starovoitov /* JMP */ 435f5bffecda951b59d0d3cdd616d68952abc52bc40Alexei Starovoitov JMP_JA: 436f5bffecda951b59d0d3cdd616d68952abc52bc40Alexei Starovoitov insn += insn->off; 437f5bffecda951b59d0d3cdd616d68952abc52bc40Alexei Starovoitov CONT; 438f5bffecda951b59d0d3cdd616d68952abc52bc40Alexei Starovoitov JMP_JEQ_X: 439f5bffecda951b59d0d3cdd616d68952abc52bc40Alexei Starovoitov if (DST == SRC) { 440f5bffecda951b59d0d3cdd616d68952abc52bc40Alexei Starovoitov insn += insn->off; 441f5bffecda951b59d0d3cdd616d68952abc52bc40Alexei Starovoitov CONT_JMP; 442f5bffecda951b59d0d3cdd616d68952abc52bc40Alexei Starovoitov } 443f5bffecda951b59d0d3cdd616d68952abc52bc40Alexei Starovoitov CONT; 444f5bffecda951b59d0d3cdd616d68952abc52bc40Alexei Starovoitov JMP_JEQ_K: 445f5bffecda951b59d0d3cdd616d68952abc52bc40Alexei Starovoitov if (DST == IMM) { 446f5bffecda951b59d0d3cdd616d68952abc52bc40Alexei Starovoitov insn += insn->off; 447f5bffecda951b59d0d3cdd616d68952abc52bc40Alexei Starovoitov CONT_JMP; 448f5bffecda951b59d0d3cdd616d68952abc52bc40Alexei Starovoitov } 449f5bffecda951b59d0d3cdd616d68952abc52bc40Alexei Starovoitov CONT; 450f5bffecda951b59d0d3cdd616d68952abc52bc40Alexei Starovoitov JMP_JNE_X: 451f5bffecda951b59d0d3cdd616d68952abc52bc40Alexei Starovoitov if (DST != SRC) { 452f5bffecda951b59d0d3cdd616d68952abc52bc40Alexei Starovoitov insn += insn->off; 453f5bffecda951b59d0d3cdd616d68952abc52bc40Alexei Starovoitov CONT_JMP; 454f5bffecda951b59d0d3cdd616d68952abc52bc40Alexei Starovoitov } 455f5bffecda951b59d0d3cdd616d68952abc52bc40Alexei Starovoitov CONT; 456f5bffecda951b59d0d3cdd616d68952abc52bc40Alexei Starovoitov JMP_JNE_K: 457f5bffecda951b59d0d3cdd616d68952abc52bc40Alexei Starovoitov if (DST != IMM) { 458f5bffecda951b59d0d3cdd616d68952abc52bc40Alexei Starovoitov insn += insn->off; 459f5bffecda951b59d0d3cdd616d68952abc52bc40Alexei Starovoitov CONT_JMP; 460f5bffecda951b59d0d3cdd616d68952abc52bc40Alexei Starovoitov } 461f5bffecda951b59d0d3cdd616d68952abc52bc40Alexei Starovoitov CONT; 462f5bffecda951b59d0d3cdd616d68952abc52bc40Alexei Starovoitov JMP_JGT_X: 463f5bffecda951b59d0d3cdd616d68952abc52bc40Alexei Starovoitov if (DST > SRC) { 464f5bffecda951b59d0d3cdd616d68952abc52bc40Alexei Starovoitov insn += insn->off; 465f5bffecda951b59d0d3cdd616d68952abc52bc40Alexei Starovoitov CONT_JMP; 466f5bffecda951b59d0d3cdd616d68952abc52bc40Alexei Starovoitov } 467f5bffecda951b59d0d3cdd616d68952abc52bc40Alexei Starovoitov CONT; 468f5bffecda951b59d0d3cdd616d68952abc52bc40Alexei Starovoitov JMP_JGT_K: 469f5bffecda951b59d0d3cdd616d68952abc52bc40Alexei Starovoitov if (DST > IMM) { 470f5bffecda951b59d0d3cdd616d68952abc52bc40Alexei Starovoitov insn += insn->off; 471f5bffecda951b59d0d3cdd616d68952abc52bc40Alexei Starovoitov CONT_JMP; 472f5bffecda951b59d0d3cdd616d68952abc52bc40Alexei Starovoitov } 473f5bffecda951b59d0d3cdd616d68952abc52bc40Alexei Starovoitov CONT; 474f5bffecda951b59d0d3cdd616d68952abc52bc40Alexei Starovoitov JMP_JGE_X: 475f5bffecda951b59d0d3cdd616d68952abc52bc40Alexei Starovoitov if (DST >= SRC) { 476f5bffecda951b59d0d3cdd616d68952abc52bc40Alexei Starovoitov insn += insn->off; 477f5bffecda951b59d0d3cdd616d68952abc52bc40Alexei Starovoitov CONT_JMP; 478f5bffecda951b59d0d3cdd616d68952abc52bc40Alexei Starovoitov } 479f5bffecda951b59d0d3cdd616d68952abc52bc40Alexei Starovoitov CONT; 480f5bffecda951b59d0d3cdd616d68952abc52bc40Alexei Starovoitov JMP_JGE_K: 481f5bffecda951b59d0d3cdd616d68952abc52bc40Alexei Starovoitov if (DST >= IMM) { 482f5bffecda951b59d0d3cdd616d68952abc52bc40Alexei Starovoitov insn += insn->off; 483f5bffecda951b59d0d3cdd616d68952abc52bc40Alexei Starovoitov CONT_JMP; 484f5bffecda951b59d0d3cdd616d68952abc52bc40Alexei Starovoitov } 485f5bffecda951b59d0d3cdd616d68952abc52bc40Alexei Starovoitov CONT; 486f5bffecda951b59d0d3cdd616d68952abc52bc40Alexei Starovoitov JMP_JSGT_X: 487f5bffecda951b59d0d3cdd616d68952abc52bc40Alexei Starovoitov if (((s64) DST) > ((s64) SRC)) { 488f5bffecda951b59d0d3cdd616d68952abc52bc40Alexei Starovoitov insn += insn->off; 489f5bffecda951b59d0d3cdd616d68952abc52bc40Alexei Starovoitov CONT_JMP; 490f5bffecda951b59d0d3cdd616d68952abc52bc40Alexei Starovoitov } 491f5bffecda951b59d0d3cdd616d68952abc52bc40Alexei Starovoitov CONT; 492f5bffecda951b59d0d3cdd616d68952abc52bc40Alexei Starovoitov JMP_JSGT_K: 493f5bffecda951b59d0d3cdd616d68952abc52bc40Alexei Starovoitov if (((s64) DST) > ((s64) IMM)) { 494f5bffecda951b59d0d3cdd616d68952abc52bc40Alexei Starovoitov insn += insn->off; 495f5bffecda951b59d0d3cdd616d68952abc52bc40Alexei Starovoitov CONT_JMP; 496f5bffecda951b59d0d3cdd616d68952abc52bc40Alexei Starovoitov } 497f5bffecda951b59d0d3cdd616d68952abc52bc40Alexei Starovoitov CONT; 498f5bffecda951b59d0d3cdd616d68952abc52bc40Alexei Starovoitov JMP_JSGE_X: 499f5bffecda951b59d0d3cdd616d68952abc52bc40Alexei Starovoitov if (((s64) DST) >= ((s64) SRC)) { 500f5bffecda951b59d0d3cdd616d68952abc52bc40Alexei Starovoitov insn += insn->off; 501f5bffecda951b59d0d3cdd616d68952abc52bc40Alexei Starovoitov CONT_JMP; 502f5bffecda951b59d0d3cdd616d68952abc52bc40Alexei Starovoitov } 503f5bffecda951b59d0d3cdd616d68952abc52bc40Alexei Starovoitov CONT; 504f5bffecda951b59d0d3cdd616d68952abc52bc40Alexei Starovoitov JMP_JSGE_K: 505f5bffecda951b59d0d3cdd616d68952abc52bc40Alexei Starovoitov if (((s64) DST) >= ((s64) IMM)) { 506f5bffecda951b59d0d3cdd616d68952abc52bc40Alexei Starovoitov insn += insn->off; 507f5bffecda951b59d0d3cdd616d68952abc52bc40Alexei Starovoitov CONT_JMP; 508f5bffecda951b59d0d3cdd616d68952abc52bc40Alexei Starovoitov } 509f5bffecda951b59d0d3cdd616d68952abc52bc40Alexei Starovoitov CONT; 510f5bffecda951b59d0d3cdd616d68952abc52bc40Alexei Starovoitov JMP_JSET_X: 511f5bffecda951b59d0d3cdd616d68952abc52bc40Alexei Starovoitov if (DST & SRC) { 512f5bffecda951b59d0d3cdd616d68952abc52bc40Alexei Starovoitov insn += insn->off; 513f5bffecda951b59d0d3cdd616d68952abc52bc40Alexei Starovoitov CONT_JMP; 514f5bffecda951b59d0d3cdd616d68952abc52bc40Alexei Starovoitov } 515f5bffecda951b59d0d3cdd616d68952abc52bc40Alexei Starovoitov CONT; 516f5bffecda951b59d0d3cdd616d68952abc52bc40Alexei Starovoitov JMP_JSET_K: 517f5bffecda951b59d0d3cdd616d68952abc52bc40Alexei Starovoitov if (DST & IMM) { 518f5bffecda951b59d0d3cdd616d68952abc52bc40Alexei Starovoitov insn += insn->off; 519f5bffecda951b59d0d3cdd616d68952abc52bc40Alexei Starovoitov CONT_JMP; 520f5bffecda951b59d0d3cdd616d68952abc52bc40Alexei Starovoitov } 521f5bffecda951b59d0d3cdd616d68952abc52bc40Alexei Starovoitov CONT; 522f5bffecda951b59d0d3cdd616d68952abc52bc40Alexei Starovoitov JMP_EXIT: 523f5bffecda951b59d0d3cdd616d68952abc52bc40Alexei Starovoitov return BPF_R0; 524f5bffecda951b59d0d3cdd616d68952abc52bc40Alexei Starovoitov 525f5bffecda951b59d0d3cdd616d68952abc52bc40Alexei Starovoitov /* STX and ST and LDX*/ 526f5bffecda951b59d0d3cdd616d68952abc52bc40Alexei Starovoitov#define LDST(SIZEOP, SIZE) \ 527f5bffecda951b59d0d3cdd616d68952abc52bc40Alexei Starovoitov STX_MEM_##SIZEOP: \ 528f5bffecda951b59d0d3cdd616d68952abc52bc40Alexei Starovoitov *(SIZE *)(unsigned long) (DST + insn->off) = SRC; \ 529f5bffecda951b59d0d3cdd616d68952abc52bc40Alexei Starovoitov CONT; \ 530f5bffecda951b59d0d3cdd616d68952abc52bc40Alexei Starovoitov ST_MEM_##SIZEOP: \ 531f5bffecda951b59d0d3cdd616d68952abc52bc40Alexei Starovoitov *(SIZE *)(unsigned long) (DST + insn->off) = IMM; \ 532f5bffecda951b59d0d3cdd616d68952abc52bc40Alexei Starovoitov CONT; \ 533f5bffecda951b59d0d3cdd616d68952abc52bc40Alexei Starovoitov LDX_MEM_##SIZEOP: \ 534f5bffecda951b59d0d3cdd616d68952abc52bc40Alexei Starovoitov DST = *(SIZE *)(unsigned long) (SRC + insn->off); \ 535f5bffecda951b59d0d3cdd616d68952abc52bc40Alexei Starovoitov CONT; 536f5bffecda951b59d0d3cdd616d68952abc52bc40Alexei Starovoitov 537f5bffecda951b59d0d3cdd616d68952abc52bc40Alexei Starovoitov LDST(B, u8) 538f5bffecda951b59d0d3cdd616d68952abc52bc40Alexei Starovoitov LDST(H, u16) 539f5bffecda951b59d0d3cdd616d68952abc52bc40Alexei Starovoitov LDST(W, u32) 540f5bffecda951b59d0d3cdd616d68952abc52bc40Alexei Starovoitov LDST(DW, u64) 541f5bffecda951b59d0d3cdd616d68952abc52bc40Alexei Starovoitov#undef LDST 542f5bffecda951b59d0d3cdd616d68952abc52bc40Alexei Starovoitov STX_XADD_W: /* lock xadd *(u32 *)(dst_reg + off16) += src_reg */ 543f5bffecda951b59d0d3cdd616d68952abc52bc40Alexei Starovoitov atomic_add((u32) SRC, (atomic_t *)(unsigned long) 544f5bffecda951b59d0d3cdd616d68952abc52bc40Alexei Starovoitov (DST + insn->off)); 545f5bffecda951b59d0d3cdd616d68952abc52bc40Alexei Starovoitov CONT; 546f5bffecda951b59d0d3cdd616d68952abc52bc40Alexei Starovoitov STX_XADD_DW: /* lock xadd *(u64 *)(dst_reg + off16) += src_reg */ 547f5bffecda951b59d0d3cdd616d68952abc52bc40Alexei Starovoitov atomic64_add((u64) SRC, (atomic64_t *)(unsigned long) 548f5bffecda951b59d0d3cdd616d68952abc52bc40Alexei Starovoitov (DST + insn->off)); 549f5bffecda951b59d0d3cdd616d68952abc52bc40Alexei Starovoitov CONT; 550f5bffecda951b59d0d3cdd616d68952abc52bc40Alexei Starovoitov LD_ABS_W: /* BPF_R0 = ntohl(*(u32 *) (skb->data + imm32)) */ 551f5bffecda951b59d0d3cdd616d68952abc52bc40Alexei Starovoitov off = IMM; 552f5bffecda951b59d0d3cdd616d68952abc52bc40Alexei Starovoitovload_word: 553f5bffecda951b59d0d3cdd616d68952abc52bc40Alexei Starovoitov /* BPF_LD + BPD_ABS and BPF_LD + BPF_IND insns are 554f5bffecda951b59d0d3cdd616d68952abc52bc40Alexei Starovoitov * only appearing in the programs where ctx == 555f5bffecda951b59d0d3cdd616d68952abc52bc40Alexei Starovoitov * skb. All programs keep 'ctx' in regs[BPF_REG_CTX] 5568fb575ca396bc31d9fa99c26336e2432b41d1bfcAlexei Starovoitov * == BPF_R6, bpf_convert_filter() saves it in BPF_R6, 557f5bffecda951b59d0d3cdd616d68952abc52bc40Alexei Starovoitov * internal BPF verifier will check that BPF_R6 == 558f5bffecda951b59d0d3cdd616d68952abc52bc40Alexei Starovoitov * ctx. 559f5bffecda951b59d0d3cdd616d68952abc52bc40Alexei Starovoitov * 560f5bffecda951b59d0d3cdd616d68952abc52bc40Alexei Starovoitov * BPF_ABS and BPF_IND are wrappers of function calls, 561f5bffecda951b59d0d3cdd616d68952abc52bc40Alexei Starovoitov * so they scratch BPF_R1-BPF_R5 registers, preserve 562f5bffecda951b59d0d3cdd616d68952abc52bc40Alexei Starovoitov * BPF_R6-BPF_R9, and store return value into BPF_R0. 563f5bffecda951b59d0d3cdd616d68952abc52bc40Alexei Starovoitov * 564f5bffecda951b59d0d3cdd616d68952abc52bc40Alexei Starovoitov * Implicit input: 565f5bffecda951b59d0d3cdd616d68952abc52bc40Alexei Starovoitov * ctx == skb == BPF_R6 == CTX 566f5bffecda951b59d0d3cdd616d68952abc52bc40Alexei Starovoitov * 567f5bffecda951b59d0d3cdd616d68952abc52bc40Alexei Starovoitov * Explicit input: 568f5bffecda951b59d0d3cdd616d68952abc52bc40Alexei Starovoitov * SRC == any register 569f5bffecda951b59d0d3cdd616d68952abc52bc40Alexei Starovoitov * IMM == 32-bit immediate 570f5bffecda951b59d0d3cdd616d68952abc52bc40Alexei Starovoitov * 571f5bffecda951b59d0d3cdd616d68952abc52bc40Alexei Starovoitov * Output: 572f5bffecda951b59d0d3cdd616d68952abc52bc40Alexei Starovoitov * BPF_R0 - 8/16/32-bit skb data converted to cpu endianness 573f5bffecda951b59d0d3cdd616d68952abc52bc40Alexei Starovoitov */ 574f5bffecda951b59d0d3cdd616d68952abc52bc40Alexei Starovoitov 575f5bffecda951b59d0d3cdd616d68952abc52bc40Alexei Starovoitov ptr = bpf_load_pointer((struct sk_buff *) (unsigned long) CTX, off, 4, &tmp); 576f5bffecda951b59d0d3cdd616d68952abc52bc40Alexei Starovoitov if (likely(ptr != NULL)) { 577f5bffecda951b59d0d3cdd616d68952abc52bc40Alexei Starovoitov BPF_R0 = get_unaligned_be32(ptr); 578f5bffecda951b59d0d3cdd616d68952abc52bc40Alexei Starovoitov CONT; 579f5bffecda951b59d0d3cdd616d68952abc52bc40Alexei Starovoitov } 580f5bffecda951b59d0d3cdd616d68952abc52bc40Alexei Starovoitov 581f5bffecda951b59d0d3cdd616d68952abc52bc40Alexei Starovoitov return 0; 582f5bffecda951b59d0d3cdd616d68952abc52bc40Alexei Starovoitov LD_ABS_H: /* BPF_R0 = ntohs(*(u16 *) (skb->data + imm32)) */ 583f5bffecda951b59d0d3cdd616d68952abc52bc40Alexei Starovoitov off = IMM; 584f5bffecda951b59d0d3cdd616d68952abc52bc40Alexei Starovoitovload_half: 585f5bffecda951b59d0d3cdd616d68952abc52bc40Alexei Starovoitov ptr = bpf_load_pointer((struct sk_buff *) (unsigned long) CTX, off, 2, &tmp); 586f5bffecda951b59d0d3cdd616d68952abc52bc40Alexei Starovoitov if (likely(ptr != NULL)) { 587f5bffecda951b59d0d3cdd616d68952abc52bc40Alexei Starovoitov BPF_R0 = get_unaligned_be16(ptr); 588f5bffecda951b59d0d3cdd616d68952abc52bc40Alexei Starovoitov CONT; 589f5bffecda951b59d0d3cdd616d68952abc52bc40Alexei Starovoitov } 590f5bffecda951b59d0d3cdd616d68952abc52bc40Alexei Starovoitov 591f5bffecda951b59d0d3cdd616d68952abc52bc40Alexei Starovoitov return 0; 592f5bffecda951b59d0d3cdd616d68952abc52bc40Alexei Starovoitov LD_ABS_B: /* BPF_R0 = *(u8 *) (skb->data + imm32) */ 593f5bffecda951b59d0d3cdd616d68952abc52bc40Alexei Starovoitov off = IMM; 594f5bffecda951b59d0d3cdd616d68952abc52bc40Alexei Starovoitovload_byte: 595f5bffecda951b59d0d3cdd616d68952abc52bc40Alexei Starovoitov ptr = bpf_load_pointer((struct sk_buff *) (unsigned long) CTX, off, 1, &tmp); 596f5bffecda951b59d0d3cdd616d68952abc52bc40Alexei Starovoitov if (likely(ptr != NULL)) { 597f5bffecda951b59d0d3cdd616d68952abc52bc40Alexei Starovoitov BPF_R0 = *(u8 *)ptr; 598f5bffecda951b59d0d3cdd616d68952abc52bc40Alexei Starovoitov CONT; 599f5bffecda951b59d0d3cdd616d68952abc52bc40Alexei Starovoitov } 600f5bffecda951b59d0d3cdd616d68952abc52bc40Alexei Starovoitov 601f5bffecda951b59d0d3cdd616d68952abc52bc40Alexei Starovoitov return 0; 602f5bffecda951b59d0d3cdd616d68952abc52bc40Alexei Starovoitov LD_IND_W: /* BPF_R0 = ntohl(*(u32 *) (skb->data + src_reg + imm32)) */ 603f5bffecda951b59d0d3cdd616d68952abc52bc40Alexei Starovoitov off = IMM + SRC; 604f5bffecda951b59d0d3cdd616d68952abc52bc40Alexei Starovoitov goto load_word; 605f5bffecda951b59d0d3cdd616d68952abc52bc40Alexei Starovoitov LD_IND_H: /* BPF_R0 = ntohs(*(u16 *) (skb->data + src_reg + imm32)) */ 606f5bffecda951b59d0d3cdd616d68952abc52bc40Alexei Starovoitov off = IMM + SRC; 607f5bffecda951b59d0d3cdd616d68952abc52bc40Alexei Starovoitov goto load_half; 608f5bffecda951b59d0d3cdd616d68952abc52bc40Alexei Starovoitov LD_IND_B: /* BPF_R0 = *(u8 *) (skb->data + src_reg + imm32) */ 609f5bffecda951b59d0d3cdd616d68952abc52bc40Alexei Starovoitov off = IMM + SRC; 610f5bffecda951b59d0d3cdd616d68952abc52bc40Alexei Starovoitov goto load_byte; 611f5bffecda951b59d0d3cdd616d68952abc52bc40Alexei Starovoitov 612f5bffecda951b59d0d3cdd616d68952abc52bc40Alexei Starovoitov default_label: 613f5bffecda951b59d0d3cdd616d68952abc52bc40Alexei Starovoitov /* If we ever reach this, we have a bug somewhere. */ 614f5bffecda951b59d0d3cdd616d68952abc52bc40Alexei Starovoitov WARN_RATELIMIT(1, "unknown opcode %02x\n", insn->code); 615f5bffecda951b59d0d3cdd616d68952abc52bc40Alexei Starovoitov return 0; 616f5bffecda951b59d0d3cdd616d68952abc52bc40Alexei Starovoitov} 617f5bffecda951b59d0d3cdd616d68952abc52bc40Alexei Starovoitov 6187ae457c1e5b45a1b826fad9d62b32191d2bdcfdbAlexei Starovoitovvoid __weak bpf_int_jit_compile(struct bpf_prog *prog) 619f5bffecda951b59d0d3cdd616d68952abc52bc40Alexei Starovoitov{ 620f5bffecda951b59d0d3cdd616d68952abc52bc40Alexei Starovoitov} 621f5bffecda951b59d0d3cdd616d68952abc52bc40Alexei Starovoitov 622f5bffecda951b59d0d3cdd616d68952abc52bc40Alexei Starovoitov/** 6237ae457c1e5b45a1b826fad9d62b32191d2bdcfdbAlexei Starovoitov * bpf_prog_select_runtime - select execution runtime for BPF program 6247ae457c1e5b45a1b826fad9d62b32191d2bdcfdbAlexei Starovoitov * @fp: bpf_prog populated with internal BPF program 625f5bffecda951b59d0d3cdd616d68952abc52bc40Alexei Starovoitov * 626f5bffecda951b59d0d3cdd616d68952abc52bc40Alexei Starovoitov * try to JIT internal BPF program, if JIT is not available select interpreter 6277ae457c1e5b45a1b826fad9d62b32191d2bdcfdbAlexei Starovoitov * BPF program will be executed via BPF_PROG_RUN() macro 628f5bffecda951b59d0d3cdd616d68952abc52bc40Alexei Starovoitov */ 6297ae457c1e5b45a1b826fad9d62b32191d2bdcfdbAlexei Starovoitovvoid bpf_prog_select_runtime(struct bpf_prog *fp) 630f5bffecda951b59d0d3cdd616d68952abc52bc40Alexei Starovoitov{ 6317ae457c1e5b45a1b826fad9d62b32191d2bdcfdbAlexei Starovoitov fp->bpf_func = (void *) __bpf_prog_run; 632f5bffecda951b59d0d3cdd616d68952abc52bc40Alexei Starovoitov 633f5bffecda951b59d0d3cdd616d68952abc52bc40Alexei Starovoitov /* Probe if internal BPF can be JITed */ 634f5bffecda951b59d0d3cdd616d68952abc52bc40Alexei Starovoitov bpf_int_jit_compile(fp); 63560a3b2253c413cf601783b070507d7dd6620c954Daniel Borkmann /* Lock whole bpf_prog as read-only */ 63660a3b2253c413cf601783b070507d7dd6620c954Daniel Borkmann bpf_prog_lock_ro(fp); 637f5bffecda951b59d0d3cdd616d68952abc52bc40Alexei Starovoitov} 6387ae457c1e5b45a1b826fad9d62b32191d2bdcfdbAlexei StarovoitovEXPORT_SYMBOL_GPL(bpf_prog_select_runtime); 639f5bffecda951b59d0d3cdd616d68952abc52bc40Alexei Starovoitov 64060a3b2253c413cf601783b070507d7dd6620c954Daniel Borkmannstatic void bpf_prog_free_deferred(struct work_struct *work) 64160a3b2253c413cf601783b070507d7dd6620c954Daniel Borkmann{ 64209756af46893c18839062976c3252e93a1beeba7Alexei Starovoitov struct bpf_prog_aux *aux; 64360a3b2253c413cf601783b070507d7dd6620c954Daniel Borkmann 64409756af46893c18839062976c3252e93a1beeba7Alexei Starovoitov aux = container_of(work, struct bpf_prog_aux, work); 64509756af46893c18839062976c3252e93a1beeba7Alexei Starovoitov bpf_jit_free(aux->prog); 64660a3b2253c413cf601783b070507d7dd6620c954Daniel Borkmann} 64760a3b2253c413cf601783b070507d7dd6620c954Daniel Borkmann 64860a3b2253c413cf601783b070507d7dd6620c954Daniel Borkmann/* Free internal BPF program */ 6497ae457c1e5b45a1b826fad9d62b32191d2bdcfdbAlexei Starovoitovvoid bpf_prog_free(struct bpf_prog *fp) 650f5bffecda951b59d0d3cdd616d68952abc52bc40Alexei Starovoitov{ 65109756af46893c18839062976c3252e93a1beeba7Alexei Starovoitov struct bpf_prog_aux *aux = fp->aux; 65260a3b2253c413cf601783b070507d7dd6620c954Daniel Borkmann 65309756af46893c18839062976c3252e93a1beeba7Alexei Starovoitov INIT_WORK(&aux->work, bpf_prog_free_deferred); 65409756af46893c18839062976c3252e93a1beeba7Alexei Starovoitov aux->prog = fp; 65509756af46893c18839062976c3252e93a1beeba7Alexei Starovoitov schedule_work(&aux->work); 656f5bffecda951b59d0d3cdd616d68952abc52bc40Alexei Starovoitov} 6577ae457c1e5b45a1b826fad9d62b32191d2bdcfdbAlexei StarovoitovEXPORT_SYMBOL_GPL(bpf_prog_free); 658f89b7755f517cdbb755d7543eef986ee9d54e654Alexei Starovoitov 659f89b7755f517cdbb755d7543eef986ee9d54e654Alexei Starovoitov/* To execute LD_ABS/LD_IND instructions __bpf_prog_run() may call 660f89b7755f517cdbb755d7543eef986ee9d54e654Alexei Starovoitov * skb_copy_bits(), so provide a weak definition of it for NET-less config. 661f89b7755f517cdbb755d7543eef986ee9d54e654Alexei Starovoitov */ 662f89b7755f517cdbb755d7543eef986ee9d54e654Alexei Starovoitovint __weak skb_copy_bits(const struct sk_buff *skb, int offset, void *to, 663f89b7755f517cdbb755d7543eef986ee9d54e654Alexei Starovoitov int len) 664f89b7755f517cdbb755d7543eef986ee9d54e654Alexei Starovoitov{ 665f89b7755f517cdbb755d7543eef986ee9d54e654Alexei Starovoitov return -EFAULT; 666f89b7755f517cdbb755d7543eef986ee9d54e654Alexei Starovoitov} 667