1c6610de353da5ca6eee5b8960e838a87a90ead0cMarkos Chandras/* 2c6610de353da5ca6eee5b8960e838a87a90ead0cMarkos Chandras * Just-In-Time compiler for BPF filters on MIPS 3c6610de353da5ca6eee5b8960e838a87a90ead0cMarkos Chandras * 4c6610de353da5ca6eee5b8960e838a87a90ead0cMarkos Chandras * Copyright (c) 2014 Imagination Technologies Ltd. 5c6610de353da5ca6eee5b8960e838a87a90ead0cMarkos Chandras * Author: Markos Chandras <markos.chandras@imgtec.com> 6c6610de353da5ca6eee5b8960e838a87a90ead0cMarkos Chandras * 7c6610de353da5ca6eee5b8960e838a87a90ead0cMarkos Chandras * This program is free software; you can redistribute it and/or modify it 8c6610de353da5ca6eee5b8960e838a87a90ead0cMarkos Chandras * under the terms of the GNU General Public License as published by the 9c6610de353da5ca6eee5b8960e838a87a90ead0cMarkos Chandras * Free Software Foundation; version 2 of the License. 10c6610de353da5ca6eee5b8960e838a87a90ead0cMarkos Chandras */ 11c6610de353da5ca6eee5b8960e838a87a90ead0cMarkos Chandras 12c6610de353da5ca6eee5b8960e838a87a90ead0cMarkos Chandras#include <linux/bitops.h> 13c6610de353da5ca6eee5b8960e838a87a90ead0cMarkos Chandras#include <linux/compiler.h> 14c6610de353da5ca6eee5b8960e838a87a90ead0cMarkos Chandras#include <linux/errno.h> 15c6610de353da5ca6eee5b8960e838a87a90ead0cMarkos Chandras#include <linux/filter.h> 16c6610de353da5ca6eee5b8960e838a87a90ead0cMarkos Chandras#include <linux/if_vlan.h> 17c6610de353da5ca6eee5b8960e838a87a90ead0cMarkos Chandras#include <linux/kconfig.h> 18c6610de353da5ca6eee5b8960e838a87a90ead0cMarkos Chandras#include <linux/moduleloader.h> 19c6610de353da5ca6eee5b8960e838a87a90ead0cMarkos Chandras#include <linux/netdevice.h> 20c6610de353da5ca6eee5b8960e838a87a90ead0cMarkos Chandras#include <linux/string.h> 21c6610de353da5ca6eee5b8960e838a87a90ead0cMarkos Chandras#include <linux/slab.h> 22c6610de353da5ca6eee5b8960e838a87a90ead0cMarkos Chandras#include <linux/types.h> 23c6610de353da5ca6eee5b8960e838a87a90ead0cMarkos Chandras#include <asm/bitops.h> 24c6610de353da5ca6eee5b8960e838a87a90ead0cMarkos Chandras#include <asm/cacheflush.h> 25c6610de353da5ca6eee5b8960e838a87a90ead0cMarkos Chandras#include <asm/cpu-features.h> 26c6610de353da5ca6eee5b8960e838a87a90ead0cMarkos Chandras#include <asm/uasm.h> 27c6610de353da5ca6eee5b8960e838a87a90ead0cMarkos Chandras 28c6610de353da5ca6eee5b8960e838a87a90ead0cMarkos Chandras#include "bpf_jit.h" 29c6610de353da5ca6eee5b8960e838a87a90ead0cMarkos Chandras 30c6610de353da5ca6eee5b8960e838a87a90ead0cMarkos Chandras/* ABI 31c6610de353da5ca6eee5b8960e838a87a90ead0cMarkos Chandras * 32c6610de353da5ca6eee5b8960e838a87a90ead0cMarkos Chandras * s0 1st scratch register 33c6610de353da5ca6eee5b8960e838a87a90ead0cMarkos Chandras * s1 2nd scratch register 34c6610de353da5ca6eee5b8960e838a87a90ead0cMarkos Chandras * s2 offset register 35c6610de353da5ca6eee5b8960e838a87a90ead0cMarkos Chandras * s3 BPF register A 36c6610de353da5ca6eee5b8960e838a87a90ead0cMarkos Chandras * s4 BPF register X 37c6610de353da5ca6eee5b8960e838a87a90ead0cMarkos Chandras * s5 *skb 38c6610de353da5ca6eee5b8960e838a87a90ead0cMarkos Chandras * s6 *scratch memory 39c6610de353da5ca6eee5b8960e838a87a90ead0cMarkos Chandras * 40c6610de353da5ca6eee5b8960e838a87a90ead0cMarkos Chandras * On entry (*bpf_func)(*skb, *filter) 41c6610de353da5ca6eee5b8960e838a87a90ead0cMarkos Chandras * a0 = MIPS_R_A0 = skb; 42c6610de353da5ca6eee5b8960e838a87a90ead0cMarkos Chandras * a1 = MIPS_R_A1 = filter; 43c6610de353da5ca6eee5b8960e838a87a90ead0cMarkos Chandras * 44c6610de353da5ca6eee5b8960e838a87a90ead0cMarkos Chandras * Stack 45c6610de353da5ca6eee5b8960e838a87a90ead0cMarkos Chandras * ... 46c6610de353da5ca6eee5b8960e838a87a90ead0cMarkos Chandras * M[15] 47c6610de353da5ca6eee5b8960e838a87a90ead0cMarkos Chandras * M[14] 48c6610de353da5ca6eee5b8960e838a87a90ead0cMarkos Chandras * M[13] 49c6610de353da5ca6eee5b8960e838a87a90ead0cMarkos Chandras * ... 50c6610de353da5ca6eee5b8960e838a87a90ead0cMarkos Chandras * M[0] <-- r_M 51c6610de353da5ca6eee5b8960e838a87a90ead0cMarkos Chandras * saved reg k-1 52c6610de353da5ca6eee5b8960e838a87a90ead0cMarkos Chandras * saved reg k-2 53c6610de353da5ca6eee5b8960e838a87a90ead0cMarkos Chandras * ... 54c6610de353da5ca6eee5b8960e838a87a90ead0cMarkos Chandras * saved reg 0 <-- r_sp 55c6610de353da5ca6eee5b8960e838a87a90ead0cMarkos Chandras * <no argument area> 56c6610de353da5ca6eee5b8960e838a87a90ead0cMarkos Chandras * 57c6610de353da5ca6eee5b8960e838a87a90ead0cMarkos Chandras * Packet layout 58c6610de353da5ca6eee5b8960e838a87a90ead0cMarkos Chandras * 59c6610de353da5ca6eee5b8960e838a87a90ead0cMarkos Chandras * <--------------------- len ------------------------> 60c6610de353da5ca6eee5b8960e838a87a90ead0cMarkos Chandras * <--skb-len(r_skb_hl)-->< ----- skb->data_len ------> 61c6610de353da5ca6eee5b8960e838a87a90ead0cMarkos Chandras * ---------------------------------------------------- 62c6610de353da5ca6eee5b8960e838a87a90ead0cMarkos Chandras * | skb->data | 63c6610de353da5ca6eee5b8960e838a87a90ead0cMarkos Chandras * ---------------------------------------------------- 64c6610de353da5ca6eee5b8960e838a87a90ead0cMarkos Chandras */ 65c6610de353da5ca6eee5b8960e838a87a90ead0cMarkos Chandras 66c6610de353da5ca6eee5b8960e838a87a90ead0cMarkos Chandras#define RSIZE (sizeof(unsigned long)) 67c6610de353da5ca6eee5b8960e838a87a90ead0cMarkos Chandras#define ptr typeof(unsigned long) 68c6610de353da5ca6eee5b8960e838a87a90ead0cMarkos Chandras 69c6610de353da5ca6eee5b8960e838a87a90ead0cMarkos Chandras/* ABI specific return values */ 70c6610de353da5ca6eee5b8960e838a87a90ead0cMarkos Chandras#ifdef CONFIG_32BIT /* O32 */ 71c6610de353da5ca6eee5b8960e838a87a90ead0cMarkos Chandras#ifdef CONFIG_CPU_LITTLE_ENDIAN 72c6610de353da5ca6eee5b8960e838a87a90ead0cMarkos Chandras#define r_err MIPS_R_V1 73c6610de353da5ca6eee5b8960e838a87a90ead0cMarkos Chandras#define r_val MIPS_R_V0 74c6610de353da5ca6eee5b8960e838a87a90ead0cMarkos Chandras#else /* CONFIG_CPU_LITTLE_ENDIAN */ 75c6610de353da5ca6eee5b8960e838a87a90ead0cMarkos Chandras#define r_err MIPS_R_V0 76c6610de353da5ca6eee5b8960e838a87a90ead0cMarkos Chandras#define r_val MIPS_R_V1 77c6610de353da5ca6eee5b8960e838a87a90ead0cMarkos Chandras#endif 78c6610de353da5ca6eee5b8960e838a87a90ead0cMarkos Chandras#else /* N64 */ 79c6610de353da5ca6eee5b8960e838a87a90ead0cMarkos Chandras#define r_err MIPS_R_V0 80c6610de353da5ca6eee5b8960e838a87a90ead0cMarkos Chandras#define r_val MIPS_R_V0 81c6610de353da5ca6eee5b8960e838a87a90ead0cMarkos Chandras#endif 82c6610de353da5ca6eee5b8960e838a87a90ead0cMarkos Chandras 83c6610de353da5ca6eee5b8960e838a87a90ead0cMarkos Chandras#define r_ret MIPS_R_V0 84c6610de353da5ca6eee5b8960e838a87a90ead0cMarkos Chandras 85c6610de353da5ca6eee5b8960e838a87a90ead0cMarkos Chandras/* 86c6610de353da5ca6eee5b8960e838a87a90ead0cMarkos Chandras * Use 2 scratch registers to avoid pipeline interlocks. 87c6610de353da5ca6eee5b8960e838a87a90ead0cMarkos Chandras * There is no overhead during epilogue and prologue since 88c6610de353da5ca6eee5b8960e838a87a90ead0cMarkos Chandras * any of the $s0-$s6 registers will only be preserved if 89c6610de353da5ca6eee5b8960e838a87a90ead0cMarkos Chandras * they are going to actually be used. 90c6610de353da5ca6eee5b8960e838a87a90ead0cMarkos Chandras */ 91c6610de353da5ca6eee5b8960e838a87a90ead0cMarkos Chandras#define r_s0 MIPS_R_S0 /* scratch reg 1 */ 92c6610de353da5ca6eee5b8960e838a87a90ead0cMarkos Chandras#define r_s1 MIPS_R_S1 /* scratch reg 2 */ 93c6610de353da5ca6eee5b8960e838a87a90ead0cMarkos Chandras#define r_off MIPS_R_S2 94c6610de353da5ca6eee5b8960e838a87a90ead0cMarkos Chandras#define r_A MIPS_R_S3 95c6610de353da5ca6eee5b8960e838a87a90ead0cMarkos Chandras#define r_X MIPS_R_S4 96c6610de353da5ca6eee5b8960e838a87a90ead0cMarkos Chandras#define r_skb MIPS_R_S5 97c6610de353da5ca6eee5b8960e838a87a90ead0cMarkos Chandras#define r_M MIPS_R_S6 98c6610de353da5ca6eee5b8960e838a87a90ead0cMarkos Chandras#define r_tmp_imm MIPS_R_T6 /* No need to preserve this */ 99c6610de353da5ca6eee5b8960e838a87a90ead0cMarkos Chandras#define r_tmp MIPS_R_T7 /* No need to preserve this */ 100c6610de353da5ca6eee5b8960e838a87a90ead0cMarkos Chandras#define r_zero MIPS_R_ZERO 101c6610de353da5ca6eee5b8960e838a87a90ead0cMarkos Chandras#define r_sp MIPS_R_SP 102c6610de353da5ca6eee5b8960e838a87a90ead0cMarkos Chandras#define r_ra MIPS_R_RA 103c6610de353da5ca6eee5b8960e838a87a90ead0cMarkos Chandras 104c6610de353da5ca6eee5b8960e838a87a90ead0cMarkos Chandras#define SCRATCH_OFF(k) (4 * (k)) 105c6610de353da5ca6eee5b8960e838a87a90ead0cMarkos Chandras 106c6610de353da5ca6eee5b8960e838a87a90ead0cMarkos Chandras/* JIT flags */ 107c6610de353da5ca6eee5b8960e838a87a90ead0cMarkos Chandras#define SEEN_CALL (1 << BPF_MEMWORDS) 108c6610de353da5ca6eee5b8960e838a87a90ead0cMarkos Chandras#define SEEN_SREG_SFT (BPF_MEMWORDS + 1) 109c6610de353da5ca6eee5b8960e838a87a90ead0cMarkos Chandras#define SEEN_SREG_BASE (1 << SEEN_SREG_SFT) 110c6610de353da5ca6eee5b8960e838a87a90ead0cMarkos Chandras#define SEEN_SREG(x) (SEEN_SREG_BASE << (x)) 111c6610de353da5ca6eee5b8960e838a87a90ead0cMarkos Chandras#define SEEN_S0 SEEN_SREG(0) 112c6610de353da5ca6eee5b8960e838a87a90ead0cMarkos Chandras#define SEEN_S1 SEEN_SREG(1) 113c6610de353da5ca6eee5b8960e838a87a90ead0cMarkos Chandras#define SEEN_OFF SEEN_SREG(2) 114c6610de353da5ca6eee5b8960e838a87a90ead0cMarkos Chandras#define SEEN_A SEEN_SREG(3) 115c6610de353da5ca6eee5b8960e838a87a90ead0cMarkos Chandras#define SEEN_X SEEN_SREG(4) 116c6610de353da5ca6eee5b8960e838a87a90ead0cMarkos Chandras#define SEEN_SKB SEEN_SREG(5) 117c6610de353da5ca6eee5b8960e838a87a90ead0cMarkos Chandras#define SEEN_MEM SEEN_SREG(6) 118c6610de353da5ca6eee5b8960e838a87a90ead0cMarkos Chandras 119c6610de353da5ca6eee5b8960e838a87a90ead0cMarkos Chandras/* Arguments used by JIT */ 120c6610de353da5ca6eee5b8960e838a87a90ead0cMarkos Chandras#define ARGS_USED_BY_JIT 2 /* only applicable to 64-bit */ 121c6610de353da5ca6eee5b8960e838a87a90ead0cMarkos Chandras 122c6610de353da5ca6eee5b8960e838a87a90ead0cMarkos Chandras#define SBIT(x) (1 << (x)) /* Signed version of BIT() */ 123c6610de353da5ca6eee5b8960e838a87a90ead0cMarkos Chandras 124c6610de353da5ca6eee5b8960e838a87a90ead0cMarkos Chandras/** 125c6610de353da5ca6eee5b8960e838a87a90ead0cMarkos Chandras * struct jit_ctx - JIT context 126c6610de353da5ca6eee5b8960e838a87a90ead0cMarkos Chandras * @skf: The sk_filter 127c6610de353da5ca6eee5b8960e838a87a90ead0cMarkos Chandras * @prologue_bytes: Number of bytes for prologue 128c6610de353da5ca6eee5b8960e838a87a90ead0cMarkos Chandras * @idx: Instruction index 129c6610de353da5ca6eee5b8960e838a87a90ead0cMarkos Chandras * @flags: JIT flags 130c6610de353da5ca6eee5b8960e838a87a90ead0cMarkos Chandras * @offsets: Instruction offsets 131c6610de353da5ca6eee5b8960e838a87a90ead0cMarkos Chandras * @target: Memory location for the compiled filter 132c6610de353da5ca6eee5b8960e838a87a90ead0cMarkos Chandras */ 133c6610de353da5ca6eee5b8960e838a87a90ead0cMarkos Chandrasstruct jit_ctx { 1347ae457c1e5b45a1b826fad9d62b32191d2bdcfdbAlexei Starovoitov const struct bpf_prog *skf; 135c6610de353da5ca6eee5b8960e838a87a90ead0cMarkos Chandras unsigned int prologue_bytes; 136c6610de353da5ca6eee5b8960e838a87a90ead0cMarkos Chandras u32 idx; 137c6610de353da5ca6eee5b8960e838a87a90ead0cMarkos Chandras u32 flags; 138c6610de353da5ca6eee5b8960e838a87a90ead0cMarkos Chandras u32 *offsets; 139c6610de353da5ca6eee5b8960e838a87a90ead0cMarkos Chandras u32 *target; 140c6610de353da5ca6eee5b8960e838a87a90ead0cMarkos Chandras}; 141c6610de353da5ca6eee5b8960e838a87a90ead0cMarkos Chandras 142c6610de353da5ca6eee5b8960e838a87a90ead0cMarkos Chandras 143c6610de353da5ca6eee5b8960e838a87a90ead0cMarkos Chandrasstatic inline int optimize_div(u32 *k) 144c6610de353da5ca6eee5b8960e838a87a90ead0cMarkos Chandras{ 145c6610de353da5ca6eee5b8960e838a87a90ead0cMarkos Chandras /* power of 2 divides can be implemented with right shift */ 146c6610de353da5ca6eee5b8960e838a87a90ead0cMarkos Chandras if (!(*k & (*k-1))) { 147c6610de353da5ca6eee5b8960e838a87a90ead0cMarkos Chandras *k = ilog2(*k); 148c6610de353da5ca6eee5b8960e838a87a90ead0cMarkos Chandras return 1; 149c6610de353da5ca6eee5b8960e838a87a90ead0cMarkos Chandras } 150c6610de353da5ca6eee5b8960e838a87a90ead0cMarkos Chandras 151c6610de353da5ca6eee5b8960e838a87a90ead0cMarkos Chandras return 0; 152c6610de353da5ca6eee5b8960e838a87a90ead0cMarkos Chandras} 153c6610de353da5ca6eee5b8960e838a87a90ead0cMarkos Chandras 15495782bf434437b3292f5cb9ce21b53bdbc1beda1Markos Chandrasstatic inline void emit_jit_reg_move(ptr dst, ptr src, struct jit_ctx *ctx); 15595782bf434437b3292f5cb9ce21b53bdbc1beda1Markos Chandras 156c6610de353da5ca6eee5b8960e838a87a90ead0cMarkos Chandras/* Simply emit the instruction if the JIT memory space has been allocated */ 157c6610de353da5ca6eee5b8960e838a87a90ead0cMarkos Chandras#define emit_instr(ctx, func, ...) \ 158c6610de353da5ca6eee5b8960e838a87a90ead0cMarkos Chandrasdo { \ 159c6610de353da5ca6eee5b8960e838a87a90ead0cMarkos Chandras if ((ctx)->target != NULL) { \ 160c6610de353da5ca6eee5b8960e838a87a90ead0cMarkos Chandras u32 *p = &(ctx)->target[ctx->idx]; \ 161c6610de353da5ca6eee5b8960e838a87a90ead0cMarkos Chandras uasm_i_##func(&p, ##__VA_ARGS__); \ 162c6610de353da5ca6eee5b8960e838a87a90ead0cMarkos Chandras } \ 163c6610de353da5ca6eee5b8960e838a87a90ead0cMarkos Chandras (ctx)->idx++; \ 164c6610de353da5ca6eee5b8960e838a87a90ead0cMarkos Chandras} while (0) 165c6610de353da5ca6eee5b8960e838a87a90ead0cMarkos Chandras 16639bcb7969a84aab3dcdb857b261a87d5d888f5a7Markos Chandras/* 16739bcb7969a84aab3dcdb857b261a87d5d888f5a7Markos Chandras * Similar to emit_instr but it must be used when we need to emit 16839bcb7969a84aab3dcdb857b261a87d5d888f5a7Markos Chandras * 32-bit or 64-bit instructions 16939bcb7969a84aab3dcdb857b261a87d5d888f5a7Markos Chandras */ 17039bcb7969a84aab3dcdb857b261a87d5d888f5a7Markos Chandras#define emit_long_instr(ctx, func, ...) \ 17139bcb7969a84aab3dcdb857b261a87d5d888f5a7Markos Chandrasdo { \ 17239bcb7969a84aab3dcdb857b261a87d5d888f5a7Markos Chandras if ((ctx)->target != NULL) { \ 17339bcb7969a84aab3dcdb857b261a87d5d888f5a7Markos Chandras u32 *p = &(ctx)->target[ctx->idx]; \ 17439bcb7969a84aab3dcdb857b261a87d5d888f5a7Markos Chandras UASM_i_##func(&p, ##__VA_ARGS__); \ 17539bcb7969a84aab3dcdb857b261a87d5d888f5a7Markos Chandras } \ 17639bcb7969a84aab3dcdb857b261a87d5d888f5a7Markos Chandras (ctx)->idx++; \ 17739bcb7969a84aab3dcdb857b261a87d5d888f5a7Markos Chandras} while (0) 17839bcb7969a84aab3dcdb857b261a87d5d888f5a7Markos Chandras 179c6610de353da5ca6eee5b8960e838a87a90ead0cMarkos Chandras/* Determine if immediate is within the 16-bit signed range */ 180c6610de353da5ca6eee5b8960e838a87a90ead0cMarkos Chandrasstatic inline bool is_range16(s32 imm) 181c6610de353da5ca6eee5b8960e838a87a90ead0cMarkos Chandras{ 18210c4d614d2ffcfc17add01f9648c3e530fb308d1Markos Chandras return !(imm >= SBIT(15) || imm < -SBIT(15)); 183c6610de353da5ca6eee5b8960e838a87a90ead0cMarkos Chandras} 184c6610de353da5ca6eee5b8960e838a87a90ead0cMarkos Chandras 185c6610de353da5ca6eee5b8960e838a87a90ead0cMarkos Chandrasstatic inline void emit_addu(unsigned int dst, unsigned int src1, 186c6610de353da5ca6eee5b8960e838a87a90ead0cMarkos Chandras unsigned int src2, struct jit_ctx *ctx) 187c6610de353da5ca6eee5b8960e838a87a90ead0cMarkos Chandras{ 188c6610de353da5ca6eee5b8960e838a87a90ead0cMarkos Chandras emit_instr(ctx, addu, dst, src1, src2); 189c6610de353da5ca6eee5b8960e838a87a90ead0cMarkos Chandras} 190c6610de353da5ca6eee5b8960e838a87a90ead0cMarkos Chandras 191c6610de353da5ca6eee5b8960e838a87a90ead0cMarkos Chandrasstatic inline void emit_nop(struct jit_ctx *ctx) 192c6610de353da5ca6eee5b8960e838a87a90ead0cMarkos Chandras{ 193c6610de353da5ca6eee5b8960e838a87a90ead0cMarkos Chandras emit_instr(ctx, nop); 194c6610de353da5ca6eee5b8960e838a87a90ead0cMarkos Chandras} 195c6610de353da5ca6eee5b8960e838a87a90ead0cMarkos Chandras 196c6610de353da5ca6eee5b8960e838a87a90ead0cMarkos Chandras/* Load a u32 immediate to a register */ 197c6610de353da5ca6eee5b8960e838a87a90ead0cMarkos Chandrasstatic inline void emit_load_imm(unsigned int dst, u32 imm, struct jit_ctx *ctx) 198c6610de353da5ca6eee5b8960e838a87a90ead0cMarkos Chandras{ 199c6610de353da5ca6eee5b8960e838a87a90ead0cMarkos Chandras if (ctx->target != NULL) { 200c6610de353da5ca6eee5b8960e838a87a90ead0cMarkos Chandras /* addiu can only handle s16 */ 20110c4d614d2ffcfc17add01f9648c3e530fb308d1Markos Chandras if (!is_range16(imm)) { 202c6610de353da5ca6eee5b8960e838a87a90ead0cMarkos Chandras u32 *p = &ctx->target[ctx->idx]; 203c6610de353da5ca6eee5b8960e838a87a90ead0cMarkos Chandras uasm_i_lui(&p, r_tmp_imm, (s32)imm >> 16); 204c6610de353da5ca6eee5b8960e838a87a90ead0cMarkos Chandras p = &ctx->target[ctx->idx + 1]; 205c6610de353da5ca6eee5b8960e838a87a90ead0cMarkos Chandras uasm_i_ori(&p, dst, r_tmp_imm, imm & 0xffff); 206c6610de353da5ca6eee5b8960e838a87a90ead0cMarkos Chandras } else { 207c6610de353da5ca6eee5b8960e838a87a90ead0cMarkos Chandras u32 *p = &ctx->target[ctx->idx]; 208c6610de353da5ca6eee5b8960e838a87a90ead0cMarkos Chandras uasm_i_addiu(&p, dst, r_zero, imm); 209c6610de353da5ca6eee5b8960e838a87a90ead0cMarkos Chandras } 210c6610de353da5ca6eee5b8960e838a87a90ead0cMarkos Chandras } 211c6610de353da5ca6eee5b8960e838a87a90ead0cMarkos Chandras ctx->idx++; 212c6610de353da5ca6eee5b8960e838a87a90ead0cMarkos Chandras 21310c4d614d2ffcfc17add01f9648c3e530fb308d1Markos Chandras if (!is_range16(imm)) 214c6610de353da5ca6eee5b8960e838a87a90ead0cMarkos Chandras ctx->idx++; 215c6610de353da5ca6eee5b8960e838a87a90ead0cMarkos Chandras} 216c6610de353da5ca6eee5b8960e838a87a90ead0cMarkos Chandras 217c6610de353da5ca6eee5b8960e838a87a90ead0cMarkos Chandrasstatic inline void emit_or(unsigned int dst, unsigned int src1, 218c6610de353da5ca6eee5b8960e838a87a90ead0cMarkos Chandras unsigned int src2, struct jit_ctx *ctx) 219c6610de353da5ca6eee5b8960e838a87a90ead0cMarkos Chandras{ 220c6610de353da5ca6eee5b8960e838a87a90ead0cMarkos Chandras emit_instr(ctx, or, dst, src1, src2); 221c6610de353da5ca6eee5b8960e838a87a90ead0cMarkos Chandras} 222c6610de353da5ca6eee5b8960e838a87a90ead0cMarkos Chandras 223c6610de353da5ca6eee5b8960e838a87a90ead0cMarkos Chandrasstatic inline void emit_ori(unsigned int dst, unsigned src, u32 imm, 224c6610de353da5ca6eee5b8960e838a87a90ead0cMarkos Chandras struct jit_ctx *ctx) 225c6610de353da5ca6eee5b8960e838a87a90ead0cMarkos Chandras{ 226c6610de353da5ca6eee5b8960e838a87a90ead0cMarkos Chandras if (imm >= BIT(16)) { 227c6610de353da5ca6eee5b8960e838a87a90ead0cMarkos Chandras emit_load_imm(r_tmp, imm, ctx); 228c6610de353da5ca6eee5b8960e838a87a90ead0cMarkos Chandras emit_or(dst, src, r_tmp, ctx); 229c6610de353da5ca6eee5b8960e838a87a90ead0cMarkos Chandras } else { 230c6610de353da5ca6eee5b8960e838a87a90ead0cMarkos Chandras emit_instr(ctx, ori, dst, src, imm); 231c6610de353da5ca6eee5b8960e838a87a90ead0cMarkos Chandras } 232c6610de353da5ca6eee5b8960e838a87a90ead0cMarkos Chandras} 233c6610de353da5ca6eee5b8960e838a87a90ead0cMarkos Chandras 234c6610de353da5ca6eee5b8960e838a87a90ead0cMarkos Chandrasstatic inline void emit_daddiu(unsigned int dst, unsigned int src, 235c6610de353da5ca6eee5b8960e838a87a90ead0cMarkos Chandras int imm, struct jit_ctx *ctx) 236c6610de353da5ca6eee5b8960e838a87a90ead0cMarkos Chandras{ 237c6610de353da5ca6eee5b8960e838a87a90ead0cMarkos Chandras /* 238c6610de353da5ca6eee5b8960e838a87a90ead0cMarkos Chandras * Only used for stack, so the imm is relatively small 239c6610de353da5ca6eee5b8960e838a87a90ead0cMarkos Chandras * and it fits in 15-bits 240c6610de353da5ca6eee5b8960e838a87a90ead0cMarkos Chandras */ 241c6610de353da5ca6eee5b8960e838a87a90ead0cMarkos Chandras emit_instr(ctx, daddiu, dst, src, imm); 242c6610de353da5ca6eee5b8960e838a87a90ead0cMarkos Chandras} 243c6610de353da5ca6eee5b8960e838a87a90ead0cMarkos Chandras 244c6610de353da5ca6eee5b8960e838a87a90ead0cMarkos Chandrasstatic inline void emit_addiu(unsigned int dst, unsigned int src, 245c6610de353da5ca6eee5b8960e838a87a90ead0cMarkos Chandras u32 imm, struct jit_ctx *ctx) 246c6610de353da5ca6eee5b8960e838a87a90ead0cMarkos Chandras{ 24710c4d614d2ffcfc17add01f9648c3e530fb308d1Markos Chandras if (!is_range16(imm)) { 248c6610de353da5ca6eee5b8960e838a87a90ead0cMarkos Chandras emit_load_imm(r_tmp, imm, ctx); 249c6610de353da5ca6eee5b8960e838a87a90ead0cMarkos Chandras emit_addu(dst, r_tmp, src, ctx); 250c6610de353da5ca6eee5b8960e838a87a90ead0cMarkos Chandras } else { 251c6610de353da5ca6eee5b8960e838a87a90ead0cMarkos Chandras emit_instr(ctx, addiu, dst, src, imm); 252c6610de353da5ca6eee5b8960e838a87a90ead0cMarkos Chandras } 253c6610de353da5ca6eee5b8960e838a87a90ead0cMarkos Chandras} 254c6610de353da5ca6eee5b8960e838a87a90ead0cMarkos Chandras 255c6610de353da5ca6eee5b8960e838a87a90ead0cMarkos Chandrasstatic inline void emit_and(unsigned int dst, unsigned int src1, 256c6610de353da5ca6eee5b8960e838a87a90ead0cMarkos Chandras unsigned int src2, struct jit_ctx *ctx) 257c6610de353da5ca6eee5b8960e838a87a90ead0cMarkos Chandras{ 258c6610de353da5ca6eee5b8960e838a87a90ead0cMarkos Chandras emit_instr(ctx, and, dst, src1, src2); 259c6610de353da5ca6eee5b8960e838a87a90ead0cMarkos Chandras} 260c6610de353da5ca6eee5b8960e838a87a90ead0cMarkos Chandras 261c6610de353da5ca6eee5b8960e838a87a90ead0cMarkos Chandrasstatic inline void emit_andi(unsigned int dst, unsigned int src, 262c6610de353da5ca6eee5b8960e838a87a90ead0cMarkos Chandras u32 imm, struct jit_ctx *ctx) 263c6610de353da5ca6eee5b8960e838a87a90ead0cMarkos Chandras{ 264c6610de353da5ca6eee5b8960e838a87a90ead0cMarkos Chandras /* If imm does not fit in u16 then load it to register */ 265c6610de353da5ca6eee5b8960e838a87a90ead0cMarkos Chandras if (imm >= BIT(16)) { 266c6610de353da5ca6eee5b8960e838a87a90ead0cMarkos Chandras emit_load_imm(r_tmp, imm, ctx); 267c6610de353da5ca6eee5b8960e838a87a90ead0cMarkos Chandras emit_and(dst, src, r_tmp, ctx); 268c6610de353da5ca6eee5b8960e838a87a90ead0cMarkos Chandras } else { 269c6610de353da5ca6eee5b8960e838a87a90ead0cMarkos Chandras emit_instr(ctx, andi, dst, src, imm); 270c6610de353da5ca6eee5b8960e838a87a90ead0cMarkos Chandras } 271c6610de353da5ca6eee5b8960e838a87a90ead0cMarkos Chandras} 272c6610de353da5ca6eee5b8960e838a87a90ead0cMarkos Chandras 273c6610de353da5ca6eee5b8960e838a87a90ead0cMarkos Chandrasstatic inline void emit_xor(unsigned int dst, unsigned int src1, 274c6610de353da5ca6eee5b8960e838a87a90ead0cMarkos Chandras unsigned int src2, struct jit_ctx *ctx) 275c6610de353da5ca6eee5b8960e838a87a90ead0cMarkos Chandras{ 276c6610de353da5ca6eee5b8960e838a87a90ead0cMarkos Chandras emit_instr(ctx, xor, dst, src1, src2); 277c6610de353da5ca6eee5b8960e838a87a90ead0cMarkos Chandras} 278c6610de353da5ca6eee5b8960e838a87a90ead0cMarkos Chandras 279c6610de353da5ca6eee5b8960e838a87a90ead0cMarkos Chandrasstatic inline void emit_xori(ptr dst, ptr src, u32 imm, struct jit_ctx *ctx) 280c6610de353da5ca6eee5b8960e838a87a90ead0cMarkos Chandras{ 281c6610de353da5ca6eee5b8960e838a87a90ead0cMarkos Chandras /* If imm does not fit in u16 then load it to register */ 282c6610de353da5ca6eee5b8960e838a87a90ead0cMarkos Chandras if (imm >= BIT(16)) { 283c6610de353da5ca6eee5b8960e838a87a90ead0cMarkos Chandras emit_load_imm(r_tmp, imm, ctx); 284c6610de353da5ca6eee5b8960e838a87a90ead0cMarkos Chandras emit_xor(dst, src, r_tmp, ctx); 285c6610de353da5ca6eee5b8960e838a87a90ead0cMarkos Chandras } else { 286c6610de353da5ca6eee5b8960e838a87a90ead0cMarkos Chandras emit_instr(ctx, xori, dst, src, imm); 287c6610de353da5ca6eee5b8960e838a87a90ead0cMarkos Chandras } 288c6610de353da5ca6eee5b8960e838a87a90ead0cMarkos Chandras} 289c6610de353da5ca6eee5b8960e838a87a90ead0cMarkos Chandras 290c6610de353da5ca6eee5b8960e838a87a90ead0cMarkos Chandrasstatic inline void emit_stack_offset(int offset, struct jit_ctx *ctx) 291c6610de353da5ca6eee5b8960e838a87a90ead0cMarkos Chandras{ 29239bcb7969a84aab3dcdb857b261a87d5d888f5a7Markos Chandras emit_long_instr(ctx, ADDIU, r_sp, r_sp, offset); 293c6610de353da5ca6eee5b8960e838a87a90ead0cMarkos Chandras} 294c6610de353da5ca6eee5b8960e838a87a90ead0cMarkos Chandras 295c6610de353da5ca6eee5b8960e838a87a90ead0cMarkos Chandrasstatic inline void emit_subu(unsigned int dst, unsigned int src1, 296c6610de353da5ca6eee5b8960e838a87a90ead0cMarkos Chandras unsigned int src2, struct jit_ctx *ctx) 297c6610de353da5ca6eee5b8960e838a87a90ead0cMarkos Chandras{ 298c6610de353da5ca6eee5b8960e838a87a90ead0cMarkos Chandras emit_instr(ctx, subu, dst, src1, src2); 299c6610de353da5ca6eee5b8960e838a87a90ead0cMarkos Chandras} 300c6610de353da5ca6eee5b8960e838a87a90ead0cMarkos Chandras 301c6610de353da5ca6eee5b8960e838a87a90ead0cMarkos Chandrasstatic inline void emit_neg(unsigned int reg, struct jit_ctx *ctx) 302c6610de353da5ca6eee5b8960e838a87a90ead0cMarkos Chandras{ 303c6610de353da5ca6eee5b8960e838a87a90ead0cMarkos Chandras emit_subu(reg, r_zero, reg, ctx); 304c6610de353da5ca6eee5b8960e838a87a90ead0cMarkos Chandras} 305c6610de353da5ca6eee5b8960e838a87a90ead0cMarkos Chandras 306c6610de353da5ca6eee5b8960e838a87a90ead0cMarkos Chandrasstatic inline void emit_sllv(unsigned int dst, unsigned int src, 307c6610de353da5ca6eee5b8960e838a87a90ead0cMarkos Chandras unsigned int sa, struct jit_ctx *ctx) 308c6610de353da5ca6eee5b8960e838a87a90ead0cMarkos Chandras{ 309c6610de353da5ca6eee5b8960e838a87a90ead0cMarkos Chandras emit_instr(ctx, sllv, dst, src, sa); 310c6610de353da5ca6eee5b8960e838a87a90ead0cMarkos Chandras} 311c6610de353da5ca6eee5b8960e838a87a90ead0cMarkos Chandras 312c6610de353da5ca6eee5b8960e838a87a90ead0cMarkos Chandrasstatic inline void emit_sll(unsigned int dst, unsigned int src, 313c6610de353da5ca6eee5b8960e838a87a90ead0cMarkos Chandras unsigned int sa, struct jit_ctx *ctx) 314c6610de353da5ca6eee5b8960e838a87a90ead0cMarkos Chandras{ 315c6610de353da5ca6eee5b8960e838a87a90ead0cMarkos Chandras /* sa is 5-bits long */ 31695782bf434437b3292f5cb9ce21b53bdbc1beda1Markos Chandras if (sa >= BIT(5)) 31795782bf434437b3292f5cb9ce21b53bdbc1beda1Markos Chandras /* Shifting >= 32 results in zero */ 31895782bf434437b3292f5cb9ce21b53bdbc1beda1Markos Chandras emit_jit_reg_move(dst, r_zero, ctx); 31995782bf434437b3292f5cb9ce21b53bdbc1beda1Markos Chandras else 32095782bf434437b3292f5cb9ce21b53bdbc1beda1Markos Chandras emit_instr(ctx, sll, dst, src, sa); 321c6610de353da5ca6eee5b8960e838a87a90ead0cMarkos Chandras} 322c6610de353da5ca6eee5b8960e838a87a90ead0cMarkos Chandras 323c6610de353da5ca6eee5b8960e838a87a90ead0cMarkos Chandrasstatic inline void emit_srlv(unsigned int dst, unsigned int src, 324c6610de353da5ca6eee5b8960e838a87a90ead0cMarkos Chandras unsigned int sa, struct jit_ctx *ctx) 325c6610de353da5ca6eee5b8960e838a87a90ead0cMarkos Chandras{ 326c6610de353da5ca6eee5b8960e838a87a90ead0cMarkos Chandras emit_instr(ctx, srlv, dst, src, sa); 327c6610de353da5ca6eee5b8960e838a87a90ead0cMarkos Chandras} 328c6610de353da5ca6eee5b8960e838a87a90ead0cMarkos Chandras 329c6610de353da5ca6eee5b8960e838a87a90ead0cMarkos Chandrasstatic inline void emit_srl(unsigned int dst, unsigned int src, 330c6610de353da5ca6eee5b8960e838a87a90ead0cMarkos Chandras unsigned int sa, struct jit_ctx *ctx) 331c6610de353da5ca6eee5b8960e838a87a90ead0cMarkos Chandras{ 332c6610de353da5ca6eee5b8960e838a87a90ead0cMarkos Chandras /* sa is 5-bits long */ 33395782bf434437b3292f5cb9ce21b53bdbc1beda1Markos Chandras if (sa >= BIT(5)) 33495782bf434437b3292f5cb9ce21b53bdbc1beda1Markos Chandras /* Shifting >= 32 results in zero */ 33595782bf434437b3292f5cb9ce21b53bdbc1beda1Markos Chandras emit_jit_reg_move(dst, r_zero, ctx); 33695782bf434437b3292f5cb9ce21b53bdbc1beda1Markos Chandras else 33795782bf434437b3292f5cb9ce21b53bdbc1beda1Markos Chandras emit_instr(ctx, srl, dst, src, sa); 338c6610de353da5ca6eee5b8960e838a87a90ead0cMarkos Chandras} 339c6610de353da5ca6eee5b8960e838a87a90ead0cMarkos Chandras 34055393ee535496f7db15f3b2e9d3cf418f772f71aMarkos Chandrasstatic inline void emit_slt(unsigned int dst, unsigned int src1, 34155393ee535496f7db15f3b2e9d3cf418f772f71aMarkos Chandras unsigned int src2, struct jit_ctx *ctx) 34255393ee535496f7db15f3b2e9d3cf418f772f71aMarkos Chandras{ 34355393ee535496f7db15f3b2e9d3cf418f772f71aMarkos Chandras emit_instr(ctx, slt, dst, src1, src2); 34455393ee535496f7db15f3b2e9d3cf418f772f71aMarkos Chandras} 34555393ee535496f7db15f3b2e9d3cf418f772f71aMarkos Chandras 346c6610de353da5ca6eee5b8960e838a87a90ead0cMarkos Chandrasstatic inline void emit_sltu(unsigned int dst, unsigned int src1, 347c6610de353da5ca6eee5b8960e838a87a90ead0cMarkos Chandras unsigned int src2, struct jit_ctx *ctx) 348c6610de353da5ca6eee5b8960e838a87a90ead0cMarkos Chandras{ 349c6610de353da5ca6eee5b8960e838a87a90ead0cMarkos Chandras emit_instr(ctx, sltu, dst, src1, src2); 350c6610de353da5ca6eee5b8960e838a87a90ead0cMarkos Chandras} 351c6610de353da5ca6eee5b8960e838a87a90ead0cMarkos Chandras 352c6610de353da5ca6eee5b8960e838a87a90ead0cMarkos Chandrasstatic inline void emit_sltiu(unsigned dst, unsigned int src, 353c6610de353da5ca6eee5b8960e838a87a90ead0cMarkos Chandras unsigned int imm, struct jit_ctx *ctx) 354c6610de353da5ca6eee5b8960e838a87a90ead0cMarkos Chandras{ 355c6610de353da5ca6eee5b8960e838a87a90ead0cMarkos Chandras /* 16 bit immediate */ 35610c4d614d2ffcfc17add01f9648c3e530fb308d1Markos Chandras if (!is_range16((s32)imm)) { 357c6610de353da5ca6eee5b8960e838a87a90ead0cMarkos Chandras emit_load_imm(r_tmp, imm, ctx); 358c6610de353da5ca6eee5b8960e838a87a90ead0cMarkos Chandras emit_sltu(dst, src, r_tmp, ctx); 359c6610de353da5ca6eee5b8960e838a87a90ead0cMarkos Chandras } else { 360c6610de353da5ca6eee5b8960e838a87a90ead0cMarkos Chandras emit_instr(ctx, sltiu, dst, src, imm); 361c6610de353da5ca6eee5b8960e838a87a90ead0cMarkos Chandras } 362c6610de353da5ca6eee5b8960e838a87a90ead0cMarkos Chandras 363c6610de353da5ca6eee5b8960e838a87a90ead0cMarkos Chandras} 364c6610de353da5ca6eee5b8960e838a87a90ead0cMarkos Chandras 365c6610de353da5ca6eee5b8960e838a87a90ead0cMarkos Chandras/* Store register on the stack */ 366c6610de353da5ca6eee5b8960e838a87a90ead0cMarkos Chandrasstatic inline void emit_store_stack_reg(ptr reg, ptr base, 367c6610de353da5ca6eee5b8960e838a87a90ead0cMarkos Chandras unsigned int offset, 368c6610de353da5ca6eee5b8960e838a87a90ead0cMarkos Chandras struct jit_ctx *ctx) 369c6610de353da5ca6eee5b8960e838a87a90ead0cMarkos Chandras{ 37039bcb7969a84aab3dcdb857b261a87d5d888f5a7Markos Chandras emit_long_instr(ctx, SW, reg, offset, base); 371c6610de353da5ca6eee5b8960e838a87a90ead0cMarkos Chandras} 372c6610de353da5ca6eee5b8960e838a87a90ead0cMarkos Chandras 373c6610de353da5ca6eee5b8960e838a87a90ead0cMarkos Chandrasstatic inline void emit_store(ptr reg, ptr base, unsigned int offset, 374c6610de353da5ca6eee5b8960e838a87a90ead0cMarkos Chandras struct jit_ctx *ctx) 375c6610de353da5ca6eee5b8960e838a87a90ead0cMarkos Chandras{ 376c6610de353da5ca6eee5b8960e838a87a90ead0cMarkos Chandras emit_instr(ctx, sw, reg, offset, base); 377c6610de353da5ca6eee5b8960e838a87a90ead0cMarkos Chandras} 378c6610de353da5ca6eee5b8960e838a87a90ead0cMarkos Chandras 379c6610de353da5ca6eee5b8960e838a87a90ead0cMarkos Chandrasstatic inline void emit_load_stack_reg(ptr reg, ptr base, 380c6610de353da5ca6eee5b8960e838a87a90ead0cMarkos Chandras unsigned int offset, 381c6610de353da5ca6eee5b8960e838a87a90ead0cMarkos Chandras struct jit_ctx *ctx) 382c6610de353da5ca6eee5b8960e838a87a90ead0cMarkos Chandras{ 38339bcb7969a84aab3dcdb857b261a87d5d888f5a7Markos Chandras emit_long_instr(ctx, LW, reg, offset, base); 384c6610de353da5ca6eee5b8960e838a87a90ead0cMarkos Chandras} 385c6610de353da5ca6eee5b8960e838a87a90ead0cMarkos Chandras 386c6610de353da5ca6eee5b8960e838a87a90ead0cMarkos Chandrasstatic inline void emit_load(unsigned int reg, unsigned int base, 387c6610de353da5ca6eee5b8960e838a87a90ead0cMarkos Chandras unsigned int offset, struct jit_ctx *ctx) 388c6610de353da5ca6eee5b8960e838a87a90ead0cMarkos Chandras{ 389c6610de353da5ca6eee5b8960e838a87a90ead0cMarkos Chandras emit_instr(ctx, lw, reg, offset, base); 390c6610de353da5ca6eee5b8960e838a87a90ead0cMarkos Chandras} 391c6610de353da5ca6eee5b8960e838a87a90ead0cMarkos Chandras 392c6610de353da5ca6eee5b8960e838a87a90ead0cMarkos Chandrasstatic inline void emit_load_byte(unsigned int reg, unsigned int base, 393c6610de353da5ca6eee5b8960e838a87a90ead0cMarkos Chandras unsigned int offset, struct jit_ctx *ctx) 394c6610de353da5ca6eee5b8960e838a87a90ead0cMarkos Chandras{ 395c6610de353da5ca6eee5b8960e838a87a90ead0cMarkos Chandras emit_instr(ctx, lb, reg, offset, base); 396c6610de353da5ca6eee5b8960e838a87a90ead0cMarkos Chandras} 397c6610de353da5ca6eee5b8960e838a87a90ead0cMarkos Chandras 398c6610de353da5ca6eee5b8960e838a87a90ead0cMarkos Chandrasstatic inline void emit_half_load(unsigned int reg, unsigned int base, 399c6610de353da5ca6eee5b8960e838a87a90ead0cMarkos Chandras unsigned int offset, struct jit_ctx *ctx) 400c6610de353da5ca6eee5b8960e838a87a90ead0cMarkos Chandras{ 401c6610de353da5ca6eee5b8960e838a87a90ead0cMarkos Chandras emit_instr(ctx, lh, reg, offset, base); 402c6610de353da5ca6eee5b8960e838a87a90ead0cMarkos Chandras} 403c6610de353da5ca6eee5b8960e838a87a90ead0cMarkos Chandras 404c6610de353da5ca6eee5b8960e838a87a90ead0cMarkos Chandrasstatic inline void emit_mul(unsigned int dst, unsigned int src1, 405c6610de353da5ca6eee5b8960e838a87a90ead0cMarkos Chandras unsigned int src2, struct jit_ctx *ctx) 406c6610de353da5ca6eee5b8960e838a87a90ead0cMarkos Chandras{ 407c6610de353da5ca6eee5b8960e838a87a90ead0cMarkos Chandras emit_instr(ctx, mul, dst, src1, src2); 408c6610de353da5ca6eee5b8960e838a87a90ead0cMarkos Chandras} 409c6610de353da5ca6eee5b8960e838a87a90ead0cMarkos Chandras 410c6610de353da5ca6eee5b8960e838a87a90ead0cMarkos Chandrasstatic inline void emit_div(unsigned int dst, unsigned int src, 411c6610de353da5ca6eee5b8960e838a87a90ead0cMarkos Chandras struct jit_ctx *ctx) 412c6610de353da5ca6eee5b8960e838a87a90ead0cMarkos Chandras{ 413c6610de353da5ca6eee5b8960e838a87a90ead0cMarkos Chandras if (ctx->target != NULL) { 414c6610de353da5ca6eee5b8960e838a87a90ead0cMarkos Chandras u32 *p = &ctx->target[ctx->idx]; 415c6610de353da5ca6eee5b8960e838a87a90ead0cMarkos Chandras uasm_i_divu(&p, dst, src); 416c6610de353da5ca6eee5b8960e838a87a90ead0cMarkos Chandras p = &ctx->target[ctx->idx + 1]; 41735a8e16abe36d385d602997e1500a668d2b9c5cfMarkos Chandras uasm_i_mflo(&p, dst); 418c6610de353da5ca6eee5b8960e838a87a90ead0cMarkos Chandras } 419c6610de353da5ca6eee5b8960e838a87a90ead0cMarkos Chandras ctx->idx += 2; /* 2 insts */ 420c6610de353da5ca6eee5b8960e838a87a90ead0cMarkos Chandras} 421c6610de353da5ca6eee5b8960e838a87a90ead0cMarkos Chandras 422c6610de353da5ca6eee5b8960e838a87a90ead0cMarkos Chandrasstatic inline void emit_mod(unsigned int dst, unsigned int src, 423c6610de353da5ca6eee5b8960e838a87a90ead0cMarkos Chandras struct jit_ctx *ctx) 424c6610de353da5ca6eee5b8960e838a87a90ead0cMarkos Chandras{ 425c6610de353da5ca6eee5b8960e838a87a90ead0cMarkos Chandras if (ctx->target != NULL) { 426c6610de353da5ca6eee5b8960e838a87a90ead0cMarkos Chandras u32 *p = &ctx->target[ctx->idx]; 427c6610de353da5ca6eee5b8960e838a87a90ead0cMarkos Chandras uasm_i_divu(&p, dst, src); 428c6610de353da5ca6eee5b8960e838a87a90ead0cMarkos Chandras p = &ctx->target[ctx->idx + 1]; 429c6610de353da5ca6eee5b8960e838a87a90ead0cMarkos Chandras uasm_i_mflo(&p, dst); 430c6610de353da5ca6eee5b8960e838a87a90ead0cMarkos Chandras } 431c6610de353da5ca6eee5b8960e838a87a90ead0cMarkos Chandras ctx->idx += 2; /* 2 insts */ 432c6610de353da5ca6eee5b8960e838a87a90ead0cMarkos Chandras} 433c6610de353da5ca6eee5b8960e838a87a90ead0cMarkos Chandras 434c6610de353da5ca6eee5b8960e838a87a90ead0cMarkos Chandrasstatic inline void emit_dsll(unsigned int dst, unsigned int src, 435c6610de353da5ca6eee5b8960e838a87a90ead0cMarkos Chandras unsigned int sa, struct jit_ctx *ctx) 436c6610de353da5ca6eee5b8960e838a87a90ead0cMarkos Chandras{ 437c6610de353da5ca6eee5b8960e838a87a90ead0cMarkos Chandras emit_instr(ctx, dsll, dst, src, sa); 438c6610de353da5ca6eee5b8960e838a87a90ead0cMarkos Chandras} 439c6610de353da5ca6eee5b8960e838a87a90ead0cMarkos Chandras 440c6610de353da5ca6eee5b8960e838a87a90ead0cMarkos Chandrasstatic inline void emit_dsrl32(unsigned int dst, unsigned int src, 441c6610de353da5ca6eee5b8960e838a87a90ead0cMarkos Chandras unsigned int sa, struct jit_ctx *ctx) 442c6610de353da5ca6eee5b8960e838a87a90ead0cMarkos Chandras{ 443c6610de353da5ca6eee5b8960e838a87a90ead0cMarkos Chandras emit_instr(ctx, dsrl32, dst, src, sa); 444c6610de353da5ca6eee5b8960e838a87a90ead0cMarkos Chandras} 445c6610de353da5ca6eee5b8960e838a87a90ead0cMarkos Chandras 446c6610de353da5ca6eee5b8960e838a87a90ead0cMarkos Chandrasstatic inline void emit_wsbh(unsigned int dst, unsigned int src, 447c6610de353da5ca6eee5b8960e838a87a90ead0cMarkos Chandras struct jit_ctx *ctx) 448c6610de353da5ca6eee5b8960e838a87a90ead0cMarkos Chandras{ 449c6610de353da5ca6eee5b8960e838a87a90ead0cMarkos Chandras emit_instr(ctx, wsbh, dst, src); 450c6610de353da5ca6eee5b8960e838a87a90ead0cMarkos Chandras} 451c6610de353da5ca6eee5b8960e838a87a90ead0cMarkos Chandras 452b6a14a9845259eb21c9d8121330c4c3b22de182eMarkos Chandras/* load pointer to register */ 453b6a14a9845259eb21c9d8121330c4c3b22de182eMarkos Chandrasstatic inline void emit_load_ptr(unsigned int dst, unsigned int src, 454b6a14a9845259eb21c9d8121330c4c3b22de182eMarkos Chandras int imm, struct jit_ctx *ctx) 455b6a14a9845259eb21c9d8121330c4c3b22de182eMarkos Chandras{ 456b6a14a9845259eb21c9d8121330c4c3b22de182eMarkos Chandras /* src contains the base addr of the 32/64-pointer */ 45739bcb7969a84aab3dcdb857b261a87d5d888f5a7Markos Chandras emit_long_instr(ctx, LW, dst, imm, src); 458b6a14a9845259eb21c9d8121330c4c3b22de182eMarkos Chandras} 459b6a14a9845259eb21c9d8121330c4c3b22de182eMarkos Chandras 460c6610de353da5ca6eee5b8960e838a87a90ead0cMarkos Chandras/* load a function pointer to register */ 461c6610de353da5ca6eee5b8960e838a87a90ead0cMarkos Chandrasstatic inline void emit_load_func(unsigned int reg, ptr imm, 462c6610de353da5ca6eee5b8960e838a87a90ead0cMarkos Chandras struct jit_ctx *ctx) 463c6610de353da5ca6eee5b8960e838a87a90ead0cMarkos Chandras{ 464c6610de353da5ca6eee5b8960e838a87a90ead0cMarkos Chandras if (config_enabled(CONFIG_64BIT)) { 465c6610de353da5ca6eee5b8960e838a87a90ead0cMarkos Chandras /* At this point imm is always 64-bit */ 466c6610de353da5ca6eee5b8960e838a87a90ead0cMarkos Chandras emit_load_imm(r_tmp, (u64)imm >> 32, ctx); 467c6610de353da5ca6eee5b8960e838a87a90ead0cMarkos Chandras emit_dsll(r_tmp_imm, r_tmp, 16, ctx); /* left shift by 16 */ 468c6610de353da5ca6eee5b8960e838a87a90ead0cMarkos Chandras emit_ori(r_tmp, r_tmp_imm, (imm >> 16) & 0xffff, ctx); 469c6610de353da5ca6eee5b8960e838a87a90ead0cMarkos Chandras emit_dsll(r_tmp_imm, r_tmp, 16, ctx); /* left shift by 16 */ 470c6610de353da5ca6eee5b8960e838a87a90ead0cMarkos Chandras emit_ori(reg, r_tmp_imm, imm & 0xffff, ctx); 471c6610de353da5ca6eee5b8960e838a87a90ead0cMarkos Chandras } else { 472c6610de353da5ca6eee5b8960e838a87a90ead0cMarkos Chandras emit_load_imm(reg, imm, ctx); 473c6610de353da5ca6eee5b8960e838a87a90ead0cMarkos Chandras } 474c6610de353da5ca6eee5b8960e838a87a90ead0cMarkos Chandras} 475c6610de353da5ca6eee5b8960e838a87a90ead0cMarkos Chandras 476c6610de353da5ca6eee5b8960e838a87a90ead0cMarkos Chandras/* Move to real MIPS register */ 477c6610de353da5ca6eee5b8960e838a87a90ead0cMarkos Chandrasstatic inline void emit_reg_move(ptr dst, ptr src, struct jit_ctx *ctx) 478c6610de353da5ca6eee5b8960e838a87a90ead0cMarkos Chandras{ 47939bcb7969a84aab3dcdb857b261a87d5d888f5a7Markos Chandras emit_long_instr(ctx, ADDU, dst, src, r_zero); 480c6610de353da5ca6eee5b8960e838a87a90ead0cMarkos Chandras} 481c6610de353da5ca6eee5b8960e838a87a90ead0cMarkos Chandras 482c6610de353da5ca6eee5b8960e838a87a90ead0cMarkos Chandras/* Move to JIT (32-bit) register */ 483c6610de353da5ca6eee5b8960e838a87a90ead0cMarkos Chandrasstatic inline void emit_jit_reg_move(ptr dst, ptr src, struct jit_ctx *ctx) 484c6610de353da5ca6eee5b8960e838a87a90ead0cMarkos Chandras{ 485c6610de353da5ca6eee5b8960e838a87a90ead0cMarkos Chandras emit_addu(dst, src, r_zero, ctx); 486c6610de353da5ca6eee5b8960e838a87a90ead0cMarkos Chandras} 487c6610de353da5ca6eee5b8960e838a87a90ead0cMarkos Chandras 488c6610de353da5ca6eee5b8960e838a87a90ead0cMarkos Chandras/* Compute the immediate value for PC-relative branches. */ 489c6610de353da5ca6eee5b8960e838a87a90ead0cMarkos Chandrasstatic inline u32 b_imm(unsigned int tgt, struct jit_ctx *ctx) 490c6610de353da5ca6eee5b8960e838a87a90ead0cMarkos Chandras{ 491c6610de353da5ca6eee5b8960e838a87a90ead0cMarkos Chandras if (ctx->target == NULL) 492c6610de353da5ca6eee5b8960e838a87a90ead0cMarkos Chandras return 0; 493c6610de353da5ca6eee5b8960e838a87a90ead0cMarkos Chandras 494c6610de353da5ca6eee5b8960e838a87a90ead0cMarkos Chandras /* 495c6610de353da5ca6eee5b8960e838a87a90ead0cMarkos Chandras * We want a pc-relative branch. We only do forward branches 496c6610de353da5ca6eee5b8960e838a87a90ead0cMarkos Chandras * so tgt is always after pc. tgt is the instruction offset 497c6610de353da5ca6eee5b8960e838a87a90ead0cMarkos Chandras * we want to jump to. 498c6610de353da5ca6eee5b8960e838a87a90ead0cMarkos Chandras 499c6610de353da5ca6eee5b8960e838a87a90ead0cMarkos Chandras * Branch on MIPS: 500c6610de353da5ca6eee5b8960e838a87a90ead0cMarkos Chandras * I: target_offset <- sign_extend(offset) 501c6610de353da5ca6eee5b8960e838a87a90ead0cMarkos Chandras * I+1: PC += target_offset (delay slot) 502c6610de353da5ca6eee5b8960e838a87a90ead0cMarkos Chandras * 503c6610de353da5ca6eee5b8960e838a87a90ead0cMarkos Chandras * ctx->idx currently points to the branch instruction 504c6610de353da5ca6eee5b8960e838a87a90ead0cMarkos Chandras * but the offset is added to the delay slot so we need 505c6610de353da5ca6eee5b8960e838a87a90ead0cMarkos Chandras * to subtract 4. 506c6610de353da5ca6eee5b8960e838a87a90ead0cMarkos Chandras */ 507c6610de353da5ca6eee5b8960e838a87a90ead0cMarkos Chandras return ctx->offsets[tgt] - 508c6610de353da5ca6eee5b8960e838a87a90ead0cMarkos Chandras (ctx->idx * 4 - ctx->prologue_bytes) - 4; 509c6610de353da5ca6eee5b8960e838a87a90ead0cMarkos Chandras} 510c6610de353da5ca6eee5b8960e838a87a90ead0cMarkos Chandras 511c6610de353da5ca6eee5b8960e838a87a90ead0cMarkos Chandrasstatic inline void emit_bcond(int cond, unsigned int reg1, unsigned int reg2, 512c6610de353da5ca6eee5b8960e838a87a90ead0cMarkos Chandras unsigned int imm, struct jit_ctx *ctx) 513c6610de353da5ca6eee5b8960e838a87a90ead0cMarkos Chandras{ 514c6610de353da5ca6eee5b8960e838a87a90ead0cMarkos Chandras if (ctx->target != NULL) { 515c6610de353da5ca6eee5b8960e838a87a90ead0cMarkos Chandras u32 *p = &ctx->target[ctx->idx]; 516c6610de353da5ca6eee5b8960e838a87a90ead0cMarkos Chandras 517c6610de353da5ca6eee5b8960e838a87a90ead0cMarkos Chandras switch (cond) { 518c6610de353da5ca6eee5b8960e838a87a90ead0cMarkos Chandras case MIPS_COND_EQ: 519c6610de353da5ca6eee5b8960e838a87a90ead0cMarkos Chandras uasm_i_beq(&p, reg1, reg2, imm); 520c6610de353da5ca6eee5b8960e838a87a90ead0cMarkos Chandras break; 521c6610de353da5ca6eee5b8960e838a87a90ead0cMarkos Chandras case MIPS_COND_NE: 522c6610de353da5ca6eee5b8960e838a87a90ead0cMarkos Chandras uasm_i_bne(&p, reg1, reg2, imm); 523c6610de353da5ca6eee5b8960e838a87a90ead0cMarkos Chandras break; 524c6610de353da5ca6eee5b8960e838a87a90ead0cMarkos Chandras case MIPS_COND_ALL: 525c6610de353da5ca6eee5b8960e838a87a90ead0cMarkos Chandras uasm_i_b(&p, imm); 526c6610de353da5ca6eee5b8960e838a87a90ead0cMarkos Chandras break; 527c6610de353da5ca6eee5b8960e838a87a90ead0cMarkos Chandras default: 528c6610de353da5ca6eee5b8960e838a87a90ead0cMarkos Chandras pr_warn("%s: Unhandled branch conditional: %d\n", 529c6610de353da5ca6eee5b8960e838a87a90ead0cMarkos Chandras __func__, cond); 530c6610de353da5ca6eee5b8960e838a87a90ead0cMarkos Chandras } 531c6610de353da5ca6eee5b8960e838a87a90ead0cMarkos Chandras } 532c6610de353da5ca6eee5b8960e838a87a90ead0cMarkos Chandras ctx->idx++; 533c6610de353da5ca6eee5b8960e838a87a90ead0cMarkos Chandras} 534c6610de353da5ca6eee5b8960e838a87a90ead0cMarkos Chandras 535c6610de353da5ca6eee5b8960e838a87a90ead0cMarkos Chandrasstatic inline void emit_b(unsigned int imm, struct jit_ctx *ctx) 536c6610de353da5ca6eee5b8960e838a87a90ead0cMarkos Chandras{ 537c6610de353da5ca6eee5b8960e838a87a90ead0cMarkos Chandras emit_bcond(MIPS_COND_ALL, r_zero, r_zero, imm, ctx); 538c6610de353da5ca6eee5b8960e838a87a90ead0cMarkos Chandras} 539c6610de353da5ca6eee5b8960e838a87a90ead0cMarkos Chandras 540c6610de353da5ca6eee5b8960e838a87a90ead0cMarkos Chandrasstatic inline void emit_jalr(unsigned int link, unsigned int reg, 541c6610de353da5ca6eee5b8960e838a87a90ead0cMarkos Chandras struct jit_ctx *ctx) 542c6610de353da5ca6eee5b8960e838a87a90ead0cMarkos Chandras{ 543c6610de353da5ca6eee5b8960e838a87a90ead0cMarkos Chandras emit_instr(ctx, jalr, link, reg); 544c6610de353da5ca6eee5b8960e838a87a90ead0cMarkos Chandras} 545c6610de353da5ca6eee5b8960e838a87a90ead0cMarkos Chandras 546c6610de353da5ca6eee5b8960e838a87a90ead0cMarkos Chandrasstatic inline void emit_jr(unsigned int reg, struct jit_ctx *ctx) 547c6610de353da5ca6eee5b8960e838a87a90ead0cMarkos Chandras{ 548c6610de353da5ca6eee5b8960e838a87a90ead0cMarkos Chandras emit_instr(ctx, jr, reg); 549c6610de353da5ca6eee5b8960e838a87a90ead0cMarkos Chandras} 550c6610de353da5ca6eee5b8960e838a87a90ead0cMarkos Chandras 551c6610de353da5ca6eee5b8960e838a87a90ead0cMarkos Chandrasstatic inline u16 align_sp(unsigned int num) 552c6610de353da5ca6eee5b8960e838a87a90ead0cMarkos Chandras{ 553c6610de353da5ca6eee5b8960e838a87a90ead0cMarkos Chandras /* Double word alignment for 32-bit, quadword for 64-bit */ 554c6610de353da5ca6eee5b8960e838a87a90ead0cMarkos Chandras unsigned int align = config_enabled(CONFIG_64BIT) ? 16 : 8; 555c6610de353da5ca6eee5b8960e838a87a90ead0cMarkos Chandras num = (num + (align - 1)) & -align; 556c6610de353da5ca6eee5b8960e838a87a90ead0cMarkos Chandras return num; 557c6610de353da5ca6eee5b8960e838a87a90ead0cMarkos Chandras} 558c6610de353da5ca6eee5b8960e838a87a90ead0cMarkos Chandras 559c6610de353da5ca6eee5b8960e838a87a90ead0cMarkos Chandrasstatic bool is_load_to_a(u16 inst) 560c6610de353da5ca6eee5b8960e838a87a90ead0cMarkos Chandras{ 561c6610de353da5ca6eee5b8960e838a87a90ead0cMarkos Chandras switch (inst) { 562a83d081ed14e4281d2620d182c6044c0c21c551eDaniel Borkmann case BPF_LD | BPF_W | BPF_LEN: 563a83d081ed14e4281d2620d182c6044c0c21c551eDaniel Borkmann case BPF_LD | BPF_W | BPF_ABS: 564a83d081ed14e4281d2620d182c6044c0c21c551eDaniel Borkmann case BPF_LD | BPF_H | BPF_ABS: 565a83d081ed14e4281d2620d182c6044c0c21c551eDaniel Borkmann case BPF_LD | BPF_B | BPF_ABS: 566c6610de353da5ca6eee5b8960e838a87a90ead0cMarkos Chandras return true; 567c6610de353da5ca6eee5b8960e838a87a90ead0cMarkos Chandras default: 568c6610de353da5ca6eee5b8960e838a87a90ead0cMarkos Chandras return false; 569c6610de353da5ca6eee5b8960e838a87a90ead0cMarkos Chandras } 570c6610de353da5ca6eee5b8960e838a87a90ead0cMarkos Chandras} 571c6610de353da5ca6eee5b8960e838a87a90ead0cMarkos Chandras 572c6610de353da5ca6eee5b8960e838a87a90ead0cMarkos Chandrasstatic void save_bpf_jit_regs(struct jit_ctx *ctx, unsigned offset) 573c6610de353da5ca6eee5b8960e838a87a90ead0cMarkos Chandras{ 574c6610de353da5ca6eee5b8960e838a87a90ead0cMarkos Chandras int i = 0, real_off = 0; 575c6610de353da5ca6eee5b8960e838a87a90ead0cMarkos Chandras u32 sflags, tmp_flags; 576c6610de353da5ca6eee5b8960e838a87a90ead0cMarkos Chandras 577c6610de353da5ca6eee5b8960e838a87a90ead0cMarkos Chandras /* Adjust the stack pointer */ 578c6610de353da5ca6eee5b8960e838a87a90ead0cMarkos Chandras emit_stack_offset(-align_sp(offset), ctx); 579c6610de353da5ca6eee5b8960e838a87a90ead0cMarkos Chandras 580c6610de353da5ca6eee5b8960e838a87a90ead0cMarkos Chandras if (ctx->flags & SEEN_CALL) { 581c6610de353da5ca6eee5b8960e838a87a90ead0cMarkos Chandras /* Argument save area */ 582c6610de353da5ca6eee5b8960e838a87a90ead0cMarkos Chandras if (config_enabled(CONFIG_64BIT)) 583c6610de353da5ca6eee5b8960e838a87a90ead0cMarkos Chandras /* Bottom of current frame */ 584c6610de353da5ca6eee5b8960e838a87a90ead0cMarkos Chandras real_off = align_sp(offset) - RSIZE; 585c6610de353da5ca6eee5b8960e838a87a90ead0cMarkos Chandras else 586c6610de353da5ca6eee5b8960e838a87a90ead0cMarkos Chandras /* Top of previous frame */ 587c6610de353da5ca6eee5b8960e838a87a90ead0cMarkos Chandras real_off = align_sp(offset) + RSIZE; 588c6610de353da5ca6eee5b8960e838a87a90ead0cMarkos Chandras emit_store_stack_reg(MIPS_R_A0, r_sp, real_off, ctx); 589c6610de353da5ca6eee5b8960e838a87a90ead0cMarkos Chandras emit_store_stack_reg(MIPS_R_A1, r_sp, real_off + RSIZE, ctx); 590c6610de353da5ca6eee5b8960e838a87a90ead0cMarkos Chandras 591c6610de353da5ca6eee5b8960e838a87a90ead0cMarkos Chandras real_off = 0; 592c6610de353da5ca6eee5b8960e838a87a90ead0cMarkos Chandras } 593c6610de353da5ca6eee5b8960e838a87a90ead0cMarkos Chandras 594c6610de353da5ca6eee5b8960e838a87a90ead0cMarkos Chandras tmp_flags = sflags = ctx->flags >> SEEN_SREG_SFT; 595c6610de353da5ca6eee5b8960e838a87a90ead0cMarkos Chandras /* sflags is essentially a bitmap */ 596c6610de353da5ca6eee5b8960e838a87a90ead0cMarkos Chandras while (tmp_flags) { 597c6610de353da5ca6eee5b8960e838a87a90ead0cMarkos Chandras if ((sflags >> i) & 0x1) { 598c6610de353da5ca6eee5b8960e838a87a90ead0cMarkos Chandras emit_store_stack_reg(MIPS_R_S0 + i, r_sp, real_off, 599c6610de353da5ca6eee5b8960e838a87a90ead0cMarkos Chandras ctx); 600c6610de353da5ca6eee5b8960e838a87a90ead0cMarkos Chandras real_off += RSIZE; 601c6610de353da5ca6eee5b8960e838a87a90ead0cMarkos Chandras } 602c6610de353da5ca6eee5b8960e838a87a90ead0cMarkos Chandras i++; 603c6610de353da5ca6eee5b8960e838a87a90ead0cMarkos Chandras tmp_flags >>= 1; 604c6610de353da5ca6eee5b8960e838a87a90ead0cMarkos Chandras } 605c6610de353da5ca6eee5b8960e838a87a90ead0cMarkos Chandras 606c6610de353da5ca6eee5b8960e838a87a90ead0cMarkos Chandras /* save return address */ 607c6610de353da5ca6eee5b8960e838a87a90ead0cMarkos Chandras if (ctx->flags & SEEN_CALL) { 608c6610de353da5ca6eee5b8960e838a87a90ead0cMarkos Chandras emit_store_stack_reg(r_ra, r_sp, real_off, ctx); 609c6610de353da5ca6eee5b8960e838a87a90ead0cMarkos Chandras real_off += RSIZE; 610c6610de353da5ca6eee5b8960e838a87a90ead0cMarkos Chandras } 611c6610de353da5ca6eee5b8960e838a87a90ead0cMarkos Chandras 612c6610de353da5ca6eee5b8960e838a87a90ead0cMarkos Chandras /* Setup r_M leaving the alignment gap if necessary */ 613c6610de353da5ca6eee5b8960e838a87a90ead0cMarkos Chandras if (ctx->flags & SEEN_MEM) { 614c6610de353da5ca6eee5b8960e838a87a90ead0cMarkos Chandras if (real_off % (RSIZE * 2)) 615c6610de353da5ca6eee5b8960e838a87a90ead0cMarkos Chandras real_off += RSIZE; 61639bcb7969a84aab3dcdb857b261a87d5d888f5a7Markos Chandras emit_long_instr(ctx, ADDIU, r_M, r_sp, real_off); 617c6610de353da5ca6eee5b8960e838a87a90ead0cMarkos Chandras } 618c6610de353da5ca6eee5b8960e838a87a90ead0cMarkos Chandras} 619c6610de353da5ca6eee5b8960e838a87a90ead0cMarkos Chandras 620c6610de353da5ca6eee5b8960e838a87a90ead0cMarkos Chandrasstatic void restore_bpf_jit_regs(struct jit_ctx *ctx, 621c6610de353da5ca6eee5b8960e838a87a90ead0cMarkos Chandras unsigned int offset) 622c6610de353da5ca6eee5b8960e838a87a90ead0cMarkos Chandras{ 623c6610de353da5ca6eee5b8960e838a87a90ead0cMarkos Chandras int i, real_off = 0; 624c6610de353da5ca6eee5b8960e838a87a90ead0cMarkos Chandras u32 sflags, tmp_flags; 625c6610de353da5ca6eee5b8960e838a87a90ead0cMarkos Chandras 626c6610de353da5ca6eee5b8960e838a87a90ead0cMarkos Chandras if (ctx->flags & SEEN_CALL) { 627c6610de353da5ca6eee5b8960e838a87a90ead0cMarkos Chandras if (config_enabled(CONFIG_64BIT)) 628c6610de353da5ca6eee5b8960e838a87a90ead0cMarkos Chandras /* Bottom of current frame */ 629c6610de353da5ca6eee5b8960e838a87a90ead0cMarkos Chandras real_off = align_sp(offset) - RSIZE; 630c6610de353da5ca6eee5b8960e838a87a90ead0cMarkos Chandras else 631c6610de353da5ca6eee5b8960e838a87a90ead0cMarkos Chandras /* Top of previous frame */ 632c6610de353da5ca6eee5b8960e838a87a90ead0cMarkos Chandras real_off = align_sp(offset) + RSIZE; 633c6610de353da5ca6eee5b8960e838a87a90ead0cMarkos Chandras emit_load_stack_reg(MIPS_R_A0, r_sp, real_off, ctx); 634c6610de353da5ca6eee5b8960e838a87a90ead0cMarkos Chandras emit_load_stack_reg(MIPS_R_A1, r_sp, real_off + RSIZE, ctx); 635c6610de353da5ca6eee5b8960e838a87a90ead0cMarkos Chandras 636c6610de353da5ca6eee5b8960e838a87a90ead0cMarkos Chandras real_off = 0; 637c6610de353da5ca6eee5b8960e838a87a90ead0cMarkos Chandras } 638c6610de353da5ca6eee5b8960e838a87a90ead0cMarkos Chandras 639c6610de353da5ca6eee5b8960e838a87a90ead0cMarkos Chandras tmp_flags = sflags = ctx->flags >> SEEN_SREG_SFT; 640c6610de353da5ca6eee5b8960e838a87a90ead0cMarkos Chandras /* sflags is a bitmap */ 641c6610de353da5ca6eee5b8960e838a87a90ead0cMarkos Chandras i = 0; 642c6610de353da5ca6eee5b8960e838a87a90ead0cMarkos Chandras while (tmp_flags) { 643c6610de353da5ca6eee5b8960e838a87a90ead0cMarkos Chandras if ((sflags >> i) & 0x1) { 644c6610de353da5ca6eee5b8960e838a87a90ead0cMarkos Chandras emit_load_stack_reg(MIPS_R_S0 + i, r_sp, real_off, 645c6610de353da5ca6eee5b8960e838a87a90ead0cMarkos Chandras ctx); 646c6610de353da5ca6eee5b8960e838a87a90ead0cMarkos Chandras real_off += RSIZE; 647c6610de353da5ca6eee5b8960e838a87a90ead0cMarkos Chandras } 648c6610de353da5ca6eee5b8960e838a87a90ead0cMarkos Chandras i++; 649c6610de353da5ca6eee5b8960e838a87a90ead0cMarkos Chandras tmp_flags >>= 1; 650c6610de353da5ca6eee5b8960e838a87a90ead0cMarkos Chandras } 651c6610de353da5ca6eee5b8960e838a87a90ead0cMarkos Chandras 652c6610de353da5ca6eee5b8960e838a87a90ead0cMarkos Chandras /* restore return address */ 653c6610de353da5ca6eee5b8960e838a87a90ead0cMarkos Chandras if (ctx->flags & SEEN_CALL) 654c6610de353da5ca6eee5b8960e838a87a90ead0cMarkos Chandras emit_load_stack_reg(r_ra, r_sp, real_off, ctx); 655c6610de353da5ca6eee5b8960e838a87a90ead0cMarkos Chandras 656c6610de353da5ca6eee5b8960e838a87a90ead0cMarkos Chandras /* Restore the sp and discard the scrach memory */ 657c6610de353da5ca6eee5b8960e838a87a90ead0cMarkos Chandras emit_stack_offset(align_sp(offset), ctx); 658c6610de353da5ca6eee5b8960e838a87a90ead0cMarkos Chandras} 659c6610de353da5ca6eee5b8960e838a87a90ead0cMarkos Chandras 660c6610de353da5ca6eee5b8960e838a87a90ead0cMarkos Chandrasstatic unsigned int get_stack_depth(struct jit_ctx *ctx) 661c6610de353da5ca6eee5b8960e838a87a90ead0cMarkos Chandras{ 662c6610de353da5ca6eee5b8960e838a87a90ead0cMarkos Chandras int sp_off = 0; 663c6610de353da5ca6eee5b8960e838a87a90ead0cMarkos Chandras 664c6610de353da5ca6eee5b8960e838a87a90ead0cMarkos Chandras 665c6610de353da5ca6eee5b8960e838a87a90ead0cMarkos Chandras /* How may s* regs do we need to preserved? */ 666c6610de353da5ca6eee5b8960e838a87a90ead0cMarkos Chandras sp_off += hweight32(ctx->flags >> SEEN_SREG_SFT) * RSIZE; 667c6610de353da5ca6eee5b8960e838a87a90ead0cMarkos Chandras 668c6610de353da5ca6eee5b8960e838a87a90ead0cMarkos Chandras if (ctx->flags & SEEN_MEM) 669c6610de353da5ca6eee5b8960e838a87a90ead0cMarkos Chandras sp_off += 4 * BPF_MEMWORDS; /* BPF_MEMWORDS are 32-bit */ 670c6610de353da5ca6eee5b8960e838a87a90ead0cMarkos Chandras 671c6610de353da5ca6eee5b8960e838a87a90ead0cMarkos Chandras if (ctx->flags & SEEN_CALL) 672c6610de353da5ca6eee5b8960e838a87a90ead0cMarkos Chandras /* 673c6610de353da5ca6eee5b8960e838a87a90ead0cMarkos Chandras * The JIT code make calls to external functions using 2 674c6610de353da5ca6eee5b8960e838a87a90ead0cMarkos Chandras * arguments. Therefore, for o32 we don't need to allocate 675c6610de353da5ca6eee5b8960e838a87a90ead0cMarkos Chandras * space because we don't care if the argumetns are lost 676c6610de353da5ca6eee5b8960e838a87a90ead0cMarkos Chandras * across calls. We do need however to preserve incoming 677c6610de353da5ca6eee5b8960e838a87a90ead0cMarkos Chandras * arguments but the space is already allocated for us by 678c6610de353da5ca6eee5b8960e838a87a90ead0cMarkos Chandras * the caller. On the other hand, for n64, we need to allocate 679c6610de353da5ca6eee5b8960e838a87a90ead0cMarkos Chandras * this space ourselves. We need to preserve $ra as well. 680c6610de353da5ca6eee5b8960e838a87a90ead0cMarkos Chandras */ 681c6610de353da5ca6eee5b8960e838a87a90ead0cMarkos Chandras sp_off += config_enabled(CONFIG_64BIT) ? 682c6610de353da5ca6eee5b8960e838a87a90ead0cMarkos Chandras (ARGS_USED_BY_JIT + 1) * RSIZE : RSIZE; 683c6610de353da5ca6eee5b8960e838a87a90ead0cMarkos Chandras 684c6610de353da5ca6eee5b8960e838a87a90ead0cMarkos Chandras /* 685c6610de353da5ca6eee5b8960e838a87a90ead0cMarkos Chandras * Subtract the bytes for the last registers since we only care about 686c6610de353da5ca6eee5b8960e838a87a90ead0cMarkos Chandras * the location on the stack pointer. 687c6610de353da5ca6eee5b8960e838a87a90ead0cMarkos Chandras */ 688c6610de353da5ca6eee5b8960e838a87a90ead0cMarkos Chandras return sp_off - RSIZE; 689c6610de353da5ca6eee5b8960e838a87a90ead0cMarkos Chandras} 690c6610de353da5ca6eee5b8960e838a87a90ead0cMarkos Chandras 691c6610de353da5ca6eee5b8960e838a87a90ead0cMarkos Chandrasstatic void build_prologue(struct jit_ctx *ctx) 692c6610de353da5ca6eee5b8960e838a87a90ead0cMarkos Chandras{ 693c6610de353da5ca6eee5b8960e838a87a90ead0cMarkos Chandras u16 first_inst = ctx->skf->insns[0].code; 694c6610de353da5ca6eee5b8960e838a87a90ead0cMarkos Chandras int sp_off; 695c6610de353da5ca6eee5b8960e838a87a90ead0cMarkos Chandras 696c6610de353da5ca6eee5b8960e838a87a90ead0cMarkos Chandras /* Calculate the total offset for the stack pointer */ 697c6610de353da5ca6eee5b8960e838a87a90ead0cMarkos Chandras sp_off = get_stack_depth(ctx); 698c6610de353da5ca6eee5b8960e838a87a90ead0cMarkos Chandras save_bpf_jit_regs(ctx, sp_off); 699c6610de353da5ca6eee5b8960e838a87a90ead0cMarkos Chandras 700c6610de353da5ca6eee5b8960e838a87a90ead0cMarkos Chandras if (ctx->flags & SEEN_SKB) 701c6610de353da5ca6eee5b8960e838a87a90ead0cMarkos Chandras emit_reg_move(r_skb, MIPS_R_A0, ctx); 702c6610de353da5ca6eee5b8960e838a87a90ead0cMarkos Chandras 703e5bb48b0553d75918094c5a6f7b60a4359887218Markos Chandras if (ctx->flags & SEEN_X) 704c6610de353da5ca6eee5b8960e838a87a90ead0cMarkos Chandras emit_jit_reg_move(r_X, r_zero, ctx); 705c6610de353da5ca6eee5b8960e838a87a90ead0cMarkos Chandras 706c6610de353da5ca6eee5b8960e838a87a90ead0cMarkos Chandras /* Do not leak kernel data to userspace */ 707a83d081ed14e4281d2620d182c6044c0c21c551eDaniel Borkmann if ((first_inst != (BPF_RET | BPF_K)) && !(is_load_to_a(first_inst))) 708c6610de353da5ca6eee5b8960e838a87a90ead0cMarkos Chandras emit_jit_reg_move(r_A, r_zero, ctx); 709c6610de353da5ca6eee5b8960e838a87a90ead0cMarkos Chandras} 710c6610de353da5ca6eee5b8960e838a87a90ead0cMarkos Chandras 711c6610de353da5ca6eee5b8960e838a87a90ead0cMarkos Chandrasstatic void build_epilogue(struct jit_ctx *ctx) 712c6610de353da5ca6eee5b8960e838a87a90ead0cMarkos Chandras{ 713c6610de353da5ca6eee5b8960e838a87a90ead0cMarkos Chandras unsigned int sp_off; 714c6610de353da5ca6eee5b8960e838a87a90ead0cMarkos Chandras 715c6610de353da5ca6eee5b8960e838a87a90ead0cMarkos Chandras /* Calculate the total offset for the stack pointer */ 716c6610de353da5ca6eee5b8960e838a87a90ead0cMarkos Chandras 717c6610de353da5ca6eee5b8960e838a87a90ead0cMarkos Chandras sp_off = get_stack_depth(ctx); 718c6610de353da5ca6eee5b8960e838a87a90ead0cMarkos Chandras restore_bpf_jit_regs(ctx, sp_off); 719c6610de353da5ca6eee5b8960e838a87a90ead0cMarkos Chandras 720c6610de353da5ca6eee5b8960e838a87a90ead0cMarkos Chandras /* Return */ 721c6610de353da5ca6eee5b8960e838a87a90ead0cMarkos Chandras emit_jr(r_ra, ctx); 722c6610de353da5ca6eee5b8960e838a87a90ead0cMarkos Chandras emit_nop(ctx); 723c6610de353da5ca6eee5b8960e838a87a90ead0cMarkos Chandras} 724c6610de353da5ca6eee5b8960e838a87a90ead0cMarkos Chandras 725c6610de353da5ca6eee5b8960e838a87a90ead0cMarkos Chandrasstatic u64 jit_get_skb_b(struct sk_buff *skb, unsigned offset) 726c6610de353da5ca6eee5b8960e838a87a90ead0cMarkos Chandras{ 727c6610de353da5ca6eee5b8960e838a87a90ead0cMarkos Chandras u8 ret; 728c6610de353da5ca6eee5b8960e838a87a90ead0cMarkos Chandras int err; 729c6610de353da5ca6eee5b8960e838a87a90ead0cMarkos Chandras 730c6610de353da5ca6eee5b8960e838a87a90ead0cMarkos Chandras err = skb_copy_bits(skb, offset, &ret, 1); 731c6610de353da5ca6eee5b8960e838a87a90ead0cMarkos Chandras 732c6610de353da5ca6eee5b8960e838a87a90ead0cMarkos Chandras return (u64)err << 32 | ret; 733c6610de353da5ca6eee5b8960e838a87a90ead0cMarkos Chandras} 734c6610de353da5ca6eee5b8960e838a87a90ead0cMarkos Chandras 735c6610de353da5ca6eee5b8960e838a87a90ead0cMarkos Chandrasstatic u64 jit_get_skb_h(struct sk_buff *skb, unsigned offset) 736c6610de353da5ca6eee5b8960e838a87a90ead0cMarkos Chandras{ 737c6610de353da5ca6eee5b8960e838a87a90ead0cMarkos Chandras u16 ret; 738c6610de353da5ca6eee5b8960e838a87a90ead0cMarkos Chandras int err; 739c6610de353da5ca6eee5b8960e838a87a90ead0cMarkos Chandras 740c6610de353da5ca6eee5b8960e838a87a90ead0cMarkos Chandras err = skb_copy_bits(skb, offset, &ret, 2); 741c6610de353da5ca6eee5b8960e838a87a90ead0cMarkos Chandras 742c6610de353da5ca6eee5b8960e838a87a90ead0cMarkos Chandras return (u64)err << 32 | ntohs(ret); 743c6610de353da5ca6eee5b8960e838a87a90ead0cMarkos Chandras} 744c6610de353da5ca6eee5b8960e838a87a90ead0cMarkos Chandras 745c6610de353da5ca6eee5b8960e838a87a90ead0cMarkos Chandrasstatic u64 jit_get_skb_w(struct sk_buff *skb, unsigned offset) 746c6610de353da5ca6eee5b8960e838a87a90ead0cMarkos Chandras{ 747c6610de353da5ca6eee5b8960e838a87a90ead0cMarkos Chandras u32 ret; 748c6610de353da5ca6eee5b8960e838a87a90ead0cMarkos Chandras int err; 749c6610de353da5ca6eee5b8960e838a87a90ead0cMarkos Chandras 750c6610de353da5ca6eee5b8960e838a87a90ead0cMarkos Chandras err = skb_copy_bits(skb, offset, &ret, 4); 751c6610de353da5ca6eee5b8960e838a87a90ead0cMarkos Chandras 752c6610de353da5ca6eee5b8960e838a87a90ead0cMarkos Chandras return (u64)err << 32 | ntohl(ret); 753c6610de353da5ca6eee5b8960e838a87a90ead0cMarkos Chandras} 754c6610de353da5ca6eee5b8960e838a87a90ead0cMarkos Chandras 755c6610de353da5ca6eee5b8960e838a87a90ead0cMarkos Chandrasstatic int build_body(struct jit_ctx *ctx) 756c6610de353da5ca6eee5b8960e838a87a90ead0cMarkos Chandras{ 757c6610de353da5ca6eee5b8960e838a87a90ead0cMarkos Chandras void *load_func[] = {jit_get_skb_b, jit_get_skb_h, jit_get_skb_w}; 7587ae457c1e5b45a1b826fad9d62b32191d2bdcfdbAlexei Starovoitov const struct bpf_prog *prog = ctx->skf; 759c6610de353da5ca6eee5b8960e838a87a90ead0cMarkos Chandras const struct sock_filter *inst; 760c6610de353da5ca6eee5b8960e838a87a90ead0cMarkos Chandras unsigned int i, off, load_order, condt; 761c6610de353da5ca6eee5b8960e838a87a90ead0cMarkos Chandras u32 k, b_off __maybe_unused; 762c6610de353da5ca6eee5b8960e838a87a90ead0cMarkos Chandras 763c6610de353da5ca6eee5b8960e838a87a90ead0cMarkos Chandras for (i = 0; i < prog->len; i++) { 764a83d081ed14e4281d2620d182c6044c0c21c551eDaniel Borkmann u16 code; 765a83d081ed14e4281d2620d182c6044c0c21c551eDaniel Borkmann 766c6610de353da5ca6eee5b8960e838a87a90ead0cMarkos Chandras inst = &(prog->insns[i]); 767c6610de353da5ca6eee5b8960e838a87a90ead0cMarkos Chandras pr_debug("%s: code->0x%02x, jt->0x%x, jf->0x%x, k->0x%x\n", 768c6610de353da5ca6eee5b8960e838a87a90ead0cMarkos Chandras __func__, inst->code, inst->jt, inst->jf, inst->k); 769c6610de353da5ca6eee5b8960e838a87a90ead0cMarkos Chandras k = inst->k; 770a83d081ed14e4281d2620d182c6044c0c21c551eDaniel Borkmann code = bpf_anc_helper(inst); 771c6610de353da5ca6eee5b8960e838a87a90ead0cMarkos Chandras 772c6610de353da5ca6eee5b8960e838a87a90ead0cMarkos Chandras if (ctx->target == NULL) 773c6610de353da5ca6eee5b8960e838a87a90ead0cMarkos Chandras ctx->offsets[i] = ctx->idx * 4; 774c6610de353da5ca6eee5b8960e838a87a90ead0cMarkos Chandras 775a83d081ed14e4281d2620d182c6044c0c21c551eDaniel Borkmann switch (code) { 776a83d081ed14e4281d2620d182c6044c0c21c551eDaniel Borkmann case BPF_LD | BPF_IMM: 777c6610de353da5ca6eee5b8960e838a87a90ead0cMarkos Chandras /* A <- k ==> li r_A, k */ 778c6610de353da5ca6eee5b8960e838a87a90ead0cMarkos Chandras ctx->flags |= SEEN_A; 779c6610de353da5ca6eee5b8960e838a87a90ead0cMarkos Chandras emit_load_imm(r_A, k, ctx); 780c6610de353da5ca6eee5b8960e838a87a90ead0cMarkos Chandras break; 781a83d081ed14e4281d2620d182c6044c0c21c551eDaniel Borkmann case BPF_LD | BPF_W | BPF_LEN: 782c6610de353da5ca6eee5b8960e838a87a90ead0cMarkos Chandras BUILD_BUG_ON(FIELD_SIZEOF(struct sk_buff, len) != 4); 783c6610de353da5ca6eee5b8960e838a87a90ead0cMarkos Chandras /* A <- len ==> lw r_A, offset(skb) */ 784c6610de353da5ca6eee5b8960e838a87a90ead0cMarkos Chandras ctx->flags |= SEEN_SKB | SEEN_A; 785c6610de353da5ca6eee5b8960e838a87a90ead0cMarkos Chandras off = offsetof(struct sk_buff, len); 786c6610de353da5ca6eee5b8960e838a87a90ead0cMarkos Chandras emit_load(r_A, r_skb, off, ctx); 787c6610de353da5ca6eee5b8960e838a87a90ead0cMarkos Chandras break; 788a83d081ed14e4281d2620d182c6044c0c21c551eDaniel Borkmann case BPF_LD | BPF_MEM: 789c6610de353da5ca6eee5b8960e838a87a90ead0cMarkos Chandras /* A <- M[k] ==> lw r_A, offset(M) */ 790c6610de353da5ca6eee5b8960e838a87a90ead0cMarkos Chandras ctx->flags |= SEEN_MEM | SEEN_A; 791c6610de353da5ca6eee5b8960e838a87a90ead0cMarkos Chandras emit_load(r_A, r_M, SCRATCH_OFF(k), ctx); 792c6610de353da5ca6eee5b8960e838a87a90ead0cMarkos Chandras break; 793a83d081ed14e4281d2620d182c6044c0c21c551eDaniel Borkmann case BPF_LD | BPF_W | BPF_ABS: 794c6610de353da5ca6eee5b8960e838a87a90ead0cMarkos Chandras /* A <- P[k:4] */ 795c6610de353da5ca6eee5b8960e838a87a90ead0cMarkos Chandras load_order = 2; 796c6610de353da5ca6eee5b8960e838a87a90ead0cMarkos Chandras goto load; 797a83d081ed14e4281d2620d182c6044c0c21c551eDaniel Borkmann case BPF_LD | BPF_H | BPF_ABS: 798c6610de353da5ca6eee5b8960e838a87a90ead0cMarkos Chandras /* A <- P[k:2] */ 799c6610de353da5ca6eee5b8960e838a87a90ead0cMarkos Chandras load_order = 1; 800c6610de353da5ca6eee5b8960e838a87a90ead0cMarkos Chandras goto load; 801a83d081ed14e4281d2620d182c6044c0c21c551eDaniel Borkmann case BPF_LD | BPF_B | BPF_ABS: 802c6610de353da5ca6eee5b8960e838a87a90ead0cMarkos Chandras /* A <- P[k:1] */ 803c6610de353da5ca6eee5b8960e838a87a90ead0cMarkos Chandras load_order = 0; 804c6610de353da5ca6eee5b8960e838a87a90ead0cMarkos Chandrasload: 80555393ee535496f7db15f3b2e9d3cf418f772f71aMarkos Chandras /* the interpreter will deal with the negative K */ 80655393ee535496f7db15f3b2e9d3cf418f772f71aMarkos Chandras if ((int)k < 0) 80755393ee535496f7db15f3b2e9d3cf418f772f71aMarkos Chandras return -ENOTSUPP; 80855393ee535496f7db15f3b2e9d3cf418f772f71aMarkos Chandras 809c6610de353da5ca6eee5b8960e838a87a90ead0cMarkos Chandras emit_load_imm(r_off, k, ctx); 810c6610de353da5ca6eee5b8960e838a87a90ead0cMarkos Chandrasload_common: 81155393ee535496f7db15f3b2e9d3cf418f772f71aMarkos Chandras /* 81255393ee535496f7db15f3b2e9d3cf418f772f71aMarkos Chandras * We may got here from the indirect loads so 81355393ee535496f7db15f3b2e9d3cf418f772f71aMarkos Chandras * return if offset is negative. 81455393ee535496f7db15f3b2e9d3cf418f772f71aMarkos Chandras */ 81555393ee535496f7db15f3b2e9d3cf418f772f71aMarkos Chandras emit_slt(r_s0, r_off, r_zero, ctx); 81655393ee535496f7db15f3b2e9d3cf418f772f71aMarkos Chandras emit_bcond(MIPS_COND_NE, r_s0, r_zero, 81755393ee535496f7db15f3b2e9d3cf418f772f71aMarkos Chandras b_imm(prog->len, ctx), ctx); 81855393ee535496f7db15f3b2e9d3cf418f772f71aMarkos Chandras emit_reg_move(r_ret, r_zero, ctx); 81955393ee535496f7db15f3b2e9d3cf418f772f71aMarkos Chandras 820c6610de353da5ca6eee5b8960e838a87a90ead0cMarkos Chandras ctx->flags |= SEEN_CALL | SEEN_OFF | SEEN_S0 | 821c6610de353da5ca6eee5b8960e838a87a90ead0cMarkos Chandras SEEN_SKB | SEEN_A; 822c6610de353da5ca6eee5b8960e838a87a90ead0cMarkos Chandras 823c6610de353da5ca6eee5b8960e838a87a90ead0cMarkos Chandras emit_load_func(r_s0, (ptr)load_func[load_order], 824c6610de353da5ca6eee5b8960e838a87a90ead0cMarkos Chandras ctx); 825c6610de353da5ca6eee5b8960e838a87a90ead0cMarkos Chandras emit_reg_move(MIPS_R_A0, r_skb, ctx); 826c6610de353da5ca6eee5b8960e838a87a90ead0cMarkos Chandras emit_jalr(MIPS_R_RA, r_s0, ctx); 827c6610de353da5ca6eee5b8960e838a87a90ead0cMarkos Chandras /* Load second argument to delay slot */ 828c6610de353da5ca6eee5b8960e838a87a90ead0cMarkos Chandras emit_reg_move(MIPS_R_A1, r_off, ctx); 829c6610de353da5ca6eee5b8960e838a87a90ead0cMarkos Chandras /* Check the error value */ 830c6610de353da5ca6eee5b8960e838a87a90ead0cMarkos Chandras if (config_enabled(CONFIG_64BIT)) { 831c6610de353da5ca6eee5b8960e838a87a90ead0cMarkos Chandras /* Get error code from the top 32-bits */ 832c6610de353da5ca6eee5b8960e838a87a90ead0cMarkos Chandras emit_dsrl32(r_s0, r_val, 0, ctx); 833c6610de353da5ca6eee5b8960e838a87a90ead0cMarkos Chandras /* Branch to 3 instructions ahead */ 834c6610de353da5ca6eee5b8960e838a87a90ead0cMarkos Chandras emit_bcond(MIPS_COND_NE, r_s0, r_zero, 3 << 2, 835c6610de353da5ca6eee5b8960e838a87a90ead0cMarkos Chandras ctx); 836c6610de353da5ca6eee5b8960e838a87a90ead0cMarkos Chandras } else { 837c6610de353da5ca6eee5b8960e838a87a90ead0cMarkos Chandras /* Branch to 3 instructions ahead */ 838c6610de353da5ca6eee5b8960e838a87a90ead0cMarkos Chandras emit_bcond(MIPS_COND_NE, r_err, r_zero, 3 << 2, 839c6610de353da5ca6eee5b8960e838a87a90ead0cMarkos Chandras ctx); 840c6610de353da5ca6eee5b8960e838a87a90ead0cMarkos Chandras } 841c6610de353da5ca6eee5b8960e838a87a90ead0cMarkos Chandras emit_nop(ctx); 842c6610de353da5ca6eee5b8960e838a87a90ead0cMarkos Chandras /* We are good */ 843c6610de353da5ca6eee5b8960e838a87a90ead0cMarkos Chandras emit_b(b_imm(i + 1, ctx), ctx); 844c6610de353da5ca6eee5b8960e838a87a90ead0cMarkos Chandras emit_jit_reg_move(r_A, r_val, ctx); 845c6610de353da5ca6eee5b8960e838a87a90ead0cMarkos Chandras /* Return with error */ 846c6610de353da5ca6eee5b8960e838a87a90ead0cMarkos Chandras emit_b(b_imm(prog->len, ctx), ctx); 847c6610de353da5ca6eee5b8960e838a87a90ead0cMarkos Chandras emit_reg_move(r_ret, r_zero, ctx); 848c6610de353da5ca6eee5b8960e838a87a90ead0cMarkos Chandras break; 849a83d081ed14e4281d2620d182c6044c0c21c551eDaniel Borkmann case BPF_LD | BPF_W | BPF_IND: 850c6610de353da5ca6eee5b8960e838a87a90ead0cMarkos Chandras /* A <- P[X + k:4] */ 851c6610de353da5ca6eee5b8960e838a87a90ead0cMarkos Chandras load_order = 2; 852c6610de353da5ca6eee5b8960e838a87a90ead0cMarkos Chandras goto load_ind; 853a83d081ed14e4281d2620d182c6044c0c21c551eDaniel Borkmann case BPF_LD | BPF_H | BPF_IND: 854c6610de353da5ca6eee5b8960e838a87a90ead0cMarkos Chandras /* A <- P[X + k:2] */ 855c6610de353da5ca6eee5b8960e838a87a90ead0cMarkos Chandras load_order = 1; 856c6610de353da5ca6eee5b8960e838a87a90ead0cMarkos Chandras goto load_ind; 857a83d081ed14e4281d2620d182c6044c0c21c551eDaniel Borkmann case BPF_LD | BPF_B | BPF_IND: 858c6610de353da5ca6eee5b8960e838a87a90ead0cMarkos Chandras /* A <- P[X + k:1] */ 859c6610de353da5ca6eee5b8960e838a87a90ead0cMarkos Chandras load_order = 0; 860c6610de353da5ca6eee5b8960e838a87a90ead0cMarkos Chandrasload_ind: 861c6610de353da5ca6eee5b8960e838a87a90ead0cMarkos Chandras ctx->flags |= SEEN_OFF | SEEN_X; 862c6610de353da5ca6eee5b8960e838a87a90ead0cMarkos Chandras emit_addiu(r_off, r_X, k, ctx); 863c6610de353da5ca6eee5b8960e838a87a90ead0cMarkos Chandras goto load_common; 864a83d081ed14e4281d2620d182c6044c0c21c551eDaniel Borkmann case BPF_LDX | BPF_IMM: 865c6610de353da5ca6eee5b8960e838a87a90ead0cMarkos Chandras /* X <- k */ 866c6610de353da5ca6eee5b8960e838a87a90ead0cMarkos Chandras ctx->flags |= SEEN_X; 867c6610de353da5ca6eee5b8960e838a87a90ead0cMarkos Chandras emit_load_imm(r_X, k, ctx); 868c6610de353da5ca6eee5b8960e838a87a90ead0cMarkos Chandras break; 869a83d081ed14e4281d2620d182c6044c0c21c551eDaniel Borkmann case BPF_LDX | BPF_MEM: 870c6610de353da5ca6eee5b8960e838a87a90ead0cMarkos Chandras /* X <- M[k] */ 871c6610de353da5ca6eee5b8960e838a87a90ead0cMarkos Chandras ctx->flags |= SEEN_X | SEEN_MEM; 872c6610de353da5ca6eee5b8960e838a87a90ead0cMarkos Chandras emit_load(r_X, r_M, SCRATCH_OFF(k), ctx); 873c6610de353da5ca6eee5b8960e838a87a90ead0cMarkos Chandras break; 874a83d081ed14e4281d2620d182c6044c0c21c551eDaniel Borkmann case BPF_LDX | BPF_W | BPF_LEN: 875c6610de353da5ca6eee5b8960e838a87a90ead0cMarkos Chandras /* X <- len */ 876c6610de353da5ca6eee5b8960e838a87a90ead0cMarkos Chandras ctx->flags |= SEEN_X | SEEN_SKB; 877c6610de353da5ca6eee5b8960e838a87a90ead0cMarkos Chandras off = offsetof(struct sk_buff, len); 878c6610de353da5ca6eee5b8960e838a87a90ead0cMarkos Chandras emit_load(r_X, r_skb, off, ctx); 879c6610de353da5ca6eee5b8960e838a87a90ead0cMarkos Chandras break; 880a83d081ed14e4281d2620d182c6044c0c21c551eDaniel Borkmann case BPF_LDX | BPF_B | BPF_MSH: 88155393ee535496f7db15f3b2e9d3cf418f772f71aMarkos Chandras /* the interpreter will deal with the negative K */ 88255393ee535496f7db15f3b2e9d3cf418f772f71aMarkos Chandras if ((int)k < 0) 88355393ee535496f7db15f3b2e9d3cf418f772f71aMarkos Chandras return -ENOTSUPP; 88455393ee535496f7db15f3b2e9d3cf418f772f71aMarkos Chandras 885c6610de353da5ca6eee5b8960e838a87a90ead0cMarkos Chandras /* X <- 4 * (P[k:1] & 0xf) */ 886c6610de353da5ca6eee5b8960e838a87a90ead0cMarkos Chandras ctx->flags |= SEEN_X | SEEN_CALL | SEEN_S0 | SEEN_SKB; 887c6610de353da5ca6eee5b8960e838a87a90ead0cMarkos Chandras /* Load offset to a1 */ 888c6610de353da5ca6eee5b8960e838a87a90ead0cMarkos Chandras emit_load_func(r_s0, (ptr)jit_get_skb_b, ctx); 889c6610de353da5ca6eee5b8960e838a87a90ead0cMarkos Chandras /* 890c6610de353da5ca6eee5b8960e838a87a90ead0cMarkos Chandras * This may emit two instructions so it may not fit 891c6610de353da5ca6eee5b8960e838a87a90ead0cMarkos Chandras * in the delay slot. So use a0 in the delay slot. 892c6610de353da5ca6eee5b8960e838a87a90ead0cMarkos Chandras */ 893c6610de353da5ca6eee5b8960e838a87a90ead0cMarkos Chandras emit_load_imm(MIPS_R_A1, k, ctx); 894c6610de353da5ca6eee5b8960e838a87a90ead0cMarkos Chandras emit_jalr(MIPS_R_RA, r_s0, ctx); 895c6610de353da5ca6eee5b8960e838a87a90ead0cMarkos Chandras emit_reg_move(MIPS_R_A0, r_skb, ctx); /* delay slot */ 896c6610de353da5ca6eee5b8960e838a87a90ead0cMarkos Chandras /* Check the error value */ 897c6610de353da5ca6eee5b8960e838a87a90ead0cMarkos Chandras if (config_enabled(CONFIG_64BIT)) { 898c6610de353da5ca6eee5b8960e838a87a90ead0cMarkos Chandras /* Top 32-bits of $v0 on 64-bit */ 899c6610de353da5ca6eee5b8960e838a87a90ead0cMarkos Chandras emit_dsrl32(r_s0, r_val, 0, ctx); 900c6610de353da5ca6eee5b8960e838a87a90ead0cMarkos Chandras emit_bcond(MIPS_COND_NE, r_s0, r_zero, 901c6610de353da5ca6eee5b8960e838a87a90ead0cMarkos Chandras 3 << 2, ctx); 902c6610de353da5ca6eee5b8960e838a87a90ead0cMarkos Chandras } else { 903c6610de353da5ca6eee5b8960e838a87a90ead0cMarkos Chandras emit_bcond(MIPS_COND_NE, r_err, r_zero, 904c6610de353da5ca6eee5b8960e838a87a90ead0cMarkos Chandras 3 << 2, ctx); 905c6610de353da5ca6eee5b8960e838a87a90ead0cMarkos Chandras } 906c6610de353da5ca6eee5b8960e838a87a90ead0cMarkos Chandras /* No need for delay slot */ 907c6610de353da5ca6eee5b8960e838a87a90ead0cMarkos Chandras /* We are good */ 908c6610de353da5ca6eee5b8960e838a87a90ead0cMarkos Chandras /* X <- P[1:K] & 0xf */ 909c6610de353da5ca6eee5b8960e838a87a90ead0cMarkos Chandras emit_andi(r_X, r_val, 0xf, ctx); 910c6610de353da5ca6eee5b8960e838a87a90ead0cMarkos Chandras /* X << 2 */ 911c6610de353da5ca6eee5b8960e838a87a90ead0cMarkos Chandras emit_b(b_imm(i + 1, ctx), ctx); 912c6610de353da5ca6eee5b8960e838a87a90ead0cMarkos Chandras emit_sll(r_X, r_X, 2, ctx); /* delay slot */ 913c6610de353da5ca6eee5b8960e838a87a90ead0cMarkos Chandras /* Return with error */ 914c6610de353da5ca6eee5b8960e838a87a90ead0cMarkos Chandras emit_b(b_imm(prog->len, ctx), ctx); 915c6610de353da5ca6eee5b8960e838a87a90ead0cMarkos Chandras emit_load_imm(r_ret, 0, ctx); /* delay slot */ 916c6610de353da5ca6eee5b8960e838a87a90ead0cMarkos Chandras break; 917a83d081ed14e4281d2620d182c6044c0c21c551eDaniel Borkmann case BPF_ST: 918c6610de353da5ca6eee5b8960e838a87a90ead0cMarkos Chandras /* M[k] <- A */ 919c6610de353da5ca6eee5b8960e838a87a90ead0cMarkos Chandras ctx->flags |= SEEN_MEM | SEEN_A; 920c6610de353da5ca6eee5b8960e838a87a90ead0cMarkos Chandras emit_store(r_A, r_M, SCRATCH_OFF(k), ctx); 921c6610de353da5ca6eee5b8960e838a87a90ead0cMarkos Chandras break; 922a83d081ed14e4281d2620d182c6044c0c21c551eDaniel Borkmann case BPF_STX: 923c6610de353da5ca6eee5b8960e838a87a90ead0cMarkos Chandras /* M[k] <- X */ 924c6610de353da5ca6eee5b8960e838a87a90ead0cMarkos Chandras ctx->flags |= SEEN_MEM | SEEN_X; 925c6610de353da5ca6eee5b8960e838a87a90ead0cMarkos Chandras emit_store(r_X, r_M, SCRATCH_OFF(k), ctx); 926c6610de353da5ca6eee5b8960e838a87a90ead0cMarkos Chandras break; 927a83d081ed14e4281d2620d182c6044c0c21c551eDaniel Borkmann case BPF_ALU | BPF_ADD | BPF_K: 928c6610de353da5ca6eee5b8960e838a87a90ead0cMarkos Chandras /* A += K */ 929c6610de353da5ca6eee5b8960e838a87a90ead0cMarkos Chandras ctx->flags |= SEEN_A; 930c6610de353da5ca6eee5b8960e838a87a90ead0cMarkos Chandras emit_addiu(r_A, r_A, k, ctx); 931c6610de353da5ca6eee5b8960e838a87a90ead0cMarkos Chandras break; 932a83d081ed14e4281d2620d182c6044c0c21c551eDaniel Borkmann case BPF_ALU | BPF_ADD | BPF_X: 933c6610de353da5ca6eee5b8960e838a87a90ead0cMarkos Chandras /* A += X */ 934c6610de353da5ca6eee5b8960e838a87a90ead0cMarkos Chandras ctx->flags |= SEEN_A | SEEN_X; 935c6610de353da5ca6eee5b8960e838a87a90ead0cMarkos Chandras emit_addu(r_A, r_A, r_X, ctx); 936c6610de353da5ca6eee5b8960e838a87a90ead0cMarkos Chandras break; 937a83d081ed14e4281d2620d182c6044c0c21c551eDaniel Borkmann case BPF_ALU | BPF_SUB | BPF_K: 938c6610de353da5ca6eee5b8960e838a87a90ead0cMarkos Chandras /* A -= K */ 939c6610de353da5ca6eee5b8960e838a87a90ead0cMarkos Chandras ctx->flags |= SEEN_A; 940c6610de353da5ca6eee5b8960e838a87a90ead0cMarkos Chandras emit_addiu(r_A, r_A, -k, ctx); 941c6610de353da5ca6eee5b8960e838a87a90ead0cMarkos Chandras break; 942a83d081ed14e4281d2620d182c6044c0c21c551eDaniel Borkmann case BPF_ALU | BPF_SUB | BPF_X: 943c6610de353da5ca6eee5b8960e838a87a90ead0cMarkos Chandras /* A -= X */ 944c6610de353da5ca6eee5b8960e838a87a90ead0cMarkos Chandras ctx->flags |= SEEN_A | SEEN_X; 945c6610de353da5ca6eee5b8960e838a87a90ead0cMarkos Chandras emit_subu(r_A, r_A, r_X, ctx); 946c6610de353da5ca6eee5b8960e838a87a90ead0cMarkos Chandras break; 947a83d081ed14e4281d2620d182c6044c0c21c551eDaniel Borkmann case BPF_ALU | BPF_MUL | BPF_K: 948c6610de353da5ca6eee5b8960e838a87a90ead0cMarkos Chandras /* A *= K */ 949c6610de353da5ca6eee5b8960e838a87a90ead0cMarkos Chandras /* Load K to scratch register before MUL */ 950c6610de353da5ca6eee5b8960e838a87a90ead0cMarkos Chandras ctx->flags |= SEEN_A | SEEN_S0; 951c6610de353da5ca6eee5b8960e838a87a90ead0cMarkos Chandras emit_load_imm(r_s0, k, ctx); 952c6610de353da5ca6eee5b8960e838a87a90ead0cMarkos Chandras emit_mul(r_A, r_A, r_s0, ctx); 953c6610de353da5ca6eee5b8960e838a87a90ead0cMarkos Chandras break; 954a83d081ed14e4281d2620d182c6044c0c21c551eDaniel Borkmann case BPF_ALU | BPF_MUL | BPF_X: 955c6610de353da5ca6eee5b8960e838a87a90ead0cMarkos Chandras /* A *= X */ 956c6610de353da5ca6eee5b8960e838a87a90ead0cMarkos Chandras ctx->flags |= SEEN_A | SEEN_X; 957c6610de353da5ca6eee5b8960e838a87a90ead0cMarkos Chandras emit_mul(r_A, r_A, r_X, ctx); 958c6610de353da5ca6eee5b8960e838a87a90ead0cMarkos Chandras break; 959a83d081ed14e4281d2620d182c6044c0c21c551eDaniel Borkmann case BPF_ALU | BPF_DIV | BPF_K: 960c6610de353da5ca6eee5b8960e838a87a90ead0cMarkos Chandras /* A /= k */ 961c6610de353da5ca6eee5b8960e838a87a90ead0cMarkos Chandras if (k == 1) 962c6610de353da5ca6eee5b8960e838a87a90ead0cMarkos Chandras break; 963c6610de353da5ca6eee5b8960e838a87a90ead0cMarkos Chandras if (optimize_div(&k)) { 964c6610de353da5ca6eee5b8960e838a87a90ead0cMarkos Chandras ctx->flags |= SEEN_A; 965c6610de353da5ca6eee5b8960e838a87a90ead0cMarkos Chandras emit_srl(r_A, r_A, k, ctx); 966c6610de353da5ca6eee5b8960e838a87a90ead0cMarkos Chandras break; 967c6610de353da5ca6eee5b8960e838a87a90ead0cMarkos Chandras } 968c6610de353da5ca6eee5b8960e838a87a90ead0cMarkos Chandras ctx->flags |= SEEN_A | SEEN_S0; 969c6610de353da5ca6eee5b8960e838a87a90ead0cMarkos Chandras emit_load_imm(r_s0, k, ctx); 970c6610de353da5ca6eee5b8960e838a87a90ead0cMarkos Chandras emit_div(r_A, r_s0, ctx); 971c6610de353da5ca6eee5b8960e838a87a90ead0cMarkos Chandras break; 972a83d081ed14e4281d2620d182c6044c0c21c551eDaniel Borkmann case BPF_ALU | BPF_MOD | BPF_K: 973c6610de353da5ca6eee5b8960e838a87a90ead0cMarkos Chandras /* A %= k */ 974c6610de353da5ca6eee5b8960e838a87a90ead0cMarkos Chandras if (k == 1 || optimize_div(&k)) { 975c6610de353da5ca6eee5b8960e838a87a90ead0cMarkos Chandras ctx->flags |= SEEN_A; 976c6610de353da5ca6eee5b8960e838a87a90ead0cMarkos Chandras emit_jit_reg_move(r_A, r_zero, ctx); 977c6610de353da5ca6eee5b8960e838a87a90ead0cMarkos Chandras } else { 978c6610de353da5ca6eee5b8960e838a87a90ead0cMarkos Chandras ctx->flags |= SEEN_A | SEEN_S0; 979c6610de353da5ca6eee5b8960e838a87a90ead0cMarkos Chandras emit_load_imm(r_s0, k, ctx); 980c6610de353da5ca6eee5b8960e838a87a90ead0cMarkos Chandras emit_mod(r_A, r_s0, ctx); 981c6610de353da5ca6eee5b8960e838a87a90ead0cMarkos Chandras } 982c6610de353da5ca6eee5b8960e838a87a90ead0cMarkos Chandras break; 983a83d081ed14e4281d2620d182c6044c0c21c551eDaniel Borkmann case BPF_ALU | BPF_DIV | BPF_X: 984c6610de353da5ca6eee5b8960e838a87a90ead0cMarkos Chandras /* A /= X */ 985c6610de353da5ca6eee5b8960e838a87a90ead0cMarkos Chandras ctx->flags |= SEEN_X | SEEN_A; 986c6610de353da5ca6eee5b8960e838a87a90ead0cMarkos Chandras /* Check if r_X is zero */ 987c6610de353da5ca6eee5b8960e838a87a90ead0cMarkos Chandras emit_bcond(MIPS_COND_EQ, r_X, r_zero, 988c6610de353da5ca6eee5b8960e838a87a90ead0cMarkos Chandras b_imm(prog->len, ctx), ctx); 989c6610de353da5ca6eee5b8960e838a87a90ead0cMarkos Chandras emit_load_imm(r_val, 0, ctx); /* delay slot */ 990c6610de353da5ca6eee5b8960e838a87a90ead0cMarkos Chandras emit_div(r_A, r_X, ctx); 991c6610de353da5ca6eee5b8960e838a87a90ead0cMarkos Chandras break; 992a83d081ed14e4281d2620d182c6044c0c21c551eDaniel Borkmann case BPF_ALU | BPF_MOD | BPF_X: 993c6610de353da5ca6eee5b8960e838a87a90ead0cMarkos Chandras /* A %= X */ 994c6610de353da5ca6eee5b8960e838a87a90ead0cMarkos Chandras ctx->flags |= SEEN_X | SEEN_A; 995c6610de353da5ca6eee5b8960e838a87a90ead0cMarkos Chandras /* Check if r_X is zero */ 996c6610de353da5ca6eee5b8960e838a87a90ead0cMarkos Chandras emit_bcond(MIPS_COND_EQ, r_X, r_zero, 997c6610de353da5ca6eee5b8960e838a87a90ead0cMarkos Chandras b_imm(prog->len, ctx), ctx); 998c6610de353da5ca6eee5b8960e838a87a90ead0cMarkos Chandras emit_load_imm(r_val, 0, ctx); /* delay slot */ 999c6610de353da5ca6eee5b8960e838a87a90ead0cMarkos Chandras emit_mod(r_A, r_X, ctx); 1000c6610de353da5ca6eee5b8960e838a87a90ead0cMarkos Chandras break; 1001a83d081ed14e4281d2620d182c6044c0c21c551eDaniel Borkmann case BPF_ALU | BPF_OR | BPF_K: 1002c6610de353da5ca6eee5b8960e838a87a90ead0cMarkos Chandras /* A |= K */ 1003c6610de353da5ca6eee5b8960e838a87a90ead0cMarkos Chandras ctx->flags |= SEEN_A; 1004c6610de353da5ca6eee5b8960e838a87a90ead0cMarkos Chandras emit_ori(r_A, r_A, k, ctx); 1005c6610de353da5ca6eee5b8960e838a87a90ead0cMarkos Chandras break; 1006a83d081ed14e4281d2620d182c6044c0c21c551eDaniel Borkmann case BPF_ALU | BPF_OR | BPF_X: 1007c6610de353da5ca6eee5b8960e838a87a90ead0cMarkos Chandras /* A |= X */ 1008c6610de353da5ca6eee5b8960e838a87a90ead0cMarkos Chandras ctx->flags |= SEEN_A; 1009c6610de353da5ca6eee5b8960e838a87a90ead0cMarkos Chandras emit_ori(r_A, r_A, r_X, ctx); 1010c6610de353da5ca6eee5b8960e838a87a90ead0cMarkos Chandras break; 1011a83d081ed14e4281d2620d182c6044c0c21c551eDaniel Borkmann case BPF_ALU | BPF_XOR | BPF_K: 1012c6610de353da5ca6eee5b8960e838a87a90ead0cMarkos Chandras /* A ^= k */ 1013c6610de353da5ca6eee5b8960e838a87a90ead0cMarkos Chandras ctx->flags |= SEEN_A; 1014c6610de353da5ca6eee5b8960e838a87a90ead0cMarkos Chandras emit_xori(r_A, r_A, k, ctx); 1015c6610de353da5ca6eee5b8960e838a87a90ead0cMarkos Chandras break; 1016a83d081ed14e4281d2620d182c6044c0c21c551eDaniel Borkmann case BPF_ANC | SKF_AD_ALU_XOR_X: 1017a83d081ed14e4281d2620d182c6044c0c21c551eDaniel Borkmann case BPF_ALU | BPF_XOR | BPF_X: 1018c6610de353da5ca6eee5b8960e838a87a90ead0cMarkos Chandras /* A ^= X */ 1019c6610de353da5ca6eee5b8960e838a87a90ead0cMarkos Chandras ctx->flags |= SEEN_A; 1020c6610de353da5ca6eee5b8960e838a87a90ead0cMarkos Chandras emit_xor(r_A, r_A, r_X, ctx); 1021c6610de353da5ca6eee5b8960e838a87a90ead0cMarkos Chandras break; 1022a83d081ed14e4281d2620d182c6044c0c21c551eDaniel Borkmann case BPF_ALU | BPF_AND | BPF_K: 1023c6610de353da5ca6eee5b8960e838a87a90ead0cMarkos Chandras /* A &= K */ 1024c6610de353da5ca6eee5b8960e838a87a90ead0cMarkos Chandras ctx->flags |= SEEN_A; 1025c6610de353da5ca6eee5b8960e838a87a90ead0cMarkos Chandras emit_andi(r_A, r_A, k, ctx); 1026c6610de353da5ca6eee5b8960e838a87a90ead0cMarkos Chandras break; 1027a83d081ed14e4281d2620d182c6044c0c21c551eDaniel Borkmann case BPF_ALU | BPF_AND | BPF_X: 1028c6610de353da5ca6eee5b8960e838a87a90ead0cMarkos Chandras /* A &= X */ 1029c6610de353da5ca6eee5b8960e838a87a90ead0cMarkos Chandras ctx->flags |= SEEN_A | SEEN_X; 1030c6610de353da5ca6eee5b8960e838a87a90ead0cMarkos Chandras emit_and(r_A, r_A, r_X, ctx); 1031c6610de353da5ca6eee5b8960e838a87a90ead0cMarkos Chandras break; 1032a83d081ed14e4281d2620d182c6044c0c21c551eDaniel Borkmann case BPF_ALU | BPF_LSH | BPF_K: 1033c6610de353da5ca6eee5b8960e838a87a90ead0cMarkos Chandras /* A <<= K */ 1034c6610de353da5ca6eee5b8960e838a87a90ead0cMarkos Chandras ctx->flags |= SEEN_A; 1035c6610de353da5ca6eee5b8960e838a87a90ead0cMarkos Chandras emit_sll(r_A, r_A, k, ctx); 1036c6610de353da5ca6eee5b8960e838a87a90ead0cMarkos Chandras break; 1037a83d081ed14e4281d2620d182c6044c0c21c551eDaniel Borkmann case BPF_ALU | BPF_LSH | BPF_X: 1038c6610de353da5ca6eee5b8960e838a87a90ead0cMarkos Chandras /* A <<= X */ 1039c6610de353da5ca6eee5b8960e838a87a90ead0cMarkos Chandras ctx->flags |= SEEN_A | SEEN_X; 1040c6610de353da5ca6eee5b8960e838a87a90ead0cMarkos Chandras emit_sllv(r_A, r_A, r_X, ctx); 1041c6610de353da5ca6eee5b8960e838a87a90ead0cMarkos Chandras break; 1042a83d081ed14e4281d2620d182c6044c0c21c551eDaniel Borkmann case BPF_ALU | BPF_RSH | BPF_K: 1043c6610de353da5ca6eee5b8960e838a87a90ead0cMarkos Chandras /* A >>= K */ 1044c6610de353da5ca6eee5b8960e838a87a90ead0cMarkos Chandras ctx->flags |= SEEN_A; 1045c6610de353da5ca6eee5b8960e838a87a90ead0cMarkos Chandras emit_srl(r_A, r_A, k, ctx); 1046c6610de353da5ca6eee5b8960e838a87a90ead0cMarkos Chandras break; 1047a83d081ed14e4281d2620d182c6044c0c21c551eDaniel Borkmann case BPF_ALU | BPF_RSH | BPF_X: 1048c6610de353da5ca6eee5b8960e838a87a90ead0cMarkos Chandras ctx->flags |= SEEN_A | SEEN_X; 1049c6610de353da5ca6eee5b8960e838a87a90ead0cMarkos Chandras emit_srlv(r_A, r_A, r_X, ctx); 1050c6610de353da5ca6eee5b8960e838a87a90ead0cMarkos Chandras break; 1051a83d081ed14e4281d2620d182c6044c0c21c551eDaniel Borkmann case BPF_ALU | BPF_NEG: 1052c6610de353da5ca6eee5b8960e838a87a90ead0cMarkos Chandras /* A = -A */ 1053c6610de353da5ca6eee5b8960e838a87a90ead0cMarkos Chandras ctx->flags |= SEEN_A; 1054c6610de353da5ca6eee5b8960e838a87a90ead0cMarkos Chandras emit_neg(r_A, ctx); 1055c6610de353da5ca6eee5b8960e838a87a90ead0cMarkos Chandras break; 1056a83d081ed14e4281d2620d182c6044c0c21c551eDaniel Borkmann case BPF_JMP | BPF_JA: 1057c6610de353da5ca6eee5b8960e838a87a90ead0cMarkos Chandras /* pc += K */ 1058c6610de353da5ca6eee5b8960e838a87a90ead0cMarkos Chandras emit_b(b_imm(i + k + 1, ctx), ctx); 1059c6610de353da5ca6eee5b8960e838a87a90ead0cMarkos Chandras emit_nop(ctx); 1060c6610de353da5ca6eee5b8960e838a87a90ead0cMarkos Chandras break; 1061a83d081ed14e4281d2620d182c6044c0c21c551eDaniel Borkmann case BPF_JMP | BPF_JEQ | BPF_K: 1062c6610de353da5ca6eee5b8960e838a87a90ead0cMarkos Chandras /* pc += ( A == K ) ? pc->jt : pc->jf */ 1063c6610de353da5ca6eee5b8960e838a87a90ead0cMarkos Chandras condt = MIPS_COND_EQ | MIPS_COND_K; 1064c6610de353da5ca6eee5b8960e838a87a90ead0cMarkos Chandras goto jmp_cmp; 1065a83d081ed14e4281d2620d182c6044c0c21c551eDaniel Borkmann case BPF_JMP | BPF_JEQ | BPF_X: 1066c6610de353da5ca6eee5b8960e838a87a90ead0cMarkos Chandras ctx->flags |= SEEN_X; 1067c6610de353da5ca6eee5b8960e838a87a90ead0cMarkos Chandras /* pc += ( A == X ) ? pc->jt : pc->jf */ 1068c6610de353da5ca6eee5b8960e838a87a90ead0cMarkos Chandras condt = MIPS_COND_EQ | MIPS_COND_X; 1069c6610de353da5ca6eee5b8960e838a87a90ead0cMarkos Chandras goto jmp_cmp; 1070a83d081ed14e4281d2620d182c6044c0c21c551eDaniel Borkmann case BPF_JMP | BPF_JGE | BPF_K: 1071c6610de353da5ca6eee5b8960e838a87a90ead0cMarkos Chandras /* pc += ( A >= K ) ? pc->jt : pc->jf */ 1072c6610de353da5ca6eee5b8960e838a87a90ead0cMarkos Chandras condt = MIPS_COND_GE | MIPS_COND_K; 1073c6610de353da5ca6eee5b8960e838a87a90ead0cMarkos Chandras goto jmp_cmp; 1074a83d081ed14e4281d2620d182c6044c0c21c551eDaniel Borkmann case BPF_JMP | BPF_JGE | BPF_X: 1075c6610de353da5ca6eee5b8960e838a87a90ead0cMarkos Chandras ctx->flags |= SEEN_X; 1076c6610de353da5ca6eee5b8960e838a87a90ead0cMarkos Chandras /* pc += ( A >= X ) ? pc->jt : pc->jf */ 1077c6610de353da5ca6eee5b8960e838a87a90ead0cMarkos Chandras condt = MIPS_COND_GE | MIPS_COND_X; 1078c6610de353da5ca6eee5b8960e838a87a90ead0cMarkos Chandras goto jmp_cmp; 1079a83d081ed14e4281d2620d182c6044c0c21c551eDaniel Borkmann case BPF_JMP | BPF_JGT | BPF_K: 1080c6610de353da5ca6eee5b8960e838a87a90ead0cMarkos Chandras /* pc += ( A > K ) ? pc->jt : pc->jf */ 1081c6610de353da5ca6eee5b8960e838a87a90ead0cMarkos Chandras condt = MIPS_COND_GT | MIPS_COND_K; 1082c6610de353da5ca6eee5b8960e838a87a90ead0cMarkos Chandras goto jmp_cmp; 1083a83d081ed14e4281d2620d182c6044c0c21c551eDaniel Borkmann case BPF_JMP | BPF_JGT | BPF_X: 1084c6610de353da5ca6eee5b8960e838a87a90ead0cMarkos Chandras ctx->flags |= SEEN_X; 1085c6610de353da5ca6eee5b8960e838a87a90ead0cMarkos Chandras /* pc += ( A > X ) ? pc->jt : pc->jf */ 1086c6610de353da5ca6eee5b8960e838a87a90ead0cMarkos Chandras condt = MIPS_COND_GT | MIPS_COND_X; 1087c6610de353da5ca6eee5b8960e838a87a90ead0cMarkos Chandrasjmp_cmp: 1088c6610de353da5ca6eee5b8960e838a87a90ead0cMarkos Chandras /* Greater or Equal */ 1089c6610de353da5ca6eee5b8960e838a87a90ead0cMarkos Chandras if ((condt & MIPS_COND_GE) || 1090c6610de353da5ca6eee5b8960e838a87a90ead0cMarkos Chandras (condt & MIPS_COND_GT)) { 1091c6610de353da5ca6eee5b8960e838a87a90ead0cMarkos Chandras if (condt & MIPS_COND_K) { /* K */ 1092c6610de353da5ca6eee5b8960e838a87a90ead0cMarkos Chandras ctx->flags |= SEEN_S0 | SEEN_A; 1093c6610de353da5ca6eee5b8960e838a87a90ead0cMarkos Chandras emit_sltiu(r_s0, r_A, k, ctx); 1094c6610de353da5ca6eee5b8960e838a87a90ead0cMarkos Chandras } else { /* X */ 1095c6610de353da5ca6eee5b8960e838a87a90ead0cMarkos Chandras ctx->flags |= SEEN_S0 | SEEN_A | 1096c6610de353da5ca6eee5b8960e838a87a90ead0cMarkos Chandras SEEN_X; 1097c6610de353da5ca6eee5b8960e838a87a90ead0cMarkos Chandras emit_sltu(r_s0, r_A, r_X, ctx); 1098c6610de353da5ca6eee5b8960e838a87a90ead0cMarkos Chandras } 1099c6610de353da5ca6eee5b8960e838a87a90ead0cMarkos Chandras /* A < (K|X) ? r_scrach = 1 */ 1100c6610de353da5ca6eee5b8960e838a87a90ead0cMarkos Chandras b_off = b_imm(i + inst->jf + 1, ctx); 11011ab24a4e3de1ec37d8ed255841d2d94d77e8a4f4Markos Chandras emit_bcond(MIPS_COND_NE, r_s0, r_zero, b_off, 1102c6610de353da5ca6eee5b8960e838a87a90ead0cMarkos Chandras ctx); 1103c6610de353da5ca6eee5b8960e838a87a90ead0cMarkos Chandras emit_nop(ctx); 1104c6610de353da5ca6eee5b8960e838a87a90ead0cMarkos Chandras /* A > (K|X) ? scratch = 0 */ 1105c6610de353da5ca6eee5b8960e838a87a90ead0cMarkos Chandras if (condt & MIPS_COND_GT) { 1106c6610de353da5ca6eee5b8960e838a87a90ead0cMarkos Chandras /* Checking for equality */ 1107c6610de353da5ca6eee5b8960e838a87a90ead0cMarkos Chandras ctx->flags |= SEEN_S0 | SEEN_A | SEEN_X; 1108c6610de353da5ca6eee5b8960e838a87a90ead0cMarkos Chandras if (condt & MIPS_COND_K) 1109c6610de353da5ca6eee5b8960e838a87a90ead0cMarkos Chandras emit_load_imm(r_s0, k, ctx); 1110c6610de353da5ca6eee5b8960e838a87a90ead0cMarkos Chandras else 1111c6610de353da5ca6eee5b8960e838a87a90ead0cMarkos Chandras emit_jit_reg_move(r_s0, r_X, 1112c6610de353da5ca6eee5b8960e838a87a90ead0cMarkos Chandras ctx); 1113c6610de353da5ca6eee5b8960e838a87a90ead0cMarkos Chandras b_off = b_imm(i + inst->jf + 1, ctx); 1114c6610de353da5ca6eee5b8960e838a87a90ead0cMarkos Chandras emit_bcond(MIPS_COND_EQ, r_A, r_s0, 1115c6610de353da5ca6eee5b8960e838a87a90ead0cMarkos Chandras b_off, ctx); 1116c6610de353da5ca6eee5b8960e838a87a90ead0cMarkos Chandras emit_nop(ctx); 1117c6610de353da5ca6eee5b8960e838a87a90ead0cMarkos Chandras /* Finally, A > K|X */ 1118c6610de353da5ca6eee5b8960e838a87a90ead0cMarkos Chandras b_off = b_imm(i + inst->jt + 1, ctx); 1119c6610de353da5ca6eee5b8960e838a87a90ead0cMarkos Chandras emit_b(b_off, ctx); 1120c6610de353da5ca6eee5b8960e838a87a90ead0cMarkos Chandras emit_nop(ctx); 1121c6610de353da5ca6eee5b8960e838a87a90ead0cMarkos Chandras } else { 1122c6610de353da5ca6eee5b8960e838a87a90ead0cMarkos Chandras /* A >= (K|X) so jump */ 1123c6610de353da5ca6eee5b8960e838a87a90ead0cMarkos Chandras b_off = b_imm(i + inst->jt + 1, ctx); 1124c6610de353da5ca6eee5b8960e838a87a90ead0cMarkos Chandras emit_b(b_off, ctx); 1125c6610de353da5ca6eee5b8960e838a87a90ead0cMarkos Chandras emit_nop(ctx); 1126c6610de353da5ca6eee5b8960e838a87a90ead0cMarkos Chandras } 1127c6610de353da5ca6eee5b8960e838a87a90ead0cMarkos Chandras } else { 1128c6610de353da5ca6eee5b8960e838a87a90ead0cMarkos Chandras /* A == K|X */ 1129c6610de353da5ca6eee5b8960e838a87a90ead0cMarkos Chandras if (condt & MIPS_COND_K) { /* K */ 1130c6610de353da5ca6eee5b8960e838a87a90ead0cMarkos Chandras ctx->flags |= SEEN_S0 | SEEN_A; 1131c6610de353da5ca6eee5b8960e838a87a90ead0cMarkos Chandras emit_load_imm(r_s0, k, ctx); 1132c6610de353da5ca6eee5b8960e838a87a90ead0cMarkos Chandras /* jump true */ 1133c6610de353da5ca6eee5b8960e838a87a90ead0cMarkos Chandras b_off = b_imm(i + inst->jt + 1, ctx); 1134c6610de353da5ca6eee5b8960e838a87a90ead0cMarkos Chandras emit_bcond(MIPS_COND_EQ, r_A, r_s0, 1135c6610de353da5ca6eee5b8960e838a87a90ead0cMarkos Chandras b_off, ctx); 1136c6610de353da5ca6eee5b8960e838a87a90ead0cMarkos Chandras emit_nop(ctx); 1137c6610de353da5ca6eee5b8960e838a87a90ead0cMarkos Chandras /* jump false */ 1138c6610de353da5ca6eee5b8960e838a87a90ead0cMarkos Chandras b_off = b_imm(i + inst->jf + 1, 1139c6610de353da5ca6eee5b8960e838a87a90ead0cMarkos Chandras ctx); 1140c6610de353da5ca6eee5b8960e838a87a90ead0cMarkos Chandras emit_bcond(MIPS_COND_NE, r_A, r_s0, 1141c6610de353da5ca6eee5b8960e838a87a90ead0cMarkos Chandras b_off, ctx); 1142c6610de353da5ca6eee5b8960e838a87a90ead0cMarkos Chandras emit_nop(ctx); 1143c6610de353da5ca6eee5b8960e838a87a90ead0cMarkos Chandras } else { /* X */ 1144c6610de353da5ca6eee5b8960e838a87a90ead0cMarkos Chandras /* jump true */ 1145c6610de353da5ca6eee5b8960e838a87a90ead0cMarkos Chandras ctx->flags |= SEEN_A | SEEN_X; 1146c6610de353da5ca6eee5b8960e838a87a90ead0cMarkos Chandras b_off = b_imm(i + inst->jt + 1, 1147c6610de353da5ca6eee5b8960e838a87a90ead0cMarkos Chandras ctx); 1148c6610de353da5ca6eee5b8960e838a87a90ead0cMarkos Chandras emit_bcond(MIPS_COND_EQ, r_A, r_X, 1149c6610de353da5ca6eee5b8960e838a87a90ead0cMarkos Chandras b_off, ctx); 1150c6610de353da5ca6eee5b8960e838a87a90ead0cMarkos Chandras emit_nop(ctx); 1151c6610de353da5ca6eee5b8960e838a87a90ead0cMarkos Chandras /* jump false */ 1152c6610de353da5ca6eee5b8960e838a87a90ead0cMarkos Chandras b_off = b_imm(i + inst->jf + 1, ctx); 1153c6610de353da5ca6eee5b8960e838a87a90ead0cMarkos Chandras emit_bcond(MIPS_COND_NE, r_A, r_X, 1154c6610de353da5ca6eee5b8960e838a87a90ead0cMarkos Chandras b_off, ctx); 1155c6610de353da5ca6eee5b8960e838a87a90ead0cMarkos Chandras emit_nop(ctx); 1156c6610de353da5ca6eee5b8960e838a87a90ead0cMarkos Chandras } 1157c6610de353da5ca6eee5b8960e838a87a90ead0cMarkos Chandras } 1158c6610de353da5ca6eee5b8960e838a87a90ead0cMarkos Chandras break; 1159a83d081ed14e4281d2620d182c6044c0c21c551eDaniel Borkmann case BPF_JMP | BPF_JSET | BPF_K: 1160c6610de353da5ca6eee5b8960e838a87a90ead0cMarkos Chandras ctx->flags |= SEEN_S0 | SEEN_S1 | SEEN_A; 1161c6610de353da5ca6eee5b8960e838a87a90ead0cMarkos Chandras /* pc += (A & K) ? pc -> jt : pc -> jf */ 1162c6610de353da5ca6eee5b8960e838a87a90ead0cMarkos Chandras emit_load_imm(r_s1, k, ctx); 1163c6610de353da5ca6eee5b8960e838a87a90ead0cMarkos Chandras emit_and(r_s0, r_A, r_s1, ctx); 1164c6610de353da5ca6eee5b8960e838a87a90ead0cMarkos Chandras /* jump true */ 1165c6610de353da5ca6eee5b8960e838a87a90ead0cMarkos Chandras b_off = b_imm(i + inst->jt + 1, ctx); 1166c6610de353da5ca6eee5b8960e838a87a90ead0cMarkos Chandras emit_bcond(MIPS_COND_NE, r_s0, r_zero, b_off, ctx); 1167c6610de353da5ca6eee5b8960e838a87a90ead0cMarkos Chandras emit_nop(ctx); 1168c6610de353da5ca6eee5b8960e838a87a90ead0cMarkos Chandras /* jump false */ 1169c6610de353da5ca6eee5b8960e838a87a90ead0cMarkos Chandras b_off = b_imm(i + inst->jf + 1, ctx); 1170c6610de353da5ca6eee5b8960e838a87a90ead0cMarkos Chandras emit_b(b_off, ctx); 1171c6610de353da5ca6eee5b8960e838a87a90ead0cMarkos Chandras emit_nop(ctx); 1172c6610de353da5ca6eee5b8960e838a87a90ead0cMarkos Chandras break; 1173a83d081ed14e4281d2620d182c6044c0c21c551eDaniel Borkmann case BPF_JMP | BPF_JSET | BPF_X: 1174c6610de353da5ca6eee5b8960e838a87a90ead0cMarkos Chandras ctx->flags |= SEEN_S0 | SEEN_X | SEEN_A; 1175c6610de353da5ca6eee5b8960e838a87a90ead0cMarkos Chandras /* pc += (A & X) ? pc -> jt : pc -> jf */ 1176c6610de353da5ca6eee5b8960e838a87a90ead0cMarkos Chandras emit_and(r_s0, r_A, r_X, ctx); 1177c6610de353da5ca6eee5b8960e838a87a90ead0cMarkos Chandras /* jump true */ 1178c6610de353da5ca6eee5b8960e838a87a90ead0cMarkos Chandras b_off = b_imm(i + inst->jt + 1, ctx); 1179c6610de353da5ca6eee5b8960e838a87a90ead0cMarkos Chandras emit_bcond(MIPS_COND_NE, r_s0, r_zero, b_off, ctx); 1180c6610de353da5ca6eee5b8960e838a87a90ead0cMarkos Chandras emit_nop(ctx); 1181c6610de353da5ca6eee5b8960e838a87a90ead0cMarkos Chandras /* jump false */ 1182c6610de353da5ca6eee5b8960e838a87a90ead0cMarkos Chandras b_off = b_imm(i + inst->jf + 1, ctx); 1183c6610de353da5ca6eee5b8960e838a87a90ead0cMarkos Chandras emit_b(b_off, ctx); 1184c6610de353da5ca6eee5b8960e838a87a90ead0cMarkos Chandras emit_nop(ctx); 1185c6610de353da5ca6eee5b8960e838a87a90ead0cMarkos Chandras break; 1186a83d081ed14e4281d2620d182c6044c0c21c551eDaniel Borkmann case BPF_RET | BPF_A: 1187c6610de353da5ca6eee5b8960e838a87a90ead0cMarkos Chandras ctx->flags |= SEEN_A; 1188c6610de353da5ca6eee5b8960e838a87a90ead0cMarkos Chandras if (i != prog->len - 1) 1189c6610de353da5ca6eee5b8960e838a87a90ead0cMarkos Chandras /* 1190c6610de353da5ca6eee5b8960e838a87a90ead0cMarkos Chandras * If this is not the last instruction 1191c6610de353da5ca6eee5b8960e838a87a90ead0cMarkos Chandras * then jump to the epilogue 1192c6610de353da5ca6eee5b8960e838a87a90ead0cMarkos Chandras */ 1193c6610de353da5ca6eee5b8960e838a87a90ead0cMarkos Chandras emit_b(b_imm(prog->len, ctx), ctx); 1194c6610de353da5ca6eee5b8960e838a87a90ead0cMarkos Chandras emit_reg_move(r_ret, r_A, ctx); /* delay slot */ 1195c6610de353da5ca6eee5b8960e838a87a90ead0cMarkos Chandras break; 1196a83d081ed14e4281d2620d182c6044c0c21c551eDaniel Borkmann case BPF_RET | BPF_K: 1197c6610de353da5ca6eee5b8960e838a87a90ead0cMarkos Chandras /* 1198c6610de353da5ca6eee5b8960e838a87a90ead0cMarkos Chandras * It can emit two instructions so it does not fit on 1199c6610de353da5ca6eee5b8960e838a87a90ead0cMarkos Chandras * the delay slot. 1200c6610de353da5ca6eee5b8960e838a87a90ead0cMarkos Chandras */ 1201c6610de353da5ca6eee5b8960e838a87a90ead0cMarkos Chandras emit_load_imm(r_ret, k, ctx); 1202c6610de353da5ca6eee5b8960e838a87a90ead0cMarkos Chandras if (i != prog->len - 1) { 1203c6610de353da5ca6eee5b8960e838a87a90ead0cMarkos Chandras /* 1204c6610de353da5ca6eee5b8960e838a87a90ead0cMarkos Chandras * If this is not the last instruction 1205c6610de353da5ca6eee5b8960e838a87a90ead0cMarkos Chandras * then jump to the epilogue 1206c6610de353da5ca6eee5b8960e838a87a90ead0cMarkos Chandras */ 1207c6610de353da5ca6eee5b8960e838a87a90ead0cMarkos Chandras emit_b(b_imm(prog->len, ctx), ctx); 1208c6610de353da5ca6eee5b8960e838a87a90ead0cMarkos Chandras emit_nop(ctx); 1209c6610de353da5ca6eee5b8960e838a87a90ead0cMarkos Chandras } 1210c6610de353da5ca6eee5b8960e838a87a90ead0cMarkos Chandras break; 1211a83d081ed14e4281d2620d182c6044c0c21c551eDaniel Borkmann case BPF_MISC | BPF_TAX: 1212c6610de353da5ca6eee5b8960e838a87a90ead0cMarkos Chandras /* X = A */ 1213c6610de353da5ca6eee5b8960e838a87a90ead0cMarkos Chandras ctx->flags |= SEEN_X | SEEN_A; 1214c6610de353da5ca6eee5b8960e838a87a90ead0cMarkos Chandras emit_jit_reg_move(r_X, r_A, ctx); 1215c6610de353da5ca6eee5b8960e838a87a90ead0cMarkos Chandras break; 1216a83d081ed14e4281d2620d182c6044c0c21c551eDaniel Borkmann case BPF_MISC | BPF_TXA: 1217c6610de353da5ca6eee5b8960e838a87a90ead0cMarkos Chandras /* A = X */ 1218c6610de353da5ca6eee5b8960e838a87a90ead0cMarkos Chandras ctx->flags |= SEEN_A | SEEN_X; 1219c6610de353da5ca6eee5b8960e838a87a90ead0cMarkos Chandras emit_jit_reg_move(r_A, r_X, ctx); 1220c6610de353da5ca6eee5b8960e838a87a90ead0cMarkos Chandras break; 1221c6610de353da5ca6eee5b8960e838a87a90ead0cMarkos Chandras /* AUX */ 1222a83d081ed14e4281d2620d182c6044c0c21c551eDaniel Borkmann case BPF_ANC | SKF_AD_PROTOCOL: 1223c6610de353da5ca6eee5b8960e838a87a90ead0cMarkos Chandras /* A = ntohs(skb->protocol */ 1224c6610de353da5ca6eee5b8960e838a87a90ead0cMarkos Chandras ctx->flags |= SEEN_SKB | SEEN_OFF | SEEN_A; 1225c6610de353da5ca6eee5b8960e838a87a90ead0cMarkos Chandras BUILD_BUG_ON(FIELD_SIZEOF(struct sk_buff, 1226c6610de353da5ca6eee5b8960e838a87a90ead0cMarkos Chandras protocol) != 2); 1227c6610de353da5ca6eee5b8960e838a87a90ead0cMarkos Chandras off = offsetof(struct sk_buff, protocol); 1228c6610de353da5ca6eee5b8960e838a87a90ead0cMarkos Chandras emit_half_load(r_A, r_skb, off, ctx); 1229c6610de353da5ca6eee5b8960e838a87a90ead0cMarkos Chandras#ifdef CONFIG_CPU_LITTLE_ENDIAN 1230c6610de353da5ca6eee5b8960e838a87a90ead0cMarkos Chandras /* This needs little endian fixup */ 12313c09bae43ba92a07a6a7b7d42360deb32d289cc0Chen Jie if (cpu_has_wsbh) { 1232b4f16c938eed87aac733972e735c5bea700948aaRalf Baechle /* R2 and later have the wsbh instruction */ 1233b4f16c938eed87aac733972e735c5bea700948aaRalf Baechle emit_wsbh(r_A, r_A, ctx); 1234b4f16c938eed87aac733972e735c5bea700948aaRalf Baechle } else { 1235c6610de353da5ca6eee5b8960e838a87a90ead0cMarkos Chandras /* Get first byte */ 1236c6610de353da5ca6eee5b8960e838a87a90ead0cMarkos Chandras emit_andi(r_tmp_imm, r_A, 0xff, ctx); 1237c6610de353da5ca6eee5b8960e838a87a90ead0cMarkos Chandras /* Shift it */ 1238c6610de353da5ca6eee5b8960e838a87a90ead0cMarkos Chandras emit_sll(r_tmp, r_tmp_imm, 8, ctx); 1239c6610de353da5ca6eee5b8960e838a87a90ead0cMarkos Chandras /* Get second byte */ 1240c6610de353da5ca6eee5b8960e838a87a90ead0cMarkos Chandras emit_srl(r_tmp_imm, r_A, 8, ctx); 1241c6610de353da5ca6eee5b8960e838a87a90ead0cMarkos Chandras emit_andi(r_tmp_imm, r_tmp_imm, 0xff, ctx); 1242c6610de353da5ca6eee5b8960e838a87a90ead0cMarkos Chandras /* Put everyting together in r_A */ 1243c6610de353da5ca6eee5b8960e838a87a90ead0cMarkos Chandras emit_or(r_A, r_tmp, r_tmp_imm, ctx); 1244c6610de353da5ca6eee5b8960e838a87a90ead0cMarkos Chandras } 1245c6610de353da5ca6eee5b8960e838a87a90ead0cMarkos Chandras#endif 1246c6610de353da5ca6eee5b8960e838a87a90ead0cMarkos Chandras break; 1247a83d081ed14e4281d2620d182c6044c0c21c551eDaniel Borkmann case BPF_ANC | SKF_AD_CPU: 1248c6610de353da5ca6eee5b8960e838a87a90ead0cMarkos Chandras ctx->flags |= SEEN_A | SEEN_OFF; 1249c6610de353da5ca6eee5b8960e838a87a90ead0cMarkos Chandras /* A = current_thread_info()->cpu */ 1250c6610de353da5ca6eee5b8960e838a87a90ead0cMarkos Chandras BUILD_BUG_ON(FIELD_SIZEOF(struct thread_info, 1251c6610de353da5ca6eee5b8960e838a87a90ead0cMarkos Chandras cpu) != 4); 1252c6610de353da5ca6eee5b8960e838a87a90ead0cMarkos Chandras off = offsetof(struct thread_info, cpu); 1253c6610de353da5ca6eee5b8960e838a87a90ead0cMarkos Chandras /* $28/gp points to the thread_info struct */ 1254c6610de353da5ca6eee5b8960e838a87a90ead0cMarkos Chandras emit_load(r_A, 28, off, ctx); 1255c6610de353da5ca6eee5b8960e838a87a90ead0cMarkos Chandras break; 1256a83d081ed14e4281d2620d182c6044c0c21c551eDaniel Borkmann case BPF_ANC | SKF_AD_IFINDEX: 1257c6610de353da5ca6eee5b8960e838a87a90ead0cMarkos Chandras /* A = skb->dev->ifindex */ 1258c6610de353da5ca6eee5b8960e838a87a90ead0cMarkos Chandras ctx->flags |= SEEN_SKB | SEEN_A | SEEN_S0; 1259c6610de353da5ca6eee5b8960e838a87a90ead0cMarkos Chandras off = offsetof(struct sk_buff, dev); 1260b6a14a9845259eb21c9d8121330c4c3b22de182eMarkos Chandras /* Load *dev pointer */ 1261b6a14a9845259eb21c9d8121330c4c3b22de182eMarkos Chandras emit_load_ptr(r_s0, r_skb, off, ctx); 1262c6610de353da5ca6eee5b8960e838a87a90ead0cMarkos Chandras /* error (0) in the delay slot */ 1263c6610de353da5ca6eee5b8960e838a87a90ead0cMarkos Chandras emit_bcond(MIPS_COND_EQ, r_s0, r_zero, 1264c6610de353da5ca6eee5b8960e838a87a90ead0cMarkos Chandras b_imm(prog->len, ctx), ctx); 1265c6610de353da5ca6eee5b8960e838a87a90ead0cMarkos Chandras emit_reg_move(r_ret, r_zero, ctx); 1266c6610de353da5ca6eee5b8960e838a87a90ead0cMarkos Chandras BUILD_BUG_ON(FIELD_SIZEOF(struct net_device, 1267c6610de353da5ca6eee5b8960e838a87a90ead0cMarkos Chandras ifindex) != 4); 1268c6610de353da5ca6eee5b8960e838a87a90ead0cMarkos Chandras off = offsetof(struct net_device, ifindex); 1269c6610de353da5ca6eee5b8960e838a87a90ead0cMarkos Chandras emit_load(r_A, r_s0, off, ctx); 1270c6610de353da5ca6eee5b8960e838a87a90ead0cMarkos Chandras break; 1271a83d081ed14e4281d2620d182c6044c0c21c551eDaniel Borkmann case BPF_ANC | SKF_AD_MARK: 1272c6610de353da5ca6eee5b8960e838a87a90ead0cMarkos Chandras ctx->flags |= SEEN_SKB | SEEN_A; 1273c6610de353da5ca6eee5b8960e838a87a90ead0cMarkos Chandras BUILD_BUG_ON(FIELD_SIZEOF(struct sk_buff, mark) != 4); 1274c6610de353da5ca6eee5b8960e838a87a90ead0cMarkos Chandras off = offsetof(struct sk_buff, mark); 1275c6610de353da5ca6eee5b8960e838a87a90ead0cMarkos Chandras emit_load(r_A, r_skb, off, ctx); 1276c6610de353da5ca6eee5b8960e838a87a90ead0cMarkos Chandras break; 1277a83d081ed14e4281d2620d182c6044c0c21c551eDaniel Borkmann case BPF_ANC | SKF_AD_RXHASH: 1278c6610de353da5ca6eee5b8960e838a87a90ead0cMarkos Chandras ctx->flags |= SEEN_SKB | SEEN_A; 1279c6610de353da5ca6eee5b8960e838a87a90ead0cMarkos Chandras BUILD_BUG_ON(FIELD_SIZEOF(struct sk_buff, hash) != 4); 1280c6610de353da5ca6eee5b8960e838a87a90ead0cMarkos Chandras off = offsetof(struct sk_buff, hash); 1281c6610de353da5ca6eee5b8960e838a87a90ead0cMarkos Chandras emit_load(r_A, r_skb, off, ctx); 1282c6610de353da5ca6eee5b8960e838a87a90ead0cMarkos Chandras break; 1283a83d081ed14e4281d2620d182c6044c0c21c551eDaniel Borkmann case BPF_ANC | SKF_AD_VLAN_TAG: 1284a83d081ed14e4281d2620d182c6044c0c21c551eDaniel Borkmann case BPF_ANC | SKF_AD_VLAN_TAG_PRESENT: 1285c6610de353da5ca6eee5b8960e838a87a90ead0cMarkos Chandras ctx->flags |= SEEN_SKB | SEEN_S0 | SEEN_A; 1286c6610de353da5ca6eee5b8960e838a87a90ead0cMarkos Chandras BUILD_BUG_ON(FIELD_SIZEOF(struct sk_buff, 1287c6610de353da5ca6eee5b8960e838a87a90ead0cMarkos Chandras vlan_tci) != 2); 1288c6610de353da5ca6eee5b8960e838a87a90ead0cMarkos Chandras off = offsetof(struct sk_buff, vlan_tci); 1289c6610de353da5ca6eee5b8960e838a87a90ead0cMarkos Chandras emit_half_load(r_s0, r_skb, off, ctx); 129091a41d7f972b1d78b4bcbb61ada4a33c9d7ba8a3Markos Chandras if (code == (BPF_ANC | SKF_AD_VLAN_TAG)) { 12916e86c59d4d0d04e7ebefd05b8af245c968892f81Markos Chandras emit_andi(r_A, r_s0, (u16)~VLAN_TAG_PRESENT, ctx); 129291a41d7f972b1d78b4bcbb61ada4a33c9d7ba8a3Markos Chandras } else { 12939ee1606e8a6316287029c86ef48ddcfe4dec595dMarkos Chandras emit_andi(r_A, r_s0, VLAN_TAG_PRESENT, ctx); 129491a41d7f972b1d78b4bcbb61ada4a33c9d7ba8a3Markos Chandras /* return 1 if present */ 129591a41d7f972b1d78b4bcbb61ada4a33c9d7ba8a3Markos Chandras emit_sltu(r_A, r_zero, r_A, ctx); 129691a41d7f972b1d78b4bcbb61ada4a33c9d7ba8a3Markos Chandras } 1297c6610de353da5ca6eee5b8960e838a87a90ead0cMarkos Chandras break; 1298a83d081ed14e4281d2620d182c6044c0c21c551eDaniel Borkmann case BPF_ANC | SKF_AD_PKTTYPE: 12999eebfe478d1b83389c3ac1bd73a45b6665e356dbMarkos Chandras ctx->flags |= SEEN_SKB; 13009eebfe478d1b83389c3ac1bd73a45b6665e356dbMarkos Chandras 1301233577a22089facf5271ab5e845b2262047c971fHannes Frederic Sowa emit_load_byte(r_tmp, r_skb, PKT_TYPE_OFFSET(), ctx); 1302c6610de353da5ca6eee5b8960e838a87a90ead0cMarkos Chandras /* Keep only the last 3 bits */ 1303c6610de353da5ca6eee5b8960e838a87a90ead0cMarkos Chandras emit_andi(r_A, r_tmp, PKT_TYPE_MAX, ctx); 1304b4fe0ec86dae91abfa9f932cd0e2e9d50e336c8bMarkos Chandras#ifdef __BIG_ENDIAN_BITFIELD 1305b4fe0ec86dae91abfa9f932cd0e2e9d50e336c8bMarkos Chandras /* Get the actual packet type to the lower 3 bits */ 1306b4fe0ec86dae91abfa9f932cd0e2e9d50e336c8bMarkos Chandras emit_srl(r_A, r_A, 5, ctx); 1307b4fe0ec86dae91abfa9f932cd0e2e9d50e336c8bMarkos Chandras#endif 1308c6610de353da5ca6eee5b8960e838a87a90ead0cMarkos Chandras break; 1309a83d081ed14e4281d2620d182c6044c0c21c551eDaniel Borkmann case BPF_ANC | SKF_AD_QUEUE: 1310c6610de353da5ca6eee5b8960e838a87a90ead0cMarkos Chandras ctx->flags |= SEEN_SKB | SEEN_A; 1311c6610de353da5ca6eee5b8960e838a87a90ead0cMarkos Chandras BUILD_BUG_ON(FIELD_SIZEOF(struct sk_buff, 1312c6610de353da5ca6eee5b8960e838a87a90ead0cMarkos Chandras queue_mapping) != 2); 1313c6610de353da5ca6eee5b8960e838a87a90ead0cMarkos Chandras BUILD_BUG_ON(offsetof(struct sk_buff, 1314c6610de353da5ca6eee5b8960e838a87a90ead0cMarkos Chandras queue_mapping) > 0xff); 1315c6610de353da5ca6eee5b8960e838a87a90ead0cMarkos Chandras off = offsetof(struct sk_buff, queue_mapping); 1316c6610de353da5ca6eee5b8960e838a87a90ead0cMarkos Chandras emit_half_load(r_A, r_skb, off, ctx); 1317c6610de353da5ca6eee5b8960e838a87a90ead0cMarkos Chandras break; 1318c6610de353da5ca6eee5b8960e838a87a90ead0cMarkos Chandras default: 131978b95b662c4c633206c997fe2bd25a9c680e047aMarkos Chandras pr_debug("%s: Unhandled opcode: 0x%02x\n", __FILE__, 132078b95b662c4c633206c997fe2bd25a9c680e047aMarkos Chandras inst->code); 1321c6610de353da5ca6eee5b8960e838a87a90ead0cMarkos Chandras return -1; 1322c6610de353da5ca6eee5b8960e838a87a90ead0cMarkos Chandras } 1323c6610de353da5ca6eee5b8960e838a87a90ead0cMarkos Chandras } 1324c6610de353da5ca6eee5b8960e838a87a90ead0cMarkos Chandras 1325c6610de353da5ca6eee5b8960e838a87a90ead0cMarkos Chandras /* compute offsets only during the first pass */ 1326c6610de353da5ca6eee5b8960e838a87a90ead0cMarkos Chandras if (ctx->target == NULL) 1327c6610de353da5ca6eee5b8960e838a87a90ead0cMarkos Chandras ctx->offsets[i] = ctx->idx * 4; 1328c6610de353da5ca6eee5b8960e838a87a90ead0cMarkos Chandras 1329c6610de353da5ca6eee5b8960e838a87a90ead0cMarkos Chandras return 0; 1330c6610de353da5ca6eee5b8960e838a87a90ead0cMarkos Chandras} 1331c6610de353da5ca6eee5b8960e838a87a90ead0cMarkos Chandras 1332c6610de353da5ca6eee5b8960e838a87a90ead0cMarkos Chandrasint bpf_jit_enable __read_mostly; 1333c6610de353da5ca6eee5b8960e838a87a90ead0cMarkos Chandras 13347ae457c1e5b45a1b826fad9d62b32191d2bdcfdbAlexei Starovoitovvoid bpf_jit_compile(struct bpf_prog *fp) 1335c6610de353da5ca6eee5b8960e838a87a90ead0cMarkos Chandras{ 1336c6610de353da5ca6eee5b8960e838a87a90ead0cMarkos Chandras struct jit_ctx ctx; 1337c6610de353da5ca6eee5b8960e838a87a90ead0cMarkos Chandras unsigned int alloc_size, tmp_idx; 1338c6610de353da5ca6eee5b8960e838a87a90ead0cMarkos Chandras 1339c6610de353da5ca6eee5b8960e838a87a90ead0cMarkos Chandras if (!bpf_jit_enable) 1340c6610de353da5ca6eee5b8960e838a87a90ead0cMarkos Chandras return; 1341c6610de353da5ca6eee5b8960e838a87a90ead0cMarkos Chandras 1342c6610de353da5ca6eee5b8960e838a87a90ead0cMarkos Chandras memset(&ctx, 0, sizeof(ctx)); 1343c6610de353da5ca6eee5b8960e838a87a90ead0cMarkos Chandras 1344c6610de353da5ca6eee5b8960e838a87a90ead0cMarkos Chandras ctx.offsets = kcalloc(fp->len, sizeof(*ctx.offsets), GFP_KERNEL); 1345c6610de353da5ca6eee5b8960e838a87a90ead0cMarkos Chandras if (ctx.offsets == NULL) 1346c6610de353da5ca6eee5b8960e838a87a90ead0cMarkos Chandras return; 1347c6610de353da5ca6eee5b8960e838a87a90ead0cMarkos Chandras 1348c6610de353da5ca6eee5b8960e838a87a90ead0cMarkos Chandras ctx.skf = fp; 1349c6610de353da5ca6eee5b8960e838a87a90ead0cMarkos Chandras 1350c6610de353da5ca6eee5b8960e838a87a90ead0cMarkos Chandras if (build_body(&ctx)) 1351c6610de353da5ca6eee5b8960e838a87a90ead0cMarkos Chandras goto out; 1352c6610de353da5ca6eee5b8960e838a87a90ead0cMarkos Chandras 1353c6610de353da5ca6eee5b8960e838a87a90ead0cMarkos Chandras tmp_idx = ctx.idx; 1354c6610de353da5ca6eee5b8960e838a87a90ead0cMarkos Chandras build_prologue(&ctx); 1355c6610de353da5ca6eee5b8960e838a87a90ead0cMarkos Chandras ctx.prologue_bytes = (ctx.idx - tmp_idx) * 4; 1356c6610de353da5ca6eee5b8960e838a87a90ead0cMarkos Chandras /* just to complete the ctx.idx count */ 1357c6610de353da5ca6eee5b8960e838a87a90ead0cMarkos Chandras build_epilogue(&ctx); 1358c6610de353da5ca6eee5b8960e838a87a90ead0cMarkos Chandras 1359c6610de353da5ca6eee5b8960e838a87a90ead0cMarkos Chandras alloc_size = 4 * ctx.idx; 1360c6610de353da5ca6eee5b8960e838a87a90ead0cMarkos Chandras ctx.target = module_alloc(alloc_size); 1361c6610de353da5ca6eee5b8960e838a87a90ead0cMarkos Chandras if (ctx.target == NULL) 1362c6610de353da5ca6eee5b8960e838a87a90ead0cMarkos Chandras goto out; 1363c6610de353da5ca6eee5b8960e838a87a90ead0cMarkos Chandras 1364c6610de353da5ca6eee5b8960e838a87a90ead0cMarkos Chandras /* Clean it */ 1365c6610de353da5ca6eee5b8960e838a87a90ead0cMarkos Chandras memset(ctx.target, 0, alloc_size); 1366c6610de353da5ca6eee5b8960e838a87a90ead0cMarkos Chandras 1367c6610de353da5ca6eee5b8960e838a87a90ead0cMarkos Chandras ctx.idx = 0; 1368c6610de353da5ca6eee5b8960e838a87a90ead0cMarkos Chandras 1369c6610de353da5ca6eee5b8960e838a87a90ead0cMarkos Chandras /* Generate the actual JIT code */ 1370c6610de353da5ca6eee5b8960e838a87a90ead0cMarkos Chandras build_prologue(&ctx); 1371c6610de353da5ca6eee5b8960e838a87a90ead0cMarkos Chandras build_body(&ctx); 1372c6610de353da5ca6eee5b8960e838a87a90ead0cMarkos Chandras build_epilogue(&ctx); 1373c6610de353da5ca6eee5b8960e838a87a90ead0cMarkos Chandras 1374c6610de353da5ca6eee5b8960e838a87a90ead0cMarkos Chandras /* Update the icache */ 1375c6610de353da5ca6eee5b8960e838a87a90ead0cMarkos Chandras flush_icache_range((ptr)ctx.target, (ptr)(ctx.target + ctx.idx)); 1376c6610de353da5ca6eee5b8960e838a87a90ead0cMarkos Chandras 1377c6610de353da5ca6eee5b8960e838a87a90ead0cMarkos Chandras if (bpf_jit_enable > 1) 1378c6610de353da5ca6eee5b8960e838a87a90ead0cMarkos Chandras /* Dump JIT code */ 1379c6610de353da5ca6eee5b8960e838a87a90ead0cMarkos Chandras bpf_jit_dump(fp->len, alloc_size, 2, ctx.target); 1380c6610de353da5ca6eee5b8960e838a87a90ead0cMarkos Chandras 1381c6610de353da5ca6eee5b8960e838a87a90ead0cMarkos Chandras fp->bpf_func = (void *)ctx.target; 1382286aad3c4014ca825c447e07e24f8929e6d266d2Daniel Borkmann fp->jited = true; 1383c6610de353da5ca6eee5b8960e838a87a90ead0cMarkos Chandras 1384c6610de353da5ca6eee5b8960e838a87a90ead0cMarkos Chandrasout: 1385c6610de353da5ca6eee5b8960e838a87a90ead0cMarkos Chandras kfree(ctx.offsets); 1386c6610de353da5ca6eee5b8960e838a87a90ead0cMarkos Chandras} 1387c6610de353da5ca6eee5b8960e838a87a90ead0cMarkos Chandras 13887ae457c1e5b45a1b826fad9d62b32191d2bdcfdbAlexei Starovoitovvoid bpf_jit_free(struct bpf_prog *fp) 1389c6610de353da5ca6eee5b8960e838a87a90ead0cMarkos Chandras{ 1390c6610de353da5ca6eee5b8960e838a87a90ead0cMarkos Chandras if (fp->jited) 1391c6610de353da5ca6eee5b8960e838a87a90ead0cMarkos Chandras module_free(NULL, fp->bpf_func); 139260a3b2253c413cf601783b070507d7dd6620c954Daniel Borkmann 139360a3b2253c413cf601783b070507d7dd6620c954Daniel Borkmann bpf_prog_unlock_free(fp); 1394c6610de353da5ca6eee5b8960e838a87a90ead0cMarkos Chandras} 1395