| #include <asm/ptrace.h> | |
| #include "bpf_jit.h" | |
| #ifdef CONFIG_SPARC64 | |
| #define SAVE_SZ 176 | |
| #define SCRATCH_OFF STACK_BIAS + 128 | |
| #define BE_PTR(label) be,pn %xcc, label | |
| #define SIGN_EXTEND(reg) sra reg, 0, reg | |
| #else | |
| #define SAVE_SZ 96 | |
| #define SCRATCH_OFF 72 | |
| #define BE_PTR(label) be label | |
| #define SIGN_EXTEND(reg) | |
| #endif | |
| #define SKF_MAX_NEG_OFF (-0x200000) /* SKF_LL_OFF from filter.h */ | |
| .text | |
| .globl bpf_jit_load_word | |
| bpf_jit_load_word: | |
| cmp r_OFF, 0 | |
| bl bpf_slow_path_word_neg | |
| nop | |
| .globl bpf_jit_load_word_positive_offset | |
| bpf_jit_load_word_positive_offset: | |
| sub r_HEADLEN, r_OFF, r_TMP | |
| cmp r_TMP, 3 | |
| ble bpf_slow_path_word | |
| add r_SKB_DATA, r_OFF, r_TMP | |
| andcc r_TMP, 3, %g0 | |
| bne load_word_unaligned | |
| nop | |
| retl | |
| ld [r_TMP], r_A | |
| load_word_unaligned: | |
| ldub [r_TMP + 0x0], r_OFF | |
| ldub [r_TMP + 0x1], r_TMP2 | |
| sll r_OFF, 8, r_OFF | |
| or r_OFF, r_TMP2, r_OFF | |
| ldub [r_TMP + 0x2], r_TMP2 | |
| sll r_OFF, 8, r_OFF | |
| or r_OFF, r_TMP2, r_OFF | |
| ldub [r_TMP + 0x3], r_TMP2 | |
| sll r_OFF, 8, r_OFF | |
| retl | |
| or r_OFF, r_TMP2, r_A | |
| .globl bpf_jit_load_half | |
| bpf_jit_load_half: | |
| cmp r_OFF, 0 | |
| bl bpf_slow_path_half_neg | |
| nop | |
| .globl bpf_jit_load_half_positive_offset | |
| bpf_jit_load_half_positive_offset: | |
| sub r_HEADLEN, r_OFF, r_TMP | |
| cmp r_TMP, 1 | |
| ble bpf_slow_path_half | |
| add r_SKB_DATA, r_OFF, r_TMP | |
| andcc r_TMP, 1, %g0 | |
| bne load_half_unaligned | |
| nop | |
| retl | |
| lduh [r_TMP], r_A | |
| load_half_unaligned: | |
| ldub [r_TMP + 0x0], r_OFF | |
| ldub [r_TMP + 0x1], r_TMP2 | |
| sll r_OFF, 8, r_OFF | |
| retl | |
| or r_OFF, r_TMP2, r_A | |
| .globl bpf_jit_load_byte | |
| bpf_jit_load_byte: | |
| cmp r_OFF, 0 | |
| bl bpf_slow_path_byte_neg | |
| nop | |
| .globl bpf_jit_load_byte_positive_offset | |
| bpf_jit_load_byte_positive_offset: | |
| cmp r_OFF, r_HEADLEN | |
| bge bpf_slow_path_byte | |
| nop | |
| retl | |
| ldub [r_SKB_DATA + r_OFF], r_A | |
| .globl bpf_jit_load_byte_msh | |
| bpf_jit_load_byte_msh: | |
| cmp r_OFF, 0 | |
| bl bpf_slow_path_byte_msh_neg | |
| nop | |
| .globl bpf_jit_load_byte_msh_positive_offset | |
| bpf_jit_load_byte_msh_positive_offset: | |
| cmp r_OFF, r_HEADLEN | |
| bge bpf_slow_path_byte_msh | |
| nop | |
| ldub [r_SKB_DATA + r_OFF], r_OFF | |
| and r_OFF, 0xf, r_OFF | |
| retl | |
| sll r_OFF, 2, r_X | |
| #define bpf_slow_path_common(LEN) \ | |
| save %sp, -SAVE_SZ, %sp; \ | |
| mov %i0, %o0; \ | |
| mov r_OFF, %o1; \ | |
| add %fp, SCRATCH_OFF, %o2; \ | |
| call skb_copy_bits; \ | |
| mov (LEN), %o3; \ | |
| cmp %o0, 0; \ | |
| restore; | |
| bpf_slow_path_word: | |
| bpf_slow_path_common(4) | |
| bl bpf_error | |
| ld [%sp + SCRATCH_OFF], r_A | |
| retl | |
| nop | |
| bpf_slow_path_half: | |
| bpf_slow_path_common(2) | |
| bl bpf_error | |
| lduh [%sp + SCRATCH_OFF], r_A | |
| retl | |
| nop | |
| bpf_slow_path_byte: | |
| bpf_slow_path_common(1) | |
| bl bpf_error | |
| ldub [%sp + SCRATCH_OFF], r_A | |
| retl | |
| nop | |
| bpf_slow_path_byte_msh: | |
| bpf_slow_path_common(1) | |
| bl bpf_error | |
| ldub [%sp + SCRATCH_OFF], r_A | |
| and r_OFF, 0xf, r_OFF | |
| retl | |
| sll r_OFF, 2, r_X | |
| #define bpf_negative_common(LEN) \ | |
| save %sp, -SAVE_SZ, %sp; \ | |
| mov %i0, %o0; \ | |
| mov r_OFF, %o1; \ | |
| SIGN_EXTEND(%o1); \ | |
| call bpf_internal_load_pointer_neg_helper; \ | |
| mov (LEN), %o2; \ | |
| mov %o0, r_TMP; \ | |
| cmp %o0, 0; \ | |
| BE_PTR(bpf_error); \ | |
| restore; | |
| bpf_slow_path_word_neg: | |
| sethi %hi(SKF_MAX_NEG_OFF), r_TMP | |
| cmp r_OFF, r_TMP | |
| bl bpf_error | |
| nop | |
| .globl bpf_jit_load_word_negative_offset | |
| bpf_jit_load_word_negative_offset: | |
| bpf_negative_common(4) | |
| andcc r_TMP, 3, %g0 | |
| bne load_word_unaligned | |
| nop | |
| retl | |
| ld [r_TMP], r_A | |
| bpf_slow_path_half_neg: | |
| sethi %hi(SKF_MAX_NEG_OFF), r_TMP | |
| cmp r_OFF, r_TMP | |
| bl bpf_error | |
| nop | |
| .globl bpf_jit_load_half_negative_offset | |
| bpf_jit_load_half_negative_offset: | |
| bpf_negative_common(2) | |
| andcc r_TMP, 1, %g0 | |
| bne load_half_unaligned | |
| nop | |
| retl | |
| lduh [r_TMP], r_A | |
| bpf_slow_path_byte_neg: | |
| sethi %hi(SKF_MAX_NEG_OFF), r_TMP | |
| cmp r_OFF, r_TMP | |
| bl bpf_error | |
| nop | |
| .globl bpf_jit_load_byte_negative_offset | |
| bpf_jit_load_byte_negative_offset: | |
| bpf_negative_common(1) | |
| retl | |
| ldub [r_TMP], r_A | |
| bpf_slow_path_byte_msh_neg: | |
| sethi %hi(SKF_MAX_NEG_OFF), r_TMP | |
| cmp r_OFF, r_TMP | |
| bl bpf_error | |
| nop | |
| .globl bpf_jit_load_byte_msh_negative_offset | |
| bpf_jit_load_byte_msh_negative_offset: | |
| bpf_negative_common(1) | |
| ldub [r_TMP], r_OFF | |
| and r_OFF, 0xf, r_OFF | |
| retl | |
| sll r_OFF, 2, r_X | |
| bpf_error: | |
| /* Make the JIT program return zero. The JIT epilogue | |
| * stores away the original %o7 into r_saved_O7. The | |
| * normal leaf function return is to use "retl" which | |
| * would evalute to "jmpl %o7 + 8, %g0" but we want to | |
| * use the saved value thus the sequence you see here. | |
| */ | |
| jmpl r_saved_O7 + 8, %g0 | |
| clr %o0 |