bpf jit: Let the x86 jit handle negative offsets

[karo-tx-linux.git] / arch / x86 / net / bpf_jit.S

diff --git a/arch/x86/net/bpf_jit.S b/arch/x86/net/bpf_jit.S
index 66870223f8c5e001e7fc5bbe02b659c8778def86..877b9a1b21523183d06973b60a9beb2459fff895 100644 (file)
--- a/arch/x86/net/bpf_jit.S
+++ b/arch/x86/net/bpf_jit.S
@@ -18,17 +18,17 @@
  * r9d : hlen = skb->len - skb->data_len
  */
 #define SKBDATA        %r8
-
-sk_load_word_ind:
-       .globl  sk_load_word_ind
-
-       add     %ebx,%esi       /* offset += X */
-#      test    %esi,%esi       /* if (offset < 0) goto bpf_error; */
-       js      bpf_error
+#define SKF_MAX_NEG_OFF    $(-0x200000) /* SKF_LL_OFF from filter.h */
 
 sk_load_word:
        .globl  sk_load_word
 
+       test    %esi,%esi
+       js      bpf_slow_path_word_neg
+
+sk_load_word_positive_offset:
+       .globl  sk_load_word_positive_offset
+
        mov     %r9d,%eax               # hlen
        sub     %esi,%eax               # hlen - offset
        cmp     $3,%eax
@@ -37,16 +37,15 @@ sk_load_word:
        bswap   %eax                    /* ntohl() */
        ret
 
-
-sk_load_half_ind:
-       .globl sk_load_half_ind
-
-       add     %ebx,%esi       /* offset += X */
-       js      bpf_error
-
 sk_load_half:
        .globl  sk_load_half
 
+       test    %esi,%esi
+       js      bpf_slow_path_half_neg
+
+sk_load_half_positive_offset:
+       .globl  sk_load_half_positive_offset
+
        mov     %r9d,%eax
        sub     %esi,%eax               #       hlen - offset
        cmp     $1,%eax
@@ -55,14 +54,15 @@ sk_load_half:
        rol     $8,%ax                  # ntohs()
        ret
 
-sk_load_byte_ind:
-       .globl sk_load_byte_ind
-       add     %ebx,%esi       /* offset += X */
-       js      bpf_error
-
 sk_load_byte:
        .globl  sk_load_byte
 
+       test    %esi,%esi
+       js      bpf_slow_path_byte_neg
+
+sk_load_byte_positive_offset:
+       .globl  sk_load_byte_positive_offset
+
        cmp     %esi,%r9d   /* if (offset >= hlen) goto bpf_slow_path_byte */
        jle     bpf_slow_path_byte
        movzbl  (SKBDATA,%rsi),%eax
@@ -73,25 +73,21 @@ sk_load_byte:
  *
  * Implements BPF_S_LDX_B_MSH : ldxb  4*([offset]&0xf)
  * Must preserve A accumulator (%eax)
- * Inputs : %esi is the offset value, already known positive
+ * Inputs : %esi is the offset value
  */
-ENTRY(sk_load_byte_msh)
-       CFI_STARTPROC
+sk_load_byte_msh:
+       .globl  sk_load_byte_msh
+       test    %esi,%esi
+       js      bpf_slow_path_byte_msh_neg
+
+sk_load_byte_msh_positive_offset:
+       .globl  sk_load_byte_msh_positive_offset
        cmp     %esi,%r9d      /* if (offset >= hlen) goto bpf_slow_path_byte_msh */
        jle     bpf_slow_path_byte_msh
        movzbl  (SKBDATA,%rsi),%ebx
        and     $15,%bl
        shl     $2,%bl
        ret
-       CFI_ENDPROC
-ENDPROC(sk_load_byte_msh)
-
-bpf_error:
-# force a return 0 from jit handler
-       xor             %eax,%eax
-       mov             -8(%rbp),%rbx
-       leaveq
-       ret
 
 /* rsi contains offset and can be scratched */
 #define bpf_slow_path_common(LEN)              \
@@ -138,3 +134,67 @@ bpf_slow_path_byte_msh:
        shl     $2,%al
        xchg    %eax,%ebx
        ret
+
+#define sk_negative_common(SIZE)                               \
+       push    %rdi;   /* save skb */                          \
+       push    %r9;                                            \
+       push    SKBDATA;                                        \
+/* rsi already has offset */                                   \
+       mov     $SIZE,%ecx;     /* size */                      \
+       call    bpf_internal_load_pointer_neg_helper;           \
+       test    %rax,%rax;                                      \
+       pop     SKBDATA;                                        \
+       pop     %r9;                                            \
+       pop     %rdi;                                           \
+       jz      bpf_error
+
+
+bpf_slow_path_word_neg:
+       cmp     SKF_MAX_NEG_OFF, %esi   /* test range */
+       jl      bpf_error       /* offset lower -> error  */
+sk_load_word_negative_offset:
+       .globl  sk_load_word_negative_offset
+       sk_negative_common(4)
+       mov     (%rax), %eax
+       bswap   %eax
+       ret
+
+bpf_slow_path_half_neg:
+       cmp     SKF_MAX_NEG_OFF, %esi
+       jl      bpf_error
+sk_load_half_negative_offset:
+       .globl  sk_load_half_negative_offset
+       sk_negative_common(2)
+       mov     (%rax),%ax
+       rol     $8,%ax
+       movzwl  %ax,%eax
+       ret
+
+bpf_slow_path_byte_neg:
+       cmp     SKF_MAX_NEG_OFF, %esi
+       jl      bpf_error
+sk_load_byte_negative_offset:
+       .globl  sk_load_byte_negative_offset
+       sk_negative_common(1)
+       movzbl  (%rax), %eax
+       ret
+
+bpf_slow_path_byte_msh_neg:
+       cmp     SKF_MAX_NEG_OFF, %esi
+       jl      bpf_error
+sk_load_byte_msh_negative_offset:
+       .globl  sk_load_byte_msh_negative_offset
+       xchg    %eax,%ebx /* dont lose A , X is about to be scratched */
+       sk_negative_common(1)
+       movzbl  (%rax),%eax
+       and     $15,%al
+       shl     $2,%al
+       xchg    %eax,%ebx
+       ret
+
+bpf_error:
+# force a return 0 from jit handler
+       xor             %eax,%eax
+       mov             -8(%rbp),%rbx
+       leaveq
+       ret