- case BPF_ST: /* mem[K] = A */
- jit->seen |= SEEN_MEM;
- /* st %r5,<K>(%r15) */
- EMIT4_DISP(0x5050f000,
- (jit->seen & SEEN_DATAREF) ? 160 + K*4 : K*4);
- break;
- case BPF_STX: /* mem[K] = X : mov %ebx,off8(%rbp) */
- jit->seen |= SEEN_XREG | SEEN_MEM;
- /* st %r12,<K>(%r15) */
- EMIT4_DISP(0x50c0f000,
- (jit->seen & SEEN_DATAREF) ? 160 + K*4 : K*4);
- break;
- case BPF_ANC | SKF_AD_PROTOCOL: /* A = ntohs(skb->protocol); */
- BUILD_BUG_ON(FIELD_SIZEOF(struct sk_buff, protocol) != 2);
- /* lhi %r5,0 */
- EMIT4(0xa7580000);
- /* icm %r5,3,<d(protocol)>(%r2) */
- EMIT4_DISP(0xbf532000, offsetof(struct sk_buff, protocol));
- break;
- case BPF_ANC | SKF_AD_IFINDEX: /* if (!skb->dev) return 0;
- * A = skb->dev->ifindex */
- BUILD_BUG_ON(FIELD_SIZEOF(struct net_device, ifindex) != 4);
- jit->seen |= SEEN_RET0;
- /* lg %r1,<d(dev)>(%r2) */
- EMIT6_DISP(0xe3102000, 0x0004, offsetof(struct sk_buff, dev));
- /* ltgr %r1,%r1 */
- EMIT4(0xb9020011);
- /* jz <ret0> */
- EMIT4_PCREL(0xa7840000, jit->ret0_ip - jit->prg);
- /* l %r5,<d(ifindex)>(%r1) */
- EMIT4_DISP(0x58501000, offsetof(struct net_device, ifindex));
- break;
- case BPF_ANC | SKF_AD_MARK: /* A = skb->mark */
- BUILD_BUG_ON(FIELD_SIZEOF(struct sk_buff, mark) != 4);
- /* l %r5,<d(mark)>(%r2) */
- EMIT4_DISP(0x58502000, offsetof(struct sk_buff, mark));
- break;
- case BPF_ANC | SKF_AD_QUEUE: /* A = skb->queue_mapping */
- BUILD_BUG_ON(FIELD_SIZEOF(struct sk_buff, queue_mapping) != 2);
- /* lhi %r5,0 */
- EMIT4(0xa7580000);
- /* icm %r5,3,<d(queue_mapping)>(%r2) */
- EMIT4_DISP(0xbf532000, offsetof(struct sk_buff, queue_mapping));
- break;
- case BPF_ANC | SKF_AD_HATYPE: /* if (!skb->dev) return 0;
- * A = skb->dev->type */
- BUILD_BUG_ON(FIELD_SIZEOF(struct net_device, type) != 2);
- jit->seen |= SEEN_RET0;
- /* lg %r1,<d(dev)>(%r2) */
- EMIT6_DISP(0xe3102000, 0x0004, offsetof(struct sk_buff, dev));
- /* ltgr %r1,%r1 */
- EMIT4(0xb9020011);
- /* jz <ret0> */
- EMIT4_PCREL(0xa7840000, jit->ret0_ip - jit->prg);
- /* lhi %r5,0 */
- EMIT4(0xa7580000);
- /* icm %r5,3,<d(type)>(%r1) */
- EMIT4_DISP(0xbf531000, offsetof(struct net_device, type));
- break;
- case BPF_ANC | SKF_AD_RXHASH: /* A = skb->hash */
- BUILD_BUG_ON(FIELD_SIZEOF(struct sk_buff, hash) != 4);
- /* l %r5,<d(hash)>(%r2) */
- EMIT4_DISP(0x58502000, offsetof(struct sk_buff, hash));
- break;
- case BPF_ANC | SKF_AD_VLAN_TAG:
- case BPF_ANC | SKF_AD_VLAN_TAG_PRESENT:
- BUILD_BUG_ON(FIELD_SIZEOF(struct sk_buff, vlan_tci) != 2);
- BUILD_BUG_ON(VLAN_TAG_PRESENT != 0x1000);
- /* lhi %r5,0 */
- EMIT4(0xa7580000);
- /* icm %r5,3,<d(vlan_tci)>(%r2) */
- EMIT4_DISP(0xbf532000, offsetof(struct sk_buff, vlan_tci));
- if (code == (BPF_ANC | SKF_AD_VLAN_TAG)) {
- /* nill %r5,0xefff */
- EMIT4_IMM(0xa5570000, ~VLAN_TAG_PRESENT);
- } else {
- /* nill %r5,0x1000 */
- EMIT4_IMM(0xa5570000, VLAN_TAG_PRESENT);
- /* srl %r5,12 */
- EMIT4_DISP(0x88500000, 12);
- }
+ /*
+ * Branch relative (number of skipped instructions) to offset on
+ * condition.
+ *
+ * Condition code to mask mapping:
+ *
+ * CC | Description | Mask
+ * ------------------------------
+ * 0 | Operands equal | 8
+ * 1 | First operand low | 4
+ * 2 | First operand high | 2
+ * 3 | Unused | 1
+ *
+ * For s390x relative branches: ip = ip + off_bytes
+ * For BPF relative branches: insn = insn + off_insns + 1
+ *
+ * For example for s390x with offset 0 we jump to the branch
+ * instruction itself (loop) and for BPF with offset 0 we
+ * branch to the instruction behind the branch.
+ */
+ case BPF_JMP | BPF_JA: /* if (true) */
+ mask = 0xf000; /* j */
+ goto branch_oc;
+ case BPF_JMP | BPF_JSGT | BPF_K: /* ((s64) dst > (s64) imm) */
+ mask = 0x2000; /* jh */
+ goto branch_ks;
+ case BPF_JMP | BPF_JSGE | BPF_K: /* ((s64) dst >= (s64) imm) */
+ mask = 0xa000; /* jhe */
+ goto branch_ks;
+ case BPF_JMP | BPF_JGT | BPF_K: /* (dst_reg > imm) */
+ mask = 0x2000; /* jh */
+ goto branch_ku;
+ case BPF_JMP | BPF_JGE | BPF_K: /* (dst_reg >= imm) */
+ mask = 0xa000; /* jhe */
+ goto branch_ku;
+ case BPF_JMP | BPF_JNE | BPF_K: /* (dst_reg != imm) */
+ mask = 0x7000; /* jne */
+ goto branch_ku;
+ case BPF_JMP | BPF_JEQ | BPF_K: /* (dst_reg == imm) */
+ mask = 0x8000; /* je */
+ goto branch_ku;
+ case BPF_JMP | BPF_JSET | BPF_K: /* (dst_reg & imm) */
+ mask = 0x7000; /* jnz */
+ /* lgfi %w1,imm (load sign extend imm) */
+ EMIT6_IMM(0xc0010000, REG_W1, imm);
+ /* ngr %w1,%dst */
+ EMIT4(0xb9800000, REG_W1, dst_reg);
+ goto branch_oc;
+
+ case BPF_JMP | BPF_JSGT | BPF_X: /* ((s64) dst > (s64) src) */
+ mask = 0x2000; /* jh */
+ goto branch_xs;
+ case BPF_JMP | BPF_JSGE | BPF_X: /* ((s64) dst >= (s64) src) */
+ mask = 0xa000; /* jhe */
+ goto branch_xs;
+ case BPF_JMP | BPF_JGT | BPF_X: /* (dst > src) */
+ mask = 0x2000; /* jh */
+ goto branch_xu;
+ case BPF_JMP | BPF_JGE | BPF_X: /* (dst >= src) */
+ mask = 0xa000; /* jhe */
+ goto branch_xu;
+ case BPF_JMP | BPF_JNE | BPF_X: /* (dst != src) */
+ mask = 0x7000; /* jne */
+ goto branch_xu;
+ case BPF_JMP | BPF_JEQ | BPF_X: /* (dst == src) */
+ mask = 0x8000; /* je */
+ goto branch_xu;
+ case BPF_JMP | BPF_JSET | BPF_X: /* (dst & src) */
+ mask = 0x7000; /* jnz */
+ /* ngrk %w1,%dst,%src */
+ EMIT4_RRF(0xb9e40000, REG_W1, dst_reg, src_reg);
+ goto branch_oc;
+branch_ks:
+ /* lgfi %w1,imm (load sign extend imm) */
+ EMIT6_IMM(0xc0010000, REG_W1, imm);
+ /* cgrj %dst,%w1,mask,off */
+ EMIT6_PCREL(0xec000000, 0x0064, dst_reg, REG_W1, i, off, mask);
+ break;
+branch_ku:
+ /* lgfi %w1,imm (load sign extend imm) */
+ EMIT6_IMM(0xc0010000, REG_W1, imm);
+ /* clgrj %dst,%w1,mask,off */
+ EMIT6_PCREL(0xec000000, 0x0065, dst_reg, REG_W1, i, off, mask);
+ break;
+branch_xs:
+ /* cgrj %dst,%src,mask,off */
+ EMIT6_PCREL(0xec000000, 0x0064, dst_reg, src_reg, i, off, mask);
+ break;
+branch_xu:
+ /* clgrj %dst,%src,mask,off */
+ EMIT6_PCREL(0xec000000, 0x0065, dst_reg, src_reg, i, off, mask);
+ break;
+branch_oc:
+ /* brc mask,jmp_off (branch instruction needs 4 bytes) */
+ jmp_off = addrs[i + off + 1] - (addrs[i + 1] - 4);
+ EMIT4_PCREL(0xa7040000 | mask << 8, jmp_off);