1 /******************************************************************************
4 * Generic x86 (32-bit and 64-bit) instruction decoder and emulator.
6 * Copyright (c) 2005 Keir Fraser
8 * Linux coding style, mod r/m decoder, segment base fixes, real-mode
9 * privileged instructions:
11 * Copyright (C) 2006 Qumranet
12 * Copyright 2010 Red Hat, Inc. and/or its affiliates.
14 * Avi Kivity <avi@qumranet.com>
15 * Yaniv Kamay <yaniv@qumranet.com>
17 * This work is licensed under the terms of the GNU GPL, version 2. See
18 * the COPYING file in the top-level directory.
20 * From: xen-unstable 10676:af9809f51f81a3c43f276f00c81a52ef558afda4
23 #include <linux/kvm_host.h>
24 #include "kvm_cache_regs.h"
25 #include <linux/module.h>
26 #include <asm/kvm_emulate.h>
27 #include <linux/stringify.h>
36 #define OpImplicit 1ull /* No generic decode */
37 #define OpReg 2ull /* Register */
38 #define OpMem 3ull /* Memory */
39 #define OpAcc 4ull /* Accumulator: AL/AX/EAX/RAX */
40 #define OpDI 5ull /* ES:DI/EDI/RDI */
41 #define OpMem64 6ull /* Memory, 64-bit */
42 #define OpImmUByte 7ull /* Zero-extended 8-bit immediate */
43 #define OpDX 8ull /* DX register */
44 #define OpCL 9ull /* CL register (for shifts) */
45 #define OpImmByte 10ull /* 8-bit sign extended immediate */
46 #define OpOne 11ull /* Implied 1 */
47 #define OpImm 12ull /* Sign extended up to 32-bit immediate */
48 #define OpMem16 13ull /* Memory operand (16-bit). */
49 #define OpMem32 14ull /* Memory operand (32-bit). */
50 #define OpImmU 15ull /* Immediate operand, zero extended */
51 #define OpSI 16ull /* SI/ESI/RSI */
52 #define OpImmFAddr 17ull /* Immediate far address */
53 #define OpMemFAddr 18ull /* Far address in memory */
54 #define OpImmU16 19ull /* Immediate operand, 16 bits, zero extended */
55 #define OpES 20ull /* ES */
56 #define OpCS 21ull /* CS */
57 #define OpSS 22ull /* SS */
58 #define OpDS 23ull /* DS */
59 #define OpFS 24ull /* FS */
60 #define OpGS 25ull /* GS */
61 #define OpMem8 26ull /* 8-bit zero extended memory operand */
62 #define OpImm64 27ull /* Sign extended 16/32/64-bit immediate */
64 #define OpBits 5 /* Width of operand field */
65 #define OpMask ((1ull << OpBits) - 1)
68 * Opcode effective-address decode tables.
69 * Note that we only emulate instructions that have at least one memory
70 * operand (excluding implicit stack references). We assume that stack
71 * references and instruction fetches will never occur in special memory
72 * areas that require emulation. So, for example, 'mov <imm>,<reg>' need
76 /* Operand sizes: 8-bit operands or specified/overridden size. */
77 #define ByteOp (1<<0) /* 8-bit operands. */
78 /* Destination operand type. */
80 #define ImplicitOps (OpImplicit << DstShift)
81 #define DstReg (OpReg << DstShift)
82 #define DstMem (OpMem << DstShift)
83 #define DstAcc (OpAcc << DstShift)
84 #define DstDI (OpDI << DstShift)
85 #define DstMem64 (OpMem64 << DstShift)
86 #define DstImmUByte (OpImmUByte << DstShift)
87 #define DstDX (OpDX << DstShift)
88 #define DstMask (OpMask << DstShift)
89 /* Source operand type. */
91 #define SrcNone (OpNone << SrcShift)
92 #define SrcReg (OpReg << SrcShift)
93 #define SrcMem (OpMem << SrcShift)
94 #define SrcMem16 (OpMem16 << SrcShift)
95 #define SrcMem32 (OpMem32 << SrcShift)
96 #define SrcImm (OpImm << SrcShift)
97 #define SrcImmByte (OpImmByte << SrcShift)
98 #define SrcOne (OpOne << SrcShift)
99 #define SrcImmUByte (OpImmUByte << SrcShift)
100 #define SrcImmU (OpImmU << SrcShift)
101 #define SrcSI (OpSI << SrcShift)
102 #define SrcImmFAddr (OpImmFAddr << SrcShift)
103 #define SrcMemFAddr (OpMemFAddr << SrcShift)
104 #define SrcAcc (OpAcc << SrcShift)
105 #define SrcImmU16 (OpImmU16 << SrcShift)
106 #define SrcImm64 (OpImm64 << SrcShift)
107 #define SrcDX (OpDX << SrcShift)
108 #define SrcMem8 (OpMem8 << SrcShift)
109 #define SrcMask (OpMask << SrcShift)
110 #define BitOp (1<<11)
111 #define MemAbs (1<<12) /* Memory operand is absolute displacement */
112 #define String (1<<13) /* String instruction (rep capable) */
113 #define Stack (1<<14) /* Stack instruction (push/pop) */
114 #define GroupMask (7<<15) /* Opcode uses one of the group mechanisms */
115 #define Group (1<<15) /* Bits 3:5 of modrm byte extend opcode */
116 #define GroupDual (2<<15) /* Alternate decoding of mod == 3 */
117 #define Prefix (3<<15) /* Instruction varies with 66/f2/f3 prefix */
118 #define RMExt (4<<15) /* Opcode extension in ModRM r/m if mod == 3 */
119 #define Escape (5<<15) /* Escape to coprocessor instruction */
120 #define Sse (1<<18) /* SSE Vector instruction */
121 /* Generic ModRM decode. */
122 #define ModRM (1<<19)
123 /* Destination is only written; never read. */
126 #define Prot (1<<21) /* instruction generates #UD if not in prot-mode */
127 #define VendorSpecific (1<<22) /* Vendor specific instruction */
128 #define NoAccess (1<<23) /* Don't access memory (lea/invlpg/verr etc) */
129 #define Op3264 (1<<24) /* Operand is 64b in long mode, 32b otherwise */
130 #define Undefined (1<<25) /* No Such Instruction */
131 #define Lock (1<<26) /* lock prefix is allowed for the instruction */
132 #define Priv (1<<27) /* instruction generates #GP if current CPL != 0 */
134 #define PageTable (1 << 29) /* instruction used to write page table */
135 /* Source 2 operand type */
136 #define Src2Shift (30)
137 #define Src2None (OpNone << Src2Shift)
138 #define Src2CL (OpCL << Src2Shift)
139 #define Src2ImmByte (OpImmByte << Src2Shift)
140 #define Src2One (OpOne << Src2Shift)
141 #define Src2Imm (OpImm << Src2Shift)
142 #define Src2ES (OpES << Src2Shift)
143 #define Src2CS (OpCS << Src2Shift)
144 #define Src2SS (OpSS << Src2Shift)
145 #define Src2DS (OpDS << Src2Shift)
146 #define Src2FS (OpFS << Src2Shift)
147 #define Src2GS (OpGS << Src2Shift)
148 #define Src2Mask (OpMask << Src2Shift)
149 #define Mmx ((u64)1 << 40) /* MMX Vector instruction */
150 #define Aligned ((u64)1 << 41) /* Explicitly aligned (e.g. MOVDQA) */
151 #define Unaligned ((u64)1 << 42) /* Explicitly unaligned (e.g. MOVDQU) */
152 #define Avx ((u64)1 << 43) /* Advanced Vector Extensions */
153 #define Fastop ((u64)1 << 44) /* Use opcode::u.fastop */
154 #define NoWrite ((u64)1 << 45) /* No writeback */
156 #define X2(x...) x, x
157 #define X3(x...) X2(x), x
158 #define X4(x...) X2(x), X2(x)
159 #define X5(x...) X4(x), x
160 #define X6(x...) X4(x), X2(x)
161 #define X7(x...) X4(x), X3(x)
162 #define X8(x...) X4(x), X4(x)
163 #define X16(x...) X8(x), X8(x)
165 #define NR_FASTOP (ilog2(sizeof(ulong)) + 1)
166 #define FASTOP_SIZE 8
169 * fastop functions have a special calling convention:
171 * dst: [rdx]:rax (in/out)
174 * flags: rflags (in/out)
176 * Moreover, they are all exactly FASTOP_SIZE bytes long, so functions for
177 * different operand sizes can be reached by calculation, rather than a jump
178 * table (which would be bigger than the code).
180 * fastop functions are declared as taking a never-defined fastop parameter,
181 * so they can't be called from C directly.
190 int (*execute)(struct x86_emulate_ctxt *ctxt);
191 const struct opcode *group;
192 const struct group_dual *gdual;
193 const struct gprefix *gprefix;
194 const struct escape *esc;
195 void (*fastop)(struct fastop *fake);
197 int (*check_perm)(struct x86_emulate_ctxt *ctxt);
201 struct opcode mod012[8];
202 struct opcode mod3[8];
206 struct opcode pfx_no;
207 struct opcode pfx_66;
208 struct opcode pfx_f2;
209 struct opcode pfx_f3;
214 struct opcode high[64];
217 /* EFLAGS bit definitions. */
218 #define EFLG_ID (1<<21)
219 #define EFLG_VIP (1<<20)
220 #define EFLG_VIF (1<<19)
221 #define EFLG_AC (1<<18)
222 #define EFLG_VM (1<<17)
223 #define EFLG_RF (1<<16)
224 #define EFLG_IOPL (3<<12)
225 #define EFLG_NT (1<<14)
226 #define EFLG_OF (1<<11)
227 #define EFLG_DF (1<<10)
228 #define EFLG_IF (1<<9)
229 #define EFLG_TF (1<<8)
230 #define EFLG_SF (1<<7)
231 #define EFLG_ZF (1<<6)
232 #define EFLG_AF (1<<4)
233 #define EFLG_PF (1<<2)
234 #define EFLG_CF (1<<0)
236 #define EFLG_RESERVED_ZEROS_MASK 0xffc0802a
237 #define EFLG_RESERVED_ONE_MASK 2
239 static ulong reg_read(struct x86_emulate_ctxt *ctxt, unsigned nr)
241 if (!(ctxt->regs_valid & (1 << nr))) {
242 ctxt->regs_valid |= 1 << nr;
243 ctxt->_regs[nr] = ctxt->ops->read_gpr(ctxt, nr);
245 return ctxt->_regs[nr];
248 static ulong *reg_write(struct x86_emulate_ctxt *ctxt, unsigned nr)
250 ctxt->regs_valid |= 1 << nr;
251 ctxt->regs_dirty |= 1 << nr;
252 return &ctxt->_regs[nr];
255 static ulong *reg_rmw(struct x86_emulate_ctxt *ctxt, unsigned nr)
258 return reg_write(ctxt, nr);
261 static void writeback_registers(struct x86_emulate_ctxt *ctxt)
265 for_each_set_bit(reg, (ulong *)&ctxt->regs_dirty, 16)
266 ctxt->ops->write_gpr(ctxt, reg, ctxt->_regs[reg]);
269 static void invalidate_registers(struct x86_emulate_ctxt *ctxt)
271 ctxt->regs_dirty = 0;
272 ctxt->regs_valid = 0;
276 * Instruction emulation:
277 * Most instructions are emulated directly via a fragment of inline assembly
278 * code. This allows us to save/restore EFLAGS and thus very easily pick up
279 * any modified flags.
282 #if defined(CONFIG_X86_64)
283 #define _LO32 "k" /* force 32-bit operand */
284 #define _STK "%%rsp" /* stack pointer */
285 #elif defined(__i386__)
286 #define _LO32 "" /* force 32-bit operand */
287 #define _STK "%%esp" /* stack pointer */
291 * These EFLAGS bits are restored from saved value during emulation, and
292 * any changes are written back to the saved value after emulation.
294 #define EFLAGS_MASK (EFLG_OF|EFLG_SF|EFLG_ZF|EFLG_AF|EFLG_PF|EFLG_CF)
296 /* Before executing instruction: restore necessary bits in EFLAGS. */
297 #define _PRE_EFLAGS(_sav, _msk, _tmp) \
298 /* EFLAGS = (_sav & _msk) | (EFLAGS & ~_msk); _sav &= ~_msk; */ \
299 "movl %"_sav",%"_LO32 _tmp"; " \
302 "movl %"_msk",%"_LO32 _tmp"; " \
303 "andl %"_LO32 _tmp",("_STK"); " \
305 "notl %"_LO32 _tmp"; " \
306 "andl %"_LO32 _tmp",("_STK"); " \
307 "andl %"_LO32 _tmp","__stringify(BITS_PER_LONG/4)"("_STK"); " \
309 "orl %"_LO32 _tmp",("_STK"); " \
313 /* After executing instruction: write-back necessary bits in EFLAGS. */
314 #define _POST_EFLAGS(_sav, _msk, _tmp) \
315 /* _sav |= EFLAGS & _msk; */ \
318 "andl %"_msk",%"_LO32 _tmp"; " \
319 "orl %"_LO32 _tmp",%"_sav"; "
327 #define ____emulate_2op(ctxt, _op, _x, _y, _suffix, _dsttype) \
329 __asm__ __volatile__ ( \
330 _PRE_EFLAGS("0", "4", "2") \
331 _op _suffix " %"_x"3,%1; " \
332 _POST_EFLAGS("0", "4", "2") \
333 : "=m" ((ctxt)->eflags), \
334 "+q" (*(_dsttype*)&(ctxt)->dst.val), \
336 : _y ((ctxt)->src.val), "i" (EFLAGS_MASK)); \
340 /* Raw emulation: instruction has two explicit operands. */
341 #define __emulate_2op_nobyte(ctxt,_op,_wx,_wy,_lx,_ly,_qx,_qy) \
343 unsigned long _tmp; \
345 switch ((ctxt)->dst.bytes) { \
347 ____emulate_2op(ctxt,_op,_wx,_wy,"w",u16); \
350 ____emulate_2op(ctxt,_op,_lx,_ly,"l",u32); \
353 ON64(____emulate_2op(ctxt,_op,_qx,_qy,"q",u64)); \
358 #define __emulate_2op(ctxt,_op,_bx,_by,_wx,_wy,_lx,_ly,_qx,_qy) \
360 unsigned long _tmp; \
361 switch ((ctxt)->dst.bytes) { \
363 ____emulate_2op(ctxt,_op,_bx,_by,"b",u8); \
366 __emulate_2op_nobyte(ctxt, _op, \
367 _wx, _wy, _lx, _ly, _qx, _qy); \
372 /* Source operand is byte-sized and may be restricted to just %cl. */
373 #define emulate_2op_SrcB(ctxt, _op) \
374 __emulate_2op(ctxt, _op, "b", "c", "b", "c", "b", "c", "b", "c")
376 /* Source operand is byte, word, long or quad sized. */
377 #define emulate_2op_SrcV(ctxt, _op) \
378 __emulate_2op(ctxt, _op, "b", "q", "w", "r", _LO32, "r", "", "r")
380 /* Source operand is word, long or quad sized. */
381 #define emulate_2op_SrcV_nobyte(ctxt, _op) \
382 __emulate_2op_nobyte(ctxt, _op, "w", "r", _LO32, "r", "", "r")
384 /* Instruction has three operands and one operand is stored in ECX register */
385 #define __emulate_2op_cl(ctxt, _op, _suffix, _type) \
387 unsigned long _tmp; \
388 _type _clv = (ctxt)->src2.val; \
389 _type _srcv = (ctxt)->src.val; \
390 _type _dstv = (ctxt)->dst.val; \
392 __asm__ __volatile__ ( \
393 _PRE_EFLAGS("0", "5", "2") \
394 _op _suffix " %4,%1 \n" \
395 _POST_EFLAGS("0", "5", "2") \
396 : "=m" ((ctxt)->eflags), "+r" (_dstv), "=&r" (_tmp) \
397 : "c" (_clv) , "r" (_srcv), "i" (EFLAGS_MASK) \
400 (ctxt)->src2.val = (unsigned long) _clv; \
401 (ctxt)->src2.val = (unsigned long) _srcv; \
402 (ctxt)->dst.val = (unsigned long) _dstv; \
405 #define emulate_2op_cl(ctxt, _op) \
407 switch ((ctxt)->dst.bytes) { \
409 __emulate_2op_cl(ctxt, _op, "w", u16); \
412 __emulate_2op_cl(ctxt, _op, "l", u32); \
415 ON64(__emulate_2op_cl(ctxt, _op, "q", ulong)); \
420 #define __emulate_1op(ctxt, _op, _suffix) \
422 unsigned long _tmp; \
424 __asm__ __volatile__ ( \
425 _PRE_EFLAGS("0", "3", "2") \
426 _op _suffix " %1; " \
427 _POST_EFLAGS("0", "3", "2") \
428 : "=m" ((ctxt)->eflags), "+m" ((ctxt)->dst.val), \
430 : "i" (EFLAGS_MASK)); \
433 /* Instruction has only one explicit operand (no source operand). */
434 #define emulate_1op(ctxt, _op) \
436 switch ((ctxt)->dst.bytes) { \
437 case 1: __emulate_1op(ctxt, _op, "b"); break; \
438 case 2: __emulate_1op(ctxt, _op, "w"); break; \
439 case 4: __emulate_1op(ctxt, _op, "l"); break; \
440 case 8: ON64(__emulate_1op(ctxt, _op, "q")); break; \
444 static int fastop(struct x86_emulate_ctxt *ctxt, void (*fop)(struct fastop *));
446 #define FOP_ALIGN ".align " __stringify(FASTOP_SIZE) " \n\t"
447 #define FOP_RET "ret \n\t"
449 #define FOP_START(op) \
450 extern void em_##op(struct fastop *fake); \
451 asm(".pushsection .text, \"ax\" \n\t" \
452 ".global em_" #op " \n\t" \
459 #define FOPNOP() FOP_ALIGN FOP_RET
461 #define FOP1E(op, dst) \
462 FOP_ALIGN #op " %" #dst " \n\t" FOP_RET
464 #define FASTOP1(op) \
469 ON64(FOP1E(op##q, rax)) \
472 #define FOP2E(op, dst, src) \
473 FOP_ALIGN #op " %" #src ", %" #dst " \n\t" FOP_RET
475 #define FASTOP2(op) \
477 FOP2E(op##b, al, bl) \
478 FOP2E(op##w, ax, bx) \
479 FOP2E(op##l, eax, ebx) \
480 ON64(FOP2E(op##q, rax, rbx)) \
483 /* 2 operand, word only */
484 #define FASTOP2W(op) \
487 FOP2E(op##w, ax, bx) \
488 FOP2E(op##l, eax, ebx) \
489 ON64(FOP2E(op##q, rax, rbx)) \
492 /* 2 operand, src is CL */
493 #define FASTOP2CL(op) \
495 FOP2E(op##b, al, cl) \
496 FOP2E(op##w, ax, cl) \
497 FOP2E(op##l, eax, cl) \
498 ON64(FOP2E(op##q, rax, cl)) \
501 #define FOP3E(op, dst, src, src2) \
502 FOP_ALIGN #op " %" #src2 ", %" #src ", %" #dst " \n\t" FOP_RET
504 /* 3-operand, word-only, src2=cl */
505 #define FASTOP3WCL(op) \
508 FOP3E(op##w, ax, bx, cl) \
509 FOP3E(op##l, eax, ebx, cl) \
510 ON64(FOP3E(op##q, rax, rbx, cl)) \
513 /* Special case for SETcc - 1 instruction per cc */
514 #define FOP_SETCC(op) ".align 4; " #op " %al; ret \n\t"
535 #define __emulate_1op_rax_rdx(ctxt, _op, _suffix, _ex) \
537 unsigned long _tmp; \
538 ulong *rax = reg_rmw((ctxt), VCPU_REGS_RAX); \
539 ulong *rdx = reg_rmw((ctxt), VCPU_REGS_RDX); \
541 __asm__ __volatile__ ( \
542 _PRE_EFLAGS("0", "5", "1") \
544 _op _suffix " %6; " \
546 _POST_EFLAGS("0", "5", "1") \
547 ".pushsection .fixup,\"ax\" \n\t" \
548 "3: movb $1, %4 \n\t" \
551 _ASM_EXTABLE(1b, 3b) \
552 : "=m" ((ctxt)->eflags), "=&r" (_tmp), \
553 "+a" (*rax), "+d" (*rdx), "+qm"(_ex) \
554 : "i" (EFLAGS_MASK), "m" ((ctxt)->src.val)); \
557 /* instruction has only one source operand, destination is implicit (e.g. mul, div, imul, idiv) */
558 #define emulate_1op_rax_rdx(ctxt, _op, _ex) \
560 switch((ctxt)->src.bytes) { \
562 __emulate_1op_rax_rdx(ctxt, _op, "b", _ex); \
565 __emulate_1op_rax_rdx(ctxt, _op, "w", _ex); \
568 __emulate_1op_rax_rdx(ctxt, _op, "l", _ex); \
571 __emulate_1op_rax_rdx(ctxt, _op, "q", _ex)); \
576 static int emulator_check_intercept(struct x86_emulate_ctxt *ctxt,
577 enum x86_intercept intercept,
578 enum x86_intercept_stage stage)
580 struct x86_instruction_info info = {
581 .intercept = intercept,
582 .rep_prefix = ctxt->rep_prefix,
583 .modrm_mod = ctxt->modrm_mod,
584 .modrm_reg = ctxt->modrm_reg,
585 .modrm_rm = ctxt->modrm_rm,
586 .src_val = ctxt->src.val64,
587 .src_bytes = ctxt->src.bytes,
588 .dst_bytes = ctxt->dst.bytes,
589 .ad_bytes = ctxt->ad_bytes,
590 .next_rip = ctxt->eip,
593 return ctxt->ops->intercept(ctxt, &info, stage);
596 static void assign_masked(ulong *dest, ulong src, ulong mask)
598 *dest = (*dest & ~mask) | (src & mask);
601 static inline unsigned long ad_mask(struct x86_emulate_ctxt *ctxt)
603 return (1UL << (ctxt->ad_bytes << 3)) - 1;
606 static ulong stack_mask(struct x86_emulate_ctxt *ctxt)
609 struct desc_struct ss;
611 if (ctxt->mode == X86EMUL_MODE_PROT64)
613 ctxt->ops->get_segment(ctxt, &sel, &ss, NULL, VCPU_SREG_SS);
614 return ~0U >> ((ss.d ^ 1) * 16); /* d=0: 0xffff; d=1: 0xffffffff */
617 static int stack_size(struct x86_emulate_ctxt *ctxt)
619 return (__fls(stack_mask(ctxt)) + 1) >> 3;
622 /* Access/update address held in a register, based on addressing mode. */
623 static inline unsigned long
624 address_mask(struct x86_emulate_ctxt *ctxt, unsigned long reg)
626 if (ctxt->ad_bytes == sizeof(unsigned long))
629 return reg & ad_mask(ctxt);
632 static inline unsigned long
633 register_address(struct x86_emulate_ctxt *ctxt, unsigned long reg)
635 return address_mask(ctxt, reg);
638 static void masked_increment(ulong *reg, ulong mask, int inc)
640 assign_masked(reg, *reg + inc, mask);
644 register_address_increment(struct x86_emulate_ctxt *ctxt, unsigned long *reg, int inc)
648 if (ctxt->ad_bytes == sizeof(unsigned long))
651 mask = ad_mask(ctxt);
652 masked_increment(reg, mask, inc);
655 static void rsp_increment(struct x86_emulate_ctxt *ctxt, int inc)
657 masked_increment(reg_rmw(ctxt, VCPU_REGS_RSP), stack_mask(ctxt), inc);
660 static inline void jmp_rel(struct x86_emulate_ctxt *ctxt, int rel)
662 register_address_increment(ctxt, &ctxt->_eip, rel);
665 static u32 desc_limit_scaled(struct desc_struct *desc)
667 u32 limit = get_desc_limit(desc);
669 return desc->g ? (limit << 12) | 0xfff : limit;
672 static void set_seg_override(struct x86_emulate_ctxt *ctxt, int seg)
674 ctxt->has_seg_override = true;
675 ctxt->seg_override = seg;
678 static unsigned long seg_base(struct x86_emulate_ctxt *ctxt, int seg)
680 if (ctxt->mode == X86EMUL_MODE_PROT64 && seg < VCPU_SREG_FS)
683 return ctxt->ops->get_cached_segment_base(ctxt, seg);
686 static unsigned seg_override(struct x86_emulate_ctxt *ctxt)
688 if (!ctxt->has_seg_override)
691 return ctxt->seg_override;
694 static int emulate_exception(struct x86_emulate_ctxt *ctxt, int vec,
695 u32 error, bool valid)
697 ctxt->exception.vector = vec;
698 ctxt->exception.error_code = error;
699 ctxt->exception.error_code_valid = valid;
700 return X86EMUL_PROPAGATE_FAULT;
703 static int emulate_db(struct x86_emulate_ctxt *ctxt)
705 return emulate_exception(ctxt, DB_VECTOR, 0, false);
708 static int emulate_gp(struct x86_emulate_ctxt *ctxt, int err)
710 return emulate_exception(ctxt, GP_VECTOR, err, true);
713 static int emulate_ss(struct x86_emulate_ctxt *ctxt, int err)
715 return emulate_exception(ctxt, SS_VECTOR, err, true);
718 static int emulate_ud(struct x86_emulate_ctxt *ctxt)
720 return emulate_exception(ctxt, UD_VECTOR, 0, false);
723 static int emulate_ts(struct x86_emulate_ctxt *ctxt, int err)
725 return emulate_exception(ctxt, TS_VECTOR, err, true);
728 static int emulate_de(struct x86_emulate_ctxt *ctxt)
730 return emulate_exception(ctxt, DE_VECTOR, 0, false);
733 static int emulate_nm(struct x86_emulate_ctxt *ctxt)
735 return emulate_exception(ctxt, NM_VECTOR, 0, false);
738 static u16 get_segment_selector(struct x86_emulate_ctxt *ctxt, unsigned seg)
741 struct desc_struct desc;
743 ctxt->ops->get_segment(ctxt, &selector, &desc, NULL, seg);
747 static void set_segment_selector(struct x86_emulate_ctxt *ctxt, u16 selector,
752 struct desc_struct desc;
754 ctxt->ops->get_segment(ctxt, &dummy, &desc, &base3, seg);
755 ctxt->ops->set_segment(ctxt, selector, &desc, base3, seg);
759 * x86 defines three classes of vector instructions: explicitly
760 * aligned, explicitly unaligned, and the rest, which change behaviour
761 * depending on whether they're AVX encoded or not.
763 * Also included is CMPXCHG16B which is not a vector instruction, yet it is
764 * subject to the same check.
766 static bool insn_aligned(struct x86_emulate_ctxt *ctxt, unsigned size)
768 if (likely(size < 16))
771 if (ctxt->d & Aligned)
773 else if (ctxt->d & Unaligned)
775 else if (ctxt->d & Avx)
781 static int __linearize(struct x86_emulate_ctxt *ctxt,
782 struct segmented_address addr,
783 unsigned size, bool write, bool fetch,
786 struct desc_struct desc;
793 la = seg_base(ctxt, addr.seg) + addr.ea;
794 switch (ctxt->mode) {
795 case X86EMUL_MODE_PROT64:
796 if (((signed long)la << 16) >> 16 != la)
797 return emulate_gp(ctxt, 0);
800 usable = ctxt->ops->get_segment(ctxt, &sel, &desc, NULL,
804 /* code segment in protected mode or read-only data segment */
805 if ((((ctxt->mode != X86EMUL_MODE_REAL) && (desc.type & 8))
806 || !(desc.type & 2)) && write)
808 /* unreadable code segment */
809 if (!fetch && (desc.type & 8) && !(desc.type & 2))
811 lim = desc_limit_scaled(&desc);
812 if ((desc.type & 8) || !(desc.type & 4)) {
813 /* expand-up segment */
814 if (addr.ea > lim || (u32)(addr.ea + size - 1) > lim)
817 /* expand-down segment */
818 if (addr.ea <= lim || (u32)(addr.ea + size - 1) <= lim)
820 lim = desc.d ? 0xffffffff : 0xffff;
821 if (addr.ea > lim || (u32)(addr.ea + size - 1) > lim)
824 cpl = ctxt->ops->cpl(ctxt);
825 if (!(desc.type & 8)) {
829 } else if ((desc.type & 8) && !(desc.type & 4)) {
830 /* nonconforming code segment */
833 } else if ((desc.type & 8) && (desc.type & 4)) {
834 /* conforming code segment */
840 if (fetch ? ctxt->mode != X86EMUL_MODE_PROT64 : ctxt->ad_bytes != 8)
842 if (insn_aligned(ctxt, size) && ((la & (size - 1)) != 0))
843 return emulate_gp(ctxt, 0);
845 return X86EMUL_CONTINUE;
847 if (addr.seg == VCPU_SREG_SS)
848 return emulate_ss(ctxt, sel);
850 return emulate_gp(ctxt, sel);
853 static int linearize(struct x86_emulate_ctxt *ctxt,
854 struct segmented_address addr,
855 unsigned size, bool write,
858 return __linearize(ctxt, addr, size, write, false, linear);
862 static int segmented_read_std(struct x86_emulate_ctxt *ctxt,
863 struct segmented_address addr,
870 rc = linearize(ctxt, addr, size, false, &linear);
871 if (rc != X86EMUL_CONTINUE)
873 return ctxt->ops->read_std(ctxt, linear, data, size, &ctxt->exception);
877 * Fetch the next byte of the instruction being emulated which is pointed to
878 * by ctxt->_eip, then increment ctxt->_eip.
880 * Also prefetch the remaining bytes of the instruction without crossing page
881 * boundary if they are not in fetch_cache yet.
883 static int do_insn_fetch_byte(struct x86_emulate_ctxt *ctxt, u8 *dest)
885 struct fetch_cache *fc = &ctxt->fetch;
889 if (ctxt->_eip == fc->end) {
890 unsigned long linear;
891 struct segmented_address addr = { .seg = VCPU_SREG_CS,
893 cur_size = fc->end - fc->start;
894 size = min(15UL - cur_size,
895 PAGE_SIZE - offset_in_page(ctxt->_eip));
896 rc = __linearize(ctxt, addr, size, false, true, &linear);
897 if (unlikely(rc != X86EMUL_CONTINUE))
899 rc = ctxt->ops->fetch(ctxt, linear, fc->data + cur_size,
900 size, &ctxt->exception);
901 if (unlikely(rc != X86EMUL_CONTINUE))
905 *dest = fc->data[ctxt->_eip - fc->start];
907 return X86EMUL_CONTINUE;
910 static int do_insn_fetch(struct x86_emulate_ctxt *ctxt,
911 void *dest, unsigned size)
915 /* x86 instructions are limited to 15 bytes. */
916 if (unlikely(ctxt->_eip + size - ctxt->eip > 15))
917 return X86EMUL_UNHANDLEABLE;
919 rc = do_insn_fetch_byte(ctxt, dest++);
920 if (rc != X86EMUL_CONTINUE)
923 return X86EMUL_CONTINUE;
926 /* Fetch next part of the instruction being emulated. */
927 #define insn_fetch(_type, _ctxt) \
928 ({ unsigned long _x; \
929 rc = do_insn_fetch(_ctxt, &_x, sizeof(_type)); \
930 if (rc != X86EMUL_CONTINUE) \
935 #define insn_fetch_arr(_arr, _size, _ctxt) \
936 ({ rc = do_insn_fetch(_ctxt, _arr, (_size)); \
937 if (rc != X86EMUL_CONTINUE) \
942 * Given the 'reg' portion of a ModRM byte, and a register block, return a
943 * pointer into the block that addresses the relevant register.
944 * @highbyte_regs specifies whether to decode AH,CH,DH,BH.
946 static void *decode_register(struct x86_emulate_ctxt *ctxt, u8 modrm_reg,
951 if (highbyte_regs && modrm_reg >= 4 && modrm_reg < 8)
952 p = (unsigned char *)reg_rmw(ctxt, modrm_reg & 3) + 1;
954 p = reg_rmw(ctxt, modrm_reg);
958 static int read_descriptor(struct x86_emulate_ctxt *ctxt,
959 struct segmented_address addr,
960 u16 *size, unsigned long *address, int op_bytes)
967 rc = segmented_read_std(ctxt, addr, size, 2);
968 if (rc != X86EMUL_CONTINUE)
971 rc = segmented_read_std(ctxt, addr, address, op_bytes);
1010 static u8 test_cc(unsigned int condition, unsigned long flags)
1013 void (*fop)(void) = (void *)em_setcc + 4 * (condition & 0xf);
1015 flags = (flags & EFLAGS_MASK) | X86_EFLAGS_IF;
1016 asm("pushq %[flags]; popf; call *%[fastop]"
1017 : "=a"(rc) : [fastop]"r"(fop), [flags]"r"(flags));
1021 static void fetch_register_operand(struct operand *op)
1023 switch (op->bytes) {
1025 op->val = *(u8 *)op->addr.reg;
1028 op->val = *(u16 *)op->addr.reg;
1031 op->val = *(u32 *)op->addr.reg;
1034 op->val = *(u64 *)op->addr.reg;
1039 static void read_sse_reg(struct x86_emulate_ctxt *ctxt, sse128_t *data, int reg)
1041 ctxt->ops->get_fpu(ctxt);
1043 case 0: asm("movdqa %%xmm0, %0" : "=m"(*data)); break;
1044 case 1: asm("movdqa %%xmm1, %0" : "=m"(*data)); break;
1045 case 2: asm("movdqa %%xmm2, %0" : "=m"(*data)); break;
1046 case 3: asm("movdqa %%xmm3, %0" : "=m"(*data)); break;
1047 case 4: asm("movdqa %%xmm4, %0" : "=m"(*data)); break;
1048 case 5: asm("movdqa %%xmm5, %0" : "=m"(*data)); break;
1049 case 6: asm("movdqa %%xmm6, %0" : "=m"(*data)); break;
1050 case 7: asm("movdqa %%xmm7, %0" : "=m"(*data)); break;
1051 #ifdef CONFIG_X86_64
1052 case 8: asm("movdqa %%xmm8, %0" : "=m"(*data)); break;
1053 case 9: asm("movdqa %%xmm9, %0" : "=m"(*data)); break;
1054 case 10: asm("movdqa %%xmm10, %0" : "=m"(*data)); break;
1055 case 11: asm("movdqa %%xmm11, %0" : "=m"(*data)); break;
1056 case 12: asm("movdqa %%xmm12, %0" : "=m"(*data)); break;
1057 case 13: asm("movdqa %%xmm13, %0" : "=m"(*data)); break;
1058 case 14: asm("movdqa %%xmm14, %0" : "=m"(*data)); break;
1059 case 15: asm("movdqa %%xmm15, %0" : "=m"(*data)); break;
1063 ctxt->ops->put_fpu(ctxt);
1066 static void write_sse_reg(struct x86_emulate_ctxt *ctxt, sse128_t *data,
1069 ctxt->ops->get_fpu(ctxt);
1071 case 0: asm("movdqa %0, %%xmm0" : : "m"(*data)); break;
1072 case 1: asm("movdqa %0, %%xmm1" : : "m"(*data)); break;
1073 case 2: asm("movdqa %0, %%xmm2" : : "m"(*data)); break;
1074 case 3: asm("movdqa %0, %%xmm3" : : "m"(*data)); break;
1075 case 4: asm("movdqa %0, %%xmm4" : : "m"(*data)); break;
1076 case 5: asm("movdqa %0, %%xmm5" : : "m"(*data)); break;
1077 case 6: asm("movdqa %0, %%xmm6" : : "m"(*data)); break;
1078 case 7: asm("movdqa %0, %%xmm7" : : "m"(*data)); break;
1079 #ifdef CONFIG_X86_64
1080 case 8: asm("movdqa %0, %%xmm8" : : "m"(*data)); break;
1081 case 9: asm("movdqa %0, %%xmm9" : : "m"(*data)); break;
1082 case 10: asm("movdqa %0, %%xmm10" : : "m"(*data)); break;
1083 case 11: asm("movdqa %0, %%xmm11" : : "m"(*data)); break;
1084 case 12: asm("movdqa %0, %%xmm12" : : "m"(*data)); break;
1085 case 13: asm("movdqa %0, %%xmm13" : : "m"(*data)); break;
1086 case 14: asm("movdqa %0, %%xmm14" : : "m"(*data)); break;
1087 case 15: asm("movdqa %0, %%xmm15" : : "m"(*data)); break;
1091 ctxt->ops->put_fpu(ctxt);
1094 static void read_mmx_reg(struct x86_emulate_ctxt *ctxt, u64 *data, int reg)
1096 ctxt->ops->get_fpu(ctxt);
1098 case 0: asm("movq %%mm0, %0" : "=m"(*data)); break;
1099 case 1: asm("movq %%mm1, %0" : "=m"(*data)); break;
1100 case 2: asm("movq %%mm2, %0" : "=m"(*data)); break;
1101 case 3: asm("movq %%mm3, %0" : "=m"(*data)); break;
1102 case 4: asm("movq %%mm4, %0" : "=m"(*data)); break;
1103 case 5: asm("movq %%mm5, %0" : "=m"(*data)); break;
1104 case 6: asm("movq %%mm6, %0" : "=m"(*data)); break;
1105 case 7: asm("movq %%mm7, %0" : "=m"(*data)); break;
1108 ctxt->ops->put_fpu(ctxt);
1111 static void write_mmx_reg(struct x86_emulate_ctxt *ctxt, u64 *data, int reg)
1113 ctxt->ops->get_fpu(ctxt);
1115 case 0: asm("movq %0, %%mm0" : : "m"(*data)); break;
1116 case 1: asm("movq %0, %%mm1" : : "m"(*data)); break;
1117 case 2: asm("movq %0, %%mm2" : : "m"(*data)); break;
1118 case 3: asm("movq %0, %%mm3" : : "m"(*data)); break;
1119 case 4: asm("movq %0, %%mm4" : : "m"(*data)); break;
1120 case 5: asm("movq %0, %%mm5" : : "m"(*data)); break;
1121 case 6: asm("movq %0, %%mm6" : : "m"(*data)); break;
1122 case 7: asm("movq %0, %%mm7" : : "m"(*data)); break;
1125 ctxt->ops->put_fpu(ctxt);
1128 static int em_fninit(struct x86_emulate_ctxt *ctxt)
1130 if (ctxt->ops->get_cr(ctxt, 0) & (X86_CR0_TS | X86_CR0_EM))
1131 return emulate_nm(ctxt);
1133 ctxt->ops->get_fpu(ctxt);
1134 asm volatile("fninit");
1135 ctxt->ops->put_fpu(ctxt);
1136 return X86EMUL_CONTINUE;
1139 static int em_fnstcw(struct x86_emulate_ctxt *ctxt)
1143 if (ctxt->ops->get_cr(ctxt, 0) & (X86_CR0_TS | X86_CR0_EM))
1144 return emulate_nm(ctxt);
1146 ctxt->ops->get_fpu(ctxt);
1147 asm volatile("fnstcw %0": "+m"(fcw));
1148 ctxt->ops->put_fpu(ctxt);
1150 /* force 2 byte destination */
1151 ctxt->dst.bytes = 2;
1152 ctxt->dst.val = fcw;
1154 return X86EMUL_CONTINUE;
1157 static int em_fnstsw(struct x86_emulate_ctxt *ctxt)
1161 if (ctxt->ops->get_cr(ctxt, 0) & (X86_CR0_TS | X86_CR0_EM))
1162 return emulate_nm(ctxt);
1164 ctxt->ops->get_fpu(ctxt);
1165 asm volatile("fnstsw %0": "+m"(fsw));
1166 ctxt->ops->put_fpu(ctxt);
1168 /* force 2 byte destination */
1169 ctxt->dst.bytes = 2;
1170 ctxt->dst.val = fsw;
1172 return X86EMUL_CONTINUE;
1175 static void decode_register_operand(struct x86_emulate_ctxt *ctxt,
1178 unsigned reg = ctxt->modrm_reg;
1179 int highbyte_regs = ctxt->rex_prefix == 0;
1181 if (!(ctxt->d & ModRM))
1182 reg = (ctxt->b & 7) | ((ctxt->rex_prefix & 1) << 3);
1184 if (ctxt->d & Sse) {
1188 read_sse_reg(ctxt, &op->vec_val, reg);
1191 if (ctxt->d & Mmx) {
1200 if (ctxt->d & ByteOp) {
1201 op->addr.reg = decode_register(ctxt, reg, highbyte_regs);
1204 op->addr.reg = decode_register(ctxt, reg, 0);
1205 op->bytes = ctxt->op_bytes;
1207 fetch_register_operand(op);
1208 op->orig_val = op->val;
1211 static void adjust_modrm_seg(struct x86_emulate_ctxt *ctxt, int base_reg)
1213 if (base_reg == VCPU_REGS_RSP || base_reg == VCPU_REGS_RBP)
1214 ctxt->modrm_seg = VCPU_SREG_SS;
1217 static int decode_modrm(struct x86_emulate_ctxt *ctxt,
1221 int index_reg = 0, base_reg = 0, scale;
1222 int rc = X86EMUL_CONTINUE;
1225 if (ctxt->rex_prefix) {
1226 ctxt->modrm_reg = (ctxt->rex_prefix & 4) << 1; /* REX.R */
1227 index_reg = (ctxt->rex_prefix & 2) << 2; /* REX.X */
1228 ctxt->modrm_rm = base_reg = (ctxt->rex_prefix & 1) << 3; /* REG.B */
1231 ctxt->modrm_mod |= (ctxt->modrm & 0xc0) >> 6;
1232 ctxt->modrm_reg |= (ctxt->modrm & 0x38) >> 3;
1233 ctxt->modrm_rm |= (ctxt->modrm & 0x07);
1234 ctxt->modrm_seg = VCPU_SREG_DS;
1236 if (ctxt->modrm_mod == 3) {
1238 op->bytes = (ctxt->d & ByteOp) ? 1 : ctxt->op_bytes;
1239 op->addr.reg = decode_register(ctxt, ctxt->modrm_rm, ctxt->d & ByteOp);
1240 if (ctxt->d & Sse) {
1243 op->addr.xmm = ctxt->modrm_rm;
1244 read_sse_reg(ctxt, &op->vec_val, ctxt->modrm_rm);
1247 if (ctxt->d & Mmx) {
1250 op->addr.xmm = ctxt->modrm_rm & 7;
1253 fetch_register_operand(op);
1259 if (ctxt->ad_bytes == 2) {
1260 unsigned bx = reg_read(ctxt, VCPU_REGS_RBX);
1261 unsigned bp = reg_read(ctxt, VCPU_REGS_RBP);
1262 unsigned si = reg_read(ctxt, VCPU_REGS_RSI);
1263 unsigned di = reg_read(ctxt, VCPU_REGS_RDI);
1265 /* 16-bit ModR/M decode. */
1266 switch (ctxt->modrm_mod) {
1268 if (ctxt->modrm_rm == 6)
1269 modrm_ea += insn_fetch(u16, ctxt);
1272 modrm_ea += insn_fetch(s8, ctxt);
1275 modrm_ea += insn_fetch(u16, ctxt);
1278 switch (ctxt->modrm_rm) {
1280 modrm_ea += bx + si;
1283 modrm_ea += bx + di;
1286 modrm_ea += bp + si;
1289 modrm_ea += bp + di;
1298 if (ctxt->modrm_mod != 0)
1305 if (ctxt->modrm_rm == 2 || ctxt->modrm_rm == 3 ||
1306 (ctxt->modrm_rm == 6 && ctxt->modrm_mod != 0))
1307 ctxt->modrm_seg = VCPU_SREG_SS;
1308 modrm_ea = (u16)modrm_ea;
1310 /* 32/64-bit ModR/M decode. */
1311 if ((ctxt->modrm_rm & 7) == 4) {
1312 sib = insn_fetch(u8, ctxt);
1313 index_reg |= (sib >> 3) & 7;
1314 base_reg |= sib & 7;
1317 if ((base_reg & 7) == 5 && ctxt->modrm_mod == 0)
1318 modrm_ea += insn_fetch(s32, ctxt);
1320 modrm_ea += reg_read(ctxt, base_reg);
1321 adjust_modrm_seg(ctxt, base_reg);
1324 modrm_ea += reg_read(ctxt, index_reg) << scale;
1325 } else if ((ctxt->modrm_rm & 7) == 5 && ctxt->modrm_mod == 0) {
1326 if (ctxt->mode == X86EMUL_MODE_PROT64)
1327 ctxt->rip_relative = 1;
1329 base_reg = ctxt->modrm_rm;
1330 modrm_ea += reg_read(ctxt, base_reg);
1331 adjust_modrm_seg(ctxt, base_reg);
1333 switch (ctxt->modrm_mod) {
1335 if (ctxt->modrm_rm == 5)
1336 modrm_ea += insn_fetch(s32, ctxt);
1339 modrm_ea += insn_fetch(s8, ctxt);
1342 modrm_ea += insn_fetch(s32, ctxt);
1346 op->addr.mem.ea = modrm_ea;
1351 static int decode_abs(struct x86_emulate_ctxt *ctxt,
1354 int rc = X86EMUL_CONTINUE;
1357 switch (ctxt->ad_bytes) {
1359 op->addr.mem.ea = insn_fetch(u16, ctxt);
1362 op->addr.mem.ea = insn_fetch(u32, ctxt);
1365 op->addr.mem.ea = insn_fetch(u64, ctxt);
1372 static void fetch_bit_operand(struct x86_emulate_ctxt *ctxt)
1376 if (ctxt->dst.type == OP_MEM && ctxt->src.type == OP_REG) {
1377 mask = ~(ctxt->dst.bytes * 8 - 1);
1379 if (ctxt->src.bytes == 2)
1380 sv = (s16)ctxt->src.val & (s16)mask;
1381 else if (ctxt->src.bytes == 4)
1382 sv = (s32)ctxt->src.val & (s32)mask;
1384 ctxt->dst.addr.mem.ea += (sv >> 3);
1387 /* only subword offset */
1388 ctxt->src.val &= (ctxt->dst.bytes << 3) - 1;
1391 static int read_emulated(struct x86_emulate_ctxt *ctxt,
1392 unsigned long addr, void *dest, unsigned size)
1395 struct read_cache *mc = &ctxt->mem_read;
1397 if (mc->pos < mc->end)
1400 WARN_ON((mc->end + size) >= sizeof(mc->data));
1402 rc = ctxt->ops->read_emulated(ctxt, addr, mc->data + mc->end, size,
1404 if (rc != X86EMUL_CONTINUE)
1410 memcpy(dest, mc->data + mc->pos, size);
1412 return X86EMUL_CONTINUE;
1415 static int segmented_read(struct x86_emulate_ctxt *ctxt,
1416 struct segmented_address addr,
1423 rc = linearize(ctxt, addr, size, false, &linear);
1424 if (rc != X86EMUL_CONTINUE)
1426 return read_emulated(ctxt, linear, data, size);
1429 static int segmented_write(struct x86_emulate_ctxt *ctxt,
1430 struct segmented_address addr,
1437 rc = linearize(ctxt, addr, size, true, &linear);
1438 if (rc != X86EMUL_CONTINUE)
1440 return ctxt->ops->write_emulated(ctxt, linear, data, size,
1444 static int segmented_cmpxchg(struct x86_emulate_ctxt *ctxt,
1445 struct segmented_address addr,
1446 const void *orig_data, const void *data,
1452 rc = linearize(ctxt, addr, size, true, &linear);
1453 if (rc != X86EMUL_CONTINUE)
1455 return ctxt->ops->cmpxchg_emulated(ctxt, linear, orig_data, data,
1456 size, &ctxt->exception);
1459 static int pio_in_emulated(struct x86_emulate_ctxt *ctxt,
1460 unsigned int size, unsigned short port,
1463 struct read_cache *rc = &ctxt->io_read;
1465 if (rc->pos == rc->end) { /* refill pio read ahead */
1466 unsigned int in_page, n;
1467 unsigned int count = ctxt->rep_prefix ?
1468 address_mask(ctxt, reg_read(ctxt, VCPU_REGS_RCX)) : 1;
1469 in_page = (ctxt->eflags & EFLG_DF) ?
1470 offset_in_page(reg_read(ctxt, VCPU_REGS_RDI)) :
1471 PAGE_SIZE - offset_in_page(reg_read(ctxt, VCPU_REGS_RDI));
1472 n = min(min(in_page, (unsigned int)sizeof(rc->data)) / size,
1476 rc->pos = rc->end = 0;
1477 if (!ctxt->ops->pio_in_emulated(ctxt, size, port, rc->data, n))
1482 if (ctxt->rep_prefix && !(ctxt->eflags & EFLG_DF)) {
1483 ctxt->dst.data = rc->data + rc->pos;
1484 ctxt->dst.type = OP_MEM_STR;
1485 ctxt->dst.count = (rc->end - rc->pos) / size;
1488 memcpy(dest, rc->data + rc->pos, size);
1494 static int read_interrupt_descriptor(struct x86_emulate_ctxt *ctxt,
1495 u16 index, struct desc_struct *desc)
1500 ctxt->ops->get_idt(ctxt, &dt);
1502 if (dt.size < index * 8 + 7)
1503 return emulate_gp(ctxt, index << 3 | 0x2);
1505 addr = dt.address + index * 8;
1506 return ctxt->ops->read_std(ctxt, addr, desc, sizeof *desc,
1510 static void get_descriptor_table_ptr(struct x86_emulate_ctxt *ctxt,
1511 u16 selector, struct desc_ptr *dt)
1513 const struct x86_emulate_ops *ops = ctxt->ops;
1515 if (selector & 1 << 2) {
1516 struct desc_struct desc;
1519 memset (dt, 0, sizeof *dt);
1520 if (!ops->get_segment(ctxt, &sel, &desc, NULL, VCPU_SREG_LDTR))
1523 dt->size = desc_limit_scaled(&desc); /* what if limit > 65535? */
1524 dt->address = get_desc_base(&desc);
1526 ops->get_gdt(ctxt, dt);
1529 /* allowed just for 8 bytes segments */
1530 static int read_segment_descriptor(struct x86_emulate_ctxt *ctxt,
1531 u16 selector, struct desc_struct *desc,
1535 u16 index = selector >> 3;
1538 get_descriptor_table_ptr(ctxt, selector, &dt);
1540 if (dt.size < index * 8 + 7)
1541 return emulate_gp(ctxt, selector & 0xfffc);
1543 *desc_addr_p = addr = dt.address + index * 8;
1544 return ctxt->ops->read_std(ctxt, addr, desc, sizeof *desc,
1548 /* allowed just for 8 bytes segments */
1549 static int write_segment_descriptor(struct x86_emulate_ctxt *ctxt,
1550 u16 selector, struct desc_struct *desc)
1553 u16 index = selector >> 3;
1556 get_descriptor_table_ptr(ctxt, selector, &dt);
1558 if (dt.size < index * 8 + 7)
1559 return emulate_gp(ctxt, selector & 0xfffc);
1561 addr = dt.address + index * 8;
1562 return ctxt->ops->write_std(ctxt, addr, desc, sizeof *desc,
1566 /* Does not support long mode */
1567 static int load_segment_descriptor(struct x86_emulate_ctxt *ctxt,
1568 u16 selector, int seg)
1570 struct desc_struct seg_desc, old_desc;
1572 unsigned err_vec = GP_VECTOR;
1574 bool null_selector = !(selector & ~0x3); /* 0000-0003 are null */
1579 memset(&seg_desc, 0, sizeof seg_desc);
1581 if ((seg <= VCPU_SREG_GS && ctxt->mode == X86EMUL_MODE_VM86)
1582 || ctxt->mode == X86EMUL_MODE_REAL) {
1583 /* set real mode segment descriptor */
1584 ctxt->ops->get_segment(ctxt, &dummy, &seg_desc, NULL, seg);
1585 set_desc_base(&seg_desc, selector << 4);
1590 cpl = ctxt->ops->cpl(ctxt);
1592 /* NULL selector is not valid for TR, CS and SS (except for long mode) */
1593 if ((seg == VCPU_SREG_CS
1594 || (seg == VCPU_SREG_SS
1595 && (ctxt->mode != X86EMUL_MODE_PROT64 || rpl != cpl))
1596 || seg == VCPU_SREG_TR)
1600 /* TR should be in GDT only */
1601 if (seg == VCPU_SREG_TR && (selector & (1 << 2)))
1604 if (null_selector) /* for NULL selector skip all following checks */
1607 ret = read_segment_descriptor(ctxt, selector, &seg_desc, &desc_addr);
1608 if (ret != X86EMUL_CONTINUE)
1611 err_code = selector & 0xfffc;
1612 err_vec = GP_VECTOR;
1614 /* can't load system descriptor into segment selector */
1615 if (seg <= VCPU_SREG_GS && !seg_desc.s)
1619 err_vec = (seg == VCPU_SREG_SS) ? SS_VECTOR : NP_VECTOR;
1628 * segment is not a writable data segment or segment
1629 * selector's RPL != CPL or segment selector's RPL != CPL
1631 if (rpl != cpl || (seg_desc.type & 0xa) != 0x2 || dpl != cpl)
1635 if (!(seg_desc.type & 8))
1638 if (seg_desc.type & 4) {
1644 if (rpl > cpl || dpl != cpl)
1647 /* CS(RPL) <- CPL */
1648 selector = (selector & 0xfffc) | cpl;
1651 if (seg_desc.s || (seg_desc.type != 1 && seg_desc.type != 9))
1653 old_desc = seg_desc;
1654 seg_desc.type |= 2; /* busy */
1655 ret = ctxt->ops->cmpxchg_emulated(ctxt, desc_addr, &old_desc, &seg_desc,
1656 sizeof(seg_desc), &ctxt->exception);
1657 if (ret != X86EMUL_CONTINUE)
1660 case VCPU_SREG_LDTR:
1661 if (seg_desc.s || seg_desc.type != 2)
1664 default: /* DS, ES, FS, or GS */
1666 * segment is not a data or readable code segment or
1667 * ((segment is a data or nonconforming code segment)
1668 * and (both RPL and CPL > DPL))
1670 if ((seg_desc.type & 0xa) == 0x8 ||
1671 (((seg_desc.type & 0xc) != 0xc) &&
1672 (rpl > dpl && cpl > dpl)))
1678 /* mark segment as accessed */
1680 ret = write_segment_descriptor(ctxt, selector, &seg_desc);
1681 if (ret != X86EMUL_CONTINUE)
1685 ctxt->ops->set_segment(ctxt, selector, &seg_desc, 0, seg);
1686 return X86EMUL_CONTINUE;
1688 emulate_exception(ctxt, err_vec, err_code, true);
1689 return X86EMUL_PROPAGATE_FAULT;
1692 static void write_register_operand(struct operand *op)
1694 /* The 4-byte case *is* correct: in 64-bit mode we zero-extend. */
1695 switch (op->bytes) {
1697 *(u8 *)op->addr.reg = (u8)op->val;
1700 *(u16 *)op->addr.reg = (u16)op->val;
1703 *op->addr.reg = (u32)op->val;
1704 break; /* 64b: zero-extend */
1706 *op->addr.reg = op->val;
1711 static int writeback(struct x86_emulate_ctxt *ctxt)
1715 if (ctxt->d & NoWrite)
1716 return X86EMUL_CONTINUE;
1718 switch (ctxt->dst.type) {
1720 write_register_operand(&ctxt->dst);
1723 if (ctxt->lock_prefix)
1724 rc = segmented_cmpxchg(ctxt,
1726 &ctxt->dst.orig_val,
1730 rc = segmented_write(ctxt,
1734 if (rc != X86EMUL_CONTINUE)
1738 rc = segmented_write(ctxt,
1741 ctxt->dst.bytes * ctxt->dst.count);
1742 if (rc != X86EMUL_CONTINUE)
1746 write_sse_reg(ctxt, &ctxt->dst.vec_val, ctxt->dst.addr.xmm);
1749 write_mmx_reg(ctxt, &ctxt->dst.mm_val, ctxt->dst.addr.mm);
1757 return X86EMUL_CONTINUE;
1760 static int push(struct x86_emulate_ctxt *ctxt, void *data, int bytes)
1762 struct segmented_address addr;
1764 rsp_increment(ctxt, -bytes);
1765 addr.ea = reg_read(ctxt, VCPU_REGS_RSP) & stack_mask(ctxt);
1766 addr.seg = VCPU_SREG_SS;
1768 return segmented_write(ctxt, addr, data, bytes);
1771 static int em_push(struct x86_emulate_ctxt *ctxt)
1773 /* Disable writeback. */
1774 ctxt->dst.type = OP_NONE;
1775 return push(ctxt, &ctxt->src.val, ctxt->op_bytes);
1778 static int emulate_pop(struct x86_emulate_ctxt *ctxt,
1779 void *dest, int len)
1782 struct segmented_address addr;
1784 addr.ea = reg_read(ctxt, VCPU_REGS_RSP) & stack_mask(ctxt);
1785 addr.seg = VCPU_SREG_SS;
1786 rc = segmented_read(ctxt, addr, dest, len);
1787 if (rc != X86EMUL_CONTINUE)
1790 rsp_increment(ctxt, len);
1794 static int em_pop(struct x86_emulate_ctxt *ctxt)
1796 return emulate_pop(ctxt, &ctxt->dst.val, ctxt->op_bytes);
1799 static int emulate_popf(struct x86_emulate_ctxt *ctxt,
1800 void *dest, int len)
1803 unsigned long val, change_mask;
1804 int iopl = (ctxt->eflags & X86_EFLAGS_IOPL) >> IOPL_SHIFT;
1805 int cpl = ctxt->ops->cpl(ctxt);
1807 rc = emulate_pop(ctxt, &val, len);
1808 if (rc != X86EMUL_CONTINUE)
1811 change_mask = EFLG_CF | EFLG_PF | EFLG_AF | EFLG_ZF | EFLG_SF | EFLG_OF
1812 | EFLG_TF | EFLG_DF | EFLG_NT | EFLG_RF | EFLG_AC | EFLG_ID;
1814 switch(ctxt->mode) {
1815 case X86EMUL_MODE_PROT64:
1816 case X86EMUL_MODE_PROT32:
1817 case X86EMUL_MODE_PROT16:
1819 change_mask |= EFLG_IOPL;
1821 change_mask |= EFLG_IF;
1823 case X86EMUL_MODE_VM86:
1825 return emulate_gp(ctxt, 0);
1826 change_mask |= EFLG_IF;
1828 default: /* real mode */
1829 change_mask |= (EFLG_IOPL | EFLG_IF);
1833 *(unsigned long *)dest =
1834 (ctxt->eflags & ~change_mask) | (val & change_mask);
1839 static int em_popf(struct x86_emulate_ctxt *ctxt)
1841 ctxt->dst.type = OP_REG;
1842 ctxt->dst.addr.reg = &ctxt->eflags;
1843 ctxt->dst.bytes = ctxt->op_bytes;
1844 return emulate_popf(ctxt, &ctxt->dst.val, ctxt->op_bytes);
1847 static int em_enter(struct x86_emulate_ctxt *ctxt)
1850 unsigned frame_size = ctxt->src.val;
1851 unsigned nesting_level = ctxt->src2.val & 31;
1855 return X86EMUL_UNHANDLEABLE;
1857 rbp = reg_read(ctxt, VCPU_REGS_RBP);
1858 rc = push(ctxt, &rbp, stack_size(ctxt));
1859 if (rc != X86EMUL_CONTINUE)
1861 assign_masked(reg_rmw(ctxt, VCPU_REGS_RBP), reg_read(ctxt, VCPU_REGS_RSP),
1863 assign_masked(reg_rmw(ctxt, VCPU_REGS_RSP),
1864 reg_read(ctxt, VCPU_REGS_RSP) - frame_size,
1866 return X86EMUL_CONTINUE;
1869 static int em_leave(struct x86_emulate_ctxt *ctxt)
1871 assign_masked(reg_rmw(ctxt, VCPU_REGS_RSP), reg_read(ctxt, VCPU_REGS_RBP),
1873 return emulate_pop(ctxt, reg_rmw(ctxt, VCPU_REGS_RBP), ctxt->op_bytes);
1876 static int em_push_sreg(struct x86_emulate_ctxt *ctxt)
1878 int seg = ctxt->src2.val;
1880 ctxt->src.val = get_segment_selector(ctxt, seg);
1882 return em_push(ctxt);
1885 static int em_pop_sreg(struct x86_emulate_ctxt *ctxt)
1887 int seg = ctxt->src2.val;
1888 unsigned long selector;
1891 rc = emulate_pop(ctxt, &selector, ctxt->op_bytes);
1892 if (rc != X86EMUL_CONTINUE)
1895 rc = load_segment_descriptor(ctxt, (u16)selector, seg);
1899 static int em_pusha(struct x86_emulate_ctxt *ctxt)
1901 unsigned long old_esp = reg_read(ctxt, VCPU_REGS_RSP);
1902 int rc = X86EMUL_CONTINUE;
1903 int reg = VCPU_REGS_RAX;
1905 while (reg <= VCPU_REGS_RDI) {
1906 (reg == VCPU_REGS_RSP) ?
1907 (ctxt->src.val = old_esp) : (ctxt->src.val = reg_read(ctxt, reg));
1910 if (rc != X86EMUL_CONTINUE)
1919 static int em_pushf(struct x86_emulate_ctxt *ctxt)
1921 ctxt->src.val = (unsigned long)ctxt->eflags;
1922 return em_push(ctxt);
1925 static int em_popa(struct x86_emulate_ctxt *ctxt)
1927 int rc = X86EMUL_CONTINUE;
1928 int reg = VCPU_REGS_RDI;
1930 while (reg >= VCPU_REGS_RAX) {
1931 if (reg == VCPU_REGS_RSP) {
1932 rsp_increment(ctxt, ctxt->op_bytes);
1936 rc = emulate_pop(ctxt, reg_rmw(ctxt, reg), ctxt->op_bytes);
1937 if (rc != X86EMUL_CONTINUE)
1944 static int __emulate_int_real(struct x86_emulate_ctxt *ctxt, int irq)
1946 const struct x86_emulate_ops *ops = ctxt->ops;
1953 /* TODO: Add limit checks */
1954 ctxt->src.val = ctxt->eflags;
1956 if (rc != X86EMUL_CONTINUE)
1959 ctxt->eflags &= ~(EFLG_IF | EFLG_TF | EFLG_AC);
1961 ctxt->src.val = get_segment_selector(ctxt, VCPU_SREG_CS);
1963 if (rc != X86EMUL_CONTINUE)
1966 ctxt->src.val = ctxt->_eip;
1968 if (rc != X86EMUL_CONTINUE)
1971 ops->get_idt(ctxt, &dt);
1973 eip_addr = dt.address + (irq << 2);
1974 cs_addr = dt.address + (irq << 2) + 2;
1976 rc = ops->read_std(ctxt, cs_addr, &cs, 2, &ctxt->exception);
1977 if (rc != X86EMUL_CONTINUE)
1980 rc = ops->read_std(ctxt, eip_addr, &eip, 2, &ctxt->exception);
1981 if (rc != X86EMUL_CONTINUE)
1984 rc = load_segment_descriptor(ctxt, cs, VCPU_SREG_CS);
1985 if (rc != X86EMUL_CONTINUE)
1993 int emulate_int_real(struct x86_emulate_ctxt *ctxt, int irq)
1997 invalidate_registers(ctxt);
1998 rc = __emulate_int_real(ctxt, irq);
1999 if (rc == X86EMUL_CONTINUE)
2000 writeback_registers(ctxt);
2004 static int emulate_int(struct x86_emulate_ctxt *ctxt, int irq)
2006 switch(ctxt->mode) {
2007 case X86EMUL_MODE_REAL:
2008 return __emulate_int_real(ctxt, irq);
2009 case X86EMUL_MODE_VM86:
2010 case X86EMUL_MODE_PROT16:
2011 case X86EMUL_MODE_PROT32:
2012 case X86EMUL_MODE_PROT64:
2014 /* Protected mode interrupts unimplemented yet */
2015 return X86EMUL_UNHANDLEABLE;
2019 static int emulate_iret_real(struct x86_emulate_ctxt *ctxt)
2021 int rc = X86EMUL_CONTINUE;
2022 unsigned long temp_eip = 0;
2023 unsigned long temp_eflags = 0;
2024 unsigned long cs = 0;
2025 unsigned long mask = EFLG_CF | EFLG_PF | EFLG_AF | EFLG_ZF | EFLG_SF | EFLG_TF |
2026 EFLG_IF | EFLG_DF | EFLG_OF | EFLG_IOPL | EFLG_NT | EFLG_RF |
2027 EFLG_AC | EFLG_ID | (1 << 1); /* Last one is the reserved bit */
2028 unsigned long vm86_mask = EFLG_VM | EFLG_VIF | EFLG_VIP;
2030 /* TODO: Add stack limit check */
2032 rc = emulate_pop(ctxt, &temp_eip, ctxt->op_bytes);
2034 if (rc != X86EMUL_CONTINUE)
2037 if (temp_eip & ~0xffff)
2038 return emulate_gp(ctxt, 0);
2040 rc = emulate_pop(ctxt, &cs, ctxt->op_bytes);
2042 if (rc != X86EMUL_CONTINUE)
2045 rc = emulate_pop(ctxt, &temp_eflags, ctxt->op_bytes);
2047 if (rc != X86EMUL_CONTINUE)
2050 rc = load_segment_descriptor(ctxt, (u16)cs, VCPU_SREG_CS);
2052 if (rc != X86EMUL_CONTINUE)
2055 ctxt->_eip = temp_eip;
2058 if (ctxt->op_bytes == 4)
2059 ctxt->eflags = ((temp_eflags & mask) | (ctxt->eflags & vm86_mask));
2060 else if (ctxt->op_bytes == 2) {
2061 ctxt->eflags &= ~0xffff;
2062 ctxt->eflags |= temp_eflags;
2065 ctxt->eflags &= ~EFLG_RESERVED_ZEROS_MASK; /* Clear reserved zeros */
2066 ctxt->eflags |= EFLG_RESERVED_ONE_MASK;
2071 static int em_iret(struct x86_emulate_ctxt *ctxt)
2073 switch(ctxt->mode) {
2074 case X86EMUL_MODE_REAL:
2075 return emulate_iret_real(ctxt);
2076 case X86EMUL_MODE_VM86:
2077 case X86EMUL_MODE_PROT16:
2078 case X86EMUL_MODE_PROT32:
2079 case X86EMUL_MODE_PROT64:
2081 /* iret from protected mode unimplemented yet */
2082 return X86EMUL_UNHANDLEABLE;
2086 static int em_jmp_far(struct x86_emulate_ctxt *ctxt)
2091 memcpy(&sel, ctxt->src.valptr + ctxt->op_bytes, 2);
2093 rc = load_segment_descriptor(ctxt, sel, VCPU_SREG_CS);
2094 if (rc != X86EMUL_CONTINUE)
2098 memcpy(&ctxt->_eip, ctxt->src.valptr, ctxt->op_bytes);
2099 return X86EMUL_CONTINUE;
2102 static int em_mul_ex(struct x86_emulate_ctxt *ctxt)
2106 emulate_1op_rax_rdx(ctxt, "mul", ex);
2107 return X86EMUL_CONTINUE;
2110 static int em_imul_ex(struct x86_emulate_ctxt *ctxt)
2114 emulate_1op_rax_rdx(ctxt, "imul", ex);
2115 return X86EMUL_CONTINUE;
2118 static int em_div_ex(struct x86_emulate_ctxt *ctxt)
2122 emulate_1op_rax_rdx(ctxt, "div", de);
2124 return emulate_de(ctxt);
2125 return X86EMUL_CONTINUE;
2128 static int em_idiv_ex(struct x86_emulate_ctxt *ctxt)
2132 emulate_1op_rax_rdx(ctxt, "idiv", de);
2134 return emulate_de(ctxt);
2135 return X86EMUL_CONTINUE;
2138 static int em_grp45(struct x86_emulate_ctxt *ctxt)
2140 int rc = X86EMUL_CONTINUE;
2142 switch (ctxt->modrm_reg) {
2143 case 2: /* call near abs */ {
2145 old_eip = ctxt->_eip;
2146 ctxt->_eip = ctxt->src.val;
2147 ctxt->src.val = old_eip;
2151 case 4: /* jmp abs */
2152 ctxt->_eip = ctxt->src.val;
2154 case 5: /* jmp far */
2155 rc = em_jmp_far(ctxt);
2164 static int em_cmpxchg8b(struct x86_emulate_ctxt *ctxt)
2166 u64 old = ctxt->dst.orig_val64;
2168 if (((u32) (old >> 0) != (u32) reg_read(ctxt, VCPU_REGS_RAX)) ||
2169 ((u32) (old >> 32) != (u32) reg_read(ctxt, VCPU_REGS_RDX))) {
2170 *reg_write(ctxt, VCPU_REGS_RAX) = (u32) (old >> 0);
2171 *reg_write(ctxt, VCPU_REGS_RDX) = (u32) (old >> 32);
2172 ctxt->eflags &= ~EFLG_ZF;
2174 ctxt->dst.val64 = ((u64)reg_read(ctxt, VCPU_REGS_RCX) << 32) |
2175 (u32) reg_read(ctxt, VCPU_REGS_RBX);
2177 ctxt->eflags |= EFLG_ZF;
2179 return X86EMUL_CONTINUE;
2182 static int em_ret(struct x86_emulate_ctxt *ctxt)
2184 ctxt->dst.type = OP_REG;
2185 ctxt->dst.addr.reg = &ctxt->_eip;
2186 ctxt->dst.bytes = ctxt->op_bytes;
2187 return em_pop(ctxt);
2190 static int em_ret_far(struct x86_emulate_ctxt *ctxt)
2195 rc = emulate_pop(ctxt, &ctxt->_eip, ctxt->op_bytes);
2196 if (rc != X86EMUL_CONTINUE)
2198 if (ctxt->op_bytes == 4)
2199 ctxt->_eip = (u32)ctxt->_eip;
2200 rc = emulate_pop(ctxt, &cs, ctxt->op_bytes);
2201 if (rc != X86EMUL_CONTINUE)
2203 rc = load_segment_descriptor(ctxt, (u16)cs, VCPU_SREG_CS);
2207 static int em_cmpxchg(struct x86_emulate_ctxt *ctxt)
2209 /* Save real source value, then compare EAX against destination. */
2210 ctxt->src.orig_val = ctxt->src.val;
2211 ctxt->src.val = reg_read(ctxt, VCPU_REGS_RAX);
2212 fastop(ctxt, em_cmp);
2214 if (ctxt->eflags & EFLG_ZF) {
2215 /* Success: write back to memory. */
2216 ctxt->dst.val = ctxt->src.orig_val;
2218 /* Failure: write the value we saw to EAX. */
2219 ctxt->dst.type = OP_REG;
2220 ctxt->dst.addr.reg = reg_rmw(ctxt, VCPU_REGS_RAX);
2222 return X86EMUL_CONTINUE;
2225 static int em_lseg(struct x86_emulate_ctxt *ctxt)
2227 int seg = ctxt->src2.val;
2231 memcpy(&sel, ctxt->src.valptr + ctxt->op_bytes, 2);
2233 rc = load_segment_descriptor(ctxt, sel, seg);
2234 if (rc != X86EMUL_CONTINUE)
2237 ctxt->dst.val = ctxt->src.val;
2242 setup_syscalls_segments(struct x86_emulate_ctxt *ctxt,
2243 struct desc_struct *cs, struct desc_struct *ss)
2245 cs->l = 0; /* will be adjusted later */
2246 set_desc_base(cs, 0); /* flat segment */
2247 cs->g = 1; /* 4kb granularity */
2248 set_desc_limit(cs, 0xfffff); /* 4GB limit */
2249 cs->type = 0x0b; /* Read, Execute, Accessed */
2251 cs->dpl = 0; /* will be adjusted later */
2256 set_desc_base(ss, 0); /* flat segment */
2257 set_desc_limit(ss, 0xfffff); /* 4GB limit */
2258 ss->g = 1; /* 4kb granularity */
2260 ss->type = 0x03; /* Read/Write, Accessed */
2261 ss->d = 1; /* 32bit stack segment */
2268 static bool vendor_intel(struct x86_emulate_ctxt *ctxt)
2270 u32 eax, ebx, ecx, edx;
2273 ctxt->ops->get_cpuid(ctxt, &eax, &ebx, &ecx, &edx);
2274 return ebx == X86EMUL_CPUID_VENDOR_GenuineIntel_ebx
2275 && ecx == X86EMUL_CPUID_VENDOR_GenuineIntel_ecx
2276 && edx == X86EMUL_CPUID_VENDOR_GenuineIntel_edx;
2279 static bool em_syscall_is_enabled(struct x86_emulate_ctxt *ctxt)
2281 const struct x86_emulate_ops *ops = ctxt->ops;
2282 u32 eax, ebx, ecx, edx;
2285 * syscall should always be enabled in longmode - so only become
2286 * vendor specific (cpuid) if other modes are active...
2288 if (ctxt->mode == X86EMUL_MODE_PROT64)
2293 ops->get_cpuid(ctxt, &eax, &ebx, &ecx, &edx);
2295 * Intel ("GenuineIntel")
2296 * remark: Intel CPUs only support "syscall" in 64bit
2297 * longmode. Also an 64bit guest with a
2298 * 32bit compat-app running will #UD !! While this
2299 * behaviour can be fixed (by emulating) into AMD
2300 * response - CPUs of AMD can't behave like Intel.
2302 if (ebx == X86EMUL_CPUID_VENDOR_GenuineIntel_ebx &&
2303 ecx == X86EMUL_CPUID_VENDOR_GenuineIntel_ecx &&
2304 edx == X86EMUL_CPUID_VENDOR_GenuineIntel_edx)
2307 /* AMD ("AuthenticAMD") */
2308 if (ebx == X86EMUL_CPUID_VENDOR_AuthenticAMD_ebx &&
2309 ecx == X86EMUL_CPUID_VENDOR_AuthenticAMD_ecx &&
2310 edx == X86EMUL_CPUID_VENDOR_AuthenticAMD_edx)
2313 /* AMD ("AMDisbetter!") */
2314 if (ebx == X86EMUL_CPUID_VENDOR_AMDisbetterI_ebx &&
2315 ecx == X86EMUL_CPUID_VENDOR_AMDisbetterI_ecx &&
2316 edx == X86EMUL_CPUID_VENDOR_AMDisbetterI_edx)
2319 /* default: (not Intel, not AMD), apply Intel's stricter rules... */
2323 static int em_syscall(struct x86_emulate_ctxt *ctxt)
2325 const struct x86_emulate_ops *ops = ctxt->ops;
2326 struct desc_struct cs, ss;
2331 /* syscall is not available in real mode */
2332 if (ctxt->mode == X86EMUL_MODE_REAL ||
2333 ctxt->mode == X86EMUL_MODE_VM86)
2334 return emulate_ud(ctxt);
2336 if (!(em_syscall_is_enabled(ctxt)))
2337 return emulate_ud(ctxt);
2339 ops->get_msr(ctxt, MSR_EFER, &efer);
2340 setup_syscalls_segments(ctxt, &cs, &ss);
2342 if (!(efer & EFER_SCE))
2343 return emulate_ud(ctxt);
2345 ops->get_msr(ctxt, MSR_STAR, &msr_data);
2347 cs_sel = (u16)(msr_data & 0xfffc);
2348 ss_sel = (u16)(msr_data + 8);
2350 if (efer & EFER_LMA) {
2354 ops->set_segment(ctxt, cs_sel, &cs, 0, VCPU_SREG_CS);
2355 ops->set_segment(ctxt, ss_sel, &ss, 0, VCPU_SREG_SS);
2357 *reg_write(ctxt, VCPU_REGS_RCX) = ctxt->_eip;
2358 if (efer & EFER_LMA) {
2359 #ifdef CONFIG_X86_64
2360 *reg_write(ctxt, VCPU_REGS_R11) = ctxt->eflags & ~EFLG_RF;
2363 ctxt->mode == X86EMUL_MODE_PROT64 ?
2364 MSR_LSTAR : MSR_CSTAR, &msr_data);
2365 ctxt->_eip = msr_data;
2367 ops->get_msr(ctxt, MSR_SYSCALL_MASK, &msr_data);
2368 ctxt->eflags &= ~(msr_data | EFLG_RF);
2372 ops->get_msr(ctxt, MSR_STAR, &msr_data);
2373 ctxt->_eip = (u32)msr_data;
2375 ctxt->eflags &= ~(EFLG_VM | EFLG_IF | EFLG_RF);
2378 return X86EMUL_CONTINUE;
2381 static int em_sysenter(struct x86_emulate_ctxt *ctxt)
2383 const struct x86_emulate_ops *ops = ctxt->ops;
2384 struct desc_struct cs, ss;
2389 ops->get_msr(ctxt, MSR_EFER, &efer);
2390 /* inject #GP if in real mode */
2391 if (ctxt->mode == X86EMUL_MODE_REAL)
2392 return emulate_gp(ctxt, 0);
2395 * Not recognized on AMD in compat mode (but is recognized in legacy
2398 if ((ctxt->mode == X86EMUL_MODE_PROT32) && (efer & EFER_LMA)
2399 && !vendor_intel(ctxt))
2400 return emulate_ud(ctxt);
2402 /* XXX sysenter/sysexit have not been tested in 64bit mode.
2403 * Therefore, we inject an #UD.
2405 if (ctxt->mode == X86EMUL_MODE_PROT64)
2406 return emulate_ud(ctxt);
2408 setup_syscalls_segments(ctxt, &cs, &ss);
2410 ops->get_msr(ctxt, MSR_IA32_SYSENTER_CS, &msr_data);
2411 switch (ctxt->mode) {
2412 case X86EMUL_MODE_PROT32:
2413 if ((msr_data & 0xfffc) == 0x0)
2414 return emulate_gp(ctxt, 0);
2416 case X86EMUL_MODE_PROT64:
2417 if (msr_data == 0x0)
2418 return emulate_gp(ctxt, 0);
2424 ctxt->eflags &= ~(EFLG_VM | EFLG_IF | EFLG_RF);
2425 cs_sel = (u16)msr_data;
2426 cs_sel &= ~SELECTOR_RPL_MASK;
2427 ss_sel = cs_sel + 8;
2428 ss_sel &= ~SELECTOR_RPL_MASK;
2429 if (ctxt->mode == X86EMUL_MODE_PROT64 || (efer & EFER_LMA)) {
2434 ops->set_segment(ctxt, cs_sel, &cs, 0, VCPU_SREG_CS);
2435 ops->set_segment(ctxt, ss_sel, &ss, 0, VCPU_SREG_SS);
2437 ops->get_msr(ctxt, MSR_IA32_SYSENTER_EIP, &msr_data);
2438 ctxt->_eip = msr_data;
2440 ops->get_msr(ctxt, MSR_IA32_SYSENTER_ESP, &msr_data);
2441 *reg_write(ctxt, VCPU_REGS_RSP) = msr_data;
2443 return X86EMUL_CONTINUE;
2446 static int em_sysexit(struct x86_emulate_ctxt *ctxt)
2448 const struct x86_emulate_ops *ops = ctxt->ops;
2449 struct desc_struct cs, ss;
2452 u16 cs_sel = 0, ss_sel = 0;
2454 /* inject #GP if in real mode or Virtual 8086 mode */
2455 if (ctxt->mode == X86EMUL_MODE_REAL ||
2456 ctxt->mode == X86EMUL_MODE_VM86)
2457 return emulate_gp(ctxt, 0);
2459 setup_syscalls_segments(ctxt, &cs, &ss);
2461 if ((ctxt->rex_prefix & 0x8) != 0x0)
2462 usermode = X86EMUL_MODE_PROT64;
2464 usermode = X86EMUL_MODE_PROT32;
2468 ops->get_msr(ctxt, MSR_IA32_SYSENTER_CS, &msr_data);
2470 case X86EMUL_MODE_PROT32:
2471 cs_sel = (u16)(msr_data + 16);
2472 if ((msr_data & 0xfffc) == 0x0)
2473 return emulate_gp(ctxt, 0);
2474 ss_sel = (u16)(msr_data + 24);
2476 case X86EMUL_MODE_PROT64:
2477 cs_sel = (u16)(msr_data + 32);
2478 if (msr_data == 0x0)
2479 return emulate_gp(ctxt, 0);
2480 ss_sel = cs_sel + 8;
2485 cs_sel |= SELECTOR_RPL_MASK;
2486 ss_sel |= SELECTOR_RPL_MASK;
2488 ops->set_segment(ctxt, cs_sel, &cs, 0, VCPU_SREG_CS);
2489 ops->set_segment(ctxt, ss_sel, &ss, 0, VCPU_SREG_SS);
2491 ctxt->_eip = reg_read(ctxt, VCPU_REGS_RDX);
2492 *reg_write(ctxt, VCPU_REGS_RSP) = reg_read(ctxt, VCPU_REGS_RCX);
2494 return X86EMUL_CONTINUE;
2497 static bool emulator_bad_iopl(struct x86_emulate_ctxt *ctxt)
2500 if (ctxt->mode == X86EMUL_MODE_REAL)
2502 if (ctxt->mode == X86EMUL_MODE_VM86)
2504 iopl = (ctxt->eflags & X86_EFLAGS_IOPL) >> IOPL_SHIFT;
2505 return ctxt->ops->cpl(ctxt) > iopl;
2508 static bool emulator_io_port_access_allowed(struct x86_emulate_ctxt *ctxt,
2511 const struct x86_emulate_ops *ops = ctxt->ops;
2512 struct desc_struct tr_seg;
2515 u16 tr, io_bitmap_ptr, perm, bit_idx = port & 0x7;
2516 unsigned mask = (1 << len) - 1;
2519 ops->get_segment(ctxt, &tr, &tr_seg, &base3, VCPU_SREG_TR);
2522 if (desc_limit_scaled(&tr_seg) < 103)
2524 base = get_desc_base(&tr_seg);
2525 #ifdef CONFIG_X86_64
2526 base |= ((u64)base3) << 32;
2528 r = ops->read_std(ctxt, base + 102, &io_bitmap_ptr, 2, NULL);
2529 if (r != X86EMUL_CONTINUE)
2531 if (io_bitmap_ptr + port/8 > desc_limit_scaled(&tr_seg))
2533 r = ops->read_std(ctxt, base + io_bitmap_ptr + port/8, &perm, 2, NULL);
2534 if (r != X86EMUL_CONTINUE)
2536 if ((perm >> bit_idx) & mask)
2541 static bool emulator_io_permited(struct x86_emulate_ctxt *ctxt,
2547 if (emulator_bad_iopl(ctxt))
2548 if (!emulator_io_port_access_allowed(ctxt, port, len))
2551 ctxt->perm_ok = true;
2556 static void save_state_to_tss16(struct x86_emulate_ctxt *ctxt,
2557 struct tss_segment_16 *tss)
2559 tss->ip = ctxt->_eip;
2560 tss->flag = ctxt->eflags;
2561 tss->ax = reg_read(ctxt, VCPU_REGS_RAX);
2562 tss->cx = reg_read(ctxt, VCPU_REGS_RCX);
2563 tss->dx = reg_read(ctxt, VCPU_REGS_RDX);
2564 tss->bx = reg_read(ctxt, VCPU_REGS_RBX);
2565 tss->sp = reg_read(ctxt, VCPU_REGS_RSP);
2566 tss->bp = reg_read(ctxt, VCPU_REGS_RBP);
2567 tss->si = reg_read(ctxt, VCPU_REGS_RSI);
2568 tss->di = reg_read(ctxt, VCPU_REGS_RDI);
2570 tss->es = get_segment_selector(ctxt, VCPU_SREG_ES);
2571 tss->cs = get_segment_selector(ctxt, VCPU_SREG_CS);
2572 tss->ss = get_segment_selector(ctxt, VCPU_SREG_SS);
2573 tss->ds = get_segment_selector(ctxt, VCPU_SREG_DS);
2574 tss->ldt = get_segment_selector(ctxt, VCPU_SREG_LDTR);
2577 static int load_state_from_tss16(struct x86_emulate_ctxt *ctxt,
2578 struct tss_segment_16 *tss)
2582 ctxt->_eip = tss->ip;
2583 ctxt->eflags = tss->flag | 2;
2584 *reg_write(ctxt, VCPU_REGS_RAX) = tss->ax;
2585 *reg_write(ctxt, VCPU_REGS_RCX) = tss->cx;
2586 *reg_write(ctxt, VCPU_REGS_RDX) = tss->dx;
2587 *reg_write(ctxt, VCPU_REGS_RBX) = tss->bx;
2588 *reg_write(ctxt, VCPU_REGS_RSP) = tss->sp;
2589 *reg_write(ctxt, VCPU_REGS_RBP) = tss->bp;
2590 *reg_write(ctxt, VCPU_REGS_RSI) = tss->si;
2591 *reg_write(ctxt, VCPU_REGS_RDI) = tss->di;
2594 * SDM says that segment selectors are loaded before segment
2597 set_segment_selector(ctxt, tss->ldt, VCPU_SREG_LDTR);
2598 set_segment_selector(ctxt, tss->es, VCPU_SREG_ES);
2599 set_segment_selector(ctxt, tss->cs, VCPU_SREG_CS);
2600 set_segment_selector(ctxt, tss->ss, VCPU_SREG_SS);
2601 set_segment_selector(ctxt, tss->ds, VCPU_SREG_DS);
2604 * Now load segment descriptors. If fault happens at this stage
2605 * it is handled in a context of new task
2607 ret = load_segment_descriptor(ctxt, tss->ldt, VCPU_SREG_LDTR);
2608 if (ret != X86EMUL_CONTINUE)
2610 ret = load_segment_descriptor(ctxt, tss->es, VCPU_SREG_ES);
2611 if (ret != X86EMUL_CONTINUE)
2613 ret = load_segment_descriptor(ctxt, tss->cs, VCPU_SREG_CS);
2614 if (ret != X86EMUL_CONTINUE)
2616 ret = load_segment_descriptor(ctxt, tss->ss, VCPU_SREG_SS);
2617 if (ret != X86EMUL_CONTINUE)
2619 ret = load_segment_descriptor(ctxt, tss->ds, VCPU_SREG_DS);
2620 if (ret != X86EMUL_CONTINUE)
2623 return X86EMUL_CONTINUE;
2626 static int task_switch_16(struct x86_emulate_ctxt *ctxt,
2627 u16 tss_selector, u16 old_tss_sel,
2628 ulong old_tss_base, struct desc_struct *new_desc)
2630 const struct x86_emulate_ops *ops = ctxt->ops;
2631 struct tss_segment_16 tss_seg;
2633 u32 new_tss_base = get_desc_base(new_desc);
2635 ret = ops->read_std(ctxt, old_tss_base, &tss_seg, sizeof tss_seg,
2637 if (ret != X86EMUL_CONTINUE)
2638 /* FIXME: need to provide precise fault address */
2641 save_state_to_tss16(ctxt, &tss_seg);
2643 ret = ops->write_std(ctxt, old_tss_base, &tss_seg, sizeof tss_seg,
2645 if (ret != X86EMUL_CONTINUE)
2646 /* FIXME: need to provide precise fault address */
2649 ret = ops->read_std(ctxt, new_tss_base, &tss_seg, sizeof tss_seg,
2651 if (ret != X86EMUL_CONTINUE)
2652 /* FIXME: need to provide precise fault address */
2655 if (old_tss_sel != 0xffff) {
2656 tss_seg.prev_task_link = old_tss_sel;
2658 ret = ops->write_std(ctxt, new_tss_base,
2659 &tss_seg.prev_task_link,
2660 sizeof tss_seg.prev_task_link,
2662 if (ret != X86EMUL_CONTINUE)
2663 /* FIXME: need to provide precise fault address */
2667 return load_state_from_tss16(ctxt, &tss_seg);
2670 static void save_state_to_tss32(struct x86_emulate_ctxt *ctxt,
2671 struct tss_segment_32 *tss)
2673 tss->cr3 = ctxt->ops->get_cr(ctxt, 3);
2674 tss->eip = ctxt->_eip;
2675 tss->eflags = ctxt->eflags;
2676 tss->eax = reg_read(ctxt, VCPU_REGS_RAX);
2677 tss->ecx = reg_read(ctxt, VCPU_REGS_RCX);
2678 tss->edx = reg_read(ctxt, VCPU_REGS_RDX);
2679 tss->ebx = reg_read(ctxt, VCPU_REGS_RBX);
2680 tss->esp = reg_read(ctxt, VCPU_REGS_RSP);
2681 tss->ebp = reg_read(ctxt, VCPU_REGS_RBP);
2682 tss->esi = reg_read(ctxt, VCPU_REGS_RSI);
2683 tss->edi = reg_read(ctxt, VCPU_REGS_RDI);
2685 tss->es = get_segment_selector(ctxt, VCPU_SREG_ES);
2686 tss->cs = get_segment_selector(ctxt, VCPU_SREG_CS);
2687 tss->ss = get_segment_selector(ctxt, VCPU_SREG_SS);
2688 tss->ds = get_segment_selector(ctxt, VCPU_SREG_DS);
2689 tss->fs = get_segment_selector(ctxt, VCPU_SREG_FS);
2690 tss->gs = get_segment_selector(ctxt, VCPU_SREG_GS);
2691 tss->ldt_selector = get_segment_selector(ctxt, VCPU_SREG_LDTR);
2694 static int load_state_from_tss32(struct x86_emulate_ctxt *ctxt,
2695 struct tss_segment_32 *tss)
2699 if (ctxt->ops->set_cr(ctxt, 3, tss->cr3))
2700 return emulate_gp(ctxt, 0);
2701 ctxt->_eip = tss->eip;
2702 ctxt->eflags = tss->eflags | 2;
2704 /* General purpose registers */
2705 *reg_write(ctxt, VCPU_REGS_RAX) = tss->eax;
2706 *reg_write(ctxt, VCPU_REGS_RCX) = tss->ecx;
2707 *reg_write(ctxt, VCPU_REGS_RDX) = tss->edx;
2708 *reg_write(ctxt, VCPU_REGS_RBX) = tss->ebx;
2709 *reg_write(ctxt, VCPU_REGS_RSP) = tss->esp;
2710 *reg_write(ctxt, VCPU_REGS_RBP) = tss->ebp;
2711 *reg_write(ctxt, VCPU_REGS_RSI) = tss->esi;
2712 *reg_write(ctxt, VCPU_REGS_RDI) = tss->edi;
2715 * SDM says that segment selectors are loaded before segment
2718 set_segment_selector(ctxt, tss->ldt_selector, VCPU_SREG_LDTR);
2719 set_segment_selector(ctxt, tss->es, VCPU_SREG_ES);
2720 set_segment_selector(ctxt, tss->cs, VCPU_SREG_CS);
2721 set_segment_selector(ctxt, tss->ss, VCPU_SREG_SS);
2722 set_segment_selector(ctxt, tss->ds, VCPU_SREG_DS);
2723 set_segment_selector(ctxt, tss->fs, VCPU_SREG_FS);
2724 set_segment_selector(ctxt, tss->gs, VCPU_SREG_GS);
2727 * If we're switching between Protected Mode and VM86, we need to make
2728 * sure to update the mode before loading the segment descriptors so
2729 * that the selectors are interpreted correctly.
2731 * Need to get rflags to the vcpu struct immediately because it
2732 * influences the CPL which is checked at least when loading the segment
2733 * descriptors and when pushing an error code to the new kernel stack.
2735 * TODO Introduce a separate ctxt->ops->set_cpl callback
2737 if (ctxt->eflags & X86_EFLAGS_VM)
2738 ctxt->mode = X86EMUL_MODE_VM86;
2740 ctxt->mode = X86EMUL_MODE_PROT32;
2742 ctxt->ops->set_rflags(ctxt, ctxt->eflags);
2745 * Now load segment descriptors. If fault happenes at this stage
2746 * it is handled in a context of new task
2748 ret = load_segment_descriptor(ctxt, tss->ldt_selector, VCPU_SREG_LDTR);
2749 if (ret != X86EMUL_CONTINUE)
2751 ret = load_segment_descriptor(ctxt, tss->es, VCPU_SREG_ES);
2752 if (ret != X86EMUL_CONTINUE)
2754 ret = load_segment_descriptor(ctxt, tss->cs, VCPU_SREG_CS);
2755 if (ret != X86EMUL_CONTINUE)
2757 ret = load_segment_descriptor(ctxt, tss->ss, VCPU_SREG_SS);
2758 if (ret != X86EMUL_CONTINUE)
2760 ret = load_segment_descriptor(ctxt, tss->ds, VCPU_SREG_DS);
2761 if (ret != X86EMUL_CONTINUE)
2763 ret = load_segment_descriptor(ctxt, tss->fs, VCPU_SREG_FS);
2764 if (ret != X86EMUL_CONTINUE)
2766 ret = load_segment_descriptor(ctxt, tss->gs, VCPU_SREG_GS);
2767 if (ret != X86EMUL_CONTINUE)
2770 return X86EMUL_CONTINUE;
2773 static int task_switch_32(struct x86_emulate_ctxt *ctxt,
2774 u16 tss_selector, u16 old_tss_sel,
2775 ulong old_tss_base, struct desc_struct *new_desc)
2777 const struct x86_emulate_ops *ops = ctxt->ops;
2778 struct tss_segment_32 tss_seg;
2780 u32 new_tss_base = get_desc_base(new_desc);
2782 ret = ops->read_std(ctxt, old_tss_base, &tss_seg, sizeof tss_seg,
2784 if (ret != X86EMUL_CONTINUE)
2785 /* FIXME: need to provide precise fault address */
2788 save_state_to_tss32(ctxt, &tss_seg);
2790 ret = ops->write_std(ctxt, old_tss_base, &tss_seg, sizeof tss_seg,
2792 if (ret != X86EMUL_CONTINUE)
2793 /* FIXME: need to provide precise fault address */
2796 ret = ops->read_std(ctxt, new_tss_base, &tss_seg, sizeof tss_seg,
2798 if (ret != X86EMUL_CONTINUE)
2799 /* FIXME: need to provide precise fault address */
2802 if (old_tss_sel != 0xffff) {
2803 tss_seg.prev_task_link = old_tss_sel;
2805 ret = ops->write_std(ctxt, new_tss_base,
2806 &tss_seg.prev_task_link,
2807 sizeof tss_seg.prev_task_link,
2809 if (ret != X86EMUL_CONTINUE)
2810 /* FIXME: need to provide precise fault address */
2814 return load_state_from_tss32(ctxt, &tss_seg);
2817 static int emulator_do_task_switch(struct x86_emulate_ctxt *ctxt,
2818 u16 tss_selector, int idt_index, int reason,
2819 bool has_error_code, u32 error_code)
2821 const struct x86_emulate_ops *ops = ctxt->ops;
2822 struct desc_struct curr_tss_desc, next_tss_desc;
2824 u16 old_tss_sel = get_segment_selector(ctxt, VCPU_SREG_TR);
2825 ulong old_tss_base =
2826 ops->get_cached_segment_base(ctxt, VCPU_SREG_TR);
2830 /* FIXME: old_tss_base == ~0 ? */
2832 ret = read_segment_descriptor(ctxt, tss_selector, &next_tss_desc, &desc_addr);
2833 if (ret != X86EMUL_CONTINUE)
2835 ret = read_segment_descriptor(ctxt, old_tss_sel, &curr_tss_desc, &desc_addr);
2836 if (ret != X86EMUL_CONTINUE)
2839 /* FIXME: check that next_tss_desc is tss */
2842 * Check privileges. The three cases are task switch caused by...
2844 * 1. jmp/call/int to task gate: Check against DPL of the task gate
2845 * 2. Exception/IRQ/iret: No check is performed
2846 * 3. jmp/call to TSS: Check against DPL of the TSS
2848 if (reason == TASK_SWITCH_GATE) {
2849 if (idt_index != -1) {
2850 /* Software interrupts */
2851 struct desc_struct task_gate_desc;
2854 ret = read_interrupt_descriptor(ctxt, idt_index,
2856 if (ret != X86EMUL_CONTINUE)
2859 dpl = task_gate_desc.dpl;
2860 if ((tss_selector & 3) > dpl || ops->cpl(ctxt) > dpl)
2861 return emulate_gp(ctxt, (idt_index << 3) | 0x2);
2863 } else if (reason != TASK_SWITCH_IRET) {
2864 int dpl = next_tss_desc.dpl;
2865 if ((tss_selector & 3) > dpl || ops->cpl(ctxt) > dpl)
2866 return emulate_gp(ctxt, tss_selector);
2870 desc_limit = desc_limit_scaled(&next_tss_desc);
2871 if (!next_tss_desc.p ||
2872 ((desc_limit < 0x67 && (next_tss_desc.type & 8)) ||
2873 desc_limit < 0x2b)) {
2874 emulate_ts(ctxt, tss_selector & 0xfffc);
2875 return X86EMUL_PROPAGATE_FAULT;
2878 if (reason == TASK_SWITCH_IRET || reason == TASK_SWITCH_JMP) {
2879 curr_tss_desc.type &= ~(1 << 1); /* clear busy flag */
2880 write_segment_descriptor(ctxt, old_tss_sel, &curr_tss_desc);
2883 if (reason == TASK_SWITCH_IRET)
2884 ctxt->eflags = ctxt->eflags & ~X86_EFLAGS_NT;
2886 /* set back link to prev task only if NT bit is set in eflags
2887 note that old_tss_sel is not used after this point */
2888 if (reason != TASK_SWITCH_CALL && reason != TASK_SWITCH_GATE)
2889 old_tss_sel = 0xffff;
2891 if (next_tss_desc.type & 8)
2892 ret = task_switch_32(ctxt, tss_selector, old_tss_sel,
2893 old_tss_base, &next_tss_desc);
2895 ret = task_switch_16(ctxt, tss_selector, old_tss_sel,
2896 old_tss_base, &next_tss_desc);
2897 if (ret != X86EMUL_CONTINUE)
2900 if (reason == TASK_SWITCH_CALL || reason == TASK_SWITCH_GATE)
2901 ctxt->eflags = ctxt->eflags | X86_EFLAGS_NT;
2903 if (reason != TASK_SWITCH_IRET) {
2904 next_tss_desc.type |= (1 << 1); /* set busy flag */
2905 write_segment_descriptor(ctxt, tss_selector, &next_tss_desc);
2908 ops->set_cr(ctxt, 0, ops->get_cr(ctxt, 0) | X86_CR0_TS);
2909 ops->set_segment(ctxt, tss_selector, &next_tss_desc, 0, VCPU_SREG_TR);
2911 if (has_error_code) {
2912 ctxt->op_bytes = ctxt->ad_bytes = (next_tss_desc.type & 8) ? 4 : 2;
2913 ctxt->lock_prefix = 0;
2914 ctxt->src.val = (unsigned long) error_code;
2915 ret = em_push(ctxt);
2921 int emulator_task_switch(struct x86_emulate_ctxt *ctxt,
2922 u16 tss_selector, int idt_index, int reason,
2923 bool has_error_code, u32 error_code)
2927 invalidate_registers(ctxt);
2928 ctxt->_eip = ctxt->eip;
2929 ctxt->dst.type = OP_NONE;
2931 rc = emulator_do_task_switch(ctxt, tss_selector, idt_index, reason,
2932 has_error_code, error_code);
2934 if (rc == X86EMUL_CONTINUE) {
2935 ctxt->eip = ctxt->_eip;
2936 writeback_registers(ctxt);
2939 return (rc == X86EMUL_UNHANDLEABLE) ? EMULATION_FAILED : EMULATION_OK;
2942 static void string_addr_inc(struct x86_emulate_ctxt *ctxt, int reg,
2945 int df = (ctxt->eflags & EFLG_DF) ? -op->count : op->count;
2947 register_address_increment(ctxt, reg_rmw(ctxt, reg), df * op->bytes);
2948 op->addr.mem.ea = register_address(ctxt, reg_read(ctxt, reg));
2951 static int em_das(struct x86_emulate_ctxt *ctxt)
2954 bool af, cf, old_cf;
2956 cf = ctxt->eflags & X86_EFLAGS_CF;
2962 af = ctxt->eflags & X86_EFLAGS_AF;
2963 if ((al & 0x0f) > 9 || af) {
2965 cf = old_cf | (al >= 250);
2970 if (old_al > 0x99 || old_cf) {
2976 /* Set PF, ZF, SF */
2977 ctxt->src.type = OP_IMM;
2979 ctxt->src.bytes = 1;
2980 fastop(ctxt, em_or);
2981 ctxt->eflags &= ~(X86_EFLAGS_AF | X86_EFLAGS_CF);
2983 ctxt->eflags |= X86_EFLAGS_CF;
2985 ctxt->eflags |= X86_EFLAGS_AF;
2986 return X86EMUL_CONTINUE;
2989 static int em_aad(struct x86_emulate_ctxt *ctxt)
2991 u8 al = ctxt->dst.val & 0xff;
2992 u8 ah = (ctxt->dst.val >> 8) & 0xff;
2994 al = (al + (ah * ctxt->src.val)) & 0xff;
2996 ctxt->dst.val = (ctxt->dst.val & 0xffff0000) | al;
2998 ctxt->eflags &= ~(X86_EFLAGS_PF | X86_EFLAGS_SF | X86_EFLAGS_ZF);
3001 ctxt->eflags |= X86_EFLAGS_ZF;
3003 ctxt->eflags |= X86_EFLAGS_PF;
3005 ctxt->eflags |= X86_EFLAGS_SF;
3007 return X86EMUL_CONTINUE;
3010 static int em_call(struct x86_emulate_ctxt *ctxt)
3012 long rel = ctxt->src.val;
3014 ctxt->src.val = (unsigned long)ctxt->_eip;
3016 return em_push(ctxt);
3019 static int em_call_far(struct x86_emulate_ctxt *ctxt)
3025 old_cs = get_segment_selector(ctxt, VCPU_SREG_CS);
3026 old_eip = ctxt->_eip;
3028 memcpy(&sel, ctxt->src.valptr + ctxt->op_bytes, 2);
3029 if (load_segment_descriptor(ctxt, sel, VCPU_SREG_CS))
3030 return X86EMUL_CONTINUE;
3033 memcpy(&ctxt->_eip, ctxt->src.valptr, ctxt->op_bytes);
3035 ctxt->src.val = old_cs;
3037 if (rc != X86EMUL_CONTINUE)
3040 ctxt->src.val = old_eip;
3041 return em_push(ctxt);
3044 static int em_ret_near_imm(struct x86_emulate_ctxt *ctxt)
3048 ctxt->dst.type = OP_REG;
3049 ctxt->dst.addr.reg = &ctxt->_eip;
3050 ctxt->dst.bytes = ctxt->op_bytes;
3051 rc = emulate_pop(ctxt, &ctxt->dst.val, ctxt->op_bytes);
3052 if (rc != X86EMUL_CONTINUE)
3054 rsp_increment(ctxt, ctxt->src.val);
3055 return X86EMUL_CONTINUE;
3058 static int em_xchg(struct x86_emulate_ctxt *ctxt)
3060 /* Write back the register source. */
3061 ctxt->src.val = ctxt->dst.val;
3062 write_register_operand(&ctxt->src);
3064 /* Write back the memory destination with implicit LOCK prefix. */
3065 ctxt->dst.val = ctxt->src.orig_val;
3066 ctxt->lock_prefix = 1;
3067 return X86EMUL_CONTINUE;
3070 static int em_imul_3op(struct x86_emulate_ctxt *ctxt)
3072 ctxt->dst.val = ctxt->src2.val;
3073 return fastop(ctxt, em_imul);
3076 static int em_cwd(struct x86_emulate_ctxt *ctxt)
3078 ctxt->dst.type = OP_REG;
3079 ctxt->dst.bytes = ctxt->src.bytes;
3080 ctxt->dst.addr.reg = reg_rmw(ctxt, VCPU_REGS_RDX);
3081 ctxt->dst.val = ~((ctxt->src.val >> (ctxt->src.bytes * 8 - 1)) - 1);
3083 return X86EMUL_CONTINUE;
3086 static int em_rdtsc(struct x86_emulate_ctxt *ctxt)
3090 ctxt->ops->get_msr(ctxt, MSR_IA32_TSC, &tsc);
3091 *reg_write(ctxt, VCPU_REGS_RAX) = (u32)tsc;
3092 *reg_write(ctxt, VCPU_REGS_RDX) = tsc >> 32;
3093 return X86EMUL_CONTINUE;
3096 static int em_rdpmc(struct x86_emulate_ctxt *ctxt)
3100 if (ctxt->ops->read_pmc(ctxt, reg_read(ctxt, VCPU_REGS_RCX), &pmc))
3101 return emulate_gp(ctxt, 0);
3102 *reg_write(ctxt, VCPU_REGS_RAX) = (u32)pmc;
3103 *reg_write(ctxt, VCPU_REGS_RDX) = pmc >> 32;
3104 return X86EMUL_CONTINUE;
3107 static int em_mov(struct x86_emulate_ctxt *ctxt)
3109 memcpy(ctxt->dst.valptr, ctxt->src.valptr, ctxt->op_bytes);
3110 return X86EMUL_CONTINUE;
3113 static int em_cr_write(struct x86_emulate_ctxt *ctxt)
3115 if (ctxt->ops->set_cr(ctxt, ctxt->modrm_reg, ctxt->src.val))
3116 return emulate_gp(ctxt, 0);
3118 /* Disable writeback. */
3119 ctxt->dst.type = OP_NONE;
3120 return X86EMUL_CONTINUE;
3123 static int em_dr_write(struct x86_emulate_ctxt *ctxt)
3127 if (ctxt->mode == X86EMUL_MODE_PROT64)
3128 val = ctxt->src.val & ~0ULL;
3130 val = ctxt->src.val & ~0U;
3132 /* #UD condition is already handled. */
3133 if (ctxt->ops->set_dr(ctxt, ctxt->modrm_reg, val) < 0)
3134 return emulate_gp(ctxt, 0);
3136 /* Disable writeback. */
3137 ctxt->dst.type = OP_NONE;
3138 return X86EMUL_CONTINUE;
3141 static int em_wrmsr(struct x86_emulate_ctxt *ctxt)
3145 msr_data = (u32)reg_read(ctxt, VCPU_REGS_RAX)
3146 | ((u64)reg_read(ctxt, VCPU_REGS_RDX) << 32);
3147 if (ctxt->ops->set_msr(ctxt, reg_read(ctxt, VCPU_REGS_RCX), msr_data))
3148 return emulate_gp(ctxt, 0);
3150 return X86EMUL_CONTINUE;
3153 static int em_rdmsr(struct x86_emulate_ctxt *ctxt)
3157 if (ctxt->ops->get_msr(ctxt, reg_read(ctxt, VCPU_REGS_RCX), &msr_data))
3158 return emulate_gp(ctxt, 0);
3160 *reg_write(ctxt, VCPU_REGS_RAX) = (u32)msr_data;
3161 *reg_write(ctxt, VCPU_REGS_RDX) = msr_data >> 32;
3162 return X86EMUL_CONTINUE;
3165 static int em_mov_rm_sreg(struct x86_emulate_ctxt *ctxt)
3167 if (ctxt->modrm_reg > VCPU_SREG_GS)
3168 return emulate_ud(ctxt);
3170 ctxt->dst.val = get_segment_selector(ctxt, ctxt->modrm_reg);
3171 return X86EMUL_CONTINUE;
3174 static int em_mov_sreg_rm(struct x86_emulate_ctxt *ctxt)
3176 u16 sel = ctxt->src.val;
3178 if (ctxt->modrm_reg == VCPU_SREG_CS || ctxt->modrm_reg > VCPU_SREG_GS)
3179 return emulate_ud(ctxt);
3181 if (ctxt->modrm_reg == VCPU_SREG_SS)
3182 ctxt->interruptibility = KVM_X86_SHADOW_INT_MOV_SS;
3184 /* Disable writeback. */
3185 ctxt->dst.type = OP_NONE;
3186 return load_segment_descriptor(ctxt, sel, ctxt->modrm_reg);
3189 static int em_lldt(struct x86_emulate_ctxt *ctxt)
3191 u16 sel = ctxt->src.val;
3193 /* Disable writeback. */
3194 ctxt->dst.type = OP_NONE;
3195 return load_segment_descriptor(ctxt, sel, VCPU_SREG_LDTR);
3198 static int em_ltr(struct x86_emulate_ctxt *ctxt)
3200 u16 sel = ctxt->src.val;
3202 /* Disable writeback. */
3203 ctxt->dst.type = OP_NONE;
3204 return load_segment_descriptor(ctxt, sel, VCPU_SREG_TR);
3207 static int em_invlpg(struct x86_emulate_ctxt *ctxt)
3212 rc = linearize(ctxt, ctxt->src.addr.mem, 1, false, &linear);
3213 if (rc == X86EMUL_CONTINUE)
3214 ctxt->ops->invlpg(ctxt, linear);
3215 /* Disable writeback. */
3216 ctxt->dst.type = OP_NONE;
3217 return X86EMUL_CONTINUE;
3220 static int em_clts(struct x86_emulate_ctxt *ctxt)
3224 cr0 = ctxt->ops->get_cr(ctxt, 0);
3226 ctxt->ops->set_cr(ctxt, 0, cr0);
3227 return X86EMUL_CONTINUE;
3230 static int em_vmcall(struct x86_emulate_ctxt *ctxt)
3234 if (ctxt->modrm_mod != 3 || ctxt->modrm_rm != 1)
3235 return X86EMUL_UNHANDLEABLE;
3237 rc = ctxt->ops->fix_hypercall(ctxt);
3238 if (rc != X86EMUL_CONTINUE)
3241 /* Let the processor re-execute the fixed hypercall */
3242 ctxt->_eip = ctxt->eip;
3243 /* Disable writeback. */
3244 ctxt->dst.type = OP_NONE;
3245 return X86EMUL_CONTINUE;
3248 static int emulate_store_desc_ptr(struct x86_emulate_ctxt *ctxt,
3249 void (*get)(struct x86_emulate_ctxt *ctxt,
3250 struct desc_ptr *ptr))
3252 struct desc_ptr desc_ptr;
3254 if (ctxt->mode == X86EMUL_MODE_PROT64)
3256 get(ctxt, &desc_ptr);
3257 if (ctxt->op_bytes == 2) {
3259 desc_ptr.address &= 0x00ffffff;
3261 /* Disable writeback. */
3262 ctxt->dst.type = OP_NONE;
3263 return segmented_write(ctxt, ctxt->dst.addr.mem,
3264 &desc_ptr, 2 + ctxt->op_bytes);
3267 static int em_sgdt(struct x86_emulate_ctxt *ctxt)
3269 return emulate_store_desc_ptr(ctxt, ctxt->ops->get_gdt);
3272 static int em_sidt(struct x86_emulate_ctxt *ctxt)
3274 return emulate_store_desc_ptr(ctxt, ctxt->ops->get_idt);
3277 static int em_lgdt(struct x86_emulate_ctxt *ctxt)
3279 struct desc_ptr desc_ptr;
3282 if (ctxt->mode == X86EMUL_MODE_PROT64)
3284 rc = read_descriptor(ctxt, ctxt->src.addr.mem,
3285 &desc_ptr.size, &desc_ptr.address,
3287 if (rc != X86EMUL_CONTINUE)
3289 ctxt->ops->set_gdt(ctxt, &desc_ptr);
3290 /* Disable writeback. */
3291 ctxt->dst.type = OP_NONE;
3292 return X86EMUL_CONTINUE;
3295 static int em_vmmcall(struct x86_emulate_ctxt *ctxt)
3299 rc = ctxt->ops->fix_hypercall(ctxt);
3301 /* Disable writeback. */
3302 ctxt->dst.type = OP_NONE;
3306 static int em_lidt(struct x86_emulate_ctxt *ctxt)
3308 struct desc_ptr desc_ptr;
3311 if (ctxt->mode == X86EMUL_MODE_PROT64)
3313 rc = read_descriptor(ctxt, ctxt->src.addr.mem,
3314 &desc_ptr.size, &desc_ptr.address,
3316 if (rc != X86EMUL_CONTINUE)
3318 ctxt->ops->set_idt(ctxt, &desc_ptr);
3319 /* Disable writeback. */
3320 ctxt->dst.type = OP_NONE;
3321 return X86EMUL_CONTINUE;
3324 static int em_smsw(struct x86_emulate_ctxt *ctxt)
3326 ctxt->dst.bytes = 2;
3327 ctxt->dst.val = ctxt->ops->get_cr(ctxt, 0);
3328 return X86EMUL_CONTINUE;
3331 static int em_lmsw(struct x86_emulate_ctxt *ctxt)
3333 ctxt->ops->set_cr(ctxt, 0, (ctxt->ops->get_cr(ctxt, 0) & ~0x0eul)
3334 | (ctxt->src.val & 0x0f));
3335 ctxt->dst.type = OP_NONE;
3336 return X86EMUL_CONTINUE;
3339 static int em_loop(struct x86_emulate_ctxt *ctxt)
3341 register_address_increment(ctxt, reg_rmw(ctxt, VCPU_REGS_RCX), -1);
3342 if ((address_mask(ctxt, reg_read(ctxt, VCPU_REGS_RCX)) != 0) &&
3343 (ctxt->b == 0xe2 || test_cc(ctxt->b ^ 0x5, ctxt->eflags)))
3344 jmp_rel(ctxt, ctxt->src.val);
3346 return X86EMUL_CONTINUE;
3349 static int em_jcxz(struct x86_emulate_ctxt *ctxt)
3351 if (address_mask(ctxt, reg_read(ctxt, VCPU_REGS_RCX)) == 0)
3352 jmp_rel(ctxt, ctxt->src.val);
3354 return X86EMUL_CONTINUE;
3357 static int em_in(struct x86_emulate_ctxt *ctxt)
3359 if (!pio_in_emulated(ctxt, ctxt->dst.bytes, ctxt->src.val,
3361 return X86EMUL_IO_NEEDED;
3363 return X86EMUL_CONTINUE;
3366 static int em_out(struct x86_emulate_ctxt *ctxt)
3368 ctxt->ops->pio_out_emulated(ctxt, ctxt->src.bytes, ctxt->dst.val,
3370 /* Disable writeback. */
3371 ctxt->dst.type = OP_NONE;
3372 return X86EMUL_CONTINUE;
3375 static int em_cli(struct x86_emulate_ctxt *ctxt)
3377 if (emulator_bad_iopl(ctxt))
3378 return emulate_gp(ctxt, 0);
3380 ctxt->eflags &= ~X86_EFLAGS_IF;
3381 return X86EMUL_CONTINUE;
3384 static int em_sti(struct x86_emulate_ctxt *ctxt)
3386 if (emulator_bad_iopl(ctxt))
3387 return emulate_gp(ctxt, 0);
3389 ctxt->interruptibility = KVM_X86_SHADOW_INT_STI;
3390 ctxt->eflags |= X86_EFLAGS_IF;
3391 return X86EMUL_CONTINUE;
3394 static int em_cpuid(struct x86_emulate_ctxt *ctxt)
3396 u32 eax, ebx, ecx, edx;
3398 eax = reg_read(ctxt, VCPU_REGS_RAX);
3399 ecx = reg_read(ctxt, VCPU_REGS_RCX);
3400 ctxt->ops->get_cpuid(ctxt, &eax, &ebx, &ecx, &edx);
3401 *reg_write(ctxt, VCPU_REGS_RAX) = eax;
3402 *reg_write(ctxt, VCPU_REGS_RBX) = ebx;
3403 *reg_write(ctxt, VCPU_REGS_RCX) = ecx;
3404 *reg_write(ctxt, VCPU_REGS_RDX) = edx;
3405 return X86EMUL_CONTINUE;
3408 static int em_lahf(struct x86_emulate_ctxt *ctxt)
3410 *reg_rmw(ctxt, VCPU_REGS_RAX) &= ~0xff00UL;
3411 *reg_rmw(ctxt, VCPU_REGS_RAX) |= (ctxt->eflags & 0xff) << 8;
3412 return X86EMUL_CONTINUE;
3415 static int em_bswap(struct x86_emulate_ctxt *ctxt)
3417 switch (ctxt->op_bytes) {
3418 #ifdef CONFIG_X86_64
3420 asm("bswap %0" : "+r"(ctxt->dst.val));
3424 asm("bswap %0" : "+r"(*(u32 *)&ctxt->dst.val));
3427 return X86EMUL_CONTINUE;
3430 static bool valid_cr(int nr)
3442 static int check_cr_read(struct x86_emulate_ctxt *ctxt)
3444 if (!valid_cr(ctxt->modrm_reg))
3445 return emulate_ud(ctxt);
3447 return X86EMUL_CONTINUE;
3450 static int check_cr_write(struct x86_emulate_ctxt *ctxt)
3452 u64 new_val = ctxt->src.val64;
3453 int cr = ctxt->modrm_reg;
3456 static u64 cr_reserved_bits[] = {
3457 0xffffffff00000000ULL,
3458 0, 0, 0, /* CR3 checked later */
3465 return emulate_ud(ctxt);
3467 if (new_val & cr_reserved_bits[cr])
3468 return emulate_gp(ctxt, 0);
3473 if (((new_val & X86_CR0_PG) && !(new_val & X86_CR0_PE)) ||
3474 ((new_val & X86_CR0_NW) && !(new_val & X86_CR0_CD)))
3475 return emulate_gp(ctxt, 0);
3477 cr4 = ctxt->ops->get_cr(ctxt, 4);
3478 ctxt->ops->get_msr(ctxt, MSR_EFER, &efer);
3480 if ((new_val & X86_CR0_PG) && (efer & EFER_LME) &&
3481 !(cr4 & X86_CR4_PAE))
3482 return emulate_gp(ctxt, 0);
3489 ctxt->ops->get_msr(ctxt, MSR_EFER, &efer);
3490 if (efer & EFER_LMA)
3491 rsvd = CR3_L_MODE_RESERVED_BITS;
3492 else if (ctxt->ops->get_cr(ctxt, 4) & X86_CR4_PAE)
3493 rsvd = CR3_PAE_RESERVED_BITS;
3494 else if (ctxt->ops->get_cr(ctxt, 0) & X86_CR0_PG)
3495 rsvd = CR3_NONPAE_RESERVED_BITS;
3498 return emulate_gp(ctxt, 0);
3503 ctxt->ops->get_msr(ctxt, MSR_EFER, &efer);
3505 if ((efer & EFER_LMA) && !(new_val & X86_CR4_PAE))
3506 return emulate_gp(ctxt, 0);
3512 return X86EMUL_CONTINUE;
3515 static int check_dr7_gd(struct x86_emulate_ctxt *ctxt)
3519 ctxt->ops->get_dr(ctxt, 7, &dr7);
3521 /* Check if DR7.Global_Enable is set */
3522 return dr7 & (1 << 13);
3525 static int check_dr_read(struct x86_emulate_ctxt *ctxt)
3527 int dr = ctxt->modrm_reg;
3531 return emulate_ud(ctxt);
3533 cr4 = ctxt->ops->get_cr(ctxt, 4);
3534 if ((cr4 & X86_CR4_DE) && (dr == 4 || dr == 5))
3535 return emulate_ud(ctxt);
3537 if (check_dr7_gd(ctxt))
3538 return emulate_db(ctxt);
3540 return X86EMUL_CONTINUE;
3543 static int check_dr_write(struct x86_emulate_ctxt *ctxt)
3545 u64 new_val = ctxt->src.val64;
3546 int dr = ctxt->modrm_reg;
3548 if ((dr == 6 || dr == 7) && (new_val & 0xffffffff00000000ULL))
3549 return emulate_gp(ctxt, 0);
3551 return check_dr_read(ctxt);
3554 static int check_svme(struct x86_emulate_ctxt *ctxt)
3558 ctxt->ops->get_msr(ctxt, MSR_EFER, &efer);
3560 if (!(efer & EFER_SVME))
3561 return emulate_ud(ctxt);
3563 return X86EMUL_CONTINUE;
3566 static int check_svme_pa(struct x86_emulate_ctxt *ctxt)
3568 u64 rax = reg_read(ctxt, VCPU_REGS_RAX);
3570 /* Valid physical address? */
3571 if (rax & 0xffff000000000000ULL)
3572 return emulate_gp(ctxt, 0);
3574 return check_svme(ctxt);
3577 static int check_rdtsc(struct x86_emulate_ctxt *ctxt)
3579 u64 cr4 = ctxt->ops->get_cr(ctxt, 4);
3581 if (cr4 & X86_CR4_TSD && ctxt->ops->cpl(ctxt))
3582 return emulate_ud(ctxt);
3584 return X86EMUL_CONTINUE;
3587 static int check_rdpmc(struct x86_emulate_ctxt *ctxt)
3589 u64 cr4 = ctxt->ops->get_cr(ctxt, 4);
3590 u64 rcx = reg_read(ctxt, VCPU_REGS_RCX);
3592 if ((!(cr4 & X86_CR4_PCE) && ctxt->ops->cpl(ctxt)) ||
3594 return emulate_gp(ctxt, 0);
3596 return X86EMUL_CONTINUE;
3599 static int check_perm_in(struct x86_emulate_ctxt *ctxt)
3601 ctxt->dst.bytes = min(ctxt->dst.bytes, 4u);
3602 if (!emulator_io_permited(ctxt, ctxt->src.val, ctxt->dst.bytes))
3603 return emulate_gp(ctxt, 0);
3605 return X86EMUL_CONTINUE;
3608 static int check_perm_out(struct x86_emulate_ctxt *ctxt)
3610 ctxt->src.bytes = min(ctxt->src.bytes, 4u);
3611 if (!emulator_io_permited(ctxt, ctxt->dst.val, ctxt->src.bytes))
3612 return emulate_gp(ctxt, 0);
3614 return X86EMUL_CONTINUE;
3617 #define D(_y) { .flags = (_y) }
3618 #define DI(_y, _i) { .flags = (_y), .intercept = x86_intercept_##_i }
3619 #define DIP(_y, _i, _p) { .flags = (_y), .intercept = x86_intercept_##_i, \
3620 .check_perm = (_p) }
3622 #define EXT(_f, _e) { .flags = ((_f) | RMExt), .u.group = (_e) }
3623 #define G(_f, _g) { .flags = ((_f) | Group | ModRM), .u.group = (_g) }
3624 #define GD(_f, _g) { .flags = ((_f) | GroupDual | ModRM), .u.gdual = (_g) }
3625 #define E(_f, _e) { .flags = ((_f) | Escape | ModRM), .u.esc = (_e) }
3626 #define I(_f, _e) { .flags = (_f), .u.execute = (_e) }
3627 #define F(_f, _e) { .flags = (_f) | Fastop, .u.fastop = (_e) }
3628 #define II(_f, _e, _i) \
3629 { .flags = (_f), .u.execute = (_e), .intercept = x86_intercept_##_i }
3630 #define IIP(_f, _e, _i, _p) \
3631 { .flags = (_f), .u.execute = (_e), .intercept = x86_intercept_##_i, \
3632 .check_perm = (_p) }
3633 #define GP(_f, _g) { .flags = ((_f) | Prefix), .u.gprefix = (_g) }
3635 #define D2bv(_f) D((_f) | ByteOp), D(_f)
3636 #define D2bvIP(_f, _i, _p) DIP((_f) | ByteOp, _i, _p), DIP(_f, _i, _p)
3637 #define I2bv(_f, _e) I((_f) | ByteOp, _e), I(_f, _e)
3638 #define F2bv(_f, _e) F((_f) | ByteOp, _e), F(_f, _e)
3639 #define I2bvIP(_f, _e, _i, _p) \
3640 IIP((_f) | ByteOp, _e, _i, _p), IIP(_f, _e, _i, _p)
3642 #define F6ALU(_f, _e) F2bv((_f) | DstMem | SrcReg | ModRM, _e), \
3643 F2bv(((_f) | DstReg | SrcMem | ModRM) & ~Lock, _e), \
3644 F2bv(((_f) & ~Lock) | DstAcc | SrcImm, _e)
3646 static const struct opcode group7_rm1[] = {
3647 DI(SrcNone | Priv, monitor),
3648 DI(SrcNone | Priv, mwait),
3652 static const struct opcode group7_rm3[] = {
3653 DIP(SrcNone | Prot | Priv, vmrun, check_svme_pa),
3654 II(SrcNone | Prot | VendorSpecific, em_vmmcall, vmmcall),
3655 DIP(SrcNone | Prot | Priv, vmload, check_svme_pa),
3656 DIP(SrcNone | Prot | Priv, vmsave, check_svme_pa),
3657 DIP(SrcNone | Prot | Priv, stgi, check_svme),
3658 DIP(SrcNone | Prot | Priv, clgi, check_svme),
3659 DIP(SrcNone | Prot | Priv, skinit, check_svme),
3660 DIP(SrcNone | Prot | Priv, invlpga, check_svme),
3663 static const struct opcode group7_rm7[] = {
3665 DIP(SrcNone, rdtscp, check_rdtsc),
3669 static const struct opcode group1[] = {
3671 F(Lock | PageTable, em_or),
3674 F(Lock | PageTable, em_and),
3680 static const struct opcode group1A[] = {
3681 I(DstMem | SrcNone | Mov | Stack, em_pop), N, N, N, N, N, N, N,
3684 static const struct opcode group2[] = {
3685 F(DstMem | ModRM, em_rol),
3686 F(DstMem | ModRM, em_ror),
3687 F(DstMem | ModRM, em_rcl),
3688 F(DstMem | ModRM, em_rcr),
3689 F(DstMem | ModRM, em_shl),
3690 F(DstMem | ModRM, em_shr),
3691 F(DstMem | ModRM, em_shl),
3692 F(DstMem | ModRM, em_sar),
3695 static const struct opcode group3[] = {
3696 F(DstMem | SrcImm | NoWrite, em_test),
3697 F(DstMem | SrcImm | NoWrite, em_test),
3698 F(DstMem | SrcNone | Lock, em_not),
3699 F(DstMem | SrcNone | Lock, em_neg),
3700 I(SrcMem, em_mul_ex),
3701 I(SrcMem, em_imul_ex),
3702 I(SrcMem, em_div_ex),
3703 I(SrcMem, em_idiv_ex),
3706 static const struct opcode group4[] = {
3707 F(ByteOp | DstMem | SrcNone | Lock, em_inc),
3708 F(ByteOp | DstMem | SrcNone | Lock, em_dec),
3712 static const struct opcode group5[] = {
3713 F(DstMem | SrcNone | Lock, em_inc),
3714 F(DstMem | SrcNone | Lock, em_dec),
3715 I(SrcMem | Stack, em_grp45),
3716 I(SrcMemFAddr | ImplicitOps | Stack, em_call_far),
3717 I(SrcMem | Stack, em_grp45),
3718 I(SrcMemFAddr | ImplicitOps, em_grp45),
3719 I(SrcMem | Stack, em_grp45), N,
3722 static const struct opcode group6[] = {
3725 II(Prot | Priv | SrcMem16, em_lldt, lldt),
3726 II(Prot | Priv | SrcMem16, em_ltr, ltr),
3730 static const struct group_dual group7 = { {
3731 II(Mov | DstMem | Priv, em_sgdt, sgdt),
3732 II(Mov | DstMem | Priv, em_sidt, sidt),
3733 II(SrcMem | Priv, em_lgdt, lgdt),
3734 II(SrcMem | Priv, em_lidt, lidt),
3735 II(SrcNone | DstMem | Mov, em_smsw, smsw), N,
3736 II(SrcMem16 | Mov | Priv, em_lmsw, lmsw),
3737 II(SrcMem | ByteOp | Priv | NoAccess, em_invlpg, invlpg),
3739 I(SrcNone | Priv | VendorSpecific, em_vmcall),
3741 N, EXT(0, group7_rm3),
3742 II(SrcNone | DstMem | Mov, em_smsw, smsw), N,
3743 II(SrcMem16 | Mov | Priv, em_lmsw, lmsw),
3747 static const struct opcode group8[] = {
3749 F(DstMem | SrcImmByte | NoWrite, em_bt),
3750 F(DstMem | SrcImmByte | Lock | PageTable, em_bts),
3751 F(DstMem | SrcImmByte | Lock, em_btr),
3752 F(DstMem | SrcImmByte | Lock | PageTable, em_btc),
3755 static const struct group_dual group9 = { {
3756 N, I(DstMem64 | Lock | PageTable, em_cmpxchg8b), N, N, N, N, N, N,
3758 N, N, N, N, N, N, N, N,
3761 static const struct opcode group11[] = {
3762 I(DstMem | SrcImm | Mov | PageTable, em_mov),
3766 static const struct gprefix pfx_0f_6f_0f_7f = {
3767 I(Mmx, em_mov), I(Sse | Aligned, em_mov), N, I(Sse | Unaligned, em_mov),
3770 static const struct gprefix pfx_vmovntpx = {
3771 I(0, em_mov), N, N, N,
3774 static const struct escape escape_d9 = { {
3775 N, N, N, N, N, N, N, I(DstMem, em_fnstcw),
3778 N, N, N, N, N, N, N, N,
3780 N, N, N, N, N, N, N, N,
3782 N, N, N, N, N, N, N, N,
3784 N, N, N, N, N, N, N, N,
3786 N, N, N, N, N, N, N, N,
3788 N, N, N, N, N, N, N, N,
3790 N, N, N, N, N, N, N, N,
3792 N, N, N, N, N, N, N, N,
3795 static const struct escape escape_db = { {
3796 N, N, N, N, N, N, N, N,
3799 N, N, N, N, N, N, N, N,
3801 N, N, N, N, N, N, N, N,
3803 N, N, N, N, N, N, N, N,
3805 N, N, N, N, N, N, N, N,
3807 N, N, N, I(ImplicitOps, em_fninit), N, N, N, N,
3809 N, N, N, N, N, N, N, N,
3811 N, N, N, N, N, N, N, N,
3813 N, N, N, N, N, N, N, N,
3816 static const struct escape escape_dd = { {
3817 N, N, N, N, N, N, N, I(DstMem, em_fnstsw),
3820 N, N, N, N, N, N, N, N,
3822 N, N, N, N, N, N, N, N,
3824 N, N, N, N, N, N, N, N,
3826 N, N, N, N, N, N, N, N,
3828 N, N, N, N, N, N, N, N,
3830 N, N, N, N, N, N, N, N,
3832 N, N, N, N, N, N, N, N,
3834 N, N, N, N, N, N, N, N,
3837 static const struct opcode opcode_table[256] = {
3839 F6ALU(Lock, em_add),
3840 I(ImplicitOps | Stack | No64 | Src2ES, em_push_sreg),
3841 I(ImplicitOps | Stack | No64 | Src2ES, em_pop_sreg),
3843 F6ALU(Lock | PageTable, em_or),
3844 I(ImplicitOps | Stack | No64 | Src2CS, em_push_sreg),
3847 F6ALU(Lock, em_adc),
3848 I(ImplicitOps | Stack | No64 | Src2SS, em_push_sreg),
3849 I(ImplicitOps | Stack | No64 | Src2SS, em_pop_sreg),
3851 F6ALU(Lock, em_sbb),
3852 I(ImplicitOps | Stack | No64 | Src2DS, em_push_sreg),
3853 I(ImplicitOps | Stack | No64 | Src2DS, em_pop_sreg),
3855 F6ALU(Lock | PageTable, em_and), N, N,
3857 F6ALU(Lock, em_sub), N, I(ByteOp | DstAcc | No64, em_das),
3859 F6ALU(Lock, em_xor), N, N,
3861 F6ALU(NoWrite, em_cmp), N, N,
3863 X8(F(DstReg, em_inc)), X8(F(DstReg, em_dec)),
3865 X8(I(SrcReg | Stack, em_push)),
3867 X8(I(DstReg | Stack, em_pop)),
3869 I(ImplicitOps | Stack | No64, em_pusha),
3870 I(ImplicitOps | Stack | No64, em_popa),
3871 N, D(DstReg | SrcMem32 | ModRM | Mov) /* movsxd (x86/64) */ ,
3874 I(SrcImm | Mov | Stack, em_push),
3875 I(DstReg | SrcMem | ModRM | Src2Imm, em_imul_3op),
3876 I(SrcImmByte | Mov | Stack, em_push),
3877 I(DstReg | SrcMem | ModRM | Src2ImmByte, em_imul_3op),
3878 I2bvIP(DstDI | SrcDX | Mov | String | Unaligned, em_in, ins, check_perm_in), /* insb, insw/insd */
3879 I2bvIP(SrcSI | DstDX | String, em_out, outs, check_perm_out), /* outsb, outsw/outsd */
3883 G(ByteOp | DstMem | SrcImm, group1),
3884 G(DstMem | SrcImm, group1),
3885 G(ByteOp | DstMem | SrcImm | No64, group1),
3886 G(DstMem | SrcImmByte, group1),
3887 F2bv(DstMem | SrcReg | ModRM | NoWrite, em_test),
3888 I2bv(DstMem | SrcReg | ModRM | Lock | PageTable, em_xchg),
3890 I2bv(DstMem | SrcReg | ModRM | Mov | PageTable, em_mov),
3891 I2bv(DstReg | SrcMem | ModRM | Mov, em_mov),
3892 I(DstMem | SrcNone | ModRM | Mov | PageTable, em_mov_rm_sreg),
3893 D(ModRM | SrcMem | NoAccess | DstReg),
3894 I(ImplicitOps | SrcMem16 | ModRM, em_mov_sreg_rm),
3897 DI(SrcAcc | DstReg, pause), X7(D(SrcAcc | DstReg)),
3899 D(DstAcc | SrcNone), I(ImplicitOps | SrcAcc, em_cwd),
3900 I(SrcImmFAddr | No64, em_call_far), N,
3901 II(ImplicitOps | Stack, em_pushf, pushf),
3902 II(ImplicitOps | Stack, em_popf, popf), N, I(ImplicitOps, em_lahf),
3904 I2bv(DstAcc | SrcMem | Mov | MemAbs, em_mov),
3905 I2bv(DstMem | SrcAcc | Mov | MemAbs | PageTable, em_mov),
3906 I2bv(SrcSI | DstDI | Mov | String, em_mov),
3907 F2bv(SrcSI | DstDI | String | NoWrite, em_cmp),
3909 F2bv(DstAcc | SrcImm | NoWrite, em_test),
3910 I2bv(SrcAcc | DstDI | Mov | String, em_mov),
3911 I2bv(SrcSI | DstAcc | Mov | String, em_mov),
3912 F2bv(SrcAcc | DstDI | String | NoWrite, em_cmp),
3914 X8(I(ByteOp | DstReg | SrcImm | Mov, em_mov)),
3916 X8(I(DstReg | SrcImm64 | Mov, em_mov)),
3918 G(ByteOp | Src2ImmByte, group2), G(Src2ImmByte, group2),
3919 I(ImplicitOps | Stack | SrcImmU16, em_ret_near_imm),
3920 I(ImplicitOps | Stack, em_ret),
3921 I(DstReg | SrcMemFAddr | ModRM | No64 | Src2ES, em_lseg),
3922 I(DstReg | SrcMemFAddr | ModRM | No64 | Src2DS, em_lseg),
3923 G(ByteOp, group11), G(0, group11),
3925 I(Stack | SrcImmU16 | Src2ImmByte, em_enter), I(Stack, em_leave),
3926 N, I(ImplicitOps | Stack, em_ret_far),
3927 D(ImplicitOps), DI(SrcImmByte, intn),
3928 D(ImplicitOps | No64), II(ImplicitOps, em_iret, iret),
3930 G(Src2One | ByteOp, group2), G(Src2One, group2),
3931 G(Src2CL | ByteOp, group2), G(Src2CL, group2),
3932 N, I(DstAcc | SrcImmByte | No64, em_aad), N, N,
3934 N, E(0, &escape_d9), N, E(0, &escape_db), N, E(0, &escape_dd), N, N,
3936 X3(I(SrcImmByte, em_loop)),
3937 I(SrcImmByte, em_jcxz),
3938 I2bvIP(SrcImmUByte | DstAcc, em_in, in, check_perm_in),
3939 I2bvIP(SrcAcc | DstImmUByte, em_out, out, check_perm_out),
3941 I(SrcImm | Stack, em_call), D(SrcImm | ImplicitOps),
3942 I(SrcImmFAddr | No64, em_jmp_far), D(SrcImmByte | ImplicitOps),
3943 I2bvIP(SrcDX | DstAcc, em_in, in, check_perm_in),
3944 I2bvIP(SrcAcc | DstDX, em_out, out, check_perm_out),
3946 N, DI(ImplicitOps, icebp), N, N,
3947 DI(ImplicitOps | Priv, hlt), D(ImplicitOps),
3948 G(ByteOp, group3), G(0, group3),
3950 D(ImplicitOps), D(ImplicitOps),
3951 I(ImplicitOps, em_cli), I(ImplicitOps, em_sti),
3952 D(ImplicitOps), D(ImplicitOps), G(0, group4), G(0, group5),
3955 static const struct opcode twobyte_table[256] = {
3957 G(0, group6), GD(0, &group7), N, N,
3958 N, I(ImplicitOps | VendorSpecific, em_syscall),
3959 II(ImplicitOps | Priv, em_clts, clts), N,
3960 DI(ImplicitOps | Priv, invd), DI(ImplicitOps | Priv, wbinvd), N, N,
3961 N, D(ImplicitOps | ModRM), N, N,
3963 N, N, N, N, N, N, N, N, D(ImplicitOps | ModRM), N, N, N, N, N, N, N,
3965 DIP(ModRM | DstMem | Priv | Op3264, cr_read, check_cr_read),
3966 DIP(ModRM | DstMem | Priv | Op3264, dr_read, check_dr_read),
3967 IIP(ModRM | SrcMem | Priv | Op3264, em_cr_write, cr_write, check_cr_write),
3968 IIP(ModRM | SrcMem | Priv | Op3264, em_dr_write, dr_write, check_dr_write),
3970 N, N, N, GP(ModRM | DstMem | SrcReg | Sse | Mov | Aligned, &pfx_vmovntpx),
3973 II(ImplicitOps | Priv, em_wrmsr, wrmsr),
3974 IIP(ImplicitOps, em_rdtsc, rdtsc, check_rdtsc),
3975 II(ImplicitOps | Priv, em_rdmsr, rdmsr),
3976 IIP(ImplicitOps, em_rdpmc, rdpmc, check_rdpmc),
3977 I(ImplicitOps | VendorSpecific, em_sysenter),
3978 I(ImplicitOps | Priv | VendorSpecific, em_sysexit),
3980 N, N, N, N, N, N, N, N,
3982 X16(D(DstReg | SrcMem | ModRM | Mov)),
3984 N, N, N, N, N, N, N, N, N, N, N, N, N, N, N, N,
3989 N, N, N, GP(SrcMem | DstReg | ModRM | Mov, &pfx_0f_6f_0f_7f),
3994 N, N, N, GP(SrcReg | DstMem | ModRM | Mov, &pfx_0f_6f_0f_7f),
3998 X16(D(ByteOp | DstMem | SrcNone | ModRM| Mov)),
4000 I(Stack | Src2FS, em_push_sreg), I(Stack | Src2FS, em_pop_sreg),
4001 II(ImplicitOps, em_cpuid, cpuid),
4002 F(DstMem | SrcReg | ModRM | BitOp | NoWrite, em_bt),
4003 F(DstMem | SrcReg | Src2ImmByte | ModRM, em_shld),
4004 F(DstMem | SrcReg | Src2CL | ModRM, em_shld), N, N,
4006 I(Stack | Src2GS, em_push_sreg), I(Stack | Src2GS, em_pop_sreg),
4007 DI(ImplicitOps, rsm),
4008 F(DstMem | SrcReg | ModRM | BitOp | Lock | PageTable, em_bts),
4009 F(DstMem | SrcReg | Src2ImmByte | ModRM, em_shrd),
4010 F(DstMem | SrcReg | Src2CL | ModRM, em_shrd),
4011 D(ModRM), F(DstReg | SrcMem | ModRM, em_imul),
4013 I2bv(DstMem | SrcReg | ModRM | Lock | PageTable, em_cmpxchg),
4014 I(DstReg | SrcMemFAddr | ModRM | Src2SS, em_lseg),
4015 F(DstMem | SrcReg | ModRM | BitOp | Lock, em_btr),
4016 I(DstReg | SrcMemFAddr | ModRM | Src2FS, em_lseg),
4017 I(DstReg | SrcMemFAddr | ModRM | Src2GS, em_lseg),
4018 D(DstReg | SrcMem8 | ModRM | Mov), D(DstReg | SrcMem16 | ModRM | Mov),
4022 F(DstMem | SrcReg | ModRM | BitOp | Lock | PageTable, em_btc),
4023 F(DstReg | SrcMem | ModRM, em_bsf), F(DstReg | SrcMem | ModRM, em_bsr),
4024 D(DstReg | SrcMem8 | ModRM | Mov), D(DstReg | SrcMem16 | ModRM | Mov),
4026 D2bv(DstMem | SrcReg | ModRM | Lock),
4027 N, D(DstMem | SrcReg | ModRM | Mov),
4028 N, N, N, GD(0, &group9),
4030 X8(I(DstReg, em_bswap)),
4032 N, N, N, N, N, N, N, N, N, N, N, N, N, N, N, N,
4034 N, N, N, N, N, N, N, N, N, N, N, N, N, N, N, N,
4036 N, N, N, N, N, N, N, N, N, N, N, N, N, N, N, N
4053 static unsigned imm_size(struct x86_emulate_ctxt *ctxt)
4057 size = (ctxt->d & ByteOp) ? 1 : ctxt->op_bytes;
4063 static int decode_imm(struct x86_emulate_ctxt *ctxt, struct operand *op,
4064 unsigned size, bool sign_extension)
4066 int rc = X86EMUL_CONTINUE;
4070 op->addr.mem.ea = ctxt->_eip;
4071 /* NB. Immediates are sign-extended as necessary. */
4072 switch (op->bytes) {
4074 op->val = insn_fetch(s8, ctxt);
4077 op->val = insn_fetch(s16, ctxt);
4080 op->val = insn_fetch(s32, ctxt);
4083 op->val = insn_fetch(s64, ctxt);
4086 if (!sign_extension) {
4087 switch (op->bytes) {
4095 op->val &= 0xffffffff;
4103 static int decode_operand(struct x86_emulate_ctxt *ctxt, struct operand *op,
4106 int rc = X86EMUL_CONTINUE;
4110 decode_register_operand(ctxt, op);
4113 rc = decode_imm(ctxt, op, 1, false);
4116 ctxt->memop.bytes = (ctxt->d & ByteOp) ? 1 : ctxt->op_bytes;
4120 if ((ctxt->d & BitOp) && op == &ctxt->dst)
4121 fetch_bit_operand(ctxt);
4122 op->orig_val = op->val;
4125 ctxt->memop.bytes = 8;
4129 op->bytes = (ctxt->d & ByteOp) ? 1 : ctxt->op_bytes;
4130 op->addr.reg = reg_rmw(ctxt, VCPU_REGS_RAX);
4131 fetch_register_operand(op);
4132 op->orig_val = op->val;
4136 op->bytes = (ctxt->d & ByteOp) ? 1 : ctxt->op_bytes;
4138 register_address(ctxt, reg_read(ctxt, VCPU_REGS_RDI));
4139 op->addr.mem.seg = VCPU_SREG_ES;
4146 op->addr.reg = reg_rmw(ctxt, VCPU_REGS_RDX);
4147 fetch_register_operand(op);
4151 op->val = reg_read(ctxt, VCPU_REGS_RCX) & 0xff;
4154 rc = decode_imm(ctxt, op, 1, true);
4161 rc = decode_imm(ctxt, op, imm_size(ctxt), true);
4164 rc = decode_imm(ctxt, op, ctxt->op_bytes, true);
4167 ctxt->memop.bytes = 1;
4170 ctxt->memop.bytes = 2;
4173 ctxt->memop.bytes = 4;
4176 rc = decode_imm(ctxt, op, 2, false);
4179 rc = decode_imm(ctxt, op, imm_size(ctxt), false);
4183 op->bytes = (ctxt->d & ByteOp) ? 1 : ctxt->op_bytes;
4185 register_address(ctxt, reg_read(ctxt, VCPU_REGS_RSI));
4186 op->addr.mem.seg = seg_override(ctxt);
4192 op->addr.mem.ea = ctxt->_eip;
4193 op->bytes = ctxt->op_bytes + 2;
4194 insn_fetch_arr(op->valptr, op->bytes, ctxt);
4197 ctxt->memop.bytes = ctxt->op_bytes + 2;
4200 op->val = VCPU_SREG_ES;
4203 op->val = VCPU_SREG_CS;
4206 op->val = VCPU_SREG_SS;
4209 op->val = VCPU_SREG_DS;
4212 op->val = VCPU_SREG_FS;
4215 op->val = VCPU_SREG_GS;
4218 /* Special instructions do their own operand decoding. */
4220 op->type = OP_NONE; /* Disable writeback. */
4228 int x86_decode_insn(struct x86_emulate_ctxt *ctxt, void *insn, int insn_len)
4230 int rc = X86EMUL_CONTINUE;
4231 int mode = ctxt->mode;
4232 int def_op_bytes, def_ad_bytes, goffset, simd_prefix;
4233 bool op_prefix = false;
4234 struct opcode opcode;
4236 ctxt->memop.type = OP_NONE;
4237 ctxt->memopp = NULL;
4238 ctxt->_eip = ctxt->eip;
4239 ctxt->fetch.start = ctxt->_eip;
4240 ctxt->fetch.end = ctxt->fetch.start + insn_len;
4242 memcpy(ctxt->fetch.data, insn, insn_len);
4245 case X86EMUL_MODE_REAL:
4246 case X86EMUL_MODE_VM86:
4247 case X86EMUL_MODE_PROT16:
4248 def_op_bytes = def_ad_bytes = 2;
4250 case X86EMUL_MODE_PROT32:
4251 def_op_bytes = def_ad_bytes = 4;
4253 #ifdef CONFIG_X86_64
4254 case X86EMUL_MODE_PROT64:
4260 return EMULATION_FAILED;
4263 ctxt->op_bytes = def_op_bytes;
4264 ctxt->ad_bytes = def_ad_bytes;
4266 /* Legacy prefixes. */
4268 switch (ctxt->b = insn_fetch(u8, ctxt)) {
4269 case 0x66: /* operand-size override */
4271 /* switch between 2/4 bytes */
4272 ctxt->op_bytes = def_op_bytes ^ 6;
4274 case 0x67: /* address-size override */
4275 if (mode == X86EMUL_MODE_PROT64)
4276 /* switch between 4/8 bytes */
4277 ctxt->ad_bytes = def_ad_bytes ^ 12;
4279 /* switch between 2/4 bytes */
4280 ctxt->ad_bytes = def_ad_bytes ^ 6;
4282 case 0x26: /* ES override */
4283 case 0x2e: /* CS override */
4284 case 0x36: /* SS override */
4285 case 0x3e: /* DS override */
4286 set_seg_override(ctxt, (ctxt->b >> 3) & 3);
4288 case 0x64: /* FS override */
4289 case 0x65: /* GS override */
4290 set_seg_override(ctxt, ctxt->b & 7);
4292 case 0x40 ... 0x4f: /* REX */
4293 if (mode != X86EMUL_MODE_PROT64)
4295 ctxt->rex_prefix = ctxt->b;
4297 case 0xf0: /* LOCK */
4298 ctxt->lock_prefix = 1;
4300 case 0xf2: /* REPNE/REPNZ */
4301 case 0xf3: /* REP/REPE/REPZ */
4302 ctxt->rep_prefix = ctxt->b;
4308 /* Any legacy prefix after a REX prefix nullifies its effect. */
4310 ctxt->rex_prefix = 0;
4316 if (ctxt->rex_prefix & 8)
4317 ctxt->op_bytes = 8; /* REX.W */
4319 /* Opcode byte(s). */
4320 opcode = opcode_table[ctxt->b];
4321 /* Two-byte opcode? */
4322 if (ctxt->b == 0x0f) {
4324 ctxt->b = insn_fetch(u8, ctxt);
4325 opcode = twobyte_table[ctxt->b];
4327 ctxt->d = opcode.flags;
4329 if (ctxt->d & ModRM)
4330 ctxt->modrm = insn_fetch(u8, ctxt);
4332 while (ctxt->d & GroupMask) {
4333 switch (ctxt->d & GroupMask) {
4335 goffset = (ctxt->modrm >> 3) & 7;
4336 opcode = opcode.u.group[goffset];
4339 goffset = (ctxt->modrm >> 3) & 7;
4340 if ((ctxt->modrm >> 6) == 3)
4341 opcode = opcode.u.gdual->mod3[goffset];
4343 opcode = opcode.u.gdual->mod012[goffset];
4346 goffset = ctxt->modrm & 7;
4347 opcode = opcode.u.group[goffset];
4350 if (ctxt->rep_prefix && op_prefix)
4351 return EMULATION_FAILED;
4352 simd_prefix = op_prefix ? 0x66 : ctxt->rep_prefix;
4353 switch (simd_prefix) {
4354 case 0x00: opcode = opcode.u.gprefix->pfx_no; break;
4355 case 0x66: opcode = opcode.u.gprefix->pfx_66; break;
4356 case 0xf2: opcode = opcode.u.gprefix->pfx_f2; break;
4357 case 0xf3: opcode = opcode.u.gprefix->pfx_f3; break;
4361 if (ctxt->modrm > 0xbf)
4362 opcode = opcode.u.esc->high[ctxt->modrm - 0xc0];
4364 opcode = opcode.u.esc->op[(ctxt->modrm >> 3) & 7];
4367 return EMULATION_FAILED;
4370 ctxt->d &= ~(u64)GroupMask;
4371 ctxt->d |= opcode.flags;
4374 ctxt->execute = opcode.u.execute;
4375 ctxt->check_perm = opcode.check_perm;
4376 ctxt->intercept = opcode.intercept;
4379 if (ctxt->d == 0 || (ctxt->d & Undefined))
4380 return EMULATION_FAILED;
4382 if (!(ctxt->d & VendorSpecific) && ctxt->only_vendor_specific_insn)
4383 return EMULATION_FAILED;
4385 if (mode == X86EMUL_MODE_PROT64 && (ctxt->d & Stack))
4388 if (ctxt->d & Op3264) {
4389 if (mode == X86EMUL_MODE_PROT64)
4396 ctxt->op_bytes = 16;
4397 else if (ctxt->d & Mmx)
4400 /* ModRM and SIB bytes. */
4401 if (ctxt->d & ModRM) {
4402 rc = decode_modrm(ctxt, &ctxt->memop);
4403 if (!ctxt->has_seg_override)
4404 set_seg_override(ctxt, ctxt->modrm_seg);
4405 } else if (ctxt->d & MemAbs)
4406 rc = decode_abs(ctxt, &ctxt->memop);
4407 if (rc != X86EMUL_CONTINUE)
4410 if (!ctxt->has_seg_override)
4411 set_seg_override(ctxt, VCPU_SREG_DS);
4413 ctxt->memop.addr.mem.seg = seg_override(ctxt);
4415 if (ctxt->memop.type == OP_MEM && ctxt->ad_bytes != 8)
4416 ctxt->memop.addr.mem.ea = (u32)ctxt->memop.addr.mem.ea;
4419 * Decode and fetch the source operand: register, memory
4422 rc = decode_operand(ctxt, &ctxt->src, (ctxt->d >> SrcShift) & OpMask);
4423 if (rc != X86EMUL_CONTINUE)
4427 * Decode and fetch the second source operand: register, memory
4430 rc = decode_operand(ctxt, &ctxt->src2, (ctxt->d >> Src2Shift) & OpMask);
4431 if (rc != X86EMUL_CONTINUE)
4434 /* Decode and fetch the destination operand: register or memory. */
4435 rc = decode_operand(ctxt, &ctxt->dst, (ctxt->d >> DstShift) & OpMask);
4438 if (ctxt->memopp && ctxt->memopp->type == OP_MEM && ctxt->rip_relative)
4439 ctxt->memopp->addr.mem.ea += ctxt->_eip;
4441 return (rc != X86EMUL_CONTINUE) ? EMULATION_FAILED : EMULATION_OK;
4444 bool x86_page_table_writing_insn(struct x86_emulate_ctxt *ctxt)
4446 return ctxt->d & PageTable;
4449 static bool string_insn_completed(struct x86_emulate_ctxt *ctxt)
4451 /* The second termination condition only applies for REPE
4452 * and REPNE. Test if the repeat string operation prefix is
4453 * REPE/REPZ or REPNE/REPNZ and if it's the case it tests the
4454 * corresponding termination condition according to:
4455 * - if REPE/REPZ and ZF = 0 then done
4456 * - if REPNE/REPNZ and ZF = 1 then done
4458 if (((ctxt->b == 0xa6) || (ctxt->b == 0xa7) ||
4459 (ctxt->b == 0xae) || (ctxt->b == 0xaf))
4460 && (((ctxt->rep_prefix == REPE_PREFIX) &&
4461 ((ctxt->eflags & EFLG_ZF) == 0))
4462 || ((ctxt->rep_prefix == REPNE_PREFIX) &&
4463 ((ctxt->eflags & EFLG_ZF) == EFLG_ZF))))
4469 static int flush_pending_x87_faults(struct x86_emulate_ctxt *ctxt)
4473 ctxt->ops->get_fpu(ctxt);
4474 asm volatile("1: fwait \n\t"
4476 ".pushsection .fixup,\"ax\" \n\t"
4478 "movb $1, %[fault] \n\t"
4481 _ASM_EXTABLE(1b, 3b)
4482 : [fault]"+qm"(fault));
4483 ctxt->ops->put_fpu(ctxt);
4485 if (unlikely(fault))
4486 return emulate_exception(ctxt, MF_VECTOR, 0, false);
4488 return X86EMUL_CONTINUE;
4491 static void fetch_possible_mmx_operand(struct x86_emulate_ctxt *ctxt,
4494 if (op->type == OP_MM)
4495 read_mmx_reg(ctxt, &op->mm_val, op->addr.mm);
4498 static int fastop(struct x86_emulate_ctxt *ctxt, void (*fop)(struct fastop *))
4500 ulong flags = (ctxt->eflags & EFLAGS_MASK) | X86_EFLAGS_IF;
4501 fop += __ffs(ctxt->dst.bytes) * FASTOP_SIZE;
4502 asm("push %[flags]; popf; call *%[fastop]; pushf; pop %[flags]\n"
4503 : "+a"(ctxt->dst.val), "+b"(ctxt->src.val), [flags]"+D"(flags)
4504 : "c"(ctxt->src2.val), [fastop]"S"(fop));
4505 ctxt->eflags = (ctxt->eflags & ~EFLAGS_MASK) | (flags & EFLAGS_MASK);
4506 return X86EMUL_CONTINUE;
4509 int x86_emulate_insn(struct x86_emulate_ctxt *ctxt)
4511 const struct x86_emulate_ops *ops = ctxt->ops;
4512 int rc = X86EMUL_CONTINUE;
4513 int saved_dst_type = ctxt->dst.type;
4515 ctxt->mem_read.pos = 0;
4517 if (ctxt->mode == X86EMUL_MODE_PROT64 && (ctxt->d & No64)) {
4518 rc = emulate_ud(ctxt);
4522 /* LOCK prefix is allowed only with some instructions */
4523 if (ctxt->lock_prefix && (!(ctxt->d & Lock) || ctxt->dst.type != OP_MEM)) {
4524 rc = emulate_ud(ctxt);
4528 if ((ctxt->d & SrcMask) == SrcMemFAddr && ctxt->src.type != OP_MEM) {
4529 rc = emulate_ud(ctxt);
4533 if (((ctxt->d & (Sse|Mmx)) && ((ops->get_cr(ctxt, 0) & X86_CR0_EM)))
4534 || ((ctxt->d & Sse) && !(ops->get_cr(ctxt, 4) & X86_CR4_OSFXSR))) {
4535 rc = emulate_ud(ctxt);
4539 if ((ctxt->d & (Sse|Mmx)) && (ops->get_cr(ctxt, 0) & X86_CR0_TS)) {
4540 rc = emulate_nm(ctxt);
4544 if (ctxt->d & Mmx) {
4545 rc = flush_pending_x87_faults(ctxt);
4546 if (rc != X86EMUL_CONTINUE)
4549 * Now that we know the fpu is exception safe, we can fetch
4552 fetch_possible_mmx_operand(ctxt, &ctxt->src);
4553 fetch_possible_mmx_operand(ctxt, &ctxt->src2);
4554 if (!(ctxt->d & Mov))
4555 fetch_possible_mmx_operand(ctxt, &ctxt->dst);
4558 if (unlikely(ctxt->guest_mode) && ctxt->intercept) {
4559 rc = emulator_check_intercept(ctxt, ctxt->intercept,
4560 X86_ICPT_PRE_EXCEPT);
4561 if (rc != X86EMUL_CONTINUE)
4565 /* Privileged instruction can be executed only in CPL=0 */
4566 if ((ctxt->d & Priv) && ops->cpl(ctxt)) {
4567 rc = emulate_gp(ctxt, 0);
4571 /* Instruction can only be executed in protected mode */
4572 if ((ctxt->d & Prot) && ctxt->mode < X86EMUL_MODE_PROT16) {
4573 rc = emulate_ud(ctxt);
4577 /* Do instruction specific permission checks */
4578 if (ctxt->check_perm) {
4579 rc = ctxt->check_perm(ctxt);
4580 if (rc != X86EMUL_CONTINUE)
4584 if (unlikely(ctxt->guest_mode) && ctxt->intercept) {
4585 rc = emulator_check_intercept(ctxt, ctxt->intercept,
4586 X86_ICPT_POST_EXCEPT);
4587 if (rc != X86EMUL_CONTINUE)
4591 if (ctxt->rep_prefix && (ctxt->d & String)) {
4592 /* All REP prefixes have the same first termination condition */
4593 if (address_mask(ctxt, reg_read(ctxt, VCPU_REGS_RCX)) == 0) {
4594 ctxt->eip = ctxt->_eip;
4599 if ((ctxt->src.type == OP_MEM) && !(ctxt->d & NoAccess)) {
4600 rc = segmented_read(ctxt, ctxt->src.addr.mem,
4601 ctxt->src.valptr, ctxt->src.bytes);
4602 if (rc != X86EMUL_CONTINUE)
4604 ctxt->src.orig_val64 = ctxt->src.val64;
4607 if (ctxt->src2.type == OP_MEM) {
4608 rc = segmented_read(ctxt, ctxt->src2.addr.mem,
4609 &ctxt->src2.val, ctxt->src2.bytes);
4610 if (rc != X86EMUL_CONTINUE)
4614 if ((ctxt->d & DstMask) == ImplicitOps)
4618 if ((ctxt->dst.type == OP_MEM) && !(ctxt->d & Mov)) {
4619 /* optimisation - avoid slow emulated read if Mov */
4620 rc = segmented_read(ctxt, ctxt->dst.addr.mem,
4621 &ctxt->dst.val, ctxt->dst.bytes);
4622 if (rc != X86EMUL_CONTINUE)
4625 ctxt->dst.orig_val = ctxt->dst.val;
4629 if (unlikely(ctxt->guest_mode) && ctxt->intercept) {
4630 rc = emulator_check_intercept(ctxt, ctxt->intercept,
4631 X86_ICPT_POST_MEMACCESS);
4632 if (rc != X86EMUL_CONTINUE)
4636 if (ctxt->execute) {
4637 if (ctxt->d & Fastop) {
4638 void (*fop)(struct fastop *) = (void *)ctxt->execute;
4639 rc = fastop(ctxt, fop);
4640 if (rc != X86EMUL_CONTINUE)
4644 rc = ctxt->execute(ctxt);
4645 if (rc != X86EMUL_CONTINUE)
4654 case 0x63: /* movsxd */
4655 if (ctxt->mode != X86EMUL_MODE_PROT64)
4656 goto cannot_emulate;
4657 ctxt->dst.val = (s32) ctxt->src.val;
4659 case 0x70 ... 0x7f: /* jcc (short) */
4660 if (test_cc(ctxt->b, ctxt->eflags))
4661 jmp_rel(ctxt, ctxt->src.val);
4663 case 0x8d: /* lea r16/r32, m */
4664 ctxt->dst.val = ctxt->src.addr.mem.ea;
4666 case 0x90 ... 0x97: /* nop / xchg reg, rax */
4667 if (ctxt->dst.addr.reg == reg_rmw(ctxt, VCPU_REGS_RAX))
4671 case 0x98: /* cbw/cwde/cdqe */
4672 switch (ctxt->op_bytes) {
4673 case 2: ctxt->dst.val = (s8)ctxt->dst.val; break;
4674 case 4: ctxt->dst.val = (s16)ctxt->dst.val; break;
4675 case 8: ctxt->dst.val = (s32)ctxt->dst.val; break;
4678 case 0xcc: /* int3 */
4679 rc = emulate_int(ctxt, 3);
4681 case 0xcd: /* int n */
4682 rc = emulate_int(ctxt, ctxt->src.val);
4684 case 0xce: /* into */
4685 if (ctxt->eflags & EFLG_OF)
4686 rc = emulate_int(ctxt, 4);
4688 case 0xe9: /* jmp rel */
4689 case 0xeb: /* jmp rel short */
4690 jmp_rel(ctxt, ctxt->src.val);
4691 ctxt->dst.type = OP_NONE; /* Disable writeback. */
4693 case 0xf4: /* hlt */
4694 ctxt->ops->halt(ctxt);
4696 case 0xf5: /* cmc */
4697 /* complement carry flag from eflags reg */
4698 ctxt->eflags ^= EFLG_CF;
4700 case 0xf8: /* clc */
4701 ctxt->eflags &= ~EFLG_CF;
4703 case 0xf9: /* stc */
4704 ctxt->eflags |= EFLG_CF;
4706 case 0xfc: /* cld */
4707 ctxt->eflags &= ~EFLG_DF;
4709 case 0xfd: /* std */
4710 ctxt->eflags |= EFLG_DF;
4713 goto cannot_emulate;
4716 if (rc != X86EMUL_CONTINUE)
4720 rc = writeback(ctxt);
4721 if (rc != X86EMUL_CONTINUE)
4725 * restore dst type in case the decoding will be reused
4726 * (happens for string instruction )
4728 ctxt->dst.type = saved_dst_type;
4730 if ((ctxt->d & SrcMask) == SrcSI)
4731 string_addr_inc(ctxt, VCPU_REGS_RSI, &ctxt->src);
4733 if ((ctxt->d & DstMask) == DstDI)
4734 string_addr_inc(ctxt, VCPU_REGS_RDI, &ctxt->dst);
4736 if (ctxt->rep_prefix && (ctxt->d & String)) {
4738 struct read_cache *r = &ctxt->io_read;
4739 if ((ctxt->d & SrcMask) == SrcSI)
4740 count = ctxt->src.count;
4742 count = ctxt->dst.count;
4743 register_address_increment(ctxt, reg_rmw(ctxt, VCPU_REGS_RCX),
4746 if (!string_insn_completed(ctxt)) {
4748 * Re-enter guest when pio read ahead buffer is empty
4749 * or, if it is not used, after each 1024 iteration.
4751 if ((r->end != 0 || reg_read(ctxt, VCPU_REGS_RCX) & 0x3ff) &&
4752 (r->end == 0 || r->end != r->pos)) {
4754 * Reset read cache. Usually happens before
4755 * decode, but since instruction is restarted
4756 * we have to do it here.
4758 ctxt->mem_read.end = 0;
4759 writeback_registers(ctxt);
4760 return EMULATION_RESTART;
4762 goto done; /* skip rip writeback */
4766 ctxt->eip = ctxt->_eip;
4769 if (rc == X86EMUL_PROPAGATE_FAULT)
4770 ctxt->have_exception = true;
4771 if (rc == X86EMUL_INTERCEPTED)
4772 return EMULATION_INTERCEPTED;
4774 if (rc == X86EMUL_CONTINUE)
4775 writeback_registers(ctxt);
4777 return (rc == X86EMUL_UNHANDLEABLE) ? EMULATION_FAILED : EMULATION_OK;
4781 case 0x09: /* wbinvd */
4782 (ctxt->ops->wbinvd)(ctxt);
4784 case 0x08: /* invd */
4785 case 0x0d: /* GrpP (prefetch) */
4786 case 0x18: /* Grp16 (prefetch/nop) */
4788 case 0x20: /* mov cr, reg */
4789 ctxt->dst.val = ops->get_cr(ctxt, ctxt->modrm_reg);
4791 case 0x21: /* mov from dr to reg */
4792 ops->get_dr(ctxt, ctxt->modrm_reg, &ctxt->dst.val);
4794 case 0x40 ... 0x4f: /* cmov */
4795 ctxt->dst.val = ctxt->dst.orig_val = ctxt->src.val;
4796 if (!test_cc(ctxt->b, ctxt->eflags))
4797 ctxt->dst.type = OP_NONE; /* no writeback */
4799 case 0x80 ... 0x8f: /* jnz rel, etc*/
4800 if (test_cc(ctxt->b, ctxt->eflags))
4801 jmp_rel(ctxt, ctxt->src.val);
4803 case 0x90 ... 0x9f: /* setcc r/m8 */
4804 ctxt->dst.val = test_cc(ctxt->b, ctxt->eflags);
4806 case 0xae: /* clflush */
4808 case 0xb6 ... 0xb7: /* movzx */
4809 ctxt->dst.bytes = ctxt->op_bytes;
4810 ctxt->dst.val = (ctxt->src.bytes == 1) ? (u8) ctxt->src.val
4811 : (u16) ctxt->src.val;
4813 case 0xbe ... 0xbf: /* movsx */
4814 ctxt->dst.bytes = ctxt->op_bytes;
4815 ctxt->dst.val = (ctxt->src.bytes == 1) ? (s8) ctxt->src.val :
4816 (s16) ctxt->src.val;
4818 case 0xc0 ... 0xc1: /* xadd */
4819 fastop(ctxt, em_add);
4820 /* Write back the register source. */
4821 ctxt->src.val = ctxt->dst.orig_val;
4822 write_register_operand(&ctxt->src);
4824 case 0xc3: /* movnti */
4825 ctxt->dst.bytes = ctxt->op_bytes;
4826 ctxt->dst.val = (ctxt->op_bytes == 4) ? (u32) ctxt->src.val :
4827 (u64) ctxt->src.val;
4830 goto cannot_emulate;
4833 if (rc != X86EMUL_CONTINUE)
4839 return EMULATION_FAILED;
4842 void emulator_invalidate_register_cache(struct x86_emulate_ctxt *ctxt)
4844 invalidate_registers(ctxt);
4847 void emulator_writeback_register_cache(struct x86_emulate_ctxt *ctxt)
4849 writeback_registers(ctxt);