1 /******************************************************************************
4 * Generic x86 (32-bit and 64-bit) instruction decoder and emulator.
6 * Copyright (c) 2005 Keir Fraser
8 * Linux coding style, mod r/m decoder, segment base fixes, real-mode
9 * privileged instructions:
11 * Copyright (C) 2006 Qumranet
12 * Copyright 2010 Red Hat, Inc. and/or its affiliates.
14 * Avi Kivity <avi@qumranet.com>
15 * Yaniv Kamay <yaniv@qumranet.com>
17 * This work is licensed under the terms of the GNU GPL, version 2. See
18 * the COPYING file in the top-level directory.
20 * From: xen-unstable 10676:af9809f51f81a3c43f276f00c81a52ef558afda4
23 #include <linux/kvm_host.h>
24 #include "kvm_cache_regs.h"
25 #include <linux/module.h>
26 #include <asm/kvm_emulate.h>
27 #include <linux/stringify.h>
36 #define OpImplicit 1ull /* No generic decode */
37 #define OpReg 2ull /* Register */
38 #define OpMem 3ull /* Memory */
39 #define OpAcc 4ull /* Accumulator: AL/AX/EAX/RAX */
40 #define OpDI 5ull /* ES:DI/EDI/RDI */
41 #define OpMem64 6ull /* Memory, 64-bit */
42 #define OpImmUByte 7ull /* Zero-extended 8-bit immediate */
43 #define OpDX 8ull /* DX register */
44 #define OpCL 9ull /* CL register (for shifts) */
45 #define OpImmByte 10ull /* 8-bit sign extended immediate */
46 #define OpOne 11ull /* Implied 1 */
47 #define OpImm 12ull /* Sign extended up to 32-bit immediate */
48 #define OpMem16 13ull /* Memory operand (16-bit). */
49 #define OpMem32 14ull /* Memory operand (32-bit). */
50 #define OpImmU 15ull /* Immediate operand, zero extended */
51 #define OpSI 16ull /* SI/ESI/RSI */
52 #define OpImmFAddr 17ull /* Immediate far address */
53 #define OpMemFAddr 18ull /* Far address in memory */
54 #define OpImmU16 19ull /* Immediate operand, 16 bits, zero extended */
55 #define OpES 20ull /* ES */
56 #define OpCS 21ull /* CS */
57 #define OpSS 22ull /* SS */
58 #define OpDS 23ull /* DS */
59 #define OpFS 24ull /* FS */
60 #define OpGS 25ull /* GS */
61 #define OpMem8 26ull /* 8-bit zero extended memory operand */
62 #define OpImm64 27ull /* Sign extended 16/32/64-bit immediate */
64 #define OpBits 5 /* Width of operand field */
65 #define OpMask ((1ull << OpBits) - 1)
68 * Opcode effective-address decode tables.
69 * Note that we only emulate instructions that have at least one memory
70 * operand (excluding implicit stack references). We assume that stack
71 * references and instruction fetches will never occur in special memory
72 * areas that require emulation. So, for example, 'mov <imm>,<reg>' need
76 /* Operand sizes: 8-bit operands or specified/overridden size. */
77 #define ByteOp (1<<0) /* 8-bit operands. */
78 /* Destination operand type. */
80 #define ImplicitOps (OpImplicit << DstShift)
81 #define DstReg (OpReg << DstShift)
82 #define DstMem (OpMem << DstShift)
83 #define DstAcc (OpAcc << DstShift)
84 #define DstDI (OpDI << DstShift)
85 #define DstMem64 (OpMem64 << DstShift)
86 #define DstImmUByte (OpImmUByte << DstShift)
87 #define DstDX (OpDX << DstShift)
88 #define DstMask (OpMask << DstShift)
89 /* Source operand type. */
91 #define SrcNone (OpNone << SrcShift)
92 #define SrcReg (OpReg << SrcShift)
93 #define SrcMem (OpMem << SrcShift)
94 #define SrcMem16 (OpMem16 << SrcShift)
95 #define SrcMem32 (OpMem32 << SrcShift)
96 #define SrcImm (OpImm << SrcShift)
97 #define SrcImmByte (OpImmByte << SrcShift)
98 #define SrcOne (OpOne << SrcShift)
99 #define SrcImmUByte (OpImmUByte << SrcShift)
100 #define SrcImmU (OpImmU << SrcShift)
101 #define SrcSI (OpSI << SrcShift)
102 #define SrcImmFAddr (OpImmFAddr << SrcShift)
103 #define SrcMemFAddr (OpMemFAddr << SrcShift)
104 #define SrcAcc (OpAcc << SrcShift)
105 #define SrcImmU16 (OpImmU16 << SrcShift)
106 #define SrcImm64 (OpImm64 << SrcShift)
107 #define SrcDX (OpDX << SrcShift)
108 #define SrcMem8 (OpMem8 << SrcShift)
109 #define SrcMask (OpMask << SrcShift)
110 #define BitOp (1<<11)
111 #define MemAbs (1<<12) /* Memory operand is absolute displacement */
112 #define String (1<<13) /* String instruction (rep capable) */
113 #define Stack (1<<14) /* Stack instruction (push/pop) */
114 #define GroupMask (7<<15) /* Opcode uses one of the group mechanisms */
115 #define Group (1<<15) /* Bits 3:5 of modrm byte extend opcode */
116 #define GroupDual (2<<15) /* Alternate decoding of mod == 3 */
117 #define Prefix (3<<15) /* Instruction varies with 66/f2/f3 prefix */
118 #define RMExt (4<<15) /* Opcode extension in ModRM r/m if mod == 3 */
119 #define Escape (5<<15) /* Escape to coprocessor instruction */
120 #define Sse (1<<18) /* SSE Vector instruction */
121 /* Generic ModRM decode. */
122 #define ModRM (1<<19)
123 /* Destination is only written; never read. */
126 #define Prot (1<<21) /* instruction generates #UD if not in prot-mode */
127 #define VendorSpecific (1<<22) /* Vendor specific instruction */
128 #define NoAccess (1<<23) /* Don't access memory (lea/invlpg/verr etc) */
129 #define Op3264 (1<<24) /* Operand is 64b in long mode, 32b otherwise */
130 #define Undefined (1<<25) /* No Such Instruction */
131 #define Lock (1<<26) /* lock prefix is allowed for the instruction */
132 #define Priv (1<<27) /* instruction generates #GP if current CPL != 0 */
134 #define PageTable (1 << 29) /* instruction used to write page table */
135 /* Source 2 operand type */
136 #define Src2Shift (30)
137 #define Src2None (OpNone << Src2Shift)
138 #define Src2CL (OpCL << Src2Shift)
139 #define Src2ImmByte (OpImmByte << Src2Shift)
140 #define Src2One (OpOne << Src2Shift)
141 #define Src2Imm (OpImm << Src2Shift)
142 #define Src2ES (OpES << Src2Shift)
143 #define Src2CS (OpCS << Src2Shift)
144 #define Src2SS (OpSS << Src2Shift)
145 #define Src2DS (OpDS << Src2Shift)
146 #define Src2FS (OpFS << Src2Shift)
147 #define Src2GS (OpGS << Src2Shift)
148 #define Src2Mask (OpMask << Src2Shift)
149 #define Mmx ((u64)1 << 40) /* MMX Vector instruction */
150 #define Aligned ((u64)1 << 41) /* Explicitly aligned (e.g. MOVDQA) */
151 #define Unaligned ((u64)1 << 42) /* Explicitly unaligned (e.g. MOVDQU) */
152 #define Avx ((u64)1 << 43) /* Advanced Vector Extensions */
153 #define Fastop ((u64)1 << 44) /* Use opcode::u.fastop */
155 #define X2(x...) x, x
156 #define X3(x...) X2(x), x
157 #define X4(x...) X2(x), X2(x)
158 #define X5(x...) X4(x), x
159 #define X6(x...) X4(x), X2(x)
160 #define X7(x...) X4(x), X3(x)
161 #define X8(x...) X4(x), X4(x)
162 #define X16(x...) X8(x), X8(x)
164 #define NR_FASTOP (ilog2(sizeof(ulong)) + 1)
165 #define FASTOP_SIZE 8
168 * fastop functions have a special calling convention:
170 * dst: [rdx]:rax (in/out)
173 * flags: rflags (in/out)
175 * Moreover, they are all exactly FASTOP_SIZE bytes long, so functions for
176 * different operand sizes can be reached by calculation, rather than a jump
177 * table (which would be bigger than the code).
179 * fastop functions are declared as taking a never-defined fastop parameter,
180 * so they can't be called from C directly.
189 int (*execute)(struct x86_emulate_ctxt *ctxt);
190 const struct opcode *group;
191 const struct group_dual *gdual;
192 const struct gprefix *gprefix;
193 const struct escape *esc;
194 void (*fastop)(struct fastop *fake);
196 int (*check_perm)(struct x86_emulate_ctxt *ctxt);
200 struct opcode mod012[8];
201 struct opcode mod3[8];
205 struct opcode pfx_no;
206 struct opcode pfx_66;
207 struct opcode pfx_f2;
208 struct opcode pfx_f3;
213 struct opcode high[64];
216 /* EFLAGS bit definitions. */
217 #define EFLG_ID (1<<21)
218 #define EFLG_VIP (1<<20)
219 #define EFLG_VIF (1<<19)
220 #define EFLG_AC (1<<18)
221 #define EFLG_VM (1<<17)
222 #define EFLG_RF (1<<16)
223 #define EFLG_IOPL (3<<12)
224 #define EFLG_NT (1<<14)
225 #define EFLG_OF (1<<11)
226 #define EFLG_DF (1<<10)
227 #define EFLG_IF (1<<9)
228 #define EFLG_TF (1<<8)
229 #define EFLG_SF (1<<7)
230 #define EFLG_ZF (1<<6)
231 #define EFLG_AF (1<<4)
232 #define EFLG_PF (1<<2)
233 #define EFLG_CF (1<<0)
235 #define EFLG_RESERVED_ZEROS_MASK 0xffc0802a
236 #define EFLG_RESERVED_ONE_MASK 2
238 static ulong reg_read(struct x86_emulate_ctxt *ctxt, unsigned nr)
240 if (!(ctxt->regs_valid & (1 << nr))) {
241 ctxt->regs_valid |= 1 << nr;
242 ctxt->_regs[nr] = ctxt->ops->read_gpr(ctxt, nr);
244 return ctxt->_regs[nr];
247 static ulong *reg_write(struct x86_emulate_ctxt *ctxt, unsigned nr)
249 ctxt->regs_valid |= 1 << nr;
250 ctxt->regs_dirty |= 1 << nr;
251 return &ctxt->_regs[nr];
254 static ulong *reg_rmw(struct x86_emulate_ctxt *ctxt, unsigned nr)
257 return reg_write(ctxt, nr);
260 static void writeback_registers(struct x86_emulate_ctxt *ctxt)
264 for_each_set_bit(reg, (ulong *)&ctxt->regs_dirty, 16)
265 ctxt->ops->write_gpr(ctxt, reg, ctxt->_regs[reg]);
268 static void invalidate_registers(struct x86_emulate_ctxt *ctxt)
270 ctxt->regs_dirty = 0;
271 ctxt->regs_valid = 0;
275 * Instruction emulation:
276 * Most instructions are emulated directly via a fragment of inline assembly
277 * code. This allows us to save/restore EFLAGS and thus very easily pick up
278 * any modified flags.
281 #if defined(CONFIG_X86_64)
282 #define _LO32 "k" /* force 32-bit operand */
283 #define _STK "%%rsp" /* stack pointer */
284 #elif defined(__i386__)
285 #define _LO32 "" /* force 32-bit operand */
286 #define _STK "%%esp" /* stack pointer */
290 * These EFLAGS bits are restored from saved value during emulation, and
291 * any changes are written back to the saved value after emulation.
293 #define EFLAGS_MASK (EFLG_OF|EFLG_SF|EFLG_ZF|EFLG_AF|EFLG_PF|EFLG_CF)
295 /* Before executing instruction: restore necessary bits in EFLAGS. */
296 #define _PRE_EFLAGS(_sav, _msk, _tmp) \
297 /* EFLAGS = (_sav & _msk) | (EFLAGS & ~_msk); _sav &= ~_msk; */ \
298 "movl %"_sav",%"_LO32 _tmp"; " \
301 "movl %"_msk",%"_LO32 _tmp"; " \
302 "andl %"_LO32 _tmp",("_STK"); " \
304 "notl %"_LO32 _tmp"; " \
305 "andl %"_LO32 _tmp",("_STK"); " \
306 "andl %"_LO32 _tmp","__stringify(BITS_PER_LONG/4)"("_STK"); " \
308 "orl %"_LO32 _tmp",("_STK"); " \
312 /* After executing instruction: write-back necessary bits in EFLAGS. */
313 #define _POST_EFLAGS(_sav, _msk, _tmp) \
314 /* _sav |= EFLAGS & _msk; */ \
317 "andl %"_msk",%"_LO32 _tmp"; " \
318 "orl %"_LO32 _tmp",%"_sav"; "
326 #define ____emulate_2op(ctxt, _op, _x, _y, _suffix, _dsttype) \
328 __asm__ __volatile__ ( \
329 _PRE_EFLAGS("0", "4", "2") \
330 _op _suffix " %"_x"3,%1; " \
331 _POST_EFLAGS("0", "4", "2") \
332 : "=m" ((ctxt)->eflags), \
333 "+q" (*(_dsttype*)&(ctxt)->dst.val), \
335 : _y ((ctxt)->src.val), "i" (EFLAGS_MASK)); \
339 /* Raw emulation: instruction has two explicit operands. */
340 #define __emulate_2op_nobyte(ctxt,_op,_wx,_wy,_lx,_ly,_qx,_qy) \
342 unsigned long _tmp; \
344 switch ((ctxt)->dst.bytes) { \
346 ____emulate_2op(ctxt,_op,_wx,_wy,"w",u16); \
349 ____emulate_2op(ctxt,_op,_lx,_ly,"l",u32); \
352 ON64(____emulate_2op(ctxt,_op,_qx,_qy,"q",u64)); \
357 #define __emulate_2op(ctxt,_op,_bx,_by,_wx,_wy,_lx,_ly,_qx,_qy) \
359 unsigned long _tmp; \
360 switch ((ctxt)->dst.bytes) { \
362 ____emulate_2op(ctxt,_op,_bx,_by,"b",u8); \
365 __emulate_2op_nobyte(ctxt, _op, \
366 _wx, _wy, _lx, _ly, _qx, _qy); \
371 /* Source operand is byte-sized and may be restricted to just %cl. */
372 #define emulate_2op_SrcB(ctxt, _op) \
373 __emulate_2op(ctxt, _op, "b", "c", "b", "c", "b", "c", "b", "c")
375 /* Source operand is byte, word, long or quad sized. */
376 #define emulate_2op_SrcV(ctxt, _op) \
377 __emulate_2op(ctxt, _op, "b", "q", "w", "r", _LO32, "r", "", "r")
379 /* Source operand is word, long or quad sized. */
380 #define emulate_2op_SrcV_nobyte(ctxt, _op) \
381 __emulate_2op_nobyte(ctxt, _op, "w", "r", _LO32, "r", "", "r")
383 /* Instruction has three operands and one operand is stored in ECX register */
384 #define __emulate_2op_cl(ctxt, _op, _suffix, _type) \
386 unsigned long _tmp; \
387 _type _clv = (ctxt)->src2.val; \
388 _type _srcv = (ctxt)->src.val; \
389 _type _dstv = (ctxt)->dst.val; \
391 __asm__ __volatile__ ( \
392 _PRE_EFLAGS("0", "5", "2") \
393 _op _suffix " %4,%1 \n" \
394 _POST_EFLAGS("0", "5", "2") \
395 : "=m" ((ctxt)->eflags), "+r" (_dstv), "=&r" (_tmp) \
396 : "c" (_clv) , "r" (_srcv), "i" (EFLAGS_MASK) \
399 (ctxt)->src2.val = (unsigned long) _clv; \
400 (ctxt)->src2.val = (unsigned long) _srcv; \
401 (ctxt)->dst.val = (unsigned long) _dstv; \
404 #define emulate_2op_cl(ctxt, _op) \
406 switch ((ctxt)->dst.bytes) { \
408 __emulate_2op_cl(ctxt, _op, "w", u16); \
411 __emulate_2op_cl(ctxt, _op, "l", u32); \
414 ON64(__emulate_2op_cl(ctxt, _op, "q", ulong)); \
419 #define __emulate_1op(ctxt, _op, _suffix) \
421 unsigned long _tmp; \
423 __asm__ __volatile__ ( \
424 _PRE_EFLAGS("0", "3", "2") \
425 _op _suffix " %1; " \
426 _POST_EFLAGS("0", "3", "2") \
427 : "=m" ((ctxt)->eflags), "+m" ((ctxt)->dst.val), \
429 : "i" (EFLAGS_MASK)); \
432 /* Instruction has only one explicit operand (no source operand). */
433 #define emulate_1op(ctxt, _op) \
435 switch ((ctxt)->dst.bytes) { \
436 case 1: __emulate_1op(ctxt, _op, "b"); break; \
437 case 2: __emulate_1op(ctxt, _op, "w"); break; \
438 case 4: __emulate_1op(ctxt, _op, "l"); break; \
439 case 8: ON64(__emulate_1op(ctxt, _op, "q")); break; \
443 #define FOP_ALIGN ".align " __stringify(FASTOP_SIZE) " \n\t"
444 #define FOP_RET "ret \n\t"
446 #define FOP_START(op) \
447 extern void em_##op(struct fastop *fake); \
448 asm(".pushsection .text, \"ax\" \n\t" \
449 ".global em_" #op " \n\t" \
456 #define FOP1E(op, dst) \
457 FOP_ALIGN #op " %" #dst " \n\t" FOP_RET
459 #define FASTOP1(op) \
464 ON64(FOP1E(op##q, rax)) \
467 #define __emulate_1op_rax_rdx(ctxt, _op, _suffix, _ex) \
469 unsigned long _tmp; \
470 ulong *rax = reg_rmw((ctxt), VCPU_REGS_RAX); \
471 ulong *rdx = reg_rmw((ctxt), VCPU_REGS_RDX); \
473 __asm__ __volatile__ ( \
474 _PRE_EFLAGS("0", "5", "1") \
476 _op _suffix " %6; " \
478 _POST_EFLAGS("0", "5", "1") \
479 ".pushsection .fixup,\"ax\" \n\t" \
480 "3: movb $1, %4 \n\t" \
483 _ASM_EXTABLE(1b, 3b) \
484 : "=m" ((ctxt)->eflags), "=&r" (_tmp), \
485 "+a" (*rax), "+d" (*rdx), "+qm"(_ex) \
486 : "i" (EFLAGS_MASK), "m" ((ctxt)->src.val)); \
489 /* instruction has only one source operand, destination is implicit (e.g. mul, div, imul, idiv) */
490 #define emulate_1op_rax_rdx(ctxt, _op, _ex) \
492 switch((ctxt)->src.bytes) { \
494 __emulate_1op_rax_rdx(ctxt, _op, "b", _ex); \
497 __emulate_1op_rax_rdx(ctxt, _op, "w", _ex); \
500 __emulate_1op_rax_rdx(ctxt, _op, "l", _ex); \
503 __emulate_1op_rax_rdx(ctxt, _op, "q", _ex)); \
508 static int emulator_check_intercept(struct x86_emulate_ctxt *ctxt,
509 enum x86_intercept intercept,
510 enum x86_intercept_stage stage)
512 struct x86_instruction_info info = {
513 .intercept = intercept,
514 .rep_prefix = ctxt->rep_prefix,
515 .modrm_mod = ctxt->modrm_mod,
516 .modrm_reg = ctxt->modrm_reg,
517 .modrm_rm = ctxt->modrm_rm,
518 .src_val = ctxt->src.val64,
519 .src_bytes = ctxt->src.bytes,
520 .dst_bytes = ctxt->dst.bytes,
521 .ad_bytes = ctxt->ad_bytes,
522 .next_rip = ctxt->eip,
525 return ctxt->ops->intercept(ctxt, &info, stage);
528 static void assign_masked(ulong *dest, ulong src, ulong mask)
530 *dest = (*dest & ~mask) | (src & mask);
533 static inline unsigned long ad_mask(struct x86_emulate_ctxt *ctxt)
535 return (1UL << (ctxt->ad_bytes << 3)) - 1;
538 static ulong stack_mask(struct x86_emulate_ctxt *ctxt)
541 struct desc_struct ss;
543 if (ctxt->mode == X86EMUL_MODE_PROT64)
545 ctxt->ops->get_segment(ctxt, &sel, &ss, NULL, VCPU_SREG_SS);
546 return ~0U >> ((ss.d ^ 1) * 16); /* d=0: 0xffff; d=1: 0xffffffff */
549 static int stack_size(struct x86_emulate_ctxt *ctxt)
551 return (__fls(stack_mask(ctxt)) + 1) >> 3;
554 /* Access/update address held in a register, based on addressing mode. */
555 static inline unsigned long
556 address_mask(struct x86_emulate_ctxt *ctxt, unsigned long reg)
558 if (ctxt->ad_bytes == sizeof(unsigned long))
561 return reg & ad_mask(ctxt);
564 static inline unsigned long
565 register_address(struct x86_emulate_ctxt *ctxt, unsigned long reg)
567 return address_mask(ctxt, reg);
570 static void masked_increment(ulong *reg, ulong mask, int inc)
572 assign_masked(reg, *reg + inc, mask);
576 register_address_increment(struct x86_emulate_ctxt *ctxt, unsigned long *reg, int inc)
580 if (ctxt->ad_bytes == sizeof(unsigned long))
583 mask = ad_mask(ctxt);
584 masked_increment(reg, mask, inc);
587 static void rsp_increment(struct x86_emulate_ctxt *ctxt, int inc)
589 masked_increment(reg_rmw(ctxt, VCPU_REGS_RSP), stack_mask(ctxt), inc);
592 static inline void jmp_rel(struct x86_emulate_ctxt *ctxt, int rel)
594 register_address_increment(ctxt, &ctxt->_eip, rel);
597 static u32 desc_limit_scaled(struct desc_struct *desc)
599 u32 limit = get_desc_limit(desc);
601 return desc->g ? (limit << 12) | 0xfff : limit;
604 static void set_seg_override(struct x86_emulate_ctxt *ctxt, int seg)
606 ctxt->has_seg_override = true;
607 ctxt->seg_override = seg;
610 static unsigned long seg_base(struct x86_emulate_ctxt *ctxt, int seg)
612 if (ctxt->mode == X86EMUL_MODE_PROT64 && seg < VCPU_SREG_FS)
615 return ctxt->ops->get_cached_segment_base(ctxt, seg);
618 static unsigned seg_override(struct x86_emulate_ctxt *ctxt)
620 if (!ctxt->has_seg_override)
623 return ctxt->seg_override;
626 static int emulate_exception(struct x86_emulate_ctxt *ctxt, int vec,
627 u32 error, bool valid)
629 ctxt->exception.vector = vec;
630 ctxt->exception.error_code = error;
631 ctxt->exception.error_code_valid = valid;
632 return X86EMUL_PROPAGATE_FAULT;
635 static int emulate_db(struct x86_emulate_ctxt *ctxt)
637 return emulate_exception(ctxt, DB_VECTOR, 0, false);
640 static int emulate_gp(struct x86_emulate_ctxt *ctxt, int err)
642 return emulate_exception(ctxt, GP_VECTOR, err, true);
645 static int emulate_ss(struct x86_emulate_ctxt *ctxt, int err)
647 return emulate_exception(ctxt, SS_VECTOR, err, true);
650 static int emulate_ud(struct x86_emulate_ctxt *ctxt)
652 return emulate_exception(ctxt, UD_VECTOR, 0, false);
655 static int emulate_ts(struct x86_emulate_ctxt *ctxt, int err)
657 return emulate_exception(ctxt, TS_VECTOR, err, true);
660 static int emulate_de(struct x86_emulate_ctxt *ctxt)
662 return emulate_exception(ctxt, DE_VECTOR, 0, false);
665 static int emulate_nm(struct x86_emulate_ctxt *ctxt)
667 return emulate_exception(ctxt, NM_VECTOR, 0, false);
670 static u16 get_segment_selector(struct x86_emulate_ctxt *ctxt, unsigned seg)
673 struct desc_struct desc;
675 ctxt->ops->get_segment(ctxt, &selector, &desc, NULL, seg);
679 static void set_segment_selector(struct x86_emulate_ctxt *ctxt, u16 selector,
684 struct desc_struct desc;
686 ctxt->ops->get_segment(ctxt, &dummy, &desc, &base3, seg);
687 ctxt->ops->set_segment(ctxt, selector, &desc, base3, seg);
691 * x86 defines three classes of vector instructions: explicitly
692 * aligned, explicitly unaligned, and the rest, which change behaviour
693 * depending on whether they're AVX encoded or not.
695 * Also included is CMPXCHG16B which is not a vector instruction, yet it is
696 * subject to the same check.
698 static bool insn_aligned(struct x86_emulate_ctxt *ctxt, unsigned size)
700 if (likely(size < 16))
703 if (ctxt->d & Aligned)
705 else if (ctxt->d & Unaligned)
707 else if (ctxt->d & Avx)
713 static int __linearize(struct x86_emulate_ctxt *ctxt,
714 struct segmented_address addr,
715 unsigned size, bool write, bool fetch,
718 struct desc_struct desc;
725 la = seg_base(ctxt, addr.seg) + addr.ea;
726 switch (ctxt->mode) {
727 case X86EMUL_MODE_PROT64:
728 if (((signed long)la << 16) >> 16 != la)
729 return emulate_gp(ctxt, 0);
732 usable = ctxt->ops->get_segment(ctxt, &sel, &desc, NULL,
736 /* code segment in protected mode or read-only data segment */
737 if ((((ctxt->mode != X86EMUL_MODE_REAL) && (desc.type & 8))
738 || !(desc.type & 2)) && write)
740 /* unreadable code segment */
741 if (!fetch && (desc.type & 8) && !(desc.type & 2))
743 lim = desc_limit_scaled(&desc);
744 if ((desc.type & 8) || !(desc.type & 4)) {
745 /* expand-up segment */
746 if (addr.ea > lim || (u32)(addr.ea + size - 1) > lim)
749 /* expand-down segment */
750 if (addr.ea <= lim || (u32)(addr.ea + size - 1) <= lim)
752 lim = desc.d ? 0xffffffff : 0xffff;
753 if (addr.ea > lim || (u32)(addr.ea + size - 1) > lim)
756 cpl = ctxt->ops->cpl(ctxt);
757 if (!(desc.type & 8)) {
761 } else if ((desc.type & 8) && !(desc.type & 4)) {
762 /* nonconforming code segment */
765 } else if ((desc.type & 8) && (desc.type & 4)) {
766 /* conforming code segment */
772 if (fetch ? ctxt->mode != X86EMUL_MODE_PROT64 : ctxt->ad_bytes != 8)
774 if (insn_aligned(ctxt, size) && ((la & (size - 1)) != 0))
775 return emulate_gp(ctxt, 0);
777 return X86EMUL_CONTINUE;
779 if (addr.seg == VCPU_SREG_SS)
780 return emulate_ss(ctxt, sel);
782 return emulate_gp(ctxt, sel);
785 static int linearize(struct x86_emulate_ctxt *ctxt,
786 struct segmented_address addr,
787 unsigned size, bool write,
790 return __linearize(ctxt, addr, size, write, false, linear);
794 static int segmented_read_std(struct x86_emulate_ctxt *ctxt,
795 struct segmented_address addr,
802 rc = linearize(ctxt, addr, size, false, &linear);
803 if (rc != X86EMUL_CONTINUE)
805 return ctxt->ops->read_std(ctxt, linear, data, size, &ctxt->exception);
809 * Fetch the next byte of the instruction being emulated which is pointed to
810 * by ctxt->_eip, then increment ctxt->_eip.
812 * Also prefetch the remaining bytes of the instruction without crossing page
813 * boundary if they are not in fetch_cache yet.
815 static int do_insn_fetch_byte(struct x86_emulate_ctxt *ctxt, u8 *dest)
817 struct fetch_cache *fc = &ctxt->fetch;
821 if (ctxt->_eip == fc->end) {
822 unsigned long linear;
823 struct segmented_address addr = { .seg = VCPU_SREG_CS,
825 cur_size = fc->end - fc->start;
826 size = min(15UL - cur_size,
827 PAGE_SIZE - offset_in_page(ctxt->_eip));
828 rc = __linearize(ctxt, addr, size, false, true, &linear);
829 if (unlikely(rc != X86EMUL_CONTINUE))
831 rc = ctxt->ops->fetch(ctxt, linear, fc->data + cur_size,
832 size, &ctxt->exception);
833 if (unlikely(rc != X86EMUL_CONTINUE))
837 *dest = fc->data[ctxt->_eip - fc->start];
839 return X86EMUL_CONTINUE;
842 static int do_insn_fetch(struct x86_emulate_ctxt *ctxt,
843 void *dest, unsigned size)
847 /* x86 instructions are limited to 15 bytes. */
848 if (unlikely(ctxt->_eip + size - ctxt->eip > 15))
849 return X86EMUL_UNHANDLEABLE;
851 rc = do_insn_fetch_byte(ctxt, dest++);
852 if (rc != X86EMUL_CONTINUE)
855 return X86EMUL_CONTINUE;
858 /* Fetch next part of the instruction being emulated. */
859 #define insn_fetch(_type, _ctxt) \
860 ({ unsigned long _x; \
861 rc = do_insn_fetch(_ctxt, &_x, sizeof(_type)); \
862 if (rc != X86EMUL_CONTINUE) \
867 #define insn_fetch_arr(_arr, _size, _ctxt) \
868 ({ rc = do_insn_fetch(_ctxt, _arr, (_size)); \
869 if (rc != X86EMUL_CONTINUE) \
874 * Given the 'reg' portion of a ModRM byte, and a register block, return a
875 * pointer into the block that addresses the relevant register.
876 * @highbyte_regs specifies whether to decode AH,CH,DH,BH.
878 static void *decode_register(struct x86_emulate_ctxt *ctxt, u8 modrm_reg,
883 if (highbyte_regs && modrm_reg >= 4 && modrm_reg < 8)
884 p = (unsigned char *)reg_rmw(ctxt, modrm_reg & 3) + 1;
886 p = reg_rmw(ctxt, modrm_reg);
890 static int read_descriptor(struct x86_emulate_ctxt *ctxt,
891 struct segmented_address addr,
892 u16 *size, unsigned long *address, int op_bytes)
899 rc = segmented_read_std(ctxt, addr, size, 2);
900 if (rc != X86EMUL_CONTINUE)
903 rc = segmented_read_std(ctxt, addr, address, op_bytes);
907 static int test_cc(unsigned int condition, unsigned int flags)
911 switch ((condition & 15) >> 1) {
913 rc |= (flags & EFLG_OF);
915 case 1: /* b/c/nae */
916 rc |= (flags & EFLG_CF);
919 rc |= (flags & EFLG_ZF);
922 rc |= (flags & (EFLG_CF|EFLG_ZF));
925 rc |= (flags & EFLG_SF);
928 rc |= (flags & EFLG_PF);
931 rc |= (flags & EFLG_ZF);
934 rc |= (!(flags & EFLG_SF) != !(flags & EFLG_OF));
938 /* Odd condition identifiers (lsb == 1) have inverted sense. */
939 return (!!rc ^ (condition & 1));
942 static void fetch_register_operand(struct operand *op)
946 op->val = *(u8 *)op->addr.reg;
949 op->val = *(u16 *)op->addr.reg;
952 op->val = *(u32 *)op->addr.reg;
955 op->val = *(u64 *)op->addr.reg;
960 static void read_sse_reg(struct x86_emulate_ctxt *ctxt, sse128_t *data, int reg)
962 ctxt->ops->get_fpu(ctxt);
964 case 0: asm("movdqa %%xmm0, %0" : "=m"(*data)); break;
965 case 1: asm("movdqa %%xmm1, %0" : "=m"(*data)); break;
966 case 2: asm("movdqa %%xmm2, %0" : "=m"(*data)); break;
967 case 3: asm("movdqa %%xmm3, %0" : "=m"(*data)); break;
968 case 4: asm("movdqa %%xmm4, %0" : "=m"(*data)); break;
969 case 5: asm("movdqa %%xmm5, %0" : "=m"(*data)); break;
970 case 6: asm("movdqa %%xmm6, %0" : "=m"(*data)); break;
971 case 7: asm("movdqa %%xmm7, %0" : "=m"(*data)); break;
973 case 8: asm("movdqa %%xmm8, %0" : "=m"(*data)); break;
974 case 9: asm("movdqa %%xmm9, %0" : "=m"(*data)); break;
975 case 10: asm("movdqa %%xmm10, %0" : "=m"(*data)); break;
976 case 11: asm("movdqa %%xmm11, %0" : "=m"(*data)); break;
977 case 12: asm("movdqa %%xmm12, %0" : "=m"(*data)); break;
978 case 13: asm("movdqa %%xmm13, %0" : "=m"(*data)); break;
979 case 14: asm("movdqa %%xmm14, %0" : "=m"(*data)); break;
980 case 15: asm("movdqa %%xmm15, %0" : "=m"(*data)); break;
984 ctxt->ops->put_fpu(ctxt);
987 static void write_sse_reg(struct x86_emulate_ctxt *ctxt, sse128_t *data,
990 ctxt->ops->get_fpu(ctxt);
992 case 0: asm("movdqa %0, %%xmm0" : : "m"(*data)); break;
993 case 1: asm("movdqa %0, %%xmm1" : : "m"(*data)); break;
994 case 2: asm("movdqa %0, %%xmm2" : : "m"(*data)); break;
995 case 3: asm("movdqa %0, %%xmm3" : : "m"(*data)); break;
996 case 4: asm("movdqa %0, %%xmm4" : : "m"(*data)); break;
997 case 5: asm("movdqa %0, %%xmm5" : : "m"(*data)); break;
998 case 6: asm("movdqa %0, %%xmm6" : : "m"(*data)); break;
999 case 7: asm("movdqa %0, %%xmm7" : : "m"(*data)); break;
1000 #ifdef CONFIG_X86_64
1001 case 8: asm("movdqa %0, %%xmm8" : : "m"(*data)); break;
1002 case 9: asm("movdqa %0, %%xmm9" : : "m"(*data)); break;
1003 case 10: asm("movdqa %0, %%xmm10" : : "m"(*data)); break;
1004 case 11: asm("movdqa %0, %%xmm11" : : "m"(*data)); break;
1005 case 12: asm("movdqa %0, %%xmm12" : : "m"(*data)); break;
1006 case 13: asm("movdqa %0, %%xmm13" : : "m"(*data)); break;
1007 case 14: asm("movdqa %0, %%xmm14" : : "m"(*data)); break;
1008 case 15: asm("movdqa %0, %%xmm15" : : "m"(*data)); break;
1012 ctxt->ops->put_fpu(ctxt);
1015 static void read_mmx_reg(struct x86_emulate_ctxt *ctxt, u64 *data, int reg)
1017 ctxt->ops->get_fpu(ctxt);
1019 case 0: asm("movq %%mm0, %0" : "=m"(*data)); break;
1020 case 1: asm("movq %%mm1, %0" : "=m"(*data)); break;
1021 case 2: asm("movq %%mm2, %0" : "=m"(*data)); break;
1022 case 3: asm("movq %%mm3, %0" : "=m"(*data)); break;
1023 case 4: asm("movq %%mm4, %0" : "=m"(*data)); break;
1024 case 5: asm("movq %%mm5, %0" : "=m"(*data)); break;
1025 case 6: asm("movq %%mm6, %0" : "=m"(*data)); break;
1026 case 7: asm("movq %%mm7, %0" : "=m"(*data)); break;
1029 ctxt->ops->put_fpu(ctxt);
1032 static void write_mmx_reg(struct x86_emulate_ctxt *ctxt, u64 *data, int reg)
1034 ctxt->ops->get_fpu(ctxt);
1036 case 0: asm("movq %0, %%mm0" : : "m"(*data)); break;
1037 case 1: asm("movq %0, %%mm1" : : "m"(*data)); break;
1038 case 2: asm("movq %0, %%mm2" : : "m"(*data)); break;
1039 case 3: asm("movq %0, %%mm3" : : "m"(*data)); break;
1040 case 4: asm("movq %0, %%mm4" : : "m"(*data)); break;
1041 case 5: asm("movq %0, %%mm5" : : "m"(*data)); break;
1042 case 6: asm("movq %0, %%mm6" : : "m"(*data)); break;
1043 case 7: asm("movq %0, %%mm7" : : "m"(*data)); break;
1046 ctxt->ops->put_fpu(ctxt);
1049 static int em_fninit(struct x86_emulate_ctxt *ctxt)
1051 if (ctxt->ops->get_cr(ctxt, 0) & (X86_CR0_TS | X86_CR0_EM))
1052 return emulate_nm(ctxt);
1054 ctxt->ops->get_fpu(ctxt);
1055 asm volatile("fninit");
1056 ctxt->ops->put_fpu(ctxt);
1057 return X86EMUL_CONTINUE;
1060 static int em_fnstcw(struct x86_emulate_ctxt *ctxt)
1064 if (ctxt->ops->get_cr(ctxt, 0) & (X86_CR0_TS | X86_CR0_EM))
1065 return emulate_nm(ctxt);
1067 ctxt->ops->get_fpu(ctxt);
1068 asm volatile("fnstcw %0": "+m"(fcw));
1069 ctxt->ops->put_fpu(ctxt);
1071 /* force 2 byte destination */
1072 ctxt->dst.bytes = 2;
1073 ctxt->dst.val = fcw;
1075 return X86EMUL_CONTINUE;
1078 static int em_fnstsw(struct x86_emulate_ctxt *ctxt)
1082 if (ctxt->ops->get_cr(ctxt, 0) & (X86_CR0_TS | X86_CR0_EM))
1083 return emulate_nm(ctxt);
1085 ctxt->ops->get_fpu(ctxt);
1086 asm volatile("fnstsw %0": "+m"(fsw));
1087 ctxt->ops->put_fpu(ctxt);
1089 /* force 2 byte destination */
1090 ctxt->dst.bytes = 2;
1091 ctxt->dst.val = fsw;
1093 return X86EMUL_CONTINUE;
1096 static void decode_register_operand(struct x86_emulate_ctxt *ctxt,
1099 unsigned reg = ctxt->modrm_reg;
1100 int highbyte_regs = ctxt->rex_prefix == 0;
1102 if (!(ctxt->d & ModRM))
1103 reg = (ctxt->b & 7) | ((ctxt->rex_prefix & 1) << 3);
1105 if (ctxt->d & Sse) {
1109 read_sse_reg(ctxt, &op->vec_val, reg);
1112 if (ctxt->d & Mmx) {
1121 if (ctxt->d & ByteOp) {
1122 op->addr.reg = decode_register(ctxt, reg, highbyte_regs);
1125 op->addr.reg = decode_register(ctxt, reg, 0);
1126 op->bytes = ctxt->op_bytes;
1128 fetch_register_operand(op);
1129 op->orig_val = op->val;
1132 static void adjust_modrm_seg(struct x86_emulate_ctxt *ctxt, int base_reg)
1134 if (base_reg == VCPU_REGS_RSP || base_reg == VCPU_REGS_RBP)
1135 ctxt->modrm_seg = VCPU_SREG_SS;
1138 static int decode_modrm(struct x86_emulate_ctxt *ctxt,
1142 int index_reg = 0, base_reg = 0, scale;
1143 int rc = X86EMUL_CONTINUE;
1146 if (ctxt->rex_prefix) {
1147 ctxt->modrm_reg = (ctxt->rex_prefix & 4) << 1; /* REX.R */
1148 index_reg = (ctxt->rex_prefix & 2) << 2; /* REX.X */
1149 ctxt->modrm_rm = base_reg = (ctxt->rex_prefix & 1) << 3; /* REG.B */
1152 ctxt->modrm_mod |= (ctxt->modrm & 0xc0) >> 6;
1153 ctxt->modrm_reg |= (ctxt->modrm & 0x38) >> 3;
1154 ctxt->modrm_rm |= (ctxt->modrm & 0x07);
1155 ctxt->modrm_seg = VCPU_SREG_DS;
1157 if (ctxt->modrm_mod == 3) {
1159 op->bytes = (ctxt->d & ByteOp) ? 1 : ctxt->op_bytes;
1160 op->addr.reg = decode_register(ctxt, ctxt->modrm_rm, ctxt->d & ByteOp);
1161 if (ctxt->d & Sse) {
1164 op->addr.xmm = ctxt->modrm_rm;
1165 read_sse_reg(ctxt, &op->vec_val, ctxt->modrm_rm);
1168 if (ctxt->d & Mmx) {
1171 op->addr.xmm = ctxt->modrm_rm & 7;
1174 fetch_register_operand(op);
1180 if (ctxt->ad_bytes == 2) {
1181 unsigned bx = reg_read(ctxt, VCPU_REGS_RBX);
1182 unsigned bp = reg_read(ctxt, VCPU_REGS_RBP);
1183 unsigned si = reg_read(ctxt, VCPU_REGS_RSI);
1184 unsigned di = reg_read(ctxt, VCPU_REGS_RDI);
1186 /* 16-bit ModR/M decode. */
1187 switch (ctxt->modrm_mod) {
1189 if (ctxt->modrm_rm == 6)
1190 modrm_ea += insn_fetch(u16, ctxt);
1193 modrm_ea += insn_fetch(s8, ctxt);
1196 modrm_ea += insn_fetch(u16, ctxt);
1199 switch (ctxt->modrm_rm) {
1201 modrm_ea += bx + si;
1204 modrm_ea += bx + di;
1207 modrm_ea += bp + si;
1210 modrm_ea += bp + di;
1219 if (ctxt->modrm_mod != 0)
1226 if (ctxt->modrm_rm == 2 || ctxt->modrm_rm == 3 ||
1227 (ctxt->modrm_rm == 6 && ctxt->modrm_mod != 0))
1228 ctxt->modrm_seg = VCPU_SREG_SS;
1229 modrm_ea = (u16)modrm_ea;
1231 /* 32/64-bit ModR/M decode. */
1232 if ((ctxt->modrm_rm & 7) == 4) {
1233 sib = insn_fetch(u8, ctxt);
1234 index_reg |= (sib >> 3) & 7;
1235 base_reg |= sib & 7;
1238 if ((base_reg & 7) == 5 && ctxt->modrm_mod == 0)
1239 modrm_ea += insn_fetch(s32, ctxt);
1241 modrm_ea += reg_read(ctxt, base_reg);
1242 adjust_modrm_seg(ctxt, base_reg);
1245 modrm_ea += reg_read(ctxt, index_reg) << scale;
1246 } else if ((ctxt->modrm_rm & 7) == 5 && ctxt->modrm_mod == 0) {
1247 if (ctxt->mode == X86EMUL_MODE_PROT64)
1248 ctxt->rip_relative = 1;
1250 base_reg = ctxt->modrm_rm;
1251 modrm_ea += reg_read(ctxt, base_reg);
1252 adjust_modrm_seg(ctxt, base_reg);
1254 switch (ctxt->modrm_mod) {
1256 if (ctxt->modrm_rm == 5)
1257 modrm_ea += insn_fetch(s32, ctxt);
1260 modrm_ea += insn_fetch(s8, ctxt);
1263 modrm_ea += insn_fetch(s32, ctxt);
1267 op->addr.mem.ea = modrm_ea;
1272 static int decode_abs(struct x86_emulate_ctxt *ctxt,
1275 int rc = X86EMUL_CONTINUE;
1278 switch (ctxt->ad_bytes) {
1280 op->addr.mem.ea = insn_fetch(u16, ctxt);
1283 op->addr.mem.ea = insn_fetch(u32, ctxt);
1286 op->addr.mem.ea = insn_fetch(u64, ctxt);
1293 static void fetch_bit_operand(struct x86_emulate_ctxt *ctxt)
1297 if (ctxt->dst.type == OP_MEM && ctxt->src.type == OP_REG) {
1298 mask = ~(ctxt->dst.bytes * 8 - 1);
1300 if (ctxt->src.bytes == 2)
1301 sv = (s16)ctxt->src.val & (s16)mask;
1302 else if (ctxt->src.bytes == 4)
1303 sv = (s32)ctxt->src.val & (s32)mask;
1305 ctxt->dst.addr.mem.ea += (sv >> 3);
1308 /* only subword offset */
1309 ctxt->src.val &= (ctxt->dst.bytes << 3) - 1;
1312 static int read_emulated(struct x86_emulate_ctxt *ctxt,
1313 unsigned long addr, void *dest, unsigned size)
1316 struct read_cache *mc = &ctxt->mem_read;
1318 if (mc->pos < mc->end)
1321 WARN_ON((mc->end + size) >= sizeof(mc->data));
1323 rc = ctxt->ops->read_emulated(ctxt, addr, mc->data + mc->end, size,
1325 if (rc != X86EMUL_CONTINUE)
1331 memcpy(dest, mc->data + mc->pos, size);
1333 return X86EMUL_CONTINUE;
1336 static int segmented_read(struct x86_emulate_ctxt *ctxt,
1337 struct segmented_address addr,
1344 rc = linearize(ctxt, addr, size, false, &linear);
1345 if (rc != X86EMUL_CONTINUE)
1347 return read_emulated(ctxt, linear, data, size);
1350 static int segmented_write(struct x86_emulate_ctxt *ctxt,
1351 struct segmented_address addr,
1358 rc = linearize(ctxt, addr, size, true, &linear);
1359 if (rc != X86EMUL_CONTINUE)
1361 return ctxt->ops->write_emulated(ctxt, linear, data, size,
1365 static int segmented_cmpxchg(struct x86_emulate_ctxt *ctxt,
1366 struct segmented_address addr,
1367 const void *orig_data, const void *data,
1373 rc = linearize(ctxt, addr, size, true, &linear);
1374 if (rc != X86EMUL_CONTINUE)
1376 return ctxt->ops->cmpxchg_emulated(ctxt, linear, orig_data, data,
1377 size, &ctxt->exception);
1380 static int pio_in_emulated(struct x86_emulate_ctxt *ctxt,
1381 unsigned int size, unsigned short port,
1384 struct read_cache *rc = &ctxt->io_read;
1386 if (rc->pos == rc->end) { /* refill pio read ahead */
1387 unsigned int in_page, n;
1388 unsigned int count = ctxt->rep_prefix ?
1389 address_mask(ctxt, reg_read(ctxt, VCPU_REGS_RCX)) : 1;
1390 in_page = (ctxt->eflags & EFLG_DF) ?
1391 offset_in_page(reg_read(ctxt, VCPU_REGS_RDI)) :
1392 PAGE_SIZE - offset_in_page(reg_read(ctxt, VCPU_REGS_RDI));
1393 n = min(min(in_page, (unsigned int)sizeof(rc->data)) / size,
1397 rc->pos = rc->end = 0;
1398 if (!ctxt->ops->pio_in_emulated(ctxt, size, port, rc->data, n))
1403 if (ctxt->rep_prefix && !(ctxt->eflags & EFLG_DF)) {
1404 ctxt->dst.data = rc->data + rc->pos;
1405 ctxt->dst.type = OP_MEM_STR;
1406 ctxt->dst.count = (rc->end - rc->pos) / size;
1409 memcpy(dest, rc->data + rc->pos, size);
1415 static int read_interrupt_descriptor(struct x86_emulate_ctxt *ctxt,
1416 u16 index, struct desc_struct *desc)
1421 ctxt->ops->get_idt(ctxt, &dt);
1423 if (dt.size < index * 8 + 7)
1424 return emulate_gp(ctxt, index << 3 | 0x2);
1426 addr = dt.address + index * 8;
1427 return ctxt->ops->read_std(ctxt, addr, desc, sizeof *desc,
1431 static void get_descriptor_table_ptr(struct x86_emulate_ctxt *ctxt,
1432 u16 selector, struct desc_ptr *dt)
1434 const struct x86_emulate_ops *ops = ctxt->ops;
1436 if (selector & 1 << 2) {
1437 struct desc_struct desc;
1440 memset (dt, 0, sizeof *dt);
1441 if (!ops->get_segment(ctxt, &sel, &desc, NULL, VCPU_SREG_LDTR))
1444 dt->size = desc_limit_scaled(&desc); /* what if limit > 65535? */
1445 dt->address = get_desc_base(&desc);
1447 ops->get_gdt(ctxt, dt);
1450 /* allowed just for 8 bytes segments */
1451 static int read_segment_descriptor(struct x86_emulate_ctxt *ctxt,
1452 u16 selector, struct desc_struct *desc,
1456 u16 index = selector >> 3;
1459 get_descriptor_table_ptr(ctxt, selector, &dt);
1461 if (dt.size < index * 8 + 7)
1462 return emulate_gp(ctxt, selector & 0xfffc);
1464 *desc_addr_p = addr = dt.address + index * 8;
1465 return ctxt->ops->read_std(ctxt, addr, desc, sizeof *desc,
1469 /* allowed just for 8 bytes segments */
1470 static int write_segment_descriptor(struct x86_emulate_ctxt *ctxt,
1471 u16 selector, struct desc_struct *desc)
1474 u16 index = selector >> 3;
1477 get_descriptor_table_ptr(ctxt, selector, &dt);
1479 if (dt.size < index * 8 + 7)
1480 return emulate_gp(ctxt, selector & 0xfffc);
1482 addr = dt.address + index * 8;
1483 return ctxt->ops->write_std(ctxt, addr, desc, sizeof *desc,
1487 /* Does not support long mode */
1488 static int load_segment_descriptor(struct x86_emulate_ctxt *ctxt,
1489 u16 selector, int seg)
1491 struct desc_struct seg_desc, old_desc;
1493 unsigned err_vec = GP_VECTOR;
1495 bool null_selector = !(selector & ~0x3); /* 0000-0003 are null */
1500 memset(&seg_desc, 0, sizeof seg_desc);
1502 if ((seg <= VCPU_SREG_GS && ctxt->mode == X86EMUL_MODE_VM86)
1503 || ctxt->mode == X86EMUL_MODE_REAL) {
1504 /* set real mode segment descriptor */
1505 ctxt->ops->get_segment(ctxt, &dummy, &seg_desc, NULL, seg);
1506 set_desc_base(&seg_desc, selector << 4);
1511 cpl = ctxt->ops->cpl(ctxt);
1513 /* NULL selector is not valid for TR, CS and SS (except for long mode) */
1514 if ((seg == VCPU_SREG_CS
1515 || (seg == VCPU_SREG_SS
1516 && (ctxt->mode != X86EMUL_MODE_PROT64 || rpl != cpl))
1517 || seg == VCPU_SREG_TR)
1521 /* TR should be in GDT only */
1522 if (seg == VCPU_SREG_TR && (selector & (1 << 2)))
1525 if (null_selector) /* for NULL selector skip all following checks */
1528 ret = read_segment_descriptor(ctxt, selector, &seg_desc, &desc_addr);
1529 if (ret != X86EMUL_CONTINUE)
1532 err_code = selector & 0xfffc;
1533 err_vec = GP_VECTOR;
1535 /* can't load system descriptor into segment selector */
1536 if (seg <= VCPU_SREG_GS && !seg_desc.s)
1540 err_vec = (seg == VCPU_SREG_SS) ? SS_VECTOR : NP_VECTOR;
1549 * segment is not a writable data segment or segment
1550 * selector's RPL != CPL or segment selector's RPL != CPL
1552 if (rpl != cpl || (seg_desc.type & 0xa) != 0x2 || dpl != cpl)
1556 if (!(seg_desc.type & 8))
1559 if (seg_desc.type & 4) {
1565 if (rpl > cpl || dpl != cpl)
1568 /* CS(RPL) <- CPL */
1569 selector = (selector & 0xfffc) | cpl;
1572 if (seg_desc.s || (seg_desc.type != 1 && seg_desc.type != 9))
1574 old_desc = seg_desc;
1575 seg_desc.type |= 2; /* busy */
1576 ret = ctxt->ops->cmpxchg_emulated(ctxt, desc_addr, &old_desc, &seg_desc,
1577 sizeof(seg_desc), &ctxt->exception);
1578 if (ret != X86EMUL_CONTINUE)
1581 case VCPU_SREG_LDTR:
1582 if (seg_desc.s || seg_desc.type != 2)
1585 default: /* DS, ES, FS, or GS */
1587 * segment is not a data or readable code segment or
1588 * ((segment is a data or nonconforming code segment)
1589 * and (both RPL and CPL > DPL))
1591 if ((seg_desc.type & 0xa) == 0x8 ||
1592 (((seg_desc.type & 0xc) != 0xc) &&
1593 (rpl > dpl && cpl > dpl)))
1599 /* mark segment as accessed */
1601 ret = write_segment_descriptor(ctxt, selector, &seg_desc);
1602 if (ret != X86EMUL_CONTINUE)
1606 ctxt->ops->set_segment(ctxt, selector, &seg_desc, 0, seg);
1607 return X86EMUL_CONTINUE;
1609 emulate_exception(ctxt, err_vec, err_code, true);
1610 return X86EMUL_PROPAGATE_FAULT;
1613 static void write_register_operand(struct operand *op)
1615 /* The 4-byte case *is* correct: in 64-bit mode we zero-extend. */
1616 switch (op->bytes) {
1618 *(u8 *)op->addr.reg = (u8)op->val;
1621 *(u16 *)op->addr.reg = (u16)op->val;
1624 *op->addr.reg = (u32)op->val;
1625 break; /* 64b: zero-extend */
1627 *op->addr.reg = op->val;
1632 static int writeback(struct x86_emulate_ctxt *ctxt)
1636 switch (ctxt->dst.type) {
1638 write_register_operand(&ctxt->dst);
1641 if (ctxt->lock_prefix)
1642 rc = segmented_cmpxchg(ctxt,
1644 &ctxt->dst.orig_val,
1648 rc = segmented_write(ctxt,
1652 if (rc != X86EMUL_CONTINUE)
1656 rc = segmented_write(ctxt,
1659 ctxt->dst.bytes * ctxt->dst.count);
1660 if (rc != X86EMUL_CONTINUE)
1664 write_sse_reg(ctxt, &ctxt->dst.vec_val, ctxt->dst.addr.xmm);
1667 write_mmx_reg(ctxt, &ctxt->dst.mm_val, ctxt->dst.addr.mm);
1675 return X86EMUL_CONTINUE;
1678 static int push(struct x86_emulate_ctxt *ctxt, void *data, int bytes)
1680 struct segmented_address addr;
1682 rsp_increment(ctxt, -bytes);
1683 addr.ea = reg_read(ctxt, VCPU_REGS_RSP) & stack_mask(ctxt);
1684 addr.seg = VCPU_SREG_SS;
1686 return segmented_write(ctxt, addr, data, bytes);
1689 static int em_push(struct x86_emulate_ctxt *ctxt)
1691 /* Disable writeback. */
1692 ctxt->dst.type = OP_NONE;
1693 return push(ctxt, &ctxt->src.val, ctxt->op_bytes);
1696 static int emulate_pop(struct x86_emulate_ctxt *ctxt,
1697 void *dest, int len)
1700 struct segmented_address addr;
1702 addr.ea = reg_read(ctxt, VCPU_REGS_RSP) & stack_mask(ctxt);
1703 addr.seg = VCPU_SREG_SS;
1704 rc = segmented_read(ctxt, addr, dest, len);
1705 if (rc != X86EMUL_CONTINUE)
1708 rsp_increment(ctxt, len);
1712 static int em_pop(struct x86_emulate_ctxt *ctxt)
1714 return emulate_pop(ctxt, &ctxt->dst.val, ctxt->op_bytes);
1717 static int emulate_popf(struct x86_emulate_ctxt *ctxt,
1718 void *dest, int len)
1721 unsigned long val, change_mask;
1722 int iopl = (ctxt->eflags & X86_EFLAGS_IOPL) >> IOPL_SHIFT;
1723 int cpl = ctxt->ops->cpl(ctxt);
1725 rc = emulate_pop(ctxt, &val, len);
1726 if (rc != X86EMUL_CONTINUE)
1729 change_mask = EFLG_CF | EFLG_PF | EFLG_AF | EFLG_ZF | EFLG_SF | EFLG_OF
1730 | EFLG_TF | EFLG_DF | EFLG_NT | EFLG_RF | EFLG_AC | EFLG_ID;
1732 switch(ctxt->mode) {
1733 case X86EMUL_MODE_PROT64:
1734 case X86EMUL_MODE_PROT32:
1735 case X86EMUL_MODE_PROT16:
1737 change_mask |= EFLG_IOPL;
1739 change_mask |= EFLG_IF;
1741 case X86EMUL_MODE_VM86:
1743 return emulate_gp(ctxt, 0);
1744 change_mask |= EFLG_IF;
1746 default: /* real mode */
1747 change_mask |= (EFLG_IOPL | EFLG_IF);
1751 *(unsigned long *)dest =
1752 (ctxt->eflags & ~change_mask) | (val & change_mask);
1757 static int em_popf(struct x86_emulate_ctxt *ctxt)
1759 ctxt->dst.type = OP_REG;
1760 ctxt->dst.addr.reg = &ctxt->eflags;
1761 ctxt->dst.bytes = ctxt->op_bytes;
1762 return emulate_popf(ctxt, &ctxt->dst.val, ctxt->op_bytes);
1765 static int em_enter(struct x86_emulate_ctxt *ctxt)
1768 unsigned frame_size = ctxt->src.val;
1769 unsigned nesting_level = ctxt->src2.val & 31;
1773 return X86EMUL_UNHANDLEABLE;
1775 rbp = reg_read(ctxt, VCPU_REGS_RBP);
1776 rc = push(ctxt, &rbp, stack_size(ctxt));
1777 if (rc != X86EMUL_CONTINUE)
1779 assign_masked(reg_rmw(ctxt, VCPU_REGS_RBP), reg_read(ctxt, VCPU_REGS_RSP),
1781 assign_masked(reg_rmw(ctxt, VCPU_REGS_RSP),
1782 reg_read(ctxt, VCPU_REGS_RSP) - frame_size,
1784 return X86EMUL_CONTINUE;
1787 static int em_leave(struct x86_emulate_ctxt *ctxt)
1789 assign_masked(reg_rmw(ctxt, VCPU_REGS_RSP), reg_read(ctxt, VCPU_REGS_RBP),
1791 return emulate_pop(ctxt, reg_rmw(ctxt, VCPU_REGS_RBP), ctxt->op_bytes);
1794 static int em_push_sreg(struct x86_emulate_ctxt *ctxt)
1796 int seg = ctxt->src2.val;
1798 ctxt->src.val = get_segment_selector(ctxt, seg);
1800 return em_push(ctxt);
1803 static int em_pop_sreg(struct x86_emulate_ctxt *ctxt)
1805 int seg = ctxt->src2.val;
1806 unsigned long selector;
1809 rc = emulate_pop(ctxt, &selector, ctxt->op_bytes);
1810 if (rc != X86EMUL_CONTINUE)
1813 rc = load_segment_descriptor(ctxt, (u16)selector, seg);
1817 static int em_pusha(struct x86_emulate_ctxt *ctxt)
1819 unsigned long old_esp = reg_read(ctxt, VCPU_REGS_RSP);
1820 int rc = X86EMUL_CONTINUE;
1821 int reg = VCPU_REGS_RAX;
1823 while (reg <= VCPU_REGS_RDI) {
1824 (reg == VCPU_REGS_RSP) ?
1825 (ctxt->src.val = old_esp) : (ctxt->src.val = reg_read(ctxt, reg));
1828 if (rc != X86EMUL_CONTINUE)
1837 static int em_pushf(struct x86_emulate_ctxt *ctxt)
1839 ctxt->src.val = (unsigned long)ctxt->eflags;
1840 return em_push(ctxt);
1843 static int em_popa(struct x86_emulate_ctxt *ctxt)
1845 int rc = X86EMUL_CONTINUE;
1846 int reg = VCPU_REGS_RDI;
1848 while (reg >= VCPU_REGS_RAX) {
1849 if (reg == VCPU_REGS_RSP) {
1850 rsp_increment(ctxt, ctxt->op_bytes);
1854 rc = emulate_pop(ctxt, reg_rmw(ctxt, reg), ctxt->op_bytes);
1855 if (rc != X86EMUL_CONTINUE)
1862 static int __emulate_int_real(struct x86_emulate_ctxt *ctxt, int irq)
1864 const struct x86_emulate_ops *ops = ctxt->ops;
1871 /* TODO: Add limit checks */
1872 ctxt->src.val = ctxt->eflags;
1874 if (rc != X86EMUL_CONTINUE)
1877 ctxt->eflags &= ~(EFLG_IF | EFLG_TF | EFLG_AC);
1879 ctxt->src.val = get_segment_selector(ctxt, VCPU_SREG_CS);
1881 if (rc != X86EMUL_CONTINUE)
1884 ctxt->src.val = ctxt->_eip;
1886 if (rc != X86EMUL_CONTINUE)
1889 ops->get_idt(ctxt, &dt);
1891 eip_addr = dt.address + (irq << 2);
1892 cs_addr = dt.address + (irq << 2) + 2;
1894 rc = ops->read_std(ctxt, cs_addr, &cs, 2, &ctxt->exception);
1895 if (rc != X86EMUL_CONTINUE)
1898 rc = ops->read_std(ctxt, eip_addr, &eip, 2, &ctxt->exception);
1899 if (rc != X86EMUL_CONTINUE)
1902 rc = load_segment_descriptor(ctxt, cs, VCPU_SREG_CS);
1903 if (rc != X86EMUL_CONTINUE)
1911 int emulate_int_real(struct x86_emulate_ctxt *ctxt, int irq)
1915 invalidate_registers(ctxt);
1916 rc = __emulate_int_real(ctxt, irq);
1917 if (rc == X86EMUL_CONTINUE)
1918 writeback_registers(ctxt);
1922 static int emulate_int(struct x86_emulate_ctxt *ctxt, int irq)
1924 switch(ctxt->mode) {
1925 case X86EMUL_MODE_REAL:
1926 return __emulate_int_real(ctxt, irq);
1927 case X86EMUL_MODE_VM86:
1928 case X86EMUL_MODE_PROT16:
1929 case X86EMUL_MODE_PROT32:
1930 case X86EMUL_MODE_PROT64:
1932 /* Protected mode interrupts unimplemented yet */
1933 return X86EMUL_UNHANDLEABLE;
1937 static int emulate_iret_real(struct x86_emulate_ctxt *ctxt)
1939 int rc = X86EMUL_CONTINUE;
1940 unsigned long temp_eip = 0;
1941 unsigned long temp_eflags = 0;
1942 unsigned long cs = 0;
1943 unsigned long mask = EFLG_CF | EFLG_PF | EFLG_AF | EFLG_ZF | EFLG_SF | EFLG_TF |
1944 EFLG_IF | EFLG_DF | EFLG_OF | EFLG_IOPL | EFLG_NT | EFLG_RF |
1945 EFLG_AC | EFLG_ID | (1 << 1); /* Last one is the reserved bit */
1946 unsigned long vm86_mask = EFLG_VM | EFLG_VIF | EFLG_VIP;
1948 /* TODO: Add stack limit check */
1950 rc = emulate_pop(ctxt, &temp_eip, ctxt->op_bytes);
1952 if (rc != X86EMUL_CONTINUE)
1955 if (temp_eip & ~0xffff)
1956 return emulate_gp(ctxt, 0);
1958 rc = emulate_pop(ctxt, &cs, ctxt->op_bytes);
1960 if (rc != X86EMUL_CONTINUE)
1963 rc = emulate_pop(ctxt, &temp_eflags, ctxt->op_bytes);
1965 if (rc != X86EMUL_CONTINUE)
1968 rc = load_segment_descriptor(ctxt, (u16)cs, VCPU_SREG_CS);
1970 if (rc != X86EMUL_CONTINUE)
1973 ctxt->_eip = temp_eip;
1976 if (ctxt->op_bytes == 4)
1977 ctxt->eflags = ((temp_eflags & mask) | (ctxt->eflags & vm86_mask));
1978 else if (ctxt->op_bytes == 2) {
1979 ctxt->eflags &= ~0xffff;
1980 ctxt->eflags |= temp_eflags;
1983 ctxt->eflags &= ~EFLG_RESERVED_ZEROS_MASK; /* Clear reserved zeros */
1984 ctxt->eflags |= EFLG_RESERVED_ONE_MASK;
1989 static int em_iret(struct x86_emulate_ctxt *ctxt)
1991 switch(ctxt->mode) {
1992 case X86EMUL_MODE_REAL:
1993 return emulate_iret_real(ctxt);
1994 case X86EMUL_MODE_VM86:
1995 case X86EMUL_MODE_PROT16:
1996 case X86EMUL_MODE_PROT32:
1997 case X86EMUL_MODE_PROT64:
1999 /* iret from protected mode unimplemented yet */
2000 return X86EMUL_UNHANDLEABLE;
2004 static int em_jmp_far(struct x86_emulate_ctxt *ctxt)
2009 memcpy(&sel, ctxt->src.valptr + ctxt->op_bytes, 2);
2011 rc = load_segment_descriptor(ctxt, sel, VCPU_SREG_CS);
2012 if (rc != X86EMUL_CONTINUE)
2016 memcpy(&ctxt->_eip, ctxt->src.valptr, ctxt->op_bytes);
2017 return X86EMUL_CONTINUE;
2020 static int em_grp2(struct x86_emulate_ctxt *ctxt)
2022 switch (ctxt->modrm_reg) {
2024 emulate_2op_SrcB(ctxt, "rol");
2027 emulate_2op_SrcB(ctxt, "ror");
2030 emulate_2op_SrcB(ctxt, "rcl");
2033 emulate_2op_SrcB(ctxt, "rcr");
2035 case 4: /* sal/shl */
2036 case 6: /* sal/shl */
2037 emulate_2op_SrcB(ctxt, "sal");
2040 emulate_2op_SrcB(ctxt, "shr");
2043 emulate_2op_SrcB(ctxt, "sar");
2046 return X86EMUL_CONTINUE;
2049 static int em_not(struct x86_emulate_ctxt *ctxt)
2051 ctxt->dst.val = ~ctxt->dst.val;
2052 return X86EMUL_CONTINUE;
2055 static int em_neg(struct x86_emulate_ctxt *ctxt)
2057 emulate_1op(ctxt, "neg");
2058 return X86EMUL_CONTINUE;
2061 static int em_mul_ex(struct x86_emulate_ctxt *ctxt)
2065 emulate_1op_rax_rdx(ctxt, "mul", ex);
2066 return X86EMUL_CONTINUE;
2069 static int em_imul_ex(struct x86_emulate_ctxt *ctxt)
2073 emulate_1op_rax_rdx(ctxt, "imul", ex);
2074 return X86EMUL_CONTINUE;
2077 static int em_div_ex(struct x86_emulate_ctxt *ctxt)
2081 emulate_1op_rax_rdx(ctxt, "div", de);
2083 return emulate_de(ctxt);
2084 return X86EMUL_CONTINUE;
2087 static int em_idiv_ex(struct x86_emulate_ctxt *ctxt)
2091 emulate_1op_rax_rdx(ctxt, "idiv", de);
2093 return emulate_de(ctxt);
2094 return X86EMUL_CONTINUE;
2097 static int em_grp45(struct x86_emulate_ctxt *ctxt)
2099 int rc = X86EMUL_CONTINUE;
2101 switch (ctxt->modrm_reg) {
2103 emulate_1op(ctxt, "inc");
2106 emulate_1op(ctxt, "dec");
2108 case 2: /* call near abs */ {
2110 old_eip = ctxt->_eip;
2111 ctxt->_eip = ctxt->src.val;
2112 ctxt->src.val = old_eip;
2116 case 4: /* jmp abs */
2117 ctxt->_eip = ctxt->src.val;
2119 case 5: /* jmp far */
2120 rc = em_jmp_far(ctxt);
2129 static int em_cmpxchg8b(struct x86_emulate_ctxt *ctxt)
2131 u64 old = ctxt->dst.orig_val64;
2133 if (((u32) (old >> 0) != (u32) reg_read(ctxt, VCPU_REGS_RAX)) ||
2134 ((u32) (old >> 32) != (u32) reg_read(ctxt, VCPU_REGS_RDX))) {
2135 *reg_write(ctxt, VCPU_REGS_RAX) = (u32) (old >> 0);
2136 *reg_write(ctxt, VCPU_REGS_RDX) = (u32) (old >> 32);
2137 ctxt->eflags &= ~EFLG_ZF;
2139 ctxt->dst.val64 = ((u64)reg_read(ctxt, VCPU_REGS_RCX) << 32) |
2140 (u32) reg_read(ctxt, VCPU_REGS_RBX);
2142 ctxt->eflags |= EFLG_ZF;
2144 return X86EMUL_CONTINUE;
2147 static int em_ret(struct x86_emulate_ctxt *ctxt)
2149 ctxt->dst.type = OP_REG;
2150 ctxt->dst.addr.reg = &ctxt->_eip;
2151 ctxt->dst.bytes = ctxt->op_bytes;
2152 return em_pop(ctxt);
2155 static int em_ret_far(struct x86_emulate_ctxt *ctxt)
2160 rc = emulate_pop(ctxt, &ctxt->_eip, ctxt->op_bytes);
2161 if (rc != X86EMUL_CONTINUE)
2163 if (ctxt->op_bytes == 4)
2164 ctxt->_eip = (u32)ctxt->_eip;
2165 rc = emulate_pop(ctxt, &cs, ctxt->op_bytes);
2166 if (rc != X86EMUL_CONTINUE)
2168 rc = load_segment_descriptor(ctxt, (u16)cs, VCPU_SREG_CS);
2172 static int em_cmpxchg(struct x86_emulate_ctxt *ctxt)
2174 /* Save real source value, then compare EAX against destination. */
2175 ctxt->src.orig_val = ctxt->src.val;
2176 ctxt->src.val = reg_read(ctxt, VCPU_REGS_RAX);
2177 emulate_2op_SrcV(ctxt, "cmp");
2179 if (ctxt->eflags & EFLG_ZF) {
2180 /* Success: write back to memory. */
2181 ctxt->dst.val = ctxt->src.orig_val;
2183 /* Failure: write the value we saw to EAX. */
2184 ctxt->dst.type = OP_REG;
2185 ctxt->dst.addr.reg = reg_rmw(ctxt, VCPU_REGS_RAX);
2187 return X86EMUL_CONTINUE;
2190 static int em_lseg(struct x86_emulate_ctxt *ctxt)
2192 int seg = ctxt->src2.val;
2196 memcpy(&sel, ctxt->src.valptr + ctxt->op_bytes, 2);
2198 rc = load_segment_descriptor(ctxt, sel, seg);
2199 if (rc != X86EMUL_CONTINUE)
2202 ctxt->dst.val = ctxt->src.val;
2207 setup_syscalls_segments(struct x86_emulate_ctxt *ctxt,
2208 struct desc_struct *cs, struct desc_struct *ss)
2210 cs->l = 0; /* will be adjusted later */
2211 set_desc_base(cs, 0); /* flat segment */
2212 cs->g = 1; /* 4kb granularity */
2213 set_desc_limit(cs, 0xfffff); /* 4GB limit */
2214 cs->type = 0x0b; /* Read, Execute, Accessed */
2216 cs->dpl = 0; /* will be adjusted later */
2221 set_desc_base(ss, 0); /* flat segment */
2222 set_desc_limit(ss, 0xfffff); /* 4GB limit */
2223 ss->g = 1; /* 4kb granularity */
2225 ss->type = 0x03; /* Read/Write, Accessed */
2226 ss->d = 1; /* 32bit stack segment */
2233 static bool vendor_intel(struct x86_emulate_ctxt *ctxt)
2235 u32 eax, ebx, ecx, edx;
2238 ctxt->ops->get_cpuid(ctxt, &eax, &ebx, &ecx, &edx);
2239 return ebx == X86EMUL_CPUID_VENDOR_GenuineIntel_ebx
2240 && ecx == X86EMUL_CPUID_VENDOR_GenuineIntel_ecx
2241 && edx == X86EMUL_CPUID_VENDOR_GenuineIntel_edx;
2244 static bool em_syscall_is_enabled(struct x86_emulate_ctxt *ctxt)
2246 const struct x86_emulate_ops *ops = ctxt->ops;
2247 u32 eax, ebx, ecx, edx;
2250 * syscall should always be enabled in longmode - so only become
2251 * vendor specific (cpuid) if other modes are active...
2253 if (ctxt->mode == X86EMUL_MODE_PROT64)
2258 ops->get_cpuid(ctxt, &eax, &ebx, &ecx, &edx);
2260 * Intel ("GenuineIntel")
2261 * remark: Intel CPUs only support "syscall" in 64bit
2262 * longmode. Also an 64bit guest with a
2263 * 32bit compat-app running will #UD !! While this
2264 * behaviour can be fixed (by emulating) into AMD
2265 * response - CPUs of AMD can't behave like Intel.
2267 if (ebx == X86EMUL_CPUID_VENDOR_GenuineIntel_ebx &&
2268 ecx == X86EMUL_CPUID_VENDOR_GenuineIntel_ecx &&
2269 edx == X86EMUL_CPUID_VENDOR_GenuineIntel_edx)
2272 /* AMD ("AuthenticAMD") */
2273 if (ebx == X86EMUL_CPUID_VENDOR_AuthenticAMD_ebx &&
2274 ecx == X86EMUL_CPUID_VENDOR_AuthenticAMD_ecx &&
2275 edx == X86EMUL_CPUID_VENDOR_AuthenticAMD_edx)
2278 /* AMD ("AMDisbetter!") */
2279 if (ebx == X86EMUL_CPUID_VENDOR_AMDisbetterI_ebx &&
2280 ecx == X86EMUL_CPUID_VENDOR_AMDisbetterI_ecx &&
2281 edx == X86EMUL_CPUID_VENDOR_AMDisbetterI_edx)
2284 /* default: (not Intel, not AMD), apply Intel's stricter rules... */
2288 static int em_syscall(struct x86_emulate_ctxt *ctxt)
2290 const struct x86_emulate_ops *ops = ctxt->ops;
2291 struct desc_struct cs, ss;
2296 /* syscall is not available in real mode */
2297 if (ctxt->mode == X86EMUL_MODE_REAL ||
2298 ctxt->mode == X86EMUL_MODE_VM86)
2299 return emulate_ud(ctxt);
2301 if (!(em_syscall_is_enabled(ctxt)))
2302 return emulate_ud(ctxt);
2304 ops->get_msr(ctxt, MSR_EFER, &efer);
2305 setup_syscalls_segments(ctxt, &cs, &ss);
2307 if (!(efer & EFER_SCE))
2308 return emulate_ud(ctxt);
2310 ops->get_msr(ctxt, MSR_STAR, &msr_data);
2312 cs_sel = (u16)(msr_data & 0xfffc);
2313 ss_sel = (u16)(msr_data + 8);
2315 if (efer & EFER_LMA) {
2319 ops->set_segment(ctxt, cs_sel, &cs, 0, VCPU_SREG_CS);
2320 ops->set_segment(ctxt, ss_sel, &ss, 0, VCPU_SREG_SS);
2322 *reg_write(ctxt, VCPU_REGS_RCX) = ctxt->_eip;
2323 if (efer & EFER_LMA) {
2324 #ifdef CONFIG_X86_64
2325 *reg_write(ctxt, VCPU_REGS_R11) = ctxt->eflags & ~EFLG_RF;
2328 ctxt->mode == X86EMUL_MODE_PROT64 ?
2329 MSR_LSTAR : MSR_CSTAR, &msr_data);
2330 ctxt->_eip = msr_data;
2332 ops->get_msr(ctxt, MSR_SYSCALL_MASK, &msr_data);
2333 ctxt->eflags &= ~(msr_data | EFLG_RF);
2337 ops->get_msr(ctxt, MSR_STAR, &msr_data);
2338 ctxt->_eip = (u32)msr_data;
2340 ctxt->eflags &= ~(EFLG_VM | EFLG_IF | EFLG_RF);
2343 return X86EMUL_CONTINUE;
2346 static int em_sysenter(struct x86_emulate_ctxt *ctxt)
2348 const struct x86_emulate_ops *ops = ctxt->ops;
2349 struct desc_struct cs, ss;
2354 ops->get_msr(ctxt, MSR_EFER, &efer);
2355 /* inject #GP if in real mode */
2356 if (ctxt->mode == X86EMUL_MODE_REAL)
2357 return emulate_gp(ctxt, 0);
2360 * Not recognized on AMD in compat mode (but is recognized in legacy
2363 if ((ctxt->mode == X86EMUL_MODE_PROT32) && (efer & EFER_LMA)
2364 && !vendor_intel(ctxt))
2365 return emulate_ud(ctxt);
2367 /* XXX sysenter/sysexit have not been tested in 64bit mode.
2368 * Therefore, we inject an #UD.
2370 if (ctxt->mode == X86EMUL_MODE_PROT64)
2371 return emulate_ud(ctxt);
2373 setup_syscalls_segments(ctxt, &cs, &ss);
2375 ops->get_msr(ctxt, MSR_IA32_SYSENTER_CS, &msr_data);
2376 switch (ctxt->mode) {
2377 case X86EMUL_MODE_PROT32:
2378 if ((msr_data & 0xfffc) == 0x0)
2379 return emulate_gp(ctxt, 0);
2381 case X86EMUL_MODE_PROT64:
2382 if (msr_data == 0x0)
2383 return emulate_gp(ctxt, 0);
2389 ctxt->eflags &= ~(EFLG_VM | EFLG_IF | EFLG_RF);
2390 cs_sel = (u16)msr_data;
2391 cs_sel &= ~SELECTOR_RPL_MASK;
2392 ss_sel = cs_sel + 8;
2393 ss_sel &= ~SELECTOR_RPL_MASK;
2394 if (ctxt->mode == X86EMUL_MODE_PROT64 || (efer & EFER_LMA)) {
2399 ops->set_segment(ctxt, cs_sel, &cs, 0, VCPU_SREG_CS);
2400 ops->set_segment(ctxt, ss_sel, &ss, 0, VCPU_SREG_SS);
2402 ops->get_msr(ctxt, MSR_IA32_SYSENTER_EIP, &msr_data);
2403 ctxt->_eip = msr_data;
2405 ops->get_msr(ctxt, MSR_IA32_SYSENTER_ESP, &msr_data);
2406 *reg_write(ctxt, VCPU_REGS_RSP) = msr_data;
2408 return X86EMUL_CONTINUE;
2411 static int em_sysexit(struct x86_emulate_ctxt *ctxt)
2413 const struct x86_emulate_ops *ops = ctxt->ops;
2414 struct desc_struct cs, ss;
2417 u16 cs_sel = 0, ss_sel = 0;
2419 /* inject #GP if in real mode or Virtual 8086 mode */
2420 if (ctxt->mode == X86EMUL_MODE_REAL ||
2421 ctxt->mode == X86EMUL_MODE_VM86)
2422 return emulate_gp(ctxt, 0);
2424 setup_syscalls_segments(ctxt, &cs, &ss);
2426 if ((ctxt->rex_prefix & 0x8) != 0x0)
2427 usermode = X86EMUL_MODE_PROT64;
2429 usermode = X86EMUL_MODE_PROT32;
2433 ops->get_msr(ctxt, MSR_IA32_SYSENTER_CS, &msr_data);
2435 case X86EMUL_MODE_PROT32:
2436 cs_sel = (u16)(msr_data + 16);
2437 if ((msr_data & 0xfffc) == 0x0)
2438 return emulate_gp(ctxt, 0);
2439 ss_sel = (u16)(msr_data + 24);
2441 case X86EMUL_MODE_PROT64:
2442 cs_sel = (u16)(msr_data + 32);
2443 if (msr_data == 0x0)
2444 return emulate_gp(ctxt, 0);
2445 ss_sel = cs_sel + 8;
2450 cs_sel |= SELECTOR_RPL_MASK;
2451 ss_sel |= SELECTOR_RPL_MASK;
2453 ops->set_segment(ctxt, cs_sel, &cs, 0, VCPU_SREG_CS);
2454 ops->set_segment(ctxt, ss_sel, &ss, 0, VCPU_SREG_SS);
2456 ctxt->_eip = reg_read(ctxt, VCPU_REGS_RDX);
2457 *reg_write(ctxt, VCPU_REGS_RSP) = reg_read(ctxt, VCPU_REGS_RCX);
2459 return X86EMUL_CONTINUE;
2462 static bool emulator_bad_iopl(struct x86_emulate_ctxt *ctxt)
2465 if (ctxt->mode == X86EMUL_MODE_REAL)
2467 if (ctxt->mode == X86EMUL_MODE_VM86)
2469 iopl = (ctxt->eflags & X86_EFLAGS_IOPL) >> IOPL_SHIFT;
2470 return ctxt->ops->cpl(ctxt) > iopl;
2473 static bool emulator_io_port_access_allowed(struct x86_emulate_ctxt *ctxt,
2476 const struct x86_emulate_ops *ops = ctxt->ops;
2477 struct desc_struct tr_seg;
2480 u16 tr, io_bitmap_ptr, perm, bit_idx = port & 0x7;
2481 unsigned mask = (1 << len) - 1;
2484 ops->get_segment(ctxt, &tr, &tr_seg, &base3, VCPU_SREG_TR);
2487 if (desc_limit_scaled(&tr_seg) < 103)
2489 base = get_desc_base(&tr_seg);
2490 #ifdef CONFIG_X86_64
2491 base |= ((u64)base3) << 32;
2493 r = ops->read_std(ctxt, base + 102, &io_bitmap_ptr, 2, NULL);
2494 if (r != X86EMUL_CONTINUE)
2496 if (io_bitmap_ptr + port/8 > desc_limit_scaled(&tr_seg))
2498 r = ops->read_std(ctxt, base + io_bitmap_ptr + port/8, &perm, 2, NULL);
2499 if (r != X86EMUL_CONTINUE)
2501 if ((perm >> bit_idx) & mask)
2506 static bool emulator_io_permited(struct x86_emulate_ctxt *ctxt,
2512 if (emulator_bad_iopl(ctxt))
2513 if (!emulator_io_port_access_allowed(ctxt, port, len))
2516 ctxt->perm_ok = true;
2521 static void save_state_to_tss16(struct x86_emulate_ctxt *ctxt,
2522 struct tss_segment_16 *tss)
2524 tss->ip = ctxt->_eip;
2525 tss->flag = ctxt->eflags;
2526 tss->ax = reg_read(ctxt, VCPU_REGS_RAX);
2527 tss->cx = reg_read(ctxt, VCPU_REGS_RCX);
2528 tss->dx = reg_read(ctxt, VCPU_REGS_RDX);
2529 tss->bx = reg_read(ctxt, VCPU_REGS_RBX);
2530 tss->sp = reg_read(ctxt, VCPU_REGS_RSP);
2531 tss->bp = reg_read(ctxt, VCPU_REGS_RBP);
2532 tss->si = reg_read(ctxt, VCPU_REGS_RSI);
2533 tss->di = reg_read(ctxt, VCPU_REGS_RDI);
2535 tss->es = get_segment_selector(ctxt, VCPU_SREG_ES);
2536 tss->cs = get_segment_selector(ctxt, VCPU_SREG_CS);
2537 tss->ss = get_segment_selector(ctxt, VCPU_SREG_SS);
2538 tss->ds = get_segment_selector(ctxt, VCPU_SREG_DS);
2539 tss->ldt = get_segment_selector(ctxt, VCPU_SREG_LDTR);
2542 static int load_state_from_tss16(struct x86_emulate_ctxt *ctxt,
2543 struct tss_segment_16 *tss)
2547 ctxt->_eip = tss->ip;
2548 ctxt->eflags = tss->flag | 2;
2549 *reg_write(ctxt, VCPU_REGS_RAX) = tss->ax;
2550 *reg_write(ctxt, VCPU_REGS_RCX) = tss->cx;
2551 *reg_write(ctxt, VCPU_REGS_RDX) = tss->dx;
2552 *reg_write(ctxt, VCPU_REGS_RBX) = tss->bx;
2553 *reg_write(ctxt, VCPU_REGS_RSP) = tss->sp;
2554 *reg_write(ctxt, VCPU_REGS_RBP) = tss->bp;
2555 *reg_write(ctxt, VCPU_REGS_RSI) = tss->si;
2556 *reg_write(ctxt, VCPU_REGS_RDI) = tss->di;
2559 * SDM says that segment selectors are loaded before segment
2562 set_segment_selector(ctxt, tss->ldt, VCPU_SREG_LDTR);
2563 set_segment_selector(ctxt, tss->es, VCPU_SREG_ES);
2564 set_segment_selector(ctxt, tss->cs, VCPU_SREG_CS);
2565 set_segment_selector(ctxt, tss->ss, VCPU_SREG_SS);
2566 set_segment_selector(ctxt, tss->ds, VCPU_SREG_DS);
2569 * Now load segment descriptors. If fault happens at this stage
2570 * it is handled in a context of new task
2572 ret = load_segment_descriptor(ctxt, tss->ldt, VCPU_SREG_LDTR);
2573 if (ret != X86EMUL_CONTINUE)
2575 ret = load_segment_descriptor(ctxt, tss->es, VCPU_SREG_ES);
2576 if (ret != X86EMUL_CONTINUE)
2578 ret = load_segment_descriptor(ctxt, tss->cs, VCPU_SREG_CS);
2579 if (ret != X86EMUL_CONTINUE)
2581 ret = load_segment_descriptor(ctxt, tss->ss, VCPU_SREG_SS);
2582 if (ret != X86EMUL_CONTINUE)
2584 ret = load_segment_descriptor(ctxt, tss->ds, VCPU_SREG_DS);
2585 if (ret != X86EMUL_CONTINUE)
2588 return X86EMUL_CONTINUE;
2591 static int task_switch_16(struct x86_emulate_ctxt *ctxt,
2592 u16 tss_selector, u16 old_tss_sel,
2593 ulong old_tss_base, struct desc_struct *new_desc)
2595 const struct x86_emulate_ops *ops = ctxt->ops;
2596 struct tss_segment_16 tss_seg;
2598 u32 new_tss_base = get_desc_base(new_desc);
2600 ret = ops->read_std(ctxt, old_tss_base, &tss_seg, sizeof tss_seg,
2602 if (ret != X86EMUL_CONTINUE)
2603 /* FIXME: need to provide precise fault address */
2606 save_state_to_tss16(ctxt, &tss_seg);
2608 ret = ops->write_std(ctxt, old_tss_base, &tss_seg, sizeof tss_seg,
2610 if (ret != X86EMUL_CONTINUE)
2611 /* FIXME: need to provide precise fault address */
2614 ret = ops->read_std(ctxt, new_tss_base, &tss_seg, sizeof tss_seg,
2616 if (ret != X86EMUL_CONTINUE)
2617 /* FIXME: need to provide precise fault address */
2620 if (old_tss_sel != 0xffff) {
2621 tss_seg.prev_task_link = old_tss_sel;
2623 ret = ops->write_std(ctxt, new_tss_base,
2624 &tss_seg.prev_task_link,
2625 sizeof tss_seg.prev_task_link,
2627 if (ret != X86EMUL_CONTINUE)
2628 /* FIXME: need to provide precise fault address */
2632 return load_state_from_tss16(ctxt, &tss_seg);
2635 static void save_state_to_tss32(struct x86_emulate_ctxt *ctxt,
2636 struct tss_segment_32 *tss)
2638 tss->cr3 = ctxt->ops->get_cr(ctxt, 3);
2639 tss->eip = ctxt->_eip;
2640 tss->eflags = ctxt->eflags;
2641 tss->eax = reg_read(ctxt, VCPU_REGS_RAX);
2642 tss->ecx = reg_read(ctxt, VCPU_REGS_RCX);
2643 tss->edx = reg_read(ctxt, VCPU_REGS_RDX);
2644 tss->ebx = reg_read(ctxt, VCPU_REGS_RBX);
2645 tss->esp = reg_read(ctxt, VCPU_REGS_RSP);
2646 tss->ebp = reg_read(ctxt, VCPU_REGS_RBP);
2647 tss->esi = reg_read(ctxt, VCPU_REGS_RSI);
2648 tss->edi = reg_read(ctxt, VCPU_REGS_RDI);
2650 tss->es = get_segment_selector(ctxt, VCPU_SREG_ES);
2651 tss->cs = get_segment_selector(ctxt, VCPU_SREG_CS);
2652 tss->ss = get_segment_selector(ctxt, VCPU_SREG_SS);
2653 tss->ds = get_segment_selector(ctxt, VCPU_SREG_DS);
2654 tss->fs = get_segment_selector(ctxt, VCPU_SREG_FS);
2655 tss->gs = get_segment_selector(ctxt, VCPU_SREG_GS);
2656 tss->ldt_selector = get_segment_selector(ctxt, VCPU_SREG_LDTR);
2659 static int load_state_from_tss32(struct x86_emulate_ctxt *ctxt,
2660 struct tss_segment_32 *tss)
2664 if (ctxt->ops->set_cr(ctxt, 3, tss->cr3))
2665 return emulate_gp(ctxt, 0);
2666 ctxt->_eip = tss->eip;
2667 ctxt->eflags = tss->eflags | 2;
2669 /* General purpose registers */
2670 *reg_write(ctxt, VCPU_REGS_RAX) = tss->eax;
2671 *reg_write(ctxt, VCPU_REGS_RCX) = tss->ecx;
2672 *reg_write(ctxt, VCPU_REGS_RDX) = tss->edx;
2673 *reg_write(ctxt, VCPU_REGS_RBX) = tss->ebx;
2674 *reg_write(ctxt, VCPU_REGS_RSP) = tss->esp;
2675 *reg_write(ctxt, VCPU_REGS_RBP) = tss->ebp;
2676 *reg_write(ctxt, VCPU_REGS_RSI) = tss->esi;
2677 *reg_write(ctxt, VCPU_REGS_RDI) = tss->edi;
2680 * SDM says that segment selectors are loaded before segment
2683 set_segment_selector(ctxt, tss->ldt_selector, VCPU_SREG_LDTR);
2684 set_segment_selector(ctxt, tss->es, VCPU_SREG_ES);
2685 set_segment_selector(ctxt, tss->cs, VCPU_SREG_CS);
2686 set_segment_selector(ctxt, tss->ss, VCPU_SREG_SS);
2687 set_segment_selector(ctxt, tss->ds, VCPU_SREG_DS);
2688 set_segment_selector(ctxt, tss->fs, VCPU_SREG_FS);
2689 set_segment_selector(ctxt, tss->gs, VCPU_SREG_GS);
2692 * If we're switching between Protected Mode and VM86, we need to make
2693 * sure to update the mode before loading the segment descriptors so
2694 * that the selectors are interpreted correctly.
2696 * Need to get rflags to the vcpu struct immediately because it
2697 * influences the CPL which is checked at least when loading the segment
2698 * descriptors and when pushing an error code to the new kernel stack.
2700 * TODO Introduce a separate ctxt->ops->set_cpl callback
2702 if (ctxt->eflags & X86_EFLAGS_VM)
2703 ctxt->mode = X86EMUL_MODE_VM86;
2705 ctxt->mode = X86EMUL_MODE_PROT32;
2707 ctxt->ops->set_rflags(ctxt, ctxt->eflags);
2710 * Now load segment descriptors. If fault happenes at this stage
2711 * it is handled in a context of new task
2713 ret = load_segment_descriptor(ctxt, tss->ldt_selector, VCPU_SREG_LDTR);
2714 if (ret != X86EMUL_CONTINUE)
2716 ret = load_segment_descriptor(ctxt, tss->es, VCPU_SREG_ES);
2717 if (ret != X86EMUL_CONTINUE)
2719 ret = load_segment_descriptor(ctxt, tss->cs, VCPU_SREG_CS);
2720 if (ret != X86EMUL_CONTINUE)
2722 ret = load_segment_descriptor(ctxt, tss->ss, VCPU_SREG_SS);
2723 if (ret != X86EMUL_CONTINUE)
2725 ret = load_segment_descriptor(ctxt, tss->ds, VCPU_SREG_DS);
2726 if (ret != X86EMUL_CONTINUE)
2728 ret = load_segment_descriptor(ctxt, tss->fs, VCPU_SREG_FS);
2729 if (ret != X86EMUL_CONTINUE)
2731 ret = load_segment_descriptor(ctxt, tss->gs, VCPU_SREG_GS);
2732 if (ret != X86EMUL_CONTINUE)
2735 return X86EMUL_CONTINUE;
2738 static int task_switch_32(struct x86_emulate_ctxt *ctxt,
2739 u16 tss_selector, u16 old_tss_sel,
2740 ulong old_tss_base, struct desc_struct *new_desc)
2742 const struct x86_emulate_ops *ops = ctxt->ops;
2743 struct tss_segment_32 tss_seg;
2745 u32 new_tss_base = get_desc_base(new_desc);
2747 ret = ops->read_std(ctxt, old_tss_base, &tss_seg, sizeof tss_seg,
2749 if (ret != X86EMUL_CONTINUE)
2750 /* FIXME: need to provide precise fault address */
2753 save_state_to_tss32(ctxt, &tss_seg);
2755 ret = ops->write_std(ctxt, old_tss_base, &tss_seg, sizeof tss_seg,
2757 if (ret != X86EMUL_CONTINUE)
2758 /* FIXME: need to provide precise fault address */
2761 ret = ops->read_std(ctxt, new_tss_base, &tss_seg, sizeof tss_seg,
2763 if (ret != X86EMUL_CONTINUE)
2764 /* FIXME: need to provide precise fault address */
2767 if (old_tss_sel != 0xffff) {
2768 tss_seg.prev_task_link = old_tss_sel;
2770 ret = ops->write_std(ctxt, new_tss_base,
2771 &tss_seg.prev_task_link,
2772 sizeof tss_seg.prev_task_link,
2774 if (ret != X86EMUL_CONTINUE)
2775 /* FIXME: need to provide precise fault address */
2779 return load_state_from_tss32(ctxt, &tss_seg);
2782 static int emulator_do_task_switch(struct x86_emulate_ctxt *ctxt,
2783 u16 tss_selector, int idt_index, int reason,
2784 bool has_error_code, u32 error_code)
2786 const struct x86_emulate_ops *ops = ctxt->ops;
2787 struct desc_struct curr_tss_desc, next_tss_desc;
2789 u16 old_tss_sel = get_segment_selector(ctxt, VCPU_SREG_TR);
2790 ulong old_tss_base =
2791 ops->get_cached_segment_base(ctxt, VCPU_SREG_TR);
2795 /* FIXME: old_tss_base == ~0 ? */
2797 ret = read_segment_descriptor(ctxt, tss_selector, &next_tss_desc, &desc_addr);
2798 if (ret != X86EMUL_CONTINUE)
2800 ret = read_segment_descriptor(ctxt, old_tss_sel, &curr_tss_desc, &desc_addr);
2801 if (ret != X86EMUL_CONTINUE)
2804 /* FIXME: check that next_tss_desc is tss */
2807 * Check privileges. The three cases are task switch caused by...
2809 * 1. jmp/call/int to task gate: Check against DPL of the task gate
2810 * 2. Exception/IRQ/iret: No check is performed
2811 * 3. jmp/call to TSS: Check against DPL of the TSS
2813 if (reason == TASK_SWITCH_GATE) {
2814 if (idt_index != -1) {
2815 /* Software interrupts */
2816 struct desc_struct task_gate_desc;
2819 ret = read_interrupt_descriptor(ctxt, idt_index,
2821 if (ret != X86EMUL_CONTINUE)
2824 dpl = task_gate_desc.dpl;
2825 if ((tss_selector & 3) > dpl || ops->cpl(ctxt) > dpl)
2826 return emulate_gp(ctxt, (idt_index << 3) | 0x2);
2828 } else if (reason != TASK_SWITCH_IRET) {
2829 int dpl = next_tss_desc.dpl;
2830 if ((tss_selector & 3) > dpl || ops->cpl(ctxt) > dpl)
2831 return emulate_gp(ctxt, tss_selector);
2835 desc_limit = desc_limit_scaled(&next_tss_desc);
2836 if (!next_tss_desc.p ||
2837 ((desc_limit < 0x67 && (next_tss_desc.type & 8)) ||
2838 desc_limit < 0x2b)) {
2839 emulate_ts(ctxt, tss_selector & 0xfffc);
2840 return X86EMUL_PROPAGATE_FAULT;
2843 if (reason == TASK_SWITCH_IRET || reason == TASK_SWITCH_JMP) {
2844 curr_tss_desc.type &= ~(1 << 1); /* clear busy flag */
2845 write_segment_descriptor(ctxt, old_tss_sel, &curr_tss_desc);
2848 if (reason == TASK_SWITCH_IRET)
2849 ctxt->eflags = ctxt->eflags & ~X86_EFLAGS_NT;
2851 /* set back link to prev task only if NT bit is set in eflags
2852 note that old_tss_sel is not used after this point */
2853 if (reason != TASK_SWITCH_CALL && reason != TASK_SWITCH_GATE)
2854 old_tss_sel = 0xffff;
2856 if (next_tss_desc.type & 8)
2857 ret = task_switch_32(ctxt, tss_selector, old_tss_sel,
2858 old_tss_base, &next_tss_desc);
2860 ret = task_switch_16(ctxt, tss_selector, old_tss_sel,
2861 old_tss_base, &next_tss_desc);
2862 if (ret != X86EMUL_CONTINUE)
2865 if (reason == TASK_SWITCH_CALL || reason == TASK_SWITCH_GATE)
2866 ctxt->eflags = ctxt->eflags | X86_EFLAGS_NT;
2868 if (reason != TASK_SWITCH_IRET) {
2869 next_tss_desc.type |= (1 << 1); /* set busy flag */
2870 write_segment_descriptor(ctxt, tss_selector, &next_tss_desc);
2873 ops->set_cr(ctxt, 0, ops->get_cr(ctxt, 0) | X86_CR0_TS);
2874 ops->set_segment(ctxt, tss_selector, &next_tss_desc, 0, VCPU_SREG_TR);
2876 if (has_error_code) {
2877 ctxt->op_bytes = ctxt->ad_bytes = (next_tss_desc.type & 8) ? 4 : 2;
2878 ctxt->lock_prefix = 0;
2879 ctxt->src.val = (unsigned long) error_code;
2880 ret = em_push(ctxt);
2886 int emulator_task_switch(struct x86_emulate_ctxt *ctxt,
2887 u16 tss_selector, int idt_index, int reason,
2888 bool has_error_code, u32 error_code)
2892 invalidate_registers(ctxt);
2893 ctxt->_eip = ctxt->eip;
2894 ctxt->dst.type = OP_NONE;
2896 rc = emulator_do_task_switch(ctxt, tss_selector, idt_index, reason,
2897 has_error_code, error_code);
2899 if (rc == X86EMUL_CONTINUE) {
2900 ctxt->eip = ctxt->_eip;
2901 writeback_registers(ctxt);
2904 return (rc == X86EMUL_UNHANDLEABLE) ? EMULATION_FAILED : EMULATION_OK;
2907 static void string_addr_inc(struct x86_emulate_ctxt *ctxt, int reg,
2910 int df = (ctxt->eflags & EFLG_DF) ? -op->count : op->count;
2912 register_address_increment(ctxt, reg_rmw(ctxt, reg), df * op->bytes);
2913 op->addr.mem.ea = register_address(ctxt, reg_read(ctxt, reg));
2916 static int em_das(struct x86_emulate_ctxt *ctxt)
2919 bool af, cf, old_cf;
2921 cf = ctxt->eflags & X86_EFLAGS_CF;
2927 af = ctxt->eflags & X86_EFLAGS_AF;
2928 if ((al & 0x0f) > 9 || af) {
2930 cf = old_cf | (al >= 250);
2935 if (old_al > 0x99 || old_cf) {
2941 /* Set PF, ZF, SF */
2942 ctxt->src.type = OP_IMM;
2944 ctxt->src.bytes = 1;
2945 emulate_2op_SrcV(ctxt, "or");
2946 ctxt->eflags &= ~(X86_EFLAGS_AF | X86_EFLAGS_CF);
2948 ctxt->eflags |= X86_EFLAGS_CF;
2950 ctxt->eflags |= X86_EFLAGS_AF;
2951 return X86EMUL_CONTINUE;
2954 static int em_aad(struct x86_emulate_ctxt *ctxt)
2956 u8 al = ctxt->dst.val & 0xff;
2957 u8 ah = (ctxt->dst.val >> 8) & 0xff;
2959 al = (al + (ah * ctxt->src.val)) & 0xff;
2961 ctxt->dst.val = (ctxt->dst.val & 0xffff0000) | al;
2963 ctxt->eflags &= ~(X86_EFLAGS_PF | X86_EFLAGS_SF | X86_EFLAGS_ZF);
2966 ctxt->eflags |= X86_EFLAGS_ZF;
2968 ctxt->eflags |= X86_EFLAGS_PF;
2970 ctxt->eflags |= X86_EFLAGS_SF;
2972 return X86EMUL_CONTINUE;
2975 static int em_call(struct x86_emulate_ctxt *ctxt)
2977 long rel = ctxt->src.val;
2979 ctxt->src.val = (unsigned long)ctxt->_eip;
2981 return em_push(ctxt);
2984 static int em_call_far(struct x86_emulate_ctxt *ctxt)
2990 old_cs = get_segment_selector(ctxt, VCPU_SREG_CS);
2991 old_eip = ctxt->_eip;
2993 memcpy(&sel, ctxt->src.valptr + ctxt->op_bytes, 2);
2994 if (load_segment_descriptor(ctxt, sel, VCPU_SREG_CS))
2995 return X86EMUL_CONTINUE;
2998 memcpy(&ctxt->_eip, ctxt->src.valptr, ctxt->op_bytes);
3000 ctxt->src.val = old_cs;
3002 if (rc != X86EMUL_CONTINUE)
3005 ctxt->src.val = old_eip;
3006 return em_push(ctxt);
3009 static int em_ret_near_imm(struct x86_emulate_ctxt *ctxt)
3013 ctxt->dst.type = OP_REG;
3014 ctxt->dst.addr.reg = &ctxt->_eip;
3015 ctxt->dst.bytes = ctxt->op_bytes;
3016 rc = emulate_pop(ctxt, &ctxt->dst.val, ctxt->op_bytes);
3017 if (rc != X86EMUL_CONTINUE)
3019 rsp_increment(ctxt, ctxt->src.val);
3020 return X86EMUL_CONTINUE;
3023 static int em_add(struct x86_emulate_ctxt *ctxt)
3025 emulate_2op_SrcV(ctxt, "add");
3026 return X86EMUL_CONTINUE;
3029 static int em_or(struct x86_emulate_ctxt *ctxt)
3031 emulate_2op_SrcV(ctxt, "or");
3032 return X86EMUL_CONTINUE;
3035 static int em_adc(struct x86_emulate_ctxt *ctxt)
3037 emulate_2op_SrcV(ctxt, "adc");
3038 return X86EMUL_CONTINUE;
3041 static int em_sbb(struct x86_emulate_ctxt *ctxt)
3043 emulate_2op_SrcV(ctxt, "sbb");
3044 return X86EMUL_CONTINUE;
3047 static int em_and(struct x86_emulate_ctxt *ctxt)
3049 emulate_2op_SrcV(ctxt, "and");
3050 return X86EMUL_CONTINUE;
3053 static int em_sub(struct x86_emulate_ctxt *ctxt)
3055 emulate_2op_SrcV(ctxt, "sub");
3056 return X86EMUL_CONTINUE;
3059 static int em_xor(struct x86_emulate_ctxt *ctxt)
3061 emulate_2op_SrcV(ctxt, "xor");
3062 return X86EMUL_CONTINUE;
3065 static int em_cmp(struct x86_emulate_ctxt *ctxt)
3067 emulate_2op_SrcV(ctxt, "cmp");
3068 /* Disable writeback. */
3069 ctxt->dst.type = OP_NONE;
3070 return X86EMUL_CONTINUE;
3073 static int em_test(struct x86_emulate_ctxt *ctxt)
3075 emulate_2op_SrcV(ctxt, "test");
3076 /* Disable writeback. */
3077 ctxt->dst.type = OP_NONE;
3078 return X86EMUL_CONTINUE;
3081 static int em_xchg(struct x86_emulate_ctxt *ctxt)
3083 /* Write back the register source. */
3084 ctxt->src.val = ctxt->dst.val;
3085 write_register_operand(&ctxt->src);
3087 /* Write back the memory destination with implicit LOCK prefix. */
3088 ctxt->dst.val = ctxt->src.orig_val;
3089 ctxt->lock_prefix = 1;
3090 return X86EMUL_CONTINUE;
3093 static int em_imul(struct x86_emulate_ctxt *ctxt)
3095 emulate_2op_SrcV_nobyte(ctxt, "imul");
3096 return X86EMUL_CONTINUE;
3099 static int em_imul_3op(struct x86_emulate_ctxt *ctxt)
3101 ctxt->dst.val = ctxt->src2.val;
3102 return em_imul(ctxt);
3105 static int em_cwd(struct x86_emulate_ctxt *ctxt)
3107 ctxt->dst.type = OP_REG;
3108 ctxt->dst.bytes = ctxt->src.bytes;
3109 ctxt->dst.addr.reg = reg_rmw(ctxt, VCPU_REGS_RDX);
3110 ctxt->dst.val = ~((ctxt->src.val >> (ctxt->src.bytes * 8 - 1)) - 1);
3112 return X86EMUL_CONTINUE;
3115 static int em_rdtsc(struct x86_emulate_ctxt *ctxt)
3119 ctxt->ops->get_msr(ctxt, MSR_IA32_TSC, &tsc);
3120 *reg_write(ctxt, VCPU_REGS_RAX) = (u32)tsc;
3121 *reg_write(ctxt, VCPU_REGS_RDX) = tsc >> 32;
3122 return X86EMUL_CONTINUE;
3125 static int em_rdpmc(struct x86_emulate_ctxt *ctxt)
3129 if (ctxt->ops->read_pmc(ctxt, reg_read(ctxt, VCPU_REGS_RCX), &pmc))
3130 return emulate_gp(ctxt, 0);
3131 *reg_write(ctxt, VCPU_REGS_RAX) = (u32)pmc;
3132 *reg_write(ctxt, VCPU_REGS_RDX) = pmc >> 32;
3133 return X86EMUL_CONTINUE;
3136 static int em_mov(struct x86_emulate_ctxt *ctxt)
3138 memcpy(ctxt->dst.valptr, ctxt->src.valptr, ctxt->op_bytes);
3139 return X86EMUL_CONTINUE;
3142 static int em_cr_write(struct x86_emulate_ctxt *ctxt)
3144 if (ctxt->ops->set_cr(ctxt, ctxt->modrm_reg, ctxt->src.val))
3145 return emulate_gp(ctxt, 0);
3147 /* Disable writeback. */
3148 ctxt->dst.type = OP_NONE;
3149 return X86EMUL_CONTINUE;
3152 static int em_dr_write(struct x86_emulate_ctxt *ctxt)
3156 if (ctxt->mode == X86EMUL_MODE_PROT64)
3157 val = ctxt->src.val & ~0ULL;
3159 val = ctxt->src.val & ~0U;
3161 /* #UD condition is already handled. */
3162 if (ctxt->ops->set_dr(ctxt, ctxt->modrm_reg, val) < 0)
3163 return emulate_gp(ctxt, 0);
3165 /* Disable writeback. */
3166 ctxt->dst.type = OP_NONE;
3167 return X86EMUL_CONTINUE;
3170 static int em_wrmsr(struct x86_emulate_ctxt *ctxt)
3174 msr_data = (u32)reg_read(ctxt, VCPU_REGS_RAX)
3175 | ((u64)reg_read(ctxt, VCPU_REGS_RDX) << 32);
3176 if (ctxt->ops->set_msr(ctxt, reg_read(ctxt, VCPU_REGS_RCX), msr_data))
3177 return emulate_gp(ctxt, 0);
3179 return X86EMUL_CONTINUE;
3182 static int em_rdmsr(struct x86_emulate_ctxt *ctxt)
3186 if (ctxt->ops->get_msr(ctxt, reg_read(ctxt, VCPU_REGS_RCX), &msr_data))
3187 return emulate_gp(ctxt, 0);
3189 *reg_write(ctxt, VCPU_REGS_RAX) = (u32)msr_data;
3190 *reg_write(ctxt, VCPU_REGS_RDX) = msr_data >> 32;
3191 return X86EMUL_CONTINUE;
3194 static int em_mov_rm_sreg(struct x86_emulate_ctxt *ctxt)
3196 if (ctxt->modrm_reg > VCPU_SREG_GS)
3197 return emulate_ud(ctxt);
3199 ctxt->dst.val = get_segment_selector(ctxt, ctxt->modrm_reg);
3200 return X86EMUL_CONTINUE;
3203 static int em_mov_sreg_rm(struct x86_emulate_ctxt *ctxt)
3205 u16 sel = ctxt->src.val;
3207 if (ctxt->modrm_reg == VCPU_SREG_CS || ctxt->modrm_reg > VCPU_SREG_GS)
3208 return emulate_ud(ctxt);
3210 if (ctxt->modrm_reg == VCPU_SREG_SS)
3211 ctxt->interruptibility = KVM_X86_SHADOW_INT_MOV_SS;
3213 /* Disable writeback. */
3214 ctxt->dst.type = OP_NONE;
3215 return load_segment_descriptor(ctxt, sel, ctxt->modrm_reg);
3218 static int em_lldt(struct x86_emulate_ctxt *ctxt)
3220 u16 sel = ctxt->src.val;
3222 /* Disable writeback. */
3223 ctxt->dst.type = OP_NONE;
3224 return load_segment_descriptor(ctxt, sel, VCPU_SREG_LDTR);
3227 static int em_ltr(struct x86_emulate_ctxt *ctxt)
3229 u16 sel = ctxt->src.val;
3231 /* Disable writeback. */
3232 ctxt->dst.type = OP_NONE;
3233 return load_segment_descriptor(ctxt, sel, VCPU_SREG_TR);
3236 static int em_invlpg(struct x86_emulate_ctxt *ctxt)
3241 rc = linearize(ctxt, ctxt->src.addr.mem, 1, false, &linear);
3242 if (rc == X86EMUL_CONTINUE)
3243 ctxt->ops->invlpg(ctxt, linear);
3244 /* Disable writeback. */
3245 ctxt->dst.type = OP_NONE;
3246 return X86EMUL_CONTINUE;
3249 static int em_clts(struct x86_emulate_ctxt *ctxt)
3253 cr0 = ctxt->ops->get_cr(ctxt, 0);
3255 ctxt->ops->set_cr(ctxt, 0, cr0);
3256 return X86EMUL_CONTINUE;
3259 static int em_vmcall(struct x86_emulate_ctxt *ctxt)
3263 if (ctxt->modrm_mod != 3 || ctxt->modrm_rm != 1)
3264 return X86EMUL_UNHANDLEABLE;
3266 rc = ctxt->ops->fix_hypercall(ctxt);
3267 if (rc != X86EMUL_CONTINUE)
3270 /* Let the processor re-execute the fixed hypercall */
3271 ctxt->_eip = ctxt->eip;
3272 /* Disable writeback. */
3273 ctxt->dst.type = OP_NONE;
3274 return X86EMUL_CONTINUE;
3277 static int emulate_store_desc_ptr(struct x86_emulate_ctxt *ctxt,
3278 void (*get)(struct x86_emulate_ctxt *ctxt,
3279 struct desc_ptr *ptr))
3281 struct desc_ptr desc_ptr;
3283 if (ctxt->mode == X86EMUL_MODE_PROT64)
3285 get(ctxt, &desc_ptr);
3286 if (ctxt->op_bytes == 2) {
3288 desc_ptr.address &= 0x00ffffff;
3290 /* Disable writeback. */
3291 ctxt->dst.type = OP_NONE;
3292 return segmented_write(ctxt, ctxt->dst.addr.mem,
3293 &desc_ptr, 2 + ctxt->op_bytes);
3296 static int em_sgdt(struct x86_emulate_ctxt *ctxt)
3298 return emulate_store_desc_ptr(ctxt, ctxt->ops->get_gdt);
3301 static int em_sidt(struct x86_emulate_ctxt *ctxt)
3303 return emulate_store_desc_ptr(ctxt, ctxt->ops->get_idt);
3306 static int em_lgdt(struct x86_emulate_ctxt *ctxt)
3308 struct desc_ptr desc_ptr;
3311 if (ctxt->mode == X86EMUL_MODE_PROT64)
3313 rc = read_descriptor(ctxt, ctxt->src.addr.mem,
3314 &desc_ptr.size, &desc_ptr.address,
3316 if (rc != X86EMUL_CONTINUE)
3318 ctxt->ops->set_gdt(ctxt, &desc_ptr);
3319 /* Disable writeback. */
3320 ctxt->dst.type = OP_NONE;
3321 return X86EMUL_CONTINUE;
3324 static int em_vmmcall(struct x86_emulate_ctxt *ctxt)
3328 rc = ctxt->ops->fix_hypercall(ctxt);
3330 /* Disable writeback. */
3331 ctxt->dst.type = OP_NONE;
3335 static int em_lidt(struct x86_emulate_ctxt *ctxt)
3337 struct desc_ptr desc_ptr;
3340 if (ctxt->mode == X86EMUL_MODE_PROT64)
3342 rc = read_descriptor(ctxt, ctxt->src.addr.mem,
3343 &desc_ptr.size, &desc_ptr.address,
3345 if (rc != X86EMUL_CONTINUE)
3347 ctxt->ops->set_idt(ctxt, &desc_ptr);
3348 /* Disable writeback. */
3349 ctxt->dst.type = OP_NONE;
3350 return X86EMUL_CONTINUE;
3353 static int em_smsw(struct x86_emulate_ctxt *ctxt)
3355 ctxt->dst.bytes = 2;
3356 ctxt->dst.val = ctxt->ops->get_cr(ctxt, 0);
3357 return X86EMUL_CONTINUE;
3360 static int em_lmsw(struct x86_emulate_ctxt *ctxt)
3362 ctxt->ops->set_cr(ctxt, 0, (ctxt->ops->get_cr(ctxt, 0) & ~0x0eul)
3363 | (ctxt->src.val & 0x0f));
3364 ctxt->dst.type = OP_NONE;
3365 return X86EMUL_CONTINUE;
3368 static int em_loop(struct x86_emulate_ctxt *ctxt)
3370 register_address_increment(ctxt, reg_rmw(ctxt, VCPU_REGS_RCX), -1);
3371 if ((address_mask(ctxt, reg_read(ctxt, VCPU_REGS_RCX)) != 0) &&
3372 (ctxt->b == 0xe2 || test_cc(ctxt->b ^ 0x5, ctxt->eflags)))
3373 jmp_rel(ctxt, ctxt->src.val);
3375 return X86EMUL_CONTINUE;
3378 static int em_jcxz(struct x86_emulate_ctxt *ctxt)
3380 if (address_mask(ctxt, reg_read(ctxt, VCPU_REGS_RCX)) == 0)
3381 jmp_rel(ctxt, ctxt->src.val);
3383 return X86EMUL_CONTINUE;
3386 static int em_in(struct x86_emulate_ctxt *ctxt)
3388 if (!pio_in_emulated(ctxt, ctxt->dst.bytes, ctxt->src.val,
3390 return X86EMUL_IO_NEEDED;
3392 return X86EMUL_CONTINUE;
3395 static int em_out(struct x86_emulate_ctxt *ctxt)
3397 ctxt->ops->pio_out_emulated(ctxt, ctxt->src.bytes, ctxt->dst.val,
3399 /* Disable writeback. */
3400 ctxt->dst.type = OP_NONE;
3401 return X86EMUL_CONTINUE;
3404 static int em_cli(struct x86_emulate_ctxt *ctxt)
3406 if (emulator_bad_iopl(ctxt))
3407 return emulate_gp(ctxt, 0);
3409 ctxt->eflags &= ~X86_EFLAGS_IF;
3410 return X86EMUL_CONTINUE;
3413 static int em_sti(struct x86_emulate_ctxt *ctxt)
3415 if (emulator_bad_iopl(ctxt))
3416 return emulate_gp(ctxt, 0);
3418 ctxt->interruptibility = KVM_X86_SHADOW_INT_STI;
3419 ctxt->eflags |= X86_EFLAGS_IF;
3420 return X86EMUL_CONTINUE;
3423 static int em_bt(struct x86_emulate_ctxt *ctxt)
3425 /* Disable writeback. */
3426 ctxt->dst.type = OP_NONE;
3427 /* only subword offset */
3428 ctxt->src.val &= (ctxt->dst.bytes << 3) - 1;
3430 emulate_2op_SrcV_nobyte(ctxt, "bt");
3431 return X86EMUL_CONTINUE;
3434 static int em_bts(struct x86_emulate_ctxt *ctxt)
3436 emulate_2op_SrcV_nobyte(ctxt, "bts");
3437 return X86EMUL_CONTINUE;
3440 static int em_btr(struct x86_emulate_ctxt *ctxt)
3442 emulate_2op_SrcV_nobyte(ctxt, "btr");
3443 return X86EMUL_CONTINUE;
3446 static int em_btc(struct x86_emulate_ctxt *ctxt)
3448 emulate_2op_SrcV_nobyte(ctxt, "btc");
3449 return X86EMUL_CONTINUE;
3452 static int em_bsf(struct x86_emulate_ctxt *ctxt)
3454 emulate_2op_SrcV_nobyte(ctxt, "bsf");
3455 return X86EMUL_CONTINUE;
3458 static int em_bsr(struct x86_emulate_ctxt *ctxt)
3460 emulate_2op_SrcV_nobyte(ctxt, "bsr");
3461 return X86EMUL_CONTINUE;
3464 static int em_cpuid(struct x86_emulate_ctxt *ctxt)
3466 u32 eax, ebx, ecx, edx;
3468 eax = reg_read(ctxt, VCPU_REGS_RAX);
3469 ecx = reg_read(ctxt, VCPU_REGS_RCX);
3470 ctxt->ops->get_cpuid(ctxt, &eax, &ebx, &ecx, &edx);
3471 *reg_write(ctxt, VCPU_REGS_RAX) = eax;
3472 *reg_write(ctxt, VCPU_REGS_RBX) = ebx;
3473 *reg_write(ctxt, VCPU_REGS_RCX) = ecx;
3474 *reg_write(ctxt, VCPU_REGS_RDX) = edx;
3475 return X86EMUL_CONTINUE;
3478 static int em_lahf(struct x86_emulate_ctxt *ctxt)
3480 *reg_rmw(ctxt, VCPU_REGS_RAX) &= ~0xff00UL;
3481 *reg_rmw(ctxt, VCPU_REGS_RAX) |= (ctxt->eflags & 0xff) << 8;
3482 return X86EMUL_CONTINUE;
3485 static int em_bswap(struct x86_emulate_ctxt *ctxt)
3487 switch (ctxt->op_bytes) {
3488 #ifdef CONFIG_X86_64
3490 asm("bswap %0" : "+r"(ctxt->dst.val));
3494 asm("bswap %0" : "+r"(*(u32 *)&ctxt->dst.val));
3497 return X86EMUL_CONTINUE;
3500 static bool valid_cr(int nr)
3512 static int check_cr_read(struct x86_emulate_ctxt *ctxt)
3514 if (!valid_cr(ctxt->modrm_reg))
3515 return emulate_ud(ctxt);
3517 return X86EMUL_CONTINUE;
3520 static int check_cr_write(struct x86_emulate_ctxt *ctxt)
3522 u64 new_val = ctxt->src.val64;
3523 int cr = ctxt->modrm_reg;
3526 static u64 cr_reserved_bits[] = {
3527 0xffffffff00000000ULL,
3528 0, 0, 0, /* CR3 checked later */
3535 return emulate_ud(ctxt);
3537 if (new_val & cr_reserved_bits[cr])
3538 return emulate_gp(ctxt, 0);
3543 if (((new_val & X86_CR0_PG) && !(new_val & X86_CR0_PE)) ||
3544 ((new_val & X86_CR0_NW) && !(new_val & X86_CR0_CD)))
3545 return emulate_gp(ctxt, 0);
3547 cr4 = ctxt->ops->get_cr(ctxt, 4);
3548 ctxt->ops->get_msr(ctxt, MSR_EFER, &efer);
3550 if ((new_val & X86_CR0_PG) && (efer & EFER_LME) &&
3551 !(cr4 & X86_CR4_PAE))
3552 return emulate_gp(ctxt, 0);
3559 ctxt->ops->get_msr(ctxt, MSR_EFER, &efer);
3560 if (efer & EFER_LMA)
3561 rsvd = CR3_L_MODE_RESERVED_BITS;
3562 else if (ctxt->ops->get_cr(ctxt, 4) & X86_CR4_PAE)
3563 rsvd = CR3_PAE_RESERVED_BITS;
3564 else if (ctxt->ops->get_cr(ctxt, 0) & X86_CR0_PG)
3565 rsvd = CR3_NONPAE_RESERVED_BITS;
3568 return emulate_gp(ctxt, 0);
3573 ctxt->ops->get_msr(ctxt, MSR_EFER, &efer);
3575 if ((efer & EFER_LMA) && !(new_val & X86_CR4_PAE))
3576 return emulate_gp(ctxt, 0);
3582 return X86EMUL_CONTINUE;
3585 static int check_dr7_gd(struct x86_emulate_ctxt *ctxt)
3589 ctxt->ops->get_dr(ctxt, 7, &dr7);
3591 /* Check if DR7.Global_Enable is set */
3592 return dr7 & (1 << 13);
3595 static int check_dr_read(struct x86_emulate_ctxt *ctxt)
3597 int dr = ctxt->modrm_reg;
3601 return emulate_ud(ctxt);
3603 cr4 = ctxt->ops->get_cr(ctxt, 4);
3604 if ((cr4 & X86_CR4_DE) && (dr == 4 || dr == 5))
3605 return emulate_ud(ctxt);
3607 if (check_dr7_gd(ctxt))
3608 return emulate_db(ctxt);
3610 return X86EMUL_CONTINUE;
3613 static int check_dr_write(struct x86_emulate_ctxt *ctxt)
3615 u64 new_val = ctxt->src.val64;
3616 int dr = ctxt->modrm_reg;
3618 if ((dr == 6 || dr == 7) && (new_val & 0xffffffff00000000ULL))
3619 return emulate_gp(ctxt, 0);
3621 return check_dr_read(ctxt);
3624 static int check_svme(struct x86_emulate_ctxt *ctxt)
3628 ctxt->ops->get_msr(ctxt, MSR_EFER, &efer);
3630 if (!(efer & EFER_SVME))
3631 return emulate_ud(ctxt);
3633 return X86EMUL_CONTINUE;
3636 static int check_svme_pa(struct x86_emulate_ctxt *ctxt)
3638 u64 rax = reg_read(ctxt, VCPU_REGS_RAX);
3640 /* Valid physical address? */
3641 if (rax & 0xffff000000000000ULL)
3642 return emulate_gp(ctxt, 0);
3644 return check_svme(ctxt);
3647 static int check_rdtsc(struct x86_emulate_ctxt *ctxt)
3649 u64 cr4 = ctxt->ops->get_cr(ctxt, 4);
3651 if (cr4 & X86_CR4_TSD && ctxt->ops->cpl(ctxt))
3652 return emulate_ud(ctxt);
3654 return X86EMUL_CONTINUE;
3657 static int check_rdpmc(struct x86_emulate_ctxt *ctxt)
3659 u64 cr4 = ctxt->ops->get_cr(ctxt, 4);
3660 u64 rcx = reg_read(ctxt, VCPU_REGS_RCX);
3662 if ((!(cr4 & X86_CR4_PCE) && ctxt->ops->cpl(ctxt)) ||
3664 return emulate_gp(ctxt, 0);
3666 return X86EMUL_CONTINUE;
3669 static int check_perm_in(struct x86_emulate_ctxt *ctxt)
3671 ctxt->dst.bytes = min(ctxt->dst.bytes, 4u);
3672 if (!emulator_io_permited(ctxt, ctxt->src.val, ctxt->dst.bytes))
3673 return emulate_gp(ctxt, 0);
3675 return X86EMUL_CONTINUE;
3678 static int check_perm_out(struct x86_emulate_ctxt *ctxt)
3680 ctxt->src.bytes = min(ctxt->src.bytes, 4u);
3681 if (!emulator_io_permited(ctxt, ctxt->dst.val, ctxt->src.bytes))
3682 return emulate_gp(ctxt, 0);
3684 return X86EMUL_CONTINUE;
3687 #define D(_y) { .flags = (_y) }
3688 #define DI(_y, _i) { .flags = (_y), .intercept = x86_intercept_##_i }
3689 #define DIP(_y, _i, _p) { .flags = (_y), .intercept = x86_intercept_##_i, \
3690 .check_perm = (_p) }
3692 #define EXT(_f, _e) { .flags = ((_f) | RMExt), .u.group = (_e) }
3693 #define G(_f, _g) { .flags = ((_f) | Group | ModRM), .u.group = (_g) }
3694 #define GD(_f, _g) { .flags = ((_f) | GroupDual | ModRM), .u.gdual = (_g) }
3695 #define E(_f, _e) { .flags = ((_f) | Escape | ModRM), .u.esc = (_e) }
3696 #define I(_f, _e) { .flags = (_f), .u.execute = (_e) }
3697 #define F(_f, _e) { .flags = (_f) | Fastop, .u.fastop = (_e) }
3698 #define II(_f, _e, _i) \
3699 { .flags = (_f), .u.execute = (_e), .intercept = x86_intercept_##_i }
3700 #define IIP(_f, _e, _i, _p) \
3701 { .flags = (_f), .u.execute = (_e), .intercept = x86_intercept_##_i, \
3702 .check_perm = (_p) }
3703 #define GP(_f, _g) { .flags = ((_f) | Prefix), .u.gprefix = (_g) }
3705 #define D2bv(_f) D((_f) | ByteOp), D(_f)
3706 #define D2bvIP(_f, _i, _p) DIP((_f) | ByteOp, _i, _p), DIP(_f, _i, _p)
3707 #define I2bv(_f, _e) I((_f) | ByteOp, _e), I(_f, _e)
3708 #define I2bvIP(_f, _e, _i, _p) \
3709 IIP((_f) | ByteOp, _e, _i, _p), IIP(_f, _e, _i, _p)
3711 #define I6ALU(_f, _e) I2bv((_f) | DstMem | SrcReg | ModRM, _e), \
3712 I2bv(((_f) | DstReg | SrcMem | ModRM) & ~Lock, _e), \
3713 I2bv(((_f) & ~Lock) | DstAcc | SrcImm, _e)
3715 static const struct opcode group7_rm1[] = {
3716 DI(SrcNone | Priv, monitor),
3717 DI(SrcNone | Priv, mwait),
3721 static const struct opcode group7_rm3[] = {
3722 DIP(SrcNone | Prot | Priv, vmrun, check_svme_pa),
3723 II(SrcNone | Prot | VendorSpecific, em_vmmcall, vmmcall),
3724 DIP(SrcNone | Prot | Priv, vmload, check_svme_pa),
3725 DIP(SrcNone | Prot | Priv, vmsave, check_svme_pa),
3726 DIP(SrcNone | Prot | Priv, stgi, check_svme),
3727 DIP(SrcNone | Prot | Priv, clgi, check_svme),
3728 DIP(SrcNone | Prot | Priv, skinit, check_svme),
3729 DIP(SrcNone | Prot | Priv, invlpga, check_svme),
3732 static const struct opcode group7_rm7[] = {
3734 DIP(SrcNone, rdtscp, check_rdtsc),
3738 static const struct opcode group1[] = {
3740 I(Lock | PageTable, em_or),
3743 I(Lock | PageTable, em_and),
3749 static const struct opcode group1A[] = {
3750 I(DstMem | SrcNone | Mov | Stack, em_pop), N, N, N, N, N, N, N,
3753 static const struct opcode group3[] = {
3754 I(DstMem | SrcImm, em_test),
3755 I(DstMem | SrcImm, em_test),
3756 I(DstMem | SrcNone | Lock, em_not),
3757 I(DstMem | SrcNone | Lock, em_neg),
3758 I(SrcMem, em_mul_ex),
3759 I(SrcMem, em_imul_ex),
3760 I(SrcMem, em_div_ex),
3761 I(SrcMem, em_idiv_ex),
3764 static const struct opcode group4[] = {
3765 I(ByteOp | DstMem | SrcNone | Lock, em_grp45),
3766 I(ByteOp | DstMem | SrcNone | Lock, em_grp45),
3770 static const struct opcode group5[] = {
3771 I(DstMem | SrcNone | Lock, em_grp45),
3772 I(DstMem | SrcNone | Lock, em_grp45),
3773 I(SrcMem | Stack, em_grp45),
3774 I(SrcMemFAddr | ImplicitOps | Stack, em_call_far),
3775 I(SrcMem | Stack, em_grp45),
3776 I(SrcMemFAddr | ImplicitOps, em_grp45),
3777 I(SrcMem | Stack, em_grp45), N,
3780 static const struct opcode group6[] = {
3783 II(Prot | Priv | SrcMem16, em_lldt, lldt),
3784 II(Prot | Priv | SrcMem16, em_ltr, ltr),
3788 static const struct group_dual group7 = { {
3789 II(Mov | DstMem | Priv, em_sgdt, sgdt),
3790 II(Mov | DstMem | Priv, em_sidt, sidt),
3791 II(SrcMem | Priv, em_lgdt, lgdt),
3792 II(SrcMem | Priv, em_lidt, lidt),
3793 II(SrcNone | DstMem | Mov, em_smsw, smsw), N,
3794 II(SrcMem16 | Mov | Priv, em_lmsw, lmsw),
3795 II(SrcMem | ByteOp | Priv | NoAccess, em_invlpg, invlpg),
3797 I(SrcNone | Priv | VendorSpecific, em_vmcall),
3799 N, EXT(0, group7_rm3),
3800 II(SrcNone | DstMem | Mov, em_smsw, smsw), N,
3801 II(SrcMem16 | Mov | Priv, em_lmsw, lmsw),
3805 static const struct opcode group8[] = {
3807 I(DstMem | SrcImmByte, em_bt),
3808 I(DstMem | SrcImmByte | Lock | PageTable, em_bts),
3809 I(DstMem | SrcImmByte | Lock, em_btr),
3810 I(DstMem | SrcImmByte | Lock | PageTable, em_btc),
3813 static const struct group_dual group9 = { {
3814 N, I(DstMem64 | Lock | PageTable, em_cmpxchg8b), N, N, N, N, N, N,
3816 N, N, N, N, N, N, N, N,
3819 static const struct opcode group11[] = {
3820 I(DstMem | SrcImm | Mov | PageTable, em_mov),
3824 static const struct gprefix pfx_0f_6f_0f_7f = {
3825 I(Mmx, em_mov), I(Sse | Aligned, em_mov), N, I(Sse | Unaligned, em_mov),
3828 static const struct gprefix pfx_vmovntpx = {
3829 I(0, em_mov), N, N, N,
3832 static const struct escape escape_d9 = { {
3833 N, N, N, N, N, N, N, I(DstMem, em_fnstcw),
3836 N, N, N, N, N, N, N, N,
3838 N, N, N, N, N, N, N, N,
3840 N, N, N, N, N, N, N, N,
3842 N, N, N, N, N, N, N, N,
3844 N, N, N, N, N, N, N, N,
3846 N, N, N, N, N, N, N, N,
3848 N, N, N, N, N, N, N, N,
3850 N, N, N, N, N, N, N, N,
3853 static const struct escape escape_db = { {
3854 N, N, N, N, N, N, N, N,
3857 N, N, N, N, N, N, N, N,
3859 N, N, N, N, N, N, N, N,
3861 N, N, N, N, N, N, N, N,
3863 N, N, N, N, N, N, N, N,
3865 N, N, N, I(ImplicitOps, em_fninit), N, N, N, N,
3867 N, N, N, N, N, N, N, N,
3869 N, N, N, N, N, N, N, N,
3871 N, N, N, N, N, N, N, N,
3874 static const struct escape escape_dd = { {
3875 N, N, N, N, N, N, N, I(DstMem, em_fnstsw),
3878 N, N, N, N, N, N, N, N,
3880 N, N, N, N, N, N, N, N,
3882 N, N, N, N, N, N, N, N,
3884 N, N, N, N, N, N, N, N,
3886 N, N, N, N, N, N, N, N,
3888 N, N, N, N, N, N, N, N,
3890 N, N, N, N, N, N, N, N,
3892 N, N, N, N, N, N, N, N,
3895 static const struct opcode opcode_table[256] = {
3897 I6ALU(Lock, em_add),
3898 I(ImplicitOps | Stack | No64 | Src2ES, em_push_sreg),
3899 I(ImplicitOps | Stack | No64 | Src2ES, em_pop_sreg),
3901 I6ALU(Lock | PageTable, em_or),
3902 I(ImplicitOps | Stack | No64 | Src2CS, em_push_sreg),
3905 I6ALU(Lock, em_adc),
3906 I(ImplicitOps | Stack | No64 | Src2SS, em_push_sreg),
3907 I(ImplicitOps | Stack | No64 | Src2SS, em_pop_sreg),
3909 I6ALU(Lock, em_sbb),
3910 I(ImplicitOps | Stack | No64 | Src2DS, em_push_sreg),
3911 I(ImplicitOps | Stack | No64 | Src2DS, em_pop_sreg),
3913 I6ALU(Lock | PageTable, em_and), N, N,
3915 I6ALU(Lock, em_sub), N, I(ByteOp | DstAcc | No64, em_das),
3917 I6ALU(Lock, em_xor), N, N,
3919 I6ALU(0, em_cmp), N, N,
3923 X8(I(SrcReg | Stack, em_push)),
3925 X8(I(DstReg | Stack, em_pop)),
3927 I(ImplicitOps | Stack | No64, em_pusha),
3928 I(ImplicitOps | Stack | No64, em_popa),
3929 N, D(DstReg | SrcMem32 | ModRM | Mov) /* movsxd (x86/64) */ ,
3932 I(SrcImm | Mov | Stack, em_push),
3933 I(DstReg | SrcMem | ModRM | Src2Imm, em_imul_3op),
3934 I(SrcImmByte | Mov | Stack, em_push),
3935 I(DstReg | SrcMem | ModRM | Src2ImmByte, em_imul_3op),
3936 I2bvIP(DstDI | SrcDX | Mov | String | Unaligned, em_in, ins, check_perm_in), /* insb, insw/insd */
3937 I2bvIP(SrcSI | DstDX | String, em_out, outs, check_perm_out), /* outsb, outsw/outsd */
3941 G(ByteOp | DstMem | SrcImm, group1),
3942 G(DstMem | SrcImm, group1),
3943 G(ByteOp | DstMem | SrcImm | No64, group1),
3944 G(DstMem | SrcImmByte, group1),
3945 I2bv(DstMem | SrcReg | ModRM, em_test),
3946 I2bv(DstMem | SrcReg | ModRM | Lock | PageTable, em_xchg),
3948 I2bv(DstMem | SrcReg | ModRM | Mov | PageTable, em_mov),
3949 I2bv(DstReg | SrcMem | ModRM | Mov, em_mov),
3950 I(DstMem | SrcNone | ModRM | Mov | PageTable, em_mov_rm_sreg),
3951 D(ModRM | SrcMem | NoAccess | DstReg),
3952 I(ImplicitOps | SrcMem16 | ModRM, em_mov_sreg_rm),
3955 DI(SrcAcc | DstReg, pause), X7(D(SrcAcc | DstReg)),
3957 D(DstAcc | SrcNone), I(ImplicitOps | SrcAcc, em_cwd),
3958 I(SrcImmFAddr | No64, em_call_far), N,
3959 II(ImplicitOps | Stack, em_pushf, pushf),
3960 II(ImplicitOps | Stack, em_popf, popf), N, I(ImplicitOps, em_lahf),
3962 I2bv(DstAcc | SrcMem | Mov | MemAbs, em_mov),
3963 I2bv(DstMem | SrcAcc | Mov | MemAbs | PageTable, em_mov),
3964 I2bv(SrcSI | DstDI | Mov | String, em_mov),
3965 I2bv(SrcSI | DstDI | String, em_cmp),
3967 I2bv(DstAcc | SrcImm, em_test),
3968 I2bv(SrcAcc | DstDI | Mov | String, em_mov),
3969 I2bv(SrcSI | DstAcc | Mov | String, em_mov),
3970 I2bv(SrcAcc | DstDI | String, em_cmp),
3972 X8(I(ByteOp | DstReg | SrcImm | Mov, em_mov)),
3974 X8(I(DstReg | SrcImm64 | Mov, em_mov)),
3976 D2bv(DstMem | SrcImmByte | ModRM),
3977 I(ImplicitOps | Stack | SrcImmU16, em_ret_near_imm),
3978 I(ImplicitOps | Stack, em_ret),
3979 I(DstReg | SrcMemFAddr | ModRM | No64 | Src2ES, em_lseg),
3980 I(DstReg | SrcMemFAddr | ModRM | No64 | Src2DS, em_lseg),
3981 G(ByteOp, group11), G(0, group11),
3983 I(Stack | SrcImmU16 | Src2ImmByte, em_enter), I(Stack, em_leave),
3984 N, I(ImplicitOps | Stack, em_ret_far),
3985 D(ImplicitOps), DI(SrcImmByte, intn),
3986 D(ImplicitOps | No64), II(ImplicitOps, em_iret, iret),
3988 D2bv(DstMem | SrcOne | ModRM), D2bv(DstMem | ModRM),
3989 N, I(DstAcc | SrcImmByte | No64, em_aad), N, N,
3991 N, E(0, &escape_d9), N, E(0, &escape_db), N, E(0, &escape_dd), N, N,
3993 X3(I(SrcImmByte, em_loop)),
3994 I(SrcImmByte, em_jcxz),
3995 I2bvIP(SrcImmUByte | DstAcc, em_in, in, check_perm_in),
3996 I2bvIP(SrcAcc | DstImmUByte, em_out, out, check_perm_out),
3998 I(SrcImm | Stack, em_call), D(SrcImm | ImplicitOps),
3999 I(SrcImmFAddr | No64, em_jmp_far), D(SrcImmByte | ImplicitOps),
4000 I2bvIP(SrcDX | DstAcc, em_in, in, check_perm_in),
4001 I2bvIP(SrcAcc | DstDX, em_out, out, check_perm_out),
4003 N, DI(ImplicitOps, icebp), N, N,
4004 DI(ImplicitOps | Priv, hlt), D(ImplicitOps),
4005 G(ByteOp, group3), G(0, group3),
4007 D(ImplicitOps), D(ImplicitOps),
4008 I(ImplicitOps, em_cli), I(ImplicitOps, em_sti),
4009 D(ImplicitOps), D(ImplicitOps), G(0, group4), G(0, group5),
4012 static const struct opcode twobyte_table[256] = {
4014 G(0, group6), GD(0, &group7), N, N,
4015 N, I(ImplicitOps | VendorSpecific, em_syscall),
4016 II(ImplicitOps | Priv, em_clts, clts), N,
4017 DI(ImplicitOps | Priv, invd), DI(ImplicitOps | Priv, wbinvd), N, N,
4018 N, D(ImplicitOps | ModRM), N, N,
4020 N, N, N, N, N, N, N, N, D(ImplicitOps | ModRM), N, N, N, N, N, N, N,
4022 DIP(ModRM | DstMem | Priv | Op3264, cr_read, check_cr_read),
4023 DIP(ModRM | DstMem | Priv | Op3264, dr_read, check_dr_read),
4024 IIP(ModRM | SrcMem | Priv | Op3264, em_cr_write, cr_write, check_cr_write),
4025 IIP(ModRM | SrcMem | Priv | Op3264, em_dr_write, dr_write, check_dr_write),
4027 N, N, N, GP(ModRM | DstMem | SrcReg | Sse | Mov | Aligned, &pfx_vmovntpx),
4030 II(ImplicitOps | Priv, em_wrmsr, wrmsr),
4031 IIP(ImplicitOps, em_rdtsc, rdtsc, check_rdtsc),
4032 II(ImplicitOps | Priv, em_rdmsr, rdmsr),
4033 IIP(ImplicitOps, em_rdpmc, rdpmc, check_rdpmc),
4034 I(ImplicitOps | VendorSpecific, em_sysenter),
4035 I(ImplicitOps | Priv | VendorSpecific, em_sysexit),
4037 N, N, N, N, N, N, N, N,
4039 X16(D(DstReg | SrcMem | ModRM | Mov)),
4041 N, N, N, N, N, N, N, N, N, N, N, N, N, N, N, N,
4046 N, N, N, GP(SrcMem | DstReg | ModRM | Mov, &pfx_0f_6f_0f_7f),
4051 N, N, N, GP(SrcReg | DstMem | ModRM | Mov, &pfx_0f_6f_0f_7f),
4055 X16(D(ByteOp | DstMem | SrcNone | ModRM| Mov)),
4057 I(Stack | Src2FS, em_push_sreg), I(Stack | Src2FS, em_pop_sreg),
4058 II(ImplicitOps, em_cpuid, cpuid), I(DstMem | SrcReg | ModRM | BitOp, em_bt),
4059 D(DstMem | SrcReg | Src2ImmByte | ModRM),
4060 D(DstMem | SrcReg | Src2CL | ModRM), N, N,
4062 I(Stack | Src2GS, em_push_sreg), I(Stack | Src2GS, em_pop_sreg),
4063 DI(ImplicitOps, rsm),
4064 I(DstMem | SrcReg | ModRM | BitOp | Lock | PageTable, em_bts),
4065 D(DstMem | SrcReg | Src2ImmByte | ModRM),
4066 D(DstMem | SrcReg | Src2CL | ModRM),
4067 D(ModRM), I(DstReg | SrcMem | ModRM, em_imul),
4069 I2bv(DstMem | SrcReg | ModRM | Lock | PageTable, em_cmpxchg),
4070 I(DstReg | SrcMemFAddr | ModRM | Src2SS, em_lseg),
4071 I(DstMem | SrcReg | ModRM | BitOp | Lock, em_btr),
4072 I(DstReg | SrcMemFAddr | ModRM | Src2FS, em_lseg),
4073 I(DstReg | SrcMemFAddr | ModRM | Src2GS, em_lseg),
4074 D(DstReg | SrcMem8 | ModRM | Mov), D(DstReg | SrcMem16 | ModRM | Mov),
4078 I(DstMem | SrcReg | ModRM | BitOp | Lock | PageTable, em_btc),
4079 I(DstReg | SrcMem | ModRM, em_bsf), I(DstReg | SrcMem | ModRM, em_bsr),
4080 D(DstReg | SrcMem8 | ModRM | Mov), D(DstReg | SrcMem16 | ModRM | Mov),
4082 D2bv(DstMem | SrcReg | ModRM | Lock),
4083 N, D(DstMem | SrcReg | ModRM | Mov),
4084 N, N, N, GD(0, &group9),
4086 X8(I(DstReg, em_bswap)),
4088 N, N, N, N, N, N, N, N, N, N, N, N, N, N, N, N,
4090 N, N, N, N, N, N, N, N, N, N, N, N, N, N, N, N,
4092 N, N, N, N, N, N, N, N, N, N, N, N, N, N, N, N
4109 static unsigned imm_size(struct x86_emulate_ctxt *ctxt)
4113 size = (ctxt->d & ByteOp) ? 1 : ctxt->op_bytes;
4119 static int decode_imm(struct x86_emulate_ctxt *ctxt, struct operand *op,
4120 unsigned size, bool sign_extension)
4122 int rc = X86EMUL_CONTINUE;
4126 op->addr.mem.ea = ctxt->_eip;
4127 /* NB. Immediates are sign-extended as necessary. */
4128 switch (op->bytes) {
4130 op->val = insn_fetch(s8, ctxt);
4133 op->val = insn_fetch(s16, ctxt);
4136 op->val = insn_fetch(s32, ctxt);
4139 op->val = insn_fetch(s64, ctxt);
4142 if (!sign_extension) {
4143 switch (op->bytes) {
4151 op->val &= 0xffffffff;
4159 static int decode_operand(struct x86_emulate_ctxt *ctxt, struct operand *op,
4162 int rc = X86EMUL_CONTINUE;
4166 decode_register_operand(ctxt, op);
4169 rc = decode_imm(ctxt, op, 1, false);
4172 ctxt->memop.bytes = (ctxt->d & ByteOp) ? 1 : ctxt->op_bytes;
4176 if ((ctxt->d & BitOp) && op == &ctxt->dst)
4177 fetch_bit_operand(ctxt);
4178 op->orig_val = op->val;
4181 ctxt->memop.bytes = 8;
4185 op->bytes = (ctxt->d & ByteOp) ? 1 : ctxt->op_bytes;
4186 op->addr.reg = reg_rmw(ctxt, VCPU_REGS_RAX);
4187 fetch_register_operand(op);
4188 op->orig_val = op->val;
4192 op->bytes = (ctxt->d & ByteOp) ? 1 : ctxt->op_bytes;
4194 register_address(ctxt, reg_read(ctxt, VCPU_REGS_RDI));
4195 op->addr.mem.seg = VCPU_SREG_ES;
4202 op->addr.reg = reg_rmw(ctxt, VCPU_REGS_RDX);
4203 fetch_register_operand(op);
4207 op->val = reg_read(ctxt, VCPU_REGS_RCX) & 0xff;
4210 rc = decode_imm(ctxt, op, 1, true);
4217 rc = decode_imm(ctxt, op, imm_size(ctxt), true);
4220 rc = decode_imm(ctxt, op, ctxt->op_bytes, true);
4223 ctxt->memop.bytes = 1;
4226 ctxt->memop.bytes = 2;
4229 ctxt->memop.bytes = 4;
4232 rc = decode_imm(ctxt, op, 2, false);
4235 rc = decode_imm(ctxt, op, imm_size(ctxt), false);
4239 op->bytes = (ctxt->d & ByteOp) ? 1 : ctxt->op_bytes;
4241 register_address(ctxt, reg_read(ctxt, VCPU_REGS_RSI));
4242 op->addr.mem.seg = seg_override(ctxt);
4248 op->addr.mem.ea = ctxt->_eip;
4249 op->bytes = ctxt->op_bytes + 2;
4250 insn_fetch_arr(op->valptr, op->bytes, ctxt);
4253 ctxt->memop.bytes = ctxt->op_bytes + 2;
4256 op->val = VCPU_SREG_ES;
4259 op->val = VCPU_SREG_CS;
4262 op->val = VCPU_SREG_SS;
4265 op->val = VCPU_SREG_DS;
4268 op->val = VCPU_SREG_FS;
4271 op->val = VCPU_SREG_GS;
4274 /* Special instructions do their own operand decoding. */
4276 op->type = OP_NONE; /* Disable writeback. */
4284 int x86_decode_insn(struct x86_emulate_ctxt *ctxt, void *insn, int insn_len)
4286 int rc = X86EMUL_CONTINUE;
4287 int mode = ctxt->mode;
4288 int def_op_bytes, def_ad_bytes, goffset, simd_prefix;
4289 bool op_prefix = false;
4290 struct opcode opcode;
4292 ctxt->memop.type = OP_NONE;
4293 ctxt->memopp = NULL;
4294 ctxt->_eip = ctxt->eip;
4295 ctxt->fetch.start = ctxt->_eip;
4296 ctxt->fetch.end = ctxt->fetch.start + insn_len;
4298 memcpy(ctxt->fetch.data, insn, insn_len);
4301 case X86EMUL_MODE_REAL:
4302 case X86EMUL_MODE_VM86:
4303 case X86EMUL_MODE_PROT16:
4304 def_op_bytes = def_ad_bytes = 2;
4306 case X86EMUL_MODE_PROT32:
4307 def_op_bytes = def_ad_bytes = 4;
4309 #ifdef CONFIG_X86_64
4310 case X86EMUL_MODE_PROT64:
4316 return EMULATION_FAILED;
4319 ctxt->op_bytes = def_op_bytes;
4320 ctxt->ad_bytes = def_ad_bytes;
4322 /* Legacy prefixes. */
4324 switch (ctxt->b = insn_fetch(u8, ctxt)) {
4325 case 0x66: /* operand-size override */
4327 /* switch between 2/4 bytes */
4328 ctxt->op_bytes = def_op_bytes ^ 6;
4330 case 0x67: /* address-size override */
4331 if (mode == X86EMUL_MODE_PROT64)
4332 /* switch between 4/8 bytes */
4333 ctxt->ad_bytes = def_ad_bytes ^ 12;
4335 /* switch between 2/4 bytes */
4336 ctxt->ad_bytes = def_ad_bytes ^ 6;
4338 case 0x26: /* ES override */
4339 case 0x2e: /* CS override */
4340 case 0x36: /* SS override */
4341 case 0x3e: /* DS override */
4342 set_seg_override(ctxt, (ctxt->b >> 3) & 3);
4344 case 0x64: /* FS override */
4345 case 0x65: /* GS override */
4346 set_seg_override(ctxt, ctxt->b & 7);
4348 case 0x40 ... 0x4f: /* REX */
4349 if (mode != X86EMUL_MODE_PROT64)
4351 ctxt->rex_prefix = ctxt->b;
4353 case 0xf0: /* LOCK */
4354 ctxt->lock_prefix = 1;
4356 case 0xf2: /* REPNE/REPNZ */
4357 case 0xf3: /* REP/REPE/REPZ */
4358 ctxt->rep_prefix = ctxt->b;
4364 /* Any legacy prefix after a REX prefix nullifies its effect. */
4366 ctxt->rex_prefix = 0;
4372 if (ctxt->rex_prefix & 8)
4373 ctxt->op_bytes = 8; /* REX.W */
4375 /* Opcode byte(s). */
4376 opcode = opcode_table[ctxt->b];
4377 /* Two-byte opcode? */
4378 if (ctxt->b == 0x0f) {
4380 ctxt->b = insn_fetch(u8, ctxt);
4381 opcode = twobyte_table[ctxt->b];
4383 ctxt->d = opcode.flags;
4385 if (ctxt->d & ModRM)
4386 ctxt->modrm = insn_fetch(u8, ctxt);
4388 while (ctxt->d & GroupMask) {
4389 switch (ctxt->d & GroupMask) {
4391 goffset = (ctxt->modrm >> 3) & 7;
4392 opcode = opcode.u.group[goffset];
4395 goffset = (ctxt->modrm >> 3) & 7;
4396 if ((ctxt->modrm >> 6) == 3)
4397 opcode = opcode.u.gdual->mod3[goffset];
4399 opcode = opcode.u.gdual->mod012[goffset];
4402 goffset = ctxt->modrm & 7;
4403 opcode = opcode.u.group[goffset];
4406 if (ctxt->rep_prefix && op_prefix)
4407 return EMULATION_FAILED;
4408 simd_prefix = op_prefix ? 0x66 : ctxt->rep_prefix;
4409 switch (simd_prefix) {
4410 case 0x00: opcode = opcode.u.gprefix->pfx_no; break;
4411 case 0x66: opcode = opcode.u.gprefix->pfx_66; break;
4412 case 0xf2: opcode = opcode.u.gprefix->pfx_f2; break;
4413 case 0xf3: opcode = opcode.u.gprefix->pfx_f3; break;
4417 if (ctxt->modrm > 0xbf)
4418 opcode = opcode.u.esc->high[ctxt->modrm - 0xc0];
4420 opcode = opcode.u.esc->op[(ctxt->modrm >> 3) & 7];
4423 return EMULATION_FAILED;
4426 ctxt->d &= ~(u64)GroupMask;
4427 ctxt->d |= opcode.flags;
4430 ctxt->execute = opcode.u.execute;
4431 ctxt->check_perm = opcode.check_perm;
4432 ctxt->intercept = opcode.intercept;
4435 if (ctxt->d == 0 || (ctxt->d & Undefined))
4436 return EMULATION_FAILED;
4438 if (!(ctxt->d & VendorSpecific) && ctxt->only_vendor_specific_insn)
4439 return EMULATION_FAILED;
4441 if (mode == X86EMUL_MODE_PROT64 && (ctxt->d & Stack))
4444 if (ctxt->d & Op3264) {
4445 if (mode == X86EMUL_MODE_PROT64)
4452 ctxt->op_bytes = 16;
4453 else if (ctxt->d & Mmx)
4456 /* ModRM and SIB bytes. */
4457 if (ctxt->d & ModRM) {
4458 rc = decode_modrm(ctxt, &ctxt->memop);
4459 if (!ctxt->has_seg_override)
4460 set_seg_override(ctxt, ctxt->modrm_seg);
4461 } else if (ctxt->d & MemAbs)
4462 rc = decode_abs(ctxt, &ctxt->memop);
4463 if (rc != X86EMUL_CONTINUE)
4466 if (!ctxt->has_seg_override)
4467 set_seg_override(ctxt, VCPU_SREG_DS);
4469 ctxt->memop.addr.mem.seg = seg_override(ctxt);
4471 if (ctxt->memop.type == OP_MEM && ctxt->ad_bytes != 8)
4472 ctxt->memop.addr.mem.ea = (u32)ctxt->memop.addr.mem.ea;
4475 * Decode and fetch the source operand: register, memory
4478 rc = decode_operand(ctxt, &ctxt->src, (ctxt->d >> SrcShift) & OpMask);
4479 if (rc != X86EMUL_CONTINUE)
4483 * Decode and fetch the second source operand: register, memory
4486 rc = decode_operand(ctxt, &ctxt->src2, (ctxt->d >> Src2Shift) & OpMask);
4487 if (rc != X86EMUL_CONTINUE)
4490 /* Decode and fetch the destination operand: register or memory. */
4491 rc = decode_operand(ctxt, &ctxt->dst, (ctxt->d >> DstShift) & OpMask);
4494 if (ctxt->memopp && ctxt->memopp->type == OP_MEM && ctxt->rip_relative)
4495 ctxt->memopp->addr.mem.ea += ctxt->_eip;
4497 return (rc != X86EMUL_CONTINUE) ? EMULATION_FAILED : EMULATION_OK;
4500 bool x86_page_table_writing_insn(struct x86_emulate_ctxt *ctxt)
4502 return ctxt->d & PageTable;
4505 static bool string_insn_completed(struct x86_emulate_ctxt *ctxt)
4507 /* The second termination condition only applies for REPE
4508 * and REPNE. Test if the repeat string operation prefix is
4509 * REPE/REPZ or REPNE/REPNZ and if it's the case it tests the
4510 * corresponding termination condition according to:
4511 * - if REPE/REPZ and ZF = 0 then done
4512 * - if REPNE/REPNZ and ZF = 1 then done
4514 if (((ctxt->b == 0xa6) || (ctxt->b == 0xa7) ||
4515 (ctxt->b == 0xae) || (ctxt->b == 0xaf))
4516 && (((ctxt->rep_prefix == REPE_PREFIX) &&
4517 ((ctxt->eflags & EFLG_ZF) == 0))
4518 || ((ctxt->rep_prefix == REPNE_PREFIX) &&
4519 ((ctxt->eflags & EFLG_ZF) == EFLG_ZF))))
4525 static int flush_pending_x87_faults(struct x86_emulate_ctxt *ctxt)
4529 ctxt->ops->get_fpu(ctxt);
4530 asm volatile("1: fwait \n\t"
4532 ".pushsection .fixup,\"ax\" \n\t"
4534 "movb $1, %[fault] \n\t"
4537 _ASM_EXTABLE(1b, 3b)
4538 : [fault]"+qm"(fault));
4539 ctxt->ops->put_fpu(ctxt);
4541 if (unlikely(fault))
4542 return emulate_exception(ctxt, MF_VECTOR, 0, false);
4544 return X86EMUL_CONTINUE;
4547 static void fetch_possible_mmx_operand(struct x86_emulate_ctxt *ctxt,
4550 if (op->type == OP_MM)
4551 read_mmx_reg(ctxt, &op->mm_val, op->addr.mm);
4554 static int fastop(struct x86_emulate_ctxt *ctxt, void (*fop)(struct fastop *))
4556 ulong flags = (ctxt->eflags & EFLAGS_MASK) | X86_EFLAGS_IF;
4557 fop += __ffs(ctxt->dst.bytes) * FASTOP_SIZE;
4558 asm("push %[flags]; popf; call *%[fastop]; pushf; pop %[flags]\n"
4559 : "+a"(ctxt->dst.val), "+b"(ctxt->src.val), [flags]"+D"(flags)
4560 : "c"(ctxt->src2.val), [fastop]"S"(fop));
4561 ctxt->eflags = (ctxt->eflags & ~EFLAGS_MASK) | (flags & EFLAGS_MASK);
4562 return X86EMUL_CONTINUE;
4565 int x86_emulate_insn(struct x86_emulate_ctxt *ctxt)
4567 const struct x86_emulate_ops *ops = ctxt->ops;
4568 int rc = X86EMUL_CONTINUE;
4569 int saved_dst_type = ctxt->dst.type;
4571 ctxt->mem_read.pos = 0;
4573 if (ctxt->mode == X86EMUL_MODE_PROT64 && (ctxt->d & No64)) {
4574 rc = emulate_ud(ctxt);
4578 /* LOCK prefix is allowed only with some instructions */
4579 if (ctxt->lock_prefix && (!(ctxt->d & Lock) || ctxt->dst.type != OP_MEM)) {
4580 rc = emulate_ud(ctxt);
4584 if ((ctxt->d & SrcMask) == SrcMemFAddr && ctxt->src.type != OP_MEM) {
4585 rc = emulate_ud(ctxt);
4589 if (((ctxt->d & (Sse|Mmx)) && ((ops->get_cr(ctxt, 0) & X86_CR0_EM)))
4590 || ((ctxt->d & Sse) && !(ops->get_cr(ctxt, 4) & X86_CR4_OSFXSR))) {
4591 rc = emulate_ud(ctxt);
4595 if ((ctxt->d & (Sse|Mmx)) && (ops->get_cr(ctxt, 0) & X86_CR0_TS)) {
4596 rc = emulate_nm(ctxt);
4600 if (ctxt->d & Mmx) {
4601 rc = flush_pending_x87_faults(ctxt);
4602 if (rc != X86EMUL_CONTINUE)
4605 * Now that we know the fpu is exception safe, we can fetch
4608 fetch_possible_mmx_operand(ctxt, &ctxt->src);
4609 fetch_possible_mmx_operand(ctxt, &ctxt->src2);
4610 if (!(ctxt->d & Mov))
4611 fetch_possible_mmx_operand(ctxt, &ctxt->dst);
4614 if (unlikely(ctxt->guest_mode) && ctxt->intercept) {
4615 rc = emulator_check_intercept(ctxt, ctxt->intercept,
4616 X86_ICPT_PRE_EXCEPT);
4617 if (rc != X86EMUL_CONTINUE)
4621 /* Privileged instruction can be executed only in CPL=0 */
4622 if ((ctxt->d & Priv) && ops->cpl(ctxt)) {
4623 rc = emulate_gp(ctxt, 0);
4627 /* Instruction can only be executed in protected mode */
4628 if ((ctxt->d & Prot) && ctxt->mode < X86EMUL_MODE_PROT16) {
4629 rc = emulate_ud(ctxt);
4633 /* Do instruction specific permission checks */
4634 if (ctxt->check_perm) {
4635 rc = ctxt->check_perm(ctxt);
4636 if (rc != X86EMUL_CONTINUE)
4640 if (unlikely(ctxt->guest_mode) && ctxt->intercept) {
4641 rc = emulator_check_intercept(ctxt, ctxt->intercept,
4642 X86_ICPT_POST_EXCEPT);
4643 if (rc != X86EMUL_CONTINUE)
4647 if (ctxt->rep_prefix && (ctxt->d & String)) {
4648 /* All REP prefixes have the same first termination condition */
4649 if (address_mask(ctxt, reg_read(ctxt, VCPU_REGS_RCX)) == 0) {
4650 ctxt->eip = ctxt->_eip;
4655 if ((ctxt->src.type == OP_MEM) && !(ctxt->d & NoAccess)) {
4656 rc = segmented_read(ctxt, ctxt->src.addr.mem,
4657 ctxt->src.valptr, ctxt->src.bytes);
4658 if (rc != X86EMUL_CONTINUE)
4660 ctxt->src.orig_val64 = ctxt->src.val64;
4663 if (ctxt->src2.type == OP_MEM) {
4664 rc = segmented_read(ctxt, ctxt->src2.addr.mem,
4665 &ctxt->src2.val, ctxt->src2.bytes);
4666 if (rc != X86EMUL_CONTINUE)
4670 if ((ctxt->d & DstMask) == ImplicitOps)
4674 if ((ctxt->dst.type == OP_MEM) && !(ctxt->d & Mov)) {
4675 /* optimisation - avoid slow emulated read if Mov */
4676 rc = segmented_read(ctxt, ctxt->dst.addr.mem,
4677 &ctxt->dst.val, ctxt->dst.bytes);
4678 if (rc != X86EMUL_CONTINUE)
4681 ctxt->dst.orig_val = ctxt->dst.val;
4685 if (unlikely(ctxt->guest_mode) && ctxt->intercept) {
4686 rc = emulator_check_intercept(ctxt, ctxt->intercept,
4687 X86_ICPT_POST_MEMACCESS);
4688 if (rc != X86EMUL_CONTINUE)
4692 if (ctxt->execute) {
4693 if (ctxt->d & Fastop) {
4694 void (*fop)(struct fastop *) = (void *)ctxt->execute;
4695 rc = fastop(ctxt, fop);
4696 if (rc != X86EMUL_CONTINUE)
4700 rc = ctxt->execute(ctxt);
4701 if (rc != X86EMUL_CONTINUE)
4710 case 0x40 ... 0x47: /* inc r16/r32 */
4711 emulate_1op(ctxt, "inc");
4713 case 0x48 ... 0x4f: /* dec r16/r32 */
4714 emulate_1op(ctxt, "dec");
4716 case 0x63: /* movsxd */
4717 if (ctxt->mode != X86EMUL_MODE_PROT64)
4718 goto cannot_emulate;
4719 ctxt->dst.val = (s32) ctxt->src.val;
4721 case 0x70 ... 0x7f: /* jcc (short) */
4722 if (test_cc(ctxt->b, ctxt->eflags))
4723 jmp_rel(ctxt, ctxt->src.val);
4725 case 0x8d: /* lea r16/r32, m */
4726 ctxt->dst.val = ctxt->src.addr.mem.ea;
4728 case 0x90 ... 0x97: /* nop / xchg reg, rax */
4729 if (ctxt->dst.addr.reg == reg_rmw(ctxt, VCPU_REGS_RAX))
4733 case 0x98: /* cbw/cwde/cdqe */
4734 switch (ctxt->op_bytes) {
4735 case 2: ctxt->dst.val = (s8)ctxt->dst.val; break;
4736 case 4: ctxt->dst.val = (s16)ctxt->dst.val; break;
4737 case 8: ctxt->dst.val = (s32)ctxt->dst.val; break;
4743 case 0xcc: /* int3 */
4744 rc = emulate_int(ctxt, 3);
4746 case 0xcd: /* int n */
4747 rc = emulate_int(ctxt, ctxt->src.val);
4749 case 0xce: /* into */
4750 if (ctxt->eflags & EFLG_OF)
4751 rc = emulate_int(ctxt, 4);
4753 case 0xd0 ... 0xd1: /* Grp2 */
4756 case 0xd2 ... 0xd3: /* Grp2 */
4757 ctxt->src.val = reg_read(ctxt, VCPU_REGS_RCX);
4760 case 0xe9: /* jmp rel */
4761 case 0xeb: /* jmp rel short */
4762 jmp_rel(ctxt, ctxt->src.val);
4763 ctxt->dst.type = OP_NONE; /* Disable writeback. */
4765 case 0xf4: /* hlt */
4766 ctxt->ops->halt(ctxt);
4768 case 0xf5: /* cmc */
4769 /* complement carry flag from eflags reg */
4770 ctxt->eflags ^= EFLG_CF;
4772 case 0xf8: /* clc */
4773 ctxt->eflags &= ~EFLG_CF;
4775 case 0xf9: /* stc */
4776 ctxt->eflags |= EFLG_CF;
4778 case 0xfc: /* cld */
4779 ctxt->eflags &= ~EFLG_DF;
4781 case 0xfd: /* std */
4782 ctxt->eflags |= EFLG_DF;
4785 goto cannot_emulate;
4788 if (rc != X86EMUL_CONTINUE)
4792 rc = writeback(ctxt);
4793 if (rc != X86EMUL_CONTINUE)
4797 * restore dst type in case the decoding will be reused
4798 * (happens for string instruction )
4800 ctxt->dst.type = saved_dst_type;
4802 if ((ctxt->d & SrcMask) == SrcSI)
4803 string_addr_inc(ctxt, VCPU_REGS_RSI, &ctxt->src);
4805 if ((ctxt->d & DstMask) == DstDI)
4806 string_addr_inc(ctxt, VCPU_REGS_RDI, &ctxt->dst);
4808 if (ctxt->rep_prefix && (ctxt->d & String)) {
4810 struct read_cache *r = &ctxt->io_read;
4811 if ((ctxt->d & SrcMask) == SrcSI)
4812 count = ctxt->src.count;
4814 count = ctxt->dst.count;
4815 register_address_increment(ctxt, reg_rmw(ctxt, VCPU_REGS_RCX),
4818 if (!string_insn_completed(ctxt)) {
4820 * Re-enter guest when pio read ahead buffer is empty
4821 * or, if it is not used, after each 1024 iteration.
4823 if ((r->end != 0 || reg_read(ctxt, VCPU_REGS_RCX) & 0x3ff) &&
4824 (r->end == 0 || r->end != r->pos)) {
4826 * Reset read cache. Usually happens before
4827 * decode, but since instruction is restarted
4828 * we have to do it here.
4830 ctxt->mem_read.end = 0;
4831 writeback_registers(ctxt);
4832 return EMULATION_RESTART;
4834 goto done; /* skip rip writeback */
4838 ctxt->eip = ctxt->_eip;
4841 if (rc == X86EMUL_PROPAGATE_FAULT)
4842 ctxt->have_exception = true;
4843 if (rc == X86EMUL_INTERCEPTED)
4844 return EMULATION_INTERCEPTED;
4846 if (rc == X86EMUL_CONTINUE)
4847 writeback_registers(ctxt);
4849 return (rc == X86EMUL_UNHANDLEABLE) ? EMULATION_FAILED : EMULATION_OK;
4853 case 0x09: /* wbinvd */
4854 (ctxt->ops->wbinvd)(ctxt);
4856 case 0x08: /* invd */
4857 case 0x0d: /* GrpP (prefetch) */
4858 case 0x18: /* Grp16 (prefetch/nop) */
4860 case 0x20: /* mov cr, reg */
4861 ctxt->dst.val = ops->get_cr(ctxt, ctxt->modrm_reg);
4863 case 0x21: /* mov from dr to reg */
4864 ops->get_dr(ctxt, ctxt->modrm_reg, &ctxt->dst.val);
4866 case 0x40 ... 0x4f: /* cmov */
4867 ctxt->dst.val = ctxt->dst.orig_val = ctxt->src.val;
4868 if (!test_cc(ctxt->b, ctxt->eflags))
4869 ctxt->dst.type = OP_NONE; /* no writeback */
4871 case 0x80 ... 0x8f: /* jnz rel, etc*/
4872 if (test_cc(ctxt->b, ctxt->eflags))
4873 jmp_rel(ctxt, ctxt->src.val);
4875 case 0x90 ... 0x9f: /* setcc r/m8 */
4876 ctxt->dst.val = test_cc(ctxt->b, ctxt->eflags);
4878 case 0xa4: /* shld imm8, r, r/m */
4879 case 0xa5: /* shld cl, r, r/m */
4880 emulate_2op_cl(ctxt, "shld");
4882 case 0xac: /* shrd imm8, r, r/m */
4883 case 0xad: /* shrd cl, r, r/m */
4884 emulate_2op_cl(ctxt, "shrd");
4886 case 0xae: /* clflush */
4888 case 0xb6 ... 0xb7: /* movzx */
4889 ctxt->dst.bytes = ctxt->op_bytes;
4890 ctxt->dst.val = (ctxt->src.bytes == 1) ? (u8) ctxt->src.val
4891 : (u16) ctxt->src.val;
4893 case 0xbe ... 0xbf: /* movsx */
4894 ctxt->dst.bytes = ctxt->op_bytes;
4895 ctxt->dst.val = (ctxt->src.bytes == 1) ? (s8) ctxt->src.val :
4896 (s16) ctxt->src.val;
4898 case 0xc0 ... 0xc1: /* xadd */
4899 emulate_2op_SrcV(ctxt, "add");
4900 /* Write back the register source. */
4901 ctxt->src.val = ctxt->dst.orig_val;
4902 write_register_operand(&ctxt->src);
4904 case 0xc3: /* movnti */
4905 ctxt->dst.bytes = ctxt->op_bytes;
4906 ctxt->dst.val = (ctxt->op_bytes == 4) ? (u32) ctxt->src.val :
4907 (u64) ctxt->src.val;
4910 goto cannot_emulate;
4913 if (rc != X86EMUL_CONTINUE)
4919 return EMULATION_FAILED;
4922 void emulator_invalidate_register_cache(struct x86_emulate_ctxt *ctxt)
4924 invalidate_registers(ctxt);
4927 void emulator_writeback_register_cache(struct x86_emulate_ctxt *ctxt)
4929 writeback_registers(ctxt);