1 /******************************************************************************
4 * Generic x86 (32-bit and 64-bit) instruction decoder and emulator.
6 * Copyright (c) 2005 Keir Fraser
8 * Linux coding style, mod r/m decoder, segment base fixes, real-mode
9 * privileged instructions:
11 * Copyright (C) 2006 Qumranet
12 * Copyright 2010 Red Hat, Inc. and/or its affiliates.
14 * Avi Kivity <avi@qumranet.com>
15 * Yaniv Kamay <yaniv@qumranet.com>
17 * This work is licensed under the terms of the GNU GPL, version 2. See
18 * the COPYING file in the top-level directory.
20 * From: xen-unstable 10676:af9809f51f81a3c43f276f00c81a52ef558afda4
23 #include <linux/kvm_host.h>
24 #include "kvm_cache_regs.h"
25 #include <linux/module.h>
26 #include <asm/kvm_emulate.h>
27 #include <linux/stringify.h>
36 #define OpImplicit 1ull /* No generic decode */
37 #define OpReg 2ull /* Register */
38 #define OpMem 3ull /* Memory */
39 #define OpAcc 4ull /* Accumulator: AL/AX/EAX/RAX */
40 #define OpDI 5ull /* ES:DI/EDI/RDI */
41 #define OpMem64 6ull /* Memory, 64-bit */
42 #define OpImmUByte 7ull /* Zero-extended 8-bit immediate */
43 #define OpDX 8ull /* DX register */
44 #define OpCL 9ull /* CL register (for shifts) */
45 #define OpImmByte 10ull /* 8-bit sign extended immediate */
46 #define OpOne 11ull /* Implied 1 */
47 #define OpImm 12ull /* Sign extended up to 32-bit immediate */
48 #define OpMem16 13ull /* Memory operand (16-bit). */
49 #define OpMem32 14ull /* Memory operand (32-bit). */
50 #define OpImmU 15ull /* Immediate operand, zero extended */
51 #define OpSI 16ull /* SI/ESI/RSI */
52 #define OpImmFAddr 17ull /* Immediate far address */
53 #define OpMemFAddr 18ull /* Far address in memory */
54 #define OpImmU16 19ull /* Immediate operand, 16 bits, zero extended */
55 #define OpES 20ull /* ES */
56 #define OpCS 21ull /* CS */
57 #define OpSS 22ull /* SS */
58 #define OpDS 23ull /* DS */
59 #define OpFS 24ull /* FS */
60 #define OpGS 25ull /* GS */
61 #define OpMem8 26ull /* 8-bit zero extended memory operand */
62 #define OpImm64 27ull /* Sign extended 16/32/64-bit immediate */
63 #define OpXLat 28ull /* memory at BX/EBX/RBX + zero-extended AL */
64 #define OpAccLo 29ull /* Low part of extended acc (AX/AX/EAX/RAX) */
65 #define OpAccHi 30ull /* High part of extended acc (-/DX/EDX/RDX) */
67 #define OpBits 5 /* Width of operand field */
68 #define OpMask ((1ull << OpBits) - 1)
71 * Opcode effective-address decode tables.
72 * Note that we only emulate instructions that have at least one memory
73 * operand (excluding implicit stack references). We assume that stack
74 * references and instruction fetches will never occur in special memory
75 * areas that require emulation. So, for example, 'mov <imm>,<reg>' need
79 /* Operand sizes: 8-bit operands or specified/overridden size. */
80 #define ByteOp (1<<0) /* 8-bit operands. */
81 /* Destination operand type. */
83 #define ImplicitOps (OpImplicit << DstShift)
84 #define DstReg (OpReg << DstShift)
85 #define DstMem (OpMem << DstShift)
86 #define DstAcc (OpAcc << DstShift)
87 #define DstDI (OpDI << DstShift)
88 #define DstMem64 (OpMem64 << DstShift)
89 #define DstMem16 (OpMem16 << DstShift)
90 #define DstImmUByte (OpImmUByte << DstShift)
91 #define DstDX (OpDX << DstShift)
92 #define DstAccLo (OpAccLo << DstShift)
93 #define DstMask (OpMask << DstShift)
94 /* Source operand type. */
96 #define SrcNone (OpNone << SrcShift)
97 #define SrcReg (OpReg << SrcShift)
98 #define SrcMem (OpMem << SrcShift)
99 #define SrcMem16 (OpMem16 << SrcShift)
100 #define SrcMem32 (OpMem32 << SrcShift)
101 #define SrcImm (OpImm << SrcShift)
102 #define SrcImmByte (OpImmByte << SrcShift)
103 #define SrcOne (OpOne << SrcShift)
104 #define SrcImmUByte (OpImmUByte << SrcShift)
105 #define SrcImmU (OpImmU << SrcShift)
106 #define SrcSI (OpSI << SrcShift)
107 #define SrcXLat (OpXLat << SrcShift)
108 #define SrcImmFAddr (OpImmFAddr << SrcShift)
109 #define SrcMemFAddr (OpMemFAddr << SrcShift)
110 #define SrcAcc (OpAcc << SrcShift)
111 #define SrcImmU16 (OpImmU16 << SrcShift)
112 #define SrcImm64 (OpImm64 << SrcShift)
113 #define SrcDX (OpDX << SrcShift)
114 #define SrcMem8 (OpMem8 << SrcShift)
115 #define SrcAccHi (OpAccHi << SrcShift)
116 #define SrcMask (OpMask << SrcShift)
117 #define BitOp (1<<11)
118 #define MemAbs (1<<12) /* Memory operand is absolute displacement */
119 #define String (1<<13) /* String instruction (rep capable) */
120 #define Stack (1<<14) /* Stack instruction (push/pop) */
121 #define GroupMask (7<<15) /* Opcode uses one of the group mechanisms */
122 #define Group (1<<15) /* Bits 3:5 of modrm byte extend opcode */
123 #define GroupDual (2<<15) /* Alternate decoding of mod == 3 */
124 #define Prefix (3<<15) /* Instruction varies with 66/f2/f3 prefix */
125 #define RMExt (4<<15) /* Opcode extension in ModRM r/m if mod == 3 */
126 #define Escape (5<<15) /* Escape to coprocessor instruction */
127 #define InstrDual (6<<15) /* Alternate instruction decoding of mod == 3 */
128 #define ModeDual (7<<15) /* Different instruction for 32/64 bit */
129 #define Sse (1<<18) /* SSE Vector instruction */
130 /* Generic ModRM decode. */
131 #define ModRM (1<<19)
132 /* Destination is only written; never read. */
135 #define Prot (1<<21) /* instruction generates #UD if not in prot-mode */
136 #define EmulateOnUD (1<<22) /* Emulate if unsupported by the host */
137 #define NoAccess (1<<23) /* Don't access memory (lea/invlpg/verr etc) */
138 #define Op3264 (1<<24) /* Operand is 64b in long mode, 32b otherwise */
139 #define Undefined (1<<25) /* No Such Instruction */
140 #define Lock (1<<26) /* lock prefix is allowed for the instruction */
141 #define Priv (1<<27) /* instruction generates #GP if current CPL != 0 */
143 #define PageTable (1 << 29) /* instruction used to write page table */
144 #define NotImpl (1 << 30) /* instruction is not implemented */
145 /* Source 2 operand type */
146 #define Src2Shift (31)
147 #define Src2None (OpNone << Src2Shift)
148 #define Src2Mem (OpMem << Src2Shift)
149 #define Src2CL (OpCL << Src2Shift)
150 #define Src2ImmByte (OpImmByte << Src2Shift)
151 #define Src2One (OpOne << Src2Shift)
152 #define Src2Imm (OpImm << Src2Shift)
153 #define Src2ES (OpES << Src2Shift)
154 #define Src2CS (OpCS << Src2Shift)
155 #define Src2SS (OpSS << Src2Shift)
156 #define Src2DS (OpDS << Src2Shift)
157 #define Src2FS (OpFS << Src2Shift)
158 #define Src2GS (OpGS << Src2Shift)
159 #define Src2Mask (OpMask << Src2Shift)
160 #define Mmx ((u64)1 << 40) /* MMX Vector instruction */
161 #define Aligned ((u64)1 << 41) /* Explicitly aligned (e.g. MOVDQA) */
162 #define Unaligned ((u64)1 << 42) /* Explicitly unaligned (e.g. MOVDQU) */
163 #define Avx ((u64)1 << 43) /* Advanced Vector Extensions */
164 #define Fastop ((u64)1 << 44) /* Use opcode::u.fastop */
165 #define NoWrite ((u64)1 << 45) /* No writeback */
166 #define SrcWrite ((u64)1 << 46) /* Write back src operand */
167 #define NoMod ((u64)1 << 47) /* Mod field is ignored */
168 #define Intercept ((u64)1 << 48) /* Has valid intercept field */
169 #define CheckPerm ((u64)1 << 49) /* Has valid check_perm field */
170 #define PrivUD ((u64)1 << 51) /* #UD instead of #GP on CPL > 0 */
171 #define NearBranch ((u64)1 << 52) /* Near branches */
172 #define No16 ((u64)1 << 53) /* No 16 bit operand */
173 #define IncSP ((u64)1 << 54) /* SP is incremented before ModRM calc */
175 #define DstXacc (DstAccLo | SrcAccHi | SrcWrite)
177 #define X2(x...) x, x
178 #define X3(x...) X2(x), x
179 #define X4(x...) X2(x), X2(x)
180 #define X5(x...) X4(x), x
181 #define X6(x...) X4(x), X2(x)
182 #define X7(x...) X4(x), X3(x)
183 #define X8(x...) X4(x), X4(x)
184 #define X16(x...) X8(x), X8(x)
186 #define NR_FASTOP (ilog2(sizeof(ulong)) + 1)
187 #define FASTOP_SIZE 8
190 * fastop functions have a special calling convention:
195 * flags: rflags (in/out)
196 * ex: rsi (in:fastop pointer, out:zero if exception)
198 * Moreover, they are all exactly FASTOP_SIZE bytes long, so functions for
199 * different operand sizes can be reached by calculation, rather than a jump
200 * table (which would be bigger than the code).
202 * fastop functions are declared as taking a never-defined fastop parameter,
203 * so they can't be called from C directly.
212 int (*execute)(struct x86_emulate_ctxt *ctxt);
213 const struct opcode *group;
214 const struct group_dual *gdual;
215 const struct gprefix *gprefix;
216 const struct escape *esc;
217 const struct instr_dual *idual;
218 const struct mode_dual *mdual;
219 void (*fastop)(struct fastop *fake);
221 int (*check_perm)(struct x86_emulate_ctxt *ctxt);
225 struct opcode mod012[8];
226 struct opcode mod3[8];
230 struct opcode pfx_no;
231 struct opcode pfx_66;
232 struct opcode pfx_f2;
233 struct opcode pfx_f3;
238 struct opcode high[64];
242 struct opcode mod012;
247 struct opcode mode32;
248 struct opcode mode64;
251 /* EFLAGS bit definitions. */
252 #define EFLG_ID (1<<21)
253 #define EFLG_VIP (1<<20)
254 #define EFLG_VIF (1<<19)
255 #define EFLG_AC (1<<18)
256 #define EFLG_VM (1<<17)
257 #define EFLG_RF (1<<16)
258 #define EFLG_IOPL (3<<12)
259 #define EFLG_NT (1<<14)
260 #define EFLG_OF (1<<11)
261 #define EFLG_DF (1<<10)
262 #define EFLG_IF (1<<9)
263 #define EFLG_TF (1<<8)
264 #define EFLG_SF (1<<7)
265 #define EFLG_ZF (1<<6)
266 #define EFLG_AF (1<<4)
267 #define EFLG_PF (1<<2)
268 #define EFLG_CF (1<<0)
270 #define EFLG_RESERVED_ZEROS_MASK 0xffc0802a
271 #define EFLG_RESERVED_ONE_MASK 2
273 enum x86_transfer_type {
275 X86_TRANSFER_CALL_JMP,
277 X86_TRANSFER_TASK_SWITCH,
280 static ulong reg_read(struct x86_emulate_ctxt *ctxt, unsigned nr)
282 if (!(ctxt->regs_valid & (1 << nr))) {
283 ctxt->regs_valid |= 1 << nr;
284 ctxt->_regs[nr] = ctxt->ops->read_gpr(ctxt, nr);
286 return ctxt->_regs[nr];
289 static ulong *reg_write(struct x86_emulate_ctxt *ctxt, unsigned nr)
291 ctxt->regs_valid |= 1 << nr;
292 ctxt->regs_dirty |= 1 << nr;
293 return &ctxt->_regs[nr];
296 static ulong *reg_rmw(struct x86_emulate_ctxt *ctxt, unsigned nr)
299 return reg_write(ctxt, nr);
302 static void writeback_registers(struct x86_emulate_ctxt *ctxt)
306 for_each_set_bit(reg, (ulong *)&ctxt->regs_dirty, 16)
307 ctxt->ops->write_gpr(ctxt, reg, ctxt->_regs[reg]);
310 static void invalidate_registers(struct x86_emulate_ctxt *ctxt)
312 ctxt->regs_dirty = 0;
313 ctxt->regs_valid = 0;
317 * These EFLAGS bits are restored from saved value during emulation, and
318 * any changes are written back to the saved value after emulation.
320 #define EFLAGS_MASK (EFLG_OF|EFLG_SF|EFLG_ZF|EFLG_AF|EFLG_PF|EFLG_CF)
328 static int fastop(struct x86_emulate_ctxt *ctxt, void (*fop)(struct fastop *));
330 #define FOP_ALIGN ".align " __stringify(FASTOP_SIZE) " \n\t"
331 #define FOP_RET "ret \n\t"
333 #define FOP_START(op) \
334 extern void em_##op(struct fastop *fake); \
335 asm(".pushsection .text, \"ax\" \n\t" \
336 ".global em_" #op " \n\t" \
343 #define FOPNOP() FOP_ALIGN FOP_RET
345 #define FOP1E(op, dst) \
346 FOP_ALIGN "10: " #op " %" #dst " \n\t" FOP_RET
348 #define FOP1EEX(op, dst) \
349 FOP1E(op, dst) _ASM_EXTABLE(10b, kvm_fastop_exception)
351 #define FASTOP1(op) \
356 ON64(FOP1E(op##q, rax)) \
359 /* 1-operand, using src2 (for MUL/DIV r/m) */
360 #define FASTOP1SRC2(op, name) \
365 ON64(FOP1E(op, rcx)) \
368 /* 1-operand, using src2 (for MUL/DIV r/m), with exceptions */
369 #define FASTOP1SRC2EX(op, name) \
374 ON64(FOP1EEX(op, rcx)) \
377 #define FOP2E(op, dst, src) \
378 FOP_ALIGN #op " %" #src ", %" #dst " \n\t" FOP_RET
380 #define FASTOP2(op) \
382 FOP2E(op##b, al, dl) \
383 FOP2E(op##w, ax, dx) \
384 FOP2E(op##l, eax, edx) \
385 ON64(FOP2E(op##q, rax, rdx)) \
388 /* 2 operand, word only */
389 #define FASTOP2W(op) \
392 FOP2E(op##w, ax, dx) \
393 FOP2E(op##l, eax, edx) \
394 ON64(FOP2E(op##q, rax, rdx)) \
397 /* 2 operand, src is CL */
398 #define FASTOP2CL(op) \
400 FOP2E(op##b, al, cl) \
401 FOP2E(op##w, ax, cl) \
402 FOP2E(op##l, eax, cl) \
403 ON64(FOP2E(op##q, rax, cl)) \
406 /* 2 operand, src and dest are reversed */
407 #define FASTOP2R(op, name) \
409 FOP2E(op##b, dl, al) \
410 FOP2E(op##w, dx, ax) \
411 FOP2E(op##l, edx, eax) \
412 ON64(FOP2E(op##q, rdx, rax)) \
415 #define FOP3E(op, dst, src, src2) \
416 FOP_ALIGN #op " %" #src2 ", %" #src ", %" #dst " \n\t" FOP_RET
418 /* 3-operand, word-only, src2=cl */
419 #define FASTOP3WCL(op) \
422 FOP3E(op##w, ax, dx, cl) \
423 FOP3E(op##l, eax, edx, cl) \
424 ON64(FOP3E(op##q, rax, rdx, cl)) \
427 /* Special case for SETcc - 1 instruction per cc */
428 #define FOP_SETCC(op) ".align 4; " #op " %al; ret \n\t"
430 asm(".global kvm_fastop_exception \n"
431 "kvm_fastop_exception: xor %esi, %esi; ret");
452 FOP_START(salc) "pushf; sbb %al, %al; popf \n\t" FOP_RET
455 static int emulator_check_intercept(struct x86_emulate_ctxt *ctxt,
456 enum x86_intercept intercept,
457 enum x86_intercept_stage stage)
459 struct x86_instruction_info info = {
460 .intercept = intercept,
461 .rep_prefix = ctxt->rep_prefix,
462 .modrm_mod = ctxt->modrm_mod,
463 .modrm_reg = ctxt->modrm_reg,
464 .modrm_rm = ctxt->modrm_rm,
465 .src_val = ctxt->src.val64,
466 .dst_val = ctxt->dst.val64,
467 .src_bytes = ctxt->src.bytes,
468 .dst_bytes = ctxt->dst.bytes,
469 .ad_bytes = ctxt->ad_bytes,
470 .next_rip = ctxt->eip,
473 return ctxt->ops->intercept(ctxt, &info, stage);
476 static void assign_masked(ulong *dest, ulong src, ulong mask)
478 *dest = (*dest & ~mask) | (src & mask);
481 static void assign_register(unsigned long *reg, u64 val, int bytes)
483 /* The 4-byte case *is* correct: in 64-bit mode we zero-extend. */
486 *(u8 *)reg = (u8)val;
489 *(u16 *)reg = (u16)val;
493 break; /* 64b: zero-extend */
500 static inline unsigned long ad_mask(struct x86_emulate_ctxt *ctxt)
502 return (1UL << (ctxt->ad_bytes << 3)) - 1;
505 static ulong stack_mask(struct x86_emulate_ctxt *ctxt)
508 struct desc_struct ss;
510 if (ctxt->mode == X86EMUL_MODE_PROT64)
512 ctxt->ops->get_segment(ctxt, &sel, &ss, NULL, VCPU_SREG_SS);
513 return ~0U >> ((ss.d ^ 1) * 16); /* d=0: 0xffff; d=1: 0xffffffff */
516 static int stack_size(struct x86_emulate_ctxt *ctxt)
518 return (__fls(stack_mask(ctxt)) + 1) >> 3;
521 /* Access/update address held in a register, based on addressing mode. */
522 static inline unsigned long
523 address_mask(struct x86_emulate_ctxt *ctxt, unsigned long reg)
525 if (ctxt->ad_bytes == sizeof(unsigned long))
528 return reg & ad_mask(ctxt);
531 static inline unsigned long
532 register_address(struct x86_emulate_ctxt *ctxt, int reg)
534 return address_mask(ctxt, reg_read(ctxt, reg));
537 static void masked_increment(ulong *reg, ulong mask, int inc)
539 assign_masked(reg, *reg + inc, mask);
543 register_address_increment(struct x86_emulate_ctxt *ctxt, int reg, int inc)
547 if (ctxt->ad_bytes == sizeof(unsigned long))
550 mask = ad_mask(ctxt);
551 masked_increment(reg_rmw(ctxt, reg), mask, inc);
554 static void rsp_increment(struct x86_emulate_ctxt *ctxt, int inc)
556 masked_increment(reg_rmw(ctxt, VCPU_REGS_RSP), stack_mask(ctxt), inc);
559 static u32 desc_limit_scaled(struct desc_struct *desc)
561 u32 limit = get_desc_limit(desc);
563 return desc->g ? (limit << 12) | 0xfff : limit;
566 static unsigned long seg_base(struct x86_emulate_ctxt *ctxt, int seg)
568 if (ctxt->mode == X86EMUL_MODE_PROT64 && seg < VCPU_SREG_FS)
571 return ctxt->ops->get_cached_segment_base(ctxt, seg);
574 static int emulate_exception(struct x86_emulate_ctxt *ctxt, int vec,
575 u32 error, bool valid)
578 ctxt->exception.vector = vec;
579 ctxt->exception.error_code = error;
580 ctxt->exception.error_code_valid = valid;
581 return X86EMUL_PROPAGATE_FAULT;
584 static int emulate_db(struct x86_emulate_ctxt *ctxt)
586 return emulate_exception(ctxt, DB_VECTOR, 0, false);
589 static int emulate_gp(struct x86_emulate_ctxt *ctxt, int err)
591 return emulate_exception(ctxt, GP_VECTOR, err, true);
594 static int emulate_ss(struct x86_emulate_ctxt *ctxt, int err)
596 return emulate_exception(ctxt, SS_VECTOR, err, true);
599 static int emulate_ud(struct x86_emulate_ctxt *ctxt)
601 return emulate_exception(ctxt, UD_VECTOR, 0, false);
604 static int emulate_ts(struct x86_emulate_ctxt *ctxt, int err)
606 return emulate_exception(ctxt, TS_VECTOR, err, true);
609 static int emulate_de(struct x86_emulate_ctxt *ctxt)
611 return emulate_exception(ctxt, DE_VECTOR, 0, false);
614 static int emulate_nm(struct x86_emulate_ctxt *ctxt)
616 return emulate_exception(ctxt, NM_VECTOR, 0, false);
619 static u16 get_segment_selector(struct x86_emulate_ctxt *ctxt, unsigned seg)
622 struct desc_struct desc;
624 ctxt->ops->get_segment(ctxt, &selector, &desc, NULL, seg);
628 static void set_segment_selector(struct x86_emulate_ctxt *ctxt, u16 selector,
633 struct desc_struct desc;
635 ctxt->ops->get_segment(ctxt, &dummy, &desc, &base3, seg);
636 ctxt->ops->set_segment(ctxt, selector, &desc, base3, seg);
640 * x86 defines three classes of vector instructions: explicitly
641 * aligned, explicitly unaligned, and the rest, which change behaviour
642 * depending on whether they're AVX encoded or not.
644 * Also included is CMPXCHG16B which is not a vector instruction, yet it is
645 * subject to the same check.
647 static bool insn_aligned(struct x86_emulate_ctxt *ctxt, unsigned size)
649 if (likely(size < 16))
652 if (ctxt->d & Aligned)
654 else if (ctxt->d & Unaligned)
656 else if (ctxt->d & Avx)
662 static __always_inline int __linearize(struct x86_emulate_ctxt *ctxt,
663 struct segmented_address addr,
664 unsigned *max_size, unsigned size,
665 bool write, bool fetch,
666 enum x86emul_mode mode, ulong *linear)
668 struct desc_struct desc;
674 la = seg_base(ctxt, addr.seg) + addr.ea;
677 case X86EMUL_MODE_PROT64:
678 if (is_noncanonical_address(la))
681 *max_size = min_t(u64, ~0u, (1ull << 48) - la);
682 if (size > *max_size)
686 usable = ctxt->ops->get_segment(ctxt, &sel, &desc, NULL,
690 /* code segment in protected mode or read-only data segment */
691 if ((((ctxt->mode != X86EMUL_MODE_REAL) && (desc.type & 8))
692 || !(desc.type & 2)) && write)
694 /* unreadable code segment */
695 if (!fetch && (desc.type & 8) && !(desc.type & 2))
697 lim = desc_limit_scaled(&desc);
698 if (!(desc.type & 8) && (desc.type & 4)) {
699 /* expand-down segment */
702 lim = desc.d ? 0xffffffff : 0xffff;
706 if (lim == 0xffffffff)
709 *max_size = (u64)lim + 1 - addr.ea;
710 if (size > *max_size)
716 if (insn_aligned(ctxt, size) && ((la & (size - 1)) != 0))
717 return emulate_gp(ctxt, 0);
719 return X86EMUL_CONTINUE;
721 if (addr.seg == VCPU_SREG_SS)
722 return emulate_ss(ctxt, 0);
724 return emulate_gp(ctxt, 0);
727 static int linearize(struct x86_emulate_ctxt *ctxt,
728 struct segmented_address addr,
729 unsigned size, bool write,
733 return __linearize(ctxt, addr, &max_size, size, write, false,
737 static inline int assign_eip(struct x86_emulate_ctxt *ctxt, ulong dst,
738 enum x86emul_mode mode)
743 struct segmented_address addr = { .seg = VCPU_SREG_CS,
746 if (ctxt->op_bytes != sizeof(unsigned long))
747 addr.ea = dst & ((1UL << (ctxt->op_bytes << 3)) - 1);
748 rc = __linearize(ctxt, addr, &max_size, 1, false, true, mode, &linear);
749 if (rc == X86EMUL_CONTINUE)
750 ctxt->_eip = addr.ea;
754 static inline int assign_eip_near(struct x86_emulate_ctxt *ctxt, ulong dst)
756 return assign_eip(ctxt, dst, ctxt->mode);
759 static int assign_eip_far(struct x86_emulate_ctxt *ctxt, ulong dst,
760 const struct desc_struct *cs_desc)
762 enum x86emul_mode mode = ctxt->mode;
766 if (ctxt->mode >= X86EMUL_MODE_PROT16) {
770 ctxt->ops->get_msr(ctxt, MSR_EFER, &efer);
772 mode = X86EMUL_MODE_PROT64;
774 mode = X86EMUL_MODE_PROT32; /* temporary value */
777 if (mode == X86EMUL_MODE_PROT16 || mode == X86EMUL_MODE_PROT32)
778 mode = cs_desc->d ? X86EMUL_MODE_PROT32 : X86EMUL_MODE_PROT16;
779 rc = assign_eip(ctxt, dst, mode);
780 if (rc == X86EMUL_CONTINUE)
785 static inline int jmp_rel(struct x86_emulate_ctxt *ctxt, int rel)
787 return assign_eip_near(ctxt, ctxt->_eip + rel);
790 static int segmented_read_std(struct x86_emulate_ctxt *ctxt,
791 struct segmented_address addr,
798 rc = linearize(ctxt, addr, size, false, &linear);
799 if (rc != X86EMUL_CONTINUE)
801 return ctxt->ops->read_std(ctxt, linear, data, size, &ctxt->exception);
805 * Prefetch the remaining bytes of the instruction without crossing page
806 * boundary if they are not in fetch_cache yet.
808 static int __do_insn_fetch_bytes(struct x86_emulate_ctxt *ctxt, int op_size)
811 unsigned size, max_size;
812 unsigned long linear;
813 int cur_size = ctxt->fetch.end - ctxt->fetch.data;
814 struct segmented_address addr = { .seg = VCPU_SREG_CS,
815 .ea = ctxt->eip + cur_size };
818 * We do not know exactly how many bytes will be needed, and
819 * __linearize is expensive, so fetch as much as possible. We
820 * just have to avoid going beyond the 15 byte limit, the end
821 * of the segment, or the end of the page.
823 * __linearize is called with size 0 so that it does not do any
824 * boundary check itself. Instead, we use max_size to check
827 rc = __linearize(ctxt, addr, &max_size, 0, false, true, ctxt->mode,
829 if (unlikely(rc != X86EMUL_CONTINUE))
832 size = min_t(unsigned, 15UL ^ cur_size, max_size);
833 size = min_t(unsigned, size, PAGE_SIZE - offset_in_page(linear));
836 * One instruction can only straddle two pages,
837 * and one has been loaded at the beginning of
838 * x86_decode_insn. So, if not enough bytes
839 * still, we must have hit the 15-byte boundary.
841 if (unlikely(size < op_size))
842 return emulate_gp(ctxt, 0);
844 rc = ctxt->ops->fetch(ctxt, linear, ctxt->fetch.end,
845 size, &ctxt->exception);
846 if (unlikely(rc != X86EMUL_CONTINUE))
848 ctxt->fetch.end += size;
849 return X86EMUL_CONTINUE;
852 static __always_inline int do_insn_fetch_bytes(struct x86_emulate_ctxt *ctxt,
855 unsigned done_size = ctxt->fetch.end - ctxt->fetch.ptr;
857 if (unlikely(done_size < size))
858 return __do_insn_fetch_bytes(ctxt, size - done_size);
860 return X86EMUL_CONTINUE;
863 /* Fetch next part of the instruction being emulated. */
864 #define insn_fetch(_type, _ctxt) \
867 rc = do_insn_fetch_bytes(_ctxt, sizeof(_type)); \
868 if (rc != X86EMUL_CONTINUE) \
870 ctxt->_eip += sizeof(_type); \
871 _x = *(_type __aligned(1) *) ctxt->fetch.ptr; \
872 ctxt->fetch.ptr += sizeof(_type); \
876 #define insn_fetch_arr(_arr, _size, _ctxt) \
878 rc = do_insn_fetch_bytes(_ctxt, _size); \
879 if (rc != X86EMUL_CONTINUE) \
881 ctxt->_eip += (_size); \
882 memcpy(_arr, ctxt->fetch.ptr, _size); \
883 ctxt->fetch.ptr += (_size); \
887 * Given the 'reg' portion of a ModRM byte, and a register block, return a
888 * pointer into the block that addresses the relevant register.
889 * @highbyte_regs specifies whether to decode AH,CH,DH,BH.
891 static void *decode_register(struct x86_emulate_ctxt *ctxt, u8 modrm_reg,
895 int highbyte_regs = (ctxt->rex_prefix == 0) && byteop;
897 if (highbyte_regs && modrm_reg >= 4 && modrm_reg < 8)
898 p = (unsigned char *)reg_rmw(ctxt, modrm_reg & 3) + 1;
900 p = reg_rmw(ctxt, modrm_reg);
904 static int read_descriptor(struct x86_emulate_ctxt *ctxt,
905 struct segmented_address addr,
906 u16 *size, unsigned long *address, int op_bytes)
913 rc = segmented_read_std(ctxt, addr, size, 2);
914 if (rc != X86EMUL_CONTINUE)
917 rc = segmented_read_std(ctxt, addr, address, op_bytes);
931 FASTOP1SRC2(mul, mul_ex);
932 FASTOP1SRC2(imul, imul_ex);
933 FASTOP1SRC2EX(div, div_ex);
934 FASTOP1SRC2EX(idiv, idiv_ex);
963 FASTOP2R(cmp, cmp_r);
965 static int em_bsf_c(struct x86_emulate_ctxt *ctxt)
967 /* If src is zero, do not writeback, but update flags */
968 if (ctxt->src.val == 0)
969 ctxt->dst.type = OP_NONE;
970 return fastop(ctxt, em_bsf);
973 static int em_bsr_c(struct x86_emulate_ctxt *ctxt)
975 /* If src is zero, do not writeback, but update flags */
976 if (ctxt->src.val == 0)
977 ctxt->dst.type = OP_NONE;
978 return fastop(ctxt, em_bsr);
981 static u8 test_cc(unsigned int condition, unsigned long flags)
984 void (*fop)(void) = (void *)em_setcc + 4 * (condition & 0xf);
986 flags = (flags & EFLAGS_MASK) | X86_EFLAGS_IF;
987 asm("push %[flags]; popf; call *%[fastop]"
988 : "=a"(rc) : [fastop]"r"(fop), [flags]"r"(flags));
992 static void fetch_register_operand(struct operand *op)
996 op->val = *(u8 *)op->addr.reg;
999 op->val = *(u16 *)op->addr.reg;
1002 op->val = *(u32 *)op->addr.reg;
1005 op->val = *(u64 *)op->addr.reg;
1010 static void read_sse_reg(struct x86_emulate_ctxt *ctxt, sse128_t *data, int reg)
1012 ctxt->ops->get_fpu(ctxt);
1014 case 0: asm("movdqa %%xmm0, %0" : "=m"(*data)); break;
1015 case 1: asm("movdqa %%xmm1, %0" : "=m"(*data)); break;
1016 case 2: asm("movdqa %%xmm2, %0" : "=m"(*data)); break;
1017 case 3: asm("movdqa %%xmm3, %0" : "=m"(*data)); break;
1018 case 4: asm("movdqa %%xmm4, %0" : "=m"(*data)); break;
1019 case 5: asm("movdqa %%xmm5, %0" : "=m"(*data)); break;
1020 case 6: asm("movdqa %%xmm6, %0" : "=m"(*data)); break;
1021 case 7: asm("movdqa %%xmm7, %0" : "=m"(*data)); break;
1022 #ifdef CONFIG_X86_64
1023 case 8: asm("movdqa %%xmm8, %0" : "=m"(*data)); break;
1024 case 9: asm("movdqa %%xmm9, %0" : "=m"(*data)); break;
1025 case 10: asm("movdqa %%xmm10, %0" : "=m"(*data)); break;
1026 case 11: asm("movdqa %%xmm11, %0" : "=m"(*data)); break;
1027 case 12: asm("movdqa %%xmm12, %0" : "=m"(*data)); break;
1028 case 13: asm("movdqa %%xmm13, %0" : "=m"(*data)); break;
1029 case 14: asm("movdqa %%xmm14, %0" : "=m"(*data)); break;
1030 case 15: asm("movdqa %%xmm15, %0" : "=m"(*data)); break;
1034 ctxt->ops->put_fpu(ctxt);
1037 static void write_sse_reg(struct x86_emulate_ctxt *ctxt, sse128_t *data,
1040 ctxt->ops->get_fpu(ctxt);
1042 case 0: asm("movdqa %0, %%xmm0" : : "m"(*data)); break;
1043 case 1: asm("movdqa %0, %%xmm1" : : "m"(*data)); break;
1044 case 2: asm("movdqa %0, %%xmm2" : : "m"(*data)); break;
1045 case 3: asm("movdqa %0, %%xmm3" : : "m"(*data)); break;
1046 case 4: asm("movdqa %0, %%xmm4" : : "m"(*data)); break;
1047 case 5: asm("movdqa %0, %%xmm5" : : "m"(*data)); break;
1048 case 6: asm("movdqa %0, %%xmm6" : : "m"(*data)); break;
1049 case 7: asm("movdqa %0, %%xmm7" : : "m"(*data)); break;
1050 #ifdef CONFIG_X86_64
1051 case 8: asm("movdqa %0, %%xmm8" : : "m"(*data)); break;
1052 case 9: asm("movdqa %0, %%xmm9" : : "m"(*data)); break;
1053 case 10: asm("movdqa %0, %%xmm10" : : "m"(*data)); break;
1054 case 11: asm("movdqa %0, %%xmm11" : : "m"(*data)); break;
1055 case 12: asm("movdqa %0, %%xmm12" : : "m"(*data)); break;
1056 case 13: asm("movdqa %0, %%xmm13" : : "m"(*data)); break;
1057 case 14: asm("movdqa %0, %%xmm14" : : "m"(*data)); break;
1058 case 15: asm("movdqa %0, %%xmm15" : : "m"(*data)); break;
1062 ctxt->ops->put_fpu(ctxt);
1065 static void read_mmx_reg(struct x86_emulate_ctxt *ctxt, u64 *data, int reg)
1067 ctxt->ops->get_fpu(ctxt);
1069 case 0: asm("movq %%mm0, %0" : "=m"(*data)); break;
1070 case 1: asm("movq %%mm1, %0" : "=m"(*data)); break;
1071 case 2: asm("movq %%mm2, %0" : "=m"(*data)); break;
1072 case 3: asm("movq %%mm3, %0" : "=m"(*data)); break;
1073 case 4: asm("movq %%mm4, %0" : "=m"(*data)); break;
1074 case 5: asm("movq %%mm5, %0" : "=m"(*data)); break;
1075 case 6: asm("movq %%mm6, %0" : "=m"(*data)); break;
1076 case 7: asm("movq %%mm7, %0" : "=m"(*data)); break;
1079 ctxt->ops->put_fpu(ctxt);
1082 static void write_mmx_reg(struct x86_emulate_ctxt *ctxt, u64 *data, int reg)
1084 ctxt->ops->get_fpu(ctxt);
1086 case 0: asm("movq %0, %%mm0" : : "m"(*data)); break;
1087 case 1: asm("movq %0, %%mm1" : : "m"(*data)); break;
1088 case 2: asm("movq %0, %%mm2" : : "m"(*data)); break;
1089 case 3: asm("movq %0, %%mm3" : : "m"(*data)); break;
1090 case 4: asm("movq %0, %%mm4" : : "m"(*data)); break;
1091 case 5: asm("movq %0, %%mm5" : : "m"(*data)); break;
1092 case 6: asm("movq %0, %%mm6" : : "m"(*data)); break;
1093 case 7: asm("movq %0, %%mm7" : : "m"(*data)); break;
1096 ctxt->ops->put_fpu(ctxt);
1099 static int em_fninit(struct x86_emulate_ctxt *ctxt)
1101 if (ctxt->ops->get_cr(ctxt, 0) & (X86_CR0_TS | X86_CR0_EM))
1102 return emulate_nm(ctxt);
1104 ctxt->ops->get_fpu(ctxt);
1105 asm volatile("fninit");
1106 ctxt->ops->put_fpu(ctxt);
1107 return X86EMUL_CONTINUE;
1110 static int em_fnstcw(struct x86_emulate_ctxt *ctxt)
1114 if (ctxt->ops->get_cr(ctxt, 0) & (X86_CR0_TS | X86_CR0_EM))
1115 return emulate_nm(ctxt);
1117 ctxt->ops->get_fpu(ctxt);
1118 asm volatile("fnstcw %0": "+m"(fcw));
1119 ctxt->ops->put_fpu(ctxt);
1121 ctxt->dst.val = fcw;
1123 return X86EMUL_CONTINUE;
1126 static int em_fnstsw(struct x86_emulate_ctxt *ctxt)
1130 if (ctxt->ops->get_cr(ctxt, 0) & (X86_CR0_TS | X86_CR0_EM))
1131 return emulate_nm(ctxt);
1133 ctxt->ops->get_fpu(ctxt);
1134 asm volatile("fnstsw %0": "+m"(fsw));
1135 ctxt->ops->put_fpu(ctxt);
1137 ctxt->dst.val = fsw;
1139 return X86EMUL_CONTINUE;
1142 static void decode_register_operand(struct x86_emulate_ctxt *ctxt,
1145 unsigned reg = ctxt->modrm_reg;
1147 if (!(ctxt->d & ModRM))
1148 reg = (ctxt->b & 7) | ((ctxt->rex_prefix & 1) << 3);
1150 if (ctxt->d & Sse) {
1154 read_sse_reg(ctxt, &op->vec_val, reg);
1157 if (ctxt->d & Mmx) {
1166 op->bytes = (ctxt->d & ByteOp) ? 1 : ctxt->op_bytes;
1167 op->addr.reg = decode_register(ctxt, reg, ctxt->d & ByteOp);
1169 fetch_register_operand(op);
1170 op->orig_val = op->val;
1173 static void adjust_modrm_seg(struct x86_emulate_ctxt *ctxt, int base_reg)
1175 if (base_reg == VCPU_REGS_RSP || base_reg == VCPU_REGS_RBP)
1176 ctxt->modrm_seg = VCPU_SREG_SS;
1179 static int decode_modrm(struct x86_emulate_ctxt *ctxt,
1183 int index_reg, base_reg, scale;
1184 int rc = X86EMUL_CONTINUE;
1187 ctxt->modrm_reg = ((ctxt->rex_prefix << 1) & 8); /* REX.R */
1188 index_reg = (ctxt->rex_prefix << 2) & 8; /* REX.X */
1189 base_reg = (ctxt->rex_prefix << 3) & 8; /* REX.B */
1191 ctxt->modrm_mod = (ctxt->modrm & 0xc0) >> 6;
1192 ctxt->modrm_reg |= (ctxt->modrm & 0x38) >> 3;
1193 ctxt->modrm_rm = base_reg | (ctxt->modrm & 0x07);
1194 ctxt->modrm_seg = VCPU_SREG_DS;
1196 if (ctxt->modrm_mod == 3 || (ctxt->d & NoMod)) {
1198 op->bytes = (ctxt->d & ByteOp) ? 1 : ctxt->op_bytes;
1199 op->addr.reg = decode_register(ctxt, ctxt->modrm_rm,
1201 if (ctxt->d & Sse) {
1204 op->addr.xmm = ctxt->modrm_rm;
1205 read_sse_reg(ctxt, &op->vec_val, ctxt->modrm_rm);
1208 if (ctxt->d & Mmx) {
1211 op->addr.mm = ctxt->modrm_rm & 7;
1214 fetch_register_operand(op);
1220 if (ctxt->ad_bytes == 2) {
1221 unsigned bx = reg_read(ctxt, VCPU_REGS_RBX);
1222 unsigned bp = reg_read(ctxt, VCPU_REGS_RBP);
1223 unsigned si = reg_read(ctxt, VCPU_REGS_RSI);
1224 unsigned di = reg_read(ctxt, VCPU_REGS_RDI);
1226 /* 16-bit ModR/M decode. */
1227 switch (ctxt->modrm_mod) {
1229 if (ctxt->modrm_rm == 6)
1230 modrm_ea += insn_fetch(u16, ctxt);
1233 modrm_ea += insn_fetch(s8, ctxt);
1236 modrm_ea += insn_fetch(u16, ctxt);
1239 switch (ctxt->modrm_rm) {
1241 modrm_ea += bx + si;
1244 modrm_ea += bx + di;
1247 modrm_ea += bp + si;
1250 modrm_ea += bp + di;
1259 if (ctxt->modrm_mod != 0)
1266 if (ctxt->modrm_rm == 2 || ctxt->modrm_rm == 3 ||
1267 (ctxt->modrm_rm == 6 && ctxt->modrm_mod != 0))
1268 ctxt->modrm_seg = VCPU_SREG_SS;
1269 modrm_ea = (u16)modrm_ea;
1271 /* 32/64-bit ModR/M decode. */
1272 if ((ctxt->modrm_rm & 7) == 4) {
1273 sib = insn_fetch(u8, ctxt);
1274 index_reg |= (sib >> 3) & 7;
1275 base_reg |= sib & 7;
1278 if ((base_reg & 7) == 5 && ctxt->modrm_mod == 0)
1279 modrm_ea += insn_fetch(s32, ctxt);
1281 modrm_ea += reg_read(ctxt, base_reg);
1282 adjust_modrm_seg(ctxt, base_reg);
1283 /* Increment ESP on POP [ESP] */
1284 if ((ctxt->d & IncSP) &&
1285 base_reg == VCPU_REGS_RSP)
1286 modrm_ea += ctxt->op_bytes;
1289 modrm_ea += reg_read(ctxt, index_reg) << scale;
1290 } else if ((ctxt->modrm_rm & 7) == 5 && ctxt->modrm_mod == 0) {
1291 modrm_ea += insn_fetch(s32, ctxt);
1292 if (ctxt->mode == X86EMUL_MODE_PROT64)
1293 ctxt->rip_relative = 1;
1295 base_reg = ctxt->modrm_rm;
1296 modrm_ea += reg_read(ctxt, base_reg);
1297 adjust_modrm_seg(ctxt, base_reg);
1299 switch (ctxt->modrm_mod) {
1301 modrm_ea += insn_fetch(s8, ctxt);
1304 modrm_ea += insn_fetch(s32, ctxt);
1308 op->addr.mem.ea = modrm_ea;
1309 if (ctxt->ad_bytes != 8)
1310 ctxt->memop.addr.mem.ea = (u32)ctxt->memop.addr.mem.ea;
1316 static int decode_abs(struct x86_emulate_ctxt *ctxt,
1319 int rc = X86EMUL_CONTINUE;
1322 switch (ctxt->ad_bytes) {
1324 op->addr.mem.ea = insn_fetch(u16, ctxt);
1327 op->addr.mem.ea = insn_fetch(u32, ctxt);
1330 op->addr.mem.ea = insn_fetch(u64, ctxt);
1337 static void fetch_bit_operand(struct x86_emulate_ctxt *ctxt)
1341 if (ctxt->dst.type == OP_MEM && ctxt->src.type == OP_REG) {
1342 mask = ~((long)ctxt->dst.bytes * 8 - 1);
1344 if (ctxt->src.bytes == 2)
1345 sv = (s16)ctxt->src.val & (s16)mask;
1346 else if (ctxt->src.bytes == 4)
1347 sv = (s32)ctxt->src.val & (s32)mask;
1349 sv = (s64)ctxt->src.val & (s64)mask;
1351 ctxt->dst.addr.mem.ea = address_mask(ctxt,
1352 ctxt->dst.addr.mem.ea + (sv >> 3));
1355 /* only subword offset */
1356 ctxt->src.val &= (ctxt->dst.bytes << 3) - 1;
1359 static int read_emulated(struct x86_emulate_ctxt *ctxt,
1360 unsigned long addr, void *dest, unsigned size)
1363 struct read_cache *mc = &ctxt->mem_read;
1365 if (mc->pos < mc->end)
1368 WARN_ON((mc->end + size) >= sizeof(mc->data));
1370 rc = ctxt->ops->read_emulated(ctxt, addr, mc->data + mc->end, size,
1372 if (rc != X86EMUL_CONTINUE)
1378 memcpy(dest, mc->data + mc->pos, size);
1380 return X86EMUL_CONTINUE;
1383 static int segmented_read(struct x86_emulate_ctxt *ctxt,
1384 struct segmented_address addr,
1391 rc = linearize(ctxt, addr, size, false, &linear);
1392 if (rc != X86EMUL_CONTINUE)
1394 return read_emulated(ctxt, linear, data, size);
1397 static int segmented_write(struct x86_emulate_ctxt *ctxt,
1398 struct segmented_address addr,
1405 rc = linearize(ctxt, addr, size, true, &linear);
1406 if (rc != X86EMUL_CONTINUE)
1408 return ctxt->ops->write_emulated(ctxt, linear, data, size,
1412 static int segmented_cmpxchg(struct x86_emulate_ctxt *ctxt,
1413 struct segmented_address addr,
1414 const void *orig_data, const void *data,
1420 rc = linearize(ctxt, addr, size, true, &linear);
1421 if (rc != X86EMUL_CONTINUE)
1423 return ctxt->ops->cmpxchg_emulated(ctxt, linear, orig_data, data,
1424 size, &ctxt->exception);
1427 static int pio_in_emulated(struct x86_emulate_ctxt *ctxt,
1428 unsigned int size, unsigned short port,
1431 struct read_cache *rc = &ctxt->io_read;
1433 if (rc->pos == rc->end) { /* refill pio read ahead */
1434 unsigned int in_page, n;
1435 unsigned int count = ctxt->rep_prefix ?
1436 address_mask(ctxt, reg_read(ctxt, VCPU_REGS_RCX)) : 1;
1437 in_page = (ctxt->eflags & EFLG_DF) ?
1438 offset_in_page(reg_read(ctxt, VCPU_REGS_RDI)) :
1439 PAGE_SIZE - offset_in_page(reg_read(ctxt, VCPU_REGS_RDI));
1440 n = min3(in_page, (unsigned int)sizeof(rc->data) / size, count);
1443 rc->pos = rc->end = 0;
1444 if (!ctxt->ops->pio_in_emulated(ctxt, size, port, rc->data, n))
1449 if (ctxt->rep_prefix && (ctxt->d & String) &&
1450 !(ctxt->eflags & EFLG_DF)) {
1451 ctxt->dst.data = rc->data + rc->pos;
1452 ctxt->dst.type = OP_MEM_STR;
1453 ctxt->dst.count = (rc->end - rc->pos) / size;
1456 memcpy(dest, rc->data + rc->pos, size);
1462 static int read_interrupt_descriptor(struct x86_emulate_ctxt *ctxt,
1463 u16 index, struct desc_struct *desc)
1468 ctxt->ops->get_idt(ctxt, &dt);
1470 if (dt.size < index * 8 + 7)
1471 return emulate_gp(ctxt, index << 3 | 0x2);
1473 addr = dt.address + index * 8;
1474 return ctxt->ops->read_std(ctxt, addr, desc, sizeof *desc,
1478 static void get_descriptor_table_ptr(struct x86_emulate_ctxt *ctxt,
1479 u16 selector, struct desc_ptr *dt)
1481 const struct x86_emulate_ops *ops = ctxt->ops;
1484 if (selector & 1 << 2) {
1485 struct desc_struct desc;
1488 memset (dt, 0, sizeof *dt);
1489 if (!ops->get_segment(ctxt, &sel, &desc, &base3,
1493 dt->size = desc_limit_scaled(&desc); /* what if limit > 65535? */
1494 dt->address = get_desc_base(&desc) | ((u64)base3 << 32);
1496 ops->get_gdt(ctxt, dt);
1499 static int get_descriptor_ptr(struct x86_emulate_ctxt *ctxt,
1500 u16 selector, ulong *desc_addr_p)
1503 u16 index = selector >> 3;
1506 get_descriptor_table_ptr(ctxt, selector, &dt);
1508 if (dt.size < index * 8 + 7)
1509 return emulate_gp(ctxt, selector & 0xfffc);
1511 addr = dt.address + index * 8;
1513 #ifdef CONFIG_X86_64
1514 if (addr >> 32 != 0) {
1517 ctxt->ops->get_msr(ctxt, MSR_EFER, &efer);
1518 if (!(efer & EFER_LMA))
1523 *desc_addr_p = addr;
1524 return X86EMUL_CONTINUE;
1527 /* allowed just for 8 bytes segments */
1528 static int read_segment_descriptor(struct x86_emulate_ctxt *ctxt,
1529 u16 selector, struct desc_struct *desc,
1534 rc = get_descriptor_ptr(ctxt, selector, desc_addr_p);
1535 if (rc != X86EMUL_CONTINUE)
1538 return ctxt->ops->read_std(ctxt, *desc_addr_p, desc, sizeof(*desc),
1542 /* allowed just for 8 bytes segments */
1543 static int write_segment_descriptor(struct x86_emulate_ctxt *ctxt,
1544 u16 selector, struct desc_struct *desc)
1549 rc = get_descriptor_ptr(ctxt, selector, &addr);
1550 if (rc != X86EMUL_CONTINUE)
1553 return ctxt->ops->write_std(ctxt, addr, desc, sizeof *desc,
1557 /* Does not support long mode */
1558 static int __load_segment_descriptor(struct x86_emulate_ctxt *ctxt,
1559 u16 selector, int seg, u8 cpl,
1560 enum x86_transfer_type transfer,
1561 struct desc_struct *desc)
1563 struct desc_struct seg_desc, old_desc;
1565 unsigned err_vec = GP_VECTOR;
1567 bool null_selector = !(selector & ~0x3); /* 0000-0003 are null */
1573 memset(&seg_desc, 0, sizeof seg_desc);
1575 if (ctxt->mode == X86EMUL_MODE_REAL) {
1576 /* set real mode segment descriptor (keep limit etc. for
1578 ctxt->ops->get_segment(ctxt, &dummy, &seg_desc, NULL, seg);
1579 set_desc_base(&seg_desc, selector << 4);
1581 } else if (seg <= VCPU_SREG_GS && ctxt->mode == X86EMUL_MODE_VM86) {
1582 /* VM86 needs a clean new segment descriptor */
1583 set_desc_base(&seg_desc, selector << 4);
1584 set_desc_limit(&seg_desc, 0xffff);
1594 /* NULL selector is not valid for TR, CS and SS (except for long mode) */
1595 if ((seg == VCPU_SREG_CS
1596 || (seg == VCPU_SREG_SS
1597 && (ctxt->mode != X86EMUL_MODE_PROT64 || rpl != cpl))
1598 || seg == VCPU_SREG_TR)
1602 /* TR should be in GDT only */
1603 if (seg == VCPU_SREG_TR && (selector & (1 << 2)))
1606 if (null_selector) /* for NULL selector skip all following checks */
1609 ret = read_segment_descriptor(ctxt, selector, &seg_desc, &desc_addr);
1610 if (ret != X86EMUL_CONTINUE)
1613 err_code = selector & 0xfffc;
1614 err_vec = (transfer == X86_TRANSFER_TASK_SWITCH) ? TS_VECTOR :
1617 /* can't load system descriptor into segment selector */
1618 if (seg <= VCPU_SREG_GS && !seg_desc.s) {
1619 if (transfer == X86_TRANSFER_CALL_JMP)
1620 return X86EMUL_UNHANDLEABLE;
1625 err_vec = (seg == VCPU_SREG_SS) ? SS_VECTOR : NP_VECTOR;
1634 * segment is not a writable data segment or segment
1635 * selector's RPL != CPL or segment selector's RPL != CPL
1637 if (rpl != cpl || (seg_desc.type & 0xa) != 0x2 || dpl != cpl)
1641 if (!(seg_desc.type & 8))
1644 if (seg_desc.type & 4) {
1650 if (rpl > cpl || dpl != cpl)
1653 /* in long-mode d/b must be clear if l is set */
1654 if (seg_desc.d && seg_desc.l) {
1657 ctxt->ops->get_msr(ctxt, MSR_EFER, &efer);
1658 if (efer & EFER_LMA)
1662 /* CS(RPL) <- CPL */
1663 selector = (selector & 0xfffc) | cpl;
1666 if (seg_desc.s || (seg_desc.type != 1 && seg_desc.type != 9))
1668 old_desc = seg_desc;
1669 seg_desc.type |= 2; /* busy */
1670 ret = ctxt->ops->cmpxchg_emulated(ctxt, desc_addr, &old_desc, &seg_desc,
1671 sizeof(seg_desc), &ctxt->exception);
1672 if (ret != X86EMUL_CONTINUE)
1675 case VCPU_SREG_LDTR:
1676 if (seg_desc.s || seg_desc.type != 2)
1679 default: /* DS, ES, FS, or GS */
1681 * segment is not a data or readable code segment or
1682 * ((segment is a data or nonconforming code segment)
1683 * and (both RPL and CPL > DPL))
1685 if ((seg_desc.type & 0xa) == 0x8 ||
1686 (((seg_desc.type & 0xc) != 0xc) &&
1687 (rpl > dpl && cpl > dpl)))
1693 /* mark segment as accessed */
1694 if (!(seg_desc.type & 1)) {
1696 ret = write_segment_descriptor(ctxt, selector,
1698 if (ret != X86EMUL_CONTINUE)
1701 } else if (ctxt->mode == X86EMUL_MODE_PROT64) {
1702 ret = ctxt->ops->read_std(ctxt, desc_addr+8, &base3,
1703 sizeof(base3), &ctxt->exception);
1704 if (ret != X86EMUL_CONTINUE)
1706 if (is_noncanonical_address(get_desc_base(&seg_desc) |
1707 ((u64)base3 << 32)))
1708 return emulate_gp(ctxt, 0);
1711 ctxt->ops->set_segment(ctxt, selector, &seg_desc, base3, seg);
1714 return X86EMUL_CONTINUE;
1716 return emulate_exception(ctxt, err_vec, err_code, true);
1719 static int load_segment_descriptor(struct x86_emulate_ctxt *ctxt,
1720 u16 selector, int seg)
1722 u8 cpl = ctxt->ops->cpl(ctxt);
1723 return __load_segment_descriptor(ctxt, selector, seg, cpl,
1724 X86_TRANSFER_NONE, NULL);
1727 static void write_register_operand(struct operand *op)
1729 return assign_register(op->addr.reg, op->val, op->bytes);
1732 static int writeback(struct x86_emulate_ctxt *ctxt, struct operand *op)
1736 write_register_operand(op);
1739 if (ctxt->lock_prefix)
1740 return segmented_cmpxchg(ctxt,
1746 return segmented_write(ctxt,
1752 return segmented_write(ctxt,
1755 op->bytes * op->count);
1758 write_sse_reg(ctxt, &op->vec_val, op->addr.xmm);
1761 write_mmx_reg(ctxt, &op->mm_val, op->addr.mm);
1769 return X86EMUL_CONTINUE;
1772 static int push(struct x86_emulate_ctxt *ctxt, void *data, int bytes)
1774 struct segmented_address addr;
1776 rsp_increment(ctxt, -bytes);
1777 addr.ea = reg_read(ctxt, VCPU_REGS_RSP) & stack_mask(ctxt);
1778 addr.seg = VCPU_SREG_SS;
1780 return segmented_write(ctxt, addr, data, bytes);
1783 static int em_push(struct x86_emulate_ctxt *ctxt)
1785 /* Disable writeback. */
1786 ctxt->dst.type = OP_NONE;
1787 return push(ctxt, &ctxt->src.val, ctxt->op_bytes);
1790 static int emulate_pop(struct x86_emulate_ctxt *ctxt,
1791 void *dest, int len)
1794 struct segmented_address addr;
1796 addr.ea = reg_read(ctxt, VCPU_REGS_RSP) & stack_mask(ctxt);
1797 addr.seg = VCPU_SREG_SS;
1798 rc = segmented_read(ctxt, addr, dest, len);
1799 if (rc != X86EMUL_CONTINUE)
1802 rsp_increment(ctxt, len);
1806 static int em_pop(struct x86_emulate_ctxt *ctxt)
1808 return emulate_pop(ctxt, &ctxt->dst.val, ctxt->op_bytes);
1811 static int emulate_popf(struct x86_emulate_ctxt *ctxt,
1812 void *dest, int len)
1815 unsigned long val, change_mask;
1816 int iopl = (ctxt->eflags & X86_EFLAGS_IOPL) >> IOPL_SHIFT;
1817 int cpl = ctxt->ops->cpl(ctxt);
1819 rc = emulate_pop(ctxt, &val, len);
1820 if (rc != X86EMUL_CONTINUE)
1823 change_mask = EFLG_CF | EFLG_PF | EFLG_AF | EFLG_ZF | EFLG_SF | EFLG_OF
1824 | EFLG_TF | EFLG_DF | EFLG_NT | EFLG_AC | EFLG_ID;
1826 switch(ctxt->mode) {
1827 case X86EMUL_MODE_PROT64:
1828 case X86EMUL_MODE_PROT32:
1829 case X86EMUL_MODE_PROT16:
1831 change_mask |= EFLG_IOPL;
1833 change_mask |= EFLG_IF;
1835 case X86EMUL_MODE_VM86:
1837 return emulate_gp(ctxt, 0);
1838 change_mask |= EFLG_IF;
1840 default: /* real mode */
1841 change_mask |= (EFLG_IOPL | EFLG_IF);
1845 *(unsigned long *)dest =
1846 (ctxt->eflags & ~change_mask) | (val & change_mask);
1851 static int em_popf(struct x86_emulate_ctxt *ctxt)
1853 ctxt->dst.type = OP_REG;
1854 ctxt->dst.addr.reg = &ctxt->eflags;
1855 ctxt->dst.bytes = ctxt->op_bytes;
1856 return emulate_popf(ctxt, &ctxt->dst.val, ctxt->op_bytes);
1859 static int em_enter(struct x86_emulate_ctxt *ctxt)
1862 unsigned frame_size = ctxt->src.val;
1863 unsigned nesting_level = ctxt->src2.val & 31;
1867 return X86EMUL_UNHANDLEABLE;
1869 rbp = reg_read(ctxt, VCPU_REGS_RBP);
1870 rc = push(ctxt, &rbp, stack_size(ctxt));
1871 if (rc != X86EMUL_CONTINUE)
1873 assign_masked(reg_rmw(ctxt, VCPU_REGS_RBP), reg_read(ctxt, VCPU_REGS_RSP),
1875 assign_masked(reg_rmw(ctxt, VCPU_REGS_RSP),
1876 reg_read(ctxt, VCPU_REGS_RSP) - frame_size,
1878 return X86EMUL_CONTINUE;
1881 static int em_leave(struct x86_emulate_ctxt *ctxt)
1883 assign_masked(reg_rmw(ctxt, VCPU_REGS_RSP), reg_read(ctxt, VCPU_REGS_RBP),
1885 return emulate_pop(ctxt, reg_rmw(ctxt, VCPU_REGS_RBP), ctxt->op_bytes);
1888 static int em_push_sreg(struct x86_emulate_ctxt *ctxt)
1890 int seg = ctxt->src2.val;
1892 ctxt->src.val = get_segment_selector(ctxt, seg);
1893 if (ctxt->op_bytes == 4) {
1894 rsp_increment(ctxt, -2);
1898 return em_push(ctxt);
1901 static int em_pop_sreg(struct x86_emulate_ctxt *ctxt)
1903 int seg = ctxt->src2.val;
1904 unsigned long selector;
1907 rc = emulate_pop(ctxt, &selector, 2);
1908 if (rc != X86EMUL_CONTINUE)
1911 if (ctxt->modrm_reg == VCPU_SREG_SS)
1912 ctxt->interruptibility = KVM_X86_SHADOW_INT_MOV_SS;
1913 if (ctxt->op_bytes > 2)
1914 rsp_increment(ctxt, ctxt->op_bytes - 2);
1916 rc = load_segment_descriptor(ctxt, (u16)selector, seg);
1920 static int em_pusha(struct x86_emulate_ctxt *ctxt)
1922 unsigned long old_esp = reg_read(ctxt, VCPU_REGS_RSP);
1923 int rc = X86EMUL_CONTINUE;
1924 int reg = VCPU_REGS_RAX;
1926 while (reg <= VCPU_REGS_RDI) {
1927 (reg == VCPU_REGS_RSP) ?
1928 (ctxt->src.val = old_esp) : (ctxt->src.val = reg_read(ctxt, reg));
1931 if (rc != X86EMUL_CONTINUE)
1940 static int em_pushf(struct x86_emulate_ctxt *ctxt)
1942 ctxt->src.val = (unsigned long)ctxt->eflags & ~EFLG_VM;
1943 return em_push(ctxt);
1946 static int em_popa(struct x86_emulate_ctxt *ctxt)
1948 int rc = X86EMUL_CONTINUE;
1949 int reg = VCPU_REGS_RDI;
1952 while (reg >= VCPU_REGS_RAX) {
1953 if (reg == VCPU_REGS_RSP) {
1954 rsp_increment(ctxt, ctxt->op_bytes);
1958 rc = emulate_pop(ctxt, &val, ctxt->op_bytes);
1959 if (rc != X86EMUL_CONTINUE)
1961 assign_register(reg_rmw(ctxt, reg), val, ctxt->op_bytes);
1967 static int __emulate_int_real(struct x86_emulate_ctxt *ctxt, int irq)
1969 const struct x86_emulate_ops *ops = ctxt->ops;
1976 /* TODO: Add limit checks */
1977 ctxt->src.val = ctxt->eflags;
1979 if (rc != X86EMUL_CONTINUE)
1982 ctxt->eflags &= ~(EFLG_IF | EFLG_TF | EFLG_AC);
1984 ctxt->src.val = get_segment_selector(ctxt, VCPU_SREG_CS);
1986 if (rc != X86EMUL_CONTINUE)
1989 ctxt->src.val = ctxt->_eip;
1991 if (rc != X86EMUL_CONTINUE)
1994 ops->get_idt(ctxt, &dt);
1996 eip_addr = dt.address + (irq << 2);
1997 cs_addr = dt.address + (irq << 2) + 2;
1999 rc = ops->read_std(ctxt, cs_addr, &cs, 2, &ctxt->exception);
2000 if (rc != X86EMUL_CONTINUE)
2003 rc = ops->read_std(ctxt, eip_addr, &eip, 2, &ctxt->exception);
2004 if (rc != X86EMUL_CONTINUE)
2007 rc = load_segment_descriptor(ctxt, cs, VCPU_SREG_CS);
2008 if (rc != X86EMUL_CONTINUE)
2016 int emulate_int_real(struct x86_emulate_ctxt *ctxt, int irq)
2020 invalidate_registers(ctxt);
2021 rc = __emulate_int_real(ctxt, irq);
2022 if (rc == X86EMUL_CONTINUE)
2023 writeback_registers(ctxt);
2027 static int emulate_int(struct x86_emulate_ctxt *ctxt, int irq)
2029 switch(ctxt->mode) {
2030 case X86EMUL_MODE_REAL:
2031 return __emulate_int_real(ctxt, irq);
2032 case X86EMUL_MODE_VM86:
2033 case X86EMUL_MODE_PROT16:
2034 case X86EMUL_MODE_PROT32:
2035 case X86EMUL_MODE_PROT64:
2037 /* Protected mode interrupts unimplemented yet */
2038 return X86EMUL_UNHANDLEABLE;
2042 static int emulate_iret_real(struct x86_emulate_ctxt *ctxt)
2044 int rc = X86EMUL_CONTINUE;
2045 unsigned long temp_eip = 0;
2046 unsigned long temp_eflags = 0;
2047 unsigned long cs = 0;
2048 unsigned long mask = EFLG_CF | EFLG_PF | EFLG_AF | EFLG_ZF | EFLG_SF | EFLG_TF |
2049 EFLG_IF | EFLG_DF | EFLG_OF | EFLG_IOPL | EFLG_NT | EFLG_RF |
2050 EFLG_AC | EFLG_ID | (1 << 1); /* Last one is the reserved bit */
2051 unsigned long vm86_mask = EFLG_VM | EFLG_VIF | EFLG_VIP;
2053 /* TODO: Add stack limit check */
2055 rc = emulate_pop(ctxt, &temp_eip, ctxt->op_bytes);
2057 if (rc != X86EMUL_CONTINUE)
2060 if (temp_eip & ~0xffff)
2061 return emulate_gp(ctxt, 0);
2063 rc = emulate_pop(ctxt, &cs, ctxt->op_bytes);
2065 if (rc != X86EMUL_CONTINUE)
2068 rc = emulate_pop(ctxt, &temp_eflags, ctxt->op_bytes);
2070 if (rc != X86EMUL_CONTINUE)
2073 rc = load_segment_descriptor(ctxt, (u16)cs, VCPU_SREG_CS);
2075 if (rc != X86EMUL_CONTINUE)
2078 ctxt->_eip = temp_eip;
2081 if (ctxt->op_bytes == 4)
2082 ctxt->eflags = ((temp_eflags & mask) | (ctxt->eflags & vm86_mask));
2083 else if (ctxt->op_bytes == 2) {
2084 ctxt->eflags &= ~0xffff;
2085 ctxt->eflags |= temp_eflags;
2088 ctxt->eflags &= ~EFLG_RESERVED_ZEROS_MASK; /* Clear reserved zeros */
2089 ctxt->eflags |= EFLG_RESERVED_ONE_MASK;
2090 ctxt->ops->set_nmi_mask(ctxt, false);
2095 static int em_iret(struct x86_emulate_ctxt *ctxt)
2097 switch(ctxt->mode) {
2098 case X86EMUL_MODE_REAL:
2099 return emulate_iret_real(ctxt);
2100 case X86EMUL_MODE_VM86:
2101 case X86EMUL_MODE_PROT16:
2102 case X86EMUL_MODE_PROT32:
2103 case X86EMUL_MODE_PROT64:
2105 /* iret from protected mode unimplemented yet */
2106 return X86EMUL_UNHANDLEABLE;
2110 static int em_jmp_far(struct x86_emulate_ctxt *ctxt)
2113 unsigned short sel, old_sel;
2114 struct desc_struct old_desc, new_desc;
2115 const struct x86_emulate_ops *ops = ctxt->ops;
2116 u8 cpl = ctxt->ops->cpl(ctxt);
2118 /* Assignment of RIP may only fail in 64-bit mode */
2119 if (ctxt->mode == X86EMUL_MODE_PROT64)
2120 ops->get_segment(ctxt, &old_sel, &old_desc, NULL,
2123 memcpy(&sel, ctxt->src.valptr + ctxt->op_bytes, 2);
2125 rc = __load_segment_descriptor(ctxt, sel, VCPU_SREG_CS, cpl,
2126 X86_TRANSFER_CALL_JMP,
2128 if (rc != X86EMUL_CONTINUE)
2131 rc = assign_eip_far(ctxt, ctxt->src.val, &new_desc);
2132 if (rc != X86EMUL_CONTINUE) {
2133 WARN_ON(ctxt->mode != X86EMUL_MODE_PROT64);
2134 /* assigning eip failed; restore the old cs */
2135 ops->set_segment(ctxt, old_sel, &old_desc, 0, VCPU_SREG_CS);
2141 static int em_jmp_abs(struct x86_emulate_ctxt *ctxt)
2143 return assign_eip_near(ctxt, ctxt->src.val);
2146 static int em_call_near_abs(struct x86_emulate_ctxt *ctxt)
2151 old_eip = ctxt->_eip;
2152 rc = assign_eip_near(ctxt, ctxt->src.val);
2153 if (rc != X86EMUL_CONTINUE)
2155 ctxt->src.val = old_eip;
2160 static int em_cmpxchg8b(struct x86_emulate_ctxt *ctxt)
2162 u64 old = ctxt->dst.orig_val64;
2164 if (ctxt->dst.bytes == 16)
2165 return X86EMUL_UNHANDLEABLE;
2167 if (((u32) (old >> 0) != (u32) reg_read(ctxt, VCPU_REGS_RAX)) ||
2168 ((u32) (old >> 32) != (u32) reg_read(ctxt, VCPU_REGS_RDX))) {
2169 *reg_write(ctxt, VCPU_REGS_RAX) = (u32) (old >> 0);
2170 *reg_write(ctxt, VCPU_REGS_RDX) = (u32) (old >> 32);
2171 ctxt->eflags &= ~EFLG_ZF;
2173 ctxt->dst.val64 = ((u64)reg_read(ctxt, VCPU_REGS_RCX) << 32) |
2174 (u32) reg_read(ctxt, VCPU_REGS_RBX);
2176 ctxt->eflags |= EFLG_ZF;
2178 return X86EMUL_CONTINUE;
2181 static int em_ret(struct x86_emulate_ctxt *ctxt)
2186 rc = emulate_pop(ctxt, &eip, ctxt->op_bytes);
2187 if (rc != X86EMUL_CONTINUE)
2190 return assign_eip_near(ctxt, eip);
2193 static int em_ret_far(struct x86_emulate_ctxt *ctxt)
2196 unsigned long eip, cs;
2198 int cpl = ctxt->ops->cpl(ctxt);
2199 struct desc_struct old_desc, new_desc;
2200 const struct x86_emulate_ops *ops = ctxt->ops;
2202 if (ctxt->mode == X86EMUL_MODE_PROT64)
2203 ops->get_segment(ctxt, &old_cs, &old_desc, NULL,
2206 rc = emulate_pop(ctxt, &eip, ctxt->op_bytes);
2207 if (rc != X86EMUL_CONTINUE)
2209 rc = emulate_pop(ctxt, &cs, ctxt->op_bytes);
2210 if (rc != X86EMUL_CONTINUE)
2212 /* Outer-privilege level return is not implemented */
2213 if (ctxt->mode >= X86EMUL_MODE_PROT16 && (cs & 3) > cpl)
2214 return X86EMUL_UNHANDLEABLE;
2215 rc = __load_segment_descriptor(ctxt, (u16)cs, VCPU_SREG_CS, cpl,
2218 if (rc != X86EMUL_CONTINUE)
2220 rc = assign_eip_far(ctxt, eip, &new_desc);
2221 if (rc != X86EMUL_CONTINUE) {
2222 WARN_ON(ctxt->mode != X86EMUL_MODE_PROT64);
2223 ops->set_segment(ctxt, old_cs, &old_desc, 0, VCPU_SREG_CS);
2228 static int em_ret_far_imm(struct x86_emulate_ctxt *ctxt)
2232 rc = em_ret_far(ctxt);
2233 if (rc != X86EMUL_CONTINUE)
2235 rsp_increment(ctxt, ctxt->src.val);
2236 return X86EMUL_CONTINUE;
2239 static int em_cmpxchg(struct x86_emulate_ctxt *ctxt)
2241 /* Save real source value, then compare EAX against destination. */
2242 ctxt->dst.orig_val = ctxt->dst.val;
2243 ctxt->dst.val = reg_read(ctxt, VCPU_REGS_RAX);
2244 ctxt->src.orig_val = ctxt->src.val;
2245 ctxt->src.val = ctxt->dst.orig_val;
2246 fastop(ctxt, em_cmp);
2248 if (ctxt->eflags & EFLG_ZF) {
2249 /* Success: write back to memory; no update of EAX */
2250 ctxt->src.type = OP_NONE;
2251 ctxt->dst.val = ctxt->src.orig_val;
2253 /* Failure: write the value we saw to EAX. */
2254 ctxt->src.type = OP_REG;
2255 ctxt->src.addr.reg = reg_rmw(ctxt, VCPU_REGS_RAX);
2256 ctxt->src.val = ctxt->dst.orig_val;
2257 /* Create write-cycle to dest by writing the same value */
2258 ctxt->dst.val = ctxt->dst.orig_val;
2260 return X86EMUL_CONTINUE;
2263 static int em_lseg(struct x86_emulate_ctxt *ctxt)
2265 int seg = ctxt->src2.val;
2269 memcpy(&sel, ctxt->src.valptr + ctxt->op_bytes, 2);
2271 rc = load_segment_descriptor(ctxt, sel, seg);
2272 if (rc != X86EMUL_CONTINUE)
2275 ctxt->dst.val = ctxt->src.val;
2280 setup_syscalls_segments(struct x86_emulate_ctxt *ctxt,
2281 struct desc_struct *cs, struct desc_struct *ss)
2283 cs->l = 0; /* will be adjusted later */
2284 set_desc_base(cs, 0); /* flat segment */
2285 cs->g = 1; /* 4kb granularity */
2286 set_desc_limit(cs, 0xfffff); /* 4GB limit */
2287 cs->type = 0x0b; /* Read, Execute, Accessed */
2289 cs->dpl = 0; /* will be adjusted later */
2294 set_desc_base(ss, 0); /* flat segment */
2295 set_desc_limit(ss, 0xfffff); /* 4GB limit */
2296 ss->g = 1; /* 4kb granularity */
2298 ss->type = 0x03; /* Read/Write, Accessed */
2299 ss->d = 1; /* 32bit stack segment */
2306 static bool vendor_intel(struct x86_emulate_ctxt *ctxt)
2308 u32 eax, ebx, ecx, edx;
2311 ctxt->ops->get_cpuid(ctxt, &eax, &ebx, &ecx, &edx);
2312 return ebx == X86EMUL_CPUID_VENDOR_GenuineIntel_ebx
2313 && ecx == X86EMUL_CPUID_VENDOR_GenuineIntel_ecx
2314 && edx == X86EMUL_CPUID_VENDOR_GenuineIntel_edx;
2317 static bool em_syscall_is_enabled(struct x86_emulate_ctxt *ctxt)
2319 const struct x86_emulate_ops *ops = ctxt->ops;
2320 u32 eax, ebx, ecx, edx;
2323 * syscall should always be enabled in longmode - so only become
2324 * vendor specific (cpuid) if other modes are active...
2326 if (ctxt->mode == X86EMUL_MODE_PROT64)
2331 ops->get_cpuid(ctxt, &eax, &ebx, &ecx, &edx);
2333 * Intel ("GenuineIntel")
2334 * remark: Intel CPUs only support "syscall" in 64bit
2335 * longmode. Also an 64bit guest with a
2336 * 32bit compat-app running will #UD !! While this
2337 * behaviour can be fixed (by emulating) into AMD
2338 * response - CPUs of AMD can't behave like Intel.
2340 if (ebx == X86EMUL_CPUID_VENDOR_GenuineIntel_ebx &&
2341 ecx == X86EMUL_CPUID_VENDOR_GenuineIntel_ecx &&
2342 edx == X86EMUL_CPUID_VENDOR_GenuineIntel_edx)
2345 /* AMD ("AuthenticAMD") */
2346 if (ebx == X86EMUL_CPUID_VENDOR_AuthenticAMD_ebx &&
2347 ecx == X86EMUL_CPUID_VENDOR_AuthenticAMD_ecx &&
2348 edx == X86EMUL_CPUID_VENDOR_AuthenticAMD_edx)
2351 /* AMD ("AMDisbetter!") */
2352 if (ebx == X86EMUL_CPUID_VENDOR_AMDisbetterI_ebx &&
2353 ecx == X86EMUL_CPUID_VENDOR_AMDisbetterI_ecx &&
2354 edx == X86EMUL_CPUID_VENDOR_AMDisbetterI_edx)
2357 /* default: (not Intel, not AMD), apply Intel's stricter rules... */
2361 static int em_syscall(struct x86_emulate_ctxt *ctxt)
2363 const struct x86_emulate_ops *ops = ctxt->ops;
2364 struct desc_struct cs, ss;
2369 /* syscall is not available in real mode */
2370 if (ctxt->mode == X86EMUL_MODE_REAL ||
2371 ctxt->mode == X86EMUL_MODE_VM86)
2372 return emulate_ud(ctxt);
2374 if (!(em_syscall_is_enabled(ctxt)))
2375 return emulate_ud(ctxt);
2377 ops->get_msr(ctxt, MSR_EFER, &efer);
2378 setup_syscalls_segments(ctxt, &cs, &ss);
2380 if (!(efer & EFER_SCE))
2381 return emulate_ud(ctxt);
2383 ops->get_msr(ctxt, MSR_STAR, &msr_data);
2385 cs_sel = (u16)(msr_data & 0xfffc);
2386 ss_sel = (u16)(msr_data + 8);
2388 if (efer & EFER_LMA) {
2392 ops->set_segment(ctxt, cs_sel, &cs, 0, VCPU_SREG_CS);
2393 ops->set_segment(ctxt, ss_sel, &ss, 0, VCPU_SREG_SS);
2395 *reg_write(ctxt, VCPU_REGS_RCX) = ctxt->_eip;
2396 if (efer & EFER_LMA) {
2397 #ifdef CONFIG_X86_64
2398 *reg_write(ctxt, VCPU_REGS_R11) = ctxt->eflags;
2401 ctxt->mode == X86EMUL_MODE_PROT64 ?
2402 MSR_LSTAR : MSR_CSTAR, &msr_data);
2403 ctxt->_eip = msr_data;
2405 ops->get_msr(ctxt, MSR_SYSCALL_MASK, &msr_data);
2406 ctxt->eflags &= ~msr_data;
2407 ctxt->eflags |= EFLG_RESERVED_ONE_MASK;
2411 ops->get_msr(ctxt, MSR_STAR, &msr_data);
2412 ctxt->_eip = (u32)msr_data;
2414 ctxt->eflags &= ~(EFLG_VM | EFLG_IF);
2417 return X86EMUL_CONTINUE;
2420 static int em_sysenter(struct x86_emulate_ctxt *ctxt)
2422 const struct x86_emulate_ops *ops = ctxt->ops;
2423 struct desc_struct cs, ss;
2428 ops->get_msr(ctxt, MSR_EFER, &efer);
2429 /* inject #GP if in real mode */
2430 if (ctxt->mode == X86EMUL_MODE_REAL)
2431 return emulate_gp(ctxt, 0);
2434 * Not recognized on AMD in compat mode (but is recognized in legacy
2437 if ((ctxt->mode != X86EMUL_MODE_PROT64) && (efer & EFER_LMA)
2438 && !vendor_intel(ctxt))
2439 return emulate_ud(ctxt);
2441 /* sysenter/sysexit have not been tested in 64bit mode. */
2442 if (ctxt->mode == X86EMUL_MODE_PROT64)
2443 return X86EMUL_UNHANDLEABLE;
2445 setup_syscalls_segments(ctxt, &cs, &ss);
2447 ops->get_msr(ctxt, MSR_IA32_SYSENTER_CS, &msr_data);
2448 if ((msr_data & 0xfffc) == 0x0)
2449 return emulate_gp(ctxt, 0);
2451 ctxt->eflags &= ~(EFLG_VM | EFLG_IF);
2452 cs_sel = (u16)msr_data & ~SELECTOR_RPL_MASK;
2453 ss_sel = cs_sel + 8;
2454 if (efer & EFER_LMA) {
2459 ops->set_segment(ctxt, cs_sel, &cs, 0, VCPU_SREG_CS);
2460 ops->set_segment(ctxt, ss_sel, &ss, 0, VCPU_SREG_SS);
2462 ops->get_msr(ctxt, MSR_IA32_SYSENTER_EIP, &msr_data);
2463 ctxt->_eip = (efer & EFER_LMA) ? msr_data : (u32)msr_data;
2465 ops->get_msr(ctxt, MSR_IA32_SYSENTER_ESP, &msr_data);
2466 *reg_write(ctxt, VCPU_REGS_RSP) = (efer & EFER_LMA) ? msr_data :
2469 return X86EMUL_CONTINUE;
2472 static int em_sysexit(struct x86_emulate_ctxt *ctxt)
2474 const struct x86_emulate_ops *ops = ctxt->ops;
2475 struct desc_struct cs, ss;
2476 u64 msr_data, rcx, rdx;
2478 u16 cs_sel = 0, ss_sel = 0;
2480 /* inject #GP if in real mode or Virtual 8086 mode */
2481 if (ctxt->mode == X86EMUL_MODE_REAL ||
2482 ctxt->mode == X86EMUL_MODE_VM86)
2483 return emulate_gp(ctxt, 0);
2485 setup_syscalls_segments(ctxt, &cs, &ss);
2487 if ((ctxt->rex_prefix & 0x8) != 0x0)
2488 usermode = X86EMUL_MODE_PROT64;
2490 usermode = X86EMUL_MODE_PROT32;
2492 rcx = reg_read(ctxt, VCPU_REGS_RCX);
2493 rdx = reg_read(ctxt, VCPU_REGS_RDX);
2497 ops->get_msr(ctxt, MSR_IA32_SYSENTER_CS, &msr_data);
2499 case X86EMUL_MODE_PROT32:
2500 cs_sel = (u16)(msr_data + 16);
2501 if ((msr_data & 0xfffc) == 0x0)
2502 return emulate_gp(ctxt, 0);
2503 ss_sel = (u16)(msr_data + 24);
2507 case X86EMUL_MODE_PROT64:
2508 cs_sel = (u16)(msr_data + 32);
2509 if (msr_data == 0x0)
2510 return emulate_gp(ctxt, 0);
2511 ss_sel = cs_sel + 8;
2514 if (is_noncanonical_address(rcx) ||
2515 is_noncanonical_address(rdx))
2516 return emulate_gp(ctxt, 0);
2519 cs_sel |= SELECTOR_RPL_MASK;
2520 ss_sel |= SELECTOR_RPL_MASK;
2522 ops->set_segment(ctxt, cs_sel, &cs, 0, VCPU_SREG_CS);
2523 ops->set_segment(ctxt, ss_sel, &ss, 0, VCPU_SREG_SS);
2526 *reg_write(ctxt, VCPU_REGS_RSP) = rcx;
2528 return X86EMUL_CONTINUE;
2531 static bool emulator_bad_iopl(struct x86_emulate_ctxt *ctxt)
2534 if (ctxt->mode == X86EMUL_MODE_REAL)
2536 if (ctxt->mode == X86EMUL_MODE_VM86)
2538 iopl = (ctxt->eflags & X86_EFLAGS_IOPL) >> IOPL_SHIFT;
2539 return ctxt->ops->cpl(ctxt) > iopl;
2542 static bool emulator_io_port_access_allowed(struct x86_emulate_ctxt *ctxt,
2545 const struct x86_emulate_ops *ops = ctxt->ops;
2546 struct desc_struct tr_seg;
2549 u16 tr, io_bitmap_ptr, perm, bit_idx = port & 0x7;
2550 unsigned mask = (1 << len) - 1;
2553 ops->get_segment(ctxt, &tr, &tr_seg, &base3, VCPU_SREG_TR);
2556 if (desc_limit_scaled(&tr_seg) < 103)
2558 base = get_desc_base(&tr_seg);
2559 #ifdef CONFIG_X86_64
2560 base |= ((u64)base3) << 32;
2562 r = ops->read_std(ctxt, base + 102, &io_bitmap_ptr, 2, NULL);
2563 if (r != X86EMUL_CONTINUE)
2565 if (io_bitmap_ptr + port/8 > desc_limit_scaled(&tr_seg))
2567 r = ops->read_std(ctxt, base + io_bitmap_ptr + port/8, &perm, 2, NULL);
2568 if (r != X86EMUL_CONTINUE)
2570 if ((perm >> bit_idx) & mask)
2575 static bool emulator_io_permited(struct x86_emulate_ctxt *ctxt,
2581 if (emulator_bad_iopl(ctxt))
2582 if (!emulator_io_port_access_allowed(ctxt, port, len))
2585 ctxt->perm_ok = true;
2590 static void save_state_to_tss16(struct x86_emulate_ctxt *ctxt,
2591 struct tss_segment_16 *tss)
2593 tss->ip = ctxt->_eip;
2594 tss->flag = ctxt->eflags;
2595 tss->ax = reg_read(ctxt, VCPU_REGS_RAX);
2596 tss->cx = reg_read(ctxt, VCPU_REGS_RCX);
2597 tss->dx = reg_read(ctxt, VCPU_REGS_RDX);
2598 tss->bx = reg_read(ctxt, VCPU_REGS_RBX);
2599 tss->sp = reg_read(ctxt, VCPU_REGS_RSP);
2600 tss->bp = reg_read(ctxt, VCPU_REGS_RBP);
2601 tss->si = reg_read(ctxt, VCPU_REGS_RSI);
2602 tss->di = reg_read(ctxt, VCPU_REGS_RDI);
2604 tss->es = get_segment_selector(ctxt, VCPU_SREG_ES);
2605 tss->cs = get_segment_selector(ctxt, VCPU_SREG_CS);
2606 tss->ss = get_segment_selector(ctxt, VCPU_SREG_SS);
2607 tss->ds = get_segment_selector(ctxt, VCPU_SREG_DS);
2608 tss->ldt = get_segment_selector(ctxt, VCPU_SREG_LDTR);
2611 static int load_state_from_tss16(struct x86_emulate_ctxt *ctxt,
2612 struct tss_segment_16 *tss)
2617 ctxt->_eip = tss->ip;
2618 ctxt->eflags = tss->flag | 2;
2619 *reg_write(ctxt, VCPU_REGS_RAX) = tss->ax;
2620 *reg_write(ctxt, VCPU_REGS_RCX) = tss->cx;
2621 *reg_write(ctxt, VCPU_REGS_RDX) = tss->dx;
2622 *reg_write(ctxt, VCPU_REGS_RBX) = tss->bx;
2623 *reg_write(ctxt, VCPU_REGS_RSP) = tss->sp;
2624 *reg_write(ctxt, VCPU_REGS_RBP) = tss->bp;
2625 *reg_write(ctxt, VCPU_REGS_RSI) = tss->si;
2626 *reg_write(ctxt, VCPU_REGS_RDI) = tss->di;
2629 * SDM says that segment selectors are loaded before segment
2632 set_segment_selector(ctxt, tss->ldt, VCPU_SREG_LDTR);
2633 set_segment_selector(ctxt, tss->es, VCPU_SREG_ES);
2634 set_segment_selector(ctxt, tss->cs, VCPU_SREG_CS);
2635 set_segment_selector(ctxt, tss->ss, VCPU_SREG_SS);
2636 set_segment_selector(ctxt, tss->ds, VCPU_SREG_DS);
2641 * Now load segment descriptors. If fault happens at this stage
2642 * it is handled in a context of new task
2644 ret = __load_segment_descriptor(ctxt, tss->ldt, VCPU_SREG_LDTR, cpl,
2645 X86_TRANSFER_TASK_SWITCH, NULL);
2646 if (ret != X86EMUL_CONTINUE)
2648 ret = __load_segment_descriptor(ctxt, tss->es, VCPU_SREG_ES, cpl,
2649 X86_TRANSFER_TASK_SWITCH, NULL);
2650 if (ret != X86EMUL_CONTINUE)
2652 ret = __load_segment_descriptor(ctxt, tss->cs, VCPU_SREG_CS, cpl,
2653 X86_TRANSFER_TASK_SWITCH, NULL);
2654 if (ret != X86EMUL_CONTINUE)
2656 ret = __load_segment_descriptor(ctxt, tss->ss, VCPU_SREG_SS, cpl,
2657 X86_TRANSFER_TASK_SWITCH, NULL);
2658 if (ret != X86EMUL_CONTINUE)
2660 ret = __load_segment_descriptor(ctxt, tss->ds, VCPU_SREG_DS, cpl,
2661 X86_TRANSFER_TASK_SWITCH, NULL);
2662 if (ret != X86EMUL_CONTINUE)
2665 return X86EMUL_CONTINUE;
2668 static int task_switch_16(struct x86_emulate_ctxt *ctxt,
2669 u16 tss_selector, u16 old_tss_sel,
2670 ulong old_tss_base, struct desc_struct *new_desc)
2672 const struct x86_emulate_ops *ops = ctxt->ops;
2673 struct tss_segment_16 tss_seg;
2675 u32 new_tss_base = get_desc_base(new_desc);
2677 ret = ops->read_std(ctxt, old_tss_base, &tss_seg, sizeof tss_seg,
2679 if (ret != X86EMUL_CONTINUE)
2682 save_state_to_tss16(ctxt, &tss_seg);
2684 ret = ops->write_std(ctxt, old_tss_base, &tss_seg, sizeof tss_seg,
2686 if (ret != X86EMUL_CONTINUE)
2689 ret = ops->read_std(ctxt, new_tss_base, &tss_seg, sizeof tss_seg,
2691 if (ret != X86EMUL_CONTINUE)
2694 if (old_tss_sel != 0xffff) {
2695 tss_seg.prev_task_link = old_tss_sel;
2697 ret = ops->write_std(ctxt, new_tss_base,
2698 &tss_seg.prev_task_link,
2699 sizeof tss_seg.prev_task_link,
2701 if (ret != X86EMUL_CONTINUE)
2705 return load_state_from_tss16(ctxt, &tss_seg);
2708 static void save_state_to_tss32(struct x86_emulate_ctxt *ctxt,
2709 struct tss_segment_32 *tss)
2711 /* CR3 and ldt selector are not saved intentionally */
2712 tss->eip = ctxt->_eip;
2713 tss->eflags = ctxt->eflags;
2714 tss->eax = reg_read(ctxt, VCPU_REGS_RAX);
2715 tss->ecx = reg_read(ctxt, VCPU_REGS_RCX);
2716 tss->edx = reg_read(ctxt, VCPU_REGS_RDX);
2717 tss->ebx = reg_read(ctxt, VCPU_REGS_RBX);
2718 tss->esp = reg_read(ctxt, VCPU_REGS_RSP);
2719 tss->ebp = reg_read(ctxt, VCPU_REGS_RBP);
2720 tss->esi = reg_read(ctxt, VCPU_REGS_RSI);
2721 tss->edi = reg_read(ctxt, VCPU_REGS_RDI);
2723 tss->es = get_segment_selector(ctxt, VCPU_SREG_ES);
2724 tss->cs = get_segment_selector(ctxt, VCPU_SREG_CS);
2725 tss->ss = get_segment_selector(ctxt, VCPU_SREG_SS);
2726 tss->ds = get_segment_selector(ctxt, VCPU_SREG_DS);
2727 tss->fs = get_segment_selector(ctxt, VCPU_SREG_FS);
2728 tss->gs = get_segment_selector(ctxt, VCPU_SREG_GS);
2731 static int load_state_from_tss32(struct x86_emulate_ctxt *ctxt,
2732 struct tss_segment_32 *tss)
2737 if (ctxt->ops->set_cr(ctxt, 3, tss->cr3))
2738 return emulate_gp(ctxt, 0);
2739 ctxt->_eip = tss->eip;
2740 ctxt->eflags = tss->eflags | 2;
2742 /* General purpose registers */
2743 *reg_write(ctxt, VCPU_REGS_RAX) = tss->eax;
2744 *reg_write(ctxt, VCPU_REGS_RCX) = tss->ecx;
2745 *reg_write(ctxt, VCPU_REGS_RDX) = tss->edx;
2746 *reg_write(ctxt, VCPU_REGS_RBX) = tss->ebx;
2747 *reg_write(ctxt, VCPU_REGS_RSP) = tss->esp;
2748 *reg_write(ctxt, VCPU_REGS_RBP) = tss->ebp;
2749 *reg_write(ctxt, VCPU_REGS_RSI) = tss->esi;
2750 *reg_write(ctxt, VCPU_REGS_RDI) = tss->edi;
2753 * SDM says that segment selectors are loaded before segment
2754 * descriptors. This is important because CPL checks will
2757 set_segment_selector(ctxt, tss->ldt_selector, VCPU_SREG_LDTR);
2758 set_segment_selector(ctxt, tss->es, VCPU_SREG_ES);
2759 set_segment_selector(ctxt, tss->cs, VCPU_SREG_CS);
2760 set_segment_selector(ctxt, tss->ss, VCPU_SREG_SS);
2761 set_segment_selector(ctxt, tss->ds, VCPU_SREG_DS);
2762 set_segment_selector(ctxt, tss->fs, VCPU_SREG_FS);
2763 set_segment_selector(ctxt, tss->gs, VCPU_SREG_GS);
2766 * If we're switching between Protected Mode and VM86, we need to make
2767 * sure to update the mode before loading the segment descriptors so
2768 * that the selectors are interpreted correctly.
2770 if (ctxt->eflags & X86_EFLAGS_VM) {
2771 ctxt->mode = X86EMUL_MODE_VM86;
2774 ctxt->mode = X86EMUL_MODE_PROT32;
2779 * Now load segment descriptors. If fault happenes at this stage
2780 * it is handled in a context of new task
2782 ret = __load_segment_descriptor(ctxt, tss->ldt_selector, VCPU_SREG_LDTR,
2783 cpl, X86_TRANSFER_TASK_SWITCH, NULL);
2784 if (ret != X86EMUL_CONTINUE)
2786 ret = __load_segment_descriptor(ctxt, tss->es, VCPU_SREG_ES, cpl,
2787 X86_TRANSFER_TASK_SWITCH, NULL);
2788 if (ret != X86EMUL_CONTINUE)
2790 ret = __load_segment_descriptor(ctxt, tss->cs, VCPU_SREG_CS, cpl,
2791 X86_TRANSFER_TASK_SWITCH, NULL);
2792 if (ret != X86EMUL_CONTINUE)
2794 ret = __load_segment_descriptor(ctxt, tss->ss, VCPU_SREG_SS, cpl,
2795 X86_TRANSFER_TASK_SWITCH, NULL);
2796 if (ret != X86EMUL_CONTINUE)
2798 ret = __load_segment_descriptor(ctxt, tss->ds, VCPU_SREG_DS, cpl,
2799 X86_TRANSFER_TASK_SWITCH, NULL);
2800 if (ret != X86EMUL_CONTINUE)
2802 ret = __load_segment_descriptor(ctxt, tss->fs, VCPU_SREG_FS, cpl,
2803 X86_TRANSFER_TASK_SWITCH, NULL);
2804 if (ret != X86EMUL_CONTINUE)
2806 ret = __load_segment_descriptor(ctxt, tss->gs, VCPU_SREG_GS, cpl,
2807 X86_TRANSFER_TASK_SWITCH, NULL);
2808 if (ret != X86EMUL_CONTINUE)
2811 return X86EMUL_CONTINUE;
2814 static int task_switch_32(struct x86_emulate_ctxt *ctxt,
2815 u16 tss_selector, u16 old_tss_sel,
2816 ulong old_tss_base, struct desc_struct *new_desc)
2818 const struct x86_emulate_ops *ops = ctxt->ops;
2819 struct tss_segment_32 tss_seg;
2821 u32 new_tss_base = get_desc_base(new_desc);
2822 u32 eip_offset = offsetof(struct tss_segment_32, eip);
2823 u32 ldt_sel_offset = offsetof(struct tss_segment_32, ldt_selector);
2825 ret = ops->read_std(ctxt, old_tss_base, &tss_seg, sizeof tss_seg,
2827 if (ret != X86EMUL_CONTINUE)
2830 save_state_to_tss32(ctxt, &tss_seg);
2832 /* Only GP registers and segment selectors are saved */
2833 ret = ops->write_std(ctxt, old_tss_base + eip_offset, &tss_seg.eip,
2834 ldt_sel_offset - eip_offset, &ctxt->exception);
2835 if (ret != X86EMUL_CONTINUE)
2838 ret = ops->read_std(ctxt, new_tss_base, &tss_seg, sizeof tss_seg,
2840 if (ret != X86EMUL_CONTINUE)
2843 if (old_tss_sel != 0xffff) {
2844 tss_seg.prev_task_link = old_tss_sel;
2846 ret = ops->write_std(ctxt, new_tss_base,
2847 &tss_seg.prev_task_link,
2848 sizeof tss_seg.prev_task_link,
2850 if (ret != X86EMUL_CONTINUE)
2854 return load_state_from_tss32(ctxt, &tss_seg);
2857 static int emulator_do_task_switch(struct x86_emulate_ctxt *ctxt,
2858 u16 tss_selector, int idt_index, int reason,
2859 bool has_error_code, u32 error_code)
2861 const struct x86_emulate_ops *ops = ctxt->ops;
2862 struct desc_struct curr_tss_desc, next_tss_desc;
2864 u16 old_tss_sel = get_segment_selector(ctxt, VCPU_SREG_TR);
2865 ulong old_tss_base =
2866 ops->get_cached_segment_base(ctxt, VCPU_SREG_TR);
2870 /* FIXME: old_tss_base == ~0 ? */
2872 ret = read_segment_descriptor(ctxt, tss_selector, &next_tss_desc, &desc_addr);
2873 if (ret != X86EMUL_CONTINUE)
2875 ret = read_segment_descriptor(ctxt, old_tss_sel, &curr_tss_desc, &desc_addr);
2876 if (ret != X86EMUL_CONTINUE)
2879 /* FIXME: check that next_tss_desc is tss */
2882 * Check privileges. The three cases are task switch caused by...
2884 * 1. jmp/call/int to task gate: Check against DPL of the task gate
2885 * 2. Exception/IRQ/iret: No check is performed
2886 * 3. jmp/call to TSS/task-gate: No check is performed since the
2887 * hardware checks it before exiting.
2889 if (reason == TASK_SWITCH_GATE) {
2890 if (idt_index != -1) {
2891 /* Software interrupts */
2892 struct desc_struct task_gate_desc;
2895 ret = read_interrupt_descriptor(ctxt, idt_index,
2897 if (ret != X86EMUL_CONTINUE)
2900 dpl = task_gate_desc.dpl;
2901 if ((tss_selector & 3) > dpl || ops->cpl(ctxt) > dpl)
2902 return emulate_gp(ctxt, (idt_index << 3) | 0x2);
2906 desc_limit = desc_limit_scaled(&next_tss_desc);
2907 if (!next_tss_desc.p ||
2908 ((desc_limit < 0x67 && (next_tss_desc.type & 8)) ||
2909 desc_limit < 0x2b)) {
2910 return emulate_ts(ctxt, tss_selector & 0xfffc);
2913 if (reason == TASK_SWITCH_IRET || reason == TASK_SWITCH_JMP) {
2914 curr_tss_desc.type &= ~(1 << 1); /* clear busy flag */
2915 write_segment_descriptor(ctxt, old_tss_sel, &curr_tss_desc);
2918 if (reason == TASK_SWITCH_IRET)
2919 ctxt->eflags = ctxt->eflags & ~X86_EFLAGS_NT;
2921 /* set back link to prev task only if NT bit is set in eflags
2922 note that old_tss_sel is not used after this point */
2923 if (reason != TASK_SWITCH_CALL && reason != TASK_SWITCH_GATE)
2924 old_tss_sel = 0xffff;
2926 if (next_tss_desc.type & 8)
2927 ret = task_switch_32(ctxt, tss_selector, old_tss_sel,
2928 old_tss_base, &next_tss_desc);
2930 ret = task_switch_16(ctxt, tss_selector, old_tss_sel,
2931 old_tss_base, &next_tss_desc);
2932 if (ret != X86EMUL_CONTINUE)
2935 if (reason == TASK_SWITCH_CALL || reason == TASK_SWITCH_GATE)
2936 ctxt->eflags = ctxt->eflags | X86_EFLAGS_NT;
2938 if (reason != TASK_SWITCH_IRET) {
2939 next_tss_desc.type |= (1 << 1); /* set busy flag */
2940 write_segment_descriptor(ctxt, tss_selector, &next_tss_desc);
2943 ops->set_cr(ctxt, 0, ops->get_cr(ctxt, 0) | X86_CR0_TS);
2944 ops->set_segment(ctxt, tss_selector, &next_tss_desc, 0, VCPU_SREG_TR);
2946 if (has_error_code) {
2947 ctxt->op_bytes = ctxt->ad_bytes = (next_tss_desc.type & 8) ? 4 : 2;
2948 ctxt->lock_prefix = 0;
2949 ctxt->src.val = (unsigned long) error_code;
2950 ret = em_push(ctxt);
2956 int emulator_task_switch(struct x86_emulate_ctxt *ctxt,
2957 u16 tss_selector, int idt_index, int reason,
2958 bool has_error_code, u32 error_code)
2962 invalidate_registers(ctxt);
2963 ctxt->_eip = ctxt->eip;
2964 ctxt->dst.type = OP_NONE;
2966 rc = emulator_do_task_switch(ctxt, tss_selector, idt_index, reason,
2967 has_error_code, error_code);
2969 if (rc == X86EMUL_CONTINUE) {
2970 ctxt->eip = ctxt->_eip;
2971 writeback_registers(ctxt);
2974 return (rc == X86EMUL_UNHANDLEABLE) ? EMULATION_FAILED : EMULATION_OK;
2977 static void string_addr_inc(struct x86_emulate_ctxt *ctxt, int reg,
2980 int df = (ctxt->eflags & EFLG_DF) ? -op->count : op->count;
2982 register_address_increment(ctxt, reg, df * op->bytes);
2983 op->addr.mem.ea = register_address(ctxt, reg);
2986 static int em_das(struct x86_emulate_ctxt *ctxt)
2989 bool af, cf, old_cf;
2991 cf = ctxt->eflags & X86_EFLAGS_CF;
2997 af = ctxt->eflags & X86_EFLAGS_AF;
2998 if ((al & 0x0f) > 9 || af) {
3000 cf = old_cf | (al >= 250);
3005 if (old_al > 0x99 || old_cf) {
3011 /* Set PF, ZF, SF */
3012 ctxt->src.type = OP_IMM;
3014 ctxt->src.bytes = 1;
3015 fastop(ctxt, em_or);
3016 ctxt->eflags &= ~(X86_EFLAGS_AF | X86_EFLAGS_CF);
3018 ctxt->eflags |= X86_EFLAGS_CF;
3020 ctxt->eflags |= X86_EFLAGS_AF;
3021 return X86EMUL_CONTINUE;
3024 static int em_aam(struct x86_emulate_ctxt *ctxt)
3028 if (ctxt->src.val == 0)
3029 return emulate_de(ctxt);
3031 al = ctxt->dst.val & 0xff;
3032 ah = al / ctxt->src.val;
3033 al %= ctxt->src.val;
3035 ctxt->dst.val = (ctxt->dst.val & 0xffff0000) | al | (ah << 8);
3037 /* Set PF, ZF, SF */
3038 ctxt->src.type = OP_IMM;
3040 ctxt->src.bytes = 1;
3041 fastop(ctxt, em_or);
3043 return X86EMUL_CONTINUE;
3046 static int em_aad(struct x86_emulate_ctxt *ctxt)
3048 u8 al = ctxt->dst.val & 0xff;
3049 u8 ah = (ctxt->dst.val >> 8) & 0xff;
3051 al = (al + (ah * ctxt->src.val)) & 0xff;
3053 ctxt->dst.val = (ctxt->dst.val & 0xffff0000) | al;
3055 /* Set PF, ZF, SF */
3056 ctxt->src.type = OP_IMM;
3058 ctxt->src.bytes = 1;
3059 fastop(ctxt, em_or);
3061 return X86EMUL_CONTINUE;
3064 static int em_call(struct x86_emulate_ctxt *ctxt)
3067 long rel = ctxt->src.val;
3069 ctxt->src.val = (unsigned long)ctxt->_eip;
3070 rc = jmp_rel(ctxt, rel);
3071 if (rc != X86EMUL_CONTINUE)
3073 return em_push(ctxt);
3076 static int em_call_far(struct x86_emulate_ctxt *ctxt)
3081 struct desc_struct old_desc, new_desc;
3082 const struct x86_emulate_ops *ops = ctxt->ops;
3083 int cpl = ctxt->ops->cpl(ctxt);
3084 enum x86emul_mode prev_mode = ctxt->mode;
3086 old_eip = ctxt->_eip;
3087 ops->get_segment(ctxt, &old_cs, &old_desc, NULL, VCPU_SREG_CS);
3089 memcpy(&sel, ctxt->src.valptr + ctxt->op_bytes, 2);
3090 rc = __load_segment_descriptor(ctxt, sel, VCPU_SREG_CS, cpl,
3091 X86_TRANSFER_CALL_JMP, &new_desc);
3092 if (rc != X86EMUL_CONTINUE)
3095 rc = assign_eip_far(ctxt, ctxt->src.val, &new_desc);
3096 if (rc != X86EMUL_CONTINUE)
3099 ctxt->src.val = old_cs;
3101 if (rc != X86EMUL_CONTINUE)
3104 ctxt->src.val = old_eip;
3106 /* If we failed, we tainted the memory, but the very least we should
3108 if (rc != X86EMUL_CONTINUE) {
3109 pr_warn_once("faulting far call emulation tainted memory\n");
3114 ops->set_segment(ctxt, old_cs, &old_desc, 0, VCPU_SREG_CS);
3115 ctxt->mode = prev_mode;
3120 static int em_ret_near_imm(struct x86_emulate_ctxt *ctxt)
3125 rc = emulate_pop(ctxt, &eip, ctxt->op_bytes);
3126 if (rc != X86EMUL_CONTINUE)
3128 rc = assign_eip_near(ctxt, eip);
3129 if (rc != X86EMUL_CONTINUE)
3131 rsp_increment(ctxt, ctxt->src.val);
3132 return X86EMUL_CONTINUE;
3135 static int em_xchg(struct x86_emulate_ctxt *ctxt)
3137 /* Write back the register source. */
3138 ctxt->src.val = ctxt->dst.val;
3139 write_register_operand(&ctxt->src);
3141 /* Write back the memory destination with implicit LOCK prefix. */
3142 ctxt->dst.val = ctxt->src.orig_val;
3143 ctxt->lock_prefix = 1;
3144 return X86EMUL_CONTINUE;
3147 static int em_imul_3op(struct x86_emulate_ctxt *ctxt)
3149 ctxt->dst.val = ctxt->src2.val;
3150 return fastop(ctxt, em_imul);
3153 static int em_cwd(struct x86_emulate_ctxt *ctxt)
3155 ctxt->dst.type = OP_REG;
3156 ctxt->dst.bytes = ctxt->src.bytes;
3157 ctxt->dst.addr.reg = reg_rmw(ctxt, VCPU_REGS_RDX);
3158 ctxt->dst.val = ~((ctxt->src.val >> (ctxt->src.bytes * 8 - 1)) - 1);
3160 return X86EMUL_CONTINUE;
3163 static int em_rdtsc(struct x86_emulate_ctxt *ctxt)
3167 ctxt->ops->get_msr(ctxt, MSR_IA32_TSC, &tsc);
3168 *reg_write(ctxt, VCPU_REGS_RAX) = (u32)tsc;
3169 *reg_write(ctxt, VCPU_REGS_RDX) = tsc >> 32;
3170 return X86EMUL_CONTINUE;
3173 static int em_rdpmc(struct x86_emulate_ctxt *ctxt)
3177 if (ctxt->ops->read_pmc(ctxt, reg_read(ctxt, VCPU_REGS_RCX), &pmc))
3178 return emulate_gp(ctxt, 0);
3179 *reg_write(ctxt, VCPU_REGS_RAX) = (u32)pmc;
3180 *reg_write(ctxt, VCPU_REGS_RDX) = pmc >> 32;
3181 return X86EMUL_CONTINUE;
3184 static int em_mov(struct x86_emulate_ctxt *ctxt)
3186 memcpy(ctxt->dst.valptr, ctxt->src.valptr, sizeof(ctxt->src.valptr));
3187 return X86EMUL_CONTINUE;
3190 #define FFL(x) bit(X86_FEATURE_##x)
3192 static int em_movbe(struct x86_emulate_ctxt *ctxt)
3194 u32 ebx, ecx, edx, eax = 1;
3198 * Check MOVBE is set in the guest-visible CPUID leaf.
3200 ctxt->ops->get_cpuid(ctxt, &eax, &ebx, &ecx, &edx);
3201 if (!(ecx & FFL(MOVBE)))
3202 return emulate_ud(ctxt);
3204 switch (ctxt->op_bytes) {
3207 * From MOVBE definition: "...When the operand size is 16 bits,
3208 * the upper word of the destination register remains unchanged
3211 * Both casting ->valptr and ->val to u16 breaks strict aliasing
3212 * rules so we have to do the operation almost per hand.
3214 tmp = (u16)ctxt->src.val;
3215 ctxt->dst.val &= ~0xffffUL;
3216 ctxt->dst.val |= (unsigned long)swab16(tmp);
3219 ctxt->dst.val = swab32((u32)ctxt->src.val);
3222 ctxt->dst.val = swab64(ctxt->src.val);
3227 return X86EMUL_CONTINUE;
3230 static int em_cr_write(struct x86_emulate_ctxt *ctxt)
3232 if (ctxt->ops->set_cr(ctxt, ctxt->modrm_reg, ctxt->src.val))
3233 return emulate_gp(ctxt, 0);
3235 /* Disable writeback. */
3236 ctxt->dst.type = OP_NONE;
3237 return X86EMUL_CONTINUE;
3240 static int em_dr_write(struct x86_emulate_ctxt *ctxt)
3244 if (ctxt->mode == X86EMUL_MODE_PROT64)
3245 val = ctxt->src.val & ~0ULL;
3247 val = ctxt->src.val & ~0U;
3249 /* #UD condition is already handled. */
3250 if (ctxt->ops->set_dr(ctxt, ctxt->modrm_reg, val) < 0)
3251 return emulate_gp(ctxt, 0);
3253 /* Disable writeback. */
3254 ctxt->dst.type = OP_NONE;
3255 return X86EMUL_CONTINUE;
3258 static int em_wrmsr(struct x86_emulate_ctxt *ctxt)
3262 msr_data = (u32)reg_read(ctxt, VCPU_REGS_RAX)
3263 | ((u64)reg_read(ctxt, VCPU_REGS_RDX) << 32);
3264 if (ctxt->ops->set_msr(ctxt, reg_read(ctxt, VCPU_REGS_RCX), msr_data))
3265 return emulate_gp(ctxt, 0);
3267 return X86EMUL_CONTINUE;
3270 static int em_rdmsr(struct x86_emulate_ctxt *ctxt)
3274 if (ctxt->ops->get_msr(ctxt, reg_read(ctxt, VCPU_REGS_RCX), &msr_data))
3275 return emulate_gp(ctxt, 0);
3277 *reg_write(ctxt, VCPU_REGS_RAX) = (u32)msr_data;
3278 *reg_write(ctxt, VCPU_REGS_RDX) = msr_data >> 32;
3279 return X86EMUL_CONTINUE;
3282 static int em_mov_rm_sreg(struct x86_emulate_ctxt *ctxt)
3284 if (ctxt->modrm_reg > VCPU_SREG_GS)
3285 return emulate_ud(ctxt);
3287 ctxt->dst.val = get_segment_selector(ctxt, ctxt->modrm_reg);
3288 if (ctxt->dst.bytes == 4 && ctxt->dst.type == OP_MEM)
3289 ctxt->dst.bytes = 2;
3290 return X86EMUL_CONTINUE;
3293 static int em_mov_sreg_rm(struct x86_emulate_ctxt *ctxt)
3295 u16 sel = ctxt->src.val;
3297 if (ctxt->modrm_reg == VCPU_SREG_CS || ctxt->modrm_reg > VCPU_SREG_GS)
3298 return emulate_ud(ctxt);
3300 if (ctxt->modrm_reg == VCPU_SREG_SS)
3301 ctxt->interruptibility = KVM_X86_SHADOW_INT_MOV_SS;
3303 /* Disable writeback. */
3304 ctxt->dst.type = OP_NONE;
3305 return load_segment_descriptor(ctxt, sel, ctxt->modrm_reg);
3308 static int em_lldt(struct x86_emulate_ctxt *ctxt)
3310 u16 sel = ctxt->src.val;
3312 /* Disable writeback. */
3313 ctxt->dst.type = OP_NONE;
3314 return load_segment_descriptor(ctxt, sel, VCPU_SREG_LDTR);
3317 static int em_ltr(struct x86_emulate_ctxt *ctxt)
3319 u16 sel = ctxt->src.val;
3321 /* Disable writeback. */
3322 ctxt->dst.type = OP_NONE;
3323 return load_segment_descriptor(ctxt, sel, VCPU_SREG_TR);
3326 static int em_invlpg(struct x86_emulate_ctxt *ctxt)
3331 rc = linearize(ctxt, ctxt->src.addr.mem, 1, false, &linear);
3332 if (rc == X86EMUL_CONTINUE)
3333 ctxt->ops->invlpg(ctxt, linear);
3334 /* Disable writeback. */
3335 ctxt->dst.type = OP_NONE;
3336 return X86EMUL_CONTINUE;
3339 static int em_clts(struct x86_emulate_ctxt *ctxt)
3343 cr0 = ctxt->ops->get_cr(ctxt, 0);
3345 ctxt->ops->set_cr(ctxt, 0, cr0);
3346 return X86EMUL_CONTINUE;
3349 static int em_hypercall(struct x86_emulate_ctxt *ctxt)
3351 int rc = ctxt->ops->fix_hypercall(ctxt);
3353 if (rc != X86EMUL_CONTINUE)
3356 /* Let the processor re-execute the fixed hypercall */
3357 ctxt->_eip = ctxt->eip;
3358 /* Disable writeback. */
3359 ctxt->dst.type = OP_NONE;
3360 return X86EMUL_CONTINUE;
3363 static int emulate_store_desc_ptr(struct x86_emulate_ctxt *ctxt,
3364 void (*get)(struct x86_emulate_ctxt *ctxt,
3365 struct desc_ptr *ptr))
3367 struct desc_ptr desc_ptr;
3369 if (ctxt->mode == X86EMUL_MODE_PROT64)
3371 get(ctxt, &desc_ptr);
3372 if (ctxt->op_bytes == 2) {
3374 desc_ptr.address &= 0x00ffffff;
3376 /* Disable writeback. */
3377 ctxt->dst.type = OP_NONE;
3378 return segmented_write(ctxt, ctxt->dst.addr.mem,
3379 &desc_ptr, 2 + ctxt->op_bytes);
3382 static int em_sgdt(struct x86_emulate_ctxt *ctxt)
3384 return emulate_store_desc_ptr(ctxt, ctxt->ops->get_gdt);
3387 static int em_sidt(struct x86_emulate_ctxt *ctxt)
3389 return emulate_store_desc_ptr(ctxt, ctxt->ops->get_idt);
3392 static int em_lgdt_lidt(struct x86_emulate_ctxt *ctxt, bool lgdt)
3394 struct desc_ptr desc_ptr;
3397 if (ctxt->mode == X86EMUL_MODE_PROT64)
3399 rc = read_descriptor(ctxt, ctxt->src.addr.mem,
3400 &desc_ptr.size, &desc_ptr.address,
3402 if (rc != X86EMUL_CONTINUE)
3404 if (ctxt->mode == X86EMUL_MODE_PROT64 &&
3405 is_noncanonical_address(desc_ptr.address))
3406 return emulate_gp(ctxt, 0);
3408 ctxt->ops->set_gdt(ctxt, &desc_ptr);
3410 ctxt->ops->set_idt(ctxt, &desc_ptr);
3411 /* Disable writeback. */
3412 ctxt->dst.type = OP_NONE;
3413 return X86EMUL_CONTINUE;
3416 static int em_lgdt(struct x86_emulate_ctxt *ctxt)
3418 return em_lgdt_lidt(ctxt, true);
3421 static int em_lidt(struct x86_emulate_ctxt *ctxt)
3423 return em_lgdt_lidt(ctxt, false);
3426 static int em_smsw(struct x86_emulate_ctxt *ctxt)
3428 if (ctxt->dst.type == OP_MEM)
3429 ctxt->dst.bytes = 2;
3430 ctxt->dst.val = ctxt->ops->get_cr(ctxt, 0);
3431 return X86EMUL_CONTINUE;
3434 static int em_lmsw(struct x86_emulate_ctxt *ctxt)
3436 ctxt->ops->set_cr(ctxt, 0, (ctxt->ops->get_cr(ctxt, 0) & ~0x0eul)
3437 | (ctxt->src.val & 0x0f));
3438 ctxt->dst.type = OP_NONE;
3439 return X86EMUL_CONTINUE;
3442 static int em_loop(struct x86_emulate_ctxt *ctxt)
3444 int rc = X86EMUL_CONTINUE;
3446 register_address_increment(ctxt, VCPU_REGS_RCX, -1);
3447 if ((address_mask(ctxt, reg_read(ctxt, VCPU_REGS_RCX)) != 0) &&
3448 (ctxt->b == 0xe2 || test_cc(ctxt->b ^ 0x5, ctxt->eflags)))
3449 rc = jmp_rel(ctxt, ctxt->src.val);
3454 static int em_jcxz(struct x86_emulate_ctxt *ctxt)
3456 int rc = X86EMUL_CONTINUE;
3458 if (address_mask(ctxt, reg_read(ctxt, VCPU_REGS_RCX)) == 0)
3459 rc = jmp_rel(ctxt, ctxt->src.val);
3464 static int em_in(struct x86_emulate_ctxt *ctxt)
3466 if (!pio_in_emulated(ctxt, ctxt->dst.bytes, ctxt->src.val,
3468 return X86EMUL_IO_NEEDED;
3470 return X86EMUL_CONTINUE;
3473 static int em_out(struct x86_emulate_ctxt *ctxt)
3475 ctxt->ops->pio_out_emulated(ctxt, ctxt->src.bytes, ctxt->dst.val,
3477 /* Disable writeback. */
3478 ctxt->dst.type = OP_NONE;
3479 return X86EMUL_CONTINUE;
3482 static int em_cli(struct x86_emulate_ctxt *ctxt)
3484 if (emulator_bad_iopl(ctxt))
3485 return emulate_gp(ctxt, 0);
3487 ctxt->eflags &= ~X86_EFLAGS_IF;
3488 return X86EMUL_CONTINUE;
3491 static int em_sti(struct x86_emulate_ctxt *ctxt)
3493 if (emulator_bad_iopl(ctxt))
3494 return emulate_gp(ctxt, 0);
3496 ctxt->interruptibility = KVM_X86_SHADOW_INT_STI;
3497 ctxt->eflags |= X86_EFLAGS_IF;
3498 return X86EMUL_CONTINUE;
3501 static int em_cpuid(struct x86_emulate_ctxt *ctxt)
3503 u32 eax, ebx, ecx, edx;
3505 eax = reg_read(ctxt, VCPU_REGS_RAX);
3506 ecx = reg_read(ctxt, VCPU_REGS_RCX);
3507 ctxt->ops->get_cpuid(ctxt, &eax, &ebx, &ecx, &edx);
3508 *reg_write(ctxt, VCPU_REGS_RAX) = eax;
3509 *reg_write(ctxt, VCPU_REGS_RBX) = ebx;
3510 *reg_write(ctxt, VCPU_REGS_RCX) = ecx;
3511 *reg_write(ctxt, VCPU_REGS_RDX) = edx;
3512 return X86EMUL_CONTINUE;
3515 static int em_sahf(struct x86_emulate_ctxt *ctxt)
3519 flags = EFLG_CF | EFLG_PF | EFLG_AF | EFLG_ZF | EFLG_SF;
3520 flags &= *reg_rmw(ctxt, VCPU_REGS_RAX) >> 8;
3522 ctxt->eflags &= ~0xffUL;
3523 ctxt->eflags |= flags | X86_EFLAGS_FIXED;
3524 return X86EMUL_CONTINUE;
3527 static int em_lahf(struct x86_emulate_ctxt *ctxt)
3529 *reg_rmw(ctxt, VCPU_REGS_RAX) &= ~0xff00UL;
3530 *reg_rmw(ctxt, VCPU_REGS_RAX) |= (ctxt->eflags & 0xff) << 8;
3531 return X86EMUL_CONTINUE;
3534 static int em_bswap(struct x86_emulate_ctxt *ctxt)
3536 switch (ctxt->op_bytes) {
3537 #ifdef CONFIG_X86_64
3539 asm("bswap %0" : "+r"(ctxt->dst.val));
3543 asm("bswap %0" : "+r"(*(u32 *)&ctxt->dst.val));
3546 return X86EMUL_CONTINUE;
3549 static int em_clflush(struct x86_emulate_ctxt *ctxt)
3551 /* emulating clflush regardless of cpuid */
3552 return X86EMUL_CONTINUE;
3555 static int em_movsxd(struct x86_emulate_ctxt *ctxt)
3557 ctxt->dst.val = (s32) ctxt->src.val;
3558 return X86EMUL_CONTINUE;
3561 static bool valid_cr(int nr)
3573 static int check_cr_read(struct x86_emulate_ctxt *ctxt)
3575 if (!valid_cr(ctxt->modrm_reg))
3576 return emulate_ud(ctxt);
3578 return X86EMUL_CONTINUE;
3581 static int check_cr_write(struct x86_emulate_ctxt *ctxt)
3583 u64 new_val = ctxt->src.val64;
3584 int cr = ctxt->modrm_reg;
3587 static u64 cr_reserved_bits[] = {
3588 0xffffffff00000000ULL,
3589 0, 0, 0, /* CR3 checked later */
3596 return emulate_ud(ctxt);
3598 if (new_val & cr_reserved_bits[cr])
3599 return emulate_gp(ctxt, 0);
3604 if (((new_val & X86_CR0_PG) && !(new_val & X86_CR0_PE)) ||
3605 ((new_val & X86_CR0_NW) && !(new_val & X86_CR0_CD)))
3606 return emulate_gp(ctxt, 0);
3608 cr4 = ctxt->ops->get_cr(ctxt, 4);
3609 ctxt->ops->get_msr(ctxt, MSR_EFER, &efer);
3611 if ((new_val & X86_CR0_PG) && (efer & EFER_LME) &&
3612 !(cr4 & X86_CR4_PAE))
3613 return emulate_gp(ctxt, 0);
3620 ctxt->ops->get_msr(ctxt, MSR_EFER, &efer);
3621 if (efer & EFER_LMA)
3622 rsvd = CR3_L_MODE_RESERVED_BITS & ~CR3_PCID_INVD;
3625 return emulate_gp(ctxt, 0);
3630 ctxt->ops->get_msr(ctxt, MSR_EFER, &efer);
3632 if ((efer & EFER_LMA) && !(new_val & X86_CR4_PAE))
3633 return emulate_gp(ctxt, 0);
3639 return X86EMUL_CONTINUE;
3642 static int check_dr7_gd(struct x86_emulate_ctxt *ctxt)
3646 ctxt->ops->get_dr(ctxt, 7, &dr7);
3648 /* Check if DR7.Global_Enable is set */
3649 return dr7 & (1 << 13);
3652 static int check_dr_read(struct x86_emulate_ctxt *ctxt)
3654 int dr = ctxt->modrm_reg;
3658 return emulate_ud(ctxt);
3660 cr4 = ctxt->ops->get_cr(ctxt, 4);
3661 if ((cr4 & X86_CR4_DE) && (dr == 4 || dr == 5))
3662 return emulate_ud(ctxt);
3664 if (check_dr7_gd(ctxt)) {
3667 ctxt->ops->get_dr(ctxt, 6, &dr6);
3669 dr6 |= DR6_BD | DR6_RTM;
3670 ctxt->ops->set_dr(ctxt, 6, dr6);
3671 return emulate_db(ctxt);
3674 return X86EMUL_CONTINUE;
3677 static int check_dr_write(struct x86_emulate_ctxt *ctxt)
3679 u64 new_val = ctxt->src.val64;
3680 int dr = ctxt->modrm_reg;
3682 if ((dr == 6 || dr == 7) && (new_val & 0xffffffff00000000ULL))
3683 return emulate_gp(ctxt, 0);
3685 return check_dr_read(ctxt);
3688 static int check_svme(struct x86_emulate_ctxt *ctxt)
3692 ctxt->ops->get_msr(ctxt, MSR_EFER, &efer);
3694 if (!(efer & EFER_SVME))
3695 return emulate_ud(ctxt);
3697 return X86EMUL_CONTINUE;
3700 static int check_svme_pa(struct x86_emulate_ctxt *ctxt)
3702 u64 rax = reg_read(ctxt, VCPU_REGS_RAX);
3704 /* Valid physical address? */
3705 if (rax & 0xffff000000000000ULL)
3706 return emulate_gp(ctxt, 0);
3708 return check_svme(ctxt);
3711 static int check_rdtsc(struct x86_emulate_ctxt *ctxt)
3713 u64 cr4 = ctxt->ops->get_cr(ctxt, 4);
3715 if (cr4 & X86_CR4_TSD && ctxt->ops->cpl(ctxt))
3716 return emulate_ud(ctxt);
3718 return X86EMUL_CONTINUE;
3721 static int check_rdpmc(struct x86_emulate_ctxt *ctxt)
3723 u64 cr4 = ctxt->ops->get_cr(ctxt, 4);
3724 u64 rcx = reg_read(ctxt, VCPU_REGS_RCX);
3726 if ((!(cr4 & X86_CR4_PCE) && ctxt->ops->cpl(ctxt)) ||
3727 ctxt->ops->check_pmc(ctxt, rcx))
3728 return emulate_gp(ctxt, 0);
3730 return X86EMUL_CONTINUE;
3733 static int check_perm_in(struct x86_emulate_ctxt *ctxt)
3735 ctxt->dst.bytes = min(ctxt->dst.bytes, 4u);
3736 if (!emulator_io_permited(ctxt, ctxt->src.val, ctxt->dst.bytes))
3737 return emulate_gp(ctxt, 0);
3739 return X86EMUL_CONTINUE;
3742 static int check_perm_out(struct x86_emulate_ctxt *ctxt)
3744 ctxt->src.bytes = min(ctxt->src.bytes, 4u);
3745 if (!emulator_io_permited(ctxt, ctxt->dst.val, ctxt->src.bytes))
3746 return emulate_gp(ctxt, 0);
3748 return X86EMUL_CONTINUE;
3751 #define D(_y) { .flags = (_y) }
3752 #define DI(_y, _i) { .flags = (_y)|Intercept, .intercept = x86_intercept_##_i }
3753 #define DIP(_y, _i, _p) { .flags = (_y)|Intercept|CheckPerm, \
3754 .intercept = x86_intercept_##_i, .check_perm = (_p) }
3755 #define N D(NotImpl)
3756 #define EXT(_f, _e) { .flags = ((_f) | RMExt), .u.group = (_e) }
3757 #define G(_f, _g) { .flags = ((_f) | Group | ModRM), .u.group = (_g) }
3758 #define GD(_f, _g) { .flags = ((_f) | GroupDual | ModRM), .u.gdual = (_g) }
3759 #define ID(_f, _i) { .flags = ((_f) | InstrDual | ModRM), .u.idual = (_i) }
3760 #define MD(_f, _m) { .flags = ((_f) | ModeDual), .u.mdual = (_m) }
3761 #define E(_f, _e) { .flags = ((_f) | Escape | ModRM), .u.esc = (_e) }
3762 #define I(_f, _e) { .flags = (_f), .u.execute = (_e) }
3763 #define F(_f, _e) { .flags = (_f) | Fastop, .u.fastop = (_e) }
3764 #define II(_f, _e, _i) \
3765 { .flags = (_f)|Intercept, .u.execute = (_e), .intercept = x86_intercept_##_i }
3766 #define IIP(_f, _e, _i, _p) \
3767 { .flags = (_f)|Intercept|CheckPerm, .u.execute = (_e), \
3768 .intercept = x86_intercept_##_i, .check_perm = (_p) }
3769 #define GP(_f, _g) { .flags = ((_f) | Prefix), .u.gprefix = (_g) }
3771 #define D2bv(_f) D((_f) | ByteOp), D(_f)
3772 #define D2bvIP(_f, _i, _p) DIP((_f) | ByteOp, _i, _p), DIP(_f, _i, _p)
3773 #define I2bv(_f, _e) I((_f) | ByteOp, _e), I(_f, _e)
3774 #define F2bv(_f, _e) F((_f) | ByteOp, _e), F(_f, _e)
3775 #define I2bvIP(_f, _e, _i, _p) \
3776 IIP((_f) | ByteOp, _e, _i, _p), IIP(_f, _e, _i, _p)
3778 #define F6ALU(_f, _e) F2bv((_f) | DstMem | SrcReg | ModRM, _e), \
3779 F2bv(((_f) | DstReg | SrcMem | ModRM) & ~Lock, _e), \
3780 F2bv(((_f) & ~Lock) | DstAcc | SrcImm, _e)
3782 static const struct opcode group7_rm0[] = {
3784 I(SrcNone | Priv | EmulateOnUD, em_hypercall),
3788 static const struct opcode group7_rm1[] = {
3789 DI(SrcNone | Priv, monitor),
3790 DI(SrcNone | Priv, mwait),
3794 static const struct opcode group7_rm3[] = {
3795 DIP(SrcNone | Prot | Priv, vmrun, check_svme_pa),
3796 II(SrcNone | Prot | EmulateOnUD, em_hypercall, vmmcall),
3797 DIP(SrcNone | Prot | Priv, vmload, check_svme_pa),
3798 DIP(SrcNone | Prot | Priv, vmsave, check_svme_pa),
3799 DIP(SrcNone | Prot | Priv, stgi, check_svme),
3800 DIP(SrcNone | Prot | Priv, clgi, check_svme),
3801 DIP(SrcNone | Prot | Priv, skinit, check_svme),
3802 DIP(SrcNone | Prot | Priv, invlpga, check_svme),
3805 static const struct opcode group7_rm7[] = {
3807 DIP(SrcNone, rdtscp, check_rdtsc),
3811 static const struct opcode group1[] = {
3813 F(Lock | PageTable, em_or),
3816 F(Lock | PageTable, em_and),
3822 static const struct opcode group1A[] = {
3823 I(DstMem | SrcNone | Mov | Stack | IncSP, em_pop), N, N, N, N, N, N, N,
3826 static const struct opcode group2[] = {
3827 F(DstMem | ModRM, em_rol),
3828 F(DstMem | ModRM, em_ror),
3829 F(DstMem | ModRM, em_rcl),
3830 F(DstMem | ModRM, em_rcr),
3831 F(DstMem | ModRM, em_shl),
3832 F(DstMem | ModRM, em_shr),
3833 F(DstMem | ModRM, em_shl),
3834 F(DstMem | ModRM, em_sar),
3837 static const struct opcode group3[] = {
3838 F(DstMem | SrcImm | NoWrite, em_test),
3839 F(DstMem | SrcImm | NoWrite, em_test),
3840 F(DstMem | SrcNone | Lock, em_not),
3841 F(DstMem | SrcNone | Lock, em_neg),
3842 F(DstXacc | Src2Mem, em_mul_ex),
3843 F(DstXacc | Src2Mem, em_imul_ex),
3844 F(DstXacc | Src2Mem, em_div_ex),
3845 F(DstXacc | Src2Mem, em_idiv_ex),
3848 static const struct opcode group4[] = {
3849 F(ByteOp | DstMem | SrcNone | Lock, em_inc),
3850 F(ByteOp | DstMem | SrcNone | Lock, em_dec),
3854 static const struct opcode group5[] = {
3855 F(DstMem | SrcNone | Lock, em_inc),
3856 F(DstMem | SrcNone | Lock, em_dec),
3857 I(SrcMem | NearBranch, em_call_near_abs),
3858 I(SrcMemFAddr | ImplicitOps | Stack, em_call_far),
3859 I(SrcMem | NearBranch, em_jmp_abs),
3860 I(SrcMemFAddr | ImplicitOps, em_jmp_far),
3861 I(SrcMem | Stack, em_push), D(Undefined),
3864 static const struct opcode group6[] = {
3865 DI(Prot | DstMem, sldt),
3866 DI(Prot | DstMem, str),
3867 II(Prot | Priv | SrcMem16, em_lldt, lldt),
3868 II(Prot | Priv | SrcMem16, em_ltr, ltr),
3872 static const struct group_dual group7 = { {
3873 II(Mov | DstMem, em_sgdt, sgdt),
3874 II(Mov | DstMem, em_sidt, sidt),
3875 II(SrcMem | Priv, em_lgdt, lgdt),
3876 II(SrcMem | Priv, em_lidt, lidt),
3877 II(SrcNone | DstMem | Mov, em_smsw, smsw), N,
3878 II(SrcMem16 | Mov | Priv, em_lmsw, lmsw),
3879 II(SrcMem | ByteOp | Priv | NoAccess, em_invlpg, invlpg),
3883 N, EXT(0, group7_rm3),
3884 II(SrcNone | DstMem | Mov, em_smsw, smsw), N,
3885 II(SrcMem16 | Mov | Priv, em_lmsw, lmsw),
3889 static const struct opcode group8[] = {
3891 F(DstMem | SrcImmByte | NoWrite, em_bt),
3892 F(DstMem | SrcImmByte | Lock | PageTable, em_bts),
3893 F(DstMem | SrcImmByte | Lock, em_btr),
3894 F(DstMem | SrcImmByte | Lock | PageTable, em_btc),
3897 static const struct group_dual group9 = { {
3898 N, I(DstMem64 | Lock | PageTable, em_cmpxchg8b), N, N, N, N, N, N,
3900 N, N, N, N, N, N, N, N,
3903 static const struct opcode group11[] = {
3904 I(DstMem | SrcImm | Mov | PageTable, em_mov),
3908 static const struct gprefix pfx_0f_ae_7 = {
3909 I(SrcMem | ByteOp, em_clflush), N, N, N,
3912 static const struct group_dual group15 = { {
3913 N, N, N, N, N, N, N, GP(0, &pfx_0f_ae_7),
3915 N, N, N, N, N, N, N, N,
3918 static const struct gprefix pfx_0f_6f_0f_7f = {
3919 I(Mmx, em_mov), I(Sse | Aligned, em_mov), N, I(Sse | Unaligned, em_mov),
3922 static const struct instr_dual instr_dual_0f_2b = {
3926 static const struct gprefix pfx_0f_2b = {
3927 ID(0, &instr_dual_0f_2b), ID(0, &instr_dual_0f_2b), N, N,
3930 static const struct gprefix pfx_0f_28_0f_29 = {
3931 I(Aligned, em_mov), I(Aligned, em_mov), N, N,
3934 static const struct gprefix pfx_0f_e7 = {
3935 N, I(Sse, em_mov), N, N,
3938 static const struct escape escape_d9 = { {
3939 N, N, N, N, N, N, N, I(DstMem16 | Mov, em_fnstcw),
3942 N, N, N, N, N, N, N, N,
3944 N, N, N, N, N, N, N, N,
3946 N, N, N, N, N, N, N, N,
3948 N, N, N, N, N, N, N, N,
3950 N, N, N, N, N, N, N, N,
3952 N, N, N, N, N, N, N, N,
3954 N, N, N, N, N, N, N, N,
3956 N, N, N, N, N, N, N, N,
3959 static const struct escape escape_db = { {
3960 N, N, N, N, N, N, N, N,
3963 N, N, N, N, N, N, N, N,
3965 N, N, N, N, N, N, N, N,
3967 N, N, N, N, N, N, N, N,
3969 N, N, N, N, N, N, N, N,
3971 N, N, N, I(ImplicitOps, em_fninit), N, N, N, N,
3973 N, N, N, N, N, N, N, N,
3975 N, N, N, N, N, N, N, N,
3977 N, N, N, N, N, N, N, N,
3980 static const struct escape escape_dd = { {
3981 N, N, N, N, N, N, N, I(DstMem16 | Mov, em_fnstsw),
3984 N, N, N, N, N, N, N, N,
3986 N, N, N, N, N, N, N, N,
3988 N, N, N, N, N, N, N, N,
3990 N, N, N, N, N, N, N, N,
3992 N, N, N, N, N, N, N, N,
3994 N, N, N, N, N, N, N, N,
3996 N, N, N, N, N, N, N, N,
3998 N, N, N, N, N, N, N, N,
4001 static const struct instr_dual instr_dual_0f_c3 = {
4002 I(DstMem | SrcReg | ModRM | No16 | Mov, em_mov), N
4005 static const struct mode_dual mode_dual_63 = {
4006 N, I(DstReg | SrcMem32 | ModRM | Mov, em_movsxd)
4009 static const struct opcode opcode_table[256] = {
4011 F6ALU(Lock, em_add),
4012 I(ImplicitOps | Stack | No64 | Src2ES, em_push_sreg),
4013 I(ImplicitOps | Stack | No64 | Src2ES, em_pop_sreg),
4015 F6ALU(Lock | PageTable, em_or),
4016 I(ImplicitOps | Stack | No64 | Src2CS, em_push_sreg),
4019 F6ALU(Lock, em_adc),
4020 I(ImplicitOps | Stack | No64 | Src2SS, em_push_sreg),
4021 I(ImplicitOps | Stack | No64 | Src2SS, em_pop_sreg),
4023 F6ALU(Lock, em_sbb),
4024 I(ImplicitOps | Stack | No64 | Src2DS, em_push_sreg),
4025 I(ImplicitOps | Stack | No64 | Src2DS, em_pop_sreg),
4027 F6ALU(Lock | PageTable, em_and), N, N,
4029 F6ALU(Lock, em_sub), N, I(ByteOp | DstAcc | No64, em_das),
4031 F6ALU(Lock, em_xor), N, N,
4033 F6ALU(NoWrite, em_cmp), N, N,
4035 X8(F(DstReg, em_inc)), X8(F(DstReg, em_dec)),
4037 X8(I(SrcReg | Stack, em_push)),
4039 X8(I(DstReg | Stack, em_pop)),
4041 I(ImplicitOps | Stack | No64, em_pusha),
4042 I(ImplicitOps | Stack | No64, em_popa),
4043 N, MD(ModRM, &mode_dual_63),
4046 I(SrcImm | Mov | Stack, em_push),
4047 I(DstReg | SrcMem | ModRM | Src2Imm, em_imul_3op),
4048 I(SrcImmByte | Mov | Stack, em_push),
4049 I(DstReg | SrcMem | ModRM | Src2ImmByte, em_imul_3op),
4050 I2bvIP(DstDI | SrcDX | Mov | String | Unaligned, em_in, ins, check_perm_in), /* insb, insw/insd */
4051 I2bvIP(SrcSI | DstDX | String, em_out, outs, check_perm_out), /* outsb, outsw/outsd */
4053 X16(D(SrcImmByte | NearBranch)),
4055 G(ByteOp | DstMem | SrcImm, group1),
4056 G(DstMem | SrcImm, group1),
4057 G(ByteOp | DstMem | SrcImm | No64, group1),
4058 G(DstMem | SrcImmByte, group1),
4059 F2bv(DstMem | SrcReg | ModRM | NoWrite, em_test),
4060 I2bv(DstMem | SrcReg | ModRM | Lock | PageTable, em_xchg),
4062 I2bv(DstMem | SrcReg | ModRM | Mov | PageTable, em_mov),
4063 I2bv(DstReg | SrcMem | ModRM | Mov, em_mov),
4064 I(DstMem | SrcNone | ModRM | Mov | PageTable, em_mov_rm_sreg),
4065 D(ModRM | SrcMem | NoAccess | DstReg),
4066 I(ImplicitOps | SrcMem16 | ModRM, em_mov_sreg_rm),
4069 DI(SrcAcc | DstReg, pause), X7(D(SrcAcc | DstReg)),
4071 D(DstAcc | SrcNone), I(ImplicitOps | SrcAcc, em_cwd),
4072 I(SrcImmFAddr | No64, em_call_far), N,
4073 II(ImplicitOps | Stack, em_pushf, pushf),
4074 II(ImplicitOps | Stack, em_popf, popf),
4075 I(ImplicitOps, em_sahf), I(ImplicitOps, em_lahf),
4077 I2bv(DstAcc | SrcMem | Mov | MemAbs, em_mov),
4078 I2bv(DstMem | SrcAcc | Mov | MemAbs | PageTable, em_mov),
4079 I2bv(SrcSI | DstDI | Mov | String, em_mov),
4080 F2bv(SrcSI | DstDI | String | NoWrite, em_cmp_r),
4082 F2bv(DstAcc | SrcImm | NoWrite, em_test),
4083 I2bv(SrcAcc | DstDI | Mov | String, em_mov),
4084 I2bv(SrcSI | DstAcc | Mov | String, em_mov),
4085 F2bv(SrcAcc | DstDI | String | NoWrite, em_cmp_r),
4087 X8(I(ByteOp | DstReg | SrcImm | Mov, em_mov)),
4089 X8(I(DstReg | SrcImm64 | Mov, em_mov)),
4091 G(ByteOp | Src2ImmByte, group2), G(Src2ImmByte, group2),
4092 I(ImplicitOps | NearBranch | SrcImmU16, em_ret_near_imm),
4093 I(ImplicitOps | NearBranch, em_ret),
4094 I(DstReg | SrcMemFAddr | ModRM | No64 | Src2ES, em_lseg),
4095 I(DstReg | SrcMemFAddr | ModRM | No64 | Src2DS, em_lseg),
4096 G(ByteOp, group11), G(0, group11),
4098 I(Stack | SrcImmU16 | Src2ImmByte, em_enter), I(Stack, em_leave),
4099 I(ImplicitOps | SrcImmU16, em_ret_far_imm),
4100 I(ImplicitOps, em_ret_far),
4101 D(ImplicitOps), DI(SrcImmByte, intn),
4102 D(ImplicitOps | No64), II(ImplicitOps, em_iret, iret),
4104 G(Src2One | ByteOp, group2), G(Src2One, group2),
4105 G(Src2CL | ByteOp, group2), G(Src2CL, group2),
4106 I(DstAcc | SrcImmUByte | No64, em_aam),
4107 I(DstAcc | SrcImmUByte | No64, em_aad),
4108 F(DstAcc | ByteOp | No64, em_salc),
4109 I(DstAcc | SrcXLat | ByteOp, em_mov),
4111 N, E(0, &escape_d9), N, E(0, &escape_db), N, E(0, &escape_dd), N, N,
4113 X3(I(SrcImmByte | NearBranch, em_loop)),
4114 I(SrcImmByte | NearBranch, em_jcxz),
4115 I2bvIP(SrcImmUByte | DstAcc, em_in, in, check_perm_in),
4116 I2bvIP(SrcAcc | DstImmUByte, em_out, out, check_perm_out),
4118 I(SrcImm | NearBranch, em_call), D(SrcImm | ImplicitOps | NearBranch),
4119 I(SrcImmFAddr | No64, em_jmp_far),
4120 D(SrcImmByte | ImplicitOps | NearBranch),
4121 I2bvIP(SrcDX | DstAcc, em_in, in, check_perm_in),
4122 I2bvIP(SrcAcc | DstDX, em_out, out, check_perm_out),
4124 N, DI(ImplicitOps, icebp), N, N,
4125 DI(ImplicitOps | Priv, hlt), D(ImplicitOps),
4126 G(ByteOp, group3), G(0, group3),
4128 D(ImplicitOps), D(ImplicitOps),
4129 I(ImplicitOps, em_cli), I(ImplicitOps, em_sti),
4130 D(ImplicitOps), D(ImplicitOps), G(0, group4), G(0, group5),
4133 static const struct opcode twobyte_table[256] = {
4135 G(0, group6), GD(0, &group7), N, N,
4136 N, I(ImplicitOps | EmulateOnUD, em_syscall),
4137 II(ImplicitOps | Priv, em_clts, clts), N,
4138 DI(ImplicitOps | Priv, invd), DI(ImplicitOps | Priv, wbinvd), N, N,
4139 N, D(ImplicitOps | ModRM | SrcMem | NoAccess), N, N,
4141 N, N, N, N, N, N, N, N,
4142 D(ImplicitOps | ModRM | SrcMem | NoAccess),
4143 N, N, N, N, N, N, D(ImplicitOps | ModRM | SrcMem | NoAccess),
4145 DIP(ModRM | DstMem | Priv | Op3264 | NoMod, cr_read, check_cr_read),
4146 DIP(ModRM | DstMem | Priv | Op3264 | NoMod, dr_read, check_dr_read),
4147 IIP(ModRM | SrcMem | Priv | Op3264 | NoMod, em_cr_write, cr_write,
4149 IIP(ModRM | SrcMem | Priv | Op3264 | NoMod, em_dr_write, dr_write,
4152 GP(ModRM | DstReg | SrcMem | Mov | Sse, &pfx_0f_28_0f_29),
4153 GP(ModRM | DstMem | SrcReg | Mov | Sse, &pfx_0f_28_0f_29),
4154 N, GP(ModRM | DstMem | SrcReg | Mov | Sse, &pfx_0f_2b),
4157 II(ImplicitOps | Priv, em_wrmsr, wrmsr),
4158 IIP(ImplicitOps, em_rdtsc, rdtsc, check_rdtsc),
4159 II(ImplicitOps | Priv, em_rdmsr, rdmsr),
4160 IIP(ImplicitOps, em_rdpmc, rdpmc, check_rdpmc),
4161 I(ImplicitOps | EmulateOnUD, em_sysenter),
4162 I(ImplicitOps | Priv | EmulateOnUD, em_sysexit),
4164 N, N, N, N, N, N, N, N,
4166 X16(D(DstReg | SrcMem | ModRM)),
4168 N, N, N, N, N, N, N, N, N, N, N, N, N, N, N, N,
4173 N, N, N, GP(SrcMem | DstReg | ModRM | Mov, &pfx_0f_6f_0f_7f),
4178 N, N, N, GP(SrcReg | DstMem | ModRM | Mov, &pfx_0f_6f_0f_7f),
4180 X16(D(SrcImm | NearBranch)),
4182 X16(D(ByteOp | DstMem | SrcNone | ModRM| Mov)),
4184 I(Stack | Src2FS, em_push_sreg), I(Stack | Src2FS, em_pop_sreg),
4185 II(ImplicitOps, em_cpuid, cpuid),
4186 F(DstMem | SrcReg | ModRM | BitOp | NoWrite, em_bt),
4187 F(DstMem | SrcReg | Src2ImmByte | ModRM, em_shld),
4188 F(DstMem | SrcReg | Src2CL | ModRM, em_shld), N, N,
4190 I(Stack | Src2GS, em_push_sreg), I(Stack | Src2GS, em_pop_sreg),
4191 DI(ImplicitOps, rsm),
4192 F(DstMem | SrcReg | ModRM | BitOp | Lock | PageTable, em_bts),
4193 F(DstMem | SrcReg | Src2ImmByte | ModRM, em_shrd),
4194 F(DstMem | SrcReg | Src2CL | ModRM, em_shrd),
4195 GD(0, &group15), F(DstReg | SrcMem | ModRM, em_imul),
4197 I2bv(DstMem | SrcReg | ModRM | Lock | PageTable | SrcWrite, em_cmpxchg),
4198 I(DstReg | SrcMemFAddr | ModRM | Src2SS, em_lseg),
4199 F(DstMem | SrcReg | ModRM | BitOp | Lock, em_btr),
4200 I(DstReg | SrcMemFAddr | ModRM | Src2FS, em_lseg),
4201 I(DstReg | SrcMemFAddr | ModRM | Src2GS, em_lseg),
4202 D(DstReg | SrcMem8 | ModRM | Mov), D(DstReg | SrcMem16 | ModRM | Mov),
4206 F(DstMem | SrcReg | ModRM | BitOp | Lock | PageTable, em_btc),
4207 I(DstReg | SrcMem | ModRM, em_bsf_c),
4208 I(DstReg | SrcMem | ModRM, em_bsr_c),
4209 D(DstReg | SrcMem8 | ModRM | Mov), D(DstReg | SrcMem16 | ModRM | Mov),
4211 F2bv(DstMem | SrcReg | ModRM | SrcWrite | Lock, em_xadd),
4212 N, ID(0, &instr_dual_0f_c3),
4213 N, N, N, GD(0, &group9),
4215 X8(I(DstReg, em_bswap)),
4217 N, N, N, N, N, N, N, N, N, N, N, N, N, N, N, N,
4219 N, N, N, N, N, N, N, GP(SrcReg | DstMem | ModRM | Mov, &pfx_0f_e7),
4220 N, N, N, N, N, N, N, N,
4222 N, N, N, N, N, N, N, N, N, N, N, N, N, N, N, N
4225 static const struct instr_dual instr_dual_0f_38_f0 = {
4226 I(DstReg | SrcMem | Mov, em_movbe), N
4229 static const struct instr_dual instr_dual_0f_38_f1 = {
4230 I(DstMem | SrcReg | Mov, em_movbe), N
4233 static const struct gprefix three_byte_0f_38_f0 = {
4234 ID(0, &instr_dual_0f_38_f0), N, N, N
4237 static const struct gprefix three_byte_0f_38_f1 = {
4238 ID(0, &instr_dual_0f_38_f1), N, N, N
4242 * Insns below are selected by the prefix which indexed by the third opcode
4245 static const struct opcode opcode_map_0f_38[256] = {
4247 X16(N), X16(N), X16(N), X16(N), X16(N), X16(N), X16(N), X16(N),
4249 X16(N), X16(N), X16(N), X16(N), X16(N), X16(N), X16(N),
4251 GP(EmulateOnUD | ModRM, &three_byte_0f_38_f0),
4252 GP(EmulateOnUD | ModRM, &three_byte_0f_38_f1),
4273 static unsigned imm_size(struct x86_emulate_ctxt *ctxt)
4277 size = (ctxt->d & ByteOp) ? 1 : ctxt->op_bytes;
4283 static int decode_imm(struct x86_emulate_ctxt *ctxt, struct operand *op,
4284 unsigned size, bool sign_extension)
4286 int rc = X86EMUL_CONTINUE;
4290 op->addr.mem.ea = ctxt->_eip;
4291 /* NB. Immediates are sign-extended as necessary. */
4292 switch (op->bytes) {
4294 op->val = insn_fetch(s8, ctxt);
4297 op->val = insn_fetch(s16, ctxt);
4300 op->val = insn_fetch(s32, ctxt);
4303 op->val = insn_fetch(s64, ctxt);
4306 if (!sign_extension) {
4307 switch (op->bytes) {
4315 op->val &= 0xffffffff;
4323 static int decode_operand(struct x86_emulate_ctxt *ctxt, struct operand *op,
4326 int rc = X86EMUL_CONTINUE;
4330 decode_register_operand(ctxt, op);
4333 rc = decode_imm(ctxt, op, 1, false);
4336 ctxt->memop.bytes = (ctxt->d & ByteOp) ? 1 : ctxt->op_bytes;
4340 if (ctxt->d & BitOp)
4341 fetch_bit_operand(ctxt);
4342 op->orig_val = op->val;
4345 ctxt->memop.bytes = (ctxt->op_bytes == 8) ? 16 : 8;
4349 op->bytes = (ctxt->d & ByteOp) ? 1 : ctxt->op_bytes;
4350 op->addr.reg = reg_rmw(ctxt, VCPU_REGS_RAX);
4351 fetch_register_operand(op);
4352 op->orig_val = op->val;
4356 op->bytes = (ctxt->d & ByteOp) ? 2 : ctxt->op_bytes;
4357 op->addr.reg = reg_rmw(ctxt, VCPU_REGS_RAX);
4358 fetch_register_operand(op);
4359 op->orig_val = op->val;
4362 if (ctxt->d & ByteOp) {
4367 op->bytes = ctxt->op_bytes;
4368 op->addr.reg = reg_rmw(ctxt, VCPU_REGS_RDX);
4369 fetch_register_operand(op);
4370 op->orig_val = op->val;
4374 op->bytes = (ctxt->d & ByteOp) ? 1 : ctxt->op_bytes;
4376 register_address(ctxt, VCPU_REGS_RDI);
4377 op->addr.mem.seg = VCPU_SREG_ES;
4384 op->addr.reg = reg_rmw(ctxt, VCPU_REGS_RDX);
4385 fetch_register_operand(op);
4390 op->val = reg_read(ctxt, VCPU_REGS_RCX) & 0xff;
4393 rc = decode_imm(ctxt, op, 1, true);
4401 rc = decode_imm(ctxt, op, imm_size(ctxt), true);
4404 rc = decode_imm(ctxt, op, ctxt->op_bytes, true);
4407 ctxt->memop.bytes = 1;
4408 if (ctxt->memop.type == OP_REG) {
4409 ctxt->memop.addr.reg = decode_register(ctxt,
4410 ctxt->modrm_rm, true);
4411 fetch_register_operand(&ctxt->memop);
4415 ctxt->memop.bytes = 2;
4418 ctxt->memop.bytes = 4;
4421 rc = decode_imm(ctxt, op, 2, false);
4424 rc = decode_imm(ctxt, op, imm_size(ctxt), false);
4428 op->bytes = (ctxt->d & ByteOp) ? 1 : ctxt->op_bytes;
4430 register_address(ctxt, VCPU_REGS_RSI);
4431 op->addr.mem.seg = ctxt->seg_override;
4437 op->bytes = (ctxt->d & ByteOp) ? 1 : ctxt->op_bytes;
4440 reg_read(ctxt, VCPU_REGS_RBX) +
4441 (reg_read(ctxt, VCPU_REGS_RAX) & 0xff));
4442 op->addr.mem.seg = ctxt->seg_override;
4447 op->addr.mem.ea = ctxt->_eip;
4448 op->bytes = ctxt->op_bytes + 2;
4449 insn_fetch_arr(op->valptr, op->bytes, ctxt);
4452 ctxt->memop.bytes = ctxt->op_bytes + 2;
4456 op->val = VCPU_SREG_ES;
4460 op->val = VCPU_SREG_CS;
4464 op->val = VCPU_SREG_SS;
4468 op->val = VCPU_SREG_DS;
4472 op->val = VCPU_SREG_FS;
4476 op->val = VCPU_SREG_GS;
4479 /* Special instructions do their own operand decoding. */
4481 op->type = OP_NONE; /* Disable writeback. */
4489 int x86_decode_insn(struct x86_emulate_ctxt *ctxt, void *insn, int insn_len)
4491 int rc = X86EMUL_CONTINUE;
4492 int mode = ctxt->mode;
4493 int def_op_bytes, def_ad_bytes, goffset, simd_prefix;
4494 bool op_prefix = false;
4495 bool has_seg_override = false;
4496 struct opcode opcode;
4498 ctxt->memop.type = OP_NONE;
4499 ctxt->memopp = NULL;
4500 ctxt->_eip = ctxt->eip;
4501 ctxt->fetch.ptr = ctxt->fetch.data;
4502 ctxt->fetch.end = ctxt->fetch.data + insn_len;
4503 ctxt->opcode_len = 1;
4505 memcpy(ctxt->fetch.data, insn, insn_len);
4507 rc = __do_insn_fetch_bytes(ctxt, 1);
4508 if (rc != X86EMUL_CONTINUE)
4513 case X86EMUL_MODE_REAL:
4514 case X86EMUL_MODE_VM86:
4515 case X86EMUL_MODE_PROT16:
4516 def_op_bytes = def_ad_bytes = 2;
4518 case X86EMUL_MODE_PROT32:
4519 def_op_bytes = def_ad_bytes = 4;
4521 #ifdef CONFIG_X86_64
4522 case X86EMUL_MODE_PROT64:
4528 return EMULATION_FAILED;
4531 ctxt->op_bytes = def_op_bytes;
4532 ctxt->ad_bytes = def_ad_bytes;
4534 /* Legacy prefixes. */
4536 switch (ctxt->b = insn_fetch(u8, ctxt)) {
4537 case 0x66: /* operand-size override */
4539 /* switch between 2/4 bytes */
4540 ctxt->op_bytes = def_op_bytes ^ 6;
4542 case 0x67: /* address-size override */
4543 if (mode == X86EMUL_MODE_PROT64)
4544 /* switch between 4/8 bytes */
4545 ctxt->ad_bytes = def_ad_bytes ^ 12;
4547 /* switch between 2/4 bytes */
4548 ctxt->ad_bytes = def_ad_bytes ^ 6;
4550 case 0x26: /* ES override */
4551 case 0x2e: /* CS override */
4552 case 0x36: /* SS override */
4553 case 0x3e: /* DS override */
4554 has_seg_override = true;
4555 ctxt->seg_override = (ctxt->b >> 3) & 3;
4557 case 0x64: /* FS override */
4558 case 0x65: /* GS override */
4559 has_seg_override = true;
4560 ctxt->seg_override = ctxt->b & 7;
4562 case 0x40 ... 0x4f: /* REX */
4563 if (mode != X86EMUL_MODE_PROT64)
4565 ctxt->rex_prefix = ctxt->b;
4567 case 0xf0: /* LOCK */
4568 ctxt->lock_prefix = 1;
4570 case 0xf2: /* REPNE/REPNZ */
4571 case 0xf3: /* REP/REPE/REPZ */
4572 ctxt->rep_prefix = ctxt->b;
4578 /* Any legacy prefix after a REX prefix nullifies its effect. */
4580 ctxt->rex_prefix = 0;
4586 if (ctxt->rex_prefix & 8)
4587 ctxt->op_bytes = 8; /* REX.W */
4589 /* Opcode byte(s). */
4590 opcode = opcode_table[ctxt->b];
4591 /* Two-byte opcode? */
4592 if (ctxt->b == 0x0f) {
4593 ctxt->opcode_len = 2;
4594 ctxt->b = insn_fetch(u8, ctxt);
4595 opcode = twobyte_table[ctxt->b];
4597 /* 0F_38 opcode map */
4598 if (ctxt->b == 0x38) {
4599 ctxt->opcode_len = 3;
4600 ctxt->b = insn_fetch(u8, ctxt);
4601 opcode = opcode_map_0f_38[ctxt->b];
4604 ctxt->d = opcode.flags;
4606 if (ctxt->d & ModRM)
4607 ctxt->modrm = insn_fetch(u8, ctxt);
4609 /* vex-prefix instructions are not implemented */
4610 if (ctxt->opcode_len == 1 && (ctxt->b == 0xc5 || ctxt->b == 0xc4) &&
4611 (mode == X86EMUL_MODE_PROT64 || (ctxt->modrm & 0xc0) == 0xc0)) {
4615 while (ctxt->d & GroupMask) {
4616 switch (ctxt->d & GroupMask) {
4618 goffset = (ctxt->modrm >> 3) & 7;
4619 opcode = opcode.u.group[goffset];
4622 goffset = (ctxt->modrm >> 3) & 7;
4623 if ((ctxt->modrm >> 6) == 3)
4624 opcode = opcode.u.gdual->mod3[goffset];
4626 opcode = opcode.u.gdual->mod012[goffset];
4629 goffset = ctxt->modrm & 7;
4630 opcode = opcode.u.group[goffset];
4633 if (ctxt->rep_prefix && op_prefix)
4634 return EMULATION_FAILED;
4635 simd_prefix = op_prefix ? 0x66 : ctxt->rep_prefix;
4636 switch (simd_prefix) {
4637 case 0x00: opcode = opcode.u.gprefix->pfx_no; break;
4638 case 0x66: opcode = opcode.u.gprefix->pfx_66; break;
4639 case 0xf2: opcode = opcode.u.gprefix->pfx_f2; break;
4640 case 0xf3: opcode = opcode.u.gprefix->pfx_f3; break;
4644 if (ctxt->modrm > 0xbf)
4645 opcode = opcode.u.esc->high[ctxt->modrm - 0xc0];
4647 opcode = opcode.u.esc->op[(ctxt->modrm >> 3) & 7];
4650 if ((ctxt->modrm >> 6) == 3)
4651 opcode = opcode.u.idual->mod3;
4653 opcode = opcode.u.idual->mod012;
4656 if (ctxt->mode == X86EMUL_MODE_PROT64)
4657 opcode = opcode.u.mdual->mode64;
4659 opcode = opcode.u.mdual->mode32;
4662 return EMULATION_FAILED;
4665 ctxt->d &= ~(u64)GroupMask;
4666 ctxt->d |= opcode.flags;
4671 return EMULATION_FAILED;
4673 ctxt->execute = opcode.u.execute;
4675 if (unlikely(ctxt->ud) && likely(!(ctxt->d & EmulateOnUD)))
4676 return EMULATION_FAILED;
4678 if (unlikely(ctxt->d &
4679 (NotImpl|Stack|Op3264|Sse|Mmx|Intercept|CheckPerm|NearBranch|
4682 * These are copied unconditionally here, and checked unconditionally
4683 * in x86_emulate_insn.
4685 ctxt->check_perm = opcode.check_perm;
4686 ctxt->intercept = opcode.intercept;
4688 if (ctxt->d & NotImpl)
4689 return EMULATION_FAILED;
4691 if (mode == X86EMUL_MODE_PROT64) {
4692 if (ctxt->op_bytes == 4 && (ctxt->d & Stack))
4694 else if (ctxt->d & NearBranch)
4698 if (ctxt->d & Op3264) {
4699 if (mode == X86EMUL_MODE_PROT64)
4705 if ((ctxt->d & No16) && ctxt->op_bytes == 2)
4709 ctxt->op_bytes = 16;
4710 else if (ctxt->d & Mmx)
4714 /* ModRM and SIB bytes. */
4715 if (ctxt->d & ModRM) {
4716 rc = decode_modrm(ctxt, &ctxt->memop);
4717 if (!has_seg_override) {
4718 has_seg_override = true;
4719 ctxt->seg_override = ctxt->modrm_seg;
4721 } else if (ctxt->d & MemAbs)
4722 rc = decode_abs(ctxt, &ctxt->memop);
4723 if (rc != X86EMUL_CONTINUE)
4726 if (!has_seg_override)
4727 ctxt->seg_override = VCPU_SREG_DS;
4729 ctxt->memop.addr.mem.seg = ctxt->seg_override;
4732 * Decode and fetch the source operand: register, memory
4735 rc = decode_operand(ctxt, &ctxt->src, (ctxt->d >> SrcShift) & OpMask);
4736 if (rc != X86EMUL_CONTINUE)
4740 * Decode and fetch the second source operand: register, memory
4743 rc = decode_operand(ctxt, &ctxt->src2, (ctxt->d >> Src2Shift) & OpMask);
4744 if (rc != X86EMUL_CONTINUE)
4747 /* Decode and fetch the destination operand: register or memory. */
4748 rc = decode_operand(ctxt, &ctxt->dst, (ctxt->d >> DstShift) & OpMask);
4750 if (ctxt->rip_relative)
4751 ctxt->memopp->addr.mem.ea = address_mask(ctxt,
4752 ctxt->memopp->addr.mem.ea + ctxt->_eip);
4755 return (rc != X86EMUL_CONTINUE) ? EMULATION_FAILED : EMULATION_OK;
4758 bool x86_page_table_writing_insn(struct x86_emulate_ctxt *ctxt)
4760 return ctxt->d & PageTable;
4763 static bool string_insn_completed(struct x86_emulate_ctxt *ctxt)
4765 /* The second termination condition only applies for REPE
4766 * and REPNE. Test if the repeat string operation prefix is
4767 * REPE/REPZ or REPNE/REPNZ and if it's the case it tests the
4768 * corresponding termination condition according to:
4769 * - if REPE/REPZ and ZF = 0 then done
4770 * - if REPNE/REPNZ and ZF = 1 then done
4772 if (((ctxt->b == 0xa6) || (ctxt->b == 0xa7) ||
4773 (ctxt->b == 0xae) || (ctxt->b == 0xaf))
4774 && (((ctxt->rep_prefix == REPE_PREFIX) &&
4775 ((ctxt->eflags & EFLG_ZF) == 0))
4776 || ((ctxt->rep_prefix == REPNE_PREFIX) &&
4777 ((ctxt->eflags & EFLG_ZF) == EFLG_ZF))))
4783 static int flush_pending_x87_faults(struct x86_emulate_ctxt *ctxt)
4787 ctxt->ops->get_fpu(ctxt);
4788 asm volatile("1: fwait \n\t"
4790 ".pushsection .fixup,\"ax\" \n\t"
4792 "movb $1, %[fault] \n\t"
4795 _ASM_EXTABLE(1b, 3b)
4796 : [fault]"+qm"(fault));
4797 ctxt->ops->put_fpu(ctxt);
4799 if (unlikely(fault))
4800 return emulate_exception(ctxt, MF_VECTOR, 0, false);
4802 return X86EMUL_CONTINUE;
4805 static void fetch_possible_mmx_operand(struct x86_emulate_ctxt *ctxt,
4808 if (op->type == OP_MM)
4809 read_mmx_reg(ctxt, &op->mm_val, op->addr.mm);
4812 static int fastop(struct x86_emulate_ctxt *ctxt, void (*fop)(struct fastop *))
4814 ulong flags = (ctxt->eflags & EFLAGS_MASK) | X86_EFLAGS_IF;
4815 if (!(ctxt->d & ByteOp))
4816 fop += __ffs(ctxt->dst.bytes) * FASTOP_SIZE;
4817 asm("push %[flags]; popf; call *%[fastop]; pushf; pop %[flags]\n"
4818 : "+a"(ctxt->dst.val), "+d"(ctxt->src.val), [flags]"+D"(flags),
4820 : "c"(ctxt->src2.val));
4821 ctxt->eflags = (ctxt->eflags & ~EFLAGS_MASK) | (flags & EFLAGS_MASK);
4822 if (!fop) /* exception is returned in fop variable */
4823 return emulate_de(ctxt);
4824 return X86EMUL_CONTINUE;
4827 void init_decode_cache(struct x86_emulate_ctxt *ctxt)
4829 memset(&ctxt->rip_relative, 0,
4830 (void *)&ctxt->modrm - (void *)&ctxt->rip_relative);
4832 ctxt->io_read.pos = 0;
4833 ctxt->io_read.end = 0;
4834 ctxt->mem_read.end = 0;
4837 int x86_emulate_insn(struct x86_emulate_ctxt *ctxt)
4839 const struct x86_emulate_ops *ops = ctxt->ops;
4840 int rc = X86EMUL_CONTINUE;
4841 int saved_dst_type = ctxt->dst.type;
4843 ctxt->mem_read.pos = 0;
4845 /* LOCK prefix is allowed only with some instructions */
4846 if (ctxt->lock_prefix && (!(ctxt->d & Lock) || ctxt->dst.type != OP_MEM)) {
4847 rc = emulate_ud(ctxt);
4851 if ((ctxt->d & SrcMask) == SrcMemFAddr && ctxt->src.type != OP_MEM) {
4852 rc = emulate_ud(ctxt);
4856 if (unlikely(ctxt->d &
4857 (No64|Undefined|Sse|Mmx|Intercept|CheckPerm|Priv|Prot|String))) {
4858 if ((ctxt->mode == X86EMUL_MODE_PROT64 && (ctxt->d & No64)) ||
4859 (ctxt->d & Undefined)) {
4860 rc = emulate_ud(ctxt);
4864 if (((ctxt->d & (Sse|Mmx)) && ((ops->get_cr(ctxt, 0) & X86_CR0_EM)))
4865 || ((ctxt->d & Sse) && !(ops->get_cr(ctxt, 4) & X86_CR4_OSFXSR))) {
4866 rc = emulate_ud(ctxt);
4870 if ((ctxt->d & (Sse|Mmx)) && (ops->get_cr(ctxt, 0) & X86_CR0_TS)) {
4871 rc = emulate_nm(ctxt);
4875 if (ctxt->d & Mmx) {
4876 rc = flush_pending_x87_faults(ctxt);
4877 if (rc != X86EMUL_CONTINUE)
4880 * Now that we know the fpu is exception safe, we can fetch
4883 fetch_possible_mmx_operand(ctxt, &ctxt->src);
4884 fetch_possible_mmx_operand(ctxt, &ctxt->src2);
4885 if (!(ctxt->d & Mov))
4886 fetch_possible_mmx_operand(ctxt, &ctxt->dst);
4889 if (unlikely(ctxt->guest_mode) && (ctxt->d & Intercept)) {
4890 rc = emulator_check_intercept(ctxt, ctxt->intercept,
4891 X86_ICPT_PRE_EXCEPT);
4892 if (rc != X86EMUL_CONTINUE)
4896 /* Instruction can only be executed in protected mode */
4897 if ((ctxt->d & Prot) && ctxt->mode < X86EMUL_MODE_PROT16) {
4898 rc = emulate_ud(ctxt);
4902 /* Privileged instruction can be executed only in CPL=0 */
4903 if ((ctxt->d & Priv) && ops->cpl(ctxt)) {
4904 if (ctxt->d & PrivUD)
4905 rc = emulate_ud(ctxt);
4907 rc = emulate_gp(ctxt, 0);
4911 /* Do instruction specific permission checks */
4912 if (ctxt->d & CheckPerm) {
4913 rc = ctxt->check_perm(ctxt);
4914 if (rc != X86EMUL_CONTINUE)
4918 if (unlikely(ctxt->guest_mode) && (ctxt->d & Intercept)) {
4919 rc = emulator_check_intercept(ctxt, ctxt->intercept,
4920 X86_ICPT_POST_EXCEPT);
4921 if (rc != X86EMUL_CONTINUE)
4925 if (ctxt->rep_prefix && (ctxt->d & String)) {
4926 /* All REP prefixes have the same first termination condition */
4927 if (address_mask(ctxt, reg_read(ctxt, VCPU_REGS_RCX)) == 0) {
4928 ctxt->eip = ctxt->_eip;
4929 ctxt->eflags &= ~EFLG_RF;
4935 if ((ctxt->src.type == OP_MEM) && !(ctxt->d & NoAccess)) {
4936 rc = segmented_read(ctxt, ctxt->src.addr.mem,
4937 ctxt->src.valptr, ctxt->src.bytes);
4938 if (rc != X86EMUL_CONTINUE)
4940 ctxt->src.orig_val64 = ctxt->src.val64;
4943 if (ctxt->src2.type == OP_MEM) {
4944 rc = segmented_read(ctxt, ctxt->src2.addr.mem,
4945 &ctxt->src2.val, ctxt->src2.bytes);
4946 if (rc != X86EMUL_CONTINUE)
4950 if ((ctxt->d & DstMask) == ImplicitOps)
4954 if ((ctxt->dst.type == OP_MEM) && !(ctxt->d & Mov)) {
4955 /* optimisation - avoid slow emulated read if Mov */
4956 rc = segmented_read(ctxt, ctxt->dst.addr.mem,
4957 &ctxt->dst.val, ctxt->dst.bytes);
4958 if (rc != X86EMUL_CONTINUE) {
4959 if (!(ctxt->d & NoWrite) &&
4960 rc == X86EMUL_PROPAGATE_FAULT &&
4961 ctxt->exception.vector == PF_VECTOR)
4962 ctxt->exception.error_code |= PFERR_WRITE_MASK;
4966 /* Copy full 64-bit value for CMPXCHG8B. */
4967 ctxt->dst.orig_val64 = ctxt->dst.val64;
4971 if (unlikely(ctxt->guest_mode) && (ctxt->d & Intercept)) {
4972 rc = emulator_check_intercept(ctxt, ctxt->intercept,
4973 X86_ICPT_POST_MEMACCESS);
4974 if (rc != X86EMUL_CONTINUE)
4978 if (ctxt->rep_prefix && (ctxt->d & String))
4979 ctxt->eflags |= EFLG_RF;
4981 ctxt->eflags &= ~EFLG_RF;
4983 if (ctxt->execute) {
4984 if (ctxt->d & Fastop) {
4985 void (*fop)(struct fastop *) = (void *)ctxt->execute;
4986 rc = fastop(ctxt, fop);
4987 if (rc != X86EMUL_CONTINUE)
4991 rc = ctxt->execute(ctxt);
4992 if (rc != X86EMUL_CONTINUE)
4997 if (ctxt->opcode_len == 2)
4999 else if (ctxt->opcode_len == 3)
5000 goto threebyte_insn;
5003 case 0x70 ... 0x7f: /* jcc (short) */
5004 if (test_cc(ctxt->b, ctxt->eflags))
5005 rc = jmp_rel(ctxt, ctxt->src.val);
5007 case 0x8d: /* lea r16/r32, m */
5008 ctxt->dst.val = ctxt->src.addr.mem.ea;
5010 case 0x90 ... 0x97: /* nop / xchg reg, rax */
5011 if (ctxt->dst.addr.reg == reg_rmw(ctxt, VCPU_REGS_RAX))
5012 ctxt->dst.type = OP_NONE;
5016 case 0x98: /* cbw/cwde/cdqe */
5017 switch (ctxt->op_bytes) {
5018 case 2: ctxt->dst.val = (s8)ctxt->dst.val; break;
5019 case 4: ctxt->dst.val = (s16)ctxt->dst.val; break;
5020 case 8: ctxt->dst.val = (s32)ctxt->dst.val; break;
5023 case 0xcc: /* int3 */
5024 rc = emulate_int(ctxt, 3);
5026 case 0xcd: /* int n */
5027 rc = emulate_int(ctxt, ctxt->src.val);
5029 case 0xce: /* into */
5030 if (ctxt->eflags & EFLG_OF)
5031 rc = emulate_int(ctxt, 4);
5033 case 0xe9: /* jmp rel */
5034 case 0xeb: /* jmp rel short */
5035 rc = jmp_rel(ctxt, ctxt->src.val);
5036 ctxt->dst.type = OP_NONE; /* Disable writeback. */
5038 case 0xf4: /* hlt */
5039 ctxt->ops->halt(ctxt);
5041 case 0xf5: /* cmc */
5042 /* complement carry flag from eflags reg */
5043 ctxt->eflags ^= EFLG_CF;
5045 case 0xf8: /* clc */
5046 ctxt->eflags &= ~EFLG_CF;
5048 case 0xf9: /* stc */
5049 ctxt->eflags |= EFLG_CF;
5051 case 0xfc: /* cld */
5052 ctxt->eflags &= ~EFLG_DF;
5054 case 0xfd: /* std */
5055 ctxt->eflags |= EFLG_DF;
5058 goto cannot_emulate;
5061 if (rc != X86EMUL_CONTINUE)
5065 if (ctxt->d & SrcWrite) {
5066 BUG_ON(ctxt->src.type == OP_MEM || ctxt->src.type == OP_MEM_STR);
5067 rc = writeback(ctxt, &ctxt->src);
5068 if (rc != X86EMUL_CONTINUE)
5071 if (!(ctxt->d & NoWrite)) {
5072 rc = writeback(ctxt, &ctxt->dst);
5073 if (rc != X86EMUL_CONTINUE)
5078 * restore dst type in case the decoding will be reused
5079 * (happens for string instruction )
5081 ctxt->dst.type = saved_dst_type;
5083 if ((ctxt->d & SrcMask) == SrcSI)
5084 string_addr_inc(ctxt, VCPU_REGS_RSI, &ctxt->src);
5086 if ((ctxt->d & DstMask) == DstDI)
5087 string_addr_inc(ctxt, VCPU_REGS_RDI, &ctxt->dst);
5089 if (ctxt->rep_prefix && (ctxt->d & String)) {
5091 struct read_cache *r = &ctxt->io_read;
5092 if ((ctxt->d & SrcMask) == SrcSI)
5093 count = ctxt->src.count;
5095 count = ctxt->dst.count;
5096 register_address_increment(ctxt, VCPU_REGS_RCX, -count);
5098 if (!string_insn_completed(ctxt)) {
5100 * Re-enter guest when pio read ahead buffer is empty
5101 * or, if it is not used, after each 1024 iteration.
5103 if ((r->end != 0 || reg_read(ctxt, VCPU_REGS_RCX) & 0x3ff) &&
5104 (r->end == 0 || r->end != r->pos)) {
5106 * Reset read cache. Usually happens before
5107 * decode, but since instruction is restarted
5108 * we have to do it here.
5110 ctxt->mem_read.end = 0;
5111 writeback_registers(ctxt);
5112 return EMULATION_RESTART;
5114 goto done; /* skip rip writeback */
5116 ctxt->eflags &= ~EFLG_RF;
5119 ctxt->eip = ctxt->_eip;
5122 if (rc == X86EMUL_PROPAGATE_FAULT) {
5123 WARN_ON(ctxt->exception.vector > 0x1f);
5124 ctxt->have_exception = true;
5126 if (rc == X86EMUL_INTERCEPTED)
5127 return EMULATION_INTERCEPTED;
5129 if (rc == X86EMUL_CONTINUE)
5130 writeback_registers(ctxt);
5132 return (rc == X86EMUL_UNHANDLEABLE) ? EMULATION_FAILED : EMULATION_OK;
5136 case 0x09: /* wbinvd */
5137 (ctxt->ops->wbinvd)(ctxt);
5139 case 0x08: /* invd */
5140 case 0x0d: /* GrpP (prefetch) */
5141 case 0x18: /* Grp16 (prefetch/nop) */
5142 case 0x1f: /* nop */
5144 case 0x20: /* mov cr, reg */
5145 ctxt->dst.val = ops->get_cr(ctxt, ctxt->modrm_reg);
5147 case 0x21: /* mov from dr to reg */
5148 ops->get_dr(ctxt, ctxt->modrm_reg, &ctxt->dst.val);
5150 case 0x40 ... 0x4f: /* cmov */
5151 if (test_cc(ctxt->b, ctxt->eflags))
5152 ctxt->dst.val = ctxt->src.val;
5153 else if (ctxt->op_bytes != 4)
5154 ctxt->dst.type = OP_NONE; /* no writeback */
5156 case 0x80 ... 0x8f: /* jnz rel, etc*/
5157 if (test_cc(ctxt->b, ctxt->eflags))
5158 rc = jmp_rel(ctxt, ctxt->src.val);
5160 case 0x90 ... 0x9f: /* setcc r/m8 */
5161 ctxt->dst.val = test_cc(ctxt->b, ctxt->eflags);
5163 case 0xb6 ... 0xb7: /* movzx */
5164 ctxt->dst.bytes = ctxt->op_bytes;
5165 ctxt->dst.val = (ctxt->src.bytes == 1) ? (u8) ctxt->src.val
5166 : (u16) ctxt->src.val;
5168 case 0xbe ... 0xbf: /* movsx */
5169 ctxt->dst.bytes = ctxt->op_bytes;
5170 ctxt->dst.val = (ctxt->src.bytes == 1) ? (s8) ctxt->src.val :
5171 (s16) ctxt->src.val;
5174 goto cannot_emulate;
5179 if (rc != X86EMUL_CONTINUE)
5185 return EMULATION_FAILED;
5188 void emulator_invalidate_register_cache(struct x86_emulate_ctxt *ctxt)
5190 invalidate_registers(ctxt);
5193 void emulator_writeback_register_cache(struct x86_emulate_ctxt *ctxt)
5195 writeback_registers(ctxt);