1 /******************************************************************************
4 * Generic x86 (32-bit and 64-bit) instruction decoder and emulator.
6 * Copyright (c) 2005 Keir Fraser
8 * Linux coding style, mod r/m decoder, segment base fixes, real-mode
9 * privileged instructions:
11 * Copyright (C) 2006 Qumranet
12 * Copyright 2010 Red Hat, Inc. and/or its affiliates.
14 * Avi Kivity <avi@qumranet.com>
15 * Yaniv Kamay <yaniv@qumranet.com>
17 * This work is licensed under the terms of the GNU GPL, version 2. See
18 * the COPYING file in the top-level directory.
20 * From: xen-unstable 10676:af9809f51f81a3c43f276f00c81a52ef558afda4
23 #include <linux/kvm_host.h>
24 #include "kvm_cache_regs.h"
25 #include <linux/module.h>
26 #include <asm/kvm_emulate.h>
27 #include <linux/stringify.h>
36 #define OpImplicit 1ull /* No generic decode */
37 #define OpReg 2ull /* Register */
38 #define OpMem 3ull /* Memory */
39 #define OpAcc 4ull /* Accumulator: AL/AX/EAX/RAX */
40 #define OpDI 5ull /* ES:DI/EDI/RDI */
41 #define OpMem64 6ull /* Memory, 64-bit */
42 #define OpImmUByte 7ull /* Zero-extended 8-bit immediate */
43 #define OpDX 8ull /* DX register */
44 #define OpCL 9ull /* CL register (for shifts) */
45 #define OpImmByte 10ull /* 8-bit sign extended immediate */
46 #define OpOne 11ull /* Implied 1 */
47 #define OpImm 12ull /* Sign extended up to 32-bit immediate */
48 #define OpMem16 13ull /* Memory operand (16-bit). */
49 #define OpMem32 14ull /* Memory operand (32-bit). */
50 #define OpImmU 15ull /* Immediate operand, zero extended */
51 #define OpSI 16ull /* SI/ESI/RSI */
52 #define OpImmFAddr 17ull /* Immediate far address */
53 #define OpMemFAddr 18ull /* Far address in memory */
54 #define OpImmU16 19ull /* Immediate operand, 16 bits, zero extended */
55 #define OpES 20ull /* ES */
56 #define OpCS 21ull /* CS */
57 #define OpSS 22ull /* SS */
58 #define OpDS 23ull /* DS */
59 #define OpFS 24ull /* FS */
60 #define OpGS 25ull /* GS */
61 #define OpMem8 26ull /* 8-bit zero extended memory operand */
62 #define OpImm64 27ull /* Sign extended 16/32/64-bit immediate */
63 #define OpXLat 28ull /* memory at BX/EBX/RBX + zero-extended AL */
64 #define OpAccLo 29ull /* Low part of extended acc (AX/AX/EAX/RAX) */
65 #define OpAccHi 30ull /* High part of extended acc (-/DX/EDX/RDX) */
67 #define OpBits 5 /* Width of operand field */
68 #define OpMask ((1ull << OpBits) - 1)
71 * Opcode effective-address decode tables.
72 * Note that we only emulate instructions that have at least one memory
73 * operand (excluding implicit stack references). We assume that stack
74 * references and instruction fetches will never occur in special memory
75 * areas that require emulation. So, for example, 'mov <imm>,<reg>' need
79 /* Operand sizes: 8-bit operands or specified/overridden size. */
80 #define ByteOp (1<<0) /* 8-bit operands. */
81 /* Destination operand type. */
83 #define ImplicitOps (OpImplicit << DstShift)
84 #define DstReg (OpReg << DstShift)
85 #define DstMem (OpMem << DstShift)
86 #define DstAcc (OpAcc << DstShift)
87 #define DstDI (OpDI << DstShift)
88 #define DstMem64 (OpMem64 << DstShift)
89 #define DstMem16 (OpMem16 << DstShift)
90 #define DstImmUByte (OpImmUByte << DstShift)
91 #define DstDX (OpDX << DstShift)
92 #define DstAccLo (OpAccLo << DstShift)
93 #define DstMask (OpMask << DstShift)
94 /* Source operand type. */
96 #define SrcNone (OpNone << SrcShift)
97 #define SrcReg (OpReg << SrcShift)
98 #define SrcMem (OpMem << SrcShift)
99 #define SrcMem16 (OpMem16 << SrcShift)
100 #define SrcMem32 (OpMem32 << SrcShift)
101 #define SrcImm (OpImm << SrcShift)
102 #define SrcImmByte (OpImmByte << SrcShift)
103 #define SrcOne (OpOne << SrcShift)
104 #define SrcImmUByte (OpImmUByte << SrcShift)
105 #define SrcImmU (OpImmU << SrcShift)
106 #define SrcSI (OpSI << SrcShift)
107 #define SrcXLat (OpXLat << SrcShift)
108 #define SrcImmFAddr (OpImmFAddr << SrcShift)
109 #define SrcMemFAddr (OpMemFAddr << SrcShift)
110 #define SrcAcc (OpAcc << SrcShift)
111 #define SrcImmU16 (OpImmU16 << SrcShift)
112 #define SrcImm64 (OpImm64 << SrcShift)
113 #define SrcDX (OpDX << SrcShift)
114 #define SrcMem8 (OpMem8 << SrcShift)
115 #define SrcAccHi (OpAccHi << SrcShift)
116 #define SrcMask (OpMask << SrcShift)
117 #define BitOp (1<<11)
118 #define MemAbs (1<<12) /* Memory operand is absolute displacement */
119 #define String (1<<13) /* String instruction (rep capable) */
120 #define Stack (1<<14) /* Stack instruction (push/pop) */
121 #define GroupMask (7<<15) /* Opcode uses one of the group mechanisms */
122 #define Group (1<<15) /* Bits 3:5 of modrm byte extend opcode */
123 #define GroupDual (2<<15) /* Alternate decoding of mod == 3 */
124 #define Prefix (3<<15) /* Instruction varies with 66/f2/f3 prefix */
125 #define RMExt (4<<15) /* Opcode extension in ModRM r/m if mod == 3 */
126 #define Escape (5<<15) /* Escape to coprocessor instruction */
127 #define InstrDual (6<<15) /* Alternate instruction decoding of mod == 3 */
128 #define ModeDual (7<<15) /* Different instruction for 32/64 bit */
129 #define Sse (1<<18) /* SSE Vector instruction */
130 /* Generic ModRM decode. */
131 #define ModRM (1<<19)
132 /* Destination is only written; never read. */
135 #define Prot (1<<21) /* instruction generates #UD if not in prot-mode */
136 #define EmulateOnUD (1<<22) /* Emulate if unsupported by the host */
137 #define NoAccess (1<<23) /* Don't access memory (lea/invlpg/verr etc) */
138 #define Op3264 (1<<24) /* Operand is 64b in long mode, 32b otherwise */
139 #define Undefined (1<<25) /* No Such Instruction */
140 #define Lock (1<<26) /* lock prefix is allowed for the instruction */
141 #define Priv (1<<27) /* instruction generates #GP if current CPL != 0 */
143 #define PageTable (1 << 29) /* instruction used to write page table */
144 #define NotImpl (1 << 30) /* instruction is not implemented */
145 /* Source 2 operand type */
146 #define Src2Shift (31)
147 #define Src2None (OpNone << Src2Shift)
148 #define Src2Mem (OpMem << Src2Shift)
149 #define Src2CL (OpCL << Src2Shift)
150 #define Src2ImmByte (OpImmByte << Src2Shift)
151 #define Src2One (OpOne << Src2Shift)
152 #define Src2Imm (OpImm << Src2Shift)
153 #define Src2ES (OpES << Src2Shift)
154 #define Src2CS (OpCS << Src2Shift)
155 #define Src2SS (OpSS << Src2Shift)
156 #define Src2DS (OpDS << Src2Shift)
157 #define Src2FS (OpFS << Src2Shift)
158 #define Src2GS (OpGS << Src2Shift)
159 #define Src2Mask (OpMask << Src2Shift)
160 #define Mmx ((u64)1 << 40) /* MMX Vector instruction */
161 #define Aligned ((u64)1 << 41) /* Explicitly aligned (e.g. MOVDQA) */
162 #define Unaligned ((u64)1 << 42) /* Explicitly unaligned (e.g. MOVDQU) */
163 #define Avx ((u64)1 << 43) /* Advanced Vector Extensions */
164 #define Fastop ((u64)1 << 44) /* Use opcode::u.fastop */
165 #define NoWrite ((u64)1 << 45) /* No writeback */
166 #define SrcWrite ((u64)1 << 46) /* Write back src operand */
167 #define NoMod ((u64)1 << 47) /* Mod field is ignored */
168 #define Intercept ((u64)1 << 48) /* Has valid intercept field */
169 #define CheckPerm ((u64)1 << 49) /* Has valid check_perm field */
170 #define PrivUD ((u64)1 << 51) /* #UD instead of #GP on CPL > 0 */
171 #define NearBranch ((u64)1 << 52) /* Near branches */
172 #define No16 ((u64)1 << 53) /* No 16 bit operand */
173 #define IncSP ((u64)1 << 54) /* SP is incremented before ModRM calc */
175 #define DstXacc (DstAccLo | SrcAccHi | SrcWrite)
177 #define X2(x...) x, x
178 #define X3(x...) X2(x), x
179 #define X4(x...) X2(x), X2(x)
180 #define X5(x...) X4(x), x
181 #define X6(x...) X4(x), X2(x)
182 #define X7(x...) X4(x), X3(x)
183 #define X8(x...) X4(x), X4(x)
184 #define X16(x...) X8(x), X8(x)
186 #define NR_FASTOP (ilog2(sizeof(ulong)) + 1)
187 #define FASTOP_SIZE 8
190 * fastop functions have a special calling convention:
195 * flags: rflags (in/out)
196 * ex: rsi (in:fastop pointer, out:zero if exception)
198 * Moreover, they are all exactly FASTOP_SIZE bytes long, so functions for
199 * different operand sizes can be reached by calculation, rather than a jump
200 * table (which would be bigger than the code).
202 * fastop functions are declared as taking a never-defined fastop parameter,
203 * so they can't be called from C directly.
212 int (*execute)(struct x86_emulate_ctxt *ctxt);
213 const struct opcode *group;
214 const struct group_dual *gdual;
215 const struct gprefix *gprefix;
216 const struct escape *esc;
217 const struct instr_dual *idual;
218 const struct mode_dual *mdual;
219 void (*fastop)(struct fastop *fake);
221 int (*check_perm)(struct x86_emulate_ctxt *ctxt);
225 struct opcode mod012[8];
226 struct opcode mod3[8];
230 struct opcode pfx_no;
231 struct opcode pfx_66;
232 struct opcode pfx_f2;
233 struct opcode pfx_f3;
238 struct opcode high[64];
242 struct opcode mod012;
247 struct opcode mode32;
248 struct opcode mode64;
251 /* EFLAGS bit definitions. */
252 #define EFLG_ID (1<<21)
253 #define EFLG_VIP (1<<20)
254 #define EFLG_VIF (1<<19)
255 #define EFLG_AC (1<<18)
256 #define EFLG_VM (1<<17)
257 #define EFLG_RF (1<<16)
258 #define EFLG_IOPL (3<<12)
259 #define EFLG_NT (1<<14)
260 #define EFLG_OF (1<<11)
261 #define EFLG_DF (1<<10)
262 #define EFLG_IF (1<<9)
263 #define EFLG_TF (1<<8)
264 #define EFLG_SF (1<<7)
265 #define EFLG_ZF (1<<6)
266 #define EFLG_AF (1<<4)
267 #define EFLG_PF (1<<2)
268 #define EFLG_CF (1<<0)
270 #define EFLG_RESERVED_ZEROS_MASK 0xffc0802a
271 #define EFLG_RESERVED_ONE_MASK 2
273 enum x86_transfer_type {
275 X86_TRANSFER_CALL_JMP,
277 X86_TRANSFER_TASK_SWITCH,
280 static ulong reg_read(struct x86_emulate_ctxt *ctxt, unsigned nr)
282 if (!(ctxt->regs_valid & (1 << nr))) {
283 ctxt->regs_valid |= 1 << nr;
284 ctxt->_regs[nr] = ctxt->ops->read_gpr(ctxt, nr);
286 return ctxt->_regs[nr];
289 static ulong *reg_write(struct x86_emulate_ctxt *ctxt, unsigned nr)
291 ctxt->regs_valid |= 1 << nr;
292 ctxt->regs_dirty |= 1 << nr;
293 return &ctxt->_regs[nr];
296 static ulong *reg_rmw(struct x86_emulate_ctxt *ctxt, unsigned nr)
299 return reg_write(ctxt, nr);
302 static void writeback_registers(struct x86_emulate_ctxt *ctxt)
306 for_each_set_bit(reg, (ulong *)&ctxt->regs_dirty, 16)
307 ctxt->ops->write_gpr(ctxt, reg, ctxt->_regs[reg]);
310 static void invalidate_registers(struct x86_emulate_ctxt *ctxt)
312 ctxt->regs_dirty = 0;
313 ctxt->regs_valid = 0;
317 * These EFLAGS bits are restored from saved value during emulation, and
318 * any changes are written back to the saved value after emulation.
320 #define EFLAGS_MASK (EFLG_OF|EFLG_SF|EFLG_ZF|EFLG_AF|EFLG_PF|EFLG_CF)
328 static int fastop(struct x86_emulate_ctxt *ctxt, void (*fop)(struct fastop *));
330 #define FOP_ALIGN ".align " __stringify(FASTOP_SIZE) " \n\t"
331 #define FOP_RET "ret \n\t"
333 #define FOP_START(op) \
334 extern void em_##op(struct fastop *fake); \
335 asm(".pushsection .text, \"ax\" \n\t" \
336 ".global em_" #op " \n\t" \
343 #define FOPNOP() FOP_ALIGN FOP_RET
345 #define FOP1E(op, dst) \
346 FOP_ALIGN "10: " #op " %" #dst " \n\t" FOP_RET
348 #define FOP1EEX(op, dst) \
349 FOP1E(op, dst) _ASM_EXTABLE(10b, kvm_fastop_exception)
351 #define FASTOP1(op) \
356 ON64(FOP1E(op##q, rax)) \
359 /* 1-operand, using src2 (for MUL/DIV r/m) */
360 #define FASTOP1SRC2(op, name) \
365 ON64(FOP1E(op, rcx)) \
368 /* 1-operand, using src2 (for MUL/DIV r/m), with exceptions */
369 #define FASTOP1SRC2EX(op, name) \
374 ON64(FOP1EEX(op, rcx)) \
377 #define FOP2E(op, dst, src) \
378 FOP_ALIGN #op " %" #src ", %" #dst " \n\t" FOP_RET
380 #define FASTOP2(op) \
382 FOP2E(op##b, al, dl) \
383 FOP2E(op##w, ax, dx) \
384 FOP2E(op##l, eax, edx) \
385 ON64(FOP2E(op##q, rax, rdx)) \
388 /* 2 operand, word only */
389 #define FASTOP2W(op) \
392 FOP2E(op##w, ax, dx) \
393 FOP2E(op##l, eax, edx) \
394 ON64(FOP2E(op##q, rax, rdx)) \
397 /* 2 operand, src is CL */
398 #define FASTOP2CL(op) \
400 FOP2E(op##b, al, cl) \
401 FOP2E(op##w, ax, cl) \
402 FOP2E(op##l, eax, cl) \
403 ON64(FOP2E(op##q, rax, cl)) \
406 /* 2 operand, src and dest are reversed */
407 #define FASTOP2R(op, name) \
409 FOP2E(op##b, dl, al) \
410 FOP2E(op##w, dx, ax) \
411 FOP2E(op##l, edx, eax) \
412 ON64(FOP2E(op##q, rdx, rax)) \
415 #define FOP3E(op, dst, src, src2) \
416 FOP_ALIGN #op " %" #src2 ", %" #src ", %" #dst " \n\t" FOP_RET
418 /* 3-operand, word-only, src2=cl */
419 #define FASTOP3WCL(op) \
422 FOP3E(op##w, ax, dx, cl) \
423 FOP3E(op##l, eax, edx, cl) \
424 ON64(FOP3E(op##q, rax, rdx, cl)) \
427 /* Special case for SETcc - 1 instruction per cc */
428 #define FOP_SETCC(op) ".align 4; " #op " %al; ret \n\t"
430 asm(".global kvm_fastop_exception \n"
431 "kvm_fastop_exception: xor %esi, %esi; ret");
452 FOP_START(salc) "pushf; sbb %al, %al; popf \n\t" FOP_RET
455 static int emulator_check_intercept(struct x86_emulate_ctxt *ctxt,
456 enum x86_intercept intercept,
457 enum x86_intercept_stage stage)
459 struct x86_instruction_info info = {
460 .intercept = intercept,
461 .rep_prefix = ctxt->rep_prefix,
462 .modrm_mod = ctxt->modrm_mod,
463 .modrm_reg = ctxt->modrm_reg,
464 .modrm_rm = ctxt->modrm_rm,
465 .src_val = ctxt->src.val64,
466 .dst_val = ctxt->dst.val64,
467 .src_bytes = ctxt->src.bytes,
468 .dst_bytes = ctxt->dst.bytes,
469 .ad_bytes = ctxt->ad_bytes,
470 .next_rip = ctxt->eip,
473 return ctxt->ops->intercept(ctxt, &info, stage);
476 static void assign_masked(ulong *dest, ulong src, ulong mask)
478 *dest = (*dest & ~mask) | (src & mask);
481 static inline unsigned long ad_mask(struct x86_emulate_ctxt *ctxt)
483 return (1UL << (ctxt->ad_bytes << 3)) - 1;
486 static ulong stack_mask(struct x86_emulate_ctxt *ctxt)
489 struct desc_struct ss;
491 if (ctxt->mode == X86EMUL_MODE_PROT64)
493 ctxt->ops->get_segment(ctxt, &sel, &ss, NULL, VCPU_SREG_SS);
494 return ~0U >> ((ss.d ^ 1) * 16); /* d=0: 0xffff; d=1: 0xffffffff */
497 static int stack_size(struct x86_emulate_ctxt *ctxt)
499 return (__fls(stack_mask(ctxt)) + 1) >> 3;
502 /* Access/update address held in a register, based on addressing mode. */
503 static inline unsigned long
504 address_mask(struct x86_emulate_ctxt *ctxt, unsigned long reg)
506 if (ctxt->ad_bytes == sizeof(unsigned long))
509 return reg & ad_mask(ctxt);
512 static inline unsigned long
513 register_address(struct x86_emulate_ctxt *ctxt, int reg)
515 return address_mask(ctxt, reg_read(ctxt, reg));
518 static void masked_increment(ulong *reg, ulong mask, int inc)
520 assign_masked(reg, *reg + inc, mask);
524 register_address_increment(struct x86_emulate_ctxt *ctxt, int reg, int inc)
528 if (ctxt->ad_bytes == sizeof(unsigned long))
531 mask = ad_mask(ctxt);
532 masked_increment(reg_rmw(ctxt, reg), mask, inc);
535 static void rsp_increment(struct x86_emulate_ctxt *ctxt, int inc)
537 masked_increment(reg_rmw(ctxt, VCPU_REGS_RSP), stack_mask(ctxt), inc);
540 static u32 desc_limit_scaled(struct desc_struct *desc)
542 u32 limit = get_desc_limit(desc);
544 return desc->g ? (limit << 12) | 0xfff : limit;
547 static unsigned long seg_base(struct x86_emulate_ctxt *ctxt, int seg)
549 if (ctxt->mode == X86EMUL_MODE_PROT64 && seg < VCPU_SREG_FS)
552 return ctxt->ops->get_cached_segment_base(ctxt, seg);
555 static int emulate_exception(struct x86_emulate_ctxt *ctxt, int vec,
556 u32 error, bool valid)
559 ctxt->exception.vector = vec;
560 ctxt->exception.error_code = error;
561 ctxt->exception.error_code_valid = valid;
562 return X86EMUL_PROPAGATE_FAULT;
565 static int emulate_db(struct x86_emulate_ctxt *ctxt)
567 return emulate_exception(ctxt, DB_VECTOR, 0, false);
570 static int emulate_gp(struct x86_emulate_ctxt *ctxt, int err)
572 return emulate_exception(ctxt, GP_VECTOR, err, true);
575 static int emulate_ss(struct x86_emulate_ctxt *ctxt, int err)
577 return emulate_exception(ctxt, SS_VECTOR, err, true);
580 static int emulate_ud(struct x86_emulate_ctxt *ctxt)
582 return emulate_exception(ctxt, UD_VECTOR, 0, false);
585 static int emulate_ts(struct x86_emulate_ctxt *ctxt, int err)
587 return emulate_exception(ctxt, TS_VECTOR, err, true);
590 static int emulate_de(struct x86_emulate_ctxt *ctxt)
592 return emulate_exception(ctxt, DE_VECTOR, 0, false);
595 static int emulate_nm(struct x86_emulate_ctxt *ctxt)
597 return emulate_exception(ctxt, NM_VECTOR, 0, false);
600 static u16 get_segment_selector(struct x86_emulate_ctxt *ctxt, unsigned seg)
603 struct desc_struct desc;
605 ctxt->ops->get_segment(ctxt, &selector, &desc, NULL, seg);
609 static void set_segment_selector(struct x86_emulate_ctxt *ctxt, u16 selector,
614 struct desc_struct desc;
616 ctxt->ops->get_segment(ctxt, &dummy, &desc, &base3, seg);
617 ctxt->ops->set_segment(ctxt, selector, &desc, base3, seg);
621 * x86 defines three classes of vector instructions: explicitly
622 * aligned, explicitly unaligned, and the rest, which change behaviour
623 * depending on whether they're AVX encoded or not.
625 * Also included is CMPXCHG16B which is not a vector instruction, yet it is
626 * subject to the same check.
628 static bool insn_aligned(struct x86_emulate_ctxt *ctxt, unsigned size)
630 if (likely(size < 16))
633 if (ctxt->d & Aligned)
635 else if (ctxt->d & Unaligned)
637 else if (ctxt->d & Avx)
643 static __always_inline int __linearize(struct x86_emulate_ctxt *ctxt,
644 struct segmented_address addr,
645 unsigned *max_size, unsigned size,
646 bool write, bool fetch,
647 enum x86emul_mode mode, ulong *linear)
649 struct desc_struct desc;
655 la = seg_base(ctxt, addr.seg) + addr.ea;
658 case X86EMUL_MODE_PROT64:
659 if (is_noncanonical_address(la))
662 *max_size = min_t(u64, ~0u, (1ull << 48) - la);
663 if (size > *max_size)
667 usable = ctxt->ops->get_segment(ctxt, &sel, &desc, NULL,
671 /* code segment in protected mode or read-only data segment */
672 if ((((ctxt->mode != X86EMUL_MODE_REAL) && (desc.type & 8))
673 || !(desc.type & 2)) && write)
675 /* unreadable code segment */
676 if (!fetch && (desc.type & 8) && !(desc.type & 2))
678 lim = desc_limit_scaled(&desc);
679 if (!(desc.type & 8) && (desc.type & 4)) {
680 /* expand-down segment */
683 lim = desc.d ? 0xffffffff : 0xffff;
687 if (lim == 0xffffffff)
690 *max_size = (u64)lim + 1 - addr.ea;
691 if (size > *max_size)
697 if (insn_aligned(ctxt, size) && ((la & (size - 1)) != 0))
698 return emulate_gp(ctxt, 0);
700 return X86EMUL_CONTINUE;
702 if (addr.seg == VCPU_SREG_SS)
703 return emulate_ss(ctxt, 0);
705 return emulate_gp(ctxt, 0);
708 static int linearize(struct x86_emulate_ctxt *ctxt,
709 struct segmented_address addr,
710 unsigned size, bool write,
714 return __linearize(ctxt, addr, &max_size, size, write, false,
718 static inline int assign_eip(struct x86_emulate_ctxt *ctxt, ulong dst,
719 enum x86emul_mode mode)
724 struct segmented_address addr = { .seg = VCPU_SREG_CS,
727 if (ctxt->op_bytes != sizeof(unsigned long))
728 addr.ea = dst & ((1UL << (ctxt->op_bytes << 3)) - 1);
729 rc = __linearize(ctxt, addr, &max_size, 1, false, true, mode, &linear);
730 if (rc == X86EMUL_CONTINUE)
731 ctxt->_eip = addr.ea;
735 static inline int assign_eip_near(struct x86_emulate_ctxt *ctxt, ulong dst)
737 return assign_eip(ctxt, dst, ctxt->mode);
740 static int assign_eip_far(struct x86_emulate_ctxt *ctxt, ulong dst,
741 const struct desc_struct *cs_desc)
743 enum x86emul_mode mode = ctxt->mode;
747 if (ctxt->mode >= X86EMUL_MODE_PROT16) {
751 ctxt->ops->get_msr(ctxt, MSR_EFER, &efer);
753 mode = X86EMUL_MODE_PROT64;
755 mode = X86EMUL_MODE_PROT32; /* temporary value */
758 if (mode == X86EMUL_MODE_PROT16 || mode == X86EMUL_MODE_PROT32)
759 mode = cs_desc->d ? X86EMUL_MODE_PROT32 : X86EMUL_MODE_PROT16;
760 rc = assign_eip(ctxt, dst, mode);
761 if (rc == X86EMUL_CONTINUE)
766 static inline int jmp_rel(struct x86_emulate_ctxt *ctxt, int rel)
768 return assign_eip_near(ctxt, ctxt->_eip + rel);
771 static int segmented_read_std(struct x86_emulate_ctxt *ctxt,
772 struct segmented_address addr,
779 rc = linearize(ctxt, addr, size, false, &linear);
780 if (rc != X86EMUL_CONTINUE)
782 return ctxt->ops->read_std(ctxt, linear, data, size, &ctxt->exception);
786 * Prefetch the remaining bytes of the instruction without crossing page
787 * boundary if they are not in fetch_cache yet.
789 static int __do_insn_fetch_bytes(struct x86_emulate_ctxt *ctxt, int op_size)
792 unsigned size, max_size;
793 unsigned long linear;
794 int cur_size = ctxt->fetch.end - ctxt->fetch.data;
795 struct segmented_address addr = { .seg = VCPU_SREG_CS,
796 .ea = ctxt->eip + cur_size };
799 * We do not know exactly how many bytes will be needed, and
800 * __linearize is expensive, so fetch as much as possible. We
801 * just have to avoid going beyond the 15 byte limit, the end
802 * of the segment, or the end of the page.
804 * __linearize is called with size 0 so that it does not do any
805 * boundary check itself. Instead, we use max_size to check
808 rc = __linearize(ctxt, addr, &max_size, 0, false, true, ctxt->mode,
810 if (unlikely(rc != X86EMUL_CONTINUE))
813 size = min_t(unsigned, 15UL ^ cur_size, max_size);
814 size = min_t(unsigned, size, PAGE_SIZE - offset_in_page(linear));
817 * One instruction can only straddle two pages,
818 * and one has been loaded at the beginning of
819 * x86_decode_insn. So, if not enough bytes
820 * still, we must have hit the 15-byte boundary.
822 if (unlikely(size < op_size))
823 return emulate_gp(ctxt, 0);
825 rc = ctxt->ops->fetch(ctxt, linear, ctxt->fetch.end,
826 size, &ctxt->exception);
827 if (unlikely(rc != X86EMUL_CONTINUE))
829 ctxt->fetch.end += size;
830 return X86EMUL_CONTINUE;
833 static __always_inline int do_insn_fetch_bytes(struct x86_emulate_ctxt *ctxt,
836 unsigned done_size = ctxt->fetch.end - ctxt->fetch.ptr;
838 if (unlikely(done_size < size))
839 return __do_insn_fetch_bytes(ctxt, size - done_size);
841 return X86EMUL_CONTINUE;
844 /* Fetch next part of the instruction being emulated. */
845 #define insn_fetch(_type, _ctxt) \
848 rc = do_insn_fetch_bytes(_ctxt, sizeof(_type)); \
849 if (rc != X86EMUL_CONTINUE) \
851 ctxt->_eip += sizeof(_type); \
852 _x = *(_type __aligned(1) *) ctxt->fetch.ptr; \
853 ctxt->fetch.ptr += sizeof(_type); \
857 #define insn_fetch_arr(_arr, _size, _ctxt) \
859 rc = do_insn_fetch_bytes(_ctxt, _size); \
860 if (rc != X86EMUL_CONTINUE) \
862 ctxt->_eip += (_size); \
863 memcpy(_arr, ctxt->fetch.ptr, _size); \
864 ctxt->fetch.ptr += (_size); \
868 * Given the 'reg' portion of a ModRM byte, and a register block, return a
869 * pointer into the block that addresses the relevant register.
870 * @highbyte_regs specifies whether to decode AH,CH,DH,BH.
872 static void *decode_register(struct x86_emulate_ctxt *ctxt, u8 modrm_reg,
876 int highbyte_regs = (ctxt->rex_prefix == 0) && byteop;
878 if (highbyte_regs && modrm_reg >= 4 && modrm_reg < 8)
879 p = (unsigned char *)reg_rmw(ctxt, modrm_reg & 3) + 1;
881 p = reg_rmw(ctxt, modrm_reg);
885 static int read_descriptor(struct x86_emulate_ctxt *ctxt,
886 struct segmented_address addr,
887 u16 *size, unsigned long *address, int op_bytes)
894 rc = segmented_read_std(ctxt, addr, size, 2);
895 if (rc != X86EMUL_CONTINUE)
898 rc = segmented_read_std(ctxt, addr, address, op_bytes);
912 FASTOP1SRC2(mul, mul_ex);
913 FASTOP1SRC2(imul, imul_ex);
914 FASTOP1SRC2EX(div, div_ex);
915 FASTOP1SRC2EX(idiv, idiv_ex);
944 FASTOP2R(cmp, cmp_r);
946 static u8 test_cc(unsigned int condition, unsigned long flags)
949 void (*fop)(void) = (void *)em_setcc + 4 * (condition & 0xf);
951 flags = (flags & EFLAGS_MASK) | X86_EFLAGS_IF;
952 asm("push %[flags]; popf; call *%[fastop]"
953 : "=a"(rc) : [fastop]"r"(fop), [flags]"r"(flags));
957 static void fetch_register_operand(struct operand *op)
961 op->val = *(u8 *)op->addr.reg;
964 op->val = *(u16 *)op->addr.reg;
967 op->val = *(u32 *)op->addr.reg;
970 op->val = *(u64 *)op->addr.reg;
975 static void read_sse_reg(struct x86_emulate_ctxt *ctxt, sse128_t *data, int reg)
977 ctxt->ops->get_fpu(ctxt);
979 case 0: asm("movdqa %%xmm0, %0" : "=m"(*data)); break;
980 case 1: asm("movdqa %%xmm1, %0" : "=m"(*data)); break;
981 case 2: asm("movdqa %%xmm2, %0" : "=m"(*data)); break;
982 case 3: asm("movdqa %%xmm3, %0" : "=m"(*data)); break;
983 case 4: asm("movdqa %%xmm4, %0" : "=m"(*data)); break;
984 case 5: asm("movdqa %%xmm5, %0" : "=m"(*data)); break;
985 case 6: asm("movdqa %%xmm6, %0" : "=m"(*data)); break;
986 case 7: asm("movdqa %%xmm7, %0" : "=m"(*data)); break;
988 case 8: asm("movdqa %%xmm8, %0" : "=m"(*data)); break;
989 case 9: asm("movdqa %%xmm9, %0" : "=m"(*data)); break;
990 case 10: asm("movdqa %%xmm10, %0" : "=m"(*data)); break;
991 case 11: asm("movdqa %%xmm11, %0" : "=m"(*data)); break;
992 case 12: asm("movdqa %%xmm12, %0" : "=m"(*data)); break;
993 case 13: asm("movdqa %%xmm13, %0" : "=m"(*data)); break;
994 case 14: asm("movdqa %%xmm14, %0" : "=m"(*data)); break;
995 case 15: asm("movdqa %%xmm15, %0" : "=m"(*data)); break;
999 ctxt->ops->put_fpu(ctxt);
1002 static void write_sse_reg(struct x86_emulate_ctxt *ctxt, sse128_t *data,
1005 ctxt->ops->get_fpu(ctxt);
1007 case 0: asm("movdqa %0, %%xmm0" : : "m"(*data)); break;
1008 case 1: asm("movdqa %0, %%xmm1" : : "m"(*data)); break;
1009 case 2: asm("movdqa %0, %%xmm2" : : "m"(*data)); break;
1010 case 3: asm("movdqa %0, %%xmm3" : : "m"(*data)); break;
1011 case 4: asm("movdqa %0, %%xmm4" : : "m"(*data)); break;
1012 case 5: asm("movdqa %0, %%xmm5" : : "m"(*data)); break;
1013 case 6: asm("movdqa %0, %%xmm6" : : "m"(*data)); break;
1014 case 7: asm("movdqa %0, %%xmm7" : : "m"(*data)); break;
1015 #ifdef CONFIG_X86_64
1016 case 8: asm("movdqa %0, %%xmm8" : : "m"(*data)); break;
1017 case 9: asm("movdqa %0, %%xmm9" : : "m"(*data)); break;
1018 case 10: asm("movdqa %0, %%xmm10" : : "m"(*data)); break;
1019 case 11: asm("movdqa %0, %%xmm11" : : "m"(*data)); break;
1020 case 12: asm("movdqa %0, %%xmm12" : : "m"(*data)); break;
1021 case 13: asm("movdqa %0, %%xmm13" : : "m"(*data)); break;
1022 case 14: asm("movdqa %0, %%xmm14" : : "m"(*data)); break;
1023 case 15: asm("movdqa %0, %%xmm15" : : "m"(*data)); break;
1027 ctxt->ops->put_fpu(ctxt);
1030 static void read_mmx_reg(struct x86_emulate_ctxt *ctxt, u64 *data, int reg)
1032 ctxt->ops->get_fpu(ctxt);
1034 case 0: asm("movq %%mm0, %0" : "=m"(*data)); break;
1035 case 1: asm("movq %%mm1, %0" : "=m"(*data)); break;
1036 case 2: asm("movq %%mm2, %0" : "=m"(*data)); break;
1037 case 3: asm("movq %%mm3, %0" : "=m"(*data)); break;
1038 case 4: asm("movq %%mm4, %0" : "=m"(*data)); break;
1039 case 5: asm("movq %%mm5, %0" : "=m"(*data)); break;
1040 case 6: asm("movq %%mm6, %0" : "=m"(*data)); break;
1041 case 7: asm("movq %%mm7, %0" : "=m"(*data)); break;
1044 ctxt->ops->put_fpu(ctxt);
1047 static void write_mmx_reg(struct x86_emulate_ctxt *ctxt, u64 *data, int reg)
1049 ctxt->ops->get_fpu(ctxt);
1051 case 0: asm("movq %0, %%mm0" : : "m"(*data)); break;
1052 case 1: asm("movq %0, %%mm1" : : "m"(*data)); break;
1053 case 2: asm("movq %0, %%mm2" : : "m"(*data)); break;
1054 case 3: asm("movq %0, %%mm3" : : "m"(*data)); break;
1055 case 4: asm("movq %0, %%mm4" : : "m"(*data)); break;
1056 case 5: asm("movq %0, %%mm5" : : "m"(*data)); break;
1057 case 6: asm("movq %0, %%mm6" : : "m"(*data)); break;
1058 case 7: asm("movq %0, %%mm7" : : "m"(*data)); break;
1061 ctxt->ops->put_fpu(ctxt);
1064 static int em_fninit(struct x86_emulate_ctxt *ctxt)
1066 if (ctxt->ops->get_cr(ctxt, 0) & (X86_CR0_TS | X86_CR0_EM))
1067 return emulate_nm(ctxt);
1069 ctxt->ops->get_fpu(ctxt);
1070 asm volatile("fninit");
1071 ctxt->ops->put_fpu(ctxt);
1072 return X86EMUL_CONTINUE;
1075 static int em_fnstcw(struct x86_emulate_ctxt *ctxt)
1079 if (ctxt->ops->get_cr(ctxt, 0) & (X86_CR0_TS | X86_CR0_EM))
1080 return emulate_nm(ctxt);
1082 ctxt->ops->get_fpu(ctxt);
1083 asm volatile("fnstcw %0": "+m"(fcw));
1084 ctxt->ops->put_fpu(ctxt);
1086 ctxt->dst.val = fcw;
1088 return X86EMUL_CONTINUE;
1091 static int em_fnstsw(struct x86_emulate_ctxt *ctxt)
1095 if (ctxt->ops->get_cr(ctxt, 0) & (X86_CR0_TS | X86_CR0_EM))
1096 return emulate_nm(ctxt);
1098 ctxt->ops->get_fpu(ctxt);
1099 asm volatile("fnstsw %0": "+m"(fsw));
1100 ctxt->ops->put_fpu(ctxt);
1102 ctxt->dst.val = fsw;
1104 return X86EMUL_CONTINUE;
1107 static void decode_register_operand(struct x86_emulate_ctxt *ctxt,
1110 unsigned reg = ctxt->modrm_reg;
1112 if (!(ctxt->d & ModRM))
1113 reg = (ctxt->b & 7) | ((ctxt->rex_prefix & 1) << 3);
1115 if (ctxt->d & Sse) {
1119 read_sse_reg(ctxt, &op->vec_val, reg);
1122 if (ctxt->d & Mmx) {
1131 op->bytes = (ctxt->d & ByteOp) ? 1 : ctxt->op_bytes;
1132 op->addr.reg = decode_register(ctxt, reg, ctxt->d & ByteOp);
1134 fetch_register_operand(op);
1135 op->orig_val = op->val;
1138 static void adjust_modrm_seg(struct x86_emulate_ctxt *ctxt, int base_reg)
1140 if (base_reg == VCPU_REGS_RSP || base_reg == VCPU_REGS_RBP)
1141 ctxt->modrm_seg = VCPU_SREG_SS;
1144 static int decode_modrm(struct x86_emulate_ctxt *ctxt,
1148 int index_reg, base_reg, scale;
1149 int rc = X86EMUL_CONTINUE;
1152 ctxt->modrm_reg = ((ctxt->rex_prefix << 1) & 8); /* REX.R */
1153 index_reg = (ctxt->rex_prefix << 2) & 8; /* REX.X */
1154 base_reg = (ctxt->rex_prefix << 3) & 8; /* REX.B */
1156 ctxt->modrm_mod = (ctxt->modrm & 0xc0) >> 6;
1157 ctxt->modrm_reg |= (ctxt->modrm & 0x38) >> 3;
1158 ctxt->modrm_rm = base_reg | (ctxt->modrm & 0x07);
1159 ctxt->modrm_seg = VCPU_SREG_DS;
1161 if (ctxt->modrm_mod == 3 || (ctxt->d & NoMod)) {
1163 op->bytes = (ctxt->d & ByteOp) ? 1 : ctxt->op_bytes;
1164 op->addr.reg = decode_register(ctxt, ctxt->modrm_rm,
1166 if (ctxt->d & Sse) {
1169 op->addr.xmm = ctxt->modrm_rm;
1170 read_sse_reg(ctxt, &op->vec_val, ctxt->modrm_rm);
1173 if (ctxt->d & Mmx) {
1176 op->addr.mm = ctxt->modrm_rm & 7;
1179 fetch_register_operand(op);
1185 if (ctxt->ad_bytes == 2) {
1186 unsigned bx = reg_read(ctxt, VCPU_REGS_RBX);
1187 unsigned bp = reg_read(ctxt, VCPU_REGS_RBP);
1188 unsigned si = reg_read(ctxt, VCPU_REGS_RSI);
1189 unsigned di = reg_read(ctxt, VCPU_REGS_RDI);
1191 /* 16-bit ModR/M decode. */
1192 switch (ctxt->modrm_mod) {
1194 if (ctxt->modrm_rm == 6)
1195 modrm_ea += insn_fetch(u16, ctxt);
1198 modrm_ea += insn_fetch(s8, ctxt);
1201 modrm_ea += insn_fetch(u16, ctxt);
1204 switch (ctxt->modrm_rm) {
1206 modrm_ea += bx + si;
1209 modrm_ea += bx + di;
1212 modrm_ea += bp + si;
1215 modrm_ea += bp + di;
1224 if (ctxt->modrm_mod != 0)
1231 if (ctxt->modrm_rm == 2 || ctxt->modrm_rm == 3 ||
1232 (ctxt->modrm_rm == 6 && ctxt->modrm_mod != 0))
1233 ctxt->modrm_seg = VCPU_SREG_SS;
1234 modrm_ea = (u16)modrm_ea;
1236 /* 32/64-bit ModR/M decode. */
1237 if ((ctxt->modrm_rm & 7) == 4) {
1238 sib = insn_fetch(u8, ctxt);
1239 index_reg |= (sib >> 3) & 7;
1240 base_reg |= sib & 7;
1243 if ((base_reg & 7) == 5 && ctxt->modrm_mod == 0)
1244 modrm_ea += insn_fetch(s32, ctxt);
1246 modrm_ea += reg_read(ctxt, base_reg);
1247 adjust_modrm_seg(ctxt, base_reg);
1248 /* Increment ESP on POP [ESP] */
1249 if ((ctxt->d & IncSP) &&
1250 base_reg == VCPU_REGS_RSP)
1251 modrm_ea += ctxt->op_bytes;
1254 modrm_ea += reg_read(ctxt, index_reg) << scale;
1255 } else if ((ctxt->modrm_rm & 7) == 5 && ctxt->modrm_mod == 0) {
1256 modrm_ea += insn_fetch(s32, ctxt);
1257 if (ctxt->mode == X86EMUL_MODE_PROT64)
1258 ctxt->rip_relative = 1;
1260 base_reg = ctxt->modrm_rm;
1261 modrm_ea += reg_read(ctxt, base_reg);
1262 adjust_modrm_seg(ctxt, base_reg);
1264 switch (ctxt->modrm_mod) {
1266 modrm_ea += insn_fetch(s8, ctxt);
1269 modrm_ea += insn_fetch(s32, ctxt);
1273 op->addr.mem.ea = modrm_ea;
1274 if (ctxt->ad_bytes != 8)
1275 ctxt->memop.addr.mem.ea = (u32)ctxt->memop.addr.mem.ea;
1281 static int decode_abs(struct x86_emulate_ctxt *ctxt,
1284 int rc = X86EMUL_CONTINUE;
1287 switch (ctxt->ad_bytes) {
1289 op->addr.mem.ea = insn_fetch(u16, ctxt);
1292 op->addr.mem.ea = insn_fetch(u32, ctxt);
1295 op->addr.mem.ea = insn_fetch(u64, ctxt);
1302 static void fetch_bit_operand(struct x86_emulate_ctxt *ctxt)
1306 if (ctxt->dst.type == OP_MEM && ctxt->src.type == OP_REG) {
1307 mask = ~((long)ctxt->dst.bytes * 8 - 1);
1309 if (ctxt->src.bytes == 2)
1310 sv = (s16)ctxt->src.val & (s16)mask;
1311 else if (ctxt->src.bytes == 4)
1312 sv = (s32)ctxt->src.val & (s32)mask;
1314 sv = (s64)ctxt->src.val & (s64)mask;
1316 ctxt->dst.addr.mem.ea = address_mask(ctxt,
1317 ctxt->dst.addr.mem.ea + (sv >> 3));
1320 /* only subword offset */
1321 ctxt->src.val &= (ctxt->dst.bytes << 3) - 1;
1324 static int read_emulated(struct x86_emulate_ctxt *ctxt,
1325 unsigned long addr, void *dest, unsigned size)
1328 struct read_cache *mc = &ctxt->mem_read;
1330 if (mc->pos < mc->end)
1333 WARN_ON((mc->end + size) >= sizeof(mc->data));
1335 rc = ctxt->ops->read_emulated(ctxt, addr, mc->data + mc->end, size,
1337 if (rc != X86EMUL_CONTINUE)
1343 memcpy(dest, mc->data + mc->pos, size);
1345 return X86EMUL_CONTINUE;
1348 static int segmented_read(struct x86_emulate_ctxt *ctxt,
1349 struct segmented_address addr,
1356 rc = linearize(ctxt, addr, size, false, &linear);
1357 if (rc != X86EMUL_CONTINUE)
1359 return read_emulated(ctxt, linear, data, size);
1362 static int segmented_write(struct x86_emulate_ctxt *ctxt,
1363 struct segmented_address addr,
1370 rc = linearize(ctxt, addr, size, true, &linear);
1371 if (rc != X86EMUL_CONTINUE)
1373 return ctxt->ops->write_emulated(ctxt, linear, data, size,
1377 static int segmented_cmpxchg(struct x86_emulate_ctxt *ctxt,
1378 struct segmented_address addr,
1379 const void *orig_data, const void *data,
1385 rc = linearize(ctxt, addr, size, true, &linear);
1386 if (rc != X86EMUL_CONTINUE)
1388 return ctxt->ops->cmpxchg_emulated(ctxt, linear, orig_data, data,
1389 size, &ctxt->exception);
1392 static int pio_in_emulated(struct x86_emulate_ctxt *ctxt,
1393 unsigned int size, unsigned short port,
1396 struct read_cache *rc = &ctxt->io_read;
1398 if (rc->pos == rc->end) { /* refill pio read ahead */
1399 unsigned int in_page, n;
1400 unsigned int count = ctxt->rep_prefix ?
1401 address_mask(ctxt, reg_read(ctxt, VCPU_REGS_RCX)) : 1;
1402 in_page = (ctxt->eflags & EFLG_DF) ?
1403 offset_in_page(reg_read(ctxt, VCPU_REGS_RDI)) :
1404 PAGE_SIZE - offset_in_page(reg_read(ctxt, VCPU_REGS_RDI));
1405 n = min3(in_page, (unsigned int)sizeof(rc->data) / size, count);
1408 rc->pos = rc->end = 0;
1409 if (!ctxt->ops->pio_in_emulated(ctxt, size, port, rc->data, n))
1414 if (ctxt->rep_prefix && (ctxt->d & String) &&
1415 !(ctxt->eflags & EFLG_DF)) {
1416 ctxt->dst.data = rc->data + rc->pos;
1417 ctxt->dst.type = OP_MEM_STR;
1418 ctxt->dst.count = (rc->end - rc->pos) / size;
1421 memcpy(dest, rc->data + rc->pos, size);
1427 static int read_interrupt_descriptor(struct x86_emulate_ctxt *ctxt,
1428 u16 index, struct desc_struct *desc)
1433 ctxt->ops->get_idt(ctxt, &dt);
1435 if (dt.size < index * 8 + 7)
1436 return emulate_gp(ctxt, index << 3 | 0x2);
1438 addr = dt.address + index * 8;
1439 return ctxt->ops->read_std(ctxt, addr, desc, sizeof *desc,
1443 static void get_descriptor_table_ptr(struct x86_emulate_ctxt *ctxt,
1444 u16 selector, struct desc_ptr *dt)
1446 const struct x86_emulate_ops *ops = ctxt->ops;
1449 if (selector & 1 << 2) {
1450 struct desc_struct desc;
1453 memset (dt, 0, sizeof *dt);
1454 if (!ops->get_segment(ctxt, &sel, &desc, &base3,
1458 dt->size = desc_limit_scaled(&desc); /* what if limit > 65535? */
1459 dt->address = get_desc_base(&desc) | ((u64)base3 << 32);
1461 ops->get_gdt(ctxt, dt);
1464 static int get_descriptor_ptr(struct x86_emulate_ctxt *ctxt,
1465 u16 selector, ulong *desc_addr_p)
1468 u16 index = selector >> 3;
1471 get_descriptor_table_ptr(ctxt, selector, &dt);
1473 if (dt.size < index * 8 + 7)
1474 return emulate_gp(ctxt, selector & 0xfffc);
1476 addr = dt.address + index * 8;
1478 #ifdef CONFIG_X86_64
1479 if (addr >> 32 != 0) {
1482 ctxt->ops->get_msr(ctxt, MSR_EFER, &efer);
1483 if (!(efer & EFER_LMA))
1488 *desc_addr_p = addr;
1489 return X86EMUL_CONTINUE;
1492 /* allowed just for 8 bytes segments */
1493 static int read_segment_descriptor(struct x86_emulate_ctxt *ctxt,
1494 u16 selector, struct desc_struct *desc,
1499 rc = get_descriptor_ptr(ctxt, selector, desc_addr_p);
1500 if (rc != X86EMUL_CONTINUE)
1503 return ctxt->ops->read_std(ctxt, *desc_addr_p, desc, sizeof(*desc),
1507 /* allowed just for 8 bytes segments */
1508 static int write_segment_descriptor(struct x86_emulate_ctxt *ctxt,
1509 u16 selector, struct desc_struct *desc)
1514 rc = get_descriptor_ptr(ctxt, selector, &addr);
1515 if (rc != X86EMUL_CONTINUE)
1518 return ctxt->ops->write_std(ctxt, addr, desc, sizeof *desc,
1522 /* Does not support long mode */
1523 static int __load_segment_descriptor(struct x86_emulate_ctxt *ctxt,
1524 u16 selector, int seg, u8 cpl,
1525 enum x86_transfer_type transfer,
1526 struct desc_struct *desc)
1528 struct desc_struct seg_desc, old_desc;
1530 unsigned err_vec = GP_VECTOR;
1532 bool null_selector = !(selector & ~0x3); /* 0000-0003 are null */
1538 memset(&seg_desc, 0, sizeof seg_desc);
1540 if (ctxt->mode == X86EMUL_MODE_REAL) {
1541 /* set real mode segment descriptor (keep limit etc. for
1543 ctxt->ops->get_segment(ctxt, &dummy, &seg_desc, NULL, seg);
1544 set_desc_base(&seg_desc, selector << 4);
1546 } else if (seg <= VCPU_SREG_GS && ctxt->mode == X86EMUL_MODE_VM86) {
1547 /* VM86 needs a clean new segment descriptor */
1548 set_desc_base(&seg_desc, selector << 4);
1549 set_desc_limit(&seg_desc, 0xffff);
1559 /* NULL selector is not valid for TR, CS and SS (except for long mode) */
1560 if ((seg == VCPU_SREG_CS
1561 || (seg == VCPU_SREG_SS
1562 && (ctxt->mode != X86EMUL_MODE_PROT64 || rpl != cpl))
1563 || seg == VCPU_SREG_TR)
1567 /* TR should be in GDT only */
1568 if (seg == VCPU_SREG_TR && (selector & (1 << 2)))
1571 if (null_selector) /* for NULL selector skip all following checks */
1574 ret = read_segment_descriptor(ctxt, selector, &seg_desc, &desc_addr);
1575 if (ret != X86EMUL_CONTINUE)
1578 err_code = selector & 0xfffc;
1579 err_vec = (transfer == X86_TRANSFER_TASK_SWITCH) ? TS_VECTOR :
1582 /* can't load system descriptor into segment selector */
1583 if (seg <= VCPU_SREG_GS && !seg_desc.s) {
1584 if (transfer == X86_TRANSFER_CALL_JMP)
1585 return X86EMUL_UNHANDLEABLE;
1590 err_vec = (seg == VCPU_SREG_SS) ? SS_VECTOR : NP_VECTOR;
1599 * segment is not a writable data segment or segment
1600 * selector's RPL != CPL or segment selector's RPL != CPL
1602 if (rpl != cpl || (seg_desc.type & 0xa) != 0x2 || dpl != cpl)
1606 if (!(seg_desc.type & 8))
1609 if (seg_desc.type & 4) {
1615 if (rpl > cpl || dpl != cpl)
1618 /* in long-mode d/b must be clear if l is set */
1619 if (seg_desc.d && seg_desc.l) {
1622 ctxt->ops->get_msr(ctxt, MSR_EFER, &efer);
1623 if (efer & EFER_LMA)
1627 /* CS(RPL) <- CPL */
1628 selector = (selector & 0xfffc) | cpl;
1631 if (seg_desc.s || (seg_desc.type != 1 && seg_desc.type != 9))
1633 old_desc = seg_desc;
1634 seg_desc.type |= 2; /* busy */
1635 ret = ctxt->ops->cmpxchg_emulated(ctxt, desc_addr, &old_desc, &seg_desc,
1636 sizeof(seg_desc), &ctxt->exception);
1637 if (ret != X86EMUL_CONTINUE)
1640 case VCPU_SREG_LDTR:
1641 if (seg_desc.s || seg_desc.type != 2)
1644 default: /* DS, ES, FS, or GS */
1646 * segment is not a data or readable code segment or
1647 * ((segment is a data or nonconforming code segment)
1648 * and (both RPL and CPL > DPL))
1650 if ((seg_desc.type & 0xa) == 0x8 ||
1651 (((seg_desc.type & 0xc) != 0xc) &&
1652 (rpl > dpl && cpl > dpl)))
1658 /* mark segment as accessed */
1659 if (!(seg_desc.type & 1)) {
1661 ret = write_segment_descriptor(ctxt, selector,
1663 if (ret != X86EMUL_CONTINUE)
1666 } else if (ctxt->mode == X86EMUL_MODE_PROT64) {
1667 ret = ctxt->ops->read_std(ctxt, desc_addr+8, &base3,
1668 sizeof(base3), &ctxt->exception);
1669 if (ret != X86EMUL_CONTINUE)
1671 if (is_noncanonical_address(get_desc_base(&seg_desc) |
1672 ((u64)base3 << 32)))
1673 return emulate_gp(ctxt, 0);
1676 ctxt->ops->set_segment(ctxt, selector, &seg_desc, base3, seg);
1679 return X86EMUL_CONTINUE;
1681 return emulate_exception(ctxt, err_vec, err_code, true);
1684 static int load_segment_descriptor(struct x86_emulate_ctxt *ctxt,
1685 u16 selector, int seg)
1687 u8 cpl = ctxt->ops->cpl(ctxt);
1688 return __load_segment_descriptor(ctxt, selector, seg, cpl,
1689 X86_TRANSFER_NONE, NULL);
1692 static void write_register_operand(struct operand *op)
1694 /* The 4-byte case *is* correct: in 64-bit mode we zero-extend. */
1695 switch (op->bytes) {
1697 *(u8 *)op->addr.reg = (u8)op->val;
1700 *(u16 *)op->addr.reg = (u16)op->val;
1703 *op->addr.reg = (u32)op->val;
1704 break; /* 64b: zero-extend */
1706 *op->addr.reg = op->val;
1711 static int writeback(struct x86_emulate_ctxt *ctxt, struct operand *op)
1715 write_register_operand(op);
1718 if (ctxt->lock_prefix)
1719 return segmented_cmpxchg(ctxt,
1725 return segmented_write(ctxt,
1731 return segmented_write(ctxt,
1734 op->bytes * op->count);
1737 write_sse_reg(ctxt, &op->vec_val, op->addr.xmm);
1740 write_mmx_reg(ctxt, &op->mm_val, op->addr.mm);
1748 return X86EMUL_CONTINUE;
1751 static int push(struct x86_emulate_ctxt *ctxt, void *data, int bytes)
1753 struct segmented_address addr;
1755 rsp_increment(ctxt, -bytes);
1756 addr.ea = reg_read(ctxt, VCPU_REGS_RSP) & stack_mask(ctxt);
1757 addr.seg = VCPU_SREG_SS;
1759 return segmented_write(ctxt, addr, data, bytes);
1762 static int em_push(struct x86_emulate_ctxt *ctxt)
1764 /* Disable writeback. */
1765 ctxt->dst.type = OP_NONE;
1766 return push(ctxt, &ctxt->src.val, ctxt->op_bytes);
1769 static int emulate_pop(struct x86_emulate_ctxt *ctxt,
1770 void *dest, int len)
1773 struct segmented_address addr;
1775 addr.ea = reg_read(ctxt, VCPU_REGS_RSP) & stack_mask(ctxt);
1776 addr.seg = VCPU_SREG_SS;
1777 rc = segmented_read(ctxt, addr, dest, len);
1778 if (rc != X86EMUL_CONTINUE)
1781 rsp_increment(ctxt, len);
1785 static int em_pop(struct x86_emulate_ctxt *ctxt)
1787 return emulate_pop(ctxt, &ctxt->dst.val, ctxt->op_bytes);
1790 static int emulate_popf(struct x86_emulate_ctxt *ctxt,
1791 void *dest, int len)
1794 unsigned long val, change_mask;
1795 int iopl = (ctxt->eflags & X86_EFLAGS_IOPL) >> IOPL_SHIFT;
1796 int cpl = ctxt->ops->cpl(ctxt);
1798 rc = emulate_pop(ctxt, &val, len);
1799 if (rc != X86EMUL_CONTINUE)
1802 change_mask = EFLG_CF | EFLG_PF | EFLG_AF | EFLG_ZF | EFLG_SF | EFLG_OF
1803 | EFLG_TF | EFLG_DF | EFLG_NT | EFLG_AC | EFLG_ID;
1805 switch(ctxt->mode) {
1806 case X86EMUL_MODE_PROT64:
1807 case X86EMUL_MODE_PROT32:
1808 case X86EMUL_MODE_PROT16:
1810 change_mask |= EFLG_IOPL;
1812 change_mask |= EFLG_IF;
1814 case X86EMUL_MODE_VM86:
1816 return emulate_gp(ctxt, 0);
1817 change_mask |= EFLG_IF;
1819 default: /* real mode */
1820 change_mask |= (EFLG_IOPL | EFLG_IF);
1824 *(unsigned long *)dest =
1825 (ctxt->eflags & ~change_mask) | (val & change_mask);
1830 static int em_popf(struct x86_emulate_ctxt *ctxt)
1832 ctxt->dst.type = OP_REG;
1833 ctxt->dst.addr.reg = &ctxt->eflags;
1834 ctxt->dst.bytes = ctxt->op_bytes;
1835 return emulate_popf(ctxt, &ctxt->dst.val, ctxt->op_bytes);
1838 static int em_enter(struct x86_emulate_ctxt *ctxt)
1841 unsigned frame_size = ctxt->src.val;
1842 unsigned nesting_level = ctxt->src2.val & 31;
1846 return X86EMUL_UNHANDLEABLE;
1848 rbp = reg_read(ctxt, VCPU_REGS_RBP);
1849 rc = push(ctxt, &rbp, stack_size(ctxt));
1850 if (rc != X86EMUL_CONTINUE)
1852 assign_masked(reg_rmw(ctxt, VCPU_REGS_RBP), reg_read(ctxt, VCPU_REGS_RSP),
1854 assign_masked(reg_rmw(ctxt, VCPU_REGS_RSP),
1855 reg_read(ctxt, VCPU_REGS_RSP) - frame_size,
1857 return X86EMUL_CONTINUE;
1860 static int em_leave(struct x86_emulate_ctxt *ctxt)
1862 assign_masked(reg_rmw(ctxt, VCPU_REGS_RSP), reg_read(ctxt, VCPU_REGS_RBP),
1864 return emulate_pop(ctxt, reg_rmw(ctxt, VCPU_REGS_RBP), ctxt->op_bytes);
1867 static int em_push_sreg(struct x86_emulate_ctxt *ctxt)
1869 int seg = ctxt->src2.val;
1871 ctxt->src.val = get_segment_selector(ctxt, seg);
1872 if (ctxt->op_bytes == 4) {
1873 rsp_increment(ctxt, -2);
1877 return em_push(ctxt);
1880 static int em_pop_sreg(struct x86_emulate_ctxt *ctxt)
1882 int seg = ctxt->src2.val;
1883 unsigned long selector;
1886 rc = emulate_pop(ctxt, &selector, 2);
1887 if (rc != X86EMUL_CONTINUE)
1890 if (ctxt->modrm_reg == VCPU_SREG_SS)
1891 ctxt->interruptibility = KVM_X86_SHADOW_INT_MOV_SS;
1892 if (ctxt->op_bytes > 2)
1893 rsp_increment(ctxt, ctxt->op_bytes - 2);
1895 rc = load_segment_descriptor(ctxt, (u16)selector, seg);
1899 static int em_pusha(struct x86_emulate_ctxt *ctxt)
1901 unsigned long old_esp = reg_read(ctxt, VCPU_REGS_RSP);
1902 int rc = X86EMUL_CONTINUE;
1903 int reg = VCPU_REGS_RAX;
1905 while (reg <= VCPU_REGS_RDI) {
1906 (reg == VCPU_REGS_RSP) ?
1907 (ctxt->src.val = old_esp) : (ctxt->src.val = reg_read(ctxt, reg));
1910 if (rc != X86EMUL_CONTINUE)
1919 static int em_pushf(struct x86_emulate_ctxt *ctxt)
1921 ctxt->src.val = (unsigned long)ctxt->eflags & ~EFLG_VM;
1922 return em_push(ctxt);
1925 static int em_popa(struct x86_emulate_ctxt *ctxt)
1927 int rc = X86EMUL_CONTINUE;
1928 int reg = VCPU_REGS_RDI;
1930 while (reg >= VCPU_REGS_RAX) {
1931 if (reg == VCPU_REGS_RSP) {
1932 rsp_increment(ctxt, ctxt->op_bytes);
1936 rc = emulate_pop(ctxt, reg_rmw(ctxt, reg), ctxt->op_bytes);
1937 if (rc != X86EMUL_CONTINUE)
1944 static int __emulate_int_real(struct x86_emulate_ctxt *ctxt, int irq)
1946 const struct x86_emulate_ops *ops = ctxt->ops;
1953 /* TODO: Add limit checks */
1954 ctxt->src.val = ctxt->eflags;
1956 if (rc != X86EMUL_CONTINUE)
1959 ctxt->eflags &= ~(EFLG_IF | EFLG_TF | EFLG_AC);
1961 ctxt->src.val = get_segment_selector(ctxt, VCPU_SREG_CS);
1963 if (rc != X86EMUL_CONTINUE)
1966 ctxt->src.val = ctxt->_eip;
1968 if (rc != X86EMUL_CONTINUE)
1971 ops->get_idt(ctxt, &dt);
1973 eip_addr = dt.address + (irq << 2);
1974 cs_addr = dt.address + (irq << 2) + 2;
1976 rc = ops->read_std(ctxt, cs_addr, &cs, 2, &ctxt->exception);
1977 if (rc != X86EMUL_CONTINUE)
1980 rc = ops->read_std(ctxt, eip_addr, &eip, 2, &ctxt->exception);
1981 if (rc != X86EMUL_CONTINUE)
1984 rc = load_segment_descriptor(ctxt, cs, VCPU_SREG_CS);
1985 if (rc != X86EMUL_CONTINUE)
1993 int emulate_int_real(struct x86_emulate_ctxt *ctxt, int irq)
1997 invalidate_registers(ctxt);
1998 rc = __emulate_int_real(ctxt, irq);
1999 if (rc == X86EMUL_CONTINUE)
2000 writeback_registers(ctxt);
2004 static int emulate_int(struct x86_emulate_ctxt *ctxt, int irq)
2006 switch(ctxt->mode) {
2007 case X86EMUL_MODE_REAL:
2008 return __emulate_int_real(ctxt, irq);
2009 case X86EMUL_MODE_VM86:
2010 case X86EMUL_MODE_PROT16:
2011 case X86EMUL_MODE_PROT32:
2012 case X86EMUL_MODE_PROT64:
2014 /* Protected mode interrupts unimplemented yet */
2015 return X86EMUL_UNHANDLEABLE;
2019 static int emulate_iret_real(struct x86_emulate_ctxt *ctxt)
2021 int rc = X86EMUL_CONTINUE;
2022 unsigned long temp_eip = 0;
2023 unsigned long temp_eflags = 0;
2024 unsigned long cs = 0;
2025 unsigned long mask = EFLG_CF | EFLG_PF | EFLG_AF | EFLG_ZF | EFLG_SF | EFLG_TF |
2026 EFLG_IF | EFLG_DF | EFLG_OF | EFLG_IOPL | EFLG_NT | EFLG_RF |
2027 EFLG_AC | EFLG_ID | (1 << 1); /* Last one is the reserved bit */
2028 unsigned long vm86_mask = EFLG_VM | EFLG_VIF | EFLG_VIP;
2030 /* TODO: Add stack limit check */
2032 rc = emulate_pop(ctxt, &temp_eip, ctxt->op_bytes);
2034 if (rc != X86EMUL_CONTINUE)
2037 if (temp_eip & ~0xffff)
2038 return emulate_gp(ctxt, 0);
2040 rc = emulate_pop(ctxt, &cs, ctxt->op_bytes);
2042 if (rc != X86EMUL_CONTINUE)
2045 rc = emulate_pop(ctxt, &temp_eflags, ctxt->op_bytes);
2047 if (rc != X86EMUL_CONTINUE)
2050 rc = load_segment_descriptor(ctxt, (u16)cs, VCPU_SREG_CS);
2052 if (rc != X86EMUL_CONTINUE)
2055 ctxt->_eip = temp_eip;
2058 if (ctxt->op_bytes == 4)
2059 ctxt->eflags = ((temp_eflags & mask) | (ctxt->eflags & vm86_mask));
2060 else if (ctxt->op_bytes == 2) {
2061 ctxt->eflags &= ~0xffff;
2062 ctxt->eflags |= temp_eflags;
2065 ctxt->eflags &= ~EFLG_RESERVED_ZEROS_MASK; /* Clear reserved zeros */
2066 ctxt->eflags |= EFLG_RESERVED_ONE_MASK;
2067 ctxt->ops->set_nmi_mask(ctxt, false);
2072 static int em_iret(struct x86_emulate_ctxt *ctxt)
2074 switch(ctxt->mode) {
2075 case X86EMUL_MODE_REAL:
2076 return emulate_iret_real(ctxt);
2077 case X86EMUL_MODE_VM86:
2078 case X86EMUL_MODE_PROT16:
2079 case X86EMUL_MODE_PROT32:
2080 case X86EMUL_MODE_PROT64:
2082 /* iret from protected mode unimplemented yet */
2083 return X86EMUL_UNHANDLEABLE;
2087 static int em_jmp_far(struct x86_emulate_ctxt *ctxt)
2090 unsigned short sel, old_sel;
2091 struct desc_struct old_desc, new_desc;
2092 const struct x86_emulate_ops *ops = ctxt->ops;
2093 u8 cpl = ctxt->ops->cpl(ctxt);
2095 /* Assignment of RIP may only fail in 64-bit mode */
2096 if (ctxt->mode == X86EMUL_MODE_PROT64)
2097 ops->get_segment(ctxt, &old_sel, &old_desc, NULL,
2100 memcpy(&sel, ctxt->src.valptr + ctxt->op_bytes, 2);
2102 rc = __load_segment_descriptor(ctxt, sel, VCPU_SREG_CS, cpl,
2103 X86_TRANSFER_CALL_JMP,
2105 if (rc != X86EMUL_CONTINUE)
2108 rc = assign_eip_far(ctxt, ctxt->src.val, &new_desc);
2109 if (rc != X86EMUL_CONTINUE) {
2110 WARN_ON(ctxt->mode != X86EMUL_MODE_PROT64);
2111 /* assigning eip failed; restore the old cs */
2112 ops->set_segment(ctxt, old_sel, &old_desc, 0, VCPU_SREG_CS);
2118 static int em_jmp_abs(struct x86_emulate_ctxt *ctxt)
2120 return assign_eip_near(ctxt, ctxt->src.val);
2123 static int em_call_near_abs(struct x86_emulate_ctxt *ctxt)
2128 old_eip = ctxt->_eip;
2129 rc = assign_eip_near(ctxt, ctxt->src.val);
2130 if (rc != X86EMUL_CONTINUE)
2132 ctxt->src.val = old_eip;
2137 static int em_cmpxchg8b(struct x86_emulate_ctxt *ctxt)
2139 u64 old = ctxt->dst.orig_val64;
2141 if (ctxt->dst.bytes == 16)
2142 return X86EMUL_UNHANDLEABLE;
2144 if (((u32) (old >> 0) != (u32) reg_read(ctxt, VCPU_REGS_RAX)) ||
2145 ((u32) (old >> 32) != (u32) reg_read(ctxt, VCPU_REGS_RDX))) {
2146 *reg_write(ctxt, VCPU_REGS_RAX) = (u32) (old >> 0);
2147 *reg_write(ctxt, VCPU_REGS_RDX) = (u32) (old >> 32);
2148 ctxt->eflags &= ~EFLG_ZF;
2150 ctxt->dst.val64 = ((u64)reg_read(ctxt, VCPU_REGS_RCX) << 32) |
2151 (u32) reg_read(ctxt, VCPU_REGS_RBX);
2153 ctxt->eflags |= EFLG_ZF;
2155 return X86EMUL_CONTINUE;
2158 static int em_ret(struct x86_emulate_ctxt *ctxt)
2163 rc = emulate_pop(ctxt, &eip, ctxt->op_bytes);
2164 if (rc != X86EMUL_CONTINUE)
2167 return assign_eip_near(ctxt, eip);
2170 static int em_ret_far(struct x86_emulate_ctxt *ctxt)
2173 unsigned long eip, cs;
2175 int cpl = ctxt->ops->cpl(ctxt);
2176 struct desc_struct old_desc, new_desc;
2177 const struct x86_emulate_ops *ops = ctxt->ops;
2179 if (ctxt->mode == X86EMUL_MODE_PROT64)
2180 ops->get_segment(ctxt, &old_cs, &old_desc, NULL,
2183 rc = emulate_pop(ctxt, &eip, ctxt->op_bytes);
2184 if (rc != X86EMUL_CONTINUE)
2186 rc = emulate_pop(ctxt, &cs, ctxt->op_bytes);
2187 if (rc != X86EMUL_CONTINUE)
2189 /* Outer-privilege level return is not implemented */
2190 if (ctxt->mode >= X86EMUL_MODE_PROT16 && (cs & 3) > cpl)
2191 return X86EMUL_UNHANDLEABLE;
2192 rc = __load_segment_descriptor(ctxt, (u16)cs, VCPU_SREG_CS, cpl,
2195 if (rc != X86EMUL_CONTINUE)
2197 rc = assign_eip_far(ctxt, eip, &new_desc);
2198 if (rc != X86EMUL_CONTINUE) {
2199 WARN_ON(ctxt->mode != X86EMUL_MODE_PROT64);
2200 ops->set_segment(ctxt, old_cs, &old_desc, 0, VCPU_SREG_CS);
2205 static int em_ret_far_imm(struct x86_emulate_ctxt *ctxt)
2209 rc = em_ret_far(ctxt);
2210 if (rc != X86EMUL_CONTINUE)
2212 rsp_increment(ctxt, ctxt->src.val);
2213 return X86EMUL_CONTINUE;
2216 static int em_cmpxchg(struct x86_emulate_ctxt *ctxt)
2218 /* Save real source value, then compare EAX against destination. */
2219 ctxt->dst.orig_val = ctxt->dst.val;
2220 ctxt->dst.val = reg_read(ctxt, VCPU_REGS_RAX);
2221 ctxt->src.orig_val = ctxt->src.val;
2222 ctxt->src.val = ctxt->dst.orig_val;
2223 fastop(ctxt, em_cmp);
2225 if (ctxt->eflags & EFLG_ZF) {
2226 /* Success: write back to memory; no update of EAX */
2227 ctxt->src.type = OP_NONE;
2228 ctxt->dst.val = ctxt->src.orig_val;
2230 /* Failure: write the value we saw to EAX. */
2231 ctxt->src.type = OP_REG;
2232 ctxt->src.addr.reg = reg_rmw(ctxt, VCPU_REGS_RAX);
2233 ctxt->src.val = ctxt->dst.orig_val;
2234 /* Create write-cycle to dest by writing the same value */
2235 ctxt->dst.val = ctxt->dst.orig_val;
2237 return X86EMUL_CONTINUE;
2240 static int em_lseg(struct x86_emulate_ctxt *ctxt)
2242 int seg = ctxt->src2.val;
2246 memcpy(&sel, ctxt->src.valptr + ctxt->op_bytes, 2);
2248 rc = load_segment_descriptor(ctxt, sel, seg);
2249 if (rc != X86EMUL_CONTINUE)
2252 ctxt->dst.val = ctxt->src.val;
2257 setup_syscalls_segments(struct x86_emulate_ctxt *ctxt,
2258 struct desc_struct *cs, struct desc_struct *ss)
2260 cs->l = 0; /* will be adjusted later */
2261 set_desc_base(cs, 0); /* flat segment */
2262 cs->g = 1; /* 4kb granularity */
2263 set_desc_limit(cs, 0xfffff); /* 4GB limit */
2264 cs->type = 0x0b; /* Read, Execute, Accessed */
2266 cs->dpl = 0; /* will be adjusted later */
2271 set_desc_base(ss, 0); /* flat segment */
2272 set_desc_limit(ss, 0xfffff); /* 4GB limit */
2273 ss->g = 1; /* 4kb granularity */
2275 ss->type = 0x03; /* Read/Write, Accessed */
2276 ss->d = 1; /* 32bit stack segment */
2283 static bool vendor_intel(struct x86_emulate_ctxt *ctxt)
2285 u32 eax, ebx, ecx, edx;
2288 ctxt->ops->get_cpuid(ctxt, &eax, &ebx, &ecx, &edx);
2289 return ebx == X86EMUL_CPUID_VENDOR_GenuineIntel_ebx
2290 && ecx == X86EMUL_CPUID_VENDOR_GenuineIntel_ecx
2291 && edx == X86EMUL_CPUID_VENDOR_GenuineIntel_edx;
2294 static bool em_syscall_is_enabled(struct x86_emulate_ctxt *ctxt)
2296 const struct x86_emulate_ops *ops = ctxt->ops;
2297 u32 eax, ebx, ecx, edx;
2300 * syscall should always be enabled in longmode - so only become
2301 * vendor specific (cpuid) if other modes are active...
2303 if (ctxt->mode == X86EMUL_MODE_PROT64)
2308 ops->get_cpuid(ctxt, &eax, &ebx, &ecx, &edx);
2310 * Intel ("GenuineIntel")
2311 * remark: Intel CPUs only support "syscall" in 64bit
2312 * longmode. Also an 64bit guest with a
2313 * 32bit compat-app running will #UD !! While this
2314 * behaviour can be fixed (by emulating) into AMD
2315 * response - CPUs of AMD can't behave like Intel.
2317 if (ebx == X86EMUL_CPUID_VENDOR_GenuineIntel_ebx &&
2318 ecx == X86EMUL_CPUID_VENDOR_GenuineIntel_ecx &&
2319 edx == X86EMUL_CPUID_VENDOR_GenuineIntel_edx)
2322 /* AMD ("AuthenticAMD") */
2323 if (ebx == X86EMUL_CPUID_VENDOR_AuthenticAMD_ebx &&
2324 ecx == X86EMUL_CPUID_VENDOR_AuthenticAMD_ecx &&
2325 edx == X86EMUL_CPUID_VENDOR_AuthenticAMD_edx)
2328 /* AMD ("AMDisbetter!") */
2329 if (ebx == X86EMUL_CPUID_VENDOR_AMDisbetterI_ebx &&
2330 ecx == X86EMUL_CPUID_VENDOR_AMDisbetterI_ecx &&
2331 edx == X86EMUL_CPUID_VENDOR_AMDisbetterI_edx)
2334 /* default: (not Intel, not AMD), apply Intel's stricter rules... */
2338 static int em_syscall(struct x86_emulate_ctxt *ctxt)
2340 const struct x86_emulate_ops *ops = ctxt->ops;
2341 struct desc_struct cs, ss;
2346 /* syscall is not available in real mode */
2347 if (ctxt->mode == X86EMUL_MODE_REAL ||
2348 ctxt->mode == X86EMUL_MODE_VM86)
2349 return emulate_ud(ctxt);
2351 if (!(em_syscall_is_enabled(ctxt)))
2352 return emulate_ud(ctxt);
2354 ops->get_msr(ctxt, MSR_EFER, &efer);
2355 setup_syscalls_segments(ctxt, &cs, &ss);
2357 if (!(efer & EFER_SCE))
2358 return emulate_ud(ctxt);
2360 ops->get_msr(ctxt, MSR_STAR, &msr_data);
2362 cs_sel = (u16)(msr_data & 0xfffc);
2363 ss_sel = (u16)(msr_data + 8);
2365 if (efer & EFER_LMA) {
2369 ops->set_segment(ctxt, cs_sel, &cs, 0, VCPU_SREG_CS);
2370 ops->set_segment(ctxt, ss_sel, &ss, 0, VCPU_SREG_SS);
2372 *reg_write(ctxt, VCPU_REGS_RCX) = ctxt->_eip;
2373 if (efer & EFER_LMA) {
2374 #ifdef CONFIG_X86_64
2375 *reg_write(ctxt, VCPU_REGS_R11) = ctxt->eflags;
2378 ctxt->mode == X86EMUL_MODE_PROT64 ?
2379 MSR_LSTAR : MSR_CSTAR, &msr_data);
2380 ctxt->_eip = msr_data;
2382 ops->get_msr(ctxt, MSR_SYSCALL_MASK, &msr_data);
2383 ctxt->eflags &= ~msr_data;
2384 ctxt->eflags |= EFLG_RESERVED_ONE_MASK;
2388 ops->get_msr(ctxt, MSR_STAR, &msr_data);
2389 ctxt->_eip = (u32)msr_data;
2391 ctxt->eflags &= ~(EFLG_VM | EFLG_IF);
2394 return X86EMUL_CONTINUE;
2397 static int em_sysenter(struct x86_emulate_ctxt *ctxt)
2399 const struct x86_emulate_ops *ops = ctxt->ops;
2400 struct desc_struct cs, ss;
2405 ops->get_msr(ctxt, MSR_EFER, &efer);
2406 /* inject #GP if in real mode */
2407 if (ctxt->mode == X86EMUL_MODE_REAL)
2408 return emulate_gp(ctxt, 0);
2411 * Not recognized on AMD in compat mode (but is recognized in legacy
2414 if ((ctxt->mode != X86EMUL_MODE_PROT64) && (efer & EFER_LMA)
2415 && !vendor_intel(ctxt))
2416 return emulate_ud(ctxt);
2418 /* sysenter/sysexit have not been tested in 64bit mode. */
2419 if (ctxt->mode == X86EMUL_MODE_PROT64)
2420 return X86EMUL_UNHANDLEABLE;
2422 setup_syscalls_segments(ctxt, &cs, &ss);
2424 ops->get_msr(ctxt, MSR_IA32_SYSENTER_CS, &msr_data);
2425 if ((msr_data & 0xfffc) == 0x0)
2426 return emulate_gp(ctxt, 0);
2428 ctxt->eflags &= ~(EFLG_VM | EFLG_IF);
2429 cs_sel = (u16)msr_data & ~SELECTOR_RPL_MASK;
2430 ss_sel = cs_sel + 8;
2431 if (efer & EFER_LMA) {
2436 ops->set_segment(ctxt, cs_sel, &cs, 0, VCPU_SREG_CS);
2437 ops->set_segment(ctxt, ss_sel, &ss, 0, VCPU_SREG_SS);
2439 ops->get_msr(ctxt, MSR_IA32_SYSENTER_EIP, &msr_data);
2440 ctxt->_eip = (efer & EFER_LMA) ? msr_data : (u32)msr_data;
2442 ops->get_msr(ctxt, MSR_IA32_SYSENTER_ESP, &msr_data);
2443 *reg_write(ctxt, VCPU_REGS_RSP) = (efer & EFER_LMA) ? msr_data :
2446 return X86EMUL_CONTINUE;
2449 static int em_sysexit(struct x86_emulate_ctxt *ctxt)
2451 const struct x86_emulate_ops *ops = ctxt->ops;
2452 struct desc_struct cs, ss;
2453 u64 msr_data, rcx, rdx;
2455 u16 cs_sel = 0, ss_sel = 0;
2457 /* inject #GP if in real mode or Virtual 8086 mode */
2458 if (ctxt->mode == X86EMUL_MODE_REAL ||
2459 ctxt->mode == X86EMUL_MODE_VM86)
2460 return emulate_gp(ctxt, 0);
2462 setup_syscalls_segments(ctxt, &cs, &ss);
2464 if ((ctxt->rex_prefix & 0x8) != 0x0)
2465 usermode = X86EMUL_MODE_PROT64;
2467 usermode = X86EMUL_MODE_PROT32;
2469 rcx = reg_read(ctxt, VCPU_REGS_RCX);
2470 rdx = reg_read(ctxt, VCPU_REGS_RDX);
2474 ops->get_msr(ctxt, MSR_IA32_SYSENTER_CS, &msr_data);
2476 case X86EMUL_MODE_PROT32:
2477 cs_sel = (u16)(msr_data + 16);
2478 if ((msr_data & 0xfffc) == 0x0)
2479 return emulate_gp(ctxt, 0);
2480 ss_sel = (u16)(msr_data + 24);
2484 case X86EMUL_MODE_PROT64:
2485 cs_sel = (u16)(msr_data + 32);
2486 if (msr_data == 0x0)
2487 return emulate_gp(ctxt, 0);
2488 ss_sel = cs_sel + 8;
2491 if (is_noncanonical_address(rcx) ||
2492 is_noncanonical_address(rdx))
2493 return emulate_gp(ctxt, 0);
2496 cs_sel |= SELECTOR_RPL_MASK;
2497 ss_sel |= SELECTOR_RPL_MASK;
2499 ops->set_segment(ctxt, cs_sel, &cs, 0, VCPU_SREG_CS);
2500 ops->set_segment(ctxt, ss_sel, &ss, 0, VCPU_SREG_SS);
2503 *reg_write(ctxt, VCPU_REGS_RSP) = rcx;
2505 return X86EMUL_CONTINUE;
2508 static bool emulator_bad_iopl(struct x86_emulate_ctxt *ctxt)
2511 if (ctxt->mode == X86EMUL_MODE_REAL)
2513 if (ctxt->mode == X86EMUL_MODE_VM86)
2515 iopl = (ctxt->eflags & X86_EFLAGS_IOPL) >> IOPL_SHIFT;
2516 return ctxt->ops->cpl(ctxt) > iopl;
2519 static bool emulator_io_port_access_allowed(struct x86_emulate_ctxt *ctxt,
2522 const struct x86_emulate_ops *ops = ctxt->ops;
2523 struct desc_struct tr_seg;
2526 u16 tr, io_bitmap_ptr, perm, bit_idx = port & 0x7;
2527 unsigned mask = (1 << len) - 1;
2530 ops->get_segment(ctxt, &tr, &tr_seg, &base3, VCPU_SREG_TR);
2533 if (desc_limit_scaled(&tr_seg) < 103)
2535 base = get_desc_base(&tr_seg);
2536 #ifdef CONFIG_X86_64
2537 base |= ((u64)base3) << 32;
2539 r = ops->read_std(ctxt, base + 102, &io_bitmap_ptr, 2, NULL);
2540 if (r != X86EMUL_CONTINUE)
2542 if (io_bitmap_ptr + port/8 > desc_limit_scaled(&tr_seg))
2544 r = ops->read_std(ctxt, base + io_bitmap_ptr + port/8, &perm, 2, NULL);
2545 if (r != X86EMUL_CONTINUE)
2547 if ((perm >> bit_idx) & mask)
2552 static bool emulator_io_permited(struct x86_emulate_ctxt *ctxt,
2558 if (emulator_bad_iopl(ctxt))
2559 if (!emulator_io_port_access_allowed(ctxt, port, len))
2562 ctxt->perm_ok = true;
2567 static void save_state_to_tss16(struct x86_emulate_ctxt *ctxt,
2568 struct tss_segment_16 *tss)
2570 tss->ip = ctxt->_eip;
2571 tss->flag = ctxt->eflags;
2572 tss->ax = reg_read(ctxt, VCPU_REGS_RAX);
2573 tss->cx = reg_read(ctxt, VCPU_REGS_RCX);
2574 tss->dx = reg_read(ctxt, VCPU_REGS_RDX);
2575 tss->bx = reg_read(ctxt, VCPU_REGS_RBX);
2576 tss->sp = reg_read(ctxt, VCPU_REGS_RSP);
2577 tss->bp = reg_read(ctxt, VCPU_REGS_RBP);
2578 tss->si = reg_read(ctxt, VCPU_REGS_RSI);
2579 tss->di = reg_read(ctxt, VCPU_REGS_RDI);
2581 tss->es = get_segment_selector(ctxt, VCPU_SREG_ES);
2582 tss->cs = get_segment_selector(ctxt, VCPU_SREG_CS);
2583 tss->ss = get_segment_selector(ctxt, VCPU_SREG_SS);
2584 tss->ds = get_segment_selector(ctxt, VCPU_SREG_DS);
2585 tss->ldt = get_segment_selector(ctxt, VCPU_SREG_LDTR);
2588 static int load_state_from_tss16(struct x86_emulate_ctxt *ctxt,
2589 struct tss_segment_16 *tss)
2594 ctxt->_eip = tss->ip;
2595 ctxt->eflags = tss->flag | 2;
2596 *reg_write(ctxt, VCPU_REGS_RAX) = tss->ax;
2597 *reg_write(ctxt, VCPU_REGS_RCX) = tss->cx;
2598 *reg_write(ctxt, VCPU_REGS_RDX) = tss->dx;
2599 *reg_write(ctxt, VCPU_REGS_RBX) = tss->bx;
2600 *reg_write(ctxt, VCPU_REGS_RSP) = tss->sp;
2601 *reg_write(ctxt, VCPU_REGS_RBP) = tss->bp;
2602 *reg_write(ctxt, VCPU_REGS_RSI) = tss->si;
2603 *reg_write(ctxt, VCPU_REGS_RDI) = tss->di;
2606 * SDM says that segment selectors are loaded before segment
2609 set_segment_selector(ctxt, tss->ldt, VCPU_SREG_LDTR);
2610 set_segment_selector(ctxt, tss->es, VCPU_SREG_ES);
2611 set_segment_selector(ctxt, tss->cs, VCPU_SREG_CS);
2612 set_segment_selector(ctxt, tss->ss, VCPU_SREG_SS);
2613 set_segment_selector(ctxt, tss->ds, VCPU_SREG_DS);
2618 * Now load segment descriptors. If fault happens at this stage
2619 * it is handled in a context of new task
2621 ret = __load_segment_descriptor(ctxt, tss->ldt, VCPU_SREG_LDTR, cpl,
2622 X86_TRANSFER_TASK_SWITCH, NULL);
2623 if (ret != X86EMUL_CONTINUE)
2625 ret = __load_segment_descriptor(ctxt, tss->es, VCPU_SREG_ES, cpl,
2626 X86_TRANSFER_TASK_SWITCH, NULL);
2627 if (ret != X86EMUL_CONTINUE)
2629 ret = __load_segment_descriptor(ctxt, tss->cs, VCPU_SREG_CS, cpl,
2630 X86_TRANSFER_TASK_SWITCH, NULL);
2631 if (ret != X86EMUL_CONTINUE)
2633 ret = __load_segment_descriptor(ctxt, tss->ss, VCPU_SREG_SS, cpl,
2634 X86_TRANSFER_TASK_SWITCH, NULL);
2635 if (ret != X86EMUL_CONTINUE)
2637 ret = __load_segment_descriptor(ctxt, tss->ds, VCPU_SREG_DS, cpl,
2638 X86_TRANSFER_TASK_SWITCH, NULL);
2639 if (ret != X86EMUL_CONTINUE)
2642 return X86EMUL_CONTINUE;
2645 static int task_switch_16(struct x86_emulate_ctxt *ctxt,
2646 u16 tss_selector, u16 old_tss_sel,
2647 ulong old_tss_base, struct desc_struct *new_desc)
2649 const struct x86_emulate_ops *ops = ctxt->ops;
2650 struct tss_segment_16 tss_seg;
2652 u32 new_tss_base = get_desc_base(new_desc);
2654 ret = ops->read_std(ctxt, old_tss_base, &tss_seg, sizeof tss_seg,
2656 if (ret != X86EMUL_CONTINUE)
2659 save_state_to_tss16(ctxt, &tss_seg);
2661 ret = ops->write_std(ctxt, old_tss_base, &tss_seg, sizeof tss_seg,
2663 if (ret != X86EMUL_CONTINUE)
2666 ret = ops->read_std(ctxt, new_tss_base, &tss_seg, sizeof tss_seg,
2668 if (ret != X86EMUL_CONTINUE)
2671 if (old_tss_sel != 0xffff) {
2672 tss_seg.prev_task_link = old_tss_sel;
2674 ret = ops->write_std(ctxt, new_tss_base,
2675 &tss_seg.prev_task_link,
2676 sizeof tss_seg.prev_task_link,
2678 if (ret != X86EMUL_CONTINUE)
2682 return load_state_from_tss16(ctxt, &tss_seg);
2685 static void save_state_to_tss32(struct x86_emulate_ctxt *ctxt,
2686 struct tss_segment_32 *tss)
2688 /* CR3 and ldt selector are not saved intentionally */
2689 tss->eip = ctxt->_eip;
2690 tss->eflags = ctxt->eflags;
2691 tss->eax = reg_read(ctxt, VCPU_REGS_RAX);
2692 tss->ecx = reg_read(ctxt, VCPU_REGS_RCX);
2693 tss->edx = reg_read(ctxt, VCPU_REGS_RDX);
2694 tss->ebx = reg_read(ctxt, VCPU_REGS_RBX);
2695 tss->esp = reg_read(ctxt, VCPU_REGS_RSP);
2696 tss->ebp = reg_read(ctxt, VCPU_REGS_RBP);
2697 tss->esi = reg_read(ctxt, VCPU_REGS_RSI);
2698 tss->edi = reg_read(ctxt, VCPU_REGS_RDI);
2700 tss->es = get_segment_selector(ctxt, VCPU_SREG_ES);
2701 tss->cs = get_segment_selector(ctxt, VCPU_SREG_CS);
2702 tss->ss = get_segment_selector(ctxt, VCPU_SREG_SS);
2703 tss->ds = get_segment_selector(ctxt, VCPU_SREG_DS);
2704 tss->fs = get_segment_selector(ctxt, VCPU_SREG_FS);
2705 tss->gs = get_segment_selector(ctxt, VCPU_SREG_GS);
2708 static int load_state_from_tss32(struct x86_emulate_ctxt *ctxt,
2709 struct tss_segment_32 *tss)
2714 if (ctxt->ops->set_cr(ctxt, 3, tss->cr3))
2715 return emulate_gp(ctxt, 0);
2716 ctxt->_eip = tss->eip;
2717 ctxt->eflags = tss->eflags | 2;
2719 /* General purpose registers */
2720 *reg_write(ctxt, VCPU_REGS_RAX) = tss->eax;
2721 *reg_write(ctxt, VCPU_REGS_RCX) = tss->ecx;
2722 *reg_write(ctxt, VCPU_REGS_RDX) = tss->edx;
2723 *reg_write(ctxt, VCPU_REGS_RBX) = tss->ebx;
2724 *reg_write(ctxt, VCPU_REGS_RSP) = tss->esp;
2725 *reg_write(ctxt, VCPU_REGS_RBP) = tss->ebp;
2726 *reg_write(ctxt, VCPU_REGS_RSI) = tss->esi;
2727 *reg_write(ctxt, VCPU_REGS_RDI) = tss->edi;
2730 * SDM says that segment selectors are loaded before segment
2731 * descriptors. This is important because CPL checks will
2734 set_segment_selector(ctxt, tss->ldt_selector, VCPU_SREG_LDTR);
2735 set_segment_selector(ctxt, tss->es, VCPU_SREG_ES);
2736 set_segment_selector(ctxt, tss->cs, VCPU_SREG_CS);
2737 set_segment_selector(ctxt, tss->ss, VCPU_SREG_SS);
2738 set_segment_selector(ctxt, tss->ds, VCPU_SREG_DS);
2739 set_segment_selector(ctxt, tss->fs, VCPU_SREG_FS);
2740 set_segment_selector(ctxt, tss->gs, VCPU_SREG_GS);
2743 * If we're switching between Protected Mode and VM86, we need to make
2744 * sure to update the mode before loading the segment descriptors so
2745 * that the selectors are interpreted correctly.
2747 if (ctxt->eflags & X86_EFLAGS_VM) {
2748 ctxt->mode = X86EMUL_MODE_VM86;
2751 ctxt->mode = X86EMUL_MODE_PROT32;
2756 * Now load segment descriptors. If fault happenes at this stage
2757 * it is handled in a context of new task
2759 ret = __load_segment_descriptor(ctxt, tss->ldt_selector, VCPU_SREG_LDTR,
2760 cpl, X86_TRANSFER_TASK_SWITCH, NULL);
2761 if (ret != X86EMUL_CONTINUE)
2763 ret = __load_segment_descriptor(ctxt, tss->es, VCPU_SREG_ES, cpl,
2764 X86_TRANSFER_TASK_SWITCH, NULL);
2765 if (ret != X86EMUL_CONTINUE)
2767 ret = __load_segment_descriptor(ctxt, tss->cs, VCPU_SREG_CS, cpl,
2768 X86_TRANSFER_TASK_SWITCH, NULL);
2769 if (ret != X86EMUL_CONTINUE)
2771 ret = __load_segment_descriptor(ctxt, tss->ss, VCPU_SREG_SS, cpl,
2772 X86_TRANSFER_TASK_SWITCH, NULL);
2773 if (ret != X86EMUL_CONTINUE)
2775 ret = __load_segment_descriptor(ctxt, tss->ds, VCPU_SREG_DS, cpl,
2776 X86_TRANSFER_TASK_SWITCH, NULL);
2777 if (ret != X86EMUL_CONTINUE)
2779 ret = __load_segment_descriptor(ctxt, tss->fs, VCPU_SREG_FS, cpl,
2780 X86_TRANSFER_TASK_SWITCH, NULL);
2781 if (ret != X86EMUL_CONTINUE)
2783 ret = __load_segment_descriptor(ctxt, tss->gs, VCPU_SREG_GS, cpl,
2784 X86_TRANSFER_TASK_SWITCH, NULL);
2785 if (ret != X86EMUL_CONTINUE)
2788 return X86EMUL_CONTINUE;
2791 static int task_switch_32(struct x86_emulate_ctxt *ctxt,
2792 u16 tss_selector, u16 old_tss_sel,
2793 ulong old_tss_base, struct desc_struct *new_desc)
2795 const struct x86_emulate_ops *ops = ctxt->ops;
2796 struct tss_segment_32 tss_seg;
2798 u32 new_tss_base = get_desc_base(new_desc);
2799 u32 eip_offset = offsetof(struct tss_segment_32, eip);
2800 u32 ldt_sel_offset = offsetof(struct tss_segment_32, ldt_selector);
2802 ret = ops->read_std(ctxt, old_tss_base, &tss_seg, sizeof tss_seg,
2804 if (ret != X86EMUL_CONTINUE)
2807 save_state_to_tss32(ctxt, &tss_seg);
2809 /* Only GP registers and segment selectors are saved */
2810 ret = ops->write_std(ctxt, old_tss_base + eip_offset, &tss_seg.eip,
2811 ldt_sel_offset - eip_offset, &ctxt->exception);
2812 if (ret != X86EMUL_CONTINUE)
2815 ret = ops->read_std(ctxt, new_tss_base, &tss_seg, sizeof tss_seg,
2817 if (ret != X86EMUL_CONTINUE)
2820 if (old_tss_sel != 0xffff) {
2821 tss_seg.prev_task_link = old_tss_sel;
2823 ret = ops->write_std(ctxt, new_tss_base,
2824 &tss_seg.prev_task_link,
2825 sizeof tss_seg.prev_task_link,
2827 if (ret != X86EMUL_CONTINUE)
2831 return load_state_from_tss32(ctxt, &tss_seg);
2834 static int emulator_do_task_switch(struct x86_emulate_ctxt *ctxt,
2835 u16 tss_selector, int idt_index, int reason,
2836 bool has_error_code, u32 error_code)
2838 const struct x86_emulate_ops *ops = ctxt->ops;
2839 struct desc_struct curr_tss_desc, next_tss_desc;
2841 u16 old_tss_sel = get_segment_selector(ctxt, VCPU_SREG_TR);
2842 ulong old_tss_base =
2843 ops->get_cached_segment_base(ctxt, VCPU_SREG_TR);
2847 /* FIXME: old_tss_base == ~0 ? */
2849 ret = read_segment_descriptor(ctxt, tss_selector, &next_tss_desc, &desc_addr);
2850 if (ret != X86EMUL_CONTINUE)
2852 ret = read_segment_descriptor(ctxt, old_tss_sel, &curr_tss_desc, &desc_addr);
2853 if (ret != X86EMUL_CONTINUE)
2856 /* FIXME: check that next_tss_desc is tss */
2859 * Check privileges. The three cases are task switch caused by...
2861 * 1. jmp/call/int to task gate: Check against DPL of the task gate
2862 * 2. Exception/IRQ/iret: No check is performed
2863 * 3. jmp/call to TSS/task-gate: No check is performed since the
2864 * hardware checks it before exiting.
2866 if (reason == TASK_SWITCH_GATE) {
2867 if (idt_index != -1) {
2868 /* Software interrupts */
2869 struct desc_struct task_gate_desc;
2872 ret = read_interrupt_descriptor(ctxt, idt_index,
2874 if (ret != X86EMUL_CONTINUE)
2877 dpl = task_gate_desc.dpl;
2878 if ((tss_selector & 3) > dpl || ops->cpl(ctxt) > dpl)
2879 return emulate_gp(ctxt, (idt_index << 3) | 0x2);
2883 desc_limit = desc_limit_scaled(&next_tss_desc);
2884 if (!next_tss_desc.p ||
2885 ((desc_limit < 0x67 && (next_tss_desc.type & 8)) ||
2886 desc_limit < 0x2b)) {
2887 return emulate_ts(ctxt, tss_selector & 0xfffc);
2890 if (reason == TASK_SWITCH_IRET || reason == TASK_SWITCH_JMP) {
2891 curr_tss_desc.type &= ~(1 << 1); /* clear busy flag */
2892 write_segment_descriptor(ctxt, old_tss_sel, &curr_tss_desc);
2895 if (reason == TASK_SWITCH_IRET)
2896 ctxt->eflags = ctxt->eflags & ~X86_EFLAGS_NT;
2898 /* set back link to prev task only if NT bit is set in eflags
2899 note that old_tss_sel is not used after this point */
2900 if (reason != TASK_SWITCH_CALL && reason != TASK_SWITCH_GATE)
2901 old_tss_sel = 0xffff;
2903 if (next_tss_desc.type & 8)
2904 ret = task_switch_32(ctxt, tss_selector, old_tss_sel,
2905 old_tss_base, &next_tss_desc);
2907 ret = task_switch_16(ctxt, tss_selector, old_tss_sel,
2908 old_tss_base, &next_tss_desc);
2909 if (ret != X86EMUL_CONTINUE)
2912 if (reason == TASK_SWITCH_CALL || reason == TASK_SWITCH_GATE)
2913 ctxt->eflags = ctxt->eflags | X86_EFLAGS_NT;
2915 if (reason != TASK_SWITCH_IRET) {
2916 next_tss_desc.type |= (1 << 1); /* set busy flag */
2917 write_segment_descriptor(ctxt, tss_selector, &next_tss_desc);
2920 ops->set_cr(ctxt, 0, ops->get_cr(ctxt, 0) | X86_CR0_TS);
2921 ops->set_segment(ctxt, tss_selector, &next_tss_desc, 0, VCPU_SREG_TR);
2923 if (has_error_code) {
2924 ctxt->op_bytes = ctxt->ad_bytes = (next_tss_desc.type & 8) ? 4 : 2;
2925 ctxt->lock_prefix = 0;
2926 ctxt->src.val = (unsigned long) error_code;
2927 ret = em_push(ctxt);
2933 int emulator_task_switch(struct x86_emulate_ctxt *ctxt,
2934 u16 tss_selector, int idt_index, int reason,
2935 bool has_error_code, u32 error_code)
2939 invalidate_registers(ctxt);
2940 ctxt->_eip = ctxt->eip;
2941 ctxt->dst.type = OP_NONE;
2943 rc = emulator_do_task_switch(ctxt, tss_selector, idt_index, reason,
2944 has_error_code, error_code);
2946 if (rc == X86EMUL_CONTINUE) {
2947 ctxt->eip = ctxt->_eip;
2948 writeback_registers(ctxt);
2951 return (rc == X86EMUL_UNHANDLEABLE) ? EMULATION_FAILED : EMULATION_OK;
2954 static void string_addr_inc(struct x86_emulate_ctxt *ctxt, int reg,
2957 int df = (ctxt->eflags & EFLG_DF) ? -op->count : op->count;
2959 register_address_increment(ctxt, reg, df * op->bytes);
2960 op->addr.mem.ea = register_address(ctxt, reg);
2963 static int em_das(struct x86_emulate_ctxt *ctxt)
2966 bool af, cf, old_cf;
2968 cf = ctxt->eflags & X86_EFLAGS_CF;
2974 af = ctxt->eflags & X86_EFLAGS_AF;
2975 if ((al & 0x0f) > 9 || af) {
2977 cf = old_cf | (al >= 250);
2982 if (old_al > 0x99 || old_cf) {
2988 /* Set PF, ZF, SF */
2989 ctxt->src.type = OP_IMM;
2991 ctxt->src.bytes = 1;
2992 fastop(ctxt, em_or);
2993 ctxt->eflags &= ~(X86_EFLAGS_AF | X86_EFLAGS_CF);
2995 ctxt->eflags |= X86_EFLAGS_CF;
2997 ctxt->eflags |= X86_EFLAGS_AF;
2998 return X86EMUL_CONTINUE;
3001 static int em_aam(struct x86_emulate_ctxt *ctxt)
3005 if (ctxt->src.val == 0)
3006 return emulate_de(ctxt);
3008 al = ctxt->dst.val & 0xff;
3009 ah = al / ctxt->src.val;
3010 al %= ctxt->src.val;
3012 ctxt->dst.val = (ctxt->dst.val & 0xffff0000) | al | (ah << 8);
3014 /* Set PF, ZF, SF */
3015 ctxt->src.type = OP_IMM;
3017 ctxt->src.bytes = 1;
3018 fastop(ctxt, em_or);
3020 return X86EMUL_CONTINUE;
3023 static int em_aad(struct x86_emulate_ctxt *ctxt)
3025 u8 al = ctxt->dst.val & 0xff;
3026 u8 ah = (ctxt->dst.val >> 8) & 0xff;
3028 al = (al + (ah * ctxt->src.val)) & 0xff;
3030 ctxt->dst.val = (ctxt->dst.val & 0xffff0000) | al;
3032 /* Set PF, ZF, SF */
3033 ctxt->src.type = OP_IMM;
3035 ctxt->src.bytes = 1;
3036 fastop(ctxt, em_or);
3038 return X86EMUL_CONTINUE;
3041 static int em_call(struct x86_emulate_ctxt *ctxt)
3044 long rel = ctxt->src.val;
3046 ctxt->src.val = (unsigned long)ctxt->_eip;
3047 rc = jmp_rel(ctxt, rel);
3048 if (rc != X86EMUL_CONTINUE)
3050 return em_push(ctxt);
3053 static int em_call_far(struct x86_emulate_ctxt *ctxt)
3058 struct desc_struct old_desc, new_desc;
3059 const struct x86_emulate_ops *ops = ctxt->ops;
3060 int cpl = ctxt->ops->cpl(ctxt);
3061 enum x86emul_mode prev_mode = ctxt->mode;
3063 old_eip = ctxt->_eip;
3064 ops->get_segment(ctxt, &old_cs, &old_desc, NULL, VCPU_SREG_CS);
3066 memcpy(&sel, ctxt->src.valptr + ctxt->op_bytes, 2);
3067 rc = __load_segment_descriptor(ctxt, sel, VCPU_SREG_CS, cpl,
3068 X86_TRANSFER_CALL_JMP, &new_desc);
3069 if (rc != X86EMUL_CONTINUE)
3072 rc = assign_eip_far(ctxt, ctxt->src.val, &new_desc);
3073 if (rc != X86EMUL_CONTINUE)
3076 ctxt->src.val = old_cs;
3078 if (rc != X86EMUL_CONTINUE)
3081 ctxt->src.val = old_eip;
3083 /* If we failed, we tainted the memory, but the very least we should
3085 if (rc != X86EMUL_CONTINUE) {
3086 pr_warn_once("faulting far call emulation tainted memory\n");
3091 ops->set_segment(ctxt, old_cs, &old_desc, 0, VCPU_SREG_CS);
3092 ctxt->mode = prev_mode;
3097 static int em_ret_near_imm(struct x86_emulate_ctxt *ctxt)
3102 rc = emulate_pop(ctxt, &eip, ctxt->op_bytes);
3103 if (rc != X86EMUL_CONTINUE)
3105 rc = assign_eip_near(ctxt, eip);
3106 if (rc != X86EMUL_CONTINUE)
3108 rsp_increment(ctxt, ctxt->src.val);
3109 return X86EMUL_CONTINUE;
3112 static int em_xchg(struct x86_emulate_ctxt *ctxt)
3114 /* Write back the register source. */
3115 ctxt->src.val = ctxt->dst.val;
3116 write_register_operand(&ctxt->src);
3118 /* Write back the memory destination with implicit LOCK prefix. */
3119 ctxt->dst.val = ctxt->src.orig_val;
3120 ctxt->lock_prefix = 1;
3121 return X86EMUL_CONTINUE;
3124 static int em_imul_3op(struct x86_emulate_ctxt *ctxt)
3126 ctxt->dst.val = ctxt->src2.val;
3127 return fastop(ctxt, em_imul);
3130 static int em_cwd(struct x86_emulate_ctxt *ctxt)
3132 ctxt->dst.type = OP_REG;
3133 ctxt->dst.bytes = ctxt->src.bytes;
3134 ctxt->dst.addr.reg = reg_rmw(ctxt, VCPU_REGS_RDX);
3135 ctxt->dst.val = ~((ctxt->src.val >> (ctxt->src.bytes * 8 - 1)) - 1);
3137 return X86EMUL_CONTINUE;
3140 static int em_rdtsc(struct x86_emulate_ctxt *ctxt)
3144 ctxt->ops->get_msr(ctxt, MSR_IA32_TSC, &tsc);
3145 *reg_write(ctxt, VCPU_REGS_RAX) = (u32)tsc;
3146 *reg_write(ctxt, VCPU_REGS_RDX) = tsc >> 32;
3147 return X86EMUL_CONTINUE;
3150 static int em_rdpmc(struct x86_emulate_ctxt *ctxt)
3154 if (ctxt->ops->read_pmc(ctxt, reg_read(ctxt, VCPU_REGS_RCX), &pmc))
3155 return emulate_gp(ctxt, 0);
3156 *reg_write(ctxt, VCPU_REGS_RAX) = (u32)pmc;
3157 *reg_write(ctxt, VCPU_REGS_RDX) = pmc >> 32;
3158 return X86EMUL_CONTINUE;
3161 static int em_mov(struct x86_emulate_ctxt *ctxt)
3163 memcpy(ctxt->dst.valptr, ctxt->src.valptr, sizeof(ctxt->src.valptr));
3164 return X86EMUL_CONTINUE;
3167 #define FFL(x) bit(X86_FEATURE_##x)
3169 static int em_movbe(struct x86_emulate_ctxt *ctxt)
3171 u32 ebx, ecx, edx, eax = 1;
3175 * Check MOVBE is set in the guest-visible CPUID leaf.
3177 ctxt->ops->get_cpuid(ctxt, &eax, &ebx, &ecx, &edx);
3178 if (!(ecx & FFL(MOVBE)))
3179 return emulate_ud(ctxt);
3181 switch (ctxt->op_bytes) {
3184 * From MOVBE definition: "...When the operand size is 16 bits,
3185 * the upper word of the destination register remains unchanged
3188 * Both casting ->valptr and ->val to u16 breaks strict aliasing
3189 * rules so we have to do the operation almost per hand.
3191 tmp = (u16)ctxt->src.val;
3192 ctxt->dst.val &= ~0xffffUL;
3193 ctxt->dst.val |= (unsigned long)swab16(tmp);
3196 ctxt->dst.val = swab32((u32)ctxt->src.val);
3199 ctxt->dst.val = swab64(ctxt->src.val);
3204 return X86EMUL_CONTINUE;
3207 static int em_cr_write(struct x86_emulate_ctxt *ctxt)
3209 if (ctxt->ops->set_cr(ctxt, ctxt->modrm_reg, ctxt->src.val))
3210 return emulate_gp(ctxt, 0);
3212 /* Disable writeback. */
3213 ctxt->dst.type = OP_NONE;
3214 return X86EMUL_CONTINUE;
3217 static int em_dr_write(struct x86_emulate_ctxt *ctxt)
3221 if (ctxt->mode == X86EMUL_MODE_PROT64)
3222 val = ctxt->src.val & ~0ULL;
3224 val = ctxt->src.val & ~0U;
3226 /* #UD condition is already handled. */
3227 if (ctxt->ops->set_dr(ctxt, ctxt->modrm_reg, val) < 0)
3228 return emulate_gp(ctxt, 0);
3230 /* Disable writeback. */
3231 ctxt->dst.type = OP_NONE;
3232 return X86EMUL_CONTINUE;
3235 static int em_wrmsr(struct x86_emulate_ctxt *ctxt)
3239 msr_data = (u32)reg_read(ctxt, VCPU_REGS_RAX)
3240 | ((u64)reg_read(ctxt, VCPU_REGS_RDX) << 32);
3241 if (ctxt->ops->set_msr(ctxt, reg_read(ctxt, VCPU_REGS_RCX), msr_data))
3242 return emulate_gp(ctxt, 0);
3244 return X86EMUL_CONTINUE;
3247 static int em_rdmsr(struct x86_emulate_ctxt *ctxt)
3251 if (ctxt->ops->get_msr(ctxt, reg_read(ctxt, VCPU_REGS_RCX), &msr_data))
3252 return emulate_gp(ctxt, 0);
3254 *reg_write(ctxt, VCPU_REGS_RAX) = (u32)msr_data;
3255 *reg_write(ctxt, VCPU_REGS_RDX) = msr_data >> 32;
3256 return X86EMUL_CONTINUE;
3259 static int em_mov_rm_sreg(struct x86_emulate_ctxt *ctxt)
3261 if (ctxt->modrm_reg > VCPU_SREG_GS)
3262 return emulate_ud(ctxt);
3264 ctxt->dst.val = get_segment_selector(ctxt, ctxt->modrm_reg);
3265 if (ctxt->dst.bytes == 4 && ctxt->dst.type == OP_MEM)
3266 ctxt->dst.bytes = 2;
3267 return X86EMUL_CONTINUE;
3270 static int em_mov_sreg_rm(struct x86_emulate_ctxt *ctxt)
3272 u16 sel = ctxt->src.val;
3274 if (ctxt->modrm_reg == VCPU_SREG_CS || ctxt->modrm_reg > VCPU_SREG_GS)
3275 return emulate_ud(ctxt);
3277 if (ctxt->modrm_reg == VCPU_SREG_SS)
3278 ctxt->interruptibility = KVM_X86_SHADOW_INT_MOV_SS;
3280 /* Disable writeback. */
3281 ctxt->dst.type = OP_NONE;
3282 return load_segment_descriptor(ctxt, sel, ctxt->modrm_reg);
3285 static int em_lldt(struct x86_emulate_ctxt *ctxt)
3287 u16 sel = ctxt->src.val;
3289 /* Disable writeback. */
3290 ctxt->dst.type = OP_NONE;
3291 return load_segment_descriptor(ctxt, sel, VCPU_SREG_LDTR);
3294 static int em_ltr(struct x86_emulate_ctxt *ctxt)
3296 u16 sel = ctxt->src.val;
3298 /* Disable writeback. */
3299 ctxt->dst.type = OP_NONE;
3300 return load_segment_descriptor(ctxt, sel, VCPU_SREG_TR);
3303 static int em_invlpg(struct x86_emulate_ctxt *ctxt)
3308 rc = linearize(ctxt, ctxt->src.addr.mem, 1, false, &linear);
3309 if (rc == X86EMUL_CONTINUE)
3310 ctxt->ops->invlpg(ctxt, linear);
3311 /* Disable writeback. */
3312 ctxt->dst.type = OP_NONE;
3313 return X86EMUL_CONTINUE;
3316 static int em_clts(struct x86_emulate_ctxt *ctxt)
3320 cr0 = ctxt->ops->get_cr(ctxt, 0);
3322 ctxt->ops->set_cr(ctxt, 0, cr0);
3323 return X86EMUL_CONTINUE;
3326 static int em_vmcall(struct x86_emulate_ctxt *ctxt)
3328 int rc = ctxt->ops->fix_hypercall(ctxt);
3330 if (rc != X86EMUL_CONTINUE)
3333 /* Let the processor re-execute the fixed hypercall */
3334 ctxt->_eip = ctxt->eip;
3335 /* Disable writeback. */
3336 ctxt->dst.type = OP_NONE;
3337 return X86EMUL_CONTINUE;
3340 static int emulate_store_desc_ptr(struct x86_emulate_ctxt *ctxt,
3341 void (*get)(struct x86_emulate_ctxt *ctxt,
3342 struct desc_ptr *ptr))
3344 struct desc_ptr desc_ptr;
3346 if (ctxt->mode == X86EMUL_MODE_PROT64)
3348 get(ctxt, &desc_ptr);
3349 if (ctxt->op_bytes == 2) {
3351 desc_ptr.address &= 0x00ffffff;
3353 /* Disable writeback. */
3354 ctxt->dst.type = OP_NONE;
3355 return segmented_write(ctxt, ctxt->dst.addr.mem,
3356 &desc_ptr, 2 + ctxt->op_bytes);
3359 static int em_sgdt(struct x86_emulate_ctxt *ctxt)
3361 return emulate_store_desc_ptr(ctxt, ctxt->ops->get_gdt);
3364 static int em_sidt(struct x86_emulate_ctxt *ctxt)
3366 return emulate_store_desc_ptr(ctxt, ctxt->ops->get_idt);
3369 static int em_lgdt_lidt(struct x86_emulate_ctxt *ctxt, bool lgdt)
3371 struct desc_ptr desc_ptr;
3374 if (ctxt->mode == X86EMUL_MODE_PROT64)
3376 rc = read_descriptor(ctxt, ctxt->src.addr.mem,
3377 &desc_ptr.size, &desc_ptr.address,
3379 if (rc != X86EMUL_CONTINUE)
3381 if (ctxt->mode == X86EMUL_MODE_PROT64 &&
3382 is_noncanonical_address(desc_ptr.address))
3383 return emulate_gp(ctxt, 0);
3385 ctxt->ops->set_gdt(ctxt, &desc_ptr);
3387 ctxt->ops->set_idt(ctxt, &desc_ptr);
3388 /* Disable writeback. */
3389 ctxt->dst.type = OP_NONE;
3390 return X86EMUL_CONTINUE;
3393 static int em_lgdt(struct x86_emulate_ctxt *ctxt)
3395 return em_lgdt_lidt(ctxt, true);
3398 static int em_vmmcall(struct x86_emulate_ctxt *ctxt)
3402 rc = ctxt->ops->fix_hypercall(ctxt);
3404 /* Disable writeback. */
3405 ctxt->dst.type = OP_NONE;
3409 static int em_lidt(struct x86_emulate_ctxt *ctxt)
3411 return em_lgdt_lidt(ctxt, false);
3414 static int em_smsw(struct x86_emulate_ctxt *ctxt)
3416 if (ctxt->dst.type == OP_MEM)
3417 ctxt->dst.bytes = 2;
3418 ctxt->dst.val = ctxt->ops->get_cr(ctxt, 0);
3419 return X86EMUL_CONTINUE;
3422 static int em_lmsw(struct x86_emulate_ctxt *ctxt)
3424 ctxt->ops->set_cr(ctxt, 0, (ctxt->ops->get_cr(ctxt, 0) & ~0x0eul)
3425 | (ctxt->src.val & 0x0f));
3426 ctxt->dst.type = OP_NONE;
3427 return X86EMUL_CONTINUE;
3430 static int em_loop(struct x86_emulate_ctxt *ctxt)
3432 int rc = X86EMUL_CONTINUE;
3434 register_address_increment(ctxt, VCPU_REGS_RCX, -1);
3435 if ((address_mask(ctxt, reg_read(ctxt, VCPU_REGS_RCX)) != 0) &&
3436 (ctxt->b == 0xe2 || test_cc(ctxt->b ^ 0x5, ctxt->eflags)))
3437 rc = jmp_rel(ctxt, ctxt->src.val);
3442 static int em_jcxz(struct x86_emulate_ctxt *ctxt)
3444 int rc = X86EMUL_CONTINUE;
3446 if (address_mask(ctxt, reg_read(ctxt, VCPU_REGS_RCX)) == 0)
3447 rc = jmp_rel(ctxt, ctxt->src.val);
3452 static int em_in(struct x86_emulate_ctxt *ctxt)
3454 if (!pio_in_emulated(ctxt, ctxt->dst.bytes, ctxt->src.val,
3456 return X86EMUL_IO_NEEDED;
3458 return X86EMUL_CONTINUE;
3461 static int em_out(struct x86_emulate_ctxt *ctxt)
3463 ctxt->ops->pio_out_emulated(ctxt, ctxt->src.bytes, ctxt->dst.val,
3465 /* Disable writeback. */
3466 ctxt->dst.type = OP_NONE;
3467 return X86EMUL_CONTINUE;
3470 static int em_cli(struct x86_emulate_ctxt *ctxt)
3472 if (emulator_bad_iopl(ctxt))
3473 return emulate_gp(ctxt, 0);
3475 ctxt->eflags &= ~X86_EFLAGS_IF;
3476 return X86EMUL_CONTINUE;
3479 static int em_sti(struct x86_emulate_ctxt *ctxt)
3481 if (emulator_bad_iopl(ctxt))
3482 return emulate_gp(ctxt, 0);
3484 ctxt->interruptibility = KVM_X86_SHADOW_INT_STI;
3485 ctxt->eflags |= X86_EFLAGS_IF;
3486 return X86EMUL_CONTINUE;
3489 static int em_cpuid(struct x86_emulate_ctxt *ctxt)
3491 u32 eax, ebx, ecx, edx;
3493 eax = reg_read(ctxt, VCPU_REGS_RAX);
3494 ecx = reg_read(ctxt, VCPU_REGS_RCX);
3495 ctxt->ops->get_cpuid(ctxt, &eax, &ebx, &ecx, &edx);
3496 *reg_write(ctxt, VCPU_REGS_RAX) = eax;
3497 *reg_write(ctxt, VCPU_REGS_RBX) = ebx;
3498 *reg_write(ctxt, VCPU_REGS_RCX) = ecx;
3499 *reg_write(ctxt, VCPU_REGS_RDX) = edx;
3500 return X86EMUL_CONTINUE;
3503 static int em_sahf(struct x86_emulate_ctxt *ctxt)
3507 flags = EFLG_CF | EFLG_PF | EFLG_AF | EFLG_ZF | EFLG_SF;
3508 flags &= *reg_rmw(ctxt, VCPU_REGS_RAX) >> 8;
3510 ctxt->eflags &= ~0xffUL;
3511 ctxt->eflags |= flags | X86_EFLAGS_FIXED;
3512 return X86EMUL_CONTINUE;
3515 static int em_lahf(struct x86_emulate_ctxt *ctxt)
3517 *reg_rmw(ctxt, VCPU_REGS_RAX) &= ~0xff00UL;
3518 *reg_rmw(ctxt, VCPU_REGS_RAX) |= (ctxt->eflags & 0xff) << 8;
3519 return X86EMUL_CONTINUE;
3522 static int em_bswap(struct x86_emulate_ctxt *ctxt)
3524 switch (ctxt->op_bytes) {
3525 #ifdef CONFIG_X86_64
3527 asm("bswap %0" : "+r"(ctxt->dst.val));
3531 asm("bswap %0" : "+r"(*(u32 *)&ctxt->dst.val));
3534 return X86EMUL_CONTINUE;
3537 static int em_clflush(struct x86_emulate_ctxt *ctxt)
3539 /* emulating clflush regardless of cpuid */
3540 return X86EMUL_CONTINUE;
3543 static int em_movsxd(struct x86_emulate_ctxt *ctxt)
3545 ctxt->dst.val = (s32) ctxt->src.val;
3546 return X86EMUL_CONTINUE;
3549 static bool valid_cr(int nr)
3561 static int check_cr_read(struct x86_emulate_ctxt *ctxt)
3563 if (!valid_cr(ctxt->modrm_reg))
3564 return emulate_ud(ctxt);
3566 return X86EMUL_CONTINUE;
3569 static int check_cr_write(struct x86_emulate_ctxt *ctxt)
3571 u64 new_val = ctxt->src.val64;
3572 int cr = ctxt->modrm_reg;
3575 static u64 cr_reserved_bits[] = {
3576 0xffffffff00000000ULL,
3577 0, 0, 0, /* CR3 checked later */
3584 return emulate_ud(ctxt);
3586 if (new_val & cr_reserved_bits[cr])
3587 return emulate_gp(ctxt, 0);
3592 if (((new_val & X86_CR0_PG) && !(new_val & X86_CR0_PE)) ||
3593 ((new_val & X86_CR0_NW) && !(new_val & X86_CR0_CD)))
3594 return emulate_gp(ctxt, 0);
3596 cr4 = ctxt->ops->get_cr(ctxt, 4);
3597 ctxt->ops->get_msr(ctxt, MSR_EFER, &efer);
3599 if ((new_val & X86_CR0_PG) && (efer & EFER_LME) &&
3600 !(cr4 & X86_CR4_PAE))
3601 return emulate_gp(ctxt, 0);
3608 ctxt->ops->get_msr(ctxt, MSR_EFER, &efer);
3609 if (efer & EFER_LMA)
3610 rsvd = CR3_L_MODE_RESERVED_BITS & ~CR3_PCID_INVD;
3613 return emulate_gp(ctxt, 0);
3618 ctxt->ops->get_msr(ctxt, MSR_EFER, &efer);
3620 if ((efer & EFER_LMA) && !(new_val & X86_CR4_PAE))
3621 return emulate_gp(ctxt, 0);
3627 return X86EMUL_CONTINUE;
3630 static int check_dr7_gd(struct x86_emulate_ctxt *ctxt)
3634 ctxt->ops->get_dr(ctxt, 7, &dr7);
3636 /* Check if DR7.Global_Enable is set */
3637 return dr7 & (1 << 13);
3640 static int check_dr_read(struct x86_emulate_ctxt *ctxt)
3642 int dr = ctxt->modrm_reg;
3646 return emulate_ud(ctxt);
3648 cr4 = ctxt->ops->get_cr(ctxt, 4);
3649 if ((cr4 & X86_CR4_DE) && (dr == 4 || dr == 5))
3650 return emulate_ud(ctxt);
3652 if (check_dr7_gd(ctxt)) {
3655 ctxt->ops->get_dr(ctxt, 6, &dr6);
3657 dr6 |= DR6_BD | DR6_RTM;
3658 ctxt->ops->set_dr(ctxt, 6, dr6);
3659 return emulate_db(ctxt);
3662 return X86EMUL_CONTINUE;
3665 static int check_dr_write(struct x86_emulate_ctxt *ctxt)
3667 u64 new_val = ctxt->src.val64;
3668 int dr = ctxt->modrm_reg;
3670 if ((dr == 6 || dr == 7) && (new_val & 0xffffffff00000000ULL))
3671 return emulate_gp(ctxt, 0);
3673 return check_dr_read(ctxt);
3676 static int check_svme(struct x86_emulate_ctxt *ctxt)
3680 ctxt->ops->get_msr(ctxt, MSR_EFER, &efer);
3682 if (!(efer & EFER_SVME))
3683 return emulate_ud(ctxt);
3685 return X86EMUL_CONTINUE;
3688 static int check_svme_pa(struct x86_emulate_ctxt *ctxt)
3690 u64 rax = reg_read(ctxt, VCPU_REGS_RAX);
3692 /* Valid physical address? */
3693 if (rax & 0xffff000000000000ULL)
3694 return emulate_gp(ctxt, 0);
3696 return check_svme(ctxt);
3699 static int check_rdtsc(struct x86_emulate_ctxt *ctxt)
3701 u64 cr4 = ctxt->ops->get_cr(ctxt, 4);
3703 if (cr4 & X86_CR4_TSD && ctxt->ops->cpl(ctxt))
3704 return emulate_ud(ctxt);
3706 return X86EMUL_CONTINUE;
3709 static int check_rdpmc(struct x86_emulate_ctxt *ctxt)
3711 u64 cr4 = ctxt->ops->get_cr(ctxt, 4);
3712 u64 rcx = reg_read(ctxt, VCPU_REGS_RCX);
3714 if ((!(cr4 & X86_CR4_PCE) && ctxt->ops->cpl(ctxt)) ||
3715 ctxt->ops->check_pmc(ctxt, rcx))
3716 return emulate_gp(ctxt, 0);
3718 return X86EMUL_CONTINUE;
3721 static int check_perm_in(struct x86_emulate_ctxt *ctxt)
3723 ctxt->dst.bytes = min(ctxt->dst.bytes, 4u);
3724 if (!emulator_io_permited(ctxt, ctxt->src.val, ctxt->dst.bytes))
3725 return emulate_gp(ctxt, 0);
3727 return X86EMUL_CONTINUE;
3730 static int check_perm_out(struct x86_emulate_ctxt *ctxt)
3732 ctxt->src.bytes = min(ctxt->src.bytes, 4u);
3733 if (!emulator_io_permited(ctxt, ctxt->dst.val, ctxt->src.bytes))
3734 return emulate_gp(ctxt, 0);
3736 return X86EMUL_CONTINUE;
3739 #define D(_y) { .flags = (_y) }
3740 #define DI(_y, _i) { .flags = (_y)|Intercept, .intercept = x86_intercept_##_i }
3741 #define DIP(_y, _i, _p) { .flags = (_y)|Intercept|CheckPerm, \
3742 .intercept = x86_intercept_##_i, .check_perm = (_p) }
3743 #define N D(NotImpl)
3744 #define EXT(_f, _e) { .flags = ((_f) | RMExt), .u.group = (_e) }
3745 #define G(_f, _g) { .flags = ((_f) | Group | ModRM), .u.group = (_g) }
3746 #define GD(_f, _g) { .flags = ((_f) | GroupDual | ModRM), .u.gdual = (_g) }
3747 #define ID(_f, _i) { .flags = ((_f) | InstrDual | ModRM), .u.idual = (_i) }
3748 #define MD(_f, _m) { .flags = ((_f) | ModeDual), .u.mdual = (_m) }
3749 #define E(_f, _e) { .flags = ((_f) | Escape | ModRM), .u.esc = (_e) }
3750 #define I(_f, _e) { .flags = (_f), .u.execute = (_e) }
3751 #define F(_f, _e) { .flags = (_f) | Fastop, .u.fastop = (_e) }
3752 #define II(_f, _e, _i) \
3753 { .flags = (_f)|Intercept, .u.execute = (_e), .intercept = x86_intercept_##_i }
3754 #define IIP(_f, _e, _i, _p) \
3755 { .flags = (_f)|Intercept|CheckPerm, .u.execute = (_e), \
3756 .intercept = x86_intercept_##_i, .check_perm = (_p) }
3757 #define GP(_f, _g) { .flags = ((_f) | Prefix), .u.gprefix = (_g) }
3759 #define D2bv(_f) D((_f) | ByteOp), D(_f)
3760 #define D2bvIP(_f, _i, _p) DIP((_f) | ByteOp, _i, _p), DIP(_f, _i, _p)
3761 #define I2bv(_f, _e) I((_f) | ByteOp, _e), I(_f, _e)
3762 #define F2bv(_f, _e) F((_f) | ByteOp, _e), F(_f, _e)
3763 #define I2bvIP(_f, _e, _i, _p) \
3764 IIP((_f) | ByteOp, _e, _i, _p), IIP(_f, _e, _i, _p)
3766 #define F6ALU(_f, _e) F2bv((_f) | DstMem | SrcReg | ModRM, _e), \
3767 F2bv(((_f) | DstReg | SrcMem | ModRM) & ~Lock, _e), \
3768 F2bv(((_f) & ~Lock) | DstAcc | SrcImm, _e)
3770 static const struct opcode group7_rm0[] = {
3772 I(SrcNone | Priv | EmulateOnUD, em_vmcall),
3776 static const struct opcode group7_rm1[] = {
3777 DI(SrcNone | Priv, monitor),
3778 DI(SrcNone | Priv, mwait),
3782 static const struct opcode group7_rm3[] = {
3783 DIP(SrcNone | Prot | Priv, vmrun, check_svme_pa),
3784 II(SrcNone | Prot | EmulateOnUD, em_vmmcall, vmmcall),
3785 DIP(SrcNone | Prot | Priv, vmload, check_svme_pa),
3786 DIP(SrcNone | Prot | Priv, vmsave, check_svme_pa),
3787 DIP(SrcNone | Prot | Priv, stgi, check_svme),
3788 DIP(SrcNone | Prot | Priv, clgi, check_svme),
3789 DIP(SrcNone | Prot | Priv, skinit, check_svme),
3790 DIP(SrcNone | Prot | Priv, invlpga, check_svme),
3793 static const struct opcode group7_rm7[] = {
3795 DIP(SrcNone, rdtscp, check_rdtsc),
3799 static const struct opcode group1[] = {
3801 F(Lock | PageTable, em_or),
3804 F(Lock | PageTable, em_and),
3810 static const struct opcode group1A[] = {
3811 I(DstMem | SrcNone | Mov | Stack | IncSP, em_pop), N, N, N, N, N, N, N,
3814 static const struct opcode group2[] = {
3815 F(DstMem | ModRM, em_rol),
3816 F(DstMem | ModRM, em_ror),
3817 F(DstMem | ModRM, em_rcl),
3818 F(DstMem | ModRM, em_rcr),
3819 F(DstMem | ModRM, em_shl),
3820 F(DstMem | ModRM, em_shr),
3821 F(DstMem | ModRM, em_shl),
3822 F(DstMem | ModRM, em_sar),
3825 static const struct opcode group3[] = {
3826 F(DstMem | SrcImm | NoWrite, em_test),
3827 F(DstMem | SrcImm | NoWrite, em_test),
3828 F(DstMem | SrcNone | Lock, em_not),
3829 F(DstMem | SrcNone | Lock, em_neg),
3830 F(DstXacc | Src2Mem, em_mul_ex),
3831 F(DstXacc | Src2Mem, em_imul_ex),
3832 F(DstXacc | Src2Mem, em_div_ex),
3833 F(DstXacc | Src2Mem, em_idiv_ex),
3836 static const struct opcode group4[] = {
3837 F(ByteOp | DstMem | SrcNone | Lock, em_inc),
3838 F(ByteOp | DstMem | SrcNone | Lock, em_dec),
3842 static const struct opcode group5[] = {
3843 F(DstMem | SrcNone | Lock, em_inc),
3844 F(DstMem | SrcNone | Lock, em_dec),
3845 I(SrcMem | NearBranch, em_call_near_abs),
3846 I(SrcMemFAddr | ImplicitOps | Stack, em_call_far),
3847 I(SrcMem | NearBranch, em_jmp_abs),
3848 I(SrcMemFAddr | ImplicitOps, em_jmp_far),
3849 I(SrcMem | Stack, em_push), D(Undefined),
3852 static const struct opcode group6[] = {
3853 DI(Prot | DstMem, sldt),
3854 DI(Prot | DstMem, str),
3855 II(Prot | Priv | SrcMem16, em_lldt, lldt),
3856 II(Prot | Priv | SrcMem16, em_ltr, ltr),
3860 static const struct group_dual group7 = { {
3861 II(Mov | DstMem, em_sgdt, sgdt),
3862 II(Mov | DstMem, em_sidt, sidt),
3863 II(SrcMem | Priv, em_lgdt, lgdt),
3864 II(SrcMem | Priv, em_lidt, lidt),
3865 II(SrcNone | DstMem | Mov, em_smsw, smsw), N,
3866 II(SrcMem16 | Mov | Priv, em_lmsw, lmsw),
3867 II(SrcMem | ByteOp | Priv | NoAccess, em_invlpg, invlpg),
3871 N, EXT(0, group7_rm3),
3872 II(SrcNone | DstMem | Mov, em_smsw, smsw), N,
3873 II(SrcMem16 | Mov | Priv, em_lmsw, lmsw),
3877 static const struct opcode group8[] = {
3879 F(DstMem | SrcImmByte | NoWrite, em_bt),
3880 F(DstMem | SrcImmByte | Lock | PageTable, em_bts),
3881 F(DstMem | SrcImmByte | Lock, em_btr),
3882 F(DstMem | SrcImmByte | Lock | PageTable, em_btc),
3885 static const struct group_dual group9 = { {
3886 N, I(DstMem64 | Lock | PageTable, em_cmpxchg8b), N, N, N, N, N, N,
3888 N, N, N, N, N, N, N, N,
3891 static const struct opcode group11[] = {
3892 I(DstMem | SrcImm | Mov | PageTable, em_mov),
3896 static const struct gprefix pfx_0f_ae_7 = {
3897 I(SrcMem | ByteOp, em_clflush), N, N, N,
3900 static const struct group_dual group15 = { {
3901 N, N, N, N, N, N, N, GP(0, &pfx_0f_ae_7),
3903 N, N, N, N, N, N, N, N,
3906 static const struct gprefix pfx_0f_6f_0f_7f = {
3907 I(Mmx, em_mov), I(Sse | Aligned, em_mov), N, I(Sse | Unaligned, em_mov),
3910 static const struct instr_dual instr_dual_0f_2b = {
3914 static const struct gprefix pfx_0f_2b = {
3915 ID(0, &instr_dual_0f_2b), ID(0, &instr_dual_0f_2b), N, N,
3918 static const struct gprefix pfx_0f_28_0f_29 = {
3919 I(Aligned, em_mov), I(Aligned, em_mov), N, N,
3922 static const struct gprefix pfx_0f_e7 = {
3923 N, I(Sse, em_mov), N, N,
3926 static const struct escape escape_d9 = { {
3927 N, N, N, N, N, N, N, I(DstMem16 | Mov, em_fnstcw),
3930 N, N, N, N, N, N, N, N,
3932 N, N, N, N, N, N, N, N,
3934 N, N, N, N, N, N, N, N,
3936 N, N, N, N, N, N, N, N,
3938 N, N, N, N, N, N, N, N,
3940 N, N, N, N, N, N, N, N,
3942 N, N, N, N, N, N, N, N,
3944 N, N, N, N, N, N, N, N,
3947 static const struct escape escape_db = { {
3948 N, N, N, N, N, N, N, N,
3951 N, N, N, N, N, N, N, N,
3953 N, N, N, N, N, N, N, N,
3955 N, N, N, N, N, N, N, N,
3957 N, N, N, N, N, N, N, N,
3959 N, N, N, I(ImplicitOps, em_fninit), N, N, N, N,
3961 N, N, N, N, N, N, N, N,
3963 N, N, N, N, N, N, N, N,
3965 N, N, N, N, N, N, N, N,
3968 static const struct escape escape_dd = { {
3969 N, N, N, N, N, N, N, I(DstMem16 | Mov, em_fnstsw),
3972 N, N, N, N, N, N, N, N,
3974 N, N, N, N, N, N, N, N,
3976 N, N, N, N, N, N, N, N,
3978 N, N, N, N, N, N, N, N,
3980 N, N, N, N, N, N, N, N,
3982 N, N, N, N, N, N, N, N,
3984 N, N, N, N, N, N, N, N,
3986 N, N, N, N, N, N, N, N,
3989 static const struct instr_dual instr_dual_0f_c3 = {
3990 I(DstMem | SrcReg | ModRM | No16 | Mov, em_mov), N
3993 static const struct mode_dual mode_dual_63 = {
3994 N, I(DstReg | SrcMem32 | ModRM | Mov, em_movsxd)
3997 static const struct opcode opcode_table[256] = {
3999 F6ALU(Lock, em_add),
4000 I(ImplicitOps | Stack | No64 | Src2ES, em_push_sreg),
4001 I(ImplicitOps | Stack | No64 | Src2ES, em_pop_sreg),
4003 F6ALU(Lock | PageTable, em_or),
4004 I(ImplicitOps | Stack | No64 | Src2CS, em_push_sreg),
4007 F6ALU(Lock, em_adc),
4008 I(ImplicitOps | Stack | No64 | Src2SS, em_push_sreg),
4009 I(ImplicitOps | Stack | No64 | Src2SS, em_pop_sreg),
4011 F6ALU(Lock, em_sbb),
4012 I(ImplicitOps | Stack | No64 | Src2DS, em_push_sreg),
4013 I(ImplicitOps | Stack | No64 | Src2DS, em_pop_sreg),
4015 F6ALU(Lock | PageTable, em_and), N, N,
4017 F6ALU(Lock, em_sub), N, I(ByteOp | DstAcc | No64, em_das),
4019 F6ALU(Lock, em_xor), N, N,
4021 F6ALU(NoWrite, em_cmp), N, N,
4023 X8(F(DstReg, em_inc)), X8(F(DstReg, em_dec)),
4025 X8(I(SrcReg | Stack, em_push)),
4027 X8(I(DstReg | Stack, em_pop)),
4029 I(ImplicitOps | Stack | No64, em_pusha),
4030 I(ImplicitOps | Stack | No64, em_popa),
4031 N, MD(ModRM, &mode_dual_63),
4034 I(SrcImm | Mov | Stack, em_push),
4035 I(DstReg | SrcMem | ModRM | Src2Imm, em_imul_3op),
4036 I(SrcImmByte | Mov | Stack, em_push),
4037 I(DstReg | SrcMem | ModRM | Src2ImmByte, em_imul_3op),
4038 I2bvIP(DstDI | SrcDX | Mov | String | Unaligned, em_in, ins, check_perm_in), /* insb, insw/insd */
4039 I2bvIP(SrcSI | DstDX | String, em_out, outs, check_perm_out), /* outsb, outsw/outsd */
4041 X16(D(SrcImmByte | NearBranch)),
4043 G(ByteOp | DstMem | SrcImm, group1),
4044 G(DstMem | SrcImm, group1),
4045 G(ByteOp | DstMem | SrcImm | No64, group1),
4046 G(DstMem | SrcImmByte, group1),
4047 F2bv(DstMem | SrcReg | ModRM | NoWrite, em_test),
4048 I2bv(DstMem | SrcReg | ModRM | Lock | PageTable, em_xchg),
4050 I2bv(DstMem | SrcReg | ModRM | Mov | PageTable, em_mov),
4051 I2bv(DstReg | SrcMem | ModRM | Mov, em_mov),
4052 I(DstMem | SrcNone | ModRM | Mov | PageTable, em_mov_rm_sreg),
4053 D(ModRM | SrcMem | NoAccess | DstReg),
4054 I(ImplicitOps | SrcMem16 | ModRM, em_mov_sreg_rm),
4057 DI(SrcAcc | DstReg, pause), X7(D(SrcAcc | DstReg)),
4059 D(DstAcc | SrcNone), I(ImplicitOps | SrcAcc, em_cwd),
4060 I(SrcImmFAddr | No64, em_call_far), N,
4061 II(ImplicitOps | Stack, em_pushf, pushf),
4062 II(ImplicitOps | Stack, em_popf, popf),
4063 I(ImplicitOps, em_sahf), I(ImplicitOps, em_lahf),
4065 I2bv(DstAcc | SrcMem | Mov | MemAbs, em_mov),
4066 I2bv(DstMem | SrcAcc | Mov | MemAbs | PageTable, em_mov),
4067 I2bv(SrcSI | DstDI | Mov | String, em_mov),
4068 F2bv(SrcSI | DstDI | String | NoWrite, em_cmp_r),
4070 F2bv(DstAcc | SrcImm | NoWrite, em_test),
4071 I2bv(SrcAcc | DstDI | Mov | String, em_mov),
4072 I2bv(SrcSI | DstAcc | Mov | String, em_mov),
4073 F2bv(SrcAcc | DstDI | String | NoWrite, em_cmp_r),
4075 X8(I(ByteOp | DstReg | SrcImm | Mov, em_mov)),
4077 X8(I(DstReg | SrcImm64 | Mov, em_mov)),
4079 G(ByteOp | Src2ImmByte, group2), G(Src2ImmByte, group2),
4080 I(ImplicitOps | NearBranch | SrcImmU16, em_ret_near_imm),
4081 I(ImplicitOps | NearBranch, em_ret),
4082 I(DstReg | SrcMemFAddr | ModRM | No64 | Src2ES, em_lseg),
4083 I(DstReg | SrcMemFAddr | ModRM | No64 | Src2DS, em_lseg),
4084 G(ByteOp, group11), G(0, group11),
4086 I(Stack | SrcImmU16 | Src2ImmByte, em_enter), I(Stack, em_leave),
4087 I(ImplicitOps | SrcImmU16, em_ret_far_imm),
4088 I(ImplicitOps, em_ret_far),
4089 D(ImplicitOps), DI(SrcImmByte, intn),
4090 D(ImplicitOps | No64), II(ImplicitOps, em_iret, iret),
4092 G(Src2One | ByteOp, group2), G(Src2One, group2),
4093 G(Src2CL | ByteOp, group2), G(Src2CL, group2),
4094 I(DstAcc | SrcImmUByte | No64, em_aam),
4095 I(DstAcc | SrcImmUByte | No64, em_aad),
4096 F(DstAcc | ByteOp | No64, em_salc),
4097 I(DstAcc | SrcXLat | ByteOp, em_mov),
4099 N, E(0, &escape_d9), N, E(0, &escape_db), N, E(0, &escape_dd), N, N,
4101 X3(I(SrcImmByte | NearBranch, em_loop)),
4102 I(SrcImmByte | NearBranch, em_jcxz),
4103 I2bvIP(SrcImmUByte | DstAcc, em_in, in, check_perm_in),
4104 I2bvIP(SrcAcc | DstImmUByte, em_out, out, check_perm_out),
4106 I(SrcImm | NearBranch, em_call), D(SrcImm | ImplicitOps | NearBranch),
4107 I(SrcImmFAddr | No64, em_jmp_far),
4108 D(SrcImmByte | ImplicitOps | NearBranch),
4109 I2bvIP(SrcDX | DstAcc, em_in, in, check_perm_in),
4110 I2bvIP(SrcAcc | DstDX, em_out, out, check_perm_out),
4112 N, DI(ImplicitOps, icebp), N, N,
4113 DI(ImplicitOps | Priv, hlt), D(ImplicitOps),
4114 G(ByteOp, group3), G(0, group3),
4116 D(ImplicitOps), D(ImplicitOps),
4117 I(ImplicitOps, em_cli), I(ImplicitOps, em_sti),
4118 D(ImplicitOps), D(ImplicitOps), G(0, group4), G(0, group5),
4121 static const struct opcode twobyte_table[256] = {
4123 G(0, group6), GD(0, &group7), N, N,
4124 N, I(ImplicitOps | EmulateOnUD, em_syscall),
4125 II(ImplicitOps | Priv, em_clts, clts), N,
4126 DI(ImplicitOps | Priv, invd), DI(ImplicitOps | Priv, wbinvd), N, N,
4127 N, D(ImplicitOps | ModRM | SrcMem | NoAccess), N, N,
4129 N, N, N, N, N, N, N, N,
4130 D(ImplicitOps | ModRM | SrcMem | NoAccess),
4131 N, N, N, N, N, N, D(ImplicitOps | ModRM | SrcMem | NoAccess),
4133 DIP(ModRM | DstMem | Priv | Op3264 | NoMod, cr_read, check_cr_read),
4134 DIP(ModRM | DstMem | Priv | Op3264 | NoMod, dr_read, check_dr_read),
4135 IIP(ModRM | SrcMem | Priv | Op3264 | NoMod, em_cr_write, cr_write,
4137 IIP(ModRM | SrcMem | Priv | Op3264 | NoMod, em_dr_write, dr_write,
4140 GP(ModRM | DstReg | SrcMem | Mov | Sse, &pfx_0f_28_0f_29),
4141 GP(ModRM | DstMem | SrcReg | Mov | Sse, &pfx_0f_28_0f_29),
4142 N, GP(ModRM | DstMem | SrcReg | Mov | Sse, &pfx_0f_2b),
4145 II(ImplicitOps | Priv, em_wrmsr, wrmsr),
4146 IIP(ImplicitOps, em_rdtsc, rdtsc, check_rdtsc),
4147 II(ImplicitOps | Priv, em_rdmsr, rdmsr),
4148 IIP(ImplicitOps, em_rdpmc, rdpmc, check_rdpmc),
4149 I(ImplicitOps | EmulateOnUD, em_sysenter),
4150 I(ImplicitOps | Priv | EmulateOnUD, em_sysexit),
4152 N, N, N, N, N, N, N, N,
4154 X16(D(DstReg | SrcMem | ModRM)),
4156 N, N, N, N, N, N, N, N, N, N, N, N, N, N, N, N,
4161 N, N, N, GP(SrcMem | DstReg | ModRM | Mov, &pfx_0f_6f_0f_7f),
4166 N, N, N, GP(SrcReg | DstMem | ModRM | Mov, &pfx_0f_6f_0f_7f),
4168 X16(D(SrcImm | NearBranch)),
4170 X16(D(ByteOp | DstMem | SrcNone | ModRM| Mov)),
4172 I(Stack | Src2FS, em_push_sreg), I(Stack | Src2FS, em_pop_sreg),
4173 II(ImplicitOps, em_cpuid, cpuid),
4174 F(DstMem | SrcReg | ModRM | BitOp | NoWrite, em_bt),
4175 F(DstMem | SrcReg | Src2ImmByte | ModRM, em_shld),
4176 F(DstMem | SrcReg | Src2CL | ModRM, em_shld), N, N,
4178 I(Stack | Src2GS, em_push_sreg), I(Stack | Src2GS, em_pop_sreg),
4179 DI(ImplicitOps, rsm),
4180 F(DstMem | SrcReg | ModRM | BitOp | Lock | PageTable, em_bts),
4181 F(DstMem | SrcReg | Src2ImmByte | ModRM, em_shrd),
4182 F(DstMem | SrcReg | Src2CL | ModRM, em_shrd),
4183 GD(0, &group15), F(DstReg | SrcMem | ModRM, em_imul),
4185 I2bv(DstMem | SrcReg | ModRM | Lock | PageTable | SrcWrite, em_cmpxchg),
4186 I(DstReg | SrcMemFAddr | ModRM | Src2SS, em_lseg),
4187 F(DstMem | SrcReg | ModRM | BitOp | Lock, em_btr),
4188 I(DstReg | SrcMemFAddr | ModRM | Src2FS, em_lseg),
4189 I(DstReg | SrcMemFAddr | ModRM | Src2GS, em_lseg),
4190 D(DstReg | SrcMem8 | ModRM | Mov), D(DstReg | SrcMem16 | ModRM | Mov),
4194 F(DstMem | SrcReg | ModRM | BitOp | Lock | PageTable, em_btc),
4195 F(DstReg | SrcMem | ModRM, em_bsf), F(DstReg | SrcMem | ModRM, em_bsr),
4196 D(DstReg | SrcMem8 | ModRM | Mov), D(DstReg | SrcMem16 | ModRM | Mov),
4198 F2bv(DstMem | SrcReg | ModRM | SrcWrite | Lock, em_xadd),
4199 N, ID(0, &instr_dual_0f_c3),
4200 N, N, N, GD(0, &group9),
4202 X8(I(DstReg, em_bswap)),
4204 N, N, N, N, N, N, N, N, N, N, N, N, N, N, N, N,
4206 N, N, N, N, N, N, N, GP(SrcReg | DstMem | ModRM | Mov, &pfx_0f_e7),
4207 N, N, N, N, N, N, N, N,
4209 N, N, N, N, N, N, N, N, N, N, N, N, N, N, N, N
4212 static const struct instr_dual instr_dual_0f_38_f0 = {
4213 I(DstReg | SrcMem | Mov, em_movbe), N
4216 static const struct instr_dual instr_dual_0f_38_f1 = {
4217 I(DstMem | SrcReg | Mov, em_movbe), N
4220 static const struct gprefix three_byte_0f_38_f0 = {
4221 ID(0, &instr_dual_0f_38_f0), N, N, N
4224 static const struct gprefix three_byte_0f_38_f1 = {
4225 ID(0, &instr_dual_0f_38_f1), N, N, N
4229 * Insns below are selected by the prefix which indexed by the third opcode
4232 static const struct opcode opcode_map_0f_38[256] = {
4234 X16(N), X16(N), X16(N), X16(N), X16(N), X16(N), X16(N), X16(N),
4236 X16(N), X16(N), X16(N), X16(N), X16(N), X16(N), X16(N),
4238 GP(EmulateOnUD | ModRM, &three_byte_0f_38_f0),
4239 GP(EmulateOnUD | ModRM, &three_byte_0f_38_f1),
4260 static unsigned imm_size(struct x86_emulate_ctxt *ctxt)
4264 size = (ctxt->d & ByteOp) ? 1 : ctxt->op_bytes;
4270 static int decode_imm(struct x86_emulate_ctxt *ctxt, struct operand *op,
4271 unsigned size, bool sign_extension)
4273 int rc = X86EMUL_CONTINUE;
4277 op->addr.mem.ea = ctxt->_eip;
4278 /* NB. Immediates are sign-extended as necessary. */
4279 switch (op->bytes) {
4281 op->val = insn_fetch(s8, ctxt);
4284 op->val = insn_fetch(s16, ctxt);
4287 op->val = insn_fetch(s32, ctxt);
4290 op->val = insn_fetch(s64, ctxt);
4293 if (!sign_extension) {
4294 switch (op->bytes) {
4302 op->val &= 0xffffffff;
4310 static int decode_operand(struct x86_emulate_ctxt *ctxt, struct operand *op,
4313 int rc = X86EMUL_CONTINUE;
4317 decode_register_operand(ctxt, op);
4320 rc = decode_imm(ctxt, op, 1, false);
4323 ctxt->memop.bytes = (ctxt->d & ByteOp) ? 1 : ctxt->op_bytes;
4327 if (ctxt->d & BitOp)
4328 fetch_bit_operand(ctxt);
4329 op->orig_val = op->val;
4332 ctxt->memop.bytes = (ctxt->op_bytes == 8) ? 16 : 8;
4336 op->bytes = (ctxt->d & ByteOp) ? 1 : ctxt->op_bytes;
4337 op->addr.reg = reg_rmw(ctxt, VCPU_REGS_RAX);
4338 fetch_register_operand(op);
4339 op->orig_val = op->val;
4343 op->bytes = (ctxt->d & ByteOp) ? 2 : ctxt->op_bytes;
4344 op->addr.reg = reg_rmw(ctxt, VCPU_REGS_RAX);
4345 fetch_register_operand(op);
4346 op->orig_val = op->val;
4349 if (ctxt->d & ByteOp) {
4354 op->bytes = ctxt->op_bytes;
4355 op->addr.reg = reg_rmw(ctxt, VCPU_REGS_RDX);
4356 fetch_register_operand(op);
4357 op->orig_val = op->val;
4361 op->bytes = (ctxt->d & ByteOp) ? 1 : ctxt->op_bytes;
4363 register_address(ctxt, VCPU_REGS_RDI);
4364 op->addr.mem.seg = VCPU_SREG_ES;
4371 op->addr.reg = reg_rmw(ctxt, VCPU_REGS_RDX);
4372 fetch_register_operand(op);
4377 op->val = reg_read(ctxt, VCPU_REGS_RCX) & 0xff;
4380 rc = decode_imm(ctxt, op, 1, true);
4388 rc = decode_imm(ctxt, op, imm_size(ctxt), true);
4391 rc = decode_imm(ctxt, op, ctxt->op_bytes, true);
4394 ctxt->memop.bytes = 1;
4395 if (ctxt->memop.type == OP_REG) {
4396 ctxt->memop.addr.reg = decode_register(ctxt,
4397 ctxt->modrm_rm, true);
4398 fetch_register_operand(&ctxt->memop);
4402 ctxt->memop.bytes = 2;
4405 ctxt->memop.bytes = 4;
4408 rc = decode_imm(ctxt, op, 2, false);
4411 rc = decode_imm(ctxt, op, imm_size(ctxt), false);
4415 op->bytes = (ctxt->d & ByteOp) ? 1 : ctxt->op_bytes;
4417 register_address(ctxt, VCPU_REGS_RSI);
4418 op->addr.mem.seg = ctxt->seg_override;
4424 op->bytes = (ctxt->d & ByteOp) ? 1 : ctxt->op_bytes;
4427 reg_read(ctxt, VCPU_REGS_RBX) +
4428 (reg_read(ctxt, VCPU_REGS_RAX) & 0xff));
4429 op->addr.mem.seg = ctxt->seg_override;
4434 op->addr.mem.ea = ctxt->_eip;
4435 op->bytes = ctxt->op_bytes + 2;
4436 insn_fetch_arr(op->valptr, op->bytes, ctxt);
4439 ctxt->memop.bytes = ctxt->op_bytes + 2;
4443 op->val = VCPU_SREG_ES;
4447 op->val = VCPU_SREG_CS;
4451 op->val = VCPU_SREG_SS;
4455 op->val = VCPU_SREG_DS;
4459 op->val = VCPU_SREG_FS;
4463 op->val = VCPU_SREG_GS;
4466 /* Special instructions do their own operand decoding. */
4468 op->type = OP_NONE; /* Disable writeback. */
4476 int x86_decode_insn(struct x86_emulate_ctxt *ctxt, void *insn, int insn_len)
4478 int rc = X86EMUL_CONTINUE;
4479 int mode = ctxt->mode;
4480 int def_op_bytes, def_ad_bytes, goffset, simd_prefix;
4481 bool op_prefix = false;
4482 bool has_seg_override = false;
4483 struct opcode opcode;
4485 ctxt->memop.type = OP_NONE;
4486 ctxt->memopp = NULL;
4487 ctxt->_eip = ctxt->eip;
4488 ctxt->fetch.ptr = ctxt->fetch.data;
4489 ctxt->fetch.end = ctxt->fetch.data + insn_len;
4490 ctxt->opcode_len = 1;
4492 memcpy(ctxt->fetch.data, insn, insn_len);
4494 rc = __do_insn_fetch_bytes(ctxt, 1);
4495 if (rc != X86EMUL_CONTINUE)
4500 case X86EMUL_MODE_REAL:
4501 case X86EMUL_MODE_VM86:
4502 case X86EMUL_MODE_PROT16:
4503 def_op_bytes = def_ad_bytes = 2;
4505 case X86EMUL_MODE_PROT32:
4506 def_op_bytes = def_ad_bytes = 4;
4508 #ifdef CONFIG_X86_64
4509 case X86EMUL_MODE_PROT64:
4515 return EMULATION_FAILED;
4518 ctxt->op_bytes = def_op_bytes;
4519 ctxt->ad_bytes = def_ad_bytes;
4521 /* Legacy prefixes. */
4523 switch (ctxt->b = insn_fetch(u8, ctxt)) {
4524 case 0x66: /* operand-size override */
4526 /* switch between 2/4 bytes */
4527 ctxt->op_bytes = def_op_bytes ^ 6;
4529 case 0x67: /* address-size override */
4530 if (mode == X86EMUL_MODE_PROT64)
4531 /* switch between 4/8 bytes */
4532 ctxt->ad_bytes = def_ad_bytes ^ 12;
4534 /* switch between 2/4 bytes */
4535 ctxt->ad_bytes = def_ad_bytes ^ 6;
4537 case 0x26: /* ES override */
4538 case 0x2e: /* CS override */
4539 case 0x36: /* SS override */
4540 case 0x3e: /* DS override */
4541 has_seg_override = true;
4542 ctxt->seg_override = (ctxt->b >> 3) & 3;
4544 case 0x64: /* FS override */
4545 case 0x65: /* GS override */
4546 has_seg_override = true;
4547 ctxt->seg_override = ctxt->b & 7;
4549 case 0x40 ... 0x4f: /* REX */
4550 if (mode != X86EMUL_MODE_PROT64)
4552 ctxt->rex_prefix = ctxt->b;
4554 case 0xf0: /* LOCK */
4555 ctxt->lock_prefix = 1;
4557 case 0xf2: /* REPNE/REPNZ */
4558 case 0xf3: /* REP/REPE/REPZ */
4559 ctxt->rep_prefix = ctxt->b;
4565 /* Any legacy prefix after a REX prefix nullifies its effect. */
4567 ctxt->rex_prefix = 0;
4573 if (ctxt->rex_prefix & 8)
4574 ctxt->op_bytes = 8; /* REX.W */
4576 /* Opcode byte(s). */
4577 opcode = opcode_table[ctxt->b];
4578 /* Two-byte opcode? */
4579 if (ctxt->b == 0x0f) {
4580 ctxt->opcode_len = 2;
4581 ctxt->b = insn_fetch(u8, ctxt);
4582 opcode = twobyte_table[ctxt->b];
4584 /* 0F_38 opcode map */
4585 if (ctxt->b == 0x38) {
4586 ctxt->opcode_len = 3;
4587 ctxt->b = insn_fetch(u8, ctxt);
4588 opcode = opcode_map_0f_38[ctxt->b];
4591 ctxt->d = opcode.flags;
4593 if (ctxt->d & ModRM)
4594 ctxt->modrm = insn_fetch(u8, ctxt);
4596 /* vex-prefix instructions are not implemented */
4597 if (ctxt->opcode_len == 1 && (ctxt->b == 0xc5 || ctxt->b == 0xc4) &&
4598 (mode == X86EMUL_MODE_PROT64 || (ctxt->modrm & 0xc0) == 0xc0)) {
4602 while (ctxt->d & GroupMask) {
4603 switch (ctxt->d & GroupMask) {
4605 goffset = (ctxt->modrm >> 3) & 7;
4606 opcode = opcode.u.group[goffset];
4609 goffset = (ctxt->modrm >> 3) & 7;
4610 if ((ctxt->modrm >> 6) == 3)
4611 opcode = opcode.u.gdual->mod3[goffset];
4613 opcode = opcode.u.gdual->mod012[goffset];
4616 goffset = ctxt->modrm & 7;
4617 opcode = opcode.u.group[goffset];
4620 if (ctxt->rep_prefix && op_prefix)
4621 return EMULATION_FAILED;
4622 simd_prefix = op_prefix ? 0x66 : ctxt->rep_prefix;
4623 switch (simd_prefix) {
4624 case 0x00: opcode = opcode.u.gprefix->pfx_no; break;
4625 case 0x66: opcode = opcode.u.gprefix->pfx_66; break;
4626 case 0xf2: opcode = opcode.u.gprefix->pfx_f2; break;
4627 case 0xf3: opcode = opcode.u.gprefix->pfx_f3; break;
4631 if (ctxt->modrm > 0xbf)
4632 opcode = opcode.u.esc->high[ctxt->modrm - 0xc0];
4634 opcode = opcode.u.esc->op[(ctxt->modrm >> 3) & 7];
4637 if ((ctxt->modrm >> 6) == 3)
4638 opcode = opcode.u.idual->mod3;
4640 opcode = opcode.u.idual->mod012;
4643 if (ctxt->mode == X86EMUL_MODE_PROT64)
4644 opcode = opcode.u.mdual->mode64;
4646 opcode = opcode.u.mdual->mode32;
4649 return EMULATION_FAILED;
4652 ctxt->d &= ~(u64)GroupMask;
4653 ctxt->d |= opcode.flags;
4658 return EMULATION_FAILED;
4660 ctxt->execute = opcode.u.execute;
4662 if (unlikely(ctxt->ud) && likely(!(ctxt->d & EmulateOnUD)))
4663 return EMULATION_FAILED;
4665 if (unlikely(ctxt->d &
4666 (NotImpl|Stack|Op3264|Sse|Mmx|Intercept|CheckPerm|NearBranch|
4669 * These are copied unconditionally here, and checked unconditionally
4670 * in x86_emulate_insn.
4672 ctxt->check_perm = opcode.check_perm;
4673 ctxt->intercept = opcode.intercept;
4675 if (ctxt->d & NotImpl)
4676 return EMULATION_FAILED;
4678 if (mode == X86EMUL_MODE_PROT64) {
4679 if (ctxt->op_bytes == 4 && (ctxt->d & Stack))
4681 else if (ctxt->d & NearBranch)
4685 if (ctxt->d & Op3264) {
4686 if (mode == X86EMUL_MODE_PROT64)
4692 if ((ctxt->d & No16) && ctxt->op_bytes == 2)
4696 ctxt->op_bytes = 16;
4697 else if (ctxt->d & Mmx)
4701 /* ModRM and SIB bytes. */
4702 if (ctxt->d & ModRM) {
4703 rc = decode_modrm(ctxt, &ctxt->memop);
4704 if (!has_seg_override) {
4705 has_seg_override = true;
4706 ctxt->seg_override = ctxt->modrm_seg;
4708 } else if (ctxt->d & MemAbs)
4709 rc = decode_abs(ctxt, &ctxt->memop);
4710 if (rc != X86EMUL_CONTINUE)
4713 if (!has_seg_override)
4714 ctxt->seg_override = VCPU_SREG_DS;
4716 ctxt->memop.addr.mem.seg = ctxt->seg_override;
4719 * Decode and fetch the source operand: register, memory
4722 rc = decode_operand(ctxt, &ctxt->src, (ctxt->d >> SrcShift) & OpMask);
4723 if (rc != X86EMUL_CONTINUE)
4727 * Decode and fetch the second source operand: register, memory
4730 rc = decode_operand(ctxt, &ctxt->src2, (ctxt->d >> Src2Shift) & OpMask);
4731 if (rc != X86EMUL_CONTINUE)
4734 /* Decode and fetch the destination operand: register or memory. */
4735 rc = decode_operand(ctxt, &ctxt->dst, (ctxt->d >> DstShift) & OpMask);
4737 if (ctxt->rip_relative)
4738 ctxt->memopp->addr.mem.ea = address_mask(ctxt,
4739 ctxt->memopp->addr.mem.ea + ctxt->_eip);
4742 return (rc != X86EMUL_CONTINUE) ? EMULATION_FAILED : EMULATION_OK;
4745 bool x86_page_table_writing_insn(struct x86_emulate_ctxt *ctxt)
4747 return ctxt->d & PageTable;
4750 static bool string_insn_completed(struct x86_emulate_ctxt *ctxt)
4752 /* The second termination condition only applies for REPE
4753 * and REPNE. Test if the repeat string operation prefix is
4754 * REPE/REPZ or REPNE/REPNZ and if it's the case it tests the
4755 * corresponding termination condition according to:
4756 * - if REPE/REPZ and ZF = 0 then done
4757 * - if REPNE/REPNZ and ZF = 1 then done
4759 if (((ctxt->b == 0xa6) || (ctxt->b == 0xa7) ||
4760 (ctxt->b == 0xae) || (ctxt->b == 0xaf))
4761 && (((ctxt->rep_prefix == REPE_PREFIX) &&
4762 ((ctxt->eflags & EFLG_ZF) == 0))
4763 || ((ctxt->rep_prefix == REPNE_PREFIX) &&
4764 ((ctxt->eflags & EFLG_ZF) == EFLG_ZF))))
4770 static int flush_pending_x87_faults(struct x86_emulate_ctxt *ctxt)
4774 ctxt->ops->get_fpu(ctxt);
4775 asm volatile("1: fwait \n\t"
4777 ".pushsection .fixup,\"ax\" \n\t"
4779 "movb $1, %[fault] \n\t"
4782 _ASM_EXTABLE(1b, 3b)
4783 : [fault]"+qm"(fault));
4784 ctxt->ops->put_fpu(ctxt);
4786 if (unlikely(fault))
4787 return emulate_exception(ctxt, MF_VECTOR, 0, false);
4789 return X86EMUL_CONTINUE;
4792 static void fetch_possible_mmx_operand(struct x86_emulate_ctxt *ctxt,
4795 if (op->type == OP_MM)
4796 read_mmx_reg(ctxt, &op->mm_val, op->addr.mm);
4799 static int fastop(struct x86_emulate_ctxt *ctxt, void (*fop)(struct fastop *))
4801 ulong flags = (ctxt->eflags & EFLAGS_MASK) | X86_EFLAGS_IF;
4802 if (!(ctxt->d & ByteOp))
4803 fop += __ffs(ctxt->dst.bytes) * FASTOP_SIZE;
4804 asm("push %[flags]; popf; call *%[fastop]; pushf; pop %[flags]\n"
4805 : "+a"(ctxt->dst.val), "+d"(ctxt->src.val), [flags]"+D"(flags),
4807 : "c"(ctxt->src2.val));
4808 ctxt->eflags = (ctxt->eflags & ~EFLAGS_MASK) | (flags & EFLAGS_MASK);
4809 if (!fop) /* exception is returned in fop variable */
4810 return emulate_de(ctxt);
4811 return X86EMUL_CONTINUE;
4814 void init_decode_cache(struct x86_emulate_ctxt *ctxt)
4816 memset(&ctxt->rip_relative, 0,
4817 (void *)&ctxt->modrm - (void *)&ctxt->rip_relative);
4819 ctxt->io_read.pos = 0;
4820 ctxt->io_read.end = 0;
4821 ctxt->mem_read.end = 0;
4824 int x86_emulate_insn(struct x86_emulate_ctxt *ctxt)
4826 const struct x86_emulate_ops *ops = ctxt->ops;
4827 int rc = X86EMUL_CONTINUE;
4828 int saved_dst_type = ctxt->dst.type;
4830 ctxt->mem_read.pos = 0;
4832 /* LOCK prefix is allowed only with some instructions */
4833 if (ctxt->lock_prefix && (!(ctxt->d & Lock) || ctxt->dst.type != OP_MEM)) {
4834 rc = emulate_ud(ctxt);
4838 if ((ctxt->d & SrcMask) == SrcMemFAddr && ctxt->src.type != OP_MEM) {
4839 rc = emulate_ud(ctxt);
4843 if (unlikely(ctxt->d &
4844 (No64|Undefined|Sse|Mmx|Intercept|CheckPerm|Priv|Prot|String))) {
4845 if ((ctxt->mode == X86EMUL_MODE_PROT64 && (ctxt->d & No64)) ||
4846 (ctxt->d & Undefined)) {
4847 rc = emulate_ud(ctxt);
4851 if (((ctxt->d & (Sse|Mmx)) && ((ops->get_cr(ctxt, 0) & X86_CR0_EM)))
4852 || ((ctxt->d & Sse) && !(ops->get_cr(ctxt, 4) & X86_CR4_OSFXSR))) {
4853 rc = emulate_ud(ctxt);
4857 if ((ctxt->d & (Sse|Mmx)) && (ops->get_cr(ctxt, 0) & X86_CR0_TS)) {
4858 rc = emulate_nm(ctxt);
4862 if (ctxt->d & Mmx) {
4863 rc = flush_pending_x87_faults(ctxt);
4864 if (rc != X86EMUL_CONTINUE)
4867 * Now that we know the fpu is exception safe, we can fetch
4870 fetch_possible_mmx_operand(ctxt, &ctxt->src);
4871 fetch_possible_mmx_operand(ctxt, &ctxt->src2);
4872 if (!(ctxt->d & Mov))
4873 fetch_possible_mmx_operand(ctxt, &ctxt->dst);
4876 if (unlikely(ctxt->guest_mode) && (ctxt->d & Intercept)) {
4877 rc = emulator_check_intercept(ctxt, ctxt->intercept,
4878 X86_ICPT_PRE_EXCEPT);
4879 if (rc != X86EMUL_CONTINUE)
4883 /* Instruction can only be executed in protected mode */
4884 if ((ctxt->d & Prot) && ctxt->mode < X86EMUL_MODE_PROT16) {
4885 rc = emulate_ud(ctxt);
4889 /* Privileged instruction can be executed only in CPL=0 */
4890 if ((ctxt->d & Priv) && ops->cpl(ctxt)) {
4891 if (ctxt->d & PrivUD)
4892 rc = emulate_ud(ctxt);
4894 rc = emulate_gp(ctxt, 0);
4898 /* Do instruction specific permission checks */
4899 if (ctxt->d & CheckPerm) {
4900 rc = ctxt->check_perm(ctxt);
4901 if (rc != X86EMUL_CONTINUE)
4905 if (unlikely(ctxt->guest_mode) && (ctxt->d & Intercept)) {
4906 rc = emulator_check_intercept(ctxt, ctxt->intercept,
4907 X86_ICPT_POST_EXCEPT);
4908 if (rc != X86EMUL_CONTINUE)
4912 if (ctxt->rep_prefix && (ctxt->d & String)) {
4913 /* All REP prefixes have the same first termination condition */
4914 if (address_mask(ctxt, reg_read(ctxt, VCPU_REGS_RCX)) == 0) {
4915 ctxt->eip = ctxt->_eip;
4916 ctxt->eflags &= ~EFLG_RF;
4922 if ((ctxt->src.type == OP_MEM) && !(ctxt->d & NoAccess)) {
4923 rc = segmented_read(ctxt, ctxt->src.addr.mem,
4924 ctxt->src.valptr, ctxt->src.bytes);
4925 if (rc != X86EMUL_CONTINUE)
4927 ctxt->src.orig_val64 = ctxt->src.val64;
4930 if (ctxt->src2.type == OP_MEM) {
4931 rc = segmented_read(ctxt, ctxt->src2.addr.mem,
4932 &ctxt->src2.val, ctxt->src2.bytes);
4933 if (rc != X86EMUL_CONTINUE)
4937 if ((ctxt->d & DstMask) == ImplicitOps)
4941 if ((ctxt->dst.type == OP_MEM) && !(ctxt->d & Mov)) {
4942 /* optimisation - avoid slow emulated read if Mov */
4943 rc = segmented_read(ctxt, ctxt->dst.addr.mem,
4944 &ctxt->dst.val, ctxt->dst.bytes);
4945 if (rc != X86EMUL_CONTINUE) {
4946 if (!(ctxt->d & NoWrite) &&
4947 rc == X86EMUL_PROPAGATE_FAULT &&
4948 ctxt->exception.vector == PF_VECTOR)
4949 ctxt->exception.error_code |= PFERR_WRITE_MASK;
4953 ctxt->dst.orig_val = ctxt->dst.val;
4957 if (unlikely(ctxt->guest_mode) && (ctxt->d & Intercept)) {
4958 rc = emulator_check_intercept(ctxt, ctxt->intercept,
4959 X86_ICPT_POST_MEMACCESS);
4960 if (rc != X86EMUL_CONTINUE)
4964 if (ctxt->rep_prefix && (ctxt->d & String))
4965 ctxt->eflags |= EFLG_RF;
4967 ctxt->eflags &= ~EFLG_RF;
4969 if (ctxt->execute) {
4970 if (ctxt->d & Fastop) {
4971 void (*fop)(struct fastop *) = (void *)ctxt->execute;
4972 rc = fastop(ctxt, fop);
4973 if (rc != X86EMUL_CONTINUE)
4977 rc = ctxt->execute(ctxt);
4978 if (rc != X86EMUL_CONTINUE)
4983 if (ctxt->opcode_len == 2)
4985 else if (ctxt->opcode_len == 3)
4986 goto threebyte_insn;
4989 case 0x70 ... 0x7f: /* jcc (short) */
4990 if (test_cc(ctxt->b, ctxt->eflags))
4991 rc = jmp_rel(ctxt, ctxt->src.val);
4993 case 0x8d: /* lea r16/r32, m */
4994 ctxt->dst.val = ctxt->src.addr.mem.ea;
4996 case 0x90 ... 0x97: /* nop / xchg reg, rax */
4997 if (ctxt->dst.addr.reg == reg_rmw(ctxt, VCPU_REGS_RAX))
4998 ctxt->dst.type = OP_NONE;
5002 case 0x98: /* cbw/cwde/cdqe */
5003 switch (ctxt->op_bytes) {
5004 case 2: ctxt->dst.val = (s8)ctxt->dst.val; break;
5005 case 4: ctxt->dst.val = (s16)ctxt->dst.val; break;
5006 case 8: ctxt->dst.val = (s32)ctxt->dst.val; break;
5009 case 0xcc: /* int3 */
5010 rc = emulate_int(ctxt, 3);
5012 case 0xcd: /* int n */
5013 rc = emulate_int(ctxt, ctxt->src.val);
5015 case 0xce: /* into */
5016 if (ctxt->eflags & EFLG_OF)
5017 rc = emulate_int(ctxt, 4);
5019 case 0xe9: /* jmp rel */
5020 case 0xeb: /* jmp rel short */
5021 rc = jmp_rel(ctxt, ctxt->src.val);
5022 ctxt->dst.type = OP_NONE; /* Disable writeback. */
5024 case 0xf4: /* hlt */
5025 ctxt->ops->halt(ctxt);
5027 case 0xf5: /* cmc */
5028 /* complement carry flag from eflags reg */
5029 ctxt->eflags ^= EFLG_CF;
5031 case 0xf8: /* clc */
5032 ctxt->eflags &= ~EFLG_CF;
5034 case 0xf9: /* stc */
5035 ctxt->eflags |= EFLG_CF;
5037 case 0xfc: /* cld */
5038 ctxt->eflags &= ~EFLG_DF;
5040 case 0xfd: /* std */
5041 ctxt->eflags |= EFLG_DF;
5044 goto cannot_emulate;
5047 if (rc != X86EMUL_CONTINUE)
5051 if (ctxt->d & SrcWrite) {
5052 BUG_ON(ctxt->src.type == OP_MEM || ctxt->src.type == OP_MEM_STR);
5053 rc = writeback(ctxt, &ctxt->src);
5054 if (rc != X86EMUL_CONTINUE)
5057 if (!(ctxt->d & NoWrite)) {
5058 rc = writeback(ctxt, &ctxt->dst);
5059 if (rc != X86EMUL_CONTINUE)
5064 * restore dst type in case the decoding will be reused
5065 * (happens for string instruction )
5067 ctxt->dst.type = saved_dst_type;
5069 if ((ctxt->d & SrcMask) == SrcSI)
5070 string_addr_inc(ctxt, VCPU_REGS_RSI, &ctxt->src);
5072 if ((ctxt->d & DstMask) == DstDI)
5073 string_addr_inc(ctxt, VCPU_REGS_RDI, &ctxt->dst);
5075 if (ctxt->rep_prefix && (ctxt->d & String)) {
5077 struct read_cache *r = &ctxt->io_read;
5078 if ((ctxt->d & SrcMask) == SrcSI)
5079 count = ctxt->src.count;
5081 count = ctxt->dst.count;
5082 register_address_increment(ctxt, VCPU_REGS_RCX, -count);
5084 if (!string_insn_completed(ctxt)) {
5086 * Re-enter guest when pio read ahead buffer is empty
5087 * or, if it is not used, after each 1024 iteration.
5089 if ((r->end != 0 || reg_read(ctxt, VCPU_REGS_RCX) & 0x3ff) &&
5090 (r->end == 0 || r->end != r->pos)) {
5092 * Reset read cache. Usually happens before
5093 * decode, but since instruction is restarted
5094 * we have to do it here.
5096 ctxt->mem_read.end = 0;
5097 writeback_registers(ctxt);
5098 return EMULATION_RESTART;
5100 goto done; /* skip rip writeback */
5102 ctxt->eflags &= ~EFLG_RF;
5105 ctxt->eip = ctxt->_eip;
5108 if (rc == X86EMUL_PROPAGATE_FAULT) {
5109 WARN_ON(ctxt->exception.vector > 0x1f);
5110 ctxt->have_exception = true;
5112 if (rc == X86EMUL_INTERCEPTED)
5113 return EMULATION_INTERCEPTED;
5115 if (rc == X86EMUL_CONTINUE)
5116 writeback_registers(ctxt);
5118 return (rc == X86EMUL_UNHANDLEABLE) ? EMULATION_FAILED : EMULATION_OK;
5122 case 0x09: /* wbinvd */
5123 (ctxt->ops->wbinvd)(ctxt);
5125 case 0x08: /* invd */
5126 case 0x0d: /* GrpP (prefetch) */
5127 case 0x18: /* Grp16 (prefetch/nop) */
5128 case 0x1f: /* nop */
5130 case 0x20: /* mov cr, reg */
5131 ctxt->dst.val = ops->get_cr(ctxt, ctxt->modrm_reg);
5133 case 0x21: /* mov from dr to reg */
5134 ops->get_dr(ctxt, ctxt->modrm_reg, &ctxt->dst.val);
5136 case 0x40 ... 0x4f: /* cmov */
5137 if (test_cc(ctxt->b, ctxt->eflags))
5138 ctxt->dst.val = ctxt->src.val;
5139 else if (ctxt->mode != X86EMUL_MODE_PROT64 ||
5140 ctxt->op_bytes != 4)
5141 ctxt->dst.type = OP_NONE; /* no writeback */
5143 case 0x80 ... 0x8f: /* jnz rel, etc*/
5144 if (test_cc(ctxt->b, ctxt->eflags))
5145 rc = jmp_rel(ctxt, ctxt->src.val);
5147 case 0x90 ... 0x9f: /* setcc r/m8 */
5148 ctxt->dst.val = test_cc(ctxt->b, ctxt->eflags);
5150 case 0xb6 ... 0xb7: /* movzx */
5151 ctxt->dst.bytes = ctxt->op_bytes;
5152 ctxt->dst.val = (ctxt->src.bytes == 1) ? (u8) ctxt->src.val
5153 : (u16) ctxt->src.val;
5155 case 0xbe ... 0xbf: /* movsx */
5156 ctxt->dst.bytes = ctxt->op_bytes;
5157 ctxt->dst.val = (ctxt->src.bytes == 1) ? (s8) ctxt->src.val :
5158 (s16) ctxt->src.val;
5161 goto cannot_emulate;
5166 if (rc != X86EMUL_CONTINUE)
5172 return EMULATION_FAILED;
5175 void emulator_invalidate_register_cache(struct x86_emulate_ctxt *ctxt)
5177 invalidate_registers(ctxt);
5180 void emulator_writeback_register_cache(struct x86_emulate_ctxt *ctxt)
5182 writeback_registers(ctxt);