1 /******************************************************************************
4 * Generic x86 (32-bit and 64-bit) instruction decoder and emulator.
6 * Copyright (c) 2005 Keir Fraser
8 * Linux coding style, mod r/m decoder, segment base fixes, real-mode
9 * privileged instructions:
11 * Copyright (C) 2006 Qumranet
12 * Copyright 2010 Red Hat, Inc. and/or its affiliates.
14 * Avi Kivity <avi@qumranet.com>
15 * Yaniv Kamay <yaniv@qumranet.com>
17 * This work is licensed under the terms of the GNU GPL, version 2. See
18 * the COPYING file in the top-level directory.
20 * From: xen-unstable 10676:af9809f51f81a3c43f276f00c81a52ef558afda4
23 #include <linux/kvm_host.h>
24 #include "kvm_cache_regs.h"
25 #include <linux/module.h>
26 #include <asm/kvm_emulate.h>
27 #include <linux/stringify.h>
36 #define OpImplicit 1ull /* No generic decode */
37 #define OpReg 2ull /* Register */
38 #define OpMem 3ull /* Memory */
39 #define OpAcc 4ull /* Accumulator: AL/AX/EAX/RAX */
40 #define OpDI 5ull /* ES:DI/EDI/RDI */
41 #define OpMem64 6ull /* Memory, 64-bit */
42 #define OpImmUByte 7ull /* Zero-extended 8-bit immediate */
43 #define OpDX 8ull /* DX register */
44 #define OpCL 9ull /* CL register (for shifts) */
45 #define OpImmByte 10ull /* 8-bit sign extended immediate */
46 #define OpOne 11ull /* Implied 1 */
47 #define OpImm 12ull /* Sign extended up to 32-bit immediate */
48 #define OpMem16 13ull /* Memory operand (16-bit). */
49 #define OpMem32 14ull /* Memory operand (32-bit). */
50 #define OpImmU 15ull /* Immediate operand, zero extended */
51 #define OpSI 16ull /* SI/ESI/RSI */
52 #define OpImmFAddr 17ull /* Immediate far address */
53 #define OpMemFAddr 18ull /* Far address in memory */
54 #define OpImmU16 19ull /* Immediate operand, 16 bits, zero extended */
55 #define OpES 20ull /* ES */
56 #define OpCS 21ull /* CS */
57 #define OpSS 22ull /* SS */
58 #define OpDS 23ull /* DS */
59 #define OpFS 24ull /* FS */
60 #define OpGS 25ull /* GS */
61 #define OpMem8 26ull /* 8-bit zero extended memory operand */
62 #define OpImm64 27ull /* Sign extended 16/32/64-bit immediate */
63 #define OpXLat 28ull /* memory at BX/EBX/RBX + zero-extended AL */
64 #define OpAccLo 29ull /* Low part of extended acc (AX/AX/EAX/RAX) */
65 #define OpAccHi 30ull /* High part of extended acc (-/DX/EDX/RDX) */
67 #define OpBits 5 /* Width of operand field */
68 #define OpMask ((1ull << OpBits) - 1)
71 * Opcode effective-address decode tables.
72 * Note that we only emulate instructions that have at least one memory
73 * operand (excluding implicit stack references). We assume that stack
74 * references and instruction fetches will never occur in special memory
75 * areas that require emulation. So, for example, 'mov <imm>,<reg>' need
79 /* Operand sizes: 8-bit operands or specified/overridden size. */
80 #define ByteOp (1<<0) /* 8-bit operands. */
81 /* Destination operand type. */
83 #define ImplicitOps (OpImplicit << DstShift)
84 #define DstReg (OpReg << DstShift)
85 #define DstMem (OpMem << DstShift)
86 #define DstAcc (OpAcc << DstShift)
87 #define DstDI (OpDI << DstShift)
88 #define DstMem64 (OpMem64 << DstShift)
89 #define DstImmUByte (OpImmUByte << DstShift)
90 #define DstDX (OpDX << DstShift)
91 #define DstAccLo (OpAccLo << DstShift)
92 #define DstMask (OpMask << DstShift)
93 /* Source operand type. */
95 #define SrcNone (OpNone << SrcShift)
96 #define SrcReg (OpReg << SrcShift)
97 #define SrcMem (OpMem << SrcShift)
98 #define SrcMem16 (OpMem16 << SrcShift)
99 #define SrcMem32 (OpMem32 << SrcShift)
100 #define SrcImm (OpImm << SrcShift)
101 #define SrcImmByte (OpImmByte << SrcShift)
102 #define SrcOne (OpOne << SrcShift)
103 #define SrcImmUByte (OpImmUByte << SrcShift)
104 #define SrcImmU (OpImmU << SrcShift)
105 #define SrcSI (OpSI << SrcShift)
106 #define SrcXLat (OpXLat << SrcShift)
107 #define SrcImmFAddr (OpImmFAddr << SrcShift)
108 #define SrcMemFAddr (OpMemFAddr << SrcShift)
109 #define SrcAcc (OpAcc << SrcShift)
110 #define SrcImmU16 (OpImmU16 << SrcShift)
111 #define SrcImm64 (OpImm64 << SrcShift)
112 #define SrcDX (OpDX << SrcShift)
113 #define SrcMem8 (OpMem8 << SrcShift)
114 #define SrcAccHi (OpAccHi << SrcShift)
115 #define SrcMask (OpMask << SrcShift)
116 #define BitOp (1<<11)
117 #define MemAbs (1<<12) /* Memory operand is absolute displacement */
118 #define String (1<<13) /* String instruction (rep capable) */
119 #define Stack (1<<14) /* Stack instruction (push/pop) */
120 #define GroupMask (7<<15) /* Opcode uses one of the group mechanisms */
121 #define Group (1<<15) /* Bits 3:5 of modrm byte extend opcode */
122 #define GroupDual (2<<15) /* Alternate decoding of mod == 3 */
123 #define Prefix (3<<15) /* Instruction varies with 66/f2/f3 prefix */
124 #define RMExt (4<<15) /* Opcode extension in ModRM r/m if mod == 3 */
125 #define Escape (5<<15) /* Escape to coprocessor instruction */
126 #define Sse (1<<18) /* SSE Vector instruction */
127 /* Generic ModRM decode. */
128 #define ModRM (1<<19)
129 /* Destination is only written; never read. */
132 #define Prot (1<<21) /* instruction generates #UD if not in prot-mode */
133 #define EmulateOnUD (1<<22) /* Emulate if unsupported by the host */
134 #define NoAccess (1<<23) /* Don't access memory (lea/invlpg/verr etc) */
135 #define Op3264 (1<<24) /* Operand is 64b in long mode, 32b otherwise */
136 #define Undefined (1<<25) /* No Such Instruction */
137 #define Lock (1<<26) /* lock prefix is allowed for the instruction */
138 #define Priv (1<<27) /* instruction generates #GP if current CPL != 0 */
140 #define PageTable (1 << 29) /* instruction used to write page table */
141 #define NotImpl (1 << 30) /* instruction is not implemented */
142 /* Source 2 operand type */
143 #define Src2Shift (31)
144 #define Src2None (OpNone << Src2Shift)
145 #define Src2Mem (OpMem << Src2Shift)
146 #define Src2CL (OpCL << Src2Shift)
147 #define Src2ImmByte (OpImmByte << Src2Shift)
148 #define Src2One (OpOne << Src2Shift)
149 #define Src2Imm (OpImm << Src2Shift)
150 #define Src2ES (OpES << Src2Shift)
151 #define Src2CS (OpCS << Src2Shift)
152 #define Src2SS (OpSS << Src2Shift)
153 #define Src2DS (OpDS << Src2Shift)
154 #define Src2FS (OpFS << Src2Shift)
155 #define Src2GS (OpGS << Src2Shift)
156 #define Src2Mask (OpMask << Src2Shift)
157 #define Mmx ((u64)1 << 40) /* MMX Vector instruction */
158 #define Aligned ((u64)1 << 41) /* Explicitly aligned (e.g. MOVDQA) */
159 #define Unaligned ((u64)1 << 42) /* Explicitly unaligned (e.g. MOVDQU) */
160 #define Avx ((u64)1 << 43) /* Advanced Vector Extensions */
161 #define Fastop ((u64)1 << 44) /* Use opcode::u.fastop */
162 #define NoWrite ((u64)1 << 45) /* No writeback */
163 #define SrcWrite ((u64)1 << 46) /* Write back src operand */
164 #define NoMod ((u64)1 << 47) /* Mod field is ignored */
165 #define Intercept ((u64)1 << 48) /* Has valid intercept field */
166 #define CheckPerm ((u64)1 << 49) /* Has valid check_perm field */
167 #define NoBigReal ((u64)1 << 50) /* No big real mode */
168 #define PrivUD ((u64)1 << 51) /* #UD instead of #GP on CPL > 0 */
170 #define DstXacc (DstAccLo | SrcAccHi | SrcWrite)
172 #define X2(x...) x, x
173 #define X3(x...) X2(x), x
174 #define X4(x...) X2(x), X2(x)
175 #define X5(x...) X4(x), x
176 #define X6(x...) X4(x), X2(x)
177 #define X7(x...) X4(x), X3(x)
178 #define X8(x...) X4(x), X4(x)
179 #define X16(x...) X8(x), X8(x)
181 #define NR_FASTOP (ilog2(sizeof(ulong)) + 1)
182 #define FASTOP_SIZE 8
185 * fastop functions have a special calling convention:
190 * flags: rflags (in/out)
191 * ex: rsi (in:fastop pointer, out:zero if exception)
193 * Moreover, they are all exactly FASTOP_SIZE bytes long, so functions for
194 * different operand sizes can be reached by calculation, rather than a jump
195 * table (which would be bigger than the code).
197 * fastop functions are declared as taking a never-defined fastop parameter,
198 * so they can't be called from C directly.
207 int (*execute)(struct x86_emulate_ctxt *ctxt);
208 const struct opcode *group;
209 const struct group_dual *gdual;
210 const struct gprefix *gprefix;
211 const struct escape *esc;
212 void (*fastop)(struct fastop *fake);
214 int (*check_perm)(struct x86_emulate_ctxt *ctxt);
218 struct opcode mod012[8];
219 struct opcode mod3[8];
223 struct opcode pfx_no;
224 struct opcode pfx_66;
225 struct opcode pfx_f2;
226 struct opcode pfx_f3;
231 struct opcode high[64];
234 /* EFLAGS bit definitions. */
235 #define EFLG_ID (1<<21)
236 #define EFLG_VIP (1<<20)
237 #define EFLG_VIF (1<<19)
238 #define EFLG_AC (1<<18)
239 #define EFLG_VM (1<<17)
240 #define EFLG_RF (1<<16)
241 #define EFLG_IOPL (3<<12)
242 #define EFLG_NT (1<<14)
243 #define EFLG_OF (1<<11)
244 #define EFLG_DF (1<<10)
245 #define EFLG_IF (1<<9)
246 #define EFLG_TF (1<<8)
247 #define EFLG_SF (1<<7)
248 #define EFLG_ZF (1<<6)
249 #define EFLG_AF (1<<4)
250 #define EFLG_PF (1<<2)
251 #define EFLG_CF (1<<0)
253 #define EFLG_RESERVED_ZEROS_MASK 0xffc0802a
254 #define EFLG_RESERVED_ONE_MASK 2
256 static ulong reg_read(struct x86_emulate_ctxt *ctxt, unsigned nr)
258 if (!(ctxt->regs_valid & (1 << nr))) {
259 ctxt->regs_valid |= 1 << nr;
260 ctxt->_regs[nr] = ctxt->ops->read_gpr(ctxt, nr);
262 return ctxt->_regs[nr];
265 static ulong *reg_write(struct x86_emulate_ctxt *ctxt, unsigned nr)
267 ctxt->regs_valid |= 1 << nr;
268 ctxt->regs_dirty |= 1 << nr;
269 return &ctxt->_regs[nr];
272 static ulong *reg_rmw(struct x86_emulate_ctxt *ctxt, unsigned nr)
275 return reg_write(ctxt, nr);
278 static void writeback_registers(struct x86_emulate_ctxt *ctxt)
282 for_each_set_bit(reg, (ulong *)&ctxt->regs_dirty, 16)
283 ctxt->ops->write_gpr(ctxt, reg, ctxt->_regs[reg]);
286 static void invalidate_registers(struct x86_emulate_ctxt *ctxt)
288 ctxt->regs_dirty = 0;
289 ctxt->regs_valid = 0;
293 * These EFLAGS bits are restored from saved value during emulation, and
294 * any changes are written back to the saved value after emulation.
296 #define EFLAGS_MASK (EFLG_OF|EFLG_SF|EFLG_ZF|EFLG_AF|EFLG_PF|EFLG_CF)
304 static int fastop(struct x86_emulate_ctxt *ctxt, void (*fop)(struct fastop *));
306 #define FOP_ALIGN ".align " __stringify(FASTOP_SIZE) " \n\t"
307 #define FOP_RET "ret \n\t"
309 #define FOP_START(op) \
310 extern void em_##op(struct fastop *fake); \
311 asm(".pushsection .text, \"ax\" \n\t" \
312 ".global em_" #op " \n\t" \
319 #define FOPNOP() FOP_ALIGN FOP_RET
321 #define FOP1E(op, dst) \
322 FOP_ALIGN "10: " #op " %" #dst " \n\t" FOP_RET
324 #define FOP1EEX(op, dst) \
325 FOP1E(op, dst) _ASM_EXTABLE(10b, kvm_fastop_exception)
327 #define FASTOP1(op) \
332 ON64(FOP1E(op##q, rax)) \
335 /* 1-operand, using src2 (for MUL/DIV r/m) */
336 #define FASTOP1SRC2(op, name) \
341 ON64(FOP1E(op, rcx)) \
344 /* 1-operand, using src2 (for MUL/DIV r/m), with exceptions */
345 #define FASTOP1SRC2EX(op, name) \
350 ON64(FOP1EEX(op, rcx)) \
353 #define FOP2E(op, dst, src) \
354 FOP_ALIGN #op " %" #src ", %" #dst " \n\t" FOP_RET
356 #define FASTOP2(op) \
358 FOP2E(op##b, al, dl) \
359 FOP2E(op##w, ax, dx) \
360 FOP2E(op##l, eax, edx) \
361 ON64(FOP2E(op##q, rax, rdx)) \
364 /* 2 operand, word only */
365 #define FASTOP2W(op) \
368 FOP2E(op##w, ax, dx) \
369 FOP2E(op##l, eax, edx) \
370 ON64(FOP2E(op##q, rax, rdx)) \
373 /* 2 operand, src is CL */
374 #define FASTOP2CL(op) \
376 FOP2E(op##b, al, cl) \
377 FOP2E(op##w, ax, cl) \
378 FOP2E(op##l, eax, cl) \
379 ON64(FOP2E(op##q, rax, cl)) \
382 #define FOP3E(op, dst, src, src2) \
383 FOP_ALIGN #op " %" #src2 ", %" #src ", %" #dst " \n\t" FOP_RET
385 /* 3-operand, word-only, src2=cl */
386 #define FASTOP3WCL(op) \
389 FOP3E(op##w, ax, dx, cl) \
390 FOP3E(op##l, eax, edx, cl) \
391 ON64(FOP3E(op##q, rax, rdx, cl)) \
394 /* Special case for SETcc - 1 instruction per cc */
395 #define FOP_SETCC(op) ".align 4; " #op " %al; ret \n\t"
397 asm(".global kvm_fastop_exception \n"
398 "kvm_fastop_exception: xor %esi, %esi; ret");
419 FOP_START(salc) "pushf; sbb %al, %al; popf \n\t" FOP_RET
422 static int emulator_check_intercept(struct x86_emulate_ctxt *ctxt,
423 enum x86_intercept intercept,
424 enum x86_intercept_stage stage)
426 struct x86_instruction_info info = {
427 .intercept = intercept,
428 .rep_prefix = ctxt->rep_prefix,
429 .modrm_mod = ctxt->modrm_mod,
430 .modrm_reg = ctxt->modrm_reg,
431 .modrm_rm = ctxt->modrm_rm,
432 .src_val = ctxt->src.val64,
433 .dst_val = ctxt->dst.val64,
434 .src_bytes = ctxt->src.bytes,
435 .dst_bytes = ctxt->dst.bytes,
436 .ad_bytes = ctxt->ad_bytes,
437 .next_rip = ctxt->eip,
440 return ctxt->ops->intercept(ctxt, &info, stage);
443 static void assign_masked(ulong *dest, ulong src, ulong mask)
445 *dest = (*dest & ~mask) | (src & mask);
448 static inline unsigned long ad_mask(struct x86_emulate_ctxt *ctxt)
450 return (1UL << (ctxt->ad_bytes << 3)) - 1;
453 static ulong stack_mask(struct x86_emulate_ctxt *ctxt)
456 struct desc_struct ss;
458 if (ctxt->mode == X86EMUL_MODE_PROT64)
460 ctxt->ops->get_segment(ctxt, &sel, &ss, NULL, VCPU_SREG_SS);
461 return ~0U >> ((ss.d ^ 1) * 16); /* d=0: 0xffff; d=1: 0xffffffff */
464 static int stack_size(struct x86_emulate_ctxt *ctxt)
466 return (__fls(stack_mask(ctxt)) + 1) >> 3;
469 /* Access/update address held in a register, based on addressing mode. */
470 static inline unsigned long
471 address_mask(struct x86_emulate_ctxt *ctxt, unsigned long reg)
473 if (ctxt->ad_bytes == sizeof(unsigned long))
476 return reg & ad_mask(ctxt);
479 static inline unsigned long
480 register_address(struct x86_emulate_ctxt *ctxt, unsigned long reg)
482 return address_mask(ctxt, reg);
485 static void masked_increment(ulong *reg, ulong mask, int inc)
487 assign_masked(reg, *reg + inc, mask);
491 register_address_increment(struct x86_emulate_ctxt *ctxt, unsigned long *reg, int inc)
495 if (ctxt->ad_bytes == sizeof(unsigned long))
498 mask = ad_mask(ctxt);
499 masked_increment(reg, mask, inc);
502 static void rsp_increment(struct x86_emulate_ctxt *ctxt, int inc)
504 masked_increment(reg_rmw(ctxt, VCPU_REGS_RSP), stack_mask(ctxt), inc);
507 static u32 desc_limit_scaled(struct desc_struct *desc)
509 u32 limit = get_desc_limit(desc);
511 return desc->g ? (limit << 12) | 0xfff : limit;
514 static unsigned long seg_base(struct x86_emulate_ctxt *ctxt, int seg)
516 if (ctxt->mode == X86EMUL_MODE_PROT64 && seg < VCPU_SREG_FS)
519 return ctxt->ops->get_cached_segment_base(ctxt, seg);
522 static int emulate_exception(struct x86_emulate_ctxt *ctxt, int vec,
523 u32 error, bool valid)
526 ctxt->exception.vector = vec;
527 ctxt->exception.error_code = error;
528 ctxt->exception.error_code_valid = valid;
529 return X86EMUL_PROPAGATE_FAULT;
532 static int emulate_db(struct x86_emulate_ctxt *ctxt)
534 return emulate_exception(ctxt, DB_VECTOR, 0, false);
537 static int emulate_gp(struct x86_emulate_ctxt *ctxt, int err)
539 return emulate_exception(ctxt, GP_VECTOR, err, true);
542 static int emulate_ss(struct x86_emulate_ctxt *ctxt, int err)
544 return emulate_exception(ctxt, SS_VECTOR, err, true);
547 static int emulate_ud(struct x86_emulate_ctxt *ctxt)
549 return emulate_exception(ctxt, UD_VECTOR, 0, false);
552 static int emulate_ts(struct x86_emulate_ctxt *ctxt, int err)
554 return emulate_exception(ctxt, TS_VECTOR, err, true);
557 static int emulate_de(struct x86_emulate_ctxt *ctxt)
559 return emulate_exception(ctxt, DE_VECTOR, 0, false);
562 static int emulate_nm(struct x86_emulate_ctxt *ctxt)
564 return emulate_exception(ctxt, NM_VECTOR, 0, false);
567 static inline int assign_eip_far(struct x86_emulate_ctxt *ctxt, ulong dst,
570 switch (ctxt->op_bytes) {
572 ctxt->_eip = (u16)dst;
575 ctxt->_eip = (u32)dst;
578 if ((cs_l && is_noncanonical_address(dst)) ||
579 (!cs_l && (dst & ~(u32)-1)))
580 return emulate_gp(ctxt, 0);
584 WARN(1, "unsupported eip assignment size\n");
586 return X86EMUL_CONTINUE;
589 static inline int assign_eip_near(struct x86_emulate_ctxt *ctxt, ulong dst)
591 return assign_eip_far(ctxt, dst, ctxt->mode == X86EMUL_MODE_PROT64);
594 static inline int jmp_rel(struct x86_emulate_ctxt *ctxt, int rel)
596 return assign_eip_near(ctxt, ctxt->_eip + rel);
599 static u16 get_segment_selector(struct x86_emulate_ctxt *ctxt, unsigned seg)
602 struct desc_struct desc;
604 ctxt->ops->get_segment(ctxt, &selector, &desc, NULL, seg);
608 static void set_segment_selector(struct x86_emulate_ctxt *ctxt, u16 selector,
613 struct desc_struct desc;
615 ctxt->ops->get_segment(ctxt, &dummy, &desc, &base3, seg);
616 ctxt->ops->set_segment(ctxt, selector, &desc, base3, seg);
620 * x86 defines three classes of vector instructions: explicitly
621 * aligned, explicitly unaligned, and the rest, which change behaviour
622 * depending on whether they're AVX encoded or not.
624 * Also included is CMPXCHG16B which is not a vector instruction, yet it is
625 * subject to the same check.
627 static bool insn_aligned(struct x86_emulate_ctxt *ctxt, unsigned size)
629 if (likely(size < 16))
632 if (ctxt->d & Aligned)
634 else if (ctxt->d & Unaligned)
636 else if (ctxt->d & Avx)
642 static int __linearize(struct x86_emulate_ctxt *ctxt,
643 struct segmented_address addr,
644 unsigned size, bool write, bool fetch,
647 struct desc_struct desc;
654 la = seg_base(ctxt, addr.seg) + addr.ea;
655 switch (ctxt->mode) {
656 case X86EMUL_MODE_PROT64:
657 if (((signed long)la << 16) >> 16 != la)
658 return emulate_gp(ctxt, 0);
661 usable = ctxt->ops->get_segment(ctxt, &sel, &desc, NULL,
665 /* code segment in protected mode or read-only data segment */
666 if ((((ctxt->mode != X86EMUL_MODE_REAL) && (desc.type & 8))
667 || !(desc.type & 2)) && write)
669 /* unreadable code segment */
670 if (!fetch && (desc.type & 8) && !(desc.type & 2))
672 lim = desc_limit_scaled(&desc);
673 if ((ctxt->mode == X86EMUL_MODE_REAL) && !fetch &&
674 (ctxt->d & NoBigReal)) {
675 /* la is between zero and 0xffff */
676 if (la > 0xffff || (u32)(la + size - 1) > 0xffff)
678 } else if ((desc.type & 8) || !(desc.type & 4)) {
679 /* expand-up segment */
680 if (addr.ea > lim || (u32)(addr.ea + size - 1) > lim)
683 /* expand-down segment */
684 if (addr.ea <= lim || (u32)(addr.ea + size - 1) <= lim)
686 lim = desc.d ? 0xffffffff : 0xffff;
687 if (addr.ea > lim || (u32)(addr.ea + size - 1) > lim)
690 cpl = ctxt->ops->cpl(ctxt);
691 if (!(desc.type & 8)) {
695 } else if ((desc.type & 8) && !(desc.type & 4)) {
696 /* nonconforming code segment */
699 } else if ((desc.type & 8) && (desc.type & 4)) {
700 /* conforming code segment */
706 if (fetch ? ctxt->mode != X86EMUL_MODE_PROT64 : ctxt->ad_bytes != 8)
708 if (insn_aligned(ctxt, size) && ((la & (size - 1)) != 0))
709 return emulate_gp(ctxt, 0);
711 return X86EMUL_CONTINUE;
713 if (addr.seg == VCPU_SREG_SS)
714 return emulate_ss(ctxt, sel);
716 return emulate_gp(ctxt, sel);
719 static int linearize(struct x86_emulate_ctxt *ctxt,
720 struct segmented_address addr,
721 unsigned size, bool write,
724 return __linearize(ctxt, addr, size, write, false, linear);
728 static int segmented_read_std(struct x86_emulate_ctxt *ctxt,
729 struct segmented_address addr,
736 rc = linearize(ctxt, addr, size, false, &linear);
737 if (rc != X86EMUL_CONTINUE)
739 return ctxt->ops->read_std(ctxt, linear, data, size, &ctxt->exception);
743 * Prefetch the remaining bytes of the instruction without crossing page
744 * boundary if they are not in fetch_cache yet.
746 static int __do_insn_fetch_bytes(struct x86_emulate_ctxt *ctxt, int op_size)
750 unsigned long linear;
751 int cur_size = ctxt->fetch.end - ctxt->fetch.data;
752 struct segmented_address addr = { .seg = VCPU_SREG_CS,
753 .ea = ctxt->eip + cur_size };
755 size = 15UL ^ cur_size;
756 rc = __linearize(ctxt, addr, size, false, true, &linear);
757 if (unlikely(rc != X86EMUL_CONTINUE))
760 size = min_t(unsigned, size, PAGE_SIZE - offset_in_page(linear));
763 * One instruction can only straddle two pages,
764 * and one has been loaded at the beginning of
765 * x86_decode_insn. So, if not enough bytes
766 * still, we must have hit the 15-byte boundary.
768 if (unlikely(size < op_size))
769 return X86EMUL_UNHANDLEABLE;
770 rc = ctxt->ops->fetch(ctxt, linear, ctxt->fetch.end,
771 size, &ctxt->exception);
772 if (unlikely(rc != X86EMUL_CONTINUE))
774 ctxt->fetch.end += size;
775 return X86EMUL_CONTINUE;
778 static __always_inline int do_insn_fetch_bytes(struct x86_emulate_ctxt *ctxt,
781 if (unlikely(ctxt->fetch.end - ctxt->fetch.ptr < size))
782 return __do_insn_fetch_bytes(ctxt, size);
784 return X86EMUL_CONTINUE;
787 /* Fetch next part of the instruction being emulated. */
788 #define insn_fetch(_type, _ctxt) \
791 rc = do_insn_fetch_bytes(_ctxt, sizeof(_type)); \
792 if (rc != X86EMUL_CONTINUE) \
794 ctxt->_eip += sizeof(_type); \
795 _x = *(_type __aligned(1) *) ctxt->fetch.ptr; \
796 ctxt->fetch.ptr += sizeof(_type); \
800 #define insn_fetch_arr(_arr, _size, _ctxt) \
802 rc = do_insn_fetch_bytes(_ctxt, _size); \
803 if (rc != X86EMUL_CONTINUE) \
805 ctxt->_eip += (_size); \
806 memcpy(_arr, ctxt->fetch.ptr, _size); \
807 ctxt->fetch.ptr += (_size); \
811 * Given the 'reg' portion of a ModRM byte, and a register block, return a
812 * pointer into the block that addresses the relevant register.
813 * @highbyte_regs specifies whether to decode AH,CH,DH,BH.
815 static void *decode_register(struct x86_emulate_ctxt *ctxt, u8 modrm_reg,
819 int highbyte_regs = (ctxt->rex_prefix == 0) && byteop;
821 if (highbyte_regs && modrm_reg >= 4 && modrm_reg < 8)
822 p = (unsigned char *)reg_rmw(ctxt, modrm_reg & 3) + 1;
824 p = reg_rmw(ctxt, modrm_reg);
828 static int read_descriptor(struct x86_emulate_ctxt *ctxt,
829 struct segmented_address addr,
830 u16 *size, unsigned long *address, int op_bytes)
837 rc = segmented_read_std(ctxt, addr, size, 2);
838 if (rc != X86EMUL_CONTINUE)
841 rc = segmented_read_std(ctxt, addr, address, op_bytes);
855 FASTOP1SRC2(mul, mul_ex);
856 FASTOP1SRC2(imul, imul_ex);
857 FASTOP1SRC2EX(div, div_ex);
858 FASTOP1SRC2EX(idiv, idiv_ex);
887 static u8 test_cc(unsigned int condition, unsigned long flags)
890 void (*fop)(void) = (void *)em_setcc + 4 * (condition & 0xf);
892 flags = (flags & EFLAGS_MASK) | X86_EFLAGS_IF;
893 asm("push %[flags]; popf; call *%[fastop]"
894 : "=a"(rc) : [fastop]"r"(fop), [flags]"r"(flags));
898 static void fetch_register_operand(struct operand *op)
902 op->val = *(u8 *)op->addr.reg;
905 op->val = *(u16 *)op->addr.reg;
908 op->val = *(u32 *)op->addr.reg;
911 op->val = *(u64 *)op->addr.reg;
916 static void read_sse_reg(struct x86_emulate_ctxt *ctxt, sse128_t *data, int reg)
918 ctxt->ops->get_fpu(ctxt);
920 case 0: asm("movdqa %%xmm0, %0" : "=m"(*data)); break;
921 case 1: asm("movdqa %%xmm1, %0" : "=m"(*data)); break;
922 case 2: asm("movdqa %%xmm2, %0" : "=m"(*data)); break;
923 case 3: asm("movdqa %%xmm3, %0" : "=m"(*data)); break;
924 case 4: asm("movdqa %%xmm4, %0" : "=m"(*data)); break;
925 case 5: asm("movdqa %%xmm5, %0" : "=m"(*data)); break;
926 case 6: asm("movdqa %%xmm6, %0" : "=m"(*data)); break;
927 case 7: asm("movdqa %%xmm7, %0" : "=m"(*data)); break;
929 case 8: asm("movdqa %%xmm8, %0" : "=m"(*data)); break;
930 case 9: asm("movdqa %%xmm9, %0" : "=m"(*data)); break;
931 case 10: asm("movdqa %%xmm10, %0" : "=m"(*data)); break;
932 case 11: asm("movdqa %%xmm11, %0" : "=m"(*data)); break;
933 case 12: asm("movdqa %%xmm12, %0" : "=m"(*data)); break;
934 case 13: asm("movdqa %%xmm13, %0" : "=m"(*data)); break;
935 case 14: asm("movdqa %%xmm14, %0" : "=m"(*data)); break;
936 case 15: asm("movdqa %%xmm15, %0" : "=m"(*data)); break;
940 ctxt->ops->put_fpu(ctxt);
943 static void write_sse_reg(struct x86_emulate_ctxt *ctxt, sse128_t *data,
946 ctxt->ops->get_fpu(ctxt);
948 case 0: asm("movdqa %0, %%xmm0" : : "m"(*data)); break;
949 case 1: asm("movdqa %0, %%xmm1" : : "m"(*data)); break;
950 case 2: asm("movdqa %0, %%xmm2" : : "m"(*data)); break;
951 case 3: asm("movdqa %0, %%xmm3" : : "m"(*data)); break;
952 case 4: asm("movdqa %0, %%xmm4" : : "m"(*data)); break;
953 case 5: asm("movdqa %0, %%xmm5" : : "m"(*data)); break;
954 case 6: asm("movdqa %0, %%xmm6" : : "m"(*data)); break;
955 case 7: asm("movdqa %0, %%xmm7" : : "m"(*data)); break;
957 case 8: asm("movdqa %0, %%xmm8" : : "m"(*data)); break;
958 case 9: asm("movdqa %0, %%xmm9" : : "m"(*data)); break;
959 case 10: asm("movdqa %0, %%xmm10" : : "m"(*data)); break;
960 case 11: asm("movdqa %0, %%xmm11" : : "m"(*data)); break;
961 case 12: asm("movdqa %0, %%xmm12" : : "m"(*data)); break;
962 case 13: asm("movdqa %0, %%xmm13" : : "m"(*data)); break;
963 case 14: asm("movdqa %0, %%xmm14" : : "m"(*data)); break;
964 case 15: asm("movdqa %0, %%xmm15" : : "m"(*data)); break;
968 ctxt->ops->put_fpu(ctxt);
971 static void read_mmx_reg(struct x86_emulate_ctxt *ctxt, u64 *data, int reg)
973 ctxt->ops->get_fpu(ctxt);
975 case 0: asm("movq %%mm0, %0" : "=m"(*data)); break;
976 case 1: asm("movq %%mm1, %0" : "=m"(*data)); break;
977 case 2: asm("movq %%mm2, %0" : "=m"(*data)); break;
978 case 3: asm("movq %%mm3, %0" : "=m"(*data)); break;
979 case 4: asm("movq %%mm4, %0" : "=m"(*data)); break;
980 case 5: asm("movq %%mm5, %0" : "=m"(*data)); break;
981 case 6: asm("movq %%mm6, %0" : "=m"(*data)); break;
982 case 7: asm("movq %%mm7, %0" : "=m"(*data)); break;
985 ctxt->ops->put_fpu(ctxt);
988 static void write_mmx_reg(struct x86_emulate_ctxt *ctxt, u64 *data, int reg)
990 ctxt->ops->get_fpu(ctxt);
992 case 0: asm("movq %0, %%mm0" : : "m"(*data)); break;
993 case 1: asm("movq %0, %%mm1" : : "m"(*data)); break;
994 case 2: asm("movq %0, %%mm2" : : "m"(*data)); break;
995 case 3: asm("movq %0, %%mm3" : : "m"(*data)); break;
996 case 4: asm("movq %0, %%mm4" : : "m"(*data)); break;
997 case 5: asm("movq %0, %%mm5" : : "m"(*data)); break;
998 case 6: asm("movq %0, %%mm6" : : "m"(*data)); break;
999 case 7: asm("movq %0, %%mm7" : : "m"(*data)); break;
1002 ctxt->ops->put_fpu(ctxt);
1005 static int em_fninit(struct x86_emulate_ctxt *ctxt)
1007 if (ctxt->ops->get_cr(ctxt, 0) & (X86_CR0_TS | X86_CR0_EM))
1008 return emulate_nm(ctxt);
1010 ctxt->ops->get_fpu(ctxt);
1011 asm volatile("fninit");
1012 ctxt->ops->put_fpu(ctxt);
1013 return X86EMUL_CONTINUE;
1016 static int em_fnstcw(struct x86_emulate_ctxt *ctxt)
1020 if (ctxt->ops->get_cr(ctxt, 0) & (X86_CR0_TS | X86_CR0_EM))
1021 return emulate_nm(ctxt);
1023 ctxt->ops->get_fpu(ctxt);
1024 asm volatile("fnstcw %0": "+m"(fcw));
1025 ctxt->ops->put_fpu(ctxt);
1027 /* force 2 byte destination */
1028 ctxt->dst.bytes = 2;
1029 ctxt->dst.val = fcw;
1031 return X86EMUL_CONTINUE;
1034 static int em_fnstsw(struct x86_emulate_ctxt *ctxt)
1038 if (ctxt->ops->get_cr(ctxt, 0) & (X86_CR0_TS | X86_CR0_EM))
1039 return emulate_nm(ctxt);
1041 ctxt->ops->get_fpu(ctxt);
1042 asm volatile("fnstsw %0": "+m"(fsw));
1043 ctxt->ops->put_fpu(ctxt);
1045 /* force 2 byte destination */
1046 ctxt->dst.bytes = 2;
1047 ctxt->dst.val = fsw;
1049 return X86EMUL_CONTINUE;
1052 static void decode_register_operand(struct x86_emulate_ctxt *ctxt,
1055 unsigned reg = ctxt->modrm_reg;
1057 if (!(ctxt->d & ModRM))
1058 reg = (ctxt->b & 7) | ((ctxt->rex_prefix & 1) << 3);
1060 if (ctxt->d & Sse) {
1064 read_sse_reg(ctxt, &op->vec_val, reg);
1067 if (ctxt->d & Mmx) {
1076 op->bytes = (ctxt->d & ByteOp) ? 1 : ctxt->op_bytes;
1077 op->addr.reg = decode_register(ctxt, reg, ctxt->d & ByteOp);
1079 fetch_register_operand(op);
1080 op->orig_val = op->val;
1083 static void adjust_modrm_seg(struct x86_emulate_ctxt *ctxt, int base_reg)
1085 if (base_reg == VCPU_REGS_RSP || base_reg == VCPU_REGS_RBP)
1086 ctxt->modrm_seg = VCPU_SREG_SS;
1089 static int decode_modrm(struct x86_emulate_ctxt *ctxt,
1093 int index_reg, base_reg, scale;
1094 int rc = X86EMUL_CONTINUE;
1097 ctxt->modrm_reg = ((ctxt->rex_prefix << 1) & 8); /* REX.R */
1098 index_reg = (ctxt->rex_prefix << 2) & 8; /* REX.X */
1099 base_reg = (ctxt->rex_prefix << 3) & 8; /* REX.B */
1101 ctxt->modrm_mod = (ctxt->modrm & 0xc0) >> 6;
1102 ctxt->modrm_reg |= (ctxt->modrm & 0x38) >> 3;
1103 ctxt->modrm_rm = base_reg | (ctxt->modrm & 0x07);
1104 ctxt->modrm_seg = VCPU_SREG_DS;
1106 if (ctxt->modrm_mod == 3 || (ctxt->d & NoMod)) {
1108 op->bytes = (ctxt->d & ByteOp) ? 1 : ctxt->op_bytes;
1109 op->addr.reg = decode_register(ctxt, ctxt->modrm_rm,
1111 if (ctxt->d & Sse) {
1114 op->addr.xmm = ctxt->modrm_rm;
1115 read_sse_reg(ctxt, &op->vec_val, ctxt->modrm_rm);
1118 if (ctxt->d & Mmx) {
1121 op->addr.mm = ctxt->modrm_rm & 7;
1124 fetch_register_operand(op);
1130 if (ctxt->ad_bytes == 2) {
1131 unsigned bx = reg_read(ctxt, VCPU_REGS_RBX);
1132 unsigned bp = reg_read(ctxt, VCPU_REGS_RBP);
1133 unsigned si = reg_read(ctxt, VCPU_REGS_RSI);
1134 unsigned di = reg_read(ctxt, VCPU_REGS_RDI);
1136 /* 16-bit ModR/M decode. */
1137 switch (ctxt->modrm_mod) {
1139 if (ctxt->modrm_rm == 6)
1140 modrm_ea += insn_fetch(u16, ctxt);
1143 modrm_ea += insn_fetch(s8, ctxt);
1146 modrm_ea += insn_fetch(u16, ctxt);
1149 switch (ctxt->modrm_rm) {
1151 modrm_ea += bx + si;
1154 modrm_ea += bx + di;
1157 modrm_ea += bp + si;
1160 modrm_ea += bp + di;
1169 if (ctxt->modrm_mod != 0)
1176 if (ctxt->modrm_rm == 2 || ctxt->modrm_rm == 3 ||
1177 (ctxt->modrm_rm == 6 && ctxt->modrm_mod != 0))
1178 ctxt->modrm_seg = VCPU_SREG_SS;
1179 modrm_ea = (u16)modrm_ea;
1181 /* 32/64-bit ModR/M decode. */
1182 if ((ctxt->modrm_rm & 7) == 4) {
1183 sib = insn_fetch(u8, ctxt);
1184 index_reg |= (sib >> 3) & 7;
1185 base_reg |= sib & 7;
1188 if ((base_reg & 7) == 5 && ctxt->modrm_mod == 0)
1189 modrm_ea += insn_fetch(s32, ctxt);
1191 modrm_ea += reg_read(ctxt, base_reg);
1192 adjust_modrm_seg(ctxt, base_reg);
1195 modrm_ea += reg_read(ctxt, index_reg) << scale;
1196 } else if ((ctxt->modrm_rm & 7) == 5 && ctxt->modrm_mod == 0) {
1197 if (ctxt->mode == X86EMUL_MODE_PROT64)
1198 ctxt->rip_relative = 1;
1200 base_reg = ctxt->modrm_rm;
1201 modrm_ea += reg_read(ctxt, base_reg);
1202 adjust_modrm_seg(ctxt, base_reg);
1204 switch (ctxt->modrm_mod) {
1206 if (ctxt->modrm_rm == 5)
1207 modrm_ea += insn_fetch(s32, ctxt);
1210 modrm_ea += insn_fetch(s8, ctxt);
1213 modrm_ea += insn_fetch(s32, ctxt);
1217 op->addr.mem.ea = modrm_ea;
1218 if (ctxt->ad_bytes != 8)
1219 ctxt->memop.addr.mem.ea = (u32)ctxt->memop.addr.mem.ea;
1225 static int decode_abs(struct x86_emulate_ctxt *ctxt,
1228 int rc = X86EMUL_CONTINUE;
1231 switch (ctxt->ad_bytes) {
1233 op->addr.mem.ea = insn_fetch(u16, ctxt);
1236 op->addr.mem.ea = insn_fetch(u32, ctxt);
1239 op->addr.mem.ea = insn_fetch(u64, ctxt);
1246 static void fetch_bit_operand(struct x86_emulate_ctxt *ctxt)
1250 if (ctxt->dst.type == OP_MEM && ctxt->src.type == OP_REG) {
1251 mask = ~((long)ctxt->dst.bytes * 8 - 1);
1253 if (ctxt->src.bytes == 2)
1254 sv = (s16)ctxt->src.val & (s16)mask;
1255 else if (ctxt->src.bytes == 4)
1256 sv = (s32)ctxt->src.val & (s32)mask;
1258 sv = (s64)ctxt->src.val & (s64)mask;
1260 ctxt->dst.addr.mem.ea += (sv >> 3);
1263 /* only subword offset */
1264 ctxt->src.val &= (ctxt->dst.bytes << 3) - 1;
1267 static int read_emulated(struct x86_emulate_ctxt *ctxt,
1268 unsigned long addr, void *dest, unsigned size)
1271 struct read_cache *mc = &ctxt->mem_read;
1273 if (mc->pos < mc->end)
1276 WARN_ON((mc->end + size) >= sizeof(mc->data));
1278 rc = ctxt->ops->read_emulated(ctxt, addr, mc->data + mc->end, size,
1280 if (rc != X86EMUL_CONTINUE)
1286 memcpy(dest, mc->data + mc->pos, size);
1288 return X86EMUL_CONTINUE;
1291 static int segmented_read(struct x86_emulate_ctxt *ctxt,
1292 struct segmented_address addr,
1299 rc = linearize(ctxt, addr, size, false, &linear);
1300 if (rc != X86EMUL_CONTINUE)
1302 return read_emulated(ctxt, linear, data, size);
1305 static int segmented_write(struct x86_emulate_ctxt *ctxt,
1306 struct segmented_address addr,
1313 rc = linearize(ctxt, addr, size, true, &linear);
1314 if (rc != X86EMUL_CONTINUE)
1316 return ctxt->ops->write_emulated(ctxt, linear, data, size,
1320 static int segmented_cmpxchg(struct x86_emulate_ctxt *ctxt,
1321 struct segmented_address addr,
1322 const void *orig_data, const void *data,
1328 rc = linearize(ctxt, addr, size, true, &linear);
1329 if (rc != X86EMUL_CONTINUE)
1331 return ctxt->ops->cmpxchg_emulated(ctxt, linear, orig_data, data,
1332 size, &ctxt->exception);
1335 static int pio_in_emulated(struct x86_emulate_ctxt *ctxt,
1336 unsigned int size, unsigned short port,
1339 struct read_cache *rc = &ctxt->io_read;
1341 if (rc->pos == rc->end) { /* refill pio read ahead */
1342 unsigned int in_page, n;
1343 unsigned int count = ctxt->rep_prefix ?
1344 address_mask(ctxt, reg_read(ctxt, VCPU_REGS_RCX)) : 1;
1345 in_page = (ctxt->eflags & EFLG_DF) ?
1346 offset_in_page(reg_read(ctxt, VCPU_REGS_RDI)) :
1347 PAGE_SIZE - offset_in_page(reg_read(ctxt, VCPU_REGS_RDI));
1348 n = min3(in_page, (unsigned int)sizeof(rc->data) / size, count);
1351 rc->pos = rc->end = 0;
1352 if (!ctxt->ops->pio_in_emulated(ctxt, size, port, rc->data, n))
1357 if (ctxt->rep_prefix && (ctxt->d & String) &&
1358 !(ctxt->eflags & EFLG_DF)) {
1359 ctxt->dst.data = rc->data + rc->pos;
1360 ctxt->dst.type = OP_MEM_STR;
1361 ctxt->dst.count = (rc->end - rc->pos) / size;
1364 memcpy(dest, rc->data + rc->pos, size);
1370 static int read_interrupt_descriptor(struct x86_emulate_ctxt *ctxt,
1371 u16 index, struct desc_struct *desc)
1376 ctxt->ops->get_idt(ctxt, &dt);
1378 if (dt.size < index * 8 + 7)
1379 return emulate_gp(ctxt, index << 3 | 0x2);
1381 addr = dt.address + index * 8;
1382 return ctxt->ops->read_std(ctxt, addr, desc, sizeof *desc,
1386 static void get_descriptor_table_ptr(struct x86_emulate_ctxt *ctxt,
1387 u16 selector, struct desc_ptr *dt)
1389 const struct x86_emulate_ops *ops = ctxt->ops;
1392 if (selector & 1 << 2) {
1393 struct desc_struct desc;
1396 memset (dt, 0, sizeof *dt);
1397 if (!ops->get_segment(ctxt, &sel, &desc, &base3,
1401 dt->size = desc_limit_scaled(&desc); /* what if limit > 65535? */
1402 dt->address = get_desc_base(&desc) | ((u64)base3 << 32);
1404 ops->get_gdt(ctxt, dt);
1407 /* allowed just for 8 bytes segments */
1408 static int read_segment_descriptor(struct x86_emulate_ctxt *ctxt,
1409 u16 selector, struct desc_struct *desc,
1413 u16 index = selector >> 3;
1416 get_descriptor_table_ptr(ctxt, selector, &dt);
1418 if (dt.size < index * 8 + 7)
1419 return emulate_gp(ctxt, selector & 0xfffc);
1421 *desc_addr_p = addr = dt.address + index * 8;
1422 return ctxt->ops->read_std(ctxt, addr, desc, sizeof *desc,
1426 /* allowed just for 8 bytes segments */
1427 static int write_segment_descriptor(struct x86_emulate_ctxt *ctxt,
1428 u16 selector, struct desc_struct *desc)
1431 u16 index = selector >> 3;
1434 get_descriptor_table_ptr(ctxt, selector, &dt);
1436 if (dt.size < index * 8 + 7)
1437 return emulate_gp(ctxt, selector & 0xfffc);
1439 addr = dt.address + index * 8;
1440 return ctxt->ops->write_std(ctxt, addr, desc, sizeof *desc,
1444 /* Does not support long mode */
1445 static int __load_segment_descriptor(struct x86_emulate_ctxt *ctxt,
1446 u16 selector, int seg, u8 cpl, bool in_task_switch)
1448 struct desc_struct seg_desc, old_desc;
1450 unsigned err_vec = GP_VECTOR;
1452 bool null_selector = !(selector & ~0x3); /* 0000-0003 are null */
1458 memset(&seg_desc, 0, sizeof seg_desc);
1460 if (ctxt->mode == X86EMUL_MODE_REAL) {
1461 /* set real mode segment descriptor (keep limit etc. for
1463 ctxt->ops->get_segment(ctxt, &dummy, &seg_desc, NULL, seg);
1464 set_desc_base(&seg_desc, selector << 4);
1466 } else if (seg <= VCPU_SREG_GS && ctxt->mode == X86EMUL_MODE_VM86) {
1467 /* VM86 needs a clean new segment descriptor */
1468 set_desc_base(&seg_desc, selector << 4);
1469 set_desc_limit(&seg_desc, 0xffff);
1479 /* NULL selector is not valid for TR, CS and SS (except for long mode) */
1480 if ((seg == VCPU_SREG_CS
1481 || (seg == VCPU_SREG_SS
1482 && (ctxt->mode != X86EMUL_MODE_PROT64 || rpl != cpl))
1483 || seg == VCPU_SREG_TR)
1487 /* TR should be in GDT only */
1488 if (seg == VCPU_SREG_TR && (selector & (1 << 2)))
1491 if (null_selector) /* for NULL selector skip all following checks */
1494 ret = read_segment_descriptor(ctxt, selector, &seg_desc, &desc_addr);
1495 if (ret != X86EMUL_CONTINUE)
1498 err_code = selector & 0xfffc;
1499 err_vec = in_task_switch ? TS_VECTOR : GP_VECTOR;
1501 /* can't load system descriptor into segment selector */
1502 if (seg <= VCPU_SREG_GS && !seg_desc.s)
1506 err_vec = (seg == VCPU_SREG_SS) ? SS_VECTOR : NP_VECTOR;
1515 * segment is not a writable data segment or segment
1516 * selector's RPL != CPL or segment selector's RPL != CPL
1518 if (rpl != cpl || (seg_desc.type & 0xa) != 0x2 || dpl != cpl)
1522 if (!(seg_desc.type & 8))
1525 if (seg_desc.type & 4) {
1531 if (rpl > cpl || dpl != cpl)
1534 /* in long-mode d/b must be clear if l is set */
1535 if (seg_desc.d && seg_desc.l) {
1538 ctxt->ops->get_msr(ctxt, MSR_EFER, &efer);
1539 if (efer & EFER_LMA)
1543 /* CS(RPL) <- CPL */
1544 selector = (selector & 0xfffc) | cpl;
1547 if (seg_desc.s || (seg_desc.type != 1 && seg_desc.type != 9))
1549 old_desc = seg_desc;
1550 seg_desc.type |= 2; /* busy */
1551 ret = ctxt->ops->cmpxchg_emulated(ctxt, desc_addr, &old_desc, &seg_desc,
1552 sizeof(seg_desc), &ctxt->exception);
1553 if (ret != X86EMUL_CONTINUE)
1556 case VCPU_SREG_LDTR:
1557 if (seg_desc.s || seg_desc.type != 2)
1560 default: /* DS, ES, FS, or GS */
1562 * segment is not a data or readable code segment or
1563 * ((segment is a data or nonconforming code segment)
1564 * and (both RPL and CPL > DPL))
1566 if ((seg_desc.type & 0xa) == 0x8 ||
1567 (((seg_desc.type & 0xc) != 0xc) &&
1568 (rpl > dpl && cpl > dpl)))
1574 /* mark segment as accessed */
1576 ret = write_segment_descriptor(ctxt, selector, &seg_desc);
1577 if (ret != X86EMUL_CONTINUE)
1579 } else if (ctxt->mode == X86EMUL_MODE_PROT64) {
1580 ret = ctxt->ops->read_std(ctxt, desc_addr+8, &base3,
1581 sizeof(base3), &ctxt->exception);
1582 if (ret != X86EMUL_CONTINUE)
1586 ctxt->ops->set_segment(ctxt, selector, &seg_desc, base3, seg);
1587 return X86EMUL_CONTINUE;
1589 return emulate_exception(ctxt, err_vec, err_code, true);
1592 static int load_segment_descriptor(struct x86_emulate_ctxt *ctxt,
1593 u16 selector, int seg)
1595 u8 cpl = ctxt->ops->cpl(ctxt);
1596 return __load_segment_descriptor(ctxt, selector, seg, cpl, false);
1599 static void write_register_operand(struct operand *op)
1601 /* The 4-byte case *is* correct: in 64-bit mode we zero-extend. */
1602 switch (op->bytes) {
1604 *(u8 *)op->addr.reg = (u8)op->val;
1607 *(u16 *)op->addr.reg = (u16)op->val;
1610 *op->addr.reg = (u32)op->val;
1611 break; /* 64b: zero-extend */
1613 *op->addr.reg = op->val;
1618 static int writeback(struct x86_emulate_ctxt *ctxt, struct operand *op)
1622 write_register_operand(op);
1625 if (ctxt->lock_prefix)
1626 return segmented_cmpxchg(ctxt,
1632 return segmented_write(ctxt,
1638 return segmented_write(ctxt,
1641 op->bytes * op->count);
1644 write_sse_reg(ctxt, &op->vec_val, op->addr.xmm);
1647 write_mmx_reg(ctxt, &op->mm_val, op->addr.mm);
1655 return X86EMUL_CONTINUE;
1658 static int push(struct x86_emulate_ctxt *ctxt, void *data, int bytes)
1660 struct segmented_address addr;
1662 rsp_increment(ctxt, -bytes);
1663 addr.ea = reg_read(ctxt, VCPU_REGS_RSP) & stack_mask(ctxt);
1664 addr.seg = VCPU_SREG_SS;
1666 return segmented_write(ctxt, addr, data, bytes);
1669 static int em_push(struct x86_emulate_ctxt *ctxt)
1671 /* Disable writeback. */
1672 ctxt->dst.type = OP_NONE;
1673 return push(ctxt, &ctxt->src.val, ctxt->op_bytes);
1676 static int emulate_pop(struct x86_emulate_ctxt *ctxt,
1677 void *dest, int len)
1680 struct segmented_address addr;
1682 addr.ea = reg_read(ctxt, VCPU_REGS_RSP) & stack_mask(ctxt);
1683 addr.seg = VCPU_SREG_SS;
1684 rc = segmented_read(ctxt, addr, dest, len);
1685 if (rc != X86EMUL_CONTINUE)
1688 rsp_increment(ctxt, len);
1692 static int em_pop(struct x86_emulate_ctxt *ctxt)
1694 return emulate_pop(ctxt, &ctxt->dst.val, ctxt->op_bytes);
1697 static int emulate_popf(struct x86_emulate_ctxt *ctxt,
1698 void *dest, int len)
1701 unsigned long val, change_mask;
1702 int iopl = (ctxt->eflags & X86_EFLAGS_IOPL) >> IOPL_SHIFT;
1703 int cpl = ctxt->ops->cpl(ctxt);
1705 rc = emulate_pop(ctxt, &val, len);
1706 if (rc != X86EMUL_CONTINUE)
1709 change_mask = EFLG_CF | EFLG_PF | EFLG_AF | EFLG_ZF | EFLG_SF | EFLG_OF
1710 | EFLG_TF | EFLG_DF | EFLG_NT | EFLG_AC | EFLG_ID;
1712 switch(ctxt->mode) {
1713 case X86EMUL_MODE_PROT64:
1714 case X86EMUL_MODE_PROT32:
1715 case X86EMUL_MODE_PROT16:
1717 change_mask |= EFLG_IOPL;
1719 change_mask |= EFLG_IF;
1721 case X86EMUL_MODE_VM86:
1723 return emulate_gp(ctxt, 0);
1724 change_mask |= EFLG_IF;
1726 default: /* real mode */
1727 change_mask |= (EFLG_IOPL | EFLG_IF);
1731 *(unsigned long *)dest =
1732 (ctxt->eflags & ~change_mask) | (val & change_mask);
1737 static int em_popf(struct x86_emulate_ctxt *ctxt)
1739 ctxt->dst.type = OP_REG;
1740 ctxt->dst.addr.reg = &ctxt->eflags;
1741 ctxt->dst.bytes = ctxt->op_bytes;
1742 return emulate_popf(ctxt, &ctxt->dst.val, ctxt->op_bytes);
1745 static int em_enter(struct x86_emulate_ctxt *ctxt)
1748 unsigned frame_size = ctxt->src.val;
1749 unsigned nesting_level = ctxt->src2.val & 31;
1753 return X86EMUL_UNHANDLEABLE;
1755 rbp = reg_read(ctxt, VCPU_REGS_RBP);
1756 rc = push(ctxt, &rbp, stack_size(ctxt));
1757 if (rc != X86EMUL_CONTINUE)
1759 assign_masked(reg_rmw(ctxt, VCPU_REGS_RBP), reg_read(ctxt, VCPU_REGS_RSP),
1761 assign_masked(reg_rmw(ctxt, VCPU_REGS_RSP),
1762 reg_read(ctxt, VCPU_REGS_RSP) - frame_size,
1764 return X86EMUL_CONTINUE;
1767 static int em_leave(struct x86_emulate_ctxt *ctxt)
1769 assign_masked(reg_rmw(ctxt, VCPU_REGS_RSP), reg_read(ctxt, VCPU_REGS_RBP),
1771 return emulate_pop(ctxt, reg_rmw(ctxt, VCPU_REGS_RBP), ctxt->op_bytes);
1774 static int em_push_sreg(struct x86_emulate_ctxt *ctxt)
1776 int seg = ctxt->src2.val;
1778 ctxt->src.val = get_segment_selector(ctxt, seg);
1780 return em_push(ctxt);
1783 static int em_pop_sreg(struct x86_emulate_ctxt *ctxt)
1785 int seg = ctxt->src2.val;
1786 unsigned long selector;
1789 rc = emulate_pop(ctxt, &selector, ctxt->op_bytes);
1790 if (rc != X86EMUL_CONTINUE)
1793 if (ctxt->modrm_reg == VCPU_SREG_SS)
1794 ctxt->interruptibility = KVM_X86_SHADOW_INT_MOV_SS;
1796 rc = load_segment_descriptor(ctxt, (u16)selector, seg);
1800 static int em_pusha(struct x86_emulate_ctxt *ctxt)
1802 unsigned long old_esp = reg_read(ctxt, VCPU_REGS_RSP);
1803 int rc = X86EMUL_CONTINUE;
1804 int reg = VCPU_REGS_RAX;
1806 while (reg <= VCPU_REGS_RDI) {
1807 (reg == VCPU_REGS_RSP) ?
1808 (ctxt->src.val = old_esp) : (ctxt->src.val = reg_read(ctxt, reg));
1811 if (rc != X86EMUL_CONTINUE)
1820 static int em_pushf(struct x86_emulate_ctxt *ctxt)
1822 ctxt->src.val = (unsigned long)ctxt->eflags;
1823 return em_push(ctxt);
1826 static int em_popa(struct x86_emulate_ctxt *ctxt)
1828 int rc = X86EMUL_CONTINUE;
1829 int reg = VCPU_REGS_RDI;
1831 while (reg >= VCPU_REGS_RAX) {
1832 if (reg == VCPU_REGS_RSP) {
1833 rsp_increment(ctxt, ctxt->op_bytes);
1837 rc = emulate_pop(ctxt, reg_rmw(ctxt, reg), ctxt->op_bytes);
1838 if (rc != X86EMUL_CONTINUE)
1845 static int __emulate_int_real(struct x86_emulate_ctxt *ctxt, int irq)
1847 const struct x86_emulate_ops *ops = ctxt->ops;
1854 /* TODO: Add limit checks */
1855 ctxt->src.val = ctxt->eflags;
1857 if (rc != X86EMUL_CONTINUE)
1860 ctxt->eflags &= ~(EFLG_IF | EFLG_TF | EFLG_AC);
1862 ctxt->src.val = get_segment_selector(ctxt, VCPU_SREG_CS);
1864 if (rc != X86EMUL_CONTINUE)
1867 ctxt->src.val = ctxt->_eip;
1869 if (rc != X86EMUL_CONTINUE)
1872 ops->get_idt(ctxt, &dt);
1874 eip_addr = dt.address + (irq << 2);
1875 cs_addr = dt.address + (irq << 2) + 2;
1877 rc = ops->read_std(ctxt, cs_addr, &cs, 2, &ctxt->exception);
1878 if (rc != X86EMUL_CONTINUE)
1881 rc = ops->read_std(ctxt, eip_addr, &eip, 2, &ctxt->exception);
1882 if (rc != X86EMUL_CONTINUE)
1885 rc = load_segment_descriptor(ctxt, cs, VCPU_SREG_CS);
1886 if (rc != X86EMUL_CONTINUE)
1894 int emulate_int_real(struct x86_emulate_ctxt *ctxt, int irq)
1898 invalidate_registers(ctxt);
1899 rc = __emulate_int_real(ctxt, irq);
1900 if (rc == X86EMUL_CONTINUE)
1901 writeback_registers(ctxt);
1905 static int emulate_int(struct x86_emulate_ctxt *ctxt, int irq)
1907 switch(ctxt->mode) {
1908 case X86EMUL_MODE_REAL:
1909 return __emulate_int_real(ctxt, irq);
1910 case X86EMUL_MODE_VM86:
1911 case X86EMUL_MODE_PROT16:
1912 case X86EMUL_MODE_PROT32:
1913 case X86EMUL_MODE_PROT64:
1915 /* Protected mode interrupts unimplemented yet */
1916 return X86EMUL_UNHANDLEABLE;
1920 static int emulate_iret_real(struct x86_emulate_ctxt *ctxt)
1922 int rc = X86EMUL_CONTINUE;
1923 unsigned long temp_eip = 0;
1924 unsigned long temp_eflags = 0;
1925 unsigned long cs = 0;
1926 unsigned long mask = EFLG_CF | EFLG_PF | EFLG_AF | EFLG_ZF | EFLG_SF | EFLG_TF |
1927 EFLG_IF | EFLG_DF | EFLG_OF | EFLG_IOPL | EFLG_NT | EFLG_RF |
1928 EFLG_AC | EFLG_ID | (1 << 1); /* Last one is the reserved bit */
1929 unsigned long vm86_mask = EFLG_VM | EFLG_VIF | EFLG_VIP;
1931 /* TODO: Add stack limit check */
1933 rc = emulate_pop(ctxt, &temp_eip, ctxt->op_bytes);
1935 if (rc != X86EMUL_CONTINUE)
1938 if (temp_eip & ~0xffff)
1939 return emulate_gp(ctxt, 0);
1941 rc = emulate_pop(ctxt, &cs, ctxt->op_bytes);
1943 if (rc != X86EMUL_CONTINUE)
1946 rc = emulate_pop(ctxt, &temp_eflags, ctxt->op_bytes);
1948 if (rc != X86EMUL_CONTINUE)
1951 rc = load_segment_descriptor(ctxt, (u16)cs, VCPU_SREG_CS);
1953 if (rc != X86EMUL_CONTINUE)
1956 ctxt->_eip = temp_eip;
1959 if (ctxt->op_bytes == 4)
1960 ctxt->eflags = ((temp_eflags & mask) | (ctxt->eflags & vm86_mask));
1961 else if (ctxt->op_bytes == 2) {
1962 ctxt->eflags &= ~0xffff;
1963 ctxt->eflags |= temp_eflags;
1966 ctxt->eflags &= ~EFLG_RESERVED_ZEROS_MASK; /* Clear reserved zeros */
1967 ctxt->eflags |= EFLG_RESERVED_ONE_MASK;
1972 static int em_iret(struct x86_emulate_ctxt *ctxt)
1974 switch(ctxt->mode) {
1975 case X86EMUL_MODE_REAL:
1976 return emulate_iret_real(ctxt);
1977 case X86EMUL_MODE_VM86:
1978 case X86EMUL_MODE_PROT16:
1979 case X86EMUL_MODE_PROT32:
1980 case X86EMUL_MODE_PROT64:
1982 /* iret from protected mode unimplemented yet */
1983 return X86EMUL_UNHANDLEABLE;
1987 static int em_jmp_far(struct x86_emulate_ctxt *ctxt)
1992 memcpy(&sel, ctxt->src.valptr + ctxt->op_bytes, 2);
1994 rc = load_segment_descriptor(ctxt, sel, VCPU_SREG_CS);
1995 if (rc != X86EMUL_CONTINUE)
1999 memcpy(&ctxt->_eip, ctxt->src.valptr, ctxt->op_bytes);
2000 return X86EMUL_CONTINUE;
2003 static int em_grp45(struct x86_emulate_ctxt *ctxt)
2005 int rc = X86EMUL_CONTINUE;
2007 switch (ctxt->modrm_reg) {
2008 case 2: /* call near abs */ {
2010 old_eip = ctxt->_eip;
2011 rc = assign_eip_near(ctxt, ctxt->src.val);
2012 if (rc != X86EMUL_CONTINUE)
2014 ctxt->src.val = old_eip;
2018 case 4: /* jmp abs */
2019 rc = assign_eip_near(ctxt, ctxt->src.val);
2021 case 5: /* jmp far */
2022 rc = em_jmp_far(ctxt);
2031 static int em_cmpxchg8b(struct x86_emulate_ctxt *ctxt)
2033 u64 old = ctxt->dst.orig_val64;
2035 if (ctxt->dst.bytes == 16)
2036 return X86EMUL_UNHANDLEABLE;
2038 if (((u32) (old >> 0) != (u32) reg_read(ctxt, VCPU_REGS_RAX)) ||
2039 ((u32) (old >> 32) != (u32) reg_read(ctxt, VCPU_REGS_RDX))) {
2040 *reg_write(ctxt, VCPU_REGS_RAX) = (u32) (old >> 0);
2041 *reg_write(ctxt, VCPU_REGS_RDX) = (u32) (old >> 32);
2042 ctxt->eflags &= ~EFLG_ZF;
2044 ctxt->dst.val64 = ((u64)reg_read(ctxt, VCPU_REGS_RCX) << 32) |
2045 (u32) reg_read(ctxt, VCPU_REGS_RBX);
2047 ctxt->eflags |= EFLG_ZF;
2049 return X86EMUL_CONTINUE;
2052 static int em_ret(struct x86_emulate_ctxt *ctxt)
2057 rc = emulate_pop(ctxt, &eip, ctxt->op_bytes);
2058 if (rc != X86EMUL_CONTINUE)
2061 return assign_eip_near(ctxt, eip);
2064 static int em_ret_far(struct x86_emulate_ctxt *ctxt)
2068 int cpl = ctxt->ops->cpl(ctxt);
2070 rc = emulate_pop(ctxt, &ctxt->_eip, ctxt->op_bytes);
2071 if (rc != X86EMUL_CONTINUE)
2073 if (ctxt->op_bytes == 4)
2074 ctxt->_eip = (u32)ctxt->_eip;
2075 rc = emulate_pop(ctxt, &cs, ctxt->op_bytes);
2076 if (rc != X86EMUL_CONTINUE)
2078 /* Outer-privilege level return is not implemented */
2079 if (ctxt->mode >= X86EMUL_MODE_PROT16 && (cs & 3) > cpl)
2080 return X86EMUL_UNHANDLEABLE;
2081 rc = load_segment_descriptor(ctxt, (u16)cs, VCPU_SREG_CS);
2085 static int em_ret_far_imm(struct x86_emulate_ctxt *ctxt)
2089 rc = em_ret_far(ctxt);
2090 if (rc != X86EMUL_CONTINUE)
2092 rsp_increment(ctxt, ctxt->src.val);
2093 return X86EMUL_CONTINUE;
2096 static int em_cmpxchg(struct x86_emulate_ctxt *ctxt)
2098 /* Save real source value, then compare EAX against destination. */
2099 ctxt->dst.orig_val = ctxt->dst.val;
2100 ctxt->dst.val = reg_read(ctxt, VCPU_REGS_RAX);
2101 ctxt->src.orig_val = ctxt->src.val;
2102 ctxt->src.val = ctxt->dst.orig_val;
2103 fastop(ctxt, em_cmp);
2105 if (ctxt->eflags & EFLG_ZF) {
2106 /* Success: write back to memory. */
2107 ctxt->dst.val = ctxt->src.orig_val;
2109 /* Failure: write the value we saw to EAX. */
2110 ctxt->dst.type = OP_REG;
2111 ctxt->dst.addr.reg = reg_rmw(ctxt, VCPU_REGS_RAX);
2112 ctxt->dst.val = ctxt->dst.orig_val;
2114 return X86EMUL_CONTINUE;
2117 static int em_lseg(struct x86_emulate_ctxt *ctxt)
2119 int seg = ctxt->src2.val;
2123 memcpy(&sel, ctxt->src.valptr + ctxt->op_bytes, 2);
2125 rc = load_segment_descriptor(ctxt, sel, seg);
2126 if (rc != X86EMUL_CONTINUE)
2129 ctxt->dst.val = ctxt->src.val;
2134 setup_syscalls_segments(struct x86_emulate_ctxt *ctxt,
2135 struct desc_struct *cs, struct desc_struct *ss)
2137 cs->l = 0; /* will be adjusted later */
2138 set_desc_base(cs, 0); /* flat segment */
2139 cs->g = 1; /* 4kb granularity */
2140 set_desc_limit(cs, 0xfffff); /* 4GB limit */
2141 cs->type = 0x0b; /* Read, Execute, Accessed */
2143 cs->dpl = 0; /* will be adjusted later */
2148 set_desc_base(ss, 0); /* flat segment */
2149 set_desc_limit(ss, 0xfffff); /* 4GB limit */
2150 ss->g = 1; /* 4kb granularity */
2152 ss->type = 0x03; /* Read/Write, Accessed */
2153 ss->d = 1; /* 32bit stack segment */
2160 static bool vendor_intel(struct x86_emulate_ctxt *ctxt)
2162 u32 eax, ebx, ecx, edx;
2165 ctxt->ops->get_cpuid(ctxt, &eax, &ebx, &ecx, &edx);
2166 return ebx == X86EMUL_CPUID_VENDOR_GenuineIntel_ebx
2167 && ecx == X86EMUL_CPUID_VENDOR_GenuineIntel_ecx
2168 && edx == X86EMUL_CPUID_VENDOR_GenuineIntel_edx;
2171 static bool em_syscall_is_enabled(struct x86_emulate_ctxt *ctxt)
2173 const struct x86_emulate_ops *ops = ctxt->ops;
2174 u32 eax, ebx, ecx, edx;
2177 * syscall should always be enabled in longmode - so only become
2178 * vendor specific (cpuid) if other modes are active...
2180 if (ctxt->mode == X86EMUL_MODE_PROT64)
2185 ops->get_cpuid(ctxt, &eax, &ebx, &ecx, &edx);
2187 * Intel ("GenuineIntel")
2188 * remark: Intel CPUs only support "syscall" in 64bit
2189 * longmode. Also an 64bit guest with a
2190 * 32bit compat-app running will #UD !! While this
2191 * behaviour can be fixed (by emulating) into AMD
2192 * response - CPUs of AMD can't behave like Intel.
2194 if (ebx == X86EMUL_CPUID_VENDOR_GenuineIntel_ebx &&
2195 ecx == X86EMUL_CPUID_VENDOR_GenuineIntel_ecx &&
2196 edx == X86EMUL_CPUID_VENDOR_GenuineIntel_edx)
2199 /* AMD ("AuthenticAMD") */
2200 if (ebx == X86EMUL_CPUID_VENDOR_AuthenticAMD_ebx &&
2201 ecx == X86EMUL_CPUID_VENDOR_AuthenticAMD_ecx &&
2202 edx == X86EMUL_CPUID_VENDOR_AuthenticAMD_edx)
2205 /* AMD ("AMDisbetter!") */
2206 if (ebx == X86EMUL_CPUID_VENDOR_AMDisbetterI_ebx &&
2207 ecx == X86EMUL_CPUID_VENDOR_AMDisbetterI_ecx &&
2208 edx == X86EMUL_CPUID_VENDOR_AMDisbetterI_edx)
2211 /* default: (not Intel, not AMD), apply Intel's stricter rules... */
2215 static int em_syscall(struct x86_emulate_ctxt *ctxt)
2217 const struct x86_emulate_ops *ops = ctxt->ops;
2218 struct desc_struct cs, ss;
2223 /* syscall is not available in real mode */
2224 if (ctxt->mode == X86EMUL_MODE_REAL ||
2225 ctxt->mode == X86EMUL_MODE_VM86)
2226 return emulate_ud(ctxt);
2228 if (!(em_syscall_is_enabled(ctxt)))
2229 return emulate_ud(ctxt);
2231 ops->get_msr(ctxt, MSR_EFER, &efer);
2232 setup_syscalls_segments(ctxt, &cs, &ss);
2234 if (!(efer & EFER_SCE))
2235 return emulate_ud(ctxt);
2237 ops->get_msr(ctxt, MSR_STAR, &msr_data);
2239 cs_sel = (u16)(msr_data & 0xfffc);
2240 ss_sel = (u16)(msr_data + 8);
2242 if (efer & EFER_LMA) {
2246 ops->set_segment(ctxt, cs_sel, &cs, 0, VCPU_SREG_CS);
2247 ops->set_segment(ctxt, ss_sel, &ss, 0, VCPU_SREG_SS);
2249 *reg_write(ctxt, VCPU_REGS_RCX) = ctxt->_eip;
2250 if (efer & EFER_LMA) {
2251 #ifdef CONFIG_X86_64
2252 *reg_write(ctxt, VCPU_REGS_R11) = ctxt->eflags;
2255 ctxt->mode == X86EMUL_MODE_PROT64 ?
2256 MSR_LSTAR : MSR_CSTAR, &msr_data);
2257 ctxt->_eip = msr_data;
2259 ops->get_msr(ctxt, MSR_SYSCALL_MASK, &msr_data);
2260 ctxt->eflags &= ~msr_data;
2264 ops->get_msr(ctxt, MSR_STAR, &msr_data);
2265 ctxt->_eip = (u32)msr_data;
2267 ctxt->eflags &= ~(EFLG_VM | EFLG_IF);
2270 return X86EMUL_CONTINUE;
2273 static int em_sysenter(struct x86_emulate_ctxt *ctxt)
2275 const struct x86_emulate_ops *ops = ctxt->ops;
2276 struct desc_struct cs, ss;
2281 ops->get_msr(ctxt, MSR_EFER, &efer);
2282 /* inject #GP if in real mode */
2283 if (ctxt->mode == X86EMUL_MODE_REAL)
2284 return emulate_gp(ctxt, 0);
2287 * Not recognized on AMD in compat mode (but is recognized in legacy
2290 if ((ctxt->mode == X86EMUL_MODE_PROT32) && (efer & EFER_LMA)
2291 && !vendor_intel(ctxt))
2292 return emulate_ud(ctxt);
2294 /* XXX sysenter/sysexit have not been tested in 64bit mode.
2295 * Therefore, we inject an #UD.
2297 if (ctxt->mode == X86EMUL_MODE_PROT64)
2298 return emulate_ud(ctxt);
2300 setup_syscalls_segments(ctxt, &cs, &ss);
2302 ops->get_msr(ctxt, MSR_IA32_SYSENTER_CS, &msr_data);
2303 switch (ctxt->mode) {
2304 case X86EMUL_MODE_PROT32:
2305 if ((msr_data & 0xfffc) == 0x0)
2306 return emulate_gp(ctxt, 0);
2308 case X86EMUL_MODE_PROT64:
2309 if (msr_data == 0x0)
2310 return emulate_gp(ctxt, 0);
2316 ctxt->eflags &= ~(EFLG_VM | EFLG_IF);
2317 cs_sel = (u16)msr_data;
2318 cs_sel &= ~SELECTOR_RPL_MASK;
2319 ss_sel = cs_sel + 8;
2320 ss_sel &= ~SELECTOR_RPL_MASK;
2321 if (ctxt->mode == X86EMUL_MODE_PROT64 || (efer & EFER_LMA)) {
2326 ops->set_segment(ctxt, cs_sel, &cs, 0, VCPU_SREG_CS);
2327 ops->set_segment(ctxt, ss_sel, &ss, 0, VCPU_SREG_SS);
2329 ops->get_msr(ctxt, MSR_IA32_SYSENTER_EIP, &msr_data);
2330 ctxt->_eip = msr_data;
2332 ops->get_msr(ctxt, MSR_IA32_SYSENTER_ESP, &msr_data);
2333 *reg_write(ctxt, VCPU_REGS_RSP) = msr_data;
2335 return X86EMUL_CONTINUE;
2338 static int em_sysexit(struct x86_emulate_ctxt *ctxt)
2340 const struct x86_emulate_ops *ops = ctxt->ops;
2341 struct desc_struct cs, ss;
2342 u64 msr_data, rcx, rdx;
2344 u16 cs_sel = 0, ss_sel = 0;
2346 /* inject #GP if in real mode or Virtual 8086 mode */
2347 if (ctxt->mode == X86EMUL_MODE_REAL ||
2348 ctxt->mode == X86EMUL_MODE_VM86)
2349 return emulate_gp(ctxt, 0);
2351 setup_syscalls_segments(ctxt, &cs, &ss);
2353 if ((ctxt->rex_prefix & 0x8) != 0x0)
2354 usermode = X86EMUL_MODE_PROT64;
2356 usermode = X86EMUL_MODE_PROT32;
2358 rcx = reg_read(ctxt, VCPU_REGS_RCX);
2359 rdx = reg_read(ctxt, VCPU_REGS_RDX);
2363 ops->get_msr(ctxt, MSR_IA32_SYSENTER_CS, &msr_data);
2365 case X86EMUL_MODE_PROT32:
2366 cs_sel = (u16)(msr_data + 16);
2367 if ((msr_data & 0xfffc) == 0x0)
2368 return emulate_gp(ctxt, 0);
2369 ss_sel = (u16)(msr_data + 24);
2371 case X86EMUL_MODE_PROT64:
2372 cs_sel = (u16)(msr_data + 32);
2373 if (msr_data == 0x0)
2374 return emulate_gp(ctxt, 0);
2375 ss_sel = cs_sel + 8;
2378 if (is_noncanonical_address(rcx) ||
2379 is_noncanonical_address(rdx))
2380 return emulate_gp(ctxt, 0);
2383 cs_sel |= SELECTOR_RPL_MASK;
2384 ss_sel |= SELECTOR_RPL_MASK;
2386 ops->set_segment(ctxt, cs_sel, &cs, 0, VCPU_SREG_CS);
2387 ops->set_segment(ctxt, ss_sel, &ss, 0, VCPU_SREG_SS);
2390 *reg_write(ctxt, VCPU_REGS_RSP) = rcx;
2392 return X86EMUL_CONTINUE;
2395 static bool emulator_bad_iopl(struct x86_emulate_ctxt *ctxt)
2398 if (ctxt->mode == X86EMUL_MODE_REAL)
2400 if (ctxt->mode == X86EMUL_MODE_VM86)
2402 iopl = (ctxt->eflags & X86_EFLAGS_IOPL) >> IOPL_SHIFT;
2403 return ctxt->ops->cpl(ctxt) > iopl;
2406 static bool emulator_io_port_access_allowed(struct x86_emulate_ctxt *ctxt,
2409 const struct x86_emulate_ops *ops = ctxt->ops;
2410 struct desc_struct tr_seg;
2413 u16 tr, io_bitmap_ptr, perm, bit_idx = port & 0x7;
2414 unsigned mask = (1 << len) - 1;
2417 ops->get_segment(ctxt, &tr, &tr_seg, &base3, VCPU_SREG_TR);
2420 if (desc_limit_scaled(&tr_seg) < 103)
2422 base = get_desc_base(&tr_seg);
2423 #ifdef CONFIG_X86_64
2424 base |= ((u64)base3) << 32;
2426 r = ops->read_std(ctxt, base + 102, &io_bitmap_ptr, 2, NULL);
2427 if (r != X86EMUL_CONTINUE)
2429 if (io_bitmap_ptr + port/8 > desc_limit_scaled(&tr_seg))
2431 r = ops->read_std(ctxt, base + io_bitmap_ptr + port/8, &perm, 2, NULL);
2432 if (r != X86EMUL_CONTINUE)
2434 if ((perm >> bit_idx) & mask)
2439 static bool emulator_io_permited(struct x86_emulate_ctxt *ctxt,
2445 if (emulator_bad_iopl(ctxt))
2446 if (!emulator_io_port_access_allowed(ctxt, port, len))
2449 ctxt->perm_ok = true;
2454 static void save_state_to_tss16(struct x86_emulate_ctxt *ctxt,
2455 struct tss_segment_16 *tss)
2457 tss->ip = ctxt->_eip;
2458 tss->flag = ctxt->eflags;
2459 tss->ax = reg_read(ctxt, VCPU_REGS_RAX);
2460 tss->cx = reg_read(ctxt, VCPU_REGS_RCX);
2461 tss->dx = reg_read(ctxt, VCPU_REGS_RDX);
2462 tss->bx = reg_read(ctxt, VCPU_REGS_RBX);
2463 tss->sp = reg_read(ctxt, VCPU_REGS_RSP);
2464 tss->bp = reg_read(ctxt, VCPU_REGS_RBP);
2465 tss->si = reg_read(ctxt, VCPU_REGS_RSI);
2466 tss->di = reg_read(ctxt, VCPU_REGS_RDI);
2468 tss->es = get_segment_selector(ctxt, VCPU_SREG_ES);
2469 tss->cs = get_segment_selector(ctxt, VCPU_SREG_CS);
2470 tss->ss = get_segment_selector(ctxt, VCPU_SREG_SS);
2471 tss->ds = get_segment_selector(ctxt, VCPU_SREG_DS);
2472 tss->ldt = get_segment_selector(ctxt, VCPU_SREG_LDTR);
2475 static int load_state_from_tss16(struct x86_emulate_ctxt *ctxt,
2476 struct tss_segment_16 *tss)
2481 ctxt->_eip = tss->ip;
2482 ctxt->eflags = tss->flag | 2;
2483 *reg_write(ctxt, VCPU_REGS_RAX) = tss->ax;
2484 *reg_write(ctxt, VCPU_REGS_RCX) = tss->cx;
2485 *reg_write(ctxt, VCPU_REGS_RDX) = tss->dx;
2486 *reg_write(ctxt, VCPU_REGS_RBX) = tss->bx;
2487 *reg_write(ctxt, VCPU_REGS_RSP) = tss->sp;
2488 *reg_write(ctxt, VCPU_REGS_RBP) = tss->bp;
2489 *reg_write(ctxt, VCPU_REGS_RSI) = tss->si;
2490 *reg_write(ctxt, VCPU_REGS_RDI) = tss->di;
2493 * SDM says that segment selectors are loaded before segment
2496 set_segment_selector(ctxt, tss->ldt, VCPU_SREG_LDTR);
2497 set_segment_selector(ctxt, tss->es, VCPU_SREG_ES);
2498 set_segment_selector(ctxt, tss->cs, VCPU_SREG_CS);
2499 set_segment_selector(ctxt, tss->ss, VCPU_SREG_SS);
2500 set_segment_selector(ctxt, tss->ds, VCPU_SREG_DS);
2505 * Now load segment descriptors. If fault happens at this stage
2506 * it is handled in a context of new task
2508 ret = __load_segment_descriptor(ctxt, tss->ldt, VCPU_SREG_LDTR, cpl, true);
2509 if (ret != X86EMUL_CONTINUE)
2511 ret = __load_segment_descriptor(ctxt, tss->es, VCPU_SREG_ES, cpl, true);
2512 if (ret != X86EMUL_CONTINUE)
2514 ret = __load_segment_descriptor(ctxt, tss->cs, VCPU_SREG_CS, cpl, true);
2515 if (ret != X86EMUL_CONTINUE)
2517 ret = __load_segment_descriptor(ctxt, tss->ss, VCPU_SREG_SS, cpl, true);
2518 if (ret != X86EMUL_CONTINUE)
2520 ret = __load_segment_descriptor(ctxt, tss->ds, VCPU_SREG_DS, cpl, true);
2521 if (ret != X86EMUL_CONTINUE)
2524 return X86EMUL_CONTINUE;
2527 static int task_switch_16(struct x86_emulate_ctxt *ctxt,
2528 u16 tss_selector, u16 old_tss_sel,
2529 ulong old_tss_base, struct desc_struct *new_desc)
2531 const struct x86_emulate_ops *ops = ctxt->ops;
2532 struct tss_segment_16 tss_seg;
2534 u32 new_tss_base = get_desc_base(new_desc);
2536 ret = ops->read_std(ctxt, old_tss_base, &tss_seg, sizeof tss_seg,
2538 if (ret != X86EMUL_CONTINUE)
2539 /* FIXME: need to provide precise fault address */
2542 save_state_to_tss16(ctxt, &tss_seg);
2544 ret = ops->write_std(ctxt, old_tss_base, &tss_seg, sizeof tss_seg,
2546 if (ret != X86EMUL_CONTINUE)
2547 /* FIXME: need to provide precise fault address */
2550 ret = ops->read_std(ctxt, new_tss_base, &tss_seg, sizeof tss_seg,
2552 if (ret != X86EMUL_CONTINUE)
2553 /* FIXME: need to provide precise fault address */
2556 if (old_tss_sel != 0xffff) {
2557 tss_seg.prev_task_link = old_tss_sel;
2559 ret = ops->write_std(ctxt, new_tss_base,
2560 &tss_seg.prev_task_link,
2561 sizeof tss_seg.prev_task_link,
2563 if (ret != X86EMUL_CONTINUE)
2564 /* FIXME: need to provide precise fault address */
2568 return load_state_from_tss16(ctxt, &tss_seg);
2571 static void save_state_to_tss32(struct x86_emulate_ctxt *ctxt,
2572 struct tss_segment_32 *tss)
2574 /* CR3 and ldt selector are not saved intentionally */
2575 tss->eip = ctxt->_eip;
2576 tss->eflags = ctxt->eflags;
2577 tss->eax = reg_read(ctxt, VCPU_REGS_RAX);
2578 tss->ecx = reg_read(ctxt, VCPU_REGS_RCX);
2579 tss->edx = reg_read(ctxt, VCPU_REGS_RDX);
2580 tss->ebx = reg_read(ctxt, VCPU_REGS_RBX);
2581 tss->esp = reg_read(ctxt, VCPU_REGS_RSP);
2582 tss->ebp = reg_read(ctxt, VCPU_REGS_RBP);
2583 tss->esi = reg_read(ctxt, VCPU_REGS_RSI);
2584 tss->edi = reg_read(ctxt, VCPU_REGS_RDI);
2586 tss->es = get_segment_selector(ctxt, VCPU_SREG_ES);
2587 tss->cs = get_segment_selector(ctxt, VCPU_SREG_CS);
2588 tss->ss = get_segment_selector(ctxt, VCPU_SREG_SS);
2589 tss->ds = get_segment_selector(ctxt, VCPU_SREG_DS);
2590 tss->fs = get_segment_selector(ctxt, VCPU_SREG_FS);
2591 tss->gs = get_segment_selector(ctxt, VCPU_SREG_GS);
2594 static int load_state_from_tss32(struct x86_emulate_ctxt *ctxt,
2595 struct tss_segment_32 *tss)
2600 if (ctxt->ops->set_cr(ctxt, 3, tss->cr3))
2601 return emulate_gp(ctxt, 0);
2602 ctxt->_eip = tss->eip;
2603 ctxt->eflags = tss->eflags | 2;
2605 /* General purpose registers */
2606 *reg_write(ctxt, VCPU_REGS_RAX) = tss->eax;
2607 *reg_write(ctxt, VCPU_REGS_RCX) = tss->ecx;
2608 *reg_write(ctxt, VCPU_REGS_RDX) = tss->edx;
2609 *reg_write(ctxt, VCPU_REGS_RBX) = tss->ebx;
2610 *reg_write(ctxt, VCPU_REGS_RSP) = tss->esp;
2611 *reg_write(ctxt, VCPU_REGS_RBP) = tss->ebp;
2612 *reg_write(ctxt, VCPU_REGS_RSI) = tss->esi;
2613 *reg_write(ctxt, VCPU_REGS_RDI) = tss->edi;
2616 * SDM says that segment selectors are loaded before segment
2617 * descriptors. This is important because CPL checks will
2620 set_segment_selector(ctxt, tss->ldt_selector, VCPU_SREG_LDTR);
2621 set_segment_selector(ctxt, tss->es, VCPU_SREG_ES);
2622 set_segment_selector(ctxt, tss->cs, VCPU_SREG_CS);
2623 set_segment_selector(ctxt, tss->ss, VCPU_SREG_SS);
2624 set_segment_selector(ctxt, tss->ds, VCPU_SREG_DS);
2625 set_segment_selector(ctxt, tss->fs, VCPU_SREG_FS);
2626 set_segment_selector(ctxt, tss->gs, VCPU_SREG_GS);
2629 * If we're switching between Protected Mode and VM86, we need to make
2630 * sure to update the mode before loading the segment descriptors so
2631 * that the selectors are interpreted correctly.
2633 if (ctxt->eflags & X86_EFLAGS_VM) {
2634 ctxt->mode = X86EMUL_MODE_VM86;
2637 ctxt->mode = X86EMUL_MODE_PROT32;
2642 * Now load segment descriptors. If fault happenes at this stage
2643 * it is handled in a context of new task
2645 ret = __load_segment_descriptor(ctxt, tss->ldt_selector, VCPU_SREG_LDTR, cpl, true);
2646 if (ret != X86EMUL_CONTINUE)
2648 ret = __load_segment_descriptor(ctxt, tss->es, VCPU_SREG_ES, cpl, true);
2649 if (ret != X86EMUL_CONTINUE)
2651 ret = __load_segment_descriptor(ctxt, tss->cs, VCPU_SREG_CS, cpl, true);
2652 if (ret != X86EMUL_CONTINUE)
2654 ret = __load_segment_descriptor(ctxt, tss->ss, VCPU_SREG_SS, cpl, true);
2655 if (ret != X86EMUL_CONTINUE)
2657 ret = __load_segment_descriptor(ctxt, tss->ds, VCPU_SREG_DS, cpl, true);
2658 if (ret != X86EMUL_CONTINUE)
2660 ret = __load_segment_descriptor(ctxt, tss->fs, VCPU_SREG_FS, cpl, true);
2661 if (ret != X86EMUL_CONTINUE)
2663 ret = __load_segment_descriptor(ctxt, tss->gs, VCPU_SREG_GS, cpl, true);
2664 if (ret != X86EMUL_CONTINUE)
2667 return X86EMUL_CONTINUE;
2670 static int task_switch_32(struct x86_emulate_ctxt *ctxt,
2671 u16 tss_selector, u16 old_tss_sel,
2672 ulong old_tss_base, struct desc_struct *new_desc)
2674 const struct x86_emulate_ops *ops = ctxt->ops;
2675 struct tss_segment_32 tss_seg;
2677 u32 new_tss_base = get_desc_base(new_desc);
2678 u32 eip_offset = offsetof(struct tss_segment_32, eip);
2679 u32 ldt_sel_offset = offsetof(struct tss_segment_32, ldt_selector);
2681 ret = ops->read_std(ctxt, old_tss_base, &tss_seg, sizeof tss_seg,
2683 if (ret != X86EMUL_CONTINUE)
2684 /* FIXME: need to provide precise fault address */
2687 save_state_to_tss32(ctxt, &tss_seg);
2689 /* Only GP registers and segment selectors are saved */
2690 ret = ops->write_std(ctxt, old_tss_base + eip_offset, &tss_seg.eip,
2691 ldt_sel_offset - eip_offset, &ctxt->exception);
2692 if (ret != X86EMUL_CONTINUE)
2693 /* FIXME: need to provide precise fault address */
2696 ret = ops->read_std(ctxt, new_tss_base, &tss_seg, sizeof tss_seg,
2698 if (ret != X86EMUL_CONTINUE)
2699 /* FIXME: need to provide precise fault address */
2702 if (old_tss_sel != 0xffff) {
2703 tss_seg.prev_task_link = old_tss_sel;
2705 ret = ops->write_std(ctxt, new_tss_base,
2706 &tss_seg.prev_task_link,
2707 sizeof tss_seg.prev_task_link,
2709 if (ret != X86EMUL_CONTINUE)
2710 /* FIXME: need to provide precise fault address */
2714 return load_state_from_tss32(ctxt, &tss_seg);
2717 static int emulator_do_task_switch(struct x86_emulate_ctxt *ctxt,
2718 u16 tss_selector, int idt_index, int reason,
2719 bool has_error_code, u32 error_code)
2721 const struct x86_emulate_ops *ops = ctxt->ops;
2722 struct desc_struct curr_tss_desc, next_tss_desc;
2724 u16 old_tss_sel = get_segment_selector(ctxt, VCPU_SREG_TR);
2725 ulong old_tss_base =
2726 ops->get_cached_segment_base(ctxt, VCPU_SREG_TR);
2730 /* FIXME: old_tss_base == ~0 ? */
2732 ret = read_segment_descriptor(ctxt, tss_selector, &next_tss_desc, &desc_addr);
2733 if (ret != X86EMUL_CONTINUE)
2735 ret = read_segment_descriptor(ctxt, old_tss_sel, &curr_tss_desc, &desc_addr);
2736 if (ret != X86EMUL_CONTINUE)
2739 /* FIXME: check that next_tss_desc is tss */
2742 * Check privileges. The three cases are task switch caused by...
2744 * 1. jmp/call/int to task gate: Check against DPL of the task gate
2745 * 2. Exception/IRQ/iret: No check is performed
2746 * 3. jmp/call to TSS: Check against DPL of the TSS
2748 if (reason == TASK_SWITCH_GATE) {
2749 if (idt_index != -1) {
2750 /* Software interrupts */
2751 struct desc_struct task_gate_desc;
2754 ret = read_interrupt_descriptor(ctxt, idt_index,
2756 if (ret != X86EMUL_CONTINUE)
2759 dpl = task_gate_desc.dpl;
2760 if ((tss_selector & 3) > dpl || ops->cpl(ctxt) > dpl)
2761 return emulate_gp(ctxt, (idt_index << 3) | 0x2);
2763 } else if (reason != TASK_SWITCH_IRET) {
2764 int dpl = next_tss_desc.dpl;
2765 if ((tss_selector & 3) > dpl || ops->cpl(ctxt) > dpl)
2766 return emulate_gp(ctxt, tss_selector);
2770 desc_limit = desc_limit_scaled(&next_tss_desc);
2771 if (!next_tss_desc.p ||
2772 ((desc_limit < 0x67 && (next_tss_desc.type & 8)) ||
2773 desc_limit < 0x2b)) {
2774 return emulate_ts(ctxt, tss_selector & 0xfffc);
2777 if (reason == TASK_SWITCH_IRET || reason == TASK_SWITCH_JMP) {
2778 curr_tss_desc.type &= ~(1 << 1); /* clear busy flag */
2779 write_segment_descriptor(ctxt, old_tss_sel, &curr_tss_desc);
2782 if (reason == TASK_SWITCH_IRET)
2783 ctxt->eflags = ctxt->eflags & ~X86_EFLAGS_NT;
2785 /* set back link to prev task only if NT bit is set in eflags
2786 note that old_tss_sel is not used after this point */
2787 if (reason != TASK_SWITCH_CALL && reason != TASK_SWITCH_GATE)
2788 old_tss_sel = 0xffff;
2790 if (next_tss_desc.type & 8)
2791 ret = task_switch_32(ctxt, tss_selector, old_tss_sel,
2792 old_tss_base, &next_tss_desc);
2794 ret = task_switch_16(ctxt, tss_selector, old_tss_sel,
2795 old_tss_base, &next_tss_desc);
2796 if (ret != X86EMUL_CONTINUE)
2799 if (reason == TASK_SWITCH_CALL || reason == TASK_SWITCH_GATE)
2800 ctxt->eflags = ctxt->eflags | X86_EFLAGS_NT;
2802 if (reason != TASK_SWITCH_IRET) {
2803 next_tss_desc.type |= (1 << 1); /* set busy flag */
2804 write_segment_descriptor(ctxt, tss_selector, &next_tss_desc);
2807 ops->set_cr(ctxt, 0, ops->get_cr(ctxt, 0) | X86_CR0_TS);
2808 ops->set_segment(ctxt, tss_selector, &next_tss_desc, 0, VCPU_SREG_TR);
2810 if (has_error_code) {
2811 ctxt->op_bytes = ctxt->ad_bytes = (next_tss_desc.type & 8) ? 4 : 2;
2812 ctxt->lock_prefix = 0;
2813 ctxt->src.val = (unsigned long) error_code;
2814 ret = em_push(ctxt);
2820 int emulator_task_switch(struct x86_emulate_ctxt *ctxt,
2821 u16 tss_selector, int idt_index, int reason,
2822 bool has_error_code, u32 error_code)
2826 invalidate_registers(ctxt);
2827 ctxt->_eip = ctxt->eip;
2828 ctxt->dst.type = OP_NONE;
2830 rc = emulator_do_task_switch(ctxt, tss_selector, idt_index, reason,
2831 has_error_code, error_code);
2833 if (rc == X86EMUL_CONTINUE) {
2834 ctxt->eip = ctxt->_eip;
2835 writeback_registers(ctxt);
2838 return (rc == X86EMUL_UNHANDLEABLE) ? EMULATION_FAILED : EMULATION_OK;
2841 static void string_addr_inc(struct x86_emulate_ctxt *ctxt, int reg,
2844 int df = (ctxt->eflags & EFLG_DF) ? -op->count : op->count;
2846 register_address_increment(ctxt, reg_rmw(ctxt, reg), df * op->bytes);
2847 op->addr.mem.ea = register_address(ctxt, reg_read(ctxt, reg));
2850 static int em_das(struct x86_emulate_ctxt *ctxt)
2853 bool af, cf, old_cf;
2855 cf = ctxt->eflags & X86_EFLAGS_CF;
2861 af = ctxt->eflags & X86_EFLAGS_AF;
2862 if ((al & 0x0f) > 9 || af) {
2864 cf = old_cf | (al >= 250);
2869 if (old_al > 0x99 || old_cf) {
2875 /* Set PF, ZF, SF */
2876 ctxt->src.type = OP_IMM;
2878 ctxt->src.bytes = 1;
2879 fastop(ctxt, em_or);
2880 ctxt->eflags &= ~(X86_EFLAGS_AF | X86_EFLAGS_CF);
2882 ctxt->eflags |= X86_EFLAGS_CF;
2884 ctxt->eflags |= X86_EFLAGS_AF;
2885 return X86EMUL_CONTINUE;
2888 static int em_aam(struct x86_emulate_ctxt *ctxt)
2892 if (ctxt->src.val == 0)
2893 return emulate_de(ctxt);
2895 al = ctxt->dst.val & 0xff;
2896 ah = al / ctxt->src.val;
2897 al %= ctxt->src.val;
2899 ctxt->dst.val = (ctxt->dst.val & 0xffff0000) | al | (ah << 8);
2901 /* Set PF, ZF, SF */
2902 ctxt->src.type = OP_IMM;
2904 ctxt->src.bytes = 1;
2905 fastop(ctxt, em_or);
2907 return X86EMUL_CONTINUE;
2910 static int em_aad(struct x86_emulate_ctxt *ctxt)
2912 u8 al = ctxt->dst.val & 0xff;
2913 u8 ah = (ctxt->dst.val >> 8) & 0xff;
2915 al = (al + (ah * ctxt->src.val)) & 0xff;
2917 ctxt->dst.val = (ctxt->dst.val & 0xffff0000) | al;
2919 /* Set PF, ZF, SF */
2920 ctxt->src.type = OP_IMM;
2922 ctxt->src.bytes = 1;
2923 fastop(ctxt, em_or);
2925 return X86EMUL_CONTINUE;
2928 static int em_call(struct x86_emulate_ctxt *ctxt)
2931 long rel = ctxt->src.val;
2933 ctxt->src.val = (unsigned long)ctxt->_eip;
2934 rc = jmp_rel(ctxt, rel);
2935 if (rc != X86EMUL_CONTINUE)
2937 return em_push(ctxt);
2940 static int em_call_far(struct x86_emulate_ctxt *ctxt)
2946 old_cs = get_segment_selector(ctxt, VCPU_SREG_CS);
2947 old_eip = ctxt->_eip;
2949 memcpy(&sel, ctxt->src.valptr + ctxt->op_bytes, 2);
2950 if (load_segment_descriptor(ctxt, sel, VCPU_SREG_CS))
2951 return X86EMUL_CONTINUE;
2954 memcpy(&ctxt->_eip, ctxt->src.valptr, ctxt->op_bytes);
2956 ctxt->src.val = old_cs;
2958 if (rc != X86EMUL_CONTINUE)
2961 ctxt->src.val = old_eip;
2962 return em_push(ctxt);
2965 static int em_ret_near_imm(struct x86_emulate_ctxt *ctxt)
2970 rc = emulate_pop(ctxt, &eip, ctxt->op_bytes);
2971 if (rc != X86EMUL_CONTINUE)
2973 rc = assign_eip_near(ctxt, eip);
2974 if (rc != X86EMUL_CONTINUE)
2976 rsp_increment(ctxt, ctxt->src.val);
2977 return X86EMUL_CONTINUE;
2980 static int em_xchg(struct x86_emulate_ctxt *ctxt)
2982 /* Write back the register source. */
2983 ctxt->src.val = ctxt->dst.val;
2984 write_register_operand(&ctxt->src);
2986 /* Write back the memory destination with implicit LOCK prefix. */
2987 ctxt->dst.val = ctxt->src.orig_val;
2988 ctxt->lock_prefix = 1;
2989 return X86EMUL_CONTINUE;
2992 static int em_imul_3op(struct x86_emulate_ctxt *ctxt)
2994 ctxt->dst.val = ctxt->src2.val;
2995 return fastop(ctxt, em_imul);
2998 static int em_cwd(struct x86_emulate_ctxt *ctxt)
3000 ctxt->dst.type = OP_REG;
3001 ctxt->dst.bytes = ctxt->src.bytes;
3002 ctxt->dst.addr.reg = reg_rmw(ctxt, VCPU_REGS_RDX);
3003 ctxt->dst.val = ~((ctxt->src.val >> (ctxt->src.bytes * 8 - 1)) - 1);
3005 return X86EMUL_CONTINUE;
3008 static int em_rdtsc(struct x86_emulate_ctxt *ctxt)
3012 ctxt->ops->get_msr(ctxt, MSR_IA32_TSC, &tsc);
3013 *reg_write(ctxt, VCPU_REGS_RAX) = (u32)tsc;
3014 *reg_write(ctxt, VCPU_REGS_RDX) = tsc >> 32;
3015 return X86EMUL_CONTINUE;
3018 static int em_rdpmc(struct x86_emulate_ctxt *ctxt)
3022 if (ctxt->ops->read_pmc(ctxt, reg_read(ctxt, VCPU_REGS_RCX), &pmc))
3023 return emulate_gp(ctxt, 0);
3024 *reg_write(ctxt, VCPU_REGS_RAX) = (u32)pmc;
3025 *reg_write(ctxt, VCPU_REGS_RDX) = pmc >> 32;
3026 return X86EMUL_CONTINUE;
3029 static int em_mov(struct x86_emulate_ctxt *ctxt)
3031 memcpy(ctxt->dst.valptr, ctxt->src.valptr, sizeof(ctxt->src.valptr));
3032 return X86EMUL_CONTINUE;
3035 #define FFL(x) bit(X86_FEATURE_##x)
3037 static int em_movbe(struct x86_emulate_ctxt *ctxt)
3039 u32 ebx, ecx, edx, eax = 1;
3043 * Check MOVBE is set in the guest-visible CPUID leaf.
3045 ctxt->ops->get_cpuid(ctxt, &eax, &ebx, &ecx, &edx);
3046 if (!(ecx & FFL(MOVBE)))
3047 return emulate_ud(ctxt);
3049 switch (ctxt->op_bytes) {
3052 * From MOVBE definition: "...When the operand size is 16 bits,
3053 * the upper word of the destination register remains unchanged
3056 * Both casting ->valptr and ->val to u16 breaks strict aliasing
3057 * rules so we have to do the operation almost per hand.
3059 tmp = (u16)ctxt->src.val;
3060 ctxt->dst.val &= ~0xffffUL;
3061 ctxt->dst.val |= (unsigned long)swab16(tmp);
3064 ctxt->dst.val = swab32((u32)ctxt->src.val);
3067 ctxt->dst.val = swab64(ctxt->src.val);
3072 return X86EMUL_CONTINUE;
3075 static int em_cr_write(struct x86_emulate_ctxt *ctxt)
3077 if (ctxt->ops->set_cr(ctxt, ctxt->modrm_reg, ctxt->src.val))
3078 return emulate_gp(ctxt, 0);
3080 /* Disable writeback. */
3081 ctxt->dst.type = OP_NONE;
3082 return X86EMUL_CONTINUE;
3085 static int em_dr_write(struct x86_emulate_ctxt *ctxt)
3089 if (ctxt->mode == X86EMUL_MODE_PROT64)
3090 val = ctxt->src.val & ~0ULL;
3092 val = ctxt->src.val & ~0U;
3094 /* #UD condition is already handled. */
3095 if (ctxt->ops->set_dr(ctxt, ctxt->modrm_reg, val) < 0)
3096 return emulate_gp(ctxt, 0);
3098 /* Disable writeback. */
3099 ctxt->dst.type = OP_NONE;
3100 return X86EMUL_CONTINUE;
3103 static int em_wrmsr(struct x86_emulate_ctxt *ctxt)
3107 msr_data = (u32)reg_read(ctxt, VCPU_REGS_RAX)
3108 | ((u64)reg_read(ctxt, VCPU_REGS_RDX) << 32);
3109 if (ctxt->ops->set_msr(ctxt, reg_read(ctxt, VCPU_REGS_RCX), msr_data))
3110 return emulate_gp(ctxt, 0);
3112 return X86EMUL_CONTINUE;
3115 static int em_rdmsr(struct x86_emulate_ctxt *ctxt)
3119 if (ctxt->ops->get_msr(ctxt, reg_read(ctxt, VCPU_REGS_RCX), &msr_data))
3120 return emulate_gp(ctxt, 0);
3122 *reg_write(ctxt, VCPU_REGS_RAX) = (u32)msr_data;
3123 *reg_write(ctxt, VCPU_REGS_RDX) = msr_data >> 32;
3124 return X86EMUL_CONTINUE;
3127 static int em_mov_rm_sreg(struct x86_emulate_ctxt *ctxt)
3129 if (ctxt->modrm_reg > VCPU_SREG_GS)
3130 return emulate_ud(ctxt);
3132 ctxt->dst.val = get_segment_selector(ctxt, ctxt->modrm_reg);
3133 return X86EMUL_CONTINUE;
3136 static int em_mov_sreg_rm(struct x86_emulate_ctxt *ctxt)
3138 u16 sel = ctxt->src.val;
3140 if (ctxt->modrm_reg == VCPU_SREG_CS || ctxt->modrm_reg > VCPU_SREG_GS)
3141 return emulate_ud(ctxt);
3143 if (ctxt->modrm_reg == VCPU_SREG_SS)
3144 ctxt->interruptibility = KVM_X86_SHADOW_INT_MOV_SS;
3146 /* Disable writeback. */
3147 ctxt->dst.type = OP_NONE;
3148 return load_segment_descriptor(ctxt, sel, ctxt->modrm_reg);
3151 static int em_lldt(struct x86_emulate_ctxt *ctxt)
3153 u16 sel = ctxt->src.val;
3155 /* Disable writeback. */
3156 ctxt->dst.type = OP_NONE;
3157 return load_segment_descriptor(ctxt, sel, VCPU_SREG_LDTR);
3160 static int em_ltr(struct x86_emulate_ctxt *ctxt)
3162 u16 sel = ctxt->src.val;
3164 /* Disable writeback. */
3165 ctxt->dst.type = OP_NONE;
3166 return load_segment_descriptor(ctxt, sel, VCPU_SREG_TR);
3169 static int em_invlpg(struct x86_emulate_ctxt *ctxt)
3174 rc = linearize(ctxt, ctxt->src.addr.mem, 1, false, &linear);
3175 if (rc == X86EMUL_CONTINUE)
3176 ctxt->ops->invlpg(ctxt, linear);
3177 /* Disable writeback. */
3178 ctxt->dst.type = OP_NONE;
3179 return X86EMUL_CONTINUE;
3182 static int em_clts(struct x86_emulate_ctxt *ctxt)
3186 cr0 = ctxt->ops->get_cr(ctxt, 0);
3188 ctxt->ops->set_cr(ctxt, 0, cr0);
3189 return X86EMUL_CONTINUE;
3192 static int em_vmcall(struct x86_emulate_ctxt *ctxt)
3194 int rc = ctxt->ops->fix_hypercall(ctxt);
3196 if (rc != X86EMUL_CONTINUE)
3199 /* Let the processor re-execute the fixed hypercall */
3200 ctxt->_eip = ctxt->eip;
3201 /* Disable writeback. */
3202 ctxt->dst.type = OP_NONE;
3203 return X86EMUL_CONTINUE;
3206 static int emulate_store_desc_ptr(struct x86_emulate_ctxt *ctxt,
3207 void (*get)(struct x86_emulate_ctxt *ctxt,
3208 struct desc_ptr *ptr))
3210 struct desc_ptr desc_ptr;
3212 if (ctxt->mode == X86EMUL_MODE_PROT64)
3214 get(ctxt, &desc_ptr);
3215 if (ctxt->op_bytes == 2) {
3217 desc_ptr.address &= 0x00ffffff;
3219 /* Disable writeback. */
3220 ctxt->dst.type = OP_NONE;
3221 return segmented_write(ctxt, ctxt->dst.addr.mem,
3222 &desc_ptr, 2 + ctxt->op_bytes);
3225 static int em_sgdt(struct x86_emulate_ctxt *ctxt)
3227 return emulate_store_desc_ptr(ctxt, ctxt->ops->get_gdt);
3230 static int em_sidt(struct x86_emulate_ctxt *ctxt)
3232 return emulate_store_desc_ptr(ctxt, ctxt->ops->get_idt);
3235 static int em_lgdt(struct x86_emulate_ctxt *ctxt)
3237 struct desc_ptr desc_ptr;
3240 if (ctxt->mode == X86EMUL_MODE_PROT64)
3242 rc = read_descriptor(ctxt, ctxt->src.addr.mem,
3243 &desc_ptr.size, &desc_ptr.address,
3245 if (rc != X86EMUL_CONTINUE)
3247 ctxt->ops->set_gdt(ctxt, &desc_ptr);
3248 /* Disable writeback. */
3249 ctxt->dst.type = OP_NONE;
3250 return X86EMUL_CONTINUE;
3253 static int em_vmmcall(struct x86_emulate_ctxt *ctxt)
3257 rc = ctxt->ops->fix_hypercall(ctxt);
3259 /* Disable writeback. */
3260 ctxt->dst.type = OP_NONE;
3264 static int em_lidt(struct x86_emulate_ctxt *ctxt)
3266 struct desc_ptr desc_ptr;
3269 if (ctxt->mode == X86EMUL_MODE_PROT64)
3271 rc = read_descriptor(ctxt, ctxt->src.addr.mem,
3272 &desc_ptr.size, &desc_ptr.address,
3274 if (rc != X86EMUL_CONTINUE)
3276 ctxt->ops->set_idt(ctxt, &desc_ptr);
3277 /* Disable writeback. */
3278 ctxt->dst.type = OP_NONE;
3279 return X86EMUL_CONTINUE;
3282 static int em_smsw(struct x86_emulate_ctxt *ctxt)
3284 if (ctxt->dst.type == OP_MEM)
3285 ctxt->dst.bytes = 2;
3286 ctxt->dst.val = ctxt->ops->get_cr(ctxt, 0);
3287 return X86EMUL_CONTINUE;
3290 static int em_lmsw(struct x86_emulate_ctxt *ctxt)
3292 ctxt->ops->set_cr(ctxt, 0, (ctxt->ops->get_cr(ctxt, 0) & ~0x0eul)
3293 | (ctxt->src.val & 0x0f));
3294 ctxt->dst.type = OP_NONE;
3295 return X86EMUL_CONTINUE;
3298 static int em_loop(struct x86_emulate_ctxt *ctxt)
3300 int rc = X86EMUL_CONTINUE;
3302 register_address_increment(ctxt, reg_rmw(ctxt, VCPU_REGS_RCX), -1);
3303 if ((address_mask(ctxt, reg_read(ctxt, VCPU_REGS_RCX)) != 0) &&
3304 (ctxt->b == 0xe2 || test_cc(ctxt->b ^ 0x5, ctxt->eflags)))
3305 rc = jmp_rel(ctxt, ctxt->src.val);
3310 static int em_jcxz(struct x86_emulate_ctxt *ctxt)
3312 int rc = X86EMUL_CONTINUE;
3314 if (address_mask(ctxt, reg_read(ctxt, VCPU_REGS_RCX)) == 0)
3315 rc = jmp_rel(ctxt, ctxt->src.val);
3320 static int em_in(struct x86_emulate_ctxt *ctxt)
3322 if (!pio_in_emulated(ctxt, ctxt->dst.bytes, ctxt->src.val,
3324 return X86EMUL_IO_NEEDED;
3326 return X86EMUL_CONTINUE;
3329 static int em_out(struct x86_emulate_ctxt *ctxt)
3331 ctxt->ops->pio_out_emulated(ctxt, ctxt->src.bytes, ctxt->dst.val,
3333 /* Disable writeback. */
3334 ctxt->dst.type = OP_NONE;
3335 return X86EMUL_CONTINUE;
3338 static int em_cli(struct x86_emulate_ctxt *ctxt)
3340 if (emulator_bad_iopl(ctxt))
3341 return emulate_gp(ctxt, 0);
3343 ctxt->eflags &= ~X86_EFLAGS_IF;
3344 return X86EMUL_CONTINUE;
3347 static int em_sti(struct x86_emulate_ctxt *ctxt)
3349 if (emulator_bad_iopl(ctxt))
3350 return emulate_gp(ctxt, 0);
3352 ctxt->interruptibility = KVM_X86_SHADOW_INT_STI;
3353 ctxt->eflags |= X86_EFLAGS_IF;
3354 return X86EMUL_CONTINUE;
3357 static int em_cpuid(struct x86_emulate_ctxt *ctxt)
3359 u32 eax, ebx, ecx, edx;
3361 eax = reg_read(ctxt, VCPU_REGS_RAX);
3362 ecx = reg_read(ctxt, VCPU_REGS_RCX);
3363 ctxt->ops->get_cpuid(ctxt, &eax, &ebx, &ecx, &edx);
3364 *reg_write(ctxt, VCPU_REGS_RAX) = eax;
3365 *reg_write(ctxt, VCPU_REGS_RBX) = ebx;
3366 *reg_write(ctxt, VCPU_REGS_RCX) = ecx;
3367 *reg_write(ctxt, VCPU_REGS_RDX) = edx;
3368 return X86EMUL_CONTINUE;
3371 static int em_sahf(struct x86_emulate_ctxt *ctxt)
3375 flags = EFLG_CF | EFLG_PF | EFLG_AF | EFLG_ZF | EFLG_SF;
3376 flags &= *reg_rmw(ctxt, VCPU_REGS_RAX) >> 8;
3378 ctxt->eflags &= ~0xffUL;
3379 ctxt->eflags |= flags | X86_EFLAGS_FIXED;
3380 return X86EMUL_CONTINUE;
3383 static int em_lahf(struct x86_emulate_ctxt *ctxt)
3385 *reg_rmw(ctxt, VCPU_REGS_RAX) &= ~0xff00UL;
3386 *reg_rmw(ctxt, VCPU_REGS_RAX) |= (ctxt->eflags & 0xff) << 8;
3387 return X86EMUL_CONTINUE;
3390 static int em_bswap(struct x86_emulate_ctxt *ctxt)
3392 switch (ctxt->op_bytes) {
3393 #ifdef CONFIG_X86_64
3395 asm("bswap %0" : "+r"(ctxt->dst.val));
3399 asm("bswap %0" : "+r"(*(u32 *)&ctxt->dst.val));
3402 return X86EMUL_CONTINUE;
3405 static bool valid_cr(int nr)
3417 static int check_cr_read(struct x86_emulate_ctxt *ctxt)
3419 if (!valid_cr(ctxt->modrm_reg))
3420 return emulate_ud(ctxt);
3422 return X86EMUL_CONTINUE;
3425 static int check_cr_write(struct x86_emulate_ctxt *ctxt)
3427 u64 new_val = ctxt->src.val64;
3428 int cr = ctxt->modrm_reg;
3431 static u64 cr_reserved_bits[] = {
3432 0xffffffff00000000ULL,
3433 0, 0, 0, /* CR3 checked later */
3440 return emulate_ud(ctxt);
3442 if (new_val & cr_reserved_bits[cr])
3443 return emulate_gp(ctxt, 0);
3448 if (((new_val & X86_CR0_PG) && !(new_val & X86_CR0_PE)) ||
3449 ((new_val & X86_CR0_NW) && !(new_val & X86_CR0_CD)))
3450 return emulate_gp(ctxt, 0);
3452 cr4 = ctxt->ops->get_cr(ctxt, 4);
3453 ctxt->ops->get_msr(ctxt, MSR_EFER, &efer);
3455 if ((new_val & X86_CR0_PG) && (efer & EFER_LME) &&
3456 !(cr4 & X86_CR4_PAE))
3457 return emulate_gp(ctxt, 0);
3464 ctxt->ops->get_msr(ctxt, MSR_EFER, &efer);
3465 if (efer & EFER_LMA)
3466 rsvd = CR3_L_MODE_RESERVED_BITS;
3469 return emulate_gp(ctxt, 0);
3474 ctxt->ops->get_msr(ctxt, MSR_EFER, &efer);
3476 if ((efer & EFER_LMA) && !(new_val & X86_CR4_PAE))
3477 return emulate_gp(ctxt, 0);
3483 return X86EMUL_CONTINUE;
3486 static int check_dr7_gd(struct x86_emulate_ctxt *ctxt)
3490 ctxt->ops->get_dr(ctxt, 7, &dr7);
3492 /* Check if DR7.Global_Enable is set */
3493 return dr7 & (1 << 13);
3496 static int check_dr_read(struct x86_emulate_ctxt *ctxt)
3498 int dr = ctxt->modrm_reg;
3502 return emulate_ud(ctxt);
3504 cr4 = ctxt->ops->get_cr(ctxt, 4);
3505 if ((cr4 & X86_CR4_DE) && (dr == 4 || dr == 5))
3506 return emulate_ud(ctxt);
3508 if (check_dr7_gd(ctxt))
3509 return emulate_db(ctxt);
3511 return X86EMUL_CONTINUE;
3514 static int check_dr_write(struct x86_emulate_ctxt *ctxt)
3516 u64 new_val = ctxt->src.val64;
3517 int dr = ctxt->modrm_reg;
3519 if ((dr == 6 || dr == 7) && (new_val & 0xffffffff00000000ULL))
3520 return emulate_gp(ctxt, 0);
3522 return check_dr_read(ctxt);
3525 static int check_svme(struct x86_emulate_ctxt *ctxt)
3529 ctxt->ops->get_msr(ctxt, MSR_EFER, &efer);
3531 if (!(efer & EFER_SVME))
3532 return emulate_ud(ctxt);
3534 return X86EMUL_CONTINUE;
3537 static int check_svme_pa(struct x86_emulate_ctxt *ctxt)
3539 u64 rax = reg_read(ctxt, VCPU_REGS_RAX);
3541 /* Valid physical address? */
3542 if (rax & 0xffff000000000000ULL)
3543 return emulate_gp(ctxt, 0);
3545 return check_svme(ctxt);
3548 static int check_rdtsc(struct x86_emulate_ctxt *ctxt)
3550 u64 cr4 = ctxt->ops->get_cr(ctxt, 4);
3552 if (cr4 & X86_CR4_TSD && ctxt->ops->cpl(ctxt))
3553 return emulate_ud(ctxt);
3555 return X86EMUL_CONTINUE;
3558 static int check_rdpmc(struct x86_emulate_ctxt *ctxt)
3560 u64 cr4 = ctxt->ops->get_cr(ctxt, 4);
3561 u64 rcx = reg_read(ctxt, VCPU_REGS_RCX);
3563 if ((!(cr4 & X86_CR4_PCE) && ctxt->ops->cpl(ctxt)) ||
3564 ctxt->ops->check_pmc(ctxt, rcx))
3565 return emulate_gp(ctxt, 0);
3567 return X86EMUL_CONTINUE;
3570 static int check_perm_in(struct x86_emulate_ctxt *ctxt)
3572 ctxt->dst.bytes = min(ctxt->dst.bytes, 4u);
3573 if (!emulator_io_permited(ctxt, ctxt->src.val, ctxt->dst.bytes))
3574 return emulate_gp(ctxt, 0);
3576 return X86EMUL_CONTINUE;
3579 static int check_perm_out(struct x86_emulate_ctxt *ctxt)
3581 ctxt->src.bytes = min(ctxt->src.bytes, 4u);
3582 if (!emulator_io_permited(ctxt, ctxt->dst.val, ctxt->src.bytes))
3583 return emulate_gp(ctxt, 0);
3585 return X86EMUL_CONTINUE;
3588 #define D(_y) { .flags = (_y) }
3589 #define DI(_y, _i) { .flags = (_y)|Intercept, .intercept = x86_intercept_##_i }
3590 #define DIP(_y, _i, _p) { .flags = (_y)|Intercept|CheckPerm, \
3591 .intercept = x86_intercept_##_i, .check_perm = (_p) }
3592 #define N D(NotImpl)
3593 #define EXT(_f, _e) { .flags = ((_f) | RMExt), .u.group = (_e) }
3594 #define G(_f, _g) { .flags = ((_f) | Group | ModRM), .u.group = (_g) }
3595 #define GD(_f, _g) { .flags = ((_f) | GroupDual | ModRM), .u.gdual = (_g) }
3596 #define E(_f, _e) { .flags = ((_f) | Escape | ModRM), .u.esc = (_e) }
3597 #define I(_f, _e) { .flags = (_f), .u.execute = (_e) }
3598 #define F(_f, _e) { .flags = (_f) | Fastop, .u.fastop = (_e) }
3599 #define II(_f, _e, _i) \
3600 { .flags = (_f)|Intercept, .u.execute = (_e), .intercept = x86_intercept_##_i }
3601 #define IIP(_f, _e, _i, _p) \
3602 { .flags = (_f)|Intercept|CheckPerm, .u.execute = (_e), \
3603 .intercept = x86_intercept_##_i, .check_perm = (_p) }
3604 #define GP(_f, _g) { .flags = ((_f) | Prefix), .u.gprefix = (_g) }
3606 #define D2bv(_f) D((_f) | ByteOp), D(_f)
3607 #define D2bvIP(_f, _i, _p) DIP((_f) | ByteOp, _i, _p), DIP(_f, _i, _p)
3608 #define I2bv(_f, _e) I((_f) | ByteOp, _e), I(_f, _e)
3609 #define F2bv(_f, _e) F((_f) | ByteOp, _e), F(_f, _e)
3610 #define I2bvIP(_f, _e, _i, _p) \
3611 IIP((_f) | ByteOp, _e, _i, _p), IIP(_f, _e, _i, _p)
3613 #define F6ALU(_f, _e) F2bv((_f) | DstMem | SrcReg | ModRM, _e), \
3614 F2bv(((_f) | DstReg | SrcMem | ModRM) & ~Lock, _e), \
3615 F2bv(((_f) & ~Lock) | DstAcc | SrcImm, _e)
3617 static const struct opcode group7_rm0[] = {
3619 I(SrcNone | Priv | EmulateOnUD, em_vmcall),
3623 static const struct opcode group7_rm1[] = {
3624 DI(SrcNone | Priv, monitor),
3625 DI(SrcNone | Priv, mwait),
3629 static const struct opcode group7_rm3[] = {
3630 DIP(SrcNone | Prot | Priv, vmrun, check_svme_pa),
3631 II(SrcNone | Prot | EmulateOnUD, em_vmmcall, vmmcall),
3632 DIP(SrcNone | Prot | Priv, vmload, check_svme_pa),
3633 DIP(SrcNone | Prot | Priv, vmsave, check_svme_pa),
3634 DIP(SrcNone | Prot | Priv, stgi, check_svme),
3635 DIP(SrcNone | Prot | Priv, clgi, check_svme),
3636 DIP(SrcNone | Prot | Priv, skinit, check_svme),
3637 DIP(SrcNone | Prot | Priv, invlpga, check_svme),
3640 static const struct opcode group7_rm7[] = {
3642 DIP(SrcNone, rdtscp, check_rdtsc),
3646 static const struct opcode group1[] = {
3648 F(Lock | PageTable, em_or),
3651 F(Lock | PageTable, em_and),
3657 static const struct opcode group1A[] = {
3658 I(DstMem | SrcNone | Mov | Stack, em_pop), N, N, N, N, N, N, N,
3661 static const struct opcode group2[] = {
3662 F(DstMem | ModRM, em_rol),
3663 F(DstMem | ModRM, em_ror),
3664 F(DstMem | ModRM, em_rcl),
3665 F(DstMem | ModRM, em_rcr),
3666 F(DstMem | ModRM, em_shl),
3667 F(DstMem | ModRM, em_shr),
3668 F(DstMem | ModRM, em_shl),
3669 F(DstMem | ModRM, em_sar),
3672 static const struct opcode group3[] = {
3673 F(DstMem | SrcImm | NoWrite, em_test),
3674 F(DstMem | SrcImm | NoWrite, em_test),
3675 F(DstMem | SrcNone | Lock, em_not),
3676 F(DstMem | SrcNone | Lock, em_neg),
3677 F(DstXacc | Src2Mem, em_mul_ex),
3678 F(DstXacc | Src2Mem, em_imul_ex),
3679 F(DstXacc | Src2Mem, em_div_ex),
3680 F(DstXacc | Src2Mem, em_idiv_ex),
3683 static const struct opcode group4[] = {
3684 F(ByteOp | DstMem | SrcNone | Lock, em_inc),
3685 F(ByteOp | DstMem | SrcNone | Lock, em_dec),
3689 static const struct opcode group5[] = {
3690 F(DstMem | SrcNone | Lock, em_inc),
3691 F(DstMem | SrcNone | Lock, em_dec),
3692 I(SrcMem | Stack, em_grp45),
3693 I(SrcMemFAddr | ImplicitOps | Stack, em_call_far),
3694 I(SrcMem | Stack, em_grp45),
3695 I(SrcMemFAddr | ImplicitOps, em_grp45),
3696 I(SrcMem | Stack, em_grp45), D(Undefined),
3699 static const struct opcode group6[] = {
3702 II(Prot | Priv | SrcMem16, em_lldt, lldt),
3703 II(Prot | Priv | SrcMem16, em_ltr, ltr),
3707 static const struct group_dual group7 = { {
3708 II(Mov | DstMem, em_sgdt, sgdt),
3709 II(Mov | DstMem, em_sidt, sidt),
3710 II(SrcMem | Priv, em_lgdt, lgdt),
3711 II(SrcMem | Priv, em_lidt, lidt),
3712 II(SrcNone | DstMem | Mov, em_smsw, smsw), N,
3713 II(SrcMem16 | Mov | Priv, em_lmsw, lmsw),
3714 II(SrcMem | ByteOp | Priv | NoAccess, em_invlpg, invlpg),
3718 N, EXT(0, group7_rm3),
3719 II(SrcNone | DstMem | Mov, em_smsw, smsw), N,
3720 II(SrcMem16 | Mov | Priv, em_lmsw, lmsw),
3724 static const struct opcode group8[] = {
3726 F(DstMem | SrcImmByte | NoWrite, em_bt),
3727 F(DstMem | SrcImmByte | Lock | PageTable, em_bts),
3728 F(DstMem | SrcImmByte | Lock, em_btr),
3729 F(DstMem | SrcImmByte | Lock | PageTable, em_btc),
3732 static const struct group_dual group9 = { {
3733 N, I(DstMem64 | Lock | PageTable, em_cmpxchg8b), N, N, N, N, N, N,
3735 N, N, N, N, N, N, N, N,
3738 static const struct opcode group11[] = {
3739 I(DstMem | SrcImm | Mov | PageTable, em_mov),
3743 static const struct gprefix pfx_0f_6f_0f_7f = {
3744 I(Mmx, em_mov), I(Sse | Aligned, em_mov), N, I(Sse | Unaligned, em_mov),
3747 static const struct gprefix pfx_0f_2b = {
3748 I(0, em_mov), I(0, em_mov), N, N,
3751 static const struct gprefix pfx_0f_28_0f_29 = {
3752 I(Aligned, em_mov), I(Aligned, em_mov), N, N,
3755 static const struct gprefix pfx_0f_e7 = {
3756 N, I(Sse, em_mov), N, N,
3759 static const struct escape escape_d9 = { {
3760 N, N, N, N, N, N, N, I(DstMem, em_fnstcw),
3763 N, N, N, N, N, N, N, N,
3765 N, N, N, N, N, N, N, N,
3767 N, N, N, N, N, N, N, N,
3769 N, N, N, N, N, N, N, N,
3771 N, N, N, N, N, N, N, N,
3773 N, N, N, N, N, N, N, N,
3775 N, N, N, N, N, N, N, N,
3777 N, N, N, N, N, N, N, N,
3780 static const struct escape escape_db = { {
3781 N, N, N, N, N, N, N, N,
3784 N, N, N, N, N, N, N, N,
3786 N, N, N, N, N, N, N, N,
3788 N, N, N, N, N, N, N, N,
3790 N, N, N, N, N, N, N, N,
3792 N, N, N, I(ImplicitOps, em_fninit), N, N, N, N,
3794 N, N, N, N, N, N, N, N,
3796 N, N, N, N, N, N, N, N,
3798 N, N, N, N, N, N, N, N,
3801 static const struct escape escape_dd = { {
3802 N, N, N, N, N, N, N, I(DstMem, em_fnstsw),
3805 N, N, N, N, N, N, N, N,
3807 N, N, N, N, N, N, N, N,
3809 N, N, N, N, N, N, N, N,
3811 N, N, N, N, N, N, N, N,
3813 N, N, N, N, N, N, N, N,
3815 N, N, N, N, N, N, N, N,
3817 N, N, N, N, N, N, N, N,
3819 N, N, N, N, N, N, N, N,
3822 static const struct opcode opcode_table[256] = {
3824 F6ALU(Lock, em_add),
3825 I(ImplicitOps | Stack | No64 | Src2ES, em_push_sreg),
3826 I(ImplicitOps | Stack | No64 | Src2ES, em_pop_sreg),
3828 F6ALU(Lock | PageTable, em_or),
3829 I(ImplicitOps | Stack | No64 | Src2CS, em_push_sreg),
3832 F6ALU(Lock, em_adc),
3833 I(ImplicitOps | Stack | No64 | Src2SS, em_push_sreg),
3834 I(ImplicitOps | Stack | No64 | Src2SS, em_pop_sreg),
3836 F6ALU(Lock, em_sbb),
3837 I(ImplicitOps | Stack | No64 | Src2DS, em_push_sreg),
3838 I(ImplicitOps | Stack | No64 | Src2DS, em_pop_sreg),
3840 F6ALU(Lock | PageTable, em_and), N, N,
3842 F6ALU(Lock, em_sub), N, I(ByteOp | DstAcc | No64, em_das),
3844 F6ALU(Lock, em_xor), N, N,
3846 F6ALU(NoWrite, em_cmp), N, N,
3848 X8(F(DstReg, em_inc)), X8(F(DstReg, em_dec)),
3850 X8(I(SrcReg | Stack, em_push)),
3852 X8(I(DstReg | Stack, em_pop)),
3854 I(ImplicitOps | Stack | No64, em_pusha),
3855 I(ImplicitOps | Stack | No64, em_popa),
3856 N, D(DstReg | SrcMem32 | ModRM | Mov) /* movsxd (x86/64) */ ,
3859 I(SrcImm | Mov | Stack, em_push),
3860 I(DstReg | SrcMem | ModRM | Src2Imm, em_imul_3op),
3861 I(SrcImmByte | Mov | Stack, em_push),
3862 I(DstReg | SrcMem | ModRM | Src2ImmByte, em_imul_3op),
3863 I2bvIP(DstDI | SrcDX | Mov | String | Unaligned, em_in, ins, check_perm_in), /* insb, insw/insd */
3864 I2bvIP(SrcSI | DstDX | String, em_out, outs, check_perm_out), /* outsb, outsw/outsd */
3868 G(ByteOp | DstMem | SrcImm, group1),
3869 G(DstMem | SrcImm, group1),
3870 G(ByteOp | DstMem | SrcImm | No64, group1),
3871 G(DstMem | SrcImmByte, group1),
3872 F2bv(DstMem | SrcReg | ModRM | NoWrite, em_test),
3873 I2bv(DstMem | SrcReg | ModRM | Lock | PageTable, em_xchg),
3875 I2bv(DstMem | SrcReg | ModRM | Mov | PageTable, em_mov),
3876 I2bv(DstReg | SrcMem | ModRM | Mov, em_mov),
3877 I(DstMem | SrcNone | ModRM | Mov | PageTable, em_mov_rm_sreg),
3878 D(ModRM | SrcMem | NoAccess | DstReg),
3879 I(ImplicitOps | SrcMem16 | ModRM, em_mov_sreg_rm),
3882 DI(SrcAcc | DstReg, pause), X7(D(SrcAcc | DstReg)),
3884 D(DstAcc | SrcNone), I(ImplicitOps | SrcAcc, em_cwd),
3885 I(SrcImmFAddr | No64, em_call_far), N,
3886 II(ImplicitOps | Stack, em_pushf, pushf),
3887 II(ImplicitOps | Stack, em_popf, popf),
3888 I(ImplicitOps, em_sahf), I(ImplicitOps, em_lahf),
3890 I2bv(DstAcc | SrcMem | Mov | MemAbs, em_mov),
3891 I2bv(DstMem | SrcAcc | Mov | MemAbs | PageTable, em_mov),
3892 I2bv(SrcSI | DstDI | Mov | String, em_mov),
3893 F2bv(SrcSI | DstDI | String | NoWrite, em_cmp),
3895 F2bv(DstAcc | SrcImm | NoWrite, em_test),
3896 I2bv(SrcAcc | DstDI | Mov | String, em_mov),
3897 I2bv(SrcSI | DstAcc | Mov | String, em_mov),
3898 F2bv(SrcAcc | DstDI | String | NoWrite, em_cmp),
3900 X8(I(ByteOp | DstReg | SrcImm | Mov, em_mov)),
3902 X8(I(DstReg | SrcImm64 | Mov, em_mov)),
3904 G(ByteOp | Src2ImmByte, group2), G(Src2ImmByte, group2),
3905 I(ImplicitOps | Stack | SrcImmU16, em_ret_near_imm),
3906 I(ImplicitOps | Stack, em_ret),
3907 I(DstReg | SrcMemFAddr | ModRM | No64 | Src2ES, em_lseg),
3908 I(DstReg | SrcMemFAddr | ModRM | No64 | Src2DS, em_lseg),
3909 G(ByteOp, group11), G(0, group11),
3911 I(Stack | SrcImmU16 | Src2ImmByte, em_enter), I(Stack, em_leave),
3912 I(ImplicitOps | Stack | SrcImmU16, em_ret_far_imm),
3913 I(ImplicitOps | Stack, em_ret_far),
3914 D(ImplicitOps), DI(SrcImmByte, intn),
3915 D(ImplicitOps | No64), II(ImplicitOps, em_iret, iret),
3917 G(Src2One | ByteOp, group2), G(Src2One, group2),
3918 G(Src2CL | ByteOp, group2), G(Src2CL, group2),
3919 I(DstAcc | SrcImmUByte | No64, em_aam),
3920 I(DstAcc | SrcImmUByte | No64, em_aad),
3921 F(DstAcc | ByteOp | No64, em_salc),
3922 I(DstAcc | SrcXLat | ByteOp, em_mov),
3924 N, E(0, &escape_d9), N, E(0, &escape_db), N, E(0, &escape_dd), N, N,
3926 X3(I(SrcImmByte, em_loop)),
3927 I(SrcImmByte, em_jcxz),
3928 I2bvIP(SrcImmUByte | DstAcc, em_in, in, check_perm_in),
3929 I2bvIP(SrcAcc | DstImmUByte, em_out, out, check_perm_out),
3931 I(SrcImm | Stack, em_call), D(SrcImm | ImplicitOps),
3932 I(SrcImmFAddr | No64, em_jmp_far), D(SrcImmByte | ImplicitOps),
3933 I2bvIP(SrcDX | DstAcc, em_in, in, check_perm_in),
3934 I2bvIP(SrcAcc | DstDX, em_out, out, check_perm_out),
3936 N, DI(ImplicitOps, icebp), N, N,
3937 DI(ImplicitOps | Priv, hlt), D(ImplicitOps),
3938 G(ByteOp, group3), G(0, group3),
3940 D(ImplicitOps), D(ImplicitOps),
3941 I(ImplicitOps, em_cli), I(ImplicitOps, em_sti),
3942 D(ImplicitOps), D(ImplicitOps), G(0, group4), G(0, group5),
3945 static const struct opcode twobyte_table[256] = {
3947 G(0, group6), GD(0, &group7), N, N,
3948 N, I(ImplicitOps | EmulateOnUD, em_syscall),
3949 II(ImplicitOps | Priv, em_clts, clts), N,
3950 DI(ImplicitOps | Priv, invd), DI(ImplicitOps | Priv, wbinvd), N, N,
3951 N, D(ImplicitOps | ModRM), N, N,
3953 N, N, N, N, N, N, N, N,
3954 D(ImplicitOps | ModRM), N, N, N, N, N, N, D(ImplicitOps | ModRM),
3956 DIP(ModRM | DstMem | Priv | Op3264 | NoMod, cr_read, check_cr_read),
3957 DIP(ModRM | DstMem | Priv | Op3264 | NoMod, dr_read, check_dr_read),
3958 IIP(ModRM | SrcMem | Priv | Op3264 | NoMod, em_cr_write, cr_write,
3960 IIP(ModRM | SrcMem | Priv | Op3264 | NoMod, em_dr_write, dr_write,
3963 GP(ModRM | DstReg | SrcMem | Mov | Sse, &pfx_0f_28_0f_29),
3964 GP(ModRM | DstMem | SrcReg | Mov | Sse, &pfx_0f_28_0f_29),
3965 N, GP(ModRM | DstMem | SrcReg | Mov | Sse, &pfx_0f_2b),
3968 II(ImplicitOps | Priv, em_wrmsr, wrmsr),
3969 IIP(ImplicitOps, em_rdtsc, rdtsc, check_rdtsc),
3970 II(ImplicitOps | Priv, em_rdmsr, rdmsr),
3971 IIP(ImplicitOps, em_rdpmc, rdpmc, check_rdpmc),
3972 I(ImplicitOps | EmulateOnUD, em_sysenter),
3973 I(ImplicitOps | Priv | EmulateOnUD, em_sysexit),
3975 N, N, N, N, N, N, N, N,
3977 X16(D(DstReg | SrcMem | ModRM)),
3979 N, N, N, N, N, N, N, N, N, N, N, N, N, N, N, N,
3984 N, N, N, GP(SrcMem | DstReg | ModRM | Mov, &pfx_0f_6f_0f_7f),
3989 N, N, N, GP(SrcReg | DstMem | ModRM | Mov, &pfx_0f_6f_0f_7f),
3993 X16(D(ByteOp | DstMem | SrcNone | ModRM| Mov)),
3995 I(Stack | Src2FS, em_push_sreg), I(Stack | Src2FS, em_pop_sreg),
3996 II(ImplicitOps, em_cpuid, cpuid),
3997 F(DstMem | SrcReg | ModRM | BitOp | NoWrite, em_bt),
3998 F(DstMem | SrcReg | Src2ImmByte | ModRM, em_shld),
3999 F(DstMem | SrcReg | Src2CL | ModRM, em_shld), N, N,
4001 I(Stack | Src2GS, em_push_sreg), I(Stack | Src2GS, em_pop_sreg),
4002 DI(ImplicitOps, rsm),
4003 F(DstMem | SrcReg | ModRM | BitOp | Lock | PageTable, em_bts),
4004 F(DstMem | SrcReg | Src2ImmByte | ModRM, em_shrd),
4005 F(DstMem | SrcReg | Src2CL | ModRM, em_shrd),
4006 D(ModRM), F(DstReg | SrcMem | ModRM, em_imul),
4008 I2bv(DstMem | SrcReg | ModRM | Lock | PageTable, em_cmpxchg),
4009 I(DstReg | SrcMemFAddr | ModRM | Src2SS, em_lseg),
4010 F(DstMem | SrcReg | ModRM | BitOp | Lock, em_btr),
4011 I(DstReg | SrcMemFAddr | ModRM | Src2FS, em_lseg),
4012 I(DstReg | SrcMemFAddr | ModRM | Src2GS, em_lseg),
4013 D(DstReg | SrcMem8 | ModRM | Mov), D(DstReg | SrcMem16 | ModRM | Mov),
4017 F(DstMem | SrcReg | ModRM | BitOp | Lock | PageTable, em_btc),
4018 F(DstReg | SrcMem | ModRM, em_bsf), F(DstReg | SrcMem | ModRM, em_bsr),
4019 D(DstReg | SrcMem8 | ModRM | Mov), D(DstReg | SrcMem16 | ModRM | Mov),
4021 F2bv(DstMem | SrcReg | ModRM | SrcWrite | Lock, em_xadd),
4022 N, D(DstMem | SrcReg | ModRM | Mov),
4023 N, N, N, GD(0, &group9),
4025 X8(I(DstReg, em_bswap)),
4027 N, N, N, N, N, N, N, N, N, N, N, N, N, N, N, N,
4029 N, N, N, N, N, N, N, GP(SrcReg | DstMem | ModRM | Mov, &pfx_0f_e7),
4030 N, N, N, N, N, N, N, N,
4032 N, N, N, N, N, N, N, N, N, N, N, N, N, N, N, N
4035 static const struct gprefix three_byte_0f_38_f0 = {
4036 I(DstReg | SrcMem | Mov, em_movbe), N, N, N
4039 static const struct gprefix three_byte_0f_38_f1 = {
4040 I(DstMem | SrcReg | Mov, em_movbe), N, N, N
4044 * Insns below are selected by the prefix which indexed by the third opcode
4047 static const struct opcode opcode_map_0f_38[256] = {
4049 X16(N), X16(N), X16(N), X16(N), X16(N), X16(N), X16(N), X16(N),
4051 X16(N), X16(N), X16(N), X16(N), X16(N), X16(N), X16(N),
4053 GP(EmulateOnUD | ModRM | Prefix, &three_byte_0f_38_f0),
4054 GP(EmulateOnUD | ModRM | Prefix, &three_byte_0f_38_f1),
4073 static unsigned imm_size(struct x86_emulate_ctxt *ctxt)
4077 size = (ctxt->d & ByteOp) ? 1 : ctxt->op_bytes;
4083 static int decode_imm(struct x86_emulate_ctxt *ctxt, struct operand *op,
4084 unsigned size, bool sign_extension)
4086 int rc = X86EMUL_CONTINUE;
4090 op->addr.mem.ea = ctxt->_eip;
4091 /* NB. Immediates are sign-extended as necessary. */
4092 switch (op->bytes) {
4094 op->val = insn_fetch(s8, ctxt);
4097 op->val = insn_fetch(s16, ctxt);
4100 op->val = insn_fetch(s32, ctxt);
4103 op->val = insn_fetch(s64, ctxt);
4106 if (!sign_extension) {
4107 switch (op->bytes) {
4115 op->val &= 0xffffffff;
4123 static int decode_operand(struct x86_emulate_ctxt *ctxt, struct operand *op,
4126 int rc = X86EMUL_CONTINUE;
4130 decode_register_operand(ctxt, op);
4133 rc = decode_imm(ctxt, op, 1, false);
4136 ctxt->memop.bytes = (ctxt->d & ByteOp) ? 1 : ctxt->op_bytes;
4140 if (ctxt->d & BitOp)
4141 fetch_bit_operand(ctxt);
4142 op->orig_val = op->val;
4145 ctxt->memop.bytes = (ctxt->op_bytes == 8) ? 16 : 8;
4149 op->bytes = (ctxt->d & ByteOp) ? 1 : ctxt->op_bytes;
4150 op->addr.reg = reg_rmw(ctxt, VCPU_REGS_RAX);
4151 fetch_register_operand(op);
4152 op->orig_val = op->val;
4156 op->bytes = (ctxt->d & ByteOp) ? 2 : ctxt->op_bytes;
4157 op->addr.reg = reg_rmw(ctxt, VCPU_REGS_RAX);
4158 fetch_register_operand(op);
4159 op->orig_val = op->val;
4162 if (ctxt->d & ByteOp) {
4167 op->bytes = ctxt->op_bytes;
4168 op->addr.reg = reg_rmw(ctxt, VCPU_REGS_RDX);
4169 fetch_register_operand(op);
4170 op->orig_val = op->val;
4174 op->bytes = (ctxt->d & ByteOp) ? 1 : ctxt->op_bytes;
4176 register_address(ctxt, reg_read(ctxt, VCPU_REGS_RDI));
4177 op->addr.mem.seg = VCPU_SREG_ES;
4184 op->addr.reg = reg_rmw(ctxt, VCPU_REGS_RDX);
4185 fetch_register_operand(op);
4189 op->val = reg_read(ctxt, VCPU_REGS_RCX) & 0xff;
4192 rc = decode_imm(ctxt, op, 1, true);
4199 rc = decode_imm(ctxt, op, imm_size(ctxt), true);
4202 rc = decode_imm(ctxt, op, ctxt->op_bytes, true);
4205 ctxt->memop.bytes = 1;
4206 if (ctxt->memop.type == OP_REG) {
4207 ctxt->memop.addr.reg = decode_register(ctxt,
4208 ctxt->modrm_rm, true);
4209 fetch_register_operand(&ctxt->memop);
4213 ctxt->memop.bytes = 2;
4216 ctxt->memop.bytes = 4;
4219 rc = decode_imm(ctxt, op, 2, false);
4222 rc = decode_imm(ctxt, op, imm_size(ctxt), false);
4226 op->bytes = (ctxt->d & ByteOp) ? 1 : ctxt->op_bytes;
4228 register_address(ctxt, reg_read(ctxt, VCPU_REGS_RSI));
4229 op->addr.mem.seg = ctxt->seg_override;
4235 op->bytes = (ctxt->d & ByteOp) ? 1 : ctxt->op_bytes;
4237 register_address(ctxt,
4238 reg_read(ctxt, VCPU_REGS_RBX) +
4239 (reg_read(ctxt, VCPU_REGS_RAX) & 0xff));
4240 op->addr.mem.seg = ctxt->seg_override;
4245 op->addr.mem.ea = ctxt->_eip;
4246 op->bytes = ctxt->op_bytes + 2;
4247 insn_fetch_arr(op->valptr, op->bytes, ctxt);
4250 ctxt->memop.bytes = ctxt->op_bytes + 2;
4253 op->val = VCPU_SREG_ES;
4256 op->val = VCPU_SREG_CS;
4259 op->val = VCPU_SREG_SS;
4262 op->val = VCPU_SREG_DS;
4265 op->val = VCPU_SREG_FS;
4268 op->val = VCPU_SREG_GS;
4271 /* Special instructions do their own operand decoding. */
4273 op->type = OP_NONE; /* Disable writeback. */
4281 int x86_decode_insn(struct x86_emulate_ctxt *ctxt, void *insn, int insn_len)
4283 int rc = X86EMUL_CONTINUE;
4284 int mode = ctxt->mode;
4285 int def_op_bytes, def_ad_bytes, goffset, simd_prefix;
4286 bool op_prefix = false;
4287 bool has_seg_override = false;
4288 struct opcode opcode;
4290 ctxt->memop.type = OP_NONE;
4291 ctxt->memopp = NULL;
4292 ctxt->_eip = ctxt->eip;
4293 ctxt->fetch.ptr = ctxt->fetch.data;
4294 ctxt->fetch.end = ctxt->fetch.data + insn_len;
4295 ctxt->opcode_len = 1;
4297 memcpy(ctxt->fetch.data, insn, insn_len);
4299 rc = __do_insn_fetch_bytes(ctxt, 1);
4300 if (rc != X86EMUL_CONTINUE)
4305 case X86EMUL_MODE_REAL:
4306 case X86EMUL_MODE_VM86:
4307 case X86EMUL_MODE_PROT16:
4308 def_op_bytes = def_ad_bytes = 2;
4310 case X86EMUL_MODE_PROT32:
4311 def_op_bytes = def_ad_bytes = 4;
4313 #ifdef CONFIG_X86_64
4314 case X86EMUL_MODE_PROT64:
4320 return EMULATION_FAILED;
4323 ctxt->op_bytes = def_op_bytes;
4324 ctxt->ad_bytes = def_ad_bytes;
4326 /* Legacy prefixes. */
4328 switch (ctxt->b = insn_fetch(u8, ctxt)) {
4329 case 0x66: /* operand-size override */
4331 /* switch between 2/4 bytes */
4332 ctxt->op_bytes = def_op_bytes ^ 6;
4334 case 0x67: /* address-size override */
4335 if (mode == X86EMUL_MODE_PROT64)
4336 /* switch between 4/8 bytes */
4337 ctxt->ad_bytes = def_ad_bytes ^ 12;
4339 /* switch between 2/4 bytes */
4340 ctxt->ad_bytes = def_ad_bytes ^ 6;
4342 case 0x26: /* ES override */
4343 case 0x2e: /* CS override */
4344 case 0x36: /* SS override */
4345 case 0x3e: /* DS override */
4346 has_seg_override = true;
4347 ctxt->seg_override = (ctxt->b >> 3) & 3;
4349 case 0x64: /* FS override */
4350 case 0x65: /* GS override */
4351 has_seg_override = true;
4352 ctxt->seg_override = ctxt->b & 7;
4354 case 0x40 ... 0x4f: /* REX */
4355 if (mode != X86EMUL_MODE_PROT64)
4357 ctxt->rex_prefix = ctxt->b;
4359 case 0xf0: /* LOCK */
4360 ctxt->lock_prefix = 1;
4362 case 0xf2: /* REPNE/REPNZ */
4363 case 0xf3: /* REP/REPE/REPZ */
4364 ctxt->rep_prefix = ctxt->b;
4370 /* Any legacy prefix after a REX prefix nullifies its effect. */
4372 ctxt->rex_prefix = 0;
4378 if (ctxt->rex_prefix & 8)
4379 ctxt->op_bytes = 8; /* REX.W */
4381 /* Opcode byte(s). */
4382 opcode = opcode_table[ctxt->b];
4383 /* Two-byte opcode? */
4384 if (ctxt->b == 0x0f) {
4385 ctxt->opcode_len = 2;
4386 ctxt->b = insn_fetch(u8, ctxt);
4387 opcode = twobyte_table[ctxt->b];
4389 /* 0F_38 opcode map */
4390 if (ctxt->b == 0x38) {
4391 ctxt->opcode_len = 3;
4392 ctxt->b = insn_fetch(u8, ctxt);
4393 opcode = opcode_map_0f_38[ctxt->b];
4396 ctxt->d = opcode.flags;
4398 if (ctxt->d & ModRM)
4399 ctxt->modrm = insn_fetch(u8, ctxt);
4401 /* vex-prefix instructions are not implemented */
4402 if (ctxt->opcode_len == 1 && (ctxt->b == 0xc5 || ctxt->b == 0xc4) &&
4403 (mode == X86EMUL_MODE_PROT64 ||
4404 (mode >= X86EMUL_MODE_PROT16 && (ctxt->modrm & 0x80)))) {
4408 while (ctxt->d & GroupMask) {
4409 switch (ctxt->d & GroupMask) {
4411 goffset = (ctxt->modrm >> 3) & 7;
4412 opcode = opcode.u.group[goffset];
4415 goffset = (ctxt->modrm >> 3) & 7;
4416 if ((ctxt->modrm >> 6) == 3)
4417 opcode = opcode.u.gdual->mod3[goffset];
4419 opcode = opcode.u.gdual->mod012[goffset];
4422 goffset = ctxt->modrm & 7;
4423 opcode = opcode.u.group[goffset];
4426 if (ctxt->rep_prefix && op_prefix)
4427 return EMULATION_FAILED;
4428 simd_prefix = op_prefix ? 0x66 : ctxt->rep_prefix;
4429 switch (simd_prefix) {
4430 case 0x00: opcode = opcode.u.gprefix->pfx_no; break;
4431 case 0x66: opcode = opcode.u.gprefix->pfx_66; break;
4432 case 0xf2: opcode = opcode.u.gprefix->pfx_f2; break;
4433 case 0xf3: opcode = opcode.u.gprefix->pfx_f3; break;
4437 if (ctxt->modrm > 0xbf)
4438 opcode = opcode.u.esc->high[ctxt->modrm - 0xc0];
4440 opcode = opcode.u.esc->op[(ctxt->modrm >> 3) & 7];
4443 return EMULATION_FAILED;
4446 ctxt->d &= ~(u64)GroupMask;
4447 ctxt->d |= opcode.flags;
4452 return EMULATION_FAILED;
4454 ctxt->execute = opcode.u.execute;
4456 if (unlikely(ctxt->ud) && likely(!(ctxt->d & EmulateOnUD)))
4457 return EMULATION_FAILED;
4459 if (unlikely(ctxt->d &
4460 (NotImpl|Stack|Op3264|Sse|Mmx|Intercept|CheckPerm))) {
4462 * These are copied unconditionally here, and checked unconditionally
4463 * in x86_emulate_insn.
4465 ctxt->check_perm = opcode.check_perm;
4466 ctxt->intercept = opcode.intercept;
4468 if (ctxt->d & NotImpl)
4469 return EMULATION_FAILED;
4471 if (mode == X86EMUL_MODE_PROT64 && (ctxt->d & Stack))
4474 if (ctxt->d & Op3264) {
4475 if (mode == X86EMUL_MODE_PROT64)
4482 ctxt->op_bytes = 16;
4483 else if (ctxt->d & Mmx)
4487 /* ModRM and SIB bytes. */
4488 if (ctxt->d & ModRM) {
4489 rc = decode_modrm(ctxt, &ctxt->memop);
4490 if (!has_seg_override) {
4491 has_seg_override = true;
4492 ctxt->seg_override = ctxt->modrm_seg;
4494 } else if (ctxt->d & MemAbs)
4495 rc = decode_abs(ctxt, &ctxt->memop);
4496 if (rc != X86EMUL_CONTINUE)
4499 if (!has_seg_override)
4500 ctxt->seg_override = VCPU_SREG_DS;
4502 ctxt->memop.addr.mem.seg = ctxt->seg_override;
4505 * Decode and fetch the source operand: register, memory
4508 rc = decode_operand(ctxt, &ctxt->src, (ctxt->d >> SrcShift) & OpMask);
4509 if (rc != X86EMUL_CONTINUE)
4513 * Decode and fetch the second source operand: register, memory
4516 rc = decode_operand(ctxt, &ctxt->src2, (ctxt->d >> Src2Shift) & OpMask);
4517 if (rc != X86EMUL_CONTINUE)
4520 /* Decode and fetch the destination operand: register or memory. */
4521 rc = decode_operand(ctxt, &ctxt->dst, (ctxt->d >> DstShift) & OpMask);
4524 if (ctxt->rip_relative)
4525 ctxt->memopp->addr.mem.ea += ctxt->_eip;
4527 return (rc != X86EMUL_CONTINUE) ? EMULATION_FAILED : EMULATION_OK;
4530 bool x86_page_table_writing_insn(struct x86_emulate_ctxt *ctxt)
4532 return ctxt->d & PageTable;
4535 static bool string_insn_completed(struct x86_emulate_ctxt *ctxt)
4537 /* The second termination condition only applies for REPE
4538 * and REPNE. Test if the repeat string operation prefix is
4539 * REPE/REPZ or REPNE/REPNZ and if it's the case it tests the
4540 * corresponding termination condition according to:
4541 * - if REPE/REPZ and ZF = 0 then done
4542 * - if REPNE/REPNZ and ZF = 1 then done
4544 if (((ctxt->b == 0xa6) || (ctxt->b == 0xa7) ||
4545 (ctxt->b == 0xae) || (ctxt->b == 0xaf))
4546 && (((ctxt->rep_prefix == REPE_PREFIX) &&
4547 ((ctxt->eflags & EFLG_ZF) == 0))
4548 || ((ctxt->rep_prefix == REPNE_PREFIX) &&
4549 ((ctxt->eflags & EFLG_ZF) == EFLG_ZF))))
4555 static int flush_pending_x87_faults(struct x86_emulate_ctxt *ctxt)
4559 ctxt->ops->get_fpu(ctxt);
4560 asm volatile("1: fwait \n\t"
4562 ".pushsection .fixup,\"ax\" \n\t"
4564 "movb $1, %[fault] \n\t"
4567 _ASM_EXTABLE(1b, 3b)
4568 : [fault]"+qm"(fault));
4569 ctxt->ops->put_fpu(ctxt);
4571 if (unlikely(fault))
4572 return emulate_exception(ctxt, MF_VECTOR, 0, false);
4574 return X86EMUL_CONTINUE;
4577 static void fetch_possible_mmx_operand(struct x86_emulate_ctxt *ctxt,
4580 if (op->type == OP_MM)
4581 read_mmx_reg(ctxt, &op->mm_val, op->addr.mm);
4584 static int fastop(struct x86_emulate_ctxt *ctxt, void (*fop)(struct fastop *))
4586 ulong flags = (ctxt->eflags & EFLAGS_MASK) | X86_EFLAGS_IF;
4587 if (!(ctxt->d & ByteOp))
4588 fop += __ffs(ctxt->dst.bytes) * FASTOP_SIZE;
4589 asm("push %[flags]; popf; call *%[fastop]; pushf; pop %[flags]\n"
4590 : "+a"(ctxt->dst.val), "+d"(ctxt->src.val), [flags]"+D"(flags),
4592 : "c"(ctxt->src2.val));
4593 ctxt->eflags = (ctxt->eflags & ~EFLAGS_MASK) | (flags & EFLAGS_MASK);
4594 if (!fop) /* exception is returned in fop variable */
4595 return emulate_de(ctxt);
4596 return X86EMUL_CONTINUE;
4599 void init_decode_cache(struct x86_emulate_ctxt *ctxt)
4601 memset(&ctxt->rip_relative, 0,
4602 (void *)&ctxt->modrm - (void *)&ctxt->rip_relative);
4604 ctxt->io_read.pos = 0;
4605 ctxt->io_read.end = 0;
4606 ctxt->mem_read.end = 0;
4609 int x86_emulate_insn(struct x86_emulate_ctxt *ctxt)
4611 const struct x86_emulate_ops *ops = ctxt->ops;
4612 int rc = X86EMUL_CONTINUE;
4613 int saved_dst_type = ctxt->dst.type;
4615 ctxt->mem_read.pos = 0;
4617 /* LOCK prefix is allowed only with some instructions */
4618 if (ctxt->lock_prefix && (!(ctxt->d & Lock) || ctxt->dst.type != OP_MEM)) {
4619 rc = emulate_ud(ctxt);
4623 if ((ctxt->d & SrcMask) == SrcMemFAddr && ctxt->src.type != OP_MEM) {
4624 rc = emulate_ud(ctxt);
4628 if (unlikely(ctxt->d &
4629 (No64|Undefined|Sse|Mmx|Intercept|CheckPerm|Priv|Prot|String))) {
4630 if ((ctxt->mode == X86EMUL_MODE_PROT64 && (ctxt->d & No64)) ||
4631 (ctxt->d & Undefined)) {
4632 rc = emulate_ud(ctxt);
4636 if (((ctxt->d & (Sse|Mmx)) && ((ops->get_cr(ctxt, 0) & X86_CR0_EM)))
4637 || ((ctxt->d & Sse) && !(ops->get_cr(ctxt, 4) & X86_CR4_OSFXSR))) {
4638 rc = emulate_ud(ctxt);
4642 if ((ctxt->d & (Sse|Mmx)) && (ops->get_cr(ctxt, 0) & X86_CR0_TS)) {
4643 rc = emulate_nm(ctxt);
4647 if (ctxt->d & Mmx) {
4648 rc = flush_pending_x87_faults(ctxt);
4649 if (rc != X86EMUL_CONTINUE)
4652 * Now that we know the fpu is exception safe, we can fetch
4655 fetch_possible_mmx_operand(ctxt, &ctxt->src);
4656 fetch_possible_mmx_operand(ctxt, &ctxt->src2);
4657 if (!(ctxt->d & Mov))
4658 fetch_possible_mmx_operand(ctxt, &ctxt->dst);
4661 if (unlikely(ctxt->guest_mode) && (ctxt->d & Intercept)) {
4662 rc = emulator_check_intercept(ctxt, ctxt->intercept,
4663 X86_ICPT_PRE_EXCEPT);
4664 if (rc != X86EMUL_CONTINUE)
4668 /* Privileged instruction can be executed only in CPL=0 */
4669 if ((ctxt->d & Priv) && ops->cpl(ctxt)) {
4670 if (ctxt->d & PrivUD)
4671 rc = emulate_ud(ctxt);
4673 rc = emulate_gp(ctxt, 0);
4677 /* Instruction can only be executed in protected mode */
4678 if ((ctxt->d & Prot) && ctxt->mode < X86EMUL_MODE_PROT16) {
4679 rc = emulate_ud(ctxt);
4683 /* Do instruction specific permission checks */
4684 if (ctxt->d & CheckPerm) {
4685 rc = ctxt->check_perm(ctxt);
4686 if (rc != X86EMUL_CONTINUE)
4690 if (unlikely(ctxt->guest_mode) && (ctxt->d & Intercept)) {
4691 rc = emulator_check_intercept(ctxt, ctxt->intercept,
4692 X86_ICPT_POST_EXCEPT);
4693 if (rc != X86EMUL_CONTINUE)
4697 if (ctxt->rep_prefix && (ctxt->d & String)) {
4698 /* All REP prefixes have the same first termination condition */
4699 if (address_mask(ctxt, reg_read(ctxt, VCPU_REGS_RCX)) == 0) {
4700 ctxt->eip = ctxt->_eip;
4701 ctxt->eflags &= ~EFLG_RF;
4707 if ((ctxt->src.type == OP_MEM) && !(ctxt->d & NoAccess)) {
4708 rc = segmented_read(ctxt, ctxt->src.addr.mem,
4709 ctxt->src.valptr, ctxt->src.bytes);
4710 if (rc != X86EMUL_CONTINUE)
4712 ctxt->src.orig_val64 = ctxt->src.val64;
4715 if (ctxt->src2.type == OP_MEM) {
4716 rc = segmented_read(ctxt, ctxt->src2.addr.mem,
4717 &ctxt->src2.val, ctxt->src2.bytes);
4718 if (rc != X86EMUL_CONTINUE)
4722 if ((ctxt->d & DstMask) == ImplicitOps)
4726 if ((ctxt->dst.type == OP_MEM) && !(ctxt->d & Mov)) {
4727 /* optimisation - avoid slow emulated read if Mov */
4728 rc = segmented_read(ctxt, ctxt->dst.addr.mem,
4729 &ctxt->dst.val, ctxt->dst.bytes);
4730 if (rc != X86EMUL_CONTINUE)
4733 ctxt->dst.orig_val = ctxt->dst.val;
4737 if (unlikely(ctxt->guest_mode) && (ctxt->d & Intercept)) {
4738 rc = emulator_check_intercept(ctxt, ctxt->intercept,
4739 X86_ICPT_POST_MEMACCESS);
4740 if (rc != X86EMUL_CONTINUE)
4744 if (ctxt->rep_prefix && (ctxt->d & String))
4745 ctxt->eflags |= EFLG_RF;
4747 ctxt->eflags &= ~EFLG_RF;
4749 if (ctxt->execute) {
4750 if (ctxt->d & Fastop) {
4751 void (*fop)(struct fastop *) = (void *)ctxt->execute;
4752 rc = fastop(ctxt, fop);
4753 if (rc != X86EMUL_CONTINUE)
4757 rc = ctxt->execute(ctxt);
4758 if (rc != X86EMUL_CONTINUE)
4763 if (ctxt->opcode_len == 2)
4765 else if (ctxt->opcode_len == 3)
4766 goto threebyte_insn;
4769 case 0x63: /* movsxd */
4770 if (ctxt->mode != X86EMUL_MODE_PROT64)
4771 goto cannot_emulate;
4772 ctxt->dst.val = (s32) ctxt->src.val;
4774 case 0x70 ... 0x7f: /* jcc (short) */
4775 if (test_cc(ctxt->b, ctxt->eflags))
4776 rc = jmp_rel(ctxt, ctxt->src.val);
4778 case 0x8d: /* lea r16/r32, m */
4779 ctxt->dst.val = ctxt->src.addr.mem.ea;
4781 case 0x90 ... 0x97: /* nop / xchg reg, rax */
4782 if (ctxt->dst.addr.reg == reg_rmw(ctxt, VCPU_REGS_RAX))
4783 ctxt->dst.type = OP_NONE;
4787 case 0x98: /* cbw/cwde/cdqe */
4788 switch (ctxt->op_bytes) {
4789 case 2: ctxt->dst.val = (s8)ctxt->dst.val; break;
4790 case 4: ctxt->dst.val = (s16)ctxt->dst.val; break;
4791 case 8: ctxt->dst.val = (s32)ctxt->dst.val; break;
4794 case 0xcc: /* int3 */
4795 rc = emulate_int(ctxt, 3);
4797 case 0xcd: /* int n */
4798 rc = emulate_int(ctxt, ctxt->src.val);
4800 case 0xce: /* into */
4801 if (ctxt->eflags & EFLG_OF)
4802 rc = emulate_int(ctxt, 4);
4804 case 0xe9: /* jmp rel */
4805 case 0xeb: /* jmp rel short */
4806 rc = jmp_rel(ctxt, ctxt->src.val);
4807 ctxt->dst.type = OP_NONE; /* Disable writeback. */
4809 case 0xf4: /* hlt */
4810 ctxt->ops->halt(ctxt);
4812 case 0xf5: /* cmc */
4813 /* complement carry flag from eflags reg */
4814 ctxt->eflags ^= EFLG_CF;
4816 case 0xf8: /* clc */
4817 ctxt->eflags &= ~EFLG_CF;
4819 case 0xf9: /* stc */
4820 ctxt->eflags |= EFLG_CF;
4822 case 0xfc: /* cld */
4823 ctxt->eflags &= ~EFLG_DF;
4825 case 0xfd: /* std */
4826 ctxt->eflags |= EFLG_DF;
4829 goto cannot_emulate;
4832 if (rc != X86EMUL_CONTINUE)
4836 if (ctxt->d & SrcWrite) {
4837 BUG_ON(ctxt->src.type == OP_MEM || ctxt->src.type == OP_MEM_STR);
4838 rc = writeback(ctxt, &ctxt->src);
4839 if (rc != X86EMUL_CONTINUE)
4842 if (!(ctxt->d & NoWrite)) {
4843 rc = writeback(ctxt, &ctxt->dst);
4844 if (rc != X86EMUL_CONTINUE)
4849 * restore dst type in case the decoding will be reused
4850 * (happens for string instruction )
4852 ctxt->dst.type = saved_dst_type;
4854 if ((ctxt->d & SrcMask) == SrcSI)
4855 string_addr_inc(ctxt, VCPU_REGS_RSI, &ctxt->src);
4857 if ((ctxt->d & DstMask) == DstDI)
4858 string_addr_inc(ctxt, VCPU_REGS_RDI, &ctxt->dst);
4860 if (ctxt->rep_prefix && (ctxt->d & String)) {
4862 struct read_cache *r = &ctxt->io_read;
4863 if ((ctxt->d & SrcMask) == SrcSI)
4864 count = ctxt->src.count;
4866 count = ctxt->dst.count;
4867 register_address_increment(ctxt, reg_rmw(ctxt, VCPU_REGS_RCX),
4870 if (!string_insn_completed(ctxt)) {
4872 * Re-enter guest when pio read ahead buffer is empty
4873 * or, if it is not used, after each 1024 iteration.
4875 if ((r->end != 0 || reg_read(ctxt, VCPU_REGS_RCX) & 0x3ff) &&
4876 (r->end == 0 || r->end != r->pos)) {
4878 * Reset read cache. Usually happens before
4879 * decode, but since instruction is restarted
4880 * we have to do it here.
4882 ctxt->mem_read.end = 0;
4883 writeback_registers(ctxt);
4884 return EMULATION_RESTART;
4886 goto done; /* skip rip writeback */
4888 ctxt->eflags &= ~EFLG_RF;
4891 ctxt->eip = ctxt->_eip;
4894 if (rc == X86EMUL_PROPAGATE_FAULT) {
4895 WARN_ON(ctxt->exception.vector > 0x1f);
4896 ctxt->have_exception = true;
4898 if (rc == X86EMUL_INTERCEPTED)
4899 return EMULATION_INTERCEPTED;
4901 if (rc == X86EMUL_CONTINUE)
4902 writeback_registers(ctxt);
4904 return (rc == X86EMUL_UNHANDLEABLE) ? EMULATION_FAILED : EMULATION_OK;
4908 case 0x09: /* wbinvd */
4909 (ctxt->ops->wbinvd)(ctxt);
4911 case 0x08: /* invd */
4912 case 0x0d: /* GrpP (prefetch) */
4913 case 0x18: /* Grp16 (prefetch/nop) */
4914 case 0x1f: /* nop */
4916 case 0x20: /* mov cr, reg */
4917 ctxt->dst.val = ops->get_cr(ctxt, ctxt->modrm_reg);
4919 case 0x21: /* mov from dr to reg */
4920 ops->get_dr(ctxt, ctxt->modrm_reg, &ctxt->dst.val);
4922 case 0x40 ... 0x4f: /* cmov */
4923 if (test_cc(ctxt->b, ctxt->eflags))
4924 ctxt->dst.val = ctxt->src.val;
4925 else if (ctxt->mode != X86EMUL_MODE_PROT64 ||
4926 ctxt->op_bytes != 4)
4927 ctxt->dst.type = OP_NONE; /* no writeback */
4929 case 0x80 ... 0x8f: /* jnz rel, etc*/
4930 if (test_cc(ctxt->b, ctxt->eflags))
4931 rc = jmp_rel(ctxt, ctxt->src.val);
4933 case 0x90 ... 0x9f: /* setcc r/m8 */
4934 ctxt->dst.val = test_cc(ctxt->b, ctxt->eflags);
4936 case 0xae: /* clflush */
4938 case 0xb6 ... 0xb7: /* movzx */
4939 ctxt->dst.bytes = ctxt->op_bytes;
4940 ctxt->dst.val = (ctxt->src.bytes == 1) ? (u8) ctxt->src.val
4941 : (u16) ctxt->src.val;
4943 case 0xbe ... 0xbf: /* movsx */
4944 ctxt->dst.bytes = ctxt->op_bytes;
4945 ctxt->dst.val = (ctxt->src.bytes == 1) ? (s8) ctxt->src.val :
4946 (s16) ctxt->src.val;
4948 case 0xc3: /* movnti */
4949 ctxt->dst.bytes = ctxt->op_bytes;
4950 ctxt->dst.val = (ctxt->op_bytes == 8) ? (u64) ctxt->src.val :
4951 (u32) ctxt->src.val;
4954 goto cannot_emulate;
4959 if (rc != X86EMUL_CONTINUE)
4965 return EMULATION_FAILED;
4968 void emulator_invalidate_register_cache(struct x86_emulate_ctxt *ctxt)
4970 invalidate_registers(ctxt);
4973 void emulator_writeback_register_cache(struct x86_emulate_ctxt *ctxt)
4975 writeback_registers(ctxt);