xref: /openbmc/linux/arch/x86/kvm/emulate.c (revision ae3473231e77a3f1909d48cd144cebe5e1d049b3)
1 /******************************************************************************
2  * emulate.c
3  *
4  * Generic x86 (32-bit and 64-bit) instruction decoder and emulator.
5  *
6  * Copyright (c) 2005 Keir Fraser
7  *
8  * Linux coding style, mod r/m decoder, segment base fixes, real-mode
9  * privileged instructions:
10  *
11  * Copyright (C) 2006 Qumranet
12  * Copyright 2010 Red Hat, Inc. and/or its affiliates.
13  *
14  *   Avi Kivity <avi@qumranet.com>
15  *   Yaniv Kamay <yaniv@qumranet.com>
16  *
17  * This work is licensed under the terms of the GNU GPL, version 2.  See
18  * the COPYING file in the top-level directory.
19  *
20  * From: xen-unstable 10676:af9809f51f81a3c43f276f00c81a52ef558afda4
21  */
22 
23 #include <linux/kvm_host.h>
24 #include "kvm_cache_regs.h"
25 #include <asm/kvm_emulate.h>
26 #include <linux/stringify.h>
27 #include <asm/debugreg.h>
28 
29 #include "x86.h"
30 #include "tss.h"
31 
32 /*
33  * Operand types
34  */
35 #define OpNone             0ull
36 #define OpImplicit         1ull  /* No generic decode */
37 #define OpReg              2ull  /* Register */
38 #define OpMem              3ull  /* Memory */
39 #define OpAcc              4ull  /* Accumulator: AL/AX/EAX/RAX */
40 #define OpDI               5ull  /* ES:DI/EDI/RDI */
41 #define OpMem64            6ull  /* Memory, 64-bit */
42 #define OpImmUByte         7ull  /* Zero-extended 8-bit immediate */
43 #define OpDX               8ull  /* DX register */
44 #define OpCL               9ull  /* CL register (for shifts) */
45 #define OpImmByte         10ull  /* 8-bit sign extended immediate */
46 #define OpOne             11ull  /* Implied 1 */
47 #define OpImm             12ull  /* Sign extended up to 32-bit immediate */
48 #define OpMem16           13ull  /* Memory operand (16-bit). */
49 #define OpMem32           14ull  /* Memory operand (32-bit). */
50 #define OpImmU            15ull  /* Immediate operand, zero extended */
51 #define OpSI              16ull  /* SI/ESI/RSI */
52 #define OpImmFAddr        17ull  /* Immediate far address */
53 #define OpMemFAddr        18ull  /* Far address in memory */
54 #define OpImmU16          19ull  /* Immediate operand, 16 bits, zero extended */
55 #define OpES              20ull  /* ES */
56 #define OpCS              21ull  /* CS */
57 #define OpSS              22ull  /* SS */
58 #define OpDS              23ull  /* DS */
59 #define OpFS              24ull  /* FS */
60 #define OpGS              25ull  /* GS */
61 #define OpMem8            26ull  /* 8-bit zero extended memory operand */
62 #define OpImm64           27ull  /* Sign extended 16/32/64-bit immediate */
63 #define OpXLat            28ull  /* memory at BX/EBX/RBX + zero-extended AL */
64 #define OpAccLo           29ull  /* Low part of extended acc (AX/AX/EAX/RAX) */
65 #define OpAccHi           30ull  /* High part of extended acc (-/DX/EDX/RDX) */
66 
67 #define OpBits             5  /* Width of operand field */
68 #define OpMask             ((1ull << OpBits) - 1)
69 
70 /*
71  * Opcode effective-address decode tables.
72  * Note that we only emulate instructions that have at least one memory
73  * operand (excluding implicit stack references). We assume that stack
74  * references and instruction fetches will never occur in special memory
75  * areas that require emulation. So, for example, 'mov <imm>,<reg>' need
76  * not be handled.
77  */
78 
79 /* Operand sizes: 8-bit operands or specified/overridden size. */
80 #define ByteOp      (1<<0)	/* 8-bit operands. */
81 /* Destination operand type. */
82 #define DstShift    1
83 #define ImplicitOps (OpImplicit << DstShift)
84 #define DstReg      (OpReg << DstShift)
85 #define DstMem      (OpMem << DstShift)
86 #define DstAcc      (OpAcc << DstShift)
87 #define DstDI       (OpDI << DstShift)
88 #define DstMem64    (OpMem64 << DstShift)
89 #define DstMem16    (OpMem16 << DstShift)
90 #define DstImmUByte (OpImmUByte << DstShift)
91 #define DstDX       (OpDX << DstShift)
92 #define DstAccLo    (OpAccLo << DstShift)
93 #define DstMask     (OpMask << DstShift)
94 /* Source operand type. */
95 #define SrcShift    6
96 #define SrcNone     (OpNone << SrcShift)
97 #define SrcReg      (OpReg << SrcShift)
98 #define SrcMem      (OpMem << SrcShift)
99 #define SrcMem16    (OpMem16 << SrcShift)
100 #define SrcMem32    (OpMem32 << SrcShift)
101 #define SrcImm      (OpImm << SrcShift)
102 #define SrcImmByte  (OpImmByte << SrcShift)
103 #define SrcOne      (OpOne << SrcShift)
104 #define SrcImmUByte (OpImmUByte << SrcShift)
105 #define SrcImmU     (OpImmU << SrcShift)
106 #define SrcSI       (OpSI << SrcShift)
107 #define SrcXLat     (OpXLat << SrcShift)
108 #define SrcImmFAddr (OpImmFAddr << SrcShift)
109 #define SrcMemFAddr (OpMemFAddr << SrcShift)
110 #define SrcAcc      (OpAcc << SrcShift)
111 #define SrcImmU16   (OpImmU16 << SrcShift)
112 #define SrcImm64    (OpImm64 << SrcShift)
113 #define SrcDX       (OpDX << SrcShift)
114 #define SrcMem8     (OpMem8 << SrcShift)
115 #define SrcAccHi    (OpAccHi << SrcShift)
116 #define SrcMask     (OpMask << SrcShift)
117 #define BitOp       (1<<11)
118 #define MemAbs      (1<<12)      /* Memory operand is absolute displacement */
119 #define String      (1<<13)     /* String instruction (rep capable) */
120 #define Stack       (1<<14)     /* Stack instruction (push/pop) */
121 #define GroupMask   (7<<15)     /* Opcode uses one of the group mechanisms */
122 #define Group       (1<<15)     /* Bits 3:5 of modrm byte extend opcode */
123 #define GroupDual   (2<<15)     /* Alternate decoding of mod == 3 */
124 #define Prefix      (3<<15)     /* Instruction varies with 66/f2/f3 prefix */
125 #define RMExt       (4<<15)     /* Opcode extension in ModRM r/m if mod == 3 */
126 #define Escape      (5<<15)     /* Escape to coprocessor instruction */
127 #define InstrDual   (6<<15)     /* Alternate instruction decoding of mod == 3 */
128 #define ModeDual    (7<<15)     /* Different instruction for 32/64 bit */
129 #define Sse         (1<<18)     /* SSE Vector instruction */
130 /* Generic ModRM decode. */
131 #define ModRM       (1<<19)
132 /* Destination is only written; never read. */
133 #define Mov         (1<<20)
134 /* Misc flags */
135 #define Prot        (1<<21) /* instruction generates #UD if not in prot-mode */
136 #define EmulateOnUD (1<<22) /* Emulate if unsupported by the host */
137 #define NoAccess    (1<<23) /* Don't access memory (lea/invlpg/verr etc) */
138 #define Op3264      (1<<24) /* Operand is 64b in long mode, 32b otherwise */
139 #define Undefined   (1<<25) /* No Such Instruction */
140 #define Lock        (1<<26) /* lock prefix is allowed for the instruction */
141 #define Priv        (1<<27) /* instruction generates #GP if current CPL != 0 */
142 #define No64	    (1<<28)
143 #define PageTable   (1 << 29)   /* instruction used to write page table */
144 #define NotImpl     (1 << 30)   /* instruction is not implemented */
145 /* Source 2 operand type */
146 #define Src2Shift   (31)
147 #define Src2None    (OpNone << Src2Shift)
148 #define Src2Mem     (OpMem << Src2Shift)
149 #define Src2CL      (OpCL << Src2Shift)
150 #define Src2ImmByte (OpImmByte << Src2Shift)
151 #define Src2One     (OpOne << Src2Shift)
152 #define Src2Imm     (OpImm << Src2Shift)
153 #define Src2ES      (OpES << Src2Shift)
154 #define Src2CS      (OpCS << Src2Shift)
155 #define Src2SS      (OpSS << Src2Shift)
156 #define Src2DS      (OpDS << Src2Shift)
157 #define Src2FS      (OpFS << Src2Shift)
158 #define Src2GS      (OpGS << Src2Shift)
159 #define Src2Mask    (OpMask << Src2Shift)
160 #define Mmx         ((u64)1 << 40)  /* MMX Vector instruction */
161 #define AlignMask   ((u64)7 << 41)
162 #define Aligned     ((u64)1 << 41)  /* Explicitly aligned (e.g. MOVDQA) */
163 #define Unaligned   ((u64)2 << 41)  /* Explicitly unaligned (e.g. MOVDQU) */
164 #define Avx         ((u64)3 << 41)  /* Advanced Vector Extensions */
165 #define Aligned16   ((u64)4 << 41)  /* Aligned to 16 byte boundary (e.g. FXSAVE) */
166 #define Fastop      ((u64)1 << 44)  /* Use opcode::u.fastop */
167 #define NoWrite     ((u64)1 << 45)  /* No writeback */
168 #define SrcWrite    ((u64)1 << 46)  /* Write back src operand */
169 #define NoMod	    ((u64)1 << 47)  /* Mod field is ignored */
170 #define Intercept   ((u64)1 << 48)  /* Has valid intercept field */
171 #define CheckPerm   ((u64)1 << 49)  /* Has valid check_perm field */
172 #define PrivUD      ((u64)1 << 51)  /* #UD instead of #GP on CPL > 0 */
173 #define NearBranch  ((u64)1 << 52)  /* Near branches */
174 #define No16	    ((u64)1 << 53)  /* No 16 bit operand */
175 #define IncSP       ((u64)1 << 54)  /* SP is incremented before ModRM calc */
176 
177 #define DstXacc     (DstAccLo | SrcAccHi | SrcWrite)
178 
179 #define X2(x...) x, x
180 #define X3(x...) X2(x), x
181 #define X4(x...) X2(x), X2(x)
182 #define X5(x...) X4(x), x
183 #define X6(x...) X4(x), X2(x)
184 #define X7(x...) X4(x), X3(x)
185 #define X8(x...) X4(x), X4(x)
186 #define X16(x...) X8(x), X8(x)
187 
188 #define NR_FASTOP (ilog2(sizeof(ulong)) + 1)
189 #define FASTOP_SIZE 8
190 
191 /*
192  * fastop functions have a special calling convention:
193  *
194  * dst:    rax        (in/out)
195  * src:    rdx        (in/out)
196  * src2:   rcx        (in)
197  * flags:  rflags     (in/out)
198  * ex:     rsi        (in:fastop pointer, out:zero if exception)
199  *
200  * Moreover, they are all exactly FASTOP_SIZE bytes long, so functions for
201  * different operand sizes can be reached by calculation, rather than a jump
202  * table (which would be bigger than the code).
203  *
204  * fastop functions are declared as taking a never-defined fastop parameter,
205  * so they can't be called from C directly.
206  */
207 
208 struct fastop;
209 
210 struct opcode {
211 	u64 flags : 56;
212 	u64 intercept : 8;
213 	union {
214 		int (*execute)(struct x86_emulate_ctxt *ctxt);
215 		const struct opcode *group;
216 		const struct group_dual *gdual;
217 		const struct gprefix *gprefix;
218 		const struct escape *esc;
219 		const struct instr_dual *idual;
220 		const struct mode_dual *mdual;
221 		void (*fastop)(struct fastop *fake);
222 	} u;
223 	int (*check_perm)(struct x86_emulate_ctxt *ctxt);
224 };
225 
226 struct group_dual {
227 	struct opcode mod012[8];
228 	struct opcode mod3[8];
229 };
230 
231 struct gprefix {
232 	struct opcode pfx_no;
233 	struct opcode pfx_66;
234 	struct opcode pfx_f2;
235 	struct opcode pfx_f3;
236 };
237 
238 struct escape {
239 	struct opcode op[8];
240 	struct opcode high[64];
241 };
242 
243 struct instr_dual {
244 	struct opcode mod012;
245 	struct opcode mod3;
246 };
247 
248 struct mode_dual {
249 	struct opcode mode32;
250 	struct opcode mode64;
251 };
252 
253 #define EFLG_RESERVED_ZEROS_MASK 0xffc0802a
254 
255 enum x86_transfer_type {
256 	X86_TRANSFER_NONE,
257 	X86_TRANSFER_CALL_JMP,
258 	X86_TRANSFER_RET,
259 	X86_TRANSFER_TASK_SWITCH,
260 };
261 
262 static ulong reg_read(struct x86_emulate_ctxt *ctxt, unsigned nr)
263 {
264 	if (!(ctxt->regs_valid & (1 << nr))) {
265 		ctxt->regs_valid |= 1 << nr;
266 		ctxt->_regs[nr] = ctxt->ops->read_gpr(ctxt, nr);
267 	}
268 	return ctxt->_regs[nr];
269 }
270 
271 static ulong *reg_write(struct x86_emulate_ctxt *ctxt, unsigned nr)
272 {
273 	ctxt->regs_valid |= 1 << nr;
274 	ctxt->regs_dirty |= 1 << nr;
275 	return &ctxt->_regs[nr];
276 }
277 
278 static ulong *reg_rmw(struct x86_emulate_ctxt *ctxt, unsigned nr)
279 {
280 	reg_read(ctxt, nr);
281 	return reg_write(ctxt, nr);
282 }
283 
284 static void writeback_registers(struct x86_emulate_ctxt *ctxt)
285 {
286 	unsigned reg;
287 
288 	for_each_set_bit(reg, (ulong *)&ctxt->regs_dirty, 16)
289 		ctxt->ops->write_gpr(ctxt, reg, ctxt->_regs[reg]);
290 }
291 
292 static void invalidate_registers(struct x86_emulate_ctxt *ctxt)
293 {
294 	ctxt->regs_dirty = 0;
295 	ctxt->regs_valid = 0;
296 }
297 
298 /*
299  * These EFLAGS bits are restored from saved value during emulation, and
300  * any changes are written back to the saved value after emulation.
301  */
302 #define EFLAGS_MASK (X86_EFLAGS_OF|X86_EFLAGS_SF|X86_EFLAGS_ZF|X86_EFLAGS_AF|\
303 		     X86_EFLAGS_PF|X86_EFLAGS_CF)
304 
305 #ifdef CONFIG_X86_64
306 #define ON64(x) x
307 #else
308 #define ON64(x)
309 #endif
310 
311 static int fastop(struct x86_emulate_ctxt *ctxt, void (*fop)(struct fastop *));
312 
313 #define FOP_FUNC(name) \
314 	".align " __stringify(FASTOP_SIZE) " \n\t" \
315 	".type " name ", @function \n\t" \
316 	name ":\n\t"
317 
318 #define FOP_RET   "ret \n\t"
319 
320 #define FOP_START(op) \
321 	extern void em_##op(struct fastop *fake); \
322 	asm(".pushsection .text, \"ax\" \n\t" \
323 	    ".global em_" #op " \n\t" \
324 	    FOP_FUNC("em_" #op)
325 
326 #define FOP_END \
327 	    ".popsection")
328 
329 #define FOPNOP() \
330 	FOP_FUNC(__stringify(__UNIQUE_ID(nop))) \
331 	FOP_RET
332 
333 #define FOP1E(op,  dst) \
334 	FOP_FUNC(#op "_" #dst) \
335 	"10: " #op " %" #dst " \n\t" FOP_RET
336 
337 #define FOP1EEX(op,  dst) \
338 	FOP1E(op, dst) _ASM_EXTABLE(10b, kvm_fastop_exception)
339 
340 #define FASTOP1(op) \
341 	FOP_START(op) \
342 	FOP1E(op##b, al) \
343 	FOP1E(op##w, ax) \
344 	FOP1E(op##l, eax) \
345 	ON64(FOP1E(op##q, rax))	\
346 	FOP_END
347 
348 /* 1-operand, using src2 (for MUL/DIV r/m) */
349 #define FASTOP1SRC2(op, name) \
350 	FOP_START(name) \
351 	FOP1E(op, cl) \
352 	FOP1E(op, cx) \
353 	FOP1E(op, ecx) \
354 	ON64(FOP1E(op, rcx)) \
355 	FOP_END
356 
357 /* 1-operand, using src2 (for MUL/DIV r/m), with exceptions */
358 #define FASTOP1SRC2EX(op, name) \
359 	FOP_START(name) \
360 	FOP1EEX(op, cl) \
361 	FOP1EEX(op, cx) \
362 	FOP1EEX(op, ecx) \
363 	ON64(FOP1EEX(op, rcx)) \
364 	FOP_END
365 
366 #define FOP2E(op,  dst, src)	   \
367 	FOP_FUNC(#op "_" #dst "_" #src) \
368 	#op " %" #src ", %" #dst " \n\t" FOP_RET
369 
370 #define FASTOP2(op) \
371 	FOP_START(op) \
372 	FOP2E(op##b, al, dl) \
373 	FOP2E(op##w, ax, dx) \
374 	FOP2E(op##l, eax, edx) \
375 	ON64(FOP2E(op##q, rax, rdx)) \
376 	FOP_END
377 
378 /* 2 operand, word only */
379 #define FASTOP2W(op) \
380 	FOP_START(op) \
381 	FOPNOP() \
382 	FOP2E(op##w, ax, dx) \
383 	FOP2E(op##l, eax, edx) \
384 	ON64(FOP2E(op##q, rax, rdx)) \
385 	FOP_END
386 
387 /* 2 operand, src is CL */
388 #define FASTOP2CL(op) \
389 	FOP_START(op) \
390 	FOP2E(op##b, al, cl) \
391 	FOP2E(op##w, ax, cl) \
392 	FOP2E(op##l, eax, cl) \
393 	ON64(FOP2E(op##q, rax, cl)) \
394 	FOP_END
395 
396 /* 2 operand, src and dest are reversed */
397 #define FASTOP2R(op, name) \
398 	FOP_START(name) \
399 	FOP2E(op##b, dl, al) \
400 	FOP2E(op##w, dx, ax) \
401 	FOP2E(op##l, edx, eax) \
402 	ON64(FOP2E(op##q, rdx, rax)) \
403 	FOP_END
404 
405 #define FOP3E(op,  dst, src, src2) \
406 	FOP_FUNC(#op "_" #dst "_" #src "_" #src2) \
407 	#op " %" #src2 ", %" #src ", %" #dst " \n\t" FOP_RET
408 
409 /* 3-operand, word-only, src2=cl */
410 #define FASTOP3WCL(op) \
411 	FOP_START(op) \
412 	FOPNOP() \
413 	FOP3E(op##w, ax, dx, cl) \
414 	FOP3E(op##l, eax, edx, cl) \
415 	ON64(FOP3E(op##q, rax, rdx, cl)) \
416 	FOP_END
417 
418 /* Special case for SETcc - 1 instruction per cc */
419 #define FOP_SETCC(op) \
420 	".align 4 \n\t" \
421 	".type " #op ", @function \n\t" \
422 	#op ": \n\t" \
423 	#op " %al \n\t" \
424 	FOP_RET
425 
426 asm(".global kvm_fastop_exception \n"
427     "kvm_fastop_exception: xor %esi, %esi; ret");
428 
429 FOP_START(setcc)
430 FOP_SETCC(seto)
431 FOP_SETCC(setno)
432 FOP_SETCC(setc)
433 FOP_SETCC(setnc)
434 FOP_SETCC(setz)
435 FOP_SETCC(setnz)
436 FOP_SETCC(setbe)
437 FOP_SETCC(setnbe)
438 FOP_SETCC(sets)
439 FOP_SETCC(setns)
440 FOP_SETCC(setp)
441 FOP_SETCC(setnp)
442 FOP_SETCC(setl)
443 FOP_SETCC(setnl)
444 FOP_SETCC(setle)
445 FOP_SETCC(setnle)
446 FOP_END;
447 
448 FOP_START(salc) "pushf; sbb %al, %al; popf \n\t" FOP_RET
449 FOP_END;
450 
451 /*
452  * XXX: inoutclob user must know where the argument is being expanded.
453  *      Relying on CC_HAVE_ASM_GOTO would allow us to remove _fault.
454  */
455 #define asm_safe(insn, inoutclob...) \
456 ({ \
457 	int _fault = 0; \
458  \
459 	asm volatile("1:" insn "\n" \
460 	             "2:\n" \
461 	             ".pushsection .fixup, \"ax\"\n" \
462 	             "3: movl $1, %[_fault]\n" \
463 	             "   jmp  2b\n" \
464 	             ".popsection\n" \
465 	             _ASM_EXTABLE(1b, 3b) \
466 	             : [_fault] "+qm"(_fault) inoutclob ); \
467  \
468 	_fault ? X86EMUL_UNHANDLEABLE : X86EMUL_CONTINUE; \
469 })
470 
471 static int emulator_check_intercept(struct x86_emulate_ctxt *ctxt,
472 				    enum x86_intercept intercept,
473 				    enum x86_intercept_stage stage)
474 {
475 	struct x86_instruction_info info = {
476 		.intercept  = intercept,
477 		.rep_prefix = ctxt->rep_prefix,
478 		.modrm_mod  = ctxt->modrm_mod,
479 		.modrm_reg  = ctxt->modrm_reg,
480 		.modrm_rm   = ctxt->modrm_rm,
481 		.src_val    = ctxt->src.val64,
482 		.dst_val    = ctxt->dst.val64,
483 		.src_bytes  = ctxt->src.bytes,
484 		.dst_bytes  = ctxt->dst.bytes,
485 		.ad_bytes   = ctxt->ad_bytes,
486 		.next_rip   = ctxt->eip,
487 	};
488 
489 	return ctxt->ops->intercept(ctxt, &info, stage);
490 }
491 
492 static void assign_masked(ulong *dest, ulong src, ulong mask)
493 {
494 	*dest = (*dest & ~mask) | (src & mask);
495 }
496 
497 static void assign_register(unsigned long *reg, u64 val, int bytes)
498 {
499 	/* The 4-byte case *is* correct: in 64-bit mode we zero-extend. */
500 	switch (bytes) {
501 	case 1:
502 		*(u8 *)reg = (u8)val;
503 		break;
504 	case 2:
505 		*(u16 *)reg = (u16)val;
506 		break;
507 	case 4:
508 		*reg = (u32)val;
509 		break;	/* 64b: zero-extend */
510 	case 8:
511 		*reg = val;
512 		break;
513 	}
514 }
515 
516 static inline unsigned long ad_mask(struct x86_emulate_ctxt *ctxt)
517 {
518 	return (1UL << (ctxt->ad_bytes << 3)) - 1;
519 }
520 
521 static ulong stack_mask(struct x86_emulate_ctxt *ctxt)
522 {
523 	u16 sel;
524 	struct desc_struct ss;
525 
526 	if (ctxt->mode == X86EMUL_MODE_PROT64)
527 		return ~0UL;
528 	ctxt->ops->get_segment(ctxt, &sel, &ss, NULL, VCPU_SREG_SS);
529 	return ~0U >> ((ss.d ^ 1) * 16);  /* d=0: 0xffff; d=1: 0xffffffff */
530 }
531 
532 static int stack_size(struct x86_emulate_ctxt *ctxt)
533 {
534 	return (__fls(stack_mask(ctxt)) + 1) >> 3;
535 }
536 
537 /* Access/update address held in a register, based on addressing mode. */
538 static inline unsigned long
539 address_mask(struct x86_emulate_ctxt *ctxt, unsigned long reg)
540 {
541 	if (ctxt->ad_bytes == sizeof(unsigned long))
542 		return reg;
543 	else
544 		return reg & ad_mask(ctxt);
545 }
546 
547 static inline unsigned long
548 register_address(struct x86_emulate_ctxt *ctxt, int reg)
549 {
550 	return address_mask(ctxt, reg_read(ctxt, reg));
551 }
552 
553 static void masked_increment(ulong *reg, ulong mask, int inc)
554 {
555 	assign_masked(reg, *reg + inc, mask);
556 }
557 
558 static inline void
559 register_address_increment(struct x86_emulate_ctxt *ctxt, int reg, int inc)
560 {
561 	ulong *preg = reg_rmw(ctxt, reg);
562 
563 	assign_register(preg, *preg + inc, ctxt->ad_bytes);
564 }
565 
566 static void rsp_increment(struct x86_emulate_ctxt *ctxt, int inc)
567 {
568 	masked_increment(reg_rmw(ctxt, VCPU_REGS_RSP), stack_mask(ctxt), inc);
569 }
570 
571 static u32 desc_limit_scaled(struct desc_struct *desc)
572 {
573 	u32 limit = get_desc_limit(desc);
574 
575 	return desc->g ? (limit << 12) | 0xfff : limit;
576 }
577 
578 static unsigned long seg_base(struct x86_emulate_ctxt *ctxt, int seg)
579 {
580 	if (ctxt->mode == X86EMUL_MODE_PROT64 && seg < VCPU_SREG_FS)
581 		return 0;
582 
583 	return ctxt->ops->get_cached_segment_base(ctxt, seg);
584 }
585 
586 static int emulate_exception(struct x86_emulate_ctxt *ctxt, int vec,
587 			     u32 error, bool valid)
588 {
589 	WARN_ON(vec > 0x1f);
590 	ctxt->exception.vector = vec;
591 	ctxt->exception.error_code = error;
592 	ctxt->exception.error_code_valid = valid;
593 	return X86EMUL_PROPAGATE_FAULT;
594 }
595 
596 static int emulate_db(struct x86_emulate_ctxt *ctxt)
597 {
598 	return emulate_exception(ctxt, DB_VECTOR, 0, false);
599 }
600 
601 static int emulate_gp(struct x86_emulate_ctxt *ctxt, int err)
602 {
603 	return emulate_exception(ctxt, GP_VECTOR, err, true);
604 }
605 
606 static int emulate_ss(struct x86_emulate_ctxt *ctxt, int err)
607 {
608 	return emulate_exception(ctxt, SS_VECTOR, err, true);
609 }
610 
611 static int emulate_ud(struct x86_emulate_ctxt *ctxt)
612 {
613 	return emulate_exception(ctxt, UD_VECTOR, 0, false);
614 }
615 
616 static int emulate_ts(struct x86_emulate_ctxt *ctxt, int err)
617 {
618 	return emulate_exception(ctxt, TS_VECTOR, err, true);
619 }
620 
621 static int emulate_de(struct x86_emulate_ctxt *ctxt)
622 {
623 	return emulate_exception(ctxt, DE_VECTOR, 0, false);
624 }
625 
626 static int emulate_nm(struct x86_emulate_ctxt *ctxt)
627 {
628 	return emulate_exception(ctxt, NM_VECTOR, 0, false);
629 }
630 
631 static u16 get_segment_selector(struct x86_emulate_ctxt *ctxt, unsigned seg)
632 {
633 	u16 selector;
634 	struct desc_struct desc;
635 
636 	ctxt->ops->get_segment(ctxt, &selector, &desc, NULL, seg);
637 	return selector;
638 }
639 
640 static void set_segment_selector(struct x86_emulate_ctxt *ctxt, u16 selector,
641 				 unsigned seg)
642 {
643 	u16 dummy;
644 	u32 base3;
645 	struct desc_struct desc;
646 
647 	ctxt->ops->get_segment(ctxt, &dummy, &desc, &base3, seg);
648 	ctxt->ops->set_segment(ctxt, selector, &desc, base3, seg);
649 }
650 
651 /*
652  * x86 defines three classes of vector instructions: explicitly
653  * aligned, explicitly unaligned, and the rest, which change behaviour
654  * depending on whether they're AVX encoded or not.
655  *
656  * Also included is CMPXCHG16B which is not a vector instruction, yet it is
657  * subject to the same check.  FXSAVE and FXRSTOR are checked here too as their
658  * 512 bytes of data must be aligned to a 16 byte boundary.
659  */
660 static unsigned insn_alignment(struct x86_emulate_ctxt *ctxt, unsigned size)
661 {
662 	u64 alignment = ctxt->d & AlignMask;
663 
664 	if (likely(size < 16))
665 		return 1;
666 
667 	switch (alignment) {
668 	case Unaligned:
669 	case Avx:
670 		return 1;
671 	case Aligned16:
672 		return 16;
673 	case Aligned:
674 	default:
675 		return size;
676 	}
677 }
678 
679 static __always_inline int __linearize(struct x86_emulate_ctxt *ctxt,
680 				       struct segmented_address addr,
681 				       unsigned *max_size, unsigned size,
682 				       bool write, bool fetch,
683 				       enum x86emul_mode mode, ulong *linear)
684 {
685 	struct desc_struct desc;
686 	bool usable;
687 	ulong la;
688 	u32 lim;
689 	u16 sel;
690 
691 	la = seg_base(ctxt, addr.seg) + addr.ea;
692 	*max_size = 0;
693 	switch (mode) {
694 	case X86EMUL_MODE_PROT64:
695 		*linear = la;
696 		if (is_noncanonical_address(la))
697 			goto bad;
698 
699 		*max_size = min_t(u64, ~0u, (1ull << 48) - la);
700 		if (size > *max_size)
701 			goto bad;
702 		break;
703 	default:
704 		*linear = la = (u32)la;
705 		usable = ctxt->ops->get_segment(ctxt, &sel, &desc, NULL,
706 						addr.seg);
707 		if (!usable)
708 			goto bad;
709 		/* code segment in protected mode or read-only data segment */
710 		if ((((ctxt->mode != X86EMUL_MODE_REAL) && (desc.type & 8))
711 					|| !(desc.type & 2)) && write)
712 			goto bad;
713 		/* unreadable code segment */
714 		if (!fetch && (desc.type & 8) && !(desc.type & 2))
715 			goto bad;
716 		lim = desc_limit_scaled(&desc);
717 		if (!(desc.type & 8) && (desc.type & 4)) {
718 			/* expand-down segment */
719 			if (addr.ea <= lim)
720 				goto bad;
721 			lim = desc.d ? 0xffffffff : 0xffff;
722 		}
723 		if (addr.ea > lim)
724 			goto bad;
725 		if (lim == 0xffffffff)
726 			*max_size = ~0u;
727 		else {
728 			*max_size = (u64)lim + 1 - addr.ea;
729 			if (size > *max_size)
730 				goto bad;
731 		}
732 		break;
733 	}
734 	if (la & (insn_alignment(ctxt, size) - 1))
735 		return emulate_gp(ctxt, 0);
736 	return X86EMUL_CONTINUE;
737 bad:
738 	if (addr.seg == VCPU_SREG_SS)
739 		return emulate_ss(ctxt, 0);
740 	else
741 		return emulate_gp(ctxt, 0);
742 }
743 
744 static int linearize(struct x86_emulate_ctxt *ctxt,
745 		     struct segmented_address addr,
746 		     unsigned size, bool write,
747 		     ulong *linear)
748 {
749 	unsigned max_size;
750 	return __linearize(ctxt, addr, &max_size, size, write, false,
751 			   ctxt->mode, linear);
752 }
753 
754 static inline int assign_eip(struct x86_emulate_ctxt *ctxt, ulong dst,
755 			     enum x86emul_mode mode)
756 {
757 	ulong linear;
758 	int rc;
759 	unsigned max_size;
760 	struct segmented_address addr = { .seg = VCPU_SREG_CS,
761 					   .ea = dst };
762 
763 	if (ctxt->op_bytes != sizeof(unsigned long))
764 		addr.ea = dst & ((1UL << (ctxt->op_bytes << 3)) - 1);
765 	rc = __linearize(ctxt, addr, &max_size, 1, false, true, mode, &linear);
766 	if (rc == X86EMUL_CONTINUE)
767 		ctxt->_eip = addr.ea;
768 	return rc;
769 }
770 
771 static inline int assign_eip_near(struct x86_emulate_ctxt *ctxt, ulong dst)
772 {
773 	return assign_eip(ctxt, dst, ctxt->mode);
774 }
775 
776 static int assign_eip_far(struct x86_emulate_ctxt *ctxt, ulong dst,
777 			  const struct desc_struct *cs_desc)
778 {
779 	enum x86emul_mode mode = ctxt->mode;
780 	int rc;
781 
782 #ifdef CONFIG_X86_64
783 	if (ctxt->mode >= X86EMUL_MODE_PROT16) {
784 		if (cs_desc->l) {
785 			u64 efer = 0;
786 
787 			ctxt->ops->get_msr(ctxt, MSR_EFER, &efer);
788 			if (efer & EFER_LMA)
789 				mode = X86EMUL_MODE_PROT64;
790 		} else
791 			mode = X86EMUL_MODE_PROT32; /* temporary value */
792 	}
793 #endif
794 	if (mode == X86EMUL_MODE_PROT16 || mode == X86EMUL_MODE_PROT32)
795 		mode = cs_desc->d ? X86EMUL_MODE_PROT32 : X86EMUL_MODE_PROT16;
796 	rc = assign_eip(ctxt, dst, mode);
797 	if (rc == X86EMUL_CONTINUE)
798 		ctxt->mode = mode;
799 	return rc;
800 }
801 
802 static inline int jmp_rel(struct x86_emulate_ctxt *ctxt, int rel)
803 {
804 	return assign_eip_near(ctxt, ctxt->_eip + rel);
805 }
806 
807 static int segmented_read_std(struct x86_emulate_ctxt *ctxt,
808 			      struct segmented_address addr,
809 			      void *data,
810 			      unsigned size)
811 {
812 	int rc;
813 	ulong linear;
814 
815 	rc = linearize(ctxt, addr, size, false, &linear);
816 	if (rc != X86EMUL_CONTINUE)
817 		return rc;
818 	return ctxt->ops->read_std(ctxt, linear, data, size, &ctxt->exception);
819 }
820 
821 /*
822  * Prefetch the remaining bytes of the instruction without crossing page
823  * boundary if they are not in fetch_cache yet.
824  */
825 static int __do_insn_fetch_bytes(struct x86_emulate_ctxt *ctxt, int op_size)
826 {
827 	int rc;
828 	unsigned size, max_size;
829 	unsigned long linear;
830 	int cur_size = ctxt->fetch.end - ctxt->fetch.data;
831 	struct segmented_address addr = { .seg = VCPU_SREG_CS,
832 					   .ea = ctxt->eip + cur_size };
833 
834 	/*
835 	 * We do not know exactly how many bytes will be needed, and
836 	 * __linearize is expensive, so fetch as much as possible.  We
837 	 * just have to avoid going beyond the 15 byte limit, the end
838 	 * of the segment, or the end of the page.
839 	 *
840 	 * __linearize is called with size 0 so that it does not do any
841 	 * boundary check itself.  Instead, we use max_size to check
842 	 * against op_size.
843 	 */
844 	rc = __linearize(ctxt, addr, &max_size, 0, false, true, ctxt->mode,
845 			 &linear);
846 	if (unlikely(rc != X86EMUL_CONTINUE))
847 		return rc;
848 
849 	size = min_t(unsigned, 15UL ^ cur_size, max_size);
850 	size = min_t(unsigned, size, PAGE_SIZE - offset_in_page(linear));
851 
852 	/*
853 	 * One instruction can only straddle two pages,
854 	 * and one has been loaded at the beginning of
855 	 * x86_decode_insn.  So, if not enough bytes
856 	 * still, we must have hit the 15-byte boundary.
857 	 */
858 	if (unlikely(size < op_size))
859 		return emulate_gp(ctxt, 0);
860 
861 	rc = ctxt->ops->fetch(ctxt, linear, ctxt->fetch.end,
862 			      size, &ctxt->exception);
863 	if (unlikely(rc != X86EMUL_CONTINUE))
864 		return rc;
865 	ctxt->fetch.end += size;
866 	return X86EMUL_CONTINUE;
867 }
868 
869 static __always_inline int do_insn_fetch_bytes(struct x86_emulate_ctxt *ctxt,
870 					       unsigned size)
871 {
872 	unsigned done_size = ctxt->fetch.end - ctxt->fetch.ptr;
873 
874 	if (unlikely(done_size < size))
875 		return __do_insn_fetch_bytes(ctxt, size - done_size);
876 	else
877 		return X86EMUL_CONTINUE;
878 }
879 
880 /* Fetch next part of the instruction being emulated. */
881 #define insn_fetch(_type, _ctxt)					\
882 ({	_type _x;							\
883 									\
884 	rc = do_insn_fetch_bytes(_ctxt, sizeof(_type));			\
885 	if (rc != X86EMUL_CONTINUE)					\
886 		goto done;						\
887 	ctxt->_eip += sizeof(_type);					\
888 	_x = *(_type __aligned(1) *) ctxt->fetch.ptr;			\
889 	ctxt->fetch.ptr += sizeof(_type);				\
890 	_x;								\
891 })
892 
893 #define insn_fetch_arr(_arr, _size, _ctxt)				\
894 ({									\
895 	rc = do_insn_fetch_bytes(_ctxt, _size);				\
896 	if (rc != X86EMUL_CONTINUE)					\
897 		goto done;						\
898 	ctxt->_eip += (_size);						\
899 	memcpy(_arr, ctxt->fetch.ptr, _size);				\
900 	ctxt->fetch.ptr += (_size);					\
901 })
902 
903 /*
904  * Given the 'reg' portion of a ModRM byte, and a register block, return a
905  * pointer into the block that addresses the relevant register.
906  * @highbyte_regs specifies whether to decode AH,CH,DH,BH.
907  */
908 static void *decode_register(struct x86_emulate_ctxt *ctxt, u8 modrm_reg,
909 			     int byteop)
910 {
911 	void *p;
912 	int highbyte_regs = (ctxt->rex_prefix == 0) && byteop;
913 
914 	if (highbyte_regs && modrm_reg >= 4 && modrm_reg < 8)
915 		p = (unsigned char *)reg_rmw(ctxt, modrm_reg & 3) + 1;
916 	else
917 		p = reg_rmw(ctxt, modrm_reg);
918 	return p;
919 }
920 
921 static int read_descriptor(struct x86_emulate_ctxt *ctxt,
922 			   struct segmented_address addr,
923 			   u16 *size, unsigned long *address, int op_bytes)
924 {
925 	int rc;
926 
927 	if (op_bytes == 2)
928 		op_bytes = 3;
929 	*address = 0;
930 	rc = segmented_read_std(ctxt, addr, size, 2);
931 	if (rc != X86EMUL_CONTINUE)
932 		return rc;
933 	addr.ea += 2;
934 	rc = segmented_read_std(ctxt, addr, address, op_bytes);
935 	return rc;
936 }
937 
938 FASTOP2(add);
939 FASTOP2(or);
940 FASTOP2(adc);
941 FASTOP2(sbb);
942 FASTOP2(and);
943 FASTOP2(sub);
944 FASTOP2(xor);
945 FASTOP2(cmp);
946 FASTOP2(test);
947 
948 FASTOP1SRC2(mul, mul_ex);
949 FASTOP1SRC2(imul, imul_ex);
950 FASTOP1SRC2EX(div, div_ex);
951 FASTOP1SRC2EX(idiv, idiv_ex);
952 
953 FASTOP3WCL(shld);
954 FASTOP3WCL(shrd);
955 
956 FASTOP2W(imul);
957 
958 FASTOP1(not);
959 FASTOP1(neg);
960 FASTOP1(inc);
961 FASTOP1(dec);
962 
963 FASTOP2CL(rol);
964 FASTOP2CL(ror);
965 FASTOP2CL(rcl);
966 FASTOP2CL(rcr);
967 FASTOP2CL(shl);
968 FASTOP2CL(shr);
969 FASTOP2CL(sar);
970 
971 FASTOP2W(bsf);
972 FASTOP2W(bsr);
973 FASTOP2W(bt);
974 FASTOP2W(bts);
975 FASTOP2W(btr);
976 FASTOP2W(btc);
977 
978 FASTOP2(xadd);
979 
980 FASTOP2R(cmp, cmp_r);
981 
982 static int em_bsf_c(struct x86_emulate_ctxt *ctxt)
983 {
984 	/* If src is zero, do not writeback, but update flags */
985 	if (ctxt->src.val == 0)
986 		ctxt->dst.type = OP_NONE;
987 	return fastop(ctxt, em_bsf);
988 }
989 
990 static int em_bsr_c(struct x86_emulate_ctxt *ctxt)
991 {
992 	/* If src is zero, do not writeback, but update flags */
993 	if (ctxt->src.val == 0)
994 		ctxt->dst.type = OP_NONE;
995 	return fastop(ctxt, em_bsr);
996 }
997 
998 static __always_inline u8 test_cc(unsigned int condition, unsigned long flags)
999 {
1000 	u8 rc;
1001 	void (*fop)(void) = (void *)em_setcc + 4 * (condition & 0xf);
1002 
1003 	flags = (flags & EFLAGS_MASK) | X86_EFLAGS_IF;
1004 	asm("push %[flags]; popf; call *%[fastop]"
1005 	    : "=a"(rc) : [fastop]"r"(fop), [flags]"r"(flags));
1006 	return rc;
1007 }
1008 
1009 static void fetch_register_operand(struct operand *op)
1010 {
1011 	switch (op->bytes) {
1012 	case 1:
1013 		op->val = *(u8 *)op->addr.reg;
1014 		break;
1015 	case 2:
1016 		op->val = *(u16 *)op->addr.reg;
1017 		break;
1018 	case 4:
1019 		op->val = *(u32 *)op->addr.reg;
1020 		break;
1021 	case 8:
1022 		op->val = *(u64 *)op->addr.reg;
1023 		break;
1024 	}
1025 }
1026 
1027 static void read_sse_reg(struct x86_emulate_ctxt *ctxt, sse128_t *data, int reg)
1028 {
1029 	ctxt->ops->get_fpu(ctxt);
1030 	switch (reg) {
1031 	case 0: asm("movdqa %%xmm0, %0" : "=m"(*data)); break;
1032 	case 1: asm("movdqa %%xmm1, %0" : "=m"(*data)); break;
1033 	case 2: asm("movdqa %%xmm2, %0" : "=m"(*data)); break;
1034 	case 3: asm("movdqa %%xmm3, %0" : "=m"(*data)); break;
1035 	case 4: asm("movdqa %%xmm4, %0" : "=m"(*data)); break;
1036 	case 5: asm("movdqa %%xmm5, %0" : "=m"(*data)); break;
1037 	case 6: asm("movdqa %%xmm6, %0" : "=m"(*data)); break;
1038 	case 7: asm("movdqa %%xmm7, %0" : "=m"(*data)); break;
1039 #ifdef CONFIG_X86_64
1040 	case 8: asm("movdqa %%xmm8, %0" : "=m"(*data)); break;
1041 	case 9: asm("movdqa %%xmm9, %0" : "=m"(*data)); break;
1042 	case 10: asm("movdqa %%xmm10, %0" : "=m"(*data)); break;
1043 	case 11: asm("movdqa %%xmm11, %0" : "=m"(*data)); break;
1044 	case 12: asm("movdqa %%xmm12, %0" : "=m"(*data)); break;
1045 	case 13: asm("movdqa %%xmm13, %0" : "=m"(*data)); break;
1046 	case 14: asm("movdqa %%xmm14, %0" : "=m"(*data)); break;
1047 	case 15: asm("movdqa %%xmm15, %0" : "=m"(*data)); break;
1048 #endif
1049 	default: BUG();
1050 	}
1051 	ctxt->ops->put_fpu(ctxt);
1052 }
1053 
1054 static void write_sse_reg(struct x86_emulate_ctxt *ctxt, sse128_t *data,
1055 			  int reg)
1056 {
1057 	ctxt->ops->get_fpu(ctxt);
1058 	switch (reg) {
1059 	case 0: asm("movdqa %0, %%xmm0" : : "m"(*data)); break;
1060 	case 1: asm("movdqa %0, %%xmm1" : : "m"(*data)); break;
1061 	case 2: asm("movdqa %0, %%xmm2" : : "m"(*data)); break;
1062 	case 3: asm("movdqa %0, %%xmm3" : : "m"(*data)); break;
1063 	case 4: asm("movdqa %0, %%xmm4" : : "m"(*data)); break;
1064 	case 5: asm("movdqa %0, %%xmm5" : : "m"(*data)); break;
1065 	case 6: asm("movdqa %0, %%xmm6" : : "m"(*data)); break;
1066 	case 7: asm("movdqa %0, %%xmm7" : : "m"(*data)); break;
1067 #ifdef CONFIG_X86_64
1068 	case 8: asm("movdqa %0, %%xmm8" : : "m"(*data)); break;
1069 	case 9: asm("movdqa %0, %%xmm9" : : "m"(*data)); break;
1070 	case 10: asm("movdqa %0, %%xmm10" : : "m"(*data)); break;
1071 	case 11: asm("movdqa %0, %%xmm11" : : "m"(*data)); break;
1072 	case 12: asm("movdqa %0, %%xmm12" : : "m"(*data)); break;
1073 	case 13: asm("movdqa %0, %%xmm13" : : "m"(*data)); break;
1074 	case 14: asm("movdqa %0, %%xmm14" : : "m"(*data)); break;
1075 	case 15: asm("movdqa %0, %%xmm15" : : "m"(*data)); break;
1076 #endif
1077 	default: BUG();
1078 	}
1079 	ctxt->ops->put_fpu(ctxt);
1080 }
1081 
1082 static void read_mmx_reg(struct x86_emulate_ctxt *ctxt, u64 *data, int reg)
1083 {
1084 	ctxt->ops->get_fpu(ctxt);
1085 	switch (reg) {
1086 	case 0: asm("movq %%mm0, %0" : "=m"(*data)); break;
1087 	case 1: asm("movq %%mm1, %0" : "=m"(*data)); break;
1088 	case 2: asm("movq %%mm2, %0" : "=m"(*data)); break;
1089 	case 3: asm("movq %%mm3, %0" : "=m"(*data)); break;
1090 	case 4: asm("movq %%mm4, %0" : "=m"(*data)); break;
1091 	case 5: asm("movq %%mm5, %0" : "=m"(*data)); break;
1092 	case 6: asm("movq %%mm6, %0" : "=m"(*data)); break;
1093 	case 7: asm("movq %%mm7, %0" : "=m"(*data)); break;
1094 	default: BUG();
1095 	}
1096 	ctxt->ops->put_fpu(ctxt);
1097 }
1098 
1099 static void write_mmx_reg(struct x86_emulate_ctxt *ctxt, u64 *data, int reg)
1100 {
1101 	ctxt->ops->get_fpu(ctxt);
1102 	switch (reg) {
1103 	case 0: asm("movq %0, %%mm0" : : "m"(*data)); break;
1104 	case 1: asm("movq %0, %%mm1" : : "m"(*data)); break;
1105 	case 2: asm("movq %0, %%mm2" : : "m"(*data)); break;
1106 	case 3: asm("movq %0, %%mm3" : : "m"(*data)); break;
1107 	case 4: asm("movq %0, %%mm4" : : "m"(*data)); break;
1108 	case 5: asm("movq %0, %%mm5" : : "m"(*data)); break;
1109 	case 6: asm("movq %0, %%mm6" : : "m"(*data)); break;
1110 	case 7: asm("movq %0, %%mm7" : : "m"(*data)); break;
1111 	default: BUG();
1112 	}
1113 	ctxt->ops->put_fpu(ctxt);
1114 }
1115 
1116 static int em_fninit(struct x86_emulate_ctxt *ctxt)
1117 {
1118 	if (ctxt->ops->get_cr(ctxt, 0) & (X86_CR0_TS | X86_CR0_EM))
1119 		return emulate_nm(ctxt);
1120 
1121 	ctxt->ops->get_fpu(ctxt);
1122 	asm volatile("fninit");
1123 	ctxt->ops->put_fpu(ctxt);
1124 	return X86EMUL_CONTINUE;
1125 }
1126 
1127 static int em_fnstcw(struct x86_emulate_ctxt *ctxt)
1128 {
1129 	u16 fcw;
1130 
1131 	if (ctxt->ops->get_cr(ctxt, 0) & (X86_CR0_TS | X86_CR0_EM))
1132 		return emulate_nm(ctxt);
1133 
1134 	ctxt->ops->get_fpu(ctxt);
1135 	asm volatile("fnstcw %0": "+m"(fcw));
1136 	ctxt->ops->put_fpu(ctxt);
1137 
1138 	ctxt->dst.val = fcw;
1139 
1140 	return X86EMUL_CONTINUE;
1141 }
1142 
1143 static int em_fnstsw(struct x86_emulate_ctxt *ctxt)
1144 {
1145 	u16 fsw;
1146 
1147 	if (ctxt->ops->get_cr(ctxt, 0) & (X86_CR0_TS | X86_CR0_EM))
1148 		return emulate_nm(ctxt);
1149 
1150 	ctxt->ops->get_fpu(ctxt);
1151 	asm volatile("fnstsw %0": "+m"(fsw));
1152 	ctxt->ops->put_fpu(ctxt);
1153 
1154 	ctxt->dst.val = fsw;
1155 
1156 	return X86EMUL_CONTINUE;
1157 }
1158 
1159 static void decode_register_operand(struct x86_emulate_ctxt *ctxt,
1160 				    struct operand *op)
1161 {
1162 	unsigned reg = ctxt->modrm_reg;
1163 
1164 	if (!(ctxt->d & ModRM))
1165 		reg = (ctxt->b & 7) | ((ctxt->rex_prefix & 1) << 3);
1166 
1167 	if (ctxt->d & Sse) {
1168 		op->type = OP_XMM;
1169 		op->bytes = 16;
1170 		op->addr.xmm = reg;
1171 		read_sse_reg(ctxt, &op->vec_val, reg);
1172 		return;
1173 	}
1174 	if (ctxt->d & Mmx) {
1175 		reg &= 7;
1176 		op->type = OP_MM;
1177 		op->bytes = 8;
1178 		op->addr.mm = reg;
1179 		return;
1180 	}
1181 
1182 	op->type = OP_REG;
1183 	op->bytes = (ctxt->d & ByteOp) ? 1 : ctxt->op_bytes;
1184 	op->addr.reg = decode_register(ctxt, reg, ctxt->d & ByteOp);
1185 
1186 	fetch_register_operand(op);
1187 	op->orig_val = op->val;
1188 }
1189 
1190 static void adjust_modrm_seg(struct x86_emulate_ctxt *ctxt, int base_reg)
1191 {
1192 	if (base_reg == VCPU_REGS_RSP || base_reg == VCPU_REGS_RBP)
1193 		ctxt->modrm_seg = VCPU_SREG_SS;
1194 }
1195 
1196 static int decode_modrm(struct x86_emulate_ctxt *ctxt,
1197 			struct operand *op)
1198 {
1199 	u8 sib;
1200 	int index_reg, base_reg, scale;
1201 	int rc = X86EMUL_CONTINUE;
1202 	ulong modrm_ea = 0;
1203 
1204 	ctxt->modrm_reg = ((ctxt->rex_prefix << 1) & 8); /* REX.R */
1205 	index_reg = (ctxt->rex_prefix << 2) & 8; /* REX.X */
1206 	base_reg = (ctxt->rex_prefix << 3) & 8; /* REX.B */
1207 
1208 	ctxt->modrm_mod = (ctxt->modrm & 0xc0) >> 6;
1209 	ctxt->modrm_reg |= (ctxt->modrm & 0x38) >> 3;
1210 	ctxt->modrm_rm = base_reg | (ctxt->modrm & 0x07);
1211 	ctxt->modrm_seg = VCPU_SREG_DS;
1212 
1213 	if (ctxt->modrm_mod == 3 || (ctxt->d & NoMod)) {
1214 		op->type = OP_REG;
1215 		op->bytes = (ctxt->d & ByteOp) ? 1 : ctxt->op_bytes;
1216 		op->addr.reg = decode_register(ctxt, ctxt->modrm_rm,
1217 				ctxt->d & ByteOp);
1218 		if (ctxt->d & Sse) {
1219 			op->type = OP_XMM;
1220 			op->bytes = 16;
1221 			op->addr.xmm = ctxt->modrm_rm;
1222 			read_sse_reg(ctxt, &op->vec_val, ctxt->modrm_rm);
1223 			return rc;
1224 		}
1225 		if (ctxt->d & Mmx) {
1226 			op->type = OP_MM;
1227 			op->bytes = 8;
1228 			op->addr.mm = ctxt->modrm_rm & 7;
1229 			return rc;
1230 		}
1231 		fetch_register_operand(op);
1232 		return rc;
1233 	}
1234 
1235 	op->type = OP_MEM;
1236 
1237 	if (ctxt->ad_bytes == 2) {
1238 		unsigned bx = reg_read(ctxt, VCPU_REGS_RBX);
1239 		unsigned bp = reg_read(ctxt, VCPU_REGS_RBP);
1240 		unsigned si = reg_read(ctxt, VCPU_REGS_RSI);
1241 		unsigned di = reg_read(ctxt, VCPU_REGS_RDI);
1242 
1243 		/* 16-bit ModR/M decode. */
1244 		switch (ctxt->modrm_mod) {
1245 		case 0:
1246 			if (ctxt->modrm_rm == 6)
1247 				modrm_ea += insn_fetch(u16, ctxt);
1248 			break;
1249 		case 1:
1250 			modrm_ea += insn_fetch(s8, ctxt);
1251 			break;
1252 		case 2:
1253 			modrm_ea += insn_fetch(u16, ctxt);
1254 			break;
1255 		}
1256 		switch (ctxt->modrm_rm) {
1257 		case 0:
1258 			modrm_ea += bx + si;
1259 			break;
1260 		case 1:
1261 			modrm_ea += bx + di;
1262 			break;
1263 		case 2:
1264 			modrm_ea += bp + si;
1265 			break;
1266 		case 3:
1267 			modrm_ea += bp + di;
1268 			break;
1269 		case 4:
1270 			modrm_ea += si;
1271 			break;
1272 		case 5:
1273 			modrm_ea += di;
1274 			break;
1275 		case 6:
1276 			if (ctxt->modrm_mod != 0)
1277 				modrm_ea += bp;
1278 			break;
1279 		case 7:
1280 			modrm_ea += bx;
1281 			break;
1282 		}
1283 		if (ctxt->modrm_rm == 2 || ctxt->modrm_rm == 3 ||
1284 		    (ctxt->modrm_rm == 6 && ctxt->modrm_mod != 0))
1285 			ctxt->modrm_seg = VCPU_SREG_SS;
1286 		modrm_ea = (u16)modrm_ea;
1287 	} else {
1288 		/* 32/64-bit ModR/M decode. */
1289 		if ((ctxt->modrm_rm & 7) == 4) {
1290 			sib = insn_fetch(u8, ctxt);
1291 			index_reg |= (sib >> 3) & 7;
1292 			base_reg |= sib & 7;
1293 			scale = sib >> 6;
1294 
1295 			if ((base_reg & 7) == 5 && ctxt->modrm_mod == 0)
1296 				modrm_ea += insn_fetch(s32, ctxt);
1297 			else {
1298 				modrm_ea += reg_read(ctxt, base_reg);
1299 				adjust_modrm_seg(ctxt, base_reg);
1300 				/* Increment ESP on POP [ESP] */
1301 				if ((ctxt->d & IncSP) &&
1302 				    base_reg == VCPU_REGS_RSP)
1303 					modrm_ea += ctxt->op_bytes;
1304 			}
1305 			if (index_reg != 4)
1306 				modrm_ea += reg_read(ctxt, index_reg) << scale;
1307 		} else if ((ctxt->modrm_rm & 7) == 5 && ctxt->modrm_mod == 0) {
1308 			modrm_ea += insn_fetch(s32, ctxt);
1309 			if (ctxt->mode == X86EMUL_MODE_PROT64)
1310 				ctxt->rip_relative = 1;
1311 		} else {
1312 			base_reg = ctxt->modrm_rm;
1313 			modrm_ea += reg_read(ctxt, base_reg);
1314 			adjust_modrm_seg(ctxt, base_reg);
1315 		}
1316 		switch (ctxt->modrm_mod) {
1317 		case 1:
1318 			modrm_ea += insn_fetch(s8, ctxt);
1319 			break;
1320 		case 2:
1321 			modrm_ea += insn_fetch(s32, ctxt);
1322 			break;
1323 		}
1324 	}
1325 	op->addr.mem.ea = modrm_ea;
1326 	if (ctxt->ad_bytes != 8)
1327 		ctxt->memop.addr.mem.ea = (u32)ctxt->memop.addr.mem.ea;
1328 
1329 done:
1330 	return rc;
1331 }
1332 
1333 static int decode_abs(struct x86_emulate_ctxt *ctxt,
1334 		      struct operand *op)
1335 {
1336 	int rc = X86EMUL_CONTINUE;
1337 
1338 	op->type = OP_MEM;
1339 	switch (ctxt->ad_bytes) {
1340 	case 2:
1341 		op->addr.mem.ea = insn_fetch(u16, ctxt);
1342 		break;
1343 	case 4:
1344 		op->addr.mem.ea = insn_fetch(u32, ctxt);
1345 		break;
1346 	case 8:
1347 		op->addr.mem.ea = insn_fetch(u64, ctxt);
1348 		break;
1349 	}
1350 done:
1351 	return rc;
1352 }
1353 
1354 static void fetch_bit_operand(struct x86_emulate_ctxt *ctxt)
1355 {
1356 	long sv = 0, mask;
1357 
1358 	if (ctxt->dst.type == OP_MEM && ctxt->src.type == OP_REG) {
1359 		mask = ~((long)ctxt->dst.bytes * 8 - 1);
1360 
1361 		if (ctxt->src.bytes == 2)
1362 			sv = (s16)ctxt->src.val & (s16)mask;
1363 		else if (ctxt->src.bytes == 4)
1364 			sv = (s32)ctxt->src.val & (s32)mask;
1365 		else
1366 			sv = (s64)ctxt->src.val & (s64)mask;
1367 
1368 		ctxt->dst.addr.mem.ea = address_mask(ctxt,
1369 					   ctxt->dst.addr.mem.ea + (sv >> 3));
1370 	}
1371 
1372 	/* only subword offset */
1373 	ctxt->src.val &= (ctxt->dst.bytes << 3) - 1;
1374 }
1375 
1376 static int read_emulated(struct x86_emulate_ctxt *ctxt,
1377 			 unsigned long addr, void *dest, unsigned size)
1378 {
1379 	int rc;
1380 	struct read_cache *mc = &ctxt->mem_read;
1381 
1382 	if (mc->pos < mc->end)
1383 		goto read_cached;
1384 
1385 	WARN_ON((mc->end + size) >= sizeof(mc->data));
1386 
1387 	rc = ctxt->ops->read_emulated(ctxt, addr, mc->data + mc->end, size,
1388 				      &ctxt->exception);
1389 	if (rc != X86EMUL_CONTINUE)
1390 		return rc;
1391 
1392 	mc->end += size;
1393 
1394 read_cached:
1395 	memcpy(dest, mc->data + mc->pos, size);
1396 	mc->pos += size;
1397 	return X86EMUL_CONTINUE;
1398 }
1399 
1400 static int segmented_read(struct x86_emulate_ctxt *ctxt,
1401 			  struct segmented_address addr,
1402 			  void *data,
1403 			  unsigned size)
1404 {
1405 	int rc;
1406 	ulong linear;
1407 
1408 	rc = linearize(ctxt, addr, size, false, &linear);
1409 	if (rc != X86EMUL_CONTINUE)
1410 		return rc;
1411 	return read_emulated(ctxt, linear, data, size);
1412 }
1413 
1414 static int segmented_write(struct x86_emulate_ctxt *ctxt,
1415 			   struct segmented_address addr,
1416 			   const void *data,
1417 			   unsigned size)
1418 {
1419 	int rc;
1420 	ulong linear;
1421 
1422 	rc = linearize(ctxt, addr, size, true, &linear);
1423 	if (rc != X86EMUL_CONTINUE)
1424 		return rc;
1425 	return ctxt->ops->write_emulated(ctxt, linear, data, size,
1426 					 &ctxt->exception);
1427 }
1428 
1429 static int segmented_cmpxchg(struct x86_emulate_ctxt *ctxt,
1430 			     struct segmented_address addr,
1431 			     const void *orig_data, const void *data,
1432 			     unsigned size)
1433 {
1434 	int rc;
1435 	ulong linear;
1436 
1437 	rc = linearize(ctxt, addr, size, true, &linear);
1438 	if (rc != X86EMUL_CONTINUE)
1439 		return rc;
1440 	return ctxt->ops->cmpxchg_emulated(ctxt, linear, orig_data, data,
1441 					   size, &ctxt->exception);
1442 }
1443 
1444 static int pio_in_emulated(struct x86_emulate_ctxt *ctxt,
1445 			   unsigned int size, unsigned short port,
1446 			   void *dest)
1447 {
1448 	struct read_cache *rc = &ctxt->io_read;
1449 
1450 	if (rc->pos == rc->end) { /* refill pio read ahead */
1451 		unsigned int in_page, n;
1452 		unsigned int count = ctxt->rep_prefix ?
1453 			address_mask(ctxt, reg_read(ctxt, VCPU_REGS_RCX)) : 1;
1454 		in_page = (ctxt->eflags & X86_EFLAGS_DF) ?
1455 			offset_in_page(reg_read(ctxt, VCPU_REGS_RDI)) :
1456 			PAGE_SIZE - offset_in_page(reg_read(ctxt, VCPU_REGS_RDI));
1457 		n = min3(in_page, (unsigned int)sizeof(rc->data) / size, count);
1458 		if (n == 0)
1459 			n = 1;
1460 		rc->pos = rc->end = 0;
1461 		if (!ctxt->ops->pio_in_emulated(ctxt, size, port, rc->data, n))
1462 			return 0;
1463 		rc->end = n * size;
1464 	}
1465 
1466 	if (ctxt->rep_prefix && (ctxt->d & String) &&
1467 	    !(ctxt->eflags & X86_EFLAGS_DF)) {
1468 		ctxt->dst.data = rc->data + rc->pos;
1469 		ctxt->dst.type = OP_MEM_STR;
1470 		ctxt->dst.count = (rc->end - rc->pos) / size;
1471 		rc->pos = rc->end;
1472 	} else {
1473 		memcpy(dest, rc->data + rc->pos, size);
1474 		rc->pos += size;
1475 	}
1476 	return 1;
1477 }
1478 
1479 static int read_interrupt_descriptor(struct x86_emulate_ctxt *ctxt,
1480 				     u16 index, struct desc_struct *desc)
1481 {
1482 	struct desc_ptr dt;
1483 	ulong addr;
1484 
1485 	ctxt->ops->get_idt(ctxt, &dt);
1486 
1487 	if (dt.size < index * 8 + 7)
1488 		return emulate_gp(ctxt, index << 3 | 0x2);
1489 
1490 	addr = dt.address + index * 8;
1491 	return ctxt->ops->read_std(ctxt, addr, desc, sizeof *desc,
1492 				   &ctxt->exception);
1493 }
1494 
1495 static void get_descriptor_table_ptr(struct x86_emulate_ctxt *ctxt,
1496 				     u16 selector, struct desc_ptr *dt)
1497 {
1498 	const struct x86_emulate_ops *ops = ctxt->ops;
1499 	u32 base3 = 0;
1500 
1501 	if (selector & 1 << 2) {
1502 		struct desc_struct desc;
1503 		u16 sel;
1504 
1505 		memset (dt, 0, sizeof *dt);
1506 		if (!ops->get_segment(ctxt, &sel, &desc, &base3,
1507 				      VCPU_SREG_LDTR))
1508 			return;
1509 
1510 		dt->size = desc_limit_scaled(&desc); /* what if limit > 65535? */
1511 		dt->address = get_desc_base(&desc) | ((u64)base3 << 32);
1512 	} else
1513 		ops->get_gdt(ctxt, dt);
1514 }
1515 
1516 static int get_descriptor_ptr(struct x86_emulate_ctxt *ctxt,
1517 			      u16 selector, ulong *desc_addr_p)
1518 {
1519 	struct desc_ptr dt;
1520 	u16 index = selector >> 3;
1521 	ulong addr;
1522 
1523 	get_descriptor_table_ptr(ctxt, selector, &dt);
1524 
1525 	if (dt.size < index * 8 + 7)
1526 		return emulate_gp(ctxt, selector & 0xfffc);
1527 
1528 	addr = dt.address + index * 8;
1529 
1530 #ifdef CONFIG_X86_64
1531 	if (addr >> 32 != 0) {
1532 		u64 efer = 0;
1533 
1534 		ctxt->ops->get_msr(ctxt, MSR_EFER, &efer);
1535 		if (!(efer & EFER_LMA))
1536 			addr &= (u32)-1;
1537 	}
1538 #endif
1539 
1540 	*desc_addr_p = addr;
1541 	return X86EMUL_CONTINUE;
1542 }
1543 
1544 /* allowed just for 8 bytes segments */
1545 static int read_segment_descriptor(struct x86_emulate_ctxt *ctxt,
1546 				   u16 selector, struct desc_struct *desc,
1547 				   ulong *desc_addr_p)
1548 {
1549 	int rc;
1550 
1551 	rc = get_descriptor_ptr(ctxt, selector, desc_addr_p);
1552 	if (rc != X86EMUL_CONTINUE)
1553 		return rc;
1554 
1555 	return ctxt->ops->read_std(ctxt, *desc_addr_p, desc, sizeof(*desc),
1556 				   &ctxt->exception);
1557 }
1558 
1559 /* allowed just for 8 bytes segments */
1560 static int write_segment_descriptor(struct x86_emulate_ctxt *ctxt,
1561 				    u16 selector, struct desc_struct *desc)
1562 {
1563 	int rc;
1564 	ulong addr;
1565 
1566 	rc = get_descriptor_ptr(ctxt, selector, &addr);
1567 	if (rc != X86EMUL_CONTINUE)
1568 		return rc;
1569 
1570 	return ctxt->ops->write_std(ctxt, addr, desc, sizeof *desc,
1571 				    &ctxt->exception);
1572 }
1573 
1574 /* Does not support long mode */
1575 static int __load_segment_descriptor(struct x86_emulate_ctxt *ctxt,
1576 				     u16 selector, int seg, u8 cpl,
1577 				     enum x86_transfer_type transfer,
1578 				     struct desc_struct *desc)
1579 {
1580 	struct desc_struct seg_desc, old_desc;
1581 	u8 dpl, rpl;
1582 	unsigned err_vec = GP_VECTOR;
1583 	u32 err_code = 0;
1584 	bool null_selector = !(selector & ~0x3); /* 0000-0003 are null */
1585 	ulong desc_addr;
1586 	int ret;
1587 	u16 dummy;
1588 	u32 base3 = 0;
1589 
1590 	memset(&seg_desc, 0, sizeof seg_desc);
1591 
1592 	if (ctxt->mode == X86EMUL_MODE_REAL) {
1593 		/* set real mode segment descriptor (keep limit etc. for
1594 		 * unreal mode) */
1595 		ctxt->ops->get_segment(ctxt, &dummy, &seg_desc, NULL, seg);
1596 		set_desc_base(&seg_desc, selector << 4);
1597 		goto load;
1598 	} else if (seg <= VCPU_SREG_GS && ctxt->mode == X86EMUL_MODE_VM86) {
1599 		/* VM86 needs a clean new segment descriptor */
1600 		set_desc_base(&seg_desc, selector << 4);
1601 		set_desc_limit(&seg_desc, 0xffff);
1602 		seg_desc.type = 3;
1603 		seg_desc.p = 1;
1604 		seg_desc.s = 1;
1605 		seg_desc.dpl = 3;
1606 		goto load;
1607 	}
1608 
1609 	rpl = selector & 3;
1610 
1611 	/* NULL selector is not valid for TR, CS and SS (except for long mode) */
1612 	if ((seg == VCPU_SREG_CS
1613 	     || (seg == VCPU_SREG_SS
1614 		 && (ctxt->mode != X86EMUL_MODE_PROT64 || rpl != cpl))
1615 	     || seg == VCPU_SREG_TR)
1616 	    && null_selector)
1617 		goto exception;
1618 
1619 	/* TR should be in GDT only */
1620 	if (seg == VCPU_SREG_TR && (selector & (1 << 2)))
1621 		goto exception;
1622 
1623 	if (null_selector) /* for NULL selector skip all following checks */
1624 		goto load;
1625 
1626 	ret = read_segment_descriptor(ctxt, selector, &seg_desc, &desc_addr);
1627 	if (ret != X86EMUL_CONTINUE)
1628 		return ret;
1629 
1630 	err_code = selector & 0xfffc;
1631 	err_vec = (transfer == X86_TRANSFER_TASK_SWITCH) ? TS_VECTOR :
1632 							   GP_VECTOR;
1633 
1634 	/* can't load system descriptor into segment selector */
1635 	if (seg <= VCPU_SREG_GS && !seg_desc.s) {
1636 		if (transfer == X86_TRANSFER_CALL_JMP)
1637 			return X86EMUL_UNHANDLEABLE;
1638 		goto exception;
1639 	}
1640 
1641 	if (!seg_desc.p) {
1642 		err_vec = (seg == VCPU_SREG_SS) ? SS_VECTOR : NP_VECTOR;
1643 		goto exception;
1644 	}
1645 
1646 	dpl = seg_desc.dpl;
1647 
1648 	switch (seg) {
1649 	case VCPU_SREG_SS:
1650 		/*
1651 		 * segment is not a writable data segment or segment
1652 		 * selector's RPL != CPL or segment selector's RPL != CPL
1653 		 */
1654 		if (rpl != cpl || (seg_desc.type & 0xa) != 0x2 || dpl != cpl)
1655 			goto exception;
1656 		break;
1657 	case VCPU_SREG_CS:
1658 		if (!(seg_desc.type & 8))
1659 			goto exception;
1660 
1661 		if (seg_desc.type & 4) {
1662 			/* conforming */
1663 			if (dpl > cpl)
1664 				goto exception;
1665 		} else {
1666 			/* nonconforming */
1667 			if (rpl > cpl || dpl != cpl)
1668 				goto exception;
1669 		}
1670 		/* in long-mode d/b must be clear if l is set */
1671 		if (seg_desc.d && seg_desc.l) {
1672 			u64 efer = 0;
1673 
1674 			ctxt->ops->get_msr(ctxt, MSR_EFER, &efer);
1675 			if (efer & EFER_LMA)
1676 				goto exception;
1677 		}
1678 
1679 		/* CS(RPL) <- CPL */
1680 		selector = (selector & 0xfffc) | cpl;
1681 		break;
1682 	case VCPU_SREG_TR:
1683 		if (seg_desc.s || (seg_desc.type != 1 && seg_desc.type != 9))
1684 			goto exception;
1685 		old_desc = seg_desc;
1686 		seg_desc.type |= 2; /* busy */
1687 		ret = ctxt->ops->cmpxchg_emulated(ctxt, desc_addr, &old_desc, &seg_desc,
1688 						  sizeof(seg_desc), &ctxt->exception);
1689 		if (ret != X86EMUL_CONTINUE)
1690 			return ret;
1691 		break;
1692 	case VCPU_SREG_LDTR:
1693 		if (seg_desc.s || seg_desc.type != 2)
1694 			goto exception;
1695 		break;
1696 	default: /*  DS, ES, FS, or GS */
1697 		/*
1698 		 * segment is not a data or readable code segment or
1699 		 * ((segment is a data or nonconforming code segment)
1700 		 * and (both RPL and CPL > DPL))
1701 		 */
1702 		if ((seg_desc.type & 0xa) == 0x8 ||
1703 		    (((seg_desc.type & 0xc) != 0xc) &&
1704 		     (rpl > dpl && cpl > dpl)))
1705 			goto exception;
1706 		break;
1707 	}
1708 
1709 	if (seg_desc.s) {
1710 		/* mark segment as accessed */
1711 		if (!(seg_desc.type & 1)) {
1712 			seg_desc.type |= 1;
1713 			ret = write_segment_descriptor(ctxt, selector,
1714 						       &seg_desc);
1715 			if (ret != X86EMUL_CONTINUE)
1716 				return ret;
1717 		}
1718 	} else if (ctxt->mode == X86EMUL_MODE_PROT64) {
1719 		ret = ctxt->ops->read_std(ctxt, desc_addr+8, &base3,
1720 				sizeof(base3), &ctxt->exception);
1721 		if (ret != X86EMUL_CONTINUE)
1722 			return ret;
1723 		if (is_noncanonical_address(get_desc_base(&seg_desc) |
1724 					     ((u64)base3 << 32)))
1725 			return emulate_gp(ctxt, 0);
1726 	}
1727 load:
1728 	ctxt->ops->set_segment(ctxt, selector, &seg_desc, base3, seg);
1729 	if (desc)
1730 		*desc = seg_desc;
1731 	return X86EMUL_CONTINUE;
1732 exception:
1733 	return emulate_exception(ctxt, err_vec, err_code, true);
1734 }
1735 
1736 static int load_segment_descriptor(struct x86_emulate_ctxt *ctxt,
1737 				   u16 selector, int seg)
1738 {
1739 	u8 cpl = ctxt->ops->cpl(ctxt);
1740 	return __load_segment_descriptor(ctxt, selector, seg, cpl,
1741 					 X86_TRANSFER_NONE, NULL);
1742 }
1743 
1744 static void write_register_operand(struct operand *op)
1745 {
1746 	return assign_register(op->addr.reg, op->val, op->bytes);
1747 }
1748 
1749 static int writeback(struct x86_emulate_ctxt *ctxt, struct operand *op)
1750 {
1751 	switch (op->type) {
1752 	case OP_REG:
1753 		write_register_operand(op);
1754 		break;
1755 	case OP_MEM:
1756 		if (ctxt->lock_prefix)
1757 			return segmented_cmpxchg(ctxt,
1758 						 op->addr.mem,
1759 						 &op->orig_val,
1760 						 &op->val,
1761 						 op->bytes);
1762 		else
1763 			return segmented_write(ctxt,
1764 					       op->addr.mem,
1765 					       &op->val,
1766 					       op->bytes);
1767 		break;
1768 	case OP_MEM_STR:
1769 		return segmented_write(ctxt,
1770 				       op->addr.mem,
1771 				       op->data,
1772 				       op->bytes * op->count);
1773 		break;
1774 	case OP_XMM:
1775 		write_sse_reg(ctxt, &op->vec_val, op->addr.xmm);
1776 		break;
1777 	case OP_MM:
1778 		write_mmx_reg(ctxt, &op->mm_val, op->addr.mm);
1779 		break;
1780 	case OP_NONE:
1781 		/* no writeback */
1782 		break;
1783 	default:
1784 		break;
1785 	}
1786 	return X86EMUL_CONTINUE;
1787 }
1788 
1789 static int push(struct x86_emulate_ctxt *ctxt, void *data, int bytes)
1790 {
1791 	struct segmented_address addr;
1792 
1793 	rsp_increment(ctxt, -bytes);
1794 	addr.ea = reg_read(ctxt, VCPU_REGS_RSP) & stack_mask(ctxt);
1795 	addr.seg = VCPU_SREG_SS;
1796 
1797 	return segmented_write(ctxt, addr, data, bytes);
1798 }
1799 
1800 static int em_push(struct x86_emulate_ctxt *ctxt)
1801 {
1802 	/* Disable writeback. */
1803 	ctxt->dst.type = OP_NONE;
1804 	return push(ctxt, &ctxt->src.val, ctxt->op_bytes);
1805 }
1806 
1807 static int emulate_pop(struct x86_emulate_ctxt *ctxt,
1808 		       void *dest, int len)
1809 {
1810 	int rc;
1811 	struct segmented_address addr;
1812 
1813 	addr.ea = reg_read(ctxt, VCPU_REGS_RSP) & stack_mask(ctxt);
1814 	addr.seg = VCPU_SREG_SS;
1815 	rc = segmented_read(ctxt, addr, dest, len);
1816 	if (rc != X86EMUL_CONTINUE)
1817 		return rc;
1818 
1819 	rsp_increment(ctxt, len);
1820 	return rc;
1821 }
1822 
1823 static int em_pop(struct x86_emulate_ctxt *ctxt)
1824 {
1825 	return emulate_pop(ctxt, &ctxt->dst.val, ctxt->op_bytes);
1826 }
1827 
1828 static int emulate_popf(struct x86_emulate_ctxt *ctxt,
1829 			void *dest, int len)
1830 {
1831 	int rc;
1832 	unsigned long val, change_mask;
1833 	int iopl = (ctxt->eflags & X86_EFLAGS_IOPL) >> X86_EFLAGS_IOPL_BIT;
1834 	int cpl = ctxt->ops->cpl(ctxt);
1835 
1836 	rc = emulate_pop(ctxt, &val, len);
1837 	if (rc != X86EMUL_CONTINUE)
1838 		return rc;
1839 
1840 	change_mask = X86_EFLAGS_CF | X86_EFLAGS_PF | X86_EFLAGS_AF |
1841 		      X86_EFLAGS_ZF | X86_EFLAGS_SF | X86_EFLAGS_OF |
1842 		      X86_EFLAGS_TF | X86_EFLAGS_DF | X86_EFLAGS_NT |
1843 		      X86_EFLAGS_AC | X86_EFLAGS_ID;
1844 
1845 	switch(ctxt->mode) {
1846 	case X86EMUL_MODE_PROT64:
1847 	case X86EMUL_MODE_PROT32:
1848 	case X86EMUL_MODE_PROT16:
1849 		if (cpl == 0)
1850 			change_mask |= X86_EFLAGS_IOPL;
1851 		if (cpl <= iopl)
1852 			change_mask |= X86_EFLAGS_IF;
1853 		break;
1854 	case X86EMUL_MODE_VM86:
1855 		if (iopl < 3)
1856 			return emulate_gp(ctxt, 0);
1857 		change_mask |= X86_EFLAGS_IF;
1858 		break;
1859 	default: /* real mode */
1860 		change_mask |= (X86_EFLAGS_IOPL | X86_EFLAGS_IF);
1861 		break;
1862 	}
1863 
1864 	*(unsigned long *)dest =
1865 		(ctxt->eflags & ~change_mask) | (val & change_mask);
1866 
1867 	return rc;
1868 }
1869 
1870 static int em_popf(struct x86_emulate_ctxt *ctxt)
1871 {
1872 	ctxt->dst.type = OP_REG;
1873 	ctxt->dst.addr.reg = &ctxt->eflags;
1874 	ctxt->dst.bytes = ctxt->op_bytes;
1875 	return emulate_popf(ctxt, &ctxt->dst.val, ctxt->op_bytes);
1876 }
1877 
1878 static int em_enter(struct x86_emulate_ctxt *ctxt)
1879 {
1880 	int rc;
1881 	unsigned frame_size = ctxt->src.val;
1882 	unsigned nesting_level = ctxt->src2.val & 31;
1883 	ulong rbp;
1884 
1885 	if (nesting_level)
1886 		return X86EMUL_UNHANDLEABLE;
1887 
1888 	rbp = reg_read(ctxt, VCPU_REGS_RBP);
1889 	rc = push(ctxt, &rbp, stack_size(ctxt));
1890 	if (rc != X86EMUL_CONTINUE)
1891 		return rc;
1892 	assign_masked(reg_rmw(ctxt, VCPU_REGS_RBP), reg_read(ctxt, VCPU_REGS_RSP),
1893 		      stack_mask(ctxt));
1894 	assign_masked(reg_rmw(ctxt, VCPU_REGS_RSP),
1895 		      reg_read(ctxt, VCPU_REGS_RSP) - frame_size,
1896 		      stack_mask(ctxt));
1897 	return X86EMUL_CONTINUE;
1898 }
1899 
1900 static int em_leave(struct x86_emulate_ctxt *ctxt)
1901 {
1902 	assign_masked(reg_rmw(ctxt, VCPU_REGS_RSP), reg_read(ctxt, VCPU_REGS_RBP),
1903 		      stack_mask(ctxt));
1904 	return emulate_pop(ctxt, reg_rmw(ctxt, VCPU_REGS_RBP), ctxt->op_bytes);
1905 }
1906 
1907 static int em_push_sreg(struct x86_emulate_ctxt *ctxt)
1908 {
1909 	int seg = ctxt->src2.val;
1910 
1911 	ctxt->src.val = get_segment_selector(ctxt, seg);
1912 	if (ctxt->op_bytes == 4) {
1913 		rsp_increment(ctxt, -2);
1914 		ctxt->op_bytes = 2;
1915 	}
1916 
1917 	return em_push(ctxt);
1918 }
1919 
1920 static int em_pop_sreg(struct x86_emulate_ctxt *ctxt)
1921 {
1922 	int seg = ctxt->src2.val;
1923 	unsigned long selector;
1924 	int rc;
1925 
1926 	rc = emulate_pop(ctxt, &selector, 2);
1927 	if (rc != X86EMUL_CONTINUE)
1928 		return rc;
1929 
1930 	if (ctxt->modrm_reg == VCPU_SREG_SS)
1931 		ctxt->interruptibility = KVM_X86_SHADOW_INT_MOV_SS;
1932 	if (ctxt->op_bytes > 2)
1933 		rsp_increment(ctxt, ctxt->op_bytes - 2);
1934 
1935 	rc = load_segment_descriptor(ctxt, (u16)selector, seg);
1936 	return rc;
1937 }
1938 
1939 static int em_pusha(struct x86_emulate_ctxt *ctxt)
1940 {
1941 	unsigned long old_esp = reg_read(ctxt, VCPU_REGS_RSP);
1942 	int rc = X86EMUL_CONTINUE;
1943 	int reg = VCPU_REGS_RAX;
1944 
1945 	while (reg <= VCPU_REGS_RDI) {
1946 		(reg == VCPU_REGS_RSP) ?
1947 		(ctxt->src.val = old_esp) : (ctxt->src.val = reg_read(ctxt, reg));
1948 
1949 		rc = em_push(ctxt);
1950 		if (rc != X86EMUL_CONTINUE)
1951 			return rc;
1952 
1953 		++reg;
1954 	}
1955 
1956 	return rc;
1957 }
1958 
1959 static int em_pushf(struct x86_emulate_ctxt *ctxt)
1960 {
1961 	ctxt->src.val = (unsigned long)ctxt->eflags & ~X86_EFLAGS_VM;
1962 	return em_push(ctxt);
1963 }
1964 
1965 static int em_popa(struct x86_emulate_ctxt *ctxt)
1966 {
1967 	int rc = X86EMUL_CONTINUE;
1968 	int reg = VCPU_REGS_RDI;
1969 	u32 val;
1970 
1971 	while (reg >= VCPU_REGS_RAX) {
1972 		if (reg == VCPU_REGS_RSP) {
1973 			rsp_increment(ctxt, ctxt->op_bytes);
1974 			--reg;
1975 		}
1976 
1977 		rc = emulate_pop(ctxt, &val, ctxt->op_bytes);
1978 		if (rc != X86EMUL_CONTINUE)
1979 			break;
1980 		assign_register(reg_rmw(ctxt, reg), val, ctxt->op_bytes);
1981 		--reg;
1982 	}
1983 	return rc;
1984 }
1985 
1986 static int __emulate_int_real(struct x86_emulate_ctxt *ctxt, int irq)
1987 {
1988 	const struct x86_emulate_ops *ops = ctxt->ops;
1989 	int rc;
1990 	struct desc_ptr dt;
1991 	gva_t cs_addr;
1992 	gva_t eip_addr;
1993 	u16 cs, eip;
1994 
1995 	/* TODO: Add limit checks */
1996 	ctxt->src.val = ctxt->eflags;
1997 	rc = em_push(ctxt);
1998 	if (rc != X86EMUL_CONTINUE)
1999 		return rc;
2000 
2001 	ctxt->eflags &= ~(X86_EFLAGS_IF | X86_EFLAGS_TF | X86_EFLAGS_AC);
2002 
2003 	ctxt->src.val = get_segment_selector(ctxt, VCPU_SREG_CS);
2004 	rc = em_push(ctxt);
2005 	if (rc != X86EMUL_CONTINUE)
2006 		return rc;
2007 
2008 	ctxt->src.val = ctxt->_eip;
2009 	rc = em_push(ctxt);
2010 	if (rc != X86EMUL_CONTINUE)
2011 		return rc;
2012 
2013 	ops->get_idt(ctxt, &dt);
2014 
2015 	eip_addr = dt.address + (irq << 2);
2016 	cs_addr = dt.address + (irq << 2) + 2;
2017 
2018 	rc = ops->read_std(ctxt, cs_addr, &cs, 2, &ctxt->exception);
2019 	if (rc != X86EMUL_CONTINUE)
2020 		return rc;
2021 
2022 	rc = ops->read_std(ctxt, eip_addr, &eip, 2, &ctxt->exception);
2023 	if (rc != X86EMUL_CONTINUE)
2024 		return rc;
2025 
2026 	rc = load_segment_descriptor(ctxt, cs, VCPU_SREG_CS);
2027 	if (rc != X86EMUL_CONTINUE)
2028 		return rc;
2029 
2030 	ctxt->_eip = eip;
2031 
2032 	return rc;
2033 }
2034 
2035 int emulate_int_real(struct x86_emulate_ctxt *ctxt, int irq)
2036 {
2037 	int rc;
2038 
2039 	invalidate_registers(ctxt);
2040 	rc = __emulate_int_real(ctxt, irq);
2041 	if (rc == X86EMUL_CONTINUE)
2042 		writeback_registers(ctxt);
2043 	return rc;
2044 }
2045 
2046 static int emulate_int(struct x86_emulate_ctxt *ctxt, int irq)
2047 {
2048 	switch(ctxt->mode) {
2049 	case X86EMUL_MODE_REAL:
2050 		return __emulate_int_real(ctxt, irq);
2051 	case X86EMUL_MODE_VM86:
2052 	case X86EMUL_MODE_PROT16:
2053 	case X86EMUL_MODE_PROT32:
2054 	case X86EMUL_MODE_PROT64:
2055 	default:
2056 		/* Protected mode interrupts unimplemented yet */
2057 		return X86EMUL_UNHANDLEABLE;
2058 	}
2059 }
2060 
2061 static int emulate_iret_real(struct x86_emulate_ctxt *ctxt)
2062 {
2063 	int rc = X86EMUL_CONTINUE;
2064 	unsigned long temp_eip = 0;
2065 	unsigned long temp_eflags = 0;
2066 	unsigned long cs = 0;
2067 	unsigned long mask = X86_EFLAGS_CF | X86_EFLAGS_PF | X86_EFLAGS_AF |
2068 			     X86_EFLAGS_ZF | X86_EFLAGS_SF | X86_EFLAGS_TF |
2069 			     X86_EFLAGS_IF | X86_EFLAGS_DF | X86_EFLAGS_OF |
2070 			     X86_EFLAGS_IOPL | X86_EFLAGS_NT | X86_EFLAGS_RF |
2071 			     X86_EFLAGS_AC | X86_EFLAGS_ID |
2072 			     X86_EFLAGS_FIXED;
2073 	unsigned long vm86_mask = X86_EFLAGS_VM | X86_EFLAGS_VIF |
2074 				  X86_EFLAGS_VIP;
2075 
2076 	/* TODO: Add stack limit check */
2077 
2078 	rc = emulate_pop(ctxt, &temp_eip, ctxt->op_bytes);
2079 
2080 	if (rc != X86EMUL_CONTINUE)
2081 		return rc;
2082 
2083 	if (temp_eip & ~0xffff)
2084 		return emulate_gp(ctxt, 0);
2085 
2086 	rc = emulate_pop(ctxt, &cs, ctxt->op_bytes);
2087 
2088 	if (rc != X86EMUL_CONTINUE)
2089 		return rc;
2090 
2091 	rc = emulate_pop(ctxt, &temp_eflags, ctxt->op_bytes);
2092 
2093 	if (rc != X86EMUL_CONTINUE)
2094 		return rc;
2095 
2096 	rc = load_segment_descriptor(ctxt, (u16)cs, VCPU_SREG_CS);
2097 
2098 	if (rc != X86EMUL_CONTINUE)
2099 		return rc;
2100 
2101 	ctxt->_eip = temp_eip;
2102 
2103 	if (ctxt->op_bytes == 4)
2104 		ctxt->eflags = ((temp_eflags & mask) | (ctxt->eflags & vm86_mask));
2105 	else if (ctxt->op_bytes == 2) {
2106 		ctxt->eflags &= ~0xffff;
2107 		ctxt->eflags |= temp_eflags;
2108 	}
2109 
2110 	ctxt->eflags &= ~EFLG_RESERVED_ZEROS_MASK; /* Clear reserved zeros */
2111 	ctxt->eflags |= X86_EFLAGS_FIXED;
2112 	ctxt->ops->set_nmi_mask(ctxt, false);
2113 
2114 	return rc;
2115 }
2116 
2117 static int em_iret(struct x86_emulate_ctxt *ctxt)
2118 {
2119 	switch(ctxt->mode) {
2120 	case X86EMUL_MODE_REAL:
2121 		return emulate_iret_real(ctxt);
2122 	case X86EMUL_MODE_VM86:
2123 	case X86EMUL_MODE_PROT16:
2124 	case X86EMUL_MODE_PROT32:
2125 	case X86EMUL_MODE_PROT64:
2126 	default:
2127 		/* iret from protected mode unimplemented yet */
2128 		return X86EMUL_UNHANDLEABLE;
2129 	}
2130 }
2131 
2132 static int em_jmp_far(struct x86_emulate_ctxt *ctxt)
2133 {
2134 	int rc;
2135 	unsigned short sel;
2136 	struct desc_struct new_desc;
2137 	u8 cpl = ctxt->ops->cpl(ctxt);
2138 
2139 	memcpy(&sel, ctxt->src.valptr + ctxt->op_bytes, 2);
2140 
2141 	rc = __load_segment_descriptor(ctxt, sel, VCPU_SREG_CS, cpl,
2142 				       X86_TRANSFER_CALL_JMP,
2143 				       &new_desc);
2144 	if (rc != X86EMUL_CONTINUE)
2145 		return rc;
2146 
2147 	rc = assign_eip_far(ctxt, ctxt->src.val, &new_desc);
2148 	/* Error handling is not implemented. */
2149 	if (rc != X86EMUL_CONTINUE)
2150 		return X86EMUL_UNHANDLEABLE;
2151 
2152 	return rc;
2153 }
2154 
2155 static int em_jmp_abs(struct x86_emulate_ctxt *ctxt)
2156 {
2157 	return assign_eip_near(ctxt, ctxt->src.val);
2158 }
2159 
2160 static int em_call_near_abs(struct x86_emulate_ctxt *ctxt)
2161 {
2162 	int rc;
2163 	long int old_eip;
2164 
2165 	old_eip = ctxt->_eip;
2166 	rc = assign_eip_near(ctxt, ctxt->src.val);
2167 	if (rc != X86EMUL_CONTINUE)
2168 		return rc;
2169 	ctxt->src.val = old_eip;
2170 	rc = em_push(ctxt);
2171 	return rc;
2172 }
2173 
2174 static int em_cmpxchg8b(struct x86_emulate_ctxt *ctxt)
2175 {
2176 	u64 old = ctxt->dst.orig_val64;
2177 
2178 	if (ctxt->dst.bytes == 16)
2179 		return X86EMUL_UNHANDLEABLE;
2180 
2181 	if (((u32) (old >> 0) != (u32) reg_read(ctxt, VCPU_REGS_RAX)) ||
2182 	    ((u32) (old >> 32) != (u32) reg_read(ctxt, VCPU_REGS_RDX))) {
2183 		*reg_write(ctxt, VCPU_REGS_RAX) = (u32) (old >> 0);
2184 		*reg_write(ctxt, VCPU_REGS_RDX) = (u32) (old >> 32);
2185 		ctxt->eflags &= ~X86_EFLAGS_ZF;
2186 	} else {
2187 		ctxt->dst.val64 = ((u64)reg_read(ctxt, VCPU_REGS_RCX) << 32) |
2188 			(u32) reg_read(ctxt, VCPU_REGS_RBX);
2189 
2190 		ctxt->eflags |= X86_EFLAGS_ZF;
2191 	}
2192 	return X86EMUL_CONTINUE;
2193 }
2194 
2195 static int em_ret(struct x86_emulate_ctxt *ctxt)
2196 {
2197 	int rc;
2198 	unsigned long eip;
2199 
2200 	rc = emulate_pop(ctxt, &eip, ctxt->op_bytes);
2201 	if (rc != X86EMUL_CONTINUE)
2202 		return rc;
2203 
2204 	return assign_eip_near(ctxt, eip);
2205 }
2206 
2207 static int em_ret_far(struct x86_emulate_ctxt *ctxt)
2208 {
2209 	int rc;
2210 	unsigned long eip, cs;
2211 	int cpl = ctxt->ops->cpl(ctxt);
2212 	struct desc_struct new_desc;
2213 
2214 	rc = emulate_pop(ctxt, &eip, ctxt->op_bytes);
2215 	if (rc != X86EMUL_CONTINUE)
2216 		return rc;
2217 	rc = emulate_pop(ctxt, &cs, ctxt->op_bytes);
2218 	if (rc != X86EMUL_CONTINUE)
2219 		return rc;
2220 	/* Outer-privilege level return is not implemented */
2221 	if (ctxt->mode >= X86EMUL_MODE_PROT16 && (cs & 3) > cpl)
2222 		return X86EMUL_UNHANDLEABLE;
2223 	rc = __load_segment_descriptor(ctxt, (u16)cs, VCPU_SREG_CS, cpl,
2224 				       X86_TRANSFER_RET,
2225 				       &new_desc);
2226 	if (rc != X86EMUL_CONTINUE)
2227 		return rc;
2228 	rc = assign_eip_far(ctxt, eip, &new_desc);
2229 	/* Error handling is not implemented. */
2230 	if (rc != X86EMUL_CONTINUE)
2231 		return X86EMUL_UNHANDLEABLE;
2232 
2233 	return rc;
2234 }
2235 
2236 static int em_ret_far_imm(struct x86_emulate_ctxt *ctxt)
2237 {
2238         int rc;
2239 
2240         rc = em_ret_far(ctxt);
2241         if (rc != X86EMUL_CONTINUE)
2242                 return rc;
2243         rsp_increment(ctxt, ctxt->src.val);
2244         return X86EMUL_CONTINUE;
2245 }
2246 
2247 static int em_cmpxchg(struct x86_emulate_ctxt *ctxt)
2248 {
2249 	/* Save real source value, then compare EAX against destination. */
2250 	ctxt->dst.orig_val = ctxt->dst.val;
2251 	ctxt->dst.val = reg_read(ctxt, VCPU_REGS_RAX);
2252 	ctxt->src.orig_val = ctxt->src.val;
2253 	ctxt->src.val = ctxt->dst.orig_val;
2254 	fastop(ctxt, em_cmp);
2255 
2256 	if (ctxt->eflags & X86_EFLAGS_ZF) {
2257 		/* Success: write back to memory; no update of EAX */
2258 		ctxt->src.type = OP_NONE;
2259 		ctxt->dst.val = ctxt->src.orig_val;
2260 	} else {
2261 		/* Failure: write the value we saw to EAX. */
2262 		ctxt->src.type = OP_REG;
2263 		ctxt->src.addr.reg = reg_rmw(ctxt, VCPU_REGS_RAX);
2264 		ctxt->src.val = ctxt->dst.orig_val;
2265 		/* Create write-cycle to dest by writing the same value */
2266 		ctxt->dst.val = ctxt->dst.orig_val;
2267 	}
2268 	return X86EMUL_CONTINUE;
2269 }
2270 
2271 static int em_lseg(struct x86_emulate_ctxt *ctxt)
2272 {
2273 	int seg = ctxt->src2.val;
2274 	unsigned short sel;
2275 	int rc;
2276 
2277 	memcpy(&sel, ctxt->src.valptr + ctxt->op_bytes, 2);
2278 
2279 	rc = load_segment_descriptor(ctxt, sel, seg);
2280 	if (rc != X86EMUL_CONTINUE)
2281 		return rc;
2282 
2283 	ctxt->dst.val = ctxt->src.val;
2284 	return rc;
2285 }
2286 
2287 static int emulator_has_longmode(struct x86_emulate_ctxt *ctxt)
2288 {
2289 	u32 eax, ebx, ecx, edx;
2290 
2291 	eax = 0x80000001;
2292 	ecx = 0;
2293 	ctxt->ops->get_cpuid(ctxt, &eax, &ebx, &ecx, &edx);
2294 	return edx & bit(X86_FEATURE_LM);
2295 }
2296 
2297 #define GET_SMSTATE(type, smbase, offset)				  \
2298 	({								  \
2299 	 type __val;							  \
2300 	 int r = ctxt->ops->read_phys(ctxt, smbase + offset, &__val,      \
2301 				      sizeof(__val));			  \
2302 	 if (r != X86EMUL_CONTINUE)					  \
2303 		 return X86EMUL_UNHANDLEABLE;				  \
2304 	 __val;								  \
2305 	})
2306 
2307 static void rsm_set_desc_flags(struct desc_struct *desc, u32 flags)
2308 {
2309 	desc->g    = (flags >> 23) & 1;
2310 	desc->d    = (flags >> 22) & 1;
2311 	desc->l    = (flags >> 21) & 1;
2312 	desc->avl  = (flags >> 20) & 1;
2313 	desc->p    = (flags >> 15) & 1;
2314 	desc->dpl  = (flags >> 13) & 3;
2315 	desc->s    = (flags >> 12) & 1;
2316 	desc->type = (flags >>  8) & 15;
2317 }
2318 
2319 static int rsm_load_seg_32(struct x86_emulate_ctxt *ctxt, u64 smbase, int n)
2320 {
2321 	struct desc_struct desc;
2322 	int offset;
2323 	u16 selector;
2324 
2325 	selector = GET_SMSTATE(u32, smbase, 0x7fa8 + n * 4);
2326 
2327 	if (n < 3)
2328 		offset = 0x7f84 + n * 12;
2329 	else
2330 		offset = 0x7f2c + (n - 3) * 12;
2331 
2332 	set_desc_base(&desc,      GET_SMSTATE(u32, smbase, offset + 8));
2333 	set_desc_limit(&desc,     GET_SMSTATE(u32, smbase, offset + 4));
2334 	rsm_set_desc_flags(&desc, GET_SMSTATE(u32, smbase, offset));
2335 	ctxt->ops->set_segment(ctxt, selector, &desc, 0, n);
2336 	return X86EMUL_CONTINUE;
2337 }
2338 
2339 static int rsm_load_seg_64(struct x86_emulate_ctxt *ctxt, u64 smbase, int n)
2340 {
2341 	struct desc_struct desc;
2342 	int offset;
2343 	u16 selector;
2344 	u32 base3;
2345 
2346 	offset = 0x7e00 + n * 16;
2347 
2348 	selector =                GET_SMSTATE(u16, smbase, offset);
2349 	rsm_set_desc_flags(&desc, GET_SMSTATE(u16, smbase, offset + 2) << 8);
2350 	set_desc_limit(&desc,     GET_SMSTATE(u32, smbase, offset + 4));
2351 	set_desc_base(&desc,      GET_SMSTATE(u32, smbase, offset + 8));
2352 	base3 =                   GET_SMSTATE(u32, smbase, offset + 12);
2353 
2354 	ctxt->ops->set_segment(ctxt, selector, &desc, base3, n);
2355 	return X86EMUL_CONTINUE;
2356 }
2357 
2358 static int rsm_enter_protected_mode(struct x86_emulate_ctxt *ctxt,
2359 				     u64 cr0, u64 cr4)
2360 {
2361 	int bad;
2362 
2363 	/*
2364 	 * First enable PAE, long mode needs it before CR0.PG = 1 is set.
2365 	 * Then enable protected mode.	However, PCID cannot be enabled
2366 	 * if EFER.LMA=0, so set it separately.
2367 	 */
2368 	bad = ctxt->ops->set_cr(ctxt, 4, cr4 & ~X86_CR4_PCIDE);
2369 	if (bad)
2370 		return X86EMUL_UNHANDLEABLE;
2371 
2372 	bad = ctxt->ops->set_cr(ctxt, 0, cr0);
2373 	if (bad)
2374 		return X86EMUL_UNHANDLEABLE;
2375 
2376 	if (cr4 & X86_CR4_PCIDE) {
2377 		bad = ctxt->ops->set_cr(ctxt, 4, cr4);
2378 		if (bad)
2379 			return X86EMUL_UNHANDLEABLE;
2380 	}
2381 
2382 	return X86EMUL_CONTINUE;
2383 }
2384 
2385 static int rsm_load_state_32(struct x86_emulate_ctxt *ctxt, u64 smbase)
2386 {
2387 	struct desc_struct desc;
2388 	struct desc_ptr dt;
2389 	u16 selector;
2390 	u32 val, cr0, cr4;
2391 	int i;
2392 
2393 	cr0 =                      GET_SMSTATE(u32, smbase, 0x7ffc);
2394 	ctxt->ops->set_cr(ctxt, 3, GET_SMSTATE(u32, smbase, 0x7ff8));
2395 	ctxt->eflags =             GET_SMSTATE(u32, smbase, 0x7ff4) | X86_EFLAGS_FIXED;
2396 	ctxt->_eip =               GET_SMSTATE(u32, smbase, 0x7ff0);
2397 
2398 	for (i = 0; i < 8; i++)
2399 		*reg_write(ctxt, i) = GET_SMSTATE(u32, smbase, 0x7fd0 + i * 4);
2400 
2401 	val = GET_SMSTATE(u32, smbase, 0x7fcc);
2402 	ctxt->ops->set_dr(ctxt, 6, (val & DR6_VOLATILE) | DR6_FIXED_1);
2403 	val = GET_SMSTATE(u32, smbase, 0x7fc8);
2404 	ctxt->ops->set_dr(ctxt, 7, (val & DR7_VOLATILE) | DR7_FIXED_1);
2405 
2406 	selector =                 GET_SMSTATE(u32, smbase, 0x7fc4);
2407 	set_desc_base(&desc,       GET_SMSTATE(u32, smbase, 0x7f64));
2408 	set_desc_limit(&desc,      GET_SMSTATE(u32, smbase, 0x7f60));
2409 	rsm_set_desc_flags(&desc,  GET_SMSTATE(u32, smbase, 0x7f5c));
2410 	ctxt->ops->set_segment(ctxt, selector, &desc, 0, VCPU_SREG_TR);
2411 
2412 	selector =                 GET_SMSTATE(u32, smbase, 0x7fc0);
2413 	set_desc_base(&desc,       GET_SMSTATE(u32, smbase, 0x7f80));
2414 	set_desc_limit(&desc,      GET_SMSTATE(u32, smbase, 0x7f7c));
2415 	rsm_set_desc_flags(&desc,  GET_SMSTATE(u32, smbase, 0x7f78));
2416 	ctxt->ops->set_segment(ctxt, selector, &desc, 0, VCPU_SREG_LDTR);
2417 
2418 	dt.address =               GET_SMSTATE(u32, smbase, 0x7f74);
2419 	dt.size =                  GET_SMSTATE(u32, smbase, 0x7f70);
2420 	ctxt->ops->set_gdt(ctxt, &dt);
2421 
2422 	dt.address =               GET_SMSTATE(u32, smbase, 0x7f58);
2423 	dt.size =                  GET_SMSTATE(u32, smbase, 0x7f54);
2424 	ctxt->ops->set_idt(ctxt, &dt);
2425 
2426 	for (i = 0; i < 6; i++) {
2427 		int r = rsm_load_seg_32(ctxt, smbase, i);
2428 		if (r != X86EMUL_CONTINUE)
2429 			return r;
2430 	}
2431 
2432 	cr4 = GET_SMSTATE(u32, smbase, 0x7f14);
2433 
2434 	ctxt->ops->set_smbase(ctxt, GET_SMSTATE(u32, smbase, 0x7ef8));
2435 
2436 	return rsm_enter_protected_mode(ctxt, cr0, cr4);
2437 }
2438 
2439 static int rsm_load_state_64(struct x86_emulate_ctxt *ctxt, u64 smbase)
2440 {
2441 	struct desc_struct desc;
2442 	struct desc_ptr dt;
2443 	u64 val, cr0, cr4;
2444 	u32 base3;
2445 	u16 selector;
2446 	int i, r;
2447 
2448 	for (i = 0; i < 16; i++)
2449 		*reg_write(ctxt, i) = GET_SMSTATE(u64, smbase, 0x7ff8 - i * 8);
2450 
2451 	ctxt->_eip   = GET_SMSTATE(u64, smbase, 0x7f78);
2452 	ctxt->eflags = GET_SMSTATE(u32, smbase, 0x7f70) | X86_EFLAGS_FIXED;
2453 
2454 	val = GET_SMSTATE(u32, smbase, 0x7f68);
2455 	ctxt->ops->set_dr(ctxt, 6, (val & DR6_VOLATILE) | DR6_FIXED_1);
2456 	val = GET_SMSTATE(u32, smbase, 0x7f60);
2457 	ctxt->ops->set_dr(ctxt, 7, (val & DR7_VOLATILE) | DR7_FIXED_1);
2458 
2459 	cr0 =                       GET_SMSTATE(u64, smbase, 0x7f58);
2460 	ctxt->ops->set_cr(ctxt, 3,  GET_SMSTATE(u64, smbase, 0x7f50));
2461 	cr4 =                       GET_SMSTATE(u64, smbase, 0x7f48);
2462 	ctxt->ops->set_smbase(ctxt, GET_SMSTATE(u32, smbase, 0x7f00));
2463 	val =                       GET_SMSTATE(u64, smbase, 0x7ed0);
2464 	ctxt->ops->set_msr(ctxt, MSR_EFER, val & ~EFER_LMA);
2465 
2466 	selector =                  GET_SMSTATE(u32, smbase, 0x7e90);
2467 	rsm_set_desc_flags(&desc,   GET_SMSTATE(u32, smbase, 0x7e92) << 8);
2468 	set_desc_limit(&desc,       GET_SMSTATE(u32, smbase, 0x7e94));
2469 	set_desc_base(&desc,        GET_SMSTATE(u32, smbase, 0x7e98));
2470 	base3 =                     GET_SMSTATE(u32, smbase, 0x7e9c);
2471 	ctxt->ops->set_segment(ctxt, selector, &desc, base3, VCPU_SREG_TR);
2472 
2473 	dt.size =                   GET_SMSTATE(u32, smbase, 0x7e84);
2474 	dt.address =                GET_SMSTATE(u64, smbase, 0x7e88);
2475 	ctxt->ops->set_idt(ctxt, &dt);
2476 
2477 	selector =                  GET_SMSTATE(u32, smbase, 0x7e70);
2478 	rsm_set_desc_flags(&desc,   GET_SMSTATE(u32, smbase, 0x7e72) << 8);
2479 	set_desc_limit(&desc,       GET_SMSTATE(u32, smbase, 0x7e74));
2480 	set_desc_base(&desc,        GET_SMSTATE(u32, smbase, 0x7e78));
2481 	base3 =                     GET_SMSTATE(u32, smbase, 0x7e7c);
2482 	ctxt->ops->set_segment(ctxt, selector, &desc, base3, VCPU_SREG_LDTR);
2483 
2484 	dt.size =                   GET_SMSTATE(u32, smbase, 0x7e64);
2485 	dt.address =                GET_SMSTATE(u64, smbase, 0x7e68);
2486 	ctxt->ops->set_gdt(ctxt, &dt);
2487 
2488 	r = rsm_enter_protected_mode(ctxt, cr0, cr4);
2489 	if (r != X86EMUL_CONTINUE)
2490 		return r;
2491 
2492 	for (i = 0; i < 6; i++) {
2493 		r = rsm_load_seg_64(ctxt, smbase, i);
2494 		if (r != X86EMUL_CONTINUE)
2495 			return r;
2496 	}
2497 
2498 	return X86EMUL_CONTINUE;
2499 }
2500 
2501 static int em_rsm(struct x86_emulate_ctxt *ctxt)
2502 {
2503 	unsigned long cr0, cr4, efer;
2504 	u64 smbase;
2505 	int ret;
2506 
2507 	if ((ctxt->emul_flags & X86EMUL_SMM_MASK) == 0)
2508 		return emulate_ud(ctxt);
2509 
2510 	/*
2511 	 * Get back to real mode, to prepare a safe state in which to load
2512 	 * CR0/CR3/CR4/EFER.  It's all a bit more complicated if the vCPU
2513 	 * supports long mode.
2514 	 */
2515 	cr4 = ctxt->ops->get_cr(ctxt, 4);
2516 	if (emulator_has_longmode(ctxt)) {
2517 		struct desc_struct cs_desc;
2518 
2519 		/* Zero CR4.PCIDE before CR0.PG.  */
2520 		if (cr4 & X86_CR4_PCIDE) {
2521 			ctxt->ops->set_cr(ctxt, 4, cr4 & ~X86_CR4_PCIDE);
2522 			cr4 &= ~X86_CR4_PCIDE;
2523 		}
2524 
2525 		/* A 32-bit code segment is required to clear EFER.LMA.  */
2526 		memset(&cs_desc, 0, sizeof(cs_desc));
2527 		cs_desc.type = 0xb;
2528 		cs_desc.s = cs_desc.g = cs_desc.p = 1;
2529 		ctxt->ops->set_segment(ctxt, 0, &cs_desc, 0, VCPU_SREG_CS);
2530 	}
2531 
2532 	/* For the 64-bit case, this will clear EFER.LMA.  */
2533 	cr0 = ctxt->ops->get_cr(ctxt, 0);
2534 	if (cr0 & X86_CR0_PE)
2535 		ctxt->ops->set_cr(ctxt, 0, cr0 & ~(X86_CR0_PG | X86_CR0_PE));
2536 
2537 	/* Now clear CR4.PAE (which must be done before clearing EFER.LME).  */
2538 	if (cr4 & X86_CR4_PAE)
2539 		ctxt->ops->set_cr(ctxt, 4, cr4 & ~X86_CR4_PAE);
2540 
2541 	/* And finally go back to 32-bit mode.  */
2542 	efer = 0;
2543 	ctxt->ops->set_msr(ctxt, MSR_EFER, efer);
2544 
2545 	smbase = ctxt->ops->get_smbase(ctxt);
2546 	if (emulator_has_longmode(ctxt))
2547 		ret = rsm_load_state_64(ctxt, smbase + 0x8000);
2548 	else
2549 		ret = rsm_load_state_32(ctxt, smbase + 0x8000);
2550 
2551 	if (ret != X86EMUL_CONTINUE) {
2552 		/* FIXME: should triple fault */
2553 		return X86EMUL_UNHANDLEABLE;
2554 	}
2555 
2556 	if ((ctxt->emul_flags & X86EMUL_SMM_INSIDE_NMI_MASK) == 0)
2557 		ctxt->ops->set_nmi_mask(ctxt, false);
2558 
2559 	ctxt->emul_flags &= ~X86EMUL_SMM_INSIDE_NMI_MASK;
2560 	ctxt->emul_flags &= ~X86EMUL_SMM_MASK;
2561 	return X86EMUL_CONTINUE;
2562 }
2563 
2564 static void
2565 setup_syscalls_segments(struct x86_emulate_ctxt *ctxt,
2566 			struct desc_struct *cs, struct desc_struct *ss)
2567 {
2568 	cs->l = 0;		/* will be adjusted later */
2569 	set_desc_base(cs, 0);	/* flat segment */
2570 	cs->g = 1;		/* 4kb granularity */
2571 	set_desc_limit(cs, 0xfffff);	/* 4GB limit */
2572 	cs->type = 0x0b;	/* Read, Execute, Accessed */
2573 	cs->s = 1;
2574 	cs->dpl = 0;		/* will be adjusted later */
2575 	cs->p = 1;
2576 	cs->d = 1;
2577 	cs->avl = 0;
2578 
2579 	set_desc_base(ss, 0);	/* flat segment */
2580 	set_desc_limit(ss, 0xfffff);	/* 4GB limit */
2581 	ss->g = 1;		/* 4kb granularity */
2582 	ss->s = 1;
2583 	ss->type = 0x03;	/* Read/Write, Accessed */
2584 	ss->d = 1;		/* 32bit stack segment */
2585 	ss->dpl = 0;
2586 	ss->p = 1;
2587 	ss->l = 0;
2588 	ss->avl = 0;
2589 }
2590 
2591 static bool vendor_intel(struct x86_emulate_ctxt *ctxt)
2592 {
2593 	u32 eax, ebx, ecx, edx;
2594 
2595 	eax = ecx = 0;
2596 	ctxt->ops->get_cpuid(ctxt, &eax, &ebx, &ecx, &edx);
2597 	return ebx == X86EMUL_CPUID_VENDOR_GenuineIntel_ebx
2598 		&& ecx == X86EMUL_CPUID_VENDOR_GenuineIntel_ecx
2599 		&& edx == X86EMUL_CPUID_VENDOR_GenuineIntel_edx;
2600 }
2601 
2602 static bool em_syscall_is_enabled(struct x86_emulate_ctxt *ctxt)
2603 {
2604 	const struct x86_emulate_ops *ops = ctxt->ops;
2605 	u32 eax, ebx, ecx, edx;
2606 
2607 	/*
2608 	 * syscall should always be enabled in longmode - so only become
2609 	 * vendor specific (cpuid) if other modes are active...
2610 	 */
2611 	if (ctxt->mode == X86EMUL_MODE_PROT64)
2612 		return true;
2613 
2614 	eax = 0x00000000;
2615 	ecx = 0x00000000;
2616 	ops->get_cpuid(ctxt, &eax, &ebx, &ecx, &edx);
2617 	/*
2618 	 * Intel ("GenuineIntel")
2619 	 * remark: Intel CPUs only support "syscall" in 64bit
2620 	 * longmode. Also an 64bit guest with a
2621 	 * 32bit compat-app running will #UD !! While this
2622 	 * behaviour can be fixed (by emulating) into AMD
2623 	 * response - CPUs of AMD can't behave like Intel.
2624 	 */
2625 	if (ebx == X86EMUL_CPUID_VENDOR_GenuineIntel_ebx &&
2626 	    ecx == X86EMUL_CPUID_VENDOR_GenuineIntel_ecx &&
2627 	    edx == X86EMUL_CPUID_VENDOR_GenuineIntel_edx)
2628 		return false;
2629 
2630 	/* AMD ("AuthenticAMD") */
2631 	if (ebx == X86EMUL_CPUID_VENDOR_AuthenticAMD_ebx &&
2632 	    ecx == X86EMUL_CPUID_VENDOR_AuthenticAMD_ecx &&
2633 	    edx == X86EMUL_CPUID_VENDOR_AuthenticAMD_edx)
2634 		return true;
2635 
2636 	/* AMD ("AMDisbetter!") */
2637 	if (ebx == X86EMUL_CPUID_VENDOR_AMDisbetterI_ebx &&
2638 	    ecx == X86EMUL_CPUID_VENDOR_AMDisbetterI_ecx &&
2639 	    edx == X86EMUL_CPUID_VENDOR_AMDisbetterI_edx)
2640 		return true;
2641 
2642 	/* default: (not Intel, not AMD), apply Intel's stricter rules... */
2643 	return false;
2644 }
2645 
2646 static int em_syscall(struct x86_emulate_ctxt *ctxt)
2647 {
2648 	const struct x86_emulate_ops *ops = ctxt->ops;
2649 	struct desc_struct cs, ss;
2650 	u64 msr_data;
2651 	u16 cs_sel, ss_sel;
2652 	u64 efer = 0;
2653 
2654 	/* syscall is not available in real mode */
2655 	if (ctxt->mode == X86EMUL_MODE_REAL ||
2656 	    ctxt->mode == X86EMUL_MODE_VM86)
2657 		return emulate_ud(ctxt);
2658 
2659 	if (!(em_syscall_is_enabled(ctxt)))
2660 		return emulate_ud(ctxt);
2661 
2662 	ops->get_msr(ctxt, MSR_EFER, &efer);
2663 	setup_syscalls_segments(ctxt, &cs, &ss);
2664 
2665 	if (!(efer & EFER_SCE))
2666 		return emulate_ud(ctxt);
2667 
2668 	ops->get_msr(ctxt, MSR_STAR, &msr_data);
2669 	msr_data >>= 32;
2670 	cs_sel = (u16)(msr_data & 0xfffc);
2671 	ss_sel = (u16)(msr_data + 8);
2672 
2673 	if (efer & EFER_LMA) {
2674 		cs.d = 0;
2675 		cs.l = 1;
2676 	}
2677 	ops->set_segment(ctxt, cs_sel, &cs, 0, VCPU_SREG_CS);
2678 	ops->set_segment(ctxt, ss_sel, &ss, 0, VCPU_SREG_SS);
2679 
2680 	*reg_write(ctxt, VCPU_REGS_RCX) = ctxt->_eip;
2681 	if (efer & EFER_LMA) {
2682 #ifdef CONFIG_X86_64
2683 		*reg_write(ctxt, VCPU_REGS_R11) = ctxt->eflags;
2684 
2685 		ops->get_msr(ctxt,
2686 			     ctxt->mode == X86EMUL_MODE_PROT64 ?
2687 			     MSR_LSTAR : MSR_CSTAR, &msr_data);
2688 		ctxt->_eip = msr_data;
2689 
2690 		ops->get_msr(ctxt, MSR_SYSCALL_MASK, &msr_data);
2691 		ctxt->eflags &= ~msr_data;
2692 		ctxt->eflags |= X86_EFLAGS_FIXED;
2693 #endif
2694 	} else {
2695 		/* legacy mode */
2696 		ops->get_msr(ctxt, MSR_STAR, &msr_data);
2697 		ctxt->_eip = (u32)msr_data;
2698 
2699 		ctxt->eflags &= ~(X86_EFLAGS_VM | X86_EFLAGS_IF);
2700 	}
2701 
2702 	return X86EMUL_CONTINUE;
2703 }
2704 
2705 static int em_sysenter(struct x86_emulate_ctxt *ctxt)
2706 {
2707 	const struct x86_emulate_ops *ops = ctxt->ops;
2708 	struct desc_struct cs, ss;
2709 	u64 msr_data;
2710 	u16 cs_sel, ss_sel;
2711 	u64 efer = 0;
2712 
2713 	ops->get_msr(ctxt, MSR_EFER, &efer);
2714 	/* inject #GP if in real mode */
2715 	if (ctxt->mode == X86EMUL_MODE_REAL)
2716 		return emulate_gp(ctxt, 0);
2717 
2718 	/*
2719 	 * Not recognized on AMD in compat mode (but is recognized in legacy
2720 	 * mode).
2721 	 */
2722 	if ((ctxt->mode != X86EMUL_MODE_PROT64) && (efer & EFER_LMA)
2723 	    && !vendor_intel(ctxt))
2724 		return emulate_ud(ctxt);
2725 
2726 	/* sysenter/sysexit have not been tested in 64bit mode. */
2727 	if (ctxt->mode == X86EMUL_MODE_PROT64)
2728 		return X86EMUL_UNHANDLEABLE;
2729 
2730 	setup_syscalls_segments(ctxt, &cs, &ss);
2731 
2732 	ops->get_msr(ctxt, MSR_IA32_SYSENTER_CS, &msr_data);
2733 	if ((msr_data & 0xfffc) == 0x0)
2734 		return emulate_gp(ctxt, 0);
2735 
2736 	ctxt->eflags &= ~(X86_EFLAGS_VM | X86_EFLAGS_IF);
2737 	cs_sel = (u16)msr_data & ~SEGMENT_RPL_MASK;
2738 	ss_sel = cs_sel + 8;
2739 	if (efer & EFER_LMA) {
2740 		cs.d = 0;
2741 		cs.l = 1;
2742 	}
2743 
2744 	ops->set_segment(ctxt, cs_sel, &cs, 0, VCPU_SREG_CS);
2745 	ops->set_segment(ctxt, ss_sel, &ss, 0, VCPU_SREG_SS);
2746 
2747 	ops->get_msr(ctxt, MSR_IA32_SYSENTER_EIP, &msr_data);
2748 	ctxt->_eip = (efer & EFER_LMA) ? msr_data : (u32)msr_data;
2749 
2750 	ops->get_msr(ctxt, MSR_IA32_SYSENTER_ESP, &msr_data);
2751 	*reg_write(ctxt, VCPU_REGS_RSP) = (efer & EFER_LMA) ? msr_data :
2752 							      (u32)msr_data;
2753 
2754 	return X86EMUL_CONTINUE;
2755 }
2756 
2757 static int em_sysexit(struct x86_emulate_ctxt *ctxt)
2758 {
2759 	const struct x86_emulate_ops *ops = ctxt->ops;
2760 	struct desc_struct cs, ss;
2761 	u64 msr_data, rcx, rdx;
2762 	int usermode;
2763 	u16 cs_sel = 0, ss_sel = 0;
2764 
2765 	/* inject #GP if in real mode or Virtual 8086 mode */
2766 	if (ctxt->mode == X86EMUL_MODE_REAL ||
2767 	    ctxt->mode == X86EMUL_MODE_VM86)
2768 		return emulate_gp(ctxt, 0);
2769 
2770 	setup_syscalls_segments(ctxt, &cs, &ss);
2771 
2772 	if ((ctxt->rex_prefix & 0x8) != 0x0)
2773 		usermode = X86EMUL_MODE_PROT64;
2774 	else
2775 		usermode = X86EMUL_MODE_PROT32;
2776 
2777 	rcx = reg_read(ctxt, VCPU_REGS_RCX);
2778 	rdx = reg_read(ctxt, VCPU_REGS_RDX);
2779 
2780 	cs.dpl = 3;
2781 	ss.dpl = 3;
2782 	ops->get_msr(ctxt, MSR_IA32_SYSENTER_CS, &msr_data);
2783 	switch (usermode) {
2784 	case X86EMUL_MODE_PROT32:
2785 		cs_sel = (u16)(msr_data + 16);
2786 		if ((msr_data & 0xfffc) == 0x0)
2787 			return emulate_gp(ctxt, 0);
2788 		ss_sel = (u16)(msr_data + 24);
2789 		rcx = (u32)rcx;
2790 		rdx = (u32)rdx;
2791 		break;
2792 	case X86EMUL_MODE_PROT64:
2793 		cs_sel = (u16)(msr_data + 32);
2794 		if (msr_data == 0x0)
2795 			return emulate_gp(ctxt, 0);
2796 		ss_sel = cs_sel + 8;
2797 		cs.d = 0;
2798 		cs.l = 1;
2799 		if (is_noncanonical_address(rcx) ||
2800 		    is_noncanonical_address(rdx))
2801 			return emulate_gp(ctxt, 0);
2802 		break;
2803 	}
2804 	cs_sel |= SEGMENT_RPL_MASK;
2805 	ss_sel |= SEGMENT_RPL_MASK;
2806 
2807 	ops->set_segment(ctxt, cs_sel, &cs, 0, VCPU_SREG_CS);
2808 	ops->set_segment(ctxt, ss_sel, &ss, 0, VCPU_SREG_SS);
2809 
2810 	ctxt->_eip = rdx;
2811 	*reg_write(ctxt, VCPU_REGS_RSP) = rcx;
2812 
2813 	return X86EMUL_CONTINUE;
2814 }
2815 
2816 static bool emulator_bad_iopl(struct x86_emulate_ctxt *ctxt)
2817 {
2818 	int iopl;
2819 	if (ctxt->mode == X86EMUL_MODE_REAL)
2820 		return false;
2821 	if (ctxt->mode == X86EMUL_MODE_VM86)
2822 		return true;
2823 	iopl = (ctxt->eflags & X86_EFLAGS_IOPL) >> X86_EFLAGS_IOPL_BIT;
2824 	return ctxt->ops->cpl(ctxt) > iopl;
2825 }
2826 
2827 static bool emulator_io_port_access_allowed(struct x86_emulate_ctxt *ctxt,
2828 					    u16 port, u16 len)
2829 {
2830 	const struct x86_emulate_ops *ops = ctxt->ops;
2831 	struct desc_struct tr_seg;
2832 	u32 base3;
2833 	int r;
2834 	u16 tr, io_bitmap_ptr, perm, bit_idx = port & 0x7;
2835 	unsigned mask = (1 << len) - 1;
2836 	unsigned long base;
2837 
2838 	ops->get_segment(ctxt, &tr, &tr_seg, &base3, VCPU_SREG_TR);
2839 	if (!tr_seg.p)
2840 		return false;
2841 	if (desc_limit_scaled(&tr_seg) < 103)
2842 		return false;
2843 	base = get_desc_base(&tr_seg);
2844 #ifdef CONFIG_X86_64
2845 	base |= ((u64)base3) << 32;
2846 #endif
2847 	r = ops->read_std(ctxt, base + 102, &io_bitmap_ptr, 2, NULL);
2848 	if (r != X86EMUL_CONTINUE)
2849 		return false;
2850 	if (io_bitmap_ptr + port/8 > desc_limit_scaled(&tr_seg))
2851 		return false;
2852 	r = ops->read_std(ctxt, base + io_bitmap_ptr + port/8, &perm, 2, NULL);
2853 	if (r != X86EMUL_CONTINUE)
2854 		return false;
2855 	if ((perm >> bit_idx) & mask)
2856 		return false;
2857 	return true;
2858 }
2859 
2860 static bool emulator_io_permited(struct x86_emulate_ctxt *ctxt,
2861 				 u16 port, u16 len)
2862 {
2863 	if (ctxt->perm_ok)
2864 		return true;
2865 
2866 	if (emulator_bad_iopl(ctxt))
2867 		if (!emulator_io_port_access_allowed(ctxt, port, len))
2868 			return false;
2869 
2870 	ctxt->perm_ok = true;
2871 
2872 	return true;
2873 }
2874 
2875 static void string_registers_quirk(struct x86_emulate_ctxt *ctxt)
2876 {
2877 	/*
2878 	 * Intel CPUs mask the counter and pointers in quite strange
2879 	 * manner when ECX is zero due to REP-string optimizations.
2880 	 */
2881 #ifdef CONFIG_X86_64
2882 	if (ctxt->ad_bytes != 4 || !vendor_intel(ctxt))
2883 		return;
2884 
2885 	*reg_write(ctxt, VCPU_REGS_RCX) = 0;
2886 
2887 	switch (ctxt->b) {
2888 	case 0xa4:	/* movsb */
2889 	case 0xa5:	/* movsd/w */
2890 		*reg_rmw(ctxt, VCPU_REGS_RSI) &= (u32)-1;
2891 		/* fall through */
2892 	case 0xaa:	/* stosb */
2893 	case 0xab:	/* stosd/w */
2894 		*reg_rmw(ctxt, VCPU_REGS_RDI) &= (u32)-1;
2895 	}
2896 #endif
2897 }
2898 
2899 static void save_state_to_tss16(struct x86_emulate_ctxt *ctxt,
2900 				struct tss_segment_16 *tss)
2901 {
2902 	tss->ip = ctxt->_eip;
2903 	tss->flag = ctxt->eflags;
2904 	tss->ax = reg_read(ctxt, VCPU_REGS_RAX);
2905 	tss->cx = reg_read(ctxt, VCPU_REGS_RCX);
2906 	tss->dx = reg_read(ctxt, VCPU_REGS_RDX);
2907 	tss->bx = reg_read(ctxt, VCPU_REGS_RBX);
2908 	tss->sp = reg_read(ctxt, VCPU_REGS_RSP);
2909 	tss->bp = reg_read(ctxt, VCPU_REGS_RBP);
2910 	tss->si = reg_read(ctxt, VCPU_REGS_RSI);
2911 	tss->di = reg_read(ctxt, VCPU_REGS_RDI);
2912 
2913 	tss->es = get_segment_selector(ctxt, VCPU_SREG_ES);
2914 	tss->cs = get_segment_selector(ctxt, VCPU_SREG_CS);
2915 	tss->ss = get_segment_selector(ctxt, VCPU_SREG_SS);
2916 	tss->ds = get_segment_selector(ctxt, VCPU_SREG_DS);
2917 	tss->ldt = get_segment_selector(ctxt, VCPU_SREG_LDTR);
2918 }
2919 
2920 static int load_state_from_tss16(struct x86_emulate_ctxt *ctxt,
2921 				 struct tss_segment_16 *tss)
2922 {
2923 	int ret;
2924 	u8 cpl;
2925 
2926 	ctxt->_eip = tss->ip;
2927 	ctxt->eflags = tss->flag | 2;
2928 	*reg_write(ctxt, VCPU_REGS_RAX) = tss->ax;
2929 	*reg_write(ctxt, VCPU_REGS_RCX) = tss->cx;
2930 	*reg_write(ctxt, VCPU_REGS_RDX) = tss->dx;
2931 	*reg_write(ctxt, VCPU_REGS_RBX) = tss->bx;
2932 	*reg_write(ctxt, VCPU_REGS_RSP) = tss->sp;
2933 	*reg_write(ctxt, VCPU_REGS_RBP) = tss->bp;
2934 	*reg_write(ctxt, VCPU_REGS_RSI) = tss->si;
2935 	*reg_write(ctxt, VCPU_REGS_RDI) = tss->di;
2936 
2937 	/*
2938 	 * SDM says that segment selectors are loaded before segment
2939 	 * descriptors
2940 	 */
2941 	set_segment_selector(ctxt, tss->ldt, VCPU_SREG_LDTR);
2942 	set_segment_selector(ctxt, tss->es, VCPU_SREG_ES);
2943 	set_segment_selector(ctxt, tss->cs, VCPU_SREG_CS);
2944 	set_segment_selector(ctxt, tss->ss, VCPU_SREG_SS);
2945 	set_segment_selector(ctxt, tss->ds, VCPU_SREG_DS);
2946 
2947 	cpl = tss->cs & 3;
2948 
2949 	/*
2950 	 * Now load segment descriptors. If fault happens at this stage
2951 	 * it is handled in a context of new task
2952 	 */
2953 	ret = __load_segment_descriptor(ctxt, tss->ldt, VCPU_SREG_LDTR, cpl,
2954 					X86_TRANSFER_TASK_SWITCH, NULL);
2955 	if (ret != X86EMUL_CONTINUE)
2956 		return ret;
2957 	ret = __load_segment_descriptor(ctxt, tss->es, VCPU_SREG_ES, cpl,
2958 					X86_TRANSFER_TASK_SWITCH, NULL);
2959 	if (ret != X86EMUL_CONTINUE)
2960 		return ret;
2961 	ret = __load_segment_descriptor(ctxt, tss->cs, VCPU_SREG_CS, cpl,
2962 					X86_TRANSFER_TASK_SWITCH, NULL);
2963 	if (ret != X86EMUL_CONTINUE)
2964 		return ret;
2965 	ret = __load_segment_descriptor(ctxt, tss->ss, VCPU_SREG_SS, cpl,
2966 					X86_TRANSFER_TASK_SWITCH, NULL);
2967 	if (ret != X86EMUL_CONTINUE)
2968 		return ret;
2969 	ret = __load_segment_descriptor(ctxt, tss->ds, VCPU_SREG_DS, cpl,
2970 					X86_TRANSFER_TASK_SWITCH, NULL);
2971 	if (ret != X86EMUL_CONTINUE)
2972 		return ret;
2973 
2974 	return X86EMUL_CONTINUE;
2975 }
2976 
2977 static int task_switch_16(struct x86_emulate_ctxt *ctxt,
2978 			  u16 tss_selector, u16 old_tss_sel,
2979 			  ulong old_tss_base, struct desc_struct *new_desc)
2980 {
2981 	const struct x86_emulate_ops *ops = ctxt->ops;
2982 	struct tss_segment_16 tss_seg;
2983 	int ret;
2984 	u32 new_tss_base = get_desc_base(new_desc);
2985 
2986 	ret = ops->read_std(ctxt, old_tss_base, &tss_seg, sizeof tss_seg,
2987 			    &ctxt->exception);
2988 	if (ret != X86EMUL_CONTINUE)
2989 		return ret;
2990 
2991 	save_state_to_tss16(ctxt, &tss_seg);
2992 
2993 	ret = ops->write_std(ctxt, old_tss_base, &tss_seg, sizeof tss_seg,
2994 			     &ctxt->exception);
2995 	if (ret != X86EMUL_CONTINUE)
2996 		return ret;
2997 
2998 	ret = ops->read_std(ctxt, new_tss_base, &tss_seg, sizeof tss_seg,
2999 			    &ctxt->exception);
3000 	if (ret != X86EMUL_CONTINUE)
3001 		return ret;
3002 
3003 	if (old_tss_sel != 0xffff) {
3004 		tss_seg.prev_task_link = old_tss_sel;
3005 
3006 		ret = ops->write_std(ctxt, new_tss_base,
3007 				     &tss_seg.prev_task_link,
3008 				     sizeof tss_seg.prev_task_link,
3009 				     &ctxt->exception);
3010 		if (ret != X86EMUL_CONTINUE)
3011 			return ret;
3012 	}
3013 
3014 	return load_state_from_tss16(ctxt, &tss_seg);
3015 }
3016 
3017 static void save_state_to_tss32(struct x86_emulate_ctxt *ctxt,
3018 				struct tss_segment_32 *tss)
3019 {
3020 	/* CR3 and ldt selector are not saved intentionally */
3021 	tss->eip = ctxt->_eip;
3022 	tss->eflags = ctxt->eflags;
3023 	tss->eax = reg_read(ctxt, VCPU_REGS_RAX);
3024 	tss->ecx = reg_read(ctxt, VCPU_REGS_RCX);
3025 	tss->edx = reg_read(ctxt, VCPU_REGS_RDX);
3026 	tss->ebx = reg_read(ctxt, VCPU_REGS_RBX);
3027 	tss->esp = reg_read(ctxt, VCPU_REGS_RSP);
3028 	tss->ebp = reg_read(ctxt, VCPU_REGS_RBP);
3029 	tss->esi = reg_read(ctxt, VCPU_REGS_RSI);
3030 	tss->edi = reg_read(ctxt, VCPU_REGS_RDI);
3031 
3032 	tss->es = get_segment_selector(ctxt, VCPU_SREG_ES);
3033 	tss->cs = get_segment_selector(ctxt, VCPU_SREG_CS);
3034 	tss->ss = get_segment_selector(ctxt, VCPU_SREG_SS);
3035 	tss->ds = get_segment_selector(ctxt, VCPU_SREG_DS);
3036 	tss->fs = get_segment_selector(ctxt, VCPU_SREG_FS);
3037 	tss->gs = get_segment_selector(ctxt, VCPU_SREG_GS);
3038 }
3039 
3040 static int load_state_from_tss32(struct x86_emulate_ctxt *ctxt,
3041 				 struct tss_segment_32 *tss)
3042 {
3043 	int ret;
3044 	u8 cpl;
3045 
3046 	if (ctxt->ops->set_cr(ctxt, 3, tss->cr3))
3047 		return emulate_gp(ctxt, 0);
3048 	ctxt->_eip = tss->eip;
3049 	ctxt->eflags = tss->eflags | 2;
3050 
3051 	/* General purpose registers */
3052 	*reg_write(ctxt, VCPU_REGS_RAX) = tss->eax;
3053 	*reg_write(ctxt, VCPU_REGS_RCX) = tss->ecx;
3054 	*reg_write(ctxt, VCPU_REGS_RDX) = tss->edx;
3055 	*reg_write(ctxt, VCPU_REGS_RBX) = tss->ebx;
3056 	*reg_write(ctxt, VCPU_REGS_RSP) = tss->esp;
3057 	*reg_write(ctxt, VCPU_REGS_RBP) = tss->ebp;
3058 	*reg_write(ctxt, VCPU_REGS_RSI) = tss->esi;
3059 	*reg_write(ctxt, VCPU_REGS_RDI) = tss->edi;
3060 
3061 	/*
3062 	 * SDM says that segment selectors are loaded before segment
3063 	 * descriptors.  This is important because CPL checks will
3064 	 * use CS.RPL.
3065 	 */
3066 	set_segment_selector(ctxt, tss->ldt_selector, VCPU_SREG_LDTR);
3067 	set_segment_selector(ctxt, tss->es, VCPU_SREG_ES);
3068 	set_segment_selector(ctxt, tss->cs, VCPU_SREG_CS);
3069 	set_segment_selector(ctxt, tss->ss, VCPU_SREG_SS);
3070 	set_segment_selector(ctxt, tss->ds, VCPU_SREG_DS);
3071 	set_segment_selector(ctxt, tss->fs, VCPU_SREG_FS);
3072 	set_segment_selector(ctxt, tss->gs, VCPU_SREG_GS);
3073 
3074 	/*
3075 	 * If we're switching between Protected Mode and VM86, we need to make
3076 	 * sure to update the mode before loading the segment descriptors so
3077 	 * that the selectors are interpreted correctly.
3078 	 */
3079 	if (ctxt->eflags & X86_EFLAGS_VM) {
3080 		ctxt->mode = X86EMUL_MODE_VM86;
3081 		cpl = 3;
3082 	} else {
3083 		ctxt->mode = X86EMUL_MODE_PROT32;
3084 		cpl = tss->cs & 3;
3085 	}
3086 
3087 	/*
3088 	 * Now load segment descriptors. If fault happenes at this stage
3089 	 * it is handled in a context of new task
3090 	 */
3091 	ret = __load_segment_descriptor(ctxt, tss->ldt_selector, VCPU_SREG_LDTR,
3092 					cpl, X86_TRANSFER_TASK_SWITCH, NULL);
3093 	if (ret != X86EMUL_CONTINUE)
3094 		return ret;
3095 	ret = __load_segment_descriptor(ctxt, tss->es, VCPU_SREG_ES, cpl,
3096 					X86_TRANSFER_TASK_SWITCH, NULL);
3097 	if (ret != X86EMUL_CONTINUE)
3098 		return ret;
3099 	ret = __load_segment_descriptor(ctxt, tss->cs, VCPU_SREG_CS, cpl,
3100 					X86_TRANSFER_TASK_SWITCH, NULL);
3101 	if (ret != X86EMUL_CONTINUE)
3102 		return ret;
3103 	ret = __load_segment_descriptor(ctxt, tss->ss, VCPU_SREG_SS, cpl,
3104 					X86_TRANSFER_TASK_SWITCH, NULL);
3105 	if (ret != X86EMUL_CONTINUE)
3106 		return ret;
3107 	ret = __load_segment_descriptor(ctxt, tss->ds, VCPU_SREG_DS, cpl,
3108 					X86_TRANSFER_TASK_SWITCH, NULL);
3109 	if (ret != X86EMUL_CONTINUE)
3110 		return ret;
3111 	ret = __load_segment_descriptor(ctxt, tss->fs, VCPU_SREG_FS, cpl,
3112 					X86_TRANSFER_TASK_SWITCH, NULL);
3113 	if (ret != X86EMUL_CONTINUE)
3114 		return ret;
3115 	ret = __load_segment_descriptor(ctxt, tss->gs, VCPU_SREG_GS, cpl,
3116 					X86_TRANSFER_TASK_SWITCH, NULL);
3117 
3118 	return ret;
3119 }
3120 
3121 static int task_switch_32(struct x86_emulate_ctxt *ctxt,
3122 			  u16 tss_selector, u16 old_tss_sel,
3123 			  ulong old_tss_base, struct desc_struct *new_desc)
3124 {
3125 	const struct x86_emulate_ops *ops = ctxt->ops;
3126 	struct tss_segment_32 tss_seg;
3127 	int ret;
3128 	u32 new_tss_base = get_desc_base(new_desc);
3129 	u32 eip_offset = offsetof(struct tss_segment_32, eip);
3130 	u32 ldt_sel_offset = offsetof(struct tss_segment_32, ldt_selector);
3131 
3132 	ret = ops->read_std(ctxt, old_tss_base, &tss_seg, sizeof tss_seg,
3133 			    &ctxt->exception);
3134 	if (ret != X86EMUL_CONTINUE)
3135 		return ret;
3136 
3137 	save_state_to_tss32(ctxt, &tss_seg);
3138 
3139 	/* Only GP registers and segment selectors are saved */
3140 	ret = ops->write_std(ctxt, old_tss_base + eip_offset, &tss_seg.eip,
3141 			     ldt_sel_offset - eip_offset, &ctxt->exception);
3142 	if (ret != X86EMUL_CONTINUE)
3143 		return ret;
3144 
3145 	ret = ops->read_std(ctxt, new_tss_base, &tss_seg, sizeof tss_seg,
3146 			    &ctxt->exception);
3147 	if (ret != X86EMUL_CONTINUE)
3148 		return ret;
3149 
3150 	if (old_tss_sel != 0xffff) {
3151 		tss_seg.prev_task_link = old_tss_sel;
3152 
3153 		ret = ops->write_std(ctxt, new_tss_base,
3154 				     &tss_seg.prev_task_link,
3155 				     sizeof tss_seg.prev_task_link,
3156 				     &ctxt->exception);
3157 		if (ret != X86EMUL_CONTINUE)
3158 			return ret;
3159 	}
3160 
3161 	return load_state_from_tss32(ctxt, &tss_seg);
3162 }
3163 
3164 static int emulator_do_task_switch(struct x86_emulate_ctxt *ctxt,
3165 				   u16 tss_selector, int idt_index, int reason,
3166 				   bool has_error_code, u32 error_code)
3167 {
3168 	const struct x86_emulate_ops *ops = ctxt->ops;
3169 	struct desc_struct curr_tss_desc, next_tss_desc;
3170 	int ret;
3171 	u16 old_tss_sel = get_segment_selector(ctxt, VCPU_SREG_TR);
3172 	ulong old_tss_base =
3173 		ops->get_cached_segment_base(ctxt, VCPU_SREG_TR);
3174 	u32 desc_limit;
3175 	ulong desc_addr, dr7;
3176 
3177 	/* FIXME: old_tss_base == ~0 ? */
3178 
3179 	ret = read_segment_descriptor(ctxt, tss_selector, &next_tss_desc, &desc_addr);
3180 	if (ret != X86EMUL_CONTINUE)
3181 		return ret;
3182 	ret = read_segment_descriptor(ctxt, old_tss_sel, &curr_tss_desc, &desc_addr);
3183 	if (ret != X86EMUL_CONTINUE)
3184 		return ret;
3185 
3186 	/* FIXME: check that next_tss_desc is tss */
3187 
3188 	/*
3189 	 * Check privileges. The three cases are task switch caused by...
3190 	 *
3191 	 * 1. jmp/call/int to task gate: Check against DPL of the task gate
3192 	 * 2. Exception/IRQ/iret: No check is performed
3193 	 * 3. jmp/call to TSS/task-gate: No check is performed since the
3194 	 *    hardware checks it before exiting.
3195 	 */
3196 	if (reason == TASK_SWITCH_GATE) {
3197 		if (idt_index != -1) {
3198 			/* Software interrupts */
3199 			struct desc_struct task_gate_desc;
3200 			int dpl;
3201 
3202 			ret = read_interrupt_descriptor(ctxt, idt_index,
3203 							&task_gate_desc);
3204 			if (ret != X86EMUL_CONTINUE)
3205 				return ret;
3206 
3207 			dpl = task_gate_desc.dpl;
3208 			if ((tss_selector & 3) > dpl || ops->cpl(ctxt) > dpl)
3209 				return emulate_gp(ctxt, (idt_index << 3) | 0x2);
3210 		}
3211 	}
3212 
3213 	desc_limit = desc_limit_scaled(&next_tss_desc);
3214 	if (!next_tss_desc.p ||
3215 	    ((desc_limit < 0x67 && (next_tss_desc.type & 8)) ||
3216 	     desc_limit < 0x2b)) {
3217 		return emulate_ts(ctxt, tss_selector & 0xfffc);
3218 	}
3219 
3220 	if (reason == TASK_SWITCH_IRET || reason == TASK_SWITCH_JMP) {
3221 		curr_tss_desc.type &= ~(1 << 1); /* clear busy flag */
3222 		write_segment_descriptor(ctxt, old_tss_sel, &curr_tss_desc);
3223 	}
3224 
3225 	if (reason == TASK_SWITCH_IRET)
3226 		ctxt->eflags = ctxt->eflags & ~X86_EFLAGS_NT;
3227 
3228 	/* set back link to prev task only if NT bit is set in eflags
3229 	   note that old_tss_sel is not used after this point */
3230 	if (reason != TASK_SWITCH_CALL && reason != TASK_SWITCH_GATE)
3231 		old_tss_sel = 0xffff;
3232 
3233 	if (next_tss_desc.type & 8)
3234 		ret = task_switch_32(ctxt, tss_selector, old_tss_sel,
3235 				     old_tss_base, &next_tss_desc);
3236 	else
3237 		ret = task_switch_16(ctxt, tss_selector, old_tss_sel,
3238 				     old_tss_base, &next_tss_desc);
3239 	if (ret != X86EMUL_CONTINUE)
3240 		return ret;
3241 
3242 	if (reason == TASK_SWITCH_CALL || reason == TASK_SWITCH_GATE)
3243 		ctxt->eflags = ctxt->eflags | X86_EFLAGS_NT;
3244 
3245 	if (reason != TASK_SWITCH_IRET) {
3246 		next_tss_desc.type |= (1 << 1); /* set busy flag */
3247 		write_segment_descriptor(ctxt, tss_selector, &next_tss_desc);
3248 	}
3249 
3250 	ops->set_cr(ctxt, 0,  ops->get_cr(ctxt, 0) | X86_CR0_TS);
3251 	ops->set_segment(ctxt, tss_selector, &next_tss_desc, 0, VCPU_SREG_TR);
3252 
3253 	if (has_error_code) {
3254 		ctxt->op_bytes = ctxt->ad_bytes = (next_tss_desc.type & 8) ? 4 : 2;
3255 		ctxt->lock_prefix = 0;
3256 		ctxt->src.val = (unsigned long) error_code;
3257 		ret = em_push(ctxt);
3258 	}
3259 
3260 	ops->get_dr(ctxt, 7, &dr7);
3261 	ops->set_dr(ctxt, 7, dr7 & ~(DR_LOCAL_ENABLE_MASK | DR_LOCAL_SLOWDOWN));
3262 
3263 	return ret;
3264 }
3265 
3266 int emulator_task_switch(struct x86_emulate_ctxt *ctxt,
3267 			 u16 tss_selector, int idt_index, int reason,
3268 			 bool has_error_code, u32 error_code)
3269 {
3270 	int rc;
3271 
3272 	invalidate_registers(ctxt);
3273 	ctxt->_eip = ctxt->eip;
3274 	ctxt->dst.type = OP_NONE;
3275 
3276 	rc = emulator_do_task_switch(ctxt, tss_selector, idt_index, reason,
3277 				     has_error_code, error_code);
3278 
3279 	if (rc == X86EMUL_CONTINUE) {
3280 		ctxt->eip = ctxt->_eip;
3281 		writeback_registers(ctxt);
3282 	}
3283 
3284 	return (rc == X86EMUL_UNHANDLEABLE) ? EMULATION_FAILED : EMULATION_OK;
3285 }
3286 
3287 static void string_addr_inc(struct x86_emulate_ctxt *ctxt, int reg,
3288 		struct operand *op)
3289 {
3290 	int df = (ctxt->eflags & X86_EFLAGS_DF) ? -op->count : op->count;
3291 
3292 	register_address_increment(ctxt, reg, df * op->bytes);
3293 	op->addr.mem.ea = register_address(ctxt, reg);
3294 }
3295 
3296 static int em_das(struct x86_emulate_ctxt *ctxt)
3297 {
3298 	u8 al, old_al;
3299 	bool af, cf, old_cf;
3300 
3301 	cf = ctxt->eflags & X86_EFLAGS_CF;
3302 	al = ctxt->dst.val;
3303 
3304 	old_al = al;
3305 	old_cf = cf;
3306 	cf = false;
3307 	af = ctxt->eflags & X86_EFLAGS_AF;
3308 	if ((al & 0x0f) > 9 || af) {
3309 		al -= 6;
3310 		cf = old_cf | (al >= 250);
3311 		af = true;
3312 	} else {
3313 		af = false;
3314 	}
3315 	if (old_al > 0x99 || old_cf) {
3316 		al -= 0x60;
3317 		cf = true;
3318 	}
3319 
3320 	ctxt->dst.val = al;
3321 	/* Set PF, ZF, SF */
3322 	ctxt->src.type = OP_IMM;
3323 	ctxt->src.val = 0;
3324 	ctxt->src.bytes = 1;
3325 	fastop(ctxt, em_or);
3326 	ctxt->eflags &= ~(X86_EFLAGS_AF | X86_EFLAGS_CF);
3327 	if (cf)
3328 		ctxt->eflags |= X86_EFLAGS_CF;
3329 	if (af)
3330 		ctxt->eflags |= X86_EFLAGS_AF;
3331 	return X86EMUL_CONTINUE;
3332 }
3333 
3334 static int em_aam(struct x86_emulate_ctxt *ctxt)
3335 {
3336 	u8 al, ah;
3337 
3338 	if (ctxt->src.val == 0)
3339 		return emulate_de(ctxt);
3340 
3341 	al = ctxt->dst.val & 0xff;
3342 	ah = al / ctxt->src.val;
3343 	al %= ctxt->src.val;
3344 
3345 	ctxt->dst.val = (ctxt->dst.val & 0xffff0000) | al | (ah << 8);
3346 
3347 	/* Set PF, ZF, SF */
3348 	ctxt->src.type = OP_IMM;
3349 	ctxt->src.val = 0;
3350 	ctxt->src.bytes = 1;
3351 	fastop(ctxt, em_or);
3352 
3353 	return X86EMUL_CONTINUE;
3354 }
3355 
3356 static int em_aad(struct x86_emulate_ctxt *ctxt)
3357 {
3358 	u8 al = ctxt->dst.val & 0xff;
3359 	u8 ah = (ctxt->dst.val >> 8) & 0xff;
3360 
3361 	al = (al + (ah * ctxt->src.val)) & 0xff;
3362 
3363 	ctxt->dst.val = (ctxt->dst.val & 0xffff0000) | al;
3364 
3365 	/* Set PF, ZF, SF */
3366 	ctxt->src.type = OP_IMM;
3367 	ctxt->src.val = 0;
3368 	ctxt->src.bytes = 1;
3369 	fastop(ctxt, em_or);
3370 
3371 	return X86EMUL_CONTINUE;
3372 }
3373 
3374 static int em_call(struct x86_emulate_ctxt *ctxt)
3375 {
3376 	int rc;
3377 	long rel = ctxt->src.val;
3378 
3379 	ctxt->src.val = (unsigned long)ctxt->_eip;
3380 	rc = jmp_rel(ctxt, rel);
3381 	if (rc != X86EMUL_CONTINUE)
3382 		return rc;
3383 	return em_push(ctxt);
3384 }
3385 
3386 static int em_call_far(struct x86_emulate_ctxt *ctxt)
3387 {
3388 	u16 sel, old_cs;
3389 	ulong old_eip;
3390 	int rc;
3391 	struct desc_struct old_desc, new_desc;
3392 	const struct x86_emulate_ops *ops = ctxt->ops;
3393 	int cpl = ctxt->ops->cpl(ctxt);
3394 	enum x86emul_mode prev_mode = ctxt->mode;
3395 
3396 	old_eip = ctxt->_eip;
3397 	ops->get_segment(ctxt, &old_cs, &old_desc, NULL, VCPU_SREG_CS);
3398 
3399 	memcpy(&sel, ctxt->src.valptr + ctxt->op_bytes, 2);
3400 	rc = __load_segment_descriptor(ctxt, sel, VCPU_SREG_CS, cpl,
3401 				       X86_TRANSFER_CALL_JMP, &new_desc);
3402 	if (rc != X86EMUL_CONTINUE)
3403 		return rc;
3404 
3405 	rc = assign_eip_far(ctxt, ctxt->src.val, &new_desc);
3406 	if (rc != X86EMUL_CONTINUE)
3407 		goto fail;
3408 
3409 	ctxt->src.val = old_cs;
3410 	rc = em_push(ctxt);
3411 	if (rc != X86EMUL_CONTINUE)
3412 		goto fail;
3413 
3414 	ctxt->src.val = old_eip;
3415 	rc = em_push(ctxt);
3416 	/* If we failed, we tainted the memory, but the very least we should
3417 	   restore cs */
3418 	if (rc != X86EMUL_CONTINUE) {
3419 		pr_warn_once("faulting far call emulation tainted memory\n");
3420 		goto fail;
3421 	}
3422 	return rc;
3423 fail:
3424 	ops->set_segment(ctxt, old_cs, &old_desc, 0, VCPU_SREG_CS);
3425 	ctxt->mode = prev_mode;
3426 	return rc;
3427 
3428 }
3429 
3430 static int em_ret_near_imm(struct x86_emulate_ctxt *ctxt)
3431 {
3432 	int rc;
3433 	unsigned long eip;
3434 
3435 	rc = emulate_pop(ctxt, &eip, ctxt->op_bytes);
3436 	if (rc != X86EMUL_CONTINUE)
3437 		return rc;
3438 	rc = assign_eip_near(ctxt, eip);
3439 	if (rc != X86EMUL_CONTINUE)
3440 		return rc;
3441 	rsp_increment(ctxt, ctxt->src.val);
3442 	return X86EMUL_CONTINUE;
3443 }
3444 
3445 static int em_xchg(struct x86_emulate_ctxt *ctxt)
3446 {
3447 	/* Write back the register source. */
3448 	ctxt->src.val = ctxt->dst.val;
3449 	write_register_operand(&ctxt->src);
3450 
3451 	/* Write back the memory destination with implicit LOCK prefix. */
3452 	ctxt->dst.val = ctxt->src.orig_val;
3453 	ctxt->lock_prefix = 1;
3454 	return X86EMUL_CONTINUE;
3455 }
3456 
3457 static int em_imul_3op(struct x86_emulate_ctxt *ctxt)
3458 {
3459 	ctxt->dst.val = ctxt->src2.val;
3460 	return fastop(ctxt, em_imul);
3461 }
3462 
3463 static int em_cwd(struct x86_emulate_ctxt *ctxt)
3464 {
3465 	ctxt->dst.type = OP_REG;
3466 	ctxt->dst.bytes = ctxt->src.bytes;
3467 	ctxt->dst.addr.reg = reg_rmw(ctxt, VCPU_REGS_RDX);
3468 	ctxt->dst.val = ~((ctxt->src.val >> (ctxt->src.bytes * 8 - 1)) - 1);
3469 
3470 	return X86EMUL_CONTINUE;
3471 }
3472 
3473 static int em_rdtsc(struct x86_emulate_ctxt *ctxt)
3474 {
3475 	u64 tsc = 0;
3476 
3477 	ctxt->ops->get_msr(ctxt, MSR_IA32_TSC, &tsc);
3478 	*reg_write(ctxt, VCPU_REGS_RAX) = (u32)tsc;
3479 	*reg_write(ctxt, VCPU_REGS_RDX) = tsc >> 32;
3480 	return X86EMUL_CONTINUE;
3481 }
3482 
3483 static int em_rdpmc(struct x86_emulate_ctxt *ctxt)
3484 {
3485 	u64 pmc;
3486 
3487 	if (ctxt->ops->read_pmc(ctxt, reg_read(ctxt, VCPU_REGS_RCX), &pmc))
3488 		return emulate_gp(ctxt, 0);
3489 	*reg_write(ctxt, VCPU_REGS_RAX) = (u32)pmc;
3490 	*reg_write(ctxt, VCPU_REGS_RDX) = pmc >> 32;
3491 	return X86EMUL_CONTINUE;
3492 }
3493 
3494 static int em_mov(struct x86_emulate_ctxt *ctxt)
3495 {
3496 	memcpy(ctxt->dst.valptr, ctxt->src.valptr, sizeof(ctxt->src.valptr));
3497 	return X86EMUL_CONTINUE;
3498 }
3499 
3500 #define FFL(x) bit(X86_FEATURE_##x)
3501 
3502 static int em_movbe(struct x86_emulate_ctxt *ctxt)
3503 {
3504 	u32 ebx, ecx, edx, eax = 1;
3505 	u16 tmp;
3506 
3507 	/*
3508 	 * Check MOVBE is set in the guest-visible CPUID leaf.
3509 	 */
3510 	ctxt->ops->get_cpuid(ctxt, &eax, &ebx, &ecx, &edx);
3511 	if (!(ecx & FFL(MOVBE)))
3512 		return emulate_ud(ctxt);
3513 
3514 	switch (ctxt->op_bytes) {
3515 	case 2:
3516 		/*
3517 		 * From MOVBE definition: "...When the operand size is 16 bits,
3518 		 * the upper word of the destination register remains unchanged
3519 		 * ..."
3520 		 *
3521 		 * Both casting ->valptr and ->val to u16 breaks strict aliasing
3522 		 * rules so we have to do the operation almost per hand.
3523 		 */
3524 		tmp = (u16)ctxt->src.val;
3525 		ctxt->dst.val &= ~0xffffUL;
3526 		ctxt->dst.val |= (unsigned long)swab16(tmp);
3527 		break;
3528 	case 4:
3529 		ctxt->dst.val = swab32((u32)ctxt->src.val);
3530 		break;
3531 	case 8:
3532 		ctxt->dst.val = swab64(ctxt->src.val);
3533 		break;
3534 	default:
3535 		BUG();
3536 	}
3537 	return X86EMUL_CONTINUE;
3538 }
3539 
3540 static int em_cr_write(struct x86_emulate_ctxt *ctxt)
3541 {
3542 	if (ctxt->ops->set_cr(ctxt, ctxt->modrm_reg, ctxt->src.val))
3543 		return emulate_gp(ctxt, 0);
3544 
3545 	/* Disable writeback. */
3546 	ctxt->dst.type = OP_NONE;
3547 	return X86EMUL_CONTINUE;
3548 }
3549 
3550 static int em_dr_write(struct x86_emulate_ctxt *ctxt)
3551 {
3552 	unsigned long val;
3553 
3554 	if (ctxt->mode == X86EMUL_MODE_PROT64)
3555 		val = ctxt->src.val & ~0ULL;
3556 	else
3557 		val = ctxt->src.val & ~0U;
3558 
3559 	/* #UD condition is already handled. */
3560 	if (ctxt->ops->set_dr(ctxt, ctxt->modrm_reg, val) < 0)
3561 		return emulate_gp(ctxt, 0);
3562 
3563 	/* Disable writeback. */
3564 	ctxt->dst.type = OP_NONE;
3565 	return X86EMUL_CONTINUE;
3566 }
3567 
3568 static int em_wrmsr(struct x86_emulate_ctxt *ctxt)
3569 {
3570 	u64 msr_data;
3571 
3572 	msr_data = (u32)reg_read(ctxt, VCPU_REGS_RAX)
3573 		| ((u64)reg_read(ctxt, VCPU_REGS_RDX) << 32);
3574 	if (ctxt->ops->set_msr(ctxt, reg_read(ctxt, VCPU_REGS_RCX), msr_data))
3575 		return emulate_gp(ctxt, 0);
3576 
3577 	return X86EMUL_CONTINUE;
3578 }
3579 
3580 static int em_rdmsr(struct x86_emulate_ctxt *ctxt)
3581 {
3582 	u64 msr_data;
3583 
3584 	if (ctxt->ops->get_msr(ctxt, reg_read(ctxt, VCPU_REGS_RCX), &msr_data))
3585 		return emulate_gp(ctxt, 0);
3586 
3587 	*reg_write(ctxt, VCPU_REGS_RAX) = (u32)msr_data;
3588 	*reg_write(ctxt, VCPU_REGS_RDX) = msr_data >> 32;
3589 	return X86EMUL_CONTINUE;
3590 }
3591 
3592 static int em_mov_rm_sreg(struct x86_emulate_ctxt *ctxt)
3593 {
3594 	if (ctxt->modrm_reg > VCPU_SREG_GS)
3595 		return emulate_ud(ctxt);
3596 
3597 	ctxt->dst.val = get_segment_selector(ctxt, ctxt->modrm_reg);
3598 	if (ctxt->dst.bytes == 4 && ctxt->dst.type == OP_MEM)
3599 		ctxt->dst.bytes = 2;
3600 	return X86EMUL_CONTINUE;
3601 }
3602 
3603 static int em_mov_sreg_rm(struct x86_emulate_ctxt *ctxt)
3604 {
3605 	u16 sel = ctxt->src.val;
3606 
3607 	if (ctxt->modrm_reg == VCPU_SREG_CS || ctxt->modrm_reg > VCPU_SREG_GS)
3608 		return emulate_ud(ctxt);
3609 
3610 	if (ctxt->modrm_reg == VCPU_SREG_SS)
3611 		ctxt->interruptibility = KVM_X86_SHADOW_INT_MOV_SS;
3612 
3613 	/* Disable writeback. */
3614 	ctxt->dst.type = OP_NONE;
3615 	return load_segment_descriptor(ctxt, sel, ctxt->modrm_reg);
3616 }
3617 
3618 static int em_lldt(struct x86_emulate_ctxt *ctxt)
3619 {
3620 	u16 sel = ctxt->src.val;
3621 
3622 	/* Disable writeback. */
3623 	ctxt->dst.type = OP_NONE;
3624 	return load_segment_descriptor(ctxt, sel, VCPU_SREG_LDTR);
3625 }
3626 
3627 static int em_ltr(struct x86_emulate_ctxt *ctxt)
3628 {
3629 	u16 sel = ctxt->src.val;
3630 
3631 	/* Disable writeback. */
3632 	ctxt->dst.type = OP_NONE;
3633 	return load_segment_descriptor(ctxt, sel, VCPU_SREG_TR);
3634 }
3635 
3636 static int em_invlpg(struct x86_emulate_ctxt *ctxt)
3637 {
3638 	int rc;
3639 	ulong linear;
3640 
3641 	rc = linearize(ctxt, ctxt->src.addr.mem, 1, false, &linear);
3642 	if (rc == X86EMUL_CONTINUE)
3643 		ctxt->ops->invlpg(ctxt, linear);
3644 	/* Disable writeback. */
3645 	ctxt->dst.type = OP_NONE;
3646 	return X86EMUL_CONTINUE;
3647 }
3648 
3649 static int em_clts(struct x86_emulate_ctxt *ctxt)
3650 {
3651 	ulong cr0;
3652 
3653 	cr0 = ctxt->ops->get_cr(ctxt, 0);
3654 	cr0 &= ~X86_CR0_TS;
3655 	ctxt->ops->set_cr(ctxt, 0, cr0);
3656 	return X86EMUL_CONTINUE;
3657 }
3658 
3659 static int em_hypercall(struct x86_emulate_ctxt *ctxt)
3660 {
3661 	int rc = ctxt->ops->fix_hypercall(ctxt);
3662 
3663 	if (rc != X86EMUL_CONTINUE)
3664 		return rc;
3665 
3666 	/* Let the processor re-execute the fixed hypercall */
3667 	ctxt->_eip = ctxt->eip;
3668 	/* Disable writeback. */
3669 	ctxt->dst.type = OP_NONE;
3670 	return X86EMUL_CONTINUE;
3671 }
3672 
3673 static int emulate_store_desc_ptr(struct x86_emulate_ctxt *ctxt,
3674 				  void (*get)(struct x86_emulate_ctxt *ctxt,
3675 					      struct desc_ptr *ptr))
3676 {
3677 	struct desc_ptr desc_ptr;
3678 
3679 	if (ctxt->mode == X86EMUL_MODE_PROT64)
3680 		ctxt->op_bytes = 8;
3681 	get(ctxt, &desc_ptr);
3682 	if (ctxt->op_bytes == 2) {
3683 		ctxt->op_bytes = 4;
3684 		desc_ptr.address &= 0x00ffffff;
3685 	}
3686 	/* Disable writeback. */
3687 	ctxt->dst.type = OP_NONE;
3688 	return segmented_write(ctxt, ctxt->dst.addr.mem,
3689 			       &desc_ptr, 2 + ctxt->op_bytes);
3690 }
3691 
3692 static int em_sgdt(struct x86_emulate_ctxt *ctxt)
3693 {
3694 	return emulate_store_desc_ptr(ctxt, ctxt->ops->get_gdt);
3695 }
3696 
3697 static int em_sidt(struct x86_emulate_ctxt *ctxt)
3698 {
3699 	return emulate_store_desc_ptr(ctxt, ctxt->ops->get_idt);
3700 }
3701 
3702 static int em_lgdt_lidt(struct x86_emulate_ctxt *ctxt, bool lgdt)
3703 {
3704 	struct desc_ptr desc_ptr;
3705 	int rc;
3706 
3707 	if (ctxt->mode == X86EMUL_MODE_PROT64)
3708 		ctxt->op_bytes = 8;
3709 	rc = read_descriptor(ctxt, ctxt->src.addr.mem,
3710 			     &desc_ptr.size, &desc_ptr.address,
3711 			     ctxt->op_bytes);
3712 	if (rc != X86EMUL_CONTINUE)
3713 		return rc;
3714 	if (ctxt->mode == X86EMUL_MODE_PROT64 &&
3715 	    is_noncanonical_address(desc_ptr.address))
3716 		return emulate_gp(ctxt, 0);
3717 	if (lgdt)
3718 		ctxt->ops->set_gdt(ctxt, &desc_ptr);
3719 	else
3720 		ctxt->ops->set_idt(ctxt, &desc_ptr);
3721 	/* Disable writeback. */
3722 	ctxt->dst.type = OP_NONE;
3723 	return X86EMUL_CONTINUE;
3724 }
3725 
3726 static int em_lgdt(struct x86_emulate_ctxt *ctxt)
3727 {
3728 	return em_lgdt_lidt(ctxt, true);
3729 }
3730 
3731 static int em_lidt(struct x86_emulate_ctxt *ctxt)
3732 {
3733 	return em_lgdt_lidt(ctxt, false);
3734 }
3735 
3736 static int em_smsw(struct x86_emulate_ctxt *ctxt)
3737 {
3738 	if (ctxt->dst.type == OP_MEM)
3739 		ctxt->dst.bytes = 2;
3740 	ctxt->dst.val = ctxt->ops->get_cr(ctxt, 0);
3741 	return X86EMUL_CONTINUE;
3742 }
3743 
3744 static int em_lmsw(struct x86_emulate_ctxt *ctxt)
3745 {
3746 	ctxt->ops->set_cr(ctxt, 0, (ctxt->ops->get_cr(ctxt, 0) & ~0x0eul)
3747 			  | (ctxt->src.val & 0x0f));
3748 	ctxt->dst.type = OP_NONE;
3749 	return X86EMUL_CONTINUE;
3750 }
3751 
3752 static int em_loop(struct x86_emulate_ctxt *ctxt)
3753 {
3754 	int rc = X86EMUL_CONTINUE;
3755 
3756 	register_address_increment(ctxt, VCPU_REGS_RCX, -1);
3757 	if ((address_mask(ctxt, reg_read(ctxt, VCPU_REGS_RCX)) != 0) &&
3758 	    (ctxt->b == 0xe2 || test_cc(ctxt->b ^ 0x5, ctxt->eflags)))
3759 		rc = jmp_rel(ctxt, ctxt->src.val);
3760 
3761 	return rc;
3762 }
3763 
3764 static int em_jcxz(struct x86_emulate_ctxt *ctxt)
3765 {
3766 	int rc = X86EMUL_CONTINUE;
3767 
3768 	if (address_mask(ctxt, reg_read(ctxt, VCPU_REGS_RCX)) == 0)
3769 		rc = jmp_rel(ctxt, ctxt->src.val);
3770 
3771 	return rc;
3772 }
3773 
3774 static int em_in(struct x86_emulate_ctxt *ctxt)
3775 {
3776 	if (!pio_in_emulated(ctxt, ctxt->dst.bytes, ctxt->src.val,
3777 			     &ctxt->dst.val))
3778 		return X86EMUL_IO_NEEDED;
3779 
3780 	return X86EMUL_CONTINUE;
3781 }
3782 
3783 static int em_out(struct x86_emulate_ctxt *ctxt)
3784 {
3785 	ctxt->ops->pio_out_emulated(ctxt, ctxt->src.bytes, ctxt->dst.val,
3786 				    &ctxt->src.val, 1);
3787 	/* Disable writeback. */
3788 	ctxt->dst.type = OP_NONE;
3789 	return X86EMUL_CONTINUE;
3790 }
3791 
3792 static int em_cli(struct x86_emulate_ctxt *ctxt)
3793 {
3794 	if (emulator_bad_iopl(ctxt))
3795 		return emulate_gp(ctxt, 0);
3796 
3797 	ctxt->eflags &= ~X86_EFLAGS_IF;
3798 	return X86EMUL_CONTINUE;
3799 }
3800 
3801 static int em_sti(struct x86_emulate_ctxt *ctxt)
3802 {
3803 	if (emulator_bad_iopl(ctxt))
3804 		return emulate_gp(ctxt, 0);
3805 
3806 	ctxt->interruptibility = KVM_X86_SHADOW_INT_STI;
3807 	ctxt->eflags |= X86_EFLAGS_IF;
3808 	return X86EMUL_CONTINUE;
3809 }
3810 
3811 static int em_cpuid(struct x86_emulate_ctxt *ctxt)
3812 {
3813 	u32 eax, ebx, ecx, edx;
3814 
3815 	eax = reg_read(ctxt, VCPU_REGS_RAX);
3816 	ecx = reg_read(ctxt, VCPU_REGS_RCX);
3817 	ctxt->ops->get_cpuid(ctxt, &eax, &ebx, &ecx, &edx);
3818 	*reg_write(ctxt, VCPU_REGS_RAX) = eax;
3819 	*reg_write(ctxt, VCPU_REGS_RBX) = ebx;
3820 	*reg_write(ctxt, VCPU_REGS_RCX) = ecx;
3821 	*reg_write(ctxt, VCPU_REGS_RDX) = edx;
3822 	return X86EMUL_CONTINUE;
3823 }
3824 
3825 static int em_sahf(struct x86_emulate_ctxt *ctxt)
3826 {
3827 	u32 flags;
3828 
3829 	flags = X86_EFLAGS_CF | X86_EFLAGS_PF | X86_EFLAGS_AF | X86_EFLAGS_ZF |
3830 		X86_EFLAGS_SF;
3831 	flags &= *reg_rmw(ctxt, VCPU_REGS_RAX) >> 8;
3832 
3833 	ctxt->eflags &= ~0xffUL;
3834 	ctxt->eflags |= flags | X86_EFLAGS_FIXED;
3835 	return X86EMUL_CONTINUE;
3836 }
3837 
3838 static int em_lahf(struct x86_emulate_ctxt *ctxt)
3839 {
3840 	*reg_rmw(ctxt, VCPU_REGS_RAX) &= ~0xff00UL;
3841 	*reg_rmw(ctxt, VCPU_REGS_RAX) |= (ctxt->eflags & 0xff) << 8;
3842 	return X86EMUL_CONTINUE;
3843 }
3844 
3845 static int em_bswap(struct x86_emulate_ctxt *ctxt)
3846 {
3847 	switch (ctxt->op_bytes) {
3848 #ifdef CONFIG_X86_64
3849 	case 8:
3850 		asm("bswap %0" : "+r"(ctxt->dst.val));
3851 		break;
3852 #endif
3853 	default:
3854 		asm("bswap %0" : "+r"(*(u32 *)&ctxt->dst.val));
3855 		break;
3856 	}
3857 	return X86EMUL_CONTINUE;
3858 }
3859 
3860 static int em_clflush(struct x86_emulate_ctxt *ctxt)
3861 {
3862 	/* emulating clflush regardless of cpuid */
3863 	return X86EMUL_CONTINUE;
3864 }
3865 
3866 static int em_movsxd(struct x86_emulate_ctxt *ctxt)
3867 {
3868 	ctxt->dst.val = (s32) ctxt->src.val;
3869 	return X86EMUL_CONTINUE;
3870 }
3871 
3872 static int check_fxsr(struct x86_emulate_ctxt *ctxt)
3873 {
3874 	u32 eax = 1, ebx, ecx = 0, edx;
3875 
3876 	ctxt->ops->get_cpuid(ctxt, &eax, &ebx, &ecx, &edx);
3877 	if (!(edx & FFL(FXSR)))
3878 		return emulate_ud(ctxt);
3879 
3880 	if (ctxt->ops->get_cr(ctxt, 0) & (X86_CR0_TS | X86_CR0_EM))
3881 		return emulate_nm(ctxt);
3882 
3883 	/*
3884 	 * Don't emulate a case that should never be hit, instead of working
3885 	 * around a lack of fxsave64/fxrstor64 on old compilers.
3886 	 */
3887 	if (ctxt->mode >= X86EMUL_MODE_PROT64)
3888 		return X86EMUL_UNHANDLEABLE;
3889 
3890 	return X86EMUL_CONTINUE;
3891 }
3892 
3893 /*
3894  * FXSAVE and FXRSTOR have 4 different formats depending on execution mode,
3895  *  1) 16 bit mode
3896  *  2) 32 bit mode
3897  *     - like (1), but FIP and FDP (foo) are only 16 bit.  At least Intel CPUs
3898  *       preserve whole 32 bit values, though, so (1) and (2) are the same wrt.
3899  *       save and restore
3900  *  3) 64-bit mode with REX.W prefix
3901  *     - like (2), but XMM 8-15 are being saved and restored
3902  *  4) 64-bit mode without REX.W prefix
3903  *     - like (3), but FIP and FDP are 64 bit
3904  *
3905  * Emulation uses (3) for (1) and (2) and preserves XMM 8-15 to reach the
3906  * desired result.  (4) is not emulated.
3907  *
3908  * Note: Guest and host CPUID.(EAX=07H,ECX=0H):EBX[bit 13] (deprecate FPU CS
3909  * and FPU DS) should match.
3910  */
3911 static int em_fxsave(struct x86_emulate_ctxt *ctxt)
3912 {
3913 	struct fxregs_state fx_state;
3914 	size_t size;
3915 	int rc;
3916 
3917 	rc = check_fxsr(ctxt);
3918 	if (rc != X86EMUL_CONTINUE)
3919 		return rc;
3920 
3921 	ctxt->ops->get_fpu(ctxt);
3922 
3923 	rc = asm_safe("fxsave %[fx]", , [fx] "+m"(fx_state));
3924 
3925 	ctxt->ops->put_fpu(ctxt);
3926 
3927 	if (rc != X86EMUL_CONTINUE)
3928 		return rc;
3929 
3930 	if (ctxt->ops->get_cr(ctxt, 4) & X86_CR4_OSFXSR)
3931 		size = offsetof(struct fxregs_state, xmm_space[8 * 16/4]);
3932 	else
3933 		size = offsetof(struct fxregs_state, xmm_space[0]);
3934 
3935 	return segmented_write(ctxt, ctxt->memop.addr.mem, &fx_state, size);
3936 }
3937 
3938 static int fxrstor_fixup(struct x86_emulate_ctxt *ctxt,
3939 		struct fxregs_state *new)
3940 {
3941 	int rc = X86EMUL_CONTINUE;
3942 	struct fxregs_state old;
3943 
3944 	rc = asm_safe("fxsave %[fx]", , [fx] "+m"(old));
3945 	if (rc != X86EMUL_CONTINUE)
3946 		return rc;
3947 
3948 	/*
3949 	 * 64 bit host will restore XMM 8-15, which is not correct on non-64
3950 	 * bit guests.  Load the current values in order to preserve 64 bit
3951 	 * XMMs after fxrstor.
3952 	 */
3953 #ifdef CONFIG_X86_64
3954 	/* XXX: accessing XMM 8-15 very awkwardly */
3955 	memcpy(&new->xmm_space[8 * 16/4], &old.xmm_space[8 * 16/4], 8 * 16);
3956 #endif
3957 
3958 	/*
3959 	 * Hardware doesn't save and restore XMM 0-7 without CR4.OSFXSR, but
3960 	 * does save and restore MXCSR.
3961 	 */
3962 	if (!(ctxt->ops->get_cr(ctxt, 4) & X86_CR4_OSFXSR))
3963 		memcpy(new->xmm_space, old.xmm_space, 8 * 16);
3964 
3965 	return rc;
3966 }
3967 
3968 static int em_fxrstor(struct x86_emulate_ctxt *ctxt)
3969 {
3970 	struct fxregs_state fx_state;
3971 	int rc;
3972 
3973 	rc = check_fxsr(ctxt);
3974 	if (rc != X86EMUL_CONTINUE)
3975 		return rc;
3976 
3977 	rc = segmented_read(ctxt, ctxt->memop.addr.mem, &fx_state, 512);
3978 	if (rc != X86EMUL_CONTINUE)
3979 		return rc;
3980 
3981 	if (fx_state.mxcsr >> 16)
3982 		return emulate_gp(ctxt, 0);
3983 
3984 	ctxt->ops->get_fpu(ctxt);
3985 
3986 	if (ctxt->mode < X86EMUL_MODE_PROT64)
3987 		rc = fxrstor_fixup(ctxt, &fx_state);
3988 
3989 	if (rc == X86EMUL_CONTINUE)
3990 		rc = asm_safe("fxrstor %[fx]", : [fx] "m"(fx_state));
3991 
3992 	ctxt->ops->put_fpu(ctxt);
3993 
3994 	return rc;
3995 }
3996 
3997 static bool valid_cr(int nr)
3998 {
3999 	switch (nr) {
4000 	case 0:
4001 	case 2 ... 4:
4002 	case 8:
4003 		return true;
4004 	default:
4005 		return false;
4006 	}
4007 }
4008 
4009 static int check_cr_read(struct x86_emulate_ctxt *ctxt)
4010 {
4011 	if (!valid_cr(ctxt->modrm_reg))
4012 		return emulate_ud(ctxt);
4013 
4014 	return X86EMUL_CONTINUE;
4015 }
4016 
4017 static int check_cr_write(struct x86_emulate_ctxt *ctxt)
4018 {
4019 	u64 new_val = ctxt->src.val64;
4020 	int cr = ctxt->modrm_reg;
4021 	u64 efer = 0;
4022 
4023 	static u64 cr_reserved_bits[] = {
4024 		0xffffffff00000000ULL,
4025 		0, 0, 0, /* CR3 checked later */
4026 		CR4_RESERVED_BITS,
4027 		0, 0, 0,
4028 		CR8_RESERVED_BITS,
4029 	};
4030 
4031 	if (!valid_cr(cr))
4032 		return emulate_ud(ctxt);
4033 
4034 	if (new_val & cr_reserved_bits[cr])
4035 		return emulate_gp(ctxt, 0);
4036 
4037 	switch (cr) {
4038 	case 0: {
4039 		u64 cr4;
4040 		if (((new_val & X86_CR0_PG) && !(new_val & X86_CR0_PE)) ||
4041 		    ((new_val & X86_CR0_NW) && !(new_val & X86_CR0_CD)))
4042 			return emulate_gp(ctxt, 0);
4043 
4044 		cr4 = ctxt->ops->get_cr(ctxt, 4);
4045 		ctxt->ops->get_msr(ctxt, MSR_EFER, &efer);
4046 
4047 		if ((new_val & X86_CR0_PG) && (efer & EFER_LME) &&
4048 		    !(cr4 & X86_CR4_PAE))
4049 			return emulate_gp(ctxt, 0);
4050 
4051 		break;
4052 		}
4053 	case 3: {
4054 		u64 rsvd = 0;
4055 
4056 		ctxt->ops->get_msr(ctxt, MSR_EFER, &efer);
4057 		if (efer & EFER_LMA)
4058 			rsvd = CR3_L_MODE_RESERVED_BITS & ~CR3_PCID_INVD;
4059 
4060 		if (new_val & rsvd)
4061 			return emulate_gp(ctxt, 0);
4062 
4063 		break;
4064 		}
4065 	case 4: {
4066 		ctxt->ops->get_msr(ctxt, MSR_EFER, &efer);
4067 
4068 		if ((efer & EFER_LMA) && !(new_val & X86_CR4_PAE))
4069 			return emulate_gp(ctxt, 0);
4070 
4071 		break;
4072 		}
4073 	}
4074 
4075 	return X86EMUL_CONTINUE;
4076 }
4077 
4078 static int check_dr7_gd(struct x86_emulate_ctxt *ctxt)
4079 {
4080 	unsigned long dr7;
4081 
4082 	ctxt->ops->get_dr(ctxt, 7, &dr7);
4083 
4084 	/* Check if DR7.Global_Enable is set */
4085 	return dr7 & (1 << 13);
4086 }
4087 
4088 static int check_dr_read(struct x86_emulate_ctxt *ctxt)
4089 {
4090 	int dr = ctxt->modrm_reg;
4091 	u64 cr4;
4092 
4093 	if (dr > 7)
4094 		return emulate_ud(ctxt);
4095 
4096 	cr4 = ctxt->ops->get_cr(ctxt, 4);
4097 	if ((cr4 & X86_CR4_DE) && (dr == 4 || dr == 5))
4098 		return emulate_ud(ctxt);
4099 
4100 	if (check_dr7_gd(ctxt)) {
4101 		ulong dr6;
4102 
4103 		ctxt->ops->get_dr(ctxt, 6, &dr6);
4104 		dr6 &= ~15;
4105 		dr6 |= DR6_BD | DR6_RTM;
4106 		ctxt->ops->set_dr(ctxt, 6, dr6);
4107 		return emulate_db(ctxt);
4108 	}
4109 
4110 	return X86EMUL_CONTINUE;
4111 }
4112 
4113 static int check_dr_write(struct x86_emulate_ctxt *ctxt)
4114 {
4115 	u64 new_val = ctxt->src.val64;
4116 	int dr = ctxt->modrm_reg;
4117 
4118 	if ((dr == 6 || dr == 7) && (new_val & 0xffffffff00000000ULL))
4119 		return emulate_gp(ctxt, 0);
4120 
4121 	return check_dr_read(ctxt);
4122 }
4123 
4124 static int check_svme(struct x86_emulate_ctxt *ctxt)
4125 {
4126 	u64 efer;
4127 
4128 	ctxt->ops->get_msr(ctxt, MSR_EFER, &efer);
4129 
4130 	if (!(efer & EFER_SVME))
4131 		return emulate_ud(ctxt);
4132 
4133 	return X86EMUL_CONTINUE;
4134 }
4135 
4136 static int check_svme_pa(struct x86_emulate_ctxt *ctxt)
4137 {
4138 	u64 rax = reg_read(ctxt, VCPU_REGS_RAX);
4139 
4140 	/* Valid physical address? */
4141 	if (rax & 0xffff000000000000ULL)
4142 		return emulate_gp(ctxt, 0);
4143 
4144 	return check_svme(ctxt);
4145 }
4146 
4147 static int check_rdtsc(struct x86_emulate_ctxt *ctxt)
4148 {
4149 	u64 cr4 = ctxt->ops->get_cr(ctxt, 4);
4150 
4151 	if (cr4 & X86_CR4_TSD && ctxt->ops->cpl(ctxt))
4152 		return emulate_ud(ctxt);
4153 
4154 	return X86EMUL_CONTINUE;
4155 }
4156 
4157 static int check_rdpmc(struct x86_emulate_ctxt *ctxt)
4158 {
4159 	u64 cr4 = ctxt->ops->get_cr(ctxt, 4);
4160 	u64 rcx = reg_read(ctxt, VCPU_REGS_RCX);
4161 
4162 	if ((!(cr4 & X86_CR4_PCE) && ctxt->ops->cpl(ctxt)) ||
4163 	    ctxt->ops->check_pmc(ctxt, rcx))
4164 		return emulate_gp(ctxt, 0);
4165 
4166 	return X86EMUL_CONTINUE;
4167 }
4168 
4169 static int check_perm_in(struct x86_emulate_ctxt *ctxt)
4170 {
4171 	ctxt->dst.bytes = min(ctxt->dst.bytes, 4u);
4172 	if (!emulator_io_permited(ctxt, ctxt->src.val, ctxt->dst.bytes))
4173 		return emulate_gp(ctxt, 0);
4174 
4175 	return X86EMUL_CONTINUE;
4176 }
4177 
4178 static int check_perm_out(struct x86_emulate_ctxt *ctxt)
4179 {
4180 	ctxt->src.bytes = min(ctxt->src.bytes, 4u);
4181 	if (!emulator_io_permited(ctxt, ctxt->dst.val, ctxt->src.bytes))
4182 		return emulate_gp(ctxt, 0);
4183 
4184 	return X86EMUL_CONTINUE;
4185 }
4186 
4187 #define D(_y) { .flags = (_y) }
4188 #define DI(_y, _i) { .flags = (_y)|Intercept, .intercept = x86_intercept_##_i }
4189 #define DIP(_y, _i, _p) { .flags = (_y)|Intercept|CheckPerm, \
4190 		      .intercept = x86_intercept_##_i, .check_perm = (_p) }
4191 #define N    D(NotImpl)
4192 #define EXT(_f, _e) { .flags = ((_f) | RMExt), .u.group = (_e) }
4193 #define G(_f, _g) { .flags = ((_f) | Group | ModRM), .u.group = (_g) }
4194 #define GD(_f, _g) { .flags = ((_f) | GroupDual | ModRM), .u.gdual = (_g) }
4195 #define ID(_f, _i) { .flags = ((_f) | InstrDual | ModRM), .u.idual = (_i) }
4196 #define MD(_f, _m) { .flags = ((_f) | ModeDual), .u.mdual = (_m) }
4197 #define E(_f, _e) { .flags = ((_f) | Escape | ModRM), .u.esc = (_e) }
4198 #define I(_f, _e) { .flags = (_f), .u.execute = (_e) }
4199 #define F(_f, _e) { .flags = (_f) | Fastop, .u.fastop = (_e) }
4200 #define II(_f, _e, _i) \
4201 	{ .flags = (_f)|Intercept, .u.execute = (_e), .intercept = x86_intercept_##_i }
4202 #define IIP(_f, _e, _i, _p) \
4203 	{ .flags = (_f)|Intercept|CheckPerm, .u.execute = (_e), \
4204 	  .intercept = x86_intercept_##_i, .check_perm = (_p) }
4205 #define GP(_f, _g) { .flags = ((_f) | Prefix), .u.gprefix = (_g) }
4206 
4207 #define D2bv(_f)      D((_f) | ByteOp), D(_f)
4208 #define D2bvIP(_f, _i, _p) DIP((_f) | ByteOp, _i, _p), DIP(_f, _i, _p)
4209 #define I2bv(_f, _e)  I((_f) | ByteOp, _e), I(_f, _e)
4210 #define F2bv(_f, _e)  F((_f) | ByteOp, _e), F(_f, _e)
4211 #define I2bvIP(_f, _e, _i, _p) \
4212 	IIP((_f) | ByteOp, _e, _i, _p), IIP(_f, _e, _i, _p)
4213 
4214 #define F6ALU(_f, _e) F2bv((_f) | DstMem | SrcReg | ModRM, _e),		\
4215 		F2bv(((_f) | DstReg | SrcMem | ModRM) & ~Lock, _e),	\
4216 		F2bv(((_f) & ~Lock) | DstAcc | SrcImm, _e)
4217 
4218 static const struct opcode group7_rm0[] = {
4219 	N,
4220 	I(SrcNone | Priv | EmulateOnUD,	em_hypercall),
4221 	N, N, N, N, N, N,
4222 };
4223 
4224 static const struct opcode group7_rm1[] = {
4225 	DI(SrcNone | Priv, monitor),
4226 	DI(SrcNone | Priv, mwait),
4227 	N, N, N, N, N, N,
4228 };
4229 
4230 static const struct opcode group7_rm3[] = {
4231 	DIP(SrcNone | Prot | Priv,		vmrun,		check_svme_pa),
4232 	II(SrcNone  | Prot | EmulateOnUD,	em_hypercall,	vmmcall),
4233 	DIP(SrcNone | Prot | Priv,		vmload,		check_svme_pa),
4234 	DIP(SrcNone | Prot | Priv,		vmsave,		check_svme_pa),
4235 	DIP(SrcNone | Prot | Priv,		stgi,		check_svme),
4236 	DIP(SrcNone | Prot | Priv,		clgi,		check_svme),
4237 	DIP(SrcNone | Prot | Priv,		skinit,		check_svme),
4238 	DIP(SrcNone | Prot | Priv,		invlpga,	check_svme),
4239 };
4240 
4241 static const struct opcode group7_rm7[] = {
4242 	N,
4243 	DIP(SrcNone, rdtscp, check_rdtsc),
4244 	N, N, N, N, N, N,
4245 };
4246 
4247 static const struct opcode group1[] = {
4248 	F(Lock, em_add),
4249 	F(Lock | PageTable, em_or),
4250 	F(Lock, em_adc),
4251 	F(Lock, em_sbb),
4252 	F(Lock | PageTable, em_and),
4253 	F(Lock, em_sub),
4254 	F(Lock, em_xor),
4255 	F(NoWrite, em_cmp),
4256 };
4257 
4258 static const struct opcode group1A[] = {
4259 	I(DstMem | SrcNone | Mov | Stack | IncSP, em_pop), N, N, N, N, N, N, N,
4260 };
4261 
4262 static const struct opcode group2[] = {
4263 	F(DstMem | ModRM, em_rol),
4264 	F(DstMem | ModRM, em_ror),
4265 	F(DstMem | ModRM, em_rcl),
4266 	F(DstMem | ModRM, em_rcr),
4267 	F(DstMem | ModRM, em_shl),
4268 	F(DstMem | ModRM, em_shr),
4269 	F(DstMem | ModRM, em_shl),
4270 	F(DstMem | ModRM, em_sar),
4271 };
4272 
4273 static const struct opcode group3[] = {
4274 	F(DstMem | SrcImm | NoWrite, em_test),
4275 	F(DstMem | SrcImm | NoWrite, em_test),
4276 	F(DstMem | SrcNone | Lock, em_not),
4277 	F(DstMem | SrcNone | Lock, em_neg),
4278 	F(DstXacc | Src2Mem, em_mul_ex),
4279 	F(DstXacc | Src2Mem, em_imul_ex),
4280 	F(DstXacc | Src2Mem, em_div_ex),
4281 	F(DstXacc | Src2Mem, em_idiv_ex),
4282 };
4283 
4284 static const struct opcode group4[] = {
4285 	F(ByteOp | DstMem | SrcNone | Lock, em_inc),
4286 	F(ByteOp | DstMem | SrcNone | Lock, em_dec),
4287 	N, N, N, N, N, N,
4288 };
4289 
4290 static const struct opcode group5[] = {
4291 	F(DstMem | SrcNone | Lock,		em_inc),
4292 	F(DstMem | SrcNone | Lock,		em_dec),
4293 	I(SrcMem | NearBranch,			em_call_near_abs),
4294 	I(SrcMemFAddr | ImplicitOps,		em_call_far),
4295 	I(SrcMem | NearBranch,			em_jmp_abs),
4296 	I(SrcMemFAddr | ImplicitOps,		em_jmp_far),
4297 	I(SrcMem | Stack,			em_push), D(Undefined),
4298 };
4299 
4300 static const struct opcode group6[] = {
4301 	DI(Prot | DstMem,	sldt),
4302 	DI(Prot | DstMem,	str),
4303 	II(Prot | Priv | SrcMem16, em_lldt, lldt),
4304 	II(Prot | Priv | SrcMem16, em_ltr, ltr),
4305 	N, N, N, N,
4306 };
4307 
4308 static const struct group_dual group7 = { {
4309 	II(Mov | DstMem,			em_sgdt, sgdt),
4310 	II(Mov | DstMem,			em_sidt, sidt),
4311 	II(SrcMem | Priv,			em_lgdt, lgdt),
4312 	II(SrcMem | Priv,			em_lidt, lidt),
4313 	II(SrcNone | DstMem | Mov,		em_smsw, smsw), N,
4314 	II(SrcMem16 | Mov | Priv,		em_lmsw, lmsw),
4315 	II(SrcMem | ByteOp | Priv | NoAccess,	em_invlpg, invlpg),
4316 }, {
4317 	EXT(0, group7_rm0),
4318 	EXT(0, group7_rm1),
4319 	N, EXT(0, group7_rm3),
4320 	II(SrcNone | DstMem | Mov,		em_smsw, smsw), N,
4321 	II(SrcMem16 | Mov | Priv,		em_lmsw, lmsw),
4322 	EXT(0, group7_rm7),
4323 } };
4324 
4325 static const struct opcode group8[] = {
4326 	N, N, N, N,
4327 	F(DstMem | SrcImmByte | NoWrite,		em_bt),
4328 	F(DstMem | SrcImmByte | Lock | PageTable,	em_bts),
4329 	F(DstMem | SrcImmByte | Lock,			em_btr),
4330 	F(DstMem | SrcImmByte | Lock | PageTable,	em_btc),
4331 };
4332 
4333 static const struct group_dual group9 = { {
4334 	N, I(DstMem64 | Lock | PageTable, em_cmpxchg8b), N, N, N, N, N, N,
4335 }, {
4336 	N, N, N, N, N, N, N, N,
4337 } };
4338 
4339 static const struct opcode group11[] = {
4340 	I(DstMem | SrcImm | Mov | PageTable, em_mov),
4341 	X7(D(Undefined)),
4342 };
4343 
4344 static const struct gprefix pfx_0f_ae_7 = {
4345 	I(SrcMem | ByteOp, em_clflush), N, N, N,
4346 };
4347 
4348 static const struct group_dual group15 = { {
4349 	I(ModRM | Aligned16, em_fxsave),
4350 	I(ModRM | Aligned16, em_fxrstor),
4351 	N, N, N, N, N, GP(0, &pfx_0f_ae_7),
4352 }, {
4353 	N, N, N, N, N, N, N, N,
4354 } };
4355 
4356 static const struct gprefix pfx_0f_6f_0f_7f = {
4357 	I(Mmx, em_mov), I(Sse | Aligned, em_mov), N, I(Sse | Unaligned, em_mov),
4358 };
4359 
4360 static const struct instr_dual instr_dual_0f_2b = {
4361 	I(0, em_mov), N
4362 };
4363 
4364 static const struct gprefix pfx_0f_2b = {
4365 	ID(0, &instr_dual_0f_2b), ID(0, &instr_dual_0f_2b), N, N,
4366 };
4367 
4368 static const struct gprefix pfx_0f_28_0f_29 = {
4369 	I(Aligned, em_mov), I(Aligned, em_mov), N, N,
4370 };
4371 
4372 static const struct gprefix pfx_0f_e7 = {
4373 	N, I(Sse, em_mov), N, N,
4374 };
4375 
4376 static const struct escape escape_d9 = { {
4377 	N, N, N, N, N, N, N, I(DstMem16 | Mov, em_fnstcw),
4378 }, {
4379 	/* 0xC0 - 0xC7 */
4380 	N, N, N, N, N, N, N, N,
4381 	/* 0xC8 - 0xCF */
4382 	N, N, N, N, N, N, N, N,
4383 	/* 0xD0 - 0xC7 */
4384 	N, N, N, N, N, N, N, N,
4385 	/* 0xD8 - 0xDF */
4386 	N, N, N, N, N, N, N, N,
4387 	/* 0xE0 - 0xE7 */
4388 	N, N, N, N, N, N, N, N,
4389 	/* 0xE8 - 0xEF */
4390 	N, N, N, N, N, N, N, N,
4391 	/* 0xF0 - 0xF7 */
4392 	N, N, N, N, N, N, N, N,
4393 	/* 0xF8 - 0xFF */
4394 	N, N, N, N, N, N, N, N,
4395 } };
4396 
4397 static const struct escape escape_db = { {
4398 	N, N, N, N, N, N, N, N,
4399 }, {
4400 	/* 0xC0 - 0xC7 */
4401 	N, N, N, N, N, N, N, N,
4402 	/* 0xC8 - 0xCF */
4403 	N, N, N, N, N, N, N, N,
4404 	/* 0xD0 - 0xC7 */
4405 	N, N, N, N, N, N, N, N,
4406 	/* 0xD8 - 0xDF */
4407 	N, N, N, N, N, N, N, N,
4408 	/* 0xE0 - 0xE7 */
4409 	N, N, N, I(ImplicitOps, em_fninit), N, N, N, N,
4410 	/* 0xE8 - 0xEF */
4411 	N, N, N, N, N, N, N, N,
4412 	/* 0xF0 - 0xF7 */
4413 	N, N, N, N, N, N, N, N,
4414 	/* 0xF8 - 0xFF */
4415 	N, N, N, N, N, N, N, N,
4416 } };
4417 
4418 static const struct escape escape_dd = { {
4419 	N, N, N, N, N, N, N, I(DstMem16 | Mov, em_fnstsw),
4420 }, {
4421 	/* 0xC0 - 0xC7 */
4422 	N, N, N, N, N, N, N, N,
4423 	/* 0xC8 - 0xCF */
4424 	N, N, N, N, N, N, N, N,
4425 	/* 0xD0 - 0xC7 */
4426 	N, N, N, N, N, N, N, N,
4427 	/* 0xD8 - 0xDF */
4428 	N, N, N, N, N, N, N, N,
4429 	/* 0xE0 - 0xE7 */
4430 	N, N, N, N, N, N, N, N,
4431 	/* 0xE8 - 0xEF */
4432 	N, N, N, N, N, N, N, N,
4433 	/* 0xF0 - 0xF7 */
4434 	N, N, N, N, N, N, N, N,
4435 	/* 0xF8 - 0xFF */
4436 	N, N, N, N, N, N, N, N,
4437 } };
4438 
4439 static const struct instr_dual instr_dual_0f_c3 = {
4440 	I(DstMem | SrcReg | ModRM | No16 | Mov, em_mov), N
4441 };
4442 
4443 static const struct mode_dual mode_dual_63 = {
4444 	N, I(DstReg | SrcMem32 | ModRM | Mov, em_movsxd)
4445 };
4446 
4447 static const struct opcode opcode_table[256] = {
4448 	/* 0x00 - 0x07 */
4449 	F6ALU(Lock, em_add),
4450 	I(ImplicitOps | Stack | No64 | Src2ES, em_push_sreg),
4451 	I(ImplicitOps | Stack | No64 | Src2ES, em_pop_sreg),
4452 	/* 0x08 - 0x0F */
4453 	F6ALU(Lock | PageTable, em_or),
4454 	I(ImplicitOps | Stack | No64 | Src2CS, em_push_sreg),
4455 	N,
4456 	/* 0x10 - 0x17 */
4457 	F6ALU(Lock, em_adc),
4458 	I(ImplicitOps | Stack | No64 | Src2SS, em_push_sreg),
4459 	I(ImplicitOps | Stack | No64 | Src2SS, em_pop_sreg),
4460 	/* 0x18 - 0x1F */
4461 	F6ALU(Lock, em_sbb),
4462 	I(ImplicitOps | Stack | No64 | Src2DS, em_push_sreg),
4463 	I(ImplicitOps | Stack | No64 | Src2DS, em_pop_sreg),
4464 	/* 0x20 - 0x27 */
4465 	F6ALU(Lock | PageTable, em_and), N, N,
4466 	/* 0x28 - 0x2F */
4467 	F6ALU(Lock, em_sub), N, I(ByteOp | DstAcc | No64, em_das),
4468 	/* 0x30 - 0x37 */
4469 	F6ALU(Lock, em_xor), N, N,
4470 	/* 0x38 - 0x3F */
4471 	F6ALU(NoWrite, em_cmp), N, N,
4472 	/* 0x40 - 0x4F */
4473 	X8(F(DstReg, em_inc)), X8(F(DstReg, em_dec)),
4474 	/* 0x50 - 0x57 */
4475 	X8(I(SrcReg | Stack, em_push)),
4476 	/* 0x58 - 0x5F */
4477 	X8(I(DstReg | Stack, em_pop)),
4478 	/* 0x60 - 0x67 */
4479 	I(ImplicitOps | Stack | No64, em_pusha),
4480 	I(ImplicitOps | Stack | No64, em_popa),
4481 	N, MD(ModRM, &mode_dual_63),
4482 	N, N, N, N,
4483 	/* 0x68 - 0x6F */
4484 	I(SrcImm | Mov | Stack, em_push),
4485 	I(DstReg | SrcMem | ModRM | Src2Imm, em_imul_3op),
4486 	I(SrcImmByte | Mov | Stack, em_push),
4487 	I(DstReg | SrcMem | ModRM | Src2ImmByte, em_imul_3op),
4488 	I2bvIP(DstDI | SrcDX | Mov | String | Unaligned, em_in, ins, check_perm_in), /* insb, insw/insd */
4489 	I2bvIP(SrcSI | DstDX | String, em_out, outs, check_perm_out), /* outsb, outsw/outsd */
4490 	/* 0x70 - 0x7F */
4491 	X16(D(SrcImmByte | NearBranch)),
4492 	/* 0x80 - 0x87 */
4493 	G(ByteOp | DstMem | SrcImm, group1),
4494 	G(DstMem | SrcImm, group1),
4495 	G(ByteOp | DstMem | SrcImm | No64, group1),
4496 	G(DstMem | SrcImmByte, group1),
4497 	F2bv(DstMem | SrcReg | ModRM | NoWrite, em_test),
4498 	I2bv(DstMem | SrcReg | ModRM | Lock | PageTable, em_xchg),
4499 	/* 0x88 - 0x8F */
4500 	I2bv(DstMem | SrcReg | ModRM | Mov | PageTable, em_mov),
4501 	I2bv(DstReg | SrcMem | ModRM | Mov, em_mov),
4502 	I(DstMem | SrcNone | ModRM | Mov | PageTable, em_mov_rm_sreg),
4503 	D(ModRM | SrcMem | NoAccess | DstReg),
4504 	I(ImplicitOps | SrcMem16 | ModRM, em_mov_sreg_rm),
4505 	G(0, group1A),
4506 	/* 0x90 - 0x97 */
4507 	DI(SrcAcc | DstReg, pause), X7(D(SrcAcc | DstReg)),
4508 	/* 0x98 - 0x9F */
4509 	D(DstAcc | SrcNone), I(ImplicitOps | SrcAcc, em_cwd),
4510 	I(SrcImmFAddr | No64, em_call_far), N,
4511 	II(ImplicitOps | Stack, em_pushf, pushf),
4512 	II(ImplicitOps | Stack, em_popf, popf),
4513 	I(ImplicitOps, em_sahf), I(ImplicitOps, em_lahf),
4514 	/* 0xA0 - 0xA7 */
4515 	I2bv(DstAcc | SrcMem | Mov | MemAbs, em_mov),
4516 	I2bv(DstMem | SrcAcc | Mov | MemAbs | PageTable, em_mov),
4517 	I2bv(SrcSI | DstDI | Mov | String, em_mov),
4518 	F2bv(SrcSI | DstDI | String | NoWrite, em_cmp_r),
4519 	/* 0xA8 - 0xAF */
4520 	F2bv(DstAcc | SrcImm | NoWrite, em_test),
4521 	I2bv(SrcAcc | DstDI | Mov | String, em_mov),
4522 	I2bv(SrcSI | DstAcc | Mov | String, em_mov),
4523 	F2bv(SrcAcc | DstDI | String | NoWrite, em_cmp_r),
4524 	/* 0xB0 - 0xB7 */
4525 	X8(I(ByteOp | DstReg | SrcImm | Mov, em_mov)),
4526 	/* 0xB8 - 0xBF */
4527 	X8(I(DstReg | SrcImm64 | Mov, em_mov)),
4528 	/* 0xC0 - 0xC7 */
4529 	G(ByteOp | Src2ImmByte, group2), G(Src2ImmByte, group2),
4530 	I(ImplicitOps | NearBranch | SrcImmU16, em_ret_near_imm),
4531 	I(ImplicitOps | NearBranch, em_ret),
4532 	I(DstReg | SrcMemFAddr | ModRM | No64 | Src2ES, em_lseg),
4533 	I(DstReg | SrcMemFAddr | ModRM | No64 | Src2DS, em_lseg),
4534 	G(ByteOp, group11), G(0, group11),
4535 	/* 0xC8 - 0xCF */
4536 	I(Stack | SrcImmU16 | Src2ImmByte, em_enter), I(Stack, em_leave),
4537 	I(ImplicitOps | SrcImmU16, em_ret_far_imm),
4538 	I(ImplicitOps, em_ret_far),
4539 	D(ImplicitOps), DI(SrcImmByte, intn),
4540 	D(ImplicitOps | No64), II(ImplicitOps, em_iret, iret),
4541 	/* 0xD0 - 0xD7 */
4542 	G(Src2One | ByteOp, group2), G(Src2One, group2),
4543 	G(Src2CL | ByteOp, group2), G(Src2CL, group2),
4544 	I(DstAcc | SrcImmUByte | No64, em_aam),
4545 	I(DstAcc | SrcImmUByte | No64, em_aad),
4546 	F(DstAcc | ByteOp | No64, em_salc),
4547 	I(DstAcc | SrcXLat | ByteOp, em_mov),
4548 	/* 0xD8 - 0xDF */
4549 	N, E(0, &escape_d9), N, E(0, &escape_db), N, E(0, &escape_dd), N, N,
4550 	/* 0xE0 - 0xE7 */
4551 	X3(I(SrcImmByte | NearBranch, em_loop)),
4552 	I(SrcImmByte | NearBranch, em_jcxz),
4553 	I2bvIP(SrcImmUByte | DstAcc, em_in,  in,  check_perm_in),
4554 	I2bvIP(SrcAcc | DstImmUByte, em_out, out, check_perm_out),
4555 	/* 0xE8 - 0xEF */
4556 	I(SrcImm | NearBranch, em_call), D(SrcImm | ImplicitOps | NearBranch),
4557 	I(SrcImmFAddr | No64, em_jmp_far),
4558 	D(SrcImmByte | ImplicitOps | NearBranch),
4559 	I2bvIP(SrcDX | DstAcc, em_in,  in,  check_perm_in),
4560 	I2bvIP(SrcAcc | DstDX, em_out, out, check_perm_out),
4561 	/* 0xF0 - 0xF7 */
4562 	N, DI(ImplicitOps, icebp), N, N,
4563 	DI(ImplicitOps | Priv, hlt), D(ImplicitOps),
4564 	G(ByteOp, group3), G(0, group3),
4565 	/* 0xF8 - 0xFF */
4566 	D(ImplicitOps), D(ImplicitOps),
4567 	I(ImplicitOps, em_cli), I(ImplicitOps, em_sti),
4568 	D(ImplicitOps), D(ImplicitOps), G(0, group4), G(0, group5),
4569 };
4570 
4571 static const struct opcode twobyte_table[256] = {
4572 	/* 0x00 - 0x0F */
4573 	G(0, group6), GD(0, &group7), N, N,
4574 	N, I(ImplicitOps | EmulateOnUD, em_syscall),
4575 	II(ImplicitOps | Priv, em_clts, clts), N,
4576 	DI(ImplicitOps | Priv, invd), DI(ImplicitOps | Priv, wbinvd), N, N,
4577 	N, D(ImplicitOps | ModRM | SrcMem | NoAccess), N, N,
4578 	/* 0x10 - 0x1F */
4579 	N, N, N, N, N, N, N, N,
4580 	D(ImplicitOps | ModRM | SrcMem | NoAccess),
4581 	N, N, N, N, N, N, D(ImplicitOps | ModRM | SrcMem | NoAccess),
4582 	/* 0x20 - 0x2F */
4583 	DIP(ModRM | DstMem | Priv | Op3264 | NoMod, cr_read, check_cr_read),
4584 	DIP(ModRM | DstMem | Priv | Op3264 | NoMod, dr_read, check_dr_read),
4585 	IIP(ModRM | SrcMem | Priv | Op3264 | NoMod, em_cr_write, cr_write,
4586 						check_cr_write),
4587 	IIP(ModRM | SrcMem | Priv | Op3264 | NoMod, em_dr_write, dr_write,
4588 						check_dr_write),
4589 	N, N, N, N,
4590 	GP(ModRM | DstReg | SrcMem | Mov | Sse, &pfx_0f_28_0f_29),
4591 	GP(ModRM | DstMem | SrcReg | Mov | Sse, &pfx_0f_28_0f_29),
4592 	N, GP(ModRM | DstMem | SrcReg | Mov | Sse, &pfx_0f_2b),
4593 	N, N, N, N,
4594 	/* 0x30 - 0x3F */
4595 	II(ImplicitOps | Priv, em_wrmsr, wrmsr),
4596 	IIP(ImplicitOps, em_rdtsc, rdtsc, check_rdtsc),
4597 	II(ImplicitOps | Priv, em_rdmsr, rdmsr),
4598 	IIP(ImplicitOps, em_rdpmc, rdpmc, check_rdpmc),
4599 	I(ImplicitOps | EmulateOnUD, em_sysenter),
4600 	I(ImplicitOps | Priv | EmulateOnUD, em_sysexit),
4601 	N, N,
4602 	N, N, N, N, N, N, N, N,
4603 	/* 0x40 - 0x4F */
4604 	X16(D(DstReg | SrcMem | ModRM)),
4605 	/* 0x50 - 0x5F */
4606 	N, N, N, N, N, N, N, N, N, N, N, N, N, N, N, N,
4607 	/* 0x60 - 0x6F */
4608 	N, N, N, N,
4609 	N, N, N, N,
4610 	N, N, N, N,
4611 	N, N, N, GP(SrcMem | DstReg | ModRM | Mov, &pfx_0f_6f_0f_7f),
4612 	/* 0x70 - 0x7F */
4613 	N, N, N, N,
4614 	N, N, N, N,
4615 	N, N, N, N,
4616 	N, N, N, GP(SrcReg | DstMem | ModRM | Mov, &pfx_0f_6f_0f_7f),
4617 	/* 0x80 - 0x8F */
4618 	X16(D(SrcImm | NearBranch)),
4619 	/* 0x90 - 0x9F */
4620 	X16(D(ByteOp | DstMem | SrcNone | ModRM| Mov)),
4621 	/* 0xA0 - 0xA7 */
4622 	I(Stack | Src2FS, em_push_sreg), I(Stack | Src2FS, em_pop_sreg),
4623 	II(ImplicitOps, em_cpuid, cpuid),
4624 	F(DstMem | SrcReg | ModRM | BitOp | NoWrite, em_bt),
4625 	F(DstMem | SrcReg | Src2ImmByte | ModRM, em_shld),
4626 	F(DstMem | SrcReg | Src2CL | ModRM, em_shld), N, N,
4627 	/* 0xA8 - 0xAF */
4628 	I(Stack | Src2GS, em_push_sreg), I(Stack | Src2GS, em_pop_sreg),
4629 	II(EmulateOnUD | ImplicitOps, em_rsm, rsm),
4630 	F(DstMem | SrcReg | ModRM | BitOp | Lock | PageTable, em_bts),
4631 	F(DstMem | SrcReg | Src2ImmByte | ModRM, em_shrd),
4632 	F(DstMem | SrcReg | Src2CL | ModRM, em_shrd),
4633 	GD(0, &group15), F(DstReg | SrcMem | ModRM, em_imul),
4634 	/* 0xB0 - 0xB7 */
4635 	I2bv(DstMem | SrcReg | ModRM | Lock | PageTable | SrcWrite, em_cmpxchg),
4636 	I(DstReg | SrcMemFAddr | ModRM | Src2SS, em_lseg),
4637 	F(DstMem | SrcReg | ModRM | BitOp | Lock, em_btr),
4638 	I(DstReg | SrcMemFAddr | ModRM | Src2FS, em_lseg),
4639 	I(DstReg | SrcMemFAddr | ModRM | Src2GS, em_lseg),
4640 	D(DstReg | SrcMem8 | ModRM | Mov), D(DstReg | SrcMem16 | ModRM | Mov),
4641 	/* 0xB8 - 0xBF */
4642 	N, N,
4643 	G(BitOp, group8),
4644 	F(DstMem | SrcReg | ModRM | BitOp | Lock | PageTable, em_btc),
4645 	I(DstReg | SrcMem | ModRM, em_bsf_c),
4646 	I(DstReg | SrcMem | ModRM, em_bsr_c),
4647 	D(DstReg | SrcMem8 | ModRM | Mov), D(DstReg | SrcMem16 | ModRM | Mov),
4648 	/* 0xC0 - 0xC7 */
4649 	F2bv(DstMem | SrcReg | ModRM | SrcWrite | Lock, em_xadd),
4650 	N, ID(0, &instr_dual_0f_c3),
4651 	N, N, N, GD(0, &group9),
4652 	/* 0xC8 - 0xCF */
4653 	X8(I(DstReg, em_bswap)),
4654 	/* 0xD0 - 0xDF */
4655 	N, N, N, N, N, N, N, N, N, N, N, N, N, N, N, N,
4656 	/* 0xE0 - 0xEF */
4657 	N, N, N, N, N, N, N, GP(SrcReg | DstMem | ModRM | Mov, &pfx_0f_e7),
4658 	N, N, N, N, N, N, N, N,
4659 	/* 0xF0 - 0xFF */
4660 	N, N, N, N, N, N, N, N, N, N, N, N, N, N, N, N
4661 };
4662 
4663 static const struct instr_dual instr_dual_0f_38_f0 = {
4664 	I(DstReg | SrcMem | Mov, em_movbe), N
4665 };
4666 
4667 static const struct instr_dual instr_dual_0f_38_f1 = {
4668 	I(DstMem | SrcReg | Mov, em_movbe), N
4669 };
4670 
4671 static const struct gprefix three_byte_0f_38_f0 = {
4672 	ID(0, &instr_dual_0f_38_f0), N, N, N
4673 };
4674 
4675 static const struct gprefix three_byte_0f_38_f1 = {
4676 	ID(0, &instr_dual_0f_38_f1), N, N, N
4677 };
4678 
4679 /*
4680  * Insns below are selected by the prefix which indexed by the third opcode
4681  * byte.
4682  */
4683 static const struct opcode opcode_map_0f_38[256] = {
4684 	/* 0x00 - 0x7f */
4685 	X16(N), X16(N), X16(N), X16(N), X16(N), X16(N), X16(N), X16(N),
4686 	/* 0x80 - 0xef */
4687 	X16(N), X16(N), X16(N), X16(N), X16(N), X16(N), X16(N),
4688 	/* 0xf0 - 0xf1 */
4689 	GP(EmulateOnUD | ModRM, &three_byte_0f_38_f0),
4690 	GP(EmulateOnUD | ModRM, &three_byte_0f_38_f1),
4691 	/* 0xf2 - 0xff */
4692 	N, N, X4(N), X8(N)
4693 };
4694 
4695 #undef D
4696 #undef N
4697 #undef G
4698 #undef GD
4699 #undef I
4700 #undef GP
4701 #undef EXT
4702 #undef MD
4703 #undef ID
4704 
4705 #undef D2bv
4706 #undef D2bvIP
4707 #undef I2bv
4708 #undef I2bvIP
4709 #undef I6ALU
4710 
4711 static unsigned imm_size(struct x86_emulate_ctxt *ctxt)
4712 {
4713 	unsigned size;
4714 
4715 	size = (ctxt->d & ByteOp) ? 1 : ctxt->op_bytes;
4716 	if (size == 8)
4717 		size = 4;
4718 	return size;
4719 }
4720 
4721 static int decode_imm(struct x86_emulate_ctxt *ctxt, struct operand *op,
4722 		      unsigned size, bool sign_extension)
4723 {
4724 	int rc = X86EMUL_CONTINUE;
4725 
4726 	op->type = OP_IMM;
4727 	op->bytes = size;
4728 	op->addr.mem.ea = ctxt->_eip;
4729 	/* NB. Immediates are sign-extended as necessary. */
4730 	switch (op->bytes) {
4731 	case 1:
4732 		op->val = insn_fetch(s8, ctxt);
4733 		break;
4734 	case 2:
4735 		op->val = insn_fetch(s16, ctxt);
4736 		break;
4737 	case 4:
4738 		op->val = insn_fetch(s32, ctxt);
4739 		break;
4740 	case 8:
4741 		op->val = insn_fetch(s64, ctxt);
4742 		break;
4743 	}
4744 	if (!sign_extension) {
4745 		switch (op->bytes) {
4746 		case 1:
4747 			op->val &= 0xff;
4748 			break;
4749 		case 2:
4750 			op->val &= 0xffff;
4751 			break;
4752 		case 4:
4753 			op->val &= 0xffffffff;
4754 			break;
4755 		}
4756 	}
4757 done:
4758 	return rc;
4759 }
4760 
4761 static int decode_operand(struct x86_emulate_ctxt *ctxt, struct operand *op,
4762 			  unsigned d)
4763 {
4764 	int rc = X86EMUL_CONTINUE;
4765 
4766 	switch (d) {
4767 	case OpReg:
4768 		decode_register_operand(ctxt, op);
4769 		break;
4770 	case OpImmUByte:
4771 		rc = decode_imm(ctxt, op, 1, false);
4772 		break;
4773 	case OpMem:
4774 		ctxt->memop.bytes = (ctxt->d & ByteOp) ? 1 : ctxt->op_bytes;
4775 	mem_common:
4776 		*op = ctxt->memop;
4777 		ctxt->memopp = op;
4778 		if (ctxt->d & BitOp)
4779 			fetch_bit_operand(ctxt);
4780 		op->orig_val = op->val;
4781 		break;
4782 	case OpMem64:
4783 		ctxt->memop.bytes = (ctxt->op_bytes == 8) ? 16 : 8;
4784 		goto mem_common;
4785 	case OpAcc:
4786 		op->type = OP_REG;
4787 		op->bytes = (ctxt->d & ByteOp) ? 1 : ctxt->op_bytes;
4788 		op->addr.reg = reg_rmw(ctxt, VCPU_REGS_RAX);
4789 		fetch_register_operand(op);
4790 		op->orig_val = op->val;
4791 		break;
4792 	case OpAccLo:
4793 		op->type = OP_REG;
4794 		op->bytes = (ctxt->d & ByteOp) ? 2 : ctxt->op_bytes;
4795 		op->addr.reg = reg_rmw(ctxt, VCPU_REGS_RAX);
4796 		fetch_register_operand(op);
4797 		op->orig_val = op->val;
4798 		break;
4799 	case OpAccHi:
4800 		if (ctxt->d & ByteOp) {
4801 			op->type = OP_NONE;
4802 			break;
4803 		}
4804 		op->type = OP_REG;
4805 		op->bytes = ctxt->op_bytes;
4806 		op->addr.reg = reg_rmw(ctxt, VCPU_REGS_RDX);
4807 		fetch_register_operand(op);
4808 		op->orig_val = op->val;
4809 		break;
4810 	case OpDI:
4811 		op->type = OP_MEM;
4812 		op->bytes = (ctxt->d & ByteOp) ? 1 : ctxt->op_bytes;
4813 		op->addr.mem.ea =
4814 			register_address(ctxt, VCPU_REGS_RDI);
4815 		op->addr.mem.seg = VCPU_SREG_ES;
4816 		op->val = 0;
4817 		op->count = 1;
4818 		break;
4819 	case OpDX:
4820 		op->type = OP_REG;
4821 		op->bytes = 2;
4822 		op->addr.reg = reg_rmw(ctxt, VCPU_REGS_RDX);
4823 		fetch_register_operand(op);
4824 		break;
4825 	case OpCL:
4826 		op->type = OP_IMM;
4827 		op->bytes = 1;
4828 		op->val = reg_read(ctxt, VCPU_REGS_RCX) & 0xff;
4829 		break;
4830 	case OpImmByte:
4831 		rc = decode_imm(ctxt, op, 1, true);
4832 		break;
4833 	case OpOne:
4834 		op->type = OP_IMM;
4835 		op->bytes = 1;
4836 		op->val = 1;
4837 		break;
4838 	case OpImm:
4839 		rc = decode_imm(ctxt, op, imm_size(ctxt), true);
4840 		break;
4841 	case OpImm64:
4842 		rc = decode_imm(ctxt, op, ctxt->op_bytes, true);
4843 		break;
4844 	case OpMem8:
4845 		ctxt->memop.bytes = 1;
4846 		if (ctxt->memop.type == OP_REG) {
4847 			ctxt->memop.addr.reg = decode_register(ctxt,
4848 					ctxt->modrm_rm, true);
4849 			fetch_register_operand(&ctxt->memop);
4850 		}
4851 		goto mem_common;
4852 	case OpMem16:
4853 		ctxt->memop.bytes = 2;
4854 		goto mem_common;
4855 	case OpMem32:
4856 		ctxt->memop.bytes = 4;
4857 		goto mem_common;
4858 	case OpImmU16:
4859 		rc = decode_imm(ctxt, op, 2, false);
4860 		break;
4861 	case OpImmU:
4862 		rc = decode_imm(ctxt, op, imm_size(ctxt), false);
4863 		break;
4864 	case OpSI:
4865 		op->type = OP_MEM;
4866 		op->bytes = (ctxt->d & ByteOp) ? 1 : ctxt->op_bytes;
4867 		op->addr.mem.ea =
4868 			register_address(ctxt, VCPU_REGS_RSI);
4869 		op->addr.mem.seg = ctxt->seg_override;
4870 		op->val = 0;
4871 		op->count = 1;
4872 		break;
4873 	case OpXLat:
4874 		op->type = OP_MEM;
4875 		op->bytes = (ctxt->d & ByteOp) ? 1 : ctxt->op_bytes;
4876 		op->addr.mem.ea =
4877 			address_mask(ctxt,
4878 				reg_read(ctxt, VCPU_REGS_RBX) +
4879 				(reg_read(ctxt, VCPU_REGS_RAX) & 0xff));
4880 		op->addr.mem.seg = ctxt->seg_override;
4881 		op->val = 0;
4882 		break;
4883 	case OpImmFAddr:
4884 		op->type = OP_IMM;
4885 		op->addr.mem.ea = ctxt->_eip;
4886 		op->bytes = ctxt->op_bytes + 2;
4887 		insn_fetch_arr(op->valptr, op->bytes, ctxt);
4888 		break;
4889 	case OpMemFAddr:
4890 		ctxt->memop.bytes = ctxt->op_bytes + 2;
4891 		goto mem_common;
4892 	case OpES:
4893 		op->type = OP_IMM;
4894 		op->val = VCPU_SREG_ES;
4895 		break;
4896 	case OpCS:
4897 		op->type = OP_IMM;
4898 		op->val = VCPU_SREG_CS;
4899 		break;
4900 	case OpSS:
4901 		op->type = OP_IMM;
4902 		op->val = VCPU_SREG_SS;
4903 		break;
4904 	case OpDS:
4905 		op->type = OP_IMM;
4906 		op->val = VCPU_SREG_DS;
4907 		break;
4908 	case OpFS:
4909 		op->type = OP_IMM;
4910 		op->val = VCPU_SREG_FS;
4911 		break;
4912 	case OpGS:
4913 		op->type = OP_IMM;
4914 		op->val = VCPU_SREG_GS;
4915 		break;
4916 	case OpImplicit:
4917 		/* Special instructions do their own operand decoding. */
4918 	default:
4919 		op->type = OP_NONE; /* Disable writeback. */
4920 		break;
4921 	}
4922 
4923 done:
4924 	return rc;
4925 }
4926 
4927 int x86_decode_insn(struct x86_emulate_ctxt *ctxt, void *insn, int insn_len)
4928 {
4929 	int rc = X86EMUL_CONTINUE;
4930 	int mode = ctxt->mode;
4931 	int def_op_bytes, def_ad_bytes, goffset, simd_prefix;
4932 	bool op_prefix = false;
4933 	bool has_seg_override = false;
4934 	struct opcode opcode;
4935 
4936 	ctxt->memop.type = OP_NONE;
4937 	ctxt->memopp = NULL;
4938 	ctxt->_eip = ctxt->eip;
4939 	ctxt->fetch.ptr = ctxt->fetch.data;
4940 	ctxt->fetch.end = ctxt->fetch.data + insn_len;
4941 	ctxt->opcode_len = 1;
4942 	if (insn_len > 0)
4943 		memcpy(ctxt->fetch.data, insn, insn_len);
4944 	else {
4945 		rc = __do_insn_fetch_bytes(ctxt, 1);
4946 		if (rc != X86EMUL_CONTINUE)
4947 			return rc;
4948 	}
4949 
4950 	switch (mode) {
4951 	case X86EMUL_MODE_REAL:
4952 	case X86EMUL_MODE_VM86:
4953 	case X86EMUL_MODE_PROT16:
4954 		def_op_bytes = def_ad_bytes = 2;
4955 		break;
4956 	case X86EMUL_MODE_PROT32:
4957 		def_op_bytes = def_ad_bytes = 4;
4958 		break;
4959 #ifdef CONFIG_X86_64
4960 	case X86EMUL_MODE_PROT64:
4961 		def_op_bytes = 4;
4962 		def_ad_bytes = 8;
4963 		break;
4964 #endif
4965 	default:
4966 		return EMULATION_FAILED;
4967 	}
4968 
4969 	ctxt->op_bytes = def_op_bytes;
4970 	ctxt->ad_bytes = def_ad_bytes;
4971 
4972 	/* Legacy prefixes. */
4973 	for (;;) {
4974 		switch (ctxt->b = insn_fetch(u8, ctxt)) {
4975 		case 0x66:	/* operand-size override */
4976 			op_prefix = true;
4977 			/* switch between 2/4 bytes */
4978 			ctxt->op_bytes = def_op_bytes ^ 6;
4979 			break;
4980 		case 0x67:	/* address-size override */
4981 			if (mode == X86EMUL_MODE_PROT64)
4982 				/* switch between 4/8 bytes */
4983 				ctxt->ad_bytes = def_ad_bytes ^ 12;
4984 			else
4985 				/* switch between 2/4 bytes */
4986 				ctxt->ad_bytes = def_ad_bytes ^ 6;
4987 			break;
4988 		case 0x26:	/* ES override */
4989 		case 0x2e:	/* CS override */
4990 		case 0x36:	/* SS override */
4991 		case 0x3e:	/* DS override */
4992 			has_seg_override = true;
4993 			ctxt->seg_override = (ctxt->b >> 3) & 3;
4994 			break;
4995 		case 0x64:	/* FS override */
4996 		case 0x65:	/* GS override */
4997 			has_seg_override = true;
4998 			ctxt->seg_override = ctxt->b & 7;
4999 			break;
5000 		case 0x40 ... 0x4f: /* REX */
5001 			if (mode != X86EMUL_MODE_PROT64)
5002 				goto done_prefixes;
5003 			ctxt->rex_prefix = ctxt->b;
5004 			continue;
5005 		case 0xf0:	/* LOCK */
5006 			ctxt->lock_prefix = 1;
5007 			break;
5008 		case 0xf2:	/* REPNE/REPNZ */
5009 		case 0xf3:	/* REP/REPE/REPZ */
5010 			ctxt->rep_prefix = ctxt->b;
5011 			break;
5012 		default:
5013 			goto done_prefixes;
5014 		}
5015 
5016 		/* Any legacy prefix after a REX prefix nullifies its effect. */
5017 
5018 		ctxt->rex_prefix = 0;
5019 	}
5020 
5021 done_prefixes:
5022 
5023 	/* REX prefix. */
5024 	if (ctxt->rex_prefix & 8)
5025 		ctxt->op_bytes = 8;	/* REX.W */
5026 
5027 	/* Opcode byte(s). */
5028 	opcode = opcode_table[ctxt->b];
5029 	/* Two-byte opcode? */
5030 	if (ctxt->b == 0x0f) {
5031 		ctxt->opcode_len = 2;
5032 		ctxt->b = insn_fetch(u8, ctxt);
5033 		opcode = twobyte_table[ctxt->b];
5034 
5035 		/* 0F_38 opcode map */
5036 		if (ctxt->b == 0x38) {
5037 			ctxt->opcode_len = 3;
5038 			ctxt->b = insn_fetch(u8, ctxt);
5039 			opcode = opcode_map_0f_38[ctxt->b];
5040 		}
5041 	}
5042 	ctxt->d = opcode.flags;
5043 
5044 	if (ctxt->d & ModRM)
5045 		ctxt->modrm = insn_fetch(u8, ctxt);
5046 
5047 	/* vex-prefix instructions are not implemented */
5048 	if (ctxt->opcode_len == 1 && (ctxt->b == 0xc5 || ctxt->b == 0xc4) &&
5049 	    (mode == X86EMUL_MODE_PROT64 || (ctxt->modrm & 0xc0) == 0xc0)) {
5050 		ctxt->d = NotImpl;
5051 	}
5052 
5053 	while (ctxt->d & GroupMask) {
5054 		switch (ctxt->d & GroupMask) {
5055 		case Group:
5056 			goffset = (ctxt->modrm >> 3) & 7;
5057 			opcode = opcode.u.group[goffset];
5058 			break;
5059 		case GroupDual:
5060 			goffset = (ctxt->modrm >> 3) & 7;
5061 			if ((ctxt->modrm >> 6) == 3)
5062 				opcode = opcode.u.gdual->mod3[goffset];
5063 			else
5064 				opcode = opcode.u.gdual->mod012[goffset];
5065 			break;
5066 		case RMExt:
5067 			goffset = ctxt->modrm & 7;
5068 			opcode = opcode.u.group[goffset];
5069 			break;
5070 		case Prefix:
5071 			if (ctxt->rep_prefix && op_prefix)
5072 				return EMULATION_FAILED;
5073 			simd_prefix = op_prefix ? 0x66 : ctxt->rep_prefix;
5074 			switch (simd_prefix) {
5075 			case 0x00: opcode = opcode.u.gprefix->pfx_no; break;
5076 			case 0x66: opcode = opcode.u.gprefix->pfx_66; break;
5077 			case 0xf2: opcode = opcode.u.gprefix->pfx_f2; break;
5078 			case 0xf3: opcode = opcode.u.gprefix->pfx_f3; break;
5079 			}
5080 			break;
5081 		case Escape:
5082 			if (ctxt->modrm > 0xbf)
5083 				opcode = opcode.u.esc->high[ctxt->modrm - 0xc0];
5084 			else
5085 				opcode = opcode.u.esc->op[(ctxt->modrm >> 3) & 7];
5086 			break;
5087 		case InstrDual:
5088 			if ((ctxt->modrm >> 6) == 3)
5089 				opcode = opcode.u.idual->mod3;
5090 			else
5091 				opcode = opcode.u.idual->mod012;
5092 			break;
5093 		case ModeDual:
5094 			if (ctxt->mode == X86EMUL_MODE_PROT64)
5095 				opcode = opcode.u.mdual->mode64;
5096 			else
5097 				opcode = opcode.u.mdual->mode32;
5098 			break;
5099 		default:
5100 			return EMULATION_FAILED;
5101 		}
5102 
5103 		ctxt->d &= ~(u64)GroupMask;
5104 		ctxt->d |= opcode.flags;
5105 	}
5106 
5107 	/* Unrecognised? */
5108 	if (ctxt->d == 0)
5109 		return EMULATION_FAILED;
5110 
5111 	ctxt->execute = opcode.u.execute;
5112 
5113 	if (unlikely(ctxt->ud) && likely(!(ctxt->d & EmulateOnUD)))
5114 		return EMULATION_FAILED;
5115 
5116 	if (unlikely(ctxt->d &
5117 	    (NotImpl|Stack|Op3264|Sse|Mmx|Intercept|CheckPerm|NearBranch|
5118 	     No16))) {
5119 		/*
5120 		 * These are copied unconditionally here, and checked unconditionally
5121 		 * in x86_emulate_insn.
5122 		 */
5123 		ctxt->check_perm = opcode.check_perm;
5124 		ctxt->intercept = opcode.intercept;
5125 
5126 		if (ctxt->d & NotImpl)
5127 			return EMULATION_FAILED;
5128 
5129 		if (mode == X86EMUL_MODE_PROT64) {
5130 			if (ctxt->op_bytes == 4 && (ctxt->d & Stack))
5131 				ctxt->op_bytes = 8;
5132 			else if (ctxt->d & NearBranch)
5133 				ctxt->op_bytes = 8;
5134 		}
5135 
5136 		if (ctxt->d & Op3264) {
5137 			if (mode == X86EMUL_MODE_PROT64)
5138 				ctxt->op_bytes = 8;
5139 			else
5140 				ctxt->op_bytes = 4;
5141 		}
5142 
5143 		if ((ctxt->d & No16) && ctxt->op_bytes == 2)
5144 			ctxt->op_bytes = 4;
5145 
5146 		if (ctxt->d & Sse)
5147 			ctxt->op_bytes = 16;
5148 		else if (ctxt->d & Mmx)
5149 			ctxt->op_bytes = 8;
5150 	}
5151 
5152 	/* ModRM and SIB bytes. */
5153 	if (ctxt->d & ModRM) {
5154 		rc = decode_modrm(ctxt, &ctxt->memop);
5155 		if (!has_seg_override) {
5156 			has_seg_override = true;
5157 			ctxt->seg_override = ctxt->modrm_seg;
5158 		}
5159 	} else if (ctxt->d & MemAbs)
5160 		rc = decode_abs(ctxt, &ctxt->memop);
5161 	if (rc != X86EMUL_CONTINUE)
5162 		goto done;
5163 
5164 	if (!has_seg_override)
5165 		ctxt->seg_override = VCPU_SREG_DS;
5166 
5167 	ctxt->memop.addr.mem.seg = ctxt->seg_override;
5168 
5169 	/*
5170 	 * Decode and fetch the source operand: register, memory
5171 	 * or immediate.
5172 	 */
5173 	rc = decode_operand(ctxt, &ctxt->src, (ctxt->d >> SrcShift) & OpMask);
5174 	if (rc != X86EMUL_CONTINUE)
5175 		goto done;
5176 
5177 	/*
5178 	 * Decode and fetch the second source operand: register, memory
5179 	 * or immediate.
5180 	 */
5181 	rc = decode_operand(ctxt, &ctxt->src2, (ctxt->d >> Src2Shift) & OpMask);
5182 	if (rc != X86EMUL_CONTINUE)
5183 		goto done;
5184 
5185 	/* Decode and fetch the destination operand: register or memory. */
5186 	rc = decode_operand(ctxt, &ctxt->dst, (ctxt->d >> DstShift) & OpMask);
5187 
5188 	if (ctxt->rip_relative && likely(ctxt->memopp))
5189 		ctxt->memopp->addr.mem.ea = address_mask(ctxt,
5190 					ctxt->memopp->addr.mem.ea + ctxt->_eip);
5191 
5192 done:
5193 	return (rc != X86EMUL_CONTINUE) ? EMULATION_FAILED : EMULATION_OK;
5194 }
5195 
5196 bool x86_page_table_writing_insn(struct x86_emulate_ctxt *ctxt)
5197 {
5198 	return ctxt->d & PageTable;
5199 }
5200 
5201 static bool string_insn_completed(struct x86_emulate_ctxt *ctxt)
5202 {
5203 	/* The second termination condition only applies for REPE
5204 	 * and REPNE. Test if the repeat string operation prefix is
5205 	 * REPE/REPZ or REPNE/REPNZ and if it's the case it tests the
5206 	 * corresponding termination condition according to:
5207 	 * 	- if REPE/REPZ and ZF = 0 then done
5208 	 * 	- if REPNE/REPNZ and ZF = 1 then done
5209 	 */
5210 	if (((ctxt->b == 0xa6) || (ctxt->b == 0xa7) ||
5211 	     (ctxt->b == 0xae) || (ctxt->b == 0xaf))
5212 	    && (((ctxt->rep_prefix == REPE_PREFIX) &&
5213 		 ((ctxt->eflags & X86_EFLAGS_ZF) == 0))
5214 		|| ((ctxt->rep_prefix == REPNE_PREFIX) &&
5215 		    ((ctxt->eflags & X86_EFLAGS_ZF) == X86_EFLAGS_ZF))))
5216 		return true;
5217 
5218 	return false;
5219 }
5220 
5221 static int flush_pending_x87_faults(struct x86_emulate_ctxt *ctxt)
5222 {
5223 	int rc;
5224 
5225 	ctxt->ops->get_fpu(ctxt);
5226 	rc = asm_safe("fwait");
5227 	ctxt->ops->put_fpu(ctxt);
5228 
5229 	if (unlikely(rc != X86EMUL_CONTINUE))
5230 		return emulate_exception(ctxt, MF_VECTOR, 0, false);
5231 
5232 	return X86EMUL_CONTINUE;
5233 }
5234 
5235 static void fetch_possible_mmx_operand(struct x86_emulate_ctxt *ctxt,
5236 				       struct operand *op)
5237 {
5238 	if (op->type == OP_MM)
5239 		read_mmx_reg(ctxt, &op->mm_val, op->addr.mm);
5240 }
5241 
5242 static int fastop(struct x86_emulate_ctxt *ctxt, void (*fop)(struct fastop *))
5243 {
5244 	register void *__sp asm(_ASM_SP);
5245 	ulong flags = (ctxt->eflags & EFLAGS_MASK) | X86_EFLAGS_IF;
5246 
5247 	if (!(ctxt->d & ByteOp))
5248 		fop += __ffs(ctxt->dst.bytes) * FASTOP_SIZE;
5249 
5250 	asm("push %[flags]; popf; call *%[fastop]; pushf; pop %[flags]\n"
5251 	    : "+a"(ctxt->dst.val), "+d"(ctxt->src.val), [flags]"+D"(flags),
5252 	      [fastop]"+S"(fop), "+r"(__sp)
5253 	    : "c"(ctxt->src2.val));
5254 
5255 	ctxt->eflags = (ctxt->eflags & ~EFLAGS_MASK) | (flags & EFLAGS_MASK);
5256 	if (!fop) /* exception is returned in fop variable */
5257 		return emulate_de(ctxt);
5258 	return X86EMUL_CONTINUE;
5259 }
5260 
5261 void init_decode_cache(struct x86_emulate_ctxt *ctxt)
5262 {
5263 	memset(&ctxt->rip_relative, 0,
5264 	       (void *)&ctxt->modrm - (void *)&ctxt->rip_relative);
5265 
5266 	ctxt->io_read.pos = 0;
5267 	ctxt->io_read.end = 0;
5268 	ctxt->mem_read.end = 0;
5269 }
5270 
5271 int x86_emulate_insn(struct x86_emulate_ctxt *ctxt)
5272 {
5273 	const struct x86_emulate_ops *ops = ctxt->ops;
5274 	int rc = X86EMUL_CONTINUE;
5275 	int saved_dst_type = ctxt->dst.type;
5276 
5277 	ctxt->mem_read.pos = 0;
5278 
5279 	/* LOCK prefix is allowed only with some instructions */
5280 	if (ctxt->lock_prefix && (!(ctxt->d & Lock) || ctxt->dst.type != OP_MEM)) {
5281 		rc = emulate_ud(ctxt);
5282 		goto done;
5283 	}
5284 
5285 	if ((ctxt->d & SrcMask) == SrcMemFAddr && ctxt->src.type != OP_MEM) {
5286 		rc = emulate_ud(ctxt);
5287 		goto done;
5288 	}
5289 
5290 	if (unlikely(ctxt->d &
5291 		     (No64|Undefined|Sse|Mmx|Intercept|CheckPerm|Priv|Prot|String))) {
5292 		if ((ctxt->mode == X86EMUL_MODE_PROT64 && (ctxt->d & No64)) ||
5293 				(ctxt->d & Undefined)) {
5294 			rc = emulate_ud(ctxt);
5295 			goto done;
5296 		}
5297 
5298 		if (((ctxt->d & (Sse|Mmx)) && ((ops->get_cr(ctxt, 0) & X86_CR0_EM)))
5299 		    || ((ctxt->d & Sse) && !(ops->get_cr(ctxt, 4) & X86_CR4_OSFXSR))) {
5300 			rc = emulate_ud(ctxt);
5301 			goto done;
5302 		}
5303 
5304 		if ((ctxt->d & (Sse|Mmx)) && (ops->get_cr(ctxt, 0) & X86_CR0_TS)) {
5305 			rc = emulate_nm(ctxt);
5306 			goto done;
5307 		}
5308 
5309 		if (ctxt->d & Mmx) {
5310 			rc = flush_pending_x87_faults(ctxt);
5311 			if (rc != X86EMUL_CONTINUE)
5312 				goto done;
5313 			/*
5314 			 * Now that we know the fpu is exception safe, we can fetch
5315 			 * operands from it.
5316 			 */
5317 			fetch_possible_mmx_operand(ctxt, &ctxt->src);
5318 			fetch_possible_mmx_operand(ctxt, &ctxt->src2);
5319 			if (!(ctxt->d & Mov))
5320 				fetch_possible_mmx_operand(ctxt, &ctxt->dst);
5321 		}
5322 
5323 		if (unlikely(ctxt->emul_flags & X86EMUL_GUEST_MASK) && ctxt->intercept) {
5324 			rc = emulator_check_intercept(ctxt, ctxt->intercept,
5325 						      X86_ICPT_PRE_EXCEPT);
5326 			if (rc != X86EMUL_CONTINUE)
5327 				goto done;
5328 		}
5329 
5330 		/* Instruction can only be executed in protected mode */
5331 		if ((ctxt->d & Prot) && ctxt->mode < X86EMUL_MODE_PROT16) {
5332 			rc = emulate_ud(ctxt);
5333 			goto done;
5334 		}
5335 
5336 		/* Privileged instruction can be executed only in CPL=0 */
5337 		if ((ctxt->d & Priv) && ops->cpl(ctxt)) {
5338 			if (ctxt->d & PrivUD)
5339 				rc = emulate_ud(ctxt);
5340 			else
5341 				rc = emulate_gp(ctxt, 0);
5342 			goto done;
5343 		}
5344 
5345 		/* Do instruction specific permission checks */
5346 		if (ctxt->d & CheckPerm) {
5347 			rc = ctxt->check_perm(ctxt);
5348 			if (rc != X86EMUL_CONTINUE)
5349 				goto done;
5350 		}
5351 
5352 		if (unlikely(ctxt->emul_flags & X86EMUL_GUEST_MASK) && (ctxt->d & Intercept)) {
5353 			rc = emulator_check_intercept(ctxt, ctxt->intercept,
5354 						      X86_ICPT_POST_EXCEPT);
5355 			if (rc != X86EMUL_CONTINUE)
5356 				goto done;
5357 		}
5358 
5359 		if (ctxt->rep_prefix && (ctxt->d & String)) {
5360 			/* All REP prefixes have the same first termination condition */
5361 			if (address_mask(ctxt, reg_read(ctxt, VCPU_REGS_RCX)) == 0) {
5362 				string_registers_quirk(ctxt);
5363 				ctxt->eip = ctxt->_eip;
5364 				ctxt->eflags &= ~X86_EFLAGS_RF;
5365 				goto done;
5366 			}
5367 		}
5368 	}
5369 
5370 	if ((ctxt->src.type == OP_MEM) && !(ctxt->d & NoAccess)) {
5371 		rc = segmented_read(ctxt, ctxt->src.addr.mem,
5372 				    ctxt->src.valptr, ctxt->src.bytes);
5373 		if (rc != X86EMUL_CONTINUE)
5374 			goto done;
5375 		ctxt->src.orig_val64 = ctxt->src.val64;
5376 	}
5377 
5378 	if (ctxt->src2.type == OP_MEM) {
5379 		rc = segmented_read(ctxt, ctxt->src2.addr.mem,
5380 				    &ctxt->src2.val, ctxt->src2.bytes);
5381 		if (rc != X86EMUL_CONTINUE)
5382 			goto done;
5383 	}
5384 
5385 	if ((ctxt->d & DstMask) == ImplicitOps)
5386 		goto special_insn;
5387 
5388 
5389 	if ((ctxt->dst.type == OP_MEM) && !(ctxt->d & Mov)) {
5390 		/* optimisation - avoid slow emulated read if Mov */
5391 		rc = segmented_read(ctxt, ctxt->dst.addr.mem,
5392 				   &ctxt->dst.val, ctxt->dst.bytes);
5393 		if (rc != X86EMUL_CONTINUE) {
5394 			if (!(ctxt->d & NoWrite) &&
5395 			    rc == X86EMUL_PROPAGATE_FAULT &&
5396 			    ctxt->exception.vector == PF_VECTOR)
5397 				ctxt->exception.error_code |= PFERR_WRITE_MASK;
5398 			goto done;
5399 		}
5400 	}
5401 	/* Copy full 64-bit value for CMPXCHG8B.  */
5402 	ctxt->dst.orig_val64 = ctxt->dst.val64;
5403 
5404 special_insn:
5405 
5406 	if (unlikely(ctxt->emul_flags & X86EMUL_GUEST_MASK) && (ctxt->d & Intercept)) {
5407 		rc = emulator_check_intercept(ctxt, ctxt->intercept,
5408 					      X86_ICPT_POST_MEMACCESS);
5409 		if (rc != X86EMUL_CONTINUE)
5410 			goto done;
5411 	}
5412 
5413 	if (ctxt->rep_prefix && (ctxt->d & String))
5414 		ctxt->eflags |= X86_EFLAGS_RF;
5415 	else
5416 		ctxt->eflags &= ~X86_EFLAGS_RF;
5417 
5418 	if (ctxt->execute) {
5419 		if (ctxt->d & Fastop) {
5420 			void (*fop)(struct fastop *) = (void *)ctxt->execute;
5421 			rc = fastop(ctxt, fop);
5422 			if (rc != X86EMUL_CONTINUE)
5423 				goto done;
5424 			goto writeback;
5425 		}
5426 		rc = ctxt->execute(ctxt);
5427 		if (rc != X86EMUL_CONTINUE)
5428 			goto done;
5429 		goto writeback;
5430 	}
5431 
5432 	if (ctxt->opcode_len == 2)
5433 		goto twobyte_insn;
5434 	else if (ctxt->opcode_len == 3)
5435 		goto threebyte_insn;
5436 
5437 	switch (ctxt->b) {
5438 	case 0x70 ... 0x7f: /* jcc (short) */
5439 		if (test_cc(ctxt->b, ctxt->eflags))
5440 			rc = jmp_rel(ctxt, ctxt->src.val);
5441 		break;
5442 	case 0x8d: /* lea r16/r32, m */
5443 		ctxt->dst.val = ctxt->src.addr.mem.ea;
5444 		break;
5445 	case 0x90 ... 0x97: /* nop / xchg reg, rax */
5446 		if (ctxt->dst.addr.reg == reg_rmw(ctxt, VCPU_REGS_RAX))
5447 			ctxt->dst.type = OP_NONE;
5448 		else
5449 			rc = em_xchg(ctxt);
5450 		break;
5451 	case 0x98: /* cbw/cwde/cdqe */
5452 		switch (ctxt->op_bytes) {
5453 		case 2: ctxt->dst.val = (s8)ctxt->dst.val; break;
5454 		case 4: ctxt->dst.val = (s16)ctxt->dst.val; break;
5455 		case 8: ctxt->dst.val = (s32)ctxt->dst.val; break;
5456 		}
5457 		break;
5458 	case 0xcc:		/* int3 */
5459 		rc = emulate_int(ctxt, 3);
5460 		break;
5461 	case 0xcd:		/* int n */
5462 		rc = emulate_int(ctxt, ctxt->src.val);
5463 		break;
5464 	case 0xce:		/* into */
5465 		if (ctxt->eflags & X86_EFLAGS_OF)
5466 			rc = emulate_int(ctxt, 4);
5467 		break;
5468 	case 0xe9: /* jmp rel */
5469 	case 0xeb: /* jmp rel short */
5470 		rc = jmp_rel(ctxt, ctxt->src.val);
5471 		ctxt->dst.type = OP_NONE; /* Disable writeback. */
5472 		break;
5473 	case 0xf4:              /* hlt */
5474 		ctxt->ops->halt(ctxt);
5475 		break;
5476 	case 0xf5:	/* cmc */
5477 		/* complement carry flag from eflags reg */
5478 		ctxt->eflags ^= X86_EFLAGS_CF;
5479 		break;
5480 	case 0xf8: /* clc */
5481 		ctxt->eflags &= ~X86_EFLAGS_CF;
5482 		break;
5483 	case 0xf9: /* stc */
5484 		ctxt->eflags |= X86_EFLAGS_CF;
5485 		break;
5486 	case 0xfc: /* cld */
5487 		ctxt->eflags &= ~X86_EFLAGS_DF;
5488 		break;
5489 	case 0xfd: /* std */
5490 		ctxt->eflags |= X86_EFLAGS_DF;
5491 		break;
5492 	default:
5493 		goto cannot_emulate;
5494 	}
5495 
5496 	if (rc != X86EMUL_CONTINUE)
5497 		goto done;
5498 
5499 writeback:
5500 	if (ctxt->d & SrcWrite) {
5501 		BUG_ON(ctxt->src.type == OP_MEM || ctxt->src.type == OP_MEM_STR);
5502 		rc = writeback(ctxt, &ctxt->src);
5503 		if (rc != X86EMUL_CONTINUE)
5504 			goto done;
5505 	}
5506 	if (!(ctxt->d & NoWrite)) {
5507 		rc = writeback(ctxt, &ctxt->dst);
5508 		if (rc != X86EMUL_CONTINUE)
5509 			goto done;
5510 	}
5511 
5512 	/*
5513 	 * restore dst type in case the decoding will be reused
5514 	 * (happens for string instruction )
5515 	 */
5516 	ctxt->dst.type = saved_dst_type;
5517 
5518 	if ((ctxt->d & SrcMask) == SrcSI)
5519 		string_addr_inc(ctxt, VCPU_REGS_RSI, &ctxt->src);
5520 
5521 	if ((ctxt->d & DstMask) == DstDI)
5522 		string_addr_inc(ctxt, VCPU_REGS_RDI, &ctxt->dst);
5523 
5524 	if (ctxt->rep_prefix && (ctxt->d & String)) {
5525 		unsigned int count;
5526 		struct read_cache *r = &ctxt->io_read;
5527 		if ((ctxt->d & SrcMask) == SrcSI)
5528 			count = ctxt->src.count;
5529 		else
5530 			count = ctxt->dst.count;
5531 		register_address_increment(ctxt, VCPU_REGS_RCX, -count);
5532 
5533 		if (!string_insn_completed(ctxt)) {
5534 			/*
5535 			 * Re-enter guest when pio read ahead buffer is empty
5536 			 * or, if it is not used, after each 1024 iteration.
5537 			 */
5538 			if ((r->end != 0 || reg_read(ctxt, VCPU_REGS_RCX) & 0x3ff) &&
5539 			    (r->end == 0 || r->end != r->pos)) {
5540 				/*
5541 				 * Reset read cache. Usually happens before
5542 				 * decode, but since instruction is restarted
5543 				 * we have to do it here.
5544 				 */
5545 				ctxt->mem_read.end = 0;
5546 				writeback_registers(ctxt);
5547 				return EMULATION_RESTART;
5548 			}
5549 			goto done; /* skip rip writeback */
5550 		}
5551 		ctxt->eflags &= ~X86_EFLAGS_RF;
5552 	}
5553 
5554 	ctxt->eip = ctxt->_eip;
5555 
5556 done:
5557 	if (rc == X86EMUL_PROPAGATE_FAULT) {
5558 		WARN_ON(ctxt->exception.vector > 0x1f);
5559 		ctxt->have_exception = true;
5560 	}
5561 	if (rc == X86EMUL_INTERCEPTED)
5562 		return EMULATION_INTERCEPTED;
5563 
5564 	if (rc == X86EMUL_CONTINUE)
5565 		writeback_registers(ctxt);
5566 
5567 	return (rc == X86EMUL_UNHANDLEABLE) ? EMULATION_FAILED : EMULATION_OK;
5568 
5569 twobyte_insn:
5570 	switch (ctxt->b) {
5571 	case 0x09:		/* wbinvd */
5572 		(ctxt->ops->wbinvd)(ctxt);
5573 		break;
5574 	case 0x08:		/* invd */
5575 	case 0x0d:		/* GrpP (prefetch) */
5576 	case 0x18:		/* Grp16 (prefetch/nop) */
5577 	case 0x1f:		/* nop */
5578 		break;
5579 	case 0x20: /* mov cr, reg */
5580 		ctxt->dst.val = ops->get_cr(ctxt, ctxt->modrm_reg);
5581 		break;
5582 	case 0x21: /* mov from dr to reg */
5583 		ops->get_dr(ctxt, ctxt->modrm_reg, &ctxt->dst.val);
5584 		break;
5585 	case 0x40 ... 0x4f:	/* cmov */
5586 		if (test_cc(ctxt->b, ctxt->eflags))
5587 			ctxt->dst.val = ctxt->src.val;
5588 		else if (ctxt->op_bytes != 4)
5589 			ctxt->dst.type = OP_NONE; /* no writeback */
5590 		break;
5591 	case 0x80 ... 0x8f: /* jnz rel, etc*/
5592 		if (test_cc(ctxt->b, ctxt->eflags))
5593 			rc = jmp_rel(ctxt, ctxt->src.val);
5594 		break;
5595 	case 0x90 ... 0x9f:     /* setcc r/m8 */
5596 		ctxt->dst.val = test_cc(ctxt->b, ctxt->eflags);
5597 		break;
5598 	case 0xb6 ... 0xb7:	/* movzx */
5599 		ctxt->dst.bytes = ctxt->op_bytes;
5600 		ctxt->dst.val = (ctxt->src.bytes == 1) ? (u8) ctxt->src.val
5601 						       : (u16) ctxt->src.val;
5602 		break;
5603 	case 0xbe ... 0xbf:	/* movsx */
5604 		ctxt->dst.bytes = ctxt->op_bytes;
5605 		ctxt->dst.val = (ctxt->src.bytes == 1) ? (s8) ctxt->src.val :
5606 							(s16) ctxt->src.val;
5607 		break;
5608 	default:
5609 		goto cannot_emulate;
5610 	}
5611 
5612 threebyte_insn:
5613 
5614 	if (rc != X86EMUL_CONTINUE)
5615 		goto done;
5616 
5617 	goto writeback;
5618 
5619 cannot_emulate:
5620 	return EMULATION_FAILED;
5621 }
5622 
5623 void emulator_invalidate_register_cache(struct x86_emulate_ctxt *ctxt)
5624 {
5625 	invalidate_registers(ctxt);
5626 }
5627 
5628 void emulator_writeback_register_cache(struct x86_emulate_ctxt *ctxt)
5629 {
5630 	writeback_registers(ctxt);
5631 }
5632