Merge tag 'master-2014-12-01' of git://git.kernel.org/pub/scm/linux/kernel/git/linvil...

[karo-tx-linux.git] / arch / x86 / kernel / entry_64.S

diff --git a/arch/x86/kernel/entry_64.S b/arch/x86/kernel/entry_64.S
index df088bb03fb3ffec9148c7cc44cb65ef1aa36118..c0226ab541061870bb590a9c817d0770e618697c 100644 (file)
--- a/arch/x86/kernel/entry_64.S
+++ b/arch/x86/kernel/entry_64.S
@@ -828,9 +828,15 @@ ENTRY(native_iret)
        jnz native_irq_return_ldt
 #endif
 
+.global native_irq_return_iret
 native_irq_return_iret:
+       /*
+        * This may fault.  Non-paranoid faults on return to userspace are
+        * handled by fixup_bad_iret.  These include #SS, #GP, and #NP.
+        * Double-faults due to espfix64 are handled in do_double_fault.
+        * Other faults here are fatal.
+        */
        iretq
-       _ASM_EXTABLE(native_irq_return_iret, bad_iret)
 
 #ifdef CONFIG_X86_ESPFIX64
 native_irq_return_ldt:
@@ -858,25 +864,6 @@ native_irq_return_ldt:
        jmp native_irq_return_iret
 #endif
 
-       .section .fixup,"ax"
-bad_iret:
-       /*
-        * The iret traps when the %cs or %ss being restored is bogus.
-        * We've lost the original trap vector and error code.
-        * #GPF is the most likely one to get for an invalid selector.
-        * So pretend we completed the iret and took the #GPF in user mode.
-        *
-        * We are now running with the kernel GS after exception recovery.
-        * But error_entry expects us to have user GS to match the user %cs,
-        * so swap back.
-        */
-       pushq $0
-
-       SWAPGS
-       jmp general_protection
-
-       .previous
-
        /* edi: workmask, edx: work */
 retint_careful:
        CFI_RESTORE_STATE
@@ -922,37 +909,6 @@ ENTRY(retint_kernel)
        CFI_ENDPROC
 END(common_interrupt)
 
-       /*
-        * If IRET takes a fault on the espfix stack, then we
-        * end up promoting it to a doublefault.  In that case,
-        * modify the stack to make it look like we just entered
-        * the #GP handler from user space, similar to bad_iret.
-        */
-#ifdef CONFIG_X86_ESPFIX64
-       ALIGN
-__do_double_fault:
-       XCPT_FRAME 1 RDI+8
-       movq RSP(%rdi),%rax             /* Trap on the espfix stack? */
-       sarq $PGDIR_SHIFT,%rax
-       cmpl $ESPFIX_PGD_ENTRY,%eax
-       jne do_double_fault             /* No, just deliver the fault */
-       cmpl $__KERNEL_CS,CS(%rdi)
-       jne do_double_fault
-       movq RIP(%rdi),%rax
-       cmpq $native_irq_return_iret,%rax
-       jne do_double_fault             /* This shouldn't happen... */
-       movq PER_CPU_VAR(kernel_stack),%rax
-       subq $(6*8-KERNEL_STACK_OFFSET),%rax    /* Reset to original stack */
-       movq %rax,RSP(%rdi)
-       movq $0,(%rax)                  /* Missing (lost) #GP error code */
-       movq $general_protection,RIP(%rdi)
-       retq
-       CFI_ENDPROC
-END(__do_double_fault)
-#else
-# define __do_double_fault do_double_fault
-#endif
-
 /*
  * APIC interrupts.
  */
@@ -1124,7 +1080,7 @@ idtentry overflow do_overflow has_error_code=0
 idtentry bounds do_bounds has_error_code=0
 idtentry invalid_op do_invalid_op has_error_code=0
 idtentry device_not_available do_device_not_available has_error_code=0
-idtentry double_fault __do_double_fault has_error_code=1 paranoid=1
+idtentry double_fault do_double_fault has_error_code=1 paranoid=1
 idtentry coprocessor_segment_overrun do_coprocessor_segment_overrun has_error_code=0
 idtentry invalid_TSS do_invalid_TSS has_error_code=1
 idtentry segment_not_present do_segment_not_present has_error_code=1
@@ -1289,7 +1245,7 @@ apicinterrupt3 HYPERVISOR_CALLBACK_VECTOR \
 
 idtentry debug do_debug has_error_code=0 paranoid=1 shift_ist=DEBUG_STACK
 idtentry int3 do_int3 has_error_code=0 paranoid=1 shift_ist=DEBUG_STACK
-idtentry stack_segment do_stack_segment has_error_code=1 paranoid=1
+idtentry stack_segment do_stack_segment has_error_code=1
 #ifdef CONFIG_XEN
 idtentry xen_debug do_debug has_error_code=0
 idtentry xen_int3 do_int3 has_error_code=0
@@ -1399,17 +1355,16 @@ error_sti:
 
 /*
  * There are two places in the kernel that can potentially fault with
- * usergs. Handle them here. The exception handlers after iret run with
- * kernel gs again, so don't set the user space flag. B stepping K8s
- * sometimes report an truncated RIP for IRET exceptions returning to
- * compat mode. Check for these here too.
+ * usergs. Handle them here.  B stepping K8s sometimes report a
+ * truncated RIP for IRET exceptions returning to compat mode. Check
+ * for these here too.
  */
 error_kernelspace:
        CFI_REL_OFFSET rcx, RCX+8
        incl %ebx
        leaq native_irq_return_iret(%rip),%rcx
        cmpq %rcx,RIP+8(%rsp)
-       je error_swapgs
+       je error_bad_iret
        movl %ecx,%eax  /* zero extend */
        cmpq %rax,RIP+8(%rsp)
        je bstep_iret
@@ -1420,7 +1375,15 @@ error_kernelspace:
 bstep_iret:
        /* Fix truncated RIP */
        movq %rcx,RIP+8(%rsp)
-       jmp error_swapgs
+       /* fall through */
+
+error_bad_iret:
+       SWAPGS
+       mov %rsp,%rdi
+       call fixup_bad_iret
+       mov %rax,%rsp
+       decl %ebx       /* Return to usergs */
+       jmp error_sti
        CFI_ENDPROC
 END(error_entry)