Merge remote-tracking branch 'tip/auto-latest'

[karo-tx-linux.git] / arch / x86 / kernel / entry_64.S

diff --git a/arch/x86/kernel/entry_64.S b/arch/x86/kernel/entry_64.S
index 8292ff7348454e13a4c344ae7c4f052b25e722ab..4214834c6dd90dbaa663750a3656e4fb6be14cf7 100644 (file)
--- a/arch/x86/kernel/entry_64.S
+++ b/arch/x86/kernel/entry_64.S
@@ -58,6 +58,7 @@
 #include <asm/asm.h>
 #include <asm/context_tracking.h>
 #include <asm/smap.h>
+#include <asm/pgtable_types.h>
 #include <linux/err.h>
 
 /* Avoid __ASSEMBLER__'ifying <linux/audit.h> just for this.  */
@@ -487,8 +488,6 @@ ENDPROC(native_usergs_sysret64)
        TRACE_IRQS_OFF
        .endm
 
-/* save complete stack frame */
-       .pushsection .kprobes.text, "ax"
 ENTRY(save_paranoid)
        XCPT_FRAME 1 RDI+8
        cld
@@ -517,7 +516,6 @@ ENTRY(save_paranoid)
 1:     ret
        CFI_ENDPROC
 END(save_paranoid)
-       .popsection
 
 /*
  * A newly forked process directly context switches into this address.
@@ -974,10 +972,6 @@ END(interrupt)
        call \func
        .endm
 
-/*
- * Interrupt entry/exit should be protected against kprobes
- */
-       .pushsection .kprobes.text, "ax"
        /*
         * The interrupt stubs push (~vector+0x80) onto the stack and
         * then jump to common_interrupt.
@@ -1039,8 +1033,18 @@ restore_args:
        RESTORE_ARGS 1,8,1
 
 irq_return:
+       /*
+        * Are we returning to a stack segment from the LDT?  Note: in
+        * 64-bit mode SS:RSP on the exception stack is always valid.
+        */
+#ifdef CONFIG_X86_ESPFIX64
+       testb $4,(SS-RIP)(%rsp)
+       jnz irq_return_ldt
+#endif
+
+irq_return_iret:
        INTERRUPT_RETURN
-       _ASM_EXTABLE(irq_return, bad_iret)
+       _ASM_EXTABLE(irq_return_iret, bad_iret)
 
 #ifdef CONFIG_PARAVIRT
 ENTRY(native_iret)
@@ -1048,6 +1052,32 @@ ENTRY(native_iret)
        _ASM_EXTABLE(native_iret, bad_iret)
 #endif
 
+#ifdef CONFIG_X86_ESPFIX64
+irq_return_ldt:
+       pushq_cfi %rax
+       pushq_cfi %rdi
+       SWAPGS
+       movq PER_CPU_VAR(espfix_waddr),%rdi
+       movq %rax,(0*8)(%rdi)   /* RAX */
+       movq (2*8)(%rsp),%rax   /* RIP */
+       movq %rax,(1*8)(%rdi)
+       movq (3*8)(%rsp),%rax   /* CS */
+       movq %rax,(2*8)(%rdi)
+       movq (4*8)(%rsp),%rax   /* RFLAGS */
+       movq %rax,(3*8)(%rdi)
+       movq (6*8)(%rsp),%rax   /* SS */
+       movq %rax,(5*8)(%rdi)
+       movq (5*8)(%rsp),%rax   /* RSP */
+       movq %rax,(4*8)(%rdi)
+       andl $0xffff0000,%eax
+       popq_cfi %rdi
+       orq PER_CPU_VAR(espfix_stack),%rax
+       SWAPGS
+       movq %rax,%rsp
+       popq_cfi %rax
+       jmp irq_return_iret
+#endif
+
        .section .fixup,"ax"
 bad_iret:
        /*
@@ -1109,13 +1139,45 @@ ENTRY(retint_kernel)
        call preempt_schedule_irq
        jmp exit_intr
 #endif
-
        CFI_ENDPROC
 END(common_interrupt)
-/*
- * End of kprobes section
- */
-       .popsection
+
+       /*
+        * If IRET takes a fault on the espfix stack, then we
+        * end up promoting it to a doublefault.  In that case,
+        * modify the stack to make it look like we just entered
+        * the #GP handler from user space, similar to bad_iret.
+        */
+#ifdef CONFIG_X86_ESPFIX64
+       ALIGN
+__do_double_fault:
+       XCPT_FRAME 1 RDI+8
+       movq RSP(%rdi),%rax             /* Trap on the espfix stack? */
+       sarq $PGDIR_SHIFT,%rax
+       cmpl $ESPFIX_PGD_ENTRY,%eax
+       jne do_double_fault             /* No, just deliver the fault */
+       cmpl $__KERNEL_CS,CS(%rdi)
+       jne do_double_fault
+       movq RIP(%rdi),%rax
+       cmpq $irq_return_iret,%rax
+#ifdef CONFIG_PARAVIRT
+       je 1f
+       cmpq $native_iret,%rax
+#endif
+       jne do_double_fault             /* This shouldn't happen... */
+1:
+       movq PER_CPU_VAR(kernel_stack),%rax
+       subq $(6*8-KERNEL_STACK_OFFSET),%rax    /* Reset to original stack */
+       movq %rax,RSP(%rdi)
+       movq $0,(%rax)                  /* Missing (lost) #GP error code */
+       movq $general_protection,RIP(%rdi)
+       retq
+       CFI_ENDPROC
+END(__do_double_fault)
+#else
+# define __do_double_fault do_double_fault
+#endif
+
 
 /*
  * APIC interrupts.
@@ -1313,7 +1375,7 @@ zeroentry overflow do_overflow
 zeroentry bounds do_bounds
 zeroentry invalid_op do_invalid_op
 zeroentry device_not_available do_device_not_available
-paranoiderrorentry double_fault do_double_fault
+paranoiderrorentry double_fault __do_double_fault
 zeroentry coprocessor_segment_overrun do_coprocessor_segment_overrun
 errorentry invalid_TSS do_invalid_TSS
 errorentry segment_not_present do_segment_not_present
@@ -1476,11 +1538,6 @@ apicinterrupt3 HYPERVISOR_CALLBACK_VECTOR \
        hyperv_callback_vector hyperv_vector_handler
 #endif /* CONFIG_HYPERV */
 
-/*
- * Some functions should be protected against kprobes
- */
-       .pushsection .kprobes.text, "ax"
-
 paranoidzeroentry_ist debug do_debug DEBUG_STACK
 paranoidzeroentry_ist int3 do_int3 DEBUG_STACK
 paranoiderrorentry stack_segment do_stack_segment
@@ -1600,7 +1657,7 @@ error_sti:
  */
 error_kernelspace:
        incl %ebx
-       leaq irq_return(%rip),%rcx
+       leaq irq_return_iret(%rip),%rcx
        cmpq %rcx,RIP+8(%rsp)
        je error_swapgs
        movl %ecx,%eax  /* zero extend */
@@ -1897,7 +1954,3 @@ ENTRY(ignore_sysret)
        CFI_ENDPROC
 END(ignore_sysret)
 
-/*
- * End of kprobes section
- */
-       .popsection