/* Unimplemented features. */
#undef KERNEL_STACK_OVERFLOW_CHECK
-#undef PREEMPTIBLE_KERNEL
-#undef ALLOCA_EXCEPTION_IN_IRAM
/* Not well tested.
*
* a0: trashed, original value saved on stack (PT_AREG0)
* a1: a1
* a2: new stack pointer, original value in depc
- * a3: dispatch table
+ * a3: a3
* depc: a2, original value saved on stack (PT_DEPC)
- * excsave1: a3
+ * excsave1: dispatch table
*
* PT_DEPC >= VALID_DOUBLE_EXCEPTION_ADDRESS: double exception, DEPC
* < VALID_DOUBLE_EXCEPTION_ADDRESS: regular exception
ENTRY(user_exception)
- /* Save a2, a3, and depc, restore excsave_1 and set SP. */
+ /* Save a1, a2, a3, and set SP. */
- xsr a3, excsave1
rsr a0, depc
s32i a1, a2, PT_AREG1
s32i a0, a2, PT_AREG2
* a0: trashed, original value saved on stack (PT_AREG0)
* a1: a1
* a2: new stack pointer, original in DEPC
- * a3: dispatch table
+ * a3: a3
* depc: a2, original value saved on stack (PT_DEPC)
- * excsave_1: a3
+ * excsave_1: dispatch table
*
* PT_DEPC >= VALID_DOUBLE_EXCEPTION_ADDRESS: double exception, DEPC
* < VALID_DOUBLE_EXCEPTION_ADDRESS: regular exception
ENTRY(kernel_exception)
- /* Save a0, a2, a3, DEPC and set SP. */
+ /* Save a1, a2, a3, and set SP. */
- xsr a3, excsave1 # restore a3, excsave_1
rsr a0, depc # get a2
s32i a1, a2, PT_AREG1
s32i a0, a2, PT_AREG2
* exception handler and call the exception handler.
*/
- movi a4, exc_table
+ rsr a4, excsave1
mov a6, a1 # pass stack frame
mov a7, a0 # pass EXCCAUSE
addx4 a4, a0, a4
.global common_exception_return
common_exception_return:
-#ifdef CONFIG_TRACE_IRQFLAGS
- l32i a4, a1, PT_DEPC
- /* Double exception means we came here with an exception
- * while PS.EXCM was set, i.e. interrupts disabled.
- */
- bgeui a4, VALID_DOUBLE_EXCEPTION_ADDRESS, 1f
- l32i a4, a1, PT_EXCCAUSE
- bnei a4, EXCCAUSE_LEVEL1_INTERRUPT, 1f
- /* We came here with an interrupt means interrupts were enabled
- * and we'll reenable them on return.
- */
- movi a4, trace_hardirqs_on
- callx4 a4
1:
-#endif
+ rsil a2, LOCKLEVEL
/* Jump if we are returning from kernel exceptions. */
-1: l32i a3, a1, PT_PS
- _bbci.l a3, PS_UM_BIT, 4f
-
- rsil a2, 0
+ l32i a3, a1, PT_PS
+ GET_THREAD_INFO(a2, a1)
+ l32i a4, a2, TI_FLAGS
+ _bbci.l a3, PS_UM_BIT, 6f
/* Specific to a user exception exit:
* We need to check some flags for signal handling and rescheduling,
* Note that we don't disable interrupts here.
*/
- GET_THREAD_INFO(a2,a1)
- l32i a4, a2, TI_FLAGS
-
_bbsi.l a4, TIF_NEED_RESCHED, 3f
_bbsi.l a4, TIF_NOTIFY_RESUME, 2f
_bbci.l a4, TIF_SIGPENDING, 5f
/* Call do_signal() */
+ rsil a2, 0
movi a4, do_notify_resume # int do_notify_resume(struct pt_regs*)
mov a6, a1
callx4 a4
3: /* Reschedule */
+ rsil a2, 0
movi a4, schedule # void schedule (void)
callx4 a4
j 1b
+#ifdef CONFIG_PREEMPT
+6:
+ _bbci.l a4, TIF_NEED_RESCHED, 4f
+
+ /* Check current_thread_info->preempt_count */
+
+ l32i a4, a2, TI_PRE_COUNT
+ bnez a4, 4f
+ movi a4, preempt_schedule_irq
+ callx4 a4
+ j 1b
+#endif
+
5:
#ifdef CONFIG_DEBUG_TLB_SANITY
l32i a4, a1, PT_DEPC
movi a4, check_tlb_sanity
callx4 a4
#endif
-4: /* Restore optional registers. */
+6:
+4:
+#ifdef CONFIG_TRACE_IRQFLAGS
+ l32i a4, a1, PT_DEPC
+ /* Double exception means we came here with an exception
+ * while PS.EXCM was set, i.e. interrupts disabled.
+ */
+ bgeui a4, VALID_DOUBLE_EXCEPTION_ADDRESS, 1f
+ l32i a4, a1, PT_EXCCAUSE
+ bnei a4, EXCCAUSE_LEVEL1_INTERRUPT, 1f
+ /* We came here with an interrupt means interrupts were enabled
+ * and we'll reenable them on return.
+ */
+ movi a4, trace_hardirqs_on
+ callx4 a4
+1:
+#endif
+ /* Restore optional registers. */
load_xtregs_opt a1 a2 a4 a5 a6 a7 PT_XTREGS_OPT
kernel_exception_exit:
-#ifdef PREEMPTIBLE_KERNEL
-
-#ifdef CONFIG_PREEMPT
-
- /*
- * Note: We've just returned from a call4, so we have
- * at least 4 addt'l regs.
- */
-
- /* Check current_thread_info->preempt_count */
-
- GET_THREAD_INFO(a2)
- l32i a3, a2, TI_PREEMPT
- bnez a3, 1f
-
- l32i a2, a2, TI_FLAGS
-
-1:
-
-#endif
-
-#endif
-
/* Check if we have to do a movsp.
*
* We only have to do a movsp if the previous window-frame has
*
* The ALLOCA handler is entered when user code executes the MOVSP
* instruction and the caller's frame is not in the register file.
- * In this case, the caller frame's a0..a3 are on the stack just
- * below sp (a1), and this handler moves them.
*
- * For "MOVSP <ar>,<as>" without destination register a1, this routine
- * simply moves the value from <as> to <ar> without moving the save area.
+ * This algorithm was taken from the Ross Morley's RTOS Porting Layer:
+ *
+ * /home/ross/rtos/porting/XtensaRTOS-PortingLayer-20090507/xtensa_vectors.S
+ *
+ * It leverages the existing window spill/fill routines and their support for
+ * double exceptions. The 'movsp' instruction will only cause an exception if
+ * the next window needs to be loaded. In fact this ALLOCA exception may be
+ * replaced at some point by changing the hardware to do a underflow exception
+ * of the proper size instead.
+ *
+ * This algorithm simply backs out the register changes started by the user
+ * excpetion handler, makes it appear that we have started a window underflow
+ * by rotating the window back and then setting the old window base (OWB) in
+ * the 'ps' register with the rolled back window base. The 'movsp' instruction
+ * will be re-executed and this time since the next window frames is in the
+ * active AR registers it won't cause an exception.
+ *
+ * If the WindowUnderflow code gets a TLB miss the page will get mapped
+ * the the partial windeowUnderflow will be handeled in the double exception
+ * handler.
*
* Entry condition:
*
* a0: trashed, original value saved on stack (PT_AREG0)
* a1: a1
* a2: new stack pointer, original in DEPC
- * a3: dispatch table
+ * a3: a3
* depc: a2, original value saved on stack (PT_DEPC)
- * excsave_1: a3
+ * excsave_1: dispatch table
*
* PT_DEPC >= VALID_DOUBLE_EXCEPTION_ADDRESS: double exception, DEPC
* < VALID_DOUBLE_EXCEPTION_ADDRESS: regular exception
*/
-#if XCHAL_HAVE_BE
-#define _EXTUI_MOVSP_SRC(ar) extui ar, ar, 4, 4
-#define _EXTUI_MOVSP_DST(ar) extui ar, ar, 0, 4
-#else
-#define _EXTUI_MOVSP_SRC(ar) extui ar, ar, 0, 4
-#define _EXTUI_MOVSP_DST(ar) extui ar, ar, 4, 4
-#endif
-
ENTRY(fast_alloca)
+ rsr a0, windowbase
+ rotw -1
+ rsr a2, ps
+ extui a3, a2, PS_OWB_SHIFT, PS_OWB_WIDTH
+ xor a3, a3, a4
+ l32i a4, a6, PT_AREG0
+ l32i a1, a6, PT_DEPC
+ rsr a6, depc
+ wsr a1, depc
+ slli a3, a3, PS_OWB_SHIFT
+ xor a2, a2, a3
+ wsr a2, ps
+ rsync
- /* We shouldn't be in a double exception. */
-
- l32i a0, a2, PT_DEPC
- _bgeui a0, VALID_DOUBLE_EXCEPTION_ADDRESS, .Lunhandled_double
-
- rsr a0, depc # get a2
- s32i a4, a2, PT_AREG4 # save a4 and
- s32i a0, a2, PT_AREG2 # a2 to stack
-
- /* Exit critical section. */
-
- movi a0, 0
- s32i a0, a3, EXC_TABLE_FIXUP
-
- /* Restore a3, excsave_1 */
-
- xsr a3, excsave1 # make sure excsave_1 is valid for dbl.
- rsr a4, epc1 # get exception address
- s32i a3, a2, PT_AREG3 # save a3 to stack
-
-#ifdef ALLOCA_EXCEPTION_IN_IRAM
-#error iram not supported
-#else
- /* Note: l8ui not allowed in IRAM/IROM!! */
- l8ui a0, a4, 1 # read as(src) from MOVSP instruction
-#endif
- movi a3, .Lmovsp_src
- _EXTUI_MOVSP_SRC(a0) # extract source register number
- addx8 a3, a0, a3
- jx a3
-
-.Lunhandled_double:
- wsr a0, excsave1
- movi a0, unrecoverable_exception
- callx0 a0
-
- .align 8
-.Lmovsp_src:
- l32i a3, a2, PT_AREG0; _j 1f; .align 8
- mov a3, a1; _j 1f; .align 8
- l32i a3, a2, PT_AREG2; _j 1f; .align 8
- l32i a3, a2, PT_AREG3; _j 1f; .align 8
- l32i a3, a2, PT_AREG4; _j 1f; .align 8
- mov a3, a5; _j 1f; .align 8
- mov a3, a6; _j 1f; .align 8
- mov a3, a7; _j 1f; .align 8
- mov a3, a8; _j 1f; .align 8
- mov a3, a9; _j 1f; .align 8
- mov a3, a10; _j 1f; .align 8
- mov a3, a11; _j 1f; .align 8
- mov a3, a12; _j 1f; .align 8
- mov a3, a13; _j 1f; .align 8
- mov a3, a14; _j 1f; .align 8
- mov a3, a15; _j 1f; .align 8
-
-1:
-
-#ifdef ALLOCA_EXCEPTION_IN_IRAM
-#error iram not supported
-#else
- l8ui a0, a4, 0 # read ar(dst) from MOVSP instruction
-#endif
- addi a4, a4, 3 # step over movsp
- _EXTUI_MOVSP_DST(a0) # extract destination register
- wsr a4, epc1 # save new epc_1
-
- _bnei a0, 1, 1f # no 'movsp a1, ax': jump
-
- /* Move the save area. This implies the use of the L32E
- * and S32E instructions, because this move must be done with
- * the user's PS.RING privilege levels, not with ring 0
- * (kernel's) privileges currently active with PS.EXCM
- * set. Note that we have stil registered a fixup routine with the
- * double exception vector in case a double exception occurs.
- */
-
- /* a0,a4:avail a1:old user stack a2:exc. stack a3:new user stack. */
-
- l32e a0, a1, -16
- l32e a4, a1, -12
- s32e a0, a3, -16
- s32e a4, a3, -12
- l32e a0, a1, -8
- l32e a4, a1, -4
- s32e a0, a3, -8
- s32e a4, a3, -4
-
- /* Restore stack-pointer and all the other saved registers. */
-
- mov a1, a3
-
- l32i a4, a2, PT_AREG4
- l32i a3, a2, PT_AREG3
- l32i a0, a2, PT_AREG0
- l32i a2, a2, PT_AREG2
- rfe
-
- /* MOVSP <at>,<as> was invoked with <at> != a1.
- * Because the stack pointer is not being modified,
- * we should be able to just modify the pointer
- * without moving any save area.
- * The processor only traps these occurrences if the
- * caller window isn't live, so unfortunately we can't
- * use this as an alternate trap mechanism.
- * So we just do the move. This requires that we
- * resolve the destination register, not just the source,
- * so there's some extra work.
- * (PERHAPS NOT REALLY NEEDED, BUT CLEANER...)
- */
-
- /* a0 dst-reg, a1 user-stack, a2 stack, a3 value of src reg. */
-
-1: movi a4, .Lmovsp_dst
- addx8 a4, a0, a4
- jx a4
-
- .align 8
-.Lmovsp_dst:
- s32i a3, a2, PT_AREG0; _j 1f; .align 8
- mov a1, a3; _j 1f; .align 8
- s32i a3, a2, PT_AREG2; _j 1f; .align 8
- s32i a3, a2, PT_AREG3; _j 1f; .align 8
- s32i a3, a2, PT_AREG4; _j 1f; .align 8
- mov a5, a3; _j 1f; .align 8
- mov a6, a3; _j 1f; .align 8
- mov a7, a3; _j 1f; .align 8
- mov a8, a3; _j 1f; .align 8
- mov a9, a3; _j 1f; .align 8
- mov a10, a3; _j 1f; .align 8
- mov a11, a3; _j 1f; .align 8
- mov a12, a3; _j 1f; .align 8
- mov a13, a3; _j 1f; .align 8
- mov a14, a3; _j 1f; .align 8
- mov a15, a3; _j 1f; .align 8
-
-1: l32i a4, a2, PT_AREG4
- l32i a3, a2, PT_AREG3
- l32i a0, a2, PT_AREG0
- l32i a2, a2, PT_AREG2
- rfe
-
+ _bbci.l a4, 31, 4f
+ rotw -1
+ _bbci.l a8, 30, 8f
+ rotw -1
+ j _WindowUnderflow12
+8: j _WindowUnderflow8
+4: j _WindowUnderflow4
ENDPROC(fast_alloca)
/*
* a0: trashed, original value saved on stack (PT_AREG0)
* a1: a1
* a2: new stack pointer, original in DEPC
- * a3: dispatch table
+ * a3: a3
* depc: a2, original value saved on stack (PT_DEPC)
- * excsave_1: a3
+ * excsave_1: dispatch table
*/
ENTRY(fast_syscall_kernel)
l32i a0, a2, PT_AREG0 # restore a0
xsr a2, depc # restore a2, depc
- rsr a3, excsave1
wsr a0, excsave1
movi a0, unrecoverable_exception
* a0: a2 (syscall-nr), original value saved on stack (PT_AREG0)
* a1: a1
* a2: new stack pointer, original in a0 and DEPC
- * a3: dispatch table, original in excsave_1
+ * a3: a3
* a4..a15: unchanged
* depc: a2, original value saved on stack (PT_DEPC)
- * excsave_1: a3
+ * excsave_1: dispatch table
*
* PT_DEPC >= VALID_DOUBLE_EXCEPTION_ADDRESS: double exception, DEPC
* < VALID_DOUBLE_EXCEPTION_ADDRESS: regular exception
ENTRY(fast_syscall_xtensa)
- xsr a3, excsave1 # restore a3, excsave1
-
s32i a7, a2, PT_AREG7 # we need an additional register
movi a7, 4 # sizeof(unsigned int)
access_ok a3, a7, a0, a2, .Leac # a0: scratch reg, a2: sp
* a0: trashed, original value saved on stack (PT_AREG0)
* a1: a1
* a2: new stack pointer, original in DEPC
- * a3: dispatch table
+ * a3: a3
* depc: a2, original value saved on stack (PT_DEPC)
- * excsave_1: a3
+ * excsave_1: dispatch table
*
* Note: We assume the stack pointer is EXC_TABLE_KSTK in the fixup handler.
*/
/* Register a FIXUP handler (pass current wb as a parameter) */
+ xsr a3, excsave1
movi a0, fast_syscall_spill_registers_fixup
s32i a0, a3, EXC_TABLE_FIXUP
rsr a0, windowbase
s32i a0, a3, EXC_TABLE_PARAM
+ xsr a3, excsave1 # restore a3 and excsave_1
- /* Save a3 and SAR on stack. */
+ /* Save a3, a4 and SAR on stack. */
rsr a0, sar
- xsr a3, excsave1 # restore a3 and excsave_1
s32i a3, a2, PT_AREG3
s32i a4, a2, PT_AREG4
s32i a0, a2, PT_AREG5 # store SAR to PT_AREG5
* in WS, so that the exception handlers save them to the task stack.
*/
- rsr a3, excsave1 # get spill-mask
+ xsr a3, excsave1 # get spill-mask
slli a2, a3, 1 # shift left by one
slli a3, a2, 32-WSBITS
src a2, a2, a3 # a1 = xxwww1yyxxxwww1yy......
wsr a2, windowstart # set corrected windowstart
- movi a3, exc_table
+ rsr a3, excsave1
l32i a2, a3, EXC_TABLE_DOUBLE_SAVE # restore a2
l32i a3, a3, EXC_TABLE_PARAM # original WB (in user task)
/* Jump to the exception handler. */
- movi a3, exc_table
+ rsr a3, excsave1
rsr a0, exccause
addx4 a0, a0, a3 # find entry in table
l32i a0, a0, EXC_TABLE_FAST_USER # load handler
xsr a3, excsave1
movi a2, fast_syscall_spill_registers_fixup
s32i a2, a3, EXC_TABLE_FIXUP
+ s32i a0, a3, EXC_TABLE_DOUBLE_SAVE
rsr a2, windowbase
s32i a2, a3, EXC_TABLE_PARAM
l32i a2, a3, EXC_TABLE_KSTK
wsr a3, windowbase
rsync
- /* Restore a3 and return. */
-
- movi a3, exc_table
- xsr a3, excsave1
-
rfde
movi a0, 0
- movi a3, exc_table
+ rsr a3, excsave1
l32i a1, a3, EXC_TABLE_KSTK
- wsr a3, excsave1
movi a4, (1 << PS_WOE_BIT) | LOCKLEVEL
wsr a4, ps
* a0: trashed, original value saved on stack (PT_AREG0)
* a1: a1
* a2: new stack pointer, original in DEPC
- * a3: dispatch table
+ * a3: a3
* depc: a2, original value saved on stack (PT_DEPC)
- * excsave_1: a3
+ * excsave_1: dispatch table
*
* PT_DEPC >= VALID_DOUBLE_EXCEPTION_ADDRESS: double exception, DEPC
* < VALID_DOUBLE_EXCEPTION_ADDRESS: regular exception
ENTRY(fast_second_level_miss)
- /* Save a1. Note: we don't expect a double exception. */
+ /* Save a1 and a3. Note: we don't expect a double exception. */
s32i a1, a2, PT_AREG1
+ s32i a3, a2, PT_AREG3
/* We need to map the page of PTEs for the user task. Find
* the pointer to that page. Also, it's possible for tsk->mm
l32i a0, a1, TASK_MM # tsk->mm
beqz a0, 9f
-
- /* We deliberately destroy a3 that holds the exception table. */
-
8: rsr a3, excvaddr # fault address
_PGD_OFFSET(a0, a3, a1)
l32i a0, a0, 0 # read pmdval
/* Exit critical section. */
-4: movi a3, exc_table # restore a3
+4: rsr a3, excsave1
movi a0, 0
s32i a0, a3, EXC_TABLE_FIXUP
l32i a0, a2, PT_AREG0
l32i a1, a2, PT_AREG1
+ l32i a3, a2, PT_AREG3
l32i a2, a2, PT_DEPC
- xsr a3, excsave1
bgeui a2, VALID_DOUBLE_EXCEPTION_ADDRESS, 1f
2: /* Invalid PGD, default exception handling */
- movi a3, exc_table
rsr a1, depc
- xsr a3, excsave1
s32i a1, a2, PT_AREG2
- s32i a3, a2, PT_AREG3
mov a1, a2
rsr a2, ps
* a0: trashed, original value saved on stack (PT_AREG0)
* a1: a1
* a2: new stack pointer, original in DEPC
- * a3: dispatch table
+ * a3: a3
* depc: a2, original value saved on stack (PT_DEPC)
- * excsave_1: a3
+ * excsave_1: dispatch table
*
* PT_DEPC >= VALID_DOUBLE_EXCEPTION_ADDRESS: double exception, DEPC
* < VALID_DOUBLE_EXCEPTION_ADDRESS: regular exception
ENTRY(fast_store_prohibited)
- /* Save a1 and a4. */
+ /* Save a1 and a3. */
s32i a1, a2, PT_AREG1
- s32i a4, a2, PT_AREG4
+ s32i a3, a2, PT_AREG3
GET_CURRENT(a1,a2)
l32i a0, a1, TASK_MM # tsk->mm
beqz a0, 9f
8: rsr a1, excvaddr # fault address
- _PGD_OFFSET(a0, a1, a4)
+ _PGD_OFFSET(a0, a1, a3)
l32i a0, a0, 0
beqz a0, 2f
* and is not PAGE_NONE. See pgtable.h for possible PTE layouts.
*/
- _PTE_OFFSET(a0, a1, a4)
- l32i a4, a0, 0 # read pteval
+ _PTE_OFFSET(a0, a1, a3)
+ l32i a3, a0, 0 # read pteval
movi a1, _PAGE_CA_INVALID
- ball a4, a1, 2f
- bbci.l a4, _PAGE_WRITABLE_BIT, 2f
+ ball a3, a1, 2f
+ bbci.l a3, _PAGE_WRITABLE_BIT, 2f
movi a1, _PAGE_ACCESSED | _PAGE_DIRTY | _PAGE_HW_WRITE
- or a4, a4, a1
+ or a3, a3, a1
rsr a1, excvaddr
- s32i a4, a0, 0
+ s32i a3, a0, 0
/* We need to flush the cache if we have page coloring. */
#if (DCACHE_WAY_SIZE > PAGE_SIZE) && XCHAL_DCACHE_IS_WRITEBACK
dhwb a0, 0
#endif
pdtlb a0, a1
- wdtlb a4, a0
+ wdtlb a3, a0
/* Exit critical section. */
movi a0, 0
+ rsr a3, excsave1
s32i a0, a3, EXC_TABLE_FIXUP
/* Restore the working registers, and return. */
- l32i a4, a2, PT_AREG4
+ l32i a3, a2, PT_AREG3
l32i a1, a2, PT_AREG1
l32i a0, a2, PT_AREG0
l32i a2, a2, PT_DEPC
- /* Restore excsave1 and a3. */
-
- xsr a3, excsave1
bgeui a2, VALID_DOUBLE_EXCEPTION_ADDRESS, 1f
rsr a2, depc
2: /* If there was a problem, handle fault in C */
- rsr a4, depc # still holds a2
- xsr a3, excsave1
- s32i a4, a2, PT_AREG2
- s32i a3, a2, PT_AREG3
- l32i a4, a2, PT_AREG4
+ rsr a3, depc # still holds a2
+ s32i a3, a2, PT_AREG2
mov a1, a2
rsr a2, ps
s32i a0, a2, PT_DEPC # mark it as a regular exception
addx4 a0, a0, a3 # find entry in table
l32i a0, a0, EXC_TABLE_FAST_USER # load handler
+ xsr a3, excsave1 # restore a3 and dispatch table
jx a0
ENDPROC(_UserExceptionVector)
s32i a0, a2, PT_DEPC # mark it as a regular exception
addx4 a0, a0, a3 # find entry in table
l32i a0, a0, EXC_TABLE_FAST_KERNEL # load handler address
+ xsr a3, excsave1 # restore a3 and dispatch table
jx a0
ENDPROC(_KernelExceptionVector)
*
* a0: DEPC
* a1: a1
- * a2: trashed, original value in EXC_TABLE_DOUBLE_A2
+ * a2: trashed, original value in EXC_TABLE_DOUBLE_SAVE
* a3: exctable
* depc: a0
* excsave_1: a3
.section .DoubleExceptionVector.text, "ax"
.begin literal_prefix .DoubleExceptionVector
+ .globl _DoubleExceptionVector_WindowUnderflow
+ .globl _DoubleExceptionVector_WindowOverflow
ENTRY(_DoubleExceptionVector)
- /* Deliberately destroy excsave (don't assume it's value was valid). */
-
- wsr a3, excsave1 # save a3
+ xsr a3, excsave1
+ s32i a2, a3, EXC_TABLE_DOUBLE_SAVE
/* Check for kernel double exception (usually fatal). */
- rsr a3, ps
- _bbci.l a3, PS_UM_BIT, .Lksp
+ rsr a2, ps
+ _bbci.l a2, PS_UM_BIT, .Lksp
/* Check if we are currently handling a window exception. */
/* Note: We don't need to indicate that we enter a critical section. */
xsr a0, depc # get DEPC, save a0
- movi a3, WINDOW_VECTORS_VADDR
- _bltu a0, a3, .Lfixup
- addi a3, a3, WINDOW_VECTORS_SIZE
- _bgeu a0, a3, .Lfixup
+ movi a2, WINDOW_VECTORS_VADDR
+ _bltu a0, a2, .Lfixup
+ addi a2, a2, WINDOW_VECTORS_SIZE
+ _bgeu a0, a2, .Lfixup
/* Window overflow/underflow exception. Get stack pointer. */
- mov a3, a2
- /* This explicit literal and the following references to it are made
- * in order to fit DoubleExceptionVector.literals into the available
- * 16-byte gap before DoubleExceptionVector.text in the absence of
- * link time relaxation. See kernel/vmlinux.lds.S
- */
- .literal .Lexc_table, exc_table
- l32r a2, .Lexc_table
- l32i a2, a2, EXC_TABLE_KSTK
+ l32i a2, a3, EXC_TABLE_KSTK
/* Check for overflow/underflow exception, jump if overflow. */
- _bbci.l a0, 6, .Lovfl
-
- /* a0: depc, a1: a1, a2: kstk, a3: a2, depc: a0, excsave: a3 */
+ _bbci.l a0, 6, _DoubleExceptionVector_WindowOverflow
- /* Restart window underflow exception.
+ /*
+ * Restart window underflow exception.
+ * Currently:
+ * depc = orig a0,
+ * a0 = orig DEPC,
+ * a2 = new sp based on KSTK from exc_table
+ * a3 = excsave_1
+ * excsave_1 = orig a3
+ *
* We return to the instruction in user space that caused the window
* underflow exception. Therefore, we change window base to the value
* before we entered the window underflow exception and prepare the
* by changing depc (in a0).
* Note: We can trash the current window frame (a0...a3) and depc!
*/
-
+_DoubleExceptionVector_WindowUnderflow:
+ xsr a3, excsave1
wsr a2, depc # save stack pointer temporarily
rsr a0, ps
- extui a0, a0, PS_OWB_SHIFT, 4
+ extui a0, a0, PS_OWB_SHIFT, PS_OWB_WIDTH
wsr a0, windowbase
rsync
xsr a2, depc # save a2 and get stack pointer
s32i a0, a2, PT_AREG0
-
- wsr a3, excsave1 # save a3
- l32r a3, .Lexc_table
-
+ xsr a3, excsave1
rsr a0, exccause
s32i a0, a2, PT_DEPC # mark it as a regular exception
addx4 a0, a0, a3
+ xsr a3, excsave1
l32i a0, a0, EXC_TABLE_FAST_USER
jx a0
-.Lfixup:/* Check for a fixup handler or if we were in a critical section. */
+ /*
+ * We only allow the ITLB miss exception if we are in kernel space.
+ * All other exceptions are unexpected and thus unrecoverable!
+ */
+
+#ifdef CONFIG_MMU
+ .extern fast_second_level_miss_double_kernel
+
+.Lksp: /* a0: a0, a1: a1, a2: a2, a3: trashed, depc: depc, excsave: a3 */
+
+ rsr a3, exccause
+ beqi a3, EXCCAUSE_ITLB_MISS, 1f
+ addi a3, a3, -EXCCAUSE_DTLB_MISS
+ bnez a3, .Lunrecoverable
+1: movi a3, fast_second_level_miss_double_kernel
+ jx a3
+#else
+.equ .Lksp, .Lunrecoverable
+#endif
+
+ /* Critical! We can't handle this situation. PANIC! */
- /* a0: depc, a1: a1, a2: a2, a3: trashed, depc: a0, excsave1: a3 */
+ .extern unrecoverable_exception
- l32r a3, .Lexc_table
- s32i a2, a3, EXC_TABLE_DOUBLE_SAVE # temporary variable
+.Lunrecoverable_fixup:
+ l32i a2, a3, EXC_TABLE_DOUBLE_SAVE
+ xsr a0, depc
+
+.Lunrecoverable:
+ rsr a3, excsave1
+ wsr a0, excsave1
+ movi a0, unrecoverable_exception
+ callx0 a0
+
+.Lfixup:/* Check for a fixup handler or if we were in a critical section. */
+
+ /* a0: depc, a1: a1, a2: trash, a3: exctable, depc: a0, excsave1: a3 */
/* Enter critical section. */
l32i a2, a3, EXC_TABLE_FIXUP
s32i a3, a3, EXC_TABLE_FIXUP
- beq a2, a3, .Lunrecoverable_fixup # critical!
+ beq a2, a3, .Lunrecoverable_fixup # critical section
beqz a2, .Ldflt # no handler was registered
/* a0: depc, a1: a1, a2: trash, a3: exctable, depc: a0, excsave: a3 */
.Ldflt: /* Get stack pointer. */
- l32i a3, a3, EXC_TABLE_DOUBLE_SAVE
- addi a2, a3, -PT_USER_SIZE
-
-.Lovfl: /* Jump to default handlers. */
+ l32i a2, a3, EXC_TABLE_DOUBLE_SAVE
+ addi a2, a2, -PT_USER_SIZE
- /* a0: depc, a1: a1, a2: kstk, a3: a2, depc: a0, excsave: a3 */
+ /* a0: depc, a1: a1, a2: kstk, a3: exctable, depc: a0, excsave: a3 */
- xsr a3, depc
s32i a0, a2, PT_DEPC
- s32i a3, a2, PT_AREG0
+ l32i a0, a3, EXC_TABLE_DOUBLE_SAVE
+ xsr a0, depc
+ s32i a0, a2, PT_AREG0
- /* a0: avail, a1: a1, a2: kstk, a3: avail, depc: a2, excsave: a3 */
+ /* a0: avail, a1: a1, a2: kstk, a3: exctable, depc: a2, excsave: a3 */
- l32r a3, .Lexc_table
rsr a0, exccause
addx4 a0, a0, a3
+ xsr a3, excsave1
l32i a0, a0, EXC_TABLE_FAST_USER
jx a0
/*
- * We only allow the ITLB miss exception if we are in kernel space.
- * All other exceptions are unexpected and thus unrecoverable!
+ * Restart window OVERFLOW exception.
+ * Currently:
+ * depc = orig a0,
+ * a0 = orig DEPC,
+ * a2 = new sp based on KSTK from exc_table
+ * a3 = EXCSAVE_1
+ * excsave_1 = orig a3
+ *
+ * We return to the instruction in user space that caused the window
+ * overflow exception. Therefore, we change window base to the value
+ * before we entered the window overflow exception and prepare the
+ * registers to return as if we were coming from a regular exception
+ * by changing DEPC (in a0).
+ *
+ * NOTE: We CANNOT trash the current window frame (a0...a3), but we
+ * can clobber depc.
+ *
+ * The tricky part here is that overflow8 and overflow12 handlers
+ * save a0, then clobber a0. To restart the handler, we have to restore
+ * a0 if the double exception was past the point where a0 was clobbered.
+ *
+ * To keep things simple, we take advantage of the fact all overflow
+ * handlers save a0 in their very first instruction. If DEPC was past
+ * that instruction, we can safely restore a0 from where it was saved
+ * on the stack.
+ *
+ * a0: depc, a1: a1, a2: kstk, a3: exc_table, depc: a0, excsave1: a3
*/
+_DoubleExceptionVector_WindowOverflow:
+ extui a2, a0, 0, 6 # get offset into 64-byte vector handler
+ beqz a2, 1f # if at start of vector, don't restore
-#ifdef CONFIG_MMU
- .extern fast_second_level_miss_double_kernel
+ addi a0, a0, -128
+ bbsi a0, 8, 1f # don't restore except for overflow 8 and 12
+ bbsi a0, 7, 2f
-.Lksp: /* a0: a0, a1: a1, a2: a2, a3: trashed, depc: depc, excsave: a3 */
+ /*
+ * Restore a0 as saved by _WindowOverflow8().
+ *
+ * FIXME: we really need a fixup handler for this L32E,
+ * for the extremely unlikely case where the overflow handler's
+ * reference thru a0 gets a hardware TLB refill that bumps out
+ * the (distinct, aliasing) TLB entry that mapped its prior
+ * references thru a9, and where our reference now thru a9
+ * gets a 2nd-level miss exception (not hardware TLB refill).
+ */
- rsr a3, exccause
- beqi a3, EXCCAUSE_ITLB_MISS, 1f
- addi a3, a3, -EXCCAUSE_DTLB_MISS
- bnez a3, .Lunrecoverable
-1: movi a3, fast_second_level_miss_double_kernel
- jx a3
-#else
-.equ .Lksp, .Lunrecoverable
-#endif
+ l32e a2, a9, -16
+ wsr a2, depc # replace the saved a0
+ j 1f
- /* Critical! We can't handle this situation. PANIC! */
+2:
+ /*
+ * Restore a0 as saved by _WindowOverflow12().
+ *
+ * FIXME: we really need a fixup handler for this L32E,
+ * for the extremely unlikely case where the overflow handler's
+ * reference thru a0 gets a hardware TLB refill that bumps out
+ * the (distinct, aliasing) TLB entry that mapped its prior
+ * references thru a13, and where our reference now thru a13
+ * gets a 2nd-level miss exception (not hardware TLB refill).
+ */
- .extern unrecoverable_exception
+ l32e a2, a13, -16
+ wsr a2, depc # replace the saved a0
+1:
+ /*
+ * Restore WindowBase while leaving all address registers restored.
+ * We have to use ROTW for this, because WSR.WINDOWBASE requires
+ * an address register (which would prevent restore).
+ *
+ * Window Base goes from 0 ... 7 (Module 8)
+ * Window Start is 8 bits; Ex: (0b1010 1010):0x55 from series of call4s
+ */
+
+ rsr a0, ps
+ extui a0, a0, PS_OWB_SHIFT, PS_OWB_WIDTH
+ rsr a2, windowbase
+ sub a0, a2, a0
+ extui a0, a0, 0, 3
-.Lunrecoverable_fixup:
l32i a2, a3, EXC_TABLE_DOUBLE_SAVE
- xsr a0, depc
+ xsr a3, excsave1
+ beqi a0, 1, .L1pane
+ beqi a0, 3, .L3pane
-.Lunrecoverable:
- rsr a3, excsave1
- wsr a0, excsave1
- movi a0, unrecoverable_exception
- callx0 a0
+ rsr a0, depc
+ rotw -2
+
+ /*
+ * We are now in the user code's original window frame.
+ * Process the exception as a user exception as if it was
+ * taken by the user code.
+ *
+ * This is similar to the user exception vector,
+ * except that PT_DEPC isn't set to EXCCAUSE.
+ */
+1:
+ xsr a3, excsave1
+ wsr a2, depc
+ l32i a2, a3, EXC_TABLE_KSTK
+ s32i a0, a2, PT_AREG0
+ rsr a0, exccause
+
+ s32i a0, a2, PT_DEPC
+
+ addx4 a0, a0, a3
+ l32i a0, a0, EXC_TABLE_FAST_USER
+ xsr a3, excsave1
+ jx a0
+
+.L1pane:
+ rsr a0, depc
+ rotw -1
+ j 1b
+
+.L3pane:
+ rsr a0, depc
+ rotw -3
+ j 1b
.end literal_prefix
projects / karo-tx-linux.git / commitdiff