powerpc: Use enhanced touch instructions in POWER7 copy_to_user/copy_from_user

[karo-tx-linux.git] / arch / powerpc / lib / copyuser_power7.S

diff --git a/arch/powerpc/lib/copyuser_power7.S b/arch/powerpc/lib/copyuser_power7.S
index 497db7b23bb1be8be3518c12a6e0e7307ddb2fe0..9c982cdec3cffe0a51612ac4454bd5d2b7af1e6f 100644 (file)
--- a/arch/powerpc/lib/copyuser_power7.S
+++ b/arch/powerpc/lib/copyuser_power7.S
@@ -298,6 +298,37 @@ err1;      stb     r0,0(r3)
        ld      r5,STACKFRAMESIZE+64(r1)
        mtlr    r0
 
+       /*
+        * We prefetch both the source and destination using enhanced touch
+        * instructions. We use a stream ID of 0 for the load side and
+        * 1 for the store side.
+        */
+       clrrdi  r6,r4,7
+       clrrdi  r9,r3,7
+       ori     r9,r9,1         /* stream=1 */
+
+       srdi    r7,r5,7         /* length in cachelines, capped at 0x3FF */
+       cmpldi  r7,0x3FF
+       ble     1f
+       li      r7,0x3FF
+1:     lis     r0,0x0E00       /* depth=7 */
+       sldi    r7,r7,7
+       or      r7,r7,r0
+       ori     r10,r7,1        /* stream=1 */
+
+       lis     r8,0x8000       /* GO=1 */
+       clrldi  r8,r8,32
+
+.machine push
+.machine "power4"
+       dcbt    r0,r6,0b01000
+       dcbt    r0,r7,0b01010
+       dcbtst  r0,r9,0b01000
+       dcbtst  r0,r10,0b01010
+       eieio
+       dcbt    r0,r8,0b01010   /* GO */
+.machine pop
+
        beq     .Lunwind_stack_nonvmx_copy
 
        /*