arch/arm/lib/memcpy.S

   1 /*
   2  *  linux/arch/arm/lib/memcpy.S
   3  *
   4  *  Author:     Nicolas Pitre
   5  *  Created:    Sep 28, 2005
   6  *  Copyright:  MontaVista Software, Inc.
   7  *
   8  *  This program is free software; you can redistribute it and/or modify
   9  *  it under the terms of the GNU General Public License version 2 as
  10  *  published by the Free Software Foundation.
  11  */
  12
  13 #include <linux/linkage.h>
  14 #include <asm/assembler.h>
  15
  16 #ifdef CONFIG_SYS_THUMB_BUILD
  17 #define W(instr)        instr.w
  18 #else
  19 #define W(instr)        instr
  20 #endif
  21
  22 #define LDR1W_SHIFT     0
  23 #define STR1W_SHIFT     0
  24
  25         .macro ldr1w ptr reg abort
  26         W(ldr) \reg, [\ptr], #4
  27         .endm
  28
  29         .macro ldr4w ptr reg1 reg2 reg3 reg4 abort
  30         ldmia \ptr!, {\reg1, \reg2, \reg3, \reg4}
  31         .endm
  32
  33         .macro ldr8w ptr reg1 reg2 reg3 reg4 reg5 reg6 reg7 reg8 abort
  34         ldmia \ptr!, {\reg1, \reg2, \reg3, \reg4, \reg5, \reg6, \reg7, \reg8}
  35         .endm
  36
  37         .macro ldr1b ptr reg cond=al abort
  38         ldrb\cond\() \reg, [\ptr], #1
  39         .endm
  40
  41         .macro str1w ptr reg abort
  42         W(str) \reg, [\ptr], #4
  43         .endm
  44
  45         .macro str8w ptr reg1 reg2 reg3 reg4 reg5 reg6 reg7 reg8 abort
  46         stmia \ptr!, {\reg1, \reg2, \reg3, \reg4, \reg5, \reg6, \reg7, \reg8}
  47         .endm
  48
  49         .macro str1b ptr reg cond=al abort
  50         strb\cond\() \reg, [\ptr], #1
  51         .endm
  52
  53         .macro enter reg1 reg2
  54         stmdb sp!, {r0, \reg1, \reg2}
  55         .endm
  56
  57         .macro exit reg1 reg2
  58         ldmfd sp!, {r0, \reg1, \reg2}
  59         .endm
  60
  61         .text
  62
  63 /* Prototype: void *memcpy(void *dest, const void *src, size_t n); */
  64         .syntax unified
  65 #ifdef CONFIG_SYS_THUMB_BUILD
  66         .thumb
  67         .thumb_func
  68 #endif
  69 ENTRY(memcpy)
  70                 cmp     r0, r1
  71                 moveq   pc, lr
  72
  73                 enter   r4, lr
  74
  75                 subs    r2, r2, #4
  76                 blt     8f
  77                 ands    ip, r0, #3
  78         PLD(    pld     [r1, #0]                )
  79                 bne     9f
  80                 ands    ip, r1, #3
  81                 bne     10f
  82
  83 1:              subs    r2, r2, #(28)
  84                 stmfd   sp!, {r5 - r8}
  85                 blt     5f
  86
  87         CALGN(  ands    ip, r0, #31             )
  88         CALGN(  rsb     r3, ip, #32             )
  89         CALGN(  sbcsne  r4, r3, r2              )  @ C is always set here
  90         CALGN(  bcs     2f                      )
  91         CALGN(  adr     r4, 6f                  )
  92         CALGN(  subs    r2, r2, r3              )  @ C gets set
  93         CALGN(  add     pc, r4, ip              )
  94
  95         PLD(    pld     [r1, #0]                )
  96 2:      PLD(    subs    r2, r2, #96             )
  97         PLD(    pld     [r1, #28]               )
  98         PLD(    blt     4f                      )
  99         PLD(    pld     [r1, #60]               )
 100         PLD(    pld     [r1, #92]               )
 101
 102 3:      PLD(    pld     [r1, #124]              )
 103 4:              ldr8w   r1, r3, r4, r5, r6, r7, r8, ip, lr, abort=20f
 104                 subs    r2, r2, #32
 105                 str8w   r0, r3, r4, r5, r6, r7, r8, ip, lr, abort=20f
 106                 bge     3b
 107         PLD(    cmn     r2, #96                 )
 108         PLD(    bge     4b                      )
 109
 110 5:              ands    ip, r2, #28
 111                 rsb     ip, ip, #32
 112 #if LDR1W_SHIFT > 0
 113                 lsl     ip, ip, #LDR1W_SHIFT
 114 #endif
 115                 addne   pc, pc, ip              @ C is always clear here
 116                 b       7f
 117 6:
 118                 .rept   (1 << LDR1W_SHIFT)
 119                 W(nop)
 120                 .endr
 121                 ldr1w   r1, r3, abort=20f
 122                 ldr1w   r1, r4, abort=20f
 123                 ldr1w   r1, r5, abort=20f
 124                 ldr1w   r1, r6, abort=20f
 125                 ldr1w   r1, r7, abort=20f
 126                 ldr1w   r1, r8, abort=20f
 127                 ldr1w   r1, lr, abort=20f
 128
 129 #if LDR1W_SHIFT < STR1W_SHIFT
 130                 lsl     ip, ip, #STR1W_SHIFT - LDR1W_SHIFT
 131 #elif LDR1W_SHIFT > STR1W_SHIFT
 132                 lsr     ip, ip, #LDR1W_SHIFT - STR1W_SHIFT
 133 #endif
 134                 add     pc, pc, ip
 135                 nop
 136                 .rept   (1 << STR1W_SHIFT)
 137                 W(nop)
 138                 .endr
 139                 str1w   r0, r3, abort=20f
 140                 str1w   r0, r4, abort=20f
 141                 str1w   r0, r5, abort=20f
 142                 str1w   r0, r6, abort=20f
 143                 str1w   r0, r7, abort=20f
 144                 str1w   r0, r8, abort=20f
 145                 str1w   r0, lr, abort=20f
 146
 147         CALGN(  bcs     2b                      )
 148
 149 7:              ldmfd   sp!, {r5 - r8}
 150
 151 8:              movs    r2, r2, lsl #31
 152                 ldr1b   r1, r3, ne, abort=21f
 153                 ldr1b   r1, r4, cs, abort=21f
 154                 ldr1b   r1, ip, cs, abort=21f
 155                 str1b   r0, r3, ne, abort=21f
 156                 str1b   r0, r4, cs, abort=21f
 157                 str1b   r0, ip, cs, abort=21f
 158
 159                 exit    r4, pc
 160
 161 9:              rsb     ip, ip, #4
 162                 cmp     ip, #2
 163                 ldr1b   r1, r3, gt, abort=21f
 164                 ldr1b   r1, r4, ge, abort=21f
 165                 ldr1b   r1, lr, abort=21f
 166                 str1b   r0, r3, gt, abort=21f
 167                 str1b   r0, r4, ge, abort=21f
 168                 subs    r2, r2, ip
 169                 str1b   r0, lr, abort=21f
 170                 blt     8b
 171                 ands    ip, r1, #3
 172                 beq     1b
 173
 174 10:             bic     r1, r1, #3
 175                 cmp     ip, #2
 176                 ldr1w   r1, lr, abort=21f
 177                 beq     17f
 178                 bgt     18f
 179
 180
 181                 .macro  forward_copy_shift pull push
 182
 183                 subs    r2, r2, #28
 184                 blt     14f
 185
 186         CALGN(  ands    ip, r0, #31             )
 187         CALGN(  rsb     ip, ip, #32             )
 188         CALGN(  sbcsne  r4, ip, r2              )  @ C is always set here
 189         CALGN(  subcc   r2, r2, ip              )
 190         CALGN(  bcc     15f                     )
 191
 192 11:             stmfd   sp!, {r5 - r9}
 193
 194         PLD(    pld     [r1, #0]                )
 195         PLD(    subs    r2, r2, #96             )
 196         PLD(    pld     [r1, #28]               )
 197         PLD(    blt     13f                     )
 198         PLD(    pld     [r1, #60]               )
 199         PLD(    pld     [r1, #92]               )
 200
 201 12:     PLD(    pld     [r1, #124]              )
 202 13:             ldr4w   r1, r4, r5, r6, r7, abort=19f
 203                 mov     r3, lr, lspull #\pull
 204                 subs    r2, r2, #32
 205                 ldr4w   r1, r8, r9, ip, lr, abort=19f
 206                 orr     r3, r3, r4, lspush #\push
 207                 mov     r4, r4, lspull #\pull
 208                 orr     r4, r4, r5, lspush #\push
 209                 mov     r5, r5, lspull #\pull
 210                 orr     r5, r5, r6, lspush #\push
 211                 mov     r6, r6, lspull #\pull
 212                 orr     r6, r6, r7, lspush #\push
 213                 mov     r7, r7, lspull #\pull
 214                 orr     r7, r7, r8, lspush #\push
 215                 mov     r8, r8, lspull #\pull
 216                 orr     r8, r8, r9, lspush #\push
 217                 mov     r9, r9, lspull #\pull
 218                 orr     r9, r9, ip, lspush #\push
 219                 mov     ip, ip, lspull #\pull
 220                 orr     ip, ip, lr, lspush #\push
 221                 str8w   r0, r3, r4, r5, r6, r7, r8, r9, ip, , abort=19f
 222                 bge     12b
 223         PLD(    cmn     r2, #96                 )
 224         PLD(    bge     13b                     )
 225
 226                 ldmfd   sp!, {r5 - r9}
 227
 228 14:             ands    ip, r2, #28
 229                 beq     16f
 230
 231 15:             mov     r3, lr, lspull #\pull
 232                 ldr1w   r1, lr, abort=21f
 233                 subs    ip, ip, #4
 234                 orr     r3, r3, lr, lspush #\push
 235                 str1w   r0, r3, abort=21f
 236                 bgt     15b
 237         CALGN(  cmp     r2, #0                  )
 238         CALGN(  bge     11b                     )
 239
 240 16:             sub     r1, r1, #(\push / 8)
 241                 b       8b
 242
 243                 .endm
 244
 245
 246                 forward_copy_shift      pull=8  push=24
 247
 248 17:             forward_copy_shift      pull=16 push=16
 249
 250 18:             forward_copy_shift      pull=24 push=8
 251
 252
 253 /*
 254  * Abort preamble and completion macros.
 255  * If a fixup handler is required then those macros must surround it.
 256  * It is assumed that the fixup code will handle the private part of
 257  * the exit macro.
 258  */
 259
 260         .macro  copy_abort_preamble
 261 19:     ldmfd   sp!, {r5 - r9}
 262         b       21f
 263 20:     ldmfd   sp!, {r5 - r8}
 264 21:
 265         .endm
 266
 267         .macro  copy_abort_end
 268         ldmfd   sp!, {r4, pc}
 269         .endm
 270
 271 ENDPROC(memcpy)