1/* 2 * Copyright (C) 2013 The Android Open Source Project 3 * All rights reserved. 4 * 5 * Redistribution and use in source and binary forms, with or without 6 * modification, are permitted provided that the following conditions 7 * are met: 8 * * Redistributions of source code must retain the above copyright 9 * notice, this list of conditions and the following disclaimer. 10 * * Redistributions in binary form must reproduce the above copyright 11 * notice, this list of conditions and the following disclaimer in 12 * the documentation and/or other materials provided with the 13 * distribution. 14 * 15 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS 16 * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT 17 * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS 18 * FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE 19 * COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, 20 * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, 21 * BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS 22 * OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED 23 * AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, 24 * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT 25 * OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 26 * SUCH DAMAGE. 27 */ 28 29 30/* 31 * This code assumes it is running on a processor that supports all arm v7 32 * instructions, that supports neon instructions, and that has a 32 byte 33 * cache line. 34 */ 35 36// Assumes neon instructions and a cache line size of 32 bytes. 37 38ENTRY(MEMCPY_BASE) 39 .cfi_startproc 40 .save {r0, lr} 41 .cfi_def_cfa_offset 8 42 .cfi_rel_offset r0, 0 43 .cfi_rel_offset lr, 4 44 45 /* do we have at least 16-bytes to copy (needed for alignment below) */ 46 cmp r2, #16 47 blo 5f 48 49 /* align destination to cache-line for the write-buffer */ 50 rsb r3, r0, #0 51 ands r3, r3, #0xF 52 beq 2f 53 54 /* copy up to 15-bytes (count in r3) */ 55 sub r2, r2, r3 56 movs ip, r3, lsl #31 57 itt mi 58 ldrbmi lr, [r1], #1 59 strbmi lr, [r0], #1 60 itttt cs 61 ldrbcs ip, [r1], #1 62 ldrbcs lr, [r1], #1 63 strbcs ip, [r0], #1 64 strbcs lr, [r0], #1 65 movs ip, r3, lsl #29 66 bge 1f 67 // copies 4 bytes, destination 32-bits aligned 68 vld4.8 {d0[0], d1[0], d2[0], d3[0]}, [r1]! 69 vst4.8 {d0[0], d1[0], d2[0], d3[0]}, [r0, :32]! 701: bcc 2f 71 // copies 8 bytes, destination 64-bits aligned 72 vld1.8 {d0}, [r1]! 73 vst1.8 {d0}, [r0, :64]! 74 752: /* make sure we have at least 64 bytes to copy */ 76 subs r2, r2, #64 77 blo 2f 78 791: /* The main loop copies 64 bytes at a time */ 80 vld1.8 {d0 - d3}, [r1]! 81 vld1.8 {d4 - d7}, [r1]! 82 pld [r1, #(32*8)] 83 subs r2, r2, #64 84 vst1.8 {d0 - d3}, [r0, :128]! 85 vst1.8 {d4 - d7}, [r0, :128]! 86 bhs 1b 87 882: /* fix-up the remaining count and make sure we have >= 32 bytes left */ 89 adds r2, r2, #32 90 blo 4f 91 92 /* Copy 32 bytes. These cache lines were already preloaded */ 93 vld1.8 {d0 - d3}, [r1]! 94 sub r2, r2, #32 95 vst1.8 {d0 - d3}, [r0, :128]! 96 974: /* less than 32 left */ 98 add r2, r2, #32 99 tst r2, #0x10 100 beq 5f 101 // copies 16 bytes, 128-bits aligned 102 vld1.8 {d0, d1}, [r1]! 103 vst1.8 {d0, d1}, [r0, :128]! 104 1055: /* copy up to 15-bytes (count in r2) */ 106 movs ip, r2, lsl #29 107 bcc 1f 108 vld1.8 {d0}, [r1]! 109 vst1.8 {d0}, [r0]! 1101: bge 2f 111 vld4.8 {d0[0], d1[0], d2[0], d3[0]}, [r1]! 112 vst4.8 {d0[0], d1[0], d2[0], d3[0]}, [r0]! 1132: movs ip, r2, lsl #31 114 itt mi 115 ldrbmi r3, [r1], #1 116 strbmi r3, [r0], #1 117 itttt cs 118 ldrbcs ip, [r1], #1 119 ldrbcs lr, [r1], #1 120 strbcs ip, [r0], #1 121 strbcs lr, [r0], #1 122 123 ldmfd sp!, {r0, lr} 124 bx lr 125 126 .cfi_endproc 127END(MEMCPY_BASE) 128