memset.S revision eb847bc8666842a3cfc9c06e8458ad1abebebaf0
1/* 2 * Copyright (C) 2013 The Android Open Source Project 3 * All rights reserved. 4 * 5 * Redistribution and use in source and binary forms, with or without 6 * modification, are permitted provided that the following conditions 7 * are met: 8 * * Redistributions of source code must retain the above copyright 9 * notice, this list of conditions and the following disclaimer. 10 * * Redistributions in binary form must reproduce the above copyright 11 * notice, this list of conditions and the following disclaimer in 12 * the documentation and/or other materials provided with the 13 * distribution. 14 * 15 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS 16 * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT 17 * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS 18 * FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE 19 * COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, 20 * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, 21 * BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS 22 * OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED 23 * AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, 24 * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT 25 * OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 26 * SUCH DAMAGE. 27 */ 28 29#include <machine/cpu-features.h> 30#include <machine/asm.h> 31#include "private/libc_events.h" 32 33 /* 34 * Optimized memset() for ARM. 35 * 36 * memset() returns its first argument. 37 */ 38 39 .fpu neon 40 .syntax unified 41 42ENTRY(__memset_chk) 43 .cfi_startproc 44 cmp r2, r3 45 bls .L_done 46 47 // Preserve lr for backtrace. 48 .save {lr} 49 push {lr} 50 .cfi_def_cfa_offset 4 51 .cfi_rel_offset lr, 0 52 53 ldr r0, error_message 54 ldr r1, error_code 551: 56 add r0, pc 57 bl __fortify_chk_fail 58error_code: 59 .word BIONIC_EVENT_MEMSET_BUFFER_OVERFLOW 60error_message: 61 .word error_string-(1b+8) 62 63 .cfi_endproc 64END(__memset_chk) 65 66ENTRY(bzero) 67 .cfi_startproc 68 mov r2, r1 69 mov r1, #0 70.L_done: 71 // Fall through to memset... 72 .cfi_endproc 73END(bzero) 74 75ENTRY(memset) 76 .cfi_startproc 77 .save {r0} 78 stmfd sp!, {r0} 79 .cfi_def_cfa_offset 4 80 .cfi_rel_offset r0, 0 81 82 // The new algorithm is slower for copies < 16 so use the old 83 // neon code in that case. 84 cmp r2, #16 85 blo .L_set_less_than_16_unknown_align 86 87 // Use strd which requires an even and odd register so move the 88 // values so that: 89 // r0 and r1 contain the memset value 90 // r2 is the number of bytes to set 91 // r3 is the destination pointer 92 mov r3, r0 93 94 // Copy the byte value in every byte of r1. 95 mov r1, r1, lsl #24 96 orr r1, r1, r1, lsr #8 97 orr r1, r1, r1, lsr #16 98 99.L_check_alignment: 100 // Align destination to a double word to avoid the strd crossing 101 // a cache line boundary. 102 ands ip, r3, #7 103 bne .L_do_double_word_align 104 105.L_double_word_aligned: 106 mov r0, r1 107 108 subs r2, #64 109 blo .L_set_less_than_64 110 1111: // Main loop sets 64 bytes at a time. 112 .irp offset, #0, #8, #16, #24, #32, #40, #48, #56 113 strd r0, r1, [r3, \offset] 114 .endr 115 116 add r3, #64 117 subs r2, #64 118 bge 1b 119 120.L_set_less_than_64: 121 // Restore r2 to the count of bytes left to set. 122 add r2, #64 123 lsls ip, r2, #27 124 bcc .L_set_less_than_32 125 // Set 32 bytes. 126 .irp offset, #0, #8, #16, #24 127 strd r0, r1, [r3, \offset] 128 .endr 129 add r3, #32 130 131.L_set_less_than_32: 132 bpl .L_set_less_than_16 133 // Set 16 bytes. 134 .irp offset, #0, #8 135 strd r0, r1, [r3, \offset] 136 .endr 137 add r3, #16 138 139.L_set_less_than_16: 140 // Less than 16 bytes to set. 141 lsls ip, r2, #29 142 bcc .L_set_less_than_8 143 144 // Set 8 bytes. 145 strd r0, r1, [r3], #8 146 147.L_set_less_than_8: 148 bpl .L_set_less_than_4 149 // Set 4 bytes 150 str r1, [r3], #4 151 152.L_set_less_than_4: 153 lsls ip, r2, #31 154 it ne 155 strbne r1, [r3], #1 156 itt cs 157 strbcs r1, [r3], #1 158 strbcs r1, [r3] 159 160 ldmfd sp!, {r0} 161 bx lr 162 163.L_do_double_word_align: 164 rsb ip, ip, #8 165 sub r2, r2, ip 166 movs r0, ip, lsl #31 167 it mi 168 strbmi r1, [r3], #1 169 itt cs 170 strbcs r1, [r3], #1 171 strbcs r1, [r3], #1 172 173 // Dst is at least word aligned by this point. 174 cmp ip, #4 175 blo .L_double_word_aligned 176 str r1, [r3], #4 177 b .L_double_word_aligned 178 179.L_set_less_than_16_unknown_align: 180 // Set up to 15 bytes. 181 vdup.8 d0, r1 182 movs ip, r2, lsl #29 183 bcc 1f 184 vst1.8 {d0}, [r0]! 1851: bge 2f 186 vst1.32 {d0[0]}, [r0]! 1872: movs ip, r2, lsl #31 188 it mi 189 strbmi r1, [r0], #1 190 itt cs 191 strbcs r1, [r0], #1 192 strbcs r1, [r0], #1 193 ldmfd sp!, {r0} 194 bx lr 195 .cfi_endproc 196END(memset) 197 198 .data 199error_string: 200 .string "memset prevented write past end of buffer" 201