104954a43b362b8c817cc5859513efad0c344f412Christopher Ferris/* 2796cbe249bf6626b1de01664c38a7b06dcb2eb1fChristopher Ferris * Copyright (C) 2013 The Android Open Source Project 304954a43b362b8c817cc5859513efad0c344f412Christopher Ferris * All rights reserved. 404954a43b362b8c817cc5859513efad0c344f412Christopher Ferris * 504954a43b362b8c817cc5859513efad0c344f412Christopher Ferris * Redistribution and use in source and binary forms, with or without 604954a43b362b8c817cc5859513efad0c344f412Christopher Ferris * modification, are permitted provided that the following conditions 704954a43b362b8c817cc5859513efad0c344f412Christopher Ferris * are met: 804954a43b362b8c817cc5859513efad0c344f412Christopher Ferris * * Redistributions of source code must retain the above copyright 904954a43b362b8c817cc5859513efad0c344f412Christopher Ferris * notice, this list of conditions and the following disclaimer. 1004954a43b362b8c817cc5859513efad0c344f412Christopher Ferris * * Redistributions in binary form must reproduce the above copyright 1104954a43b362b8c817cc5859513efad0c344f412Christopher Ferris * notice, this list of conditions and the following disclaimer in 1204954a43b362b8c817cc5859513efad0c344f412Christopher Ferris * the documentation and/or other materials provided with the 1304954a43b362b8c817cc5859513efad0c344f412Christopher Ferris * distribution. 1404954a43b362b8c817cc5859513efad0c344f412Christopher Ferris * 1504954a43b362b8c817cc5859513efad0c344f412Christopher Ferris * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS 1604954a43b362b8c817cc5859513efad0c344f412Christopher Ferris * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT 1704954a43b362b8c817cc5859513efad0c344f412Christopher Ferris * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS 1804954a43b362b8c817cc5859513efad0c344f412Christopher Ferris * FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE 1904954a43b362b8c817cc5859513efad0c344f412Christopher Ferris * COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, 2004954a43b362b8c817cc5859513efad0c344f412Christopher Ferris * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, 2104954a43b362b8c817cc5859513efad0c344f412Christopher Ferris * BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS 2204954a43b362b8c817cc5859513efad0c344f412Christopher Ferris * OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED 2304954a43b362b8c817cc5859513efad0c344f412Christopher Ferris * AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, 2404954a43b362b8c817cc5859513efad0c344f412Christopher Ferris * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT 2504954a43b362b8c817cc5859513efad0c344f412Christopher Ferris * OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 2604954a43b362b8c817cc5859513efad0c344f412Christopher Ferris * SUCH DAMAGE. 2704954a43b362b8c817cc5859513efad0c344f412Christopher Ferris */ 2804954a43b362b8c817cc5859513efad0c344f412Christopher Ferris 2904954a43b362b8c817cc5859513efad0c344f412Christopher Ferris#include <machine/cpu-features.h> 30851e68a2402fa414544e66650e09dfdaac813e51Elliott Hughes#include <private/bionic_asm.h> 31851e68a2402fa414544e66650e09dfdaac813e51Elliott Hughes#include <private/libc_events.h> 3204954a43b362b8c817cc5859513efad0c344f412Christopher Ferris 3359a13c122ebc4191583b67c846a95d690dcda5cfChristopher Ferris /* 3459a13c122ebc4191583b67c846a95d690dcda5cfChristopher Ferris * Optimized memset() for ARM. 3504954a43b362b8c817cc5859513efad0c344f412Christopher Ferris * 3604954a43b362b8c817cc5859513efad0c344f412Christopher Ferris * memset() returns its first argument. 3759a13c122ebc4191583b67c846a95d690dcda5cfChristopher Ferris */ 3804954a43b362b8c817cc5859513efad0c344f412Christopher Ferris 39796cbe249bf6626b1de01664c38a7b06dcb2eb1fChristopher Ferris .fpu neon 40796cbe249bf6626b1de01664c38a7b06dcb2eb1fChristopher Ferris .syntax unified 4104954a43b362b8c817cc5859513efad0c344f412Christopher Ferris 4259a13c122ebc4191583b67c846a95d690dcda5cfChristopher FerrisENTRY(__memset_chk) 4359a13c122ebc4191583b67c846a95d690dcda5cfChristopher Ferris cmp r2, r3 445f45d583b0cfb4f7bed1447e8eed003a529cc69eChristopher Ferris bls .L_done 455f45d583b0cfb4f7bed1447e8eed003a529cc69eChristopher Ferris 465f45d583b0cfb4f7bed1447e8eed003a529cc69eChristopher Ferris // Preserve lr for backtrace. 475f45d583b0cfb4f7bed1447e8eed003a529cc69eChristopher Ferris push {lr} 485f45d583b0cfb4f7bed1447e8eed003a529cc69eChristopher Ferris .cfi_def_cfa_offset 4 495f45d583b0cfb4f7bed1447e8eed003a529cc69eChristopher Ferris .cfi_rel_offset lr, 0 5059a13c122ebc4191583b67c846a95d690dcda5cfChristopher Ferris 5159a13c122ebc4191583b67c846a95d690dcda5cfChristopher Ferris ldr r0, error_message 5259a13c122ebc4191583b67c846a95d690dcda5cfChristopher Ferris ldr r1, error_code 5359a13c122ebc4191583b67c846a95d690dcda5cfChristopher Ferris1: 5459a13c122ebc4191583b67c846a95d690dcda5cfChristopher Ferris add r0, pc 5559a13c122ebc4191583b67c846a95d690dcda5cfChristopher Ferris bl __fortify_chk_fail 5659a13c122ebc4191583b67c846a95d690dcda5cfChristopher Ferriserror_code: 5759a13c122ebc4191583b67c846a95d690dcda5cfChristopher Ferris .word BIONIC_EVENT_MEMSET_BUFFER_OVERFLOW 5859a13c122ebc4191583b67c846a95d690dcda5cfChristopher Ferriserror_message: 5959a13c122ebc4191583b67c846a95d690dcda5cfChristopher Ferris .word error_string-(1b+8) 6059a13c122ebc4191583b67c846a95d690dcda5cfChristopher FerrisEND(__memset_chk) 6159a13c122ebc4191583b67c846a95d690dcda5cfChristopher Ferris 6204954a43b362b8c817cc5859513efad0c344f412Christopher FerrisENTRY(bzero) 63796cbe249bf6626b1de01664c38a7b06dcb2eb1fChristopher Ferris mov r2, r1 64796cbe249bf6626b1de01664c38a7b06dcb2eb1fChristopher Ferris mov r1, #0 655f45d583b0cfb4f7bed1447e8eed003a529cc69eChristopher Ferris.L_done: 6604954a43b362b8c817cc5859513efad0c344f412Christopher Ferris // Fall through to memset... 6704954a43b362b8c817cc5859513efad0c344f412Christopher FerrisEND(bzero) 6804954a43b362b8c817cc5859513efad0c344f412Christopher Ferris 6904954a43b362b8c817cc5859513efad0c344f412Christopher FerrisENTRY(memset) 7004954a43b362b8c817cc5859513efad0c344f412Christopher Ferris stmfd sp!, {r0} 715f45d583b0cfb4f7bed1447e8eed003a529cc69eChristopher Ferris .cfi_def_cfa_offset 4 725f45d583b0cfb4f7bed1447e8eed003a529cc69eChristopher Ferris .cfi_rel_offset r0, 0 7304954a43b362b8c817cc5859513efad0c344f412Christopher Ferris 74796cbe249bf6626b1de01664c38a7b06dcb2eb1fChristopher Ferris // The new algorithm is slower for copies < 16 so use the old 75796cbe249bf6626b1de01664c38a7b06dcb2eb1fChristopher Ferris // neon code in that case. 7604954a43b362b8c817cc5859513efad0c344f412Christopher Ferris cmp r2, #16 775f45d583b0cfb4f7bed1447e8eed003a529cc69eChristopher Ferris blo .L_set_less_than_16_unknown_align 78796cbe249bf6626b1de01664c38a7b06dcb2eb1fChristopher Ferris 79796cbe249bf6626b1de01664c38a7b06dcb2eb1fChristopher Ferris // Use strd which requires an even and odd register so move the 80796cbe249bf6626b1de01664c38a7b06dcb2eb1fChristopher Ferris // values so that: 81796cbe249bf6626b1de01664c38a7b06dcb2eb1fChristopher Ferris // r0 and r1 contain the memset value 82796cbe249bf6626b1de01664c38a7b06dcb2eb1fChristopher Ferris // r2 is the number of bytes to set 83796cbe249bf6626b1de01664c38a7b06dcb2eb1fChristopher Ferris // r3 is the destination pointer 84796cbe249bf6626b1de01664c38a7b06dcb2eb1fChristopher Ferris mov r3, r0 85796cbe249bf6626b1de01664c38a7b06dcb2eb1fChristopher Ferris 86796cbe249bf6626b1de01664c38a7b06dcb2eb1fChristopher Ferris // Copy the byte value in every byte of r1. 87796cbe249bf6626b1de01664c38a7b06dcb2eb1fChristopher Ferris mov r1, r1, lsl #24 88796cbe249bf6626b1de01664c38a7b06dcb2eb1fChristopher Ferris orr r1, r1, r1, lsr #8 89796cbe249bf6626b1de01664c38a7b06dcb2eb1fChristopher Ferris orr r1, r1, r1, lsr #16 90796cbe249bf6626b1de01664c38a7b06dcb2eb1fChristopher Ferris 915f45d583b0cfb4f7bed1447e8eed003a529cc69eChristopher Ferris.L_check_alignment: 92796cbe249bf6626b1de01664c38a7b06dcb2eb1fChristopher Ferris // Align destination to a double word to avoid the strd crossing 93796cbe249bf6626b1de01664c38a7b06dcb2eb1fChristopher Ferris // a cache line boundary. 94796cbe249bf6626b1de01664c38a7b06dcb2eb1fChristopher Ferris ands ip, r3, #7 955f45d583b0cfb4f7bed1447e8eed003a529cc69eChristopher Ferris bne .L_do_double_word_align 96796cbe249bf6626b1de01664c38a7b06dcb2eb1fChristopher Ferris 975f45d583b0cfb4f7bed1447e8eed003a529cc69eChristopher Ferris.L_double_word_aligned: 98796cbe249bf6626b1de01664c38a7b06dcb2eb1fChristopher Ferris mov r0, r1 99796cbe249bf6626b1de01664c38a7b06dcb2eb1fChristopher Ferris 100796cbe249bf6626b1de01664c38a7b06dcb2eb1fChristopher Ferris subs r2, #64 1015f45d583b0cfb4f7bed1447e8eed003a529cc69eChristopher Ferris blo .L_set_less_than_64 102796cbe249bf6626b1de01664c38a7b06dcb2eb1fChristopher Ferris 103796cbe249bf6626b1de01664c38a7b06dcb2eb1fChristopher Ferris1: // Main loop sets 64 bytes at a time. 104796cbe249bf6626b1de01664c38a7b06dcb2eb1fChristopher Ferris .irp offset, #0, #8, #16, #24, #32, #40, #48, #56 105796cbe249bf6626b1de01664c38a7b06dcb2eb1fChristopher Ferris strd r0, r1, [r3, \offset] 106796cbe249bf6626b1de01664c38a7b06dcb2eb1fChristopher Ferris .endr 107796cbe249bf6626b1de01664c38a7b06dcb2eb1fChristopher Ferris 108796cbe249bf6626b1de01664c38a7b06dcb2eb1fChristopher Ferris add r3, #64 109796cbe249bf6626b1de01664c38a7b06dcb2eb1fChristopher Ferris subs r2, #64 110796cbe249bf6626b1de01664c38a7b06dcb2eb1fChristopher Ferris bge 1b 111796cbe249bf6626b1de01664c38a7b06dcb2eb1fChristopher Ferris 1125f45d583b0cfb4f7bed1447e8eed003a529cc69eChristopher Ferris.L_set_less_than_64: 113796cbe249bf6626b1de01664c38a7b06dcb2eb1fChristopher Ferris // Restore r2 to the count of bytes left to set. 114796cbe249bf6626b1de01664c38a7b06dcb2eb1fChristopher Ferris add r2, #64 115796cbe249bf6626b1de01664c38a7b06dcb2eb1fChristopher Ferris lsls ip, r2, #27 1165f45d583b0cfb4f7bed1447e8eed003a529cc69eChristopher Ferris bcc .L_set_less_than_32 117796cbe249bf6626b1de01664c38a7b06dcb2eb1fChristopher Ferris // Set 32 bytes. 118796cbe249bf6626b1de01664c38a7b06dcb2eb1fChristopher Ferris .irp offset, #0, #8, #16, #24 119796cbe249bf6626b1de01664c38a7b06dcb2eb1fChristopher Ferris strd r0, r1, [r3, \offset] 120796cbe249bf6626b1de01664c38a7b06dcb2eb1fChristopher Ferris .endr 121796cbe249bf6626b1de01664c38a7b06dcb2eb1fChristopher Ferris add r3, #32 122796cbe249bf6626b1de01664c38a7b06dcb2eb1fChristopher Ferris 1235f45d583b0cfb4f7bed1447e8eed003a529cc69eChristopher Ferris.L_set_less_than_32: 1245f45d583b0cfb4f7bed1447e8eed003a529cc69eChristopher Ferris bpl .L_set_less_than_16 125796cbe249bf6626b1de01664c38a7b06dcb2eb1fChristopher Ferris // Set 16 bytes. 126796cbe249bf6626b1de01664c38a7b06dcb2eb1fChristopher Ferris .irp offset, #0, #8 127796cbe249bf6626b1de01664c38a7b06dcb2eb1fChristopher Ferris strd r0, r1, [r3, \offset] 128796cbe249bf6626b1de01664c38a7b06dcb2eb1fChristopher Ferris .endr 129796cbe249bf6626b1de01664c38a7b06dcb2eb1fChristopher Ferris add r3, #16 130796cbe249bf6626b1de01664c38a7b06dcb2eb1fChristopher Ferris 1315f45d583b0cfb4f7bed1447e8eed003a529cc69eChristopher Ferris.L_set_less_than_16: 132796cbe249bf6626b1de01664c38a7b06dcb2eb1fChristopher Ferris // Less than 16 bytes to set. 133796cbe249bf6626b1de01664c38a7b06dcb2eb1fChristopher Ferris lsls ip, r2, #29 1345f45d583b0cfb4f7bed1447e8eed003a529cc69eChristopher Ferris bcc .L_set_less_than_8 135796cbe249bf6626b1de01664c38a7b06dcb2eb1fChristopher Ferris 136796cbe249bf6626b1de01664c38a7b06dcb2eb1fChristopher Ferris // Set 8 bytes. 137796cbe249bf6626b1de01664c38a7b06dcb2eb1fChristopher Ferris strd r0, r1, [r3], #8 138796cbe249bf6626b1de01664c38a7b06dcb2eb1fChristopher Ferris 1395f45d583b0cfb4f7bed1447e8eed003a529cc69eChristopher Ferris.L_set_less_than_8: 1405f45d583b0cfb4f7bed1447e8eed003a529cc69eChristopher Ferris bpl .L_set_less_than_4 141796cbe249bf6626b1de01664c38a7b06dcb2eb1fChristopher Ferris // Set 4 bytes 142796cbe249bf6626b1de01664c38a7b06dcb2eb1fChristopher Ferris str r1, [r3], #4 143796cbe249bf6626b1de01664c38a7b06dcb2eb1fChristopher Ferris 1445f45d583b0cfb4f7bed1447e8eed003a529cc69eChristopher Ferris.L_set_less_than_4: 145796cbe249bf6626b1de01664c38a7b06dcb2eb1fChristopher Ferris lsls ip, r2, #31 146796cbe249bf6626b1de01664c38a7b06dcb2eb1fChristopher Ferris it ne 147796cbe249bf6626b1de01664c38a7b06dcb2eb1fChristopher Ferris strbne r1, [r3], #1 148796cbe249bf6626b1de01664c38a7b06dcb2eb1fChristopher Ferris itt cs 149796cbe249bf6626b1de01664c38a7b06dcb2eb1fChristopher Ferris strbcs r1, [r3], #1 150796cbe249bf6626b1de01664c38a7b06dcb2eb1fChristopher Ferris strbcs r1, [r3] 151796cbe249bf6626b1de01664c38a7b06dcb2eb1fChristopher Ferris 152796cbe249bf6626b1de01664c38a7b06dcb2eb1fChristopher Ferris ldmfd sp!, {r0} 153796cbe249bf6626b1de01664c38a7b06dcb2eb1fChristopher Ferris bx lr 154796cbe249bf6626b1de01664c38a7b06dcb2eb1fChristopher Ferris 1555f45d583b0cfb4f7bed1447e8eed003a529cc69eChristopher Ferris.L_do_double_word_align: 156796cbe249bf6626b1de01664c38a7b06dcb2eb1fChristopher Ferris rsb ip, ip, #8 157796cbe249bf6626b1de01664c38a7b06dcb2eb1fChristopher Ferris sub r2, r2, ip 158796cbe249bf6626b1de01664c38a7b06dcb2eb1fChristopher Ferris movs r0, ip, lsl #31 159796cbe249bf6626b1de01664c38a7b06dcb2eb1fChristopher Ferris it mi 160796cbe249bf6626b1de01664c38a7b06dcb2eb1fChristopher Ferris strbmi r1, [r3], #1 161796cbe249bf6626b1de01664c38a7b06dcb2eb1fChristopher Ferris itt cs 162796cbe249bf6626b1de01664c38a7b06dcb2eb1fChristopher Ferris strbcs r1, [r3], #1 163796cbe249bf6626b1de01664c38a7b06dcb2eb1fChristopher Ferris strbcs r1, [r3], #1 164796cbe249bf6626b1de01664c38a7b06dcb2eb1fChristopher Ferris 165796cbe249bf6626b1de01664c38a7b06dcb2eb1fChristopher Ferris // Dst is at least word aligned by this point. 166796cbe249bf6626b1de01664c38a7b06dcb2eb1fChristopher Ferris cmp ip, #4 1675f45d583b0cfb4f7bed1447e8eed003a529cc69eChristopher Ferris blo .L_double_word_aligned 168796cbe249bf6626b1de01664c38a7b06dcb2eb1fChristopher Ferris str r1, [r3], #4 1695f45d583b0cfb4f7bed1447e8eed003a529cc69eChristopher Ferris b .L_double_word_aligned 170796cbe249bf6626b1de01664c38a7b06dcb2eb1fChristopher Ferris 1715f45d583b0cfb4f7bed1447e8eed003a529cc69eChristopher Ferris.L_set_less_than_16_unknown_align: 172796cbe249bf6626b1de01664c38a7b06dcb2eb1fChristopher Ferris // Set up to 15 bytes. 173796cbe249bf6626b1de01664c38a7b06dcb2eb1fChristopher Ferris vdup.8 d0, r1 17404954a43b362b8c817cc5859513efad0c344f412Christopher Ferris movs ip, r2, lsl #29 17504954a43b362b8c817cc5859513efad0c344f412Christopher Ferris bcc 1f 17604954a43b362b8c817cc5859513efad0c344f412Christopher Ferris vst1.8 {d0}, [r0]! 17704954a43b362b8c817cc5859513efad0c344f412Christopher Ferris1: bge 2f 17804954a43b362b8c817cc5859513efad0c344f412Christopher Ferris vst1.32 {d0[0]}, [r0]! 17904954a43b362b8c817cc5859513efad0c344f412Christopher Ferris2: movs ip, r2, lsl #31 180796cbe249bf6626b1de01664c38a7b06dcb2eb1fChristopher Ferris it mi 181796cbe249bf6626b1de01664c38a7b06dcb2eb1fChristopher Ferris strbmi r1, [r0], #1 182796cbe249bf6626b1de01664c38a7b06dcb2eb1fChristopher Ferris itt cs 183796cbe249bf6626b1de01664c38a7b06dcb2eb1fChristopher Ferris strbcs r1, [r0], #1 184796cbe249bf6626b1de01664c38a7b06dcb2eb1fChristopher Ferris strbcs r1, [r0], #1 18504954a43b362b8c817cc5859513efad0c344f412Christopher Ferris ldmfd sp!, {r0} 18604954a43b362b8c817cc5859513efad0c344f412Christopher Ferris bx lr 18704954a43b362b8c817cc5859513efad0c344f412Christopher FerrisEND(memset) 18859a13c122ebc4191583b67c846a95d690dcda5cfChristopher Ferris 18959a13c122ebc4191583b67c846a95d690dcda5cfChristopher Ferris .data 19059a13c122ebc4191583b67c846a95d690dcda5cfChristopher Ferriserror_string: 19168b67113a44311b3568027af5893e316f63ec556Elliott Hughes .string "memset: prevented write past end of buffer" 192