17c83a1ed81a15f3e75836c1ac7d500a952f02e10Christopher Ferris/* 27c83a1ed81a15f3e75836c1ac7d500a952f02e10Christopher Ferris * Copyright (C) 2008 The Android Open Source Project 37c83a1ed81a15f3e75836c1ac7d500a952f02e10Christopher Ferris * All rights reserved. 47c83a1ed81a15f3e75836c1ac7d500a952f02e10Christopher Ferris * 57c83a1ed81a15f3e75836c1ac7d500a952f02e10Christopher Ferris * Redistribution and use in source and binary forms, with or without 67c83a1ed81a15f3e75836c1ac7d500a952f02e10Christopher Ferris * modification, are permitted provided that the following conditions 77c83a1ed81a15f3e75836c1ac7d500a952f02e10Christopher Ferris * are met: 87c83a1ed81a15f3e75836c1ac7d500a952f02e10Christopher Ferris * * Redistributions of source code must retain the above copyright 97c83a1ed81a15f3e75836c1ac7d500a952f02e10Christopher Ferris * notice, this list of conditions and the following disclaimer. 107c83a1ed81a15f3e75836c1ac7d500a952f02e10Christopher Ferris * * Redistributions in binary form must reproduce the above copyright 117c83a1ed81a15f3e75836c1ac7d500a952f02e10Christopher Ferris * notice, this list of conditions and the following disclaimer in 127c83a1ed81a15f3e75836c1ac7d500a952f02e10Christopher Ferris * the documentation and/or other materials provided with the 137c83a1ed81a15f3e75836c1ac7d500a952f02e10Christopher Ferris * distribution. 147c83a1ed81a15f3e75836c1ac7d500a952f02e10Christopher Ferris * 157c83a1ed81a15f3e75836c1ac7d500a952f02e10Christopher Ferris * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS 167c83a1ed81a15f3e75836c1ac7d500a952f02e10Christopher Ferris * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT 177c83a1ed81a15f3e75836c1ac7d500a952f02e10Christopher Ferris * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS 187c83a1ed81a15f3e75836c1ac7d500a952f02e10Christopher Ferris * FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE 197c83a1ed81a15f3e75836c1ac7d500a952f02e10Christopher Ferris * COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, 207c83a1ed81a15f3e75836c1ac7d500a952f02e10Christopher Ferris * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, 217c83a1ed81a15f3e75836c1ac7d500a952f02e10Christopher Ferris * BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS 227c83a1ed81a15f3e75836c1ac7d500a952f02e10Christopher Ferris * OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED 237c83a1ed81a15f3e75836c1ac7d500a952f02e10Christopher Ferris * AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, 247c83a1ed81a15f3e75836c1ac7d500a952f02e10Christopher Ferris * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT 257c83a1ed81a15f3e75836c1ac7d500a952f02e10Christopher Ferris * OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 267c83a1ed81a15f3e75836c1ac7d500a952f02e10Christopher Ferris * SUCH DAMAGE. 277c83a1ed81a15f3e75836c1ac7d500a952f02e10Christopher Ferris */ 287c83a1ed81a15f3e75836c1ac7d500a952f02e10Christopher Ferris 29851e68a2402fa414544e66650e09dfdaac813e51Elliott Hughes#include <private/bionic_asm.h> 30851e68a2402fa414544e66650e09dfdaac813e51Elliott Hughes#include <private/libc_events.h> 317c83a1ed81a15f3e75836c1ac7d500a952f02e10Christopher Ferris 327c83a1ed81a15f3e75836c1ac7d500a952f02e10Christopher Ferris/* 337c83a1ed81a15f3e75836c1ac7d500a952f02e10Christopher Ferris * This code assumes it is running on a processor that supports all arm v7 347c83a1ed81a15f3e75836c1ac7d500a952f02e10Christopher Ferris * instructions and that supports neon instructions. 357c83a1ed81a15f3e75836c1ac7d500a952f02e10Christopher Ferris */ 367c83a1ed81a15f3e75836c1ac7d500a952f02e10Christopher Ferris 377c83a1ed81a15f3e75836c1ac7d500a952f02e10Christopher Ferris .fpu neon 387c83a1ed81a15f3e75836c1ac7d500a952f02e10Christopher Ferris 3959a13c122ebc4191583b67c846a95d690dcda5cfChristopher FerrisENTRY(__memset_chk) 4059a13c122ebc4191583b67c846a95d690dcda5cfChristopher Ferris cmp r2, r3 415f45d583b0cfb4f7bed1447e8eed003a529cc69eChristopher Ferris bls .L_done 425f45d583b0cfb4f7bed1447e8eed003a529cc69eChristopher Ferris 435f45d583b0cfb4f7bed1447e8eed003a529cc69eChristopher Ferris // Preserve lr for backtrace. 445f45d583b0cfb4f7bed1447e8eed003a529cc69eChristopher Ferris push {lr} 455f45d583b0cfb4f7bed1447e8eed003a529cc69eChristopher Ferris .cfi_def_cfa_offset 4 465f45d583b0cfb4f7bed1447e8eed003a529cc69eChristopher Ferris .cfi_rel_offset lr, 0 4759a13c122ebc4191583b67c846a95d690dcda5cfChristopher Ferris 4859a13c122ebc4191583b67c846a95d690dcda5cfChristopher Ferris ldr r0, error_message 4959a13c122ebc4191583b67c846a95d690dcda5cfChristopher Ferris ldr r1, error_code 5059a13c122ebc4191583b67c846a95d690dcda5cfChristopher Ferris1: 5159a13c122ebc4191583b67c846a95d690dcda5cfChristopher Ferris add r0, pc 5259a13c122ebc4191583b67c846a95d690dcda5cfChristopher Ferris bl __fortify_chk_fail 5359a13c122ebc4191583b67c846a95d690dcda5cfChristopher Ferriserror_code: 5459a13c122ebc4191583b67c846a95d690dcda5cfChristopher Ferris .word BIONIC_EVENT_MEMSET_BUFFER_OVERFLOW 5559a13c122ebc4191583b67c846a95d690dcda5cfChristopher Ferriserror_message: 5659a13c122ebc4191583b67c846a95d690dcda5cfChristopher Ferris .word error_string-(1b+8) 5759a13c122ebc4191583b67c846a95d690dcda5cfChristopher FerrisEND(__memset_chk) 5859a13c122ebc4191583b67c846a95d690dcda5cfChristopher Ferris 597c83a1ed81a15f3e75836c1ac7d500a952f02e10Christopher FerrisENTRY(bzero) 607c83a1ed81a15f3e75836c1ac7d500a952f02e10Christopher Ferris mov r2, r1 617c83a1ed81a15f3e75836c1ac7d500a952f02e10Christopher Ferris mov r1, #0 6259a13c122ebc4191583b67c846a95d690dcda5cfChristopher Ferris 635f45d583b0cfb4f7bed1447e8eed003a529cc69eChristopher Ferris.L_done: 6459a13c122ebc4191583b67c846a95d690dcda5cfChristopher Ferris // Fall through to memset... 657c83a1ed81a15f3e75836c1ac7d500a952f02e10Christopher FerrisEND(bzero) 667c83a1ed81a15f3e75836c1ac7d500a952f02e10Christopher Ferris 677c83a1ed81a15f3e75836c1ac7d500a952f02e10Christopher Ferris/* memset() returns its first argument. */ 687c83a1ed81a15f3e75836c1ac7d500a952f02e10Christopher FerrisENTRY(memset) 69507cfe2e10a6c4ad61b9638820ba10bfe881a18cChristopher Ferris // The neon memset only wins for less than 132. 707c83a1ed81a15f3e75836c1ac7d500a952f02e10Christopher Ferris cmp r2, #132 71a57c9c084bc686a35f4f494ce23cf2a9bb3d5d00Christopher Ferris bhi __memset_large_copy 727c83a1ed81a15f3e75836c1ac7d500a952f02e10Christopher Ferris 737c83a1ed81a15f3e75836c1ac7d500a952f02e10Christopher Ferris stmfd sp!, {r0} 745f45d583b0cfb4f7bed1447e8eed003a529cc69eChristopher Ferris .cfi_def_cfa_offset 4 755f45d583b0cfb4f7bed1447e8eed003a529cc69eChristopher Ferris .cfi_rel_offset r0, 0 767c83a1ed81a15f3e75836c1ac7d500a952f02e10Christopher Ferris 777c83a1ed81a15f3e75836c1ac7d500a952f02e10Christopher Ferris vdup.8 q0, r1 787c83a1ed81a15f3e75836c1ac7d500a952f02e10Christopher Ferris 797c83a1ed81a15f3e75836c1ac7d500a952f02e10Christopher Ferris /* make sure we have at least 32 bytes to write */ 807c83a1ed81a15f3e75836c1ac7d500a952f02e10Christopher Ferris subs r2, r2, #32 817c83a1ed81a15f3e75836c1ac7d500a952f02e10Christopher Ferris blo 2f 827c83a1ed81a15f3e75836c1ac7d500a952f02e10Christopher Ferris vmov q1, q0 837c83a1ed81a15f3e75836c1ac7d500a952f02e10Christopher Ferris 847c83a1ed81a15f3e75836c1ac7d500a952f02e10Christopher Ferris1: /* The main loop writes 32 bytes at a time */ 857c83a1ed81a15f3e75836c1ac7d500a952f02e10Christopher Ferris subs r2, r2, #32 867c83a1ed81a15f3e75836c1ac7d500a952f02e10Christopher Ferris vst1.8 {d0 - d3}, [r0]! 877c83a1ed81a15f3e75836c1ac7d500a952f02e10Christopher Ferris bhs 1b 887c83a1ed81a15f3e75836c1ac7d500a952f02e10Christopher Ferris 897c83a1ed81a15f3e75836c1ac7d500a952f02e10Christopher Ferris2: /* less than 32 left */ 907c83a1ed81a15f3e75836c1ac7d500a952f02e10Christopher Ferris add r2, r2, #32 917c83a1ed81a15f3e75836c1ac7d500a952f02e10Christopher Ferris tst r2, #0x10 927c83a1ed81a15f3e75836c1ac7d500a952f02e10Christopher Ferris beq 3f 937c83a1ed81a15f3e75836c1ac7d500a952f02e10Christopher Ferris 947c83a1ed81a15f3e75836c1ac7d500a952f02e10Christopher Ferris // writes 16 bytes, 128-bits aligned 957c83a1ed81a15f3e75836c1ac7d500a952f02e10Christopher Ferris vst1.8 {d0, d1}, [r0]! 967c83a1ed81a15f3e75836c1ac7d500a952f02e10Christopher Ferris3: /* write up to 15-bytes (count in r2) */ 977c83a1ed81a15f3e75836c1ac7d500a952f02e10Christopher Ferris movs ip, r2, lsl #29 987c83a1ed81a15f3e75836c1ac7d500a952f02e10Christopher Ferris bcc 1f 997c83a1ed81a15f3e75836c1ac7d500a952f02e10Christopher Ferris vst1.8 {d0}, [r0]! 1007c83a1ed81a15f3e75836c1ac7d500a952f02e10Christopher Ferris1: bge 2f 1017c83a1ed81a15f3e75836c1ac7d500a952f02e10Christopher Ferris vst1.32 {d0[0]}, [r0]! 1027c83a1ed81a15f3e75836c1ac7d500a952f02e10Christopher Ferris2: movs ip, r2, lsl #31 1037c83a1ed81a15f3e75836c1ac7d500a952f02e10Christopher Ferris strmib r1, [r0], #1 1047c83a1ed81a15f3e75836c1ac7d500a952f02e10Christopher Ferris strcsb r1, [r0], #1 1057c83a1ed81a15f3e75836c1ac7d500a952f02e10Christopher Ferris strcsb r1, [r0], #1 1067c83a1ed81a15f3e75836c1ac7d500a952f02e10Christopher Ferris ldmfd sp!, {r0} 1077c83a1ed81a15f3e75836c1ac7d500a952f02e10Christopher Ferris bx lr 108a57c9c084bc686a35f4f494ce23cf2a9bb3d5d00Christopher FerrisEND(memset) 109a57c9c084bc686a35f4f494ce23cf2a9bb3d5d00Christopher Ferris 11032bbf8a63bb43a540cc0f1dd5037736d10b70e0bNick KralevichENTRY_PRIVATE(__memset_large_copy) 1117c83a1ed81a15f3e75836c1ac7d500a952f02e10Christopher Ferris /* compute the offset to align the destination 1127c83a1ed81a15f3e75836c1ac7d500a952f02e10Christopher Ferris * offset = (4-(src&3))&3 = -src & 3 1137c83a1ed81a15f3e75836c1ac7d500a952f02e10Christopher Ferris */ 1147c83a1ed81a15f3e75836c1ac7d500a952f02e10Christopher Ferris stmfd sp!, {r0, r4-r7, lr} 1155f45d583b0cfb4f7bed1447e8eed003a529cc69eChristopher Ferris .cfi_def_cfa_offset 24 1165f45d583b0cfb4f7bed1447e8eed003a529cc69eChristopher Ferris .cfi_rel_offset r0, 0 1175f45d583b0cfb4f7bed1447e8eed003a529cc69eChristopher Ferris .cfi_rel_offset r4, 4 1185f45d583b0cfb4f7bed1447e8eed003a529cc69eChristopher Ferris .cfi_rel_offset r5, 8 1195f45d583b0cfb4f7bed1447e8eed003a529cc69eChristopher Ferris .cfi_rel_offset r6, 12 1205f45d583b0cfb4f7bed1447e8eed003a529cc69eChristopher Ferris .cfi_rel_offset r7, 16 1215f45d583b0cfb4f7bed1447e8eed003a529cc69eChristopher Ferris .cfi_rel_offset lr, 20 1225f45d583b0cfb4f7bed1447e8eed003a529cc69eChristopher Ferris 1237c83a1ed81a15f3e75836c1ac7d500a952f02e10Christopher Ferris rsb r3, r0, #0 1247c83a1ed81a15f3e75836c1ac7d500a952f02e10Christopher Ferris ands r3, r3, #3 1257c83a1ed81a15f3e75836c1ac7d500a952f02e10Christopher Ferris cmp r3, r2 1267c83a1ed81a15f3e75836c1ac7d500a952f02e10Christopher Ferris movhi r3, r2 1277c83a1ed81a15f3e75836c1ac7d500a952f02e10Christopher Ferris 1287c83a1ed81a15f3e75836c1ac7d500a952f02e10Christopher Ferris /* splat r1 */ 1297c83a1ed81a15f3e75836c1ac7d500a952f02e10Christopher Ferris mov r1, r1, lsl #24 1307c83a1ed81a15f3e75836c1ac7d500a952f02e10Christopher Ferris orr r1, r1, r1, lsr #8 1317c83a1ed81a15f3e75836c1ac7d500a952f02e10Christopher Ferris orr r1, r1, r1, lsr #16 1327c83a1ed81a15f3e75836c1ac7d500a952f02e10Christopher Ferris 1337c83a1ed81a15f3e75836c1ac7d500a952f02e10Christopher Ferris movs r12, r3, lsl #31 1347c83a1ed81a15f3e75836c1ac7d500a952f02e10Christopher Ferris strcsb r1, [r0], #1 /* can't use strh (alignment unknown) */ 1357c83a1ed81a15f3e75836c1ac7d500a952f02e10Christopher Ferris strcsb r1, [r0], #1 1367c83a1ed81a15f3e75836c1ac7d500a952f02e10Christopher Ferris strmib r1, [r0], #1 1377c83a1ed81a15f3e75836c1ac7d500a952f02e10Christopher Ferris subs r2, r2, r3 1387c83a1ed81a15f3e75836c1ac7d500a952f02e10Christopher Ferris ldmlsfd sp!, {r0, r4-r7, lr} /* return */ 1397c83a1ed81a15f3e75836c1ac7d500a952f02e10Christopher Ferris bxls lr 1407c83a1ed81a15f3e75836c1ac7d500a952f02e10Christopher Ferris 1417c83a1ed81a15f3e75836c1ac7d500a952f02e10Christopher Ferris /* align the destination to a cache-line */ 1427c83a1ed81a15f3e75836c1ac7d500a952f02e10Christopher Ferris mov r12, r1 1437c83a1ed81a15f3e75836c1ac7d500a952f02e10Christopher Ferris mov lr, r1 1447c83a1ed81a15f3e75836c1ac7d500a952f02e10Christopher Ferris mov r4, r1 1457c83a1ed81a15f3e75836c1ac7d500a952f02e10Christopher Ferris mov r5, r1 1467c83a1ed81a15f3e75836c1ac7d500a952f02e10Christopher Ferris mov r6, r1 1477c83a1ed81a15f3e75836c1ac7d500a952f02e10Christopher Ferris mov r7, r1 1487c83a1ed81a15f3e75836c1ac7d500a952f02e10Christopher Ferris 1497c83a1ed81a15f3e75836c1ac7d500a952f02e10Christopher Ferris rsb r3, r0, #0 1507c83a1ed81a15f3e75836c1ac7d500a952f02e10Christopher Ferris ands r3, r3, #0x1C 1517c83a1ed81a15f3e75836c1ac7d500a952f02e10Christopher Ferris beq 3f 1527c83a1ed81a15f3e75836c1ac7d500a952f02e10Christopher Ferris cmp r3, r2 1537c83a1ed81a15f3e75836c1ac7d500a952f02e10Christopher Ferris andhi r3, r2, #0x1C 1547c83a1ed81a15f3e75836c1ac7d500a952f02e10Christopher Ferris sub r2, r2, r3 1557c83a1ed81a15f3e75836c1ac7d500a952f02e10Christopher Ferris 1567c83a1ed81a15f3e75836c1ac7d500a952f02e10Christopher Ferris /* conditionally writes 0 to 7 words (length in r3) */ 1577c83a1ed81a15f3e75836c1ac7d500a952f02e10Christopher Ferris movs r3, r3, lsl #28 1587c83a1ed81a15f3e75836c1ac7d500a952f02e10Christopher Ferris stmcsia r0!, {r1, lr} 1597c83a1ed81a15f3e75836c1ac7d500a952f02e10Christopher Ferris stmcsia r0!, {r1, lr} 1607c83a1ed81a15f3e75836c1ac7d500a952f02e10Christopher Ferris stmmiia r0!, {r1, lr} 1617c83a1ed81a15f3e75836c1ac7d500a952f02e10Christopher Ferris movs r3, r3, lsl #2 1627c83a1ed81a15f3e75836c1ac7d500a952f02e10Christopher Ferris strcs r1, [r0], #4 1637c83a1ed81a15f3e75836c1ac7d500a952f02e10Christopher Ferris 1647c83a1ed81a15f3e75836c1ac7d500a952f02e10Christopher Ferris3: 1657c83a1ed81a15f3e75836c1ac7d500a952f02e10Christopher Ferris subs r2, r2, #32 1667c83a1ed81a15f3e75836c1ac7d500a952f02e10Christopher Ferris mov r3, r1 1677c83a1ed81a15f3e75836c1ac7d500a952f02e10Christopher Ferris bmi 2f 1687c83a1ed81a15f3e75836c1ac7d500a952f02e10Christopher Ferris1: subs r2, r2, #32 1697c83a1ed81a15f3e75836c1ac7d500a952f02e10Christopher Ferris stmia r0!, {r1,r3,r4,r5,r6,r7,r12,lr} 1707c83a1ed81a15f3e75836c1ac7d500a952f02e10Christopher Ferris bhs 1b 1717c83a1ed81a15f3e75836c1ac7d500a952f02e10Christopher Ferris2: add r2, r2, #32 1727c83a1ed81a15f3e75836c1ac7d500a952f02e10Christopher Ferris 1737c83a1ed81a15f3e75836c1ac7d500a952f02e10Christopher Ferris /* conditionally stores 0 to 31 bytes */ 1747c83a1ed81a15f3e75836c1ac7d500a952f02e10Christopher Ferris movs r2, r2, lsl #28 1757c83a1ed81a15f3e75836c1ac7d500a952f02e10Christopher Ferris stmcsia r0!, {r1,r3,r12,lr} 1767c83a1ed81a15f3e75836c1ac7d500a952f02e10Christopher Ferris stmmiia r0!, {r1, lr} 1777c83a1ed81a15f3e75836c1ac7d500a952f02e10Christopher Ferris movs r2, r2, lsl #2 1787c83a1ed81a15f3e75836c1ac7d500a952f02e10Christopher Ferris strcs r1, [r0], #4 1797c83a1ed81a15f3e75836c1ac7d500a952f02e10Christopher Ferris strmih r1, [r0], #2 1807c83a1ed81a15f3e75836c1ac7d500a952f02e10Christopher Ferris movs r2, r2, lsl #2 1817c83a1ed81a15f3e75836c1ac7d500a952f02e10Christopher Ferris strcsb r1, [r0] 1827c83a1ed81a15f3e75836c1ac7d500a952f02e10Christopher Ferris ldmfd sp!, {r0, r4-r7, lr} 1837c83a1ed81a15f3e75836c1ac7d500a952f02e10Christopher Ferris bx lr 184a57c9c084bc686a35f4f494ce23cf2a9bb3d5d00Christopher FerrisEND(__memset_large_copy) 18559a13c122ebc4191583b67c846a95d690dcda5cfChristopher Ferris 18659a13c122ebc4191583b67c846a95d690dcda5cfChristopher Ferris .data 18759a13c122ebc4191583b67c846a95d690dcda5cfChristopher Ferriserror_string: 18868b67113a44311b3568027af5893e316f63ec556Elliott Hughes .string "memset: prevented write past end of buffer" 189