17e4fa560999d07064d219a16ebb50d3691dd1b63Bernhard Rosenkraenzer/* Copyright (c) 2012, Linaro Limited 27e4fa560999d07064d219a16ebb50d3691dd1b63Bernhard Rosenkraenzer All rights reserved. 37e4fa560999d07064d219a16ebb50d3691dd1b63Bernhard Rosenkraenzer 47e4fa560999d07064d219a16ebb50d3691dd1b63Bernhard Rosenkraenzer Redistribution and use in source and binary forms, with or without 57e4fa560999d07064d219a16ebb50d3691dd1b63Bernhard Rosenkraenzer modification, are permitted provided that the following conditions are met: 67e4fa560999d07064d219a16ebb50d3691dd1b63Bernhard Rosenkraenzer * Redistributions of source code must retain the above copyright 77e4fa560999d07064d219a16ebb50d3691dd1b63Bernhard Rosenkraenzer notice, this list of conditions and the following disclaimer. 87e4fa560999d07064d219a16ebb50d3691dd1b63Bernhard Rosenkraenzer * Redistributions in binary form must reproduce the above copyright 97e4fa560999d07064d219a16ebb50d3691dd1b63Bernhard Rosenkraenzer notice, this list of conditions and the following disclaimer in the 107e4fa560999d07064d219a16ebb50d3691dd1b63Bernhard Rosenkraenzer documentation and/or other materials provided with the distribution. 117e4fa560999d07064d219a16ebb50d3691dd1b63Bernhard Rosenkraenzer * Neither the name of the Linaro nor the 127e4fa560999d07064d219a16ebb50d3691dd1b63Bernhard Rosenkraenzer names of its contributors may be used to endorse or promote products 137e4fa560999d07064d219a16ebb50d3691dd1b63Bernhard Rosenkraenzer derived from this software without specific prior written permission. 147e4fa560999d07064d219a16ebb50d3691dd1b63Bernhard Rosenkraenzer 157e4fa560999d07064d219a16ebb50d3691dd1b63Bernhard Rosenkraenzer THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS 167e4fa560999d07064d219a16ebb50d3691dd1b63Bernhard Rosenkraenzer "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT 177e4fa560999d07064d219a16ebb50d3691dd1b63Bernhard Rosenkraenzer LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR 187e4fa560999d07064d219a16ebb50d3691dd1b63Bernhard Rosenkraenzer A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT 197e4fa560999d07064d219a16ebb50d3691dd1b63Bernhard Rosenkraenzer HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, 207e4fa560999d07064d219a16ebb50d3691dd1b63Bernhard Rosenkraenzer SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT 217e4fa560999d07064d219a16ebb50d3691dd1b63Bernhard Rosenkraenzer LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 227e4fa560999d07064d219a16ebb50d3691dd1b63Bernhard Rosenkraenzer DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 237e4fa560999d07064d219a16ebb50d3691dd1b63Bernhard Rosenkraenzer THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 247e4fa560999d07064d219a16ebb50d3691dd1b63Bernhard Rosenkraenzer (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 257e4fa560999d07064d219a16ebb50d3691dd1b63Bernhard Rosenkraenzer OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 267e4fa560999d07064d219a16ebb50d3691dd1b63Bernhard Rosenkraenzer*/ 277e4fa560999d07064d219a16ebb50d3691dd1b63Bernhard Rosenkraenzer 287e4fa560999d07064d219a16ebb50d3691dd1b63Bernhard Rosenkraenzer/* Assumptions: 297e4fa560999d07064d219a16ebb50d3691dd1b63Bernhard Rosenkraenzer * 307e4fa560999d07064d219a16ebb50d3691dd1b63Bernhard Rosenkraenzer * ARMv8-a, AArch64 317e4fa560999d07064d219a16ebb50d3691dd1b63Bernhard Rosenkraenzer * Unaligned accesses 327e4fa560999d07064d219a16ebb50d3691dd1b63Bernhard Rosenkraenzer * 337e4fa560999d07064d219a16ebb50d3691dd1b63Bernhard Rosenkraenzer */ 347e4fa560999d07064d219a16ebb50d3691dd1b63Bernhard Rosenkraenzer 357e4fa560999d07064d219a16ebb50d3691dd1b63Bernhard Rosenkraenzer#include <private/bionic_asm.h> 367e4fa560999d07064d219a16ebb50d3691dd1b63Bernhard Rosenkraenzer 377e4fa560999d07064d219a16ebb50d3691dd1b63Bernhard Rosenkraenzer/* By default we assume that the DC instruction can be used to zero 387e4fa560999d07064d219a16ebb50d3691dd1b63Bernhard Rosenkraenzer data blocks more efficiently. In some circumstances this might be 397e4fa560999d07064d219a16ebb50d3691dd1b63Bernhard Rosenkraenzer unsafe, for example in an asymmetric multiprocessor environment with 407e4fa560999d07064d219a16ebb50d3691dd1b63Bernhard Rosenkraenzer different DC clear lengths (neither the upper nor lower lengths are 4162d92e1610080cad16342f87a7fb46d8388ee4bbBernhard Rosenkraenzer safe to use). 427e4fa560999d07064d219a16ebb50d3691dd1b63Bernhard Rosenkraenzer 437e4fa560999d07064d219a16ebb50d3691dd1b63Bernhard Rosenkraenzer If code may be run in a virtualized environment, then define 447e4fa560999d07064d219a16ebb50d3691dd1b63Bernhard Rosenkraenzer MAYBE_VIRT. This will cause the code to cache the system register 457e4fa560999d07064d219a16ebb50d3691dd1b63Bernhard Rosenkraenzer values rather than re-reading them each call. */ 467e4fa560999d07064d219a16ebb50d3691dd1b63Bernhard Rosenkraenzer 477e4fa560999d07064d219a16ebb50d3691dd1b63Bernhard Rosenkraenzer#define dstin x0 4862d92e1610080cad16342f87a7fb46d8388ee4bbBernhard Rosenkraenzer#ifdef BZERO 4962d92e1610080cad16342f87a7fb46d8388ee4bbBernhard Rosenkraenzer#define count x1 5062d92e1610080cad16342f87a7fb46d8388ee4bbBernhard Rosenkraenzer#else 517e4fa560999d07064d219a16ebb50d3691dd1b63Bernhard Rosenkraenzer#define count x2 5262d92e1610080cad16342f87a7fb46d8388ee4bbBernhard Rosenkraenzer#endif 5362d92e1610080cad16342f87a7fb46d8388ee4bbBernhard Rosenkraenzer#define val w1 547e4fa560999d07064d219a16ebb50d3691dd1b63Bernhard Rosenkraenzer#define tmp1 x3 557e4fa560999d07064d219a16ebb50d3691dd1b63Bernhard Rosenkraenzer#define tmp1w w3 567e4fa560999d07064d219a16ebb50d3691dd1b63Bernhard Rosenkraenzer#define tmp2 x4 577e4fa560999d07064d219a16ebb50d3691dd1b63Bernhard Rosenkraenzer#define tmp2w w4 587e4fa560999d07064d219a16ebb50d3691dd1b63Bernhard Rosenkraenzer#define zva_len_x x5 597e4fa560999d07064d219a16ebb50d3691dd1b63Bernhard Rosenkraenzer#define zva_len w5 607e4fa560999d07064d219a16ebb50d3691dd1b63Bernhard Rosenkraenzer#define zva_bits_x x6 617e4fa560999d07064d219a16ebb50d3691dd1b63Bernhard Rosenkraenzer 627e4fa560999d07064d219a16ebb50d3691dd1b63Bernhard Rosenkraenzer#define A_l x7 637e4fa560999d07064d219a16ebb50d3691dd1b63Bernhard Rosenkraenzer#define A_lw w7 647e4fa560999d07064d219a16ebb50d3691dd1b63Bernhard Rosenkraenzer#define dst x8 657e4fa560999d07064d219a16ebb50d3691dd1b63Bernhard Rosenkraenzer#define tmp3w w9 667e4fa560999d07064d219a16ebb50d3691dd1b63Bernhard Rosenkraenzer 6762d92e1610080cad16342f87a7fb46d8388ee4bbBernhard Rosenkraenzer#ifdef BZERO 6862d92e1610080cad16342f87a7fb46d8388ee4bbBernhard RosenkraenzerENTRY(bzero) 6962d92e1610080cad16342f87a7fb46d8388ee4bbBernhard Rosenkraenzer#else 707e4fa560999d07064d219a16ebb50d3691dd1b63Bernhard RosenkraenzerENTRY(memset) 7162d92e1610080cad16342f87a7fb46d8388ee4bbBernhard Rosenkraenzer#endif 727e4fa560999d07064d219a16ebb50d3691dd1b63Bernhard Rosenkraenzer 737e4fa560999d07064d219a16ebb50d3691dd1b63Bernhard Rosenkraenzer mov dst, dstin /* Preserve return value. */ 7462d92e1610080cad16342f87a7fb46d8388ee4bbBernhard Rosenkraenzer#ifdef BZERO 7562d92e1610080cad16342f87a7fb46d8388ee4bbBernhard Rosenkraenzer b .Lzero_mem 7662d92e1610080cad16342f87a7fb46d8388ee4bbBernhard Rosenkraenzer#endif 777e4fa560999d07064d219a16ebb50d3691dd1b63Bernhard Rosenkraenzer ands A_lw, val, #255 787e4fa560999d07064d219a16ebb50d3691dd1b63Bernhard Rosenkraenzer b.eq .Lzero_mem 797e4fa560999d07064d219a16ebb50d3691dd1b63Bernhard Rosenkraenzer orr A_lw, A_lw, A_lw, lsl #8 807e4fa560999d07064d219a16ebb50d3691dd1b63Bernhard Rosenkraenzer orr A_lw, A_lw, A_lw, lsl #16 817e4fa560999d07064d219a16ebb50d3691dd1b63Bernhard Rosenkraenzer orr A_l, A_l, A_l, lsl #32 827e4fa560999d07064d219a16ebb50d3691dd1b63Bernhard Rosenkraenzer.Ltail_maybe_long: 837e4fa560999d07064d219a16ebb50d3691dd1b63Bernhard Rosenkraenzer cmp count, #64 847e4fa560999d07064d219a16ebb50d3691dd1b63Bernhard Rosenkraenzer b.ge .Lnot_short 857e4fa560999d07064d219a16ebb50d3691dd1b63Bernhard Rosenkraenzer.Ltail_maybe_tiny: 867e4fa560999d07064d219a16ebb50d3691dd1b63Bernhard Rosenkraenzer cmp count, #15 877e4fa560999d07064d219a16ebb50d3691dd1b63Bernhard Rosenkraenzer b.le .Ltail15tiny 887e4fa560999d07064d219a16ebb50d3691dd1b63Bernhard Rosenkraenzer.Ltail63: 897e4fa560999d07064d219a16ebb50d3691dd1b63Bernhard Rosenkraenzer ands tmp1, count, #0x30 907e4fa560999d07064d219a16ebb50d3691dd1b63Bernhard Rosenkraenzer b.eq .Ltail15 917e4fa560999d07064d219a16ebb50d3691dd1b63Bernhard Rosenkraenzer add dst, dst, tmp1 927e4fa560999d07064d219a16ebb50d3691dd1b63Bernhard Rosenkraenzer cmp tmp1w, #0x20 937e4fa560999d07064d219a16ebb50d3691dd1b63Bernhard Rosenkraenzer b.eq 1f 947e4fa560999d07064d219a16ebb50d3691dd1b63Bernhard Rosenkraenzer b.lt 2f 957e4fa560999d07064d219a16ebb50d3691dd1b63Bernhard Rosenkraenzer stp A_l, A_l, [dst, #-48] 967e4fa560999d07064d219a16ebb50d3691dd1b63Bernhard Rosenkraenzer1: 977e4fa560999d07064d219a16ebb50d3691dd1b63Bernhard Rosenkraenzer stp A_l, A_l, [dst, #-32] 987e4fa560999d07064d219a16ebb50d3691dd1b63Bernhard Rosenkraenzer2: 997e4fa560999d07064d219a16ebb50d3691dd1b63Bernhard Rosenkraenzer stp A_l, A_l, [dst, #-16] 1007e4fa560999d07064d219a16ebb50d3691dd1b63Bernhard Rosenkraenzer 1017e4fa560999d07064d219a16ebb50d3691dd1b63Bernhard Rosenkraenzer.Ltail15: 1027e4fa560999d07064d219a16ebb50d3691dd1b63Bernhard Rosenkraenzer and count, count, #15 1037e4fa560999d07064d219a16ebb50d3691dd1b63Bernhard Rosenkraenzer add dst, dst, count 1047e4fa560999d07064d219a16ebb50d3691dd1b63Bernhard Rosenkraenzer stp A_l, A_l, [dst, #-16] /* Repeat some/all of last store. */ 1057e4fa560999d07064d219a16ebb50d3691dd1b63Bernhard Rosenkraenzer ret 1067e4fa560999d07064d219a16ebb50d3691dd1b63Bernhard Rosenkraenzer 1077e4fa560999d07064d219a16ebb50d3691dd1b63Bernhard Rosenkraenzer.Ltail15tiny: 1087e4fa560999d07064d219a16ebb50d3691dd1b63Bernhard Rosenkraenzer /* Set up to 15 bytes. Does not assume earlier memory 1097e4fa560999d07064d219a16ebb50d3691dd1b63Bernhard Rosenkraenzer being set. */ 1107e4fa560999d07064d219a16ebb50d3691dd1b63Bernhard Rosenkraenzer tbz count, #3, 1f 1117e4fa560999d07064d219a16ebb50d3691dd1b63Bernhard Rosenkraenzer str A_l, [dst], #8 1127e4fa560999d07064d219a16ebb50d3691dd1b63Bernhard Rosenkraenzer1: 1137e4fa560999d07064d219a16ebb50d3691dd1b63Bernhard Rosenkraenzer tbz count, #2, 1f 1147e4fa560999d07064d219a16ebb50d3691dd1b63Bernhard Rosenkraenzer str A_lw, [dst], #4 1157e4fa560999d07064d219a16ebb50d3691dd1b63Bernhard Rosenkraenzer1: 1167e4fa560999d07064d219a16ebb50d3691dd1b63Bernhard Rosenkraenzer tbz count, #1, 1f 1177e4fa560999d07064d219a16ebb50d3691dd1b63Bernhard Rosenkraenzer strh A_lw, [dst], #2 1187e4fa560999d07064d219a16ebb50d3691dd1b63Bernhard Rosenkraenzer1: 1197e4fa560999d07064d219a16ebb50d3691dd1b63Bernhard Rosenkraenzer tbz count, #0, 1f 1207e4fa560999d07064d219a16ebb50d3691dd1b63Bernhard Rosenkraenzer strb A_lw, [dst] 1217e4fa560999d07064d219a16ebb50d3691dd1b63Bernhard Rosenkraenzer1: 1227e4fa560999d07064d219a16ebb50d3691dd1b63Bernhard Rosenkraenzer ret 1237e4fa560999d07064d219a16ebb50d3691dd1b63Bernhard Rosenkraenzer 1247e4fa560999d07064d219a16ebb50d3691dd1b63Bernhard Rosenkraenzer /* Critical loop. Start at a new cache line boundary. Assuming 1257e4fa560999d07064d219a16ebb50d3691dd1b63Bernhard Rosenkraenzer * 64 bytes per line, this ensures the entire loop is in one line. */ 1267e4fa560999d07064d219a16ebb50d3691dd1b63Bernhard Rosenkraenzer .p2align 6 1277e4fa560999d07064d219a16ebb50d3691dd1b63Bernhard Rosenkraenzer.Lnot_short: 1287e4fa560999d07064d219a16ebb50d3691dd1b63Bernhard Rosenkraenzer neg tmp2, dst 1297e4fa560999d07064d219a16ebb50d3691dd1b63Bernhard Rosenkraenzer ands tmp2, tmp2, #15 1307e4fa560999d07064d219a16ebb50d3691dd1b63Bernhard Rosenkraenzer b.eq 2f 1317e4fa560999d07064d219a16ebb50d3691dd1b63Bernhard Rosenkraenzer /* Bring DST to 128-bit (16-byte) alignment. We know that there's 1327e4fa560999d07064d219a16ebb50d3691dd1b63Bernhard Rosenkraenzer * more than that to set, so we simply store 16 bytes and advance by 1337e4fa560999d07064d219a16ebb50d3691dd1b63Bernhard Rosenkraenzer * the amount required to reach alignment. */ 1347e4fa560999d07064d219a16ebb50d3691dd1b63Bernhard Rosenkraenzer sub count, count, tmp2 1357e4fa560999d07064d219a16ebb50d3691dd1b63Bernhard Rosenkraenzer stp A_l, A_l, [dst] 1367e4fa560999d07064d219a16ebb50d3691dd1b63Bernhard Rosenkraenzer add dst, dst, tmp2 1377e4fa560999d07064d219a16ebb50d3691dd1b63Bernhard Rosenkraenzer /* There may be less than 63 bytes to go now. */ 1387e4fa560999d07064d219a16ebb50d3691dd1b63Bernhard Rosenkraenzer cmp count, #63 1397e4fa560999d07064d219a16ebb50d3691dd1b63Bernhard Rosenkraenzer b.le .Ltail63 1407e4fa560999d07064d219a16ebb50d3691dd1b63Bernhard Rosenkraenzer2: 1417e4fa560999d07064d219a16ebb50d3691dd1b63Bernhard Rosenkraenzer sub dst, dst, #16 /* Pre-bias. */ 1427e4fa560999d07064d219a16ebb50d3691dd1b63Bernhard Rosenkraenzer sub count, count, #64 1437e4fa560999d07064d219a16ebb50d3691dd1b63Bernhard Rosenkraenzer1: 1447e4fa560999d07064d219a16ebb50d3691dd1b63Bernhard Rosenkraenzer stp A_l, A_l, [dst, #16] 1457e4fa560999d07064d219a16ebb50d3691dd1b63Bernhard Rosenkraenzer stp A_l, A_l, [dst, #32] 1467e4fa560999d07064d219a16ebb50d3691dd1b63Bernhard Rosenkraenzer stp A_l, A_l, [dst, #48] 1477e4fa560999d07064d219a16ebb50d3691dd1b63Bernhard Rosenkraenzer stp A_l, A_l, [dst, #64]! 1487e4fa560999d07064d219a16ebb50d3691dd1b63Bernhard Rosenkraenzer subs count, count, #64 1497e4fa560999d07064d219a16ebb50d3691dd1b63Bernhard Rosenkraenzer b.ge 1b 1507e4fa560999d07064d219a16ebb50d3691dd1b63Bernhard Rosenkraenzer tst count, #0x3f 1517e4fa560999d07064d219a16ebb50d3691dd1b63Bernhard Rosenkraenzer add dst, dst, #16 1527e4fa560999d07064d219a16ebb50d3691dd1b63Bernhard Rosenkraenzer b.ne .Ltail63 1537e4fa560999d07064d219a16ebb50d3691dd1b63Bernhard Rosenkraenzer ret 1547e4fa560999d07064d219a16ebb50d3691dd1b63Bernhard Rosenkraenzer 1557e4fa560999d07064d219a16ebb50d3691dd1b63Bernhard Rosenkraenzer /* For zeroing memory, check to see if we can use the ZVA feature to 1567e4fa560999d07064d219a16ebb50d3691dd1b63Bernhard Rosenkraenzer * zero entire 'cache' lines. */ 1577e4fa560999d07064d219a16ebb50d3691dd1b63Bernhard Rosenkraenzer.Lzero_mem: 1587e4fa560999d07064d219a16ebb50d3691dd1b63Bernhard Rosenkraenzer mov A_l, #0 1597e4fa560999d07064d219a16ebb50d3691dd1b63Bernhard Rosenkraenzer cmp count, #63 1607e4fa560999d07064d219a16ebb50d3691dd1b63Bernhard Rosenkraenzer b.le .Ltail_maybe_tiny 1617e4fa560999d07064d219a16ebb50d3691dd1b63Bernhard Rosenkraenzer neg tmp2, dst 1627e4fa560999d07064d219a16ebb50d3691dd1b63Bernhard Rosenkraenzer ands tmp2, tmp2, #15 1637e4fa560999d07064d219a16ebb50d3691dd1b63Bernhard Rosenkraenzer b.eq 1f 1647e4fa560999d07064d219a16ebb50d3691dd1b63Bernhard Rosenkraenzer sub count, count, tmp2 1657e4fa560999d07064d219a16ebb50d3691dd1b63Bernhard Rosenkraenzer stp A_l, A_l, [dst] 1667e4fa560999d07064d219a16ebb50d3691dd1b63Bernhard Rosenkraenzer add dst, dst, tmp2 1677e4fa560999d07064d219a16ebb50d3691dd1b63Bernhard Rosenkraenzer cmp count, #63 1687e4fa560999d07064d219a16ebb50d3691dd1b63Bernhard Rosenkraenzer b.le .Ltail63 1697e4fa560999d07064d219a16ebb50d3691dd1b63Bernhard Rosenkraenzer1: 1707e4fa560999d07064d219a16ebb50d3691dd1b63Bernhard Rosenkraenzer /* For zeroing small amounts of memory, it's not worth setting up 1717e4fa560999d07064d219a16ebb50d3691dd1b63Bernhard Rosenkraenzer * the line-clear code. */ 1727e4fa560999d07064d219a16ebb50d3691dd1b63Bernhard Rosenkraenzer cmp count, #128 1737e4fa560999d07064d219a16ebb50d3691dd1b63Bernhard Rosenkraenzer b.lt .Lnot_short 1747e4fa560999d07064d219a16ebb50d3691dd1b63Bernhard Rosenkraenzer#ifdef MAYBE_VIRT 1757e4fa560999d07064d219a16ebb50d3691dd1b63Bernhard Rosenkraenzer /* For efficiency when virtualized, we cache the ZVA capability. */ 1767e4fa560999d07064d219a16ebb50d3691dd1b63Bernhard Rosenkraenzer adrp tmp2, .Lcache_clear 1777e4fa560999d07064d219a16ebb50d3691dd1b63Bernhard Rosenkraenzer ldr zva_len, [tmp2, #:lo12:.Lcache_clear] 1787e4fa560999d07064d219a16ebb50d3691dd1b63Bernhard Rosenkraenzer tbnz zva_len, #31, .Lnot_short 1797e4fa560999d07064d219a16ebb50d3691dd1b63Bernhard Rosenkraenzer cbnz zva_len, .Lzero_by_line 1807e4fa560999d07064d219a16ebb50d3691dd1b63Bernhard Rosenkraenzer mrs tmp1, dczid_el0 1817e4fa560999d07064d219a16ebb50d3691dd1b63Bernhard Rosenkraenzer tbz tmp1, #4, 1f 1827e4fa560999d07064d219a16ebb50d3691dd1b63Bernhard Rosenkraenzer /* ZVA not available. Remember this for next time. */ 1837e4fa560999d07064d219a16ebb50d3691dd1b63Bernhard Rosenkraenzer mov zva_len, #~0 1847e4fa560999d07064d219a16ebb50d3691dd1b63Bernhard Rosenkraenzer str zva_len, [tmp2, #:lo12:.Lcache_clear] 1857e4fa560999d07064d219a16ebb50d3691dd1b63Bernhard Rosenkraenzer b .Lnot_short 1867e4fa560999d07064d219a16ebb50d3691dd1b63Bernhard Rosenkraenzer1: 1877e4fa560999d07064d219a16ebb50d3691dd1b63Bernhard Rosenkraenzer mov tmp3w, #4 1887e4fa560999d07064d219a16ebb50d3691dd1b63Bernhard Rosenkraenzer and zva_len, tmp1w, #15 /* Safety: other bits reserved. */ 1897e4fa560999d07064d219a16ebb50d3691dd1b63Bernhard Rosenkraenzer lsl zva_len, tmp3w, zva_len 1907e4fa560999d07064d219a16ebb50d3691dd1b63Bernhard Rosenkraenzer str zva_len, [tmp2, #:lo12:.Lcache_clear] 1917e4fa560999d07064d219a16ebb50d3691dd1b63Bernhard Rosenkraenzer#else 1927e4fa560999d07064d219a16ebb50d3691dd1b63Bernhard Rosenkraenzer mrs tmp1, dczid_el0 1937e4fa560999d07064d219a16ebb50d3691dd1b63Bernhard Rosenkraenzer tbnz tmp1, #4, .Lnot_short 1947e4fa560999d07064d219a16ebb50d3691dd1b63Bernhard Rosenkraenzer mov tmp3w, #4 1957e4fa560999d07064d219a16ebb50d3691dd1b63Bernhard Rosenkraenzer and zva_len, tmp1w, #15 /* Safety: other bits reserved. */ 1967e4fa560999d07064d219a16ebb50d3691dd1b63Bernhard Rosenkraenzer lsl zva_len, tmp3w, zva_len 1977e4fa560999d07064d219a16ebb50d3691dd1b63Bernhard Rosenkraenzer#endif 1987e4fa560999d07064d219a16ebb50d3691dd1b63Bernhard Rosenkraenzer 1997e4fa560999d07064d219a16ebb50d3691dd1b63Bernhard Rosenkraenzer.Lzero_by_line: 2007e4fa560999d07064d219a16ebb50d3691dd1b63Bernhard Rosenkraenzer /* Compute how far we need to go to become suitably aligned. We're 2017e4fa560999d07064d219a16ebb50d3691dd1b63Bernhard Rosenkraenzer * already at quad-word alignment. */ 2027e4fa560999d07064d219a16ebb50d3691dd1b63Bernhard Rosenkraenzer cmp count, zva_len_x 2037e4fa560999d07064d219a16ebb50d3691dd1b63Bernhard Rosenkraenzer b.lt .Lnot_short /* Not enough to reach alignment. */ 2047e4fa560999d07064d219a16ebb50d3691dd1b63Bernhard Rosenkraenzer sub zva_bits_x, zva_len_x, #1 2057e4fa560999d07064d219a16ebb50d3691dd1b63Bernhard Rosenkraenzer neg tmp2, dst 2067e4fa560999d07064d219a16ebb50d3691dd1b63Bernhard Rosenkraenzer ands tmp2, tmp2, zva_bits_x 2077e4fa560999d07064d219a16ebb50d3691dd1b63Bernhard Rosenkraenzer b.eq 1f /* Already aligned. */ 2087e4fa560999d07064d219a16ebb50d3691dd1b63Bernhard Rosenkraenzer /* Not aligned, check that there's enough to copy after alignment. */ 2097e4fa560999d07064d219a16ebb50d3691dd1b63Bernhard Rosenkraenzer sub tmp1, count, tmp2 2107e4fa560999d07064d219a16ebb50d3691dd1b63Bernhard Rosenkraenzer cmp tmp1, #64 2117e4fa560999d07064d219a16ebb50d3691dd1b63Bernhard Rosenkraenzer ccmp tmp1, zva_len_x, #8, ge /* NZCV=0b1000 */ 2127e4fa560999d07064d219a16ebb50d3691dd1b63Bernhard Rosenkraenzer b.lt .Lnot_short 2137e4fa560999d07064d219a16ebb50d3691dd1b63Bernhard Rosenkraenzer /* We know that there's at least 64 bytes to zero and that it's safe 2147e4fa560999d07064d219a16ebb50d3691dd1b63Bernhard Rosenkraenzer * to overrun by 64 bytes. */ 2157e4fa560999d07064d219a16ebb50d3691dd1b63Bernhard Rosenkraenzer mov count, tmp1 2167e4fa560999d07064d219a16ebb50d3691dd1b63Bernhard Rosenkraenzer2: 2177e4fa560999d07064d219a16ebb50d3691dd1b63Bernhard Rosenkraenzer stp A_l, A_l, [dst] 2187e4fa560999d07064d219a16ebb50d3691dd1b63Bernhard Rosenkraenzer stp A_l, A_l, [dst, #16] 2197e4fa560999d07064d219a16ebb50d3691dd1b63Bernhard Rosenkraenzer stp A_l, A_l, [dst, #32] 2207e4fa560999d07064d219a16ebb50d3691dd1b63Bernhard Rosenkraenzer subs tmp2, tmp2, #64 2217e4fa560999d07064d219a16ebb50d3691dd1b63Bernhard Rosenkraenzer stp A_l, A_l, [dst, #48] 2227e4fa560999d07064d219a16ebb50d3691dd1b63Bernhard Rosenkraenzer add dst, dst, #64 2237e4fa560999d07064d219a16ebb50d3691dd1b63Bernhard Rosenkraenzer b.ge 2b 2247e4fa560999d07064d219a16ebb50d3691dd1b63Bernhard Rosenkraenzer /* We've overrun a bit, so adjust dst downwards. */ 2257e4fa560999d07064d219a16ebb50d3691dd1b63Bernhard Rosenkraenzer add dst, dst, tmp2 2267e4fa560999d07064d219a16ebb50d3691dd1b63Bernhard Rosenkraenzer1: 2277e4fa560999d07064d219a16ebb50d3691dd1b63Bernhard Rosenkraenzer sub count, count, zva_len_x 2287e4fa560999d07064d219a16ebb50d3691dd1b63Bernhard Rosenkraenzer3: 2297e4fa560999d07064d219a16ebb50d3691dd1b63Bernhard Rosenkraenzer dc zva, dst 2307e4fa560999d07064d219a16ebb50d3691dd1b63Bernhard Rosenkraenzer add dst, dst, zva_len_x 2317e4fa560999d07064d219a16ebb50d3691dd1b63Bernhard Rosenkraenzer subs count, count, zva_len_x 2327e4fa560999d07064d219a16ebb50d3691dd1b63Bernhard Rosenkraenzer b.ge 3b 2337e4fa560999d07064d219a16ebb50d3691dd1b63Bernhard Rosenkraenzer ands count, count, zva_bits_x 2347e4fa560999d07064d219a16ebb50d3691dd1b63Bernhard Rosenkraenzer b.ne .Ltail_maybe_long 2357e4fa560999d07064d219a16ebb50d3691dd1b63Bernhard Rosenkraenzer ret 23662d92e1610080cad16342f87a7fb46d8388ee4bbBernhard Rosenkraenzer#ifdef BZERO 23762d92e1610080cad16342f87a7fb46d8388ee4bbBernhard RosenkraenzerEND(bzero) 23862d92e1610080cad16342f87a7fb46d8388ee4bbBernhard Rosenkraenzer#else 2397e4fa560999d07064d219a16ebb50d3691dd1b63Bernhard RosenkraenzerEND(memset) 24062d92e1610080cad16342f87a7fb46d8388ee4bbBernhard Rosenkraenzer#endif 2417e4fa560999d07064d219a16ebb50d3691dd1b63Bernhard Rosenkraenzer 2427e4fa560999d07064d219a16ebb50d3691dd1b63Bernhard Rosenkraenzer#ifdef MAYBE_VIRT 2437e4fa560999d07064d219a16ebb50d3691dd1b63Bernhard Rosenkraenzer .bss 2447e4fa560999d07064d219a16ebb50d3691dd1b63Bernhard Rosenkraenzer .p2align 2 2457e4fa560999d07064d219a16ebb50d3691dd1b63Bernhard Rosenkraenzer.Lcache_clear: 2467e4fa560999d07064d219a16ebb50d3691dd1b63Bernhard Rosenkraenzer .space 4 2477e4fa560999d07064d219a16ebb50d3691dd1b63Bernhard Rosenkraenzer#endif 248