1e6e8a0bd7cffcc9ae2e0e75546fb12a19213d4aeBen Cheng/* 2e6e8a0bd7cffcc9ae2e0e75546fb12a19213d4aeBen Cheng * Copyright 2010 Tilera Corporation. All Rights Reserved. 3e6e8a0bd7cffcc9ae2e0e75546fb12a19213d4aeBen Cheng * 4e6e8a0bd7cffcc9ae2e0e75546fb12a19213d4aeBen Cheng * This program is free software; you can redistribute it and/or 5e6e8a0bd7cffcc9ae2e0e75546fb12a19213d4aeBen Cheng * modify it under the terms of the GNU General Public License 6e6e8a0bd7cffcc9ae2e0e75546fb12a19213d4aeBen Cheng * as published by the Free Software Foundation, version 2. 7e6e8a0bd7cffcc9ae2e0e75546fb12a19213d4aeBen Cheng * 8e6e8a0bd7cffcc9ae2e0e75546fb12a19213d4aeBen Cheng * This program is distributed in the hope that it will be useful, but 9e6e8a0bd7cffcc9ae2e0e75546fb12a19213d4aeBen Cheng * WITHOUT ANY WARRANTY; without even the implied warranty of 10e6e8a0bd7cffcc9ae2e0e75546fb12a19213d4aeBen Cheng * MERCHANTABILITY OR FITNESS FOR A PARTICULAR PURPOSE, GOOD TITLE or 11e6e8a0bd7cffcc9ae2e0e75546fb12a19213d4aeBen Cheng * NON INFRINGEMENT. See the GNU General Public License for 12e6e8a0bd7cffcc9ae2e0e75546fb12a19213d4aeBen Cheng * more details. 13e6e8a0bd7cffcc9ae2e0e75546fb12a19213d4aeBen Cheng */ 14e6e8a0bd7cffcc9ae2e0e75546fb12a19213d4aeBen Cheng 15e6e8a0bd7cffcc9ae2e0e75546fb12a19213d4aeBen Cheng#include <arch/chip.h> 16e6e8a0bd7cffcc9ae2e0e75546fb12a19213d4aeBen Cheng 17e6e8a0bd7cffcc9ae2e0e75546fb12a19213d4aeBen Cheng 18e6e8a0bd7cffcc9ae2e0e75546fb12a19213d4aeBen Cheng/* 19e6e8a0bd7cffcc9ae2e0e75546fb12a19213d4aeBen Cheng * This file shares the implementation of the userspace memcpy and 20e6e8a0bd7cffcc9ae2e0e75546fb12a19213d4aeBen Cheng * the kernel's memcpy, copy_to_user and copy_from_user. 21e6e8a0bd7cffcc9ae2e0e75546fb12a19213d4aeBen Cheng */ 22e6e8a0bd7cffcc9ae2e0e75546fb12a19213d4aeBen Cheng 23e6e8a0bd7cffcc9ae2e0e75546fb12a19213d4aeBen Cheng#include <linux/linkage.h> 24e6e8a0bd7cffcc9ae2e0e75546fb12a19213d4aeBen Cheng 25e6e8a0bd7cffcc9ae2e0e75546fb12a19213d4aeBen Cheng#define IS_MEMCPY 0 26e6e8a0bd7cffcc9ae2e0e75546fb12a19213d4aeBen Cheng#define IS_COPY_FROM_USER 1 27e6e8a0bd7cffcc9ae2e0e75546fb12a19213d4aeBen Cheng#define IS_COPY_FROM_USER_ZEROING 2 28e6e8a0bd7cffcc9ae2e0e75546fb12a19213d4aeBen Cheng#define IS_COPY_TO_USER -1 29e6e8a0bd7cffcc9ae2e0e75546fb12a19213d4aeBen Cheng 30e6e8a0bd7cffcc9ae2e0e75546fb12a19213d4aeBen Cheng .section .text.memcpy_common, "ax" 31e6e8a0bd7cffcc9ae2e0e75546fb12a19213d4aeBen Cheng .align 64 32e6e8a0bd7cffcc9ae2e0e75546fb12a19213d4aeBen Cheng 33e6e8a0bd7cffcc9ae2e0e75546fb12a19213d4aeBen Cheng/* Use this to preface each bundle that can cause an exception so 34e6e8a0bd7cffcc9ae2e0e75546fb12a19213d4aeBen Cheng * the kernel can clean up properly. The special cleanup code should 35e6e8a0bd7cffcc9ae2e0e75546fb12a19213d4aeBen Cheng * not use these, since it knows what it is doing. 36e6e8a0bd7cffcc9ae2e0e75546fb12a19213d4aeBen Cheng */ 37e6e8a0bd7cffcc9ae2e0e75546fb12a19213d4aeBen Cheng#define EX \ 38e6e8a0bd7cffcc9ae2e0e75546fb12a19213d4aeBen Cheng .pushsection __ex_table, "a"; \ 39e6e8a0bd7cffcc9ae2e0e75546fb12a19213d4aeBen Cheng .align 4; \ 40e6e8a0bd7cffcc9ae2e0e75546fb12a19213d4aeBen Cheng .word 9f, memcpy_common_fixup; \ 41e6e8a0bd7cffcc9ae2e0e75546fb12a19213d4aeBen Cheng .popsection; \ 42e6e8a0bd7cffcc9ae2e0e75546fb12a19213d4aeBen Cheng 9 43e6e8a0bd7cffcc9ae2e0e75546fb12a19213d4aeBen Cheng 44e6e8a0bd7cffcc9ae2e0e75546fb12a19213d4aeBen Cheng 45e6e8a0bd7cffcc9ae2e0e75546fb12a19213d4aeBen Cheng/* __copy_from_user_inatomic takes the kernel target address in r0, 46e6e8a0bd7cffcc9ae2e0e75546fb12a19213d4aeBen Cheng * the user source in r1, and the bytes to copy in r2. 47e6e8a0bd7cffcc9ae2e0e75546fb12a19213d4aeBen Cheng * It returns the number of uncopiable bytes (hopefully zero) in r0. 48e6e8a0bd7cffcc9ae2e0e75546fb12a19213d4aeBen Cheng */ 49e6e8a0bd7cffcc9ae2e0e75546fb12a19213d4aeBen ChengENTRY(__copy_from_user_inatomic) 50e6e8a0bd7cffcc9ae2e0e75546fb12a19213d4aeBen Cheng.type __copy_from_user_inatomic, @function 51e6e8a0bd7cffcc9ae2e0e75546fb12a19213d4aeBen Cheng FEEDBACK_ENTER_EXPLICIT(__copy_from_user_inatomic, \ 52e6e8a0bd7cffcc9ae2e0e75546fb12a19213d4aeBen Cheng .text.memcpy_common, \ 53e6e8a0bd7cffcc9ae2e0e75546fb12a19213d4aeBen Cheng .Lend_memcpy_common - __copy_from_user_inatomic) 54e6e8a0bd7cffcc9ae2e0e75546fb12a19213d4aeBen Cheng { movei r29, IS_COPY_FROM_USER; j memcpy_common } 55e6e8a0bd7cffcc9ae2e0e75546fb12a19213d4aeBen Cheng .size __copy_from_user_inatomic, . - __copy_from_user_inatomic 56e6e8a0bd7cffcc9ae2e0e75546fb12a19213d4aeBen Cheng 57e6e8a0bd7cffcc9ae2e0e75546fb12a19213d4aeBen Cheng/* __copy_from_user_zeroing is like __copy_from_user_inatomic, but 58e6e8a0bd7cffcc9ae2e0e75546fb12a19213d4aeBen Cheng * any uncopiable bytes are zeroed in the target. 59e6e8a0bd7cffcc9ae2e0e75546fb12a19213d4aeBen Cheng */ 60e6e8a0bd7cffcc9ae2e0e75546fb12a19213d4aeBen ChengENTRY(__copy_from_user_zeroing) 61e6e8a0bd7cffcc9ae2e0e75546fb12a19213d4aeBen Cheng.type __copy_from_user_zeroing, @function 62e6e8a0bd7cffcc9ae2e0e75546fb12a19213d4aeBen Cheng FEEDBACK_REENTER(__copy_from_user_inatomic) 63e6e8a0bd7cffcc9ae2e0e75546fb12a19213d4aeBen Cheng { movei r29, IS_COPY_FROM_USER_ZEROING; j memcpy_common } 64e6e8a0bd7cffcc9ae2e0e75546fb12a19213d4aeBen Cheng .size __copy_from_user_zeroing, . - __copy_from_user_zeroing 65e6e8a0bd7cffcc9ae2e0e75546fb12a19213d4aeBen Cheng 66e6e8a0bd7cffcc9ae2e0e75546fb12a19213d4aeBen Cheng/* __copy_to_user_inatomic takes the user target address in r0, 67e6e8a0bd7cffcc9ae2e0e75546fb12a19213d4aeBen Cheng * the kernel source in r1, and the bytes to copy in r2. 68e6e8a0bd7cffcc9ae2e0e75546fb12a19213d4aeBen Cheng * It returns the number of uncopiable bytes (hopefully zero) in r0. 69e6e8a0bd7cffcc9ae2e0e75546fb12a19213d4aeBen Cheng */ 70e6e8a0bd7cffcc9ae2e0e75546fb12a19213d4aeBen ChengENTRY(__copy_to_user_inatomic) 71e6e8a0bd7cffcc9ae2e0e75546fb12a19213d4aeBen Cheng.type __copy_to_user_inatomic, @function 72e6e8a0bd7cffcc9ae2e0e75546fb12a19213d4aeBen Cheng FEEDBACK_REENTER(__copy_from_user_inatomic) 73e6e8a0bd7cffcc9ae2e0e75546fb12a19213d4aeBen Cheng { movei r29, IS_COPY_TO_USER; j memcpy_common } 74e6e8a0bd7cffcc9ae2e0e75546fb12a19213d4aeBen Cheng .size __copy_to_user_inatomic, . - __copy_to_user_inatomic 75e6e8a0bd7cffcc9ae2e0e75546fb12a19213d4aeBen Cheng 76e6e8a0bd7cffcc9ae2e0e75546fb12a19213d4aeBen ChengENTRY(memcpy) 77e6e8a0bd7cffcc9ae2e0e75546fb12a19213d4aeBen Cheng.type memcpy, @function 78e6e8a0bd7cffcc9ae2e0e75546fb12a19213d4aeBen Cheng FEEDBACK_REENTER(__copy_from_user_inatomic) 79e6e8a0bd7cffcc9ae2e0e75546fb12a19213d4aeBen Cheng { movei r29, IS_MEMCPY } 80e6e8a0bd7cffcc9ae2e0e75546fb12a19213d4aeBen Cheng .size memcpy, . - memcpy 81e6e8a0bd7cffcc9ae2e0e75546fb12a19213d4aeBen Cheng /* Fall through */ 82e6e8a0bd7cffcc9ae2e0e75546fb12a19213d4aeBen Cheng 83e6e8a0bd7cffcc9ae2e0e75546fb12a19213d4aeBen Cheng .type memcpy_common, @function 84e6e8a0bd7cffcc9ae2e0e75546fb12a19213d4aeBen Chengmemcpy_common: 85e6e8a0bd7cffcc9ae2e0e75546fb12a19213d4aeBen Cheng /* On entry, r29 holds one of the IS_* macro values from above. */ 86e6e8a0bd7cffcc9ae2e0e75546fb12a19213d4aeBen Cheng 87e6e8a0bd7cffcc9ae2e0e75546fb12a19213d4aeBen Cheng 88e6e8a0bd7cffcc9ae2e0e75546fb12a19213d4aeBen Cheng /* r0 is the dest, r1 is the source, r2 is the size. */ 89e6e8a0bd7cffcc9ae2e0e75546fb12a19213d4aeBen Cheng 90e6e8a0bd7cffcc9ae2e0e75546fb12a19213d4aeBen Cheng /* Save aside original dest so we can return it at the end. */ 91e6e8a0bd7cffcc9ae2e0e75546fb12a19213d4aeBen Cheng { sw sp, lr; move r23, r0; or r4, r0, r1 } 92e6e8a0bd7cffcc9ae2e0e75546fb12a19213d4aeBen Cheng 93e6e8a0bd7cffcc9ae2e0e75546fb12a19213d4aeBen Cheng /* Check for an empty size. */ 94e6e8a0bd7cffcc9ae2e0e75546fb12a19213d4aeBen Cheng { bz r2, .Ldone; andi r4, r4, 3 } 95e6e8a0bd7cffcc9ae2e0e75546fb12a19213d4aeBen Cheng 96e6e8a0bd7cffcc9ae2e0e75546fb12a19213d4aeBen Cheng /* Save aside original values in case of a fault. */ 97e6e8a0bd7cffcc9ae2e0e75546fb12a19213d4aeBen Cheng { move r24, r1; move r25, r2 } 98e6e8a0bd7cffcc9ae2e0e75546fb12a19213d4aeBen Cheng move r27, lr 99e6e8a0bd7cffcc9ae2e0e75546fb12a19213d4aeBen Cheng 100e6e8a0bd7cffcc9ae2e0e75546fb12a19213d4aeBen Cheng /* Check for an unaligned source or dest. */ 101e6e8a0bd7cffcc9ae2e0e75546fb12a19213d4aeBen Cheng { bnz r4, .Lcopy_unaligned_maybe_many; addli r4, r2, -256 } 102e6e8a0bd7cffcc9ae2e0e75546fb12a19213d4aeBen Cheng 103e6e8a0bd7cffcc9ae2e0e75546fb12a19213d4aeBen Cheng.Lcheck_aligned_copy_size: 104e6e8a0bd7cffcc9ae2e0e75546fb12a19213d4aeBen Cheng /* If we are copying < 256 bytes, branch to simple case. */ 105e6e8a0bd7cffcc9ae2e0e75546fb12a19213d4aeBen Cheng { blzt r4, .Lcopy_8_check; slti_u r8, r2, 8 } 106e6e8a0bd7cffcc9ae2e0e75546fb12a19213d4aeBen Cheng 107e6e8a0bd7cffcc9ae2e0e75546fb12a19213d4aeBen Cheng /* Copying >= 256 bytes, so jump to complex prefetching loop. */ 108e6e8a0bd7cffcc9ae2e0e75546fb12a19213d4aeBen Cheng { andi r6, r1, 63; j .Lcopy_many } 109e6e8a0bd7cffcc9ae2e0e75546fb12a19213d4aeBen Cheng 110e6e8a0bd7cffcc9ae2e0e75546fb12a19213d4aeBen Cheng/* 111e6e8a0bd7cffcc9ae2e0e75546fb12a19213d4aeBen Cheng * 112e6e8a0bd7cffcc9ae2e0e75546fb12a19213d4aeBen Cheng * Aligned 4 byte at a time copy loop 113e6e8a0bd7cffcc9ae2e0e75546fb12a19213d4aeBen Cheng * 114e6e8a0bd7cffcc9ae2e0e75546fb12a19213d4aeBen Cheng */ 115e6e8a0bd7cffcc9ae2e0e75546fb12a19213d4aeBen Cheng 116e6e8a0bd7cffcc9ae2e0e75546fb12a19213d4aeBen Cheng.Lcopy_8_loop: 117e6e8a0bd7cffcc9ae2e0e75546fb12a19213d4aeBen Cheng /* Copy two words at a time to hide load latency. */ 118e6e8a0bd7cffcc9ae2e0e75546fb12a19213d4aeBen ChengEX: { lw r3, r1; addi r1, r1, 4; slti_u r8, r2, 16 } 119e6e8a0bd7cffcc9ae2e0e75546fb12a19213d4aeBen ChengEX: { lw r4, r1; addi r1, r1, 4 } 120e6e8a0bd7cffcc9ae2e0e75546fb12a19213d4aeBen ChengEX: { sw r0, r3; addi r0, r0, 4; addi r2, r2, -4 } 121e6e8a0bd7cffcc9ae2e0e75546fb12a19213d4aeBen ChengEX: { sw r0, r4; addi r0, r0, 4; addi r2, r2, -4 } 122e6e8a0bd7cffcc9ae2e0e75546fb12a19213d4aeBen Cheng.Lcopy_8_check: 123e6e8a0bd7cffcc9ae2e0e75546fb12a19213d4aeBen Cheng { bzt r8, .Lcopy_8_loop; slti_u r4, r2, 4 } 124e6e8a0bd7cffcc9ae2e0e75546fb12a19213d4aeBen Cheng 125e6e8a0bd7cffcc9ae2e0e75546fb12a19213d4aeBen Cheng /* Copy odd leftover word, if any. */ 126e6e8a0bd7cffcc9ae2e0e75546fb12a19213d4aeBen Cheng { bnzt r4, .Lcheck_odd_stragglers } 127e6e8a0bd7cffcc9ae2e0e75546fb12a19213d4aeBen ChengEX: { lw r3, r1; addi r1, r1, 4 } 128e6e8a0bd7cffcc9ae2e0e75546fb12a19213d4aeBen ChengEX: { sw r0, r3; addi r0, r0, 4; addi r2, r2, -4 } 129e6e8a0bd7cffcc9ae2e0e75546fb12a19213d4aeBen Cheng 130e6e8a0bd7cffcc9ae2e0e75546fb12a19213d4aeBen Cheng.Lcheck_odd_stragglers: 131e6e8a0bd7cffcc9ae2e0e75546fb12a19213d4aeBen Cheng { bnz r2, .Lcopy_unaligned_few } 132e6e8a0bd7cffcc9ae2e0e75546fb12a19213d4aeBen Cheng 133e6e8a0bd7cffcc9ae2e0e75546fb12a19213d4aeBen Cheng.Ldone: 134e6e8a0bd7cffcc9ae2e0e75546fb12a19213d4aeBen Cheng /* For memcpy return original dest address, else zero. */ 135e6e8a0bd7cffcc9ae2e0e75546fb12a19213d4aeBen Cheng { mz r0, r29, r23; jrp lr } 136e6e8a0bd7cffcc9ae2e0e75546fb12a19213d4aeBen Cheng 137e6e8a0bd7cffcc9ae2e0e75546fb12a19213d4aeBen Cheng 138e6e8a0bd7cffcc9ae2e0e75546fb12a19213d4aeBen Cheng/* 139e6e8a0bd7cffcc9ae2e0e75546fb12a19213d4aeBen Cheng * 140e6e8a0bd7cffcc9ae2e0e75546fb12a19213d4aeBen Cheng * Prefetching multiple cache line copy handler (for large transfers). 141e6e8a0bd7cffcc9ae2e0e75546fb12a19213d4aeBen Cheng * 142e6e8a0bd7cffcc9ae2e0e75546fb12a19213d4aeBen Cheng */ 143e6e8a0bd7cffcc9ae2e0e75546fb12a19213d4aeBen Cheng 144e6e8a0bd7cffcc9ae2e0e75546fb12a19213d4aeBen Cheng /* Copy words until r1 is cache-line-aligned. */ 145e6e8a0bd7cffcc9ae2e0e75546fb12a19213d4aeBen Cheng.Lalign_loop: 146e6e8a0bd7cffcc9ae2e0e75546fb12a19213d4aeBen ChengEX: { lw r3, r1; addi r1, r1, 4 } 147e6e8a0bd7cffcc9ae2e0e75546fb12a19213d4aeBen Cheng { andi r6, r1, 63 } 148e6e8a0bd7cffcc9ae2e0e75546fb12a19213d4aeBen ChengEX: { sw r0, r3; addi r0, r0, 4; addi r2, r2, -4 } 149e6e8a0bd7cffcc9ae2e0e75546fb12a19213d4aeBen Cheng.Lcopy_many: 150e6e8a0bd7cffcc9ae2e0e75546fb12a19213d4aeBen Cheng { bnzt r6, .Lalign_loop; addi r9, r0, 63 } 151e6e8a0bd7cffcc9ae2e0e75546fb12a19213d4aeBen Cheng 152e6e8a0bd7cffcc9ae2e0e75546fb12a19213d4aeBen Cheng { addi r3, r1, 60; andi r9, r9, -64 } 153e6e8a0bd7cffcc9ae2e0e75546fb12a19213d4aeBen Cheng 154e6e8a0bd7cffcc9ae2e0e75546fb12a19213d4aeBen Cheng /* No need to prefetch dst, we'll just do the wh64 155e6e8a0bd7cffcc9ae2e0e75546fb12a19213d4aeBen Cheng * right before we copy a line. 156e6e8a0bd7cffcc9ae2e0e75546fb12a19213d4aeBen Cheng */ 157e6e8a0bd7cffcc9ae2e0e75546fb12a19213d4aeBen ChengEX: { lw r5, r3; addi r3, r3, 64; movei r4, 1 } 158e6e8a0bd7cffcc9ae2e0e75546fb12a19213d4aeBen Cheng /* Intentionally stall for a few cycles to leave L2 cache alone. */ 159e6e8a0bd7cffcc9ae2e0e75546fb12a19213d4aeBen Cheng { bnzt zero, .; move r27, lr } 160e6e8a0bd7cffcc9ae2e0e75546fb12a19213d4aeBen ChengEX: { lw r6, r3; addi r3, r3, 64 } 161e6e8a0bd7cffcc9ae2e0e75546fb12a19213d4aeBen Cheng /* Intentionally stall for a few cycles to leave L2 cache alone. */ 162e6e8a0bd7cffcc9ae2e0e75546fb12a19213d4aeBen Cheng { bnzt zero, . } 163e6e8a0bd7cffcc9ae2e0e75546fb12a19213d4aeBen ChengEX: { lw r7, r3; addi r3, r3, 64 } 164e6e8a0bd7cffcc9ae2e0e75546fb12a19213d4aeBen Cheng /* Intentionally stall for a few cycles to leave L2 cache alone. */ 165e6e8a0bd7cffcc9ae2e0e75546fb12a19213d4aeBen Cheng { bz zero, .Lbig_loop2 } 166e6e8a0bd7cffcc9ae2e0e75546fb12a19213d4aeBen Cheng 167e6e8a0bd7cffcc9ae2e0e75546fb12a19213d4aeBen Cheng /* On entry to this loop: 168e6e8a0bd7cffcc9ae2e0e75546fb12a19213d4aeBen Cheng * - r0 points to the start of dst line 0 169e6e8a0bd7cffcc9ae2e0e75546fb12a19213d4aeBen Cheng * - r1 points to start of src line 0 170e6e8a0bd7cffcc9ae2e0e75546fb12a19213d4aeBen Cheng * - r2 >= (256 - 60), only the first time the loop trips. 171e6e8a0bd7cffcc9ae2e0e75546fb12a19213d4aeBen Cheng * - r3 contains r1 + 128 + 60 [pointer to end of source line 2] 172e6e8a0bd7cffcc9ae2e0e75546fb12a19213d4aeBen Cheng * This is our prefetch address. When we get near the end 173e6e8a0bd7cffcc9ae2e0e75546fb12a19213d4aeBen Cheng * rather than prefetching off the end this is changed to point 174e6e8a0bd7cffcc9ae2e0e75546fb12a19213d4aeBen Cheng * to some "safe" recently loaded address. 175e6e8a0bd7cffcc9ae2e0e75546fb12a19213d4aeBen Cheng * - r5 contains *(r1 + 60) [i.e. last word of source line 0] 176e6e8a0bd7cffcc9ae2e0e75546fb12a19213d4aeBen Cheng * - r6 contains *(r1 + 64 + 60) [i.e. last word of source line 1] 177e6e8a0bd7cffcc9ae2e0e75546fb12a19213d4aeBen Cheng * - r9 contains ((r0 + 63) & -64) 178e6e8a0bd7cffcc9ae2e0e75546fb12a19213d4aeBen Cheng * [start of next dst cache line.] 179e6e8a0bd7cffcc9ae2e0e75546fb12a19213d4aeBen Cheng */ 180e6e8a0bd7cffcc9ae2e0e75546fb12a19213d4aeBen Cheng 181e6e8a0bd7cffcc9ae2e0e75546fb12a19213d4aeBen Cheng.Lbig_loop: 182e6e8a0bd7cffcc9ae2e0e75546fb12a19213d4aeBen Cheng { jal .Lcopy_line2; add r15, r1, r2 } 183e6e8a0bd7cffcc9ae2e0e75546fb12a19213d4aeBen Cheng 184e6e8a0bd7cffcc9ae2e0e75546fb12a19213d4aeBen Cheng.Lbig_loop2: 185e6e8a0bd7cffcc9ae2e0e75546fb12a19213d4aeBen Cheng /* Copy line 0, first stalling until r5 is ready. */ 186e6e8a0bd7cffcc9ae2e0e75546fb12a19213d4aeBen ChengEX: { move r12, r5; lw r16, r1 } 187e6e8a0bd7cffcc9ae2e0e75546fb12a19213d4aeBen Cheng { bz r4, .Lcopy_8_check; slti_u r8, r2, 8 } 188e6e8a0bd7cffcc9ae2e0e75546fb12a19213d4aeBen Cheng /* Prefetch several lines ahead. */ 189e6e8a0bd7cffcc9ae2e0e75546fb12a19213d4aeBen ChengEX: { lw r5, r3; addi r3, r3, 64 } 190e6e8a0bd7cffcc9ae2e0e75546fb12a19213d4aeBen Cheng { jal .Lcopy_line } 191e6e8a0bd7cffcc9ae2e0e75546fb12a19213d4aeBen Cheng 192e6e8a0bd7cffcc9ae2e0e75546fb12a19213d4aeBen Cheng /* Copy line 1, first stalling until r6 is ready. */ 193e6e8a0bd7cffcc9ae2e0e75546fb12a19213d4aeBen ChengEX: { move r12, r6; lw r16, r1 } 194e6e8a0bd7cffcc9ae2e0e75546fb12a19213d4aeBen Cheng { bz r4, .Lcopy_8_check; slti_u r8, r2, 8 } 195e6e8a0bd7cffcc9ae2e0e75546fb12a19213d4aeBen Cheng /* Prefetch several lines ahead. */ 196e6e8a0bd7cffcc9ae2e0e75546fb12a19213d4aeBen ChengEX: { lw r6, r3; addi r3, r3, 64 } 197e6e8a0bd7cffcc9ae2e0e75546fb12a19213d4aeBen Cheng { jal .Lcopy_line } 198e6e8a0bd7cffcc9ae2e0e75546fb12a19213d4aeBen Cheng 199e6e8a0bd7cffcc9ae2e0e75546fb12a19213d4aeBen Cheng /* Copy line 2, first stalling until r7 is ready. */ 200e6e8a0bd7cffcc9ae2e0e75546fb12a19213d4aeBen ChengEX: { move r12, r7; lw r16, r1 } 201e6e8a0bd7cffcc9ae2e0e75546fb12a19213d4aeBen Cheng { bz r4, .Lcopy_8_check; slti_u r8, r2, 8 } 202e6e8a0bd7cffcc9ae2e0e75546fb12a19213d4aeBen Cheng /* Prefetch several lines ahead. */ 203e6e8a0bd7cffcc9ae2e0e75546fb12a19213d4aeBen ChengEX: { lw r7, r3; addi r3, r3, 64 } 204e6e8a0bd7cffcc9ae2e0e75546fb12a19213d4aeBen Cheng /* Use up a caches-busy cycle by jumping back to the top of the 205e6e8a0bd7cffcc9ae2e0e75546fb12a19213d4aeBen Cheng * loop. Might as well get it out of the way now. 206e6e8a0bd7cffcc9ae2e0e75546fb12a19213d4aeBen Cheng */ 207e6e8a0bd7cffcc9ae2e0e75546fb12a19213d4aeBen Cheng { j .Lbig_loop } 208e6e8a0bd7cffcc9ae2e0e75546fb12a19213d4aeBen Cheng 209e6e8a0bd7cffcc9ae2e0e75546fb12a19213d4aeBen Cheng 210e6e8a0bd7cffcc9ae2e0e75546fb12a19213d4aeBen Cheng /* On entry: 211e6e8a0bd7cffcc9ae2e0e75546fb12a19213d4aeBen Cheng * - r0 points to the destination line. 212e6e8a0bd7cffcc9ae2e0e75546fb12a19213d4aeBen Cheng * - r1 points to the source line. 213e6e8a0bd7cffcc9ae2e0e75546fb12a19213d4aeBen Cheng * - r3 is the next prefetch address. 214e6e8a0bd7cffcc9ae2e0e75546fb12a19213d4aeBen Cheng * - r9 holds the last address used for wh64. 215e6e8a0bd7cffcc9ae2e0e75546fb12a19213d4aeBen Cheng * - r12 = WORD_15 216e6e8a0bd7cffcc9ae2e0e75546fb12a19213d4aeBen Cheng * - r16 = WORD_0. 217e6e8a0bd7cffcc9ae2e0e75546fb12a19213d4aeBen Cheng * - r17 == r1 + 16. 218e6e8a0bd7cffcc9ae2e0e75546fb12a19213d4aeBen Cheng * - r27 holds saved lr to restore. 219e6e8a0bd7cffcc9ae2e0e75546fb12a19213d4aeBen Cheng * 220e6e8a0bd7cffcc9ae2e0e75546fb12a19213d4aeBen Cheng * On exit: 221e6e8a0bd7cffcc9ae2e0e75546fb12a19213d4aeBen Cheng * - r0 is incremented by 64. 222e6e8a0bd7cffcc9ae2e0e75546fb12a19213d4aeBen Cheng * - r1 is incremented by 64, unless that would point to a word 223e6e8a0bd7cffcc9ae2e0e75546fb12a19213d4aeBen Cheng * beyond the end of the source array, in which case it is redirected 224e6e8a0bd7cffcc9ae2e0e75546fb12a19213d4aeBen Cheng * to point to an arbitrary word already in the cache. 225e6e8a0bd7cffcc9ae2e0e75546fb12a19213d4aeBen Cheng * - r2 is decremented by 64. 226e6e8a0bd7cffcc9ae2e0e75546fb12a19213d4aeBen Cheng * - r3 is unchanged, unless it points to a word beyond the 227e6e8a0bd7cffcc9ae2e0e75546fb12a19213d4aeBen Cheng * end of the source array, in which case it is redirected 228e6e8a0bd7cffcc9ae2e0e75546fb12a19213d4aeBen Cheng * to point to an arbitrary word already in the cache. 229e6e8a0bd7cffcc9ae2e0e75546fb12a19213d4aeBen Cheng * Redirecting is OK since if we are that close to the end 230e6e8a0bd7cffcc9ae2e0e75546fb12a19213d4aeBen Cheng * of the array we will not come back to this subroutine 231e6e8a0bd7cffcc9ae2e0e75546fb12a19213d4aeBen Cheng * and use the contents of the prefetched address. 232e6e8a0bd7cffcc9ae2e0e75546fb12a19213d4aeBen Cheng * - r4 is nonzero iff r2 >= 64. 233e6e8a0bd7cffcc9ae2e0e75546fb12a19213d4aeBen Cheng * - r9 is incremented by 64, unless it points beyond the 234e6e8a0bd7cffcc9ae2e0e75546fb12a19213d4aeBen Cheng * end of the last full destination cache line, in which 235e6e8a0bd7cffcc9ae2e0e75546fb12a19213d4aeBen Cheng * case it is redirected to a "safe address" that can be 236e6e8a0bd7cffcc9ae2e0e75546fb12a19213d4aeBen Cheng * clobbered (sp - 64) 237e6e8a0bd7cffcc9ae2e0e75546fb12a19213d4aeBen Cheng * - lr contains the value in r27. 238e6e8a0bd7cffcc9ae2e0e75546fb12a19213d4aeBen Cheng */ 239e6e8a0bd7cffcc9ae2e0e75546fb12a19213d4aeBen Cheng 240e6e8a0bd7cffcc9ae2e0e75546fb12a19213d4aeBen Cheng/* r26 unused */ 241e6e8a0bd7cffcc9ae2e0e75546fb12a19213d4aeBen Cheng 242e6e8a0bd7cffcc9ae2e0e75546fb12a19213d4aeBen Cheng.Lcopy_line: 243e6e8a0bd7cffcc9ae2e0e75546fb12a19213d4aeBen Cheng /* TODO: when r3 goes past the end, we would like to redirect it 244e6e8a0bd7cffcc9ae2e0e75546fb12a19213d4aeBen Cheng * to prefetch the last partial cache line (if any) just once, for the 245e6e8a0bd7cffcc9ae2e0e75546fb12a19213d4aeBen Cheng * benefit of the final cleanup loop. But we don't want to 246e6e8a0bd7cffcc9ae2e0e75546fb12a19213d4aeBen Cheng * prefetch that line more than once, or subsequent prefetches 247e6e8a0bd7cffcc9ae2e0e75546fb12a19213d4aeBen Cheng * will go into the RTF. But then .Lbig_loop should unconditionally 248e6e8a0bd7cffcc9ae2e0e75546fb12a19213d4aeBen Cheng * branch to top of loop to execute final prefetch, and its 249e6e8a0bd7cffcc9ae2e0e75546fb12a19213d4aeBen Cheng * nop should become a conditional branch. 250e6e8a0bd7cffcc9ae2e0e75546fb12a19213d4aeBen Cheng */ 251e6e8a0bd7cffcc9ae2e0e75546fb12a19213d4aeBen Cheng 252e6e8a0bd7cffcc9ae2e0e75546fb12a19213d4aeBen Cheng /* We need two non-memory cycles here to cover the resources 253e6e8a0bd7cffcc9ae2e0e75546fb12a19213d4aeBen Cheng * used by the loads initiated by the caller. 254e6e8a0bd7cffcc9ae2e0e75546fb12a19213d4aeBen Cheng */ 255e6e8a0bd7cffcc9ae2e0e75546fb12a19213d4aeBen Cheng { add r15, r1, r2 } 256e6e8a0bd7cffcc9ae2e0e75546fb12a19213d4aeBen Cheng.Lcopy_line2: 257e6e8a0bd7cffcc9ae2e0e75546fb12a19213d4aeBen Cheng { slt_u r13, r3, r15; addi r17, r1, 16 } 258e6e8a0bd7cffcc9ae2e0e75546fb12a19213d4aeBen Cheng 259e6e8a0bd7cffcc9ae2e0e75546fb12a19213d4aeBen Cheng /* NOTE: this will stall for one cycle as L1 is busy. */ 260e6e8a0bd7cffcc9ae2e0e75546fb12a19213d4aeBen Cheng 261e6e8a0bd7cffcc9ae2e0e75546fb12a19213d4aeBen Cheng /* Fill second L1D line. */ 262e6e8a0bd7cffcc9ae2e0e75546fb12a19213d4aeBen ChengEX: { lw r17, r17; addi r1, r1, 48; mvz r3, r13, r1 } /* r17 = WORD_4 */ 263e6e8a0bd7cffcc9ae2e0e75546fb12a19213d4aeBen Cheng 264e6e8a0bd7cffcc9ae2e0e75546fb12a19213d4aeBen Cheng /* Prepare destination line for writing. */ 265e6e8a0bd7cffcc9ae2e0e75546fb12a19213d4aeBen ChengEX: { wh64 r9; addi r9, r9, 64 } 266e6e8a0bd7cffcc9ae2e0e75546fb12a19213d4aeBen Cheng /* Load seven words that are L1D hits to cover wh64 L2 usage. */ 267e6e8a0bd7cffcc9ae2e0e75546fb12a19213d4aeBen Cheng 268e6e8a0bd7cffcc9ae2e0e75546fb12a19213d4aeBen Cheng /* Load the three remaining words from the last L1D line, which 269e6e8a0bd7cffcc9ae2e0e75546fb12a19213d4aeBen Cheng * we know has already filled the L1D. 270e6e8a0bd7cffcc9ae2e0e75546fb12a19213d4aeBen Cheng */ 271e6e8a0bd7cffcc9ae2e0e75546fb12a19213d4aeBen ChengEX: { lw r4, r1; addi r1, r1, 4; addi r20, r1, 16 } /* r4 = WORD_12 */ 272e6e8a0bd7cffcc9ae2e0e75546fb12a19213d4aeBen ChengEX: { lw r8, r1; addi r1, r1, 4; slt_u r13, r20, r15 }/* r8 = WORD_13 */ 273e6e8a0bd7cffcc9ae2e0e75546fb12a19213d4aeBen ChengEX: { lw r11, r1; addi r1, r1, -52; mvz r20, r13, r1 } /* r11 = WORD_14 */ 274e6e8a0bd7cffcc9ae2e0e75546fb12a19213d4aeBen Cheng 275e6e8a0bd7cffcc9ae2e0e75546fb12a19213d4aeBen Cheng /* Load the three remaining words from the first L1D line, first 276e6e8a0bd7cffcc9ae2e0e75546fb12a19213d4aeBen Cheng * stalling until it has filled by "looking at" r16. 277e6e8a0bd7cffcc9ae2e0e75546fb12a19213d4aeBen Cheng */ 278e6e8a0bd7cffcc9ae2e0e75546fb12a19213d4aeBen ChengEX: { lw r13, r1; addi r1, r1, 4; move zero, r16 } /* r13 = WORD_1 */ 279e6e8a0bd7cffcc9ae2e0e75546fb12a19213d4aeBen ChengEX: { lw r14, r1; addi r1, r1, 4 } /* r14 = WORD_2 */ 280e6e8a0bd7cffcc9ae2e0e75546fb12a19213d4aeBen ChengEX: { lw r15, r1; addi r1, r1, 8; addi r10, r0, 60 } /* r15 = WORD_3 */ 281e6e8a0bd7cffcc9ae2e0e75546fb12a19213d4aeBen Cheng 282e6e8a0bd7cffcc9ae2e0e75546fb12a19213d4aeBen Cheng /* Load second word from the second L1D line, first 283e6e8a0bd7cffcc9ae2e0e75546fb12a19213d4aeBen Cheng * stalling until it has filled by "looking at" r17. 284e6e8a0bd7cffcc9ae2e0e75546fb12a19213d4aeBen Cheng */ 285e6e8a0bd7cffcc9ae2e0e75546fb12a19213d4aeBen ChengEX: { lw r19, r1; addi r1, r1, 4; move zero, r17 } /* r19 = WORD_5 */ 286e6e8a0bd7cffcc9ae2e0e75546fb12a19213d4aeBen Cheng 287e6e8a0bd7cffcc9ae2e0e75546fb12a19213d4aeBen Cheng /* Store last word to the destination line, potentially dirtying it 288e6e8a0bd7cffcc9ae2e0e75546fb12a19213d4aeBen Cheng * for the first time, which keeps the L2 busy for two cycles. 289e6e8a0bd7cffcc9ae2e0e75546fb12a19213d4aeBen Cheng */ 290e6e8a0bd7cffcc9ae2e0e75546fb12a19213d4aeBen ChengEX: { sw r10, r12 } /* store(WORD_15) */ 291e6e8a0bd7cffcc9ae2e0e75546fb12a19213d4aeBen Cheng 292e6e8a0bd7cffcc9ae2e0e75546fb12a19213d4aeBen Cheng /* Use two L1D hits to cover the sw L2 access above. */ 293e6e8a0bd7cffcc9ae2e0e75546fb12a19213d4aeBen ChengEX: { lw r10, r1; addi r1, r1, 4 } /* r10 = WORD_6 */ 294e6e8a0bd7cffcc9ae2e0e75546fb12a19213d4aeBen ChengEX: { lw r12, r1; addi r1, r1, 4 } /* r12 = WORD_7 */ 295e6e8a0bd7cffcc9ae2e0e75546fb12a19213d4aeBen Cheng 296e6e8a0bd7cffcc9ae2e0e75546fb12a19213d4aeBen Cheng /* Fill third L1D line. */ 297e6e8a0bd7cffcc9ae2e0e75546fb12a19213d4aeBen ChengEX: { lw r18, r1; addi r1, r1, 4 } /* r18 = WORD_8 */ 298e6e8a0bd7cffcc9ae2e0e75546fb12a19213d4aeBen Cheng 299e6e8a0bd7cffcc9ae2e0e75546fb12a19213d4aeBen Cheng /* Store first L1D line. */ 300e6e8a0bd7cffcc9ae2e0e75546fb12a19213d4aeBen ChengEX: { sw r0, r16; addi r0, r0, 4; add r16, r0, r2 } /* store(WORD_0) */ 301e6e8a0bd7cffcc9ae2e0e75546fb12a19213d4aeBen ChengEX: { sw r0, r13; addi r0, r0, 4; andi r16, r16, -64 } /* store(WORD_1) */ 302e6e8a0bd7cffcc9ae2e0e75546fb12a19213d4aeBen ChengEX: { sw r0, r14; addi r0, r0, 4; slt_u r16, r9, r16 } /* store(WORD_2) */ 303e6e8a0bd7cffcc9ae2e0e75546fb12a19213d4aeBen ChengEX: { sw r0, r15; addi r0, r0, 4; addi r13, sp, -64 } /* store(WORD_3) */ 304e6e8a0bd7cffcc9ae2e0e75546fb12a19213d4aeBen Cheng /* Store second L1D line. */ 305e6e8a0bd7cffcc9ae2e0e75546fb12a19213d4aeBen ChengEX: { sw r0, r17; addi r0, r0, 4; mvz r9, r16, r13 }/* store(WORD_4) */ 306e6e8a0bd7cffcc9ae2e0e75546fb12a19213d4aeBen ChengEX: { sw r0, r19; addi r0, r0, 4 } /* store(WORD_5) */ 307e6e8a0bd7cffcc9ae2e0e75546fb12a19213d4aeBen ChengEX: { sw r0, r10; addi r0, r0, 4 } /* store(WORD_6) */ 308e6e8a0bd7cffcc9ae2e0e75546fb12a19213d4aeBen ChengEX: { sw r0, r12; addi r0, r0, 4 } /* store(WORD_7) */ 309e6e8a0bd7cffcc9ae2e0e75546fb12a19213d4aeBen Cheng 310e6e8a0bd7cffcc9ae2e0e75546fb12a19213d4aeBen ChengEX: { lw r13, r1; addi r1, r1, 4; move zero, r18 } /* r13 = WORD_9 */ 311e6e8a0bd7cffcc9ae2e0e75546fb12a19213d4aeBen ChengEX: { lw r14, r1; addi r1, r1, 4 } /* r14 = WORD_10 */ 312e6e8a0bd7cffcc9ae2e0e75546fb12a19213d4aeBen ChengEX: { lw r15, r1; move r1, r20 } /* r15 = WORD_11 */ 313e6e8a0bd7cffcc9ae2e0e75546fb12a19213d4aeBen Cheng 314e6e8a0bd7cffcc9ae2e0e75546fb12a19213d4aeBen Cheng /* Store third L1D line. */ 315e6e8a0bd7cffcc9ae2e0e75546fb12a19213d4aeBen ChengEX: { sw r0, r18; addi r0, r0, 4 } /* store(WORD_8) */ 316e6e8a0bd7cffcc9ae2e0e75546fb12a19213d4aeBen ChengEX: { sw r0, r13; addi r0, r0, 4 } /* store(WORD_9) */ 317e6e8a0bd7cffcc9ae2e0e75546fb12a19213d4aeBen ChengEX: { sw r0, r14; addi r0, r0, 4 } /* store(WORD_10) */ 318e6e8a0bd7cffcc9ae2e0e75546fb12a19213d4aeBen ChengEX: { sw r0, r15; addi r0, r0, 4 } /* store(WORD_11) */ 319e6e8a0bd7cffcc9ae2e0e75546fb12a19213d4aeBen Cheng 320e6e8a0bd7cffcc9ae2e0e75546fb12a19213d4aeBen Cheng /* Store rest of fourth L1D line. */ 321e6e8a0bd7cffcc9ae2e0e75546fb12a19213d4aeBen ChengEX: { sw r0, r4; addi r0, r0, 4 } /* store(WORD_12) */ 322e6e8a0bd7cffcc9ae2e0e75546fb12a19213d4aeBen Cheng { 323e6e8a0bd7cffcc9ae2e0e75546fb12a19213d4aeBen ChengEX: sw r0, r8 /* store(WORD_13) */ 324e6e8a0bd7cffcc9ae2e0e75546fb12a19213d4aeBen Cheng addi r0, r0, 4 325e6e8a0bd7cffcc9ae2e0e75546fb12a19213d4aeBen Cheng /* Will r2 be > 64 after we subtract 64 below? */ 326e6e8a0bd7cffcc9ae2e0e75546fb12a19213d4aeBen Cheng shri r4, r2, 7 327e6e8a0bd7cffcc9ae2e0e75546fb12a19213d4aeBen Cheng } 328e6e8a0bd7cffcc9ae2e0e75546fb12a19213d4aeBen Cheng { 329e6e8a0bd7cffcc9ae2e0e75546fb12a19213d4aeBen ChengEX: sw r0, r11 /* store(WORD_14) */ 330e6e8a0bd7cffcc9ae2e0e75546fb12a19213d4aeBen Cheng addi r0, r0, 8 331e6e8a0bd7cffcc9ae2e0e75546fb12a19213d4aeBen Cheng /* Record 64 bytes successfully copied. */ 332e6e8a0bd7cffcc9ae2e0e75546fb12a19213d4aeBen Cheng addi r2, r2, -64 333e6e8a0bd7cffcc9ae2e0e75546fb12a19213d4aeBen Cheng } 334e6e8a0bd7cffcc9ae2e0e75546fb12a19213d4aeBen Cheng 335e6e8a0bd7cffcc9ae2e0e75546fb12a19213d4aeBen Cheng { jrp lr; move lr, r27 } 336e6e8a0bd7cffcc9ae2e0e75546fb12a19213d4aeBen Cheng 337e6e8a0bd7cffcc9ae2e0e75546fb12a19213d4aeBen Cheng /* Convey to the backtrace library that the stack frame is size 338e6e8a0bd7cffcc9ae2e0e75546fb12a19213d4aeBen Cheng * zero, and the real return address is on the stack rather than 339e6e8a0bd7cffcc9ae2e0e75546fb12a19213d4aeBen Cheng * in 'lr'. 340e6e8a0bd7cffcc9ae2e0e75546fb12a19213d4aeBen Cheng */ 341e6e8a0bd7cffcc9ae2e0e75546fb12a19213d4aeBen Cheng { info 8 } 342e6e8a0bd7cffcc9ae2e0e75546fb12a19213d4aeBen Cheng 343e6e8a0bd7cffcc9ae2e0e75546fb12a19213d4aeBen Cheng .align 64 344e6e8a0bd7cffcc9ae2e0e75546fb12a19213d4aeBen Cheng.Lcopy_unaligned_maybe_many: 345e6e8a0bd7cffcc9ae2e0e75546fb12a19213d4aeBen Cheng /* Skip the setup overhead if we aren't copying many bytes. */ 346e6e8a0bd7cffcc9ae2e0e75546fb12a19213d4aeBen Cheng { slti_u r8, r2, 20; sub r4, zero, r0 } 347e6e8a0bd7cffcc9ae2e0e75546fb12a19213d4aeBen Cheng { bnzt r8, .Lcopy_unaligned_few; andi r4, r4, 3 } 348e6e8a0bd7cffcc9ae2e0e75546fb12a19213d4aeBen Cheng { bz r4, .Ldest_is_word_aligned; add r18, r1, r2 } 349e6e8a0bd7cffcc9ae2e0e75546fb12a19213d4aeBen Cheng 350e6e8a0bd7cffcc9ae2e0e75546fb12a19213d4aeBen Cheng/* 351e6e8a0bd7cffcc9ae2e0e75546fb12a19213d4aeBen Cheng * 352e6e8a0bd7cffcc9ae2e0e75546fb12a19213d4aeBen Cheng * unaligned 4 byte at a time copy handler. 353e6e8a0bd7cffcc9ae2e0e75546fb12a19213d4aeBen Cheng * 354e6e8a0bd7cffcc9ae2e0e75546fb12a19213d4aeBen Cheng */ 355e6e8a0bd7cffcc9ae2e0e75546fb12a19213d4aeBen Cheng 356e6e8a0bd7cffcc9ae2e0e75546fb12a19213d4aeBen Cheng /* Copy single bytes until r0 == 0 mod 4, so we can store words. */ 357e6e8a0bd7cffcc9ae2e0e75546fb12a19213d4aeBen Cheng.Lalign_dest_loop: 358e6e8a0bd7cffcc9ae2e0e75546fb12a19213d4aeBen ChengEX: { lb_u r3, r1; addi r1, r1, 1; addi r4, r4, -1 } 359e6e8a0bd7cffcc9ae2e0e75546fb12a19213d4aeBen ChengEX: { sb r0, r3; addi r0, r0, 1; addi r2, r2, -1 } 360e6e8a0bd7cffcc9ae2e0e75546fb12a19213d4aeBen Cheng { bnzt r4, .Lalign_dest_loop; andi r3, r1, 3 } 361e6e8a0bd7cffcc9ae2e0e75546fb12a19213d4aeBen Cheng 362e6e8a0bd7cffcc9ae2e0e75546fb12a19213d4aeBen Cheng /* If source and dest are now *both* aligned, do an aligned copy. */ 363e6e8a0bd7cffcc9ae2e0e75546fb12a19213d4aeBen Cheng { bz r3, .Lcheck_aligned_copy_size; addli r4, r2, -256 } 364e6e8a0bd7cffcc9ae2e0e75546fb12a19213d4aeBen Cheng 365e6e8a0bd7cffcc9ae2e0e75546fb12a19213d4aeBen Cheng.Ldest_is_word_aligned: 366e6e8a0bd7cffcc9ae2e0e75546fb12a19213d4aeBen Cheng 367e6e8a0bd7cffcc9ae2e0e75546fb12a19213d4aeBen ChengEX: { andi r8, r0, 63; lwadd_na r6, r1, 4} 368e6e8a0bd7cffcc9ae2e0e75546fb12a19213d4aeBen Cheng { slti_u r9, r2, 64; bz r8, .Ldest_is_L2_line_aligned } 369e6e8a0bd7cffcc9ae2e0e75546fb12a19213d4aeBen Cheng 370e6e8a0bd7cffcc9ae2e0e75546fb12a19213d4aeBen Cheng /* This copies unaligned words until either there are fewer 371e6e8a0bd7cffcc9ae2e0e75546fb12a19213d4aeBen Cheng * than 4 bytes left to copy, or until the destination pointer 372e6e8a0bd7cffcc9ae2e0e75546fb12a19213d4aeBen Cheng * is cache-aligned, whichever comes first. 373e6e8a0bd7cffcc9ae2e0e75546fb12a19213d4aeBen Cheng * 374e6e8a0bd7cffcc9ae2e0e75546fb12a19213d4aeBen Cheng * On entry: 375e6e8a0bd7cffcc9ae2e0e75546fb12a19213d4aeBen Cheng * - r0 is the next store address. 376e6e8a0bd7cffcc9ae2e0e75546fb12a19213d4aeBen Cheng * - r1 points 4 bytes past the load address corresponding to r0. 377e6e8a0bd7cffcc9ae2e0e75546fb12a19213d4aeBen Cheng * - r2 >= 4 378e6e8a0bd7cffcc9ae2e0e75546fb12a19213d4aeBen Cheng * - r6 is the next aligned word loaded. 379e6e8a0bd7cffcc9ae2e0e75546fb12a19213d4aeBen Cheng */ 380e6e8a0bd7cffcc9ae2e0e75546fb12a19213d4aeBen Cheng.Lcopy_unaligned_src_words: 381e6e8a0bd7cffcc9ae2e0e75546fb12a19213d4aeBen ChengEX: { lwadd_na r7, r1, 4; slti_u r8, r2, 4 + 4 } 382e6e8a0bd7cffcc9ae2e0e75546fb12a19213d4aeBen Cheng /* stall */ 383e6e8a0bd7cffcc9ae2e0e75546fb12a19213d4aeBen Cheng { dword_align r6, r7, r1; slti_u r9, r2, 64 + 4 } 384e6e8a0bd7cffcc9ae2e0e75546fb12a19213d4aeBen ChengEX: { swadd r0, r6, 4; addi r2, r2, -4 } 385e6e8a0bd7cffcc9ae2e0e75546fb12a19213d4aeBen Cheng { bnz r8, .Lcleanup_unaligned_words; andi r8, r0, 63 } 386e6e8a0bd7cffcc9ae2e0e75546fb12a19213d4aeBen Cheng { bnzt r8, .Lcopy_unaligned_src_words; move r6, r7 } 387e6e8a0bd7cffcc9ae2e0e75546fb12a19213d4aeBen Cheng 388e6e8a0bd7cffcc9ae2e0e75546fb12a19213d4aeBen Cheng /* On entry: 389e6e8a0bd7cffcc9ae2e0e75546fb12a19213d4aeBen Cheng * - r0 is the next store address. 390e6e8a0bd7cffcc9ae2e0e75546fb12a19213d4aeBen Cheng * - r1 points 4 bytes past the load address corresponding to r0. 391e6e8a0bd7cffcc9ae2e0e75546fb12a19213d4aeBen Cheng * - r2 >= 4 (# of bytes left to store). 392e6e8a0bd7cffcc9ae2e0e75546fb12a19213d4aeBen Cheng * - r6 is the next aligned src word value. 393e6e8a0bd7cffcc9ae2e0e75546fb12a19213d4aeBen Cheng * - r9 = (r2 < 64U). 394e6e8a0bd7cffcc9ae2e0e75546fb12a19213d4aeBen Cheng * - r18 points one byte past the end of source memory. 395e6e8a0bd7cffcc9ae2e0e75546fb12a19213d4aeBen Cheng */ 396e6e8a0bd7cffcc9ae2e0e75546fb12a19213d4aeBen Cheng.Ldest_is_L2_line_aligned: 397e6e8a0bd7cffcc9ae2e0e75546fb12a19213d4aeBen Cheng 398e6e8a0bd7cffcc9ae2e0e75546fb12a19213d4aeBen Cheng { 399e6e8a0bd7cffcc9ae2e0e75546fb12a19213d4aeBen Cheng /* Not a full cache line remains. */ 400e6e8a0bd7cffcc9ae2e0e75546fb12a19213d4aeBen Cheng bnz r9, .Lcleanup_unaligned_words 401e6e8a0bd7cffcc9ae2e0e75546fb12a19213d4aeBen Cheng move r7, r6 402e6e8a0bd7cffcc9ae2e0e75546fb12a19213d4aeBen Cheng } 403e6e8a0bd7cffcc9ae2e0e75546fb12a19213d4aeBen Cheng 404e6e8a0bd7cffcc9ae2e0e75546fb12a19213d4aeBen Cheng /* r2 >= 64 */ 405e6e8a0bd7cffcc9ae2e0e75546fb12a19213d4aeBen Cheng 406e6e8a0bd7cffcc9ae2e0e75546fb12a19213d4aeBen Cheng /* Kick off two prefetches, but don't go past the end. */ 407e6e8a0bd7cffcc9ae2e0e75546fb12a19213d4aeBen Cheng { addi r3, r1, 63 - 4; addi r8, r1, 64 + 63 - 4 } 408e6e8a0bd7cffcc9ae2e0e75546fb12a19213d4aeBen Cheng { prefetch r3; move r3, r8; slt_u r8, r8, r18 } 409e6e8a0bd7cffcc9ae2e0e75546fb12a19213d4aeBen Cheng { mvz r3, r8, r1; addi r8, r3, 64 } 410e6e8a0bd7cffcc9ae2e0e75546fb12a19213d4aeBen Cheng { prefetch r3; move r3, r8; slt_u r8, r8, r18 } 411e6e8a0bd7cffcc9ae2e0e75546fb12a19213d4aeBen Cheng { mvz r3, r8, r1; movei r17, 0 } 412e6e8a0bd7cffcc9ae2e0e75546fb12a19213d4aeBen Cheng 413e6e8a0bd7cffcc9ae2e0e75546fb12a19213d4aeBen Cheng.Lcopy_unaligned_line: 414e6e8a0bd7cffcc9ae2e0e75546fb12a19213d4aeBen Cheng /* Prefetch another line. */ 415e6e8a0bd7cffcc9ae2e0e75546fb12a19213d4aeBen Cheng { prefetch r3; addi r15, r1, 60; addi r3, r3, 64 } 416e6e8a0bd7cffcc9ae2e0e75546fb12a19213d4aeBen Cheng /* Fire off a load of the last word we are about to copy. */ 417e6e8a0bd7cffcc9ae2e0e75546fb12a19213d4aeBen ChengEX: { lw_na r15, r15; slt_u r8, r3, r18 } 418e6e8a0bd7cffcc9ae2e0e75546fb12a19213d4aeBen Cheng 419e6e8a0bd7cffcc9ae2e0e75546fb12a19213d4aeBen ChengEX: { mvz r3, r8, r1; wh64 r0 } 420e6e8a0bd7cffcc9ae2e0e75546fb12a19213d4aeBen Cheng 421e6e8a0bd7cffcc9ae2e0e75546fb12a19213d4aeBen Cheng /* This loop runs twice. 422e6e8a0bd7cffcc9ae2e0e75546fb12a19213d4aeBen Cheng * 423e6e8a0bd7cffcc9ae2e0e75546fb12a19213d4aeBen Cheng * On entry: 424e6e8a0bd7cffcc9ae2e0e75546fb12a19213d4aeBen Cheng * - r17 is even before the first iteration, and odd before 425e6e8a0bd7cffcc9ae2e0e75546fb12a19213d4aeBen Cheng * the second. It is incremented inside the loop. Encountering 426e6e8a0bd7cffcc9ae2e0e75546fb12a19213d4aeBen Cheng * an even value at the end of the loop makes it stop. 427e6e8a0bd7cffcc9ae2e0e75546fb12a19213d4aeBen Cheng */ 428e6e8a0bd7cffcc9ae2e0e75546fb12a19213d4aeBen Cheng.Lcopy_half_an_unaligned_line: 429e6e8a0bd7cffcc9ae2e0e75546fb12a19213d4aeBen ChengEX: { 430e6e8a0bd7cffcc9ae2e0e75546fb12a19213d4aeBen Cheng /* Stall until the last byte is ready. In the steady state this 431e6e8a0bd7cffcc9ae2e0e75546fb12a19213d4aeBen Cheng * guarantees all words to load below will be in the L2 cache, which 432e6e8a0bd7cffcc9ae2e0e75546fb12a19213d4aeBen Cheng * avoids shunting the loads to the RTF. 433e6e8a0bd7cffcc9ae2e0e75546fb12a19213d4aeBen Cheng */ 434e6e8a0bd7cffcc9ae2e0e75546fb12a19213d4aeBen Cheng move zero, r15 435e6e8a0bd7cffcc9ae2e0e75546fb12a19213d4aeBen Cheng lwadd_na r7, r1, 16 436e6e8a0bd7cffcc9ae2e0e75546fb12a19213d4aeBen Cheng } 437e6e8a0bd7cffcc9ae2e0e75546fb12a19213d4aeBen ChengEX: { lwadd_na r11, r1, 12 } 438e6e8a0bd7cffcc9ae2e0e75546fb12a19213d4aeBen ChengEX: { lwadd_na r14, r1, -24 } 439e6e8a0bd7cffcc9ae2e0e75546fb12a19213d4aeBen ChengEX: { lwadd_na r8, r1, 4 } 440e6e8a0bd7cffcc9ae2e0e75546fb12a19213d4aeBen ChengEX: { lwadd_na r9, r1, 4 } 441e6e8a0bd7cffcc9ae2e0e75546fb12a19213d4aeBen ChengEX: { 442e6e8a0bd7cffcc9ae2e0e75546fb12a19213d4aeBen Cheng lwadd_na r10, r1, 8 443e6e8a0bd7cffcc9ae2e0e75546fb12a19213d4aeBen Cheng /* r16 = (r2 < 64), after we subtract 32 from r2 below. */ 444e6e8a0bd7cffcc9ae2e0e75546fb12a19213d4aeBen Cheng slti_u r16, r2, 64 + 32 445e6e8a0bd7cffcc9ae2e0e75546fb12a19213d4aeBen Cheng } 446e6e8a0bd7cffcc9ae2e0e75546fb12a19213d4aeBen ChengEX: { lwadd_na r12, r1, 4; addi r17, r17, 1 } 447e6e8a0bd7cffcc9ae2e0e75546fb12a19213d4aeBen ChengEX: { lwadd_na r13, r1, 8; dword_align r6, r7, r1 } 448e6e8a0bd7cffcc9ae2e0e75546fb12a19213d4aeBen ChengEX: { swadd r0, r6, 4; dword_align r7, r8, r1 } 449e6e8a0bd7cffcc9ae2e0e75546fb12a19213d4aeBen ChengEX: { swadd r0, r7, 4; dword_align r8, r9, r1 } 450e6e8a0bd7cffcc9ae2e0e75546fb12a19213d4aeBen ChengEX: { swadd r0, r8, 4; dword_align r9, r10, r1 } 451e6e8a0bd7cffcc9ae2e0e75546fb12a19213d4aeBen ChengEX: { swadd r0, r9, 4; dword_align r10, r11, r1 } 452e6e8a0bd7cffcc9ae2e0e75546fb12a19213d4aeBen ChengEX: { swadd r0, r10, 4; dword_align r11, r12, r1 } 453e6e8a0bd7cffcc9ae2e0e75546fb12a19213d4aeBen ChengEX: { swadd r0, r11, 4; dword_align r12, r13, r1 } 454e6e8a0bd7cffcc9ae2e0e75546fb12a19213d4aeBen ChengEX: { swadd r0, r12, 4; dword_align r13, r14, r1 } 455e6e8a0bd7cffcc9ae2e0e75546fb12a19213d4aeBen ChengEX: { swadd r0, r13, 4; addi r2, r2, -32 } 456e6e8a0bd7cffcc9ae2e0e75546fb12a19213d4aeBen Cheng { move r6, r14; bbst r17, .Lcopy_half_an_unaligned_line } 457e6e8a0bd7cffcc9ae2e0e75546fb12a19213d4aeBen Cheng 458e6e8a0bd7cffcc9ae2e0e75546fb12a19213d4aeBen Cheng { bzt r16, .Lcopy_unaligned_line; move r7, r6 } 459e6e8a0bd7cffcc9ae2e0e75546fb12a19213d4aeBen Cheng 460e6e8a0bd7cffcc9ae2e0e75546fb12a19213d4aeBen Cheng /* On entry: 461e6e8a0bd7cffcc9ae2e0e75546fb12a19213d4aeBen Cheng * - r0 is the next store address. 462e6e8a0bd7cffcc9ae2e0e75546fb12a19213d4aeBen Cheng * - r1 points 4 bytes past the load address corresponding to r0. 463e6e8a0bd7cffcc9ae2e0e75546fb12a19213d4aeBen Cheng * - r2 >= 0 (# of bytes left to store). 464e6e8a0bd7cffcc9ae2e0e75546fb12a19213d4aeBen Cheng * - r7 is the next aligned src word value. 465e6e8a0bd7cffcc9ae2e0e75546fb12a19213d4aeBen Cheng */ 466e6e8a0bd7cffcc9ae2e0e75546fb12a19213d4aeBen Cheng.Lcleanup_unaligned_words: 467e6e8a0bd7cffcc9ae2e0e75546fb12a19213d4aeBen Cheng /* Handle any trailing bytes. */ 468e6e8a0bd7cffcc9ae2e0e75546fb12a19213d4aeBen Cheng { bz r2, .Lcopy_unaligned_done; slti_u r8, r2, 4 } 469e6e8a0bd7cffcc9ae2e0e75546fb12a19213d4aeBen Cheng { bzt r8, .Lcopy_unaligned_src_words; move r6, r7 } 470e6e8a0bd7cffcc9ae2e0e75546fb12a19213d4aeBen Cheng 471e6e8a0bd7cffcc9ae2e0e75546fb12a19213d4aeBen Cheng /* Move r1 back to the point where it corresponds to r0. */ 472e6e8a0bd7cffcc9ae2e0e75546fb12a19213d4aeBen Cheng { addi r1, r1, -4 } 473e6e8a0bd7cffcc9ae2e0e75546fb12a19213d4aeBen Cheng 474e6e8a0bd7cffcc9ae2e0e75546fb12a19213d4aeBen Cheng /* Fall through */ 475e6e8a0bd7cffcc9ae2e0e75546fb12a19213d4aeBen Cheng 476e6e8a0bd7cffcc9ae2e0e75546fb12a19213d4aeBen Cheng/* 477e6e8a0bd7cffcc9ae2e0e75546fb12a19213d4aeBen Cheng * 478e6e8a0bd7cffcc9ae2e0e75546fb12a19213d4aeBen Cheng * 1 byte at a time copy handler. 479e6e8a0bd7cffcc9ae2e0e75546fb12a19213d4aeBen Cheng * 480e6e8a0bd7cffcc9ae2e0e75546fb12a19213d4aeBen Cheng */ 481e6e8a0bd7cffcc9ae2e0e75546fb12a19213d4aeBen Cheng 482e6e8a0bd7cffcc9ae2e0e75546fb12a19213d4aeBen Cheng.Lcopy_unaligned_few: 483e6e8a0bd7cffcc9ae2e0e75546fb12a19213d4aeBen ChengEX: { lb_u r3, r1; addi r1, r1, 1 } 484e6e8a0bd7cffcc9ae2e0e75546fb12a19213d4aeBen ChengEX: { sb r0, r3; addi r0, r0, 1; addi r2, r2, -1 } 485e6e8a0bd7cffcc9ae2e0e75546fb12a19213d4aeBen Cheng { bnzt r2, .Lcopy_unaligned_few } 486e6e8a0bd7cffcc9ae2e0e75546fb12a19213d4aeBen Cheng 487e6e8a0bd7cffcc9ae2e0e75546fb12a19213d4aeBen Cheng.Lcopy_unaligned_done: 488e6e8a0bd7cffcc9ae2e0e75546fb12a19213d4aeBen Cheng 489e6e8a0bd7cffcc9ae2e0e75546fb12a19213d4aeBen Cheng /* For memcpy return original dest address, else zero. */ 490e6e8a0bd7cffcc9ae2e0e75546fb12a19213d4aeBen Cheng { mz r0, r29, r23; jrp lr } 491e6e8a0bd7cffcc9ae2e0e75546fb12a19213d4aeBen Cheng 492e6e8a0bd7cffcc9ae2e0e75546fb12a19213d4aeBen Cheng.Lend_memcpy_common: 493e6e8a0bd7cffcc9ae2e0e75546fb12a19213d4aeBen Cheng .size memcpy_common, .Lend_memcpy_common - memcpy_common 494e6e8a0bd7cffcc9ae2e0e75546fb12a19213d4aeBen Cheng 495e6e8a0bd7cffcc9ae2e0e75546fb12a19213d4aeBen Cheng .section .fixup,"ax" 496e6e8a0bd7cffcc9ae2e0e75546fb12a19213d4aeBen Chengmemcpy_common_fixup: 497e6e8a0bd7cffcc9ae2e0e75546fb12a19213d4aeBen Cheng .type memcpy_common_fixup, @function 498e6e8a0bd7cffcc9ae2e0e75546fb12a19213d4aeBen Cheng 499e6e8a0bd7cffcc9ae2e0e75546fb12a19213d4aeBen Cheng /* Skip any bytes we already successfully copied. 500e6e8a0bd7cffcc9ae2e0e75546fb12a19213d4aeBen Cheng * r2 (num remaining) is correct, but r0 (dst) and r1 (src) 501e6e8a0bd7cffcc9ae2e0e75546fb12a19213d4aeBen Cheng * may not be quite right because of unrolling and prefetching. 502e6e8a0bd7cffcc9ae2e0e75546fb12a19213d4aeBen Cheng * So we need to recompute their values as the address just 503e6e8a0bd7cffcc9ae2e0e75546fb12a19213d4aeBen Cheng * after the last byte we are sure was successfully loaded and 504e6e8a0bd7cffcc9ae2e0e75546fb12a19213d4aeBen Cheng * then stored. 505e6e8a0bd7cffcc9ae2e0e75546fb12a19213d4aeBen Cheng */ 506e6e8a0bd7cffcc9ae2e0e75546fb12a19213d4aeBen Cheng 507e6e8a0bd7cffcc9ae2e0e75546fb12a19213d4aeBen Cheng /* Determine how many bytes we successfully copied. */ 508e6e8a0bd7cffcc9ae2e0e75546fb12a19213d4aeBen Cheng { sub r3, r25, r2 } 509e6e8a0bd7cffcc9ae2e0e75546fb12a19213d4aeBen Cheng 510e6e8a0bd7cffcc9ae2e0e75546fb12a19213d4aeBen Cheng /* Add this to the original r0 and r1 to get their new values. */ 511e6e8a0bd7cffcc9ae2e0e75546fb12a19213d4aeBen Cheng { add r0, r23, r3; add r1, r24, r3 } 512e6e8a0bd7cffcc9ae2e0e75546fb12a19213d4aeBen Cheng 513e6e8a0bd7cffcc9ae2e0e75546fb12a19213d4aeBen Cheng { bzt r29, memcpy_fixup_loop } 514e6e8a0bd7cffcc9ae2e0e75546fb12a19213d4aeBen Cheng { blzt r29, copy_to_user_fixup_loop } 515e6e8a0bd7cffcc9ae2e0e75546fb12a19213d4aeBen Cheng 516e6e8a0bd7cffcc9ae2e0e75546fb12a19213d4aeBen Chengcopy_from_user_fixup_loop: 517e6e8a0bd7cffcc9ae2e0e75546fb12a19213d4aeBen Cheng /* Try copying the rest one byte at a time, expecting a load fault. */ 518e6e8a0bd7cffcc9ae2e0e75546fb12a19213d4aeBen Cheng.Lcfu: { lb_u r3, r1; addi r1, r1, 1 } 519e6e8a0bd7cffcc9ae2e0e75546fb12a19213d4aeBen Cheng { sb r0, r3; addi r0, r0, 1; addi r2, r2, -1 } 520e6e8a0bd7cffcc9ae2e0e75546fb12a19213d4aeBen Cheng { bnzt r2, copy_from_user_fixup_loop } 521e6e8a0bd7cffcc9ae2e0e75546fb12a19213d4aeBen Cheng 522e6e8a0bd7cffcc9ae2e0e75546fb12a19213d4aeBen Cheng.Lcopy_from_user_fixup_zero_remainder: 523e6e8a0bd7cffcc9ae2e0e75546fb12a19213d4aeBen Cheng { bbs r29, 2f } /* low bit set means IS_COPY_FROM_USER */ 524e6e8a0bd7cffcc9ae2e0e75546fb12a19213d4aeBen Cheng /* byte-at-a-time loop faulted, so zero the rest. */ 525e6e8a0bd7cffcc9ae2e0e75546fb12a19213d4aeBen Cheng { move r3, r2; bz r2, 2f /* should be impossible, but handle it. */ } 526e6e8a0bd7cffcc9ae2e0e75546fb12a19213d4aeBen Cheng1: { sb r0, zero; addi r0, r0, 1; addi r3, r3, -1 } 527e6e8a0bd7cffcc9ae2e0e75546fb12a19213d4aeBen Cheng { bnzt r3, 1b } 528e6e8a0bd7cffcc9ae2e0e75546fb12a19213d4aeBen Cheng2: move lr, r27 529e6e8a0bd7cffcc9ae2e0e75546fb12a19213d4aeBen Cheng { move r0, r2; jrp lr } 530e6e8a0bd7cffcc9ae2e0e75546fb12a19213d4aeBen Cheng 531e6e8a0bd7cffcc9ae2e0e75546fb12a19213d4aeBen Chengcopy_to_user_fixup_loop: 532e6e8a0bd7cffcc9ae2e0e75546fb12a19213d4aeBen Cheng /* Try copying the rest one byte at a time, expecting a store fault. */ 533e6e8a0bd7cffcc9ae2e0e75546fb12a19213d4aeBen Cheng { lb_u r3, r1; addi r1, r1, 1 } 534e6e8a0bd7cffcc9ae2e0e75546fb12a19213d4aeBen Cheng.Lctu: { sb r0, r3; addi r0, r0, 1; addi r2, r2, -1 } 535e6e8a0bd7cffcc9ae2e0e75546fb12a19213d4aeBen Cheng { bnzt r2, copy_to_user_fixup_loop } 536e6e8a0bd7cffcc9ae2e0e75546fb12a19213d4aeBen Cheng.Lcopy_to_user_fixup_done: 537e6e8a0bd7cffcc9ae2e0e75546fb12a19213d4aeBen Cheng move lr, r27 538e6e8a0bd7cffcc9ae2e0e75546fb12a19213d4aeBen Cheng { move r0, r2; jrp lr } 539e6e8a0bd7cffcc9ae2e0e75546fb12a19213d4aeBen Cheng 540e6e8a0bd7cffcc9ae2e0e75546fb12a19213d4aeBen Chengmemcpy_fixup_loop: 541e6e8a0bd7cffcc9ae2e0e75546fb12a19213d4aeBen Cheng /* Try copying the rest one byte at a time. We expect a disastrous 542e6e8a0bd7cffcc9ae2e0e75546fb12a19213d4aeBen Cheng * fault to happen since we are in fixup code, but let it happen. 543e6e8a0bd7cffcc9ae2e0e75546fb12a19213d4aeBen Cheng */ 544e6e8a0bd7cffcc9ae2e0e75546fb12a19213d4aeBen Cheng { lb_u r3, r1; addi r1, r1, 1 } 545e6e8a0bd7cffcc9ae2e0e75546fb12a19213d4aeBen Cheng { sb r0, r3; addi r0, r0, 1; addi r2, r2, -1 } 546e6e8a0bd7cffcc9ae2e0e75546fb12a19213d4aeBen Cheng { bnzt r2, memcpy_fixup_loop } 547e6e8a0bd7cffcc9ae2e0e75546fb12a19213d4aeBen Cheng /* This should be unreachable, we should have faulted again. 548e6e8a0bd7cffcc9ae2e0e75546fb12a19213d4aeBen Cheng * But be paranoid and handle it in case some interrupt changed 549e6e8a0bd7cffcc9ae2e0e75546fb12a19213d4aeBen Cheng * the TLB or something. 550e6e8a0bd7cffcc9ae2e0e75546fb12a19213d4aeBen Cheng */ 551e6e8a0bd7cffcc9ae2e0e75546fb12a19213d4aeBen Cheng move lr, r27 552e6e8a0bd7cffcc9ae2e0e75546fb12a19213d4aeBen Cheng { move r0, r23; jrp lr } 553e6e8a0bd7cffcc9ae2e0e75546fb12a19213d4aeBen Cheng 554e6e8a0bd7cffcc9ae2e0e75546fb12a19213d4aeBen Cheng .size memcpy_common_fixup, . - memcpy_common_fixup 555e6e8a0bd7cffcc9ae2e0e75546fb12a19213d4aeBen Cheng 556e6e8a0bd7cffcc9ae2e0e75546fb12a19213d4aeBen Cheng .section __ex_table,"a" 557e6e8a0bd7cffcc9ae2e0e75546fb12a19213d4aeBen Cheng .align 4 558e6e8a0bd7cffcc9ae2e0e75546fb12a19213d4aeBen Cheng .word .Lcfu, .Lcopy_from_user_fixup_zero_remainder 559e6e8a0bd7cffcc9ae2e0e75546fb12a19213d4aeBen Cheng .word .Lctu, .Lcopy_to_user_fixup_done 560