151533b615e605d86154ec1b4e585c8ca1b0b15b7Mikael Starvik/* 251533b615e605d86154ec1b4e585c8ca1b0b15b7Mikael Starvik * User address space access functions. 351533b615e605d86154ec1b4e585c8ca1b0b15b7Mikael Starvik * The non-inlined parts of asm-cris/uaccess.h are here. 451533b615e605d86154ec1b4e585c8ca1b0b15b7Mikael Starvik * 551533b615e605d86154ec1b4e585c8ca1b0b15b7Mikael Starvik * Copyright (C) 2000, 2003 Axis Communications AB. 651533b615e605d86154ec1b4e585c8ca1b0b15b7Mikael Starvik * 751533b615e605d86154ec1b4e585c8ca1b0b15b7Mikael Starvik * Written by Hans-Peter Nilsson. 851533b615e605d86154ec1b4e585c8ca1b0b15b7Mikael Starvik * Pieces used from memcpy, originally by Kenny Ranerup long time ago. 951533b615e605d86154ec1b4e585c8ca1b0b15b7Mikael Starvik */ 1051533b615e605d86154ec1b4e585c8ca1b0b15b7Mikael Starvik 1151533b615e605d86154ec1b4e585c8ca1b0b15b7Mikael Starvik#include <asm/uaccess.h> 1251533b615e605d86154ec1b4e585c8ca1b0b15b7Mikael Starvik 1351533b615e605d86154ec1b4e585c8ca1b0b15b7Mikael Starvik/* Asm:s have been tweaked (within the domain of correctness) to give 1451533b615e605d86154ec1b4e585c8ca1b0b15b7Mikael Starvik satisfactory results for "gcc version 3.2.1 Axis release R53/1.53-v32". 1551533b615e605d86154ec1b4e585c8ca1b0b15b7Mikael Starvik 1651533b615e605d86154ec1b4e585c8ca1b0b15b7Mikael Starvik Check regularly... 1751533b615e605d86154ec1b4e585c8ca1b0b15b7Mikael Starvik 1851533b615e605d86154ec1b4e585c8ca1b0b15b7Mikael Starvik Note that for CRISv32, the PC saved at a bus-fault is the address 1951533b615e605d86154ec1b4e585c8ca1b0b15b7Mikael Starvik *at* the faulting instruction, with a special case for instructions 2051533b615e605d86154ec1b4e585c8ca1b0b15b7Mikael Starvik in delay slots: then it's the address of the branch. Note also that 2151533b615e605d86154ec1b4e585c8ca1b0b15b7Mikael Starvik in contrast to v10, a postincrement in the instruction is *not* 2251533b615e605d86154ec1b4e585c8ca1b0b15b7Mikael Starvik performed at a bus-fault; the register is seen having the original 2351533b615e605d86154ec1b4e585c8ca1b0b15b7Mikael Starvik value in fault handlers. */ 2451533b615e605d86154ec1b4e585c8ca1b0b15b7Mikael Starvik 2551533b615e605d86154ec1b4e585c8ca1b0b15b7Mikael Starvik 2651533b615e605d86154ec1b4e585c8ca1b0b15b7Mikael Starvik/* Copy to userspace. This is based on the memcpy used for 2751533b615e605d86154ec1b4e585c8ca1b0b15b7Mikael Starvik kernel-to-kernel copying; see "string.c". */ 2851533b615e605d86154ec1b4e585c8ca1b0b15b7Mikael Starvik 2951533b615e605d86154ec1b4e585c8ca1b0b15b7Mikael Starvikunsigned long 3051533b615e605d86154ec1b4e585c8ca1b0b15b7Mikael Starvik__copy_user (void __user *pdst, const void *psrc, unsigned long pn) 3151533b615e605d86154ec1b4e585c8ca1b0b15b7Mikael Starvik{ 3251533b615e605d86154ec1b4e585c8ca1b0b15b7Mikael Starvik /* We want the parameters put in special registers. 3351533b615e605d86154ec1b4e585c8ca1b0b15b7Mikael Starvik Make sure the compiler is able to make something useful of this. 3451533b615e605d86154ec1b4e585c8ca1b0b15b7Mikael Starvik As it is now: r10 -> r13; r11 -> r11 (nop); r12 -> r12 (nop). 3551533b615e605d86154ec1b4e585c8ca1b0b15b7Mikael Starvik 3651533b615e605d86154ec1b4e585c8ca1b0b15b7Mikael Starvik FIXME: Comment for old gcc version. Check. 3749b4ff3304b52b18c490fc4deb400b61bb7ed142Simon Arlott If gcc was alright, it really would need no temporaries, and no 3851533b615e605d86154ec1b4e585c8ca1b0b15b7Mikael Starvik stack space to save stuff on. */ 3951533b615e605d86154ec1b4e585c8ca1b0b15b7Mikael Starvik 4051533b615e605d86154ec1b4e585c8ca1b0b15b7Mikael Starvik register char *dst __asm__ ("r13") = pdst; 4151533b615e605d86154ec1b4e585c8ca1b0b15b7Mikael Starvik register const char *src __asm__ ("r11") = psrc; 4251533b615e605d86154ec1b4e585c8ca1b0b15b7Mikael Starvik register int n __asm__ ("r12") = pn; 4351533b615e605d86154ec1b4e585c8ca1b0b15b7Mikael Starvik register int retn __asm__ ("r10") = 0; 4451533b615e605d86154ec1b4e585c8ca1b0b15b7Mikael Starvik 4551533b615e605d86154ec1b4e585c8ca1b0b15b7Mikael Starvik 4651533b615e605d86154ec1b4e585c8ca1b0b15b7Mikael Starvik /* When src is aligned but not dst, this makes a few extra needless 4751533b615e605d86154ec1b4e585c8ca1b0b15b7Mikael Starvik cycles. I believe it would take as many to check that the 4851533b615e605d86154ec1b4e585c8ca1b0b15b7Mikael Starvik re-alignment was unnecessary. */ 4951533b615e605d86154ec1b4e585c8ca1b0b15b7Mikael Starvik if (((unsigned long) dst & 3) != 0 5051533b615e605d86154ec1b4e585c8ca1b0b15b7Mikael Starvik /* Don't align if we wouldn't copy more than a few bytes; so we 5151533b615e605d86154ec1b4e585c8ca1b0b15b7Mikael Starvik don't have to check further for overflows. */ 5251533b615e605d86154ec1b4e585c8ca1b0b15b7Mikael Starvik && n >= 3) 5351533b615e605d86154ec1b4e585c8ca1b0b15b7Mikael Starvik { 5451533b615e605d86154ec1b4e585c8ca1b0b15b7Mikael Starvik if ((unsigned long) dst & 1) 5551533b615e605d86154ec1b4e585c8ca1b0b15b7Mikael Starvik { 5651533b615e605d86154ec1b4e585c8ca1b0b15b7Mikael Starvik __asm_copy_to_user_1 (dst, src, retn); 5751533b615e605d86154ec1b4e585c8ca1b0b15b7Mikael Starvik n--; 5851533b615e605d86154ec1b4e585c8ca1b0b15b7Mikael Starvik } 5951533b615e605d86154ec1b4e585c8ca1b0b15b7Mikael Starvik 6051533b615e605d86154ec1b4e585c8ca1b0b15b7Mikael Starvik if ((unsigned long) dst & 2) 6151533b615e605d86154ec1b4e585c8ca1b0b15b7Mikael Starvik { 6251533b615e605d86154ec1b4e585c8ca1b0b15b7Mikael Starvik __asm_copy_to_user_2 (dst, src, retn); 6351533b615e605d86154ec1b4e585c8ca1b0b15b7Mikael Starvik n -= 2; 6451533b615e605d86154ec1b4e585c8ca1b0b15b7Mikael Starvik } 6551533b615e605d86154ec1b4e585c8ca1b0b15b7Mikael Starvik } 6651533b615e605d86154ec1b4e585c8ca1b0b15b7Mikael Starvik 6751533b615e605d86154ec1b4e585c8ca1b0b15b7Mikael Starvik /* Movem is dirt cheap. The overheap is low enough to always use the 6851533b615e605d86154ec1b4e585c8ca1b0b15b7Mikael Starvik minimum possible block size as the threshold. */ 6951533b615e605d86154ec1b4e585c8ca1b0b15b7Mikael Starvik if (n >= 44) 7051533b615e605d86154ec1b4e585c8ca1b0b15b7Mikael Starvik { 7151533b615e605d86154ec1b4e585c8ca1b0b15b7Mikael Starvik /* For large copies we use 'movem'. */ 7251533b615e605d86154ec1b4e585c8ca1b0b15b7Mikael Starvik 7351533b615e605d86154ec1b4e585c8ca1b0b15b7Mikael Starvik /* It is not optimal to tell the compiler about clobbering any 7451533b615e605d86154ec1b4e585c8ca1b0b15b7Mikael Starvik registers; that will move the saving/restoring of those registers 7551533b615e605d86154ec1b4e585c8ca1b0b15b7Mikael Starvik to the function prologue/epilogue, and make non-movem sizes 7651533b615e605d86154ec1b4e585c8ca1b0b15b7Mikael Starvik suboptimal. */ 7751533b615e605d86154ec1b4e585c8ca1b0b15b7Mikael Starvik __asm__ volatile ("\ 7851533b615e605d86154ec1b4e585c8ca1b0b15b7Mikael Starvik ;; Check that the register asm declaration got right. \n\ 7951533b615e605d86154ec1b4e585c8ca1b0b15b7Mikael Starvik ;; The GCC manual explicitly says TRT will happen. \n\ 8051533b615e605d86154ec1b4e585c8ca1b0b15b7Mikael Starvik .ifnc %0%1%2%3,$r13$r11$r12$r10 \n\ 8151533b615e605d86154ec1b4e585c8ca1b0b15b7Mikael Starvik .err \n\ 8251533b615e605d86154ec1b4e585c8ca1b0b15b7Mikael Starvik .endif \n\ 8351533b615e605d86154ec1b4e585c8ca1b0b15b7Mikael Starvik \n\ 8451533b615e605d86154ec1b4e585c8ca1b0b15b7Mikael Starvik ;; Save the registers we'll use in the movem process \n\ 8551533b615e605d86154ec1b4e585c8ca1b0b15b7Mikael Starvik ;; on the stack. \n\ 8651533b615e605d86154ec1b4e585c8ca1b0b15b7Mikael Starvik subq 11*4,$sp \n\ 8751533b615e605d86154ec1b4e585c8ca1b0b15b7Mikael Starvik movem $r10,[$sp] \n\ 8851533b615e605d86154ec1b4e585c8ca1b0b15b7Mikael Starvik \n\ 8951533b615e605d86154ec1b4e585c8ca1b0b15b7Mikael Starvik ;; Now we've got this: \n\ 9051533b615e605d86154ec1b4e585c8ca1b0b15b7Mikael Starvik ;; r11 - src \n\ 9151533b615e605d86154ec1b4e585c8ca1b0b15b7Mikael Starvik ;; r13 - dst \n\ 9251533b615e605d86154ec1b4e585c8ca1b0b15b7Mikael Starvik ;; r12 - n \n\ 9351533b615e605d86154ec1b4e585c8ca1b0b15b7Mikael Starvik \n\ 9451533b615e605d86154ec1b4e585c8ca1b0b15b7Mikael Starvik ;; Update n for the first loop \n\ 9551533b615e605d86154ec1b4e585c8ca1b0b15b7Mikael Starvik subq 44,$r12 \n\ 9651533b615e605d86154ec1b4e585c8ca1b0b15b7Mikael Starvik0: \n\ 9751533b615e605d86154ec1b4e585c8ca1b0b15b7Mikael Starvik movem [$r11+],$r10 \n\ 9851533b615e605d86154ec1b4e585c8ca1b0b15b7Mikael Starvik subq 44,$r12 \n\ 9951533b615e605d86154ec1b4e585c8ca1b0b15b7Mikael Starvik1: bge 0b \n\ 10051533b615e605d86154ec1b4e585c8ca1b0b15b7Mikael Starvik movem $r10,[$r13+] \n\ 10151533b615e605d86154ec1b4e585c8ca1b0b15b7Mikael Starvik3: \n\ 10251533b615e605d86154ec1b4e585c8ca1b0b15b7Mikael Starvik addq 44,$r12 ;; compensate for last loop underflowing n \n\ 10351533b615e605d86154ec1b4e585c8ca1b0b15b7Mikael Starvik \n\ 10451533b615e605d86154ec1b4e585c8ca1b0b15b7Mikael Starvik ;; Restore registers from stack \n\ 10551533b615e605d86154ec1b4e585c8ca1b0b15b7Mikael Starvik movem [$sp+],$r10 \n\ 10651533b615e605d86154ec1b4e585c8ca1b0b15b7Mikael Starvik2: \n\ 10751533b615e605d86154ec1b4e585c8ca1b0b15b7Mikael Starvik .section .fixup,\"ax\" \n\ 10851533b615e605d86154ec1b4e585c8ca1b0b15b7Mikael Starvik4: \n\ 10951533b615e605d86154ec1b4e585c8ca1b0b15b7Mikael Starvik; When failing on any of the 1..44 bytes in a chunk, we adjust back the \n\ 11051533b615e605d86154ec1b4e585c8ca1b0b15b7Mikael Starvik; source pointer and just drop through to the by-16 and by-4 loops to \n\ 11151533b615e605d86154ec1b4e585c8ca1b0b15b7Mikael Starvik; get the correct number of failing bytes. This necessarily means a \n\ 11251533b615e605d86154ec1b4e585c8ca1b0b15b7Mikael Starvik; few extra exceptions, but invalid user pointers shouldn't happen in \n\ 11351533b615e605d86154ec1b4e585c8ca1b0b15b7Mikael Starvik; time-critical code anyway. \n\ 11451533b615e605d86154ec1b4e585c8ca1b0b15b7Mikael Starvik jump 3b \n\ 11551533b615e605d86154ec1b4e585c8ca1b0b15b7Mikael Starvik subq 44,$r11 \n\ 11651533b615e605d86154ec1b4e585c8ca1b0b15b7Mikael Starvik \n\ 11751533b615e605d86154ec1b4e585c8ca1b0b15b7Mikael Starvik .previous \n\ 11851533b615e605d86154ec1b4e585c8ca1b0b15b7Mikael Starvik .section __ex_table,\"a\" \n\ 11951533b615e605d86154ec1b4e585c8ca1b0b15b7Mikael Starvik .dword 1b,4b \n\ 12051533b615e605d86154ec1b4e585c8ca1b0b15b7Mikael Starvik .previous" 12151533b615e605d86154ec1b4e585c8ca1b0b15b7Mikael Starvik 12251533b615e605d86154ec1b4e585c8ca1b0b15b7Mikael Starvik /* Outputs */ : "=r" (dst), "=r" (src), "=r" (n), "=r" (retn) 12351533b615e605d86154ec1b4e585c8ca1b0b15b7Mikael Starvik /* Inputs */ : "0" (dst), "1" (src), "2" (n), "3" (retn)); 12451533b615e605d86154ec1b4e585c8ca1b0b15b7Mikael Starvik 12551533b615e605d86154ec1b4e585c8ca1b0b15b7Mikael Starvik } 12651533b615e605d86154ec1b4e585c8ca1b0b15b7Mikael Starvik 12751533b615e605d86154ec1b4e585c8ca1b0b15b7Mikael Starvik while (n >= 16) 12851533b615e605d86154ec1b4e585c8ca1b0b15b7Mikael Starvik { 12951533b615e605d86154ec1b4e585c8ca1b0b15b7Mikael Starvik __asm_copy_to_user_16 (dst, src, retn); 13051533b615e605d86154ec1b4e585c8ca1b0b15b7Mikael Starvik n -= 16; 13151533b615e605d86154ec1b4e585c8ca1b0b15b7Mikael Starvik } 13251533b615e605d86154ec1b4e585c8ca1b0b15b7Mikael Starvik 13351533b615e605d86154ec1b4e585c8ca1b0b15b7Mikael Starvik /* Having a separate by-four loops cuts down on cache footprint. 13451533b615e605d86154ec1b4e585c8ca1b0b15b7Mikael Starvik FIXME: Test with and without; increasing switch to be 0..15. */ 13551533b615e605d86154ec1b4e585c8ca1b0b15b7Mikael Starvik while (n >= 4) 13651533b615e605d86154ec1b4e585c8ca1b0b15b7Mikael Starvik { 13751533b615e605d86154ec1b4e585c8ca1b0b15b7Mikael Starvik __asm_copy_to_user_4 (dst, src, retn); 13851533b615e605d86154ec1b4e585c8ca1b0b15b7Mikael Starvik n -= 4; 13951533b615e605d86154ec1b4e585c8ca1b0b15b7Mikael Starvik } 14051533b615e605d86154ec1b4e585c8ca1b0b15b7Mikael Starvik 14151533b615e605d86154ec1b4e585c8ca1b0b15b7Mikael Starvik switch (n) 14251533b615e605d86154ec1b4e585c8ca1b0b15b7Mikael Starvik { 14351533b615e605d86154ec1b4e585c8ca1b0b15b7Mikael Starvik case 0: 14451533b615e605d86154ec1b4e585c8ca1b0b15b7Mikael Starvik break; 14551533b615e605d86154ec1b4e585c8ca1b0b15b7Mikael Starvik case 1: 14651533b615e605d86154ec1b4e585c8ca1b0b15b7Mikael Starvik __asm_copy_to_user_1 (dst, src, retn); 14751533b615e605d86154ec1b4e585c8ca1b0b15b7Mikael Starvik break; 14851533b615e605d86154ec1b4e585c8ca1b0b15b7Mikael Starvik case 2: 14951533b615e605d86154ec1b4e585c8ca1b0b15b7Mikael Starvik __asm_copy_to_user_2 (dst, src, retn); 15051533b615e605d86154ec1b4e585c8ca1b0b15b7Mikael Starvik break; 15151533b615e605d86154ec1b4e585c8ca1b0b15b7Mikael Starvik case 3: 15251533b615e605d86154ec1b4e585c8ca1b0b15b7Mikael Starvik __asm_copy_to_user_3 (dst, src, retn); 15351533b615e605d86154ec1b4e585c8ca1b0b15b7Mikael Starvik break; 15451533b615e605d86154ec1b4e585c8ca1b0b15b7Mikael Starvik } 15551533b615e605d86154ec1b4e585c8ca1b0b15b7Mikael Starvik 15651533b615e605d86154ec1b4e585c8ca1b0b15b7Mikael Starvik return retn; 15751533b615e605d86154ec1b4e585c8ca1b0b15b7Mikael Starvik} 15851533b615e605d86154ec1b4e585c8ca1b0b15b7Mikael Starvik 15951533b615e605d86154ec1b4e585c8ca1b0b15b7Mikael Starvik/* Copy from user to kernel, zeroing the bytes that were inaccessible in 16051533b615e605d86154ec1b4e585c8ca1b0b15b7Mikael Starvik userland. The return-value is the number of bytes that were 16151533b615e605d86154ec1b4e585c8ca1b0b15b7Mikael Starvik inaccessible. */ 16251533b615e605d86154ec1b4e585c8ca1b0b15b7Mikael Starvik 16351533b615e605d86154ec1b4e585c8ca1b0b15b7Mikael Starvikunsigned long 16407f2402b4adbcd0e6822ddc27953b63d4504faecJesper Nilsson__copy_user_zeroing(void *pdst, const void __user *psrc, unsigned long pn) 16551533b615e605d86154ec1b4e585c8ca1b0b15b7Mikael Starvik{ 16651533b615e605d86154ec1b4e585c8ca1b0b15b7Mikael Starvik /* We want the parameters put in special registers. 16751533b615e605d86154ec1b4e585c8ca1b0b15b7Mikael Starvik Make sure the compiler is able to make something useful of this. 16851533b615e605d86154ec1b4e585c8ca1b0b15b7Mikael Starvik As it is now: r10 -> r13; r11 -> r11 (nop); r12 -> r12 (nop). 16951533b615e605d86154ec1b4e585c8ca1b0b15b7Mikael Starvik 17051533b615e605d86154ec1b4e585c8ca1b0b15b7Mikael Starvik FIXME: Comment for old gcc version. Check. 17149b4ff3304b52b18c490fc4deb400b61bb7ed142Simon Arlott If gcc was alright, it really would need no temporaries, and no 17251533b615e605d86154ec1b4e585c8ca1b0b15b7Mikael Starvik stack space to save stuff on. */ 17351533b615e605d86154ec1b4e585c8ca1b0b15b7Mikael Starvik 17451533b615e605d86154ec1b4e585c8ca1b0b15b7Mikael Starvik register char *dst __asm__ ("r13") = pdst; 17551533b615e605d86154ec1b4e585c8ca1b0b15b7Mikael Starvik register const char *src __asm__ ("r11") = psrc; 17651533b615e605d86154ec1b4e585c8ca1b0b15b7Mikael Starvik register int n __asm__ ("r12") = pn; 17751533b615e605d86154ec1b4e585c8ca1b0b15b7Mikael Starvik register int retn __asm__ ("r10") = 0; 17851533b615e605d86154ec1b4e585c8ca1b0b15b7Mikael Starvik 17951533b615e605d86154ec1b4e585c8ca1b0b15b7Mikael Starvik /* The best reason to align src is that we then know that a read-fault 18051533b615e605d86154ec1b4e585c8ca1b0b15b7Mikael Starvik was for aligned bytes; there's no 1..3 remaining good bytes to 18151533b615e605d86154ec1b4e585c8ca1b0b15b7Mikael Starvik pickle. */ 18251533b615e605d86154ec1b4e585c8ca1b0b15b7Mikael Starvik if (((unsigned long) src & 3) != 0) 18351533b615e605d86154ec1b4e585c8ca1b0b15b7Mikael Starvik { 18451533b615e605d86154ec1b4e585c8ca1b0b15b7Mikael Starvik if (((unsigned long) src & 1) && n != 0) 18551533b615e605d86154ec1b4e585c8ca1b0b15b7Mikael Starvik { 18651533b615e605d86154ec1b4e585c8ca1b0b15b7Mikael Starvik __asm_copy_from_user_1 (dst, src, retn); 18751533b615e605d86154ec1b4e585c8ca1b0b15b7Mikael Starvik n--; 18851533b615e605d86154ec1b4e585c8ca1b0b15b7Mikael Starvik } 18951533b615e605d86154ec1b4e585c8ca1b0b15b7Mikael Starvik 19051533b615e605d86154ec1b4e585c8ca1b0b15b7Mikael Starvik if (((unsigned long) src & 2) && n >= 2) 19151533b615e605d86154ec1b4e585c8ca1b0b15b7Mikael Starvik { 19251533b615e605d86154ec1b4e585c8ca1b0b15b7Mikael Starvik __asm_copy_from_user_2 (dst, src, retn); 19351533b615e605d86154ec1b4e585c8ca1b0b15b7Mikael Starvik n -= 2; 19451533b615e605d86154ec1b4e585c8ca1b0b15b7Mikael Starvik } 19551533b615e605d86154ec1b4e585c8ca1b0b15b7Mikael Starvik 19651533b615e605d86154ec1b4e585c8ca1b0b15b7Mikael Starvik /* We only need one check after the unalignment-adjustments, because 19751533b615e605d86154ec1b4e585c8ca1b0b15b7Mikael Starvik if both adjustments were done, either both or neither reference 19851533b615e605d86154ec1b4e585c8ca1b0b15b7Mikael Starvik had an exception. */ 19951533b615e605d86154ec1b4e585c8ca1b0b15b7Mikael Starvik if (retn != 0) 20051533b615e605d86154ec1b4e585c8ca1b0b15b7Mikael Starvik goto copy_exception_bytes; 20151533b615e605d86154ec1b4e585c8ca1b0b15b7Mikael Starvik } 20251533b615e605d86154ec1b4e585c8ca1b0b15b7Mikael Starvik 20351533b615e605d86154ec1b4e585c8ca1b0b15b7Mikael Starvik /* Movem is dirt cheap. The overheap is low enough to always use the 20451533b615e605d86154ec1b4e585c8ca1b0b15b7Mikael Starvik minimum possible block size as the threshold. */ 20551533b615e605d86154ec1b4e585c8ca1b0b15b7Mikael Starvik if (n >= 44) 20651533b615e605d86154ec1b4e585c8ca1b0b15b7Mikael Starvik { 20751533b615e605d86154ec1b4e585c8ca1b0b15b7Mikael Starvik /* It is not optimal to tell the compiler about clobbering any 20851533b615e605d86154ec1b4e585c8ca1b0b15b7Mikael Starvik registers; that will move the saving/restoring of those registers 20951533b615e605d86154ec1b4e585c8ca1b0b15b7Mikael Starvik to the function prologue/epilogue, and make non-movem sizes 21051533b615e605d86154ec1b4e585c8ca1b0b15b7Mikael Starvik suboptimal. */ 21151533b615e605d86154ec1b4e585c8ca1b0b15b7Mikael Starvik __asm__ volatile ("\ 21251533b615e605d86154ec1b4e585c8ca1b0b15b7Mikael Starvik .ifnc %0%1%2%3,$r13$r11$r12$r10 \n\ 21351533b615e605d86154ec1b4e585c8ca1b0b15b7Mikael Starvik .err \n\ 21451533b615e605d86154ec1b4e585c8ca1b0b15b7Mikael Starvik .endif \n\ 21551533b615e605d86154ec1b4e585c8ca1b0b15b7Mikael Starvik \n\ 21651533b615e605d86154ec1b4e585c8ca1b0b15b7Mikael Starvik ;; Save the registers we'll use in the movem process \n\ 21751533b615e605d86154ec1b4e585c8ca1b0b15b7Mikael Starvik ;; on the stack. \n\ 21851533b615e605d86154ec1b4e585c8ca1b0b15b7Mikael Starvik subq 11*4,$sp \n\ 21951533b615e605d86154ec1b4e585c8ca1b0b15b7Mikael Starvik movem $r10,[$sp] \n\ 22051533b615e605d86154ec1b4e585c8ca1b0b15b7Mikael Starvik \n\ 22151533b615e605d86154ec1b4e585c8ca1b0b15b7Mikael Starvik ;; Now we've got this: \n\ 22251533b615e605d86154ec1b4e585c8ca1b0b15b7Mikael Starvik ;; r11 - src \n\ 22351533b615e605d86154ec1b4e585c8ca1b0b15b7Mikael Starvik ;; r13 - dst \n\ 22451533b615e605d86154ec1b4e585c8ca1b0b15b7Mikael Starvik ;; r12 - n \n\ 22551533b615e605d86154ec1b4e585c8ca1b0b15b7Mikael Starvik \n\ 22651533b615e605d86154ec1b4e585c8ca1b0b15b7Mikael Starvik ;; Update n for the first loop \n\ 22751533b615e605d86154ec1b4e585c8ca1b0b15b7Mikael Starvik subq 44,$r12 \n\ 22851533b615e605d86154ec1b4e585c8ca1b0b15b7Mikael Starvik0: \n\ 22951533b615e605d86154ec1b4e585c8ca1b0b15b7Mikael Starvik movem [$r11+],$r10 \n\ 23051533b615e605d86154ec1b4e585c8ca1b0b15b7Mikael Starvik \n\ 23151533b615e605d86154ec1b4e585c8ca1b0b15b7Mikael Starvik subq 44,$r12 \n\ 23251533b615e605d86154ec1b4e585c8ca1b0b15b7Mikael Starvik bge 0b \n\ 23351533b615e605d86154ec1b4e585c8ca1b0b15b7Mikael Starvik movem $r10,[$r13+] \n\ 23451533b615e605d86154ec1b4e585c8ca1b0b15b7Mikael Starvik \n\ 23551533b615e605d86154ec1b4e585c8ca1b0b15b7Mikael Starvik4: \n\ 23651533b615e605d86154ec1b4e585c8ca1b0b15b7Mikael Starvik addq 44,$r12 ;; compensate for last loop underflowing n \n\ 23751533b615e605d86154ec1b4e585c8ca1b0b15b7Mikael Starvik \n\ 23851533b615e605d86154ec1b4e585c8ca1b0b15b7Mikael Starvik ;; Restore registers from stack \n\ 23951533b615e605d86154ec1b4e585c8ca1b0b15b7Mikael Starvik movem [$sp+],$r10 \n\ 24051533b615e605d86154ec1b4e585c8ca1b0b15b7Mikael Starvik .section .fixup,\"ax\" \n\ 24151533b615e605d86154ec1b4e585c8ca1b0b15b7Mikael Starvik \n\ 24251533b615e605d86154ec1b4e585c8ca1b0b15b7Mikael Starvik;; Do not jump back into the loop if we fail. For some uses, we get a \n\ 24351533b615e605d86154ec1b4e585c8ca1b0b15b7Mikael Starvik;; page fault somewhere on the line. Without checking for page limits, \n\ 24451533b615e605d86154ec1b4e585c8ca1b0b15b7Mikael Starvik;; we don't know where, but we need to copy accurately and keep an \n\ 24551533b615e605d86154ec1b4e585c8ca1b0b15b7Mikael Starvik;; accurate count; not just clear the whole line. To do that, we fall \n\ 24651533b615e605d86154ec1b4e585c8ca1b0b15b7Mikael Starvik;; down in the code below, proceeding with smaller amounts. It should \n\ 24751533b615e605d86154ec1b4e585c8ca1b0b15b7Mikael Starvik;; be kept in mind that we have to cater to code like what at one time \n\ 24851533b615e605d86154ec1b4e585c8ca1b0b15b7Mikael Starvik;; was in fs/super.c: \n\ 24951533b615e605d86154ec1b4e585c8ca1b0b15b7Mikael Starvik;; i = size - copy_from_user((void *)page, data, size); \n\ 25051533b615e605d86154ec1b4e585c8ca1b0b15b7Mikael Starvik;; which would cause repeated faults while clearing the remainder of \n\ 25151533b615e605d86154ec1b4e585c8ca1b0b15b7Mikael Starvik;; the SIZE bytes at PAGE after the first fault. \n\ 25251533b615e605d86154ec1b4e585c8ca1b0b15b7Mikael Starvik;; A caveat here is that we must not fall through from a failing page \n\ 25351533b615e605d86154ec1b4e585c8ca1b0b15b7Mikael Starvik;; to a valid page. \n\ 25451533b615e605d86154ec1b4e585c8ca1b0b15b7Mikael Starvik \n\ 25551533b615e605d86154ec1b4e585c8ca1b0b15b7Mikael Starvik3: \n\ 25651533b615e605d86154ec1b4e585c8ca1b0b15b7Mikael Starvik jump 4b ;; Fall through, pretending the fault didn't happen. \n\ 25751533b615e605d86154ec1b4e585c8ca1b0b15b7Mikael Starvik nop \n\ 25851533b615e605d86154ec1b4e585c8ca1b0b15b7Mikael Starvik \n\ 25951533b615e605d86154ec1b4e585c8ca1b0b15b7Mikael Starvik .previous \n\ 26051533b615e605d86154ec1b4e585c8ca1b0b15b7Mikael Starvik .section __ex_table,\"a\" \n\ 26151533b615e605d86154ec1b4e585c8ca1b0b15b7Mikael Starvik .dword 0b,3b \n\ 26251533b615e605d86154ec1b4e585c8ca1b0b15b7Mikael Starvik .previous" 26351533b615e605d86154ec1b4e585c8ca1b0b15b7Mikael Starvik 26451533b615e605d86154ec1b4e585c8ca1b0b15b7Mikael Starvik /* Outputs */ : "=r" (dst), "=r" (src), "=r" (n), "=r" (retn) 26551533b615e605d86154ec1b4e585c8ca1b0b15b7Mikael Starvik /* Inputs */ : "0" (dst), "1" (src), "2" (n), "3" (retn)); 26651533b615e605d86154ec1b4e585c8ca1b0b15b7Mikael Starvik } 26751533b615e605d86154ec1b4e585c8ca1b0b15b7Mikael Starvik 26851533b615e605d86154ec1b4e585c8ca1b0b15b7Mikael Starvik /* Either we directly start copying here, using dword copying in a loop, 26951533b615e605d86154ec1b4e585c8ca1b0b15b7Mikael Starvik or we copy as much as possible with 'movem' and then the last block 27051533b615e605d86154ec1b4e585c8ca1b0b15b7Mikael Starvik (<44 bytes) is copied here. This will work since 'movem' will have 27151533b615e605d86154ec1b4e585c8ca1b0b15b7Mikael Starvik updated src, dst and n. (Except with failing src.) 27251533b615e605d86154ec1b4e585c8ca1b0b15b7Mikael Starvik 27351533b615e605d86154ec1b4e585c8ca1b0b15b7Mikael Starvik Since we want to keep src accurate, we can't use 27451533b615e605d86154ec1b4e585c8ca1b0b15b7Mikael Starvik __asm_copy_from_user_N with N != (1, 2, 4); it updates dst and 27551533b615e605d86154ec1b4e585c8ca1b0b15b7Mikael Starvik retn, but not src (by design; it's value is ignored elsewhere). */ 27651533b615e605d86154ec1b4e585c8ca1b0b15b7Mikael Starvik 27751533b615e605d86154ec1b4e585c8ca1b0b15b7Mikael Starvik while (n >= 4) 27851533b615e605d86154ec1b4e585c8ca1b0b15b7Mikael Starvik { 27951533b615e605d86154ec1b4e585c8ca1b0b15b7Mikael Starvik __asm_copy_from_user_4 (dst, src, retn); 28051533b615e605d86154ec1b4e585c8ca1b0b15b7Mikael Starvik n -= 4; 28151533b615e605d86154ec1b4e585c8ca1b0b15b7Mikael Starvik 28251533b615e605d86154ec1b4e585c8ca1b0b15b7Mikael Starvik if (retn) 28351533b615e605d86154ec1b4e585c8ca1b0b15b7Mikael Starvik goto copy_exception_bytes; 28451533b615e605d86154ec1b4e585c8ca1b0b15b7Mikael Starvik } 28551533b615e605d86154ec1b4e585c8ca1b0b15b7Mikael Starvik 28651533b615e605d86154ec1b4e585c8ca1b0b15b7Mikael Starvik /* If we get here, there were no memory read faults. */ 28751533b615e605d86154ec1b4e585c8ca1b0b15b7Mikael Starvik switch (n) 28851533b615e605d86154ec1b4e585c8ca1b0b15b7Mikael Starvik { 28951533b615e605d86154ec1b4e585c8ca1b0b15b7Mikael Starvik /* These copies are at least "naturally aligned" (so we don't have 29051533b615e605d86154ec1b4e585c8ca1b0b15b7Mikael Starvik to check each byte), due to the src alignment code before the 29151533b615e605d86154ec1b4e585c8ca1b0b15b7Mikael Starvik movem loop. The *_3 case *will* get the correct count for retn. */ 29251533b615e605d86154ec1b4e585c8ca1b0b15b7Mikael Starvik case 0: 29351533b615e605d86154ec1b4e585c8ca1b0b15b7Mikael Starvik /* This case deliberately left in (if you have doubts check the 29451533b615e605d86154ec1b4e585c8ca1b0b15b7Mikael Starvik generated assembly code). */ 29551533b615e605d86154ec1b4e585c8ca1b0b15b7Mikael Starvik break; 29651533b615e605d86154ec1b4e585c8ca1b0b15b7Mikael Starvik case 1: 29751533b615e605d86154ec1b4e585c8ca1b0b15b7Mikael Starvik __asm_copy_from_user_1 (dst, src, retn); 29851533b615e605d86154ec1b4e585c8ca1b0b15b7Mikael Starvik break; 29951533b615e605d86154ec1b4e585c8ca1b0b15b7Mikael Starvik case 2: 30051533b615e605d86154ec1b4e585c8ca1b0b15b7Mikael Starvik __asm_copy_from_user_2 (dst, src, retn); 30151533b615e605d86154ec1b4e585c8ca1b0b15b7Mikael Starvik break; 30251533b615e605d86154ec1b4e585c8ca1b0b15b7Mikael Starvik case 3: 30351533b615e605d86154ec1b4e585c8ca1b0b15b7Mikael Starvik __asm_copy_from_user_3 (dst, src, retn); 30451533b615e605d86154ec1b4e585c8ca1b0b15b7Mikael Starvik break; 30551533b615e605d86154ec1b4e585c8ca1b0b15b7Mikael Starvik } 30651533b615e605d86154ec1b4e585c8ca1b0b15b7Mikael Starvik 30751533b615e605d86154ec1b4e585c8ca1b0b15b7Mikael Starvik /* If we get here, retn correctly reflects the number of failing 30851533b615e605d86154ec1b4e585c8ca1b0b15b7Mikael Starvik bytes. */ 30951533b615e605d86154ec1b4e585c8ca1b0b15b7Mikael Starvik return retn; 31051533b615e605d86154ec1b4e585c8ca1b0b15b7Mikael Starvik 31151533b615e605d86154ec1b4e585c8ca1b0b15b7Mikael Starvikcopy_exception_bytes: 31251533b615e605d86154ec1b4e585c8ca1b0b15b7Mikael Starvik /* We already have "retn" bytes cleared, and need to clear the 31351533b615e605d86154ec1b4e585c8ca1b0b15b7Mikael Starvik remaining "n" bytes. A non-optimized simple byte-for-byte in-line 31451533b615e605d86154ec1b4e585c8ca1b0b15b7Mikael Starvik memset is preferred here, since this isn't speed-critical code and 31551533b615e605d86154ec1b4e585c8ca1b0b15b7Mikael Starvik we'd rather have this a leaf-function than calling memset. */ 31651533b615e605d86154ec1b4e585c8ca1b0b15b7Mikael Starvik { 31751533b615e605d86154ec1b4e585c8ca1b0b15b7Mikael Starvik char *endp; 31851533b615e605d86154ec1b4e585c8ca1b0b15b7Mikael Starvik for (endp = dst + n; dst < endp; dst++) 31951533b615e605d86154ec1b4e585c8ca1b0b15b7Mikael Starvik *dst = 0; 32051533b615e605d86154ec1b4e585c8ca1b0b15b7Mikael Starvik } 32151533b615e605d86154ec1b4e585c8ca1b0b15b7Mikael Starvik 32251533b615e605d86154ec1b4e585c8ca1b0b15b7Mikael Starvik return retn + n; 32351533b615e605d86154ec1b4e585c8ca1b0b15b7Mikael Starvik} 32451533b615e605d86154ec1b4e585c8ca1b0b15b7Mikael Starvik 32551533b615e605d86154ec1b4e585c8ca1b0b15b7Mikael Starvik/* Zero userspace. */ 32651533b615e605d86154ec1b4e585c8ca1b0b15b7Mikael Starvik 32751533b615e605d86154ec1b4e585c8ca1b0b15b7Mikael Starvikunsigned long 32851533b615e605d86154ec1b4e585c8ca1b0b15b7Mikael Starvik__do_clear_user (void __user *pto, unsigned long pn) 32951533b615e605d86154ec1b4e585c8ca1b0b15b7Mikael Starvik{ 33051533b615e605d86154ec1b4e585c8ca1b0b15b7Mikael Starvik /* We want the parameters put in special registers. 33151533b615e605d86154ec1b4e585c8ca1b0b15b7Mikael Starvik Make sure the compiler is able to make something useful of this. 33251533b615e605d86154ec1b4e585c8ca1b0b15b7Mikael Starvik As it is now: r10 -> r13; r11 -> r11 (nop); r12 -> r12 (nop). 33351533b615e605d86154ec1b4e585c8ca1b0b15b7Mikael Starvik 33451533b615e605d86154ec1b4e585c8ca1b0b15b7Mikael Starvik FIXME: Comment for old gcc version. Check. 33549b4ff3304b52b18c490fc4deb400b61bb7ed142Simon Arlott If gcc was alright, it really would need no temporaries, and no 33651533b615e605d86154ec1b4e585c8ca1b0b15b7Mikael Starvik stack space to save stuff on. */ 33751533b615e605d86154ec1b4e585c8ca1b0b15b7Mikael Starvik 33851533b615e605d86154ec1b4e585c8ca1b0b15b7Mikael Starvik register char *dst __asm__ ("r13") = pto; 33951533b615e605d86154ec1b4e585c8ca1b0b15b7Mikael Starvik register int n __asm__ ("r12") = pn; 34051533b615e605d86154ec1b4e585c8ca1b0b15b7Mikael Starvik register int retn __asm__ ("r10") = 0; 34151533b615e605d86154ec1b4e585c8ca1b0b15b7Mikael Starvik 34251533b615e605d86154ec1b4e585c8ca1b0b15b7Mikael Starvik 34351533b615e605d86154ec1b4e585c8ca1b0b15b7Mikael Starvik if (((unsigned long) dst & 3) != 0 34451533b615e605d86154ec1b4e585c8ca1b0b15b7Mikael Starvik /* Don't align if we wouldn't copy more than a few bytes. */ 34551533b615e605d86154ec1b4e585c8ca1b0b15b7Mikael Starvik && n >= 3) 34651533b615e605d86154ec1b4e585c8ca1b0b15b7Mikael Starvik { 34751533b615e605d86154ec1b4e585c8ca1b0b15b7Mikael Starvik if ((unsigned long) dst & 1) 34851533b615e605d86154ec1b4e585c8ca1b0b15b7Mikael Starvik { 34951533b615e605d86154ec1b4e585c8ca1b0b15b7Mikael Starvik __asm_clear_1 (dst, retn); 35051533b615e605d86154ec1b4e585c8ca1b0b15b7Mikael Starvik n--; 35151533b615e605d86154ec1b4e585c8ca1b0b15b7Mikael Starvik } 35251533b615e605d86154ec1b4e585c8ca1b0b15b7Mikael Starvik 35351533b615e605d86154ec1b4e585c8ca1b0b15b7Mikael Starvik if ((unsigned long) dst & 2) 35451533b615e605d86154ec1b4e585c8ca1b0b15b7Mikael Starvik { 35551533b615e605d86154ec1b4e585c8ca1b0b15b7Mikael Starvik __asm_clear_2 (dst, retn); 35651533b615e605d86154ec1b4e585c8ca1b0b15b7Mikael Starvik n -= 2; 35751533b615e605d86154ec1b4e585c8ca1b0b15b7Mikael Starvik } 35851533b615e605d86154ec1b4e585c8ca1b0b15b7Mikael Starvik } 35951533b615e605d86154ec1b4e585c8ca1b0b15b7Mikael Starvik 36051533b615e605d86154ec1b4e585c8ca1b0b15b7Mikael Starvik /* Decide which copying method to use. 36151533b615e605d86154ec1b4e585c8ca1b0b15b7Mikael Starvik FIXME: This number is from the "ordinary" kernel memset. */ 36251533b615e605d86154ec1b4e585c8ca1b0b15b7Mikael Starvik if (n >= 48) 36351533b615e605d86154ec1b4e585c8ca1b0b15b7Mikael Starvik { 36451533b615e605d86154ec1b4e585c8ca1b0b15b7Mikael Starvik /* For large clears we use 'movem' */ 36551533b615e605d86154ec1b4e585c8ca1b0b15b7Mikael Starvik 36651533b615e605d86154ec1b4e585c8ca1b0b15b7Mikael Starvik /* It is not optimal to tell the compiler about clobbering any 36751533b615e605d86154ec1b4e585c8ca1b0b15b7Mikael Starvik call-saved registers; that will move the saving/restoring of 36851533b615e605d86154ec1b4e585c8ca1b0b15b7Mikael Starvik those registers to the function prologue/epilogue, and make 36951533b615e605d86154ec1b4e585c8ca1b0b15b7Mikael Starvik non-movem sizes suboptimal. 37051533b615e605d86154ec1b4e585c8ca1b0b15b7Mikael Starvik 37151533b615e605d86154ec1b4e585c8ca1b0b15b7Mikael Starvik This method is not foolproof; it assumes that the "asm reg" 37251533b615e605d86154ec1b4e585c8ca1b0b15b7Mikael Starvik declarations at the beginning of the function really are used 37351533b615e605d86154ec1b4e585c8ca1b0b15b7Mikael Starvik here (beware: they may be moved to temporary registers). 37451533b615e605d86154ec1b4e585c8ca1b0b15b7Mikael Starvik This way, we do not have to save/move the registers around into 37551533b615e605d86154ec1b4e585c8ca1b0b15b7Mikael Starvik temporaries; we can safely use them straight away. 37651533b615e605d86154ec1b4e585c8ca1b0b15b7Mikael Starvik 37751533b615e605d86154ec1b4e585c8ca1b0b15b7Mikael Starvik If you want to check that the allocation was right; then 37851533b615e605d86154ec1b4e585c8ca1b0b15b7Mikael Starvik check the equalities in the first comment. It should say 37951533b615e605d86154ec1b4e585c8ca1b0b15b7Mikael Starvik something like "r13=r13, r11=r11, r12=r12". */ 38051533b615e605d86154ec1b4e585c8ca1b0b15b7Mikael Starvik __asm__ volatile ("\ 38151533b615e605d86154ec1b4e585c8ca1b0b15b7Mikael Starvik .ifnc %0%1%2,$r13$r12$r10 \n\ 38251533b615e605d86154ec1b4e585c8ca1b0b15b7Mikael Starvik .err \n\ 38351533b615e605d86154ec1b4e585c8ca1b0b15b7Mikael Starvik .endif \n\ 38451533b615e605d86154ec1b4e585c8ca1b0b15b7Mikael Starvik \n\ 38551533b615e605d86154ec1b4e585c8ca1b0b15b7Mikael Starvik ;; Save the registers we'll clobber in the movem process \n\ 38651533b615e605d86154ec1b4e585c8ca1b0b15b7Mikael Starvik ;; on the stack. Don't mention them to gcc, it will only be \n\ 38751533b615e605d86154ec1b4e585c8ca1b0b15b7Mikael Starvik ;; upset. \n\ 38851533b615e605d86154ec1b4e585c8ca1b0b15b7Mikael Starvik subq 11*4,$sp \n\ 38951533b615e605d86154ec1b4e585c8ca1b0b15b7Mikael Starvik movem $r10,[$sp] \n\ 39051533b615e605d86154ec1b4e585c8ca1b0b15b7Mikael Starvik \n\ 39151533b615e605d86154ec1b4e585c8ca1b0b15b7Mikael Starvik clear.d $r0 \n\ 39251533b615e605d86154ec1b4e585c8ca1b0b15b7Mikael Starvik clear.d $r1 \n\ 39351533b615e605d86154ec1b4e585c8ca1b0b15b7Mikael Starvik clear.d $r2 \n\ 39451533b615e605d86154ec1b4e585c8ca1b0b15b7Mikael Starvik clear.d $r3 \n\ 39551533b615e605d86154ec1b4e585c8ca1b0b15b7Mikael Starvik clear.d $r4 \n\ 39651533b615e605d86154ec1b4e585c8ca1b0b15b7Mikael Starvik clear.d $r5 \n\ 39751533b615e605d86154ec1b4e585c8ca1b0b15b7Mikael Starvik clear.d $r6 \n\ 39851533b615e605d86154ec1b4e585c8ca1b0b15b7Mikael Starvik clear.d $r7 \n\ 39951533b615e605d86154ec1b4e585c8ca1b0b15b7Mikael Starvik clear.d $r8 \n\ 40051533b615e605d86154ec1b4e585c8ca1b0b15b7Mikael Starvik clear.d $r9 \n\ 40151533b615e605d86154ec1b4e585c8ca1b0b15b7Mikael Starvik clear.d $r10 \n\ 40251533b615e605d86154ec1b4e585c8ca1b0b15b7Mikael Starvik clear.d $r11 \n\ 40351533b615e605d86154ec1b4e585c8ca1b0b15b7Mikael Starvik \n\ 40451533b615e605d86154ec1b4e585c8ca1b0b15b7Mikael Starvik ;; Now we've got this: \n\ 40551533b615e605d86154ec1b4e585c8ca1b0b15b7Mikael Starvik ;; r13 - dst \n\ 40651533b615e605d86154ec1b4e585c8ca1b0b15b7Mikael Starvik ;; r12 - n \n\ 40751533b615e605d86154ec1b4e585c8ca1b0b15b7Mikael Starvik \n\ 40851533b615e605d86154ec1b4e585c8ca1b0b15b7Mikael Starvik ;; Update n for the first loop \n\ 40951533b615e605d86154ec1b4e585c8ca1b0b15b7Mikael Starvik subq 12*4,$r12 \n\ 41051533b615e605d86154ec1b4e585c8ca1b0b15b7Mikael Starvik0: \n\ 41151533b615e605d86154ec1b4e585c8ca1b0b15b7Mikael Starvik subq 12*4,$r12 \n\ 41251533b615e605d86154ec1b4e585c8ca1b0b15b7Mikael Starvik1: \n\ 41351533b615e605d86154ec1b4e585c8ca1b0b15b7Mikael Starvik bge 0b \n\ 41451533b615e605d86154ec1b4e585c8ca1b0b15b7Mikael Starvik movem $r11,[$r13+] \n\ 41551533b615e605d86154ec1b4e585c8ca1b0b15b7Mikael Starvik \n\ 41651533b615e605d86154ec1b4e585c8ca1b0b15b7Mikael Starvik addq 12*4,$r12 ;; compensate for last loop underflowing n \n\ 41751533b615e605d86154ec1b4e585c8ca1b0b15b7Mikael Starvik \n\ 41851533b615e605d86154ec1b4e585c8ca1b0b15b7Mikael Starvik ;; Restore registers from stack \n\ 41951533b615e605d86154ec1b4e585c8ca1b0b15b7Mikael Starvik movem [$sp+],$r10 \n\ 42051533b615e605d86154ec1b4e585c8ca1b0b15b7Mikael Starvik2: \n\ 42151533b615e605d86154ec1b4e585c8ca1b0b15b7Mikael Starvik .section .fixup,\"ax\" \n\ 42251533b615e605d86154ec1b4e585c8ca1b0b15b7Mikael Starvik3: \n\ 42351533b615e605d86154ec1b4e585c8ca1b0b15b7Mikael Starvik movem [$sp],$r10 \n\ 42451533b615e605d86154ec1b4e585c8ca1b0b15b7Mikael Starvik addq 12*4,$r10 \n\ 42551533b615e605d86154ec1b4e585c8ca1b0b15b7Mikael Starvik addq 12*4,$r13 \n\ 42651533b615e605d86154ec1b4e585c8ca1b0b15b7Mikael Starvik movem $r10,[$sp] \n\ 42751533b615e605d86154ec1b4e585c8ca1b0b15b7Mikael Starvik jump 0b \n\ 42851533b615e605d86154ec1b4e585c8ca1b0b15b7Mikael Starvik clear.d $r10 \n\ 42951533b615e605d86154ec1b4e585c8ca1b0b15b7Mikael Starvik \n\ 43051533b615e605d86154ec1b4e585c8ca1b0b15b7Mikael Starvik .previous \n\ 43151533b615e605d86154ec1b4e585c8ca1b0b15b7Mikael Starvik .section __ex_table,\"a\" \n\ 43251533b615e605d86154ec1b4e585c8ca1b0b15b7Mikael Starvik .dword 1b,3b \n\ 43351533b615e605d86154ec1b4e585c8ca1b0b15b7Mikael Starvik .previous" 43451533b615e605d86154ec1b4e585c8ca1b0b15b7Mikael Starvik 43551533b615e605d86154ec1b4e585c8ca1b0b15b7Mikael Starvik /* Outputs */ : "=r" (dst), "=r" (n), "=r" (retn) 43651533b615e605d86154ec1b4e585c8ca1b0b15b7Mikael Starvik /* Inputs */ : "0" (dst), "1" (n), "2" (retn) 43751533b615e605d86154ec1b4e585c8ca1b0b15b7Mikael Starvik /* Clobber */ : "r11"); 43851533b615e605d86154ec1b4e585c8ca1b0b15b7Mikael Starvik } 43951533b615e605d86154ec1b4e585c8ca1b0b15b7Mikael Starvik 44051533b615e605d86154ec1b4e585c8ca1b0b15b7Mikael Starvik while (n >= 16) 44151533b615e605d86154ec1b4e585c8ca1b0b15b7Mikael Starvik { 44251533b615e605d86154ec1b4e585c8ca1b0b15b7Mikael Starvik __asm_clear_16 (dst, retn); 44351533b615e605d86154ec1b4e585c8ca1b0b15b7Mikael Starvik n -= 16; 44451533b615e605d86154ec1b4e585c8ca1b0b15b7Mikael Starvik } 44551533b615e605d86154ec1b4e585c8ca1b0b15b7Mikael Starvik 44651533b615e605d86154ec1b4e585c8ca1b0b15b7Mikael Starvik /* Having a separate by-four loops cuts down on cache footprint. 44751533b615e605d86154ec1b4e585c8ca1b0b15b7Mikael Starvik FIXME: Test with and without; increasing switch to be 0..15. */ 44851533b615e605d86154ec1b4e585c8ca1b0b15b7Mikael Starvik while (n >= 4) 44951533b615e605d86154ec1b4e585c8ca1b0b15b7Mikael Starvik { 45051533b615e605d86154ec1b4e585c8ca1b0b15b7Mikael Starvik __asm_clear_4 (dst, retn); 45151533b615e605d86154ec1b4e585c8ca1b0b15b7Mikael Starvik n -= 4; 45251533b615e605d86154ec1b4e585c8ca1b0b15b7Mikael Starvik } 45351533b615e605d86154ec1b4e585c8ca1b0b15b7Mikael Starvik 45451533b615e605d86154ec1b4e585c8ca1b0b15b7Mikael Starvik switch (n) 45551533b615e605d86154ec1b4e585c8ca1b0b15b7Mikael Starvik { 45651533b615e605d86154ec1b4e585c8ca1b0b15b7Mikael Starvik case 0: 45751533b615e605d86154ec1b4e585c8ca1b0b15b7Mikael Starvik break; 45851533b615e605d86154ec1b4e585c8ca1b0b15b7Mikael Starvik case 1: 45951533b615e605d86154ec1b4e585c8ca1b0b15b7Mikael Starvik __asm_clear_1 (dst, retn); 46051533b615e605d86154ec1b4e585c8ca1b0b15b7Mikael Starvik break; 46151533b615e605d86154ec1b4e585c8ca1b0b15b7Mikael Starvik case 2: 46251533b615e605d86154ec1b4e585c8ca1b0b15b7Mikael Starvik __asm_clear_2 (dst, retn); 46351533b615e605d86154ec1b4e585c8ca1b0b15b7Mikael Starvik break; 46451533b615e605d86154ec1b4e585c8ca1b0b15b7Mikael Starvik case 3: 46551533b615e605d86154ec1b4e585c8ca1b0b15b7Mikael Starvik __asm_clear_3 (dst, retn); 46651533b615e605d86154ec1b4e585c8ca1b0b15b7Mikael Starvik break; 46751533b615e605d86154ec1b4e585c8ca1b0b15b7Mikael Starvik } 46851533b615e605d86154ec1b4e585c8ca1b0b15b7Mikael Starvik 46951533b615e605d86154ec1b4e585c8ca1b0b15b7Mikael Starvik return retn; 47051533b615e605d86154ec1b4e585c8ca1b0b15b7Mikael Starvik} 471