memcpy.c revision 1da177e4c3f41524e886b7f1b8a0c1fc7321cac
11da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds/* 21da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds * Optimized memory copy routines. 31da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds * 41da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds * Copyright (C) 2004 Randolph Chung <tausq@debian.org> 51da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds * 61da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds * This program is free software; you can redistribute it and/or modify 71da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds * it under the terms of the GNU General Public License as published by 81da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds * the Free Software Foundation; either version 2, or (at your option) 91da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds * any later version. 101da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds * 111da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds * This program is distributed in the hope that it will be useful, 121da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds * but WITHOUT ANY WARRANTY; without even the implied warranty of 131da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 141da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds * GNU General Public License for more details. 151da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds * 161da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds * You should have received a copy of the GNU General Public License 171da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds * along with this program; if not, write to the Free Software 181da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA. 191da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds * 201da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds * Portions derived from the GNU C Library 211da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds * Copyright (C) 1991, 1997, 2003 Free Software Foundation, Inc. 221da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds * 231da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds * Several strategies are tried to try to get the best performance for various 241da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds * conditions. In the optimal case, we copy 64-bytes in an unrolled loop using 251da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds * fp regs. This is followed by loops that copy 32- or 16-bytes at a time using 261da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds * general registers. Unaligned copies are handled either by aligning the 271da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds * destination and then using shift-and-write method, or in a few cases by 281da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds * falling back to a byte-at-a-time copy. 291da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds * 301da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds * I chose to implement this in C because it is easier to maintain and debug, 311da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds * and in my experiments it appears that the C code generated by gcc (3.3/3.4 321da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds * at the time of writing) is fairly optimal. Unfortunately some of the 331da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds * semantics of the copy routine (exception handling) is difficult to express 341da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds * in C, so we have to play some tricks to get it to work. 351da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds * 361da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds * All the loads and stores are done via explicit asm() code in order to use 371da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds * the right space registers. 381da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds * 391da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds * Testing with various alignments and buffer sizes shows that this code is 401da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds * often >10x faster than a simple byte-at-a-time copy, even for strangely 411da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds * aligned operands. It is interesting to note that the glibc version 421da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds * of memcpy (written in C) is actually quite fast already. This routine is 431da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds * able to beat it by 30-40% for aligned copies because of the loop unrolling, 441da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds * but in some cases the glibc version is still slightly faster. This lends 451da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds * more credibility that gcc can generate very good code as long as we are 461da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds * careful. 471da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds * 481da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds * TODO: 491da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds * - cache prefetching needs more experimentation to get optimal settings 501da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds * - try not to use the post-increment address modifiers; they create additional 511da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds * interlocks 521da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds * - replace byte-copy loops with stybs sequences 531da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds */ 541da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 551da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds#ifdef __KERNEL__ 561da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds#include <linux/config.h> 571da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds#include <linux/module.h> 581da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds#include <linux/compiler.h> 591da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds#include <asm/uaccess.h> 601da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds#define s_space "%%sr1" 611da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds#define d_space "%%sr2" 621da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds#else 631da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds#include "memcpy.h" 641da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds#define s_space "%%sr0" 651da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds#define d_space "%%sr0" 661da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds#define pa_memcpy new2_copy 671da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds#endif 681da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 691da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus TorvaldsDECLARE_PER_CPU(struct exception_data, exception_data); 701da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 711da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds#define preserve_branch(label) do { \ 721da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds volatile int dummy; \ 731da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds /* The following branch is never taken, it's just here to */ \ 741da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds /* prevent gcc from optimizing away our exception code. */ \ 751da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds if (unlikely(dummy != dummy)) \ 761da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds goto label; \ 771da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds} while (0) 781da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 791da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds#define get_user_space() (segment_eq(get_fs(), KERNEL_DS) ? 0 : mfsp(3)) 801da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds#define get_kernel_space() (0) 811da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 821da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds#define MERGE(w0, sh_1, w1, sh_2) ({ \ 831da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds unsigned int _r; \ 841da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds asm volatile ( \ 851da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds "mtsar %3\n" \ 861da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds "shrpw %1, %2, %%sar, %0\n" \ 871da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds : "=r"(_r) \ 881da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds : "r"(w0), "r"(w1), "r"(sh_2) \ 891da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds ); \ 901da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds _r; \ 911da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds}) 921da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds#define THRESHOLD 16 931da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 941da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds#ifdef DEBUG_MEMCPY 951da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds#define DPRINTF(fmt, args...) do { printk(KERN_DEBUG "%s:%d:%s ", __FILE__, __LINE__, __FUNCTION__ ); printk(KERN_DEBUG fmt, ##args ); } while (0) 961da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds#else 971da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds#define DPRINTF(fmt, args...) 981da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds#endif 991da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 1001da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds#ifndef __LP64__ 1011da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds#define EXC_WORD ".word" 1021da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds#else 1031da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds#define EXC_WORD ".dword" 1041da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds#endif 1051da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 1061da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds#define def_load_ai_insn(_insn,_sz,_tt,_s,_a,_t,_e) \ 1071da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds __asm__ __volatile__ ( \ 1081da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds "1:\t" #_insn ",ma " #_sz "(" _s ",%1), %0\n" \ 1091da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds "\t.section __ex_table,\"aw\"\n" \ 1101da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds "\t" EXC_WORD "\t1b\n" \ 1111da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds "\t" EXC_WORD "\t" #_e "\n" \ 1121da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds "\t.previous\n" \ 1131da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds : _tt(_t), "+r"(_a) \ 1141da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds : \ 1151da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds : "r8") 1161da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 1171da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds#define def_store_ai_insn(_insn,_sz,_tt,_s,_a,_t,_e) \ 1181da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds __asm__ __volatile__ ( \ 1191da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds "1:\t" #_insn ",ma %1, " #_sz "(" _s ",%0)\n" \ 1201da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds "\t.section __ex_table,\"aw\"\n" \ 1211da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds "\t" EXC_WORD "\t1b\n" \ 1221da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds "\t" EXC_WORD "\t" #_e "\n" \ 1231da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds "\t.previous\n" \ 1241da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds : "+r"(_a) \ 1251da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds : _tt(_t) \ 1261da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds : "r8") 1271da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 1281da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds#define ldbma(_s, _a, _t, _e) def_load_ai_insn(ldbs,1,"=r",_s,_a,_t,_e) 1291da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds#define stbma(_s, _t, _a, _e) def_store_ai_insn(stbs,1,"r",_s,_a,_t,_e) 1301da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds#define ldwma(_s, _a, _t, _e) def_load_ai_insn(ldw,4,"=r",_s,_a,_t,_e) 1311da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds#define stwma(_s, _t, _a, _e) def_store_ai_insn(stw,4,"r",_s,_a,_t,_e) 1321da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds#define flddma(_s, _a, _t, _e) def_load_ai_insn(fldd,8,"=f",_s,_a,_t,_e) 1331da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds#define fstdma(_s, _t, _a, _e) def_store_ai_insn(fstd,8,"f",_s,_a,_t,_e) 1341da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 1351da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds#define def_load_insn(_insn,_tt,_s,_o,_a,_t,_e) \ 1361da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds __asm__ __volatile__ ( \ 1371da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds "1:\t" #_insn " " #_o "(" _s ",%1), %0\n" \ 1381da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds "\t.section __ex_table,\"aw\"\n" \ 1391da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds "\t" EXC_WORD "\t1b\n" \ 1401da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds "\t" EXC_WORD "\t" #_e "\n" \ 1411da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds "\t.previous\n" \ 1421da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds : _tt(_t) \ 1431da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds : "r"(_a) \ 1441da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds : "r8") 1451da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 1461da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds#define def_store_insn(_insn,_tt,_s,_t,_o,_a,_e) \ 1471da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds __asm__ __volatile__ ( \ 1481da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds "1:\t" #_insn " %0, " #_o "(" _s ",%1)\n" \ 1491da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds "\t.section __ex_table,\"aw\"\n" \ 1501da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds "\t" EXC_WORD "\t1b\n" \ 1511da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds "\t" EXC_WORD "\t" #_e "\n" \ 1521da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds "\t.previous\n" \ 1531da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds : \ 1541da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds : _tt(_t), "r"(_a) \ 1551da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds : "r8") 1561da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 1571da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds#define ldw(_s,_o,_a,_t,_e) def_load_insn(ldw,"=r",_s,_o,_a,_t,_e) 1581da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds#define stw(_s,_t,_o,_a,_e) def_store_insn(stw,"r",_s,_t,_o,_a,_e) 1591da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 1601da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds#ifdef CONFIG_PREFETCH 1611da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvaldsextern inline void prefetch_src(const void *addr) 1621da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds{ 1631da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds __asm__("ldw 0(" s_space ",%0), %%r0" : : "r" (addr)); 1641da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds} 1651da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 1661da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvaldsextern inline void prefetch_dst(const void *addr) 1671da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds{ 1681da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds __asm__("ldd 0(" d_space ",%0), %%r0" : : "r" (addr)); 1691da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds} 1701da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds#else 1711da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds#define prefetch_src(addr) 1721da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds#define prefetch_dst(addr) 1731da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds#endif 1741da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 1751da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds/* Copy from a not-aligned src to an aligned dst, using shifts. Handles 4 words 1761da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds * per loop. This code is derived from glibc. 1771da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds */ 1781da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvaldsstatic inline unsigned long copy_dstaligned(unsigned long dst, unsigned long src, unsigned long len, unsigned long o_dst, unsigned long o_src, unsigned long o_len) 1791da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds{ 1801da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds /* gcc complains that a2 and a3 may be uninitialized, but actually 1811da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds * they cannot be. Initialize a2/a3 to shut gcc up. 1821da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds */ 1831da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds register unsigned int a0, a1, a2 = 0, a3 = 0; 1841da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds int sh_1, sh_2; 1851da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds struct exception_data *d; 1861da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 1871da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds /* prefetch_src((const void *)src); */ 1881da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 1891da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds /* Calculate how to shift a word read at the memory operation 1901da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds aligned srcp to make it aligned for copy. */ 1911da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds sh_1 = 8 * (src % sizeof(unsigned int)); 1921da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds sh_2 = 8 * sizeof(unsigned int) - sh_1; 1931da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 1941da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds /* Make src aligned by rounding it down. */ 1951da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds src &= -sizeof(unsigned int); 1961da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 1971da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds switch (len % 4) 1981da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds { 1991da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds case 2: 2001da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds /* a1 = ((unsigned int *) src)[0]; 2011da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds a2 = ((unsigned int *) src)[1]; */ 2021da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds ldw(s_space, 0, src, a1, cda_ldw_exc); 2031da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds ldw(s_space, 4, src, a2, cda_ldw_exc); 2041da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds src -= 1 * sizeof(unsigned int); 2051da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds dst -= 3 * sizeof(unsigned int); 2061da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds len += 2; 2071da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds goto do1; 2081da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds case 3: 2091da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds /* a0 = ((unsigned int *) src)[0]; 2101da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds a1 = ((unsigned int *) src)[1]; */ 2111da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds ldw(s_space, 0, src, a0, cda_ldw_exc); 2121da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds ldw(s_space, 4, src, a1, cda_ldw_exc); 2131da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds src -= 0 * sizeof(unsigned int); 2141da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds dst -= 2 * sizeof(unsigned int); 2151da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds len += 1; 2161da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds goto do2; 2171da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds case 0: 2181da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds if (len == 0) 2191da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds return 0; 2201da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds /* a3 = ((unsigned int *) src)[0]; 2211da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds a0 = ((unsigned int *) src)[1]; */ 2221da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds ldw(s_space, 0, src, a3, cda_ldw_exc); 2231da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds ldw(s_space, 4, src, a0, cda_ldw_exc); 2241da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds src -=-1 * sizeof(unsigned int); 2251da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds dst -= 1 * sizeof(unsigned int); 2261da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds len += 0; 2271da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds goto do3; 2281da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds case 1: 2291da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds /* a2 = ((unsigned int *) src)[0]; 2301da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds a3 = ((unsigned int *) src)[1]; */ 2311da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds ldw(s_space, 0, src, a2, cda_ldw_exc); 2321da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds ldw(s_space, 4, src, a3, cda_ldw_exc); 2331da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds src -=-2 * sizeof(unsigned int); 2341da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds dst -= 0 * sizeof(unsigned int); 2351da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds len -= 1; 2361da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds if (len == 0) 2371da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds goto do0; 2381da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds goto do4; /* No-op. */ 2391da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds } 2401da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 2411da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds do 2421da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds { 2431da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds /* prefetch_src((const void *)(src + 4 * sizeof(unsigned int))); */ 2441da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvaldsdo4: 2451da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds /* a0 = ((unsigned int *) src)[0]; */ 2461da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds ldw(s_space, 0, src, a0, cda_ldw_exc); 2471da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds /* ((unsigned int *) dst)[0] = MERGE (a2, sh_1, a3, sh_2); */ 2481da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds stw(d_space, MERGE (a2, sh_1, a3, sh_2), 0, dst, cda_stw_exc); 2491da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvaldsdo3: 2501da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds /* a1 = ((unsigned int *) src)[1]; */ 2511da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds ldw(s_space, 4, src, a1, cda_ldw_exc); 2521da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds /* ((unsigned int *) dst)[1] = MERGE (a3, sh_1, a0, sh_2); */ 2531da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds stw(d_space, MERGE (a3, sh_1, a0, sh_2), 4, dst, cda_stw_exc); 2541da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvaldsdo2: 2551da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds /* a2 = ((unsigned int *) src)[2]; */ 2561da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds ldw(s_space, 8, src, a2, cda_ldw_exc); 2571da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds /* ((unsigned int *) dst)[2] = MERGE (a0, sh_1, a1, sh_2); */ 2581da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds stw(d_space, MERGE (a0, sh_1, a1, sh_2), 8, dst, cda_stw_exc); 2591da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvaldsdo1: 2601da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds /* a3 = ((unsigned int *) src)[3]; */ 2611da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds ldw(s_space, 12, src, a3, cda_ldw_exc); 2621da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds /* ((unsigned int *) dst)[3] = MERGE (a1, sh_1, a2, sh_2); */ 2631da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds stw(d_space, MERGE (a1, sh_1, a2, sh_2), 12, dst, cda_stw_exc); 2641da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 2651da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds src += 4 * sizeof(unsigned int); 2661da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds dst += 4 * sizeof(unsigned int); 2671da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds len -= 4; 2681da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds } 2691da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds while (len != 0); 2701da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 2711da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvaldsdo0: 2721da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds /* ((unsigned int *) dst)[0] = MERGE (a2, sh_1, a3, sh_2); */ 2731da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds stw(d_space, MERGE (a2, sh_1, a3, sh_2), 0, dst, cda_stw_exc); 2741da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 2751da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds preserve_branch(handle_load_error); 2761da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds preserve_branch(handle_store_error); 2771da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 2781da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds return 0; 2791da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 2801da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvaldshandle_load_error: 2811da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds __asm__ __volatile__ ("cda_ldw_exc:\n"); 2821da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds d = &__get_cpu_var(exception_data); 2831da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds DPRINTF("cda_ldw_exc: o_len=%lu fault_addr=%lu o_src=%lu ret=%lu\n", 2841da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds o_len, d->fault_addr, o_src, o_len - d->fault_addr + o_src); 2851da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds return o_len * 4 - d->fault_addr + o_src; 2861da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 2871da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvaldshandle_store_error: 2881da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds __asm__ __volatile__ ("cda_stw_exc:\n"); 2891da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds d = &__get_cpu_var(exception_data); 2901da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds DPRINTF("cda_stw_exc: o_len=%lu fault_addr=%lu o_dst=%lu ret=%lu\n", 2911da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds o_len, d->fault_addr, o_dst, o_len - d->fault_addr + o_dst); 2921da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds return o_len * 4 - d->fault_addr + o_dst; 2931da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds} 2941da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 2951da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 2961da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds/* Returns 0 for success, otherwise, returns number of bytes not transferred. */ 2971da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvaldsunsigned long pa_memcpy(void *dstp, const void *srcp, unsigned long len) 2981da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds{ 2991da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds register unsigned long src, dst, t1, t2, t3; 3001da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds register unsigned char *pcs, *pcd; 3011da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds register unsigned int *pws, *pwd; 3021da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds register double *pds, *pdd; 3031da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds unsigned long ret = 0; 3041da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds unsigned long o_dst, o_src, o_len; 3051da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds struct exception_data *d; 3061da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 3071da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds src = (unsigned long)srcp; 3081da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds dst = (unsigned long)dstp; 3091da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds pcs = (unsigned char *)srcp; 3101da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds pcd = (unsigned char *)dstp; 3111da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 3121da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds o_dst = dst; o_src = src; o_len = len; 3131da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 3141da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds /* prefetch_src((const void *)srcp); */ 3151da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 3161da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds if (len < THRESHOLD) 3171da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds goto byte_copy; 3181da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 3191da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds /* Check alignment */ 3201da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds t1 = (src ^ dst); 3211da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds if (unlikely(t1 & (sizeof(double)-1))) 3221da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds goto unaligned_copy; 3231da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 3241da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds /* src and dst have same alignment. */ 3251da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 3261da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds /* Copy bytes till we are double-aligned. */ 3271da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds t2 = src & (sizeof(double) - 1); 3281da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds if (unlikely(t2 != 0)) { 3291da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds t2 = sizeof(double) - t2; 3301da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds while (t2 && len) { 3311da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds /* *pcd++ = *pcs++; */ 3321da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds ldbma(s_space, pcs, t3, pmc_load_exc); 3331da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds len--; 3341da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds stbma(d_space, t3, pcd, pmc_store_exc); 3351da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds t2--; 3361da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds } 3371da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds } 3381da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 3391da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds pds = (double *)pcs; 3401da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds pdd = (double *)pcd; 3411da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 3421da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds /* Copy 8 doubles at a time */ 3431da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds while (len >= 8*sizeof(double)) { 3441da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds register double r1, r2, r3, r4, r5, r6, r7, r8; 3451da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds /* prefetch_src((char *)pds + L1_CACHE_BYTES); */ 3461da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds flddma(s_space, pds, r1, pmc_load_exc); 3471da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds flddma(s_space, pds, r2, pmc_load_exc); 3481da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds flddma(s_space, pds, r3, pmc_load_exc); 3491da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds flddma(s_space, pds, r4, pmc_load_exc); 3501da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds fstdma(d_space, r1, pdd, pmc_store_exc); 3511da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds fstdma(d_space, r2, pdd, pmc_store_exc); 3521da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds fstdma(d_space, r3, pdd, pmc_store_exc); 3531da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds fstdma(d_space, r4, pdd, pmc_store_exc); 3541da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 3551da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds#if 0 3561da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds if (L1_CACHE_BYTES <= 32) 3571da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds prefetch_src((char *)pds + L1_CACHE_BYTES); 3581da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds#endif 3591da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds flddma(s_space, pds, r5, pmc_load_exc); 3601da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds flddma(s_space, pds, r6, pmc_load_exc); 3611da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds flddma(s_space, pds, r7, pmc_load_exc); 3621da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds flddma(s_space, pds, r8, pmc_load_exc); 3631da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds fstdma(d_space, r5, pdd, pmc_store_exc); 3641da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds fstdma(d_space, r6, pdd, pmc_store_exc); 3651da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds fstdma(d_space, r7, pdd, pmc_store_exc); 3661da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds fstdma(d_space, r8, pdd, pmc_store_exc); 3671da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds len -= 8*sizeof(double); 3681da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds } 3691da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 3701da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds pws = (unsigned int *)pds; 3711da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds pwd = (unsigned int *)pdd; 3721da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 3731da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvaldsword_copy: 3741da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds while (len >= 8*sizeof(unsigned int)) { 3751da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds register unsigned int r1,r2,r3,r4,r5,r6,r7,r8; 3761da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds /* prefetch_src((char *)pws + L1_CACHE_BYTES); */ 3771da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds ldwma(s_space, pws, r1, pmc_load_exc); 3781da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds ldwma(s_space, pws, r2, pmc_load_exc); 3791da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds ldwma(s_space, pws, r3, pmc_load_exc); 3801da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds ldwma(s_space, pws, r4, pmc_load_exc); 3811da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds stwma(d_space, r1, pwd, pmc_store_exc); 3821da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds stwma(d_space, r2, pwd, pmc_store_exc); 3831da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds stwma(d_space, r3, pwd, pmc_store_exc); 3841da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds stwma(d_space, r4, pwd, pmc_store_exc); 3851da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 3861da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds ldwma(s_space, pws, r5, pmc_load_exc); 3871da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds ldwma(s_space, pws, r6, pmc_load_exc); 3881da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds ldwma(s_space, pws, r7, pmc_load_exc); 3891da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds ldwma(s_space, pws, r8, pmc_load_exc); 3901da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds stwma(d_space, r5, pwd, pmc_store_exc); 3911da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds stwma(d_space, r6, pwd, pmc_store_exc); 3921da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds stwma(d_space, r7, pwd, pmc_store_exc); 3931da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds stwma(d_space, r8, pwd, pmc_store_exc); 3941da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds len -= 8*sizeof(unsigned int); 3951da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds } 3961da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 3971da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds while (len >= 4*sizeof(unsigned int)) { 3981da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds register unsigned int r1,r2,r3,r4; 3991da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds ldwma(s_space, pws, r1, pmc_load_exc); 4001da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds ldwma(s_space, pws, r2, pmc_load_exc); 4011da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds ldwma(s_space, pws, r3, pmc_load_exc); 4021da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds ldwma(s_space, pws, r4, pmc_load_exc); 4031da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds stwma(d_space, r1, pwd, pmc_store_exc); 4041da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds stwma(d_space, r2, pwd, pmc_store_exc); 4051da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds stwma(d_space, r3, pwd, pmc_store_exc); 4061da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds stwma(d_space, r4, pwd, pmc_store_exc); 4071da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds len -= 4*sizeof(unsigned int); 4081da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds } 4091da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 4101da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds pcs = (unsigned char *)pws; 4111da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds pcd = (unsigned char *)pwd; 4121da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 4131da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvaldsbyte_copy: 4141da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds while (len) { 4151da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds /* *pcd++ = *pcs++; */ 4161da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds ldbma(s_space, pcs, t3, pmc_load_exc); 4171da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds stbma(d_space, t3, pcd, pmc_store_exc); 4181da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds len--; 4191da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds } 4201da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 4211da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds return 0; 4221da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 4231da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvaldsunaligned_copy: 4241da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds /* possibly we are aligned on a word, but not on a double... */ 4251da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds if (likely(t1 & (sizeof(unsigned int)-1)) == 0) { 4261da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds t2 = src & (sizeof(unsigned int) - 1); 4271da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 4281da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds if (unlikely(t2 != 0)) { 4291da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds t2 = sizeof(unsigned int) - t2; 4301da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds while (t2) { 4311da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds /* *pcd++ = *pcs++; */ 4321da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds ldbma(s_space, pcs, t3, pmc_load_exc); 4331da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds stbma(d_space, t3, pcd, pmc_store_exc); 4341da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds len--; 4351da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds t2--; 4361da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds } 4371da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds } 4381da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 4391da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds pws = (unsigned int *)pcs; 4401da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds pwd = (unsigned int *)pcd; 4411da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds goto word_copy; 4421da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds } 4431da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 4441da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds /* Align the destination. */ 4451da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds if (unlikely((dst & (sizeof(unsigned int) - 1)) != 0)) { 4461da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds t2 = sizeof(unsigned int) - (dst & (sizeof(unsigned int) - 1)); 4471da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds while (t2) { 4481da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds /* *pcd++ = *pcs++; */ 4491da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds ldbma(s_space, pcs, t3, pmc_load_exc); 4501da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds stbma(d_space, t3, pcd, pmc_store_exc); 4511da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds len--; 4521da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds t2--; 4531da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds } 4541da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds dst = (unsigned long)pcd; 4551da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds src = (unsigned long)pcs; 4561da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds } 4571da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 4581da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds ret = copy_dstaligned(dst, src, len / sizeof(unsigned int), 4591da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds o_dst, o_src, o_len); 4601da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds if (ret) 4611da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds return ret; 4621da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 4631da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds pcs += (len & -sizeof(unsigned int)); 4641da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds pcd += (len & -sizeof(unsigned int)); 4651da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds len %= sizeof(unsigned int); 4661da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 4671da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds preserve_branch(handle_load_error); 4681da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds preserve_branch(handle_store_error); 4691da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 4701da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds goto byte_copy; 4711da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 4721da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvaldshandle_load_error: 4731da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds __asm__ __volatile__ ("pmc_load_exc:\n"); 4741da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds d = &__get_cpu_var(exception_data); 4751da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds DPRINTF("pmc_load_exc: o_len=%lu fault_addr=%lu o_src=%lu ret=%lu\n", 4761da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds o_len, d->fault_addr, o_src, o_len - d->fault_addr + o_src); 4771da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds return o_len - d->fault_addr + o_src; 4781da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 4791da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvaldshandle_store_error: 4801da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds __asm__ __volatile__ ("pmc_store_exc:\n"); 4811da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds d = &__get_cpu_var(exception_data); 4821da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds DPRINTF("pmc_store_exc: o_len=%lu fault_addr=%lu o_dst=%lu ret=%lu\n", 4831da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds o_len, d->fault_addr, o_dst, o_len - d->fault_addr + o_dst); 4841da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds return o_len - d->fault_addr + o_dst; 4851da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds} 4861da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 4871da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds#ifdef __KERNEL__ 4881da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvaldsunsigned long copy_to_user(void __user *dst, const void *src, unsigned long len) 4891da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds{ 4901da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds mtsp(get_kernel_space(), 1); 4911da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds mtsp(get_user_space(), 2); 4921da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds return pa_memcpy((void __force *)dst, src, len); 4931da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds} 4941da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 4951da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvaldsunsigned long copy_from_user(void *dst, const void __user *src, unsigned long len) 4961da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds{ 4971da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds mtsp(get_user_space(), 1); 4981da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds mtsp(get_kernel_space(), 2); 4991da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds return pa_memcpy(dst, (void __force *)src, len); 5001da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds} 5011da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 5021da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvaldsunsigned long copy_in_user(void __user *dst, const void __user *src, unsigned long len) 5031da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds{ 5041da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds mtsp(get_user_space(), 1); 5051da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds mtsp(get_user_space(), 2); 5061da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds return pa_memcpy((void __force *)dst, (void __force *)src, len); 5071da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds} 5081da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 5091da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 5101da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvaldsvoid * memcpy(void * dst,const void *src, size_t count) 5111da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds{ 5121da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds mtsp(get_kernel_space(), 1); 5131da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds mtsp(get_kernel_space(), 2); 5141da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds pa_memcpy(dst, src, count); 5151da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds return dst; 5161da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds} 5171da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds 5181da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus TorvaldsEXPORT_SYMBOL(copy_to_user); 5191da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus TorvaldsEXPORT_SYMBOL(copy_from_user); 5201da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus TorvaldsEXPORT_SYMBOL(copy_in_user); 5211da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus TorvaldsEXPORT_SYMBOL(memcpy); 5221da177e4c3f41524e886b7f1b8a0c1fc7321cacLinus Torvalds#endif 523