1cf5adce11743e98739fcb97e76d688f0b0bc2199David S. Miller/* NG2memcpy.S: Niagara-2 optimized memcpy. 2cf5adce11743e98739fcb97e76d688f0b0bc2199David S. Miller * 3cf5adce11743e98739fcb97e76d688f0b0bc2199David S. Miller * Copyright (C) 2007 David S. Miller (davem@davemloft.net) 4cf5adce11743e98739fcb97e76d688f0b0bc2199David S. Miller */ 5cf5adce11743e98739fcb97e76d688f0b0bc2199David S. Miller 6cf5adce11743e98739fcb97e76d688f0b0bc2199David S. Miller#ifdef __KERNEL__ 7cf5adce11743e98739fcb97e76d688f0b0bc2199David S. Miller#include <asm/visasm.h> 8cf5adce11743e98739fcb97e76d688f0b0bc2199David S. Miller#include <asm/asi.h> 9cf5adce11743e98739fcb97e76d688f0b0bc2199David S. Miller#define GLOBAL_SPARE %g7 10cf5adce11743e98739fcb97e76d688f0b0bc2199David S. Miller#else 11cf5adce11743e98739fcb97e76d688f0b0bc2199David S. Miller#define ASI_PNF 0x82 12cf5adce11743e98739fcb97e76d688f0b0bc2199David S. Miller#define ASI_BLK_P 0xf0 13cf5adce11743e98739fcb97e76d688f0b0bc2199David S. Miller#define ASI_BLK_INIT_QUAD_LDD_P 0xe2 14cf5adce11743e98739fcb97e76d688f0b0bc2199David S. Miller#define FPRS_FEF 0x04 15cf5adce11743e98739fcb97e76d688f0b0bc2199David S. Miller#ifdef MEMCPY_DEBUG 16cf5adce11743e98739fcb97e76d688f0b0bc2199David S. Miller#define VISEntryHalf rd %fprs, %o5; wr %g0, FPRS_FEF, %fprs; \ 17cf5adce11743e98739fcb97e76d688f0b0bc2199David S. Miller clr %g1; clr %g2; clr %g3; subcc %g0, %g0, %g0; 18cf5adce11743e98739fcb97e76d688f0b0bc2199David S. Miller#define VISExitHalf and %o5, FPRS_FEF, %o5; wr %o5, 0x0, %fprs 19cf5adce11743e98739fcb97e76d688f0b0bc2199David S. Miller#else 20cf5adce11743e98739fcb97e76d688f0b0bc2199David S. Miller#define VISEntryHalf rd %fprs, %o5; wr %g0, FPRS_FEF, %fprs 21cf5adce11743e98739fcb97e76d688f0b0bc2199David S. Miller#define VISExitHalf and %o5, FPRS_FEF, %o5; wr %o5, 0x0, %fprs 22cf5adce11743e98739fcb97e76d688f0b0bc2199David S. Miller#endif 23cf5adce11743e98739fcb97e76d688f0b0bc2199David S. Miller#define GLOBAL_SPARE %g5 24cf5adce11743e98739fcb97e76d688f0b0bc2199David S. Miller#endif 25cf5adce11743e98739fcb97e76d688f0b0bc2199David S. Miller 26cf5adce11743e98739fcb97e76d688f0b0bc2199David S. Miller#ifndef STORE_ASI 27cf5adce11743e98739fcb97e76d688f0b0bc2199David S. Miller#ifndef SIMULATE_NIAGARA_ON_NON_NIAGARA 28cf5adce11743e98739fcb97e76d688f0b0bc2199David S. Miller#define STORE_ASI ASI_BLK_INIT_QUAD_LDD_P 29cf5adce11743e98739fcb97e76d688f0b0bc2199David S. Miller#else 30cf5adce11743e98739fcb97e76d688f0b0bc2199David S. Miller#define STORE_ASI 0x80 /* ASI_P */ 31cf5adce11743e98739fcb97e76d688f0b0bc2199David S. Miller#endif 32cf5adce11743e98739fcb97e76d688f0b0bc2199David S. Miller#endif 33cf5adce11743e98739fcb97e76d688f0b0bc2199David S. Miller 34cf5adce11743e98739fcb97e76d688f0b0bc2199David S. Miller#ifndef EX_LD 35cf5adce11743e98739fcb97e76d688f0b0bc2199David S. Miller#define EX_LD(x) x 36cf5adce11743e98739fcb97e76d688f0b0bc2199David S. Miller#endif 37cf5adce11743e98739fcb97e76d688f0b0bc2199David S. Miller 38cf5adce11743e98739fcb97e76d688f0b0bc2199David S. Miller#ifndef EX_ST 39cf5adce11743e98739fcb97e76d688f0b0bc2199David S. Miller#define EX_ST(x) x 40cf5adce11743e98739fcb97e76d688f0b0bc2199David S. Miller#endif 41cf5adce11743e98739fcb97e76d688f0b0bc2199David S. Miller 42cf5adce11743e98739fcb97e76d688f0b0bc2199David S. Miller#ifndef EX_RETVAL 43cf5adce11743e98739fcb97e76d688f0b0bc2199David S. Miller#define EX_RETVAL(x) x 44cf5adce11743e98739fcb97e76d688f0b0bc2199David S. Miller#endif 45cf5adce11743e98739fcb97e76d688f0b0bc2199David S. Miller 46cf5adce11743e98739fcb97e76d688f0b0bc2199David S. Miller#ifndef LOAD 47cf5adce11743e98739fcb97e76d688f0b0bc2199David S. Miller#define LOAD(type,addr,dest) type [addr], dest 48cf5adce11743e98739fcb97e76d688f0b0bc2199David S. Miller#endif 49cf5adce11743e98739fcb97e76d688f0b0bc2199David S. Miller 50cf5adce11743e98739fcb97e76d688f0b0bc2199David S. Miller#ifndef LOAD_BLK 51cf5adce11743e98739fcb97e76d688f0b0bc2199David S. Miller#define LOAD_BLK(addr,dest) ldda [addr] ASI_BLK_P, dest 52cf5adce11743e98739fcb97e76d688f0b0bc2199David S. Miller#endif 53cf5adce11743e98739fcb97e76d688f0b0bc2199David S. Miller 54cf5adce11743e98739fcb97e76d688f0b0bc2199David S. Miller#ifndef STORE 55cf5adce11743e98739fcb97e76d688f0b0bc2199David S. Miller#ifndef MEMCPY_DEBUG 56cf5adce11743e98739fcb97e76d688f0b0bc2199David S. Miller#define STORE(type,src,addr) type src, [addr] 57cf5adce11743e98739fcb97e76d688f0b0bc2199David S. Miller#else 58cf5adce11743e98739fcb97e76d688f0b0bc2199David S. Miller#define STORE(type,src,addr) type##a src, [addr] 0x80 59cf5adce11743e98739fcb97e76d688f0b0bc2199David S. Miller#endif 60cf5adce11743e98739fcb97e76d688f0b0bc2199David S. Miller#endif 61cf5adce11743e98739fcb97e76d688f0b0bc2199David S. Miller 62cf5adce11743e98739fcb97e76d688f0b0bc2199David S. Miller#ifndef STORE_BLK 63cf5adce11743e98739fcb97e76d688f0b0bc2199David S. Miller#define STORE_BLK(src,addr) stda src, [addr] ASI_BLK_P 64cf5adce11743e98739fcb97e76d688f0b0bc2199David S. Miller#endif 65cf5adce11743e98739fcb97e76d688f0b0bc2199David S. Miller 66cf5adce11743e98739fcb97e76d688f0b0bc2199David S. Miller#ifndef STORE_INIT 67cf5adce11743e98739fcb97e76d688f0b0bc2199David S. Miller#define STORE_INIT(src,addr) stxa src, [addr] STORE_ASI 68cf5adce11743e98739fcb97e76d688f0b0bc2199David S. Miller#endif 69cf5adce11743e98739fcb97e76d688f0b0bc2199David S. Miller 70cf5adce11743e98739fcb97e76d688f0b0bc2199David S. Miller#ifndef FUNC_NAME 71cf5adce11743e98739fcb97e76d688f0b0bc2199David S. Miller#define FUNC_NAME NG2memcpy 72cf5adce11743e98739fcb97e76d688f0b0bc2199David S. Miller#endif 73cf5adce11743e98739fcb97e76d688f0b0bc2199David S. Miller 74cf5adce11743e98739fcb97e76d688f0b0bc2199David S. Miller#ifndef PREAMBLE 75cf5adce11743e98739fcb97e76d688f0b0bc2199David S. Miller#define PREAMBLE 76cf5adce11743e98739fcb97e76d688f0b0bc2199David S. Miller#endif 77cf5adce11743e98739fcb97e76d688f0b0bc2199David S. Miller 78cf5adce11743e98739fcb97e76d688f0b0bc2199David S. Miller#ifndef XCC 79cf5adce11743e98739fcb97e76d688f0b0bc2199David S. Miller#define XCC xcc 80cf5adce11743e98739fcb97e76d688f0b0bc2199David S. Miller#endif 81cf5adce11743e98739fcb97e76d688f0b0bc2199David S. Miller 82cf5adce11743e98739fcb97e76d688f0b0bc2199David S. Miller#define FREG_FROB(x0, x1, x2, x3, x4, x5, x6, x7, x8) \ 83cf5adce11743e98739fcb97e76d688f0b0bc2199David S. Miller faligndata %x0, %x1, %f0; \ 84cf5adce11743e98739fcb97e76d688f0b0bc2199David S. Miller faligndata %x1, %x2, %f2; \ 85cf5adce11743e98739fcb97e76d688f0b0bc2199David S. Miller faligndata %x2, %x3, %f4; \ 86cf5adce11743e98739fcb97e76d688f0b0bc2199David S. Miller faligndata %x3, %x4, %f6; \ 87cf5adce11743e98739fcb97e76d688f0b0bc2199David S. Miller faligndata %x4, %x5, %f8; \ 88cf5adce11743e98739fcb97e76d688f0b0bc2199David S. Miller faligndata %x5, %x6, %f10; \ 89cf5adce11743e98739fcb97e76d688f0b0bc2199David S. Miller faligndata %x6, %x7, %f12; \ 90cf5adce11743e98739fcb97e76d688f0b0bc2199David S. Miller faligndata %x7, %x8, %f14; 91cf5adce11743e98739fcb97e76d688f0b0bc2199David S. Miller 92cf5adce11743e98739fcb97e76d688f0b0bc2199David S. Miller#define FREG_MOVE_1(x0) \ 93cf5adce11743e98739fcb97e76d688f0b0bc2199David S. Miller fmovd %x0, %f0; 94cf5adce11743e98739fcb97e76d688f0b0bc2199David S. Miller#define FREG_MOVE_2(x0, x1) \ 95cf5adce11743e98739fcb97e76d688f0b0bc2199David S. Miller fmovd %x0, %f0; \ 96cf5adce11743e98739fcb97e76d688f0b0bc2199David S. Miller fmovd %x1, %f2; 97cf5adce11743e98739fcb97e76d688f0b0bc2199David S. Miller#define FREG_MOVE_3(x0, x1, x2) \ 98cf5adce11743e98739fcb97e76d688f0b0bc2199David S. Miller fmovd %x0, %f0; \ 99cf5adce11743e98739fcb97e76d688f0b0bc2199David S. Miller fmovd %x1, %f2; \ 100cf5adce11743e98739fcb97e76d688f0b0bc2199David S. Miller fmovd %x2, %f4; 101cf5adce11743e98739fcb97e76d688f0b0bc2199David S. Miller#define FREG_MOVE_4(x0, x1, x2, x3) \ 102cf5adce11743e98739fcb97e76d688f0b0bc2199David S. Miller fmovd %x0, %f0; \ 103cf5adce11743e98739fcb97e76d688f0b0bc2199David S. Miller fmovd %x1, %f2; \ 104cf5adce11743e98739fcb97e76d688f0b0bc2199David S. Miller fmovd %x2, %f4; \ 105cf5adce11743e98739fcb97e76d688f0b0bc2199David S. Miller fmovd %x3, %f6; 106cf5adce11743e98739fcb97e76d688f0b0bc2199David S. Miller#define FREG_MOVE_5(x0, x1, x2, x3, x4) \ 107cf5adce11743e98739fcb97e76d688f0b0bc2199David S. Miller fmovd %x0, %f0; \ 108cf5adce11743e98739fcb97e76d688f0b0bc2199David S. Miller fmovd %x1, %f2; \ 109cf5adce11743e98739fcb97e76d688f0b0bc2199David S. Miller fmovd %x2, %f4; \ 110cf5adce11743e98739fcb97e76d688f0b0bc2199David S. Miller fmovd %x3, %f6; \ 111cf5adce11743e98739fcb97e76d688f0b0bc2199David S. Miller fmovd %x4, %f8; 112cf5adce11743e98739fcb97e76d688f0b0bc2199David S. Miller#define FREG_MOVE_6(x0, x1, x2, x3, x4, x5) \ 113cf5adce11743e98739fcb97e76d688f0b0bc2199David S. Miller fmovd %x0, %f0; \ 114cf5adce11743e98739fcb97e76d688f0b0bc2199David S. Miller fmovd %x1, %f2; \ 115cf5adce11743e98739fcb97e76d688f0b0bc2199David S. Miller fmovd %x2, %f4; \ 116cf5adce11743e98739fcb97e76d688f0b0bc2199David S. Miller fmovd %x3, %f6; \ 117cf5adce11743e98739fcb97e76d688f0b0bc2199David S. Miller fmovd %x4, %f8; \ 118cf5adce11743e98739fcb97e76d688f0b0bc2199David S. Miller fmovd %x5, %f10; 119cf5adce11743e98739fcb97e76d688f0b0bc2199David S. Miller#define FREG_MOVE_7(x0, x1, x2, x3, x4, x5, x6) \ 120cf5adce11743e98739fcb97e76d688f0b0bc2199David S. Miller fmovd %x0, %f0; \ 121cf5adce11743e98739fcb97e76d688f0b0bc2199David S. Miller fmovd %x1, %f2; \ 122cf5adce11743e98739fcb97e76d688f0b0bc2199David S. Miller fmovd %x2, %f4; \ 123cf5adce11743e98739fcb97e76d688f0b0bc2199David S. Miller fmovd %x3, %f6; \ 124cf5adce11743e98739fcb97e76d688f0b0bc2199David S. Miller fmovd %x4, %f8; \ 125cf5adce11743e98739fcb97e76d688f0b0bc2199David S. Miller fmovd %x5, %f10; \ 126cf5adce11743e98739fcb97e76d688f0b0bc2199David S. Miller fmovd %x6, %f12; 127cf5adce11743e98739fcb97e76d688f0b0bc2199David S. Miller#define FREG_MOVE_8(x0, x1, x2, x3, x4, x5, x6, x7) \ 128cf5adce11743e98739fcb97e76d688f0b0bc2199David S. Miller fmovd %x0, %f0; \ 129cf5adce11743e98739fcb97e76d688f0b0bc2199David S. Miller fmovd %x1, %f2; \ 130cf5adce11743e98739fcb97e76d688f0b0bc2199David S. Miller fmovd %x2, %f4; \ 131cf5adce11743e98739fcb97e76d688f0b0bc2199David S. Miller fmovd %x3, %f6; \ 132cf5adce11743e98739fcb97e76d688f0b0bc2199David S. Miller fmovd %x4, %f8; \ 133cf5adce11743e98739fcb97e76d688f0b0bc2199David S. Miller fmovd %x5, %f10; \ 134cf5adce11743e98739fcb97e76d688f0b0bc2199David S. Miller fmovd %x6, %f12; \ 135cf5adce11743e98739fcb97e76d688f0b0bc2199David S. Miller fmovd %x7, %f14; 136cf5adce11743e98739fcb97e76d688f0b0bc2199David S. Miller#define FREG_LOAD_1(base, x0) \ 137cf5adce11743e98739fcb97e76d688f0b0bc2199David S. Miller EX_LD(LOAD(ldd, base + 0x00, %x0)) 138cf5adce11743e98739fcb97e76d688f0b0bc2199David S. Miller#define FREG_LOAD_2(base, x0, x1) \ 139cf5adce11743e98739fcb97e76d688f0b0bc2199David S. Miller EX_LD(LOAD(ldd, base + 0x00, %x0)); \ 140cf5adce11743e98739fcb97e76d688f0b0bc2199David S. Miller EX_LD(LOAD(ldd, base + 0x08, %x1)); 141cf5adce11743e98739fcb97e76d688f0b0bc2199David S. Miller#define FREG_LOAD_3(base, x0, x1, x2) \ 142cf5adce11743e98739fcb97e76d688f0b0bc2199David S. Miller EX_LD(LOAD(ldd, base + 0x00, %x0)); \ 143cf5adce11743e98739fcb97e76d688f0b0bc2199David S. Miller EX_LD(LOAD(ldd, base + 0x08, %x1)); \ 144cf5adce11743e98739fcb97e76d688f0b0bc2199David S. Miller EX_LD(LOAD(ldd, base + 0x10, %x2)); 145cf5adce11743e98739fcb97e76d688f0b0bc2199David S. Miller#define FREG_LOAD_4(base, x0, x1, x2, x3) \ 146cf5adce11743e98739fcb97e76d688f0b0bc2199David S. Miller EX_LD(LOAD(ldd, base + 0x00, %x0)); \ 147cf5adce11743e98739fcb97e76d688f0b0bc2199David S. Miller EX_LD(LOAD(ldd, base + 0x08, %x1)); \ 148cf5adce11743e98739fcb97e76d688f0b0bc2199David S. Miller EX_LD(LOAD(ldd, base + 0x10, %x2)); \ 149cf5adce11743e98739fcb97e76d688f0b0bc2199David S. Miller EX_LD(LOAD(ldd, base + 0x18, %x3)); 150cf5adce11743e98739fcb97e76d688f0b0bc2199David S. Miller#define FREG_LOAD_5(base, x0, x1, x2, x3, x4) \ 151cf5adce11743e98739fcb97e76d688f0b0bc2199David S. Miller EX_LD(LOAD(ldd, base + 0x00, %x0)); \ 152cf5adce11743e98739fcb97e76d688f0b0bc2199David S. Miller EX_LD(LOAD(ldd, base + 0x08, %x1)); \ 153cf5adce11743e98739fcb97e76d688f0b0bc2199David S. Miller EX_LD(LOAD(ldd, base + 0x10, %x2)); \ 154cf5adce11743e98739fcb97e76d688f0b0bc2199David S. Miller EX_LD(LOAD(ldd, base + 0x18, %x3)); \ 155cf5adce11743e98739fcb97e76d688f0b0bc2199David S. Miller EX_LD(LOAD(ldd, base + 0x20, %x4)); 156cf5adce11743e98739fcb97e76d688f0b0bc2199David S. Miller#define FREG_LOAD_6(base, x0, x1, x2, x3, x4, x5) \ 157cf5adce11743e98739fcb97e76d688f0b0bc2199David S. Miller EX_LD(LOAD(ldd, base + 0x00, %x0)); \ 158cf5adce11743e98739fcb97e76d688f0b0bc2199David S. Miller EX_LD(LOAD(ldd, base + 0x08, %x1)); \ 159cf5adce11743e98739fcb97e76d688f0b0bc2199David S. Miller EX_LD(LOAD(ldd, base + 0x10, %x2)); \ 160cf5adce11743e98739fcb97e76d688f0b0bc2199David S. Miller EX_LD(LOAD(ldd, base + 0x18, %x3)); \ 161cf5adce11743e98739fcb97e76d688f0b0bc2199David S. Miller EX_LD(LOAD(ldd, base + 0x20, %x4)); \ 162cf5adce11743e98739fcb97e76d688f0b0bc2199David S. Miller EX_LD(LOAD(ldd, base + 0x28, %x5)); 163cf5adce11743e98739fcb97e76d688f0b0bc2199David S. Miller#define FREG_LOAD_7(base, x0, x1, x2, x3, x4, x5, x6) \ 164cf5adce11743e98739fcb97e76d688f0b0bc2199David S. Miller EX_LD(LOAD(ldd, base + 0x00, %x0)); \ 165cf5adce11743e98739fcb97e76d688f0b0bc2199David S. Miller EX_LD(LOAD(ldd, base + 0x08, %x1)); \ 166cf5adce11743e98739fcb97e76d688f0b0bc2199David S. Miller EX_LD(LOAD(ldd, base + 0x10, %x2)); \ 167cf5adce11743e98739fcb97e76d688f0b0bc2199David S. Miller EX_LD(LOAD(ldd, base + 0x18, %x3)); \ 168cf5adce11743e98739fcb97e76d688f0b0bc2199David S. Miller EX_LD(LOAD(ldd, base + 0x20, %x4)); \ 169cf5adce11743e98739fcb97e76d688f0b0bc2199David S. Miller EX_LD(LOAD(ldd, base + 0x28, %x5)); \ 170cf5adce11743e98739fcb97e76d688f0b0bc2199David S. Miller EX_LD(LOAD(ldd, base + 0x30, %x6)); 171cf5adce11743e98739fcb97e76d688f0b0bc2199David S. Miller 172cf5adce11743e98739fcb97e76d688f0b0bc2199David S. Miller .register %g2,#scratch 173cf5adce11743e98739fcb97e76d688f0b0bc2199David S. Miller .register %g3,#scratch 174cf5adce11743e98739fcb97e76d688f0b0bc2199David S. Miller 175cf5adce11743e98739fcb97e76d688f0b0bc2199David S. Miller .text 176cf5adce11743e98739fcb97e76d688f0b0bc2199David S. Miller .align 64 177cf5adce11743e98739fcb97e76d688f0b0bc2199David S. Miller 178cf5adce11743e98739fcb97e76d688f0b0bc2199David S. Miller .globl FUNC_NAME 179cf5adce11743e98739fcb97e76d688f0b0bc2199David S. Miller .type FUNC_NAME,#function 180cf5adce11743e98739fcb97e76d688f0b0bc2199David S. MillerFUNC_NAME: /* %o0=dst, %o1=src, %o2=len */ 181cf5adce11743e98739fcb97e76d688f0b0bc2199David S. Miller srlx %o2, 31, %g2 182cf5adce11743e98739fcb97e76d688f0b0bc2199David S. Miller cmp %g2, 0 183cf5adce11743e98739fcb97e76d688f0b0bc2199David S. Miller tne %xcc, 5 184cf5adce11743e98739fcb97e76d688f0b0bc2199David S. Miller PREAMBLE 185cf5adce11743e98739fcb97e76d688f0b0bc2199David S. Miller mov %o0, GLOBAL_SPARE 186cf5adce11743e98739fcb97e76d688f0b0bc2199David S. Miller cmp %o2, 0 187cf5adce11743e98739fcb97e76d688f0b0bc2199David S. Miller be,pn %XCC, 85f 188cf5adce11743e98739fcb97e76d688f0b0bc2199David S. Miller or %o0, %o1, %o3 189cf5adce11743e98739fcb97e76d688f0b0bc2199David S. Miller cmp %o2, 16 190cf5adce11743e98739fcb97e76d688f0b0bc2199David S. Miller blu,a,pn %XCC, 80f 191cf5adce11743e98739fcb97e76d688f0b0bc2199David S. Miller or %o3, %o2, %o3 192cf5adce11743e98739fcb97e76d688f0b0bc2199David S. Miller 193cf5adce11743e98739fcb97e76d688f0b0bc2199David S. Miller /* 2 blocks (128 bytes) is the minimum we can do the block 194cf5adce11743e98739fcb97e76d688f0b0bc2199David S. Miller * copy with. We need to ensure that we'll iterate at least 195cf5adce11743e98739fcb97e76d688f0b0bc2199David S. Miller * once in the block copy loop. At worst we'll need to align 196cf5adce11743e98739fcb97e76d688f0b0bc2199David S. Miller * the destination to a 64-byte boundary which can chew up 197cf5adce11743e98739fcb97e76d688f0b0bc2199David S. Miller * to (64 - 1) bytes from the length before we perform the 198cf5adce11743e98739fcb97e76d688f0b0bc2199David S. Miller * block copy loop. 199cf5adce11743e98739fcb97e76d688f0b0bc2199David S. Miller * 200cf5adce11743e98739fcb97e76d688f0b0bc2199David S. Miller * However, the cut-off point, performance wise, is around 201cf5adce11743e98739fcb97e76d688f0b0bc2199David S. Miller * 4 64-byte blocks. 202cf5adce11743e98739fcb97e76d688f0b0bc2199David S. Miller */ 203cf5adce11743e98739fcb97e76d688f0b0bc2199David S. Miller cmp %o2, (4 * 64) 204cf5adce11743e98739fcb97e76d688f0b0bc2199David S. Miller blu,pt %XCC, 75f 205cf5adce11743e98739fcb97e76d688f0b0bc2199David S. Miller andcc %o3, 0x7, %g0 206cf5adce11743e98739fcb97e76d688f0b0bc2199David S. Miller 207cf5adce11743e98739fcb97e76d688f0b0bc2199David S. Miller /* %o0: dst 208cf5adce11743e98739fcb97e76d688f0b0bc2199David S. Miller * %o1: src 209cf5adce11743e98739fcb97e76d688f0b0bc2199David S. Miller * %o2: len (known to be >= 128) 210cf5adce11743e98739fcb97e76d688f0b0bc2199David S. Miller * 211cf5adce11743e98739fcb97e76d688f0b0bc2199David S. Miller * The block copy loops can use %o4, %g2, %g3 as 212cf5adce11743e98739fcb97e76d688f0b0bc2199David S. Miller * temporaries while copying the data. %o5 must 213cf5adce11743e98739fcb97e76d688f0b0bc2199David S. Miller * be preserved between VISEntryHalf and VISExitHalf 214cf5adce11743e98739fcb97e76d688f0b0bc2199David S. Miller */ 215cf5adce11743e98739fcb97e76d688f0b0bc2199David S. Miller 216cf5adce11743e98739fcb97e76d688f0b0bc2199David S. Miller LOAD(prefetch, %o1 + 0x000, #one_read) 217cf5adce11743e98739fcb97e76d688f0b0bc2199David S. Miller LOAD(prefetch, %o1 + 0x040, #one_read) 218cf5adce11743e98739fcb97e76d688f0b0bc2199David S. Miller LOAD(prefetch, %o1 + 0x080, #one_read) 219cf5adce11743e98739fcb97e76d688f0b0bc2199David S. Miller 220cf5adce11743e98739fcb97e76d688f0b0bc2199David S. Miller /* Align destination on 64-byte boundary. */ 221cf5adce11743e98739fcb97e76d688f0b0bc2199David S. Miller andcc %o0, (64 - 1), %o4 222cf5adce11743e98739fcb97e76d688f0b0bc2199David S. Miller be,pt %XCC, 2f 223cf5adce11743e98739fcb97e76d688f0b0bc2199David S. Miller sub %o4, 64, %o4 224cf5adce11743e98739fcb97e76d688f0b0bc2199David S. Miller sub %g0, %o4, %o4 ! bytes to align dst 225cf5adce11743e98739fcb97e76d688f0b0bc2199David S. Miller sub %o2, %o4, %o2 226cf5adce11743e98739fcb97e76d688f0b0bc2199David S. Miller1: subcc %o4, 1, %o4 227cf5adce11743e98739fcb97e76d688f0b0bc2199David S. Miller EX_LD(LOAD(ldub, %o1, %g1)) 228cf5adce11743e98739fcb97e76d688f0b0bc2199David S. Miller EX_ST(STORE(stb, %g1, %o0)) 229cf5adce11743e98739fcb97e76d688f0b0bc2199David S. Miller add %o1, 1, %o1 230cf5adce11743e98739fcb97e76d688f0b0bc2199David S. Miller bne,pt %XCC, 1b 231cf5adce11743e98739fcb97e76d688f0b0bc2199David S. Miller add %o0, 1, %o0 232cf5adce11743e98739fcb97e76d688f0b0bc2199David S. Miller 233cf5adce11743e98739fcb97e76d688f0b0bc2199David S. Miller2: 234cf5adce11743e98739fcb97e76d688f0b0bc2199David S. Miller /* Clobbers o5/g1/g2/g3/g7/icc/xcc. We must preserve 235cf5adce11743e98739fcb97e76d688f0b0bc2199David S. Miller * o5 from here until we hit VISExitHalf. 236cf5adce11743e98739fcb97e76d688f0b0bc2199David S. Miller */ 237cf5adce11743e98739fcb97e76d688f0b0bc2199David S. Miller VISEntryHalf 238cf5adce11743e98739fcb97e76d688f0b0bc2199David S. Miller 239cf5adce11743e98739fcb97e76d688f0b0bc2199David S. Miller alignaddr %o1, %g0, %g0 240cf5adce11743e98739fcb97e76d688f0b0bc2199David S. Miller 241cf5adce11743e98739fcb97e76d688f0b0bc2199David S. Miller add %o1, (64 - 1), %o4 242cf5adce11743e98739fcb97e76d688f0b0bc2199David S. Miller andn %o4, (64 - 1), %o4 243cf5adce11743e98739fcb97e76d688f0b0bc2199David S. Miller andn %o2, (64 - 1), %g1 244cf5adce11743e98739fcb97e76d688f0b0bc2199David S. Miller sub %o2, %g1, %o2 245cf5adce11743e98739fcb97e76d688f0b0bc2199David S. Miller 246cf5adce11743e98739fcb97e76d688f0b0bc2199David S. Miller and %o1, (64 - 1), %g2 247cf5adce11743e98739fcb97e76d688f0b0bc2199David S. Miller add %o1, %g1, %o1 248cf5adce11743e98739fcb97e76d688f0b0bc2199David S. Miller sub %o0, %o4, %g3 249cf5adce11743e98739fcb97e76d688f0b0bc2199David S. Miller brz,pt %g2, 190f 250cf5adce11743e98739fcb97e76d688f0b0bc2199David S. Miller cmp %g2, 32 251cf5adce11743e98739fcb97e76d688f0b0bc2199David S. Miller blu,a 5f 252cf5adce11743e98739fcb97e76d688f0b0bc2199David S. Miller cmp %g2, 16 253cf5adce11743e98739fcb97e76d688f0b0bc2199David S. Miller cmp %g2, 48 254cf5adce11743e98739fcb97e76d688f0b0bc2199David S. Miller blu,a 4f 255cf5adce11743e98739fcb97e76d688f0b0bc2199David S. Miller cmp %g2, 40 256cf5adce11743e98739fcb97e76d688f0b0bc2199David S. Miller cmp %g2, 56 257cf5adce11743e98739fcb97e76d688f0b0bc2199David S. Miller blu 170f 258cf5adce11743e98739fcb97e76d688f0b0bc2199David S. Miller nop 259cf5adce11743e98739fcb97e76d688f0b0bc2199David S. Miller ba,a,pt %xcc, 180f 260cf5adce11743e98739fcb97e76d688f0b0bc2199David S. Miller 261cf5adce11743e98739fcb97e76d688f0b0bc2199David S. Miller4: /* 32 <= low bits < 48 */ 262cf5adce11743e98739fcb97e76d688f0b0bc2199David S. Miller blu 150f 263cf5adce11743e98739fcb97e76d688f0b0bc2199David S. Miller nop 264cf5adce11743e98739fcb97e76d688f0b0bc2199David S. Miller ba,a,pt %xcc, 160f 265cf5adce11743e98739fcb97e76d688f0b0bc2199David S. Miller5: /* 0 < low bits < 32 */ 266cf5adce11743e98739fcb97e76d688f0b0bc2199David S. Miller blu,a 6f 267cf5adce11743e98739fcb97e76d688f0b0bc2199David S. Miller cmp %g2, 8 268cf5adce11743e98739fcb97e76d688f0b0bc2199David S. Miller cmp %g2, 24 269cf5adce11743e98739fcb97e76d688f0b0bc2199David S. Miller blu 130f 270cf5adce11743e98739fcb97e76d688f0b0bc2199David S. Miller nop 271cf5adce11743e98739fcb97e76d688f0b0bc2199David S. Miller ba,a,pt %xcc, 140f 272cf5adce11743e98739fcb97e76d688f0b0bc2199David S. Miller6: /* 0 < low bits < 16 */ 273cf5adce11743e98739fcb97e76d688f0b0bc2199David S. Miller bgeu 120f 274cf5adce11743e98739fcb97e76d688f0b0bc2199David S. Miller nop 275cf5adce11743e98739fcb97e76d688f0b0bc2199David S. Miller /* fall through for 0 < low bits < 8 */ 276cf5adce11743e98739fcb97e76d688f0b0bc2199David S. Miller110: sub %o4, 64, %g2 277cf5adce11743e98739fcb97e76d688f0b0bc2199David S. Miller EX_LD(LOAD_BLK(%g2, %f0)) 278cf5adce11743e98739fcb97e76d688f0b0bc2199David S. Miller1: EX_ST(STORE_INIT(%g0, %o4 + %g3)) 279cf5adce11743e98739fcb97e76d688f0b0bc2199David S. Miller EX_LD(LOAD_BLK(%o4, %f16)) 280cf5adce11743e98739fcb97e76d688f0b0bc2199David S. Miller FREG_FROB(f0, f2, f4, f6, f8, f10, f12, f14, f16) 281cf5adce11743e98739fcb97e76d688f0b0bc2199David S. Miller EX_ST(STORE_BLK(%f0, %o4 + %g3)) 282cf5adce11743e98739fcb97e76d688f0b0bc2199David S. Miller FREG_MOVE_8(f16, f18, f20, f22, f24, f26, f28, f30) 283cf5adce11743e98739fcb97e76d688f0b0bc2199David S. Miller subcc %g1, 64, %g1 284cf5adce11743e98739fcb97e76d688f0b0bc2199David S. Miller add %o4, 64, %o4 285cf5adce11743e98739fcb97e76d688f0b0bc2199David S. Miller bne,pt %xcc, 1b 286cf5adce11743e98739fcb97e76d688f0b0bc2199David S. Miller LOAD(prefetch, %o4 + 64, #one_read) 287cf5adce11743e98739fcb97e76d688f0b0bc2199David S. Miller ba,pt %xcc, 195f 288cf5adce11743e98739fcb97e76d688f0b0bc2199David S. Miller nop 289cf5adce11743e98739fcb97e76d688f0b0bc2199David S. Miller 290cf5adce11743e98739fcb97e76d688f0b0bc2199David S. Miller120: sub %o4, 56, %g2 291cf5adce11743e98739fcb97e76d688f0b0bc2199David S. Miller FREG_LOAD_7(%g2, f0, f2, f4, f6, f8, f10, f12) 292cf5adce11743e98739fcb97e76d688f0b0bc2199David S. Miller1: EX_ST(STORE_INIT(%g0, %o4 + %g3)) 293cf5adce11743e98739fcb97e76d688f0b0bc2199David S. Miller EX_LD(LOAD_BLK(%o4, %f16)) 294cf5adce11743e98739fcb97e76d688f0b0bc2199David S. Miller FREG_FROB(f0, f2, f4, f6, f8, f10, f12, f16, f18) 295cf5adce11743e98739fcb97e76d688f0b0bc2199David S. Miller EX_ST(STORE_BLK(%f0, %o4 + %g3)) 296cf5adce11743e98739fcb97e76d688f0b0bc2199David S. Miller FREG_MOVE_7(f18, f20, f22, f24, f26, f28, f30) 297cf5adce11743e98739fcb97e76d688f0b0bc2199David S. Miller subcc %g1, 64, %g1 298cf5adce11743e98739fcb97e76d688f0b0bc2199David S. Miller add %o4, 64, %o4 299cf5adce11743e98739fcb97e76d688f0b0bc2199David S. Miller bne,pt %xcc, 1b 300cf5adce11743e98739fcb97e76d688f0b0bc2199David S. Miller LOAD(prefetch, %o4 + 64, #one_read) 301cf5adce11743e98739fcb97e76d688f0b0bc2199David S. Miller ba,pt %xcc, 195f 302cf5adce11743e98739fcb97e76d688f0b0bc2199David S. Miller nop 303cf5adce11743e98739fcb97e76d688f0b0bc2199David S. Miller 304cf5adce11743e98739fcb97e76d688f0b0bc2199David S. Miller130: sub %o4, 48, %g2 305cf5adce11743e98739fcb97e76d688f0b0bc2199David S. Miller FREG_LOAD_6(%g2, f0, f2, f4, f6, f8, f10) 306cf5adce11743e98739fcb97e76d688f0b0bc2199David S. Miller1: EX_ST(STORE_INIT(%g0, %o4 + %g3)) 307cf5adce11743e98739fcb97e76d688f0b0bc2199David S. Miller EX_LD(LOAD_BLK(%o4, %f16)) 308cf5adce11743e98739fcb97e76d688f0b0bc2199David S. Miller FREG_FROB(f0, f2, f4, f6, f8, f10, f16, f18, f20) 309cf5adce11743e98739fcb97e76d688f0b0bc2199David S. Miller EX_ST(STORE_BLK(%f0, %o4 + %g3)) 310cf5adce11743e98739fcb97e76d688f0b0bc2199David S. Miller FREG_MOVE_6(f20, f22, f24, f26, f28, f30) 311cf5adce11743e98739fcb97e76d688f0b0bc2199David S. Miller subcc %g1, 64, %g1 312cf5adce11743e98739fcb97e76d688f0b0bc2199David S. Miller add %o4, 64, %o4 313cf5adce11743e98739fcb97e76d688f0b0bc2199David S. Miller bne,pt %xcc, 1b 314cf5adce11743e98739fcb97e76d688f0b0bc2199David S. Miller LOAD(prefetch, %o4 + 64, #one_read) 315cf5adce11743e98739fcb97e76d688f0b0bc2199David S. Miller ba,pt %xcc, 195f 316cf5adce11743e98739fcb97e76d688f0b0bc2199David S. Miller nop 317cf5adce11743e98739fcb97e76d688f0b0bc2199David S. Miller 318cf5adce11743e98739fcb97e76d688f0b0bc2199David S. Miller140: sub %o4, 40, %g2 319cf5adce11743e98739fcb97e76d688f0b0bc2199David S. Miller FREG_LOAD_5(%g2, f0, f2, f4, f6, f8) 320cf5adce11743e98739fcb97e76d688f0b0bc2199David S. Miller1: EX_ST(STORE_INIT(%g0, %o4 + %g3)) 321cf5adce11743e98739fcb97e76d688f0b0bc2199David S. Miller EX_LD(LOAD_BLK(%o4, %f16)) 322cf5adce11743e98739fcb97e76d688f0b0bc2199David S. Miller FREG_FROB(f0, f2, f4, f6, f8, f16, f18, f20, f22) 323cf5adce11743e98739fcb97e76d688f0b0bc2199David S. Miller EX_ST(STORE_BLK(%f0, %o4 + %g3)) 324cf5adce11743e98739fcb97e76d688f0b0bc2199David S. Miller FREG_MOVE_5(f22, f24, f26, f28, f30) 325cf5adce11743e98739fcb97e76d688f0b0bc2199David S. Miller subcc %g1, 64, %g1 326cf5adce11743e98739fcb97e76d688f0b0bc2199David S. Miller add %o4, 64, %o4 327cf5adce11743e98739fcb97e76d688f0b0bc2199David S. Miller bne,pt %xcc, 1b 328cf5adce11743e98739fcb97e76d688f0b0bc2199David S. Miller LOAD(prefetch, %o4 + 64, #one_read) 329cf5adce11743e98739fcb97e76d688f0b0bc2199David S. Miller ba,pt %xcc, 195f 330cf5adce11743e98739fcb97e76d688f0b0bc2199David S. Miller nop 331cf5adce11743e98739fcb97e76d688f0b0bc2199David S. Miller 332cf5adce11743e98739fcb97e76d688f0b0bc2199David S. Miller150: sub %o4, 32, %g2 333cf5adce11743e98739fcb97e76d688f0b0bc2199David S. Miller FREG_LOAD_4(%g2, f0, f2, f4, f6) 334cf5adce11743e98739fcb97e76d688f0b0bc2199David S. Miller1: EX_ST(STORE_INIT(%g0, %o4 + %g3)) 335cf5adce11743e98739fcb97e76d688f0b0bc2199David S. Miller EX_LD(LOAD_BLK(%o4, %f16)) 336cf5adce11743e98739fcb97e76d688f0b0bc2199David S. Miller FREG_FROB(f0, f2, f4, f6, f16, f18, f20, f22, f24) 337cf5adce11743e98739fcb97e76d688f0b0bc2199David S. Miller EX_ST(STORE_BLK(%f0, %o4 + %g3)) 338cf5adce11743e98739fcb97e76d688f0b0bc2199David S. Miller FREG_MOVE_4(f24, f26, f28, f30) 339cf5adce11743e98739fcb97e76d688f0b0bc2199David S. Miller subcc %g1, 64, %g1 340cf5adce11743e98739fcb97e76d688f0b0bc2199David S. Miller add %o4, 64, %o4 341cf5adce11743e98739fcb97e76d688f0b0bc2199David S. Miller bne,pt %xcc, 1b 342cf5adce11743e98739fcb97e76d688f0b0bc2199David S. Miller LOAD(prefetch, %o4 + 64, #one_read) 343cf5adce11743e98739fcb97e76d688f0b0bc2199David S. Miller ba,pt %xcc, 195f 344cf5adce11743e98739fcb97e76d688f0b0bc2199David S. Miller nop 345cf5adce11743e98739fcb97e76d688f0b0bc2199David S. Miller 346cf5adce11743e98739fcb97e76d688f0b0bc2199David S. Miller160: sub %o4, 24, %g2 347cf5adce11743e98739fcb97e76d688f0b0bc2199David S. Miller FREG_LOAD_3(%g2, f0, f2, f4) 348cf5adce11743e98739fcb97e76d688f0b0bc2199David S. Miller1: EX_ST(STORE_INIT(%g0, %o4 + %g3)) 349cf5adce11743e98739fcb97e76d688f0b0bc2199David S. Miller EX_LD(LOAD_BLK(%o4, %f16)) 350cf5adce11743e98739fcb97e76d688f0b0bc2199David S. Miller FREG_FROB(f0, f2, f4, f16, f18, f20, f22, f24, f26) 351cf5adce11743e98739fcb97e76d688f0b0bc2199David S. Miller EX_ST(STORE_BLK(%f0, %o4 + %g3)) 352cf5adce11743e98739fcb97e76d688f0b0bc2199David S. Miller FREG_MOVE_3(f26, f28, f30) 353cf5adce11743e98739fcb97e76d688f0b0bc2199David S. Miller subcc %g1, 64, %g1 354cf5adce11743e98739fcb97e76d688f0b0bc2199David S. Miller add %o4, 64, %o4 355cf5adce11743e98739fcb97e76d688f0b0bc2199David S. Miller bne,pt %xcc, 1b 356cf5adce11743e98739fcb97e76d688f0b0bc2199David S. Miller LOAD(prefetch, %o4 + 64, #one_read) 357cf5adce11743e98739fcb97e76d688f0b0bc2199David S. Miller ba,pt %xcc, 195f 358cf5adce11743e98739fcb97e76d688f0b0bc2199David S. Miller nop 359cf5adce11743e98739fcb97e76d688f0b0bc2199David S. Miller 360cf5adce11743e98739fcb97e76d688f0b0bc2199David S. Miller170: sub %o4, 16, %g2 361cf5adce11743e98739fcb97e76d688f0b0bc2199David S. Miller FREG_LOAD_2(%g2, f0, f2) 362cf5adce11743e98739fcb97e76d688f0b0bc2199David S. Miller1: EX_ST(STORE_INIT(%g0, %o4 + %g3)) 363cf5adce11743e98739fcb97e76d688f0b0bc2199David S. Miller EX_LD(LOAD_BLK(%o4, %f16)) 364cf5adce11743e98739fcb97e76d688f0b0bc2199David S. Miller FREG_FROB(f0, f2, f16, f18, f20, f22, f24, f26, f28) 365cf5adce11743e98739fcb97e76d688f0b0bc2199David S. Miller EX_ST(STORE_BLK(%f0, %o4 + %g3)) 366cf5adce11743e98739fcb97e76d688f0b0bc2199David S. Miller FREG_MOVE_2(f28, f30) 367cf5adce11743e98739fcb97e76d688f0b0bc2199David S. Miller subcc %g1, 64, %g1 368cf5adce11743e98739fcb97e76d688f0b0bc2199David S. Miller add %o4, 64, %o4 369cf5adce11743e98739fcb97e76d688f0b0bc2199David S. Miller bne,pt %xcc, 1b 370cf5adce11743e98739fcb97e76d688f0b0bc2199David S. Miller LOAD(prefetch, %o4 + 64, #one_read) 371cf5adce11743e98739fcb97e76d688f0b0bc2199David S. Miller ba,pt %xcc, 195f 372cf5adce11743e98739fcb97e76d688f0b0bc2199David S. Miller nop 373cf5adce11743e98739fcb97e76d688f0b0bc2199David S. Miller 374cf5adce11743e98739fcb97e76d688f0b0bc2199David S. Miller180: sub %o4, 8, %g2 375cf5adce11743e98739fcb97e76d688f0b0bc2199David S. Miller FREG_LOAD_1(%g2, f0) 376cf5adce11743e98739fcb97e76d688f0b0bc2199David S. Miller1: EX_ST(STORE_INIT(%g0, %o4 + %g3)) 377cf5adce11743e98739fcb97e76d688f0b0bc2199David S. Miller EX_LD(LOAD_BLK(%o4, %f16)) 378cf5adce11743e98739fcb97e76d688f0b0bc2199David S. Miller FREG_FROB(f0, f16, f18, f20, f22, f24, f26, f28, f30) 379cf5adce11743e98739fcb97e76d688f0b0bc2199David S. Miller EX_ST(STORE_BLK(%f0, %o4 + %g3)) 380cf5adce11743e98739fcb97e76d688f0b0bc2199David S. Miller FREG_MOVE_1(f30) 381cf5adce11743e98739fcb97e76d688f0b0bc2199David S. Miller subcc %g1, 64, %g1 382cf5adce11743e98739fcb97e76d688f0b0bc2199David S. Miller add %o4, 64, %o4 383cf5adce11743e98739fcb97e76d688f0b0bc2199David S. Miller bne,pt %xcc, 1b 384cf5adce11743e98739fcb97e76d688f0b0bc2199David S. Miller LOAD(prefetch, %o4 + 64, #one_read) 385cf5adce11743e98739fcb97e76d688f0b0bc2199David S. Miller ba,pt %xcc, 195f 386cf5adce11743e98739fcb97e76d688f0b0bc2199David S. Miller nop 387cf5adce11743e98739fcb97e76d688f0b0bc2199David S. Miller 388cf5adce11743e98739fcb97e76d688f0b0bc2199David S. Miller190: 389cf5adce11743e98739fcb97e76d688f0b0bc2199David S. Miller1: EX_ST(STORE_INIT(%g0, %o4 + %g3)) 390cf5adce11743e98739fcb97e76d688f0b0bc2199David S. Miller subcc %g1, 64, %g1 391cf5adce11743e98739fcb97e76d688f0b0bc2199David S. Miller EX_LD(LOAD_BLK(%o4, %f0)) 392cf5adce11743e98739fcb97e76d688f0b0bc2199David S. Miller EX_ST(STORE_BLK(%f0, %o4 + %g3)) 393cf5adce11743e98739fcb97e76d688f0b0bc2199David S. Miller add %o4, 64, %o4 394cf5adce11743e98739fcb97e76d688f0b0bc2199David S. Miller bne,pt %xcc, 1b 395cf5adce11743e98739fcb97e76d688f0b0bc2199David S. Miller LOAD(prefetch, %o4 + 64, #one_read) 396cf5adce11743e98739fcb97e76d688f0b0bc2199David S. Miller 397cf5adce11743e98739fcb97e76d688f0b0bc2199David S. Miller195: 398cf5adce11743e98739fcb97e76d688f0b0bc2199David S. Miller add %o4, %g3, %o0 399cf5adce11743e98739fcb97e76d688f0b0bc2199David S. Miller membar #Sync 400cf5adce11743e98739fcb97e76d688f0b0bc2199David S. Miller 401cf5adce11743e98739fcb97e76d688f0b0bc2199David S. Miller VISExitHalf 402cf5adce11743e98739fcb97e76d688f0b0bc2199David S. Miller 403cf5adce11743e98739fcb97e76d688f0b0bc2199David S. Miller /* %o2 contains any final bytes still needed to be copied 404cf5adce11743e98739fcb97e76d688f0b0bc2199David S. Miller * over. If anything is left, we copy it one byte at a time. 405cf5adce11743e98739fcb97e76d688f0b0bc2199David S. Miller */ 406cf5adce11743e98739fcb97e76d688f0b0bc2199David S. Miller brz,pt %o2, 85f 407cf5adce11743e98739fcb97e76d688f0b0bc2199David S. Miller sub %o0, %o1, %o3 408cf5adce11743e98739fcb97e76d688f0b0bc2199David S. Miller ba,a,pt %XCC, 90f 409cf5adce11743e98739fcb97e76d688f0b0bc2199David S. Miller 410cf5adce11743e98739fcb97e76d688f0b0bc2199David S. Miller .align 64 411cf5adce11743e98739fcb97e76d688f0b0bc2199David S. Miller75: /* 16 < len <= 64 */ 412cf5adce11743e98739fcb97e76d688f0b0bc2199David S. Miller bne,pn %XCC, 75f 413cf5adce11743e98739fcb97e76d688f0b0bc2199David S. Miller sub %o0, %o1, %o3 414cf5adce11743e98739fcb97e76d688f0b0bc2199David S. Miller 415cf5adce11743e98739fcb97e76d688f0b0bc2199David S. Miller72: 416cf5adce11743e98739fcb97e76d688f0b0bc2199David S. Miller andn %o2, 0xf, %o4 417cf5adce11743e98739fcb97e76d688f0b0bc2199David S. Miller and %o2, 0xf, %o2 418cf5adce11743e98739fcb97e76d688f0b0bc2199David S. Miller1: subcc %o4, 0x10, %o4 419cf5adce11743e98739fcb97e76d688f0b0bc2199David S. Miller EX_LD(LOAD(ldx, %o1, %o5)) 420cf5adce11743e98739fcb97e76d688f0b0bc2199David S. Miller add %o1, 0x08, %o1 421cf5adce11743e98739fcb97e76d688f0b0bc2199David S. Miller EX_LD(LOAD(ldx, %o1, %g1)) 422cf5adce11743e98739fcb97e76d688f0b0bc2199David S. Miller sub %o1, 0x08, %o1 423cf5adce11743e98739fcb97e76d688f0b0bc2199David S. Miller EX_ST(STORE(stx, %o5, %o1 + %o3)) 424cf5adce11743e98739fcb97e76d688f0b0bc2199David S. Miller add %o1, 0x8, %o1 425cf5adce11743e98739fcb97e76d688f0b0bc2199David S. Miller EX_ST(STORE(stx, %g1, %o1 + %o3)) 426cf5adce11743e98739fcb97e76d688f0b0bc2199David S. Miller bgu,pt %XCC, 1b 427cf5adce11743e98739fcb97e76d688f0b0bc2199David S. Miller add %o1, 0x8, %o1 428cf5adce11743e98739fcb97e76d688f0b0bc2199David S. Miller73: andcc %o2, 0x8, %g0 429cf5adce11743e98739fcb97e76d688f0b0bc2199David S. Miller be,pt %XCC, 1f 430cf5adce11743e98739fcb97e76d688f0b0bc2199David S. Miller nop 431cf5adce11743e98739fcb97e76d688f0b0bc2199David S. Miller sub %o2, 0x8, %o2 432cf5adce11743e98739fcb97e76d688f0b0bc2199David S. Miller EX_LD(LOAD(ldx, %o1, %o5)) 433cf5adce11743e98739fcb97e76d688f0b0bc2199David S. Miller EX_ST(STORE(stx, %o5, %o1 + %o3)) 434cf5adce11743e98739fcb97e76d688f0b0bc2199David S. Miller add %o1, 0x8, %o1 435cf5adce11743e98739fcb97e76d688f0b0bc2199David S. Miller1: andcc %o2, 0x4, %g0 436cf5adce11743e98739fcb97e76d688f0b0bc2199David S. Miller be,pt %XCC, 1f 437cf5adce11743e98739fcb97e76d688f0b0bc2199David S. Miller nop 438cf5adce11743e98739fcb97e76d688f0b0bc2199David S. Miller sub %o2, 0x4, %o2 439cf5adce11743e98739fcb97e76d688f0b0bc2199David S. Miller EX_LD(LOAD(lduw, %o1, %o5)) 440cf5adce11743e98739fcb97e76d688f0b0bc2199David S. Miller EX_ST(STORE(stw, %o5, %o1 + %o3)) 441cf5adce11743e98739fcb97e76d688f0b0bc2199David S. Miller add %o1, 0x4, %o1 442cf5adce11743e98739fcb97e76d688f0b0bc2199David S. Miller1: cmp %o2, 0 443cf5adce11743e98739fcb97e76d688f0b0bc2199David S. Miller be,pt %XCC, 85f 444cf5adce11743e98739fcb97e76d688f0b0bc2199David S. Miller nop 445cf5adce11743e98739fcb97e76d688f0b0bc2199David S. Miller ba,pt %xcc, 90f 446cf5adce11743e98739fcb97e76d688f0b0bc2199David S. Miller nop 447cf5adce11743e98739fcb97e76d688f0b0bc2199David S. Miller 448cf5adce11743e98739fcb97e76d688f0b0bc2199David S. Miller75: 449cf5adce11743e98739fcb97e76d688f0b0bc2199David S. Miller andcc %o0, 0x7, %g1 450cf5adce11743e98739fcb97e76d688f0b0bc2199David S. Miller sub %g1, 0x8, %g1 451cf5adce11743e98739fcb97e76d688f0b0bc2199David S. Miller be,pn %icc, 2f 452cf5adce11743e98739fcb97e76d688f0b0bc2199David S. Miller sub %g0, %g1, %g1 453cf5adce11743e98739fcb97e76d688f0b0bc2199David S. Miller sub %o2, %g1, %o2 454cf5adce11743e98739fcb97e76d688f0b0bc2199David S. Miller 455cf5adce11743e98739fcb97e76d688f0b0bc2199David S. Miller1: subcc %g1, 1, %g1 456cf5adce11743e98739fcb97e76d688f0b0bc2199David S. Miller EX_LD(LOAD(ldub, %o1, %o5)) 457cf5adce11743e98739fcb97e76d688f0b0bc2199David S. Miller EX_ST(STORE(stb, %o5, %o1 + %o3)) 458cf5adce11743e98739fcb97e76d688f0b0bc2199David S. Miller bgu,pt %icc, 1b 459cf5adce11743e98739fcb97e76d688f0b0bc2199David S. Miller add %o1, 1, %o1 460cf5adce11743e98739fcb97e76d688f0b0bc2199David S. Miller 461cf5adce11743e98739fcb97e76d688f0b0bc2199David S. Miller2: add %o1, %o3, %o0 462cf5adce11743e98739fcb97e76d688f0b0bc2199David S. Miller andcc %o1, 0x7, %g1 463cf5adce11743e98739fcb97e76d688f0b0bc2199David S. Miller bne,pt %icc, 8f 464cf5adce11743e98739fcb97e76d688f0b0bc2199David S. Miller sll %g1, 3, %g1 465cf5adce11743e98739fcb97e76d688f0b0bc2199David S. Miller 466cf5adce11743e98739fcb97e76d688f0b0bc2199David S. Miller cmp %o2, 16 467cf5adce11743e98739fcb97e76d688f0b0bc2199David S. Miller bgeu,pt %icc, 72b 468cf5adce11743e98739fcb97e76d688f0b0bc2199David S. Miller nop 469cf5adce11743e98739fcb97e76d688f0b0bc2199David S. Miller ba,a,pt %xcc, 73b 470cf5adce11743e98739fcb97e76d688f0b0bc2199David S. Miller 471cf5adce11743e98739fcb97e76d688f0b0bc2199David S. Miller8: mov 64, %o3 472cf5adce11743e98739fcb97e76d688f0b0bc2199David S. Miller andn %o1, 0x7, %o1 473cf5adce11743e98739fcb97e76d688f0b0bc2199David S. Miller EX_LD(LOAD(ldx, %o1, %g2)) 474cf5adce11743e98739fcb97e76d688f0b0bc2199David S. Miller sub %o3, %g1, %o3 475cf5adce11743e98739fcb97e76d688f0b0bc2199David S. Miller andn %o2, 0x7, %o4 476cf5adce11743e98739fcb97e76d688f0b0bc2199David S. Miller sllx %g2, %g1, %g2 477cf5adce11743e98739fcb97e76d688f0b0bc2199David S. Miller1: add %o1, 0x8, %o1 478cf5adce11743e98739fcb97e76d688f0b0bc2199David S. Miller EX_LD(LOAD(ldx, %o1, %g3)) 479cf5adce11743e98739fcb97e76d688f0b0bc2199David S. Miller subcc %o4, 0x8, %o4 480cf5adce11743e98739fcb97e76d688f0b0bc2199David S. Miller srlx %g3, %o3, %o5 481cf5adce11743e98739fcb97e76d688f0b0bc2199David S. Miller or %o5, %g2, %o5 482cf5adce11743e98739fcb97e76d688f0b0bc2199David S. Miller EX_ST(STORE(stx, %o5, %o0)) 483cf5adce11743e98739fcb97e76d688f0b0bc2199David S. Miller add %o0, 0x8, %o0 484cf5adce11743e98739fcb97e76d688f0b0bc2199David S. Miller bgu,pt %icc, 1b 485cf5adce11743e98739fcb97e76d688f0b0bc2199David S. Miller sllx %g3, %g1, %g2 486cf5adce11743e98739fcb97e76d688f0b0bc2199David S. Miller 487cf5adce11743e98739fcb97e76d688f0b0bc2199David S. Miller srl %g1, 3, %g1 488cf5adce11743e98739fcb97e76d688f0b0bc2199David S. Miller andcc %o2, 0x7, %o2 489cf5adce11743e98739fcb97e76d688f0b0bc2199David S. Miller be,pn %icc, 85f 490cf5adce11743e98739fcb97e76d688f0b0bc2199David S. Miller add %o1, %g1, %o1 491cf5adce11743e98739fcb97e76d688f0b0bc2199David S. Miller ba,pt %xcc, 90f 492cf5adce11743e98739fcb97e76d688f0b0bc2199David S. Miller sub %o0, %o1, %o3 493cf5adce11743e98739fcb97e76d688f0b0bc2199David S. Miller 494cf5adce11743e98739fcb97e76d688f0b0bc2199David S. Miller .align 64 495cf5adce11743e98739fcb97e76d688f0b0bc2199David S. Miller80: /* 0 < len <= 16 */ 496cf5adce11743e98739fcb97e76d688f0b0bc2199David S. Miller andcc %o3, 0x3, %g0 497cf5adce11743e98739fcb97e76d688f0b0bc2199David S. Miller bne,pn %XCC, 90f 498cf5adce11743e98739fcb97e76d688f0b0bc2199David S. Miller sub %o0, %o1, %o3 499cf5adce11743e98739fcb97e76d688f0b0bc2199David S. Miller 500cf5adce11743e98739fcb97e76d688f0b0bc2199David S. Miller1: 501cf5adce11743e98739fcb97e76d688f0b0bc2199David S. Miller subcc %o2, 4, %o2 502cf5adce11743e98739fcb97e76d688f0b0bc2199David S. Miller EX_LD(LOAD(lduw, %o1, %g1)) 503cf5adce11743e98739fcb97e76d688f0b0bc2199David S. Miller EX_ST(STORE(stw, %g1, %o1 + %o3)) 504cf5adce11743e98739fcb97e76d688f0b0bc2199David S. Miller bgu,pt %XCC, 1b 505cf5adce11743e98739fcb97e76d688f0b0bc2199David S. Miller add %o1, 4, %o1 506cf5adce11743e98739fcb97e76d688f0b0bc2199David S. Miller 507cf5adce11743e98739fcb97e76d688f0b0bc2199David S. Miller85: retl 508cf5adce11743e98739fcb97e76d688f0b0bc2199David S. Miller mov EX_RETVAL(GLOBAL_SPARE), %o0 509cf5adce11743e98739fcb97e76d688f0b0bc2199David S. Miller 510cf5adce11743e98739fcb97e76d688f0b0bc2199David S. Miller .align 32 511cf5adce11743e98739fcb97e76d688f0b0bc2199David S. Miller90: 512cf5adce11743e98739fcb97e76d688f0b0bc2199David S. Miller subcc %o2, 1, %o2 513cf5adce11743e98739fcb97e76d688f0b0bc2199David S. Miller EX_LD(LOAD(ldub, %o1, %g1)) 514cf5adce11743e98739fcb97e76d688f0b0bc2199David S. Miller EX_ST(STORE(stb, %g1, %o1 + %o3)) 515cf5adce11743e98739fcb97e76d688f0b0bc2199David S. Miller bgu,pt %XCC, 90b 516cf5adce11743e98739fcb97e76d688f0b0bc2199David S. Miller add %o1, 1, %o1 517cf5adce11743e98739fcb97e76d688f0b0bc2199David S. Miller retl 518cf5adce11743e98739fcb97e76d688f0b0bc2199David S. Miller mov EX_RETVAL(GLOBAL_SPARE), %o0 519cf5adce11743e98739fcb97e76d688f0b0bc2199David S. Miller 520cf5adce11743e98739fcb97e76d688f0b0bc2199David S. Miller .size FUNC_NAME, .-FUNC_NAME 521