1fcb877ac4b05e65f64b9b3199d71dfeaae299177Prashant Patil/* 2fcb877ac4b05e65f64b9b3199d71dfeaae299177Prashant Patil * Copyright (c) 2017 Imagination Technologies. 3fcb877ac4b05e65f64b9b3199d71dfeaae299177Prashant Patil * 4fcb877ac4b05e65f64b9b3199d71dfeaae299177Prashant Patil * All rights reserved. 5fcb877ac4b05e65f64b9b3199d71dfeaae299177Prashant Patil * 6fcb877ac4b05e65f64b9b3199d71dfeaae299177Prashant Patil * Redistribution and use in source and binary forms, with or without 7fcb877ac4b05e65f64b9b3199d71dfeaae299177Prashant Patil * modification, are permitted provided that the following conditions 8fcb877ac4b05e65f64b9b3199d71dfeaae299177Prashant Patil * are met: 9fcb877ac4b05e65f64b9b3199d71dfeaae299177Prashant Patil * 10fcb877ac4b05e65f64b9b3199d71dfeaae299177Prashant Patil * * Redistributions of source code must retain the above copyright 11fcb877ac4b05e65f64b9b3199d71dfeaae299177Prashant Patil * notice, this list of conditions and the following disclaimer. 12fcb877ac4b05e65f64b9b3199d71dfeaae299177Prashant Patil * * Redistributions in binary form must reproduce the above copyright 13fcb877ac4b05e65f64b9b3199d71dfeaae299177Prashant Patil * notice, this list of conditions and the following disclaimer 14fcb877ac4b05e65f64b9b3199d71dfeaae299177Prashant Patil * in the documentation and/or other materials provided with 15fcb877ac4b05e65f64b9b3199d71dfeaae299177Prashant Patil * the distribution. 16fcb877ac4b05e65f64b9b3199d71dfeaae299177Prashant Patil * * Neither the name of Imagination Technologies nor the names of its 17fcb877ac4b05e65f64b9b3199d71dfeaae299177Prashant Patil * contributors may be used to endorse or promote products derived 18fcb877ac4b05e65f64b9b3199d71dfeaae299177Prashant Patil * from this software without specific prior written permission. 19fcb877ac4b05e65f64b9b3199d71dfeaae299177Prashant Patil * 20fcb877ac4b05e65f64b9b3199d71dfeaae299177Prashant Patil * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS 21fcb877ac4b05e65f64b9b3199d71dfeaae299177Prashant Patil * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT 22fcb877ac4b05e65f64b9b3199d71dfeaae299177Prashant Patil * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR 23fcb877ac4b05e65f64b9b3199d71dfeaae299177Prashant Patil * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT 24fcb877ac4b05e65f64b9b3199d71dfeaae299177Prashant Patil * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, 25fcb877ac4b05e65f64b9b3199d71dfeaae299177Prashant Patil * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT 26fcb877ac4b05e65f64b9b3199d71dfeaae299177Prashant Patil * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 27fcb877ac4b05e65f64b9b3199d71dfeaae299177Prashant Patil * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 28fcb877ac4b05e65f64b9b3199d71dfeaae299177Prashant Patil * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 29fcb877ac4b05e65f64b9b3199d71dfeaae299177Prashant Patil * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 30fcb877ac4b05e65f64b9b3199d71dfeaae299177Prashant Patil * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 31fcb877ac4b05e65f64b9b3199d71dfeaae299177Prashant Patil */ 32fcb877ac4b05e65f64b9b3199d71dfeaae299177Prashant Patil#include <string.h> 33fcb877ac4b05e65f64b9b3199d71dfeaae299177Prashant Patil 34fcb877ac4b05e65f64b9b3199d71dfeaae299177Prashant Patil#define ENABLE_PREFETCH 1 35fcb877ac4b05e65f64b9b3199d71dfeaae299177Prashant Patil#define op_t unsigned long int 36fcb877ac4b05e65f64b9b3199d71dfeaae299177Prashant Patil#define op_size sizeof (op_t) 37fcb877ac4b05e65f64b9b3199d71dfeaae299177Prashant Patil 38fcb877ac4b05e65f64b9b3199d71dfeaae299177Prashant Patil#if ENABLE_PREFETCH 39fcb877ac4b05e65f64b9b3199d71dfeaae299177Prashant Patil#define PREFETCH(addr) __builtin_prefetch (addr, 0, 1); 40fcb877ac4b05e65f64b9b3199d71dfeaae299177Prashant Patil#else 41fcb877ac4b05e65f64b9b3199d71dfeaae299177Prashant Patil#define PREFETCH(addr) 42fcb877ac4b05e65f64b9b3199d71dfeaae299177Prashant Patil#endif 43fcb877ac4b05e65f64b9b3199d71dfeaae299177Prashant Patil 44fcb877ac4b05e65f64b9b3199d71dfeaae299177Prashant Patil#if __mips64 || __mips_isa_rev >= 2 45fcb877ac4b05e65f64b9b3199d71dfeaae299177Prashant Patilstatic inline void * __attribute__ ((always_inline)) 46fcb877ac4b05e65f64b9b3199d71dfeaae299177Prashant Patildo_bytes (const op_t* w, op_t inval) 47fcb877ac4b05e65f64b9b3199d71dfeaae299177Prashant Patil{ 48fcb877ac4b05e65f64b9b3199d71dfeaae299177Prashant Patil const unsigned char *p = (const unsigned char *) w; 49fcb877ac4b05e65f64b9b3199d71dfeaae299177Prashant Patil op_t outval = 0; 50fcb877ac4b05e65f64b9b3199d71dfeaae299177Prashant Patil#if __mips64 51fcb877ac4b05e65f64b9b3199d71dfeaae299177Prashant Patil __asm__ volatile ( 52fcb877ac4b05e65f64b9b3199d71dfeaae299177Prashant Patil "dsbh %1, %0 \n\t" 53fcb877ac4b05e65f64b9b3199d71dfeaae299177Prashant Patil "dshd %0, %1 \n\t" 54fcb877ac4b05e65f64b9b3199d71dfeaae299177Prashant Patil "dclz %1, %0 \n\t" 55fcb877ac4b05e65f64b9b3199d71dfeaae299177Prashant Patil : "+r" (inval), "+r" (outval) 56fcb877ac4b05e65f64b9b3199d71dfeaae299177Prashant Patil ); 57fcb877ac4b05e65f64b9b3199d71dfeaae299177Prashant Patil#else 58fcb877ac4b05e65f64b9b3199d71dfeaae299177Prashant Patil __asm__ volatile ( 59fcb877ac4b05e65f64b9b3199d71dfeaae299177Prashant Patil "wsbh %1, %0 \n\t" 60fcb877ac4b05e65f64b9b3199d71dfeaae299177Prashant Patil "rotr %0, %1, 16 \n\t" 61fcb877ac4b05e65f64b9b3199d71dfeaae299177Prashant Patil "clz %1, %0 \n\t" 62fcb877ac4b05e65f64b9b3199d71dfeaae299177Prashant Patil : "+r" (inval), "+r" (outval) 63fcb877ac4b05e65f64b9b3199d71dfeaae299177Prashant Patil ); 64fcb877ac4b05e65f64b9b3199d71dfeaae299177Prashant Patil#endif 65fcb877ac4b05e65f64b9b3199d71dfeaae299177Prashant Patil p += (outval >> 3); 66fcb877ac4b05e65f64b9b3199d71dfeaae299177Prashant Patil return (void *) p; 67fcb877ac4b05e65f64b9b3199d71dfeaae299177Prashant Patil} 68fcb877ac4b05e65f64b9b3199d71dfeaae299177Prashant Patil 69fcb877ac4b05e65f64b9b3199d71dfeaae299177Prashant Patil#define DO_WORD(in, val) { \ 70fcb877ac4b05e65f64b9b3199d71dfeaae299177Prashant Patil op_t tmp = ((val - mask_1) & ~val) & mask_128; \ 71fcb877ac4b05e65f64b9b3199d71dfeaae299177Prashant Patil if (tmp != 0) \ 72fcb877ac4b05e65f64b9b3199d71dfeaae299177Prashant Patil return do_bytes(in, tmp); \ 73fcb877ac4b05e65f64b9b3199d71dfeaae299177Prashant Patil} 74fcb877ac4b05e65f64b9b3199d71dfeaae299177Prashant Patil#else 75fcb877ac4b05e65f64b9b3199d71dfeaae299177Prashant Patilstatic inline void * __attribute__ ((always_inline)) 76fcb877ac4b05e65f64b9b3199d71dfeaae299177Prashant Patildo_bytes (const op_t* w, unsigned char ch) 77fcb877ac4b05e65f64b9b3199d71dfeaae299177Prashant Patil{ 78fcb877ac4b05e65f64b9b3199d71dfeaae299177Prashant Patil const unsigned char *p = (const unsigned char *) w; 79fcb877ac4b05e65f64b9b3199d71dfeaae299177Prashant Patil for (; *p != ch; ++p); 80fcb877ac4b05e65f64b9b3199d71dfeaae299177Prashant Patil return (void *) p; 81fcb877ac4b05e65f64b9b3199d71dfeaae299177Prashant Patil} 82fcb877ac4b05e65f64b9b3199d71dfeaae299177Prashant Patil 83fcb877ac4b05e65f64b9b3199d71dfeaae299177Prashant Patil#define DO_WORD(in, val) { \ 84fcb877ac4b05e65f64b9b3199d71dfeaae299177Prashant Patil op_t tmp = ((val - mask_1) & ~val) & mask_128; \ 85fcb877ac4b05e65f64b9b3199d71dfeaae299177Prashant Patil if (tmp != 0) \ 86fcb877ac4b05e65f64b9b3199d71dfeaae299177Prashant Patil return do_bytes(in, ch); \ 87fcb877ac4b05e65f64b9b3199d71dfeaae299177Prashant Patil} 88fcb877ac4b05e65f64b9b3199d71dfeaae299177Prashant Patil#endif 89fcb877ac4b05e65f64b9b3199d71dfeaae299177Prashant Patil 90fcb877ac4b05e65f64b9b3199d71dfeaae299177Prashant Patil#define DO_WORDS(w) { \ 91fcb877ac4b05e65f64b9b3199d71dfeaae299177Prashant Patil op_t* w1 = (op_t*) w; \ 92fcb877ac4b05e65f64b9b3199d71dfeaae299177Prashant Patil op_t val0 = w1[0] ^ mask_c; \ 93fcb877ac4b05e65f64b9b3199d71dfeaae299177Prashant Patil op_t val1 = w1[1] ^ mask_c; \ 94fcb877ac4b05e65f64b9b3199d71dfeaae299177Prashant Patil op_t val2 = w1[2] ^ mask_c; \ 95fcb877ac4b05e65f64b9b3199d71dfeaae299177Prashant Patil op_t val3 = w1[3] ^ mask_c; \ 96fcb877ac4b05e65f64b9b3199d71dfeaae299177Prashant Patil DO_WORD(w1, val0) \ 97fcb877ac4b05e65f64b9b3199d71dfeaae299177Prashant Patil DO_WORD(w1 + 1, val1) \ 98fcb877ac4b05e65f64b9b3199d71dfeaae299177Prashant Patil DO_WORD(w1 + 2, val2) \ 99fcb877ac4b05e65f64b9b3199d71dfeaae299177Prashant Patil DO_WORD(w1 + 3, val3) \ 100fcb877ac4b05e65f64b9b3199d71dfeaae299177Prashant Patil} 101fcb877ac4b05e65f64b9b3199d71dfeaae299177Prashant Patil 102fcb877ac4b05e65f64b9b3199d71dfeaae299177Prashant Patilvoid * 103fcb877ac4b05e65f64b9b3199d71dfeaae299177Prashant Patilmemchr (void const *s, int c_in, size_t n) __overloadable 104fcb877ac4b05e65f64b9b3199d71dfeaae299177Prashant Patil{ 105fcb877ac4b05e65f64b9b3199d71dfeaae299177Prashant Patil if (n != 0) { 106fcb877ac4b05e65f64b9b3199d71dfeaae299177Prashant Patil const unsigned char *p = (const unsigned char *) s; 107fcb877ac4b05e65f64b9b3199d71dfeaae299177Prashant Patil const op_t *w; 108fcb877ac4b05e65f64b9b3199d71dfeaae299177Prashant Patil op_t mask_1, mask_128, mask_c; 109fcb877ac4b05e65f64b9b3199d71dfeaae299177Prashant Patil unsigned char ch = (unsigned char) c_in; 110fcb877ac4b05e65f64b9b3199d71dfeaae299177Prashant Patil 111fcb877ac4b05e65f64b9b3199d71dfeaae299177Prashant Patil /* 112fcb877ac4b05e65f64b9b3199d71dfeaae299177Prashant Patil * Check bytewize till initial alignment 113fcb877ac4b05e65f64b9b3199d71dfeaae299177Prashant Patil */ 114fcb877ac4b05e65f64b9b3199d71dfeaae299177Prashant Patil for (; n > 0 && ((size_t) p % op_size) != 0; --n, ++p) { 115fcb877ac4b05e65f64b9b3199d71dfeaae299177Prashant Patil if (*p == ch) 116fcb877ac4b05e65f64b9b3199d71dfeaae299177Prashant Patil return (void *) p; 117fcb877ac4b05e65f64b9b3199d71dfeaae299177Prashant Patil } 118fcb877ac4b05e65f64b9b3199d71dfeaae299177Prashant Patil 119fcb877ac4b05e65f64b9b3199d71dfeaae299177Prashant Patil w = (const op_t *) p; 120fcb877ac4b05e65f64b9b3199d71dfeaae299177Prashant Patil 121fcb877ac4b05e65f64b9b3199d71dfeaae299177Prashant Patil mask_c = ch | (ch << 8); 122fcb877ac4b05e65f64b9b3199d71dfeaae299177Prashant Patil mask_c |= mask_c << 16; 123fcb877ac4b05e65f64b9b3199d71dfeaae299177Prashant Patil __asm__ volatile ( 124fcb877ac4b05e65f64b9b3199d71dfeaae299177Prashant Patil "li %0, 0x01010101 \n\t" 125fcb877ac4b05e65f64b9b3199d71dfeaae299177Prashant Patil : "=r" (mask_1) 126fcb877ac4b05e65f64b9b3199d71dfeaae299177Prashant Patil ); 127fcb877ac4b05e65f64b9b3199d71dfeaae299177Prashant Patil#if __mips64 128fcb877ac4b05e65f64b9b3199d71dfeaae299177Prashant Patil mask_1 |= mask_1 << 32; 129fcb877ac4b05e65f64b9b3199d71dfeaae299177Prashant Patil mask_c |= mask_c << 32; 130fcb877ac4b05e65f64b9b3199d71dfeaae299177Prashant Patil#endif 131fcb877ac4b05e65f64b9b3199d71dfeaae299177Prashant Patil mask_128 = mask_1 << 7; 132fcb877ac4b05e65f64b9b3199d71dfeaae299177Prashant Patil 133fcb877ac4b05e65f64b9b3199d71dfeaae299177Prashant Patil /* 134fcb877ac4b05e65f64b9b3199d71dfeaae299177Prashant Patil * Check op_size byteswize after initial alignment 135fcb877ac4b05e65f64b9b3199d71dfeaae299177Prashant Patil */ 136fcb877ac4b05e65f64b9b3199d71dfeaae299177Prashant Patil#if ((_MIPS_SIM == _ABIO32) || _MIPS_TUNE_I6400) 137fcb877ac4b05e65f64b9b3199d71dfeaae299177Prashant Patil PREFETCH (w); 138fcb877ac4b05e65f64b9b3199d71dfeaae299177Prashant Patil PREFETCH (w + 8); 139fcb877ac4b05e65f64b9b3199d71dfeaae299177Prashant Patil while (n >= 24 * op_size) { 140fcb877ac4b05e65f64b9b3199d71dfeaae299177Prashant Patil PREFETCH(w + 16); 141fcb877ac4b05e65f64b9b3199d71dfeaae299177Prashant Patil DO_WORDS(w); 142fcb877ac4b05e65f64b9b3199d71dfeaae299177Prashant Patil DO_WORDS(w + 4); 143fcb877ac4b05e65f64b9b3199d71dfeaae299177Prashant Patil w += 8; 144fcb877ac4b05e65f64b9b3199d71dfeaae299177Prashant Patil n -= 8 * op_size; 145fcb877ac4b05e65f64b9b3199d71dfeaae299177Prashant Patil } 146fcb877ac4b05e65f64b9b3199d71dfeaae299177Prashant Patil while (n >= 8 * op_size) { 147fcb877ac4b05e65f64b9b3199d71dfeaae299177Prashant Patil DO_WORDS(w); 148fcb877ac4b05e65f64b9b3199d71dfeaae299177Prashant Patil DO_WORDS(w + 4); 149fcb877ac4b05e65f64b9b3199d71dfeaae299177Prashant Patil w += 8; 150fcb877ac4b05e65f64b9b3199d71dfeaae299177Prashant Patil n -= 8 * op_size; 151fcb877ac4b05e65f64b9b3199d71dfeaae299177Prashant Patil } 152fcb877ac4b05e65f64b9b3199d71dfeaae299177Prashant Patil#else 153fcb877ac4b05e65f64b9b3199d71dfeaae299177Prashant Patil PREFETCH (w); 154fcb877ac4b05e65f64b9b3199d71dfeaae299177Prashant Patil PREFETCH (w + 4); 155fcb877ac4b05e65f64b9b3199d71dfeaae299177Prashant Patil while (n >= 12 * op_size) { 156fcb877ac4b05e65f64b9b3199d71dfeaae299177Prashant Patil PREFETCH(w + 8); 157fcb877ac4b05e65f64b9b3199d71dfeaae299177Prashant Patil DO_WORDS(w); 158fcb877ac4b05e65f64b9b3199d71dfeaae299177Prashant Patil w += 4; 159fcb877ac4b05e65f64b9b3199d71dfeaae299177Prashant Patil n -= 4 * op_size; 160fcb877ac4b05e65f64b9b3199d71dfeaae299177Prashant Patil } 161fcb877ac4b05e65f64b9b3199d71dfeaae299177Prashant Patil while (n >= 4 * op_size) { 162fcb877ac4b05e65f64b9b3199d71dfeaae299177Prashant Patil DO_WORDS(w); 163fcb877ac4b05e65f64b9b3199d71dfeaae299177Prashant Patil w += 4; 164fcb877ac4b05e65f64b9b3199d71dfeaae299177Prashant Patil n -= 4 * op_size; 165fcb877ac4b05e65f64b9b3199d71dfeaae299177Prashant Patil } 166fcb877ac4b05e65f64b9b3199d71dfeaae299177Prashant Patil#endif 167fcb877ac4b05e65f64b9b3199d71dfeaae299177Prashant Patil 168fcb877ac4b05e65f64b9b3199d71dfeaae299177Prashant Patil while (n >= op_size) { 169fcb877ac4b05e65f64b9b3199d71dfeaae299177Prashant Patil op_t val = *w ^ mask_c; 170fcb877ac4b05e65f64b9b3199d71dfeaae299177Prashant Patil DO_WORD(w, val); 171fcb877ac4b05e65f64b9b3199d71dfeaae299177Prashant Patil w++; 172fcb877ac4b05e65f64b9b3199d71dfeaae299177Prashant Patil n -= op_size; 173fcb877ac4b05e65f64b9b3199d71dfeaae299177Prashant Patil } 174fcb877ac4b05e65f64b9b3199d71dfeaae299177Prashant Patil 175fcb877ac4b05e65f64b9b3199d71dfeaae299177Prashant Patil /* 176fcb877ac4b05e65f64b9b3199d71dfeaae299177Prashant Patil * Check bytewize for remaining bytes 177fcb877ac4b05e65f64b9b3199d71dfeaae299177Prashant Patil */ 178fcb877ac4b05e65f64b9b3199d71dfeaae299177Prashant Patil p = (const unsigned char *) w; 179fcb877ac4b05e65f64b9b3199d71dfeaae299177Prashant Patil for (; n > 0; --n, ++p) { 180fcb877ac4b05e65f64b9b3199d71dfeaae299177Prashant Patil if (*p == ch) 181fcb877ac4b05e65f64b9b3199d71dfeaae299177Prashant Patil return (void *) p; 182fcb877ac4b05e65f64b9b3199d71dfeaae299177Prashant Patil } 183fcb877ac4b05e65f64b9b3199d71dfeaae299177Prashant Patil } 184fcb877ac4b05e65f64b9b3199d71dfeaae299177Prashant Patil return NULL; 185fcb877ac4b05e65f64b9b3199d71dfeaae299177Prashant Patil} 186