18b23a6c7e1aee255004dd19098d4c2462b61b849The Android Open Source Project/* 28b23a6c7e1aee255004dd19098d4c2462b61b849The Android Open Source Project * ARM NEON vector operations. 38b23a6c7e1aee255004dd19098d4c2462b61b849The Android Open Source Project * 48b23a6c7e1aee255004dd19098d4c2462b61b849The Android Open Source Project * Copyright (c) 2007, 2008 CodeSourcery. 58b23a6c7e1aee255004dd19098d4c2462b61b849The Android Open Source Project * Written by Paul Brook 68b23a6c7e1aee255004dd19098d4c2462b61b849The Android Open Source Project * 78b23a6c7e1aee255004dd19098d4c2462b61b849The Android Open Source Project * This code is licenced under the GNU GPL v2. 88b23a6c7e1aee255004dd19098d4c2462b61b849The Android Open Source Project */ 98b23a6c7e1aee255004dd19098d4c2462b61b849The Android Open Source Project#include <stdlib.h> 108b23a6c7e1aee255004dd19098d4c2462b61b849The Android Open Source Project#include <stdio.h> 118b23a6c7e1aee255004dd19098d4c2462b61b849The Android Open Source Project 128b23a6c7e1aee255004dd19098d4c2462b61b849The Android Open Source Project#include "cpu.h" 135285864985be9077e58e42235af6582dee72e841David 'Digit' Turner#include "exec.h" 145285864985be9077e58e42235af6582dee72e841David 'Digit' Turner#include "helper.h" 158b23a6c7e1aee255004dd19098d4c2462b61b849The Android Open Source Project 168b23a6c7e1aee255004dd19098d4c2462b61b849The Android Open Source Project#define SIGNBIT (uint32_t)0x80000000 178b23a6c7e1aee255004dd19098d4c2462b61b849The Android Open Source Project#define SIGNBIT64 ((uint64_t)1 << 63) 188b23a6c7e1aee255004dd19098d4c2462b61b849The Android Open Source Project 198b23a6c7e1aee255004dd19098d4c2462b61b849The Android Open Source Project#define SET_QC() env->vfp.xregs[ARM_VFP_FPSCR] = CPSR_Q 208b23a6c7e1aee255004dd19098d4c2462b61b849The Android Open Source Project 215285864985be9077e58e42235af6582dee72e841David 'Digit' Turner#define NFS (&env->vfp.standard_fp_status) 228b23a6c7e1aee255004dd19098d4c2462b61b849The Android Open Source Project 238b23a6c7e1aee255004dd19098d4c2462b61b849The Android Open Source Project#define NEON_TYPE1(name, type) \ 248b23a6c7e1aee255004dd19098d4c2462b61b849The Android Open Source Projecttypedef struct \ 258b23a6c7e1aee255004dd19098d4c2462b61b849The Android Open Source Project{ \ 268b23a6c7e1aee255004dd19098d4c2462b61b849The Android Open Source Project type v1; \ 278b23a6c7e1aee255004dd19098d4c2462b61b849The Android Open Source Project} neon_##name; 2820894ae3fa98f82da925fbeb72e616eef509758aDavid 'Digit' Turner#ifdef HOST_WORDS_BIGENDIAN 298b23a6c7e1aee255004dd19098d4c2462b61b849The Android Open Source Project#define NEON_TYPE2(name, type) \ 308b23a6c7e1aee255004dd19098d4c2462b61b849The Android Open Source Projecttypedef struct \ 318b23a6c7e1aee255004dd19098d4c2462b61b849The Android Open Source Project{ \ 328b23a6c7e1aee255004dd19098d4c2462b61b849The Android Open Source Project type v2; \ 338b23a6c7e1aee255004dd19098d4c2462b61b849The Android Open Source Project type v1; \ 348b23a6c7e1aee255004dd19098d4c2462b61b849The Android Open Source Project} neon_##name; 358b23a6c7e1aee255004dd19098d4c2462b61b849The Android Open Source Project#define NEON_TYPE4(name, type) \ 368b23a6c7e1aee255004dd19098d4c2462b61b849The Android Open Source Projecttypedef struct \ 378b23a6c7e1aee255004dd19098d4c2462b61b849The Android Open Source Project{ \ 388b23a6c7e1aee255004dd19098d4c2462b61b849The Android Open Source Project type v4; \ 398b23a6c7e1aee255004dd19098d4c2462b61b849The Android Open Source Project type v3; \ 408b23a6c7e1aee255004dd19098d4c2462b61b849The Android Open Source Project type v2; \ 418b23a6c7e1aee255004dd19098d4c2462b61b849The Android Open Source Project type v1; \ 428b23a6c7e1aee255004dd19098d4c2462b61b849The Android Open Source Project} neon_##name; 438b23a6c7e1aee255004dd19098d4c2462b61b849The Android Open Source Project#else 448b23a6c7e1aee255004dd19098d4c2462b61b849The Android Open Source Project#define NEON_TYPE2(name, type) \ 458b23a6c7e1aee255004dd19098d4c2462b61b849The Android Open Source Projecttypedef struct \ 468b23a6c7e1aee255004dd19098d4c2462b61b849The Android Open Source Project{ \ 478b23a6c7e1aee255004dd19098d4c2462b61b849The Android Open Source Project type v1; \ 488b23a6c7e1aee255004dd19098d4c2462b61b849The Android Open Source Project type v2; \ 498b23a6c7e1aee255004dd19098d4c2462b61b849The Android Open Source Project} neon_##name; 508b23a6c7e1aee255004dd19098d4c2462b61b849The Android Open Source Project#define NEON_TYPE4(name, type) \ 518b23a6c7e1aee255004dd19098d4c2462b61b849The Android Open Source Projecttypedef struct \ 528b23a6c7e1aee255004dd19098d4c2462b61b849The Android Open Source Project{ \ 538b23a6c7e1aee255004dd19098d4c2462b61b849The Android Open Source Project type v1; \ 548b23a6c7e1aee255004dd19098d4c2462b61b849The Android Open Source Project type v2; \ 558b23a6c7e1aee255004dd19098d4c2462b61b849The Android Open Source Project type v3; \ 568b23a6c7e1aee255004dd19098d4c2462b61b849The Android Open Source Project type v4; \ 578b23a6c7e1aee255004dd19098d4c2462b61b849The Android Open Source Project} neon_##name; 588b23a6c7e1aee255004dd19098d4c2462b61b849The Android Open Source Project#endif 598b23a6c7e1aee255004dd19098d4c2462b61b849The Android Open Source Project 608b23a6c7e1aee255004dd19098d4c2462b61b849The Android Open Source ProjectNEON_TYPE4(s8, int8_t) 618b23a6c7e1aee255004dd19098d4c2462b61b849The Android Open Source ProjectNEON_TYPE4(u8, uint8_t) 628b23a6c7e1aee255004dd19098d4c2462b61b849The Android Open Source ProjectNEON_TYPE2(s16, int16_t) 638b23a6c7e1aee255004dd19098d4c2462b61b849The Android Open Source ProjectNEON_TYPE2(u16, uint16_t) 648b23a6c7e1aee255004dd19098d4c2462b61b849The Android Open Source ProjectNEON_TYPE1(s32, int32_t) 658b23a6c7e1aee255004dd19098d4c2462b61b849The Android Open Source ProjectNEON_TYPE1(u32, uint32_t) 668b23a6c7e1aee255004dd19098d4c2462b61b849The Android Open Source Project#undef NEON_TYPE4 678b23a6c7e1aee255004dd19098d4c2462b61b849The Android Open Source Project#undef NEON_TYPE2 688b23a6c7e1aee255004dd19098d4c2462b61b849The Android Open Source Project#undef NEON_TYPE1 698b23a6c7e1aee255004dd19098d4c2462b61b849The Android Open Source Project 708b23a6c7e1aee255004dd19098d4c2462b61b849The Android Open Source Project/* Copy from a uint32_t to a vector structure type. */ 718b23a6c7e1aee255004dd19098d4c2462b61b849The Android Open Source Project#define NEON_UNPACK(vtype, dest, val) do { \ 728b23a6c7e1aee255004dd19098d4c2462b61b849The Android Open Source Project union { \ 738b23a6c7e1aee255004dd19098d4c2462b61b849The Android Open Source Project vtype v; \ 748b23a6c7e1aee255004dd19098d4c2462b61b849The Android Open Source Project uint32_t i; \ 758b23a6c7e1aee255004dd19098d4c2462b61b849The Android Open Source Project } conv_u; \ 768b23a6c7e1aee255004dd19098d4c2462b61b849The Android Open Source Project conv_u.i = (val); \ 778b23a6c7e1aee255004dd19098d4c2462b61b849The Android Open Source Project dest = conv_u.v; \ 788b23a6c7e1aee255004dd19098d4c2462b61b849The Android Open Source Project } while(0) 798b23a6c7e1aee255004dd19098d4c2462b61b849The Android Open Source Project 808b23a6c7e1aee255004dd19098d4c2462b61b849The Android Open Source Project/* Copy from a vector structure type to a uint32_t. */ 818b23a6c7e1aee255004dd19098d4c2462b61b849The Android Open Source Project#define NEON_PACK(vtype, dest, val) do { \ 828b23a6c7e1aee255004dd19098d4c2462b61b849The Android Open Source Project union { \ 838b23a6c7e1aee255004dd19098d4c2462b61b849The Android Open Source Project vtype v; \ 848b23a6c7e1aee255004dd19098d4c2462b61b849The Android Open Source Project uint32_t i; \ 858b23a6c7e1aee255004dd19098d4c2462b61b849The Android Open Source Project } conv_u; \ 868b23a6c7e1aee255004dd19098d4c2462b61b849The Android Open Source Project conv_u.v = (val); \ 878b23a6c7e1aee255004dd19098d4c2462b61b849The Android Open Source Project dest = conv_u.i; \ 888b23a6c7e1aee255004dd19098d4c2462b61b849The Android Open Source Project } while(0) 898b23a6c7e1aee255004dd19098d4c2462b61b849The Android Open Source Project 908b23a6c7e1aee255004dd19098d4c2462b61b849The Android Open Source Project#define NEON_DO1 \ 918b23a6c7e1aee255004dd19098d4c2462b61b849The Android Open Source Project NEON_FN(vdest.v1, vsrc1.v1, vsrc2.v1); 928b23a6c7e1aee255004dd19098d4c2462b61b849The Android Open Source Project#define NEON_DO2 \ 938b23a6c7e1aee255004dd19098d4c2462b61b849The Android Open Source Project NEON_FN(vdest.v1, vsrc1.v1, vsrc2.v1); \ 948b23a6c7e1aee255004dd19098d4c2462b61b849The Android Open Source Project NEON_FN(vdest.v2, vsrc1.v2, vsrc2.v2); 958b23a6c7e1aee255004dd19098d4c2462b61b849The Android Open Source Project#define NEON_DO4 \ 968b23a6c7e1aee255004dd19098d4c2462b61b849The Android Open Source Project NEON_FN(vdest.v1, vsrc1.v1, vsrc2.v1); \ 978b23a6c7e1aee255004dd19098d4c2462b61b849The Android Open Source Project NEON_FN(vdest.v2, vsrc1.v2, vsrc2.v2); \ 988b23a6c7e1aee255004dd19098d4c2462b61b849The Android Open Source Project NEON_FN(vdest.v3, vsrc1.v3, vsrc2.v3); \ 998b23a6c7e1aee255004dd19098d4c2462b61b849The Android Open Source Project NEON_FN(vdest.v4, vsrc1.v4, vsrc2.v4); 1008b23a6c7e1aee255004dd19098d4c2462b61b849The Android Open Source Project 1018b23a6c7e1aee255004dd19098d4c2462b61b849The Android Open Source Project#define NEON_VOP_BODY(vtype, n) \ 1028b23a6c7e1aee255004dd19098d4c2462b61b849The Android Open Source Project{ \ 1038b23a6c7e1aee255004dd19098d4c2462b61b849The Android Open Source Project uint32_t res; \ 1048b23a6c7e1aee255004dd19098d4c2462b61b849The Android Open Source Project vtype vsrc1; \ 1058b23a6c7e1aee255004dd19098d4c2462b61b849The Android Open Source Project vtype vsrc2; \ 1068b23a6c7e1aee255004dd19098d4c2462b61b849The Android Open Source Project vtype vdest; \ 1078b23a6c7e1aee255004dd19098d4c2462b61b849The Android Open Source Project NEON_UNPACK(vtype, vsrc1, arg1); \ 1088b23a6c7e1aee255004dd19098d4c2462b61b849The Android Open Source Project NEON_UNPACK(vtype, vsrc2, arg2); \ 1098b23a6c7e1aee255004dd19098d4c2462b61b849The Android Open Source Project NEON_DO##n; \ 1108b23a6c7e1aee255004dd19098d4c2462b61b849The Android Open Source Project NEON_PACK(vtype, res, vdest); \ 1118b23a6c7e1aee255004dd19098d4c2462b61b849The Android Open Source Project return res; \ 1128b23a6c7e1aee255004dd19098d4c2462b61b849The Android Open Source Project} 1138b23a6c7e1aee255004dd19098d4c2462b61b849The Android Open Source Project 1148b23a6c7e1aee255004dd19098d4c2462b61b849The Android Open Source Project#define NEON_VOP(name, vtype, n) \ 1158b23a6c7e1aee255004dd19098d4c2462b61b849The Android Open Source Projectuint32_t HELPER(glue(neon_,name))(uint32_t arg1, uint32_t arg2) \ 1168b23a6c7e1aee255004dd19098d4c2462b61b849The Android Open Source ProjectNEON_VOP_BODY(vtype, n) 1178b23a6c7e1aee255004dd19098d4c2462b61b849The Android Open Source Project 1188b23a6c7e1aee255004dd19098d4c2462b61b849The Android Open Source Project/* Pairwise operations. */ 1198b23a6c7e1aee255004dd19098d4c2462b61b849The Android Open Source Project/* For 32-bit elements each segment only contains a single element, so 1208b23a6c7e1aee255004dd19098d4c2462b61b849The Android Open Source Project the elementwise and pairwise operations are the same. */ 1218b23a6c7e1aee255004dd19098d4c2462b61b849The Android Open Source Project#define NEON_PDO2 \ 1228b23a6c7e1aee255004dd19098d4c2462b61b849The Android Open Source Project NEON_FN(vdest.v1, vsrc1.v1, vsrc1.v2); \ 1238b23a6c7e1aee255004dd19098d4c2462b61b849The Android Open Source Project NEON_FN(vdest.v2, vsrc2.v1, vsrc2.v2); 1248b23a6c7e1aee255004dd19098d4c2462b61b849The Android Open Source Project#define NEON_PDO4 \ 1258b23a6c7e1aee255004dd19098d4c2462b61b849The Android Open Source Project NEON_FN(vdest.v1, vsrc1.v1, vsrc1.v2); \ 1268b23a6c7e1aee255004dd19098d4c2462b61b849The Android Open Source Project NEON_FN(vdest.v2, vsrc1.v3, vsrc1.v4); \ 1278b23a6c7e1aee255004dd19098d4c2462b61b849The Android Open Source Project NEON_FN(vdest.v3, vsrc2.v1, vsrc2.v2); \ 1288b23a6c7e1aee255004dd19098d4c2462b61b849The Android Open Source Project NEON_FN(vdest.v4, vsrc2.v3, vsrc2.v4); \ 1298b23a6c7e1aee255004dd19098d4c2462b61b849The Android Open Source Project 1308b23a6c7e1aee255004dd19098d4c2462b61b849The Android Open Source Project#define NEON_POP(name, vtype, n) \ 1318b23a6c7e1aee255004dd19098d4c2462b61b849The Android Open Source Projectuint32_t HELPER(glue(neon_,name))(uint32_t arg1, uint32_t arg2) \ 1328b23a6c7e1aee255004dd19098d4c2462b61b849The Android Open Source Project{ \ 1338b23a6c7e1aee255004dd19098d4c2462b61b849The Android Open Source Project uint32_t res; \ 1348b23a6c7e1aee255004dd19098d4c2462b61b849The Android Open Source Project vtype vsrc1; \ 1358b23a6c7e1aee255004dd19098d4c2462b61b849The Android Open Source Project vtype vsrc2; \ 1368b23a6c7e1aee255004dd19098d4c2462b61b849The Android Open Source Project vtype vdest; \ 1378b23a6c7e1aee255004dd19098d4c2462b61b849The Android Open Source Project NEON_UNPACK(vtype, vsrc1, arg1); \ 1388b23a6c7e1aee255004dd19098d4c2462b61b849The Android Open Source Project NEON_UNPACK(vtype, vsrc2, arg2); \ 1398b23a6c7e1aee255004dd19098d4c2462b61b849The Android Open Source Project NEON_PDO##n; \ 1408b23a6c7e1aee255004dd19098d4c2462b61b849The Android Open Source Project NEON_PACK(vtype, res, vdest); \ 1418b23a6c7e1aee255004dd19098d4c2462b61b849The Android Open Source Project return res; \ 1428b23a6c7e1aee255004dd19098d4c2462b61b849The Android Open Source Project} 1438b23a6c7e1aee255004dd19098d4c2462b61b849The Android Open Source Project 1448b23a6c7e1aee255004dd19098d4c2462b61b849The Android Open Source Project/* Unary operators. */ 1458b23a6c7e1aee255004dd19098d4c2462b61b849The Android Open Source Project#define NEON_VOP1(name, vtype, n) \ 1468b23a6c7e1aee255004dd19098d4c2462b61b849The Android Open Source Projectuint32_t HELPER(glue(neon_,name))(uint32_t arg) \ 1478b23a6c7e1aee255004dd19098d4c2462b61b849The Android Open Source Project{ \ 1488b23a6c7e1aee255004dd19098d4c2462b61b849The Android Open Source Project vtype vsrc1; \ 1498b23a6c7e1aee255004dd19098d4c2462b61b849The Android Open Source Project vtype vdest; \ 1508b23a6c7e1aee255004dd19098d4c2462b61b849The Android Open Source Project NEON_UNPACK(vtype, vsrc1, arg); \ 1518b23a6c7e1aee255004dd19098d4c2462b61b849The Android Open Source Project NEON_DO##n; \ 1528b23a6c7e1aee255004dd19098d4c2462b61b849The Android Open Source Project NEON_PACK(vtype, arg, vdest); \ 1538b23a6c7e1aee255004dd19098d4c2462b61b849The Android Open Source Project return arg; \ 1548b23a6c7e1aee255004dd19098d4c2462b61b849The Android Open Source Project} 1558b23a6c7e1aee255004dd19098d4c2462b61b849The Android Open Source Project 1568b23a6c7e1aee255004dd19098d4c2462b61b849The Android Open Source Project 1578b23a6c7e1aee255004dd19098d4c2462b61b849The Android Open Source Project#define NEON_USAT(dest, src1, src2, type) do { \ 1588b23a6c7e1aee255004dd19098d4c2462b61b849The Android Open Source Project uint32_t tmp = (uint32_t)src1 + (uint32_t)src2; \ 1598b23a6c7e1aee255004dd19098d4c2462b61b849The Android Open Source Project if (tmp != (type)tmp) { \ 1608b23a6c7e1aee255004dd19098d4c2462b61b849The Android Open Source Project SET_QC(); \ 1618b23a6c7e1aee255004dd19098d4c2462b61b849The Android Open Source Project dest = ~0; \ 1628b23a6c7e1aee255004dd19098d4c2462b61b849The Android Open Source Project } else { \ 1638b23a6c7e1aee255004dd19098d4c2462b61b849The Android Open Source Project dest = tmp; \ 1648b23a6c7e1aee255004dd19098d4c2462b61b849The Android Open Source Project }} while(0) 1658b23a6c7e1aee255004dd19098d4c2462b61b849The Android Open Source Project#define NEON_FN(dest, src1, src2) NEON_USAT(dest, src1, src2, uint8_t) 1665285864985be9077e58e42235af6582dee72e841David 'Digit' TurnerNEON_VOP(qadd_u8, neon_u8, 4) 1678b23a6c7e1aee255004dd19098d4c2462b61b849The Android Open Source Project#undef NEON_FN 1688b23a6c7e1aee255004dd19098d4c2462b61b849The Android Open Source Project#define NEON_FN(dest, src1, src2) NEON_USAT(dest, src1, src2, uint16_t) 1695285864985be9077e58e42235af6582dee72e841David 'Digit' TurnerNEON_VOP(qadd_u16, neon_u16, 2) 1708b23a6c7e1aee255004dd19098d4c2462b61b849The Android Open Source Project#undef NEON_FN 1718b23a6c7e1aee255004dd19098d4c2462b61b849The Android Open Source Project#undef NEON_USAT 1728b23a6c7e1aee255004dd19098d4c2462b61b849The Android Open Source Project 1735285864985be9077e58e42235af6582dee72e841David 'Digit' Turneruint32_t HELPER(neon_qadd_u32)(uint32_t a, uint32_t b) 1745285864985be9077e58e42235af6582dee72e841David 'Digit' Turner{ 1755285864985be9077e58e42235af6582dee72e841David 'Digit' Turner uint32_t res = a + b; 1765285864985be9077e58e42235af6582dee72e841David 'Digit' Turner if (res < a) { 1775285864985be9077e58e42235af6582dee72e841David 'Digit' Turner SET_QC(); 1785285864985be9077e58e42235af6582dee72e841David 'Digit' Turner res = ~0; 1795285864985be9077e58e42235af6582dee72e841David 'Digit' Turner } 1805285864985be9077e58e42235af6582dee72e841David 'Digit' Turner return res; 1815285864985be9077e58e42235af6582dee72e841David 'Digit' Turner} 1825285864985be9077e58e42235af6582dee72e841David 'Digit' Turner 1835285864985be9077e58e42235af6582dee72e841David 'Digit' Turneruint64_t HELPER(neon_qadd_u64)(uint64_t src1, uint64_t src2) 1845285864985be9077e58e42235af6582dee72e841David 'Digit' Turner{ 1855285864985be9077e58e42235af6582dee72e841David 'Digit' Turner uint64_t res; 1865285864985be9077e58e42235af6582dee72e841David 'Digit' Turner 1875285864985be9077e58e42235af6582dee72e841David 'Digit' Turner res = src1 + src2; 1885285864985be9077e58e42235af6582dee72e841David 'Digit' Turner if (res < src1) { 1895285864985be9077e58e42235af6582dee72e841David 'Digit' Turner SET_QC(); 1905285864985be9077e58e42235af6582dee72e841David 'Digit' Turner res = ~(uint64_t)0; 1915285864985be9077e58e42235af6582dee72e841David 'Digit' Turner } 1925285864985be9077e58e42235af6582dee72e841David 'Digit' Turner return res; 1935285864985be9077e58e42235af6582dee72e841David 'Digit' Turner} 1945285864985be9077e58e42235af6582dee72e841David 'Digit' Turner 1958b23a6c7e1aee255004dd19098d4c2462b61b849The Android Open Source Project#define NEON_SSAT(dest, src1, src2, type) do { \ 1968b23a6c7e1aee255004dd19098d4c2462b61b849The Android Open Source Project int32_t tmp = (uint32_t)src1 + (uint32_t)src2; \ 1978b23a6c7e1aee255004dd19098d4c2462b61b849The Android Open Source Project if (tmp != (type)tmp) { \ 1988b23a6c7e1aee255004dd19098d4c2462b61b849The Android Open Source Project SET_QC(); \ 1998b23a6c7e1aee255004dd19098d4c2462b61b849The Android Open Source Project if (src2 > 0) { \ 2008b23a6c7e1aee255004dd19098d4c2462b61b849The Android Open Source Project tmp = (1 << (sizeof(type) * 8 - 1)) - 1; \ 2018b23a6c7e1aee255004dd19098d4c2462b61b849The Android Open Source Project } else { \ 2028b23a6c7e1aee255004dd19098d4c2462b61b849The Android Open Source Project tmp = 1 << (sizeof(type) * 8 - 1); \ 2038b23a6c7e1aee255004dd19098d4c2462b61b849The Android Open Source Project } \ 2048b23a6c7e1aee255004dd19098d4c2462b61b849The Android Open Source Project } \ 2058b23a6c7e1aee255004dd19098d4c2462b61b849The Android Open Source Project dest = tmp; \ 2068b23a6c7e1aee255004dd19098d4c2462b61b849The Android Open Source Project } while(0) 2078b23a6c7e1aee255004dd19098d4c2462b61b849The Android Open Source Project#define NEON_FN(dest, src1, src2) NEON_SSAT(dest, src1, src2, int8_t) 2085285864985be9077e58e42235af6582dee72e841David 'Digit' TurnerNEON_VOP(qadd_s8, neon_s8, 4) 2098b23a6c7e1aee255004dd19098d4c2462b61b849The Android Open Source Project#undef NEON_FN 2108b23a6c7e1aee255004dd19098d4c2462b61b849The Android Open Source Project#define NEON_FN(dest, src1, src2) NEON_SSAT(dest, src1, src2, int16_t) 2115285864985be9077e58e42235af6582dee72e841David 'Digit' TurnerNEON_VOP(qadd_s16, neon_s16, 2) 2128b23a6c7e1aee255004dd19098d4c2462b61b849The Android Open Source Project#undef NEON_FN 2138b23a6c7e1aee255004dd19098d4c2462b61b849The Android Open Source Project#undef NEON_SSAT 2148b23a6c7e1aee255004dd19098d4c2462b61b849The Android Open Source Project 2155285864985be9077e58e42235af6582dee72e841David 'Digit' Turneruint32_t HELPER(neon_qadd_s32)(uint32_t a, uint32_t b) 2165285864985be9077e58e42235af6582dee72e841David 'Digit' Turner{ 2175285864985be9077e58e42235af6582dee72e841David 'Digit' Turner uint32_t res = a + b; 2185285864985be9077e58e42235af6582dee72e841David 'Digit' Turner if (((res ^ a) & SIGNBIT) && !((a ^ b) & SIGNBIT)) { 2195285864985be9077e58e42235af6582dee72e841David 'Digit' Turner SET_QC(); 2205285864985be9077e58e42235af6582dee72e841David 'Digit' Turner res = ~(((int32_t)a >> 31) ^ SIGNBIT); 2215285864985be9077e58e42235af6582dee72e841David 'Digit' Turner } 2225285864985be9077e58e42235af6582dee72e841David 'Digit' Turner return res; 2235285864985be9077e58e42235af6582dee72e841David 'Digit' Turner} 2245285864985be9077e58e42235af6582dee72e841David 'Digit' Turner 2255285864985be9077e58e42235af6582dee72e841David 'Digit' Turneruint64_t HELPER(neon_qadd_s64)(uint64_t src1, uint64_t src2) 2265285864985be9077e58e42235af6582dee72e841David 'Digit' Turner{ 2275285864985be9077e58e42235af6582dee72e841David 'Digit' Turner uint64_t res; 2285285864985be9077e58e42235af6582dee72e841David 'Digit' Turner 2295285864985be9077e58e42235af6582dee72e841David 'Digit' Turner res = src1 + src2; 2305285864985be9077e58e42235af6582dee72e841David 'Digit' Turner if (((res ^ src1) & SIGNBIT64) && !((src1 ^ src2) & SIGNBIT64)) { 2315285864985be9077e58e42235af6582dee72e841David 'Digit' Turner SET_QC(); 2325285864985be9077e58e42235af6582dee72e841David 'Digit' Turner res = ((int64_t)src1 >> 63) ^ ~SIGNBIT64; 2335285864985be9077e58e42235af6582dee72e841David 'Digit' Turner } 2345285864985be9077e58e42235af6582dee72e841David 'Digit' Turner return res; 2355285864985be9077e58e42235af6582dee72e841David 'Digit' Turner} 2365285864985be9077e58e42235af6582dee72e841David 'Digit' Turner 2378b23a6c7e1aee255004dd19098d4c2462b61b849The Android Open Source Project#define NEON_USAT(dest, src1, src2, type) do { \ 2388b23a6c7e1aee255004dd19098d4c2462b61b849The Android Open Source Project uint32_t tmp = (uint32_t)src1 - (uint32_t)src2; \ 2398b23a6c7e1aee255004dd19098d4c2462b61b849The Android Open Source Project if (tmp != (type)tmp) { \ 2408b23a6c7e1aee255004dd19098d4c2462b61b849The Android Open Source Project SET_QC(); \ 2418b23a6c7e1aee255004dd19098d4c2462b61b849The Android Open Source Project dest = 0; \ 2428b23a6c7e1aee255004dd19098d4c2462b61b849The Android Open Source Project } else { \ 2438b23a6c7e1aee255004dd19098d4c2462b61b849The Android Open Source Project dest = tmp; \ 2448b23a6c7e1aee255004dd19098d4c2462b61b849The Android Open Source Project }} while(0) 2458b23a6c7e1aee255004dd19098d4c2462b61b849The Android Open Source Project#define NEON_FN(dest, src1, src2) NEON_USAT(dest, src1, src2, uint8_t) 2465285864985be9077e58e42235af6582dee72e841David 'Digit' TurnerNEON_VOP(qsub_u8, neon_u8, 4) 2478b23a6c7e1aee255004dd19098d4c2462b61b849The Android Open Source Project#undef NEON_FN 2488b23a6c7e1aee255004dd19098d4c2462b61b849The Android Open Source Project#define NEON_FN(dest, src1, src2) NEON_USAT(dest, src1, src2, uint16_t) 2495285864985be9077e58e42235af6582dee72e841David 'Digit' TurnerNEON_VOP(qsub_u16, neon_u16, 2) 2508b23a6c7e1aee255004dd19098d4c2462b61b849The Android Open Source Project#undef NEON_FN 2518b23a6c7e1aee255004dd19098d4c2462b61b849The Android Open Source Project#undef NEON_USAT 2528b23a6c7e1aee255004dd19098d4c2462b61b849The Android Open Source Project 2535285864985be9077e58e42235af6582dee72e841David 'Digit' Turneruint32_t HELPER(neon_qsub_u32)(uint32_t a, uint32_t b) 2545285864985be9077e58e42235af6582dee72e841David 'Digit' Turner{ 2555285864985be9077e58e42235af6582dee72e841David 'Digit' Turner uint32_t res = a - b; 2565285864985be9077e58e42235af6582dee72e841David 'Digit' Turner if (res > a) { 2575285864985be9077e58e42235af6582dee72e841David 'Digit' Turner SET_QC(); 2585285864985be9077e58e42235af6582dee72e841David 'Digit' Turner res = 0; 2595285864985be9077e58e42235af6582dee72e841David 'Digit' Turner } 2605285864985be9077e58e42235af6582dee72e841David 'Digit' Turner return res; 2615285864985be9077e58e42235af6582dee72e841David 'Digit' Turner} 2625285864985be9077e58e42235af6582dee72e841David 'Digit' Turner 2635285864985be9077e58e42235af6582dee72e841David 'Digit' Turneruint64_t HELPER(neon_qsub_u64)(uint64_t src1, uint64_t src2) 2645285864985be9077e58e42235af6582dee72e841David 'Digit' Turner{ 2655285864985be9077e58e42235af6582dee72e841David 'Digit' Turner uint64_t res; 2665285864985be9077e58e42235af6582dee72e841David 'Digit' Turner 2675285864985be9077e58e42235af6582dee72e841David 'Digit' Turner if (src1 < src2) { 2685285864985be9077e58e42235af6582dee72e841David 'Digit' Turner SET_QC(); 2695285864985be9077e58e42235af6582dee72e841David 'Digit' Turner res = 0; 2705285864985be9077e58e42235af6582dee72e841David 'Digit' Turner } else { 2715285864985be9077e58e42235af6582dee72e841David 'Digit' Turner res = src1 - src2; 2725285864985be9077e58e42235af6582dee72e841David 'Digit' Turner } 2735285864985be9077e58e42235af6582dee72e841David 'Digit' Turner return res; 2745285864985be9077e58e42235af6582dee72e841David 'Digit' Turner} 2755285864985be9077e58e42235af6582dee72e841David 'Digit' Turner 2768b23a6c7e1aee255004dd19098d4c2462b61b849The Android Open Source Project#define NEON_SSAT(dest, src1, src2, type) do { \ 2778b23a6c7e1aee255004dd19098d4c2462b61b849The Android Open Source Project int32_t tmp = (uint32_t)src1 - (uint32_t)src2; \ 2788b23a6c7e1aee255004dd19098d4c2462b61b849The Android Open Source Project if (tmp != (type)tmp) { \ 2798b23a6c7e1aee255004dd19098d4c2462b61b849The Android Open Source Project SET_QC(); \ 2808b23a6c7e1aee255004dd19098d4c2462b61b849The Android Open Source Project if (src2 < 0) { \ 2818b23a6c7e1aee255004dd19098d4c2462b61b849The Android Open Source Project tmp = (1 << (sizeof(type) * 8 - 1)) - 1; \ 2828b23a6c7e1aee255004dd19098d4c2462b61b849The Android Open Source Project } else { \ 2838b23a6c7e1aee255004dd19098d4c2462b61b849The Android Open Source Project tmp = 1 << (sizeof(type) * 8 - 1); \ 2848b23a6c7e1aee255004dd19098d4c2462b61b849The Android Open Source Project } \ 2858b23a6c7e1aee255004dd19098d4c2462b61b849The Android Open Source Project } \ 2868b23a6c7e1aee255004dd19098d4c2462b61b849The Android Open Source Project dest = tmp; \ 2878b23a6c7e1aee255004dd19098d4c2462b61b849The Android Open Source Project } while(0) 2888b23a6c7e1aee255004dd19098d4c2462b61b849The Android Open Source Project#define NEON_FN(dest, src1, src2) NEON_SSAT(dest, src1, src2, int8_t) 2895285864985be9077e58e42235af6582dee72e841David 'Digit' TurnerNEON_VOP(qsub_s8, neon_s8, 4) 2908b23a6c7e1aee255004dd19098d4c2462b61b849The Android Open Source Project#undef NEON_FN 2918b23a6c7e1aee255004dd19098d4c2462b61b849The Android Open Source Project#define NEON_FN(dest, src1, src2) NEON_SSAT(dest, src1, src2, int16_t) 2925285864985be9077e58e42235af6582dee72e841David 'Digit' TurnerNEON_VOP(qsub_s16, neon_s16, 2) 2938b23a6c7e1aee255004dd19098d4c2462b61b849The Android Open Source Project#undef NEON_FN 2948b23a6c7e1aee255004dd19098d4c2462b61b849The Android Open Source Project#undef NEON_SSAT 2958b23a6c7e1aee255004dd19098d4c2462b61b849The Android Open Source Project 2965285864985be9077e58e42235af6582dee72e841David 'Digit' Turneruint32_t HELPER(neon_qsub_s32)(uint32_t a, uint32_t b) 2975285864985be9077e58e42235af6582dee72e841David 'Digit' Turner{ 2985285864985be9077e58e42235af6582dee72e841David 'Digit' Turner uint32_t res = a - b; 2995285864985be9077e58e42235af6582dee72e841David 'Digit' Turner if (((res ^ a) & SIGNBIT) && ((a ^ b) & SIGNBIT)) { 3005285864985be9077e58e42235af6582dee72e841David 'Digit' Turner SET_QC(); 3015285864985be9077e58e42235af6582dee72e841David 'Digit' Turner res = ~(((int32_t)a >> 31) ^ SIGNBIT); 3025285864985be9077e58e42235af6582dee72e841David 'Digit' Turner } 3035285864985be9077e58e42235af6582dee72e841David 'Digit' Turner return res; 3045285864985be9077e58e42235af6582dee72e841David 'Digit' Turner} 3055285864985be9077e58e42235af6582dee72e841David 'Digit' Turner 3065285864985be9077e58e42235af6582dee72e841David 'Digit' Turneruint64_t HELPER(neon_qsub_s64)(uint64_t src1, uint64_t src2) 3075285864985be9077e58e42235af6582dee72e841David 'Digit' Turner{ 3085285864985be9077e58e42235af6582dee72e841David 'Digit' Turner uint64_t res; 3095285864985be9077e58e42235af6582dee72e841David 'Digit' Turner 3105285864985be9077e58e42235af6582dee72e841David 'Digit' Turner res = src1 - src2; 3115285864985be9077e58e42235af6582dee72e841David 'Digit' Turner if (((res ^ src1) & SIGNBIT64) && ((src1 ^ src2) & SIGNBIT64)) { 3125285864985be9077e58e42235af6582dee72e841David 'Digit' Turner SET_QC(); 3135285864985be9077e58e42235af6582dee72e841David 'Digit' Turner res = ((int64_t)src1 >> 63) ^ ~SIGNBIT64; 3145285864985be9077e58e42235af6582dee72e841David 'Digit' Turner } 3155285864985be9077e58e42235af6582dee72e841David 'Digit' Turner return res; 3165285864985be9077e58e42235af6582dee72e841David 'Digit' Turner} 3175285864985be9077e58e42235af6582dee72e841David 'Digit' Turner 3188b23a6c7e1aee255004dd19098d4c2462b61b849The Android Open Source Project#define NEON_FN(dest, src1, src2) dest = (src1 + src2) >> 1 3198b23a6c7e1aee255004dd19098d4c2462b61b849The Android Open Source ProjectNEON_VOP(hadd_s8, neon_s8, 4) 3208b23a6c7e1aee255004dd19098d4c2462b61b849The Android Open Source ProjectNEON_VOP(hadd_u8, neon_u8, 4) 3218b23a6c7e1aee255004dd19098d4c2462b61b849The Android Open Source ProjectNEON_VOP(hadd_s16, neon_s16, 2) 3228b23a6c7e1aee255004dd19098d4c2462b61b849The Android Open Source ProjectNEON_VOP(hadd_u16, neon_u16, 2) 3238b23a6c7e1aee255004dd19098d4c2462b61b849The Android Open Source Project#undef NEON_FN 3248b23a6c7e1aee255004dd19098d4c2462b61b849The Android Open Source Project 3258b23a6c7e1aee255004dd19098d4c2462b61b849The Android Open Source Projectint32_t HELPER(neon_hadd_s32)(int32_t src1, int32_t src2) 3268b23a6c7e1aee255004dd19098d4c2462b61b849The Android Open Source Project{ 3278b23a6c7e1aee255004dd19098d4c2462b61b849The Android Open Source Project int32_t dest; 3288b23a6c7e1aee255004dd19098d4c2462b61b849The Android Open Source Project 3298b23a6c7e1aee255004dd19098d4c2462b61b849The Android Open Source Project dest = (src1 >> 1) + (src2 >> 1); 3308b23a6c7e1aee255004dd19098d4c2462b61b849The Android Open Source Project if (src1 & src2 & 1) 3318b23a6c7e1aee255004dd19098d4c2462b61b849The Android Open Source Project dest++; 3328b23a6c7e1aee255004dd19098d4c2462b61b849The Android Open Source Project return dest; 3338b23a6c7e1aee255004dd19098d4c2462b61b849The Android Open Source Project} 3348b23a6c7e1aee255004dd19098d4c2462b61b849The Android Open Source Project 3358b23a6c7e1aee255004dd19098d4c2462b61b849The Android Open Source Projectuint32_t HELPER(neon_hadd_u32)(uint32_t src1, uint32_t src2) 3368b23a6c7e1aee255004dd19098d4c2462b61b849The Android Open Source Project{ 3378b23a6c7e1aee255004dd19098d4c2462b61b849The Android Open Source Project uint32_t dest; 3388b23a6c7e1aee255004dd19098d4c2462b61b849The Android Open Source Project 3398b23a6c7e1aee255004dd19098d4c2462b61b849The Android Open Source Project dest = (src1 >> 1) + (src2 >> 1); 3408b23a6c7e1aee255004dd19098d4c2462b61b849The Android Open Source Project if (src1 & src2 & 1) 3418b23a6c7e1aee255004dd19098d4c2462b61b849The Android Open Source Project dest++; 3428b23a6c7e1aee255004dd19098d4c2462b61b849The Android Open Source Project return dest; 3438b23a6c7e1aee255004dd19098d4c2462b61b849The Android Open Source Project} 3448b23a6c7e1aee255004dd19098d4c2462b61b849The Android Open Source Project 3458b23a6c7e1aee255004dd19098d4c2462b61b849The Android Open Source Project#define NEON_FN(dest, src1, src2) dest = (src1 + src2 + 1) >> 1 3468b23a6c7e1aee255004dd19098d4c2462b61b849The Android Open Source ProjectNEON_VOP(rhadd_s8, neon_s8, 4) 3478b23a6c7e1aee255004dd19098d4c2462b61b849The Android Open Source ProjectNEON_VOP(rhadd_u8, neon_u8, 4) 3488b23a6c7e1aee255004dd19098d4c2462b61b849The Android Open Source ProjectNEON_VOP(rhadd_s16, neon_s16, 2) 3498b23a6c7e1aee255004dd19098d4c2462b61b849The Android Open Source ProjectNEON_VOP(rhadd_u16, neon_u16, 2) 3508b23a6c7e1aee255004dd19098d4c2462b61b849The Android Open Source Project#undef NEON_FN 3518b23a6c7e1aee255004dd19098d4c2462b61b849The Android Open Source Project 3528b23a6c7e1aee255004dd19098d4c2462b61b849The Android Open Source Projectint32_t HELPER(neon_rhadd_s32)(int32_t src1, int32_t src2) 3538b23a6c7e1aee255004dd19098d4c2462b61b849The Android Open Source Project{ 3548b23a6c7e1aee255004dd19098d4c2462b61b849The Android Open Source Project int32_t dest; 3558b23a6c7e1aee255004dd19098d4c2462b61b849The Android Open Source Project 3568b23a6c7e1aee255004dd19098d4c2462b61b849The Android Open Source Project dest = (src1 >> 1) + (src2 >> 1); 3578b23a6c7e1aee255004dd19098d4c2462b61b849The Android Open Source Project if ((src1 | src2) & 1) 3588b23a6c7e1aee255004dd19098d4c2462b61b849The Android Open Source Project dest++; 3598b23a6c7e1aee255004dd19098d4c2462b61b849The Android Open Source Project return dest; 3608b23a6c7e1aee255004dd19098d4c2462b61b849The Android Open Source Project} 3618b23a6c7e1aee255004dd19098d4c2462b61b849The Android Open Source Project 3628b23a6c7e1aee255004dd19098d4c2462b61b849The Android Open Source Projectuint32_t HELPER(neon_rhadd_u32)(uint32_t src1, uint32_t src2) 3638b23a6c7e1aee255004dd19098d4c2462b61b849The Android Open Source Project{ 3648b23a6c7e1aee255004dd19098d4c2462b61b849The Android Open Source Project uint32_t dest; 3658b23a6c7e1aee255004dd19098d4c2462b61b849The Android Open Source Project 3668b23a6c7e1aee255004dd19098d4c2462b61b849The Android Open Source Project dest = (src1 >> 1) + (src2 >> 1); 3678b23a6c7e1aee255004dd19098d4c2462b61b849The Android Open Source Project if ((src1 | src2) & 1) 3688b23a6c7e1aee255004dd19098d4c2462b61b849The Android Open Source Project dest++; 3698b23a6c7e1aee255004dd19098d4c2462b61b849The Android Open Source Project return dest; 3708b23a6c7e1aee255004dd19098d4c2462b61b849The Android Open Source Project} 3718b23a6c7e1aee255004dd19098d4c2462b61b849The Android Open Source Project 3728b23a6c7e1aee255004dd19098d4c2462b61b849The Android Open Source Project#define NEON_FN(dest, src1, src2) dest = (src1 - src2) >> 1 3738b23a6c7e1aee255004dd19098d4c2462b61b849The Android Open Source ProjectNEON_VOP(hsub_s8, neon_s8, 4) 3748b23a6c7e1aee255004dd19098d4c2462b61b849The Android Open Source ProjectNEON_VOP(hsub_u8, neon_u8, 4) 3758b23a6c7e1aee255004dd19098d4c2462b61b849The Android Open Source ProjectNEON_VOP(hsub_s16, neon_s16, 2) 3768b23a6c7e1aee255004dd19098d4c2462b61b849The Android Open Source ProjectNEON_VOP(hsub_u16, neon_u16, 2) 3778b23a6c7e1aee255004dd19098d4c2462b61b849The Android Open Source Project#undef NEON_FN 3788b23a6c7e1aee255004dd19098d4c2462b61b849The Android Open Source Project 3798b23a6c7e1aee255004dd19098d4c2462b61b849The Android Open Source Projectint32_t HELPER(neon_hsub_s32)(int32_t src1, int32_t src2) 3808b23a6c7e1aee255004dd19098d4c2462b61b849The Android Open Source Project{ 3818b23a6c7e1aee255004dd19098d4c2462b61b849The Android Open Source Project int32_t dest; 3828b23a6c7e1aee255004dd19098d4c2462b61b849The Android Open Source Project 3838b23a6c7e1aee255004dd19098d4c2462b61b849The Android Open Source Project dest = (src1 >> 1) - (src2 >> 1); 3848b23a6c7e1aee255004dd19098d4c2462b61b849The Android Open Source Project if ((~src1) & src2 & 1) 3858b23a6c7e1aee255004dd19098d4c2462b61b849The Android Open Source Project dest--; 3868b23a6c7e1aee255004dd19098d4c2462b61b849The Android Open Source Project return dest; 3878b23a6c7e1aee255004dd19098d4c2462b61b849The Android Open Source Project} 3888b23a6c7e1aee255004dd19098d4c2462b61b849The Android Open Source Project 3898b23a6c7e1aee255004dd19098d4c2462b61b849The Android Open Source Projectuint32_t HELPER(neon_hsub_u32)(uint32_t src1, uint32_t src2) 3908b23a6c7e1aee255004dd19098d4c2462b61b849The Android Open Source Project{ 3918b23a6c7e1aee255004dd19098d4c2462b61b849The Android Open Source Project uint32_t dest; 3928b23a6c7e1aee255004dd19098d4c2462b61b849The Android Open Source Project 3938b23a6c7e1aee255004dd19098d4c2462b61b849The Android Open Source Project dest = (src1 >> 1) - (src2 >> 1); 3948b23a6c7e1aee255004dd19098d4c2462b61b849The Android Open Source Project if ((~src1) & src2 & 1) 3958b23a6c7e1aee255004dd19098d4c2462b61b849The Android Open Source Project dest--; 3968b23a6c7e1aee255004dd19098d4c2462b61b849The Android Open Source Project return dest; 3978b23a6c7e1aee255004dd19098d4c2462b61b849The Android Open Source Project} 3988b23a6c7e1aee255004dd19098d4c2462b61b849The Android Open Source Project 3998b23a6c7e1aee255004dd19098d4c2462b61b849The Android Open Source Project#define NEON_FN(dest, src1, src2) dest = (src1 > src2) ? ~0 : 0 4008b23a6c7e1aee255004dd19098d4c2462b61b849The Android Open Source ProjectNEON_VOP(cgt_s8, neon_s8, 4) 4018b23a6c7e1aee255004dd19098d4c2462b61b849The Android Open Source ProjectNEON_VOP(cgt_u8, neon_u8, 4) 4028b23a6c7e1aee255004dd19098d4c2462b61b849The Android Open Source ProjectNEON_VOP(cgt_s16, neon_s16, 2) 4038b23a6c7e1aee255004dd19098d4c2462b61b849The Android Open Source ProjectNEON_VOP(cgt_u16, neon_u16, 2) 4048b23a6c7e1aee255004dd19098d4c2462b61b849The Android Open Source ProjectNEON_VOP(cgt_s32, neon_s32, 1) 4058b23a6c7e1aee255004dd19098d4c2462b61b849The Android Open Source ProjectNEON_VOP(cgt_u32, neon_u32, 1) 4068b23a6c7e1aee255004dd19098d4c2462b61b849The Android Open Source Project#undef NEON_FN 4078b23a6c7e1aee255004dd19098d4c2462b61b849The Android Open Source Project 4088b23a6c7e1aee255004dd19098d4c2462b61b849The Android Open Source Project#define NEON_FN(dest, src1, src2) dest = (src1 >= src2) ? ~0 : 0 4098b23a6c7e1aee255004dd19098d4c2462b61b849The Android Open Source ProjectNEON_VOP(cge_s8, neon_s8, 4) 4108b23a6c7e1aee255004dd19098d4c2462b61b849The Android Open Source ProjectNEON_VOP(cge_u8, neon_u8, 4) 4118b23a6c7e1aee255004dd19098d4c2462b61b849The Android Open Source ProjectNEON_VOP(cge_s16, neon_s16, 2) 4128b23a6c7e1aee255004dd19098d4c2462b61b849The Android Open Source ProjectNEON_VOP(cge_u16, neon_u16, 2) 4138b23a6c7e1aee255004dd19098d4c2462b61b849The Android Open Source ProjectNEON_VOP(cge_s32, neon_s32, 1) 4148b23a6c7e1aee255004dd19098d4c2462b61b849The Android Open Source ProjectNEON_VOP(cge_u32, neon_u32, 1) 4158b23a6c7e1aee255004dd19098d4c2462b61b849The Android Open Source Project#undef NEON_FN 4168b23a6c7e1aee255004dd19098d4c2462b61b849The Android Open Source Project 4178b23a6c7e1aee255004dd19098d4c2462b61b849The Android Open Source Project#define NEON_FN(dest, src1, src2) dest = (src1 < src2) ? src1 : src2 4188b23a6c7e1aee255004dd19098d4c2462b61b849The Android Open Source ProjectNEON_VOP(min_s8, neon_s8, 4) 4198b23a6c7e1aee255004dd19098d4c2462b61b849The Android Open Source ProjectNEON_VOP(min_u8, neon_u8, 4) 4208b23a6c7e1aee255004dd19098d4c2462b61b849The Android Open Source ProjectNEON_VOP(min_s16, neon_s16, 2) 4218b23a6c7e1aee255004dd19098d4c2462b61b849The Android Open Source ProjectNEON_VOP(min_u16, neon_u16, 2) 4228b23a6c7e1aee255004dd19098d4c2462b61b849The Android Open Source ProjectNEON_VOP(min_s32, neon_s32, 1) 4238b23a6c7e1aee255004dd19098d4c2462b61b849The Android Open Source ProjectNEON_VOP(min_u32, neon_u32, 1) 4248b23a6c7e1aee255004dd19098d4c2462b61b849The Android Open Source ProjectNEON_POP(pmin_s8, neon_s8, 4) 4258b23a6c7e1aee255004dd19098d4c2462b61b849The Android Open Source ProjectNEON_POP(pmin_u8, neon_u8, 4) 4268b23a6c7e1aee255004dd19098d4c2462b61b849The Android Open Source ProjectNEON_POP(pmin_s16, neon_s16, 2) 4278b23a6c7e1aee255004dd19098d4c2462b61b849The Android Open Source ProjectNEON_POP(pmin_u16, neon_u16, 2) 4288b23a6c7e1aee255004dd19098d4c2462b61b849The Android Open Source Project#undef NEON_FN 4298b23a6c7e1aee255004dd19098d4c2462b61b849The Android Open Source Project 4308b23a6c7e1aee255004dd19098d4c2462b61b849The Android Open Source Project#define NEON_FN(dest, src1, src2) dest = (src1 > src2) ? src1 : src2 4318b23a6c7e1aee255004dd19098d4c2462b61b849The Android Open Source ProjectNEON_VOP(max_s8, neon_s8, 4) 4328b23a6c7e1aee255004dd19098d4c2462b61b849The Android Open Source ProjectNEON_VOP(max_u8, neon_u8, 4) 4338b23a6c7e1aee255004dd19098d4c2462b61b849The Android Open Source ProjectNEON_VOP(max_s16, neon_s16, 2) 4348b23a6c7e1aee255004dd19098d4c2462b61b849The Android Open Source ProjectNEON_VOP(max_u16, neon_u16, 2) 4358b23a6c7e1aee255004dd19098d4c2462b61b849The Android Open Source ProjectNEON_VOP(max_s32, neon_s32, 1) 4368b23a6c7e1aee255004dd19098d4c2462b61b849The Android Open Source ProjectNEON_VOP(max_u32, neon_u32, 1) 4378b23a6c7e1aee255004dd19098d4c2462b61b849The Android Open Source ProjectNEON_POP(pmax_s8, neon_s8, 4) 4388b23a6c7e1aee255004dd19098d4c2462b61b849The Android Open Source ProjectNEON_POP(pmax_u8, neon_u8, 4) 4398b23a6c7e1aee255004dd19098d4c2462b61b849The Android Open Source ProjectNEON_POP(pmax_s16, neon_s16, 2) 4408b23a6c7e1aee255004dd19098d4c2462b61b849The Android Open Source ProjectNEON_POP(pmax_u16, neon_u16, 2) 4418b23a6c7e1aee255004dd19098d4c2462b61b849The Android Open Source Project#undef NEON_FN 4428b23a6c7e1aee255004dd19098d4c2462b61b849The Android Open Source Project 4438b23a6c7e1aee255004dd19098d4c2462b61b849The Android Open Source Project#define NEON_FN(dest, src1, src2) \ 4448b23a6c7e1aee255004dd19098d4c2462b61b849The Android Open Source Project dest = (src1 > src2) ? (src1 - src2) : (src2 - src1) 4458b23a6c7e1aee255004dd19098d4c2462b61b849The Android Open Source ProjectNEON_VOP(abd_s8, neon_s8, 4) 4468b23a6c7e1aee255004dd19098d4c2462b61b849The Android Open Source ProjectNEON_VOP(abd_u8, neon_u8, 4) 4478b23a6c7e1aee255004dd19098d4c2462b61b849The Android Open Source ProjectNEON_VOP(abd_s16, neon_s16, 2) 4488b23a6c7e1aee255004dd19098d4c2462b61b849The Android Open Source ProjectNEON_VOP(abd_u16, neon_u16, 2) 4498b23a6c7e1aee255004dd19098d4c2462b61b849The Android Open Source ProjectNEON_VOP(abd_s32, neon_s32, 1) 4508b23a6c7e1aee255004dd19098d4c2462b61b849The Android Open Source ProjectNEON_VOP(abd_u32, neon_u32, 1) 4518b23a6c7e1aee255004dd19098d4c2462b61b849The Android Open Source Project#undef NEON_FN 4528b23a6c7e1aee255004dd19098d4c2462b61b849The Android Open Source Project 4538b23a6c7e1aee255004dd19098d4c2462b61b849The Android Open Source Project#define NEON_FN(dest, src1, src2) do { \ 4548b23a6c7e1aee255004dd19098d4c2462b61b849The Android Open Source Project int8_t tmp; \ 4558b23a6c7e1aee255004dd19098d4c2462b61b849The Android Open Source Project tmp = (int8_t)src2; \ 4565285864985be9077e58e42235af6582dee72e841David 'Digit' Turner if (tmp >= (ssize_t)sizeof(src1) * 8 || \ 4575285864985be9077e58e42235af6582dee72e841David 'Digit' Turner tmp <= -(ssize_t)sizeof(src1) * 8) { \ 4588b23a6c7e1aee255004dd19098d4c2462b61b849The Android Open Source Project dest = 0; \ 4598b23a6c7e1aee255004dd19098d4c2462b61b849The Android Open Source Project } else if (tmp < 0) { \ 4608b23a6c7e1aee255004dd19098d4c2462b61b849The Android Open Source Project dest = src1 >> -tmp; \ 4618b23a6c7e1aee255004dd19098d4c2462b61b849The Android Open Source Project } else { \ 4628b23a6c7e1aee255004dd19098d4c2462b61b849The Android Open Source Project dest = src1 << tmp; \ 4638b23a6c7e1aee255004dd19098d4c2462b61b849The Android Open Source Project }} while (0) 4648b23a6c7e1aee255004dd19098d4c2462b61b849The Android Open Source ProjectNEON_VOP(shl_u8, neon_u8, 4) 4658b23a6c7e1aee255004dd19098d4c2462b61b849The Android Open Source ProjectNEON_VOP(shl_u16, neon_u16, 2) 4668b23a6c7e1aee255004dd19098d4c2462b61b849The Android Open Source ProjectNEON_VOP(shl_u32, neon_u32, 1) 4678b23a6c7e1aee255004dd19098d4c2462b61b849The Android Open Source Project#undef NEON_FN 4688b23a6c7e1aee255004dd19098d4c2462b61b849The Android Open Source Project 4698b23a6c7e1aee255004dd19098d4c2462b61b849The Android Open Source Projectuint64_t HELPER(neon_shl_u64)(uint64_t val, uint64_t shiftop) 4708b23a6c7e1aee255004dd19098d4c2462b61b849The Android Open Source Project{ 4718b23a6c7e1aee255004dd19098d4c2462b61b849The Android Open Source Project int8_t shift = (int8_t)shiftop; 4728b23a6c7e1aee255004dd19098d4c2462b61b849The Android Open Source Project if (shift >= 64 || shift <= -64) { 4738b23a6c7e1aee255004dd19098d4c2462b61b849The Android Open Source Project val = 0; 4748b23a6c7e1aee255004dd19098d4c2462b61b849The Android Open Source Project } else if (shift < 0) { 4758b23a6c7e1aee255004dd19098d4c2462b61b849The Android Open Source Project val >>= -shift; 4768b23a6c7e1aee255004dd19098d4c2462b61b849The Android Open Source Project } else { 4778b23a6c7e1aee255004dd19098d4c2462b61b849The Android Open Source Project val <<= shift; 4788b23a6c7e1aee255004dd19098d4c2462b61b849The Android Open Source Project } 4798b23a6c7e1aee255004dd19098d4c2462b61b849The Android Open Source Project return val; 4808b23a6c7e1aee255004dd19098d4c2462b61b849The Android Open Source Project} 4818b23a6c7e1aee255004dd19098d4c2462b61b849The Android Open Source Project 4828b23a6c7e1aee255004dd19098d4c2462b61b849The Android Open Source Project#define NEON_FN(dest, src1, src2) do { \ 4838b23a6c7e1aee255004dd19098d4c2462b61b849The Android Open Source Project int8_t tmp; \ 4848b23a6c7e1aee255004dd19098d4c2462b61b849The Android Open Source Project tmp = (int8_t)src2; \ 4855285864985be9077e58e42235af6582dee72e841David 'Digit' Turner if (tmp >= (ssize_t)sizeof(src1) * 8) { \ 4868b23a6c7e1aee255004dd19098d4c2462b61b849The Android Open Source Project dest = 0; \ 4875285864985be9077e58e42235af6582dee72e841David 'Digit' Turner } else if (tmp <= -(ssize_t)sizeof(src1) * 8) { \ 4888b23a6c7e1aee255004dd19098d4c2462b61b849The Android Open Source Project dest = src1 >> (sizeof(src1) * 8 - 1); \ 4898b23a6c7e1aee255004dd19098d4c2462b61b849The Android Open Source Project } else if (tmp < 0) { \ 4908b23a6c7e1aee255004dd19098d4c2462b61b849The Android Open Source Project dest = src1 >> -tmp; \ 4918b23a6c7e1aee255004dd19098d4c2462b61b849The Android Open Source Project } else { \ 4928b23a6c7e1aee255004dd19098d4c2462b61b849The Android Open Source Project dest = src1 << tmp; \ 4938b23a6c7e1aee255004dd19098d4c2462b61b849The Android Open Source Project }} while (0) 4948b23a6c7e1aee255004dd19098d4c2462b61b849The Android Open Source ProjectNEON_VOP(shl_s8, neon_s8, 4) 4958b23a6c7e1aee255004dd19098d4c2462b61b849The Android Open Source ProjectNEON_VOP(shl_s16, neon_s16, 2) 4968b23a6c7e1aee255004dd19098d4c2462b61b849The Android Open Source ProjectNEON_VOP(shl_s32, neon_s32, 1) 4978b23a6c7e1aee255004dd19098d4c2462b61b849The Android Open Source Project#undef NEON_FN 4988b23a6c7e1aee255004dd19098d4c2462b61b849The Android Open Source Project 4998b23a6c7e1aee255004dd19098d4c2462b61b849The Android Open Source Projectuint64_t HELPER(neon_shl_s64)(uint64_t valop, uint64_t shiftop) 5008b23a6c7e1aee255004dd19098d4c2462b61b849The Android Open Source Project{ 5018b23a6c7e1aee255004dd19098d4c2462b61b849The Android Open Source Project int8_t shift = (int8_t)shiftop; 5028b23a6c7e1aee255004dd19098d4c2462b61b849The Android Open Source Project int64_t val = valop; 5038b23a6c7e1aee255004dd19098d4c2462b61b849The Android Open Source Project if (shift >= 64) { 5048b23a6c7e1aee255004dd19098d4c2462b61b849The Android Open Source Project val = 0; 5058b23a6c7e1aee255004dd19098d4c2462b61b849The Android Open Source Project } else if (shift <= -64) { 5068b23a6c7e1aee255004dd19098d4c2462b61b849The Android Open Source Project val >>= 63; 5078b23a6c7e1aee255004dd19098d4c2462b61b849The Android Open Source Project } else if (shift < 0) { 5088b23a6c7e1aee255004dd19098d4c2462b61b849The Android Open Source Project val >>= -shift; 5098b23a6c7e1aee255004dd19098d4c2462b61b849The Android Open Source Project } else { 5108b23a6c7e1aee255004dd19098d4c2462b61b849The Android Open Source Project val <<= shift; 5118b23a6c7e1aee255004dd19098d4c2462b61b849The Android Open Source Project } 5128b23a6c7e1aee255004dd19098d4c2462b61b849The Android Open Source Project return val; 5138b23a6c7e1aee255004dd19098d4c2462b61b849The Android Open Source Project} 5148b23a6c7e1aee255004dd19098d4c2462b61b849The Android Open Source Project 5158b23a6c7e1aee255004dd19098d4c2462b61b849The Android Open Source Project#define NEON_FN(dest, src1, src2) do { \ 5168b23a6c7e1aee255004dd19098d4c2462b61b849The Android Open Source Project int8_t tmp; \ 5178b23a6c7e1aee255004dd19098d4c2462b61b849The Android Open Source Project tmp = (int8_t)src2; \ 5185285864985be9077e58e42235af6582dee72e841David 'Digit' Turner if ((tmp >= (ssize_t)sizeof(src1) * 8) \ 5195285864985be9077e58e42235af6582dee72e841David 'Digit' Turner || (tmp <= -(ssize_t)sizeof(src1) * 8)) { \ 5208b23a6c7e1aee255004dd19098d4c2462b61b849The Android Open Source Project dest = 0; \ 5218b23a6c7e1aee255004dd19098d4c2462b61b849The Android Open Source Project } else if (tmp < 0) { \ 5228b23a6c7e1aee255004dd19098d4c2462b61b849The Android Open Source Project dest = (src1 + (1 << (-1 - tmp))) >> -tmp; \ 5238b23a6c7e1aee255004dd19098d4c2462b61b849The Android Open Source Project } else { \ 5248b23a6c7e1aee255004dd19098d4c2462b61b849The Android Open Source Project dest = src1 << tmp; \ 5258b23a6c7e1aee255004dd19098d4c2462b61b849The Android Open Source Project }} while (0) 5268b23a6c7e1aee255004dd19098d4c2462b61b849The Android Open Source ProjectNEON_VOP(rshl_s8, neon_s8, 4) 5278b23a6c7e1aee255004dd19098d4c2462b61b849The Android Open Source ProjectNEON_VOP(rshl_s16, neon_s16, 2) 5288b23a6c7e1aee255004dd19098d4c2462b61b849The Android Open Source Project#undef NEON_FN 5298b23a6c7e1aee255004dd19098d4c2462b61b849The Android Open Source Project 5305285864985be9077e58e42235af6582dee72e841David 'Digit' Turner/* The addition of the rounding constant may overflow, so we use an 5315285864985be9077e58e42235af6582dee72e841David 'Digit' Turner * intermediate 64 bits accumulator. */ 5325285864985be9077e58e42235af6582dee72e841David 'Digit' Turneruint32_t HELPER(neon_rshl_s32)(uint32_t valop, uint32_t shiftop) 5335285864985be9077e58e42235af6582dee72e841David 'Digit' Turner{ 5345285864985be9077e58e42235af6582dee72e841David 'Digit' Turner int32_t dest; 5355285864985be9077e58e42235af6582dee72e841David 'Digit' Turner int32_t val = (int32_t)valop; 5365285864985be9077e58e42235af6582dee72e841David 'Digit' Turner int8_t shift = (int8_t)shiftop; 5375285864985be9077e58e42235af6582dee72e841David 'Digit' Turner if ((shift >= 32) || (shift <= -32)) { 5385285864985be9077e58e42235af6582dee72e841David 'Digit' Turner dest = 0; 5395285864985be9077e58e42235af6582dee72e841David 'Digit' Turner } else if (shift < 0) { 5405285864985be9077e58e42235af6582dee72e841David 'Digit' Turner int64_t big_dest = ((int64_t)val + (1 << (-1 - shift))); 5415285864985be9077e58e42235af6582dee72e841David 'Digit' Turner dest = big_dest >> -shift; 5425285864985be9077e58e42235af6582dee72e841David 'Digit' Turner } else { 5435285864985be9077e58e42235af6582dee72e841David 'Digit' Turner dest = val << shift; 5445285864985be9077e58e42235af6582dee72e841David 'Digit' Turner } 5455285864985be9077e58e42235af6582dee72e841David 'Digit' Turner return dest; 5465285864985be9077e58e42235af6582dee72e841David 'Digit' Turner} 5475285864985be9077e58e42235af6582dee72e841David 'Digit' Turner 5485285864985be9077e58e42235af6582dee72e841David 'Digit' Turner/* Handling addition overflow with 64 bits inputs values is more 5495285864985be9077e58e42235af6582dee72e841David 'Digit' Turner * tricky than with 32 bits values. */ 5508b23a6c7e1aee255004dd19098d4c2462b61b849The Android Open Source Projectuint64_t HELPER(neon_rshl_s64)(uint64_t valop, uint64_t shiftop) 5518b23a6c7e1aee255004dd19098d4c2462b61b849The Android Open Source Project{ 5528b23a6c7e1aee255004dd19098d4c2462b61b849The Android Open Source Project int8_t shift = (int8_t)shiftop; 5538b23a6c7e1aee255004dd19098d4c2462b61b849The Android Open Source Project int64_t val = valop; 5545285864985be9077e58e42235af6582dee72e841David 'Digit' Turner if ((shift >= 64) || (shift <= -64)) { 5558b23a6c7e1aee255004dd19098d4c2462b61b849The Android Open Source Project val = 0; 5568b23a6c7e1aee255004dd19098d4c2462b61b849The Android Open Source Project } else if (shift < 0) { 5575285864985be9077e58e42235af6582dee72e841David 'Digit' Turner val >>= (-shift - 1); 5585285864985be9077e58e42235af6582dee72e841David 'Digit' Turner if (val == INT64_MAX) { 5595285864985be9077e58e42235af6582dee72e841David 'Digit' Turner /* In this case, it means that the rounding constant is 1, 5605285864985be9077e58e42235af6582dee72e841David 'Digit' Turner * and the addition would overflow. Return the actual 5615285864985be9077e58e42235af6582dee72e841David 'Digit' Turner * result directly. */ 5625285864985be9077e58e42235af6582dee72e841David 'Digit' Turner val = 0x4000000000000000LL; 5635285864985be9077e58e42235af6582dee72e841David 'Digit' Turner } else { 5645285864985be9077e58e42235af6582dee72e841David 'Digit' Turner val++; 5655285864985be9077e58e42235af6582dee72e841David 'Digit' Turner val >>= 1; 5665285864985be9077e58e42235af6582dee72e841David 'Digit' Turner } 5678b23a6c7e1aee255004dd19098d4c2462b61b849The Android Open Source Project } else { 5688b23a6c7e1aee255004dd19098d4c2462b61b849The Android Open Source Project val <<= shift; 5698b23a6c7e1aee255004dd19098d4c2462b61b849The Android Open Source Project } 5708b23a6c7e1aee255004dd19098d4c2462b61b849The Android Open Source Project return val; 5718b23a6c7e1aee255004dd19098d4c2462b61b849The Android Open Source Project} 5728b23a6c7e1aee255004dd19098d4c2462b61b849The Android Open Source Project 5738b23a6c7e1aee255004dd19098d4c2462b61b849The Android Open Source Project#define NEON_FN(dest, src1, src2) do { \ 5748b23a6c7e1aee255004dd19098d4c2462b61b849The Android Open Source Project int8_t tmp; \ 5758b23a6c7e1aee255004dd19098d4c2462b61b849The Android Open Source Project tmp = (int8_t)src2; \ 5765285864985be9077e58e42235af6582dee72e841David 'Digit' Turner if (tmp >= (ssize_t)sizeof(src1) * 8 || \ 5775285864985be9077e58e42235af6582dee72e841David 'Digit' Turner tmp < -(ssize_t)sizeof(src1) * 8) { \ 5788b23a6c7e1aee255004dd19098d4c2462b61b849The Android Open Source Project dest = 0; \ 5795285864985be9077e58e42235af6582dee72e841David 'Digit' Turner } else if (tmp == -(ssize_t)sizeof(src1) * 8) { \ 5805285864985be9077e58e42235af6582dee72e841David 'Digit' Turner dest = src1 >> (-tmp - 1); \ 5818b23a6c7e1aee255004dd19098d4c2462b61b849The Android Open Source Project } else if (tmp < 0) { \ 5828b23a6c7e1aee255004dd19098d4c2462b61b849The Android Open Source Project dest = (src1 + (1 << (-1 - tmp))) >> -tmp; \ 5838b23a6c7e1aee255004dd19098d4c2462b61b849The Android Open Source Project } else { \ 5848b23a6c7e1aee255004dd19098d4c2462b61b849The Android Open Source Project dest = src1 << tmp; \ 5858b23a6c7e1aee255004dd19098d4c2462b61b849The Android Open Source Project }} while (0) 5868b23a6c7e1aee255004dd19098d4c2462b61b849The Android Open Source ProjectNEON_VOP(rshl_u8, neon_u8, 4) 5878b23a6c7e1aee255004dd19098d4c2462b61b849The Android Open Source ProjectNEON_VOP(rshl_u16, neon_u16, 2) 5888b23a6c7e1aee255004dd19098d4c2462b61b849The Android Open Source Project#undef NEON_FN 5898b23a6c7e1aee255004dd19098d4c2462b61b849The Android Open Source Project 5905285864985be9077e58e42235af6582dee72e841David 'Digit' Turner/* The addition of the rounding constant may overflow, so we use an 5915285864985be9077e58e42235af6582dee72e841David 'Digit' Turner * intermediate 64 bits accumulator. */ 5925285864985be9077e58e42235af6582dee72e841David 'Digit' Turneruint32_t HELPER(neon_rshl_u32)(uint32_t val, uint32_t shiftop) 5935285864985be9077e58e42235af6582dee72e841David 'Digit' Turner{ 5945285864985be9077e58e42235af6582dee72e841David 'Digit' Turner uint32_t dest; 5955285864985be9077e58e42235af6582dee72e841David 'Digit' Turner int8_t shift = (int8_t)shiftop; 5965285864985be9077e58e42235af6582dee72e841David 'Digit' Turner if (shift >= 32 || shift < -32) { 5975285864985be9077e58e42235af6582dee72e841David 'Digit' Turner dest = 0; 5985285864985be9077e58e42235af6582dee72e841David 'Digit' Turner } else if (shift == -32) { 5995285864985be9077e58e42235af6582dee72e841David 'Digit' Turner dest = val >> 31; 6005285864985be9077e58e42235af6582dee72e841David 'Digit' Turner } else if (shift < 0) { 6015285864985be9077e58e42235af6582dee72e841David 'Digit' Turner uint64_t big_dest = ((uint64_t)val + (1 << (-1 - shift))); 6025285864985be9077e58e42235af6582dee72e841David 'Digit' Turner dest = big_dest >> -shift; 6035285864985be9077e58e42235af6582dee72e841David 'Digit' Turner } else { 6045285864985be9077e58e42235af6582dee72e841David 'Digit' Turner dest = val << shift; 6055285864985be9077e58e42235af6582dee72e841David 'Digit' Turner } 6065285864985be9077e58e42235af6582dee72e841David 'Digit' Turner return dest; 6075285864985be9077e58e42235af6582dee72e841David 'Digit' Turner} 6085285864985be9077e58e42235af6582dee72e841David 'Digit' Turner 6095285864985be9077e58e42235af6582dee72e841David 'Digit' Turner/* Handling addition overflow with 64 bits inputs values is more 6105285864985be9077e58e42235af6582dee72e841David 'Digit' Turner * tricky than with 32 bits values. */ 6118b23a6c7e1aee255004dd19098d4c2462b61b849The Android Open Source Projectuint64_t HELPER(neon_rshl_u64)(uint64_t val, uint64_t shiftop) 6128b23a6c7e1aee255004dd19098d4c2462b61b849The Android Open Source Project{ 6138b23a6c7e1aee255004dd19098d4c2462b61b849The Android Open Source Project int8_t shift = (uint8_t)shiftop; 6145285864985be9077e58e42235af6582dee72e841David 'Digit' Turner if (shift >= 64 || shift < -64) { 6158b23a6c7e1aee255004dd19098d4c2462b61b849The Android Open Source Project val = 0; 6168b23a6c7e1aee255004dd19098d4c2462b61b849The Android Open Source Project } else if (shift == -64) { 6178b23a6c7e1aee255004dd19098d4c2462b61b849The Android Open Source Project /* Rounding a 1-bit result just preserves that bit. */ 6188b23a6c7e1aee255004dd19098d4c2462b61b849The Android Open Source Project val >>= 63; 6195285864985be9077e58e42235af6582dee72e841David 'Digit' Turner } else if (shift < 0) { 6205285864985be9077e58e42235af6582dee72e841David 'Digit' Turner val >>= (-shift - 1); 6215285864985be9077e58e42235af6582dee72e841David 'Digit' Turner if (val == UINT64_MAX) { 6225285864985be9077e58e42235af6582dee72e841David 'Digit' Turner /* In this case, it means that the rounding constant is 1, 6235285864985be9077e58e42235af6582dee72e841David 'Digit' Turner * and the addition would overflow. Return the actual 6245285864985be9077e58e42235af6582dee72e841David 'Digit' Turner * result directly. */ 6255285864985be9077e58e42235af6582dee72e841David 'Digit' Turner val = 0x8000000000000000ULL; 6265285864985be9077e58e42235af6582dee72e841David 'Digit' Turner } else { 6275285864985be9077e58e42235af6582dee72e841David 'Digit' Turner val++; 6285285864985be9077e58e42235af6582dee72e841David 'Digit' Turner val >>= 1; 6295285864985be9077e58e42235af6582dee72e841David 'Digit' Turner } 6308b23a6c7e1aee255004dd19098d4c2462b61b849The Android Open Source Project } else { 6318b23a6c7e1aee255004dd19098d4c2462b61b849The Android Open Source Project val <<= shift; 6328b23a6c7e1aee255004dd19098d4c2462b61b849The Android Open Source Project } 6338b23a6c7e1aee255004dd19098d4c2462b61b849The Android Open Source Project return val; 6348b23a6c7e1aee255004dd19098d4c2462b61b849The Android Open Source Project} 6358b23a6c7e1aee255004dd19098d4c2462b61b849The Android Open Source Project 6368b23a6c7e1aee255004dd19098d4c2462b61b849The Android Open Source Project#define NEON_FN(dest, src1, src2) do { \ 6378b23a6c7e1aee255004dd19098d4c2462b61b849The Android Open Source Project int8_t tmp; \ 6388b23a6c7e1aee255004dd19098d4c2462b61b849The Android Open Source Project tmp = (int8_t)src2; \ 6395285864985be9077e58e42235af6582dee72e841David 'Digit' Turner if (tmp >= (ssize_t)sizeof(src1) * 8) { \ 6408b23a6c7e1aee255004dd19098d4c2462b61b849The Android Open Source Project if (src1) { \ 6418b23a6c7e1aee255004dd19098d4c2462b61b849The Android Open Source Project SET_QC(); \ 6428b23a6c7e1aee255004dd19098d4c2462b61b849The Android Open Source Project dest = ~0; \ 6438b23a6c7e1aee255004dd19098d4c2462b61b849The Android Open Source Project } else { \ 6448b23a6c7e1aee255004dd19098d4c2462b61b849The Android Open Source Project dest = 0; \ 6458b23a6c7e1aee255004dd19098d4c2462b61b849The Android Open Source Project } \ 6465285864985be9077e58e42235af6582dee72e841David 'Digit' Turner } else if (tmp <= -(ssize_t)sizeof(src1) * 8) { \ 6478b23a6c7e1aee255004dd19098d4c2462b61b849The Android Open Source Project dest = 0; \ 6488b23a6c7e1aee255004dd19098d4c2462b61b849The Android Open Source Project } else if (tmp < 0) { \ 6498b23a6c7e1aee255004dd19098d4c2462b61b849The Android Open Source Project dest = src1 >> -tmp; \ 6508b23a6c7e1aee255004dd19098d4c2462b61b849The Android Open Source Project } else { \ 6518b23a6c7e1aee255004dd19098d4c2462b61b849The Android Open Source Project dest = src1 << tmp; \ 6528b23a6c7e1aee255004dd19098d4c2462b61b849The Android Open Source Project if ((dest >> tmp) != src1) { \ 6538b23a6c7e1aee255004dd19098d4c2462b61b849The Android Open Source Project SET_QC(); \ 6548b23a6c7e1aee255004dd19098d4c2462b61b849The Android Open Source Project dest = ~0; \ 6558b23a6c7e1aee255004dd19098d4c2462b61b849The Android Open Source Project } \ 6568b23a6c7e1aee255004dd19098d4c2462b61b849The Android Open Source Project }} while (0) 6575285864985be9077e58e42235af6582dee72e841David 'Digit' TurnerNEON_VOP(qshl_u8, neon_u8, 4) 6585285864985be9077e58e42235af6582dee72e841David 'Digit' TurnerNEON_VOP(qshl_u16, neon_u16, 2) 6595285864985be9077e58e42235af6582dee72e841David 'Digit' TurnerNEON_VOP(qshl_u32, neon_u32, 1) 6608b23a6c7e1aee255004dd19098d4c2462b61b849The Android Open Source Project#undef NEON_FN 6618b23a6c7e1aee255004dd19098d4c2462b61b849The Android Open Source Project 6625285864985be9077e58e42235af6582dee72e841David 'Digit' Turneruint64_t HELPER(neon_qshl_u64)(uint64_t val, uint64_t shiftop) 6638b23a6c7e1aee255004dd19098d4c2462b61b849The Android Open Source Project{ 6648b23a6c7e1aee255004dd19098d4c2462b61b849The Android Open Source Project int8_t shift = (int8_t)shiftop; 6658b23a6c7e1aee255004dd19098d4c2462b61b849The Android Open Source Project if (shift >= 64) { 6668b23a6c7e1aee255004dd19098d4c2462b61b849The Android Open Source Project if (val) { 6678b23a6c7e1aee255004dd19098d4c2462b61b849The Android Open Source Project val = ~(uint64_t)0; 6688b23a6c7e1aee255004dd19098d4c2462b61b849The Android Open Source Project SET_QC(); 6698b23a6c7e1aee255004dd19098d4c2462b61b849The Android Open Source Project } 6708b23a6c7e1aee255004dd19098d4c2462b61b849The Android Open Source Project } else if (shift <= -64) { 6718b23a6c7e1aee255004dd19098d4c2462b61b849The Android Open Source Project val = 0; 6728b23a6c7e1aee255004dd19098d4c2462b61b849The Android Open Source Project } else if (shift < 0) { 6738b23a6c7e1aee255004dd19098d4c2462b61b849The Android Open Source Project val >>= -shift; 6748b23a6c7e1aee255004dd19098d4c2462b61b849The Android Open Source Project } else { 6758b23a6c7e1aee255004dd19098d4c2462b61b849The Android Open Source Project uint64_t tmp = val; 6768b23a6c7e1aee255004dd19098d4c2462b61b849The Android Open Source Project val <<= shift; 6778b23a6c7e1aee255004dd19098d4c2462b61b849The Android Open Source Project if ((val >> shift) != tmp) { 6788b23a6c7e1aee255004dd19098d4c2462b61b849The Android Open Source Project SET_QC(); 6798b23a6c7e1aee255004dd19098d4c2462b61b849The Android Open Source Project val = ~(uint64_t)0; 6808b23a6c7e1aee255004dd19098d4c2462b61b849The Android Open Source Project } 6818b23a6c7e1aee255004dd19098d4c2462b61b849The Android Open Source Project } 6828b23a6c7e1aee255004dd19098d4c2462b61b849The Android Open Source Project return val; 6838b23a6c7e1aee255004dd19098d4c2462b61b849The Android Open Source Project} 6848b23a6c7e1aee255004dd19098d4c2462b61b849The Android Open Source Project 6858b23a6c7e1aee255004dd19098d4c2462b61b849The Android Open Source Project#define NEON_FN(dest, src1, src2) do { \ 6868b23a6c7e1aee255004dd19098d4c2462b61b849The Android Open Source Project int8_t tmp; \ 6878b23a6c7e1aee255004dd19098d4c2462b61b849The Android Open Source Project tmp = (int8_t)src2; \ 6885285864985be9077e58e42235af6582dee72e841David 'Digit' Turner if (tmp >= (ssize_t)sizeof(src1) * 8) { \ 6895285864985be9077e58e42235af6582dee72e841David 'Digit' Turner if (src1) { \ 6908b23a6c7e1aee255004dd19098d4c2462b61b849The Android Open Source Project SET_QC(); \ 6915285864985be9077e58e42235af6582dee72e841David 'Digit' Turner dest = (uint32_t)(1 << (sizeof(src1) * 8 - 1)); \ 6925285864985be9077e58e42235af6582dee72e841David 'Digit' Turner if (src1 > 0) { \ 6935285864985be9077e58e42235af6582dee72e841David 'Digit' Turner dest--; \ 6945285864985be9077e58e42235af6582dee72e841David 'Digit' Turner } \ 6955285864985be9077e58e42235af6582dee72e841David 'Digit' Turner } else { \ 6965285864985be9077e58e42235af6582dee72e841David 'Digit' Turner dest = src1; \ 6975285864985be9077e58e42235af6582dee72e841David 'Digit' Turner } \ 6985285864985be9077e58e42235af6582dee72e841David 'Digit' Turner } else if (tmp <= -(ssize_t)sizeof(src1) * 8) { \ 6998b23a6c7e1aee255004dd19098d4c2462b61b849The Android Open Source Project dest = src1 >> 31; \ 7008b23a6c7e1aee255004dd19098d4c2462b61b849The Android Open Source Project } else if (tmp < 0) { \ 7018b23a6c7e1aee255004dd19098d4c2462b61b849The Android Open Source Project dest = src1 >> -tmp; \ 7028b23a6c7e1aee255004dd19098d4c2462b61b849The Android Open Source Project } else { \ 7038b23a6c7e1aee255004dd19098d4c2462b61b849The Android Open Source Project dest = src1 << tmp; \ 7048b23a6c7e1aee255004dd19098d4c2462b61b849The Android Open Source Project if ((dest >> tmp) != src1) { \ 7058b23a6c7e1aee255004dd19098d4c2462b61b849The Android Open Source Project SET_QC(); \ 7065285864985be9077e58e42235af6582dee72e841David 'Digit' Turner dest = (uint32_t)(1 << (sizeof(src1) * 8 - 1)); \ 7075285864985be9077e58e42235af6582dee72e841David 'Digit' Turner if (src1 > 0) { \ 7085285864985be9077e58e42235af6582dee72e841David 'Digit' Turner dest--; \ 7095285864985be9077e58e42235af6582dee72e841David 'Digit' Turner } \ 7108b23a6c7e1aee255004dd19098d4c2462b61b849The Android Open Source Project } \ 7118b23a6c7e1aee255004dd19098d4c2462b61b849The Android Open Source Project }} while (0) 7125285864985be9077e58e42235af6582dee72e841David 'Digit' TurnerNEON_VOP(qshl_s8, neon_s8, 4) 7135285864985be9077e58e42235af6582dee72e841David 'Digit' TurnerNEON_VOP(qshl_s16, neon_s16, 2) 7145285864985be9077e58e42235af6582dee72e841David 'Digit' TurnerNEON_VOP(qshl_s32, neon_s32, 1) 7158b23a6c7e1aee255004dd19098d4c2462b61b849The Android Open Source Project#undef NEON_FN 7168b23a6c7e1aee255004dd19098d4c2462b61b849The Android Open Source Project 7175285864985be9077e58e42235af6582dee72e841David 'Digit' Turneruint64_t HELPER(neon_qshl_s64)(uint64_t valop, uint64_t shiftop) 7188b23a6c7e1aee255004dd19098d4c2462b61b849The Android Open Source Project{ 7198b23a6c7e1aee255004dd19098d4c2462b61b849The Android Open Source Project int8_t shift = (uint8_t)shiftop; 7208b23a6c7e1aee255004dd19098d4c2462b61b849The Android Open Source Project int64_t val = valop; 7218b23a6c7e1aee255004dd19098d4c2462b61b849The Android Open Source Project if (shift >= 64) { 7228b23a6c7e1aee255004dd19098d4c2462b61b849The Android Open Source Project if (val) { 7238b23a6c7e1aee255004dd19098d4c2462b61b849The Android Open Source Project SET_QC(); 7245285864985be9077e58e42235af6582dee72e841David 'Digit' Turner val = (val >> 63) ^ ~SIGNBIT64; 7258b23a6c7e1aee255004dd19098d4c2462b61b849The Android Open Source Project } 7265285864985be9077e58e42235af6582dee72e841David 'Digit' Turner } else if (shift <= -64) { 7278b23a6c7e1aee255004dd19098d4c2462b61b849The Android Open Source Project val >>= 63; 7288b23a6c7e1aee255004dd19098d4c2462b61b849The Android Open Source Project } else if (shift < 0) { 7298b23a6c7e1aee255004dd19098d4c2462b61b849The Android Open Source Project val >>= -shift; 7308b23a6c7e1aee255004dd19098d4c2462b61b849The Android Open Source Project } else { 7318b23a6c7e1aee255004dd19098d4c2462b61b849The Android Open Source Project int64_t tmp = val; 7328b23a6c7e1aee255004dd19098d4c2462b61b849The Android Open Source Project val <<= shift; 7338b23a6c7e1aee255004dd19098d4c2462b61b849The Android Open Source Project if ((val >> shift) != tmp) { 7348b23a6c7e1aee255004dd19098d4c2462b61b849The Android Open Source Project SET_QC(); 7358b23a6c7e1aee255004dd19098d4c2462b61b849The Android Open Source Project val = (tmp >> 63) ^ ~SIGNBIT64; 7368b23a6c7e1aee255004dd19098d4c2462b61b849The Android Open Source Project } 7378b23a6c7e1aee255004dd19098d4c2462b61b849The Android Open Source Project } 7388b23a6c7e1aee255004dd19098d4c2462b61b849The Android Open Source Project return val; 7398b23a6c7e1aee255004dd19098d4c2462b61b849The Android Open Source Project} 7408b23a6c7e1aee255004dd19098d4c2462b61b849The Android Open Source Project 7415285864985be9077e58e42235af6582dee72e841David 'Digit' Turner#define NEON_FN(dest, src1, src2) do { \ 7425285864985be9077e58e42235af6582dee72e841David 'Digit' Turner if (src1 & (1 << (sizeof(src1) * 8 - 1))) { \ 7435285864985be9077e58e42235af6582dee72e841David 'Digit' Turner SET_QC(); \ 7445285864985be9077e58e42235af6582dee72e841David 'Digit' Turner dest = 0; \ 7455285864985be9077e58e42235af6582dee72e841David 'Digit' Turner } else { \ 7465285864985be9077e58e42235af6582dee72e841David 'Digit' Turner int8_t tmp; \ 7475285864985be9077e58e42235af6582dee72e841David 'Digit' Turner tmp = (int8_t)src2; \ 7485285864985be9077e58e42235af6582dee72e841David 'Digit' Turner if (tmp >= (ssize_t)sizeof(src1) * 8) { \ 7495285864985be9077e58e42235af6582dee72e841David 'Digit' Turner if (src1) { \ 7505285864985be9077e58e42235af6582dee72e841David 'Digit' Turner SET_QC(); \ 7515285864985be9077e58e42235af6582dee72e841David 'Digit' Turner dest = ~0; \ 7525285864985be9077e58e42235af6582dee72e841David 'Digit' Turner } else { \ 7535285864985be9077e58e42235af6582dee72e841David 'Digit' Turner dest = 0; \ 7545285864985be9077e58e42235af6582dee72e841David 'Digit' Turner } \ 7555285864985be9077e58e42235af6582dee72e841David 'Digit' Turner } else if (tmp <= -(ssize_t)sizeof(src1) * 8) { \ 7565285864985be9077e58e42235af6582dee72e841David 'Digit' Turner dest = 0; \ 7575285864985be9077e58e42235af6582dee72e841David 'Digit' Turner } else if (tmp < 0) { \ 7585285864985be9077e58e42235af6582dee72e841David 'Digit' Turner dest = src1 >> -tmp; \ 7595285864985be9077e58e42235af6582dee72e841David 'Digit' Turner } else { \ 7605285864985be9077e58e42235af6582dee72e841David 'Digit' Turner dest = src1 << tmp; \ 7615285864985be9077e58e42235af6582dee72e841David 'Digit' Turner if ((dest >> tmp) != src1) { \ 7625285864985be9077e58e42235af6582dee72e841David 'Digit' Turner SET_QC(); \ 7635285864985be9077e58e42235af6582dee72e841David 'Digit' Turner dest = ~0; \ 7645285864985be9077e58e42235af6582dee72e841David 'Digit' Turner } \ 7655285864985be9077e58e42235af6582dee72e841David 'Digit' Turner } \ 7665285864985be9077e58e42235af6582dee72e841David 'Digit' Turner }} while (0) 7675285864985be9077e58e42235af6582dee72e841David 'Digit' TurnerNEON_VOP(qshlu_s8, neon_u8, 4) 7685285864985be9077e58e42235af6582dee72e841David 'Digit' TurnerNEON_VOP(qshlu_s16, neon_u16, 2) 7695285864985be9077e58e42235af6582dee72e841David 'Digit' Turner#undef NEON_FN 7705285864985be9077e58e42235af6582dee72e841David 'Digit' Turner 7715285864985be9077e58e42235af6582dee72e841David 'Digit' Turneruint32_t HELPER(neon_qshlu_s32)(uint32_t valop, uint32_t shiftop) 7725285864985be9077e58e42235af6582dee72e841David 'Digit' Turner{ 7735285864985be9077e58e42235af6582dee72e841David 'Digit' Turner if ((int32_t)valop < 0) { 7745285864985be9077e58e42235af6582dee72e841David 'Digit' Turner SET_QC(); 7755285864985be9077e58e42235af6582dee72e841David 'Digit' Turner return 0; 7765285864985be9077e58e42235af6582dee72e841David 'Digit' Turner } 7775285864985be9077e58e42235af6582dee72e841David 'Digit' Turner return helper_neon_qshl_u32(valop, shiftop); 7785285864985be9077e58e42235af6582dee72e841David 'Digit' Turner} 7795285864985be9077e58e42235af6582dee72e841David 'Digit' Turner 7805285864985be9077e58e42235af6582dee72e841David 'Digit' Turneruint64_t HELPER(neon_qshlu_s64)(uint64_t valop, uint64_t shiftop) 7815285864985be9077e58e42235af6582dee72e841David 'Digit' Turner{ 7825285864985be9077e58e42235af6582dee72e841David 'Digit' Turner if ((int64_t)valop < 0) { 7835285864985be9077e58e42235af6582dee72e841David 'Digit' Turner SET_QC(); 7845285864985be9077e58e42235af6582dee72e841David 'Digit' Turner return 0; 7855285864985be9077e58e42235af6582dee72e841David 'Digit' Turner } 7865285864985be9077e58e42235af6582dee72e841David 'Digit' Turner return helper_neon_qshl_u64(valop, shiftop); 7875285864985be9077e58e42235af6582dee72e841David 'Digit' Turner} 7888b23a6c7e1aee255004dd19098d4c2462b61b849The Android Open Source Project 7898b23a6c7e1aee255004dd19098d4c2462b61b849The Android Open Source Project/* FIXME: This is wrong. */ 7908b23a6c7e1aee255004dd19098d4c2462b61b849The Android Open Source Project#define NEON_FN(dest, src1, src2) do { \ 7918b23a6c7e1aee255004dd19098d4c2462b61b849The Android Open Source Project int8_t tmp; \ 7928b23a6c7e1aee255004dd19098d4c2462b61b849The Android Open Source Project tmp = (int8_t)src2; \ 7935285864985be9077e58e42235af6582dee72e841David 'Digit' Turner if (tmp >= (ssize_t)sizeof(src1) * 8) { \ 7945285864985be9077e58e42235af6582dee72e841David 'Digit' Turner if (src1) { \ 7955285864985be9077e58e42235af6582dee72e841David 'Digit' Turner SET_QC(); \ 7965285864985be9077e58e42235af6582dee72e841David 'Digit' Turner dest = ~0; \ 7975285864985be9077e58e42235af6582dee72e841David 'Digit' Turner } else { \ 7985285864985be9077e58e42235af6582dee72e841David 'Digit' Turner dest = 0; \ 7995285864985be9077e58e42235af6582dee72e841David 'Digit' Turner } \ 8005285864985be9077e58e42235af6582dee72e841David 'Digit' Turner } else if (tmp < -(ssize_t)sizeof(src1) * 8) { \ 8015285864985be9077e58e42235af6582dee72e841David 'Digit' Turner dest = 0; \ 8025285864985be9077e58e42235af6582dee72e841David 'Digit' Turner } else if (tmp == -(ssize_t)sizeof(src1) * 8) { \ 8035285864985be9077e58e42235af6582dee72e841David 'Digit' Turner dest = src1 >> (sizeof(src1) * 8 - 1); \ 8045285864985be9077e58e42235af6582dee72e841David 'Digit' Turner } else if (tmp < 0) { \ 8058b23a6c7e1aee255004dd19098d4c2462b61b849The Android Open Source Project dest = (src1 + (1 << (-1 - tmp))) >> -tmp; \ 8068b23a6c7e1aee255004dd19098d4c2462b61b849The Android Open Source Project } else { \ 8078b23a6c7e1aee255004dd19098d4c2462b61b849The Android Open Source Project dest = src1 << tmp; \ 8088b23a6c7e1aee255004dd19098d4c2462b61b849The Android Open Source Project if ((dest >> tmp) != src1) { \ 8098b23a6c7e1aee255004dd19098d4c2462b61b849The Android Open Source Project SET_QC(); \ 8108b23a6c7e1aee255004dd19098d4c2462b61b849The Android Open Source Project dest = ~0; \ 8118b23a6c7e1aee255004dd19098d4c2462b61b849The Android Open Source Project } \ 8128b23a6c7e1aee255004dd19098d4c2462b61b849The Android Open Source Project }} while (0) 8135285864985be9077e58e42235af6582dee72e841David 'Digit' TurnerNEON_VOP(qrshl_u8, neon_u8, 4) 8145285864985be9077e58e42235af6582dee72e841David 'Digit' TurnerNEON_VOP(qrshl_u16, neon_u16, 2) 8158b23a6c7e1aee255004dd19098d4c2462b61b849The Android Open Source Project#undef NEON_FN 8168b23a6c7e1aee255004dd19098d4c2462b61b849The Android Open Source Project 8175285864985be9077e58e42235af6582dee72e841David 'Digit' Turner/* The addition of the rounding constant may overflow, so we use an 8185285864985be9077e58e42235af6582dee72e841David 'Digit' Turner * intermediate 64 bits accumulator. */ 8195285864985be9077e58e42235af6582dee72e841David 'Digit' Turneruint32_t HELPER(neon_qrshl_u32)(uint32_t val, uint32_t shiftop) 8208b23a6c7e1aee255004dd19098d4c2462b61b849The Android Open Source Project{ 8215285864985be9077e58e42235af6582dee72e841David 'Digit' Turner uint32_t dest; 8228b23a6c7e1aee255004dd19098d4c2462b61b849The Android Open Source Project int8_t shift = (int8_t)shiftop; 8235285864985be9077e58e42235af6582dee72e841David 'Digit' Turner if (shift >= 32) { 8245285864985be9077e58e42235af6582dee72e841David 'Digit' Turner if (val) { 8255285864985be9077e58e42235af6582dee72e841David 'Digit' Turner SET_QC(); 8265285864985be9077e58e42235af6582dee72e841David 'Digit' Turner dest = ~0; 8275285864985be9077e58e42235af6582dee72e841David 'Digit' Turner } else { 8285285864985be9077e58e42235af6582dee72e841David 'Digit' Turner dest = 0; 8295285864985be9077e58e42235af6582dee72e841David 'Digit' Turner } 8305285864985be9077e58e42235af6582dee72e841David 'Digit' Turner } else if (shift < -32) { 8315285864985be9077e58e42235af6582dee72e841David 'Digit' Turner dest = 0; 8325285864985be9077e58e42235af6582dee72e841David 'Digit' Turner } else if (shift == -32) { 8335285864985be9077e58e42235af6582dee72e841David 'Digit' Turner dest = val >> 31; 8345285864985be9077e58e42235af6582dee72e841David 'Digit' Turner } else if (shift < 0) { 8355285864985be9077e58e42235af6582dee72e841David 'Digit' Turner uint64_t big_dest = ((uint64_t)val + (1 << (-1 - shift))); 8365285864985be9077e58e42235af6582dee72e841David 'Digit' Turner dest = big_dest >> -shift; 8375285864985be9077e58e42235af6582dee72e841David 'Digit' Turner } else { 8385285864985be9077e58e42235af6582dee72e841David 'Digit' Turner dest = val << shift; 8395285864985be9077e58e42235af6582dee72e841David 'Digit' Turner if ((dest >> shift) != val) { 8405285864985be9077e58e42235af6582dee72e841David 'Digit' Turner SET_QC(); 8415285864985be9077e58e42235af6582dee72e841David 'Digit' Turner dest = ~0; 8425285864985be9077e58e42235af6582dee72e841David 'Digit' Turner } 8435285864985be9077e58e42235af6582dee72e841David 'Digit' Turner } 8445285864985be9077e58e42235af6582dee72e841David 'Digit' Turner return dest; 8455285864985be9077e58e42235af6582dee72e841David 'Digit' Turner} 8465285864985be9077e58e42235af6582dee72e841David 'Digit' Turner 8475285864985be9077e58e42235af6582dee72e841David 'Digit' Turner/* Handling addition overflow with 64 bits inputs values is more 8485285864985be9077e58e42235af6582dee72e841David 'Digit' Turner * tricky than with 32 bits values. */ 8495285864985be9077e58e42235af6582dee72e841David 'Digit' Turneruint64_t HELPER(neon_qrshl_u64)(uint64_t val, uint64_t shiftop) 8505285864985be9077e58e42235af6582dee72e841David 'Digit' Turner{ 8515285864985be9077e58e42235af6582dee72e841David 'Digit' Turner int8_t shift = (int8_t)shiftop; 8525285864985be9077e58e42235af6582dee72e841David 'Digit' Turner if (shift >= 64) { 8535285864985be9077e58e42235af6582dee72e841David 'Digit' Turner if (val) { 8545285864985be9077e58e42235af6582dee72e841David 'Digit' Turner SET_QC(); 8555285864985be9077e58e42235af6582dee72e841David 'Digit' Turner val = ~0; 8565285864985be9077e58e42235af6582dee72e841David 'Digit' Turner } 8575285864985be9077e58e42235af6582dee72e841David 'Digit' Turner } else if (shift < -64) { 8585285864985be9077e58e42235af6582dee72e841David 'Digit' Turner val = 0; 8595285864985be9077e58e42235af6582dee72e841David 'Digit' Turner } else if (shift == -64) { 8605285864985be9077e58e42235af6582dee72e841David 'Digit' Turner val >>= 63; 8615285864985be9077e58e42235af6582dee72e841David 'Digit' Turner } else if (shift < 0) { 8625285864985be9077e58e42235af6582dee72e841David 'Digit' Turner val >>= (-shift - 1); 8635285864985be9077e58e42235af6582dee72e841David 'Digit' Turner if (val == UINT64_MAX) { 8645285864985be9077e58e42235af6582dee72e841David 'Digit' Turner /* In this case, it means that the rounding constant is 1, 8655285864985be9077e58e42235af6582dee72e841David 'Digit' Turner * and the addition would overflow. Return the actual 8665285864985be9077e58e42235af6582dee72e841David 'Digit' Turner * result directly. */ 8675285864985be9077e58e42235af6582dee72e841David 'Digit' Turner val = 0x8000000000000000ULL; 8685285864985be9077e58e42235af6582dee72e841David 'Digit' Turner } else { 8695285864985be9077e58e42235af6582dee72e841David 'Digit' Turner val++; 8705285864985be9077e58e42235af6582dee72e841David 'Digit' Turner val >>= 1; 8715285864985be9077e58e42235af6582dee72e841David 'Digit' Turner } 8728b23a6c7e1aee255004dd19098d4c2462b61b849The Android Open Source Project } else { \ 8738b23a6c7e1aee255004dd19098d4c2462b61b849The Android Open Source Project uint64_t tmp = val; 8748b23a6c7e1aee255004dd19098d4c2462b61b849The Android Open Source Project val <<= shift; 8758b23a6c7e1aee255004dd19098d4c2462b61b849The Android Open Source Project if ((val >> shift) != tmp) { 8768b23a6c7e1aee255004dd19098d4c2462b61b849The Android Open Source Project SET_QC(); 8778b23a6c7e1aee255004dd19098d4c2462b61b849The Android Open Source Project val = ~0; 8788b23a6c7e1aee255004dd19098d4c2462b61b849The Android Open Source Project } 8798b23a6c7e1aee255004dd19098d4c2462b61b849The Android Open Source Project } 8808b23a6c7e1aee255004dd19098d4c2462b61b849The Android Open Source Project return val; 8818b23a6c7e1aee255004dd19098d4c2462b61b849The Android Open Source Project} 8828b23a6c7e1aee255004dd19098d4c2462b61b849The Android Open Source Project 8838b23a6c7e1aee255004dd19098d4c2462b61b849The Android Open Source Project#define NEON_FN(dest, src1, src2) do { \ 8848b23a6c7e1aee255004dd19098d4c2462b61b849The Android Open Source Project int8_t tmp; \ 8858b23a6c7e1aee255004dd19098d4c2462b61b849The Android Open Source Project tmp = (int8_t)src2; \ 8865285864985be9077e58e42235af6582dee72e841David 'Digit' Turner if (tmp >= (ssize_t)sizeof(src1) * 8) { \ 8875285864985be9077e58e42235af6582dee72e841David 'Digit' Turner if (src1) { \ 8885285864985be9077e58e42235af6582dee72e841David 'Digit' Turner SET_QC(); \ 8895285864985be9077e58e42235af6582dee72e841David 'Digit' Turner dest = (1 << (sizeof(src1) * 8 - 1)); \ 8905285864985be9077e58e42235af6582dee72e841David 'Digit' Turner if (src1 > 0) { \ 8915285864985be9077e58e42235af6582dee72e841David 'Digit' Turner dest--; \ 8925285864985be9077e58e42235af6582dee72e841David 'Digit' Turner } \ 8935285864985be9077e58e42235af6582dee72e841David 'Digit' Turner } else { \ 8945285864985be9077e58e42235af6582dee72e841David 'Digit' Turner dest = 0; \ 8955285864985be9077e58e42235af6582dee72e841David 'Digit' Turner } \ 8965285864985be9077e58e42235af6582dee72e841David 'Digit' Turner } else if (tmp <= -(ssize_t)sizeof(src1) * 8) { \ 8975285864985be9077e58e42235af6582dee72e841David 'Digit' Turner dest = 0; \ 8985285864985be9077e58e42235af6582dee72e841David 'Digit' Turner } else if (tmp < 0) { \ 8998b23a6c7e1aee255004dd19098d4c2462b61b849The Android Open Source Project dest = (src1 + (1 << (-1 - tmp))) >> -tmp; \ 9008b23a6c7e1aee255004dd19098d4c2462b61b849The Android Open Source Project } else { \ 9018b23a6c7e1aee255004dd19098d4c2462b61b849The Android Open Source Project dest = src1 << tmp; \ 9028b23a6c7e1aee255004dd19098d4c2462b61b849The Android Open Source Project if ((dest >> tmp) != src1) { \ 9038b23a6c7e1aee255004dd19098d4c2462b61b849The Android Open Source Project SET_QC(); \ 9045285864985be9077e58e42235af6582dee72e841David 'Digit' Turner dest = (uint32_t)(1 << (sizeof(src1) * 8 - 1)); \ 9055285864985be9077e58e42235af6582dee72e841David 'Digit' Turner if (src1 > 0) { \ 9065285864985be9077e58e42235af6582dee72e841David 'Digit' Turner dest--; \ 9075285864985be9077e58e42235af6582dee72e841David 'Digit' Turner } \ 9088b23a6c7e1aee255004dd19098d4c2462b61b849The Android Open Source Project } \ 9098b23a6c7e1aee255004dd19098d4c2462b61b849The Android Open Source Project }} while (0) 9105285864985be9077e58e42235af6582dee72e841David 'Digit' TurnerNEON_VOP(qrshl_s8, neon_s8, 4) 9115285864985be9077e58e42235af6582dee72e841David 'Digit' TurnerNEON_VOP(qrshl_s16, neon_s16, 2) 9128b23a6c7e1aee255004dd19098d4c2462b61b849The Android Open Source Project#undef NEON_FN 9138b23a6c7e1aee255004dd19098d4c2462b61b849The Android Open Source Project 9145285864985be9077e58e42235af6582dee72e841David 'Digit' Turner/* The addition of the rounding constant may overflow, so we use an 9155285864985be9077e58e42235af6582dee72e841David 'Digit' Turner * intermediate 64 bits accumulator. */ 9165285864985be9077e58e42235af6582dee72e841David 'Digit' Turneruint32_t HELPER(neon_qrshl_s32)(uint32_t valop, uint32_t shiftop) 9175285864985be9077e58e42235af6582dee72e841David 'Digit' Turner{ 9185285864985be9077e58e42235af6582dee72e841David 'Digit' Turner int32_t dest; 9195285864985be9077e58e42235af6582dee72e841David 'Digit' Turner int32_t val = (int32_t)valop; 9205285864985be9077e58e42235af6582dee72e841David 'Digit' Turner int8_t shift = (int8_t)shiftop; 9215285864985be9077e58e42235af6582dee72e841David 'Digit' Turner if (shift >= 32) { 9225285864985be9077e58e42235af6582dee72e841David 'Digit' Turner if (val) { 9235285864985be9077e58e42235af6582dee72e841David 'Digit' Turner SET_QC(); 9245285864985be9077e58e42235af6582dee72e841David 'Digit' Turner dest = (val >> 31) ^ ~SIGNBIT; 9255285864985be9077e58e42235af6582dee72e841David 'Digit' Turner } else { 9265285864985be9077e58e42235af6582dee72e841David 'Digit' Turner dest = 0; 9275285864985be9077e58e42235af6582dee72e841David 'Digit' Turner } 9285285864985be9077e58e42235af6582dee72e841David 'Digit' Turner } else if (shift <= -32) { 9295285864985be9077e58e42235af6582dee72e841David 'Digit' Turner dest = 0; 9305285864985be9077e58e42235af6582dee72e841David 'Digit' Turner } else if (shift < 0) { 9315285864985be9077e58e42235af6582dee72e841David 'Digit' Turner int64_t big_dest = ((int64_t)val + (1 << (-1 - shift))); 9325285864985be9077e58e42235af6582dee72e841David 'Digit' Turner dest = big_dest >> -shift; 9335285864985be9077e58e42235af6582dee72e841David 'Digit' Turner } else { 9345285864985be9077e58e42235af6582dee72e841David 'Digit' Turner dest = val << shift; 9355285864985be9077e58e42235af6582dee72e841David 'Digit' Turner if ((dest >> shift) != val) { 9365285864985be9077e58e42235af6582dee72e841David 'Digit' Turner SET_QC(); 9375285864985be9077e58e42235af6582dee72e841David 'Digit' Turner dest = (val >> 31) ^ ~SIGNBIT; 9385285864985be9077e58e42235af6582dee72e841David 'Digit' Turner } 9395285864985be9077e58e42235af6582dee72e841David 'Digit' Turner } 9405285864985be9077e58e42235af6582dee72e841David 'Digit' Turner return dest; 9415285864985be9077e58e42235af6582dee72e841David 'Digit' Turner} 9425285864985be9077e58e42235af6582dee72e841David 'Digit' Turner 9435285864985be9077e58e42235af6582dee72e841David 'Digit' Turner/* Handling addition overflow with 64 bits inputs values is more 9445285864985be9077e58e42235af6582dee72e841David 'Digit' Turner * tricky than with 32 bits values. */ 9455285864985be9077e58e42235af6582dee72e841David 'Digit' Turneruint64_t HELPER(neon_qrshl_s64)(uint64_t valop, uint64_t shiftop) 9468b23a6c7e1aee255004dd19098d4c2462b61b849The Android Open Source Project{ 9478b23a6c7e1aee255004dd19098d4c2462b61b849The Android Open Source Project int8_t shift = (uint8_t)shiftop; 9488b23a6c7e1aee255004dd19098d4c2462b61b849The Android Open Source Project int64_t val = valop; 9498b23a6c7e1aee255004dd19098d4c2462b61b849The Android Open Source Project 9505285864985be9077e58e42235af6582dee72e841David 'Digit' Turner if (shift >= 64) { 9515285864985be9077e58e42235af6582dee72e841David 'Digit' Turner if (val) { 9525285864985be9077e58e42235af6582dee72e841David 'Digit' Turner SET_QC(); 9535285864985be9077e58e42235af6582dee72e841David 'Digit' Turner val = (val >> 63) ^ ~SIGNBIT64; 9545285864985be9077e58e42235af6582dee72e841David 'Digit' Turner } 9555285864985be9077e58e42235af6582dee72e841David 'Digit' Turner } else if (shift <= -64) { 9565285864985be9077e58e42235af6582dee72e841David 'Digit' Turner val = 0; 9575285864985be9077e58e42235af6582dee72e841David 'Digit' Turner } else if (shift < 0) { 9585285864985be9077e58e42235af6582dee72e841David 'Digit' Turner val >>= (-shift - 1); 9595285864985be9077e58e42235af6582dee72e841David 'Digit' Turner if (val == INT64_MAX) { 9605285864985be9077e58e42235af6582dee72e841David 'Digit' Turner /* In this case, it means that the rounding constant is 1, 9615285864985be9077e58e42235af6582dee72e841David 'Digit' Turner * and the addition would overflow. Return the actual 9625285864985be9077e58e42235af6582dee72e841David 'Digit' Turner * result directly. */ 9635285864985be9077e58e42235af6582dee72e841David 'Digit' Turner val = 0x4000000000000000ULL; 9645285864985be9077e58e42235af6582dee72e841David 'Digit' Turner } else { 9655285864985be9077e58e42235af6582dee72e841David 'Digit' Turner val++; 9665285864985be9077e58e42235af6582dee72e841David 'Digit' Turner val >>= 1; 9675285864985be9077e58e42235af6582dee72e841David 'Digit' Turner } 9688b23a6c7e1aee255004dd19098d4c2462b61b849The Android Open Source Project } else { 9695285864985be9077e58e42235af6582dee72e841David 'Digit' Turner int64_t tmp = val; 9708b23a6c7e1aee255004dd19098d4c2462b61b849The Android Open Source Project val <<= shift; 9718b23a6c7e1aee255004dd19098d4c2462b61b849The Android Open Source Project if ((val >> shift) != tmp) { 9728b23a6c7e1aee255004dd19098d4c2462b61b849The Android Open Source Project SET_QC(); 9735285864985be9077e58e42235af6582dee72e841David 'Digit' Turner val = (tmp >> 63) ^ ~SIGNBIT64; 9748b23a6c7e1aee255004dd19098d4c2462b61b849The Android Open Source Project } 9758b23a6c7e1aee255004dd19098d4c2462b61b849The Android Open Source Project } 9768b23a6c7e1aee255004dd19098d4c2462b61b849The Android Open Source Project return val; 9778b23a6c7e1aee255004dd19098d4c2462b61b849The Android Open Source Project} 9788b23a6c7e1aee255004dd19098d4c2462b61b849The Android Open Source Project 9798b23a6c7e1aee255004dd19098d4c2462b61b849The Android Open Source Projectuint32_t HELPER(neon_add_u8)(uint32_t a, uint32_t b) 9808b23a6c7e1aee255004dd19098d4c2462b61b849The Android Open Source Project{ 9818b23a6c7e1aee255004dd19098d4c2462b61b849The Android Open Source Project uint32_t mask; 9828b23a6c7e1aee255004dd19098d4c2462b61b849The Android Open Source Project mask = (a ^ b) & 0x80808080u; 9838b23a6c7e1aee255004dd19098d4c2462b61b849The Android Open Source Project a &= ~0x80808080u; 9848b23a6c7e1aee255004dd19098d4c2462b61b849The Android Open Source Project b &= ~0x80808080u; 9858b23a6c7e1aee255004dd19098d4c2462b61b849The Android Open Source Project return (a + b) ^ mask; 9868b23a6c7e1aee255004dd19098d4c2462b61b849The Android Open Source Project} 9878b23a6c7e1aee255004dd19098d4c2462b61b849The Android Open Source Project 9888b23a6c7e1aee255004dd19098d4c2462b61b849The Android Open Source Projectuint32_t HELPER(neon_add_u16)(uint32_t a, uint32_t b) 9898b23a6c7e1aee255004dd19098d4c2462b61b849The Android Open Source Project{ 9908b23a6c7e1aee255004dd19098d4c2462b61b849The Android Open Source Project uint32_t mask; 9918b23a6c7e1aee255004dd19098d4c2462b61b849The Android Open Source Project mask = (a ^ b) & 0x80008000u; 9928b23a6c7e1aee255004dd19098d4c2462b61b849The Android Open Source Project a &= ~0x80008000u; 9938b23a6c7e1aee255004dd19098d4c2462b61b849The Android Open Source Project b &= ~0x80008000u; 9948b23a6c7e1aee255004dd19098d4c2462b61b849The Android Open Source Project return (a + b) ^ mask; 9958b23a6c7e1aee255004dd19098d4c2462b61b849The Android Open Source Project} 9968b23a6c7e1aee255004dd19098d4c2462b61b849The Android Open Source Project 9978b23a6c7e1aee255004dd19098d4c2462b61b849The Android Open Source Project#define NEON_FN(dest, src1, src2) dest = src1 + src2 9988b23a6c7e1aee255004dd19098d4c2462b61b849The Android Open Source ProjectNEON_POP(padd_u8, neon_u8, 4) 9998b23a6c7e1aee255004dd19098d4c2462b61b849The Android Open Source ProjectNEON_POP(padd_u16, neon_u16, 2) 10008b23a6c7e1aee255004dd19098d4c2462b61b849The Android Open Source Project#undef NEON_FN 10018b23a6c7e1aee255004dd19098d4c2462b61b849The Android Open Source Project 10028b23a6c7e1aee255004dd19098d4c2462b61b849The Android Open Source Project#define NEON_FN(dest, src1, src2) dest = src1 - src2 10038b23a6c7e1aee255004dd19098d4c2462b61b849The Android Open Source ProjectNEON_VOP(sub_u8, neon_u8, 4) 10048b23a6c7e1aee255004dd19098d4c2462b61b849The Android Open Source ProjectNEON_VOP(sub_u16, neon_u16, 2) 10058b23a6c7e1aee255004dd19098d4c2462b61b849The Android Open Source Project#undef NEON_FN 10068b23a6c7e1aee255004dd19098d4c2462b61b849The Android Open Source Project 10078b23a6c7e1aee255004dd19098d4c2462b61b849The Android Open Source Project#define NEON_FN(dest, src1, src2) dest = src1 * src2 10088b23a6c7e1aee255004dd19098d4c2462b61b849The Android Open Source ProjectNEON_VOP(mul_u8, neon_u8, 4) 10098b23a6c7e1aee255004dd19098d4c2462b61b849The Android Open Source ProjectNEON_VOP(mul_u16, neon_u16, 2) 10108b23a6c7e1aee255004dd19098d4c2462b61b849The Android Open Source Project#undef NEON_FN 10118b23a6c7e1aee255004dd19098d4c2462b61b849The Android Open Source Project 10128b23a6c7e1aee255004dd19098d4c2462b61b849The Android Open Source Project/* Polynomial multiplication is like integer multiplication except the 10138b23a6c7e1aee255004dd19098d4c2462b61b849The Android Open Source Project partial products are XORed, not added. */ 10148b23a6c7e1aee255004dd19098d4c2462b61b849The Android Open Source Projectuint32_t HELPER(neon_mul_p8)(uint32_t op1, uint32_t op2) 10158b23a6c7e1aee255004dd19098d4c2462b61b849The Android Open Source Project{ 10168b23a6c7e1aee255004dd19098d4c2462b61b849The Android Open Source Project uint32_t mask; 10178b23a6c7e1aee255004dd19098d4c2462b61b849The Android Open Source Project uint32_t result; 10188b23a6c7e1aee255004dd19098d4c2462b61b849The Android Open Source Project result = 0; 10198b23a6c7e1aee255004dd19098d4c2462b61b849The Android Open Source Project while (op1) { 10208b23a6c7e1aee255004dd19098d4c2462b61b849The Android Open Source Project mask = 0; 10218b23a6c7e1aee255004dd19098d4c2462b61b849The Android Open Source Project if (op1 & 1) 10228b23a6c7e1aee255004dd19098d4c2462b61b849The Android Open Source Project mask |= 0xff; 10238b23a6c7e1aee255004dd19098d4c2462b61b849The Android Open Source Project if (op1 & (1 << 8)) 10248b23a6c7e1aee255004dd19098d4c2462b61b849The Android Open Source Project mask |= (0xff << 8); 10258b23a6c7e1aee255004dd19098d4c2462b61b849The Android Open Source Project if (op1 & (1 << 16)) 10268b23a6c7e1aee255004dd19098d4c2462b61b849The Android Open Source Project mask |= (0xff << 16); 10278b23a6c7e1aee255004dd19098d4c2462b61b849The Android Open Source Project if (op1 & (1 << 24)) 10288b23a6c7e1aee255004dd19098d4c2462b61b849The Android Open Source Project mask |= (0xff << 24); 10298b23a6c7e1aee255004dd19098d4c2462b61b849The Android Open Source Project result ^= op2 & mask; 10308b23a6c7e1aee255004dd19098d4c2462b61b849The Android Open Source Project op1 = (op1 >> 1) & 0x7f7f7f7f; 10318b23a6c7e1aee255004dd19098d4c2462b61b849The Android Open Source Project op2 = (op2 << 1) & 0xfefefefe; 10328b23a6c7e1aee255004dd19098d4c2462b61b849The Android Open Source Project } 10338b23a6c7e1aee255004dd19098d4c2462b61b849The Android Open Source Project return result; 10348b23a6c7e1aee255004dd19098d4c2462b61b849The Android Open Source Project} 10358b23a6c7e1aee255004dd19098d4c2462b61b849The Android Open Source Project 10365285864985be9077e58e42235af6582dee72e841David 'Digit' Turneruint64_t HELPER(neon_mull_p8)(uint32_t op1, uint32_t op2) 10375285864985be9077e58e42235af6582dee72e841David 'Digit' Turner{ 10385285864985be9077e58e42235af6582dee72e841David 'Digit' Turner uint64_t result = 0; 10395285864985be9077e58e42235af6582dee72e841David 'Digit' Turner uint64_t mask; 10405285864985be9077e58e42235af6582dee72e841David 'Digit' Turner uint64_t op2ex = op2; 10415285864985be9077e58e42235af6582dee72e841David 'Digit' Turner op2ex = (op2ex & 0xff) | 10425285864985be9077e58e42235af6582dee72e841David 'Digit' Turner ((op2ex & 0xff00) << 8) | 10435285864985be9077e58e42235af6582dee72e841David 'Digit' Turner ((op2ex & 0xff0000) << 16) | 10445285864985be9077e58e42235af6582dee72e841David 'Digit' Turner ((op2ex & 0xff000000) << 24); 10455285864985be9077e58e42235af6582dee72e841David 'Digit' Turner while (op1) { 10465285864985be9077e58e42235af6582dee72e841David 'Digit' Turner mask = 0; 10475285864985be9077e58e42235af6582dee72e841David 'Digit' Turner if (op1 & 1) { 10485285864985be9077e58e42235af6582dee72e841David 'Digit' Turner mask |= 0xffff; 10495285864985be9077e58e42235af6582dee72e841David 'Digit' Turner } 10505285864985be9077e58e42235af6582dee72e841David 'Digit' Turner if (op1 & (1 << 8)) { 10515285864985be9077e58e42235af6582dee72e841David 'Digit' Turner mask |= (0xffffU << 16); 10525285864985be9077e58e42235af6582dee72e841David 'Digit' Turner } 10535285864985be9077e58e42235af6582dee72e841David 'Digit' Turner if (op1 & (1 << 16)) { 10545285864985be9077e58e42235af6582dee72e841David 'Digit' Turner mask |= (0xffffULL << 32); 10555285864985be9077e58e42235af6582dee72e841David 'Digit' Turner } 10565285864985be9077e58e42235af6582dee72e841David 'Digit' Turner if (op1 & (1 << 24)) { 10575285864985be9077e58e42235af6582dee72e841David 'Digit' Turner mask |= (0xffffULL << 48); 10585285864985be9077e58e42235af6582dee72e841David 'Digit' Turner } 10595285864985be9077e58e42235af6582dee72e841David 'Digit' Turner result ^= op2ex & mask; 10605285864985be9077e58e42235af6582dee72e841David 'Digit' Turner op1 = (op1 >> 1) & 0x7f7f7f7f; 10615285864985be9077e58e42235af6582dee72e841David 'Digit' Turner op2ex <<= 1; 10625285864985be9077e58e42235af6582dee72e841David 'Digit' Turner } 10635285864985be9077e58e42235af6582dee72e841David 'Digit' Turner return result; 10645285864985be9077e58e42235af6582dee72e841David 'Digit' Turner} 10655285864985be9077e58e42235af6582dee72e841David 'Digit' Turner 10668b23a6c7e1aee255004dd19098d4c2462b61b849The Android Open Source Project#define NEON_FN(dest, src1, src2) dest = (src1 & src2) ? -1 : 0 10678b23a6c7e1aee255004dd19098d4c2462b61b849The Android Open Source ProjectNEON_VOP(tst_u8, neon_u8, 4) 10688b23a6c7e1aee255004dd19098d4c2462b61b849The Android Open Source ProjectNEON_VOP(tst_u16, neon_u16, 2) 10698b23a6c7e1aee255004dd19098d4c2462b61b849The Android Open Source ProjectNEON_VOP(tst_u32, neon_u32, 1) 10708b23a6c7e1aee255004dd19098d4c2462b61b849The Android Open Source Project#undef NEON_FN 10718b23a6c7e1aee255004dd19098d4c2462b61b849The Android Open Source Project 10728b23a6c7e1aee255004dd19098d4c2462b61b849The Android Open Source Project#define NEON_FN(dest, src1, src2) dest = (src1 == src2) ? -1 : 0 10738b23a6c7e1aee255004dd19098d4c2462b61b849The Android Open Source ProjectNEON_VOP(ceq_u8, neon_u8, 4) 10748b23a6c7e1aee255004dd19098d4c2462b61b849The Android Open Source ProjectNEON_VOP(ceq_u16, neon_u16, 2) 10758b23a6c7e1aee255004dd19098d4c2462b61b849The Android Open Source ProjectNEON_VOP(ceq_u32, neon_u32, 1) 10768b23a6c7e1aee255004dd19098d4c2462b61b849The Android Open Source Project#undef NEON_FN 10778b23a6c7e1aee255004dd19098d4c2462b61b849The Android Open Source Project 10788b23a6c7e1aee255004dd19098d4c2462b61b849The Android Open Source Project#define NEON_FN(dest, src, dummy) dest = (src < 0) ? -src : src 10798b23a6c7e1aee255004dd19098d4c2462b61b849The Android Open Source ProjectNEON_VOP1(abs_s8, neon_s8, 4) 10808b23a6c7e1aee255004dd19098d4c2462b61b849The Android Open Source ProjectNEON_VOP1(abs_s16, neon_s16, 2) 10818b23a6c7e1aee255004dd19098d4c2462b61b849The Android Open Source Project#undef NEON_FN 10828b23a6c7e1aee255004dd19098d4c2462b61b849The Android Open Source Project 10838b23a6c7e1aee255004dd19098d4c2462b61b849The Android Open Source Project/* Count Leading Sign/Zero Bits. */ 10848b23a6c7e1aee255004dd19098d4c2462b61b849The Android Open Source Projectstatic inline int do_clz8(uint8_t x) 10858b23a6c7e1aee255004dd19098d4c2462b61b849The Android Open Source Project{ 10868b23a6c7e1aee255004dd19098d4c2462b61b849The Android Open Source Project int n; 10878b23a6c7e1aee255004dd19098d4c2462b61b849The Android Open Source Project for (n = 8; x; n--) 10888b23a6c7e1aee255004dd19098d4c2462b61b849The Android Open Source Project x >>= 1; 10898b23a6c7e1aee255004dd19098d4c2462b61b849The Android Open Source Project return n; 10908b23a6c7e1aee255004dd19098d4c2462b61b849The Android Open Source Project} 10918b23a6c7e1aee255004dd19098d4c2462b61b849The Android Open Source Project 10928b23a6c7e1aee255004dd19098d4c2462b61b849The Android Open Source Projectstatic inline int do_clz16(uint16_t x) 10938b23a6c7e1aee255004dd19098d4c2462b61b849The Android Open Source Project{ 10948b23a6c7e1aee255004dd19098d4c2462b61b849The Android Open Source Project int n; 10958b23a6c7e1aee255004dd19098d4c2462b61b849The Android Open Source Project for (n = 16; x; n--) 10968b23a6c7e1aee255004dd19098d4c2462b61b849The Android Open Source Project x >>= 1; 10978b23a6c7e1aee255004dd19098d4c2462b61b849The Android Open Source Project return n; 10988b23a6c7e1aee255004dd19098d4c2462b61b849The Android Open Source Project} 10998b23a6c7e1aee255004dd19098d4c2462b61b849The Android Open Source Project 11008b23a6c7e1aee255004dd19098d4c2462b61b849The Android Open Source Project#define NEON_FN(dest, src, dummy) dest = do_clz8(src) 11018b23a6c7e1aee255004dd19098d4c2462b61b849The Android Open Source ProjectNEON_VOP1(clz_u8, neon_u8, 4) 11028b23a6c7e1aee255004dd19098d4c2462b61b849The Android Open Source Project#undef NEON_FN 11038b23a6c7e1aee255004dd19098d4c2462b61b849The Android Open Source Project 11048b23a6c7e1aee255004dd19098d4c2462b61b849The Android Open Source Project#define NEON_FN(dest, src, dummy) dest = do_clz16(src) 11058b23a6c7e1aee255004dd19098d4c2462b61b849The Android Open Source ProjectNEON_VOP1(clz_u16, neon_u16, 2) 11068b23a6c7e1aee255004dd19098d4c2462b61b849The Android Open Source Project#undef NEON_FN 11078b23a6c7e1aee255004dd19098d4c2462b61b849The Android Open Source Project 11088b23a6c7e1aee255004dd19098d4c2462b61b849The Android Open Source Project#define NEON_FN(dest, src, dummy) dest = do_clz8((src < 0) ? ~src : src) - 1 11098b23a6c7e1aee255004dd19098d4c2462b61b849The Android Open Source ProjectNEON_VOP1(cls_s8, neon_s8, 4) 11108b23a6c7e1aee255004dd19098d4c2462b61b849The Android Open Source Project#undef NEON_FN 11118b23a6c7e1aee255004dd19098d4c2462b61b849The Android Open Source Project 11128b23a6c7e1aee255004dd19098d4c2462b61b849The Android Open Source Project#define NEON_FN(dest, src, dummy) dest = do_clz16((src < 0) ? ~src : src) - 1 11138b23a6c7e1aee255004dd19098d4c2462b61b849The Android Open Source ProjectNEON_VOP1(cls_s16, neon_s16, 2) 11148b23a6c7e1aee255004dd19098d4c2462b61b849The Android Open Source Project#undef NEON_FN 11158b23a6c7e1aee255004dd19098d4c2462b61b849The Android Open Source Project 11168b23a6c7e1aee255004dd19098d4c2462b61b849The Android Open Source Projectuint32_t HELPER(neon_cls_s32)(uint32_t x) 11178b23a6c7e1aee255004dd19098d4c2462b61b849The Android Open Source Project{ 11188b23a6c7e1aee255004dd19098d4c2462b61b849The Android Open Source Project int count; 11198b23a6c7e1aee255004dd19098d4c2462b61b849The Android Open Source Project if ((int32_t)x < 0) 11208b23a6c7e1aee255004dd19098d4c2462b61b849The Android Open Source Project x = ~x; 11218b23a6c7e1aee255004dd19098d4c2462b61b849The Android Open Source Project for (count = 32; x; count--) 11228b23a6c7e1aee255004dd19098d4c2462b61b849The Android Open Source Project x = x >> 1; 11238b23a6c7e1aee255004dd19098d4c2462b61b849The Android Open Source Project return count - 1; 11248b23a6c7e1aee255004dd19098d4c2462b61b849The Android Open Source Project} 11258b23a6c7e1aee255004dd19098d4c2462b61b849The Android Open Source Project 11268b23a6c7e1aee255004dd19098d4c2462b61b849The Android Open Source Project/* Bit count. */ 11278b23a6c7e1aee255004dd19098d4c2462b61b849The Android Open Source Projectuint32_t HELPER(neon_cnt_u8)(uint32_t x) 11288b23a6c7e1aee255004dd19098d4c2462b61b849The Android Open Source Project{ 11298b23a6c7e1aee255004dd19098d4c2462b61b849The Android Open Source Project x = (x & 0x55555555) + ((x >> 1) & 0x55555555); 11308b23a6c7e1aee255004dd19098d4c2462b61b849The Android Open Source Project x = (x & 0x33333333) + ((x >> 2) & 0x33333333); 11318b23a6c7e1aee255004dd19098d4c2462b61b849The Android Open Source Project x = (x & 0x0f0f0f0f) + ((x >> 4) & 0x0f0f0f0f); 11328b23a6c7e1aee255004dd19098d4c2462b61b849The Android Open Source Project return x; 11338b23a6c7e1aee255004dd19098d4c2462b61b849The Android Open Source Project} 11348b23a6c7e1aee255004dd19098d4c2462b61b849The Android Open Source Project 11358b23a6c7e1aee255004dd19098d4c2462b61b849The Android Open Source Project#define NEON_QDMULH16(dest, src1, src2, round) do { \ 11368b23a6c7e1aee255004dd19098d4c2462b61b849The Android Open Source Project uint32_t tmp = (int32_t)(int16_t) src1 * (int16_t) src2; \ 11378b23a6c7e1aee255004dd19098d4c2462b61b849The Android Open Source Project if ((tmp ^ (tmp << 1)) & SIGNBIT) { \ 11388b23a6c7e1aee255004dd19098d4c2462b61b849The Android Open Source Project SET_QC(); \ 11398b23a6c7e1aee255004dd19098d4c2462b61b849The Android Open Source Project tmp = (tmp >> 31) ^ ~SIGNBIT; \ 11405285864985be9077e58e42235af6582dee72e841David 'Digit' Turner } else { \ 11415285864985be9077e58e42235af6582dee72e841David 'Digit' Turner tmp <<= 1; \ 11428b23a6c7e1aee255004dd19098d4c2462b61b849The Android Open Source Project } \ 11438b23a6c7e1aee255004dd19098d4c2462b61b849The Android Open Source Project if (round) { \ 11448b23a6c7e1aee255004dd19098d4c2462b61b849The Android Open Source Project int32_t old = tmp; \ 11458b23a6c7e1aee255004dd19098d4c2462b61b849The Android Open Source Project tmp += 1 << 15; \ 11468b23a6c7e1aee255004dd19098d4c2462b61b849The Android Open Source Project if ((int32_t)tmp < old) { \ 11478b23a6c7e1aee255004dd19098d4c2462b61b849The Android Open Source Project SET_QC(); \ 11488b23a6c7e1aee255004dd19098d4c2462b61b849The Android Open Source Project tmp = SIGNBIT - 1; \ 11498b23a6c7e1aee255004dd19098d4c2462b61b849The Android Open Source Project } \ 11508b23a6c7e1aee255004dd19098d4c2462b61b849The Android Open Source Project } \ 11518b23a6c7e1aee255004dd19098d4c2462b61b849The Android Open Source Project dest = tmp >> 16; \ 11528b23a6c7e1aee255004dd19098d4c2462b61b849The Android Open Source Project } while(0) 11538b23a6c7e1aee255004dd19098d4c2462b61b849The Android Open Source Project#define NEON_FN(dest, src1, src2) NEON_QDMULH16(dest, src1, src2, 0) 11545285864985be9077e58e42235af6582dee72e841David 'Digit' TurnerNEON_VOP(qdmulh_s16, neon_s16, 2) 11558b23a6c7e1aee255004dd19098d4c2462b61b849The Android Open Source Project#undef NEON_FN 11568b23a6c7e1aee255004dd19098d4c2462b61b849The Android Open Source Project#define NEON_FN(dest, src1, src2) NEON_QDMULH16(dest, src1, src2, 1) 11575285864985be9077e58e42235af6582dee72e841David 'Digit' TurnerNEON_VOP(qrdmulh_s16, neon_s16, 2) 11588b23a6c7e1aee255004dd19098d4c2462b61b849The Android Open Source Project#undef NEON_FN 11598b23a6c7e1aee255004dd19098d4c2462b61b849The Android Open Source Project#undef NEON_QDMULH16 11608b23a6c7e1aee255004dd19098d4c2462b61b849The Android Open Source Project 11618b23a6c7e1aee255004dd19098d4c2462b61b849The Android Open Source Project#define NEON_QDMULH32(dest, src1, src2, round) do { \ 11628b23a6c7e1aee255004dd19098d4c2462b61b849The Android Open Source Project uint64_t tmp = (int64_t)(int32_t) src1 * (int32_t) src2; \ 11638b23a6c7e1aee255004dd19098d4c2462b61b849The Android Open Source Project if ((tmp ^ (tmp << 1)) & SIGNBIT64) { \ 11648b23a6c7e1aee255004dd19098d4c2462b61b849The Android Open Source Project SET_QC(); \ 11658b23a6c7e1aee255004dd19098d4c2462b61b849The Android Open Source Project tmp = (tmp >> 63) ^ ~SIGNBIT64; \ 11668b23a6c7e1aee255004dd19098d4c2462b61b849The Android Open Source Project } else { \ 11678b23a6c7e1aee255004dd19098d4c2462b61b849The Android Open Source Project tmp <<= 1; \ 11688b23a6c7e1aee255004dd19098d4c2462b61b849The Android Open Source Project } \ 11698b23a6c7e1aee255004dd19098d4c2462b61b849The Android Open Source Project if (round) { \ 11708b23a6c7e1aee255004dd19098d4c2462b61b849The Android Open Source Project int64_t old = tmp; \ 11718b23a6c7e1aee255004dd19098d4c2462b61b849The Android Open Source Project tmp += (int64_t)1 << 31; \ 11728b23a6c7e1aee255004dd19098d4c2462b61b849The Android Open Source Project if ((int64_t)tmp < old) { \ 11738b23a6c7e1aee255004dd19098d4c2462b61b849The Android Open Source Project SET_QC(); \ 11748b23a6c7e1aee255004dd19098d4c2462b61b849The Android Open Source Project tmp = SIGNBIT64 - 1; \ 11758b23a6c7e1aee255004dd19098d4c2462b61b849The Android Open Source Project } \ 11768b23a6c7e1aee255004dd19098d4c2462b61b849The Android Open Source Project } \ 11778b23a6c7e1aee255004dd19098d4c2462b61b849The Android Open Source Project dest = tmp >> 32; \ 11788b23a6c7e1aee255004dd19098d4c2462b61b849The Android Open Source Project } while(0) 11798b23a6c7e1aee255004dd19098d4c2462b61b849The Android Open Source Project#define NEON_FN(dest, src1, src2) NEON_QDMULH32(dest, src1, src2, 0) 11805285864985be9077e58e42235af6582dee72e841David 'Digit' TurnerNEON_VOP(qdmulh_s32, neon_s32, 1) 11818b23a6c7e1aee255004dd19098d4c2462b61b849The Android Open Source Project#undef NEON_FN 11828b23a6c7e1aee255004dd19098d4c2462b61b849The Android Open Source Project#define NEON_FN(dest, src1, src2) NEON_QDMULH32(dest, src1, src2, 1) 11835285864985be9077e58e42235af6582dee72e841David 'Digit' TurnerNEON_VOP(qrdmulh_s32, neon_s32, 1) 11848b23a6c7e1aee255004dd19098d4c2462b61b849The Android Open Source Project#undef NEON_FN 11858b23a6c7e1aee255004dd19098d4c2462b61b849The Android Open Source Project#undef NEON_QDMULH32 11868b23a6c7e1aee255004dd19098d4c2462b61b849The Android Open Source Project 11878b23a6c7e1aee255004dd19098d4c2462b61b849The Android Open Source Projectuint32_t HELPER(neon_narrow_u8)(uint64_t x) 11888b23a6c7e1aee255004dd19098d4c2462b61b849The Android Open Source Project{ 11898b23a6c7e1aee255004dd19098d4c2462b61b849The Android Open Source Project return (x & 0xffu) | ((x >> 8) & 0xff00u) | ((x >> 16) & 0xff0000u) 11908b23a6c7e1aee255004dd19098d4c2462b61b849The Android Open Source Project | ((x >> 24) & 0xff000000u); 11918b23a6c7e1aee255004dd19098d4c2462b61b849The Android Open Source Project} 11928b23a6c7e1aee255004dd19098d4c2462b61b849The Android Open Source Project 11938b23a6c7e1aee255004dd19098d4c2462b61b849The Android Open Source Projectuint32_t HELPER(neon_narrow_u16)(uint64_t x) 11948b23a6c7e1aee255004dd19098d4c2462b61b849The Android Open Source Project{ 11958b23a6c7e1aee255004dd19098d4c2462b61b849The Android Open Source Project return (x & 0xffffu) | ((x >> 16) & 0xffff0000u); 11968b23a6c7e1aee255004dd19098d4c2462b61b849The Android Open Source Project} 11978b23a6c7e1aee255004dd19098d4c2462b61b849The Android Open Source Project 11988b23a6c7e1aee255004dd19098d4c2462b61b849The Android Open Source Projectuint32_t HELPER(neon_narrow_high_u8)(uint64_t x) 11998b23a6c7e1aee255004dd19098d4c2462b61b849The Android Open Source Project{ 12008b23a6c7e1aee255004dd19098d4c2462b61b849The Android Open Source Project return ((x >> 8) & 0xff) | ((x >> 16) & 0xff00) 12018b23a6c7e1aee255004dd19098d4c2462b61b849The Android Open Source Project | ((x >> 24) & 0xff0000) | ((x >> 32) & 0xff000000); 12028b23a6c7e1aee255004dd19098d4c2462b61b849The Android Open Source Project} 12038b23a6c7e1aee255004dd19098d4c2462b61b849The Android Open Source Project 12048b23a6c7e1aee255004dd19098d4c2462b61b849The Android Open Source Projectuint32_t HELPER(neon_narrow_high_u16)(uint64_t x) 12058b23a6c7e1aee255004dd19098d4c2462b61b849The Android Open Source Project{ 12068b23a6c7e1aee255004dd19098d4c2462b61b849The Android Open Source Project return ((x >> 16) & 0xffff) | ((x >> 32) & 0xffff0000); 12078b23a6c7e1aee255004dd19098d4c2462b61b849The Android Open Source Project} 12088b23a6c7e1aee255004dd19098d4c2462b61b849The Android Open Source Project 12098b23a6c7e1aee255004dd19098d4c2462b61b849The Android Open Source Projectuint32_t HELPER(neon_narrow_round_high_u8)(uint64_t x) 12108b23a6c7e1aee255004dd19098d4c2462b61b849The Android Open Source Project{ 12118b23a6c7e1aee255004dd19098d4c2462b61b849The Android Open Source Project x &= 0xff80ff80ff80ff80ull; 12128b23a6c7e1aee255004dd19098d4c2462b61b849The Android Open Source Project x += 0x0080008000800080ull; 12138b23a6c7e1aee255004dd19098d4c2462b61b849The Android Open Source Project return ((x >> 8) & 0xff) | ((x >> 16) & 0xff00) 12148b23a6c7e1aee255004dd19098d4c2462b61b849The Android Open Source Project | ((x >> 24) & 0xff0000) | ((x >> 32) & 0xff000000); 12158b23a6c7e1aee255004dd19098d4c2462b61b849The Android Open Source Project} 12168b23a6c7e1aee255004dd19098d4c2462b61b849The Android Open Source Project 12178b23a6c7e1aee255004dd19098d4c2462b61b849The Android Open Source Projectuint32_t HELPER(neon_narrow_round_high_u16)(uint64_t x) 12188b23a6c7e1aee255004dd19098d4c2462b61b849The Android Open Source Project{ 12198b23a6c7e1aee255004dd19098d4c2462b61b849The Android Open Source Project x &= 0xffff8000ffff8000ull; 12208b23a6c7e1aee255004dd19098d4c2462b61b849The Android Open Source Project x += 0x0000800000008000ull; 12218b23a6c7e1aee255004dd19098d4c2462b61b849The Android Open Source Project return ((x >> 16) & 0xffff) | ((x >> 32) & 0xffff0000); 12228b23a6c7e1aee255004dd19098d4c2462b61b849The Android Open Source Project} 12238b23a6c7e1aee255004dd19098d4c2462b61b849The Android Open Source Project 12245285864985be9077e58e42235af6582dee72e841David 'Digit' Turneruint32_t HELPER(neon_unarrow_sat8)(uint64_t x) 12255285864985be9077e58e42235af6582dee72e841David 'Digit' Turner{ 12265285864985be9077e58e42235af6582dee72e841David 'Digit' Turner uint16_t s; 12275285864985be9077e58e42235af6582dee72e841David 'Digit' Turner uint8_t d; 12285285864985be9077e58e42235af6582dee72e841David 'Digit' Turner uint32_t res = 0; 12295285864985be9077e58e42235af6582dee72e841David 'Digit' Turner#define SAT8(n) \ 12305285864985be9077e58e42235af6582dee72e841David 'Digit' Turner s = x >> n; \ 12315285864985be9077e58e42235af6582dee72e841David 'Digit' Turner if (s & 0x8000) { \ 12325285864985be9077e58e42235af6582dee72e841David 'Digit' Turner SET_QC(); \ 12335285864985be9077e58e42235af6582dee72e841David 'Digit' Turner } else { \ 12345285864985be9077e58e42235af6582dee72e841David 'Digit' Turner if (s > 0xff) { \ 12355285864985be9077e58e42235af6582dee72e841David 'Digit' Turner d = 0xff; \ 12365285864985be9077e58e42235af6582dee72e841David 'Digit' Turner SET_QC(); \ 12375285864985be9077e58e42235af6582dee72e841David 'Digit' Turner } else { \ 12385285864985be9077e58e42235af6582dee72e841David 'Digit' Turner d = s; \ 12395285864985be9077e58e42235af6582dee72e841David 'Digit' Turner } \ 12405285864985be9077e58e42235af6582dee72e841David 'Digit' Turner res |= (uint32_t)d << (n / 2); \ 12415285864985be9077e58e42235af6582dee72e841David 'Digit' Turner } 12425285864985be9077e58e42235af6582dee72e841David 'Digit' Turner 12435285864985be9077e58e42235af6582dee72e841David 'Digit' Turner SAT8(0); 12445285864985be9077e58e42235af6582dee72e841David 'Digit' Turner SAT8(16); 12455285864985be9077e58e42235af6582dee72e841David 'Digit' Turner SAT8(32); 12465285864985be9077e58e42235af6582dee72e841David 'Digit' Turner SAT8(48); 12475285864985be9077e58e42235af6582dee72e841David 'Digit' Turner#undef SAT8 12485285864985be9077e58e42235af6582dee72e841David 'Digit' Turner return res; 12495285864985be9077e58e42235af6582dee72e841David 'Digit' Turner} 12505285864985be9077e58e42235af6582dee72e841David 'Digit' Turner 12515285864985be9077e58e42235af6582dee72e841David 'Digit' Turneruint32_t HELPER(neon_narrow_sat_u8)(uint64_t x) 12528b23a6c7e1aee255004dd19098d4c2462b61b849The Android Open Source Project{ 12538b23a6c7e1aee255004dd19098d4c2462b61b849The Android Open Source Project uint16_t s; 12548b23a6c7e1aee255004dd19098d4c2462b61b849The Android Open Source Project uint8_t d; 12558b23a6c7e1aee255004dd19098d4c2462b61b849The Android Open Source Project uint32_t res = 0; 12568b23a6c7e1aee255004dd19098d4c2462b61b849The Android Open Source Project#define SAT8(n) \ 12578b23a6c7e1aee255004dd19098d4c2462b61b849The Android Open Source Project s = x >> n; \ 12588b23a6c7e1aee255004dd19098d4c2462b61b849The Android Open Source Project if (s > 0xff) { \ 12598b23a6c7e1aee255004dd19098d4c2462b61b849The Android Open Source Project d = 0xff; \ 12608b23a6c7e1aee255004dd19098d4c2462b61b849The Android Open Source Project SET_QC(); \ 12618b23a6c7e1aee255004dd19098d4c2462b61b849The Android Open Source Project } else { \ 12628b23a6c7e1aee255004dd19098d4c2462b61b849The Android Open Source Project d = s; \ 12638b23a6c7e1aee255004dd19098d4c2462b61b849The Android Open Source Project } \ 12648b23a6c7e1aee255004dd19098d4c2462b61b849The Android Open Source Project res |= (uint32_t)d << (n / 2); 12658b23a6c7e1aee255004dd19098d4c2462b61b849The Android Open Source Project 12668b23a6c7e1aee255004dd19098d4c2462b61b849The Android Open Source Project SAT8(0); 12678b23a6c7e1aee255004dd19098d4c2462b61b849The Android Open Source Project SAT8(16); 12688b23a6c7e1aee255004dd19098d4c2462b61b849The Android Open Source Project SAT8(32); 12698b23a6c7e1aee255004dd19098d4c2462b61b849The Android Open Source Project SAT8(48); 12708b23a6c7e1aee255004dd19098d4c2462b61b849The Android Open Source Project#undef SAT8 12718b23a6c7e1aee255004dd19098d4c2462b61b849The Android Open Source Project return res; 12728b23a6c7e1aee255004dd19098d4c2462b61b849The Android Open Source Project} 12738b23a6c7e1aee255004dd19098d4c2462b61b849The Android Open Source Project 12745285864985be9077e58e42235af6582dee72e841David 'Digit' Turneruint32_t HELPER(neon_narrow_sat_s8)(uint64_t x) 12758b23a6c7e1aee255004dd19098d4c2462b61b849The Android Open Source Project{ 12768b23a6c7e1aee255004dd19098d4c2462b61b849The Android Open Source Project int16_t s; 12778b23a6c7e1aee255004dd19098d4c2462b61b849The Android Open Source Project uint8_t d; 12788b23a6c7e1aee255004dd19098d4c2462b61b849The Android Open Source Project uint32_t res = 0; 12798b23a6c7e1aee255004dd19098d4c2462b61b849The Android Open Source Project#define SAT8(n) \ 12808b23a6c7e1aee255004dd19098d4c2462b61b849The Android Open Source Project s = x >> n; \ 12818b23a6c7e1aee255004dd19098d4c2462b61b849The Android Open Source Project if (s != (int8_t)s) { \ 12828b23a6c7e1aee255004dd19098d4c2462b61b849The Android Open Source Project d = (s >> 15) ^ 0x7f; \ 12838b23a6c7e1aee255004dd19098d4c2462b61b849The Android Open Source Project SET_QC(); \ 12848b23a6c7e1aee255004dd19098d4c2462b61b849The Android Open Source Project } else { \ 12858b23a6c7e1aee255004dd19098d4c2462b61b849The Android Open Source Project d = s; \ 12868b23a6c7e1aee255004dd19098d4c2462b61b849The Android Open Source Project } \ 12878b23a6c7e1aee255004dd19098d4c2462b61b849The Android Open Source Project res |= (uint32_t)d << (n / 2); 12888b23a6c7e1aee255004dd19098d4c2462b61b849The Android Open Source Project 12898b23a6c7e1aee255004dd19098d4c2462b61b849The Android Open Source Project SAT8(0); 12908b23a6c7e1aee255004dd19098d4c2462b61b849The Android Open Source Project SAT8(16); 12918b23a6c7e1aee255004dd19098d4c2462b61b849The Android Open Source Project SAT8(32); 12928b23a6c7e1aee255004dd19098d4c2462b61b849The Android Open Source Project SAT8(48); 12938b23a6c7e1aee255004dd19098d4c2462b61b849The Android Open Source Project#undef SAT8 12948b23a6c7e1aee255004dd19098d4c2462b61b849The Android Open Source Project return res; 12958b23a6c7e1aee255004dd19098d4c2462b61b849The Android Open Source Project} 12968b23a6c7e1aee255004dd19098d4c2462b61b849The Android Open Source Project 12975285864985be9077e58e42235af6582dee72e841David 'Digit' Turneruint32_t HELPER(neon_unarrow_sat16)(uint64_t x) 12985285864985be9077e58e42235af6582dee72e841David 'Digit' Turner{ 12995285864985be9077e58e42235af6582dee72e841David 'Digit' Turner uint32_t high; 13005285864985be9077e58e42235af6582dee72e841David 'Digit' Turner uint32_t low; 13015285864985be9077e58e42235af6582dee72e841David 'Digit' Turner low = x; 13025285864985be9077e58e42235af6582dee72e841David 'Digit' Turner if (low & 0x80000000) { 13035285864985be9077e58e42235af6582dee72e841David 'Digit' Turner low = 0; 13045285864985be9077e58e42235af6582dee72e841David 'Digit' Turner SET_QC(); 13055285864985be9077e58e42235af6582dee72e841David 'Digit' Turner } else if (low > 0xffff) { 13065285864985be9077e58e42235af6582dee72e841David 'Digit' Turner low = 0xffff; 13075285864985be9077e58e42235af6582dee72e841David 'Digit' Turner SET_QC(); 13085285864985be9077e58e42235af6582dee72e841David 'Digit' Turner } 13095285864985be9077e58e42235af6582dee72e841David 'Digit' Turner high = x >> 32; 13105285864985be9077e58e42235af6582dee72e841David 'Digit' Turner if (high & 0x80000000) { 13115285864985be9077e58e42235af6582dee72e841David 'Digit' Turner high = 0; 13125285864985be9077e58e42235af6582dee72e841David 'Digit' Turner SET_QC(); 13135285864985be9077e58e42235af6582dee72e841David 'Digit' Turner } else if (high > 0xffff) { 13145285864985be9077e58e42235af6582dee72e841David 'Digit' Turner high = 0xffff; 13155285864985be9077e58e42235af6582dee72e841David 'Digit' Turner SET_QC(); 13165285864985be9077e58e42235af6582dee72e841David 'Digit' Turner } 13175285864985be9077e58e42235af6582dee72e841David 'Digit' Turner return low | (high << 16); 13185285864985be9077e58e42235af6582dee72e841David 'Digit' Turner} 13195285864985be9077e58e42235af6582dee72e841David 'Digit' Turner 13205285864985be9077e58e42235af6582dee72e841David 'Digit' Turneruint32_t HELPER(neon_narrow_sat_u16)(uint64_t x) 13218b23a6c7e1aee255004dd19098d4c2462b61b849The Android Open Source Project{ 13228b23a6c7e1aee255004dd19098d4c2462b61b849The Android Open Source Project uint32_t high; 13238b23a6c7e1aee255004dd19098d4c2462b61b849The Android Open Source Project uint32_t low; 13248b23a6c7e1aee255004dd19098d4c2462b61b849The Android Open Source Project low = x; 13258b23a6c7e1aee255004dd19098d4c2462b61b849The Android Open Source Project if (low > 0xffff) { 13268b23a6c7e1aee255004dd19098d4c2462b61b849The Android Open Source Project low = 0xffff; 13278b23a6c7e1aee255004dd19098d4c2462b61b849The Android Open Source Project SET_QC(); 13288b23a6c7e1aee255004dd19098d4c2462b61b849The Android Open Source Project } 13298b23a6c7e1aee255004dd19098d4c2462b61b849The Android Open Source Project high = x >> 32; 13308b23a6c7e1aee255004dd19098d4c2462b61b849The Android Open Source Project if (high > 0xffff) { 13318b23a6c7e1aee255004dd19098d4c2462b61b849The Android Open Source Project high = 0xffff; 13328b23a6c7e1aee255004dd19098d4c2462b61b849The Android Open Source Project SET_QC(); 13338b23a6c7e1aee255004dd19098d4c2462b61b849The Android Open Source Project } 13348b23a6c7e1aee255004dd19098d4c2462b61b849The Android Open Source Project return low | (high << 16); 13358b23a6c7e1aee255004dd19098d4c2462b61b849The Android Open Source Project} 13368b23a6c7e1aee255004dd19098d4c2462b61b849The Android Open Source Project 13375285864985be9077e58e42235af6582dee72e841David 'Digit' Turneruint32_t HELPER(neon_narrow_sat_s16)(uint64_t x) 13388b23a6c7e1aee255004dd19098d4c2462b61b849The Android Open Source Project{ 13398b23a6c7e1aee255004dd19098d4c2462b61b849The Android Open Source Project int32_t low; 13408b23a6c7e1aee255004dd19098d4c2462b61b849The Android Open Source Project int32_t high; 13418b23a6c7e1aee255004dd19098d4c2462b61b849The Android Open Source Project low = x; 13428b23a6c7e1aee255004dd19098d4c2462b61b849The Android Open Source Project if (low != (int16_t)low) { 13438b23a6c7e1aee255004dd19098d4c2462b61b849The Android Open Source Project low = (low >> 31) ^ 0x7fff; 13448b23a6c7e1aee255004dd19098d4c2462b61b849The Android Open Source Project SET_QC(); 13458b23a6c7e1aee255004dd19098d4c2462b61b849The Android Open Source Project } 13468b23a6c7e1aee255004dd19098d4c2462b61b849The Android Open Source Project high = x >> 32; 13478b23a6c7e1aee255004dd19098d4c2462b61b849The Android Open Source Project if (high != (int16_t)high) { 13488b23a6c7e1aee255004dd19098d4c2462b61b849The Android Open Source Project high = (high >> 31) ^ 0x7fff; 13498b23a6c7e1aee255004dd19098d4c2462b61b849The Android Open Source Project SET_QC(); 13508b23a6c7e1aee255004dd19098d4c2462b61b849The Android Open Source Project } 13518b23a6c7e1aee255004dd19098d4c2462b61b849The Android Open Source Project return (uint16_t)low | (high << 16); 13528b23a6c7e1aee255004dd19098d4c2462b61b849The Android Open Source Project} 13538b23a6c7e1aee255004dd19098d4c2462b61b849The Android Open Source Project 13545285864985be9077e58e42235af6582dee72e841David 'Digit' Turneruint32_t HELPER(neon_unarrow_sat32)(uint64_t x) 13555285864985be9077e58e42235af6582dee72e841David 'Digit' Turner{ 13565285864985be9077e58e42235af6582dee72e841David 'Digit' Turner if (x & 0x8000000000000000ull) { 13575285864985be9077e58e42235af6582dee72e841David 'Digit' Turner SET_QC(); 13585285864985be9077e58e42235af6582dee72e841David 'Digit' Turner return 0; 13595285864985be9077e58e42235af6582dee72e841David 'Digit' Turner } 13605285864985be9077e58e42235af6582dee72e841David 'Digit' Turner if (x > 0xffffffffu) { 13615285864985be9077e58e42235af6582dee72e841David 'Digit' Turner SET_QC(); 13625285864985be9077e58e42235af6582dee72e841David 'Digit' Turner return 0xffffffffu; 13635285864985be9077e58e42235af6582dee72e841David 'Digit' Turner } 13645285864985be9077e58e42235af6582dee72e841David 'Digit' Turner return x; 13655285864985be9077e58e42235af6582dee72e841David 'Digit' Turner} 13665285864985be9077e58e42235af6582dee72e841David 'Digit' Turner 13675285864985be9077e58e42235af6582dee72e841David 'Digit' Turneruint32_t HELPER(neon_narrow_sat_u32)(uint64_t x) 13688b23a6c7e1aee255004dd19098d4c2462b61b849The Android Open Source Project{ 13698b23a6c7e1aee255004dd19098d4c2462b61b849The Android Open Source Project if (x > 0xffffffffu) { 13708b23a6c7e1aee255004dd19098d4c2462b61b849The Android Open Source Project SET_QC(); 13718b23a6c7e1aee255004dd19098d4c2462b61b849The Android Open Source Project return 0xffffffffu; 13728b23a6c7e1aee255004dd19098d4c2462b61b849The Android Open Source Project } 13738b23a6c7e1aee255004dd19098d4c2462b61b849The Android Open Source Project return x; 13748b23a6c7e1aee255004dd19098d4c2462b61b849The Android Open Source Project} 13758b23a6c7e1aee255004dd19098d4c2462b61b849The Android Open Source Project 13765285864985be9077e58e42235af6582dee72e841David 'Digit' Turneruint32_t HELPER(neon_narrow_sat_s32)(uint64_t x) 13778b23a6c7e1aee255004dd19098d4c2462b61b849The Android Open Source Project{ 13788b23a6c7e1aee255004dd19098d4c2462b61b849The Android Open Source Project if ((int64_t)x != (int32_t)x) { 13798b23a6c7e1aee255004dd19098d4c2462b61b849The Android Open Source Project SET_QC(); 13805285864985be9077e58e42235af6582dee72e841David 'Digit' Turner return ((int64_t)x >> 63) ^ 0x7fffffff; 13818b23a6c7e1aee255004dd19098d4c2462b61b849The Android Open Source Project } 13828b23a6c7e1aee255004dd19098d4c2462b61b849The Android Open Source Project return x; 13838b23a6c7e1aee255004dd19098d4c2462b61b849The Android Open Source Project} 13848b23a6c7e1aee255004dd19098d4c2462b61b849The Android Open Source Project 13858b23a6c7e1aee255004dd19098d4c2462b61b849The Android Open Source Projectuint64_t HELPER(neon_widen_u8)(uint32_t x) 13868b23a6c7e1aee255004dd19098d4c2462b61b849The Android Open Source Project{ 13878b23a6c7e1aee255004dd19098d4c2462b61b849The Android Open Source Project uint64_t tmp; 13888b23a6c7e1aee255004dd19098d4c2462b61b849The Android Open Source Project uint64_t ret; 13898b23a6c7e1aee255004dd19098d4c2462b61b849The Android Open Source Project ret = (uint8_t)x; 13908b23a6c7e1aee255004dd19098d4c2462b61b849The Android Open Source Project tmp = (uint8_t)(x >> 8); 13918b23a6c7e1aee255004dd19098d4c2462b61b849The Android Open Source Project ret |= tmp << 16; 13928b23a6c7e1aee255004dd19098d4c2462b61b849The Android Open Source Project tmp = (uint8_t)(x >> 16); 13938b23a6c7e1aee255004dd19098d4c2462b61b849The Android Open Source Project ret |= tmp << 32; 13948b23a6c7e1aee255004dd19098d4c2462b61b849The Android Open Source Project tmp = (uint8_t)(x >> 24); 13958b23a6c7e1aee255004dd19098d4c2462b61b849The Android Open Source Project ret |= tmp << 48; 13968b23a6c7e1aee255004dd19098d4c2462b61b849The Android Open Source Project return ret; 13978b23a6c7e1aee255004dd19098d4c2462b61b849The Android Open Source Project} 13988b23a6c7e1aee255004dd19098d4c2462b61b849The Android Open Source Project 13998b23a6c7e1aee255004dd19098d4c2462b61b849The Android Open Source Projectuint64_t HELPER(neon_widen_s8)(uint32_t x) 14008b23a6c7e1aee255004dd19098d4c2462b61b849The Android Open Source Project{ 14018b23a6c7e1aee255004dd19098d4c2462b61b849The Android Open Source Project uint64_t tmp; 14028b23a6c7e1aee255004dd19098d4c2462b61b849The Android Open Source Project uint64_t ret; 14038b23a6c7e1aee255004dd19098d4c2462b61b849The Android Open Source Project ret = (uint16_t)(int8_t)x; 14048b23a6c7e1aee255004dd19098d4c2462b61b849The Android Open Source Project tmp = (uint16_t)(int8_t)(x >> 8); 14058b23a6c7e1aee255004dd19098d4c2462b61b849The Android Open Source Project ret |= tmp << 16; 14068b23a6c7e1aee255004dd19098d4c2462b61b849The Android Open Source Project tmp = (uint16_t)(int8_t)(x >> 16); 14078b23a6c7e1aee255004dd19098d4c2462b61b849The Android Open Source Project ret |= tmp << 32; 14088b23a6c7e1aee255004dd19098d4c2462b61b849The Android Open Source Project tmp = (uint16_t)(int8_t)(x >> 24); 14098b23a6c7e1aee255004dd19098d4c2462b61b849The Android Open Source Project ret |= tmp << 48; 14108b23a6c7e1aee255004dd19098d4c2462b61b849The Android Open Source Project return ret; 14118b23a6c7e1aee255004dd19098d4c2462b61b849The Android Open Source Project} 14128b23a6c7e1aee255004dd19098d4c2462b61b849The Android Open Source Project 14138b23a6c7e1aee255004dd19098d4c2462b61b849The Android Open Source Projectuint64_t HELPER(neon_widen_u16)(uint32_t x) 14148b23a6c7e1aee255004dd19098d4c2462b61b849The Android Open Source Project{ 14158b23a6c7e1aee255004dd19098d4c2462b61b849The Android Open Source Project uint64_t high = (uint16_t)(x >> 16); 14168b23a6c7e1aee255004dd19098d4c2462b61b849The Android Open Source Project return ((uint16_t)x) | (high << 32); 14178b23a6c7e1aee255004dd19098d4c2462b61b849The Android Open Source Project} 14188b23a6c7e1aee255004dd19098d4c2462b61b849The Android Open Source Project 14198b23a6c7e1aee255004dd19098d4c2462b61b849The Android Open Source Projectuint64_t HELPER(neon_widen_s16)(uint32_t x) 14208b23a6c7e1aee255004dd19098d4c2462b61b849The Android Open Source Project{ 14218b23a6c7e1aee255004dd19098d4c2462b61b849The Android Open Source Project uint64_t high = (int16_t)(x >> 16); 14228b23a6c7e1aee255004dd19098d4c2462b61b849The Android Open Source Project return ((uint32_t)(int16_t)x) | (high << 32); 14238b23a6c7e1aee255004dd19098d4c2462b61b849The Android Open Source Project} 14248b23a6c7e1aee255004dd19098d4c2462b61b849The Android Open Source Project 14258b23a6c7e1aee255004dd19098d4c2462b61b849The Android Open Source Projectuint64_t HELPER(neon_addl_u16)(uint64_t a, uint64_t b) 14268b23a6c7e1aee255004dd19098d4c2462b61b849The Android Open Source Project{ 14278b23a6c7e1aee255004dd19098d4c2462b61b849The Android Open Source Project uint64_t mask; 14288b23a6c7e1aee255004dd19098d4c2462b61b849The Android Open Source Project mask = (a ^ b) & 0x8000800080008000ull; 14298b23a6c7e1aee255004dd19098d4c2462b61b849The Android Open Source Project a &= ~0x8000800080008000ull; 14308b23a6c7e1aee255004dd19098d4c2462b61b849The Android Open Source Project b &= ~0x8000800080008000ull; 14318b23a6c7e1aee255004dd19098d4c2462b61b849The Android Open Source Project return (a + b) ^ mask; 14328b23a6c7e1aee255004dd19098d4c2462b61b849The Android Open Source Project} 14338b23a6c7e1aee255004dd19098d4c2462b61b849The Android Open Source Project 14348b23a6c7e1aee255004dd19098d4c2462b61b849The Android Open Source Projectuint64_t HELPER(neon_addl_u32)(uint64_t a, uint64_t b) 14358b23a6c7e1aee255004dd19098d4c2462b61b849The Android Open Source Project{ 14368b23a6c7e1aee255004dd19098d4c2462b61b849The Android Open Source Project uint64_t mask; 14378b23a6c7e1aee255004dd19098d4c2462b61b849The Android Open Source Project mask = (a ^ b) & 0x8000000080000000ull; 14388b23a6c7e1aee255004dd19098d4c2462b61b849The Android Open Source Project a &= ~0x8000000080000000ull; 14398b23a6c7e1aee255004dd19098d4c2462b61b849The Android Open Source Project b &= ~0x8000000080000000ull; 14408b23a6c7e1aee255004dd19098d4c2462b61b849The Android Open Source Project return (a + b) ^ mask; 14418b23a6c7e1aee255004dd19098d4c2462b61b849The Android Open Source Project} 14428b23a6c7e1aee255004dd19098d4c2462b61b849The Android Open Source Project 14438b23a6c7e1aee255004dd19098d4c2462b61b849The Android Open Source Projectuint64_t HELPER(neon_paddl_u16)(uint64_t a, uint64_t b) 14448b23a6c7e1aee255004dd19098d4c2462b61b849The Android Open Source Project{ 14458b23a6c7e1aee255004dd19098d4c2462b61b849The Android Open Source Project uint64_t tmp; 14468b23a6c7e1aee255004dd19098d4c2462b61b849The Android Open Source Project uint64_t tmp2; 14478b23a6c7e1aee255004dd19098d4c2462b61b849The Android Open Source Project 14488b23a6c7e1aee255004dd19098d4c2462b61b849The Android Open Source Project tmp = a & 0x0000ffff0000ffffull; 14498b23a6c7e1aee255004dd19098d4c2462b61b849The Android Open Source Project tmp += (a >> 16) & 0x0000ffff0000ffffull; 14508b23a6c7e1aee255004dd19098d4c2462b61b849The Android Open Source Project tmp2 = b & 0xffff0000ffff0000ull; 14518b23a6c7e1aee255004dd19098d4c2462b61b849The Android Open Source Project tmp2 += (b << 16) & 0xffff0000ffff0000ull; 14528b23a6c7e1aee255004dd19098d4c2462b61b849The Android Open Source Project return ( tmp & 0xffff) 14538b23a6c7e1aee255004dd19098d4c2462b61b849The Android Open Source Project | ((tmp >> 16) & 0xffff0000ull) 14548b23a6c7e1aee255004dd19098d4c2462b61b849The Android Open Source Project | ((tmp2 << 16) & 0xffff00000000ull) 14558b23a6c7e1aee255004dd19098d4c2462b61b849The Android Open Source Project | ( tmp2 & 0xffff000000000000ull); 14568b23a6c7e1aee255004dd19098d4c2462b61b849The Android Open Source Project} 14578b23a6c7e1aee255004dd19098d4c2462b61b849The Android Open Source Project 14588b23a6c7e1aee255004dd19098d4c2462b61b849The Android Open Source Projectuint64_t HELPER(neon_paddl_u32)(uint64_t a, uint64_t b) 14598b23a6c7e1aee255004dd19098d4c2462b61b849The Android Open Source Project{ 14608b23a6c7e1aee255004dd19098d4c2462b61b849The Android Open Source Project uint32_t low = a + (a >> 32); 14618b23a6c7e1aee255004dd19098d4c2462b61b849The Android Open Source Project uint32_t high = b + (b >> 32); 14628b23a6c7e1aee255004dd19098d4c2462b61b849The Android Open Source Project return low + ((uint64_t)high << 32); 14638b23a6c7e1aee255004dd19098d4c2462b61b849The Android Open Source Project} 14648b23a6c7e1aee255004dd19098d4c2462b61b849The Android Open Source Project 14658b23a6c7e1aee255004dd19098d4c2462b61b849The Android Open Source Projectuint64_t HELPER(neon_subl_u16)(uint64_t a, uint64_t b) 14668b23a6c7e1aee255004dd19098d4c2462b61b849The Android Open Source Project{ 14678b23a6c7e1aee255004dd19098d4c2462b61b849The Android Open Source Project uint64_t mask; 14688b23a6c7e1aee255004dd19098d4c2462b61b849The Android Open Source Project mask = (a ^ ~b) & 0x8000800080008000ull; 14698b23a6c7e1aee255004dd19098d4c2462b61b849The Android Open Source Project a |= 0x8000800080008000ull; 14708b23a6c7e1aee255004dd19098d4c2462b61b849The Android Open Source Project b &= ~0x8000800080008000ull; 14718b23a6c7e1aee255004dd19098d4c2462b61b849The Android Open Source Project return (a - b) ^ mask; 14728b23a6c7e1aee255004dd19098d4c2462b61b849The Android Open Source Project} 14738b23a6c7e1aee255004dd19098d4c2462b61b849The Android Open Source Project 14748b23a6c7e1aee255004dd19098d4c2462b61b849The Android Open Source Projectuint64_t HELPER(neon_subl_u32)(uint64_t a, uint64_t b) 14758b23a6c7e1aee255004dd19098d4c2462b61b849The Android Open Source Project{ 14768b23a6c7e1aee255004dd19098d4c2462b61b849The Android Open Source Project uint64_t mask; 14778b23a6c7e1aee255004dd19098d4c2462b61b849The Android Open Source Project mask = (a ^ ~b) & 0x8000000080000000ull; 14788b23a6c7e1aee255004dd19098d4c2462b61b849The Android Open Source Project a |= 0x8000000080000000ull; 14798b23a6c7e1aee255004dd19098d4c2462b61b849The Android Open Source Project b &= ~0x8000000080000000ull; 14808b23a6c7e1aee255004dd19098d4c2462b61b849The Android Open Source Project return (a - b) ^ mask; 14818b23a6c7e1aee255004dd19098d4c2462b61b849The Android Open Source Project} 14828b23a6c7e1aee255004dd19098d4c2462b61b849The Android Open Source Project 14835285864985be9077e58e42235af6582dee72e841David 'Digit' Turneruint64_t HELPER(neon_addl_saturate_s32)(uint64_t a, uint64_t b) 14848b23a6c7e1aee255004dd19098d4c2462b61b849The Android Open Source Project{ 14858b23a6c7e1aee255004dd19098d4c2462b61b849The Android Open Source Project uint32_t x, y; 14868b23a6c7e1aee255004dd19098d4c2462b61b849The Android Open Source Project uint32_t low, high; 14878b23a6c7e1aee255004dd19098d4c2462b61b849The Android Open Source Project 14888b23a6c7e1aee255004dd19098d4c2462b61b849The Android Open Source Project x = a; 14898b23a6c7e1aee255004dd19098d4c2462b61b849The Android Open Source Project y = b; 14908b23a6c7e1aee255004dd19098d4c2462b61b849The Android Open Source Project low = x + y; 14918b23a6c7e1aee255004dd19098d4c2462b61b849The Android Open Source Project if (((low ^ x) & SIGNBIT) && !((x ^ y) & SIGNBIT)) { 14928b23a6c7e1aee255004dd19098d4c2462b61b849The Android Open Source Project SET_QC(); 14938b23a6c7e1aee255004dd19098d4c2462b61b849The Android Open Source Project low = ((int32_t)x >> 31) ^ ~SIGNBIT; 14948b23a6c7e1aee255004dd19098d4c2462b61b849The Android Open Source Project } 14958b23a6c7e1aee255004dd19098d4c2462b61b849The Android Open Source Project x = a >> 32; 14968b23a6c7e1aee255004dd19098d4c2462b61b849The Android Open Source Project y = b >> 32; 14978b23a6c7e1aee255004dd19098d4c2462b61b849The Android Open Source Project high = x + y; 14988b23a6c7e1aee255004dd19098d4c2462b61b849The Android Open Source Project if (((high ^ x) & SIGNBIT) && !((x ^ y) & SIGNBIT)) { 14998b23a6c7e1aee255004dd19098d4c2462b61b849The Android Open Source Project SET_QC(); 15008b23a6c7e1aee255004dd19098d4c2462b61b849The Android Open Source Project high = ((int32_t)x >> 31) ^ ~SIGNBIT; 15018b23a6c7e1aee255004dd19098d4c2462b61b849The Android Open Source Project } 15028b23a6c7e1aee255004dd19098d4c2462b61b849The Android Open Source Project return low | ((uint64_t)high << 32); 15038b23a6c7e1aee255004dd19098d4c2462b61b849The Android Open Source Project} 15048b23a6c7e1aee255004dd19098d4c2462b61b849The Android Open Source Project 15055285864985be9077e58e42235af6582dee72e841David 'Digit' Turneruint64_t HELPER(neon_addl_saturate_s64)(uint64_t a, uint64_t b) 15068b23a6c7e1aee255004dd19098d4c2462b61b849The Android Open Source Project{ 15078b23a6c7e1aee255004dd19098d4c2462b61b849The Android Open Source Project uint64_t result; 15088b23a6c7e1aee255004dd19098d4c2462b61b849The Android Open Source Project 15098b23a6c7e1aee255004dd19098d4c2462b61b849The Android Open Source Project result = a + b; 15108b23a6c7e1aee255004dd19098d4c2462b61b849The Android Open Source Project if (((result ^ a) & SIGNBIT64) && !((a ^ b) & SIGNBIT64)) { 15118b23a6c7e1aee255004dd19098d4c2462b61b849The Android Open Source Project SET_QC(); 15128b23a6c7e1aee255004dd19098d4c2462b61b849The Android Open Source Project result = ((int64_t)a >> 63) ^ ~SIGNBIT64; 15138b23a6c7e1aee255004dd19098d4c2462b61b849The Android Open Source Project } 15148b23a6c7e1aee255004dd19098d4c2462b61b849The Android Open Source Project return result; 15158b23a6c7e1aee255004dd19098d4c2462b61b849The Android Open Source Project} 15168b23a6c7e1aee255004dd19098d4c2462b61b849The Android Open Source Project 15175285864985be9077e58e42235af6582dee72e841David 'Digit' Turner/* We have to do the arithmetic in a larger type than 15185285864985be9077e58e42235af6582dee72e841David 'Digit' Turner * the input type, because for example with a signed 32 bit 15195285864985be9077e58e42235af6582dee72e841David 'Digit' Turner * op the absolute difference can overflow a signed 32 bit value. 15205285864985be9077e58e42235af6582dee72e841David 'Digit' Turner */ 15215285864985be9077e58e42235af6582dee72e841David 'Digit' Turner#define DO_ABD(dest, x, y, intype, arithtype) do { \ 15225285864985be9077e58e42235af6582dee72e841David 'Digit' Turner arithtype tmp_x = (intype)(x); \ 15235285864985be9077e58e42235af6582dee72e841David 'Digit' Turner arithtype tmp_y = (intype)(y); \ 15248b23a6c7e1aee255004dd19098d4c2462b61b849The Android Open Source Project dest = ((tmp_x > tmp_y) ? tmp_x - tmp_y : tmp_y - tmp_x); \ 15258b23a6c7e1aee255004dd19098d4c2462b61b849The Android Open Source Project } while(0) 15268b23a6c7e1aee255004dd19098d4c2462b61b849The Android Open Source Project 15278b23a6c7e1aee255004dd19098d4c2462b61b849The Android Open Source Projectuint64_t HELPER(neon_abdl_u16)(uint32_t a, uint32_t b) 15288b23a6c7e1aee255004dd19098d4c2462b61b849The Android Open Source Project{ 15298b23a6c7e1aee255004dd19098d4c2462b61b849The Android Open Source Project uint64_t tmp; 15308b23a6c7e1aee255004dd19098d4c2462b61b849The Android Open Source Project uint64_t result; 15315285864985be9077e58e42235af6582dee72e841David 'Digit' Turner DO_ABD(result, a, b, uint8_t, uint32_t); 15325285864985be9077e58e42235af6582dee72e841David 'Digit' Turner DO_ABD(tmp, a >> 8, b >> 8, uint8_t, uint32_t); 15338b23a6c7e1aee255004dd19098d4c2462b61b849The Android Open Source Project result |= tmp << 16; 15345285864985be9077e58e42235af6582dee72e841David 'Digit' Turner DO_ABD(tmp, a >> 16, b >> 16, uint8_t, uint32_t); 15358b23a6c7e1aee255004dd19098d4c2462b61b849The Android Open Source Project result |= tmp << 32; 15365285864985be9077e58e42235af6582dee72e841David 'Digit' Turner DO_ABD(tmp, a >> 24, b >> 24, uint8_t, uint32_t); 15378b23a6c7e1aee255004dd19098d4c2462b61b849The Android Open Source Project result |= tmp << 48; 15388b23a6c7e1aee255004dd19098d4c2462b61b849The Android Open Source Project return result; 15398b23a6c7e1aee255004dd19098d4c2462b61b849The Android Open Source Project} 15408b23a6c7e1aee255004dd19098d4c2462b61b849The Android Open Source Project 15418b23a6c7e1aee255004dd19098d4c2462b61b849The Android Open Source Projectuint64_t HELPER(neon_abdl_s16)(uint32_t a, uint32_t b) 15428b23a6c7e1aee255004dd19098d4c2462b61b849The Android Open Source Project{ 15438b23a6c7e1aee255004dd19098d4c2462b61b849The Android Open Source Project uint64_t tmp; 15448b23a6c7e1aee255004dd19098d4c2462b61b849The Android Open Source Project uint64_t result; 15455285864985be9077e58e42235af6582dee72e841David 'Digit' Turner DO_ABD(result, a, b, int8_t, int32_t); 15465285864985be9077e58e42235af6582dee72e841David 'Digit' Turner DO_ABD(tmp, a >> 8, b >> 8, int8_t, int32_t); 15478b23a6c7e1aee255004dd19098d4c2462b61b849The Android Open Source Project result |= tmp << 16; 15485285864985be9077e58e42235af6582dee72e841David 'Digit' Turner DO_ABD(tmp, a >> 16, b >> 16, int8_t, int32_t); 15498b23a6c7e1aee255004dd19098d4c2462b61b849The Android Open Source Project result |= tmp << 32; 15505285864985be9077e58e42235af6582dee72e841David 'Digit' Turner DO_ABD(tmp, a >> 24, b >> 24, int8_t, int32_t); 15518b23a6c7e1aee255004dd19098d4c2462b61b849The Android Open Source Project result |= tmp << 48; 15528b23a6c7e1aee255004dd19098d4c2462b61b849The Android Open Source Project return result; 15538b23a6c7e1aee255004dd19098d4c2462b61b849The Android Open Source Project} 15548b23a6c7e1aee255004dd19098d4c2462b61b849The Android Open Source Project 15558b23a6c7e1aee255004dd19098d4c2462b61b849The Android Open Source Projectuint64_t HELPER(neon_abdl_u32)(uint32_t a, uint32_t b) 15568b23a6c7e1aee255004dd19098d4c2462b61b849The Android Open Source Project{ 15578b23a6c7e1aee255004dd19098d4c2462b61b849The Android Open Source Project uint64_t tmp; 15588b23a6c7e1aee255004dd19098d4c2462b61b849The Android Open Source Project uint64_t result; 15595285864985be9077e58e42235af6582dee72e841David 'Digit' Turner DO_ABD(result, a, b, uint16_t, uint32_t); 15605285864985be9077e58e42235af6582dee72e841David 'Digit' Turner DO_ABD(tmp, a >> 16, b >> 16, uint16_t, uint32_t); 15618b23a6c7e1aee255004dd19098d4c2462b61b849The Android Open Source Project return result | (tmp << 32); 15628b23a6c7e1aee255004dd19098d4c2462b61b849The Android Open Source Project} 15638b23a6c7e1aee255004dd19098d4c2462b61b849The Android Open Source Project 15648b23a6c7e1aee255004dd19098d4c2462b61b849The Android Open Source Projectuint64_t HELPER(neon_abdl_s32)(uint32_t a, uint32_t b) 15658b23a6c7e1aee255004dd19098d4c2462b61b849The Android Open Source Project{ 15668b23a6c7e1aee255004dd19098d4c2462b61b849The Android Open Source Project uint64_t tmp; 15678b23a6c7e1aee255004dd19098d4c2462b61b849The Android Open Source Project uint64_t result; 15685285864985be9077e58e42235af6582dee72e841David 'Digit' Turner DO_ABD(result, a, b, int16_t, int32_t); 15695285864985be9077e58e42235af6582dee72e841David 'Digit' Turner DO_ABD(tmp, a >> 16, b >> 16, int16_t, int32_t); 15708b23a6c7e1aee255004dd19098d4c2462b61b849The Android Open Source Project return result | (tmp << 32); 15718b23a6c7e1aee255004dd19098d4c2462b61b849The Android Open Source Project} 15728b23a6c7e1aee255004dd19098d4c2462b61b849The Android Open Source Project 15738b23a6c7e1aee255004dd19098d4c2462b61b849The Android Open Source Projectuint64_t HELPER(neon_abdl_u64)(uint32_t a, uint32_t b) 15748b23a6c7e1aee255004dd19098d4c2462b61b849The Android Open Source Project{ 15758b23a6c7e1aee255004dd19098d4c2462b61b849The Android Open Source Project uint64_t result; 15765285864985be9077e58e42235af6582dee72e841David 'Digit' Turner DO_ABD(result, a, b, uint32_t, uint64_t); 15778b23a6c7e1aee255004dd19098d4c2462b61b849The Android Open Source Project return result; 15788b23a6c7e1aee255004dd19098d4c2462b61b849The Android Open Source Project} 15798b23a6c7e1aee255004dd19098d4c2462b61b849The Android Open Source Project 15808b23a6c7e1aee255004dd19098d4c2462b61b849The Android Open Source Projectuint64_t HELPER(neon_abdl_s64)(uint32_t a, uint32_t b) 15818b23a6c7e1aee255004dd19098d4c2462b61b849The Android Open Source Project{ 15828b23a6c7e1aee255004dd19098d4c2462b61b849The Android Open Source Project uint64_t result; 15835285864985be9077e58e42235af6582dee72e841David 'Digit' Turner DO_ABD(result, a, b, int32_t, int64_t); 15848b23a6c7e1aee255004dd19098d4c2462b61b849The Android Open Source Project return result; 15858b23a6c7e1aee255004dd19098d4c2462b61b849The Android Open Source Project} 15868b23a6c7e1aee255004dd19098d4c2462b61b849The Android Open Source Project#undef DO_ABD 15878b23a6c7e1aee255004dd19098d4c2462b61b849The Android Open Source Project 15888b23a6c7e1aee255004dd19098d4c2462b61b849The Android Open Source Project/* Widening multiply. Named type is the source type. */ 15898b23a6c7e1aee255004dd19098d4c2462b61b849The Android Open Source Project#define DO_MULL(dest, x, y, type1, type2) do { \ 15908b23a6c7e1aee255004dd19098d4c2462b61b849The Android Open Source Project type1 tmp_x = x; \ 15918b23a6c7e1aee255004dd19098d4c2462b61b849The Android Open Source Project type1 tmp_y = y; \ 15928b23a6c7e1aee255004dd19098d4c2462b61b849The Android Open Source Project dest = (type2)((type2)tmp_x * (type2)tmp_y); \ 15938b23a6c7e1aee255004dd19098d4c2462b61b849The Android Open Source Project } while(0) 15948b23a6c7e1aee255004dd19098d4c2462b61b849The Android Open Source Project 15958b23a6c7e1aee255004dd19098d4c2462b61b849The Android Open Source Projectuint64_t HELPER(neon_mull_u8)(uint32_t a, uint32_t b) 15968b23a6c7e1aee255004dd19098d4c2462b61b849The Android Open Source Project{ 15978b23a6c7e1aee255004dd19098d4c2462b61b849The Android Open Source Project uint64_t tmp; 15988b23a6c7e1aee255004dd19098d4c2462b61b849The Android Open Source Project uint64_t result; 15998b23a6c7e1aee255004dd19098d4c2462b61b849The Android Open Source Project 16008b23a6c7e1aee255004dd19098d4c2462b61b849The Android Open Source Project DO_MULL(result, a, b, uint8_t, uint16_t); 16018b23a6c7e1aee255004dd19098d4c2462b61b849The Android Open Source Project DO_MULL(tmp, a >> 8, b >> 8, uint8_t, uint16_t); 16028b23a6c7e1aee255004dd19098d4c2462b61b849The Android Open Source Project result |= tmp << 16; 16038b23a6c7e1aee255004dd19098d4c2462b61b849The Android Open Source Project DO_MULL(tmp, a >> 16, b >> 16, uint8_t, uint16_t); 16048b23a6c7e1aee255004dd19098d4c2462b61b849The Android Open Source Project result |= tmp << 32; 16058b23a6c7e1aee255004dd19098d4c2462b61b849The Android Open Source Project DO_MULL(tmp, a >> 24, b >> 24, uint8_t, uint16_t); 16068b23a6c7e1aee255004dd19098d4c2462b61b849The Android Open Source Project result |= tmp << 48; 16078b23a6c7e1aee255004dd19098d4c2462b61b849The Android Open Source Project return result; 16088b23a6c7e1aee255004dd19098d4c2462b61b849The Android Open Source Project} 16098b23a6c7e1aee255004dd19098d4c2462b61b849The Android Open Source Project 16108b23a6c7e1aee255004dd19098d4c2462b61b849The Android Open Source Projectuint64_t HELPER(neon_mull_s8)(uint32_t a, uint32_t b) 16118b23a6c7e1aee255004dd19098d4c2462b61b849The Android Open Source Project{ 16128b23a6c7e1aee255004dd19098d4c2462b61b849The Android Open Source Project uint64_t tmp; 16138b23a6c7e1aee255004dd19098d4c2462b61b849The Android Open Source Project uint64_t result; 16148b23a6c7e1aee255004dd19098d4c2462b61b849The Android Open Source Project 16158b23a6c7e1aee255004dd19098d4c2462b61b849The Android Open Source Project DO_MULL(result, a, b, int8_t, uint16_t); 16168b23a6c7e1aee255004dd19098d4c2462b61b849The Android Open Source Project DO_MULL(tmp, a >> 8, b >> 8, int8_t, uint16_t); 16178b23a6c7e1aee255004dd19098d4c2462b61b849The Android Open Source Project result |= tmp << 16; 16188b23a6c7e1aee255004dd19098d4c2462b61b849The Android Open Source Project DO_MULL(tmp, a >> 16, b >> 16, int8_t, uint16_t); 16198b23a6c7e1aee255004dd19098d4c2462b61b849The Android Open Source Project result |= tmp << 32; 16208b23a6c7e1aee255004dd19098d4c2462b61b849The Android Open Source Project DO_MULL(tmp, a >> 24, b >> 24, int8_t, uint16_t); 16218b23a6c7e1aee255004dd19098d4c2462b61b849The Android Open Source Project result |= tmp << 48; 16228b23a6c7e1aee255004dd19098d4c2462b61b849The Android Open Source Project return result; 16238b23a6c7e1aee255004dd19098d4c2462b61b849The Android Open Source Project} 16248b23a6c7e1aee255004dd19098d4c2462b61b849The Android Open Source Project 16258b23a6c7e1aee255004dd19098d4c2462b61b849The Android Open Source Projectuint64_t HELPER(neon_mull_u16)(uint32_t a, uint32_t b) 16268b23a6c7e1aee255004dd19098d4c2462b61b849The Android Open Source Project{ 16278b23a6c7e1aee255004dd19098d4c2462b61b849The Android Open Source Project uint64_t tmp; 16288b23a6c7e1aee255004dd19098d4c2462b61b849The Android Open Source Project uint64_t result; 16298b23a6c7e1aee255004dd19098d4c2462b61b849The Android Open Source Project 16308b23a6c7e1aee255004dd19098d4c2462b61b849The Android Open Source Project DO_MULL(result, a, b, uint16_t, uint32_t); 16318b23a6c7e1aee255004dd19098d4c2462b61b849The Android Open Source Project DO_MULL(tmp, a >> 16, b >> 16, uint16_t, uint32_t); 16328b23a6c7e1aee255004dd19098d4c2462b61b849The Android Open Source Project return result | (tmp << 32); 16338b23a6c7e1aee255004dd19098d4c2462b61b849The Android Open Source Project} 16348b23a6c7e1aee255004dd19098d4c2462b61b849The Android Open Source Project 16358b23a6c7e1aee255004dd19098d4c2462b61b849The Android Open Source Projectuint64_t HELPER(neon_mull_s16)(uint32_t a, uint32_t b) 16368b23a6c7e1aee255004dd19098d4c2462b61b849The Android Open Source Project{ 16378b23a6c7e1aee255004dd19098d4c2462b61b849The Android Open Source Project uint64_t tmp; 16388b23a6c7e1aee255004dd19098d4c2462b61b849The Android Open Source Project uint64_t result; 16398b23a6c7e1aee255004dd19098d4c2462b61b849The Android Open Source Project 16408b23a6c7e1aee255004dd19098d4c2462b61b849The Android Open Source Project DO_MULL(result, a, b, int16_t, uint32_t); 16418b23a6c7e1aee255004dd19098d4c2462b61b849The Android Open Source Project DO_MULL(tmp, a >> 16, b >> 16, int16_t, uint32_t); 16428b23a6c7e1aee255004dd19098d4c2462b61b849The Android Open Source Project return result | (tmp << 32); 16438b23a6c7e1aee255004dd19098d4c2462b61b849The Android Open Source Project} 16448b23a6c7e1aee255004dd19098d4c2462b61b849The Android Open Source Project 16458b23a6c7e1aee255004dd19098d4c2462b61b849The Android Open Source Projectuint64_t HELPER(neon_negl_u16)(uint64_t x) 16468b23a6c7e1aee255004dd19098d4c2462b61b849The Android Open Source Project{ 16478b23a6c7e1aee255004dd19098d4c2462b61b849The Android Open Source Project uint16_t tmp; 16488b23a6c7e1aee255004dd19098d4c2462b61b849The Android Open Source Project uint64_t result; 16498b23a6c7e1aee255004dd19098d4c2462b61b849The Android Open Source Project result = (uint16_t)-x; 16508b23a6c7e1aee255004dd19098d4c2462b61b849The Android Open Source Project tmp = -(x >> 16); 16518b23a6c7e1aee255004dd19098d4c2462b61b849The Android Open Source Project result |= (uint64_t)tmp << 16; 16528b23a6c7e1aee255004dd19098d4c2462b61b849The Android Open Source Project tmp = -(x >> 32); 16538b23a6c7e1aee255004dd19098d4c2462b61b849The Android Open Source Project result |= (uint64_t)tmp << 32; 16548b23a6c7e1aee255004dd19098d4c2462b61b849The Android Open Source Project tmp = -(x >> 48); 16558b23a6c7e1aee255004dd19098d4c2462b61b849The Android Open Source Project result |= (uint64_t)tmp << 48; 16568b23a6c7e1aee255004dd19098d4c2462b61b849The Android Open Source Project return result; 16578b23a6c7e1aee255004dd19098d4c2462b61b849The Android Open Source Project} 16588b23a6c7e1aee255004dd19098d4c2462b61b849The Android Open Source Project 16598b23a6c7e1aee255004dd19098d4c2462b61b849The Android Open Source Projectuint64_t HELPER(neon_negl_u32)(uint64_t x) 16608b23a6c7e1aee255004dd19098d4c2462b61b849The Android Open Source Project{ 16618b23a6c7e1aee255004dd19098d4c2462b61b849The Android Open Source Project uint32_t low = -x; 16628b23a6c7e1aee255004dd19098d4c2462b61b849The Android Open Source Project uint32_t high = -(x >> 32); 16638b23a6c7e1aee255004dd19098d4c2462b61b849The Android Open Source Project return low | ((uint64_t)high << 32); 16648b23a6c7e1aee255004dd19098d4c2462b61b849The Android Open Source Project} 16658b23a6c7e1aee255004dd19098d4c2462b61b849The Android Open Source Project 16668b23a6c7e1aee255004dd19098d4c2462b61b849The Android Open Source Project/* FIXME: There should be a native op for this. */ 16678b23a6c7e1aee255004dd19098d4c2462b61b849The Android Open Source Projectuint64_t HELPER(neon_negl_u64)(uint64_t x) 16688b23a6c7e1aee255004dd19098d4c2462b61b849The Android Open Source Project{ 16698b23a6c7e1aee255004dd19098d4c2462b61b849The Android Open Source Project return -x; 16708b23a6c7e1aee255004dd19098d4c2462b61b849The Android Open Source Project} 16718b23a6c7e1aee255004dd19098d4c2462b61b849The Android Open Source Project 16728b23a6c7e1aee255004dd19098d4c2462b61b849The Android Open Source Project/* Saturnating sign manuipulation. */ 16738b23a6c7e1aee255004dd19098d4c2462b61b849The Android Open Source Project/* ??? Make these use NEON_VOP1 */ 16748b23a6c7e1aee255004dd19098d4c2462b61b849The Android Open Source Project#define DO_QABS8(x) do { \ 16758b23a6c7e1aee255004dd19098d4c2462b61b849The Android Open Source Project if (x == (int8_t)0x80) { \ 16768b23a6c7e1aee255004dd19098d4c2462b61b849The Android Open Source Project x = 0x7f; \ 16778b23a6c7e1aee255004dd19098d4c2462b61b849The Android Open Source Project SET_QC(); \ 16788b23a6c7e1aee255004dd19098d4c2462b61b849The Android Open Source Project } else if (x < 0) { \ 16798b23a6c7e1aee255004dd19098d4c2462b61b849The Android Open Source Project x = -x; \ 16808b23a6c7e1aee255004dd19098d4c2462b61b849The Android Open Source Project }} while (0) 16815285864985be9077e58e42235af6582dee72e841David 'Digit' Turneruint32_t HELPER(neon_qabs_s8)(uint32_t x) 16828b23a6c7e1aee255004dd19098d4c2462b61b849The Android Open Source Project{ 16838b23a6c7e1aee255004dd19098d4c2462b61b849The Android Open Source Project neon_s8 vec; 16848b23a6c7e1aee255004dd19098d4c2462b61b849The Android Open Source Project NEON_UNPACK(neon_s8, vec, x); 16858b23a6c7e1aee255004dd19098d4c2462b61b849The Android Open Source Project DO_QABS8(vec.v1); 16868b23a6c7e1aee255004dd19098d4c2462b61b849The Android Open Source Project DO_QABS8(vec.v2); 16878b23a6c7e1aee255004dd19098d4c2462b61b849The Android Open Source Project DO_QABS8(vec.v3); 16888b23a6c7e1aee255004dd19098d4c2462b61b849The Android Open Source Project DO_QABS8(vec.v4); 16898b23a6c7e1aee255004dd19098d4c2462b61b849The Android Open Source Project NEON_PACK(neon_s8, x, vec); 16908b23a6c7e1aee255004dd19098d4c2462b61b849The Android Open Source Project return x; 16918b23a6c7e1aee255004dd19098d4c2462b61b849The Android Open Source Project} 16928b23a6c7e1aee255004dd19098d4c2462b61b849The Android Open Source Project#undef DO_QABS8 16938b23a6c7e1aee255004dd19098d4c2462b61b849The Android Open Source Project 16948b23a6c7e1aee255004dd19098d4c2462b61b849The Android Open Source Project#define DO_QNEG8(x) do { \ 16958b23a6c7e1aee255004dd19098d4c2462b61b849The Android Open Source Project if (x == (int8_t)0x80) { \ 16968b23a6c7e1aee255004dd19098d4c2462b61b849The Android Open Source Project x = 0x7f; \ 16978b23a6c7e1aee255004dd19098d4c2462b61b849The Android Open Source Project SET_QC(); \ 16988b23a6c7e1aee255004dd19098d4c2462b61b849The Android Open Source Project } else { \ 16998b23a6c7e1aee255004dd19098d4c2462b61b849The Android Open Source Project x = -x; \ 17008b23a6c7e1aee255004dd19098d4c2462b61b849The Android Open Source Project }} while (0) 17015285864985be9077e58e42235af6582dee72e841David 'Digit' Turneruint32_t HELPER(neon_qneg_s8)(uint32_t x) 17028b23a6c7e1aee255004dd19098d4c2462b61b849The Android Open Source Project{ 17038b23a6c7e1aee255004dd19098d4c2462b61b849The Android Open Source Project neon_s8 vec; 17048b23a6c7e1aee255004dd19098d4c2462b61b849The Android Open Source Project NEON_UNPACK(neon_s8, vec, x); 17058b23a6c7e1aee255004dd19098d4c2462b61b849The Android Open Source Project DO_QNEG8(vec.v1); 17068b23a6c7e1aee255004dd19098d4c2462b61b849The Android Open Source Project DO_QNEG8(vec.v2); 17078b23a6c7e1aee255004dd19098d4c2462b61b849The Android Open Source Project DO_QNEG8(vec.v3); 17088b23a6c7e1aee255004dd19098d4c2462b61b849The Android Open Source Project DO_QNEG8(vec.v4); 17098b23a6c7e1aee255004dd19098d4c2462b61b849The Android Open Source Project NEON_PACK(neon_s8, x, vec); 17108b23a6c7e1aee255004dd19098d4c2462b61b849The Android Open Source Project return x; 17118b23a6c7e1aee255004dd19098d4c2462b61b849The Android Open Source Project} 17128b23a6c7e1aee255004dd19098d4c2462b61b849The Android Open Source Project#undef DO_QNEG8 17138b23a6c7e1aee255004dd19098d4c2462b61b849The Android Open Source Project 17148b23a6c7e1aee255004dd19098d4c2462b61b849The Android Open Source Project#define DO_QABS16(x) do { \ 17158b23a6c7e1aee255004dd19098d4c2462b61b849The Android Open Source Project if (x == (int16_t)0x8000) { \ 17168b23a6c7e1aee255004dd19098d4c2462b61b849The Android Open Source Project x = 0x7fff; \ 17178b23a6c7e1aee255004dd19098d4c2462b61b849The Android Open Source Project SET_QC(); \ 17188b23a6c7e1aee255004dd19098d4c2462b61b849The Android Open Source Project } else if (x < 0) { \ 17198b23a6c7e1aee255004dd19098d4c2462b61b849The Android Open Source Project x = -x; \ 17208b23a6c7e1aee255004dd19098d4c2462b61b849The Android Open Source Project }} while (0) 17215285864985be9077e58e42235af6582dee72e841David 'Digit' Turneruint32_t HELPER(neon_qabs_s16)(uint32_t x) 17228b23a6c7e1aee255004dd19098d4c2462b61b849The Android Open Source Project{ 17238b23a6c7e1aee255004dd19098d4c2462b61b849The Android Open Source Project neon_s16 vec; 17248b23a6c7e1aee255004dd19098d4c2462b61b849The Android Open Source Project NEON_UNPACK(neon_s16, vec, x); 17258b23a6c7e1aee255004dd19098d4c2462b61b849The Android Open Source Project DO_QABS16(vec.v1); 17268b23a6c7e1aee255004dd19098d4c2462b61b849The Android Open Source Project DO_QABS16(vec.v2); 17278b23a6c7e1aee255004dd19098d4c2462b61b849The Android Open Source Project NEON_PACK(neon_s16, x, vec); 17288b23a6c7e1aee255004dd19098d4c2462b61b849The Android Open Source Project return x; 17298b23a6c7e1aee255004dd19098d4c2462b61b849The Android Open Source Project} 17308b23a6c7e1aee255004dd19098d4c2462b61b849The Android Open Source Project#undef DO_QABS16 17318b23a6c7e1aee255004dd19098d4c2462b61b849The Android Open Source Project 17328b23a6c7e1aee255004dd19098d4c2462b61b849The Android Open Source Project#define DO_QNEG16(x) do { \ 17338b23a6c7e1aee255004dd19098d4c2462b61b849The Android Open Source Project if (x == (int16_t)0x8000) { \ 17348b23a6c7e1aee255004dd19098d4c2462b61b849The Android Open Source Project x = 0x7fff; \ 17358b23a6c7e1aee255004dd19098d4c2462b61b849The Android Open Source Project SET_QC(); \ 17368b23a6c7e1aee255004dd19098d4c2462b61b849The Android Open Source Project } else { \ 17378b23a6c7e1aee255004dd19098d4c2462b61b849The Android Open Source Project x = -x; \ 17388b23a6c7e1aee255004dd19098d4c2462b61b849The Android Open Source Project }} while (0) 17395285864985be9077e58e42235af6582dee72e841David 'Digit' Turneruint32_t HELPER(neon_qneg_s16)(uint32_t x) 17408b23a6c7e1aee255004dd19098d4c2462b61b849The Android Open Source Project{ 17418b23a6c7e1aee255004dd19098d4c2462b61b849The Android Open Source Project neon_s16 vec; 17428b23a6c7e1aee255004dd19098d4c2462b61b849The Android Open Source Project NEON_UNPACK(neon_s16, vec, x); 17438b23a6c7e1aee255004dd19098d4c2462b61b849The Android Open Source Project DO_QNEG16(vec.v1); 17448b23a6c7e1aee255004dd19098d4c2462b61b849The Android Open Source Project DO_QNEG16(vec.v2); 17458b23a6c7e1aee255004dd19098d4c2462b61b849The Android Open Source Project NEON_PACK(neon_s16, x, vec); 17468b23a6c7e1aee255004dd19098d4c2462b61b849The Android Open Source Project return x; 17478b23a6c7e1aee255004dd19098d4c2462b61b849The Android Open Source Project} 17488b23a6c7e1aee255004dd19098d4c2462b61b849The Android Open Source Project#undef DO_QNEG16 17498b23a6c7e1aee255004dd19098d4c2462b61b849The Android Open Source Project 17505285864985be9077e58e42235af6582dee72e841David 'Digit' Turneruint32_t HELPER(neon_qabs_s32)(uint32_t x) 17518b23a6c7e1aee255004dd19098d4c2462b61b849The Android Open Source Project{ 17528b23a6c7e1aee255004dd19098d4c2462b61b849The Android Open Source Project if (x == SIGNBIT) { 17538b23a6c7e1aee255004dd19098d4c2462b61b849The Android Open Source Project SET_QC(); 17548b23a6c7e1aee255004dd19098d4c2462b61b849The Android Open Source Project x = ~SIGNBIT; 17558b23a6c7e1aee255004dd19098d4c2462b61b849The Android Open Source Project } else if ((int32_t)x < 0) { 17568b23a6c7e1aee255004dd19098d4c2462b61b849The Android Open Source Project x = -x; 17578b23a6c7e1aee255004dd19098d4c2462b61b849The Android Open Source Project } 17588b23a6c7e1aee255004dd19098d4c2462b61b849The Android Open Source Project return x; 17598b23a6c7e1aee255004dd19098d4c2462b61b849The Android Open Source Project} 17608b23a6c7e1aee255004dd19098d4c2462b61b849The Android Open Source Project 17615285864985be9077e58e42235af6582dee72e841David 'Digit' Turneruint32_t HELPER(neon_qneg_s32)(uint32_t x) 17628b23a6c7e1aee255004dd19098d4c2462b61b849The Android Open Source Project{ 17638b23a6c7e1aee255004dd19098d4c2462b61b849The Android Open Source Project if (x == SIGNBIT) { 17648b23a6c7e1aee255004dd19098d4c2462b61b849The Android Open Source Project SET_QC(); 17658b23a6c7e1aee255004dd19098d4c2462b61b849The Android Open Source Project x = ~SIGNBIT; 17668b23a6c7e1aee255004dd19098d4c2462b61b849The Android Open Source Project } else { 17678b23a6c7e1aee255004dd19098d4c2462b61b849The Android Open Source Project x = -x; 17688b23a6c7e1aee255004dd19098d4c2462b61b849The Android Open Source Project } 17698b23a6c7e1aee255004dd19098d4c2462b61b849The Android Open Source Project return x; 17708b23a6c7e1aee255004dd19098d4c2462b61b849The Android Open Source Project} 17718b23a6c7e1aee255004dd19098d4c2462b61b849The Android Open Source Project 17728b23a6c7e1aee255004dd19098d4c2462b61b849The Android Open Source Project/* NEON Float helpers. */ 17738b23a6c7e1aee255004dd19098d4c2462b61b849The Android Open Source Projectuint32_t HELPER(neon_min_f32)(uint32_t a, uint32_t b) 17748b23a6c7e1aee255004dd19098d4c2462b61b849The Android Open Source Project{ 17755285864985be9077e58e42235af6582dee72e841David 'Digit' Turner return float32_val(float32_min(make_float32(a), make_float32(b), NFS)); 17768b23a6c7e1aee255004dd19098d4c2462b61b849The Android Open Source Project} 17778b23a6c7e1aee255004dd19098d4c2462b61b849The Android Open Source Project 17788b23a6c7e1aee255004dd19098d4c2462b61b849The Android Open Source Projectuint32_t HELPER(neon_max_f32)(uint32_t a, uint32_t b) 17798b23a6c7e1aee255004dd19098d4c2462b61b849The Android Open Source Project{ 17805285864985be9077e58e42235af6582dee72e841David 'Digit' Turner return float32_val(float32_max(make_float32(a), make_float32(b), NFS)); 17818b23a6c7e1aee255004dd19098d4c2462b61b849The Android Open Source Project} 17828b23a6c7e1aee255004dd19098d4c2462b61b849The Android Open Source Project 17838b23a6c7e1aee255004dd19098d4c2462b61b849The Android Open Source Projectuint32_t HELPER(neon_abd_f32)(uint32_t a, uint32_t b) 17848b23a6c7e1aee255004dd19098d4c2462b61b849The Android Open Source Project{ 17855285864985be9077e58e42235af6582dee72e841David 'Digit' Turner float32 f0 = make_float32(a); 17865285864985be9077e58e42235af6582dee72e841David 'Digit' Turner float32 f1 = make_float32(b); 17875285864985be9077e58e42235af6582dee72e841David 'Digit' Turner return float32_val(float32_abs(float32_sub(f0, f1, NFS))); 17888b23a6c7e1aee255004dd19098d4c2462b61b849The Android Open Source Project} 17898b23a6c7e1aee255004dd19098d4c2462b61b849The Android Open Source Project 17908b23a6c7e1aee255004dd19098d4c2462b61b849The Android Open Source Projectuint32_t HELPER(neon_add_f32)(uint32_t a, uint32_t b) 17918b23a6c7e1aee255004dd19098d4c2462b61b849The Android Open Source Project{ 17925285864985be9077e58e42235af6582dee72e841David 'Digit' Turner return float32_val(float32_add(make_float32(a), make_float32(b), NFS)); 17938b23a6c7e1aee255004dd19098d4c2462b61b849The Android Open Source Project} 17948b23a6c7e1aee255004dd19098d4c2462b61b849The Android Open Source Project 17958b23a6c7e1aee255004dd19098d4c2462b61b849The Android Open Source Projectuint32_t HELPER(neon_sub_f32)(uint32_t a, uint32_t b) 17968b23a6c7e1aee255004dd19098d4c2462b61b849The Android Open Source Project{ 17975285864985be9077e58e42235af6582dee72e841David 'Digit' Turner return float32_val(float32_sub(make_float32(a), make_float32(b), NFS)); 17988b23a6c7e1aee255004dd19098d4c2462b61b849The Android Open Source Project} 17998b23a6c7e1aee255004dd19098d4c2462b61b849The Android Open Source Project 18008b23a6c7e1aee255004dd19098d4c2462b61b849The Android Open Source Projectuint32_t HELPER(neon_mul_f32)(uint32_t a, uint32_t b) 18018b23a6c7e1aee255004dd19098d4c2462b61b849The Android Open Source Project{ 18025285864985be9077e58e42235af6582dee72e841David 'Digit' Turner return float32_val(float32_mul(make_float32(a), make_float32(b), NFS)); 18038b23a6c7e1aee255004dd19098d4c2462b61b849The Android Open Source Project} 18048b23a6c7e1aee255004dd19098d4c2462b61b849The Android Open Source Project 18055285864985be9077e58e42235af6582dee72e841David 'Digit' Turner/* Floating point comparisons produce an integer result. 18065285864985be9077e58e42235af6582dee72e841David 'Digit' Turner * Note that EQ doesn't signal InvalidOp for QNaNs but GE and GT do. 18075285864985be9077e58e42235af6582dee72e841David 'Digit' Turner * Softfloat routines return 0/1, which we convert to the 0/-1 Neon requires. 18085285864985be9077e58e42235af6582dee72e841David 'Digit' Turner */ 18095285864985be9077e58e42235af6582dee72e841David 'Digit' Turneruint32_t HELPER(neon_ceq_f32)(uint32_t a, uint32_t b) 18105285864985be9077e58e42235af6582dee72e841David 'Digit' Turner{ 18115285864985be9077e58e42235af6582dee72e841David 'Digit' Turner return -float32_eq_quiet(make_float32(a), make_float32(b), NFS); 18125285864985be9077e58e42235af6582dee72e841David 'Digit' Turner} 18135285864985be9077e58e42235af6582dee72e841David 'Digit' Turner 18145285864985be9077e58e42235af6582dee72e841David 'Digit' Turneruint32_t HELPER(neon_cge_f32)(uint32_t a, uint32_t b) 18155285864985be9077e58e42235af6582dee72e841David 'Digit' Turner{ 18165285864985be9077e58e42235af6582dee72e841David 'Digit' Turner return -float32_le(make_float32(b), make_float32(a), NFS); 18178b23a6c7e1aee255004dd19098d4c2462b61b849The Android Open Source Project} 18188b23a6c7e1aee255004dd19098d4c2462b61b849The Android Open Source Project 18195285864985be9077e58e42235af6582dee72e841David 'Digit' Turneruint32_t HELPER(neon_cgt_f32)(uint32_t a, uint32_t b) 18205285864985be9077e58e42235af6582dee72e841David 'Digit' Turner{ 18215285864985be9077e58e42235af6582dee72e841David 'Digit' Turner return -float32_lt(make_float32(b), make_float32(a), NFS); 18225285864985be9077e58e42235af6582dee72e841David 'Digit' Turner} 18238b23a6c7e1aee255004dd19098d4c2462b61b849The Android Open Source Project 18248b23a6c7e1aee255004dd19098d4c2462b61b849The Android Open Source Projectuint32_t HELPER(neon_acge_f32)(uint32_t a, uint32_t b) 18258b23a6c7e1aee255004dd19098d4c2462b61b849The Android Open Source Project{ 18265285864985be9077e58e42235af6582dee72e841David 'Digit' Turner float32 f0 = float32_abs(make_float32(a)); 18275285864985be9077e58e42235af6582dee72e841David 'Digit' Turner float32 f1 = float32_abs(make_float32(b)); 18285285864985be9077e58e42235af6582dee72e841David 'Digit' Turner return -float32_le(f1, f0, NFS); 18298b23a6c7e1aee255004dd19098d4c2462b61b849The Android Open Source Project} 18308b23a6c7e1aee255004dd19098d4c2462b61b849The Android Open Source Project 18318b23a6c7e1aee255004dd19098d4c2462b61b849The Android Open Source Projectuint32_t HELPER(neon_acgt_f32)(uint32_t a, uint32_t b) 18328b23a6c7e1aee255004dd19098d4c2462b61b849The Android Open Source Project{ 18335285864985be9077e58e42235af6582dee72e841David 'Digit' Turner float32 f0 = float32_abs(make_float32(a)); 18345285864985be9077e58e42235af6582dee72e841David 'Digit' Turner float32 f1 = float32_abs(make_float32(b)); 18355285864985be9077e58e42235af6582dee72e841David 'Digit' Turner return -float32_lt(f1, f0, NFS); 18365285864985be9077e58e42235af6582dee72e841David 'Digit' Turner} 18375285864985be9077e58e42235af6582dee72e841David 'Digit' Turner 18385285864985be9077e58e42235af6582dee72e841David 'Digit' Turner#define ELEM(V, N, SIZE) (((V) >> ((N) * (SIZE))) & ((1ull << (SIZE)) - 1)) 18395285864985be9077e58e42235af6582dee72e841David 'Digit' Turner 18405285864985be9077e58e42235af6582dee72e841David 'Digit' Turnervoid HELPER(neon_qunzip8)(uint32_t rd, uint32_t rm) 18415285864985be9077e58e42235af6582dee72e841David 'Digit' Turner{ 18425285864985be9077e58e42235af6582dee72e841David 'Digit' Turner uint64_t zm0 = float64_val(env->vfp.regs[rm]); 18435285864985be9077e58e42235af6582dee72e841David 'Digit' Turner uint64_t zm1 = float64_val(env->vfp.regs[rm + 1]); 18445285864985be9077e58e42235af6582dee72e841David 'Digit' Turner uint64_t zd0 = float64_val(env->vfp.regs[rd]); 18455285864985be9077e58e42235af6582dee72e841David 'Digit' Turner uint64_t zd1 = float64_val(env->vfp.regs[rd + 1]); 18465285864985be9077e58e42235af6582dee72e841David 'Digit' Turner uint64_t d0 = ELEM(zd0, 0, 8) | (ELEM(zd0, 2, 8) << 8) 18475285864985be9077e58e42235af6582dee72e841David 'Digit' Turner | (ELEM(zd0, 4, 8) << 16) | (ELEM(zd0, 6, 8) << 24) 18485285864985be9077e58e42235af6582dee72e841David 'Digit' Turner | (ELEM(zd1, 0, 8) << 32) | (ELEM(zd1, 2, 8) << 40) 18495285864985be9077e58e42235af6582dee72e841David 'Digit' Turner | (ELEM(zd1, 4, 8) << 48) | (ELEM(zd1, 6, 8) << 56); 18505285864985be9077e58e42235af6582dee72e841David 'Digit' Turner uint64_t d1 = ELEM(zm0, 0, 8) | (ELEM(zm0, 2, 8) << 8) 18515285864985be9077e58e42235af6582dee72e841David 'Digit' Turner | (ELEM(zm0, 4, 8) << 16) | (ELEM(zm0, 6, 8) << 24) 18525285864985be9077e58e42235af6582dee72e841David 'Digit' Turner | (ELEM(zm1, 0, 8) << 32) | (ELEM(zm1, 2, 8) << 40) 18535285864985be9077e58e42235af6582dee72e841David 'Digit' Turner | (ELEM(zm1, 4, 8) << 48) | (ELEM(zm1, 6, 8) << 56); 18545285864985be9077e58e42235af6582dee72e841David 'Digit' Turner uint64_t m0 = ELEM(zd0, 1, 8) | (ELEM(zd0, 3, 8) << 8) 18555285864985be9077e58e42235af6582dee72e841David 'Digit' Turner | (ELEM(zd0, 5, 8) << 16) | (ELEM(zd0, 7, 8) << 24) 18565285864985be9077e58e42235af6582dee72e841David 'Digit' Turner | (ELEM(zd1, 1, 8) << 32) | (ELEM(zd1, 3, 8) << 40) 18575285864985be9077e58e42235af6582dee72e841David 'Digit' Turner | (ELEM(zd1, 5, 8) << 48) | (ELEM(zd1, 7, 8) << 56); 18585285864985be9077e58e42235af6582dee72e841David 'Digit' Turner uint64_t m1 = ELEM(zm0, 1, 8) | (ELEM(zm0, 3, 8) << 8) 18595285864985be9077e58e42235af6582dee72e841David 'Digit' Turner | (ELEM(zm0, 5, 8) << 16) | (ELEM(zm0, 7, 8) << 24) 18605285864985be9077e58e42235af6582dee72e841David 'Digit' Turner | (ELEM(zm1, 1, 8) << 32) | (ELEM(zm1, 3, 8) << 40) 18615285864985be9077e58e42235af6582dee72e841David 'Digit' Turner | (ELEM(zm1, 5, 8) << 48) | (ELEM(zm1, 7, 8) << 56); 18625285864985be9077e58e42235af6582dee72e841David 'Digit' Turner env->vfp.regs[rm] = make_float64(m0); 18635285864985be9077e58e42235af6582dee72e841David 'Digit' Turner env->vfp.regs[rm + 1] = make_float64(m1); 18645285864985be9077e58e42235af6582dee72e841David 'Digit' Turner env->vfp.regs[rd] = make_float64(d0); 18655285864985be9077e58e42235af6582dee72e841David 'Digit' Turner env->vfp.regs[rd + 1] = make_float64(d1); 18665285864985be9077e58e42235af6582dee72e841David 'Digit' Turner} 18675285864985be9077e58e42235af6582dee72e841David 'Digit' Turner 18685285864985be9077e58e42235af6582dee72e841David 'Digit' Turnervoid HELPER(neon_qunzip16)(uint32_t rd, uint32_t rm) 18695285864985be9077e58e42235af6582dee72e841David 'Digit' Turner{ 18705285864985be9077e58e42235af6582dee72e841David 'Digit' Turner uint64_t zm0 = float64_val(env->vfp.regs[rm]); 18715285864985be9077e58e42235af6582dee72e841David 'Digit' Turner uint64_t zm1 = float64_val(env->vfp.regs[rm + 1]); 18725285864985be9077e58e42235af6582dee72e841David 'Digit' Turner uint64_t zd0 = float64_val(env->vfp.regs[rd]); 18735285864985be9077e58e42235af6582dee72e841David 'Digit' Turner uint64_t zd1 = float64_val(env->vfp.regs[rd + 1]); 18745285864985be9077e58e42235af6582dee72e841David 'Digit' Turner uint64_t d0 = ELEM(zd0, 0, 16) | (ELEM(zd0, 2, 16) << 16) 18755285864985be9077e58e42235af6582dee72e841David 'Digit' Turner | (ELEM(zd1, 0, 16) << 32) | (ELEM(zd1, 2, 16) << 48); 18765285864985be9077e58e42235af6582dee72e841David 'Digit' Turner uint64_t d1 = ELEM(zm0, 0, 16) | (ELEM(zm0, 2, 16) << 16) 18775285864985be9077e58e42235af6582dee72e841David 'Digit' Turner | (ELEM(zm1, 0, 16) << 32) | (ELEM(zm1, 2, 16) << 48); 18785285864985be9077e58e42235af6582dee72e841David 'Digit' Turner uint64_t m0 = ELEM(zd0, 1, 16) | (ELEM(zd0, 3, 16) << 16) 18795285864985be9077e58e42235af6582dee72e841David 'Digit' Turner | (ELEM(zd1, 1, 16) << 32) | (ELEM(zd1, 3, 16) << 48); 18805285864985be9077e58e42235af6582dee72e841David 'Digit' Turner uint64_t m1 = ELEM(zm0, 1, 16) | (ELEM(zm0, 3, 16) << 16) 18815285864985be9077e58e42235af6582dee72e841David 'Digit' Turner | (ELEM(zm1, 1, 16) << 32) | (ELEM(zm1, 3, 16) << 48); 18825285864985be9077e58e42235af6582dee72e841David 'Digit' Turner env->vfp.regs[rm] = make_float64(m0); 18835285864985be9077e58e42235af6582dee72e841David 'Digit' Turner env->vfp.regs[rm + 1] = make_float64(m1); 18845285864985be9077e58e42235af6582dee72e841David 'Digit' Turner env->vfp.regs[rd] = make_float64(d0); 18855285864985be9077e58e42235af6582dee72e841David 'Digit' Turner env->vfp.regs[rd + 1] = make_float64(d1); 18865285864985be9077e58e42235af6582dee72e841David 'Digit' Turner} 18875285864985be9077e58e42235af6582dee72e841David 'Digit' Turner 18885285864985be9077e58e42235af6582dee72e841David 'Digit' Turnervoid HELPER(neon_qunzip32)(uint32_t rd, uint32_t rm) 18895285864985be9077e58e42235af6582dee72e841David 'Digit' Turner{ 18905285864985be9077e58e42235af6582dee72e841David 'Digit' Turner uint64_t zm0 = float64_val(env->vfp.regs[rm]); 18915285864985be9077e58e42235af6582dee72e841David 'Digit' Turner uint64_t zm1 = float64_val(env->vfp.regs[rm + 1]); 18925285864985be9077e58e42235af6582dee72e841David 'Digit' Turner uint64_t zd0 = float64_val(env->vfp.regs[rd]); 18935285864985be9077e58e42235af6582dee72e841David 'Digit' Turner uint64_t zd1 = float64_val(env->vfp.regs[rd + 1]); 18945285864985be9077e58e42235af6582dee72e841David 'Digit' Turner uint64_t d0 = ELEM(zd0, 0, 32) | (ELEM(zd1, 0, 32) << 32); 18955285864985be9077e58e42235af6582dee72e841David 'Digit' Turner uint64_t d1 = ELEM(zm0, 0, 32) | (ELEM(zm1, 0, 32) << 32); 18965285864985be9077e58e42235af6582dee72e841David 'Digit' Turner uint64_t m0 = ELEM(zd0, 1, 32) | (ELEM(zd1, 1, 32) << 32); 18975285864985be9077e58e42235af6582dee72e841David 'Digit' Turner uint64_t m1 = ELEM(zm0, 1, 32) | (ELEM(zm1, 1, 32) << 32); 18985285864985be9077e58e42235af6582dee72e841David 'Digit' Turner env->vfp.regs[rm] = make_float64(m0); 18995285864985be9077e58e42235af6582dee72e841David 'Digit' Turner env->vfp.regs[rm + 1] = make_float64(m1); 19005285864985be9077e58e42235af6582dee72e841David 'Digit' Turner env->vfp.regs[rd] = make_float64(d0); 19015285864985be9077e58e42235af6582dee72e841David 'Digit' Turner env->vfp.regs[rd + 1] = make_float64(d1); 19025285864985be9077e58e42235af6582dee72e841David 'Digit' Turner} 19035285864985be9077e58e42235af6582dee72e841David 'Digit' Turner 19045285864985be9077e58e42235af6582dee72e841David 'Digit' Turnervoid HELPER(neon_unzip8)(uint32_t rd, uint32_t rm) 19055285864985be9077e58e42235af6582dee72e841David 'Digit' Turner{ 19065285864985be9077e58e42235af6582dee72e841David 'Digit' Turner uint64_t zm = float64_val(env->vfp.regs[rm]); 19075285864985be9077e58e42235af6582dee72e841David 'Digit' Turner uint64_t zd = float64_val(env->vfp.regs[rd]); 19085285864985be9077e58e42235af6582dee72e841David 'Digit' Turner uint64_t d0 = ELEM(zd, 0, 8) | (ELEM(zd, 2, 8) << 8) 19095285864985be9077e58e42235af6582dee72e841David 'Digit' Turner | (ELEM(zd, 4, 8) << 16) | (ELEM(zd, 6, 8) << 24) 19105285864985be9077e58e42235af6582dee72e841David 'Digit' Turner | (ELEM(zm, 0, 8) << 32) | (ELEM(zm, 2, 8) << 40) 19115285864985be9077e58e42235af6582dee72e841David 'Digit' Turner | (ELEM(zm, 4, 8) << 48) | (ELEM(zm, 6, 8) << 56); 19125285864985be9077e58e42235af6582dee72e841David 'Digit' Turner uint64_t m0 = ELEM(zd, 1, 8) | (ELEM(zd, 3, 8) << 8) 19135285864985be9077e58e42235af6582dee72e841David 'Digit' Turner | (ELEM(zd, 5, 8) << 16) | (ELEM(zd, 7, 8) << 24) 19145285864985be9077e58e42235af6582dee72e841David 'Digit' Turner | (ELEM(zm, 1, 8) << 32) | (ELEM(zm, 3, 8) << 40) 19155285864985be9077e58e42235af6582dee72e841David 'Digit' Turner | (ELEM(zm, 5, 8) << 48) | (ELEM(zm, 7, 8) << 56); 19165285864985be9077e58e42235af6582dee72e841David 'Digit' Turner env->vfp.regs[rm] = make_float64(m0); 19175285864985be9077e58e42235af6582dee72e841David 'Digit' Turner env->vfp.regs[rd] = make_float64(d0); 19185285864985be9077e58e42235af6582dee72e841David 'Digit' Turner} 19195285864985be9077e58e42235af6582dee72e841David 'Digit' Turner 19205285864985be9077e58e42235af6582dee72e841David 'Digit' Turnervoid HELPER(neon_unzip16)(uint32_t rd, uint32_t rm) 19215285864985be9077e58e42235af6582dee72e841David 'Digit' Turner{ 19225285864985be9077e58e42235af6582dee72e841David 'Digit' Turner uint64_t zm = float64_val(env->vfp.regs[rm]); 19235285864985be9077e58e42235af6582dee72e841David 'Digit' Turner uint64_t zd = float64_val(env->vfp.regs[rd]); 19245285864985be9077e58e42235af6582dee72e841David 'Digit' Turner uint64_t d0 = ELEM(zd, 0, 16) | (ELEM(zd, 2, 16) << 16) 19255285864985be9077e58e42235af6582dee72e841David 'Digit' Turner | (ELEM(zm, 0, 16) << 32) | (ELEM(zm, 2, 16) << 48); 19265285864985be9077e58e42235af6582dee72e841David 'Digit' Turner uint64_t m0 = ELEM(zd, 1, 16) | (ELEM(zd, 3, 16) << 16) 19275285864985be9077e58e42235af6582dee72e841David 'Digit' Turner | (ELEM(zm, 1, 16) << 32) | (ELEM(zm, 3, 16) << 48); 19285285864985be9077e58e42235af6582dee72e841David 'Digit' Turner env->vfp.regs[rm] = make_float64(m0); 19295285864985be9077e58e42235af6582dee72e841David 'Digit' Turner env->vfp.regs[rd] = make_float64(d0); 19305285864985be9077e58e42235af6582dee72e841David 'Digit' Turner} 19315285864985be9077e58e42235af6582dee72e841David 'Digit' Turner 19325285864985be9077e58e42235af6582dee72e841David 'Digit' Turnervoid HELPER(neon_qzip8)(uint32_t rd, uint32_t rm) 19335285864985be9077e58e42235af6582dee72e841David 'Digit' Turner{ 19345285864985be9077e58e42235af6582dee72e841David 'Digit' Turner uint64_t zm0 = float64_val(env->vfp.regs[rm]); 19355285864985be9077e58e42235af6582dee72e841David 'Digit' Turner uint64_t zm1 = float64_val(env->vfp.regs[rm + 1]); 19365285864985be9077e58e42235af6582dee72e841David 'Digit' Turner uint64_t zd0 = float64_val(env->vfp.regs[rd]); 19375285864985be9077e58e42235af6582dee72e841David 'Digit' Turner uint64_t zd1 = float64_val(env->vfp.regs[rd + 1]); 19385285864985be9077e58e42235af6582dee72e841David 'Digit' Turner uint64_t d0 = ELEM(zd0, 0, 8) | (ELEM(zm0, 0, 8) << 8) 19395285864985be9077e58e42235af6582dee72e841David 'Digit' Turner | (ELEM(zd0, 1, 8) << 16) | (ELEM(zm0, 1, 8) << 24) 19405285864985be9077e58e42235af6582dee72e841David 'Digit' Turner | (ELEM(zd0, 2, 8) << 32) | (ELEM(zm0, 2, 8) << 40) 19415285864985be9077e58e42235af6582dee72e841David 'Digit' Turner | (ELEM(zd0, 3, 8) << 48) | (ELEM(zm0, 3, 8) << 56); 19425285864985be9077e58e42235af6582dee72e841David 'Digit' Turner uint64_t d1 = ELEM(zd0, 4, 8) | (ELEM(zm0, 4, 8) << 8) 19435285864985be9077e58e42235af6582dee72e841David 'Digit' Turner | (ELEM(zd0, 5, 8) << 16) | (ELEM(zm0, 5, 8) << 24) 19445285864985be9077e58e42235af6582dee72e841David 'Digit' Turner | (ELEM(zd0, 6, 8) << 32) | (ELEM(zm0, 6, 8) << 40) 19455285864985be9077e58e42235af6582dee72e841David 'Digit' Turner | (ELEM(zd0, 7, 8) << 48) | (ELEM(zm0, 7, 8) << 56); 19465285864985be9077e58e42235af6582dee72e841David 'Digit' Turner uint64_t m0 = ELEM(zd1, 0, 8) | (ELEM(zm1, 0, 8) << 8) 19475285864985be9077e58e42235af6582dee72e841David 'Digit' Turner | (ELEM(zd1, 1, 8) << 16) | (ELEM(zm1, 1, 8) << 24) 19485285864985be9077e58e42235af6582dee72e841David 'Digit' Turner | (ELEM(zd1, 2, 8) << 32) | (ELEM(zm1, 2, 8) << 40) 19495285864985be9077e58e42235af6582dee72e841David 'Digit' Turner | (ELEM(zd1, 3, 8) << 48) | (ELEM(zm1, 3, 8) << 56); 19505285864985be9077e58e42235af6582dee72e841David 'Digit' Turner uint64_t m1 = ELEM(zd1, 4, 8) | (ELEM(zm1, 4, 8) << 8) 19515285864985be9077e58e42235af6582dee72e841David 'Digit' Turner | (ELEM(zd1, 5, 8) << 16) | (ELEM(zm1, 5, 8) << 24) 19525285864985be9077e58e42235af6582dee72e841David 'Digit' Turner | (ELEM(zd1, 6, 8) << 32) | (ELEM(zm1, 6, 8) << 40) 19535285864985be9077e58e42235af6582dee72e841David 'Digit' Turner | (ELEM(zd1, 7, 8) << 48) | (ELEM(zm1, 7, 8) << 56); 19545285864985be9077e58e42235af6582dee72e841David 'Digit' Turner env->vfp.regs[rm] = make_float64(m0); 19555285864985be9077e58e42235af6582dee72e841David 'Digit' Turner env->vfp.regs[rm + 1] = make_float64(m1); 19565285864985be9077e58e42235af6582dee72e841David 'Digit' Turner env->vfp.regs[rd] = make_float64(d0); 19575285864985be9077e58e42235af6582dee72e841David 'Digit' Turner env->vfp.regs[rd + 1] = make_float64(d1); 19585285864985be9077e58e42235af6582dee72e841David 'Digit' Turner} 19595285864985be9077e58e42235af6582dee72e841David 'Digit' Turner 19605285864985be9077e58e42235af6582dee72e841David 'Digit' Turnervoid HELPER(neon_qzip16)(uint32_t rd, uint32_t rm) 19615285864985be9077e58e42235af6582dee72e841David 'Digit' Turner{ 19625285864985be9077e58e42235af6582dee72e841David 'Digit' Turner uint64_t zm0 = float64_val(env->vfp.regs[rm]); 19635285864985be9077e58e42235af6582dee72e841David 'Digit' Turner uint64_t zm1 = float64_val(env->vfp.regs[rm + 1]); 19645285864985be9077e58e42235af6582dee72e841David 'Digit' Turner uint64_t zd0 = float64_val(env->vfp.regs[rd]); 19655285864985be9077e58e42235af6582dee72e841David 'Digit' Turner uint64_t zd1 = float64_val(env->vfp.regs[rd + 1]); 19665285864985be9077e58e42235af6582dee72e841David 'Digit' Turner uint64_t d0 = ELEM(zd0, 0, 16) | (ELEM(zm0, 0, 16) << 16) 19675285864985be9077e58e42235af6582dee72e841David 'Digit' Turner | (ELEM(zd0, 1, 16) << 32) | (ELEM(zm0, 1, 16) << 48); 19685285864985be9077e58e42235af6582dee72e841David 'Digit' Turner uint64_t d1 = ELEM(zd0, 2, 16) | (ELEM(zm0, 2, 16) << 16) 19695285864985be9077e58e42235af6582dee72e841David 'Digit' Turner | (ELEM(zd0, 3, 16) << 32) | (ELEM(zm0, 3, 16) << 48); 19705285864985be9077e58e42235af6582dee72e841David 'Digit' Turner uint64_t m0 = ELEM(zd1, 0, 16) | (ELEM(zm1, 0, 16) << 16) 19715285864985be9077e58e42235af6582dee72e841David 'Digit' Turner | (ELEM(zd1, 1, 16) << 32) | (ELEM(zm1, 1, 16) << 48); 19725285864985be9077e58e42235af6582dee72e841David 'Digit' Turner uint64_t m1 = ELEM(zd1, 2, 16) | (ELEM(zm1, 2, 16) << 16) 19735285864985be9077e58e42235af6582dee72e841David 'Digit' Turner | (ELEM(zd1, 3, 16) << 32) | (ELEM(zm1, 3, 16) << 48); 19745285864985be9077e58e42235af6582dee72e841David 'Digit' Turner env->vfp.regs[rm] = make_float64(m0); 19755285864985be9077e58e42235af6582dee72e841David 'Digit' Turner env->vfp.regs[rm + 1] = make_float64(m1); 19765285864985be9077e58e42235af6582dee72e841David 'Digit' Turner env->vfp.regs[rd] = make_float64(d0); 19775285864985be9077e58e42235af6582dee72e841David 'Digit' Turner env->vfp.regs[rd + 1] = make_float64(d1); 19785285864985be9077e58e42235af6582dee72e841David 'Digit' Turner} 19795285864985be9077e58e42235af6582dee72e841David 'Digit' Turner 19805285864985be9077e58e42235af6582dee72e841David 'Digit' Turnervoid HELPER(neon_qzip32)(uint32_t rd, uint32_t rm) 19815285864985be9077e58e42235af6582dee72e841David 'Digit' Turner{ 19825285864985be9077e58e42235af6582dee72e841David 'Digit' Turner uint64_t zm0 = float64_val(env->vfp.regs[rm]); 19835285864985be9077e58e42235af6582dee72e841David 'Digit' Turner uint64_t zm1 = float64_val(env->vfp.regs[rm + 1]); 19845285864985be9077e58e42235af6582dee72e841David 'Digit' Turner uint64_t zd0 = float64_val(env->vfp.regs[rd]); 19855285864985be9077e58e42235af6582dee72e841David 'Digit' Turner uint64_t zd1 = float64_val(env->vfp.regs[rd + 1]); 19865285864985be9077e58e42235af6582dee72e841David 'Digit' Turner uint64_t d0 = ELEM(zd0, 0, 32) | (ELEM(zm0, 0, 32) << 32); 19875285864985be9077e58e42235af6582dee72e841David 'Digit' Turner uint64_t d1 = ELEM(zd0, 1, 32) | (ELEM(zm0, 1, 32) << 32); 19885285864985be9077e58e42235af6582dee72e841David 'Digit' Turner uint64_t m0 = ELEM(zd1, 0, 32) | (ELEM(zm1, 0, 32) << 32); 19895285864985be9077e58e42235af6582dee72e841David 'Digit' Turner uint64_t m1 = ELEM(zd1, 1, 32) | (ELEM(zm1, 1, 32) << 32); 19905285864985be9077e58e42235af6582dee72e841David 'Digit' Turner env->vfp.regs[rm] = make_float64(m0); 19915285864985be9077e58e42235af6582dee72e841David 'Digit' Turner env->vfp.regs[rm + 1] = make_float64(m1); 19925285864985be9077e58e42235af6582dee72e841David 'Digit' Turner env->vfp.regs[rd] = make_float64(d0); 19935285864985be9077e58e42235af6582dee72e841David 'Digit' Turner env->vfp.regs[rd + 1] = make_float64(d1); 19945285864985be9077e58e42235af6582dee72e841David 'Digit' Turner} 19955285864985be9077e58e42235af6582dee72e841David 'Digit' Turner 19965285864985be9077e58e42235af6582dee72e841David 'Digit' Turnervoid HELPER(neon_zip8)(uint32_t rd, uint32_t rm) 19975285864985be9077e58e42235af6582dee72e841David 'Digit' Turner{ 19985285864985be9077e58e42235af6582dee72e841David 'Digit' Turner uint64_t zm = float64_val(env->vfp.regs[rm]); 19995285864985be9077e58e42235af6582dee72e841David 'Digit' Turner uint64_t zd = float64_val(env->vfp.regs[rd]); 20005285864985be9077e58e42235af6582dee72e841David 'Digit' Turner uint64_t d0 = ELEM(zd, 0, 8) | (ELEM(zm, 0, 8) << 8) 20015285864985be9077e58e42235af6582dee72e841David 'Digit' Turner | (ELEM(zd, 1, 8) << 16) | (ELEM(zm, 1, 8) << 24) 20025285864985be9077e58e42235af6582dee72e841David 'Digit' Turner | (ELEM(zd, 2, 8) << 32) | (ELEM(zm, 2, 8) << 40) 20035285864985be9077e58e42235af6582dee72e841David 'Digit' Turner | (ELEM(zd, 3, 8) << 48) | (ELEM(zm, 3, 8) << 56); 20045285864985be9077e58e42235af6582dee72e841David 'Digit' Turner uint64_t m0 = ELEM(zd, 4, 8) | (ELEM(zm, 4, 8) << 8) 20055285864985be9077e58e42235af6582dee72e841David 'Digit' Turner | (ELEM(zd, 5, 8) << 16) | (ELEM(zm, 5, 8) << 24) 20065285864985be9077e58e42235af6582dee72e841David 'Digit' Turner | (ELEM(zd, 6, 8) << 32) | (ELEM(zm, 6, 8) << 40) 20075285864985be9077e58e42235af6582dee72e841David 'Digit' Turner | (ELEM(zd, 7, 8) << 48) | (ELEM(zm, 7, 8) << 56); 20085285864985be9077e58e42235af6582dee72e841David 'Digit' Turner env->vfp.regs[rm] = make_float64(m0); 20095285864985be9077e58e42235af6582dee72e841David 'Digit' Turner env->vfp.regs[rd] = make_float64(d0); 20105285864985be9077e58e42235af6582dee72e841David 'Digit' Turner} 20115285864985be9077e58e42235af6582dee72e841David 'Digit' Turner 20125285864985be9077e58e42235af6582dee72e841David 'Digit' Turnervoid HELPER(neon_zip16)(uint32_t rd, uint32_t rm) 20135285864985be9077e58e42235af6582dee72e841David 'Digit' Turner{ 20145285864985be9077e58e42235af6582dee72e841David 'Digit' Turner uint64_t zm = float64_val(env->vfp.regs[rm]); 20155285864985be9077e58e42235af6582dee72e841David 'Digit' Turner uint64_t zd = float64_val(env->vfp.regs[rd]); 20165285864985be9077e58e42235af6582dee72e841David 'Digit' Turner uint64_t d0 = ELEM(zd, 0, 16) | (ELEM(zm, 0, 16) << 16) 20175285864985be9077e58e42235af6582dee72e841David 'Digit' Turner | (ELEM(zd, 1, 16) << 32) | (ELEM(zm, 1, 16) << 48); 20185285864985be9077e58e42235af6582dee72e841David 'Digit' Turner uint64_t m0 = ELEM(zd, 2, 16) | (ELEM(zm, 2, 16) << 16) 20195285864985be9077e58e42235af6582dee72e841David 'Digit' Turner | (ELEM(zd, 3, 16) << 32) | (ELEM(zm, 3, 16) << 48); 20205285864985be9077e58e42235af6582dee72e841David 'Digit' Turner env->vfp.regs[rm] = make_float64(m0); 20215285864985be9077e58e42235af6582dee72e841David 'Digit' Turner env->vfp.regs[rd] = make_float64(d0); 20228b23a6c7e1aee255004dd19098d4c2462b61b849The Android Open Source Project} 2023