1/* Copyright (C) 2002, 2003, 2004, 2005, 2006, 2007, 2008, 2009, 2010 2 Free Software Foundation, Inc. 3 4 This file is part of GCC. 5 6 GCC is free software; you can redistribute it and/or modify 7 it under the terms of the GNU General Public License as published by 8 the Free Software Foundation; either version 3, or (at your option) 9 any later version. 10 11 GCC is distributed in the hope that it will be useful, 12 but WITHOUT ANY WARRANTY; without even the implied warranty of 13 MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 14 GNU General Public License for more details. 15 16 Under Section 7 of GPL version 3, you are granted additional 17 permissions described in the GCC Runtime Library Exception, version 18 3.1, as published by the Free Software Foundation. 19 20 You should have received a copy of the GNU General Public License and 21 a copy of the GCC Runtime Library Exception along with this program; 22 see the files COPYING3 and COPYING.RUNTIME respectively. If not, see 23 <http://www.gnu.org/licenses/>. */ 24 25/* Implemented from the specification included in the Intel C++ Compiler 26 User Guide and Reference, version 9.0. */ 27 28#ifndef _XMMINTRIN_H_INCLUDED 29#define _XMMINTRIN_H_INCLUDED 30 31#ifndef __SSE__ 32# error "SSE instruction set not enabled" 33#else 34 35/* We need type definitions from the MMX header file. */ 36#include <mmintrin.h> 37 38/* Get _mm_malloc () and _mm_free (). */ 39#include <mm_malloc.h> 40 41/* The Intel API is flexible enough that we must allow aliasing with other 42 vector types, and their scalar components. */ 43typedef float __m128 __attribute__ ((__vector_size__ (16), __may_alias__)); 44 45/* Internal data types for implementing the intrinsics. */ 46typedef float __v4sf __attribute__ ((__vector_size__ (16))); 47 48/* Create a selector for use with the SHUFPS instruction. */ 49#define _MM_SHUFFLE(fp3,fp2,fp1,fp0) \ 50 (((fp3) << 6) | ((fp2) << 4) | ((fp1) << 2) | (fp0)) 51 52/* Constants for use with _mm_prefetch. */ 53enum _mm_hint 54{ 55 _MM_HINT_T0 = 3, 56 _MM_HINT_T1 = 2, 57 _MM_HINT_T2 = 1, 58 _MM_HINT_NTA = 0 59}; 60 61/* Bits in the MXCSR. */ 62#define _MM_EXCEPT_MASK 0x003f 63#define _MM_EXCEPT_INVALID 0x0001 64#define _MM_EXCEPT_DENORM 0x0002 65#define _MM_EXCEPT_DIV_ZERO 0x0004 66#define _MM_EXCEPT_OVERFLOW 0x0008 67#define _MM_EXCEPT_UNDERFLOW 0x0010 68#define _MM_EXCEPT_INEXACT 0x0020 69 70#define _MM_MASK_MASK 0x1f80 71#define _MM_MASK_INVALID 0x0080 72#define _MM_MASK_DENORM 0x0100 73#define _MM_MASK_DIV_ZERO 0x0200 74#define _MM_MASK_OVERFLOW 0x0400 75#define _MM_MASK_UNDERFLOW 0x0800 76#define _MM_MASK_INEXACT 0x1000 77 78#define _MM_ROUND_MASK 0x6000 79#define _MM_ROUND_NEAREST 0x0000 80#define _MM_ROUND_DOWN 0x2000 81#define _MM_ROUND_UP 0x4000 82#define _MM_ROUND_TOWARD_ZERO 0x6000 83 84#define _MM_FLUSH_ZERO_MASK 0x8000 85#define _MM_FLUSH_ZERO_ON 0x8000 86#define _MM_FLUSH_ZERO_OFF 0x0000 87 88/* Create a vector of zeros. */ 89extern __inline __m128 __attribute__((__gnu_inline__, __always_inline__, __artificial__)) 90_mm_setzero_ps (void) 91{ 92 return __extension__ (__m128){ 0.0f, 0.0f, 0.0f, 0.0f }; 93} 94 95/* Perform the respective operation on the lower SPFP (single-precision 96 floating-point) values of A and B; the upper three SPFP values are 97 passed through from A. */ 98 99extern __inline __m128 __attribute__((__gnu_inline__, __always_inline__, __artificial__)) 100_mm_add_ss (__m128 __A, __m128 __B) 101{ 102 return (__m128) __builtin_ia32_addss ((__v4sf)__A, (__v4sf)__B); 103} 104 105extern __inline __m128 __attribute__((__gnu_inline__, __always_inline__, __artificial__)) 106_mm_sub_ss (__m128 __A, __m128 __B) 107{ 108 return (__m128) __builtin_ia32_subss ((__v4sf)__A, (__v4sf)__B); 109} 110 111extern __inline __m128 __attribute__((__gnu_inline__, __always_inline__, __artificial__)) 112_mm_mul_ss (__m128 __A, __m128 __B) 113{ 114 return (__m128) __builtin_ia32_mulss ((__v4sf)__A, (__v4sf)__B); 115} 116 117extern __inline __m128 __attribute__((__gnu_inline__, __always_inline__, __artificial__)) 118_mm_div_ss (__m128 __A, __m128 __B) 119{ 120 return (__m128) __builtin_ia32_divss ((__v4sf)__A, (__v4sf)__B); 121} 122 123extern __inline __m128 __attribute__((__gnu_inline__, __always_inline__, __artificial__)) 124_mm_sqrt_ss (__m128 __A) 125{ 126 return (__m128) __builtin_ia32_sqrtss ((__v4sf)__A); 127} 128 129extern __inline __m128 __attribute__((__gnu_inline__, __always_inline__, __artificial__)) 130_mm_rcp_ss (__m128 __A) 131{ 132 return (__m128) __builtin_ia32_rcpss ((__v4sf)__A); 133} 134 135extern __inline __m128 __attribute__((__gnu_inline__, __always_inline__, __artificial__)) 136_mm_rsqrt_ss (__m128 __A) 137{ 138 return (__m128) __builtin_ia32_rsqrtss ((__v4sf)__A); 139} 140 141extern __inline __m128 __attribute__((__gnu_inline__, __always_inline__, __artificial__)) 142_mm_min_ss (__m128 __A, __m128 __B) 143{ 144 return (__m128) __builtin_ia32_minss ((__v4sf)__A, (__v4sf)__B); 145} 146 147extern __inline __m128 __attribute__((__gnu_inline__, __always_inline__, __artificial__)) 148_mm_max_ss (__m128 __A, __m128 __B) 149{ 150 return (__m128) __builtin_ia32_maxss ((__v4sf)__A, (__v4sf)__B); 151} 152 153/* Perform the respective operation on the four SPFP values in A and B. */ 154 155extern __inline __m128 __attribute__((__gnu_inline__, __always_inline__, __artificial__)) 156_mm_add_ps (__m128 __A, __m128 __B) 157{ 158 return (__m128) __builtin_ia32_addps ((__v4sf)__A, (__v4sf)__B); 159} 160 161extern __inline __m128 __attribute__((__gnu_inline__, __always_inline__, __artificial__)) 162_mm_sub_ps (__m128 __A, __m128 __B) 163{ 164 return (__m128) __builtin_ia32_subps ((__v4sf)__A, (__v4sf)__B); 165} 166 167extern __inline __m128 __attribute__((__gnu_inline__, __always_inline__, __artificial__)) 168_mm_mul_ps (__m128 __A, __m128 __B) 169{ 170 return (__m128) __builtin_ia32_mulps ((__v4sf)__A, (__v4sf)__B); 171} 172 173extern __inline __m128 __attribute__((__gnu_inline__, __always_inline__, __artificial__)) 174_mm_div_ps (__m128 __A, __m128 __B) 175{ 176 return (__m128) __builtin_ia32_divps ((__v4sf)__A, (__v4sf)__B); 177} 178 179extern __inline __m128 __attribute__((__gnu_inline__, __always_inline__, __artificial__)) 180_mm_sqrt_ps (__m128 __A) 181{ 182 return (__m128) __builtin_ia32_sqrtps ((__v4sf)__A); 183} 184 185extern __inline __m128 __attribute__((__gnu_inline__, __always_inline__, __artificial__)) 186_mm_rcp_ps (__m128 __A) 187{ 188 return (__m128) __builtin_ia32_rcpps ((__v4sf)__A); 189} 190 191extern __inline __m128 __attribute__((__gnu_inline__, __always_inline__, __artificial__)) 192_mm_rsqrt_ps (__m128 __A) 193{ 194 return (__m128) __builtin_ia32_rsqrtps ((__v4sf)__A); 195} 196 197extern __inline __m128 __attribute__((__gnu_inline__, __always_inline__, __artificial__)) 198_mm_min_ps (__m128 __A, __m128 __B) 199{ 200 return (__m128) __builtin_ia32_minps ((__v4sf)__A, (__v4sf)__B); 201} 202 203extern __inline __m128 __attribute__((__gnu_inline__, __always_inline__, __artificial__)) 204_mm_max_ps (__m128 __A, __m128 __B) 205{ 206 return (__m128) __builtin_ia32_maxps ((__v4sf)__A, (__v4sf)__B); 207} 208 209/* Perform logical bit-wise operations on 128-bit values. */ 210 211extern __inline __m128 __attribute__((__gnu_inline__, __always_inline__, __artificial__)) 212_mm_and_ps (__m128 __A, __m128 __B) 213{ 214 return __builtin_ia32_andps (__A, __B); 215} 216 217extern __inline __m128 __attribute__((__gnu_inline__, __always_inline__, __artificial__)) 218_mm_andnot_ps (__m128 __A, __m128 __B) 219{ 220 return __builtin_ia32_andnps (__A, __B); 221} 222 223extern __inline __m128 __attribute__((__gnu_inline__, __always_inline__, __artificial__)) 224_mm_or_ps (__m128 __A, __m128 __B) 225{ 226 return __builtin_ia32_orps (__A, __B); 227} 228 229extern __inline __m128 __attribute__((__gnu_inline__, __always_inline__, __artificial__)) 230_mm_xor_ps (__m128 __A, __m128 __B) 231{ 232 return __builtin_ia32_xorps (__A, __B); 233} 234 235/* Perform a comparison on the lower SPFP values of A and B. If the 236 comparison is true, place a mask of all ones in the result, otherwise a 237 mask of zeros. The upper three SPFP values are passed through from A. */ 238 239extern __inline __m128 __attribute__((__gnu_inline__, __always_inline__, __artificial__)) 240_mm_cmpeq_ss (__m128 __A, __m128 __B) 241{ 242 return (__m128) __builtin_ia32_cmpeqss ((__v4sf)__A, (__v4sf)__B); 243} 244 245extern __inline __m128 __attribute__((__gnu_inline__, __always_inline__, __artificial__)) 246_mm_cmplt_ss (__m128 __A, __m128 __B) 247{ 248 return (__m128) __builtin_ia32_cmpltss ((__v4sf)__A, (__v4sf)__B); 249} 250 251extern __inline __m128 __attribute__((__gnu_inline__, __always_inline__, __artificial__)) 252_mm_cmple_ss (__m128 __A, __m128 __B) 253{ 254 return (__m128) __builtin_ia32_cmpless ((__v4sf)__A, (__v4sf)__B); 255} 256 257extern __inline __m128 __attribute__((__gnu_inline__, __always_inline__, __artificial__)) 258_mm_cmpgt_ss (__m128 __A, __m128 __B) 259{ 260 return (__m128) __builtin_ia32_movss ((__v4sf) __A, 261 (__v4sf) 262 __builtin_ia32_cmpltss ((__v4sf) __B, 263 (__v4sf) 264 __A)); 265} 266 267extern __inline __m128 __attribute__((__gnu_inline__, __always_inline__, __artificial__)) 268_mm_cmpge_ss (__m128 __A, __m128 __B) 269{ 270 return (__m128) __builtin_ia32_movss ((__v4sf) __A, 271 (__v4sf) 272 __builtin_ia32_cmpless ((__v4sf) __B, 273 (__v4sf) 274 __A)); 275} 276 277extern __inline __m128 __attribute__((__gnu_inline__, __always_inline__, __artificial__)) 278_mm_cmpneq_ss (__m128 __A, __m128 __B) 279{ 280 return (__m128) __builtin_ia32_cmpneqss ((__v4sf)__A, (__v4sf)__B); 281} 282 283extern __inline __m128 __attribute__((__gnu_inline__, __always_inline__, __artificial__)) 284_mm_cmpnlt_ss (__m128 __A, __m128 __B) 285{ 286 return (__m128) __builtin_ia32_cmpnltss ((__v4sf)__A, (__v4sf)__B); 287} 288 289extern __inline __m128 __attribute__((__gnu_inline__, __always_inline__, __artificial__)) 290_mm_cmpnle_ss (__m128 __A, __m128 __B) 291{ 292 return (__m128) __builtin_ia32_cmpnless ((__v4sf)__A, (__v4sf)__B); 293} 294 295extern __inline __m128 __attribute__((__gnu_inline__, __always_inline__, __artificial__)) 296_mm_cmpngt_ss (__m128 __A, __m128 __B) 297{ 298 return (__m128) __builtin_ia32_movss ((__v4sf) __A, 299 (__v4sf) 300 __builtin_ia32_cmpnltss ((__v4sf) __B, 301 (__v4sf) 302 __A)); 303} 304 305extern __inline __m128 __attribute__((__gnu_inline__, __always_inline__, __artificial__)) 306_mm_cmpnge_ss (__m128 __A, __m128 __B) 307{ 308 return (__m128) __builtin_ia32_movss ((__v4sf) __A, 309 (__v4sf) 310 __builtin_ia32_cmpnless ((__v4sf) __B, 311 (__v4sf) 312 __A)); 313} 314 315extern __inline __m128 __attribute__((__gnu_inline__, __always_inline__, __artificial__)) 316_mm_cmpord_ss (__m128 __A, __m128 __B) 317{ 318 return (__m128) __builtin_ia32_cmpordss ((__v4sf)__A, (__v4sf)__B); 319} 320 321extern __inline __m128 __attribute__((__gnu_inline__, __always_inline__, __artificial__)) 322_mm_cmpunord_ss (__m128 __A, __m128 __B) 323{ 324 return (__m128) __builtin_ia32_cmpunordss ((__v4sf)__A, (__v4sf)__B); 325} 326 327/* Perform a comparison on the four SPFP values of A and B. For each 328 element, if the comparison is true, place a mask of all ones in the 329 result, otherwise a mask of zeros. */ 330 331extern __inline __m128 __attribute__((__gnu_inline__, __always_inline__, __artificial__)) 332_mm_cmpeq_ps (__m128 __A, __m128 __B) 333{ 334 return (__m128) __builtin_ia32_cmpeqps ((__v4sf)__A, (__v4sf)__B); 335} 336 337extern __inline __m128 __attribute__((__gnu_inline__, __always_inline__, __artificial__)) 338_mm_cmplt_ps (__m128 __A, __m128 __B) 339{ 340 return (__m128) __builtin_ia32_cmpltps ((__v4sf)__A, (__v4sf)__B); 341} 342 343extern __inline __m128 __attribute__((__gnu_inline__, __always_inline__, __artificial__)) 344_mm_cmple_ps (__m128 __A, __m128 __B) 345{ 346 return (__m128) __builtin_ia32_cmpleps ((__v4sf)__A, (__v4sf)__B); 347} 348 349extern __inline __m128 __attribute__((__gnu_inline__, __always_inline__, __artificial__)) 350_mm_cmpgt_ps (__m128 __A, __m128 __B) 351{ 352 return (__m128) __builtin_ia32_cmpgtps ((__v4sf)__A, (__v4sf)__B); 353} 354 355extern __inline __m128 __attribute__((__gnu_inline__, __always_inline__, __artificial__)) 356_mm_cmpge_ps (__m128 __A, __m128 __B) 357{ 358 return (__m128) __builtin_ia32_cmpgeps ((__v4sf)__A, (__v4sf)__B); 359} 360 361extern __inline __m128 __attribute__((__gnu_inline__, __always_inline__, __artificial__)) 362_mm_cmpneq_ps (__m128 __A, __m128 __B) 363{ 364 return (__m128) __builtin_ia32_cmpneqps ((__v4sf)__A, (__v4sf)__B); 365} 366 367extern __inline __m128 __attribute__((__gnu_inline__, __always_inline__, __artificial__)) 368_mm_cmpnlt_ps (__m128 __A, __m128 __B) 369{ 370 return (__m128) __builtin_ia32_cmpnltps ((__v4sf)__A, (__v4sf)__B); 371} 372 373extern __inline __m128 __attribute__((__gnu_inline__, __always_inline__, __artificial__)) 374_mm_cmpnle_ps (__m128 __A, __m128 __B) 375{ 376 return (__m128) __builtin_ia32_cmpnleps ((__v4sf)__A, (__v4sf)__B); 377} 378 379extern __inline __m128 __attribute__((__gnu_inline__, __always_inline__, __artificial__)) 380_mm_cmpngt_ps (__m128 __A, __m128 __B) 381{ 382 return (__m128) __builtin_ia32_cmpngtps ((__v4sf)__A, (__v4sf)__B); 383} 384 385extern __inline __m128 __attribute__((__gnu_inline__, __always_inline__, __artificial__)) 386_mm_cmpnge_ps (__m128 __A, __m128 __B) 387{ 388 return (__m128) __builtin_ia32_cmpngeps ((__v4sf)__A, (__v4sf)__B); 389} 390 391extern __inline __m128 __attribute__((__gnu_inline__, __always_inline__, __artificial__)) 392_mm_cmpord_ps (__m128 __A, __m128 __B) 393{ 394 return (__m128) __builtin_ia32_cmpordps ((__v4sf)__A, (__v4sf)__B); 395} 396 397extern __inline __m128 __attribute__((__gnu_inline__, __always_inline__, __artificial__)) 398_mm_cmpunord_ps (__m128 __A, __m128 __B) 399{ 400 return (__m128) __builtin_ia32_cmpunordps ((__v4sf)__A, (__v4sf)__B); 401} 402 403/* Compare the lower SPFP values of A and B and return 1 if true 404 and 0 if false. */ 405 406extern __inline int __attribute__((__gnu_inline__, __always_inline__, __artificial__)) 407_mm_comieq_ss (__m128 __A, __m128 __B) 408{ 409 return __builtin_ia32_comieq ((__v4sf)__A, (__v4sf)__B); 410} 411 412extern __inline int __attribute__((__gnu_inline__, __always_inline__, __artificial__)) 413_mm_comilt_ss (__m128 __A, __m128 __B) 414{ 415 return __builtin_ia32_comilt ((__v4sf)__A, (__v4sf)__B); 416} 417 418extern __inline int __attribute__((__gnu_inline__, __always_inline__, __artificial__)) 419_mm_comile_ss (__m128 __A, __m128 __B) 420{ 421 return __builtin_ia32_comile ((__v4sf)__A, (__v4sf)__B); 422} 423 424extern __inline int __attribute__((__gnu_inline__, __always_inline__, __artificial__)) 425_mm_comigt_ss (__m128 __A, __m128 __B) 426{ 427 return __builtin_ia32_comigt ((__v4sf)__A, (__v4sf)__B); 428} 429 430extern __inline int __attribute__((__gnu_inline__, __always_inline__, __artificial__)) 431_mm_comige_ss (__m128 __A, __m128 __B) 432{ 433 return __builtin_ia32_comige ((__v4sf)__A, (__v4sf)__B); 434} 435 436extern __inline int __attribute__((__gnu_inline__, __always_inline__, __artificial__)) 437_mm_comineq_ss (__m128 __A, __m128 __B) 438{ 439 return __builtin_ia32_comineq ((__v4sf)__A, (__v4sf)__B); 440} 441 442extern __inline int __attribute__((__gnu_inline__, __always_inline__, __artificial__)) 443_mm_ucomieq_ss (__m128 __A, __m128 __B) 444{ 445 return __builtin_ia32_ucomieq ((__v4sf)__A, (__v4sf)__B); 446} 447 448extern __inline int __attribute__((__gnu_inline__, __always_inline__, __artificial__)) 449_mm_ucomilt_ss (__m128 __A, __m128 __B) 450{ 451 return __builtin_ia32_ucomilt ((__v4sf)__A, (__v4sf)__B); 452} 453 454extern __inline int __attribute__((__gnu_inline__, __always_inline__, __artificial__)) 455_mm_ucomile_ss (__m128 __A, __m128 __B) 456{ 457 return __builtin_ia32_ucomile ((__v4sf)__A, (__v4sf)__B); 458} 459 460extern __inline int __attribute__((__gnu_inline__, __always_inline__, __artificial__)) 461_mm_ucomigt_ss (__m128 __A, __m128 __B) 462{ 463 return __builtin_ia32_ucomigt ((__v4sf)__A, (__v4sf)__B); 464} 465 466extern __inline int __attribute__((__gnu_inline__, __always_inline__, __artificial__)) 467_mm_ucomige_ss (__m128 __A, __m128 __B) 468{ 469 return __builtin_ia32_ucomige ((__v4sf)__A, (__v4sf)__B); 470} 471 472extern __inline int __attribute__((__gnu_inline__, __always_inline__, __artificial__)) 473_mm_ucomineq_ss (__m128 __A, __m128 __B) 474{ 475 return __builtin_ia32_ucomineq ((__v4sf)__A, (__v4sf)__B); 476} 477 478/* Convert the lower SPFP value to a 32-bit integer according to the current 479 rounding mode. */ 480extern __inline int __attribute__((__gnu_inline__, __always_inline__, __artificial__)) 481_mm_cvtss_si32 (__m128 __A) 482{ 483 return __builtin_ia32_cvtss2si ((__v4sf) __A); 484} 485 486extern __inline int __attribute__((__gnu_inline__, __always_inline__, __artificial__)) 487_mm_cvt_ss2si (__m128 __A) 488{ 489 return _mm_cvtss_si32 (__A); 490} 491 492#ifdef __x86_64__ 493/* Convert the lower SPFP value to a 32-bit integer according to the 494 current rounding mode. */ 495 496/* Intel intrinsic. */ 497extern __inline long long __attribute__((__gnu_inline__, __always_inline__, __artificial__)) 498_mm_cvtss_si64 (__m128 __A) 499{ 500 return __builtin_ia32_cvtss2si64 ((__v4sf) __A); 501} 502 503/* Microsoft intrinsic. */ 504extern __inline long long __attribute__((__gnu_inline__, __always_inline__, __artificial__)) 505_mm_cvtss_si64x (__m128 __A) 506{ 507 return __builtin_ia32_cvtss2si64 ((__v4sf) __A); 508} 509#endif 510 511/* Convert the two lower SPFP values to 32-bit integers according to the 512 current rounding mode. Return the integers in packed form. */ 513extern __inline __m64 __attribute__((__gnu_inline__, __always_inline__, __artificial__)) 514_mm_cvtps_pi32 (__m128 __A) 515{ 516 return (__m64) __builtin_ia32_cvtps2pi ((__v4sf) __A); 517} 518 519extern __inline __m64 __attribute__((__gnu_inline__, __always_inline__, __artificial__)) 520_mm_cvt_ps2pi (__m128 __A) 521{ 522 return _mm_cvtps_pi32 (__A); 523} 524 525/* Truncate the lower SPFP value to a 32-bit integer. */ 526extern __inline int __attribute__((__gnu_inline__, __always_inline__, __artificial__)) 527_mm_cvttss_si32 (__m128 __A) 528{ 529 return __builtin_ia32_cvttss2si ((__v4sf) __A); 530} 531 532extern __inline int __attribute__((__gnu_inline__, __always_inline__, __artificial__)) 533_mm_cvtt_ss2si (__m128 __A) 534{ 535 return _mm_cvttss_si32 (__A); 536} 537 538#ifdef __x86_64__ 539/* Truncate the lower SPFP value to a 32-bit integer. */ 540 541/* Intel intrinsic. */ 542extern __inline long long __attribute__((__gnu_inline__, __always_inline__, __artificial__)) 543_mm_cvttss_si64 (__m128 __A) 544{ 545 return __builtin_ia32_cvttss2si64 ((__v4sf) __A); 546} 547 548/* Microsoft intrinsic. */ 549extern __inline long long __attribute__((__gnu_inline__, __always_inline__, __artificial__)) 550_mm_cvttss_si64x (__m128 __A) 551{ 552 return __builtin_ia32_cvttss2si64 ((__v4sf) __A); 553} 554#endif 555 556/* Truncate the two lower SPFP values to 32-bit integers. Return the 557 integers in packed form. */ 558extern __inline __m64 __attribute__((__gnu_inline__, __always_inline__, __artificial__)) 559_mm_cvttps_pi32 (__m128 __A) 560{ 561 return (__m64) __builtin_ia32_cvttps2pi ((__v4sf) __A); 562} 563 564extern __inline __m64 __attribute__((__gnu_inline__, __always_inline__, __artificial__)) 565_mm_cvtt_ps2pi (__m128 __A) 566{ 567 return _mm_cvttps_pi32 (__A); 568} 569 570/* Convert B to a SPFP value and insert it as element zero in A. */ 571extern __inline __m128 __attribute__((__gnu_inline__, __always_inline__, __artificial__)) 572_mm_cvtsi32_ss (__m128 __A, int __B) 573{ 574 return (__m128) __builtin_ia32_cvtsi2ss ((__v4sf) __A, __B); 575} 576 577extern __inline __m128 __attribute__((__gnu_inline__, __always_inline__, __artificial__)) 578_mm_cvt_si2ss (__m128 __A, int __B) 579{ 580 return _mm_cvtsi32_ss (__A, __B); 581} 582 583#ifdef __x86_64__ 584/* Convert B to a SPFP value and insert it as element zero in A. */ 585 586/* Intel intrinsic. */ 587extern __inline __m128 __attribute__((__gnu_inline__, __always_inline__, __artificial__)) 588_mm_cvtsi64_ss (__m128 __A, long long __B) 589{ 590 return (__m128) __builtin_ia32_cvtsi642ss ((__v4sf) __A, __B); 591} 592 593/* Microsoft intrinsic. */ 594extern __inline __m128 __attribute__((__gnu_inline__, __always_inline__, __artificial__)) 595_mm_cvtsi64x_ss (__m128 __A, long long __B) 596{ 597 return (__m128) __builtin_ia32_cvtsi642ss ((__v4sf) __A, __B); 598} 599#endif 600 601/* Convert the two 32-bit values in B to SPFP form and insert them 602 as the two lower elements in A. */ 603extern __inline __m128 __attribute__((__gnu_inline__, __always_inline__, __artificial__)) 604_mm_cvtpi32_ps (__m128 __A, __m64 __B) 605{ 606 return (__m128) __builtin_ia32_cvtpi2ps ((__v4sf) __A, (__v2si)__B); 607} 608 609extern __inline __m128 __attribute__((__gnu_inline__, __always_inline__, __artificial__)) 610_mm_cvt_pi2ps (__m128 __A, __m64 __B) 611{ 612 return _mm_cvtpi32_ps (__A, __B); 613} 614 615/* Convert the four signed 16-bit values in A to SPFP form. */ 616extern __inline __m128 __attribute__((__gnu_inline__, __always_inline__, __artificial__)) 617_mm_cvtpi16_ps (__m64 __A) 618{ 619 __v4hi __sign; 620 __v2si __hisi, __losi; 621 __v4sf __zero, __ra, __rb; 622 623 /* This comparison against zero gives us a mask that can be used to 624 fill in the missing sign bits in the unpack operations below, so 625 that we get signed values after unpacking. */ 626 __sign = __builtin_ia32_pcmpgtw ((__v4hi)0LL, (__v4hi)__A); 627 628 /* Convert the four words to doublewords. */ 629 __losi = (__v2si) __builtin_ia32_punpcklwd ((__v4hi)__A, __sign); 630 __hisi = (__v2si) __builtin_ia32_punpckhwd ((__v4hi)__A, __sign); 631 632 /* Convert the doublewords to floating point two at a time. */ 633 __zero = (__v4sf) _mm_setzero_ps (); 634 __ra = __builtin_ia32_cvtpi2ps (__zero, __losi); 635 __rb = __builtin_ia32_cvtpi2ps (__ra, __hisi); 636 637 return (__m128) __builtin_ia32_movlhps (__ra, __rb); 638} 639 640/* Convert the four unsigned 16-bit values in A to SPFP form. */ 641extern __inline __m128 __attribute__((__gnu_inline__, __always_inline__, __artificial__)) 642_mm_cvtpu16_ps (__m64 __A) 643{ 644 __v2si __hisi, __losi; 645 __v4sf __zero, __ra, __rb; 646 647 /* Convert the four words to doublewords. */ 648 __losi = (__v2si) __builtin_ia32_punpcklwd ((__v4hi)__A, (__v4hi)0LL); 649 __hisi = (__v2si) __builtin_ia32_punpckhwd ((__v4hi)__A, (__v4hi)0LL); 650 651 /* Convert the doublewords to floating point two at a time. */ 652 __zero = (__v4sf) _mm_setzero_ps (); 653 __ra = __builtin_ia32_cvtpi2ps (__zero, __losi); 654 __rb = __builtin_ia32_cvtpi2ps (__ra, __hisi); 655 656 return (__m128) __builtin_ia32_movlhps (__ra, __rb); 657} 658 659/* Convert the low four signed 8-bit values in A to SPFP form. */ 660extern __inline __m128 __attribute__((__gnu_inline__, __always_inline__, __artificial__)) 661_mm_cvtpi8_ps (__m64 __A) 662{ 663 __v8qi __sign; 664 665 /* This comparison against zero gives us a mask that can be used to 666 fill in the missing sign bits in the unpack operations below, so 667 that we get signed values after unpacking. */ 668 __sign = __builtin_ia32_pcmpgtb ((__v8qi)0LL, (__v8qi)__A); 669 670 /* Convert the four low bytes to words. */ 671 __A = (__m64) __builtin_ia32_punpcklbw ((__v8qi)__A, __sign); 672 673 return _mm_cvtpi16_ps(__A); 674} 675 676/* Convert the low four unsigned 8-bit values in A to SPFP form. */ 677extern __inline __m128 __attribute__((__gnu_inline__, __always_inline__, __artificial__)) 678_mm_cvtpu8_ps(__m64 __A) 679{ 680 __A = (__m64) __builtin_ia32_punpcklbw ((__v8qi)__A, (__v8qi)0LL); 681 return _mm_cvtpu16_ps(__A); 682} 683 684/* Convert the four signed 32-bit values in A and B to SPFP form. */ 685extern __inline __m128 __attribute__((__gnu_inline__, __always_inline__, __artificial__)) 686_mm_cvtpi32x2_ps(__m64 __A, __m64 __B) 687{ 688 __v4sf __zero = (__v4sf) _mm_setzero_ps (); 689 __v4sf __sfa = __builtin_ia32_cvtpi2ps (__zero, (__v2si)__A); 690 __v4sf __sfb = __builtin_ia32_cvtpi2ps (__sfa, (__v2si)__B); 691 return (__m128) __builtin_ia32_movlhps (__sfa, __sfb); 692} 693 694/* Convert the four SPFP values in A to four signed 16-bit integers. */ 695extern __inline __m64 __attribute__((__gnu_inline__, __always_inline__, __artificial__)) 696_mm_cvtps_pi16(__m128 __A) 697{ 698 __v4sf __hisf = (__v4sf)__A; 699 __v4sf __losf = __builtin_ia32_movhlps (__hisf, __hisf); 700 __v2si __hisi = __builtin_ia32_cvtps2pi (__hisf); 701 __v2si __losi = __builtin_ia32_cvtps2pi (__losf); 702 return (__m64) __builtin_ia32_packssdw (__hisi, __losi); 703} 704 705/* Convert the four SPFP values in A to four signed 8-bit integers. */ 706extern __inline __m64 __attribute__((__gnu_inline__, __always_inline__, __artificial__)) 707_mm_cvtps_pi8(__m128 __A) 708{ 709 __v4hi __tmp = (__v4hi) _mm_cvtps_pi16 (__A); 710 return (__m64) __builtin_ia32_packsswb (__tmp, (__v4hi)0LL); 711} 712 713/* Selects four specific SPFP values from A and B based on MASK. */ 714#ifdef __OPTIMIZE__ 715extern __inline __m128 __attribute__((__gnu_inline__, __always_inline__, __artificial__)) 716_mm_shuffle_ps (__m128 __A, __m128 __B, int const __mask) 717{ 718 return (__m128) __builtin_ia32_shufps ((__v4sf)__A, (__v4sf)__B, __mask); 719} 720#else 721#define _mm_shuffle_ps(A, B, MASK) \ 722 ((__m128) __builtin_ia32_shufps ((__v4sf)(__m128)(A), \ 723 (__v4sf)(__m128)(B), (int)(MASK))) 724#endif 725 726/* Selects and interleaves the upper two SPFP values from A and B. */ 727extern __inline __m128 __attribute__((__gnu_inline__, __always_inline__, __artificial__)) 728_mm_unpackhi_ps (__m128 __A, __m128 __B) 729{ 730 return (__m128) __builtin_ia32_unpckhps ((__v4sf)__A, (__v4sf)__B); 731} 732 733/* Selects and interleaves the lower two SPFP values from A and B. */ 734extern __inline __m128 __attribute__((__gnu_inline__, __always_inline__, __artificial__)) 735_mm_unpacklo_ps (__m128 __A, __m128 __B) 736{ 737 return (__m128) __builtin_ia32_unpcklps ((__v4sf)__A, (__v4sf)__B); 738} 739 740/* Sets the upper two SPFP values with 64-bits of data loaded from P; 741 the lower two values are passed through from A. */ 742extern __inline __m128 __attribute__((__gnu_inline__, __always_inline__, __artificial__)) 743_mm_loadh_pi (__m128 __A, __m64 const *__P) 744{ 745 return (__m128) __builtin_ia32_loadhps ((__v4sf)__A, (const __v2sf *)__P); 746} 747 748/* Stores the upper two SPFP values of A into P. */ 749extern __inline void __attribute__((__gnu_inline__, __always_inline__, __artificial__)) 750_mm_storeh_pi (__m64 *__P, __m128 __A) 751{ 752 __builtin_ia32_storehps ((__v2sf *)__P, (__v4sf)__A); 753} 754 755/* Moves the upper two values of B into the lower two values of A. */ 756extern __inline __m128 __attribute__((__gnu_inline__, __always_inline__, __artificial__)) 757_mm_movehl_ps (__m128 __A, __m128 __B) 758{ 759 return (__m128) __builtin_ia32_movhlps ((__v4sf)__A, (__v4sf)__B); 760} 761 762/* Moves the lower two values of B into the upper two values of A. */ 763extern __inline __m128 __attribute__((__gnu_inline__, __always_inline__, __artificial__)) 764_mm_movelh_ps (__m128 __A, __m128 __B) 765{ 766 return (__m128) __builtin_ia32_movlhps ((__v4sf)__A, (__v4sf)__B); 767} 768 769/* Sets the lower two SPFP values with 64-bits of data loaded from P; 770 the upper two values are passed through from A. */ 771extern __inline __m128 __attribute__((__gnu_inline__, __always_inline__, __artificial__)) 772_mm_loadl_pi (__m128 __A, __m64 const *__P) 773{ 774 return (__m128) __builtin_ia32_loadlps ((__v4sf)__A, (const __v2sf *)__P); 775} 776 777/* Stores the lower two SPFP values of A into P. */ 778extern __inline void __attribute__((__gnu_inline__, __always_inline__, __artificial__)) 779_mm_storel_pi (__m64 *__P, __m128 __A) 780{ 781 __builtin_ia32_storelps ((__v2sf *)__P, (__v4sf)__A); 782} 783 784/* Creates a 4-bit mask from the most significant bits of the SPFP values. */ 785extern __inline int __attribute__((__gnu_inline__, __always_inline__, __artificial__)) 786_mm_movemask_ps (__m128 __A) 787{ 788 return __builtin_ia32_movmskps ((__v4sf)__A); 789} 790 791/* Return the contents of the control register. */ 792extern __inline unsigned int __attribute__((__gnu_inline__, __always_inline__, __artificial__)) 793_mm_getcsr (void) 794{ 795 return __builtin_ia32_stmxcsr (); 796} 797 798/* Read exception bits from the control register. */ 799extern __inline unsigned int __attribute__((__gnu_inline__, __always_inline__, __artificial__)) 800_MM_GET_EXCEPTION_STATE (void) 801{ 802 return _mm_getcsr() & _MM_EXCEPT_MASK; 803} 804 805extern __inline unsigned int __attribute__((__gnu_inline__, __always_inline__, __artificial__)) 806_MM_GET_EXCEPTION_MASK (void) 807{ 808 return _mm_getcsr() & _MM_MASK_MASK; 809} 810 811extern __inline unsigned int __attribute__((__gnu_inline__, __always_inline__, __artificial__)) 812_MM_GET_ROUNDING_MODE (void) 813{ 814 return _mm_getcsr() & _MM_ROUND_MASK; 815} 816 817extern __inline unsigned int __attribute__((__gnu_inline__, __always_inline__, __artificial__)) 818_MM_GET_FLUSH_ZERO_MODE (void) 819{ 820 return _mm_getcsr() & _MM_FLUSH_ZERO_MASK; 821} 822 823/* Set the control register to I. */ 824extern __inline void __attribute__((__gnu_inline__, __always_inline__, __artificial__)) 825_mm_setcsr (unsigned int __I) 826{ 827 __builtin_ia32_ldmxcsr (__I); 828} 829 830/* Set exception bits in the control register. */ 831extern __inline void __attribute__((__gnu_inline__, __always_inline__, __artificial__)) 832_MM_SET_EXCEPTION_STATE(unsigned int __mask) 833{ 834 _mm_setcsr((_mm_getcsr() & ~_MM_EXCEPT_MASK) | __mask); 835} 836 837extern __inline void __attribute__((__gnu_inline__, __always_inline__, __artificial__)) 838_MM_SET_EXCEPTION_MASK (unsigned int __mask) 839{ 840 _mm_setcsr((_mm_getcsr() & ~_MM_MASK_MASK) | __mask); 841} 842 843extern __inline void __attribute__((__gnu_inline__, __always_inline__, __artificial__)) 844_MM_SET_ROUNDING_MODE (unsigned int __mode) 845{ 846 _mm_setcsr((_mm_getcsr() & ~_MM_ROUND_MASK) | __mode); 847} 848 849extern __inline void __attribute__((__gnu_inline__, __always_inline__, __artificial__)) 850_MM_SET_FLUSH_ZERO_MODE (unsigned int __mode) 851{ 852 _mm_setcsr((_mm_getcsr() & ~_MM_FLUSH_ZERO_MASK) | __mode); 853} 854 855/* Create a vector with element 0 as F and the rest zero. */ 856extern __inline __m128 __attribute__((__gnu_inline__, __always_inline__, __artificial__)) 857_mm_set_ss (float __F) 858{ 859 return __extension__ (__m128)(__v4sf){ __F, 0.0f, 0.0f, 0.0f }; 860} 861 862/* Create a vector with all four elements equal to F. */ 863extern __inline __m128 __attribute__((__gnu_inline__, __always_inline__, __artificial__)) 864_mm_set1_ps (float __F) 865{ 866 return __extension__ (__m128)(__v4sf){ __F, __F, __F, __F }; 867} 868 869extern __inline __m128 __attribute__((__gnu_inline__, __always_inline__, __artificial__)) 870_mm_set_ps1 (float __F) 871{ 872 return _mm_set1_ps (__F); 873} 874 875/* Create a vector with element 0 as *P and the rest zero. */ 876extern __inline __m128 __attribute__((__gnu_inline__, __always_inline__, __artificial__)) 877_mm_load_ss (float const *__P) 878{ 879 return _mm_set_ss (*__P); 880} 881 882/* Create a vector with all four elements equal to *P. */ 883extern __inline __m128 __attribute__((__gnu_inline__, __always_inline__, __artificial__)) 884_mm_load1_ps (float const *__P) 885{ 886 return _mm_set1_ps (*__P); 887} 888 889extern __inline __m128 __attribute__((__gnu_inline__, __always_inline__, __artificial__)) 890_mm_load_ps1 (float const *__P) 891{ 892 return _mm_load1_ps (__P); 893} 894 895/* Load four SPFP values from P. The address must be 16-byte aligned. */ 896extern __inline __m128 __attribute__((__gnu_inline__, __always_inline__, __artificial__)) 897_mm_load_ps (float const *__P) 898{ 899 return (__m128) *(__v4sf *)__P; 900} 901 902/* Load four SPFP values from P. The address need not be 16-byte aligned. */ 903extern __inline __m128 __attribute__((__gnu_inline__, __always_inline__, __artificial__)) 904_mm_loadu_ps (float const *__P) 905{ 906 return (__m128) __builtin_ia32_loadups (__P); 907} 908 909/* Load four SPFP values in reverse order. The address must be aligned. */ 910extern __inline __m128 __attribute__((__gnu_inline__, __always_inline__, __artificial__)) 911_mm_loadr_ps (float const *__P) 912{ 913 __v4sf __tmp = *(__v4sf *)__P; 914 return (__m128) __builtin_ia32_shufps (__tmp, __tmp, _MM_SHUFFLE (0,1,2,3)); 915} 916 917/* Create the vector [Z Y X W]. */ 918extern __inline __m128 __attribute__((__gnu_inline__, __always_inline__, __artificial__)) 919_mm_set_ps (const float __Z, const float __Y, const float __X, const float __W) 920{ 921 return __extension__ (__m128)(__v4sf){ __W, __X, __Y, __Z }; 922} 923 924/* Create the vector [W X Y Z]. */ 925extern __inline __m128 __attribute__((__gnu_inline__, __always_inline__, __artificial__)) 926_mm_setr_ps (float __Z, float __Y, float __X, float __W) 927{ 928 return __extension__ (__m128)(__v4sf){ __Z, __Y, __X, __W }; 929} 930 931/* Stores the lower SPFP value. */ 932extern __inline void __attribute__((__gnu_inline__, __always_inline__, __artificial__)) 933_mm_store_ss (float *__P, __m128 __A) 934{ 935 *__P = __builtin_ia32_vec_ext_v4sf ((__v4sf)__A, 0); 936} 937 938extern __inline float __attribute__((__gnu_inline__, __always_inline__, __artificial__)) 939_mm_cvtss_f32 (__m128 __A) 940{ 941 return __builtin_ia32_vec_ext_v4sf ((__v4sf)__A, 0); 942} 943 944/* Store four SPFP values. The address must be 16-byte aligned. */ 945extern __inline void __attribute__((__gnu_inline__, __always_inline__, __artificial__)) 946_mm_store_ps (float *__P, __m128 __A) 947{ 948 *(__v4sf *)__P = (__v4sf)__A; 949} 950 951/* Store four SPFP values. The address need not be 16-byte aligned. */ 952extern __inline void __attribute__((__gnu_inline__, __always_inline__, __artificial__)) 953_mm_storeu_ps (float *__P, __m128 __A) 954{ 955 __builtin_ia32_storeups (__P, (__v4sf)__A); 956} 957 958/* Store the lower SPFP value across four words. */ 959extern __inline void __attribute__((__gnu_inline__, __always_inline__, __artificial__)) 960_mm_store1_ps (float *__P, __m128 __A) 961{ 962 __v4sf __va = (__v4sf)__A; 963 __v4sf __tmp = __builtin_ia32_shufps (__va, __va, _MM_SHUFFLE (0,0,0,0)); 964 _mm_storeu_ps (__P, __tmp); 965} 966 967extern __inline void __attribute__((__gnu_inline__, __always_inline__, __artificial__)) 968_mm_store_ps1 (float *__P, __m128 __A) 969{ 970 _mm_store1_ps (__P, __A); 971} 972 973/* Store four SPFP values in reverse order. The address must be aligned. */ 974extern __inline void __attribute__((__gnu_inline__, __always_inline__, __artificial__)) 975_mm_storer_ps (float *__P, __m128 __A) 976{ 977 __v4sf __va = (__v4sf)__A; 978 __v4sf __tmp = __builtin_ia32_shufps (__va, __va, _MM_SHUFFLE (0,1,2,3)); 979 _mm_store_ps (__P, __tmp); 980} 981 982/* Sets the low SPFP value of A from the low value of B. */ 983extern __inline __m128 __attribute__((__gnu_inline__, __always_inline__, __artificial__)) 984_mm_move_ss (__m128 __A, __m128 __B) 985{ 986 return (__m128) __builtin_ia32_movss ((__v4sf)__A, (__v4sf)__B); 987} 988 989/* Extracts one of the four words of A. The selector N must be immediate. */ 990#ifdef __OPTIMIZE__ 991extern __inline int __attribute__((__gnu_inline__, __always_inline__, __artificial__)) 992_mm_extract_pi16 (__m64 const __A, int const __N) 993{ 994 return __builtin_ia32_vec_ext_v4hi ((__v4hi)__A, __N); 995} 996 997extern __inline int __attribute__((__gnu_inline__, __always_inline__, __artificial__)) 998_m_pextrw (__m64 const __A, int const __N) 999{ 1000 return _mm_extract_pi16 (__A, __N); 1001} 1002#else 1003#define _mm_extract_pi16(A, N) \ 1004 ((int) __builtin_ia32_vec_ext_v4hi ((__v4hi)(__m64)(A), (int)(N))) 1005 1006#define _m_pextrw(A, N) _mm_extract_pi16(A, N) 1007#endif 1008 1009/* Inserts word D into one of four words of A. The selector N must be 1010 immediate. */ 1011#ifdef __OPTIMIZE__ 1012extern __inline __m64 __attribute__((__gnu_inline__, __always_inline__, __artificial__)) 1013_mm_insert_pi16 (__m64 const __A, int const __D, int const __N) 1014{ 1015 return (__m64) __builtin_ia32_vec_set_v4hi ((__v4hi)__A, __D, __N); 1016} 1017 1018extern __inline __m64 __attribute__((__gnu_inline__, __always_inline__, __artificial__)) 1019_m_pinsrw (__m64 const __A, int const __D, int const __N) 1020{ 1021 return _mm_insert_pi16 (__A, __D, __N); 1022} 1023#else 1024#define _mm_insert_pi16(A, D, N) \ 1025 ((__m64) __builtin_ia32_vec_set_v4hi ((__v4hi)(__m64)(A), \ 1026 (int)(D), (int)(N))) 1027 1028#define _m_pinsrw(A, D, N) _mm_insert_pi16(A, D, N) 1029#endif 1030 1031/* Compute the element-wise maximum of signed 16-bit values. */ 1032extern __inline __m64 __attribute__((__gnu_inline__, __always_inline__, __artificial__)) 1033_mm_max_pi16 (__m64 __A, __m64 __B) 1034{ 1035 return (__m64) __builtin_ia32_pmaxsw ((__v4hi)__A, (__v4hi)__B); 1036} 1037 1038extern __inline __m64 __attribute__((__gnu_inline__, __always_inline__, __artificial__)) 1039_m_pmaxsw (__m64 __A, __m64 __B) 1040{ 1041 return _mm_max_pi16 (__A, __B); 1042} 1043 1044/* Compute the element-wise maximum of unsigned 8-bit values. */ 1045extern __inline __m64 __attribute__((__gnu_inline__, __always_inline__, __artificial__)) 1046_mm_max_pu8 (__m64 __A, __m64 __B) 1047{ 1048 return (__m64) __builtin_ia32_pmaxub ((__v8qi)__A, (__v8qi)__B); 1049} 1050 1051extern __inline __m64 __attribute__((__gnu_inline__, __always_inline__, __artificial__)) 1052_m_pmaxub (__m64 __A, __m64 __B) 1053{ 1054 return _mm_max_pu8 (__A, __B); 1055} 1056 1057/* Compute the element-wise minimum of signed 16-bit values. */ 1058extern __inline __m64 __attribute__((__gnu_inline__, __always_inline__, __artificial__)) 1059_mm_min_pi16 (__m64 __A, __m64 __B) 1060{ 1061 return (__m64) __builtin_ia32_pminsw ((__v4hi)__A, (__v4hi)__B); 1062} 1063 1064extern __inline __m64 __attribute__((__gnu_inline__, __always_inline__, __artificial__)) 1065_m_pminsw (__m64 __A, __m64 __B) 1066{ 1067 return _mm_min_pi16 (__A, __B); 1068} 1069 1070/* Compute the element-wise minimum of unsigned 8-bit values. */ 1071extern __inline __m64 __attribute__((__gnu_inline__, __always_inline__, __artificial__)) 1072_mm_min_pu8 (__m64 __A, __m64 __B) 1073{ 1074 return (__m64) __builtin_ia32_pminub ((__v8qi)__A, (__v8qi)__B); 1075} 1076 1077extern __inline __m64 __attribute__((__gnu_inline__, __always_inline__, __artificial__)) 1078_m_pminub (__m64 __A, __m64 __B) 1079{ 1080 return _mm_min_pu8 (__A, __B); 1081} 1082 1083/* Create an 8-bit mask of the signs of 8-bit values. */ 1084extern __inline int __attribute__((__gnu_inline__, __always_inline__, __artificial__)) 1085_mm_movemask_pi8 (__m64 __A) 1086{ 1087 return __builtin_ia32_pmovmskb ((__v8qi)__A); 1088} 1089 1090extern __inline int __attribute__((__gnu_inline__, __always_inline__, __artificial__)) 1091_m_pmovmskb (__m64 __A) 1092{ 1093 return _mm_movemask_pi8 (__A); 1094} 1095 1096/* Multiply four unsigned 16-bit values in A by four unsigned 16-bit values 1097 in B and produce the high 16 bits of the 32-bit results. */ 1098extern __inline __m64 __attribute__((__gnu_inline__, __always_inline__, __artificial__)) 1099_mm_mulhi_pu16 (__m64 __A, __m64 __B) 1100{ 1101 return (__m64) __builtin_ia32_pmulhuw ((__v4hi)__A, (__v4hi)__B); 1102} 1103 1104extern __inline __m64 __attribute__((__gnu_inline__, __always_inline__, __artificial__)) 1105_m_pmulhuw (__m64 __A, __m64 __B) 1106{ 1107 return _mm_mulhi_pu16 (__A, __B); 1108} 1109 1110/* Return a combination of the four 16-bit values in A. The selector 1111 must be an immediate. */ 1112#ifdef __OPTIMIZE__ 1113extern __inline __m64 __attribute__((__gnu_inline__, __always_inline__, __artificial__)) 1114_mm_shuffle_pi16 (__m64 __A, int const __N) 1115{ 1116 return (__m64) __builtin_ia32_pshufw ((__v4hi)__A, __N); 1117} 1118 1119extern __inline __m64 __attribute__((__gnu_inline__, __always_inline__, __artificial__)) 1120_m_pshufw (__m64 __A, int const __N) 1121{ 1122 return _mm_shuffle_pi16 (__A, __N); 1123} 1124#else 1125#define _mm_shuffle_pi16(A, N) \ 1126 ((__m64) __builtin_ia32_pshufw ((__v4hi)(__m64)(A), (int)(N))) 1127 1128#define _m_pshufw(A, N) _mm_shuffle_pi16 (A, N) 1129#endif 1130 1131/* Conditionally store byte elements of A into P. The high bit of each 1132 byte in the selector N determines whether the corresponding byte from 1133 A is stored. */ 1134extern __inline void __attribute__((__gnu_inline__, __always_inline__, __artificial__)) 1135_mm_maskmove_si64 (__m64 __A, __m64 __N, char *__P) 1136{ 1137 __builtin_ia32_maskmovq ((__v8qi)__A, (__v8qi)__N, __P); 1138} 1139 1140extern __inline void __attribute__((__gnu_inline__, __always_inline__, __artificial__)) 1141_m_maskmovq (__m64 __A, __m64 __N, char *__P) 1142{ 1143 _mm_maskmove_si64 (__A, __N, __P); 1144} 1145 1146/* Compute the rounded averages of the unsigned 8-bit values in A and B. */ 1147extern __inline __m64 __attribute__((__gnu_inline__, __always_inline__, __artificial__)) 1148_mm_avg_pu8 (__m64 __A, __m64 __B) 1149{ 1150 return (__m64) __builtin_ia32_pavgb ((__v8qi)__A, (__v8qi)__B); 1151} 1152 1153extern __inline __m64 __attribute__((__gnu_inline__, __always_inline__, __artificial__)) 1154_m_pavgb (__m64 __A, __m64 __B) 1155{ 1156 return _mm_avg_pu8 (__A, __B); 1157} 1158 1159/* Compute the rounded averages of the unsigned 16-bit values in A and B. */ 1160extern __inline __m64 __attribute__((__gnu_inline__, __always_inline__, __artificial__)) 1161_mm_avg_pu16 (__m64 __A, __m64 __B) 1162{ 1163 return (__m64) __builtin_ia32_pavgw ((__v4hi)__A, (__v4hi)__B); 1164} 1165 1166extern __inline __m64 __attribute__((__gnu_inline__, __always_inline__, __artificial__)) 1167_m_pavgw (__m64 __A, __m64 __B) 1168{ 1169 return _mm_avg_pu16 (__A, __B); 1170} 1171 1172/* Compute the sum of the absolute differences of the unsigned 8-bit 1173 values in A and B. Return the value in the lower 16-bit word; the 1174 upper words are cleared. */ 1175extern __inline __m64 __attribute__((__gnu_inline__, __always_inline__, __artificial__)) 1176_mm_sad_pu8 (__m64 __A, __m64 __B) 1177{ 1178 return (__m64) __builtin_ia32_psadbw ((__v8qi)__A, (__v8qi)__B); 1179} 1180 1181extern __inline __m64 __attribute__((__gnu_inline__, __always_inline__, __artificial__)) 1182_m_psadbw (__m64 __A, __m64 __B) 1183{ 1184 return _mm_sad_pu8 (__A, __B); 1185} 1186 1187/* Loads one cache line from address P to a location "closer" to the 1188 processor. The selector I specifies the type of prefetch operation. */ 1189#ifdef __OPTIMIZE__ 1190extern __inline void __attribute__((__gnu_inline__, __always_inline__, __artificial__)) 1191_mm_prefetch (const void *__P, enum _mm_hint __I) 1192{ 1193 __builtin_prefetch (__P, 0, __I); 1194} 1195#else 1196#define _mm_prefetch(P, I) \ 1197 __builtin_prefetch ((P), 0, (I)) 1198#endif 1199 1200/* Stores the data in A to the address P without polluting the caches. */ 1201extern __inline void __attribute__((__gnu_inline__, __always_inline__, __artificial__)) 1202_mm_stream_pi (__m64 *__P, __m64 __A) 1203{ 1204 __builtin_ia32_movntq ((unsigned long long *)__P, (unsigned long long)__A); 1205} 1206 1207/* Likewise. The address must be 16-byte aligned. */ 1208extern __inline void __attribute__((__gnu_inline__, __always_inline__, __artificial__)) 1209_mm_stream_ps (float *__P, __m128 __A) 1210{ 1211 __builtin_ia32_movntps (__P, (__v4sf)__A); 1212} 1213 1214/* Guarantees that every preceding store is globally visible before 1215 any subsequent store. */ 1216extern __inline void __attribute__((__gnu_inline__, __always_inline__, __artificial__)) 1217_mm_sfence (void) 1218{ 1219 __builtin_ia32_sfence (); 1220} 1221 1222/* The execution of the next instruction is delayed by an implementation 1223 specific amount of time. The instruction does not modify the 1224 architectural state. */ 1225extern __inline void __attribute__((__gnu_inline__, __always_inline__, __artificial__)) 1226_mm_pause (void) 1227{ 1228 __asm__ __volatile__ ("rep; nop" : : ); 1229} 1230 1231/* Transpose the 4x4 matrix composed of row[0-3]. */ 1232#define _MM_TRANSPOSE4_PS(row0, row1, row2, row3) \ 1233do { \ 1234 __v4sf __r0 = (row0), __r1 = (row1), __r2 = (row2), __r3 = (row3); \ 1235 __v4sf __t0 = __builtin_ia32_unpcklps (__r0, __r1); \ 1236 __v4sf __t1 = __builtin_ia32_unpcklps (__r2, __r3); \ 1237 __v4sf __t2 = __builtin_ia32_unpckhps (__r0, __r1); \ 1238 __v4sf __t3 = __builtin_ia32_unpckhps (__r2, __r3); \ 1239 (row0) = __builtin_ia32_movlhps (__t0, __t1); \ 1240 (row1) = __builtin_ia32_movhlps (__t1, __t0); \ 1241 (row2) = __builtin_ia32_movlhps (__t2, __t3); \ 1242 (row3) = __builtin_ia32_movhlps (__t3, __t2); \ 1243} while (0) 1244 1245/* For backward source compatibility. */ 1246#ifdef __SSE2__ 1247# include <emmintrin.h> 1248#endif 1249 1250#endif /* __SSE__ */ 1251#endif /* _XMMINTRIN_H_INCLUDED */ 1252