17e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto//===- subzero/src/IceTargetLoweringX86BaseImpl.h - x86 lowering -*- C++ -*-==// 27e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto// 37e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto// The Subzero Code Generator 47e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto// 57e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto// This file is distributed under the University of Illinois Open Source 67e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto// License. See LICENSE.TXT for details. 77e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto// 87e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto//===----------------------------------------------------------------------===// 99612d32c7e5eb2cb403686ef31172d42e075e460Andrew Scull/// 109612d32c7e5eb2cb403686ef31172d42e075e460Andrew Scull/// \file 1192a6e5b08ec68e7076d637ebc680da2fcc346a00Jim Stichnoth/// \brief Implements the TargetLoweringX86Base class, which consists almost 1257e126899b20c65ff3ea23a3b7d7a67ab30b99dcAndrew Scull/// entirely of the lowering sequence for each high-level instruction. 139612d32c7e5eb2cb403686ef31172d42e075e460Andrew Scull/// 147e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto//===----------------------------------------------------------------------===// 157e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto 167e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto#ifndef SUBZERO_SRC_ICETARGETLOWERINGX86BASEIMPL_H 177e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto#define SUBZERO_SRC_ICETARGETLOWERINGX86BASEIMPL_H 187e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto 197e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto#include "IceCfg.h" 207e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto#include "IceCfgNode.h" 217e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto#include "IceClFlags.h" 227e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto#include "IceDefs.h" 237e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto#include "IceELFObjectWriter.h" 247e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto#include "IceGlobalInits.h" 25ec3f56532be1792d04ed470221df663bb8ca9c19John Porto#include "IceInstVarIter.h" 26b9a847280e4486e566dabdd5b0d571309b4ad628Jim Stichnoth#include "IceInstX86Base.h" 277e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto#include "IceLiveness.h" 287e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto#include "IceOperand.h" 2953483691eba6e23de63afe0579b436002d06d187Jan Voung#include "IcePhiLoweringImpl.h" 307e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto#include "IceUtils.h" 31b9a847280e4486e566dabdd5b0d571309b4ad628Jim Stichnoth#include "IceVariableSplitting.h" 32b9a847280e4486e566dabdd5b0d571309b4ad628Jim Stichnoth 3367f8de9adf6439881a00d8e0f081918436c71f62John Porto#include "llvm/Support/MathExtras.h" 347e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto 3587f80c128a9e2bc95829cd1aac2bbc3c44b44ac1Andrew Scull#include <stack> 3687f80c128a9e2bc95829cd1aac2bbc3c44b44ac1Andrew Scull 377e93c62d7e223b7fd9e6e0889e4b70b635589282John Portonamespace Ice { 386b80cf109e33aaf58de116b63a0562227d096429David Sehrnamespace X86 { 396b80cf109e33aaf58de116b63a0562227d096429David Sehrtemplate <typename T> struct PoolTypeConverter {}; 406b80cf109e33aaf58de116b63a0562227d096429David Sehr 416b80cf109e33aaf58de116b63a0562227d096429David Sehrtemplate <> struct PoolTypeConverter<float> { 426b80cf109e33aaf58de116b63a0562227d096429David Sehr using PrimitiveIntType = uint32_t; 436b80cf109e33aaf58de116b63a0562227d096429David Sehr using IceType = ConstantFloat; 446b80cf109e33aaf58de116b63a0562227d096429David Sehr static const Type Ty = IceType_f32; 456b80cf109e33aaf58de116b63a0562227d096429David Sehr static const char *TypeName; 466b80cf109e33aaf58de116b63a0562227d096429David Sehr static const char *AsmTag; 476b80cf109e33aaf58de116b63a0562227d096429David Sehr static const char *PrintfString; 486b80cf109e33aaf58de116b63a0562227d096429David Sehr}; 496b80cf109e33aaf58de116b63a0562227d096429David Sehr 506b80cf109e33aaf58de116b63a0562227d096429David Sehrtemplate <> struct PoolTypeConverter<double> { 516b80cf109e33aaf58de116b63a0562227d096429David Sehr using PrimitiveIntType = uint64_t; 526b80cf109e33aaf58de116b63a0562227d096429David Sehr using IceType = ConstantDouble; 536b80cf109e33aaf58de116b63a0562227d096429David Sehr static const Type Ty = IceType_f64; 546b80cf109e33aaf58de116b63a0562227d096429David Sehr static const char *TypeName; 556b80cf109e33aaf58de116b63a0562227d096429David Sehr static const char *AsmTag; 566b80cf109e33aaf58de116b63a0562227d096429David Sehr static const char *PrintfString; 576b80cf109e33aaf58de116b63a0562227d096429David Sehr}; 586b80cf109e33aaf58de116b63a0562227d096429David Sehr 596b80cf109e33aaf58de116b63a0562227d096429David Sehr// Add converter for int type constant pooling 606b80cf109e33aaf58de116b63a0562227d096429David Sehrtemplate <> struct PoolTypeConverter<uint32_t> { 616b80cf109e33aaf58de116b63a0562227d096429David Sehr using PrimitiveIntType = uint32_t; 626b80cf109e33aaf58de116b63a0562227d096429David Sehr using IceType = ConstantInteger32; 636b80cf109e33aaf58de116b63a0562227d096429David Sehr static const Type Ty = IceType_i32; 646b80cf109e33aaf58de116b63a0562227d096429David Sehr static const char *TypeName; 656b80cf109e33aaf58de116b63a0562227d096429David Sehr static const char *AsmTag; 666b80cf109e33aaf58de116b63a0562227d096429David Sehr static const char *PrintfString; 676b80cf109e33aaf58de116b63a0562227d096429David Sehr}; 686b80cf109e33aaf58de116b63a0562227d096429David Sehr 696b80cf109e33aaf58de116b63a0562227d096429David Sehr// Add converter for int type constant pooling 706b80cf109e33aaf58de116b63a0562227d096429David Sehrtemplate <> struct PoolTypeConverter<uint16_t> { 716b80cf109e33aaf58de116b63a0562227d096429David Sehr using PrimitiveIntType = uint32_t; 726b80cf109e33aaf58de116b63a0562227d096429David Sehr using IceType = ConstantInteger32; 736b80cf109e33aaf58de116b63a0562227d096429David Sehr static const Type Ty = IceType_i16; 746b80cf109e33aaf58de116b63a0562227d096429David Sehr static const char *TypeName; 756b80cf109e33aaf58de116b63a0562227d096429David Sehr static const char *AsmTag; 766b80cf109e33aaf58de116b63a0562227d096429David Sehr static const char *PrintfString; 776b80cf109e33aaf58de116b63a0562227d096429David Sehr}; 786b80cf109e33aaf58de116b63a0562227d096429David Sehr 796b80cf109e33aaf58de116b63a0562227d096429David Sehr// Add converter for int type constant pooling 806b80cf109e33aaf58de116b63a0562227d096429David Sehrtemplate <> struct PoolTypeConverter<uint8_t> { 816b80cf109e33aaf58de116b63a0562227d096429David Sehr using PrimitiveIntType = uint32_t; 826b80cf109e33aaf58de116b63a0562227d096429David Sehr using IceType = ConstantInteger32; 836b80cf109e33aaf58de116b63a0562227d096429David Sehr static const Type Ty = IceType_i8; 846b80cf109e33aaf58de116b63a0562227d096429David Sehr static const char *TypeName; 856b80cf109e33aaf58de116b63a0562227d096429David Sehr static const char *AsmTag; 866b80cf109e33aaf58de116b63a0562227d096429David Sehr static const char *PrintfString; 876b80cf109e33aaf58de116b63a0562227d096429David Sehr}; 886b80cf109e33aaf58de116b63a0562227d096429David Sehr} // end of namespace X86 896b80cf109e33aaf58de116b63a0562227d096429David Sehr 904a56686b5b56db6803f90ad53514bf2fa190d9f7John Portonamespace X86NAMESPACE { 917e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto 92d6cf6b3890ad64a92c9688d35ca7dd29dc7255acEric Holkusing Utils::BoolFlagSaver; 937e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto 944a56686b5b56db6803f90ad53514bf2fa190d9f7John Portotemplate <typename Traits> class BoolFoldingEntry { 957e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto BoolFoldingEntry(const BoolFoldingEntry &) = delete; 967e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto 977e93c62d7e223b7fd9e6e0889e4b70b635589282John Portopublic: 987e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto BoolFoldingEntry() = default; 997e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto explicit BoolFoldingEntry(Inst *I); 1007e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto BoolFoldingEntry &operator=(const BoolFoldingEntry &) = default; 1019612d32c7e5eb2cb403686ef31172d42e075e460Andrew Scull /// Instr is the instruction producing the i1-type variable of interest. 1027e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto Inst *Instr = nullptr; 1039612d32c7e5eb2cb403686ef31172d42e075e460Andrew Scull /// IsComplex is the cached result of BoolFolding::hasComplexLowering(Instr). 1047e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto bool IsComplex = false; 1059612d32c7e5eb2cb403686ef31172d42e075e460Andrew Scull /// IsLiveOut is initialized conservatively to true, and is set to false when 10657e126899b20c65ff3ea23a3b7d7a67ab30b99dcAndrew Scull /// we encounter an instruction that ends Var's live range. We disable the 10757e126899b20c65ff3ea23a3b7d7a67ab30b99dcAndrew Scull /// folding optimization when Var is live beyond this basic block. Note that 1089612d32c7e5eb2cb403686ef31172d42e075e460Andrew Scull /// if liveness analysis is not performed (e.g. in Om1 mode), IsLiveOut will 1099612d32c7e5eb2cb403686ef31172d42e075e460Andrew Scull /// always be true and the folding optimization will never be performed. 1107e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto bool IsLiveOut = true; 1117e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto // NumUses counts the number of times Var is used as a source operand in the 11257e126899b20c65ff3ea23a3b7d7a67ab30b99dcAndrew Scull // basic block. If IsComplex is true and there is more than one use of Var, 1137e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto // then the folding optimization is disabled for Var. 1147e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto uint32_t NumUses = 0; 1157e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto}; 1167e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto 1174a56686b5b56db6803f90ad53514bf2fa190d9f7John Portotemplate <typename Traits> class BoolFolding { 1187e93c62d7e223b7fd9e6e0889e4b70b635589282John Portopublic: 1197e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto enum BoolFoldingProducerKind { 1207e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto PK_None, 1211d235425dab1f3dd059973fc53f1b1d5879469e3John Porto // TODO(jpp): PK_Icmp32 is no longer meaningful. Rename to PK_IcmpNative. 1227e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto PK_Icmp32, 1237e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto PK_Icmp64, 1247e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto PK_Fcmp, 125daf096cd1b8b9ec3d29c8d7823c3d60cd51c65c9David Sehr PK_Trunc, 1267b3d9cbb1c1697e03c616162382d945f35c37347John Porto PK_Arith // A flag-setting arithmetic instruction. 1277e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto }; 1287e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto 1299612d32c7e5eb2cb403686ef31172d42e075e460Andrew Scull /// Currently the actual enum values are not used (other than CK_None), but we 130921856d4e4bd4f0deedc7324d5f6a4ca0be3439fJohn Porto /// go ahead and produce them anyway for symmetry with the 1319612d32c7e5eb2cb403686ef31172d42e075e460Andrew Scull /// BoolFoldingProducerKind. 1327e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto enum BoolFoldingConsumerKind { CK_None, CK_Br, CK_Select, CK_Sext, CK_Zext }; 1337e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto 1347e93c62d7e223b7fd9e6e0889e4b70b635589282John Portoprivate: 1357e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto BoolFolding(const BoolFolding &) = delete; 1367e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto BoolFolding &operator=(const BoolFolding &) = delete; 1377e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto 1387e93c62d7e223b7fd9e6e0889e4b70b635589282John Portopublic: 1397e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto BoolFolding() = default; 1407e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto static BoolFoldingProducerKind getProducerKind(const Inst *Instr); 1417e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto static BoolFoldingConsumerKind getConsumerKind(const Inst *Instr); 1427e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto static bool hasComplexLowering(const Inst *Instr); 143e398428c3ef8ed2339c0f334dd30a7d3f0ed4b6eDavid Sehr static bool isValidFolding(BoolFoldingProducerKind ProducerKind, 144e398428c3ef8ed2339c0f334dd30a7d3f0ed4b6eDavid Sehr BoolFoldingConsumerKind ConsumerKind); 1457e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto void init(CfgNode *Node); 1467e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto const Inst *getProducerFor(const Operand *Opnd) const; 1477e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto void dump(const Cfg *Func) const; 1487e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto 1497e93c62d7e223b7fd9e6e0889e4b70b635589282John Portoprivate: 1509612d32c7e5eb2cb403686ef31172d42e075e460Andrew Scull /// Returns true if Producers contains a valid entry for the given VarNum. 1517e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto bool containsValid(SizeT VarNum) const { 1527e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto auto Element = Producers.find(VarNum); 1537e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto return Element != Producers.end() && Element->second.Instr != nullptr; 1547e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto } 1557e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto void setInvalid(SizeT VarNum) { Producers[VarNum].Instr = nullptr; } 156f1f773dd2c9c7118e07caa61d580bdba4447c25cJim Stichnoth void invalidateProducersOnStore(const Inst *Instr); 1579612d32c7e5eb2cb403686ef31172d42e075e460Andrew Scull /// Producers maps Variable::Number to a BoolFoldingEntry. 158e82b560e649f8a68bcb252b9b002708e74d962d3John Porto CfgUnorderedMap<SizeT, BoolFoldingEntry<Traits>> Producers; 1597e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto}; 1607e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto 1614a56686b5b56db6803f90ad53514bf2fa190d9f7John Portotemplate <typename Traits> 1624a56686b5b56db6803f90ad53514bf2fa190d9f7John PortoBoolFoldingEntry<Traits>::BoolFoldingEntry(Inst *I) 1634a56686b5b56db6803f90ad53514bf2fa190d9f7John Porto : Instr(I), IsComplex(BoolFolding<Traits>::hasComplexLowering(I)) {} 1647e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto 1654a56686b5b56db6803f90ad53514bf2fa190d9f7John Portotemplate <typename Traits> 1664a56686b5b56db6803f90ad53514bf2fa190d9f7John Portotypename BoolFolding<Traits>::BoolFoldingProducerKind 1674a56686b5b56db6803f90ad53514bf2fa190d9f7John PortoBoolFolding<Traits>::getProducerKind(const Inst *Instr) { 1687e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto if (llvm::isa<InstIcmp>(Instr)) { 1694a56686b5b56db6803f90ad53514bf2fa190d9f7John Porto if (Traits::Is64Bit || Instr->getSrc(0)->getType() != IceType_i64) 1707e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto return PK_Icmp32; 171d981025a26d962d50f7fead809b44d16be9051f3David Sehr return PK_Icmp64; 1727e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto } 1737e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto if (llvm::isa<InstFcmp>(Instr)) 1747e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto return PK_Fcmp; 175daf096cd1b8b9ec3d29c8d7823c3d60cd51c65c9David Sehr if (auto *Arith = llvm::dyn_cast<InstArithmetic>(Instr)) { 1764a56686b5b56db6803f90ad53514bf2fa190d9f7John Porto if (Traits::Is64Bit || Arith->getSrc(0)->getType() != IceType_i64) { 177daf096cd1b8b9ec3d29c8d7823c3d60cd51c65c9David Sehr switch (Arith->getOp()) { 178daf096cd1b8b9ec3d29c8d7823c3d60cd51c65c9David Sehr default: 179daf096cd1b8b9ec3d29c8d7823c3d60cd51c65c9David Sehr return PK_None; 180daf096cd1b8b9ec3d29c8d7823c3d60cd51c65c9David Sehr case InstArithmetic::And: 181daf096cd1b8b9ec3d29c8d7823c3d60cd51c65c9David Sehr case InstArithmetic::Or: 182daf096cd1b8b9ec3d29c8d7823c3d60cd51c65c9David Sehr return PK_Arith; 183daf096cd1b8b9ec3d29c8d7823c3d60cd51c65c9David Sehr } 184daf096cd1b8b9ec3d29c8d7823c3d60cd51c65c9David Sehr } 185daf096cd1b8b9ec3d29c8d7823c3d60cd51c65c9David Sehr } 186daf096cd1b8b9ec3d29c8d7823c3d60cd51c65c9David Sehr return PK_None; // TODO(stichnot): remove this 187daf096cd1b8b9ec3d29c8d7823c3d60cd51c65c9David Sehr 1887e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto if (auto *Cast = llvm::dyn_cast<InstCast>(Instr)) { 1897e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto switch (Cast->getCastKind()) { 1907e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto default: 1917e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto return PK_None; 1927e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto case InstCast::Trunc: 1937e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto return PK_Trunc; 1947e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto } 1957e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto } 1967e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto return PK_None; 1977e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto} 1987e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto 1994a56686b5b56db6803f90ad53514bf2fa190d9f7John Portotemplate <typename Traits> 2004a56686b5b56db6803f90ad53514bf2fa190d9f7John Portotypename BoolFolding<Traits>::BoolFoldingConsumerKind 2014a56686b5b56db6803f90ad53514bf2fa190d9f7John PortoBoolFolding<Traits>::getConsumerKind(const Inst *Instr) { 2027e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto if (llvm::isa<InstBr>(Instr)) 2037e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto return CK_Br; 2047e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto if (llvm::isa<InstSelect>(Instr)) 2057e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto return CK_Select; 2067e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto return CK_None; // TODO(stichnot): remove this 2077e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto 2087e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto if (auto *Cast = llvm::dyn_cast<InstCast>(Instr)) { 2097e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto switch (Cast->getCastKind()) { 2107e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto default: 2117e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto return CK_None; 2127e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto case InstCast::Sext: 2137e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto return CK_Sext; 2147e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto case InstCast::Zext: 2157e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto return CK_Zext; 2167e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto } 2177e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto } 2187e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto return CK_None; 2197e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto} 2207e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto 221921856d4e4bd4f0deedc7324d5f6a4ca0be3439fJohn Porto/// Returns true if the producing instruction has a "complex" lowering sequence. 222921856d4e4bd4f0deedc7324d5f6a4ca0be3439fJohn Porto/// This generally means that its lowering sequence requires more than one 223921856d4e4bd4f0deedc7324d5f6a4ca0be3439fJohn Porto/// conditional branch, namely 64-bit integer compares and some floating-point 22457e126899b20c65ff3ea23a3b7d7a67ab30b99dcAndrew Scull/// compares. When this is true, and there is more than one consumer, we prefer 225921856d4e4bd4f0deedc7324d5f6a4ca0be3439fJohn Porto/// to disable the folding optimization because it minimizes branches. 2264a56686b5b56db6803f90ad53514bf2fa190d9f7John Portotemplate <typename Traits> 2274a56686b5b56db6803f90ad53514bf2fa190d9f7John Portobool BoolFolding<Traits>::hasComplexLowering(const Inst *Instr) { 2287e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto switch (getProducerKind(Instr)) { 2297e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto default: 2307e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto return false; 2317e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto case PK_Icmp64: 2324a56686b5b56db6803f90ad53514bf2fa190d9f7John Porto return !Traits::Is64Bit; 2337e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto case PK_Fcmp: 2344a56686b5b56db6803f90ad53514bf2fa190d9f7John Porto return Traits::TableFcmp[llvm::cast<InstFcmp>(Instr)->getCondition()].C2 != 2354a56686b5b56db6803f90ad53514bf2fa190d9f7John Porto Traits::Cond::Br_None; 2367e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto } 2377e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto} 2387e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto 2394a56686b5b56db6803f90ad53514bf2fa190d9f7John Portotemplate <typename Traits> 2404a56686b5b56db6803f90ad53514bf2fa190d9f7John Portobool BoolFolding<Traits>::isValidFolding( 2414a56686b5b56db6803f90ad53514bf2fa190d9f7John Porto typename BoolFolding<Traits>::BoolFoldingProducerKind ProducerKind, 2424a56686b5b56db6803f90ad53514bf2fa190d9f7John Porto typename BoolFolding<Traits>::BoolFoldingConsumerKind ConsumerKind) { 243e398428c3ef8ed2339c0f334dd30a7d3f0ed4b6eDavid Sehr switch (ProducerKind) { 244e398428c3ef8ed2339c0f334dd30a7d3f0ed4b6eDavid Sehr default: 245e398428c3ef8ed2339c0f334dd30a7d3f0ed4b6eDavid Sehr return false; 246e398428c3ef8ed2339c0f334dd30a7d3f0ed4b6eDavid Sehr case PK_Icmp32: 247e398428c3ef8ed2339c0f334dd30a7d3f0ed4b6eDavid Sehr case PK_Icmp64: 248e398428c3ef8ed2339c0f334dd30a7d3f0ed4b6eDavid Sehr case PK_Fcmp: 249e398428c3ef8ed2339c0f334dd30a7d3f0ed4b6eDavid Sehr return (ConsumerKind == CK_Br) || (ConsumerKind == CK_Select); 250e398428c3ef8ed2339c0f334dd30a7d3f0ed4b6eDavid Sehr case PK_Arith: 251e398428c3ef8ed2339c0f334dd30a7d3f0ed4b6eDavid Sehr return ConsumerKind == CK_Br; 252e398428c3ef8ed2339c0f334dd30a7d3f0ed4b6eDavid Sehr } 253e398428c3ef8ed2339c0f334dd30a7d3f0ed4b6eDavid Sehr} 254e398428c3ef8ed2339c0f334dd30a7d3f0ed4b6eDavid Sehr 2554a56686b5b56db6803f90ad53514bf2fa190d9f7John Portotemplate <typename Traits> void BoolFolding<Traits>::init(CfgNode *Node) { 2567e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto Producers.clear(); 2577e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto for (Inst &Instr : Node->getInsts()) { 258f1f773dd2c9c7118e07caa61d580bdba4447c25cJim Stichnoth if (Instr.isDeleted()) 259f1f773dd2c9c7118e07caa61d580bdba4447c25cJim Stichnoth continue; 260f1f773dd2c9c7118e07caa61d580bdba4447c25cJim Stichnoth invalidateProducersOnStore(&Instr); 2617e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto // Check whether Instr is a valid producer. 2627e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto Variable *Var = Instr.getDest(); 26347b6ba6db7c8b8dd36ac54dbd19ae7b4a5b77424Nicolas Capens if (Var) { // only consider instructions with an actual dest var 26447b6ba6db7c8b8dd36ac54dbd19ae7b4a5b77424Nicolas Capens if (isBooleanType(Var->getType())) { // only bool-type dest vars 26547b6ba6db7c8b8dd36ac54dbd19ae7b4a5b77424Nicolas Capens if (getProducerKind(&Instr) != PK_None) { // white-listed instructions 26647b6ba6db7c8b8dd36ac54dbd19ae7b4a5b77424Nicolas Capens Producers[Var->getIndex()] = BoolFoldingEntry<Traits>(&Instr); 26747b6ba6db7c8b8dd36ac54dbd19ae7b4a5b77424Nicolas Capens } 26847b6ba6db7c8b8dd36ac54dbd19ae7b4a5b77424Nicolas Capens } 2697e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto } 2707e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto // Check each src variable against the map. 271ec3f56532be1792d04ed470221df663bb8ca9c19John Porto FOREACH_VAR_IN_INST(Var, Instr) { 272ec3f56532be1792d04ed470221df663bb8ca9c19John Porto SizeT VarNum = Var->getIndex(); 273e398428c3ef8ed2339c0f334dd30a7d3f0ed4b6eDavid Sehr if (!containsValid(VarNum)) 274e398428c3ef8ed2339c0f334dd30a7d3f0ed4b6eDavid Sehr continue; 275e398428c3ef8ed2339c0f334dd30a7d3f0ed4b6eDavid Sehr // All valid consumers use Var as the first source operand 276e398428c3ef8ed2339c0f334dd30a7d3f0ed4b6eDavid Sehr if (IndexOfVarOperandInInst(Var) != 0) { 277e398428c3ef8ed2339c0f334dd30a7d3f0ed4b6eDavid Sehr setInvalid(VarNum); 278e398428c3ef8ed2339c0f334dd30a7d3f0ed4b6eDavid Sehr continue; 279e398428c3ef8ed2339c0f334dd30a7d3f0ed4b6eDavid Sehr } 280e398428c3ef8ed2339c0f334dd30a7d3f0ed4b6eDavid Sehr // Consumer instructions must be white-listed 2814a56686b5b56db6803f90ad53514bf2fa190d9f7John Porto typename BoolFolding<Traits>::BoolFoldingConsumerKind ConsumerKind = 2824a56686b5b56db6803f90ad53514bf2fa190d9f7John Porto getConsumerKind(&Instr); 283e398428c3ef8ed2339c0f334dd30a7d3f0ed4b6eDavid Sehr if (ConsumerKind == CK_None) { 284e398428c3ef8ed2339c0f334dd30a7d3f0ed4b6eDavid Sehr setInvalid(VarNum); 285e398428c3ef8ed2339c0f334dd30a7d3f0ed4b6eDavid Sehr continue; 286e398428c3ef8ed2339c0f334dd30a7d3f0ed4b6eDavid Sehr } 2874a56686b5b56db6803f90ad53514bf2fa190d9f7John Porto typename BoolFolding<Traits>::BoolFoldingProducerKind ProducerKind = 2884a56686b5b56db6803f90ad53514bf2fa190d9f7John Porto getProducerKind(Producers[VarNum].Instr); 289e398428c3ef8ed2339c0f334dd30a7d3f0ed4b6eDavid Sehr if (!isValidFolding(ProducerKind, ConsumerKind)) { 290e398428c3ef8ed2339c0f334dd30a7d3f0ed4b6eDavid Sehr setInvalid(VarNum); 291e398428c3ef8ed2339c0f334dd30a7d3f0ed4b6eDavid Sehr continue; 292e398428c3ef8ed2339c0f334dd30a7d3f0ed4b6eDavid Sehr } 293e398428c3ef8ed2339c0f334dd30a7d3f0ed4b6eDavid Sehr // Avoid creating multiple copies of complex producer instructions. 294e398428c3ef8ed2339c0f334dd30a7d3f0ed4b6eDavid Sehr if (Producers[VarNum].IsComplex && Producers[VarNum].NumUses > 0) { 295e398428c3ef8ed2339c0f334dd30a7d3f0ed4b6eDavid Sehr setInvalid(VarNum); 296e398428c3ef8ed2339c0f334dd30a7d3f0ed4b6eDavid Sehr continue; 297e398428c3ef8ed2339c0f334dd30a7d3f0ed4b6eDavid Sehr } 298e398428c3ef8ed2339c0f334dd30a7d3f0ed4b6eDavid Sehr ++Producers[VarNum].NumUses; 299e398428c3ef8ed2339c0f334dd30a7d3f0ed4b6eDavid Sehr if (Instr.isLastUse(Var)) { 300e398428c3ef8ed2339c0f334dd30a7d3f0ed4b6eDavid Sehr Producers[VarNum].IsLiveOut = false; 3017e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto } 3027e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto } 3037e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto } 3047e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto for (auto &I : Producers) { 3057e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto // Ignore entries previously marked invalid. 3067e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto if (I.second.Instr == nullptr) 3077e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto continue; 3087e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto // Disable the producer if its dest may be live beyond this block. 3097e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto if (I.second.IsLiveOut) { 3107e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto setInvalid(I.first); 3117e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto continue; 3127e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto } 31357e126899b20c65ff3ea23a3b7d7a67ab30b99dcAndrew Scull // Mark as "dead" rather than outright deleting. This is so that other 314921856d4e4bd4f0deedc7324d5f6a4ca0be3439fJohn Porto // peephole style optimizations during or before lowering have access to 31557e126899b20c65ff3ea23a3b7d7a67ab30b99dcAndrew Scull // this instruction in undeleted form. See for example 316921856d4e4bd4f0deedc7324d5f6a4ca0be3439fJohn Porto // tryOptimizedCmpxchgCmpBr(). 3177e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto I.second.Instr->setDead(); 3187e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto } 3197e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto} 3207e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto 3214a56686b5b56db6803f90ad53514bf2fa190d9f7John Portotemplate <typename Traits> 3224a56686b5b56db6803f90ad53514bf2fa190d9f7John Portoconst Inst *BoolFolding<Traits>::getProducerFor(const Operand *Opnd) const { 3237e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto auto *Var = llvm::dyn_cast<const Variable>(Opnd); 3247e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto if (Var == nullptr) 3257e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto return nullptr; 3267e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto SizeT VarNum = Var->getIndex(); 3277e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto auto Element = Producers.find(VarNum); 3287e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto if (Element == Producers.end()) 3297e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto return nullptr; 3307e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto return Element->second.Instr; 3317e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto} 3327e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto 3334a56686b5b56db6803f90ad53514bf2fa190d9f7John Portotemplate <typename Traits> 3344a56686b5b56db6803f90ad53514bf2fa190d9f7John Portovoid BoolFolding<Traits>::dump(const Cfg *Func) const { 33520b71f5890ee8651983b126c5978594a01e0af96Jim Stichnoth if (!BuildDefs::dump() || !Func->isVerbose(IceV_Folding)) 3367e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto return; 3377e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto OstreamLocker L(Func->getContext()); 3387e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto Ostream &Str = Func->getContext()->getStrDump(); 3397e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto for (auto &I : Producers) { 3407e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto if (I.second.Instr == nullptr) 3417e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto continue; 3427e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto Str << "Found foldable producer:\n "; 3437e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto I.second.Instr->dump(Func); 3447e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto Str << "\n"; 3457e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto } 3467e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto} 3477e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto 348f1f773dd2c9c7118e07caa61d580bdba4447c25cJim Stichnoth/// If the given instruction has potential memory side effects (e.g. store, rmw, 349f1f773dd2c9c7118e07caa61d580bdba4447c25cJim Stichnoth/// or a call instruction with potential memory side effects), then we must not 350f1f773dd2c9c7118e07caa61d580bdba4447c25cJim Stichnoth/// allow a pre-store Producer instruction with memory operands to be folded 351f1f773dd2c9c7118e07caa61d580bdba4447c25cJim Stichnoth/// into a post-store Consumer instruction. If this is detected, the Producer 352f1f773dd2c9c7118e07caa61d580bdba4447c25cJim Stichnoth/// is invalidated. 353f1f773dd2c9c7118e07caa61d580bdba4447c25cJim Stichnoth/// 354f1f773dd2c9c7118e07caa61d580bdba4447c25cJim Stichnoth/// We use the Producer's IsLiveOut field to determine whether any potential 355f1f773dd2c9c7118e07caa61d580bdba4447c25cJim Stichnoth/// Consumers come after this store instruction. The IsLiveOut field is 356f1f773dd2c9c7118e07caa61d580bdba4447c25cJim Stichnoth/// initialized to true, and BoolFolding::init() sets IsLiveOut to false when it 357f1f773dd2c9c7118e07caa61d580bdba4447c25cJim Stichnoth/// sees the variable's definitive last use (indicating the variable is not in 358f1f773dd2c9c7118e07caa61d580bdba4447c25cJim Stichnoth/// the node's live-out set). Thus if we see here that IsLiveOut is false, we 359f1f773dd2c9c7118e07caa61d580bdba4447c25cJim Stichnoth/// know that there can be no consumers after the store, and therefore we know 360f1f773dd2c9c7118e07caa61d580bdba4447c25cJim Stichnoth/// the folding is safe despite the store instruction. 361f1f773dd2c9c7118e07caa61d580bdba4447c25cJim Stichnothtemplate <typename Traits> 362f1f773dd2c9c7118e07caa61d580bdba4447c25cJim Stichnothvoid BoolFolding<Traits>::invalidateProducersOnStore(const Inst *Instr) { 363f1f773dd2c9c7118e07caa61d580bdba4447c25cJim Stichnoth if (!Instr->isMemoryWrite()) 364f1f773dd2c9c7118e07caa61d580bdba4447c25cJim Stichnoth return; 365f1f773dd2c9c7118e07caa61d580bdba4447c25cJim Stichnoth for (auto &ProducerPair : Producers) { 366f1f773dd2c9c7118e07caa61d580bdba4447c25cJim Stichnoth if (!ProducerPair.second.IsLiveOut) 367f1f773dd2c9c7118e07caa61d580bdba4447c25cJim Stichnoth continue; 368f1f773dd2c9c7118e07caa61d580bdba4447c25cJim Stichnoth Inst *PInst = ProducerPair.second.Instr; 369f1f773dd2c9c7118e07caa61d580bdba4447c25cJim Stichnoth if (PInst == nullptr) 370f1f773dd2c9c7118e07caa61d580bdba4447c25cJim Stichnoth continue; 371f1f773dd2c9c7118e07caa61d580bdba4447c25cJim Stichnoth bool HasMemOperand = false; 372f1f773dd2c9c7118e07caa61d580bdba4447c25cJim Stichnoth const SizeT SrcSize = PInst->getSrcSize(); 373f1f773dd2c9c7118e07caa61d580bdba4447c25cJim Stichnoth for (SizeT I = 0; I < SrcSize; ++I) { 374f1f773dd2c9c7118e07caa61d580bdba4447c25cJim Stichnoth if (llvm::isa<typename Traits::X86OperandMem>(PInst->getSrc(I))) { 375f1f773dd2c9c7118e07caa61d580bdba4447c25cJim Stichnoth HasMemOperand = true; 376f1f773dd2c9c7118e07caa61d580bdba4447c25cJim Stichnoth break; 377f1f773dd2c9c7118e07caa61d580bdba4447c25cJim Stichnoth } 378f1f773dd2c9c7118e07caa61d580bdba4447c25cJim Stichnoth } 379f1f773dd2c9c7118e07caa61d580bdba4447c25cJim Stichnoth if (!HasMemOperand) 380f1f773dd2c9c7118e07caa61d580bdba4447c25cJim Stichnoth continue; 381f1f773dd2c9c7118e07caa61d580bdba4447c25cJim Stichnoth setInvalid(ProducerPair.first); 382f1f773dd2c9c7118e07caa61d580bdba4447c25cJim Stichnoth } 383f1f773dd2c9c7118e07caa61d580bdba4447c25cJim Stichnoth} 384f1f773dd2c9c7118e07caa61d580bdba4447c25cJim Stichnoth 3854a56686b5b56db6803f90ad53514bf2fa190d9f7John Portotemplate <typename TraitsType> 3864a56686b5b56db6803f90ad53514bf2fa190d9f7John Portovoid TargetX86Base<TraitsType>::initNodeForLowering(CfgNode *Node) { 3877e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto FoldingInfo.init(Node); 3887e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto FoldingInfo.dump(Func); 3897e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto} 3907e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto 3914a56686b5b56db6803f90ad53514bf2fa190d9f7John Portotemplate <typename TraitsType> 3924a56686b5b56db6803f90ad53514bf2fa190d9f7John PortoTargetX86Base<TraitsType>::TargetX86Base(Cfg *Func) 393ac2388c385afda17acb3fc242b17b9232ec2262aJohn Porto : TargetLowering(Func), NeedSandboxing(SandboxingType == ST_NaCl) { 3947e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto static_assert( 3957e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto (Traits::InstructionSet::End - Traits::InstructionSet::Begin) == 3967e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto (TargetInstructionSet::X86InstructionSet_End - 3977e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto TargetInstructionSet::X86InstructionSet_Begin), 3987e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto "Traits::InstructionSet range different from TargetInstructionSet"); 399d46999474d2b4a388e1d8a7c71f06cd4cec51bfcKarl Schimpf if (getFlags().getTargetInstructionSet() != 4007e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto TargetInstructionSet::BaseInstructionSet) { 4014a56686b5b56db6803f90ad53514bf2fa190d9f7John Porto InstructionSet = static_cast<InstructionSetEnum>( 402d46999474d2b4a388e1d8a7c71f06cd4cec51bfcKarl Schimpf (getFlags().getTargetInstructionSet() - 4037e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto TargetInstructionSet::X86InstructionSet_Begin) + 4047e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto Traits::InstructionSet::Begin); 4057e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto } 40694844f1290d7fae68ad4f9e9b9e3543639e44a15Jim Stichnoth} 40794844f1290d7fae68ad4f9e9b9e3543639e44a15Jim Stichnoth 4088ff4b2819944bc4f02fb29204a1fa5ba7dea5682Jim Stichnothtemplate <typename TraitsType> 4095403f5dc6b43f077bbbee369bbe178b660654366Karl Schimpfvoid TargetX86Base<TraitsType>::staticInit(GlobalContext *Ctx) { 4108aa396610b7baf728631a43ea16ad3d13e38397aJim Stichnoth RegNumT::setLimit(Traits::RegisterSet::Reg_NUM); 411d46999474d2b4a388e1d8a7c71f06cd4cec51bfcKarl Schimpf Traits::initRegisterSet(getFlags(), &TypeToRegisterSet, &RegisterAliases); 412b40595a17b83cca5d11f8d056a4ac5a4d8102a84Jim Stichnoth for (size_t i = 0; i < TypeToRegisterSet.size(); ++i) 413b40595a17b83cca5d11f8d056a4ac5a4d8102a84Jim Stichnoth TypeToRegisterSetUnfiltered[i] = TypeToRegisterSet[i]; 4145403f5dc6b43f077bbbee369bbe178b660654366Karl Schimpf filterTypeToRegisterSet(Ctx, Traits::RegisterSet::Reg_NUM, 4155403f5dc6b43f077bbbee369bbe178b660654366Karl Schimpf TypeToRegisterSet.data(), TypeToRegisterSet.size(), 4162544d4d2ef7e340664dc0748a5de966f0ee8ef08Jim Stichnoth Traits::getRegName, getRegClassName); 4178ff4b2819944bc4f02fb29204a1fa5ba7dea5682Jim Stichnoth PcRelFixup = Traits::FK_PcRel; 418d46999474d2b4a388e1d8a7c71f06cd4cec51bfcKarl Schimpf AbsFixup = getFlags().getUseNonsfi() ? Traits::FK_Gotoff : Traits::FK_Abs; 4197e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto} 4207e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto 421467ffe51bebcb3ae3e3ce745c38fc20e8837c31cJim Stichnothtemplate <typename TraitsType> 422467ffe51bebcb3ae3e3ce745c38fc20e8837c31cJim Stichnothbool TargetX86Base<TraitsType>::shouldBePooled(const Constant *C) { 423467ffe51bebcb3ae3e3ce745c38fc20e8837c31cJim Stichnoth if (auto *ConstFloat = llvm::dyn_cast<ConstantFloat>(C)) { 424467ffe51bebcb3ae3e3ce745c38fc20e8837c31cJim Stichnoth return !Utils::isPositiveZero(ConstFloat->getValue()); 425467ffe51bebcb3ae3e3ce745c38fc20e8837c31cJim Stichnoth } 426467ffe51bebcb3ae3e3ce745c38fc20e8837c31cJim Stichnoth if (auto *ConstDouble = llvm::dyn_cast<ConstantDouble>(C)) { 427467ffe51bebcb3ae3e3ce745c38fc20e8837c31cJim Stichnoth return !Utils::isPositiveZero(ConstDouble->getValue()); 428467ffe51bebcb3ae3e3ce745c38fc20e8837c31cJim Stichnoth } 429d46999474d2b4a388e1d8a7c71f06cd4cec51bfcKarl Schimpf if (getFlags().getRandomizeAndPoolImmediatesOption() != RPI_Pool) { 430467ffe51bebcb3ae3e3ce745c38fc20e8837c31cJim Stichnoth return false; 431467ffe51bebcb3ae3e3ce745c38fc20e8837c31cJim Stichnoth } 432467ffe51bebcb3ae3e3ce745c38fc20e8837c31cJim Stichnoth return C->shouldBeRandomizedOrPooled(); 433467ffe51bebcb3ae3e3ce745c38fc20e8837c31cJim Stichnoth} 434467ffe51bebcb3ae3e3ce745c38fc20e8837c31cJim Stichnoth 43532f9ccef8dee33110cc9ebdca2853d6a078752edNicolas Capenstemplate <typename TraitsType> 43632f9ccef8dee33110cc9ebdca2853d6a078752edNicolas Capens::Ice::Type TargetX86Base<TraitsType>::getPointerType() { 43732f9ccef8dee33110cc9ebdca2853d6a078752edNicolas Capens if (!Traits::Is64Bit || 43832f9ccef8dee33110cc9ebdca2853d6a078752edNicolas Capens ::Ice::getFlags().getApplicationBinaryInterface() == ::Ice::ABI_PNaCl) { 43932f9ccef8dee33110cc9ebdca2853d6a078752edNicolas Capens return ::Ice::IceType_i32; 44032f9ccef8dee33110cc9ebdca2853d6a078752edNicolas Capens } 44161593fb9d46f1b3d98878d4b90dcb07009b93fbcNicolas Capens return ::Ice::IceType_i64; 44232f9ccef8dee33110cc9ebdca2853d6a078752edNicolas Capens} 44332f9ccef8dee33110cc9ebdca2853d6a078752edNicolas Capens 4444a56686b5b56db6803f90ad53514bf2fa190d9f7John Portotemplate <typename TraitsType> void TargetX86Base<TraitsType>::translateO2() { 4457e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto TimerMarker T(TimerStack::TT_O2, Func); 4467e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto 447ac2388c385afda17acb3fc242b17b9232ec2262aJohn Porto if (SandboxingType != ST_None) { 448ac2388c385afda17acb3fc242b17b9232ec2262aJohn Porto initRebasePtr(); 44956958cb33d3c1d045f2844408d825442d523f59fJohn Porto } 45056958cb33d3c1d045f2844408d825442d523f59fJohn Porto 4515e0a8a71b75ec56981883587612b16f7c1635a03John Porto genTargetHelperCalls(); 45226217e3333150e66fc96aca79c01105906797960David Sehr Func->dump("After target helper call insertion"); 4535e0a8a71b75ec56981883587612b16f7c1635a03John Porto 4544318a410939b5cee89736f6f15a3185e385b5dc7David Sehr // Merge Alloca instructions, and lay out the stack. 4554318a410939b5cee89736f6f15a3185e385b5dc7David Sehr static constexpr bool SortAndCombineAllocas = true; 4564318a410939b5cee89736f6f15a3185e385b5dc7David Sehr Func->processAllocas(SortAndCombineAllocas); 4574318a410939b5cee89736f6f15a3185e385b5dc7David Sehr Func->dump("After Alloca processing"); 4584318a410939b5cee89736f6f15a3185e385b5dc7David Sehr 459f47d520c1a91bb029d73a5a5f7c0e416ed763f23Manasij Mukherjee // Run this early so it can be used to focus optimizations on potentially hot 460f47d520c1a91bb029d73a5a5f7c0e416ed763f23Manasij Mukherjee // code. 461f47d520c1a91bb029d73a5a5f7c0e416ed763f23Manasij Mukherjee // TODO(stichnot,ascull): currently only used for regalloc not 462f47d520c1a91bb029d73a5a5f7c0e416ed763f23Manasij Mukherjee // expensive high level optimizations which could be focused on potentially 463f47d520c1a91bb029d73a5a5f7c0e416ed763f23Manasij Mukherjee // hot code. 464f47d520c1a91bb029d73a5a5f7c0e416ed763f23Manasij Mukherjee Func->generateLoopInfo(); 465f47d520c1a91bb029d73a5a5f7c0e416ed763f23Manasij Mukherjee Func->dump("After loop analysis"); 466f47d520c1a91bb029d73a5a5f7c0e416ed763f23Manasij Mukherjee if (getFlags().getLoopInvariantCodeMotion()) { 467f47d520c1a91bb029d73a5a5f7c0e416ed763f23Manasij Mukherjee Func->loopInvariantCodeMotion(); 468f47d520c1a91bb029d73a5a5f7c0e416ed763f23Manasij Mukherjee Func->dump("After LICM"); 469f47d520c1a91bb029d73a5a5f7c0e416ed763f23Manasij Mukherjee } 470f47d520c1a91bb029d73a5a5f7c0e416ed763f23Manasij Mukherjee 47153c8fbdf9af4673bb3e674ed5c9d09e899edabf8Manasij Mukherjee if (getFlags().getLocalCSE() != Ice::LCSE_Disabled) { 47253c8fbdf9af4673bb3e674ed5c9d09e899edabf8Manasij Mukherjee Func->localCSE(getFlags().getLocalCSE() == Ice::LCSE_EnabledSSA); 473032c315836b8c1ed9ff6bb2be1981b6d4a1a3594Manasij Mukherjee Func->dump("After Local CSE"); 4745bcc6caf4b1890ef678d67fedf905977a2fc8576Manasij Mukherjee Func->floatConstantCSE(); 475032c315836b8c1ed9ff6bb2be1981b6d4a1a3594Manasij Mukherjee } 47645f51a2675e8367a2d7ce90e5b63e161da668856Manasij Mukherjee if (getFlags().getEnableShortCircuit()) { 47745f51a2675e8367a2d7ce90e5b63e161da668856Manasij Mukherjee Func->shortCircuitJumps(); 47845f51a2675e8367a2d7ce90e5b63e161da668856Manasij Mukherjee Func->dump("After Short Circuiting"); 47945f51a2675e8367a2d7ce90e5b63e161da668856Manasij Mukherjee } 480032c315836b8c1ed9ff6bb2be1981b6d4a1a3594Manasij Mukherjee 481d46999474d2b4a388e1d8a7c71f06cd4cec51bfcKarl Schimpf if (!getFlags().getEnablePhiEdgeSplit()) { 4827e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto // Lower Phi instructions. 4837e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto Func->placePhiLoads(); 4847e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto if (Func->hasError()) 4857e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto return; 4867e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto Func->placePhiStores(); 4877e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto if (Func->hasError()) 4887e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto return; 4897e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto Func->deletePhis(); 4907e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto if (Func->hasError()) 4917e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto return; 4927e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto Func->dump("After Phi lowering"); 4937e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto } 4947e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto 4957e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto // Address mode optimization. 4967e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto Func->getVMetadata()->init(VMK_SingleDefs); 4977e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto Func->doAddressOpt(); 498a47c11c7f17022050043d69c0802241e0747a056John Porto Func->materializeVectorShuffles(); 4997e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto 50057e126899b20c65ff3ea23a3b7d7a67ab30b99dcAndrew Scull // Find read-modify-write opportunities. Do this after address mode 5017e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto // optimization so that doAddressOpt() doesn't need to be applied to RMW 5027e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto // instructions as well. 5037e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto findRMW(); 5047e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto Func->dump("After RMW transform"); 5057e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto 5067e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto // Argument lowering 5077e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto Func->doArgLowering(); 5087e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto 50957e126899b20c65ff3ea23a3b7d7a67ab30b99dcAndrew Scull // Target lowering. This requires liveness analysis for some parts of the 51057e126899b20c65ff3ea23a3b7d7a67ab30b99dcAndrew Scull // lowering decisions, such as compare/branch fusing. If non-lightweight 511921856d4e4bd4f0deedc7324d5f6a4ca0be3439fJohn Porto // liveness analysis is used, the instructions need to be renumbered first 512921856d4e4bd4f0deedc7324d5f6a4ca0be3439fJohn Porto // TODO: This renumbering should only be necessary if we're actually 513921856d4e4bd4f0deedc7324d5f6a4ca0be3439fJohn Porto // calculating live intervals, which we only do for register allocation. 5147e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto Func->renumberInstructions(); 5157e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto if (Func->hasError()) 5167e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto return; 5177e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto 51857e126899b20c65ff3ea23a3b7d7a67ab30b99dcAndrew Scull // TODO: It should be sufficient to use the fastest liveness calculation, 51957e126899b20c65ff3ea23a3b7d7a67ab30b99dcAndrew Scull // i.e. livenessLightweight(). However, for some reason that slows down the 52057e126899b20c65ff3ea23a3b7d7a67ab30b99dcAndrew Scull // rest of the translation. Investigate. 5217e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto Func->liveness(Liveness_Basic); 5227e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto if (Func->hasError()) 5237e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto return; 5247e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto Func->dump("After x86 address mode opt"); 5257e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto 5267e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto // Disable constant blinding or pooling for load optimization. 5277e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto { 5287e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto BoolFlagSaver B(RandomizationPoolingPaused, true); 5297e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto doLoadOpt(); 5307e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto } 5317e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto Func->genCode(); 5327e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto if (Func->hasError()) 5337e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto return; 534ac2388c385afda17acb3fc242b17b9232ec2262aJohn Porto if (SandboxingType != ST_None) { 535ac2388c385afda17acb3fc242b17b9232ec2262aJohn Porto initSandbox(); 536ac2388c385afda17acb3fc242b17b9232ec2262aJohn Porto } 5377e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto Func->dump("After x86 codegen"); 538b9a847280e4486e566dabdd5b0d571309b4ad628Jim Stichnoth splitBlockLocalVariables(Func); 5397e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto 540aa6c109366408d066c4ac9e22c8c0ded6d18092fAndrew Scull // Register allocation. This requires instruction renumbering and full 541aa6c109366408d066c4ac9e22c8c0ded6d18092fAndrew Scull // liveness analysis. Loops must be identified before liveness so variable 542aa6c109366408d066c4ac9e22c8c0ded6d18092fAndrew Scull // use weights are correct. 5437e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto Func->renumberInstructions(); 5447e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto if (Func->hasError()) 5457e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto return; 5467e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto Func->liveness(Liveness_Intervals); 5477e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto if (Func->hasError()) 5487e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto return; 549921856d4e4bd4f0deedc7324d5f6a4ca0be3439fJohn Porto // The post-codegen dump is done here, after liveness analysis and associated 550921856d4e4bd4f0deedc7324d5f6a4ca0be3439fJohn Porto // cleanup, to make the dump cleaner and more useful. 551ef18fc51dd5c1b2640c291a363cf1e4712142eaaNicolas Capens Func->dump("After initial x86 codegen"); 5522943d77ca2b3c2380f34b674960f92264c473360Jim Stichnoth // Validate the live range computations. The expensive validation call is 5532943d77ca2b3c2380f34b674960f92264c473360Jim Stichnoth // deliberately only made when assertions are enabled. 5542943d77ca2b3c2380f34b674960f92264c473360Jim Stichnoth assert(Func->validateLiveness()); 5557e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto Func->getVMetadata()->init(VMK_All); 5567e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto regAlloc(RAK_Global); 5577e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto if (Func->hasError()) 5587e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto return; 5597e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto Func->dump("After linear scan regalloc"); 5607e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto 561d46999474d2b4a388e1d8a7c71f06cd4cec51bfcKarl Schimpf if (getFlags().getEnablePhiEdgeSplit()) { 562a3f57b9a6c7dbd3be7bf86dba697e9219c3413b2Jim Stichnoth Func->advancedPhiLowering(); 5637e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto Func->dump("After advanced Phi lowering"); 5647e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto } 5657e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto 5667e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto // Stack frame mapping. 5677e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto Func->genFrame(); 5687e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto if (Func->hasError()) 5697e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto return; 5707e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto Func->dump("After stack frame mapping"); 5717e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto 5727e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto Func->contractEmptyNodes(); 5737e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto Func->reorderNodes(); 5747e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto 575969f6a33c32f9e7197672fbc911918589f5218bdQining Lu // Shuffle basic block order if -reorder-basic-blocks is enabled. 576969f6a33c32f9e7197672fbc911918589f5218bdQining Lu Func->shuffleNodes(); 577969f6a33c32f9e7197672fbc911918589f5218bdQining Lu 57857e126899b20c65ff3ea23a3b7d7a67ab30b99dcAndrew Scull // Branch optimization. This needs to be done just before code emission. In 579921856d4e4bd4f0deedc7324d5f6a4ca0be3439fJohn Porto // particular, no transformations that insert or reorder CfgNodes should be 58057e126899b20c65ff3ea23a3b7d7a67ab30b99dcAndrew Scull // done after branch optimization. We go ahead and do it before nop insertion 581921856d4e4bd4f0deedc7324d5f6a4ca0be3439fJohn Porto // to reduce the amount of work needed for searching for opportunities. 5827e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto Func->doBranchOpt(); 5837e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto Func->dump("After branch optimization"); 5847e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto 585969f6a33c32f9e7197672fbc911918589f5218bdQining Lu // Nop insertion if -nop-insertion is enabled. 586969f6a33c32f9e7197672fbc911918589f5218bdQining Lu Func->doNopInsertion(); 58786df4e9e6d183f07638440afd2c225b485c03917Andrew Scull 58886df4e9e6d183f07638440afd2c225b485c03917Andrew Scull // Mark nodes that require sandbox alignment 58956958cb33d3c1d045f2844408d825442d523f59fJohn Porto if (NeedSandboxing) { 59086df4e9e6d183f07638440afd2c225b485c03917Andrew Scull Func->markNodesForSandboxing(); 59156958cb33d3c1d045f2844408d825442d523f59fJohn Porto } 5927e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto} 5937e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto 5944a56686b5b56db6803f90ad53514bf2fa190d9f7John Portotemplate <typename TraitsType> void TargetX86Base<TraitsType>::translateOm1() { 5957e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto TimerMarker T(TimerStack::TT_Om1, Func); 5967e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto 597ac2388c385afda17acb3fc242b17b9232ec2262aJohn Porto if (SandboxingType != ST_None) { 598ac2388c385afda17acb3fc242b17b9232ec2262aJohn Porto initRebasePtr(); 59956958cb33d3c1d045f2844408d825442d523f59fJohn Porto } 60056958cb33d3c1d045f2844408d825442d523f59fJohn Porto 6015e0a8a71b75ec56981883587612b16f7c1635a03John Porto genTargetHelperCalls(); 6025e0a8a71b75ec56981883587612b16f7c1635a03John Porto 6034318a410939b5cee89736f6f15a3185e385b5dc7David Sehr // Do not merge Alloca instructions, and lay out the stack. 6044318a410939b5cee89736f6f15a3185e385b5dc7David Sehr static constexpr bool SortAndCombineAllocas = false; 6054318a410939b5cee89736f6f15a3185e385b5dc7David Sehr Func->processAllocas(SortAndCombineAllocas); 6064318a410939b5cee89736f6f15a3185e385b5dc7David Sehr Func->dump("After Alloca processing"); 6074318a410939b5cee89736f6f15a3185e385b5dc7David Sehr 6087e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto Func->placePhiLoads(); 6097e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto if (Func->hasError()) 6107e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto return; 6117e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto Func->placePhiStores(); 6127e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto if (Func->hasError()) 6137e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto return; 6147e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto Func->deletePhis(); 6157e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto if (Func->hasError()) 6167e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto return; 6177e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto Func->dump("After Phi lowering"); 6187e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto 6197e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto Func->doArgLowering(); 6207e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto Func->genCode(); 6217e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto if (Func->hasError()) 6227e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto return; 623ac2388c385afda17acb3fc242b17b9232ec2262aJohn Porto if (SandboxingType != ST_None) { 624ac2388c385afda17acb3fc242b17b9232ec2262aJohn Porto initSandbox(); 625ac2388c385afda17acb3fc242b17b9232ec2262aJohn Porto } 626ef18fc51dd5c1b2640c291a363cf1e4712142eaaNicolas Capens Func->dump("After initial x86 codegen"); 6277e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto 6287e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto regAlloc(RAK_InfOnly); 6297e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto if (Func->hasError()) 6307e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto return; 6317e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto Func->dump("After regalloc of infinite-weight variables"); 6327e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto 6337e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto Func->genFrame(); 6347e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto if (Func->hasError()) 6357e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto return; 6367e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto Func->dump("After stack frame mapping"); 6377e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto 638969f6a33c32f9e7197672fbc911918589f5218bdQining Lu // Shuffle basic block order if -reorder-basic-blocks is enabled. 639969f6a33c32f9e7197672fbc911918589f5218bdQining Lu Func->shuffleNodes(); 640969f6a33c32f9e7197672fbc911918589f5218bdQining Lu 641969f6a33c32f9e7197672fbc911918589f5218bdQining Lu // Nop insertion if -nop-insertion is enabled. 642969f6a33c32f9e7197672fbc911918589f5218bdQining Lu Func->doNopInsertion(); 64386df4e9e6d183f07638440afd2c225b485c03917Andrew Scull 64486df4e9e6d183f07638440afd2c225b485c03917Andrew Scull // Mark nodes that require sandbox alignment 64556958cb33d3c1d045f2844408d825442d523f59fJohn Porto if (NeedSandboxing) 64686df4e9e6d183f07638440afd2c225b485c03917Andrew Scull Func->markNodesForSandboxing(); 6477e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto} 6487e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto 6495aeed955e17bac8cc44cc6d2b6ff7513cc714c2fJohn Portoinline bool canRMW(const InstArithmetic *Arith) { 6507e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto Type Ty = Arith->getDest()->getType(); 651921856d4e4bd4f0deedc7324d5f6a4ca0be3439fJohn Porto // X86 vector instructions write to a register and have no RMW option. 6527e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto if (isVectorType(Ty)) 6537e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto return false; 6547e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto bool isI64 = Ty == IceType_i64; 6557e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto 6567e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto switch (Arith->getOp()) { 6577e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto // Not handled for lack of simple lowering: 6587e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto // shift on i64 6597e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto // mul, udiv, urem, sdiv, srem, frem 6607e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto // Not handled for lack of RMW instructions: 6617e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto // fadd, fsub, fmul, fdiv (also vector types) 6627e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto default: 6637e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto return false; 6647e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto case InstArithmetic::Add: 6657e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto case InstArithmetic::Sub: 6667e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto case InstArithmetic::And: 6677e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto case InstArithmetic::Or: 6687e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto case InstArithmetic::Xor: 6697e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto return true; 6707e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto case InstArithmetic::Shl: 6717e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto case InstArithmetic::Lshr: 6727e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto case InstArithmetic::Ashr: 6737e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto return false; // TODO(stichnot): implement 6747e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto return !isI64; 6757e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto } 6767e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto} 6777e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto 6784a56686b5b56db6803f90ad53514bf2fa190d9f7John Portotemplate <typename TraitsType> 6797e93c62d7e223b7fd9e6e0889e4b70b635589282John Portobool isSameMemAddressOperand(const Operand *A, const Operand *B) { 6807e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto if (A == B) 6817e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto return true; 6824a56686b5b56db6803f90ad53514bf2fa190d9f7John Porto if (auto *MemA = 6834a56686b5b56db6803f90ad53514bf2fa190d9f7John Porto llvm::dyn_cast<typename TargetX86Base<TraitsType>::X86OperandMem>( 6844a56686b5b56db6803f90ad53514bf2fa190d9f7John Porto A)) { 6854a56686b5b56db6803f90ad53514bf2fa190d9f7John Porto if (auto *MemB = 6864a56686b5b56db6803f90ad53514bf2fa190d9f7John Porto llvm::dyn_cast<typename TargetX86Base<TraitsType>::X86OperandMem>( 6874a56686b5b56db6803f90ad53514bf2fa190d9f7John Porto B)) { 6887e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto return MemA->getBase() == MemB->getBase() && 6897e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto MemA->getOffset() == MemB->getOffset() && 6907e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto MemA->getIndex() == MemB->getIndex() && 6917e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto MemA->getShift() == MemB->getShift() && 6927e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto MemA->getSegmentRegister() == MemB->getSegmentRegister(); 6937e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto } 6947e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto } 6957e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto return false; 6967e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto} 6977e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto 6984a56686b5b56db6803f90ad53514bf2fa190d9f7John Portotemplate <typename TraitsType> void TargetX86Base<TraitsType>::findRMW() { 699b88d8c878538d856b845bd5cabdff7e75ef2761aJim Stichnoth TimerMarker _(TimerStack::TT_findRMW, Func); 7007e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto Func->dump("Before RMW"); 70100741a005cf242d2e9108f7cc7c454246e13741cAndrew Scull if (Func->isVerbose(IceV_RMW)) 70200741a005cf242d2e9108f7cc7c454246e13741cAndrew Scull Func->getContext()->lockStr(); 7037e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto for (CfgNode *Node : Func->getNodes()) { 7047e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto // Walk through the instructions, considering each sequence of 3 70557e126899b20c65ff3ea23a3b7d7a67ab30b99dcAndrew Scull // instructions, and look for the particular RMW pattern. Note that this 70657e126899b20c65ff3ea23a3b7d7a67ab30b99dcAndrew Scull // search can be "broken" (false negatives) if there are intervening 70757e126899b20c65ff3ea23a3b7d7a67ab30b99dcAndrew Scull // deleted instructions, or intervening instructions that could be safely 70857e126899b20c65ff3ea23a3b7d7a67ab30b99dcAndrew Scull // moved out of the way to reveal an RMW pattern. 7097e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto auto E = Node->getInsts().end(); 7107e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto auto I1 = E, I2 = E, I3 = Node->getInsts().begin(); 7117e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto for (; I3 != E; I1 = I2, I2 = I3, ++I3) { 7127e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto // Make I3 skip over deleted instructions. 7137e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto while (I3 != E && I3->isDeleted()) 7147e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto ++I3; 7157e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto if (I1 == E || I2 == E || I3 == E) 7167e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto continue; 7177e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto assert(!I1->isDeleted()); 7187e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto assert(!I2->isDeleted()); 7197e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto assert(!I3->isDeleted()); 72000741a005cf242d2e9108f7cc7c454246e13741cAndrew Scull auto *Load = llvm::dyn_cast<InstLoad>(I1); 72100741a005cf242d2e9108f7cc7c454246e13741cAndrew Scull auto *Arith = llvm::dyn_cast<InstArithmetic>(I2); 72200741a005cf242d2e9108f7cc7c454246e13741cAndrew Scull auto *Store = llvm::dyn_cast<InstStore>(I3); 72300741a005cf242d2e9108f7cc7c454246e13741cAndrew Scull if (!Load || !Arith || !Store) 72400741a005cf242d2e9108f7cc7c454246e13741cAndrew Scull continue; 72500741a005cf242d2e9108f7cc7c454246e13741cAndrew Scull // Look for: 72600741a005cf242d2e9108f7cc7c454246e13741cAndrew Scull // a = Load addr 72700741a005cf242d2e9108f7cc7c454246e13741cAndrew Scull // b = <op> a, other 72800741a005cf242d2e9108f7cc7c454246e13741cAndrew Scull // Store b, addr 72900741a005cf242d2e9108f7cc7c454246e13741cAndrew Scull // Change to: 73000741a005cf242d2e9108f7cc7c454246e13741cAndrew Scull // a = Load addr 73100741a005cf242d2e9108f7cc7c454246e13741cAndrew Scull // b = <op> a, other 73200741a005cf242d2e9108f7cc7c454246e13741cAndrew Scull // x = FakeDef 73300741a005cf242d2e9108f7cc7c454246e13741cAndrew Scull // RMW <op>, addr, other, x 73400741a005cf242d2e9108f7cc7c454246e13741cAndrew Scull // b = Store b, addr, x 735230d4101fb3c591d044406eef27d0ce43ffab53eJim Stichnoth // Note that inferTwoAddress() makes sure setDestRedefined() gets called 736230d4101fb3c591d044406eef27d0ce43ffab53eJim Stichnoth // on the updated Store instruction, to avoid liveness problems later. 73700741a005cf242d2e9108f7cc7c454246e13741cAndrew Scull // 73800741a005cf242d2e9108f7cc7c454246e13741cAndrew Scull // With this transformation, the Store instruction acquires a Dest 73900741a005cf242d2e9108f7cc7c454246e13741cAndrew Scull // variable and is now subject to dead code elimination if there are no 740230d4101fb3c591d044406eef27d0ce43ffab53eJim Stichnoth // more uses of "b". Variable "x" is a beacon for determining whether the 741230d4101fb3c591d044406eef27d0ce43ffab53eJim Stichnoth // Store instruction gets dead-code eliminated. If the Store instruction 742230d4101fb3c591d044406eef27d0ce43ffab53eJim Stichnoth // is eliminated, then it must be the case that the RMW instruction ends 743230d4101fb3c591d044406eef27d0ce43ffab53eJim Stichnoth // x's live range, and therefore the RMW instruction will be retained and 744230d4101fb3c591d044406eef27d0ce43ffab53eJim Stichnoth // later lowered. On the other hand, if the RMW instruction does not end 745230d4101fb3c591d044406eef27d0ce43ffab53eJim Stichnoth // x's live range, then the Store instruction must still be present, and 746230d4101fb3c591d044406eef27d0ce43ffab53eJim Stichnoth // therefore the RMW instruction is ignored during lowering because it is 747230d4101fb3c591d044406eef27d0ce43ffab53eJim Stichnoth // redundant with the Store instruction. 74800741a005cf242d2e9108f7cc7c454246e13741cAndrew Scull // 74900741a005cf242d2e9108f7cc7c454246e13741cAndrew Scull // Note that if "a" has further uses, the RMW transformation may still 75000741a005cf242d2e9108f7cc7c454246e13741cAndrew Scull // trigger, resulting in two loads and one store, which is worse than the 75100741a005cf242d2e9108f7cc7c454246e13741cAndrew Scull // original one load and one store. However, this is probably rare, and 75200741a005cf242d2e9108f7cc7c454246e13741cAndrew Scull // caching probably keeps it just as fast. 7534a56686b5b56db6803f90ad53514bf2fa190d9f7John Porto if (!isSameMemAddressOperand<TraitsType>(Load->getSourceAddress(), 7544a56686b5b56db6803f90ad53514bf2fa190d9f7John Porto Store->getAddr())) 75500741a005cf242d2e9108f7cc7c454246e13741cAndrew Scull continue; 75600741a005cf242d2e9108f7cc7c454246e13741cAndrew Scull Operand *ArithSrcFromLoad = Arith->getSrc(0); 75700741a005cf242d2e9108f7cc7c454246e13741cAndrew Scull Operand *ArithSrcOther = Arith->getSrc(1); 75800741a005cf242d2e9108f7cc7c454246e13741cAndrew Scull if (ArithSrcFromLoad != Load->getDest()) { 75900741a005cf242d2e9108f7cc7c454246e13741cAndrew Scull if (!Arith->isCommutative() || ArithSrcOther != Load->getDest()) 76000741a005cf242d2e9108f7cc7c454246e13741cAndrew Scull continue; 76100741a005cf242d2e9108f7cc7c454246e13741cAndrew Scull std::swap(ArithSrcFromLoad, ArithSrcOther); 76200741a005cf242d2e9108f7cc7c454246e13741cAndrew Scull } 76300741a005cf242d2e9108f7cc7c454246e13741cAndrew Scull if (Arith->getDest() != Store->getData()) 76400741a005cf242d2e9108f7cc7c454246e13741cAndrew Scull continue; 76500741a005cf242d2e9108f7cc7c454246e13741cAndrew Scull if (!canRMW(Arith)) 76600741a005cf242d2e9108f7cc7c454246e13741cAndrew Scull continue; 76700741a005cf242d2e9108f7cc7c454246e13741cAndrew Scull if (Func->isVerbose(IceV_RMW)) { 76800741a005cf242d2e9108f7cc7c454246e13741cAndrew Scull Ostream &Str = Func->getContext()->getStrDump(); 76900741a005cf242d2e9108f7cc7c454246e13741cAndrew Scull Str << "Found RMW in " << Func->getFunctionName() << ":\n "; 77000741a005cf242d2e9108f7cc7c454246e13741cAndrew Scull Load->dump(Func); 77100741a005cf242d2e9108f7cc7c454246e13741cAndrew Scull Str << "\n "; 77200741a005cf242d2e9108f7cc7c454246e13741cAndrew Scull Arith->dump(Func); 77300741a005cf242d2e9108f7cc7c454246e13741cAndrew Scull Str << "\n "; 77400741a005cf242d2e9108f7cc7c454246e13741cAndrew Scull Store->dump(Func); 77500741a005cf242d2e9108f7cc7c454246e13741cAndrew Scull Str << "\n"; 7767e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto } 77700741a005cf242d2e9108f7cc7c454246e13741cAndrew Scull Variable *Beacon = Func->makeVariable(IceType_i32); 77800741a005cf242d2e9108f7cc7c454246e13741cAndrew Scull Beacon->setMustNotHaveReg(); 77900741a005cf242d2e9108f7cc7c454246e13741cAndrew Scull Store->setRmwBeacon(Beacon); 78054f3d518805b04b3f2958b7906a7d07dedb4e1aeJim Stichnoth auto *BeaconDef = InstFakeDef::create(Func, Beacon); 78100741a005cf242d2e9108f7cc7c454246e13741cAndrew Scull Node->getInsts().insert(I3, BeaconDef); 7824a56686b5b56db6803f90ad53514bf2fa190d9f7John Porto auto *RMW = InstX86FakeRMW::create(Func, ArithSrcOther, Store->getAddr(), 7834a56686b5b56db6803f90ad53514bf2fa190d9f7John Porto Beacon, Arith->getOp()); 78400741a005cf242d2e9108f7cc7c454246e13741cAndrew Scull Node->getInsts().insert(I3, RMW); 7857e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto } 7867e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto } 78700741a005cf242d2e9108f7cc7c454246e13741cAndrew Scull if (Func->isVerbose(IceV_RMW)) 78800741a005cf242d2e9108f7cc7c454246e13741cAndrew Scull Func->getContext()->unlockStr(); 7897e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto} 7907e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto 7917e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto// Converts a ConstantInteger32 operand into its constant value, or 7927e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto// MemoryOrderInvalid if the operand is not a ConstantInteger32. 7935aeed955e17bac8cc44cc6d2b6ff7513cc714c2fJohn Portoinline uint64_t getConstantMemoryOrder(Operand *Opnd) { 7945bff61c44841990680781892036adb17b3cff0c4Jim Stichnoth if (auto *Integer = llvm::dyn_cast<ConstantInteger32>(Opnd)) 7957e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto return Integer->getValue(); 7967e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto return Intrinsics::MemoryOrderInvalid; 7977e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto} 7987e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto 79957e126899b20c65ff3ea23a3b7d7a67ab30b99dcAndrew Scull/// Determines whether the dest of a Load instruction can be folded into one of 80057e126899b20c65ff3ea23a3b7d7a67ab30b99dcAndrew Scull/// the src operands of a 2-operand instruction. This is true as long as the 80157e126899b20c65ff3ea23a3b7d7a67ab30b99dcAndrew Scull/// load dest matches exactly one of the binary instruction's src operands. 80257e126899b20c65ff3ea23a3b7d7a67ab30b99dcAndrew Scull/// Replaces Src0 or Src1 with LoadSrc if the answer is true. 8035aeed955e17bac8cc44cc6d2b6ff7513cc714c2fJohn Portoinline bool canFoldLoadIntoBinaryInst(Operand *LoadSrc, Variable *LoadDest, 8045aeed955e17bac8cc44cc6d2b6ff7513cc714c2fJohn Porto Operand *&Src0, Operand *&Src1) { 8057e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto if (Src0 == LoadDest && Src1 != LoadDest) { 8067e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto Src0 = LoadSrc; 8077e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto return true; 8087e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto } 8097e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto if (Src0 != LoadDest && Src1 == LoadDest) { 8107e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto Src1 = LoadSrc; 8117e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto return true; 8127e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto } 8137e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto return false; 8147e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto} 8157e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto 8164a56686b5b56db6803f90ad53514bf2fa190d9f7John Portotemplate <typename TraitsType> void TargetX86Base<TraitsType>::doLoadOpt() { 817b88d8c878538d856b845bd5cabdff7e75ef2761aJim Stichnoth TimerMarker _(TimerStack::TT_loadOpt, Func); 8187e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto for (CfgNode *Node : Func->getNodes()) { 8197e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto Context.init(Node); 8207e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto while (!Context.atEnd()) { 8217e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto Variable *LoadDest = nullptr; 8227e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto Operand *LoadSrc = nullptr; 823efdf412032f7622a0663696896708d75b82e92f6Jim Stichnoth Inst *CurInst = iteratorToInst(Context.getCur()); 8247e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto Inst *Next = Context.getNextInst(); 82557e126899b20c65ff3ea23a3b7d7a67ab30b99dcAndrew Scull // Determine whether the current instruction is a Load instruction or 82657e126899b20c65ff3ea23a3b7d7a67ab30b99dcAndrew Scull // equivalent. 8277e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto if (auto *Load = llvm::dyn_cast<InstLoad>(CurInst)) { 8287e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto // An InstLoad always qualifies. 8297e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto LoadDest = Load->getDest(); 8305bff61c44841990680781892036adb17b3cff0c4Jim Stichnoth constexpr bool DoLegalize = false; 8317e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto LoadSrc = formMemoryOperand(Load->getSourceAddress(), 8327e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto LoadDest->getType(), DoLegalize); 8337e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto } else if (auto *Intrin = llvm::dyn_cast<InstIntrinsicCall>(CurInst)) { 83457e126899b20c65ff3ea23a3b7d7a67ab30b99dcAndrew Scull // An AtomicLoad intrinsic qualifies as long as it has a valid memory 83557e126899b20c65ff3ea23a3b7d7a67ab30b99dcAndrew Scull // ordering, and can be implemented in a single instruction (i.e., not 83657e126899b20c65ff3ea23a3b7d7a67ab30b99dcAndrew Scull // i64 on x86-32). 8377e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto Intrinsics::IntrinsicID ID = Intrin->getIntrinsicInfo().ID; 8387e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto if (ID == Intrinsics::AtomicLoad && 8391d235425dab1f3dd059973fc53f1b1d5879469e3John Porto (Traits::Is64Bit || Intrin->getDest()->getType() != IceType_i64) && 8407e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto Intrinsics::isMemoryOrderValid( 8417e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto ID, getConstantMemoryOrder(Intrin->getArg(1)))) { 8427e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto LoadDest = Intrin->getDest(); 8435bff61c44841990680781892036adb17b3cff0c4Jim Stichnoth constexpr bool DoLegalize = false; 8447e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto LoadSrc = formMemoryOperand(Intrin->getArg(0), LoadDest->getType(), 8457e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto DoLegalize); 8467e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto } 8477e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto } 84857e126899b20c65ff3ea23a3b7d7a67ab30b99dcAndrew Scull // A Load instruction can be folded into the following instruction only 84957e126899b20c65ff3ea23a3b7d7a67ab30b99dcAndrew Scull // if the following instruction ends the Load's Dest variable's live 85057e126899b20c65ff3ea23a3b7d7a67ab30b99dcAndrew Scull // range. 8517e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto if (LoadDest && Next && Next->isLastUse(LoadDest)) { 8527e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto assert(LoadSrc); 8537e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto Inst *NewInst = nullptr; 8547e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto if (auto *Arith = llvm::dyn_cast<InstArithmetic>(Next)) { 8557e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto Operand *Src0 = Arith->getSrc(0); 8567e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto Operand *Src1 = Arith->getSrc(1); 8577e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto if (canFoldLoadIntoBinaryInst(LoadSrc, LoadDest, Src0, Src1)) { 8587e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto NewInst = InstArithmetic::create(Func, Arith->getOp(), 8597e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto Arith->getDest(), Src0, Src1); 8607e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto } 8617e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto } else if (auto *Icmp = llvm::dyn_cast<InstIcmp>(Next)) { 8627e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto Operand *Src0 = Icmp->getSrc(0); 8637e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto Operand *Src1 = Icmp->getSrc(1); 8647e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto if (canFoldLoadIntoBinaryInst(LoadSrc, LoadDest, Src0, Src1)) { 8657e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto NewInst = InstIcmp::create(Func, Icmp->getCondition(), 8667e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto Icmp->getDest(), Src0, Src1); 8677e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto } 8687e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto } else if (auto *Fcmp = llvm::dyn_cast<InstFcmp>(Next)) { 8697e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto Operand *Src0 = Fcmp->getSrc(0); 8707e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto Operand *Src1 = Fcmp->getSrc(1); 8717e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto if (canFoldLoadIntoBinaryInst(LoadSrc, LoadDest, Src0, Src1)) { 8727e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto NewInst = InstFcmp::create(Func, Fcmp->getCondition(), 8737e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto Fcmp->getDest(), Src0, Src1); 8747e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto } 8757e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto } else if (auto *Select = llvm::dyn_cast<InstSelect>(Next)) { 8767e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto Operand *Src0 = Select->getTrueOperand(); 8777e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto Operand *Src1 = Select->getFalseOperand(); 8787e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto if (canFoldLoadIntoBinaryInst(LoadSrc, LoadDest, Src0, Src1)) { 8797e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto NewInst = InstSelect::create(Func, Select->getDest(), 8807e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto Select->getCondition(), Src0, Src1); 8817e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto } 8827e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto } else if (auto *Cast = llvm::dyn_cast<InstCast>(Next)) { 88357e126899b20c65ff3ea23a3b7d7a67ab30b99dcAndrew Scull // The load dest can always be folded into a Cast instruction. 88454f3d518805b04b3f2958b7906a7d07dedb4e1aeJim Stichnoth auto *Src0 = llvm::dyn_cast<Variable>(Cast->getSrc(0)); 8857e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto if (Src0 == LoadDest) { 8867e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto NewInst = InstCast::create(Func, Cast->getCastKind(), 8877e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto Cast->getDest(), LoadSrc); 8887e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto } 8897e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto } 8907e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto if (NewInst) { 8917e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto CurInst->setDeleted(); 8927e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto Next->setDeleted(); 8937e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto Context.insert(NewInst); 89457e126899b20c65ff3ea23a3b7d7a67ab30b99dcAndrew Scull // Update NewInst->LiveRangesEnded so that target lowering may 89557e126899b20c65ff3ea23a3b7d7a67ab30b99dcAndrew Scull // benefit. Also update NewInst->HasSideEffects. 8967e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto NewInst->spliceLivenessInfo(Next, CurInst); 8977e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto } 8987e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto } 8997e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto Context.advanceCur(); 9007e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto Context.advanceNext(); 9017e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto } 9027e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto } 9037e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto Func->dump("After load optimization"); 9047e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto} 9057e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto 9064a56686b5b56db6803f90ad53514bf2fa190d9f7John Portotemplate <typename TraitsType> 9074a56686b5b56db6803f90ad53514bf2fa190d9f7John Portobool TargetX86Base<TraitsType>::doBranchOpt(Inst *I, const CfgNode *NextNode) { 9084a56686b5b56db6803f90ad53514bf2fa190d9f7John Porto if (auto *Br = llvm::dyn_cast<InstX86Br>(I)) { 9097e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto return Br->optimizeBranch(NextNode); 9107e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto } 9117e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto return false; 9127e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto} 9137e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto 9144a56686b5b56db6803f90ad53514bf2fa190d9f7John Portotemplate <typename TraitsType> 9158aa396610b7baf728631a43ea16ad3d13e38397aJim StichnothVariable *TargetX86Base<TraitsType>::getPhysicalRegister(RegNumT RegNum, 9164a56686b5b56db6803f90ad53514bf2fa190d9f7John Porto Type Ty) { 9177e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto if (Ty == IceType_void) 9187e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto Ty = IceType_i32; 9197e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto if (PhysicalRegisters[Ty].empty()) 9205d0acff3a2fa421923392aadb4df2742064b6248John Porto PhysicalRegisters[Ty].resize(Traits::RegisterSet::Reg_NUM); 9218aa396610b7baf728631a43ea16ad3d13e38397aJim Stichnoth assert(unsigned(RegNum) < PhysicalRegisters[Ty].size()); 9227e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto Variable *Reg = PhysicalRegisters[Ty][RegNum]; 9237e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto if (Reg == nullptr) { 9245aeed955e17bac8cc44cc6d2b6ff7513cc714c2fJohn Porto Reg = Func->makeVariable(Ty); 9257e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto Reg->setRegNum(RegNum); 9267e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto PhysicalRegisters[Ty][RegNum] = Reg; 927696605575c4ba11df5ec56149c561839d1bfe532Jim Stichnoth // Specially mark a named physical register as an "argument" so that it is 928696605575c4ba11df5ec56149c561839d1bfe532Jim Stichnoth // considered live upon function entry. Otherwise it's possible to get 929696605575c4ba11df5ec56149c561839d1bfe532Jim Stichnoth // liveness validation errors for saving callee-save registers. 930696605575c4ba11df5ec56149c561839d1bfe532Jim Stichnoth Func->addImplicitArg(Reg); 931696605575c4ba11df5ec56149c561839d1bfe532Jim Stichnoth // Don't bother tracking the live range of a named physical register. 932696605575c4ba11df5ec56149c561839d1bfe532Jim Stichnoth Reg->setIgnoreLiveness(); 9337e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto } 9348aa396610b7baf728631a43ea16ad3d13e38397aJim Stichnoth assert(Traits::getGprForType(Ty, RegNum) == RegNum); 9357e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto return Reg; 9367e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto} 9377e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto 9384a56686b5b56db6803f90ad53514bf2fa190d9f7John Portotemplate <typename TraitsType> 939467ffe51bebcb3ae3e3ce745c38fc20e8837c31cJim Stichnothconst char *TargetX86Base<TraitsType>::getRegName(RegNumT RegNum, 940467ffe51bebcb3ae3e3ce745c38fc20e8837c31cJim Stichnoth Type Ty) const { 941008f4ce5417c97891ea02b54b691b39aa2e2a0afJohn Porto return Traits::getRegName(Traits::getGprForType(Ty, RegNum)); 9427e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto} 9437e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto 9444a56686b5b56db6803f90ad53514bf2fa190d9f7John Portotemplate <typename TraitsType> 9454a56686b5b56db6803f90ad53514bf2fa190d9f7John Portovoid TargetX86Base<TraitsType>::emitVariable(const Variable *Var) const { 94628068adbf34a4602090efddc18b4dd123ffdeb6aJan Voung if (!BuildDefs::dump()) 94728068adbf34a4602090efddc18b4dd123ffdeb6aJan Voung return; 9487e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto Ostream &Str = Ctx->getStrEmit(); 9497e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto if (Var->hasReg()) { 95056958cb33d3c1d045f2844408d825442d523f59fJohn Porto const bool Is64BitSandboxing = Traits::Is64Bit && NeedSandboxing; 95156958cb33d3c1d045f2844408d825442d523f59fJohn Porto const Type VarType = (Var->isRematerializable() && Is64BitSandboxing) 95256958cb33d3c1d045f2844408d825442d523f59fJohn Porto ? IceType_i64 95356958cb33d3c1d045f2844408d825442d523f59fJohn Porto : Var->getType(); 95456958cb33d3c1d045f2844408d825442d523f59fJohn Porto Str << "%" << getRegName(Var->getRegNum(), VarType); 9557e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto return; 9567e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto } 95711c9a325399b282cb4ea7d1d24d42fceeec2a09aAndrew Scull if (Var->mustHaveReg()) { 958a91c34118294efbf08ebd11eed96fce83bf35f3cJim Stichnoth llvm::report_fatal_error("Infinite-weight Variable (" + Var->getName() + 95945bec54fdb3d6f6c48fc4f55fc56bdbf9dd3326eJim Stichnoth ") has no register assigned - function " + 96045bec54fdb3d6f6c48fc4f55fc56bdbf9dd3326eJim Stichnoth Func->getFunctionName()); 9617e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto } 962238b4c160169aea53c5ab87f40fd1d454f41e590Jim Stichnoth const int32_t Offset = Var->getStackOffset(); 9638aa396610b7baf728631a43ea16ad3d13e38397aJim Stichnoth auto BaseRegNum = Var->getBaseRegNum(); 9645fa0a5f7f730f33e6987527a261de8d833d7585cReed Kotler if (BaseRegNum.hasNoValue()) 96528068adbf34a4602090efddc18b4dd123ffdeb6aJan Voung BaseRegNum = getFrameOrStackReg(); 966fe62f0a229a7b2bc910eb8b7cb83d2534ed39343Jim Stichnoth 967fe62f0a229a7b2bc910eb8b7cb83d2534ed39343Jim Stichnoth // Print in the form "Offset(%reg)", omitting Offset when it is 0. 968fe62f0a229a7b2bc910eb8b7cb83d2534ed39343Jim Stichnoth if (getFlags().getDecorateAsm()) { 969fe62f0a229a7b2bc910eb8b7cb83d2534ed39343Jim Stichnoth Str << Var->getSymbolicStackOffset(); 970fe62f0a229a7b2bc910eb8b7cb83d2534ed39343Jim Stichnoth } else if (Offset != 0) { 971fe62f0a229a7b2bc910eb8b7cb83d2534ed39343Jim Stichnoth Str << Offset; 97228068adbf34a4602090efddc18b4dd123ffdeb6aJan Voung } 9731d235425dab1f3dd059973fc53f1b1d5879469e3John Porto const Type FrameSPTy = Traits::WordType; 97428068adbf34a4602090efddc18b4dd123ffdeb6aJan Voung Str << "(%" << getRegName(BaseRegNum, FrameSPTy) << ")"; 9757e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto} 9767e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto 9774a56686b5b56db6803f90ad53514bf2fa190d9f7John Portotemplate <typename TraitsType> 9784a56686b5b56db6803f90ad53514bf2fa190d9f7John Portotypename TargetX86Base<TraitsType>::X86Address 9794a56686b5b56db6803f90ad53514bf2fa190d9f7John PortoTargetX86Base<TraitsType>::stackVarToAsmOperand(const Variable *Var) const { 9807e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto if (Var->hasReg()) 9818ff4b2819944bc4f02fb29204a1fa5ba7dea5682Jim Stichnoth llvm::report_fatal_error("Stack Variable has a register assigned"); 98211c9a325399b282cb4ea7d1d24d42fceeec2a09aAndrew Scull if (Var->mustHaveReg()) { 983a91c34118294efbf08ebd11eed96fce83bf35f3cJim Stichnoth llvm::report_fatal_error("Infinite-weight Variable (" + Var->getName() + 98445bec54fdb3d6f6c48fc4f55fc56bdbf9dd3326eJim Stichnoth ") has no register assigned - function " + 98545bec54fdb3d6f6c48fc4f55fc56bdbf9dd3326eJim Stichnoth Func->getFunctionName()); 9867e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto } 9877e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto int32_t Offset = Var->getStackOffset(); 9888aa396610b7baf728631a43ea16ad3d13e38397aJim Stichnoth auto BaseRegNum = Var->getBaseRegNum(); 9895fa0a5f7f730f33e6987527a261de8d833d7585cReed Kotler if (Var->getBaseRegNum().hasNoValue()) 99028068adbf34a4602090efddc18b4dd123ffdeb6aJan Voung BaseRegNum = getFrameOrStackReg(); 9914a56686b5b56db6803f90ad53514bf2fa190d9f7John Porto return X86Address(Traits::getEncodedGPR(BaseRegNum), Offset, 9924a56686b5b56db6803f90ad53514bf2fa190d9f7John Porto AssemblerFixup::NoFixup); 9937e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto} 9947e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto 995b9a404d433ca20a86ea2398555ecb48ba2cba3b5David Sehrtemplate <typename TraitsType> 996b9a404d433ca20a86ea2398555ecb48ba2cba3b5David Sehrvoid TargetX86Base<TraitsType>::addProlog(CfgNode *Node) { 997b9a404d433ca20a86ea2398555ecb48ba2cba3b5David Sehr // Stack frame layout: 998b9a404d433ca20a86ea2398555ecb48ba2cba3b5David Sehr // 999b9a404d433ca20a86ea2398555ecb48ba2cba3b5David Sehr // +------------------------+ 1000b9a404d433ca20a86ea2398555ecb48ba2cba3b5David Sehr // | 1. return address | 1001b9a404d433ca20a86ea2398555ecb48ba2cba3b5David Sehr // +------------------------+ 1002b9a404d433ca20a86ea2398555ecb48ba2cba3b5David Sehr // | 2. preserved registers | 10034e679e51f645b99940f43c80fec7f146d59eb00fNicolas Capens // +------------------------+ <--- BasePointer (if used) 1004b9a404d433ca20a86ea2398555ecb48ba2cba3b5David Sehr // | 3. padding | 1005b9a404d433ca20a86ea2398555ecb48ba2cba3b5David Sehr // +------------------------+ 1006b9a404d433ca20a86ea2398555ecb48ba2cba3b5David Sehr // | 4. global spill area | 1007b9a404d433ca20a86ea2398555ecb48ba2cba3b5David Sehr // +------------------------+ 1008b9a404d433ca20a86ea2398555ecb48ba2cba3b5David Sehr // | 5. padding | 1009b9a404d433ca20a86ea2398555ecb48ba2cba3b5David Sehr // +------------------------+ 1010b9a404d433ca20a86ea2398555ecb48ba2cba3b5David Sehr // | 6. local spill area | 1011b9a404d433ca20a86ea2398555ecb48ba2cba3b5David Sehr // +------------------------+ 1012b9a404d433ca20a86ea2398555ecb48ba2cba3b5David Sehr // | 7. padding | 1013b9a404d433ca20a86ea2398555ecb48ba2cba3b5David Sehr // +------------------------+ 1014b9a404d433ca20a86ea2398555ecb48ba2cba3b5David Sehr // | 8. allocas | 1015b9a404d433ca20a86ea2398555ecb48ba2cba3b5David Sehr // +------------------------+ 1016b9a404d433ca20a86ea2398555ecb48ba2cba3b5David Sehr // | 9. padding | 1017b9a404d433ca20a86ea2398555ecb48ba2cba3b5David Sehr // +------------------------+ 1018b9a404d433ca20a86ea2398555ecb48ba2cba3b5David Sehr // | 10. out args | 1019b9a404d433ca20a86ea2398555ecb48ba2cba3b5David Sehr // +------------------------+ <--- StackPointer 1020b9a404d433ca20a86ea2398555ecb48ba2cba3b5David Sehr // 1021b9a404d433ca20a86ea2398555ecb48ba2cba3b5David Sehr // The following variables record the size in bytes of the given areas: 10224e679e51f645b99940f43c80fec7f146d59eb00fNicolas Capens // * X86_RET_IP_SIZE_BYTES: area 1 10234e679e51f645b99940f43c80fec7f146d59eb00fNicolas Capens // * PreservedRegsSizeBytes: area 2 10244e679e51f645b99940f43c80fec7f146d59eb00fNicolas Capens // * SpillAreaPaddingBytes: area 3 10254e679e51f645b99940f43c80fec7f146d59eb00fNicolas Capens // * GlobalsSize: area 4 10264e679e51f645b99940f43c80fec7f146d59eb00fNicolas Capens // * LocalsSlotsPaddingBytes: area 5 1027b9a404d433ca20a86ea2398555ecb48ba2cba3b5David Sehr // * GlobalsAndSubsequentPaddingSize: areas 4 - 5 10284e679e51f645b99940f43c80fec7f146d59eb00fNicolas Capens // * LocalsSpillAreaSize: area 6 10294e679e51f645b99940f43c80fec7f146d59eb00fNicolas Capens // * FixedAllocaSizeBytes: areas 7 - 8 10304e679e51f645b99940f43c80fec7f146d59eb00fNicolas Capens // * SpillAreaSizeBytes: areas 3 - 10 10314e679e51f645b99940f43c80fec7f146d59eb00fNicolas Capens // * maxOutArgsSizeBytes(): areas 9 - 10 1032b9a404d433ca20a86ea2398555ecb48ba2cba3b5David Sehr 1033b9a404d433ca20a86ea2398555ecb48ba2cba3b5David Sehr // Determine stack frame offsets for each Variable without a register 1034b9a404d433ca20a86ea2398555ecb48ba2cba3b5David Sehr // assignment. This can be done as one variable per stack slot. Or, do 1035b9a404d433ca20a86ea2398555ecb48ba2cba3b5David Sehr // coalescing by running the register allocator again with an infinite set of 1036b9a404d433ca20a86ea2398555ecb48ba2cba3b5David Sehr // registers (as a side effect, this gives variables a second chance at 1037b9a404d433ca20a86ea2398555ecb48ba2cba3b5David Sehr // physical register assignment). 1038b9a404d433ca20a86ea2398555ecb48ba2cba3b5David Sehr // 1039b9a404d433ca20a86ea2398555ecb48ba2cba3b5David Sehr // A middle ground approach is to leverage sparsity and allocate one block of 1040b9a404d433ca20a86ea2398555ecb48ba2cba3b5David Sehr // space on the frame for globals (variables with multi-block lifetime), and 1041b9a404d433ca20a86ea2398555ecb48ba2cba3b5David Sehr // one block to share for locals (single-block lifetime). 1042b9a404d433ca20a86ea2398555ecb48ba2cba3b5David Sehr 1043b9a404d433ca20a86ea2398555ecb48ba2cba3b5David Sehr Context.init(Node); 1044b9a404d433ca20a86ea2398555ecb48ba2cba3b5David Sehr Context.setInsertPoint(Context.getCur()); 1045b9a404d433ca20a86ea2398555ecb48ba2cba3b5David Sehr 1046e82b560e649f8a68bcb252b9b002708e74d962d3John Porto SmallBitVector CalleeSaves = getRegisterSet(RegSet_CalleeSave, RegSet_None); 1047e82b560e649f8a68bcb252b9b002708e74d962d3John Porto RegsUsed = SmallBitVector(CalleeSaves.size()); 1048b9a404d433ca20a86ea2398555ecb48ba2cba3b5David Sehr VarList SortedSpilledVariables, VariablesLinkedToSpillSlots; 1049b9a404d433ca20a86ea2398555ecb48ba2cba3b5David Sehr size_t GlobalsSize = 0; 1050b9a404d433ca20a86ea2398555ecb48ba2cba3b5David Sehr // If there is a separate locals area, this represents that area. Otherwise 1051b9a404d433ca20a86ea2398555ecb48ba2cba3b5David Sehr // it counts any variable not counted by GlobalsSize. 1052b9a404d433ca20a86ea2398555ecb48ba2cba3b5David Sehr SpillAreaSizeBytes = 0; 1053b9a404d433ca20a86ea2398555ecb48ba2cba3b5David Sehr // If there is a separate locals area, this specifies the alignment for it. 1054b9a404d433ca20a86ea2398555ecb48ba2cba3b5David Sehr uint32_t LocalsSlotsAlignmentBytes = 0; 1055b9a404d433ca20a86ea2398555ecb48ba2cba3b5David Sehr // The entire spill locations area gets aligned to largest natural alignment 1056b9a404d433ca20a86ea2398555ecb48ba2cba3b5David Sehr // of the variables that have a spill slot. 1057b9a404d433ca20a86ea2398555ecb48ba2cba3b5David Sehr uint32_t SpillAreaAlignmentBytes = 0; 1058b9a404d433ca20a86ea2398555ecb48ba2cba3b5David Sehr // A spill slot linked to a variable with a stack slot should reuse that 1059b9a404d433ca20a86ea2398555ecb48ba2cba3b5David Sehr // stack slot. 1060b9a404d433ca20a86ea2398555ecb48ba2cba3b5David Sehr std::function<bool(Variable *)> TargetVarHook = 1061b9a404d433ca20a86ea2398555ecb48ba2cba3b5David Sehr [&VariablesLinkedToSpillSlots](Variable *Var) { 1062b9a847280e4486e566dabdd5b0d571309b4ad628Jim Stichnoth // TODO(stichnot): Refactor this into the base class. 1063b9a847280e4486e566dabdd5b0d571309b4ad628Jim Stichnoth Variable *Root = Var->getLinkedToStackRoot(); 1064b9a847280e4486e566dabdd5b0d571309b4ad628Jim Stichnoth if (Root != nullptr) { 1065b9a847280e4486e566dabdd5b0d571309b4ad628Jim Stichnoth assert(!Root->hasReg()); 1066b9a847280e4486e566dabdd5b0d571309b4ad628Jim Stichnoth if (!Root->hasReg()) { 1067b9a404d433ca20a86ea2398555ecb48ba2cba3b5David Sehr VariablesLinkedToSpillSlots.push_back(Var); 1068b9a404d433ca20a86ea2398555ecb48ba2cba3b5David Sehr return true; 1069b9a404d433ca20a86ea2398555ecb48ba2cba3b5David Sehr } 1070b9a404d433ca20a86ea2398555ecb48ba2cba3b5David Sehr } 1071b9a404d433ca20a86ea2398555ecb48ba2cba3b5David Sehr return false; 1072b9a404d433ca20a86ea2398555ecb48ba2cba3b5David Sehr }; 1073b9a404d433ca20a86ea2398555ecb48ba2cba3b5David Sehr 1074b9a404d433ca20a86ea2398555ecb48ba2cba3b5David Sehr // Compute the list of spilled variables and bounds for GlobalsSize, etc. 1075b9a404d433ca20a86ea2398555ecb48ba2cba3b5David Sehr getVarStackSlotParams(SortedSpilledVariables, RegsUsed, &GlobalsSize, 1076b9a404d433ca20a86ea2398555ecb48ba2cba3b5David Sehr &SpillAreaSizeBytes, &SpillAreaAlignmentBytes, 1077b9a404d433ca20a86ea2398555ecb48ba2cba3b5David Sehr &LocalsSlotsAlignmentBytes, TargetVarHook); 1078b9a404d433ca20a86ea2398555ecb48ba2cba3b5David Sehr uint32_t LocalsSpillAreaSize = SpillAreaSizeBytes; 1079b9a404d433ca20a86ea2398555ecb48ba2cba3b5David Sehr SpillAreaSizeBytes += GlobalsSize; 1080b9a404d433ca20a86ea2398555ecb48ba2cba3b5David Sehr 1081b9a404d433ca20a86ea2398555ecb48ba2cba3b5David Sehr // Add push instructions for preserved registers. 1082b9a404d433ca20a86ea2398555ecb48ba2cba3b5David Sehr uint32_t NumCallee = 0; 1083b9a404d433ca20a86ea2398555ecb48ba2cba3b5David Sehr size_t PreservedRegsSizeBytes = 0; 1084e82b560e649f8a68bcb252b9b002708e74d962d3John Porto SmallBitVector Pushed(CalleeSaves.size()); 10858aa396610b7baf728631a43ea16ad3d13e38397aJim Stichnoth for (RegNumT i : RegNumBVIter(CalleeSaves)) { 10868aa396610b7baf728631a43ea16ad3d13e38397aJim Stichnoth const auto Canonical = Traits::getBaseReg(i); 1087b9a404d433ca20a86ea2398555ecb48ba2cba3b5David Sehr assert(Canonical == Traits::getBaseReg(Canonical)); 10888aa396610b7baf728631a43ea16ad3d13e38397aJim Stichnoth if (RegsUsed[i]) { 1089b9a404d433ca20a86ea2398555ecb48ba2cba3b5David Sehr Pushed[Canonical] = true; 1090b9a404d433ca20a86ea2398555ecb48ba2cba3b5David Sehr } 1091b9a404d433ca20a86ea2398555ecb48ba2cba3b5David Sehr } 10928aa396610b7baf728631a43ea16ad3d13e38397aJim Stichnoth for (RegNumT RegNum : RegNumBVIter(Pushed)) { 10938aa396610b7baf728631a43ea16ad3d13e38397aJim Stichnoth assert(RegNum == Traits::getBaseReg(RegNum)); 1094b9a404d433ca20a86ea2398555ecb48ba2cba3b5David Sehr ++NumCallee; 1095b9a404d433ca20a86ea2398555ecb48ba2cba3b5David Sehr PreservedRegsSizeBytes += typeWidthInBytes(Traits::WordType); 10968aa396610b7baf728631a43ea16ad3d13e38397aJim Stichnoth _push_reg(getPhysicalRegister(RegNum, Traits::WordType)); 1097b9a404d433ca20a86ea2398555ecb48ba2cba3b5David Sehr } 1098b9a404d433ca20a86ea2398555ecb48ba2cba3b5David Sehr Ctx->statsUpdateRegistersSaved(NumCallee); 1099b9a404d433ca20a86ea2398555ecb48ba2cba3b5David Sehr 1100b9a404d433ca20a86ea2398555ecb48ba2cba3b5David Sehr // Generate "push frameptr; mov frameptr, stackptr" 1101b9a404d433ca20a86ea2398555ecb48ba2cba3b5David Sehr if (IsEbpBasedFrame) { 1102b9a404d433ca20a86ea2398555ecb48ba2cba3b5David Sehr assert((RegsUsed & getRegisterSet(RegSet_FramePointer, RegSet_None)) 1103b9a404d433ca20a86ea2398555ecb48ba2cba3b5David Sehr .count() == 0); 1104b9a404d433ca20a86ea2398555ecb48ba2cba3b5David Sehr PreservedRegsSizeBytes += typeWidthInBytes(Traits::WordType); 1105b9a404d433ca20a86ea2398555ecb48ba2cba3b5David Sehr _link_bp(); 1106b9a404d433ca20a86ea2398555ecb48ba2cba3b5David Sehr } 1107b9a404d433ca20a86ea2398555ecb48ba2cba3b5David Sehr 1108b9a404d433ca20a86ea2398555ecb48ba2cba3b5David Sehr // Align the variables area. SpillAreaPaddingBytes is the size of the region 1109b9a404d433ca20a86ea2398555ecb48ba2cba3b5David Sehr // after the preserved registers and before the spill areas. 1110b9a404d433ca20a86ea2398555ecb48ba2cba3b5David Sehr // LocalsSlotsPaddingBytes is the amount of padding between the globals and 1111b9a404d433ca20a86ea2398555ecb48ba2cba3b5David Sehr // locals area if they are separate. 1112b9a404d433ca20a86ea2398555ecb48ba2cba3b5David Sehr assert(LocalsSlotsAlignmentBytes <= SpillAreaAlignmentBytes); 1113b9a404d433ca20a86ea2398555ecb48ba2cba3b5David Sehr uint32_t SpillAreaPaddingBytes = 0; 1114b9a404d433ca20a86ea2398555ecb48ba2cba3b5David Sehr uint32_t LocalsSlotsPaddingBytes = 0; 1115b9a404d433ca20a86ea2398555ecb48ba2cba3b5David Sehr alignStackSpillAreas(Traits::X86_RET_IP_SIZE_BYTES + PreservedRegsSizeBytes, 1116b9a404d433ca20a86ea2398555ecb48ba2cba3b5David Sehr SpillAreaAlignmentBytes, GlobalsSize, 1117b9a404d433ca20a86ea2398555ecb48ba2cba3b5David Sehr LocalsSlotsAlignmentBytes, &SpillAreaPaddingBytes, 1118b9a404d433ca20a86ea2398555ecb48ba2cba3b5David Sehr &LocalsSlotsPaddingBytes); 1119b9a404d433ca20a86ea2398555ecb48ba2cba3b5David Sehr SpillAreaSizeBytes += SpillAreaPaddingBytes + LocalsSlotsPaddingBytes; 1120b9a404d433ca20a86ea2398555ecb48ba2cba3b5David Sehr uint32_t GlobalsAndSubsequentPaddingSize = 1121b9a404d433ca20a86ea2398555ecb48ba2cba3b5David Sehr GlobalsSize + LocalsSlotsPaddingBytes; 1122b9a404d433ca20a86ea2398555ecb48ba2cba3b5David Sehr 1123b9a404d433ca20a86ea2398555ecb48ba2cba3b5David Sehr // Functions returning scalar floating point types may need to convert values 1124b9a404d433ca20a86ea2398555ecb48ba2cba3b5David Sehr // from an in-register xmm value to the top of the x87 floating point stack. 1125b9a404d433ca20a86ea2398555ecb48ba2cba3b5David Sehr // This is done by a movp[sd] and an fld[sd]. Ensure there is enough scratch 1126b9a404d433ca20a86ea2398555ecb48ba2cba3b5David Sehr // space on the stack for this. 1127b9a404d433ca20a86ea2398555ecb48ba2cba3b5David Sehr const Type ReturnType = Func->getReturnType(); 1128b9a404d433ca20a86ea2398555ecb48ba2cba3b5David Sehr if (!Traits::X86_PASS_SCALAR_FP_IN_XMM) { 1129b9a404d433ca20a86ea2398555ecb48ba2cba3b5David Sehr if (isScalarFloatingType(ReturnType)) { 1130a551dfce9bd44034b4fd588cd4b06b3b0435df8dNicolas Capens // Avoid misaligned double-precision load/store. 1131a551dfce9bd44034b4fd588cd4b06b3b0435df8dNicolas Capens RequiredStackAlignment = std::max<size_t>( 1132a551dfce9bd44034b4fd588cd4b06b3b0435df8dNicolas Capens RequiredStackAlignment, Traits::X86_STACK_ALIGNMENT_BYTES); 1133b9a404d433ca20a86ea2398555ecb48ba2cba3b5David Sehr SpillAreaSizeBytes = 1134b9a404d433ca20a86ea2398555ecb48ba2cba3b5David Sehr std::max(typeWidthInBytesOnStack(ReturnType), SpillAreaSizeBytes); 1135b9a404d433ca20a86ea2398555ecb48ba2cba3b5David Sehr } 1136b9a404d433ca20a86ea2398555ecb48ba2cba3b5David Sehr } 1137b9a404d433ca20a86ea2398555ecb48ba2cba3b5David Sehr 1138a551dfce9bd44034b4fd588cd4b06b3b0435df8dNicolas Capens RequiredStackAlignment = 1139a551dfce9bd44034b4fd588cd4b06b3b0435df8dNicolas Capens std::max<size_t>(RequiredStackAlignment, SpillAreaAlignmentBytes); 1140a551dfce9bd44034b4fd588cd4b06b3b0435df8dNicolas Capens 1141a551dfce9bd44034b4fd588cd4b06b3b0435df8dNicolas Capens if (PrologEmitsFixedAllocas) { 1142a551dfce9bd44034b4fd588cd4b06b3b0435df8dNicolas Capens RequiredStackAlignment = 1143a551dfce9bd44034b4fd588cd4b06b3b0435df8dNicolas Capens std::max(RequiredStackAlignment, FixedAllocaAlignBytes); 1144b9a404d433ca20a86ea2398555ecb48ba2cba3b5David Sehr } 1145b9a404d433ca20a86ea2398555ecb48ba2cba3b5David Sehr 1146b9a404d433ca20a86ea2398555ecb48ba2cba3b5David Sehr // Combine fixed allocations into SpillAreaSizeBytes if we are emitting the 1147b9a404d433ca20a86ea2398555ecb48ba2cba3b5David Sehr // fixed allocations in the prolog. 1148b9a404d433ca20a86ea2398555ecb48ba2cba3b5David Sehr if (PrologEmitsFixedAllocas) 1149b9a404d433ca20a86ea2398555ecb48ba2cba3b5David Sehr SpillAreaSizeBytes += FixedAllocaSizeBytes; 1150a551dfce9bd44034b4fd588cd4b06b3b0435df8dNicolas Capens 1151a551dfce9bd44034b4fd588cd4b06b3b0435df8dNicolas Capens // Entering the function has made the stack pointer unaligned. Re-align it by 1152a551dfce9bd44034b4fd588cd4b06b3b0435df8dNicolas Capens // adjusting the stack size. 1153a551dfce9bd44034b4fd588cd4b06b3b0435df8dNicolas Capens uint32_t StackOffset = Traits::X86_RET_IP_SIZE_BYTES + PreservedRegsSizeBytes; 1154a551dfce9bd44034b4fd588cd4b06b3b0435df8dNicolas Capens uint32_t StackSize = Utils::applyAlignment(StackOffset + SpillAreaSizeBytes, 1155a551dfce9bd44034b4fd588cd4b06b3b0435df8dNicolas Capens RequiredStackAlignment); 1156a551dfce9bd44034b4fd588cd4b06b3b0435df8dNicolas Capens StackSize = Utils::applyAlignment(StackSize + maxOutArgsSizeBytes(), 1157a551dfce9bd44034b4fd588cd4b06b3b0435df8dNicolas Capens RequiredStackAlignment); 1158a551dfce9bd44034b4fd588cd4b06b3b0435df8dNicolas Capens SpillAreaSizeBytes = StackSize - StackOffset; 1159a551dfce9bd44034b4fd588cd4b06b3b0435df8dNicolas Capens 1160b9a404d433ca20a86ea2398555ecb48ba2cba3b5David Sehr if (SpillAreaSizeBytes) { 1161b9a404d433ca20a86ea2398555ecb48ba2cba3b5David Sehr // Generate "sub stackptr, SpillAreaSizeBytes" 1162b9a404d433ca20a86ea2398555ecb48ba2cba3b5David Sehr _sub_sp(Ctx->getConstantInt32(SpillAreaSizeBytes)); 1163a551dfce9bd44034b4fd588cd4b06b3b0435df8dNicolas Capens } 1164a551dfce9bd44034b4fd588cd4b06b3b0435df8dNicolas Capens 1165a551dfce9bd44034b4fd588cd4b06b3b0435df8dNicolas Capens // If the required alignment is greater than the stack pointer's guaranteed 1166a551dfce9bd44034b4fd588cd4b06b3b0435df8dNicolas Capens // alignment, align the stack pointer accordingly. 1167a551dfce9bd44034b4fd588cd4b06b3b0435df8dNicolas Capens if (RequiredStackAlignment > Traits::X86_STACK_ALIGNMENT_BYTES) { 1168a551dfce9bd44034b4fd588cd4b06b3b0435df8dNicolas Capens assert(IsEbpBasedFrame); 1169a551dfce9bd44034b4fd588cd4b06b3b0435df8dNicolas Capens _and(getPhysicalRegister(getStackReg(), Traits::WordType), 1170a551dfce9bd44034b4fd588cd4b06b3b0435df8dNicolas Capens Ctx->getConstantInt32(-RequiredStackAlignment)); 1171b9a404d433ca20a86ea2398555ecb48ba2cba3b5David Sehr } 1172b9a404d433ca20a86ea2398555ecb48ba2cba3b5David Sehr 1173b9a404d433ca20a86ea2398555ecb48ba2cba3b5David Sehr // Account for known-frame-offset alloca instructions that were not already 1174b9a404d433ca20a86ea2398555ecb48ba2cba3b5David Sehr // combined into the prolog. 1175b9a404d433ca20a86ea2398555ecb48ba2cba3b5David Sehr if (!PrologEmitsFixedAllocas) 1176b9a404d433ca20a86ea2398555ecb48ba2cba3b5David Sehr SpillAreaSizeBytes += FixedAllocaSizeBytes; 1177b9a404d433ca20a86ea2398555ecb48ba2cba3b5David Sehr 1178b9a404d433ca20a86ea2398555ecb48ba2cba3b5David Sehr Ctx->statsUpdateFrameBytes(SpillAreaSizeBytes); 1179b9a404d433ca20a86ea2398555ecb48ba2cba3b5David Sehr 1180b9a404d433ca20a86ea2398555ecb48ba2cba3b5David Sehr // Fill in stack offsets for stack args, and copy args into registers for 1181b9a404d433ca20a86ea2398555ecb48ba2cba3b5David Sehr // those that were register-allocated. Args are pushed right to left, so 1182b9a404d433ca20a86ea2398555ecb48ba2cba3b5David Sehr // Arg[0] is closest to the stack/frame pointer. 11834e679e51f645b99940f43c80fec7f146d59eb00fNicolas Capens RegNumT FrameOrStackReg = IsEbpBasedFrame ? getFrameReg() : getStackReg(); 11844e679e51f645b99940f43c80fec7f146d59eb00fNicolas Capens Variable *FramePtr = getPhysicalRegister(FrameOrStackReg, Traits::WordType); 1185b9a404d433ca20a86ea2398555ecb48ba2cba3b5David Sehr size_t BasicFrameOffset = 1186b9a404d433ca20a86ea2398555ecb48ba2cba3b5David Sehr PreservedRegsSizeBytes + Traits::X86_RET_IP_SIZE_BYTES; 1187b9a404d433ca20a86ea2398555ecb48ba2cba3b5David Sehr if (!IsEbpBasedFrame) 1188b9a404d433ca20a86ea2398555ecb48ba2cba3b5David Sehr BasicFrameOffset += SpillAreaSizeBytes; 1189b9a404d433ca20a86ea2398555ecb48ba2cba3b5David Sehr 1190b9a404d433ca20a86ea2398555ecb48ba2cba3b5David Sehr emitGetIP(Node); 1191b9a404d433ca20a86ea2398555ecb48ba2cba3b5David Sehr 1192b9a404d433ca20a86ea2398555ecb48ba2cba3b5David Sehr const VarList &Args = Func->getArgs(); 1193b9a404d433ca20a86ea2398555ecb48ba2cba3b5David Sehr size_t InArgsSizeBytes = 0; 1194b9a404d433ca20a86ea2398555ecb48ba2cba3b5David Sehr unsigned NumXmmArgs = 0; 1195b9a404d433ca20a86ea2398555ecb48ba2cba3b5David Sehr unsigned NumGPRArgs = 0; 1196b9a404d433ca20a86ea2398555ecb48ba2cba3b5David Sehr for (Variable *Arg : Args) { 1197b9a404d433ca20a86ea2398555ecb48ba2cba3b5David Sehr // Skip arguments passed in registers. 1198b9a404d433ca20a86ea2398555ecb48ba2cba3b5David Sehr if (isVectorType(Arg->getType())) { 11995fa0a5f7f730f33e6987527a261de8d833d7585cReed Kotler if (Traits::getRegisterForXmmArgNum(NumXmmArgs).hasValue()) { 1200b9a404d433ca20a86ea2398555ecb48ba2cba3b5David Sehr ++NumXmmArgs; 1201b9a404d433ca20a86ea2398555ecb48ba2cba3b5David Sehr continue; 1202b9a404d433ca20a86ea2398555ecb48ba2cba3b5David Sehr } 1203b9a404d433ca20a86ea2398555ecb48ba2cba3b5David Sehr } else if (isScalarFloatingType(Arg->getType())) { 1204b9a404d433ca20a86ea2398555ecb48ba2cba3b5David Sehr if (Traits::X86_PASS_SCALAR_FP_IN_XMM && 12055fa0a5f7f730f33e6987527a261de8d833d7585cReed Kotler Traits::getRegisterForXmmArgNum(NumXmmArgs).hasValue()) { 1206b9a404d433ca20a86ea2398555ecb48ba2cba3b5David Sehr ++NumXmmArgs; 1207b9a404d433ca20a86ea2398555ecb48ba2cba3b5David Sehr continue; 1208b9a404d433ca20a86ea2398555ecb48ba2cba3b5David Sehr } 1209b9a404d433ca20a86ea2398555ecb48ba2cba3b5David Sehr } else { 1210b9a404d433ca20a86ea2398555ecb48ba2cba3b5David Sehr assert(isScalarIntegerType(Arg->getType())); 12115fa0a5f7f730f33e6987527a261de8d833d7585cReed Kotler if (Traits::getRegisterForGprArgNum(Traits::WordType, NumGPRArgs) 12125fa0a5f7f730f33e6987527a261de8d833d7585cReed Kotler .hasValue()) { 1213b9a404d433ca20a86ea2398555ecb48ba2cba3b5David Sehr ++NumGPRArgs; 1214b9a404d433ca20a86ea2398555ecb48ba2cba3b5David Sehr continue; 1215b9a404d433ca20a86ea2398555ecb48ba2cba3b5David Sehr } 1216b9a404d433ca20a86ea2398555ecb48ba2cba3b5David Sehr } 1217b9a404d433ca20a86ea2398555ecb48ba2cba3b5David Sehr // For esp-based frames where the allocas are done outside the prolog, the 1218b9a404d433ca20a86ea2398555ecb48ba2cba3b5David Sehr // esp value may not stabilize to its home value until after all the 1219b9a404d433ca20a86ea2398555ecb48ba2cba3b5David Sehr // fixed-size alloca instructions have executed. In this case, a stack 1220b9a404d433ca20a86ea2398555ecb48ba2cba3b5David Sehr // adjustment is needed when accessing in-args in order to copy them into 1221b9a404d433ca20a86ea2398555ecb48ba2cba3b5David Sehr // registers. 1222b9a404d433ca20a86ea2398555ecb48ba2cba3b5David Sehr size_t StackAdjBytes = 0; 1223b9a404d433ca20a86ea2398555ecb48ba2cba3b5David Sehr if (!IsEbpBasedFrame && !PrologEmitsFixedAllocas) 1224b9a404d433ca20a86ea2398555ecb48ba2cba3b5David Sehr StackAdjBytes -= FixedAllocaSizeBytes; 1225b9a404d433ca20a86ea2398555ecb48ba2cba3b5David Sehr finishArgumentLowering(Arg, FramePtr, BasicFrameOffset, StackAdjBytes, 1226b9a404d433ca20a86ea2398555ecb48ba2cba3b5David Sehr InArgsSizeBytes); 1227b9a404d433ca20a86ea2398555ecb48ba2cba3b5David Sehr } 1228b9a404d433ca20a86ea2398555ecb48ba2cba3b5David Sehr 1229b9a404d433ca20a86ea2398555ecb48ba2cba3b5David Sehr // Fill in stack offsets for locals. 1230b9a404d433ca20a86ea2398555ecb48ba2cba3b5David Sehr assignVarStackSlots(SortedSpilledVariables, SpillAreaPaddingBytes, 1231b9a404d433ca20a86ea2398555ecb48ba2cba3b5David Sehr SpillAreaSizeBytes, GlobalsAndSubsequentPaddingSize, 12324e679e51f645b99940f43c80fec7f146d59eb00fNicolas Capens IsEbpBasedFrame && !needsStackPointerAlignment()); 1233b9a404d433ca20a86ea2398555ecb48ba2cba3b5David Sehr // Assign stack offsets to variables that have been linked to spilled 1234b9a404d433ca20a86ea2398555ecb48ba2cba3b5David Sehr // variables. 1235b9a404d433ca20a86ea2398555ecb48ba2cba3b5David Sehr for (Variable *Var : VariablesLinkedToSpillSlots) { 1236b9a847280e4486e566dabdd5b0d571309b4ad628Jim Stichnoth const Variable *Root = Var->getLinkedToStackRoot(); 1237fe62f0a229a7b2bc910eb8b7cb83d2534ed39343Jim Stichnoth assert(Root != nullptr); 1238fe62f0a229a7b2bc910eb8b7cb83d2534ed39343Jim Stichnoth Var->setStackOffset(Root->getStackOffset()); 1239b9a404d433ca20a86ea2398555ecb48ba2cba3b5David Sehr } 1240b9a404d433ca20a86ea2398555ecb48ba2cba3b5David Sehr this->HasComputedFrame = true; 1241b9a404d433ca20a86ea2398555ecb48ba2cba3b5David Sehr 1242b9a404d433ca20a86ea2398555ecb48ba2cba3b5David Sehr if (BuildDefs::dump() && Func->isVerbose(IceV_Frame)) { 1243b9a404d433ca20a86ea2398555ecb48ba2cba3b5David Sehr OstreamLocker L(Func->getContext()); 1244b9a404d433ca20a86ea2398555ecb48ba2cba3b5David Sehr Ostream &Str = Func->getContext()->getStrDump(); 1245b9a404d433ca20a86ea2398555ecb48ba2cba3b5David Sehr 1246b9a404d433ca20a86ea2398555ecb48ba2cba3b5David Sehr Str << "Stack layout:\n"; 1247b9a404d433ca20a86ea2398555ecb48ba2cba3b5David Sehr uint32_t EspAdjustmentPaddingSize = 1248b9a404d433ca20a86ea2398555ecb48ba2cba3b5David Sehr SpillAreaSizeBytes - LocalsSpillAreaSize - 1249b9a404d433ca20a86ea2398555ecb48ba2cba3b5David Sehr GlobalsAndSubsequentPaddingSize - SpillAreaPaddingBytes - 1250b9a404d433ca20a86ea2398555ecb48ba2cba3b5David Sehr maxOutArgsSizeBytes(); 1251b9a404d433ca20a86ea2398555ecb48ba2cba3b5David Sehr Str << " in-args = " << InArgsSizeBytes << " bytes\n" 1252b9a404d433ca20a86ea2398555ecb48ba2cba3b5David Sehr << " return address = " << Traits::X86_RET_IP_SIZE_BYTES << " bytes\n" 1253b9a404d433ca20a86ea2398555ecb48ba2cba3b5David Sehr << " preserved registers = " << PreservedRegsSizeBytes << " bytes\n" 1254b9a404d433ca20a86ea2398555ecb48ba2cba3b5David Sehr << " spill area padding = " << SpillAreaPaddingBytes << " bytes\n" 1255b9a404d433ca20a86ea2398555ecb48ba2cba3b5David Sehr << " globals spill area = " << GlobalsSize << " bytes\n" 1256b9a404d433ca20a86ea2398555ecb48ba2cba3b5David Sehr << " globals-locals spill areas intermediate padding = " 1257b9a404d433ca20a86ea2398555ecb48ba2cba3b5David Sehr << GlobalsAndSubsequentPaddingSize - GlobalsSize << " bytes\n" 1258b9a404d433ca20a86ea2398555ecb48ba2cba3b5David Sehr << " locals spill area = " << LocalsSpillAreaSize << " bytes\n" 1259b9a404d433ca20a86ea2398555ecb48ba2cba3b5David Sehr << " esp alignment padding = " << EspAdjustmentPaddingSize 1260b9a404d433ca20a86ea2398555ecb48ba2cba3b5David Sehr << " bytes\n"; 1261b9a404d433ca20a86ea2398555ecb48ba2cba3b5David Sehr 1262b9a404d433ca20a86ea2398555ecb48ba2cba3b5David Sehr Str << "Stack details:\n" 1263b9a404d433ca20a86ea2398555ecb48ba2cba3b5David Sehr << " esp adjustment = " << SpillAreaSizeBytes << " bytes\n" 1264b9a404d433ca20a86ea2398555ecb48ba2cba3b5David Sehr << " spill area alignment = " << SpillAreaAlignmentBytes << " bytes\n" 1265b9a404d433ca20a86ea2398555ecb48ba2cba3b5David Sehr << " outgoing args size = " << maxOutArgsSizeBytes() << " bytes\n" 1266b9a404d433ca20a86ea2398555ecb48ba2cba3b5David Sehr << " locals spill area alignment = " << LocalsSlotsAlignmentBytes 1267b9a404d433ca20a86ea2398555ecb48ba2cba3b5David Sehr << " bytes\n" 1268b9a404d433ca20a86ea2398555ecb48ba2cba3b5David Sehr << " is ebp based = " << IsEbpBasedFrame << "\n"; 1269b9a404d433ca20a86ea2398555ecb48ba2cba3b5David Sehr } 1270b9a404d433ca20a86ea2398555ecb48ba2cba3b5David Sehr} 1271b9a404d433ca20a86ea2398555ecb48ba2cba3b5David Sehr 12729612d32c7e5eb2cb403686ef31172d42e075e460Andrew Scull/// Helper function for addProlog(). 12739612d32c7e5eb2cb403686ef31172d42e075e460Andrew Scull/// 127457e126899b20c65ff3ea23a3b7d7a67ab30b99dcAndrew Scull/// This assumes Arg is an argument passed on the stack. This sets the frame 127557e126899b20c65ff3ea23a3b7d7a67ab30b99dcAndrew Scull/// offset for Arg and updates InArgsSizeBytes according to Arg's width. For an 127657e126899b20c65ff3ea23a3b7d7a67ab30b99dcAndrew Scull/// I64 arg that has been split into Lo and Hi components, it calls itself 127757e126899b20c65ff3ea23a3b7d7a67ab30b99dcAndrew Scull/// recursively on the components, taking care to handle Lo first because of the 127857e126899b20c65ff3ea23a3b7d7a67ab30b99dcAndrew Scull/// little-endian architecture. Lastly, this function generates an instruction 127957e126899b20c65ff3ea23a3b7d7a67ab30b99dcAndrew Scull/// to copy Arg into its assigned register if applicable. 12804a56686b5b56db6803f90ad53514bf2fa190d9f7John Portotemplate <typename TraitsType> 12814a56686b5b56db6803f90ad53514bf2fa190d9f7John Portovoid TargetX86Base<TraitsType>::finishArgumentLowering( 12824a56686b5b56db6803f90ad53514bf2fa190d9f7John Porto Variable *Arg, Variable *FramePtr, size_t BasicFrameOffset, 12834a56686b5b56db6803f90ad53514bf2fa190d9f7John Porto size_t StackAdjBytes, size_t &InArgsSizeBytes) { 12846d47bcdcf16ce8eddc737d4f26baad9653604048Andrew Scull if (!Traits::Is64Bit) { 12856d47bcdcf16ce8eddc737d4f26baad9653604048Andrew Scull if (auto *Arg64On32 = llvm::dyn_cast<Variable64On32>(Arg)) { 12866d47bcdcf16ce8eddc737d4f26baad9653604048Andrew Scull Variable *Lo = Arg64On32->getLo(); 12876d47bcdcf16ce8eddc737d4f26baad9653604048Andrew Scull Variable *Hi = Arg64On32->getHi(); 128855f931f65f307116c2def40c3b11ec93998a907eJim Stichnoth finishArgumentLowering(Lo, FramePtr, BasicFrameOffset, StackAdjBytes, 128955f931f65f307116c2def40c3b11ec93998a907eJim Stichnoth InArgsSizeBytes); 129055f931f65f307116c2def40c3b11ec93998a907eJim Stichnoth finishArgumentLowering(Hi, FramePtr, BasicFrameOffset, StackAdjBytes, 129155f931f65f307116c2def40c3b11ec93998a907eJim Stichnoth InArgsSizeBytes); 12926d47bcdcf16ce8eddc737d4f26baad9653604048Andrew Scull return; 12936d47bcdcf16ce8eddc737d4f26baad9653604048Andrew Scull } 12947e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto } 12956d47bcdcf16ce8eddc737d4f26baad9653604048Andrew Scull Type Ty = Arg->getType(); 12967e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto if (isVectorType(Ty)) { 12977e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto InArgsSizeBytes = Traits::applyStackAlignment(InArgsSizeBytes); 12987e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto } 12997e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto Arg->setStackOffset(BasicFrameOffset + InArgsSizeBytes); 13007e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto InArgsSizeBytes += typeWidthInBytesOnStack(Ty); 13017e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto if (Arg->hasReg()) { 13021d235425dab1f3dd059973fc53f1b1d5879469e3John Porto assert(Ty != IceType_i64 || Traits::Is64Bit); 13034a56686b5b56db6803f90ad53514bf2fa190d9f7John Porto auto *Mem = X86OperandMem::create( 130455f931f65f307116c2def40c3b11ec93998a907eJim Stichnoth Func, Ty, FramePtr, 130555f931f65f307116c2def40c3b11ec93998a907eJim Stichnoth Ctx->getConstantInt32(Arg->getStackOffset() + StackAdjBytes)); 13067e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto if (isVectorType(Arg->getType())) { 13077e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto _movp(Arg, Mem); 13087e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto } else { 13097e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto _mov(Arg, Mem); 13107e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto } 13114a56686b5b56db6803f90ad53514bf2fa190d9f7John Porto // This argument-copying instruction uses an explicit X86OperandMem 131257e126899b20c65ff3ea23a3b7d7a67ab30b99dcAndrew Scull // operand instead of a Variable, so its fill-from-stack operation has to 131357e126899b20c65ff3ea23a3b7d7a67ab30b99dcAndrew Scull // be tracked separately for statistics. 13147e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto Ctx->statsUpdateFills(); 13157e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto } 13167e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto} 13177e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto 1318b9a404d433ca20a86ea2398555ecb48ba2cba3b5David Sehrtemplate <typename TraitsType> 1319b9a404d433ca20a86ea2398555ecb48ba2cba3b5David Sehrvoid TargetX86Base<TraitsType>::addEpilog(CfgNode *Node) { 1320b9a404d433ca20a86ea2398555ecb48ba2cba3b5David Sehr InstList &Insts = Node->getInsts(); 1321b9a404d433ca20a86ea2398555ecb48ba2cba3b5David Sehr InstList::reverse_iterator RI, E; 1322b9a404d433ca20a86ea2398555ecb48ba2cba3b5David Sehr for (RI = Insts.rbegin(), E = Insts.rend(); RI != E; ++RI) { 1323b9a404d433ca20a86ea2398555ecb48ba2cba3b5David Sehr if (llvm::isa<typename Traits::Insts::Ret>(*RI)) 1324b9a404d433ca20a86ea2398555ecb48ba2cba3b5David Sehr break; 1325b9a404d433ca20a86ea2398555ecb48ba2cba3b5David Sehr } 1326b9a404d433ca20a86ea2398555ecb48ba2cba3b5David Sehr if (RI == E) 1327b9a404d433ca20a86ea2398555ecb48ba2cba3b5David Sehr return; 1328b9a404d433ca20a86ea2398555ecb48ba2cba3b5David Sehr 1329b9a404d433ca20a86ea2398555ecb48ba2cba3b5David Sehr // Convert the reverse_iterator position into its corresponding (forward) 1330b9a404d433ca20a86ea2398555ecb48ba2cba3b5David Sehr // iterator position. 13317c9728faead1368150bcaa8313bc460df29ea1b7Jim Stichnoth InstList::iterator InsertPoint = reverseToForwardIterator(RI); 1332b9a404d433ca20a86ea2398555ecb48ba2cba3b5David Sehr --InsertPoint; 1333b9a404d433ca20a86ea2398555ecb48ba2cba3b5David Sehr Context.init(Node); 1334b9a404d433ca20a86ea2398555ecb48ba2cba3b5David Sehr Context.setInsertPoint(InsertPoint); 1335b9a404d433ca20a86ea2398555ecb48ba2cba3b5David Sehr 1336b9a404d433ca20a86ea2398555ecb48ba2cba3b5David Sehr if (IsEbpBasedFrame) { 1337b9a404d433ca20a86ea2398555ecb48ba2cba3b5David Sehr _unlink_bp(); 1338b9a404d433ca20a86ea2398555ecb48ba2cba3b5David Sehr } else { 1339b9a404d433ca20a86ea2398555ecb48ba2cba3b5David Sehr // add stackptr, SpillAreaSizeBytes 1340b9a404d433ca20a86ea2398555ecb48ba2cba3b5David Sehr if (SpillAreaSizeBytes != 0) { 1341b9a404d433ca20a86ea2398555ecb48ba2cba3b5David Sehr _add_sp(Ctx->getConstantInt32(SpillAreaSizeBytes)); 1342b9a404d433ca20a86ea2398555ecb48ba2cba3b5David Sehr } 1343b9a404d433ca20a86ea2398555ecb48ba2cba3b5David Sehr } 1344b9a404d433ca20a86ea2398555ecb48ba2cba3b5David Sehr 1345b9a404d433ca20a86ea2398555ecb48ba2cba3b5David Sehr // Add pop instructions for preserved registers. 1346e82b560e649f8a68bcb252b9b002708e74d962d3John Porto SmallBitVector CalleeSaves = getRegisterSet(RegSet_CalleeSave, RegSet_None); 1347e82b560e649f8a68bcb252b9b002708e74d962d3John Porto SmallBitVector Popped(CalleeSaves.size()); 1348b9a404d433ca20a86ea2398555ecb48ba2cba3b5David Sehr for (int32_t i = CalleeSaves.size() - 1; i >= 0; --i) { 13498aa396610b7baf728631a43ea16ad3d13e38397aJim Stichnoth const auto RegNum = RegNumT::fromInt(i); 13508aa396610b7baf728631a43ea16ad3d13e38397aJim Stichnoth if (RegNum == getFrameReg() && IsEbpBasedFrame) 1351b9a404d433ca20a86ea2398555ecb48ba2cba3b5David Sehr continue; 13528aa396610b7baf728631a43ea16ad3d13e38397aJim Stichnoth const RegNumT Canonical = Traits::getBaseReg(RegNum); 1353b9a404d433ca20a86ea2398555ecb48ba2cba3b5David Sehr if (CalleeSaves[i] && RegsUsed[i]) { 1354b9a404d433ca20a86ea2398555ecb48ba2cba3b5David Sehr Popped[Canonical] = true; 1355b9a404d433ca20a86ea2398555ecb48ba2cba3b5David Sehr } 1356b9a404d433ca20a86ea2398555ecb48ba2cba3b5David Sehr } 1357b9a404d433ca20a86ea2398555ecb48ba2cba3b5David Sehr for (int32_t i = Popped.size() - 1; i >= 0; --i) { 1358b9a404d433ca20a86ea2398555ecb48ba2cba3b5David Sehr if (!Popped[i]) 1359b9a404d433ca20a86ea2398555ecb48ba2cba3b5David Sehr continue; 13608aa396610b7baf728631a43ea16ad3d13e38397aJim Stichnoth const auto RegNum = RegNumT::fromInt(i); 13618aa396610b7baf728631a43ea16ad3d13e38397aJim Stichnoth assert(RegNum == Traits::getBaseReg(RegNum)); 13628aa396610b7baf728631a43ea16ad3d13e38397aJim Stichnoth _pop(getPhysicalRegister(RegNum, Traits::WordType)); 1363b9a404d433ca20a86ea2398555ecb48ba2cba3b5David Sehr } 1364b9a404d433ca20a86ea2398555ecb48ba2cba3b5David Sehr 1365b9a404d433ca20a86ea2398555ecb48ba2cba3b5David Sehr if (!NeedSandboxing) { 1366b9a404d433ca20a86ea2398555ecb48ba2cba3b5David Sehr return; 1367b9a404d433ca20a86ea2398555ecb48ba2cba3b5David Sehr } 1368b9a404d433ca20a86ea2398555ecb48ba2cba3b5David Sehr emitSandboxedReturn(); 1369b9a404d433ca20a86ea2398555ecb48ba2cba3b5David Sehr if (RI->getSrcSize()) { 1370b9a404d433ca20a86ea2398555ecb48ba2cba3b5David Sehr auto *RetValue = llvm::cast<Variable>(RI->getSrc(0)); 1371b9a404d433ca20a86ea2398555ecb48ba2cba3b5David Sehr Context.insert<InstFakeUse>(RetValue); 1372b9a404d433ca20a86ea2398555ecb48ba2cba3b5David Sehr } 1373b9a404d433ca20a86ea2398555ecb48ba2cba3b5David Sehr RI->setDeleted(); 1374b9a404d433ca20a86ea2398555ecb48ba2cba3b5David Sehr} 1375b9a404d433ca20a86ea2398555ecb48ba2cba3b5David Sehr 13764a56686b5b56db6803f90ad53514bf2fa190d9f7John Portotemplate <typename TraitsType> Type TargetX86Base<TraitsType>::stackSlotType() { 13771d235425dab1f3dd059973fc53f1b1d5879469e3John Porto return Traits::WordType; 13787e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto} 13797e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto 13804a56686b5b56db6803f90ad53514bf2fa190d9f7John Portotemplate <typename TraitsType> 13811d235425dab1f3dd059973fc53f1b1d5879469e3John Portotemplate <typename T> 13821d235425dab1f3dd059973fc53f1b1d5879469e3John Portotypename std::enable_if<!T::Is64Bit, Operand>::type * 13834a56686b5b56db6803f90ad53514bf2fa190d9f7John PortoTargetX86Base<TraitsType>::loOperand(Operand *Operand) { 13847e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto assert(Operand->getType() == IceType_i64 || 13857e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto Operand->getType() == IceType_f64); 13867e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto if (Operand->getType() != IceType_i64 && Operand->getType() != IceType_f64) 13877e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto return Operand; 13886d47bcdcf16ce8eddc737d4f26baad9653604048Andrew Scull if (auto *Var64On32 = llvm::dyn_cast<Variable64On32>(Operand)) 13896d47bcdcf16ce8eddc737d4f26baad9653604048Andrew Scull return Var64On32->getLo(); 1390fbdd244013aad0808220386eb6bd17eb0635dd4cJan Voung if (auto *Const = llvm::dyn_cast<ConstantInteger64>(Operand)) { 1391fbdd244013aad0808220386eb6bd17eb0635dd4cJan Voung auto *ConstInt = llvm::dyn_cast<ConstantInteger32>( 13927e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto Ctx->getConstantInt32(static_cast<int32_t>(Const->getValue()))); 1393fbdd244013aad0808220386eb6bd17eb0635dd4cJan Voung // Check if we need to blind/pool the constant. 13947e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto return legalize(ConstInt); 13957e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto } 13964a56686b5b56db6803f90ad53514bf2fa190d9f7John Porto if (auto *Mem = llvm::dyn_cast<X86OperandMem>(Operand)) { 13974a56686b5b56db6803f90ad53514bf2fa190d9f7John Porto auto *MemOperand = X86OperandMem::create( 13987e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto Func, IceType_i32, Mem->getBase(), Mem->getOffset(), Mem->getIndex(), 139956958cb33d3c1d045f2844408d825442d523f59fJohn Porto Mem->getShift(), Mem->getSegmentRegister(), Mem->getIsRebased()); 14007e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto // Test if we should randomize or pool the offset, if so randomize it or 14017e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto // pool it then create mem operand with the blinded/pooled constant. 14027e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto // Otherwise, return the mem operand as ordinary mem operand. 14037e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto return legalize(MemOperand); 14047e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto } 14057e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto llvm_unreachable("Unsupported operand type"); 14067e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto return nullptr; 14077e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto} 14087e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto 14094a56686b5b56db6803f90ad53514bf2fa190d9f7John Portotemplate <typename TraitsType> 14101d235425dab1f3dd059973fc53f1b1d5879469e3John Portotemplate <typename T> 14111d235425dab1f3dd059973fc53f1b1d5879469e3John Portotypename std::enable_if<!T::Is64Bit, Operand>::type * 14124a56686b5b56db6803f90ad53514bf2fa190d9f7John PortoTargetX86Base<TraitsType>::hiOperand(Operand *Operand) { 14137e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto assert(Operand->getType() == IceType_i64 || 14147e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto Operand->getType() == IceType_f64); 14157e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto if (Operand->getType() != IceType_i64 && Operand->getType() != IceType_f64) 14167e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto return Operand; 14176d47bcdcf16ce8eddc737d4f26baad9653604048Andrew Scull if (auto *Var64On32 = llvm::dyn_cast<Variable64On32>(Operand)) 14186d47bcdcf16ce8eddc737d4f26baad9653604048Andrew Scull return Var64On32->getHi(); 1419fbdd244013aad0808220386eb6bd17eb0635dd4cJan Voung if (auto *Const = llvm::dyn_cast<ConstantInteger64>(Operand)) { 1420fbdd244013aad0808220386eb6bd17eb0635dd4cJan Voung auto *ConstInt = llvm::dyn_cast<ConstantInteger32>( 14217e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto Ctx->getConstantInt32(static_cast<int32_t>(Const->getValue() >> 32))); 1422fbdd244013aad0808220386eb6bd17eb0635dd4cJan Voung // Check if we need to blind/pool the constant. 14237e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto return legalize(ConstInt); 14247e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto } 14254a56686b5b56db6803f90ad53514bf2fa190d9f7John Porto if (auto *Mem = llvm::dyn_cast<X86OperandMem>(Operand)) { 14267e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto Constant *Offset = Mem->getOffset(); 14277e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto if (Offset == nullptr) { 14287e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto Offset = Ctx->getConstantInt32(4); 1429fbdd244013aad0808220386eb6bd17eb0635dd4cJan Voung } else if (auto *IntOffset = llvm::dyn_cast<ConstantInteger32>(Offset)) { 14307e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto Offset = Ctx->getConstantInt32(4 + IntOffset->getValue()); 1431fbdd244013aad0808220386eb6bd17eb0635dd4cJan Voung } else if (auto *SymOffset = llvm::dyn_cast<ConstantRelocatable>(Offset)) { 14327e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto assert(!Utils::WouldOverflowAdd(SymOffset->getOffset(), 4)); 14337e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto Offset = 143498ba00666271be1bdcd45b72b3dec04419efe61bJim Stichnoth Ctx->getConstantSym(4 + SymOffset->getOffset(), SymOffset->getName()); 14357e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto } 14364a56686b5b56db6803f90ad53514bf2fa190d9f7John Porto auto *MemOperand = X86OperandMem::create( 14377e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto Func, IceType_i32, Mem->getBase(), Offset, Mem->getIndex(), 143856958cb33d3c1d045f2844408d825442d523f59fJohn Porto Mem->getShift(), Mem->getSegmentRegister(), Mem->getIsRebased()); 14397e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto // Test if the Offset is an eligible i32 constants for randomization and 14407e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto // pooling. Blind/pool it if it is. Otherwise return as oridinary mem 14417e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto // operand. 14427e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto return legalize(MemOperand); 14437e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto } 14447e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto llvm_unreachable("Unsupported operand type"); 14457e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto return nullptr; 14467e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto} 14477e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto 14484a56686b5b56db6803f90ad53514bf2fa190d9f7John Portotemplate <typename TraitsType> 1449e82b560e649f8a68bcb252b9b002708e74d962d3John PortoSmallBitVector 14504a56686b5b56db6803f90ad53514bf2fa190d9f7John PortoTargetX86Base<TraitsType>::getRegisterSet(RegSetMask Include, 14514a56686b5b56db6803f90ad53514bf2fa190d9f7John Porto RegSetMask Exclude) const { 1452d46999474d2b4a388e1d8a7c71f06cd4cec51bfcKarl Schimpf return Traits::getRegisterSet(getFlags(), Include, Exclude); 14537e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto} 14547e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto 14554a56686b5b56db6803f90ad53514bf2fa190d9f7John Portotemplate <typename TraitsType> 14568cfeb69e17190d3bfe22a8a1cbd7679a114d68cfJim Stichnothvoid TargetX86Base<TraitsType>::lowerAlloca(const InstAlloca *Instr) { 145757e126899b20c65ff3ea23a3b7d7a67ab30b99dcAndrew Scull // Conservatively require the stack to be aligned. Some stack adjustment 145857e126899b20c65ff3ea23a3b7d7a67ab30b99dcAndrew Scull // operations implemented below assume that the stack is aligned before the 145957e126899b20c65ff3ea23a3b7d7a67ab30b99dcAndrew Scull // alloca. All the alloca code ensures that the stack alignment is preserved 146057e126899b20c65ff3ea23a3b7d7a67ab30b99dcAndrew Scull // after the alloca. The stack alignment restriction can be relaxed in some 146157e126899b20c65ff3ea23a3b7d7a67ab30b99dcAndrew Scull // cases. 1462a551dfce9bd44034b4fd588cd4b06b3b0435df8dNicolas Capens RequiredStackAlignment = std::max<size_t>(RequiredStackAlignment, 1463a551dfce9bd44034b4fd588cd4b06b3b0435df8dNicolas Capens Traits::X86_STACK_ALIGNMENT_BYTES); 14647e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto 14657e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto // For default align=0, set it to the real value 1, to avoid any 14667e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto // bit-manipulation problems below. 14678cfeb69e17190d3bfe22a8a1cbd7679a114d68cfJim Stichnoth const uint32_t AlignmentParam = std::max(1u, Instr->getAlignInBytes()); 14687e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto 14697e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto // LLVM enforces power of 2 alignment. 14707e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto assert(llvm::isPowerOf2_32(AlignmentParam)); 14717e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto assert(llvm::isPowerOf2_32(Traits::X86_STACK_ALIGNMENT_BYTES)); 14727e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto 147326217e3333150e66fc96aca79c01105906797960David Sehr const uint32_t Alignment = 14747e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto std::max(AlignmentParam, Traits::X86_STACK_ALIGNMENT_BYTES); 147526217e3333150e66fc96aca79c01105906797960David Sehr const bool OverAligned = Alignment > Traits::X86_STACK_ALIGNMENT_BYTES; 1476dd6dcfaf765dc93ae64ec45d623106f4b3a3c13aJim Stichnoth const bool OptM1 = Func->getOptLevel() == Opt_m1; 14778cfeb69e17190d3bfe22a8a1cbd7679a114d68cfJim Stichnoth const bool AllocaWithKnownOffset = Instr->getKnownFrameOffset(); 147826217e3333150e66fc96aca79c01105906797960David Sehr const bool UseFramePointer = 147926217e3333150e66fc96aca79c01105906797960David Sehr hasFramePointer() || OverAligned || !AllocaWithKnownOffset || OptM1; 148026217e3333150e66fc96aca79c01105906797960David Sehr 148126217e3333150e66fc96aca79c01105906797960David Sehr if (UseFramePointer) 14824318a410939b5cee89736f6f15a3185e385b5dc7David Sehr setHasFramePointer(); 148326217e3333150e66fc96aca79c01105906797960David Sehr 1484008f4ce5417c97891ea02b54b691b39aa2e2a0afJohn Porto Variable *esp = getPhysicalRegister(getStackReg(), Traits::WordType); 148526217e3333150e66fc96aca79c01105906797960David Sehr if (OverAligned) { 14867e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto _and(esp, Ctx->getConstantInt32(-Alignment)); 14877e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto } 148826217e3333150e66fc96aca79c01105906797960David Sehr 14898cfeb69e17190d3bfe22a8a1cbd7679a114d68cfJim Stichnoth Variable *Dest = Instr->getDest(); 14908cfeb69e17190d3bfe22a8a1cbd7679a114d68cfJim Stichnoth Operand *TotalSize = legalize(Instr->getSizeInBytes()); 149126217e3333150e66fc96aca79c01105906797960David Sehr 14927e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto if (const auto *ConstantTotalSize = 14937e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto llvm::dyn_cast<ConstantInteger32>(TotalSize)) { 149426217e3333150e66fc96aca79c01105906797960David Sehr const uint32_t Value = 149526217e3333150e66fc96aca79c01105906797960David Sehr Utils::applyAlignment(ConstantTotalSize->getValue(), Alignment); 149656958cb33d3c1d045f2844408d825442d523f59fJohn Porto if (UseFramePointer) { 149756958cb33d3c1d045f2844408d825442d523f59fJohn Porto _sub_sp(Ctx->getConstantInt32(Value)); 149856958cb33d3c1d045f2844408d825442d523f59fJohn Porto } else { 149926217e3333150e66fc96aca79c01105906797960David Sehr // If we don't need a Frame Pointer, this alloca has a known offset to the 150026217e3333150e66fc96aca79c01105906797960David Sehr // stack pointer. We don't need adjust the stack pointer, nor assign any 150126217e3333150e66fc96aca79c01105906797960David Sehr // value to Dest, as Dest is rematerializable. 150226217e3333150e66fc96aca79c01105906797960David Sehr assert(Dest->isRematerializable()); 150355f931f65f307116c2def40c3b11ec93998a907eJim Stichnoth FixedAllocaSizeBytes += Value; 15041d937a8e35b7ae51ee64c32f8f34d51ba33498f4John Porto Context.insert<InstFakeDef>(Dest); 150555f931f65f307116c2def40c3b11ec93998a907eJim Stichnoth } 15067e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto } else { 150757e126899b20c65ff3ea23a3b7d7a67ab30b99dcAndrew Scull // Non-constant sizes need to be adjusted to the next highest multiple of 150857e126899b20c65ff3ea23a3b7d7a67ab30b99dcAndrew Scull // the required alignment at runtime. 150956958cb33d3c1d045f2844408d825442d523f59fJohn Porto Variable *T = nullptr; 151056958cb33d3c1d045f2844408d825442d523f59fJohn Porto if (Traits::Is64Bit && TotalSize->getType() != IceType_i64 && 151156958cb33d3c1d045f2844408d825442d523f59fJohn Porto !NeedSandboxing) { 151256958cb33d3c1d045f2844408d825442d523f59fJohn Porto T = makeReg(IceType_i64); 1513008f4ce5417c97891ea02b54b691b39aa2e2a0afJohn Porto _movzx(T, TotalSize); 1514008f4ce5417c97891ea02b54b691b39aa2e2a0afJohn Porto } else { 151556958cb33d3c1d045f2844408d825442d523f59fJohn Porto T = makeReg(IceType_i32); 1516008f4ce5417c97891ea02b54b691b39aa2e2a0afJohn Porto _mov(T, TotalSize); 1517008f4ce5417c97891ea02b54b691b39aa2e2a0afJohn Porto } 15187e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto _add(T, Ctx->getConstantInt32(Alignment - 1)); 15197e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto _and(T, Ctx->getConstantInt32(-Alignment)); 152056958cb33d3c1d045f2844408d825442d523f59fJohn Porto _sub_sp(T); 15217e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto } 152226217e3333150e66fc96aca79c01105906797960David Sehr // Add enough to the returned address to account for the out args area. 152326217e3333150e66fc96aca79c01105906797960David Sehr uint32_t OutArgsSize = maxOutArgsSizeBytes(); 152426217e3333150e66fc96aca79c01105906797960David Sehr if (OutArgsSize > 0) { 152526217e3333150e66fc96aca79c01105906797960David Sehr Variable *T = makeReg(IceType_i32); 15264a56686b5b56db6803f90ad53514bf2fa190d9f7John Porto auto *CalculateOperand = X86OperandMem::create( 152756958cb33d3c1d045f2844408d825442d523f59fJohn Porto Func, IceType_void, esp, Ctx->getConstantInt(IceType_i32, OutArgsSize)); 152826217e3333150e66fc96aca79c01105906797960David Sehr _lea(T, CalculateOperand); 152926217e3333150e66fc96aca79c01105906797960David Sehr _mov(Dest, T); 153026217e3333150e66fc96aca79c01105906797960David Sehr } else { 153126217e3333150e66fc96aca79c01105906797960David Sehr _mov(Dest, esp); 153226217e3333150e66fc96aca79c01105906797960David Sehr } 15337e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto} 15347e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto 15350c68bef895ea8987922922ede6f1918763eaa00bDavid Sehrtemplate <typename TraitsType> 15360c68bef895ea8987922922ede6f1918763eaa00bDavid Sehrvoid TargetX86Base<TraitsType>::lowerArguments() { 1537f531931fb45172157bc5816a8f55309556996dc7Jim Stichnoth const bool OptM1 = Func->getOptLevel() == Opt_m1; 15380c68bef895ea8987922922ede6f1918763eaa00bDavid Sehr VarList &Args = Func->getArgs(); 15390c68bef895ea8987922922ede6f1918763eaa00bDavid Sehr unsigned NumXmmArgs = 0; 15400c68bef895ea8987922922ede6f1918763eaa00bDavid Sehr bool XmmSlotsRemain = true; 15410c68bef895ea8987922922ede6f1918763eaa00bDavid Sehr unsigned NumGprArgs = 0; 15420c68bef895ea8987922922ede6f1918763eaa00bDavid Sehr bool GprSlotsRemain = true; 15430c68bef895ea8987922922ede6f1918763eaa00bDavid Sehr 15440c68bef895ea8987922922ede6f1918763eaa00bDavid Sehr Context.init(Func->getEntryNode()); 15450c68bef895ea8987922922ede6f1918763eaa00bDavid Sehr Context.setInsertPoint(Context.getCur()); 15460c68bef895ea8987922922ede6f1918763eaa00bDavid Sehr 15470c68bef895ea8987922922ede6f1918763eaa00bDavid Sehr for (SizeT i = 0, End = Args.size(); 15480c68bef895ea8987922922ede6f1918763eaa00bDavid Sehr i < End && (XmmSlotsRemain || GprSlotsRemain); ++i) { 15490c68bef895ea8987922922ede6f1918763eaa00bDavid Sehr Variable *Arg = Args[i]; 15500c68bef895ea8987922922ede6f1918763eaa00bDavid Sehr Type Ty = Arg->getType(); 15510c68bef895ea8987922922ede6f1918763eaa00bDavid Sehr Variable *RegisterArg = nullptr; 15525fa0a5f7f730f33e6987527a261de8d833d7585cReed Kotler RegNumT RegNum; 15530c68bef895ea8987922922ede6f1918763eaa00bDavid Sehr if (isVectorType(Ty)) { 15540c68bef895ea8987922922ede6f1918763eaa00bDavid Sehr RegNum = Traits::getRegisterForXmmArgNum(NumXmmArgs); 15555fa0a5f7f730f33e6987527a261de8d833d7585cReed Kotler if (RegNum.hasNoValue()) { 15560c68bef895ea8987922922ede6f1918763eaa00bDavid Sehr XmmSlotsRemain = false; 15570c68bef895ea8987922922ede6f1918763eaa00bDavid Sehr continue; 15580c68bef895ea8987922922ede6f1918763eaa00bDavid Sehr } 15590c68bef895ea8987922922ede6f1918763eaa00bDavid Sehr ++NumXmmArgs; 15600c68bef895ea8987922922ede6f1918763eaa00bDavid Sehr RegisterArg = Func->makeVariable(Ty); 15610c68bef895ea8987922922ede6f1918763eaa00bDavid Sehr } else if (isScalarFloatingType(Ty)) { 15620c68bef895ea8987922922ede6f1918763eaa00bDavid Sehr if (!Traits::X86_PASS_SCALAR_FP_IN_XMM) { 15630c68bef895ea8987922922ede6f1918763eaa00bDavid Sehr continue; 15640c68bef895ea8987922922ede6f1918763eaa00bDavid Sehr } 15650c68bef895ea8987922922ede6f1918763eaa00bDavid Sehr RegNum = Traits::getRegisterForXmmArgNum(NumXmmArgs); 15665fa0a5f7f730f33e6987527a261de8d833d7585cReed Kotler if (RegNum.hasNoValue()) { 15670c68bef895ea8987922922ede6f1918763eaa00bDavid Sehr XmmSlotsRemain = false; 15680c68bef895ea8987922922ede6f1918763eaa00bDavid Sehr continue; 15690c68bef895ea8987922922ede6f1918763eaa00bDavid Sehr } 15700c68bef895ea8987922922ede6f1918763eaa00bDavid Sehr ++NumXmmArgs; 15710c68bef895ea8987922922ede6f1918763eaa00bDavid Sehr RegisterArg = Func->makeVariable(Ty); 15720c68bef895ea8987922922ede6f1918763eaa00bDavid Sehr } else if (isScalarIntegerType(Ty)) { 15730c68bef895ea8987922922ede6f1918763eaa00bDavid Sehr RegNum = Traits::getRegisterForGprArgNum(Ty, NumGprArgs); 15745fa0a5f7f730f33e6987527a261de8d833d7585cReed Kotler if (RegNum.hasNoValue()) { 15750c68bef895ea8987922922ede6f1918763eaa00bDavid Sehr GprSlotsRemain = false; 15760c68bef895ea8987922922ede6f1918763eaa00bDavid Sehr continue; 15770c68bef895ea8987922922ede6f1918763eaa00bDavid Sehr } 15780c68bef895ea8987922922ede6f1918763eaa00bDavid Sehr ++NumGprArgs; 15790c68bef895ea8987922922ede6f1918763eaa00bDavid Sehr RegisterArg = Func->makeVariable(Ty); 15800c68bef895ea8987922922ede6f1918763eaa00bDavid Sehr } 15815fa0a5f7f730f33e6987527a261de8d833d7585cReed Kotler assert(RegNum.hasValue()); 15820c68bef895ea8987922922ede6f1918763eaa00bDavid Sehr assert(RegisterArg != nullptr); 15830c68bef895ea8987922922ede6f1918763eaa00bDavid Sehr // Replace Arg in the argument list with the home register. Then generate 15840c68bef895ea8987922922ede6f1918763eaa00bDavid Sehr // an instruction in the prolog to copy the home register to the assigned 15850c68bef895ea8987922922ede6f1918763eaa00bDavid Sehr // location of Arg. 15860c68bef895ea8987922922ede6f1918763eaa00bDavid Sehr if (BuildDefs::dump()) 1587a91c34118294efbf08ebd11eed96fce83bf35f3cJim Stichnoth RegisterArg->setName(Func, "home_reg:" + Arg->getName()); 15880c68bef895ea8987922922ede6f1918763eaa00bDavid Sehr RegisterArg->setRegNum(RegNum); 15890c68bef895ea8987922922ede6f1918763eaa00bDavid Sehr RegisterArg->setIsArg(); 15900c68bef895ea8987922922ede6f1918763eaa00bDavid Sehr Arg->setIsArg(false); 15910c68bef895ea8987922922ede6f1918763eaa00bDavid Sehr 15920c68bef895ea8987922922ede6f1918763eaa00bDavid Sehr Args[i] = RegisterArg; 1593f531931fb45172157bc5816a8f55309556996dc7Jim Stichnoth // When not Om1, do the assignment through a temporary, instead of directly 1594f531931fb45172157bc5816a8f55309556996dc7Jim Stichnoth // from the pre-colored variable, so that a subsequent availabilityGet() 1595f531931fb45172157bc5816a8f55309556996dc7Jim Stichnoth // call has a chance to work. (In Om1, don't bother creating extra 1596f531931fb45172157bc5816a8f55309556996dc7Jim Stichnoth // instructions with extra variables to register-allocate.) 1597f531931fb45172157bc5816a8f55309556996dc7Jim Stichnoth if (OptM1) { 1598f531931fb45172157bc5816a8f55309556996dc7Jim Stichnoth Context.insert<InstAssign>(Arg, RegisterArg); 1599f531931fb45172157bc5816a8f55309556996dc7Jim Stichnoth } else { 1600f531931fb45172157bc5816a8f55309556996dc7Jim Stichnoth Variable *Tmp = makeReg(RegisterArg->getType()); 1601f531931fb45172157bc5816a8f55309556996dc7Jim Stichnoth Context.insert<InstAssign>(Tmp, RegisterArg); 1602f531931fb45172157bc5816a8f55309556996dc7Jim Stichnoth Context.insert<InstAssign>(Arg, Tmp); 1603f531931fb45172157bc5816a8f55309556996dc7Jim Stichnoth } 16040c68bef895ea8987922922ede6f1918763eaa00bDavid Sehr } 1605f531931fb45172157bc5816a8f55309556996dc7Jim Stichnoth if (!OptM1) 1606f531931fb45172157bc5816a8f55309556996dc7Jim Stichnoth Context.availabilityUpdate(); 16070c68bef895ea8987922922ede6f1918763eaa00bDavid Sehr} 16080c68bef895ea8987922922ede6f1918763eaa00bDavid Sehr 160957e126899b20c65ff3ea23a3b7d7a67ab30b99dcAndrew Scull/// Strength-reduce scalar integer multiplication by a constant (for i32 or 161057e126899b20c65ff3ea23a3b7d7a67ab30b99dcAndrew Scull/// narrower) for certain constants. The lea instruction can be used to multiply 161157e126899b20c65ff3ea23a3b7d7a67ab30b99dcAndrew Scull/// by 3, 5, or 9, and the lsh instruction can be used to multiply by powers of 161257e126899b20c65ff3ea23a3b7d7a67ab30b99dcAndrew Scull/// 2. These can be combined such that e.g. multiplying by 100 can be done as 2 161357e126899b20c65ff3ea23a3b7d7a67ab30b99dcAndrew Scull/// lea-based multiplies by 5, combined with left-shifting by 2. 16144a56686b5b56db6803f90ad53514bf2fa190d9f7John Portotemplate <typename TraitsType> 16154a56686b5b56db6803f90ad53514bf2fa190d9f7John Portobool TargetX86Base<TraitsType>::optimizeScalarMul(Variable *Dest, Operand *Src0, 16164a56686b5b56db6803f90ad53514bf2fa190d9f7John Porto int32_t Src1) { 161757e126899b20c65ff3ea23a3b7d7a67ab30b99dcAndrew Scull // Disable this optimization for Om1 and O0, just to keep things simple 161857e126899b20c65ff3ea23a3b7d7a67ab30b99dcAndrew Scull // there. 1619dd6dcfaf765dc93ae64ec45d623106f4b3a3c13aJim Stichnoth if (Func->getOptLevel() < Opt_1) 16207e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto return false; 16217e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto Type Ty = Dest->getType(); 16227e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto if (Src1 == -1) { 162356958cb33d3c1d045f2844408d825442d523f59fJohn Porto Variable *T = nullptr; 16247e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto _mov(T, Src0); 16257e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto _neg(T); 16267e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto _mov(Dest, T); 16277e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto return true; 16287e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto } 16297e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto if (Src1 == 0) { 16307e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto _mov(Dest, Ctx->getConstantZero(Ty)); 16317e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto return true; 16327e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto } 16337e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto if (Src1 == 1) { 163456958cb33d3c1d045f2844408d825442d523f59fJohn Porto Variable *T = nullptr; 16357e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto _mov(T, Src0); 16367e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto _mov(Dest, T); 16377e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto return true; 16387e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto } 16397e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto // Don't bother with the edge case where Src1 == MININT. 16407e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto if (Src1 == -Src1) 16417e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto return false; 16427e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto const bool Src1IsNegative = Src1 < 0; 16437e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto if (Src1IsNegative) 16447e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto Src1 = -Src1; 16457e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto uint32_t Count9 = 0; 16467e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto uint32_t Count5 = 0; 16477e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto uint32_t Count3 = 0; 16487e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto uint32_t Count2 = 0; 16497e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto uint32_t CountOps = 0; 16507e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto while (Src1 > 1) { 16517e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto if (Src1 % 9 == 0) { 16527e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto ++CountOps; 16537e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto ++Count9; 16547e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto Src1 /= 9; 16557e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto } else if (Src1 % 5 == 0) { 16567e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto ++CountOps; 16577e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto ++Count5; 16587e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto Src1 /= 5; 16597e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto } else if (Src1 % 3 == 0) { 16607e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto ++CountOps; 16617e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto ++Count3; 16627e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto Src1 /= 3; 16637e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto } else if (Src1 % 2 == 0) { 16647e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto if (Count2 == 0) 16657e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto ++CountOps; 16667e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto ++Count2; 16677e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto Src1 /= 2; 16687e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto } else { 16697e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto return false; 16707e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto } 16717e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto } 167256958cb33d3c1d045f2844408d825442d523f59fJohn Porto // Lea optimization only works for i16 and i32 types, not i8. 167356958cb33d3c1d045f2844408d825442d523f59fJohn Porto if (Ty != IceType_i32 && !(Traits::Is64Bit && Ty == IceType_i64) && 167456958cb33d3c1d045f2844408d825442d523f59fJohn Porto (Count3 || Count5 || Count9)) 16757e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto return false; 167657e126899b20c65ff3ea23a3b7d7a67ab30b99dcAndrew Scull // Limit the number of lea/shl operations for a single multiply, to a 167757e126899b20c65ff3ea23a3b7d7a67ab30b99dcAndrew Scull // somewhat arbitrary choice of 3. 16785bff61c44841990680781892036adb17b3cff0c4Jim Stichnoth constexpr uint32_t MaxOpsForOptimizedMul = 3; 16797e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto if (CountOps > MaxOpsForOptimizedMul) 16807e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto return false; 168156958cb33d3c1d045f2844408d825442d523f59fJohn Porto Variable *T = makeReg(Traits::WordType); 168256958cb33d3c1d045f2844408d825442d523f59fJohn Porto if (typeWidthInBytes(Src0->getType()) < typeWidthInBytes(T->getType())) { 1683373913fa0a4c7c4dd23ada9e6b4e1780fdea0096Jim Stichnoth Operand *Src0RM = legalize(Src0, Legal_Reg | Legal_Mem); 1684373913fa0a4c7c4dd23ada9e6b4e1780fdea0096Jim Stichnoth _movzx(T, Src0RM); 168556958cb33d3c1d045f2844408d825442d523f59fJohn Porto } else { 168656958cb33d3c1d045f2844408d825442d523f59fJohn Porto _mov(T, Src0); 168756958cb33d3c1d045f2844408d825442d523f59fJohn Porto } 16887e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto Constant *Zero = Ctx->getConstantZero(IceType_i32); 16897e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto for (uint32_t i = 0; i < Count9; ++i) { 16905bff61c44841990680781892036adb17b3cff0c4Jim Stichnoth constexpr uint16_t Shift = 3; // log2(9-1) 16914a56686b5b56db6803f90ad53514bf2fa190d9f7John Porto _lea(T, X86OperandMem::create(Func, IceType_void, T, Zero, T, Shift)); 16927e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto } 16937e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto for (uint32_t i = 0; i < Count5; ++i) { 16945bff61c44841990680781892036adb17b3cff0c4Jim Stichnoth constexpr uint16_t Shift = 2; // log2(5-1) 16954a56686b5b56db6803f90ad53514bf2fa190d9f7John Porto _lea(T, X86OperandMem::create(Func, IceType_void, T, Zero, T, Shift)); 16967e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto } 16977e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto for (uint32_t i = 0; i < Count3; ++i) { 16985bff61c44841990680781892036adb17b3cff0c4Jim Stichnoth constexpr uint16_t Shift = 1; // log2(3-1) 16994a56686b5b56db6803f90ad53514bf2fa190d9f7John Porto _lea(T, X86OperandMem::create(Func, IceType_void, T, Zero, T, Shift)); 17007e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto } 17017e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto if (Count2) { 17027e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto _shl(T, Ctx->getConstantInt(Ty, Count2)); 17037e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto } 17047e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto if (Src1IsNegative) 17057e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto _neg(T); 17067e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto _mov(Dest, T); 17077e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto return true; 17087e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto} 17097e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto 17104a56686b5b56db6803f90ad53514bf2fa190d9f7John Portotemplate <typename TraitsType> 17114a56686b5b56db6803f90ad53514bf2fa190d9f7John Portovoid TargetX86Base<TraitsType>::lowerShift64(InstArithmetic::OpKind Op, 17124a56686b5b56db6803f90ad53514bf2fa190d9f7John Porto Operand *Src0Lo, Operand *Src0Hi, 17134a56686b5b56db6803f90ad53514bf2fa190d9f7John Porto Operand *Src1Lo, Variable *DestLo, 17144a56686b5b56db6803f90ad53514bf2fa190d9f7John Porto Variable *DestHi) { 1715188eae5c51cd6d8b4fd53dafe65d4ef7a1b5d867David Sehr // TODO: Refactor the similarities between Shl, Lshr, and Ashr. 1716188eae5c51cd6d8b4fd53dafe65d4ef7a1b5d867David Sehr Variable *T_1 = nullptr, *T_2 = nullptr, *T_3 = nullptr; 1717188eae5c51cd6d8b4fd53dafe65d4ef7a1b5d867David Sehr Constant *Zero = Ctx->getConstantZero(IceType_i32); 1718188eae5c51cd6d8b4fd53dafe65d4ef7a1b5d867David Sehr Constant *SignExtend = Ctx->getConstantInt32(0x1f); 1719188eae5c51cd6d8b4fd53dafe65d4ef7a1b5d867David Sehr if (auto *ConstantShiftAmount = llvm::dyn_cast<ConstantInteger32>(Src1Lo)) { 1720188eae5c51cd6d8b4fd53dafe65d4ef7a1b5d867David Sehr uint32_t ShiftAmount = ConstantShiftAmount->getValue(); 1721188eae5c51cd6d8b4fd53dafe65d4ef7a1b5d867David Sehr if (ShiftAmount > 32) { 1722188eae5c51cd6d8b4fd53dafe65d4ef7a1b5d867David Sehr Constant *ReducedShift = Ctx->getConstantInt32(ShiftAmount - 32); 1723188eae5c51cd6d8b4fd53dafe65d4ef7a1b5d867David Sehr switch (Op) { 1724188eae5c51cd6d8b4fd53dafe65d4ef7a1b5d867David Sehr default: 1725188eae5c51cd6d8b4fd53dafe65d4ef7a1b5d867David Sehr assert(0 && "non-shift op"); 1726188eae5c51cd6d8b4fd53dafe65d4ef7a1b5d867David Sehr break; 1727188eae5c51cd6d8b4fd53dafe65d4ef7a1b5d867David Sehr case InstArithmetic::Shl: { 1728188eae5c51cd6d8b4fd53dafe65d4ef7a1b5d867David Sehr // a=b<<c ==> 1729188eae5c51cd6d8b4fd53dafe65d4ef7a1b5d867David Sehr // t2 = b.lo 1730188eae5c51cd6d8b4fd53dafe65d4ef7a1b5d867David Sehr // t2 = shl t2, ShiftAmount-32 1731188eae5c51cd6d8b4fd53dafe65d4ef7a1b5d867David Sehr // t3 = t2 1732188eae5c51cd6d8b4fd53dafe65d4ef7a1b5d867David Sehr // t2 = 0 1733188eae5c51cd6d8b4fd53dafe65d4ef7a1b5d867David Sehr _mov(T_2, Src0Lo); 1734188eae5c51cd6d8b4fd53dafe65d4ef7a1b5d867David Sehr _shl(T_2, ReducedShift); 1735188eae5c51cd6d8b4fd53dafe65d4ef7a1b5d867David Sehr _mov(DestHi, T_2); 1736188eae5c51cd6d8b4fd53dafe65d4ef7a1b5d867David Sehr _mov(DestLo, Zero); 1737188eae5c51cd6d8b4fd53dafe65d4ef7a1b5d867David Sehr } break; 1738188eae5c51cd6d8b4fd53dafe65d4ef7a1b5d867David Sehr case InstArithmetic::Lshr: { 1739188eae5c51cd6d8b4fd53dafe65d4ef7a1b5d867David Sehr // a=b>>c (unsigned) ==> 1740188eae5c51cd6d8b4fd53dafe65d4ef7a1b5d867David Sehr // t2 = b.hi 1741188eae5c51cd6d8b4fd53dafe65d4ef7a1b5d867David Sehr // t2 = shr t2, ShiftAmount-32 1742188eae5c51cd6d8b4fd53dafe65d4ef7a1b5d867David Sehr // a.lo = t2 1743188eae5c51cd6d8b4fd53dafe65d4ef7a1b5d867David Sehr // a.hi = 0 1744188eae5c51cd6d8b4fd53dafe65d4ef7a1b5d867David Sehr _mov(T_2, Src0Hi); 1745188eae5c51cd6d8b4fd53dafe65d4ef7a1b5d867David Sehr _shr(T_2, ReducedShift); 1746188eae5c51cd6d8b4fd53dafe65d4ef7a1b5d867David Sehr _mov(DestLo, T_2); 1747188eae5c51cd6d8b4fd53dafe65d4ef7a1b5d867David Sehr _mov(DestHi, Zero); 1748188eae5c51cd6d8b4fd53dafe65d4ef7a1b5d867David Sehr } break; 1749188eae5c51cd6d8b4fd53dafe65d4ef7a1b5d867David Sehr case InstArithmetic::Ashr: { 1750188eae5c51cd6d8b4fd53dafe65d4ef7a1b5d867David Sehr // a=b>>c (signed) ==> 1751188eae5c51cd6d8b4fd53dafe65d4ef7a1b5d867David Sehr // t3 = b.hi 1752188eae5c51cd6d8b4fd53dafe65d4ef7a1b5d867David Sehr // t3 = sar t3, 0x1f 1753188eae5c51cd6d8b4fd53dafe65d4ef7a1b5d867David Sehr // t2 = b.hi 1754188eae5c51cd6d8b4fd53dafe65d4ef7a1b5d867David Sehr // t2 = shrd t2, t3, ShiftAmount-32 1755188eae5c51cd6d8b4fd53dafe65d4ef7a1b5d867David Sehr // a.lo = t2 1756188eae5c51cd6d8b4fd53dafe65d4ef7a1b5d867David Sehr // a.hi = t3 1757188eae5c51cd6d8b4fd53dafe65d4ef7a1b5d867David Sehr _mov(T_3, Src0Hi); 1758188eae5c51cd6d8b4fd53dafe65d4ef7a1b5d867David Sehr _sar(T_3, SignExtend); 1759188eae5c51cd6d8b4fd53dafe65d4ef7a1b5d867David Sehr _mov(T_2, Src0Hi); 1760188eae5c51cd6d8b4fd53dafe65d4ef7a1b5d867David Sehr _shrd(T_2, T_3, ReducedShift); 1761188eae5c51cd6d8b4fd53dafe65d4ef7a1b5d867David Sehr _mov(DestLo, T_2); 1762188eae5c51cd6d8b4fd53dafe65d4ef7a1b5d867David Sehr _mov(DestHi, T_3); 1763188eae5c51cd6d8b4fd53dafe65d4ef7a1b5d867David Sehr } break; 1764188eae5c51cd6d8b4fd53dafe65d4ef7a1b5d867David Sehr } 1765188eae5c51cd6d8b4fd53dafe65d4ef7a1b5d867David Sehr } else if (ShiftAmount == 32) { 1766188eae5c51cd6d8b4fd53dafe65d4ef7a1b5d867David Sehr switch (Op) { 1767188eae5c51cd6d8b4fd53dafe65d4ef7a1b5d867David Sehr default: 1768188eae5c51cd6d8b4fd53dafe65d4ef7a1b5d867David Sehr assert(0 && "non-shift op"); 1769188eae5c51cd6d8b4fd53dafe65d4ef7a1b5d867David Sehr break; 1770188eae5c51cd6d8b4fd53dafe65d4ef7a1b5d867David Sehr case InstArithmetic::Shl: { 1771188eae5c51cd6d8b4fd53dafe65d4ef7a1b5d867David Sehr // a=b<<c ==> 1772188eae5c51cd6d8b4fd53dafe65d4ef7a1b5d867David Sehr // t2 = b.lo 1773188eae5c51cd6d8b4fd53dafe65d4ef7a1b5d867David Sehr // a.hi = t2 1774188eae5c51cd6d8b4fd53dafe65d4ef7a1b5d867David Sehr // a.lo = 0 1775188eae5c51cd6d8b4fd53dafe65d4ef7a1b5d867David Sehr _mov(T_2, Src0Lo); 1776188eae5c51cd6d8b4fd53dafe65d4ef7a1b5d867David Sehr _mov(DestHi, T_2); 1777188eae5c51cd6d8b4fd53dafe65d4ef7a1b5d867David Sehr _mov(DestLo, Zero); 1778188eae5c51cd6d8b4fd53dafe65d4ef7a1b5d867David Sehr } break; 1779188eae5c51cd6d8b4fd53dafe65d4ef7a1b5d867David Sehr case InstArithmetic::Lshr: { 1780188eae5c51cd6d8b4fd53dafe65d4ef7a1b5d867David Sehr // a=b>>c (unsigned) ==> 1781188eae5c51cd6d8b4fd53dafe65d4ef7a1b5d867David Sehr // t2 = b.hi 1782188eae5c51cd6d8b4fd53dafe65d4ef7a1b5d867David Sehr // a.lo = t2 1783188eae5c51cd6d8b4fd53dafe65d4ef7a1b5d867David Sehr // a.hi = 0 1784188eae5c51cd6d8b4fd53dafe65d4ef7a1b5d867David Sehr _mov(T_2, Src0Hi); 1785188eae5c51cd6d8b4fd53dafe65d4ef7a1b5d867David Sehr _mov(DestLo, T_2); 1786188eae5c51cd6d8b4fd53dafe65d4ef7a1b5d867David Sehr _mov(DestHi, Zero); 1787188eae5c51cd6d8b4fd53dafe65d4ef7a1b5d867David Sehr } break; 1788188eae5c51cd6d8b4fd53dafe65d4ef7a1b5d867David Sehr case InstArithmetic::Ashr: { 1789188eae5c51cd6d8b4fd53dafe65d4ef7a1b5d867David Sehr // a=b>>c (signed) ==> 1790188eae5c51cd6d8b4fd53dafe65d4ef7a1b5d867David Sehr // t2 = b.hi 1791188eae5c51cd6d8b4fd53dafe65d4ef7a1b5d867David Sehr // a.lo = t2 1792188eae5c51cd6d8b4fd53dafe65d4ef7a1b5d867David Sehr // t3 = b.hi 1793188eae5c51cd6d8b4fd53dafe65d4ef7a1b5d867David Sehr // t3 = sar t3, 0x1f 1794188eae5c51cd6d8b4fd53dafe65d4ef7a1b5d867David Sehr // a.hi = t3 1795188eae5c51cd6d8b4fd53dafe65d4ef7a1b5d867David Sehr _mov(T_2, Src0Hi); 1796188eae5c51cd6d8b4fd53dafe65d4ef7a1b5d867David Sehr _mov(DestLo, T_2); 1797188eae5c51cd6d8b4fd53dafe65d4ef7a1b5d867David Sehr _mov(T_3, Src0Hi); 1798188eae5c51cd6d8b4fd53dafe65d4ef7a1b5d867David Sehr _sar(T_3, SignExtend); 1799188eae5c51cd6d8b4fd53dafe65d4ef7a1b5d867David Sehr _mov(DestHi, T_3); 1800188eae5c51cd6d8b4fd53dafe65d4ef7a1b5d867David Sehr } break; 1801188eae5c51cd6d8b4fd53dafe65d4ef7a1b5d867David Sehr } 1802188eae5c51cd6d8b4fd53dafe65d4ef7a1b5d867David Sehr } else { 1803188eae5c51cd6d8b4fd53dafe65d4ef7a1b5d867David Sehr // COMMON PREFIX OF: a=b SHIFT_OP c ==> 1804188eae5c51cd6d8b4fd53dafe65d4ef7a1b5d867David Sehr // t2 = b.lo 1805188eae5c51cd6d8b4fd53dafe65d4ef7a1b5d867David Sehr // t3 = b.hi 1806188eae5c51cd6d8b4fd53dafe65d4ef7a1b5d867David Sehr _mov(T_2, Src0Lo); 1807188eae5c51cd6d8b4fd53dafe65d4ef7a1b5d867David Sehr _mov(T_3, Src0Hi); 1808188eae5c51cd6d8b4fd53dafe65d4ef7a1b5d867David Sehr switch (Op) { 1809188eae5c51cd6d8b4fd53dafe65d4ef7a1b5d867David Sehr default: 1810188eae5c51cd6d8b4fd53dafe65d4ef7a1b5d867David Sehr assert(0 && "non-shift op"); 1811188eae5c51cd6d8b4fd53dafe65d4ef7a1b5d867David Sehr break; 1812188eae5c51cd6d8b4fd53dafe65d4ef7a1b5d867David Sehr case InstArithmetic::Shl: { 1813188eae5c51cd6d8b4fd53dafe65d4ef7a1b5d867David Sehr // a=b<<c ==> 1814188eae5c51cd6d8b4fd53dafe65d4ef7a1b5d867David Sehr // t3 = shld t3, t2, ShiftAmount 1815188eae5c51cd6d8b4fd53dafe65d4ef7a1b5d867David Sehr // t2 = shl t2, ShiftAmount 1816188eae5c51cd6d8b4fd53dafe65d4ef7a1b5d867David Sehr _shld(T_3, T_2, ConstantShiftAmount); 1817188eae5c51cd6d8b4fd53dafe65d4ef7a1b5d867David Sehr _shl(T_2, ConstantShiftAmount); 1818188eae5c51cd6d8b4fd53dafe65d4ef7a1b5d867David Sehr } break; 1819188eae5c51cd6d8b4fd53dafe65d4ef7a1b5d867David Sehr case InstArithmetic::Lshr: { 1820188eae5c51cd6d8b4fd53dafe65d4ef7a1b5d867David Sehr // a=b>>c (unsigned) ==> 1821188eae5c51cd6d8b4fd53dafe65d4ef7a1b5d867David Sehr // t2 = shrd t2, t3, ShiftAmount 1822188eae5c51cd6d8b4fd53dafe65d4ef7a1b5d867David Sehr // t3 = shr t3, ShiftAmount 1823188eae5c51cd6d8b4fd53dafe65d4ef7a1b5d867David Sehr _shrd(T_2, T_3, ConstantShiftAmount); 1824188eae5c51cd6d8b4fd53dafe65d4ef7a1b5d867David Sehr _shr(T_3, ConstantShiftAmount); 1825188eae5c51cd6d8b4fd53dafe65d4ef7a1b5d867David Sehr } break; 1826188eae5c51cd6d8b4fd53dafe65d4ef7a1b5d867David Sehr case InstArithmetic::Ashr: { 1827188eae5c51cd6d8b4fd53dafe65d4ef7a1b5d867David Sehr // a=b>>c (signed) ==> 1828188eae5c51cd6d8b4fd53dafe65d4ef7a1b5d867David Sehr // t2 = shrd t2, t3, ShiftAmount 1829188eae5c51cd6d8b4fd53dafe65d4ef7a1b5d867David Sehr // t3 = sar t3, ShiftAmount 1830188eae5c51cd6d8b4fd53dafe65d4ef7a1b5d867David Sehr _shrd(T_2, T_3, ConstantShiftAmount); 1831188eae5c51cd6d8b4fd53dafe65d4ef7a1b5d867David Sehr _sar(T_3, ConstantShiftAmount); 1832188eae5c51cd6d8b4fd53dafe65d4ef7a1b5d867David Sehr } break; 1833188eae5c51cd6d8b4fd53dafe65d4ef7a1b5d867David Sehr } 1834188eae5c51cd6d8b4fd53dafe65d4ef7a1b5d867David Sehr // COMMON SUFFIX OF: a=b SHIFT_OP c ==> 1835188eae5c51cd6d8b4fd53dafe65d4ef7a1b5d867David Sehr // a.lo = t2 1836188eae5c51cd6d8b4fd53dafe65d4ef7a1b5d867David Sehr // a.hi = t3 1837188eae5c51cd6d8b4fd53dafe65d4ef7a1b5d867David Sehr _mov(DestLo, T_2); 1838188eae5c51cd6d8b4fd53dafe65d4ef7a1b5d867David Sehr _mov(DestHi, T_3); 1839188eae5c51cd6d8b4fd53dafe65d4ef7a1b5d867David Sehr } 1840188eae5c51cd6d8b4fd53dafe65d4ef7a1b5d867David Sehr } else { 1841188eae5c51cd6d8b4fd53dafe65d4ef7a1b5d867David Sehr // NON-CONSTANT CASES. 1842188eae5c51cd6d8b4fd53dafe65d4ef7a1b5d867David Sehr Constant *BitTest = Ctx->getConstantInt32(0x20); 18434a56686b5b56db6803f90ad53514bf2fa190d9f7John Porto InstX86Label *Label = InstX86Label::create(Func, this); 1844188eae5c51cd6d8b4fd53dafe65d4ef7a1b5d867David Sehr // COMMON PREFIX OF: a=b SHIFT_OP c ==> 1845188eae5c51cd6d8b4fd53dafe65d4ef7a1b5d867David Sehr // t1:ecx = c.lo & 0xff 1846188eae5c51cd6d8b4fd53dafe65d4ef7a1b5d867David Sehr // t2 = b.lo 1847188eae5c51cd6d8b4fd53dafe65d4ef7a1b5d867David Sehr // t3 = b.hi 1848c59288b334b91f4c0b2edf0de7415c68c760aa12Jim Stichnoth T_1 = copyToReg8(Src1Lo, Traits::RegisterSet::Reg_cl); 1849188eae5c51cd6d8b4fd53dafe65d4ef7a1b5d867David Sehr _mov(T_2, Src0Lo); 1850188eae5c51cd6d8b4fd53dafe65d4ef7a1b5d867David Sehr _mov(T_3, Src0Hi); 1851188eae5c51cd6d8b4fd53dafe65d4ef7a1b5d867David Sehr switch (Op) { 1852188eae5c51cd6d8b4fd53dafe65d4ef7a1b5d867David Sehr default: 1853188eae5c51cd6d8b4fd53dafe65d4ef7a1b5d867David Sehr assert(0 && "non-shift op"); 1854188eae5c51cd6d8b4fd53dafe65d4ef7a1b5d867David Sehr break; 1855188eae5c51cd6d8b4fd53dafe65d4ef7a1b5d867David Sehr case InstArithmetic::Shl: { 1856188eae5c51cd6d8b4fd53dafe65d4ef7a1b5d867David Sehr // a=b<<c ==> 1857188eae5c51cd6d8b4fd53dafe65d4ef7a1b5d867David Sehr // t3 = shld t3, t2, t1 1858188eae5c51cd6d8b4fd53dafe65d4ef7a1b5d867David Sehr // t2 = shl t2, t1 1859188eae5c51cd6d8b4fd53dafe65d4ef7a1b5d867David Sehr // test t1, 0x20 1860188eae5c51cd6d8b4fd53dafe65d4ef7a1b5d867David Sehr // je L1 1861188eae5c51cd6d8b4fd53dafe65d4ef7a1b5d867David Sehr // use(t3) 1862188eae5c51cd6d8b4fd53dafe65d4ef7a1b5d867David Sehr // t3 = t2 1863188eae5c51cd6d8b4fd53dafe65d4ef7a1b5d867David Sehr // t2 = 0 1864188eae5c51cd6d8b4fd53dafe65d4ef7a1b5d867David Sehr _shld(T_3, T_2, T_1); 1865188eae5c51cd6d8b4fd53dafe65d4ef7a1b5d867David Sehr _shl(T_2, T_1); 1866188eae5c51cd6d8b4fd53dafe65d4ef7a1b5d867David Sehr _test(T_1, BitTest); 1867188eae5c51cd6d8b4fd53dafe65d4ef7a1b5d867David Sehr _br(Traits::Cond::Br_e, Label); 1868230d4101fb3c591d044406eef27d0ce43ffab53eJim Stichnoth // T_2 and T_3 are being assigned again because of the intra-block control 1869e398428c3ef8ed2339c0f334dd30a7d3f0ed4b6eDavid Sehr // flow, so we need to use _redefined to avoid liveness problems. 1870e398428c3ef8ed2339c0f334dd30a7d3f0ed4b6eDavid Sehr _redefined(_mov(T_3, T_2)); 1871e398428c3ef8ed2339c0f334dd30a7d3f0ed4b6eDavid Sehr _redefined(_mov(T_2, Zero)); 1872188eae5c51cd6d8b4fd53dafe65d4ef7a1b5d867David Sehr } break; 1873188eae5c51cd6d8b4fd53dafe65d4ef7a1b5d867David Sehr case InstArithmetic::Lshr: { 1874188eae5c51cd6d8b4fd53dafe65d4ef7a1b5d867David Sehr // a=b>>c (unsigned) ==> 1875188eae5c51cd6d8b4fd53dafe65d4ef7a1b5d867David Sehr // t2 = shrd t2, t3, t1 1876188eae5c51cd6d8b4fd53dafe65d4ef7a1b5d867David Sehr // t3 = shr t3, t1 1877188eae5c51cd6d8b4fd53dafe65d4ef7a1b5d867David Sehr // test t1, 0x20 1878188eae5c51cd6d8b4fd53dafe65d4ef7a1b5d867David Sehr // je L1 1879188eae5c51cd6d8b4fd53dafe65d4ef7a1b5d867David Sehr // use(t2) 1880188eae5c51cd6d8b4fd53dafe65d4ef7a1b5d867David Sehr // t2 = t3 1881188eae5c51cd6d8b4fd53dafe65d4ef7a1b5d867David Sehr // t3 = 0 1882188eae5c51cd6d8b4fd53dafe65d4ef7a1b5d867David Sehr _shrd(T_2, T_3, T_1); 1883188eae5c51cd6d8b4fd53dafe65d4ef7a1b5d867David Sehr _shr(T_3, T_1); 1884188eae5c51cd6d8b4fd53dafe65d4ef7a1b5d867David Sehr _test(T_1, BitTest); 1885188eae5c51cd6d8b4fd53dafe65d4ef7a1b5d867David Sehr _br(Traits::Cond::Br_e, Label); 1886230d4101fb3c591d044406eef27d0ce43ffab53eJim Stichnoth // T_2 and T_3 are being assigned again because of the intra-block control 1887e398428c3ef8ed2339c0f334dd30a7d3f0ed4b6eDavid Sehr // flow, so we need to use _redefined to avoid liveness problems. 1888e398428c3ef8ed2339c0f334dd30a7d3f0ed4b6eDavid Sehr _redefined(_mov(T_2, T_3)); 1889e398428c3ef8ed2339c0f334dd30a7d3f0ed4b6eDavid Sehr _redefined(_mov(T_3, Zero)); 1890188eae5c51cd6d8b4fd53dafe65d4ef7a1b5d867David Sehr } break; 1891188eae5c51cd6d8b4fd53dafe65d4ef7a1b5d867David Sehr case InstArithmetic::Ashr: { 1892188eae5c51cd6d8b4fd53dafe65d4ef7a1b5d867David Sehr // a=b>>c (signed) ==> 1893188eae5c51cd6d8b4fd53dafe65d4ef7a1b5d867David Sehr // t2 = shrd t2, t3, t1 1894188eae5c51cd6d8b4fd53dafe65d4ef7a1b5d867David Sehr // t3 = sar t3, t1 1895188eae5c51cd6d8b4fd53dafe65d4ef7a1b5d867David Sehr // test t1, 0x20 1896188eae5c51cd6d8b4fd53dafe65d4ef7a1b5d867David Sehr // je L1 1897188eae5c51cd6d8b4fd53dafe65d4ef7a1b5d867David Sehr // use(t2) 1898188eae5c51cd6d8b4fd53dafe65d4ef7a1b5d867David Sehr // t2 = t3 1899188eae5c51cd6d8b4fd53dafe65d4ef7a1b5d867David Sehr // t3 = sar t3, 0x1f 1900188eae5c51cd6d8b4fd53dafe65d4ef7a1b5d867David Sehr Constant *SignExtend = Ctx->getConstantInt32(0x1f); 1901188eae5c51cd6d8b4fd53dafe65d4ef7a1b5d867David Sehr _shrd(T_2, T_3, T_1); 1902188eae5c51cd6d8b4fd53dafe65d4ef7a1b5d867David Sehr _sar(T_3, T_1); 1903188eae5c51cd6d8b4fd53dafe65d4ef7a1b5d867David Sehr _test(T_1, BitTest); 1904188eae5c51cd6d8b4fd53dafe65d4ef7a1b5d867David Sehr _br(Traits::Cond::Br_e, Label); 1905230d4101fb3c591d044406eef27d0ce43ffab53eJim Stichnoth // T_2 and T_3 are being assigned again because of the intra-block control 1906e398428c3ef8ed2339c0f334dd30a7d3f0ed4b6eDavid Sehr // flow, so T_2 needs to use _redefined to avoid liveness problems. T_3 1907e398428c3ef8ed2339c0f334dd30a7d3f0ed4b6eDavid Sehr // doesn't need special treatment because it is reassigned via _sar 1908e398428c3ef8ed2339c0f334dd30a7d3f0ed4b6eDavid Sehr // instead of _mov. 1909e398428c3ef8ed2339c0f334dd30a7d3f0ed4b6eDavid Sehr _redefined(_mov(T_2, T_3)); 1910188eae5c51cd6d8b4fd53dafe65d4ef7a1b5d867David Sehr _sar(T_3, SignExtend); 1911188eae5c51cd6d8b4fd53dafe65d4ef7a1b5d867David Sehr } break; 1912188eae5c51cd6d8b4fd53dafe65d4ef7a1b5d867David Sehr } 1913188eae5c51cd6d8b4fd53dafe65d4ef7a1b5d867David Sehr // COMMON SUFFIX OF: a=b SHIFT_OP c ==> 1914188eae5c51cd6d8b4fd53dafe65d4ef7a1b5d867David Sehr // L1: 1915188eae5c51cd6d8b4fd53dafe65d4ef7a1b5d867David Sehr // a.lo = t2 1916188eae5c51cd6d8b4fd53dafe65d4ef7a1b5d867David Sehr // a.hi = t3 1917188eae5c51cd6d8b4fd53dafe65d4ef7a1b5d867David Sehr Context.insert(Label); 1918188eae5c51cd6d8b4fd53dafe65d4ef7a1b5d867David Sehr _mov(DestLo, T_2); 1919188eae5c51cd6d8b4fd53dafe65d4ef7a1b5d867David Sehr _mov(DestHi, T_3); 1920188eae5c51cd6d8b4fd53dafe65d4ef7a1b5d867David Sehr } 1921188eae5c51cd6d8b4fd53dafe65d4ef7a1b5d867David Sehr} 1922188eae5c51cd6d8b4fd53dafe65d4ef7a1b5d867David Sehr 19234a56686b5b56db6803f90ad53514bf2fa190d9f7John Portotemplate <typename TraitsType> 19248cfeb69e17190d3bfe22a8a1cbd7679a114d68cfJim Stichnothvoid TargetX86Base<TraitsType>::lowerArithmetic(const InstArithmetic *Instr) { 19258cfeb69e17190d3bfe22a8a1cbd7679a114d68cfJim Stichnoth Variable *Dest = Instr->getDest(); 19263607b6c924922d261cbc75b68e07a1e2a5aec5dcJim Stichnoth if (Dest->isRematerializable()) { 19271d937a8e35b7ae51ee64c32f8f34d51ba33498f4John Porto Context.insert<InstFakeDef>(Dest); 19283607b6c924922d261cbc75b68e07a1e2a5aec5dcJim Stichnoth return; 19293607b6c924922d261cbc75b68e07a1e2a5aec5dcJim Stichnoth } 1930c59288b334b91f4c0b2edf0de7415c68c760aa12Jim Stichnoth Type Ty = Dest->getType(); 19318cfeb69e17190d3bfe22a8a1cbd7679a114d68cfJim Stichnoth Operand *Src0 = legalize(Instr->getSrc(0)); 19328cfeb69e17190d3bfe22a8a1cbd7679a114d68cfJim Stichnoth Operand *Src1 = legalize(Instr->getSrc(1)); 19338cfeb69e17190d3bfe22a8a1cbd7679a114d68cfJim Stichnoth if (Instr->isCommutative()) { 1934487bad0253e0d70d0cb191e8e5c71d01bf0044eeDavid Sehr uint32_t SwapCount = 0; 1935487bad0253e0d70d0cb191e8e5c71d01bf0044eeDavid Sehr if (!llvm::isa<Variable>(Src0) && llvm::isa<Variable>(Src1)) { 19367e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto std::swap(Src0, Src1); 1937487bad0253e0d70d0cb191e8e5c71d01bf0044eeDavid Sehr ++SwapCount; 1938487bad0253e0d70d0cb191e8e5c71d01bf0044eeDavid Sehr } 1939487bad0253e0d70d0cb191e8e5c71d01bf0044eeDavid Sehr if (llvm::isa<Constant>(Src0) && !llvm::isa<Constant>(Src1)) { 1940487bad0253e0d70d0cb191e8e5c71d01bf0044eeDavid Sehr std::swap(Src0, Src1); 1941487bad0253e0d70d0cb191e8e5c71d01bf0044eeDavid Sehr ++SwapCount; 1942487bad0253e0d70d0cb191e8e5c71d01bf0044eeDavid Sehr } 1943487bad0253e0d70d0cb191e8e5c71d01bf0044eeDavid Sehr // Improve two-address code patterns by avoiding a copy to the dest 1944487bad0253e0d70d0cb191e8e5c71d01bf0044eeDavid Sehr // register when one of the source operands ends its lifetime here. 19458cfeb69e17190d3bfe22a8a1cbd7679a114d68cfJim Stichnoth if (!Instr->isLastUse(Src0) && Instr->isLastUse(Src1)) { 19467e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto std::swap(Src0, Src1); 1947487bad0253e0d70d0cb191e8e5c71d01bf0044eeDavid Sehr ++SwapCount; 1948487bad0253e0d70d0cb191e8e5c71d01bf0044eeDavid Sehr } 1949487bad0253e0d70d0cb191e8e5c71d01bf0044eeDavid Sehr assert(SwapCount <= 1); 1950a313a121cc46b81bca7472e43ed0ae72156e8974Karl Schimpf (void)SwapCount; 19517e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto } 1952c59288b334b91f4c0b2edf0de7415c68c760aa12Jim Stichnoth if (!Traits::Is64Bit && Ty == IceType_i64) { 19531d235425dab1f3dd059973fc53f1b1d5879469e3John Porto // These x86-32 helper-call-involved instructions are lowered in this 195457e126899b20c65ff3ea23a3b7d7a67ab30b99dcAndrew Scull // separate switch. This is because loOperand() and hiOperand() may insert 195557e126899b20c65ff3ea23a3b7d7a67ab30b99dcAndrew Scull // redundant instructions for constant blinding and pooling. Such redundant 195657e126899b20c65ff3ea23a3b7d7a67ab30b99dcAndrew Scull // instructions will fail liveness analysis under -Om1 setting. And, 195757e126899b20c65ff3ea23a3b7d7a67ab30b99dcAndrew Scull // actually these arguments do not need to be processed with loOperand() 195857e126899b20c65ff3ea23a3b7d7a67ab30b99dcAndrew Scull // and hiOperand() to be used. 19598cfeb69e17190d3bfe22a8a1cbd7679a114d68cfJim Stichnoth switch (Instr->getOp()) { 196026217e3333150e66fc96aca79c01105906797960David Sehr case InstArithmetic::Udiv: 196126217e3333150e66fc96aca79c01105906797960David Sehr case InstArithmetic::Sdiv: 196226217e3333150e66fc96aca79c01105906797960David Sehr case InstArithmetic::Urem: 196326217e3333150e66fc96aca79c01105906797960David Sehr case InstArithmetic::Srem: 196426217e3333150e66fc96aca79c01105906797960David Sehr llvm::report_fatal_error("Helper call was expected"); 19657e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto return; 19667e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto default: 19677e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto break; 19687e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto } 19697e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto 197054f3d518805b04b3f2958b7906a7d07dedb4e1aeJim Stichnoth auto *DestLo = llvm::cast<Variable>(loOperand(Dest)); 197154f3d518805b04b3f2958b7906a7d07dedb4e1aeJim Stichnoth auto *DestHi = llvm::cast<Variable>(hiOperand(Dest)); 19727e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto Operand *Src0Lo = loOperand(Src0); 19737e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto Operand *Src0Hi = hiOperand(Src0); 19747e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto Operand *Src1Lo = loOperand(Src1); 19757e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto Operand *Src1Hi = hiOperand(Src1); 19767e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto Variable *T_Lo = nullptr, *T_Hi = nullptr; 19778cfeb69e17190d3bfe22a8a1cbd7679a114d68cfJim Stichnoth switch (Instr->getOp()) { 19787e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto case InstArithmetic::_num: 19797e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto llvm_unreachable("Unknown arithmetic operator"); 19807e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto break; 19817e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto case InstArithmetic::Add: 19827e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto _mov(T_Lo, Src0Lo); 19837e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto _add(T_Lo, Src1Lo); 19847e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto _mov(DestLo, T_Lo); 19857e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto _mov(T_Hi, Src0Hi); 19867e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto _adc(T_Hi, Src1Hi); 19877e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto _mov(DestHi, T_Hi); 19887e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto break; 19897e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto case InstArithmetic::And: 19907e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto _mov(T_Lo, Src0Lo); 19917e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto _and(T_Lo, Src1Lo); 19927e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto _mov(DestLo, T_Lo); 19937e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto _mov(T_Hi, Src0Hi); 19947e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto _and(T_Hi, Src1Hi); 19957e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto _mov(DestHi, T_Hi); 19967e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto break; 19977e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto case InstArithmetic::Or: 19987e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto _mov(T_Lo, Src0Lo); 19997e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto _or(T_Lo, Src1Lo); 20007e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto _mov(DestLo, T_Lo); 20017e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto _mov(T_Hi, Src0Hi); 20027e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto _or(T_Hi, Src1Hi); 20037e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto _mov(DestHi, T_Hi); 20047e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto break; 20057e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto case InstArithmetic::Xor: 20067e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto _mov(T_Lo, Src0Lo); 20077e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto _xor(T_Lo, Src1Lo); 20087e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto _mov(DestLo, T_Lo); 20097e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto _mov(T_Hi, Src0Hi); 20107e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto _xor(T_Hi, Src1Hi); 20117e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto _mov(DestHi, T_Hi); 20127e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto break; 20137e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto case InstArithmetic::Sub: 20147e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto _mov(T_Lo, Src0Lo); 20157e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto _sub(T_Lo, Src1Lo); 20167e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto _mov(DestLo, T_Lo); 20177e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto _mov(T_Hi, Src0Hi); 20187e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto _sbb(T_Hi, Src1Hi); 20197e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto _mov(DestHi, T_Hi); 20207e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto break; 20217e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto case InstArithmetic::Mul: { 20227e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto Variable *T_1 = nullptr, *T_2 = nullptr, *T_3 = nullptr; 20235d0acff3a2fa421923392aadb4df2742064b6248John Porto Variable *T_4Lo = makeReg(IceType_i32, Traits::RegisterSet::Reg_eax); 20245d0acff3a2fa421923392aadb4df2742064b6248John Porto Variable *T_4Hi = makeReg(IceType_i32, Traits::RegisterSet::Reg_edx); 20257e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto // gcc does the following: 20267e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto // a=b*c ==> 20277e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto // t1 = b.hi; t1 *=(imul) c.lo 20287e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto // t2 = c.hi; t2 *=(imul) b.lo 20297e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto // t3:eax = b.lo 20307e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto // t4.hi:edx,t4.lo:eax = t3:eax *(mul) c.lo 20317e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto // a.lo = t4.lo 20327e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto // t4.hi += t1 20337e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto // t4.hi += t2 20347e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto // a.hi = t4.hi 20357e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto // The mul instruction cannot take an immediate operand. 20367e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto Src1Lo = legalize(Src1Lo, Legal_Reg | Legal_Mem); 20377e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto _mov(T_1, Src0Hi); 20387e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto _imul(T_1, Src1Lo); 20395d0acff3a2fa421923392aadb4df2742064b6248John Porto _mov(T_3, Src0Lo, Traits::RegisterSet::Reg_eax); 20407e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto _mul(T_4Lo, T_3, Src1Lo); 204157e126899b20c65ff3ea23a3b7d7a67ab30b99dcAndrew Scull // The mul instruction produces two dest variables, edx:eax. We create a 204257e126899b20c65ff3ea23a3b7d7a67ab30b99dcAndrew Scull // fake definition of edx to account for this. 20431d937a8e35b7ae51ee64c32f8f34d51ba33498f4John Porto Context.insert<InstFakeDef>(T_4Hi, T_4Lo); 204428df6bad8e70d9930040464791296e192aaa03efJim Stichnoth Context.insert<InstFakeUse>(T_4Hi); 20457e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto _mov(DestLo, T_4Lo); 20467e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto _add(T_4Hi, T_1); 2047b40595a17b83cca5d11f8d056a4ac5a4d8102a84Jim Stichnoth _mov(T_2, Src1Hi); 2048b40595a17b83cca5d11f8d056a4ac5a4d8102a84Jim Stichnoth _imul(T_2, Src0Lo); 20497e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto _add(T_4Hi, T_2); 20507e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto _mov(DestHi, T_4Hi); 20517e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto } break; 2052188eae5c51cd6d8b4fd53dafe65d4ef7a1b5d867David Sehr case InstArithmetic::Shl: 2053188eae5c51cd6d8b4fd53dafe65d4ef7a1b5d867David Sehr case InstArithmetic::Lshr: 2054188eae5c51cd6d8b4fd53dafe65d4ef7a1b5d867David Sehr case InstArithmetic::Ashr: 20558cfeb69e17190d3bfe22a8a1cbd7679a114d68cfJim Stichnoth lowerShift64(Instr->getOp(), Src0Lo, Src0Hi, Src1Lo, DestLo, DestHi); 2056188eae5c51cd6d8b4fd53dafe65d4ef7a1b5d867David Sehr break; 20577e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto case InstArithmetic::Fadd: 20587e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto case InstArithmetic::Fsub: 20597e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto case InstArithmetic::Fmul: 20607e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto case InstArithmetic::Fdiv: 20617e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto case InstArithmetic::Frem: 20627e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto llvm_unreachable("FP instruction with i64 type"); 20637e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto break; 20647e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto case InstArithmetic::Udiv: 20657e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto case InstArithmetic::Sdiv: 20667e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto case InstArithmetic::Urem: 20677e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto case InstArithmetic::Srem: 20687e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto llvm_unreachable("Call-helper-involved instruction for i64 type \ 20697e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto should have already been handled before"); 20707e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto break; 20717e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto } 20727e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto return; 20737e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto } 2074c59288b334b91f4c0b2edf0de7415c68c760aa12Jim Stichnoth if (isVectorType(Ty)) { 207557e126899b20c65ff3ea23a3b7d7a67ab30b99dcAndrew Scull // TODO: Trap on integer divide and integer modulo by zero. See: 207657e126899b20c65ff3ea23a3b7d7a67ab30b99dcAndrew Scull // https://code.google.com/p/nativeclient/issues/detail?id=3899 20774a56686b5b56db6803f90ad53514bf2fa190d9f7John Porto if (llvm::isa<X86OperandMem>(Src1)) 207897f460dcd848dcef3455ed553c1b9e107d0e6ae9Andrew Scull Src1 = legalizeToReg(Src1); 20798cfeb69e17190d3bfe22a8a1cbd7679a114d68cfJim Stichnoth switch (Instr->getOp()) { 20807e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto case InstArithmetic::_num: 20817e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto llvm_unreachable("Unknown arithmetic operator"); 20827e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto break; 20837e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto case InstArithmetic::Add: { 2084c59288b334b91f4c0b2edf0de7415c68c760aa12Jim Stichnoth Variable *T = makeReg(Ty); 20857e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto _movp(T, Src0); 20867e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto _padd(T, Src1); 20877e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto _movp(Dest, T); 20887e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto } break; 20897e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto case InstArithmetic::And: { 2090c59288b334b91f4c0b2edf0de7415c68c760aa12Jim Stichnoth Variable *T = makeReg(Ty); 20917e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto _movp(T, Src0); 20927e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto _pand(T, Src1); 20937e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto _movp(Dest, T); 20947e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto } break; 20957e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto case InstArithmetic::Or: { 2096c59288b334b91f4c0b2edf0de7415c68c760aa12Jim Stichnoth Variable *T = makeReg(Ty); 20977e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto _movp(T, Src0); 20987e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto _por(T, Src1); 20997e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto _movp(Dest, T); 21007e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto } break; 21017e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto case InstArithmetic::Xor: { 2102c59288b334b91f4c0b2edf0de7415c68c760aa12Jim Stichnoth Variable *T = makeReg(Ty); 21037e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto _movp(T, Src0); 21047e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto _pxor(T, Src1); 21057e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto _movp(Dest, T); 21067e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto } break; 21077e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto case InstArithmetic::Sub: { 2108c59288b334b91f4c0b2edf0de7415c68c760aa12Jim Stichnoth Variable *T = makeReg(Ty); 21097e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto _movp(T, Src0); 21107e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto _psub(T, Src1); 21117e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto _movp(Dest, T); 21127e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto } break; 21137e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto case InstArithmetic::Mul: { 2114c59288b334b91f4c0b2edf0de7415c68c760aa12Jim Stichnoth bool TypesAreValidForPmull = Ty == IceType_v4i32 || Ty == IceType_v8i16; 21157e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto bool InstructionSetIsValidForPmull = 2116c59288b334b91f4c0b2edf0de7415c68c760aa12Jim Stichnoth Ty == IceType_v8i16 || InstructionSet >= Traits::SSE4_1; 21177e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto if (TypesAreValidForPmull && InstructionSetIsValidForPmull) { 2118c59288b334b91f4c0b2edf0de7415c68c760aa12Jim Stichnoth Variable *T = makeReg(Ty); 21197e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto _movp(T, Src0); 2120ebbb5912415bf46798f064fea93863a95f32efd8Jim Stichnoth _pmull(T, Src0 == Src1 ? T : Src1); 21217e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto _movp(Dest, T); 2122c59288b334b91f4c0b2edf0de7415c68c760aa12Jim Stichnoth } else if (Ty == IceType_v4i32) { 21237e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto // Lowering sequence: 21247e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto // Note: The mask arguments have index 0 on the left. 21257e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto // 21267e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto // movups T1, Src0 21277e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto // pshufd T2, Src0, {1,0,3,0} 21287e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto // pshufd T3, Src1, {1,0,3,0} 21297e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto // # T1 = {Src0[0] * Src1[0], Src0[2] * Src1[2]} 21307e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto // pmuludq T1, Src1 21317e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto // # T2 = {Src0[1] * Src1[1], Src0[3] * Src1[3]} 21327e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto // pmuludq T2, T3 21337e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto // # T1 = {lo(T1[0]), lo(T1[2]), lo(T2[0]), lo(T2[2])} 21347e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto // shufps T1, T2, {0,2,0,2} 21357e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto // pshufd T4, T1, {0,2,1,3} 21367e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto // movups Dest, T4 21377e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto 21387e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto // Mask that directs pshufd to create a vector with entries 21397e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto // Src[1, 0, 3, 0] 21405bff61c44841990680781892036adb17b3cff0c4Jim Stichnoth constexpr unsigned Constant1030 = 0x31; 21417e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto Constant *Mask1030 = Ctx->getConstantInt32(Constant1030); 21427e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto // Mask that directs shufps to create a vector with entries 21437e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto // Dest[0, 2], Src[0, 2] 21445bff61c44841990680781892036adb17b3cff0c4Jim Stichnoth constexpr unsigned Mask0202 = 0x88; 21457e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto // Mask that directs pshufd to create a vector with entries 21467e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto // Src[0, 2, 1, 3] 21475bff61c44841990680781892036adb17b3cff0c4Jim Stichnoth constexpr unsigned Mask0213 = 0xd8; 21487e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto Variable *T1 = makeReg(IceType_v4i32); 21497e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto Variable *T2 = makeReg(IceType_v4i32); 21507e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto Variable *T3 = makeReg(IceType_v4i32); 21517e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto Variable *T4 = makeReg(IceType_v4i32); 21527e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto _movp(T1, Src0); 21537e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto _pshufd(T2, Src0, Mask1030); 21547e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto _pshufd(T3, Src1, Mask1030); 21557e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto _pmuludq(T1, Src1); 21567e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto _pmuludq(T2, T3); 21577e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto _shufps(T1, T2, Ctx->getConstantInt32(Mask0202)); 21587e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto _pshufd(T4, T1, Ctx->getConstantInt32(Mask0213)); 21597e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto _movp(Dest, T4); 2160c59288b334b91f4c0b2edf0de7415c68c760aa12Jim Stichnoth } else if (Ty == IceType_v16i8) { 216126217e3333150e66fc96aca79c01105906797960David Sehr llvm::report_fatal_error("Scalarized operation was expected"); 2162ebbb5912415bf46798f064fea93863a95f32efd8Jim Stichnoth } else { 2163ebbb5912415bf46798f064fea93863a95f32efd8Jim Stichnoth llvm::report_fatal_error("Invalid vector multiply type"); 21647e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto } 21657e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto } break; 2166b09353904b93e8914d459a286b6b9badb8b1983bNicolas Capens case InstArithmetic::Shl: { 2167b09353904b93e8914d459a286b6b9badb8b1983bNicolas Capens assert(llvm::isa<Constant>(Src1) && "Non-constant shift not scalarized"); 2168b09353904b93e8914d459a286b6b9badb8b1983bNicolas Capens Variable *T = makeReg(Ty); 2169b09353904b93e8914d459a286b6b9badb8b1983bNicolas Capens _movp(T, Src0); 2170b09353904b93e8914d459a286b6b9badb8b1983bNicolas Capens _psll(T, Src1); 2171b09353904b93e8914d459a286b6b9badb8b1983bNicolas Capens _movp(Dest, T); 2172b09353904b93e8914d459a286b6b9badb8b1983bNicolas Capens } break; 2173b09353904b93e8914d459a286b6b9badb8b1983bNicolas Capens case InstArithmetic::Lshr: { 2174b09353904b93e8914d459a286b6b9badb8b1983bNicolas Capens assert(llvm::isa<Constant>(Src1) && "Non-constant shift not scalarized"); 2175b09353904b93e8914d459a286b6b9badb8b1983bNicolas Capens Variable *T = makeReg(Ty); 2176b09353904b93e8914d459a286b6b9badb8b1983bNicolas Capens _movp(T, Src0); 2177b09353904b93e8914d459a286b6b9badb8b1983bNicolas Capens _psrl(T, Src1); 2178b09353904b93e8914d459a286b6b9badb8b1983bNicolas Capens _movp(Dest, T); 2179b09353904b93e8914d459a286b6b9badb8b1983bNicolas Capens } break; 2180b09353904b93e8914d459a286b6b9badb8b1983bNicolas Capens case InstArithmetic::Ashr: { 2181b09353904b93e8914d459a286b6b9badb8b1983bNicolas Capens assert(llvm::isa<Constant>(Src1) && "Non-constant shift not scalarized"); 2182b09353904b93e8914d459a286b6b9badb8b1983bNicolas Capens Variable *T = makeReg(Ty); 2183b09353904b93e8914d459a286b6b9badb8b1983bNicolas Capens _movp(T, Src0); 2184b09353904b93e8914d459a286b6b9badb8b1983bNicolas Capens _psra(T, Src1); 2185b09353904b93e8914d459a286b6b9badb8b1983bNicolas Capens _movp(Dest, T); 2186b09353904b93e8914d459a286b6b9badb8b1983bNicolas Capens } break; 21877e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto case InstArithmetic::Udiv: 21887e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto case InstArithmetic::Urem: 21897e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto case InstArithmetic::Sdiv: 21907e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto case InstArithmetic::Srem: 219126217e3333150e66fc96aca79c01105906797960David Sehr llvm::report_fatal_error("Scalarized operation was expected"); 21927e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto break; 21937e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto case InstArithmetic::Fadd: { 2194c59288b334b91f4c0b2edf0de7415c68c760aa12Jim Stichnoth Variable *T = makeReg(Ty); 21957e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto _movp(T, Src0); 21967e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto _addps(T, Src1); 21977e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto _movp(Dest, T); 21987e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto } break; 21997e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto case InstArithmetic::Fsub: { 2200c59288b334b91f4c0b2edf0de7415c68c760aa12Jim Stichnoth Variable *T = makeReg(Ty); 22017e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto _movp(T, Src0); 22027e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto _subps(T, Src1); 22037e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto _movp(Dest, T); 22047e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto } break; 22057e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto case InstArithmetic::Fmul: { 2206c59288b334b91f4c0b2edf0de7415c68c760aa12Jim Stichnoth Variable *T = makeReg(Ty); 22077e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto _movp(T, Src0); 2208ebbb5912415bf46798f064fea93863a95f32efd8Jim Stichnoth _mulps(T, Src0 == Src1 ? T : Src1); 22097e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto _movp(Dest, T); 22107e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto } break; 22117e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto case InstArithmetic::Fdiv: { 2212c59288b334b91f4c0b2edf0de7415c68c760aa12Jim Stichnoth Variable *T = makeReg(Ty); 22137e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto _movp(T, Src0); 22147e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto _divps(T, Src1); 22157e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto _movp(Dest, T); 22167e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto } break; 22177e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto case InstArithmetic::Frem: 221826217e3333150e66fc96aca79c01105906797960David Sehr llvm::report_fatal_error("Scalarized operation was expected"); 22197e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto break; 22207e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto } 22217e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto return; 22227e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto } 22237e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto Variable *T_edx = nullptr; 22247e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto Variable *T = nullptr; 22258cfeb69e17190d3bfe22a8a1cbd7679a114d68cfJim Stichnoth switch (Instr->getOp()) { 22267e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto case InstArithmetic::_num: 22277e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto llvm_unreachable("Unknown arithmetic operator"); 22287e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto break; 22295b7e1c06cfda07cb09d65fc526a39433a63f9bd4Manasij Mukherjee case InstArithmetic::Add: { 22305b7e1c06cfda07cb09d65fc526a39433a63f9bd4Manasij Mukherjee const bool ValidType = 22315b7e1c06cfda07cb09d65fc526a39433a63f9bd4Manasij Mukherjee Ty == IceType_i32 || (Ty == IceType_i64 && Traits::Is64Bit); 22325b7e1c06cfda07cb09d65fc526a39433a63f9bd4Manasij Mukherjee auto *Const = llvm::dyn_cast<Constant>(Instr->getSrc(1)); 22335b7e1c06cfda07cb09d65fc526a39433a63f9bd4Manasij Mukherjee const bool ValidKind = 22345b7e1c06cfda07cb09d65fc526a39433a63f9bd4Manasij Mukherjee Const != nullptr && (llvm::isa<ConstantInteger32>(Const) || 22355b7e1c06cfda07cb09d65fc526a39433a63f9bd4Manasij Mukherjee llvm::isa<ConstantRelocatable>(Const)); 22365b7e1c06cfda07cb09d65fc526a39433a63f9bd4Manasij Mukherjee if (getFlags().getAggressiveLea() && ValidType && ValidKind) { 22375b7e1c06cfda07cb09d65fc526a39433a63f9bd4Manasij Mukherjee auto *Var = legalizeToReg(Src0); 22385b7e1c06cfda07cb09d65fc526a39433a63f9bd4Manasij Mukherjee auto *Mem = Traits::X86OperandMem::create(Func, IceType_void, Var, Const); 22395b7e1c06cfda07cb09d65fc526a39433a63f9bd4Manasij Mukherjee T = makeReg(Ty); 22405b7e1c06cfda07cb09d65fc526a39433a63f9bd4Manasij Mukherjee _lea(T, _sandbox_mem_reference(Mem)); 22415b7e1c06cfda07cb09d65fc526a39433a63f9bd4Manasij Mukherjee _mov(Dest, T); 22425b7e1c06cfda07cb09d65fc526a39433a63f9bd4Manasij Mukherjee break; 22435b7e1c06cfda07cb09d65fc526a39433a63f9bd4Manasij Mukherjee } 22447e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto _mov(T, Src0); 22457e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto _add(T, Src1); 22467e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto _mov(Dest, T); 22475b7e1c06cfda07cb09d65fc526a39433a63f9bd4Manasij Mukherjee } break; 22487e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto case InstArithmetic::And: 22497e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto _mov(T, Src0); 22507e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto _and(T, Src1); 22517e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto _mov(Dest, T); 22527e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto break; 22537e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto case InstArithmetic::Or: 22547e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto _mov(T, Src0); 22557e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto _or(T, Src1); 22567e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto _mov(Dest, T); 22577e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto break; 22587e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto case InstArithmetic::Xor: 22597e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto _mov(T, Src0); 22607e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto _xor(T, Src1); 22617e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto _mov(Dest, T); 22627e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto break; 22637e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto case InstArithmetic::Sub: 22647e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto _mov(T, Src0); 22657e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto _sub(T, Src1); 22667e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto _mov(Dest, T); 22677e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto break; 22687e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto case InstArithmetic::Mul: 22697e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto if (auto *C = llvm::dyn_cast<ConstantInteger32>(Src1)) { 22707e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto if (optimizeScalarMul(Dest, Src0, C->getValue())) 22717e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto return; 22727e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto } 227357e126899b20c65ff3ea23a3b7d7a67ab30b99dcAndrew Scull // The 8-bit version of imul only allows the form "imul r/m8" where T must 22745bff61c44841990680781892036adb17b3cff0c4Jim Stichnoth // be in al. 2275c59288b334b91f4c0b2edf0de7415c68c760aa12Jim Stichnoth if (isByteSizedArithType(Ty)) { 22765bff61c44841990680781892036adb17b3cff0c4Jim Stichnoth _mov(T, Src0, Traits::RegisterSet::Reg_al); 22777e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto Src1 = legalize(Src1, Legal_Reg | Legal_Mem); 2278e11f878a3569f9e316c478d2e5595e0b000d0720David Sehr _imul(T, Src0 == Src1 ? T : Src1); 2279e11f878a3569f9e316c478d2e5595e0b000d0720David Sehr _mov(Dest, T); 2280e11f878a3569f9e316c478d2e5595e0b000d0720David Sehr } else if (auto *ImmConst = llvm::dyn_cast<ConstantInteger32>(Src1)) { 2281c59288b334b91f4c0b2edf0de7415c68c760aa12Jim Stichnoth T = makeReg(Ty); 2282e11f878a3569f9e316c478d2e5595e0b000d0720David Sehr _imul_imm(T, Src0, ImmConst); 2283e11f878a3569f9e316c478d2e5595e0b000d0720David Sehr _mov(Dest, T); 22847e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto } else { 22857e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto _mov(T, Src0); 2286e11f878a3569f9e316c478d2e5595e0b000d0720David Sehr _imul(T, Src0 == Src1 ? T : Src1); 2287e11f878a3569f9e316c478d2e5595e0b000d0720David Sehr _mov(Dest, T); 22887e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto } 22897e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto break; 22907e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto case InstArithmetic::Shl: 22917e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto _mov(T, Src0); 22929c2c093f37106ab952a439fa098d862101bb5854Jim Stichnoth if (!llvm::isa<ConstantInteger32>(Src1) && 22939c2c093f37106ab952a439fa098d862101bb5854Jim Stichnoth !llvm::isa<ConstantInteger64>(Src1)) 2294c59288b334b91f4c0b2edf0de7415c68c760aa12Jim Stichnoth Src1 = copyToReg8(Src1, Traits::RegisterSet::Reg_cl); 22957e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto _shl(T, Src1); 22967e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto _mov(Dest, T); 22977e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto break; 22987e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto case InstArithmetic::Lshr: 22997e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto _mov(T, Src0); 23009c2c093f37106ab952a439fa098d862101bb5854Jim Stichnoth if (!llvm::isa<ConstantInteger32>(Src1) && 23019c2c093f37106ab952a439fa098d862101bb5854Jim Stichnoth !llvm::isa<ConstantInteger64>(Src1)) 2302c59288b334b91f4c0b2edf0de7415c68c760aa12Jim Stichnoth Src1 = copyToReg8(Src1, Traits::RegisterSet::Reg_cl); 23037e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto _shr(T, Src1); 23047e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto _mov(Dest, T); 23057e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto break; 23067e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto case InstArithmetic::Ashr: 23077e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto _mov(T, Src0); 23089c2c093f37106ab952a439fa098d862101bb5854Jim Stichnoth if (!llvm::isa<ConstantInteger32>(Src1) && 23099c2c093f37106ab952a439fa098d862101bb5854Jim Stichnoth !llvm::isa<ConstantInteger64>(Src1)) 2310c59288b334b91f4c0b2edf0de7415c68c760aa12Jim Stichnoth Src1 = copyToReg8(Src1, Traits::RegisterSet::Reg_cl); 23117e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto _sar(T, Src1); 23127e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto _mov(Dest, T); 23137e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto break; 2314c59288b334b91f4c0b2edf0de7415c68c760aa12Jim Stichnoth case InstArithmetic::Udiv: { 23157e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto // div and idiv are the few arithmetic operators that do not allow 23167e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto // immediates as the operand. 23177e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto Src1 = legalize(Src1, Legal_Reg | Legal_Mem); 23188aa396610b7baf728631a43ea16ad3d13e38397aJim Stichnoth RegNumT Eax; 23198aa396610b7baf728631a43ea16ad3d13e38397aJim Stichnoth RegNumT Edx; 2320c59288b334b91f4c0b2edf0de7415c68c760aa12Jim Stichnoth switch (Ty) { 2321c59288b334b91f4c0b2edf0de7415c68c760aa12Jim Stichnoth default: 23223c275ce1e832ba9ccfb730c4235db786cf080465John Porto llvm::report_fatal_error("Bad type for udiv"); 23233c275ce1e832ba9ccfb730c4235db786cf080465John Porto case IceType_i64: 23243c275ce1e832ba9ccfb730c4235db786cf080465John Porto Eax = Traits::getRaxOrDie(); 23253c275ce1e832ba9ccfb730c4235db786cf080465John Porto Edx = Traits::getRdxOrDie(); 2326008f4ce5417c97891ea02b54b691b39aa2e2a0afJohn Porto break; 2327c59288b334b91f4c0b2edf0de7415c68c760aa12Jim Stichnoth case IceType_i32: 23283c275ce1e832ba9ccfb730c4235db786cf080465John Porto Eax = Traits::RegisterSet::Reg_eax; 23293c275ce1e832ba9ccfb730c4235db786cf080465John Porto Edx = Traits::RegisterSet::Reg_edx; 2330c59288b334b91f4c0b2edf0de7415c68c760aa12Jim Stichnoth break; 2331c59288b334b91f4c0b2edf0de7415c68c760aa12Jim Stichnoth case IceType_i16: 2332c59288b334b91f4c0b2edf0de7415c68c760aa12Jim Stichnoth Eax = Traits::RegisterSet::Reg_ax; 2333c59288b334b91f4c0b2edf0de7415c68c760aa12Jim Stichnoth Edx = Traits::RegisterSet::Reg_dx; 2334c59288b334b91f4c0b2edf0de7415c68c760aa12Jim Stichnoth break; 2335c59288b334b91f4c0b2edf0de7415c68c760aa12Jim Stichnoth case IceType_i8: 2336c59288b334b91f4c0b2edf0de7415c68c760aa12Jim Stichnoth Eax = Traits::RegisterSet::Reg_al; 2337c59288b334b91f4c0b2edf0de7415c68c760aa12Jim Stichnoth Edx = Traits::RegisterSet::Reg_ah; 2338c59288b334b91f4c0b2edf0de7415c68c760aa12Jim Stichnoth break; 23397e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto } 2340008f4ce5417c97891ea02b54b691b39aa2e2a0afJohn Porto T_edx = makeReg(Ty, Edx); 2341c59288b334b91f4c0b2edf0de7415c68c760aa12Jim Stichnoth _mov(T, Src0, Eax); 2342008f4ce5417c97891ea02b54b691b39aa2e2a0afJohn Porto _mov(T_edx, Ctx->getConstantZero(Ty)); 2343017a55389fd6ef05ad70870363911db0fc816d98Jim Stichnoth _div(T_edx, Src1, T); 2344017a55389fd6ef05ad70870363911db0fc816d98Jim Stichnoth _redefined(Context.insert<InstFakeDef>(T, T_edx)); 2345c59288b334b91f4c0b2edf0de7415c68c760aa12Jim Stichnoth _mov(Dest, T); 2346c59288b334b91f4c0b2edf0de7415c68c760aa12Jim Stichnoth } break; 23477e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto case InstArithmetic::Sdiv: 234857e126899b20c65ff3ea23a3b7d7a67ab30b99dcAndrew Scull // TODO(stichnot): Enable this after doing better performance and cross 234957e126899b20c65ff3ea23a3b7d7a67ab30b99dcAndrew Scull // testing. 2350dd6dcfaf765dc93ae64ec45d623106f4b3a3c13aJim Stichnoth if (false && Func->getOptLevel() >= Opt_1) { 235157e126899b20c65ff3ea23a3b7d7a67ab30b99dcAndrew Scull // Optimize division by constant power of 2, but not for Om1 or O0, just 235257e126899b20c65ff3ea23a3b7d7a67ab30b99dcAndrew Scull // to keep things simple there. 23537e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto if (auto *C = llvm::dyn_cast<ConstantInteger32>(Src1)) { 23542d6c82679e9786a924b76023aa30e1f44d0f8cbbJim Stichnoth const int32_t Divisor = C->getValue(); 23552d6c82679e9786a924b76023aa30e1f44d0f8cbbJim Stichnoth const uint32_t UDivisor = Divisor; 23567e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto if (Divisor > 0 && llvm::isPowerOf2_32(UDivisor)) { 23577e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto uint32_t LogDiv = llvm::Log2_32(UDivisor); 23587e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto // LLVM does the following for dest=src/(1<<log): 23597e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto // t=src 23607e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto // sar t,typewidth-1 // -1 if src is negative, 0 if not 23617e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto // shr t,typewidth-log 23627e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto // add t,src 23637e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto // sar t,log 23647e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto // dest=t 23657e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto uint32_t TypeWidth = Traits::X86_CHAR_BIT * typeWidthInBytes(Ty); 23667e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto _mov(T, Src0); 236757e126899b20c65ff3ea23a3b7d7a67ab30b99dcAndrew Scull // If for some reason we are dividing by 1, just treat it like an 236857e126899b20c65ff3ea23a3b7d7a67ab30b99dcAndrew Scull // assignment. 23697e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto if (LogDiv > 0) { 23707e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto // The initial sar is unnecessary when dividing by 2. 23717e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto if (LogDiv > 1) 23727e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto _sar(T, Ctx->getConstantInt(Ty, TypeWidth - 1)); 23737e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto _shr(T, Ctx->getConstantInt(Ty, TypeWidth - LogDiv)); 23747e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto _add(T, Src0); 23757e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto _sar(T, Ctx->getConstantInt(Ty, LogDiv)); 23767e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto } 23777e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto _mov(Dest, T); 23787e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto return; 23797e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto } 23807e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto } 23817e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto } 23827e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto Src1 = legalize(Src1, Legal_Reg | Legal_Mem); 2383c59288b334b91f4c0b2edf0de7415c68c760aa12Jim Stichnoth switch (Ty) { 23845bff61c44841990680781892036adb17b3cff0c4Jim Stichnoth default: 23853c275ce1e832ba9ccfb730c4235db786cf080465John Porto llvm::report_fatal_error("Bad type for sdiv"); 23863c275ce1e832ba9ccfb730c4235db786cf080465John Porto case IceType_i64: 23873c275ce1e832ba9ccfb730c4235db786cf080465John Porto T_edx = makeReg(Ty, Traits::getRdxOrDie()); 23883c275ce1e832ba9ccfb730c4235db786cf080465John Porto _mov(T, Src0, Traits::getRaxOrDie()); 23893c275ce1e832ba9ccfb730c4235db786cf080465John Porto break; 23905bff61c44841990680781892036adb17b3cff0c4Jim Stichnoth case IceType_i32: 23915bff61c44841990680781892036adb17b3cff0c4Jim Stichnoth T_edx = makeReg(Ty, Traits::RegisterSet::Reg_edx); 23925d0acff3a2fa421923392aadb4df2742064b6248John Porto _mov(T, Src0, Traits::RegisterSet::Reg_eax); 23935bff61c44841990680781892036adb17b3cff0c4Jim Stichnoth break; 23945bff61c44841990680781892036adb17b3cff0c4Jim Stichnoth case IceType_i16: 23955bff61c44841990680781892036adb17b3cff0c4Jim Stichnoth T_edx = makeReg(Ty, Traits::RegisterSet::Reg_dx); 23965bff61c44841990680781892036adb17b3cff0c4Jim Stichnoth _mov(T, Src0, Traits::RegisterSet::Reg_ax); 23975bff61c44841990680781892036adb17b3cff0c4Jim Stichnoth break; 23985bff61c44841990680781892036adb17b3cff0c4Jim Stichnoth case IceType_i8: 23995bff61c44841990680781892036adb17b3cff0c4Jim Stichnoth T_edx = makeReg(IceType_i16, Traits::RegisterSet::Reg_ax); 24005bff61c44841990680781892036adb17b3cff0c4Jim Stichnoth _mov(T, Src0, Traits::RegisterSet::Reg_al); 24015bff61c44841990680781892036adb17b3cff0c4Jim Stichnoth break; 24027e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto } 24035bff61c44841990680781892036adb17b3cff0c4Jim Stichnoth _cbwdq(T_edx, T); 2404017a55389fd6ef05ad70870363911db0fc816d98Jim Stichnoth _idiv(T_edx, Src1, T); 2405017a55389fd6ef05ad70870363911db0fc816d98Jim Stichnoth _redefined(Context.insert<InstFakeDef>(T, T_edx)); 24065bff61c44841990680781892036adb17b3cff0c4Jim Stichnoth _mov(Dest, T); 24077e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto break; 2408c59288b334b91f4c0b2edf0de7415c68c760aa12Jim Stichnoth case InstArithmetic::Urem: { 24097e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto Src1 = legalize(Src1, Legal_Reg | Legal_Mem); 24108aa396610b7baf728631a43ea16ad3d13e38397aJim Stichnoth RegNumT Eax; 24118aa396610b7baf728631a43ea16ad3d13e38397aJim Stichnoth RegNumT Edx; 2412c59288b334b91f4c0b2edf0de7415c68c760aa12Jim Stichnoth switch (Ty) { 2413c59288b334b91f4c0b2edf0de7415c68c760aa12Jim Stichnoth default: 24143c275ce1e832ba9ccfb730c4235db786cf080465John Porto llvm::report_fatal_error("Bad type for urem"); 24153c275ce1e832ba9ccfb730c4235db786cf080465John Porto case IceType_i64: 24163c275ce1e832ba9ccfb730c4235db786cf080465John Porto Eax = Traits::getRaxOrDie(); 24173c275ce1e832ba9ccfb730c4235db786cf080465John Porto Edx = Traits::getRdxOrDie(); 24183c275ce1e832ba9ccfb730c4235db786cf080465John Porto break; 2419c59288b334b91f4c0b2edf0de7415c68c760aa12Jim Stichnoth case IceType_i32: 24203c275ce1e832ba9ccfb730c4235db786cf080465John Porto Eax = Traits::RegisterSet::Reg_eax; 24213c275ce1e832ba9ccfb730c4235db786cf080465John Porto Edx = Traits::RegisterSet::Reg_edx; 2422c59288b334b91f4c0b2edf0de7415c68c760aa12Jim Stichnoth break; 2423c59288b334b91f4c0b2edf0de7415c68c760aa12Jim Stichnoth case IceType_i16: 2424c59288b334b91f4c0b2edf0de7415c68c760aa12Jim Stichnoth Eax = Traits::RegisterSet::Reg_ax; 2425c59288b334b91f4c0b2edf0de7415c68c760aa12Jim Stichnoth Edx = Traits::RegisterSet::Reg_dx; 2426c59288b334b91f4c0b2edf0de7415c68c760aa12Jim Stichnoth break; 2427c59288b334b91f4c0b2edf0de7415c68c760aa12Jim Stichnoth case IceType_i8: 2428c59288b334b91f4c0b2edf0de7415c68c760aa12Jim Stichnoth Eax = Traits::RegisterSet::Reg_al; 2429c59288b334b91f4c0b2edf0de7415c68c760aa12Jim Stichnoth Edx = Traits::RegisterSet::Reg_ah; 2430c59288b334b91f4c0b2edf0de7415c68c760aa12Jim Stichnoth break; 24317e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto } 2432c59288b334b91f4c0b2edf0de7415c68c760aa12Jim Stichnoth T_edx = makeReg(Ty, Edx); 2433c59288b334b91f4c0b2edf0de7415c68c760aa12Jim Stichnoth _mov(T_edx, Ctx->getConstantZero(Ty)); 2434c59288b334b91f4c0b2edf0de7415c68c760aa12Jim Stichnoth _mov(T, Src0, Eax); 2435017a55389fd6ef05ad70870363911db0fc816d98Jim Stichnoth _div(T, Src1, T_edx); 2436017a55389fd6ef05ad70870363911db0fc816d98Jim Stichnoth _redefined(Context.insert<InstFakeDef>(T_edx, T)); 24372655d9627f11b0c97f10eacf1f730bf66e5db05bJim Stichnoth if (Ty == IceType_i8) { 24382655d9627f11b0c97f10eacf1f730bf66e5db05bJim Stichnoth // Register ah must be moved into one of {al,bl,cl,dl} before it can be 24392655d9627f11b0c97f10eacf1f730bf66e5db05bJim Stichnoth // moved into a general 8-bit register. 24402655d9627f11b0c97f10eacf1f730bf66e5db05bJim Stichnoth auto *T_AhRcvr = makeReg(Ty); 24412655d9627f11b0c97f10eacf1f730bf66e5db05bJim Stichnoth T_AhRcvr->setRegClass(RCX86_IsAhRcvr); 24422655d9627f11b0c97f10eacf1f730bf66e5db05bJim Stichnoth _mov(T_AhRcvr, T_edx); 24432655d9627f11b0c97f10eacf1f730bf66e5db05bJim Stichnoth T_edx = T_AhRcvr; 24442655d9627f11b0c97f10eacf1f730bf66e5db05bJim Stichnoth } 2445c59288b334b91f4c0b2edf0de7415c68c760aa12Jim Stichnoth _mov(Dest, T_edx); 2446c59288b334b91f4c0b2edf0de7415c68c760aa12Jim Stichnoth } break; 2447c59288b334b91f4c0b2edf0de7415c68c760aa12Jim Stichnoth case InstArithmetic::Srem: { 244857e126899b20c65ff3ea23a3b7d7a67ab30b99dcAndrew Scull // TODO(stichnot): Enable this after doing better performance and cross 244957e126899b20c65ff3ea23a3b7d7a67ab30b99dcAndrew Scull // testing. 2450dd6dcfaf765dc93ae64ec45d623106f4b3a3c13aJim Stichnoth if (false && Func->getOptLevel() >= Opt_1) { 245157e126899b20c65ff3ea23a3b7d7a67ab30b99dcAndrew Scull // Optimize mod by constant power of 2, but not for Om1 or O0, just to 245257e126899b20c65ff3ea23a3b7d7a67ab30b99dcAndrew Scull // keep things simple there. 24537e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto if (auto *C = llvm::dyn_cast<ConstantInteger32>(Src1)) { 24542d6c82679e9786a924b76023aa30e1f44d0f8cbbJim Stichnoth const int32_t Divisor = C->getValue(); 24552d6c82679e9786a924b76023aa30e1f44d0f8cbbJim Stichnoth const uint32_t UDivisor = Divisor; 24567e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto if (Divisor > 0 && llvm::isPowerOf2_32(UDivisor)) { 24577e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto uint32_t LogDiv = llvm::Log2_32(UDivisor); 24587e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto // LLVM does the following for dest=src%(1<<log): 24597e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto // t=src 24607e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto // sar t,typewidth-1 // -1 if src is negative, 0 if not 24617e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto // shr t,typewidth-log 24627e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto // add t,src 24637e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto // and t, -(1<<log) 24647e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto // sub t,src 24657e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto // neg t 24667e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto // dest=t 24677e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto uint32_t TypeWidth = Traits::X86_CHAR_BIT * typeWidthInBytes(Ty); 24687e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto // If for some reason we are dividing by 1, just assign 0. 24697e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto if (LogDiv == 0) { 24707e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto _mov(Dest, Ctx->getConstantZero(Ty)); 24717e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto return; 24727e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto } 24737e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto _mov(T, Src0); 24747e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto // The initial sar is unnecessary when dividing by 2. 24757e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto if (LogDiv > 1) 24767e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto _sar(T, Ctx->getConstantInt(Ty, TypeWidth - 1)); 24777e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto _shr(T, Ctx->getConstantInt(Ty, TypeWidth - LogDiv)); 24787e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto _add(T, Src0); 24797e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto _and(T, Ctx->getConstantInt(Ty, -(1 << LogDiv))); 24807e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto _sub(T, Src0); 24817e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto _neg(T); 24827e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto _mov(Dest, T); 24837e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto return; 24847e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto } 24857e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto } 24867e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto } 24877e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto Src1 = legalize(Src1, Legal_Reg | Legal_Mem); 24888aa396610b7baf728631a43ea16ad3d13e38397aJim Stichnoth RegNumT Eax; 24898aa396610b7baf728631a43ea16ad3d13e38397aJim Stichnoth RegNumT Edx; 2490c59288b334b91f4c0b2edf0de7415c68c760aa12Jim Stichnoth switch (Ty) { 24915bff61c44841990680781892036adb17b3cff0c4Jim Stichnoth default: 24923c275ce1e832ba9ccfb730c4235db786cf080465John Porto llvm::report_fatal_error("Bad type for srem"); 24933c275ce1e832ba9ccfb730c4235db786cf080465John Porto case IceType_i64: 24943c275ce1e832ba9ccfb730c4235db786cf080465John Porto Eax = Traits::getRaxOrDie(); 24953c275ce1e832ba9ccfb730c4235db786cf080465John Porto Edx = Traits::getRdxOrDie(); 24963c275ce1e832ba9ccfb730c4235db786cf080465John Porto break; 24975bff61c44841990680781892036adb17b3cff0c4Jim Stichnoth case IceType_i32: 24983c275ce1e832ba9ccfb730c4235db786cf080465John Porto Eax = Traits::RegisterSet::Reg_eax; 24993c275ce1e832ba9ccfb730c4235db786cf080465John Porto Edx = Traits::RegisterSet::Reg_edx; 25005bff61c44841990680781892036adb17b3cff0c4Jim Stichnoth break; 25015bff61c44841990680781892036adb17b3cff0c4Jim Stichnoth case IceType_i16: 2502c59288b334b91f4c0b2edf0de7415c68c760aa12Jim Stichnoth Eax = Traits::RegisterSet::Reg_ax; 2503c59288b334b91f4c0b2edf0de7415c68c760aa12Jim Stichnoth Edx = Traits::RegisterSet::Reg_dx; 25045bff61c44841990680781892036adb17b3cff0c4Jim Stichnoth break; 25055bff61c44841990680781892036adb17b3cff0c4Jim Stichnoth case IceType_i8: 2506c59288b334b91f4c0b2edf0de7415c68c760aa12Jim Stichnoth Eax = Traits::RegisterSet::Reg_al; 2507c59288b334b91f4c0b2edf0de7415c68c760aa12Jim Stichnoth Edx = Traits::RegisterSet::Reg_ah; 25085bff61c44841990680781892036adb17b3cff0c4Jim Stichnoth break; 25097e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto } 2510c59288b334b91f4c0b2edf0de7415c68c760aa12Jim Stichnoth T_edx = makeReg(Ty, Edx); 2511c59288b334b91f4c0b2edf0de7415c68c760aa12Jim Stichnoth _mov(T, Src0, Eax); 2512c59288b334b91f4c0b2edf0de7415c68c760aa12Jim Stichnoth _cbwdq(T_edx, T); 2513017a55389fd6ef05ad70870363911db0fc816d98Jim Stichnoth _idiv(T, Src1, T_edx); 2514017a55389fd6ef05ad70870363911db0fc816d98Jim Stichnoth _redefined(Context.insert<InstFakeDef>(T_edx, T)); 25152655d9627f11b0c97f10eacf1f730bf66e5db05bJim Stichnoth if (Ty == IceType_i8) { 25162655d9627f11b0c97f10eacf1f730bf66e5db05bJim Stichnoth // Register ah must be moved into one of {al,bl,cl,dl} before it can be 25172655d9627f11b0c97f10eacf1f730bf66e5db05bJim Stichnoth // moved into a general 8-bit register. 25182655d9627f11b0c97f10eacf1f730bf66e5db05bJim Stichnoth auto *T_AhRcvr = makeReg(Ty); 25192655d9627f11b0c97f10eacf1f730bf66e5db05bJim Stichnoth T_AhRcvr->setRegClass(RCX86_IsAhRcvr); 25202655d9627f11b0c97f10eacf1f730bf66e5db05bJim Stichnoth _mov(T_AhRcvr, T_edx); 25212655d9627f11b0c97f10eacf1f730bf66e5db05bJim Stichnoth T_edx = T_AhRcvr; 25222655d9627f11b0c97f10eacf1f730bf66e5db05bJim Stichnoth } 2523c59288b334b91f4c0b2edf0de7415c68c760aa12Jim Stichnoth _mov(Dest, T_edx); 2524c59288b334b91f4c0b2edf0de7415c68c760aa12Jim Stichnoth } break; 25257e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto case InstArithmetic::Fadd: 25267e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto _mov(T, Src0); 25277e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto _addss(T, Src1); 25287e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto _mov(Dest, T); 25297e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto break; 25307e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto case InstArithmetic::Fsub: 25317e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto _mov(T, Src0); 25327e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto _subss(T, Src1); 25337e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto _mov(Dest, T); 25347e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto break; 25357e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto case InstArithmetic::Fmul: 25367e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto _mov(T, Src0); 2537ebbb5912415bf46798f064fea93863a95f32efd8Jim Stichnoth _mulss(T, Src0 == Src1 ? T : Src1); 25387e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto _mov(Dest, T); 25397e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto break; 25407e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto case InstArithmetic::Fdiv: 25417e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto _mov(T, Src0); 25427e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto _divss(T, Src1); 25437e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto _mov(Dest, T); 25447e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto break; 254526217e3333150e66fc96aca79c01105906797960David Sehr case InstArithmetic::Frem: 254626217e3333150e66fc96aca79c01105906797960David Sehr llvm::report_fatal_error("Helper call was expected"); 254726217e3333150e66fc96aca79c01105906797960David Sehr break; 25487e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto } 25497e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto} 25507e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto 25514a56686b5b56db6803f90ad53514bf2fa190d9f7John Portotemplate <typename TraitsType> 25528cfeb69e17190d3bfe22a8a1cbd7679a114d68cfJim Stichnothvoid TargetX86Base<TraitsType>::lowerAssign(const InstAssign *Instr) { 25538cfeb69e17190d3bfe22a8a1cbd7679a114d68cfJim Stichnoth Variable *Dest = Instr->getDest(); 25543607b6c924922d261cbc75b68e07a1e2a5aec5dcJim Stichnoth if (Dest->isRematerializable()) { 25551d937a8e35b7ae51ee64c32f8f34d51ba33498f4John Porto Context.insert<InstFakeDef>(Dest); 25563607b6c924922d261cbc75b68e07a1e2a5aec5dcJim Stichnoth return; 25573607b6c924922d261cbc75b68e07a1e2a5aec5dcJim Stichnoth } 25588cfeb69e17190d3bfe22a8a1cbd7679a114d68cfJim Stichnoth Operand *Src = Instr->getSrc(0); 2559e398428c3ef8ed2339c0f334dd30a7d3f0ed4b6eDavid Sehr assert(Dest->getType() == Src->getType()); 2560e398428c3ef8ed2339c0f334dd30a7d3f0ed4b6eDavid Sehr lowerMove(Dest, Src, false); 25617e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto} 25627e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto 25634a56686b5b56db6803f90ad53514bf2fa190d9f7John Portotemplate <typename TraitsType> 25644a56686b5b56db6803f90ad53514bf2fa190d9f7John Portovoid TargetX86Base<TraitsType>::lowerBr(const InstBr *Br) { 2565e398428c3ef8ed2339c0f334dd30a7d3f0ed4b6eDavid Sehr if (Br->isUnconditional()) { 2566e398428c3ef8ed2339c0f334dd30a7d3f0ed4b6eDavid Sehr _br(Br->getTargetUnconditional()); 25677e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto return; 25687e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto } 2569e398428c3ef8ed2339c0f334dd30a7d3f0ed4b6eDavid Sehr Operand *Cond = Br->getCondition(); 25707e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto 25717e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto // Handle folding opportunities. 2572e398428c3ef8ed2339c0f334dd30a7d3f0ed4b6eDavid Sehr if (const Inst *Producer = FoldingInfo.getProducerFor(Cond)) { 25737e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto assert(Producer->isDeleted()); 2574caeaa27b18e94cbad635dbe21da87e816a35c7daJim Stichnoth switch (BoolFolding<Traits>::getProducerKind(Producer)) { 25757e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto default: 25767e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto break; 2577caeaa27b18e94cbad635dbe21da87e816a35c7daJim Stichnoth case BoolFolding<Traits>::PK_Icmp32: 2578caeaa27b18e94cbad635dbe21da87e816a35c7daJim Stichnoth case BoolFolding<Traits>::PK_Icmp64: { 25792d6c82679e9786a924b76023aa30e1f44d0f8cbbJim Stichnoth lowerIcmpAndConsumer(llvm::cast<InstIcmp>(Producer), Br); 25807e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto return; 25817e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto } 2582caeaa27b18e94cbad635dbe21da87e816a35c7daJim Stichnoth case BoolFolding<Traits>::PK_Fcmp: { 25832d6c82679e9786a924b76023aa30e1f44d0f8cbbJim Stichnoth lowerFcmpAndConsumer(llvm::cast<InstFcmp>(Producer), Br); 2584daf096cd1b8b9ec3d29c8d7823c3d60cd51c65c9David Sehr return; 2585daf096cd1b8b9ec3d29c8d7823c3d60cd51c65c9David Sehr } 2586caeaa27b18e94cbad635dbe21da87e816a35c7daJim Stichnoth case BoolFolding<Traits>::PK_Arith: { 25872d6c82679e9786a924b76023aa30e1f44d0f8cbbJim Stichnoth lowerArithAndConsumer(llvm::cast<InstArithmetic>(Producer), Br); 2588daf096cd1b8b9ec3d29c8d7823c3d60cd51c65c9David Sehr return; 2589daf096cd1b8b9ec3d29c8d7823c3d60cd51c65c9David Sehr } 25907e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto } 25917e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto } 25927e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto Operand *Src0 = legalize(Cond, Legal_Reg | Legal_Mem); 25937e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto Constant *Zero = Ctx->getConstantZero(IceType_i32); 25947e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto _cmp(Src0, Zero); 2595e398428c3ef8ed2339c0f334dd30a7d3f0ed4b6eDavid Sehr _br(Traits::Cond::Br_ne, Br->getTargetTrue(), Br->getTargetFalse()); 25967e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto} 25977e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto 25980c68bef895ea8987922922ede6f1918763eaa00bDavid Sehr// constexprMax returns a (constexpr) max(S0, S1), and it is used for defining 25990c68bef895ea8987922922ede6f1918763eaa00bDavid Sehr// OperandList in lowerCall. std::max() is supposed to work, but it doesn't. 26000c68bef895ea8987922922ede6f1918763eaa00bDavid Sehrinline constexpr SizeT constexprMax(SizeT S0, SizeT S1) { 26010c68bef895ea8987922922ede6f1918763eaa00bDavid Sehr return S0 < S1 ? S1 : S0; 26020c68bef895ea8987922922ede6f1918763eaa00bDavid Sehr} 26030c68bef895ea8987922922ede6f1918763eaa00bDavid Sehr 26040c68bef895ea8987922922ede6f1918763eaa00bDavid Sehrtemplate <typename TraitsType> 26050c68bef895ea8987922922ede6f1918763eaa00bDavid Sehrvoid TargetX86Base<TraitsType>::lowerCall(const InstCall *Instr) { 26060c68bef895ea8987922922ede6f1918763eaa00bDavid Sehr // Common x86 calling convention lowering: 26070c68bef895ea8987922922ede6f1918763eaa00bDavid Sehr // 26080c68bef895ea8987922922ede6f1918763eaa00bDavid Sehr // * At the point before the call, the stack must be aligned to 16 bytes. 26090c68bef895ea8987922922ede6f1918763eaa00bDavid Sehr // 26100c68bef895ea8987922922ede6f1918763eaa00bDavid Sehr // * Non-register arguments are pushed onto the stack in right-to-left order, 26110c68bef895ea8987922922ede6f1918763eaa00bDavid Sehr // such that the left-most argument ends up on the top of the stack at the 26120c68bef895ea8987922922ede6f1918763eaa00bDavid Sehr // lowest memory address. 26130c68bef895ea8987922922ede6f1918763eaa00bDavid Sehr // 26140c68bef895ea8987922922ede6f1918763eaa00bDavid Sehr // * Stack arguments of vector type are aligned to start at the next highest 26150c68bef895ea8987922922ede6f1918763eaa00bDavid Sehr // multiple of 16 bytes. Other stack arguments are aligned to the next word 26160c68bef895ea8987922922ede6f1918763eaa00bDavid Sehr // size boundary (4 or 8 bytes, respectively). 2617a551dfce9bd44034b4fd588cd4b06b3b0435df8dNicolas Capens RequiredStackAlignment = std::max<size_t>(RequiredStackAlignment, 2618a551dfce9bd44034b4fd588cd4b06b3b0435df8dNicolas Capens Traits::X86_STACK_ALIGNMENT_BYTES); 26190c68bef895ea8987922922ede6f1918763eaa00bDavid Sehr 26200c68bef895ea8987922922ede6f1918763eaa00bDavid Sehr using OperandList = 26210c68bef895ea8987922922ede6f1918763eaa00bDavid Sehr llvm::SmallVector<Operand *, constexprMax(Traits::X86_MAX_XMM_ARGS, 26220c68bef895ea8987922922ede6f1918763eaa00bDavid Sehr Traits::X86_MAX_GPR_ARGS)>; 26230c68bef895ea8987922922ede6f1918763eaa00bDavid Sehr OperandList XmmArgs; 26240c68bef895ea8987922922ede6f1918763eaa00bDavid Sehr CfgVector<std::pair<const Type, Operand *>> GprArgs; 26250c68bef895ea8987922922ede6f1918763eaa00bDavid Sehr OperandList StackArgs, StackArgLocations; 26260c68bef895ea8987922922ede6f1918763eaa00bDavid Sehr uint32_t ParameterAreaSizeBytes = 0; 26270c68bef895ea8987922922ede6f1918763eaa00bDavid Sehr 26280c68bef895ea8987922922ede6f1918763eaa00bDavid Sehr // Classify each argument operand according to the location where the argument 26290c68bef895ea8987922922ede6f1918763eaa00bDavid Sehr // is passed. 26300c68bef895ea8987922922ede6f1918763eaa00bDavid Sehr for (SizeT i = 0, NumArgs = Instr->getNumArgs(); i < NumArgs; ++i) { 26310c68bef895ea8987922922ede6f1918763eaa00bDavid Sehr Operand *Arg = Instr->getArg(i); 26320c68bef895ea8987922922ede6f1918763eaa00bDavid Sehr const Type Ty = Arg->getType(); 26330c68bef895ea8987922922ede6f1918763eaa00bDavid Sehr // The PNaCl ABI requires the width of arguments to be at least 32 bits. 26340c68bef895ea8987922922ede6f1918763eaa00bDavid Sehr assert(typeWidthInBytes(Ty) >= 4); 26355fa0a5f7f730f33e6987527a261de8d833d7585cReed Kotler if (isVectorType(Ty) && 26365fa0a5f7f730f33e6987527a261de8d833d7585cReed Kotler Traits::getRegisterForXmmArgNum(XmmArgs.size()).hasValue()) { 26370c68bef895ea8987922922ede6f1918763eaa00bDavid Sehr XmmArgs.push_back(Arg); 26380c68bef895ea8987922922ede6f1918763eaa00bDavid Sehr } else if (isScalarFloatingType(Ty) && Traits::X86_PASS_SCALAR_FP_IN_XMM && 26395fa0a5f7f730f33e6987527a261de8d833d7585cReed Kotler Traits::getRegisterForXmmArgNum(XmmArgs.size()).hasValue()) { 26400c68bef895ea8987922922ede6f1918763eaa00bDavid Sehr XmmArgs.push_back(Arg); 26410c68bef895ea8987922922ede6f1918763eaa00bDavid Sehr } else if (isScalarIntegerType(Ty) && 26425fa0a5f7f730f33e6987527a261de8d833d7585cReed Kotler Traits::getRegisterForGprArgNum(Ty, GprArgs.size()).hasValue()) { 26430c68bef895ea8987922922ede6f1918763eaa00bDavid Sehr GprArgs.emplace_back(Ty, Arg); 26440c68bef895ea8987922922ede6f1918763eaa00bDavid Sehr } else { 26450c68bef895ea8987922922ede6f1918763eaa00bDavid Sehr // Place on stack. 26460c68bef895ea8987922922ede6f1918763eaa00bDavid Sehr StackArgs.push_back(Arg); 26470c68bef895ea8987922922ede6f1918763eaa00bDavid Sehr if (isVectorType(Arg->getType())) { 26480c68bef895ea8987922922ede6f1918763eaa00bDavid Sehr ParameterAreaSizeBytes = 26490c68bef895ea8987922922ede6f1918763eaa00bDavid Sehr Traits::applyStackAlignment(ParameterAreaSizeBytes); 26500c68bef895ea8987922922ede6f1918763eaa00bDavid Sehr } 26510c68bef895ea8987922922ede6f1918763eaa00bDavid Sehr Variable *esp = getPhysicalRegister(getStackReg(), Traits::WordType); 26520c68bef895ea8987922922ede6f1918763eaa00bDavid Sehr Constant *Loc = Ctx->getConstantInt32(ParameterAreaSizeBytes); 26530c68bef895ea8987922922ede6f1918763eaa00bDavid Sehr StackArgLocations.push_back( 26540c68bef895ea8987922922ede6f1918763eaa00bDavid Sehr Traits::X86OperandMem::create(Func, Ty, esp, Loc)); 26550c68bef895ea8987922922ede6f1918763eaa00bDavid Sehr ParameterAreaSizeBytes += typeWidthInBytesOnStack(Arg->getType()); 26560c68bef895ea8987922922ede6f1918763eaa00bDavid Sehr } 26570c68bef895ea8987922922ede6f1918763eaa00bDavid Sehr } 26580c68bef895ea8987922922ede6f1918763eaa00bDavid Sehr // Ensure there is enough space for the fstp/movs for floating returns. 26590c68bef895ea8987922922ede6f1918763eaa00bDavid Sehr Variable *Dest = Instr->getDest(); 26600c68bef895ea8987922922ede6f1918763eaa00bDavid Sehr const Type DestTy = Dest ? Dest->getType() : IceType_void; 26614ab4fbed8ef1cd0d8eafb205c86ae451ce92b3f6John Porto if (!Traits::X86_PASS_SCALAR_FP_IN_XMM) { 26620c68bef895ea8987922922ede6f1918763eaa00bDavid Sehr if (isScalarFloatingType(DestTy)) { 26630c68bef895ea8987922922ede6f1918763eaa00bDavid Sehr ParameterAreaSizeBytes = 26640c68bef895ea8987922922ede6f1918763eaa00bDavid Sehr std::max(static_cast<size_t>(ParameterAreaSizeBytes), 26650c68bef895ea8987922922ede6f1918763eaa00bDavid Sehr typeWidthInBytesOnStack(DestTy)); 26660c68bef895ea8987922922ede6f1918763eaa00bDavid Sehr } 26670c68bef895ea8987922922ede6f1918763eaa00bDavid Sehr } 26680c68bef895ea8987922922ede6f1918763eaa00bDavid Sehr // Adjust the parameter area so that the stack is aligned. It is assumed that 26690c68bef895ea8987922922ede6f1918763eaa00bDavid Sehr // the stack is already aligned at the start of the calling sequence. 26700c68bef895ea8987922922ede6f1918763eaa00bDavid Sehr ParameterAreaSizeBytes = Traits::applyStackAlignment(ParameterAreaSizeBytes); 26710c68bef895ea8987922922ede6f1918763eaa00bDavid Sehr assert(ParameterAreaSizeBytes <= maxOutArgsSizeBytes()); 26720c68bef895ea8987922922ede6f1918763eaa00bDavid Sehr // Copy arguments that are passed on the stack to the appropriate stack 2673f531931fb45172157bc5816a8f55309556996dc7Jim Stichnoth // locations. We make sure legalize() is called on each argument at this 2674f531931fb45172157bc5816a8f55309556996dc7Jim Stichnoth // point, to allow availabilityGet() to work. 26750c68bef895ea8987922922ede6f1918763eaa00bDavid Sehr for (SizeT i = 0, NumStackArgs = StackArgs.size(); i < NumStackArgs; ++i) { 2676f531931fb45172157bc5816a8f55309556996dc7Jim Stichnoth lowerStore( 2677f531931fb45172157bc5816a8f55309556996dc7Jim Stichnoth InstStore::create(Func, legalize(StackArgs[i]), StackArgLocations[i])); 26780c68bef895ea8987922922ede6f1918763eaa00bDavid Sehr } 26790c68bef895ea8987922922ede6f1918763eaa00bDavid Sehr // Copy arguments to be passed in registers to the appropriate registers. 26800c68bef895ea8987922922ede6f1918763eaa00bDavid Sehr for (SizeT i = 0, NumXmmArgs = XmmArgs.size(); i < NumXmmArgs; ++i) { 2681f531931fb45172157bc5816a8f55309556996dc7Jim Stichnoth XmmArgs[i] = 2682f531931fb45172157bc5816a8f55309556996dc7Jim Stichnoth legalizeToReg(legalize(XmmArgs[i]), Traits::getRegisterForXmmArgNum(i)); 26830c68bef895ea8987922922ede6f1918763eaa00bDavid Sehr } 26840c68bef895ea8987922922ede6f1918763eaa00bDavid Sehr // Materialize moves for arguments passed in GPRs. 26850c68bef895ea8987922922ede6f1918763eaa00bDavid Sehr for (SizeT i = 0, NumGprArgs = GprArgs.size(); i < NumGprArgs; ++i) { 26860c68bef895ea8987922922ede6f1918763eaa00bDavid Sehr const Type SignatureTy = GprArgs[i].first; 2687e450656d45ef91dc3a1cf51eda0e13fffdb7ff7dJim Stichnoth Operand *Arg = 2688e450656d45ef91dc3a1cf51eda0e13fffdb7ff7dJim Stichnoth legalize(GprArgs[i].second, Legal_Default | Legal_Rematerializable); 2689f531931fb45172157bc5816a8f55309556996dc7Jim Stichnoth GprArgs[i].second = 26900c68bef895ea8987922922ede6f1918763eaa00bDavid Sehr legalizeToReg(Arg, Traits::getRegisterForGprArgNum(Arg->getType(), i)); 26910c68bef895ea8987922922ede6f1918763eaa00bDavid Sehr assert(SignatureTy == IceType_i64 || SignatureTy == IceType_i32); 26920c68bef895ea8987922922ede6f1918763eaa00bDavid Sehr assert(SignatureTy == Arg->getType()); 26930c68bef895ea8987922922ede6f1918763eaa00bDavid Sehr (void)SignatureTy; 2694f531931fb45172157bc5816a8f55309556996dc7Jim Stichnoth } 2695f531931fb45172157bc5816a8f55309556996dc7Jim Stichnoth // Generate a FakeUse of register arguments so that they do not get dead code 2696f531931fb45172157bc5816a8f55309556996dc7Jim Stichnoth // eliminated as a result of the FakeKill of scratch registers after the call. 2697f531931fb45172157bc5816a8f55309556996dc7Jim Stichnoth // These need to be right before the call instruction. 2698f531931fb45172157bc5816a8f55309556996dc7Jim Stichnoth for (auto *Arg : XmmArgs) { 2699f531931fb45172157bc5816a8f55309556996dc7Jim Stichnoth Context.insert<InstFakeUse>(llvm::cast<Variable>(Arg)); 2700f531931fb45172157bc5816a8f55309556996dc7Jim Stichnoth } 2701f531931fb45172157bc5816a8f55309556996dc7Jim Stichnoth for (auto &ArgPair : GprArgs) { 2702f531931fb45172157bc5816a8f55309556996dc7Jim Stichnoth Context.insert<InstFakeUse>(llvm::cast<Variable>(ArgPair.second)); 27030c68bef895ea8987922922ede6f1918763eaa00bDavid Sehr } 27040c68bef895ea8987922922ede6f1918763eaa00bDavid Sehr // Generate the call instruction. Assign its result to a temporary with high 27050c68bef895ea8987922922ede6f1918763eaa00bDavid Sehr // register allocation weight. 27060c68bef895ea8987922922ede6f1918763eaa00bDavid Sehr // ReturnReg doubles as ReturnRegLo as necessary. 27070c68bef895ea8987922922ede6f1918763eaa00bDavid Sehr Variable *ReturnReg = nullptr; 27080c68bef895ea8987922922ede6f1918763eaa00bDavid Sehr Variable *ReturnRegHi = nullptr; 27090c68bef895ea8987922922ede6f1918763eaa00bDavid Sehr if (Dest) { 27100c68bef895ea8987922922ede6f1918763eaa00bDavid Sehr switch (DestTy) { 27110c68bef895ea8987922922ede6f1918763eaa00bDavid Sehr case IceType_NUM: 27120c68bef895ea8987922922ede6f1918763eaa00bDavid Sehr case IceType_void: 27130c68bef895ea8987922922ede6f1918763eaa00bDavid Sehr case IceType_i1: 27140c68bef895ea8987922922ede6f1918763eaa00bDavid Sehr case IceType_i8: 27150c68bef895ea8987922922ede6f1918763eaa00bDavid Sehr case IceType_i16: 27160c68bef895ea8987922922ede6f1918763eaa00bDavid Sehr llvm::report_fatal_error("Invalid Call dest type"); 27170c68bef895ea8987922922ede6f1918763eaa00bDavid Sehr break; 27180c68bef895ea8987922922ede6f1918763eaa00bDavid Sehr case IceType_i32: 27190c68bef895ea8987922922ede6f1918763eaa00bDavid Sehr ReturnReg = makeReg(DestTy, Traits::RegisterSet::Reg_eax); 27200c68bef895ea8987922922ede6f1918763eaa00bDavid Sehr break; 27210c68bef895ea8987922922ede6f1918763eaa00bDavid Sehr case IceType_i64: 27220c68bef895ea8987922922ede6f1918763eaa00bDavid Sehr if (Traits::Is64Bit) { 2723ee1aae820b08055583e6c9be7a3118d1e975a45dJim Stichnoth ReturnReg = makeReg(IceType_i64, Traits::getRaxOrDie()); 27240c68bef895ea8987922922ede6f1918763eaa00bDavid Sehr } else { 27250c68bef895ea8987922922ede6f1918763eaa00bDavid Sehr ReturnReg = makeReg(IceType_i32, Traits::RegisterSet::Reg_eax); 27260c68bef895ea8987922922ede6f1918763eaa00bDavid Sehr ReturnRegHi = makeReg(IceType_i32, Traits::RegisterSet::Reg_edx); 27270c68bef895ea8987922922ede6f1918763eaa00bDavid Sehr } 27280c68bef895ea8987922922ede6f1918763eaa00bDavid Sehr break; 27290c68bef895ea8987922922ede6f1918763eaa00bDavid Sehr case IceType_f32: 27300c68bef895ea8987922922ede6f1918763eaa00bDavid Sehr case IceType_f64: 27310c68bef895ea8987922922ede6f1918763eaa00bDavid Sehr if (!Traits::X86_PASS_SCALAR_FP_IN_XMM) { 27320c68bef895ea8987922922ede6f1918763eaa00bDavid Sehr // Leave ReturnReg==ReturnRegHi==nullptr, and capture the result with 27330c68bef895ea8987922922ede6f1918763eaa00bDavid Sehr // the fstp instruction. 27340c68bef895ea8987922922ede6f1918763eaa00bDavid Sehr break; 27350c68bef895ea8987922922ede6f1918763eaa00bDavid Sehr } 27360c68bef895ea8987922922ede6f1918763eaa00bDavid Sehr // Fallthrough intended. 27370c68bef895ea8987922922ede6f1918763eaa00bDavid Sehr case IceType_v4i1: 27380c68bef895ea8987922922ede6f1918763eaa00bDavid Sehr case IceType_v8i1: 27390c68bef895ea8987922922ede6f1918763eaa00bDavid Sehr case IceType_v16i1: 27400c68bef895ea8987922922ede6f1918763eaa00bDavid Sehr case IceType_v16i8: 27410c68bef895ea8987922922ede6f1918763eaa00bDavid Sehr case IceType_v8i16: 27420c68bef895ea8987922922ede6f1918763eaa00bDavid Sehr case IceType_v4i32: 27430c68bef895ea8987922922ede6f1918763eaa00bDavid Sehr case IceType_v4f32: 27440c68bef895ea8987922922ede6f1918763eaa00bDavid Sehr ReturnReg = makeReg(DestTy, Traits::RegisterSet::Reg_xmm0); 27450c68bef895ea8987922922ede6f1918763eaa00bDavid Sehr break; 27460c68bef895ea8987922922ede6f1918763eaa00bDavid Sehr } 27470c68bef895ea8987922922ede6f1918763eaa00bDavid Sehr } 27480c68bef895ea8987922922ede6f1918763eaa00bDavid Sehr // Emit the call to the function. 27490c68bef895ea8987922922ede6f1918763eaa00bDavid Sehr Operand *CallTarget = 27500c68bef895ea8987922922ede6f1918763eaa00bDavid Sehr legalize(Instr->getCallTarget(), Legal_Reg | Legal_Imm | Legal_AddrAbs); 27510c68bef895ea8987922922ede6f1918763eaa00bDavid Sehr Inst *NewCall = emitCallToTarget(CallTarget, ReturnReg); 27520c68bef895ea8987922922ede6f1918763eaa00bDavid Sehr // Keep the upper return register live on 32-bit platform. 27530c68bef895ea8987922922ede6f1918763eaa00bDavid Sehr if (ReturnRegHi) 27540c68bef895ea8987922922ede6f1918763eaa00bDavid Sehr Context.insert<InstFakeDef>(ReturnRegHi); 27550c68bef895ea8987922922ede6f1918763eaa00bDavid Sehr // Mark the call as killing all the caller-save registers. 27560c68bef895ea8987922922ede6f1918763eaa00bDavid Sehr Context.insert<InstFakeKill>(NewCall); 27570c68bef895ea8987922922ede6f1918763eaa00bDavid Sehr // Handle x86-32 floating point returns. 2758deb5a8258ff638b23af3dd6f05eb1ea8bebd131dJim Stichnoth if (Dest != nullptr && isScalarFloatingType(DestTy) && 27590c68bef895ea8987922922ede6f1918763eaa00bDavid Sehr !Traits::X86_PASS_SCALAR_FP_IN_XMM) { 27600c68bef895ea8987922922ede6f1918763eaa00bDavid Sehr // Special treatment for an FP function which returns its result in st(0). 27610c68bef895ea8987922922ede6f1918763eaa00bDavid Sehr // If Dest ends up being a physical xmm register, the fstp emit code will 27620c68bef895ea8987922922ede6f1918763eaa00bDavid Sehr // route st(0) through the space reserved in the function argument area 27630c68bef895ea8987922922ede6f1918763eaa00bDavid Sehr // we allocated. 27640c68bef895ea8987922922ede6f1918763eaa00bDavid Sehr _fstp(Dest); 27650c68bef895ea8987922922ede6f1918763eaa00bDavid Sehr // Create a fake use of Dest in case it actually isn't used, because st(0) 27660c68bef895ea8987922922ede6f1918763eaa00bDavid Sehr // still needs to be popped. 27670c68bef895ea8987922922ede6f1918763eaa00bDavid Sehr Context.insert<InstFakeUse>(Dest); 27680c68bef895ea8987922922ede6f1918763eaa00bDavid Sehr } 27690c68bef895ea8987922922ede6f1918763eaa00bDavid Sehr // Generate a FakeUse to keep the call live if necessary. 27700c68bef895ea8987922922ede6f1918763eaa00bDavid Sehr if (Instr->hasSideEffects() && ReturnReg) { 27710c68bef895ea8987922922ede6f1918763eaa00bDavid Sehr Context.insert<InstFakeUse>(ReturnReg); 27720c68bef895ea8987922922ede6f1918763eaa00bDavid Sehr } 27730c68bef895ea8987922922ede6f1918763eaa00bDavid Sehr // Process the return value, if any. 27740c68bef895ea8987922922ede6f1918763eaa00bDavid Sehr if (Dest == nullptr) 27750c68bef895ea8987922922ede6f1918763eaa00bDavid Sehr return; 2776deb5a8258ff638b23af3dd6f05eb1ea8bebd131dJim Stichnoth // Assign the result of the call to Dest. Route it through a temporary so 2777deb5a8258ff638b23af3dd6f05eb1ea8bebd131dJim Stichnoth // that the local register availability peephole can be subsequently used. 2778deb5a8258ff638b23af3dd6f05eb1ea8bebd131dJim Stichnoth Variable *Tmp = nullptr; 27790c68bef895ea8987922922ede6f1918763eaa00bDavid Sehr if (isVectorType(DestTy)) { 27800c68bef895ea8987922922ede6f1918763eaa00bDavid Sehr assert(ReturnReg && "Vector type requires a return register"); 2781deb5a8258ff638b23af3dd6f05eb1ea8bebd131dJim Stichnoth Tmp = makeReg(DestTy); 2782deb5a8258ff638b23af3dd6f05eb1ea8bebd131dJim Stichnoth _movp(Tmp, ReturnReg); 2783deb5a8258ff638b23af3dd6f05eb1ea8bebd131dJim Stichnoth _movp(Dest, Tmp); 27840c68bef895ea8987922922ede6f1918763eaa00bDavid Sehr } else if (isScalarFloatingType(DestTy)) { 27850c68bef895ea8987922922ede6f1918763eaa00bDavid Sehr if (Traits::X86_PASS_SCALAR_FP_IN_XMM) { 27860c68bef895ea8987922922ede6f1918763eaa00bDavid Sehr assert(ReturnReg && "FP type requires a return register"); 2787deb5a8258ff638b23af3dd6f05eb1ea8bebd131dJim Stichnoth _mov(Tmp, ReturnReg); 2788deb5a8258ff638b23af3dd6f05eb1ea8bebd131dJim Stichnoth _mov(Dest, Tmp); 27890c68bef895ea8987922922ede6f1918763eaa00bDavid Sehr } 27900c68bef895ea8987922922ede6f1918763eaa00bDavid Sehr } else { 27910c68bef895ea8987922922ede6f1918763eaa00bDavid Sehr assert(isScalarIntegerType(DestTy)); 27920c68bef895ea8987922922ede6f1918763eaa00bDavid Sehr assert(ReturnReg && "Integer type requires a return register"); 27930c68bef895ea8987922922ede6f1918763eaa00bDavid Sehr if (DestTy == IceType_i64 && !Traits::Is64Bit) { 27940c68bef895ea8987922922ede6f1918763eaa00bDavid Sehr assert(ReturnRegHi && "64-bit type requires two return registers"); 27950c68bef895ea8987922922ede6f1918763eaa00bDavid Sehr auto *Dest64On32 = llvm::cast<Variable64On32>(Dest); 27960c68bef895ea8987922922ede6f1918763eaa00bDavid Sehr Variable *DestLo = Dest64On32->getLo(); 27970c68bef895ea8987922922ede6f1918763eaa00bDavid Sehr Variable *DestHi = Dest64On32->getHi(); 2798deb5a8258ff638b23af3dd6f05eb1ea8bebd131dJim Stichnoth _mov(Tmp, ReturnReg); 2799deb5a8258ff638b23af3dd6f05eb1ea8bebd131dJim Stichnoth _mov(DestLo, Tmp); 2800deb5a8258ff638b23af3dd6f05eb1ea8bebd131dJim Stichnoth Variable *TmpHi = nullptr; 2801deb5a8258ff638b23af3dd6f05eb1ea8bebd131dJim Stichnoth _mov(TmpHi, ReturnRegHi); 2802deb5a8258ff638b23af3dd6f05eb1ea8bebd131dJim Stichnoth _mov(DestHi, TmpHi); 28030c68bef895ea8987922922ede6f1918763eaa00bDavid Sehr } else { 2804deb5a8258ff638b23af3dd6f05eb1ea8bebd131dJim Stichnoth _mov(Tmp, ReturnReg); 2805deb5a8258ff638b23af3dd6f05eb1ea8bebd131dJim Stichnoth _mov(Dest, Tmp); 28060c68bef895ea8987922922ede6f1918763eaa00bDavid Sehr } 28070c68bef895ea8987922922ede6f1918763eaa00bDavid Sehr } 28080c68bef895ea8987922922ede6f1918763eaa00bDavid Sehr} 28090c68bef895ea8987922922ede6f1918763eaa00bDavid Sehr 28104a56686b5b56db6803f90ad53514bf2fa190d9f7John Portotemplate <typename TraitsType> 28118cfeb69e17190d3bfe22a8a1cbd7679a114d68cfJim Stichnothvoid TargetX86Base<TraitsType>::lowerCast(const InstCast *Instr) { 28127e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto // a = cast(b) ==> t=cast(b); a=t; (link t->b, link a->t, no overlap) 28138cfeb69e17190d3bfe22a8a1cbd7679a114d68cfJim Stichnoth InstCast::OpKind CastKind = Instr->getCastKind(); 28148cfeb69e17190d3bfe22a8a1cbd7679a114d68cfJim Stichnoth Variable *Dest = Instr->getDest(); 2815c59288b334b91f4c0b2edf0de7415c68c760aa12Jim Stichnoth Type DestTy = Dest->getType(); 28167e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto switch (CastKind) { 28177e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto default: 28187e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto Func->setError("Cast type not supported"); 28197e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto return; 28207e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto case InstCast::Sext: { 28217e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto // Src0RM is the source operand legalized to physical register or memory, 28227e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto // but not immediate, since the relevant x86 native instructions don't 282357e126899b20c65ff3ea23a3b7d7a67ab30b99dcAndrew Scull // allow an immediate operand. If the operand is an immediate, we could 282457e126899b20c65ff3ea23a3b7d7a67ab30b99dcAndrew Scull // consider computing the strength-reduced result at translation time, but 282557e126899b20c65ff3ea23a3b7d7a67ab30b99dcAndrew Scull // we're unlikely to see something like that in the bitcode that the 282657e126899b20c65ff3ea23a3b7d7a67ab30b99dcAndrew Scull // optimizer wouldn't have already taken care of. 28278cfeb69e17190d3bfe22a8a1cbd7679a114d68cfJim Stichnoth Operand *Src0RM = legalize(Instr->getSrc(0), Legal_Reg | Legal_Mem); 2828c59288b334b91f4c0b2edf0de7415c68c760aa12Jim Stichnoth if (isVectorType(DestTy)) { 28297e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto if (DestTy == IceType_v16i8) { 28307e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto // onemask = materialize(1,1,...); dst = (src & onemask) > 0 2831c59288b334b91f4c0b2edf0de7415c68c760aa12Jim Stichnoth Variable *OneMask = makeVectorOfOnes(DestTy); 28327e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto Variable *T = makeReg(DestTy); 28337e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto _movp(T, Src0RM); 28347e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto _pand(T, OneMask); 2835c59288b334b91f4c0b2edf0de7415c68c760aa12Jim Stichnoth Variable *Zeros = makeVectorOfZeros(DestTy); 28367e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto _pcmpgt(T, Zeros); 28377e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto _movp(Dest, T); 28387e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto } else { 28399612d32c7e5eb2cb403686ef31172d42e075e460Andrew Scull /// width = width(elty) - 1; dest = (src << width) >> width 28407e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto SizeT ShiftAmount = 28417e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto Traits::X86_CHAR_BIT * typeWidthInBytes(typeElementType(DestTy)) - 28427e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto 1; 28437e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto Constant *ShiftConstant = Ctx->getConstantInt8(ShiftAmount); 28447e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto Variable *T = makeReg(DestTy); 28457e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto _movp(T, Src0RM); 28467e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto _psll(T, ShiftConstant); 28477e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto _psra(T, ShiftConstant); 28487e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto _movp(Dest, T); 28497e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto } 2850c59288b334b91f4c0b2edf0de7415c68c760aa12Jim Stichnoth } else if (!Traits::Is64Bit && DestTy == IceType_i64) { 28517e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto // t1=movsx src; t2=t1; t2=sar t2, 31; dst.lo=t1; dst.hi=t2 28527e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto Constant *Shift = Ctx->getConstantInt32(31); 285354f3d518805b04b3f2958b7906a7d07dedb4e1aeJim Stichnoth auto *DestLo = llvm::cast<Variable>(loOperand(Dest)); 285454f3d518805b04b3f2958b7906a7d07dedb4e1aeJim Stichnoth auto *DestHi = llvm::cast<Variable>(hiOperand(Dest)); 28557e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto Variable *T_Lo = makeReg(DestLo->getType()); 28567e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto if (Src0RM->getType() == IceType_i32) { 28577e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto _mov(T_Lo, Src0RM); 28587e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto } else if (Src0RM->getType() == IceType_i1) { 28597e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto _movzx(T_Lo, Src0RM); 28607e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto _shl(T_Lo, Shift); 28617e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto _sar(T_Lo, Shift); 28627e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto } else { 28637e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto _movsx(T_Lo, Src0RM); 28647e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto } 28657e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto _mov(DestLo, T_Lo); 28667e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto Variable *T_Hi = nullptr; 28677e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto _mov(T_Hi, T_Lo); 28687e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto if (Src0RM->getType() != IceType_i1) 28697e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto // For i1, the sar instruction is already done above. 28707e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto _sar(T_Hi, Shift); 28717e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto _mov(DestHi, T_Hi); 28727e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto } else if (Src0RM->getType() == IceType_i1) { 28737e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto // t1 = src 28747e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto // shl t1, dst_bitwidth - 1 28757e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto // sar t1, dst_bitwidth - 1 28767e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto // dst = t1 2877c59288b334b91f4c0b2edf0de7415c68c760aa12Jim Stichnoth size_t DestBits = Traits::X86_CHAR_BIT * typeWidthInBytes(DestTy); 28787e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto Constant *ShiftAmount = Ctx->getConstantInt32(DestBits - 1); 2879c59288b334b91f4c0b2edf0de7415c68c760aa12Jim Stichnoth Variable *T = makeReg(DestTy); 2880c59288b334b91f4c0b2edf0de7415c68c760aa12Jim Stichnoth if (typeWidthInBytes(DestTy) <= typeWidthInBytes(Src0RM->getType())) { 28817e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto _mov(T, Src0RM); 28827e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto } else { 288357e126899b20c65ff3ea23a3b7d7a67ab30b99dcAndrew Scull // Widen the source using movsx or movzx. (It doesn't matter which one, 288457e126899b20c65ff3ea23a3b7d7a67ab30b99dcAndrew Scull // since the following shl/sar overwrite the bits.) 28857e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto _movzx(T, Src0RM); 28867e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto } 28877e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto _shl(T, ShiftAmount); 28887e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto _sar(T, ShiftAmount); 28897e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto _mov(Dest, T); 28907e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto } else { 28917e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto // t1 = movsx src; dst = t1 2892c59288b334b91f4c0b2edf0de7415c68c760aa12Jim Stichnoth Variable *T = makeReg(DestTy); 28937e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto _movsx(T, Src0RM); 28947e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto _mov(Dest, T); 28957e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto } 28967e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto break; 28977e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto } 28987e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto case InstCast::Zext: { 28998cfeb69e17190d3bfe22a8a1cbd7679a114d68cfJim Stichnoth Operand *Src0RM = legalize(Instr->getSrc(0), Legal_Reg | Legal_Mem); 2900c59288b334b91f4c0b2edf0de7415c68c760aa12Jim Stichnoth if (isVectorType(DestTy)) { 29017e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto // onemask = materialize(1,1,...); dest = onemask & src 29027e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto Variable *OneMask = makeVectorOfOnes(DestTy); 29037e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto Variable *T = makeReg(DestTy); 29047e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto _movp(T, Src0RM); 29057e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto _pand(T, OneMask); 29067e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto _movp(Dest, T); 2907c59288b334b91f4c0b2edf0de7415c68c760aa12Jim Stichnoth } else if (!Traits::Is64Bit && DestTy == IceType_i64) { 29087e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto // t1=movzx src; dst.lo=t1; dst.hi=0 29097e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto Constant *Zero = Ctx->getConstantZero(IceType_i32); 291054f3d518805b04b3f2958b7906a7d07dedb4e1aeJim Stichnoth auto *DestLo = llvm::cast<Variable>(loOperand(Dest)); 291154f3d518805b04b3f2958b7906a7d07dedb4e1aeJim Stichnoth auto *DestHi = llvm::cast<Variable>(hiOperand(Dest)); 29127e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto Variable *Tmp = makeReg(DestLo->getType()); 29137e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto if (Src0RM->getType() == IceType_i32) { 29147e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto _mov(Tmp, Src0RM); 29157e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto } else { 29167e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto _movzx(Tmp, Src0RM); 29177e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto } 29187e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto _mov(DestLo, Tmp); 29197e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto _mov(DestHi, Zero); 29207e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto } else if (Src0RM->getType() == IceType_i1) { 2921485d0773c9fd0605cc126e7e9020464a89ade564Jim Stichnoth // t = Src0RM; Dest = t 29221d235425dab1f3dd059973fc53f1b1d5879469e3John Porto Variable *T = nullptr; 29237e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto if (DestTy == IceType_i8) { 29247e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto _mov(T, Src0RM); 29257e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto } else { 29261d235425dab1f3dd059973fc53f1b1d5879469e3John Porto assert(DestTy != IceType_i1); 29271d235425dab1f3dd059973fc53f1b1d5879469e3John Porto assert(Traits::Is64Bit || DestTy != IceType_i64); 29287e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto // Use 32-bit for both 16-bit and 32-bit, since 32-bit ops are shorter. 29291d235425dab1f3dd059973fc53f1b1d5879469e3John Porto // In x86-64 we need to widen T to 64-bits to ensure that T -- if 29301d235425dab1f3dd059973fc53f1b1d5879469e3John Porto // written to the stack (i.e., in -Om1) will be fully zero-extended. 29311d235425dab1f3dd059973fc53f1b1d5879469e3John Porto T = makeReg(DestTy == IceType_i64 ? IceType_i64 : IceType_i32); 29327e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto _movzx(T, Src0RM); 29337e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto } 29347e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto _mov(Dest, T); 29357e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto } else { 29367e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto // t1 = movzx src; dst = t1 2937c59288b334b91f4c0b2edf0de7415c68c760aa12Jim Stichnoth Variable *T = makeReg(DestTy); 29387e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto _movzx(T, Src0RM); 29397e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto _mov(Dest, T); 29407e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto } 29417e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto break; 29427e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto } 29437e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto case InstCast::Trunc: { 2944c59288b334b91f4c0b2edf0de7415c68c760aa12Jim Stichnoth if (isVectorType(DestTy)) { 29457e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto // onemask = materialize(1,1,...); dst = src & onemask 29468cfeb69e17190d3bfe22a8a1cbd7679a114d68cfJim Stichnoth Operand *Src0RM = legalize(Instr->getSrc(0), Legal_Reg | Legal_Mem); 29477e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto Type Src0Ty = Src0RM->getType(); 29487e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto Variable *OneMask = makeVectorOfOnes(Src0Ty); 2949c59288b334b91f4c0b2edf0de7415c68c760aa12Jim Stichnoth Variable *T = makeReg(DestTy); 29507e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto _movp(T, Src0RM); 29517e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto _pand(T, OneMask); 29527e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto _movp(Dest, T); 2953c59288b334b91f4c0b2edf0de7415c68c760aa12Jim Stichnoth } else if (DestTy == IceType_i1 || DestTy == IceType_i8) { 2954c59288b334b91f4c0b2edf0de7415c68c760aa12Jim Stichnoth // Make sure we truncate from and into valid registers. 29558cfeb69e17190d3bfe22a8a1cbd7679a114d68cfJim Stichnoth Operand *Src0 = legalizeUndef(Instr->getSrc(0)); 2956c59288b334b91f4c0b2edf0de7415c68c760aa12Jim Stichnoth if (!Traits::Is64Bit && Src0->getType() == IceType_i64) 2957c59288b334b91f4c0b2edf0de7415c68c760aa12Jim Stichnoth Src0 = loOperand(Src0); 2958c59288b334b91f4c0b2edf0de7415c68c760aa12Jim Stichnoth Operand *Src0RM = legalize(Src0, Legal_Reg | Legal_Mem); 2959c59288b334b91f4c0b2edf0de7415c68c760aa12Jim Stichnoth Variable *T = copyToReg8(Src0RM); 2960c59288b334b91f4c0b2edf0de7415c68c760aa12Jim Stichnoth if (DestTy == IceType_i1) 2961c59288b334b91f4c0b2edf0de7415c68c760aa12Jim Stichnoth _and(T, Ctx->getConstantInt1(1)); 2962c59288b334b91f4c0b2edf0de7415c68c760aa12Jim Stichnoth _mov(Dest, T); 29637e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto } else { 29648cfeb69e17190d3bfe22a8a1cbd7679a114d68cfJim Stichnoth Operand *Src0 = legalizeUndef(Instr->getSrc(0)); 29651d235425dab1f3dd059973fc53f1b1d5879469e3John Porto if (!Traits::Is64Bit && Src0->getType() == IceType_i64) 29667e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto Src0 = loOperand(Src0); 29677e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto Operand *Src0RM = legalize(Src0, Legal_Reg | Legal_Mem); 29687e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto // t1 = trunc Src0RM; Dest = t1 2969c59288b334b91f4c0b2edf0de7415c68c760aa12Jim Stichnoth Variable *T = makeReg(DestTy); 29707e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto _mov(T, Src0RM); 29717e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto _mov(Dest, T); 29727e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto } 29737e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto break; 29747e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto } 29757e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto case InstCast::Fptrunc: 29767e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto case InstCast::Fpext: { 29778cfeb69e17190d3bfe22a8a1cbd7679a114d68cfJim Stichnoth Operand *Src0RM = legalize(Instr->getSrc(0), Legal_Reg | Legal_Mem); 29787e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto // t1 = cvt Src0RM; Dest = t1 2979c59288b334b91f4c0b2edf0de7415c68c760aa12Jim Stichnoth Variable *T = makeReg(DestTy); 2980921856d4e4bd4f0deedc7324d5f6a4ca0be3439fJohn Porto _cvt(T, Src0RM, Traits::Insts::Cvt::Float2float); 29817e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto _mov(Dest, T); 29827e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto break; 29837e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto } 29847e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto case InstCast::Fptosi: 2985c59288b334b91f4c0b2edf0de7415c68c760aa12Jim Stichnoth if (isVectorType(DestTy)) { 2986dbf81e0c0e364173176159e0e2548e9948b197a4Nicolas Capens assert(DestTy == IceType_v4i32); 2987dbf81e0c0e364173176159e0e2548e9948b197a4Nicolas Capens assert(Instr->getSrc(0)->getType() == IceType_v4f32); 2988dbf81e0c0e364173176159e0e2548e9948b197a4Nicolas Capens Operand *Src0R = legalizeToReg(Instr->getSrc(0)); 2989c59288b334b91f4c0b2edf0de7415c68c760aa12Jim Stichnoth Variable *T = makeReg(DestTy); 2990dbf81e0c0e364173176159e0e2548e9948b197a4Nicolas Capens _cvt(T, Src0R, Traits::Insts::Cvt::Tps2dq); 29917e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto _movp(Dest, T); 2992c59288b334b91f4c0b2edf0de7415c68c760aa12Jim Stichnoth } else if (!Traits::Is64Bit && DestTy == IceType_i64) { 299326217e3333150e66fc96aca79c01105906797960David Sehr llvm::report_fatal_error("Helper call was expected"); 29947e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto } else { 29958cfeb69e17190d3bfe22a8a1cbd7679a114d68cfJim Stichnoth Operand *Src0RM = legalize(Instr->getSrc(0), Legal_Reg | Legal_Mem); 29967e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto // t1.i32 = cvt Src0RM; t2.dest_type = t1; Dest = t2.dest_type 29971d235425dab1f3dd059973fc53f1b1d5879469e3John Porto Variable *T_1 = nullptr; 2998c59288b334b91f4c0b2edf0de7415c68c760aa12Jim Stichnoth if (Traits::Is64Bit && DestTy == IceType_i64) { 29991d235425dab1f3dd059973fc53f1b1d5879469e3John Porto T_1 = makeReg(IceType_i64); 30001d235425dab1f3dd059973fc53f1b1d5879469e3John Porto } else { 3001c59288b334b91f4c0b2edf0de7415c68c760aa12Jim Stichnoth assert(DestTy != IceType_i64); 30021d235425dab1f3dd059973fc53f1b1d5879469e3John Porto T_1 = makeReg(IceType_i32); 30031d235425dab1f3dd059973fc53f1b1d5879469e3John Porto } 30041d235425dab1f3dd059973fc53f1b1d5879469e3John Porto // cvt() requires its integer argument to be a GPR. 3005c59288b334b91f4c0b2edf0de7415c68c760aa12Jim Stichnoth Variable *T_2 = makeReg(DestTy); 3006c59288b334b91f4c0b2edf0de7415c68c760aa12Jim Stichnoth if (isByteSizedType(DestTy)) { 3007c59288b334b91f4c0b2edf0de7415c68c760aa12Jim Stichnoth assert(T_1->getType() == IceType_i32); 3008c59288b334b91f4c0b2edf0de7415c68c760aa12Jim Stichnoth T_1->setRegClass(RCX86_Is32To8); 3009c59288b334b91f4c0b2edf0de7415c68c760aa12Jim Stichnoth T_2->setRegClass(RCX86_IsTrunc8Rcvr); 3010c59288b334b91f4c0b2edf0de7415c68c760aa12Jim Stichnoth } 3011921856d4e4bd4f0deedc7324d5f6a4ca0be3439fJohn Porto _cvt(T_1, Src0RM, Traits::Insts::Cvt::Tss2si); 30127e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto _mov(T_2, T_1); // T_1 and T_2 may have different integer types 3013c59288b334b91f4c0b2edf0de7415c68c760aa12Jim Stichnoth if (DestTy == IceType_i1) 30147e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto _and(T_2, Ctx->getConstantInt1(1)); 30157e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto _mov(Dest, T_2); 30167e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto } 30177e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto break; 30187e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto case InstCast::Fptoui: 3019c59288b334b91f4c0b2edf0de7415c68c760aa12Jim Stichnoth if (isVectorType(DestTy)) { 302026217e3333150e66fc96aca79c01105906797960David Sehr llvm::report_fatal_error("Helper call was expected"); 3021c59288b334b91f4c0b2edf0de7415c68c760aa12Jim Stichnoth } else if (DestTy == IceType_i64 || 3022c59288b334b91f4c0b2edf0de7415c68c760aa12Jim Stichnoth (!Traits::Is64Bit && DestTy == IceType_i32)) { 302326217e3333150e66fc96aca79c01105906797960David Sehr llvm::report_fatal_error("Helper call was expected"); 30247e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto } else { 30258cfeb69e17190d3bfe22a8a1cbd7679a114d68cfJim Stichnoth Operand *Src0RM = legalize(Instr->getSrc(0), Legal_Reg | Legal_Mem); 30267e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto // t1.i32 = cvt Src0RM; t2.dest_type = t1; Dest = t2.dest_type 3027c59288b334b91f4c0b2edf0de7415c68c760aa12Jim Stichnoth assert(DestTy != IceType_i64); 30281d235425dab1f3dd059973fc53f1b1d5879469e3John Porto Variable *T_1 = nullptr; 3029c59288b334b91f4c0b2edf0de7415c68c760aa12Jim Stichnoth if (Traits::Is64Bit && DestTy == IceType_i32) { 30301d235425dab1f3dd059973fc53f1b1d5879469e3John Porto T_1 = makeReg(IceType_i64); 30311d235425dab1f3dd059973fc53f1b1d5879469e3John Porto } else { 3032c59288b334b91f4c0b2edf0de7415c68c760aa12Jim Stichnoth assert(DestTy != IceType_i32); 30331d235425dab1f3dd059973fc53f1b1d5879469e3John Porto T_1 = makeReg(IceType_i32); 30341d235425dab1f3dd059973fc53f1b1d5879469e3John Porto } 3035c59288b334b91f4c0b2edf0de7415c68c760aa12Jim Stichnoth Variable *T_2 = makeReg(DestTy); 3036c59288b334b91f4c0b2edf0de7415c68c760aa12Jim Stichnoth if (isByteSizedType(DestTy)) { 3037c59288b334b91f4c0b2edf0de7415c68c760aa12Jim Stichnoth assert(T_1->getType() == IceType_i32); 3038c59288b334b91f4c0b2edf0de7415c68c760aa12Jim Stichnoth T_1->setRegClass(RCX86_Is32To8); 3039c59288b334b91f4c0b2edf0de7415c68c760aa12Jim Stichnoth T_2->setRegClass(RCX86_IsTrunc8Rcvr); 3040c59288b334b91f4c0b2edf0de7415c68c760aa12Jim Stichnoth } 3041921856d4e4bd4f0deedc7324d5f6a4ca0be3439fJohn Porto _cvt(T_1, Src0RM, Traits::Insts::Cvt::Tss2si); 30427e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto _mov(T_2, T_1); // T_1 and T_2 may have different integer types 3043c59288b334b91f4c0b2edf0de7415c68c760aa12Jim Stichnoth if (DestTy == IceType_i1) 30447e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto _and(T_2, Ctx->getConstantInt1(1)); 30457e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto _mov(Dest, T_2); 30467e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto } 30477e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto break; 30487e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto case InstCast::Sitofp: 3049c59288b334b91f4c0b2edf0de7415c68c760aa12Jim Stichnoth if (isVectorType(DestTy)) { 3050dbf81e0c0e364173176159e0e2548e9948b197a4Nicolas Capens assert(DestTy == IceType_v4f32); 3051dbf81e0c0e364173176159e0e2548e9948b197a4Nicolas Capens assert(Instr->getSrc(0)->getType() == IceType_v4i32); 3052dbf81e0c0e364173176159e0e2548e9948b197a4Nicolas Capens Operand *Src0R = legalizeToReg(Instr->getSrc(0)); 3053c59288b334b91f4c0b2edf0de7415c68c760aa12Jim Stichnoth Variable *T = makeReg(DestTy); 3054dbf81e0c0e364173176159e0e2548e9948b197a4Nicolas Capens _cvt(T, Src0R, Traits::Insts::Cvt::Dq2ps); 30557e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto _movp(Dest, T); 30568cfeb69e17190d3bfe22a8a1cbd7679a114d68cfJim Stichnoth } else if (!Traits::Is64Bit && Instr->getSrc(0)->getType() == IceType_i64) { 305726217e3333150e66fc96aca79c01105906797960David Sehr llvm::report_fatal_error("Helper call was expected"); 30587e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto } else { 30598cfeb69e17190d3bfe22a8a1cbd7679a114d68cfJim Stichnoth Operand *Src0RM = legalize(Instr->getSrc(0), Legal_Reg | Legal_Mem); 30607e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto // Sign-extend the operand. 30617e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto // t1.i32 = movsx Src0RM; t2 = Cvt t1.i32; Dest = t2 30621d235425dab1f3dd059973fc53f1b1d5879469e3John Porto Variable *T_1 = nullptr; 30631d235425dab1f3dd059973fc53f1b1d5879469e3John Porto if (Traits::Is64Bit && Src0RM->getType() == IceType_i64) { 30641d235425dab1f3dd059973fc53f1b1d5879469e3John Porto T_1 = makeReg(IceType_i64); 30651d235425dab1f3dd059973fc53f1b1d5879469e3John Porto } else { 30661d235425dab1f3dd059973fc53f1b1d5879469e3John Porto assert(Src0RM->getType() != IceType_i64); 30671d235425dab1f3dd059973fc53f1b1d5879469e3John Porto T_1 = makeReg(IceType_i32); 30681d235425dab1f3dd059973fc53f1b1d5879469e3John Porto } 3069c59288b334b91f4c0b2edf0de7415c68c760aa12Jim Stichnoth Variable *T_2 = makeReg(DestTy); 30701d235425dab1f3dd059973fc53f1b1d5879469e3John Porto if (Src0RM->getType() == T_1->getType()) 30717e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto _mov(T_1, Src0RM); 30727e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto else 30737e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto _movsx(T_1, Src0RM); 3074921856d4e4bd4f0deedc7324d5f6a4ca0be3439fJohn Porto _cvt(T_2, T_1, Traits::Insts::Cvt::Si2ss); 30757e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto _mov(Dest, T_2); 30767e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto } 30777e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto break; 30787e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto case InstCast::Uitofp: { 30798cfeb69e17190d3bfe22a8a1cbd7679a114d68cfJim Stichnoth Operand *Src0 = Instr->getSrc(0); 30807e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto if (isVectorType(Src0->getType())) { 308126217e3333150e66fc96aca79c01105906797960David Sehr llvm::report_fatal_error("Helper call was expected"); 30827e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto } else if (Src0->getType() == IceType_i64 || 30831d235425dab1f3dd059973fc53f1b1d5879469e3John Porto (!Traits::Is64Bit && Src0->getType() == IceType_i32)) { 308426217e3333150e66fc96aca79c01105906797960David Sehr llvm::report_fatal_error("Helper call was expected"); 30857e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto } else { 30867e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto Operand *Src0RM = legalize(Src0, Legal_Reg | Legal_Mem); 30877e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto // Zero-extend the operand. 30887e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto // t1.i32 = movzx Src0RM; t2 = Cvt t1.i32; Dest = t2 30891d235425dab1f3dd059973fc53f1b1d5879469e3John Porto Variable *T_1 = nullptr; 30901d235425dab1f3dd059973fc53f1b1d5879469e3John Porto if (Traits::Is64Bit && Src0RM->getType() == IceType_i32) { 30911d235425dab1f3dd059973fc53f1b1d5879469e3John Porto T_1 = makeReg(IceType_i64); 30921d235425dab1f3dd059973fc53f1b1d5879469e3John Porto } else { 30931d235425dab1f3dd059973fc53f1b1d5879469e3John Porto assert(Src0RM->getType() != IceType_i64); 30941d235425dab1f3dd059973fc53f1b1d5879469e3John Porto assert(Traits::Is64Bit || Src0RM->getType() != IceType_i32); 30951d235425dab1f3dd059973fc53f1b1d5879469e3John Porto T_1 = makeReg(IceType_i32); 30961d235425dab1f3dd059973fc53f1b1d5879469e3John Porto } 3097c59288b334b91f4c0b2edf0de7415c68c760aa12Jim Stichnoth Variable *T_2 = makeReg(DestTy); 30981d235425dab1f3dd059973fc53f1b1d5879469e3John Porto if (Src0RM->getType() == T_1->getType()) 30997e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto _mov(T_1, Src0RM); 31007e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto else 310174bfa8712543f32b75bbeb0ab0d43ec599d02059Jim Stichnoth _movzx(T_1, Src0RM)->setMustKeep(); 3102921856d4e4bd4f0deedc7324d5f6a4ca0be3439fJohn Porto _cvt(T_2, T_1, Traits::Insts::Cvt::Si2ss); 31037e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto _mov(Dest, T_2); 31047e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto } 31057e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto break; 31067e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto } 31077e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto case InstCast::Bitcast: { 31088cfeb69e17190d3bfe22a8a1cbd7679a114d68cfJim Stichnoth Operand *Src0 = Instr->getSrc(0); 3109c59288b334b91f4c0b2edf0de7415c68c760aa12Jim Stichnoth if (DestTy == Src0->getType()) { 311054f3d518805b04b3f2958b7906a7d07dedb4e1aeJim Stichnoth auto *Assign = InstAssign::create(Func, Dest, Src0); 31117e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto lowerAssign(Assign); 31127e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto return; 31137e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto } 3114c59288b334b91f4c0b2edf0de7415c68c760aa12Jim Stichnoth switch (DestTy) { 31157e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto default: 31167e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto llvm_unreachable("Unexpected Bitcast dest type"); 31177e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto case IceType_i8: { 311826217e3333150e66fc96aca79c01105906797960David Sehr llvm::report_fatal_error("Helper call was expected"); 31197e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto } break; 31207e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto case IceType_i16: { 312126217e3333150e66fc96aca79c01105906797960David Sehr llvm::report_fatal_error("Helper call was expected"); 31227e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto } break; 31237e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto case IceType_i32: 31247e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto case IceType_f32: { 3125a1410df95992d58e06ccdacd62896dd2a7bffb93Jim Stichnoth Variable *Src0R = legalizeToReg(Src0); 3126a1410df95992d58e06ccdacd62896dd2a7bffb93Jim Stichnoth Variable *T = makeReg(DestTy); 3127a1410df95992d58e06ccdacd62896dd2a7bffb93Jim Stichnoth _movd(T, Src0R); 3128a1410df95992d58e06ccdacd62896dd2a7bffb93Jim Stichnoth _mov(Dest, T); 31297e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto } break; 31307e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto case IceType_i64: { 31311d235425dab1f3dd059973fc53f1b1d5879469e3John Porto assert(Src0->getType() == IceType_f64); 31321d235425dab1f3dd059973fc53f1b1d5879469e3John Porto if (Traits::Is64Bit) { 31331d235425dab1f3dd059973fc53f1b1d5879469e3John Porto Variable *Src0R = legalizeToReg(Src0); 31341d235425dab1f3dd059973fc53f1b1d5879469e3John Porto Variable *T = makeReg(IceType_i64); 31351d235425dab1f3dd059973fc53f1b1d5879469e3John Porto _movd(T, Src0R); 31361d235425dab1f3dd059973fc53f1b1d5879469e3John Porto _mov(Dest, T); 31377e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto } else { 31381d235425dab1f3dd059973fc53f1b1d5879469e3John Porto Operand *Src0RM = legalize(Src0, Legal_Reg | Legal_Mem); 31391d235425dab1f3dd059973fc53f1b1d5879469e3John Porto // a.i64 = bitcast b.f64 ==> 31401d235425dab1f3dd059973fc53f1b1d5879469e3John Porto // s.f64 = spill b.f64 31411d235425dab1f3dd059973fc53f1b1d5879469e3John Porto // t_lo.i32 = lo(s.f64) 31421d235425dab1f3dd059973fc53f1b1d5879469e3John Porto // a_lo.i32 = t_lo.i32 31431d235425dab1f3dd059973fc53f1b1d5879469e3John Porto // t_hi.i32 = hi(s.f64) 31441d235425dab1f3dd059973fc53f1b1d5879469e3John Porto // a_hi.i32 = t_hi.i32 31451d235425dab1f3dd059973fc53f1b1d5879469e3John Porto Operand *SpillLo, *SpillHi; 31461d235425dab1f3dd059973fc53f1b1d5879469e3John Porto if (auto *Src0Var = llvm::dyn_cast<Variable>(Src0RM)) { 3147e343e0660aac84a8c6369f67fde19b401ca00104Jim Stichnoth Variable *Spill = Func->makeVariable(IceType_f64); 3148e343e0660aac84a8c6369f67fde19b401ca00104Jim Stichnoth Spill->setLinkedTo(Src0Var); 314911c9a325399b282cb4ea7d1d24d42fceeec2a09aAndrew Scull Spill->setMustNotHaveReg(); 31501d235425dab1f3dd059973fc53f1b1d5879469e3John Porto _movq(Spill, Src0RM); 31511d235425dab1f3dd059973fc53f1b1d5879469e3John Porto SpillLo = Traits::VariableSplit::create(Func, Spill, 31521d235425dab1f3dd059973fc53f1b1d5879469e3John Porto Traits::VariableSplit::Low); 31531d235425dab1f3dd059973fc53f1b1d5879469e3John Porto SpillHi = Traits::VariableSplit::create(Func, Spill, 31541d235425dab1f3dd059973fc53f1b1d5879469e3John Porto Traits::VariableSplit::High); 31551d235425dab1f3dd059973fc53f1b1d5879469e3John Porto } else { 31561d235425dab1f3dd059973fc53f1b1d5879469e3John Porto SpillLo = loOperand(Src0RM); 31571d235425dab1f3dd059973fc53f1b1d5879469e3John Porto SpillHi = hiOperand(Src0RM); 31581d235425dab1f3dd059973fc53f1b1d5879469e3John Porto } 31597e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto 316054f3d518805b04b3f2958b7906a7d07dedb4e1aeJim Stichnoth auto *DestLo = llvm::cast<Variable>(loOperand(Dest)); 316154f3d518805b04b3f2958b7906a7d07dedb4e1aeJim Stichnoth auto *DestHi = llvm::cast<Variable>(hiOperand(Dest)); 31621d235425dab1f3dd059973fc53f1b1d5879469e3John Porto Variable *T_Lo = makeReg(IceType_i32); 31631d235425dab1f3dd059973fc53f1b1d5879469e3John Porto Variable *T_Hi = makeReg(IceType_i32); 31647e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto 31651d235425dab1f3dd059973fc53f1b1d5879469e3John Porto _mov(T_Lo, SpillLo); 31661d235425dab1f3dd059973fc53f1b1d5879469e3John Porto _mov(DestLo, T_Lo); 31671d235425dab1f3dd059973fc53f1b1d5879469e3John Porto _mov(T_Hi, SpillHi); 31681d235425dab1f3dd059973fc53f1b1d5879469e3John Porto _mov(DestHi, T_Hi); 31691d235425dab1f3dd059973fc53f1b1d5879469e3John Porto } 31707e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto } break; 31717e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto case IceType_f64: { 31727e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto assert(Src0->getType() == IceType_i64); 31731d235425dab1f3dd059973fc53f1b1d5879469e3John Porto if (Traits::Is64Bit) { 31741d235425dab1f3dd059973fc53f1b1d5879469e3John Porto Operand *Src0RM = legalize(Src0, Legal_Reg | Legal_Mem); 31751d235425dab1f3dd059973fc53f1b1d5879469e3John Porto Variable *T = makeReg(IceType_f64); 31761d235425dab1f3dd059973fc53f1b1d5879469e3John Porto _movd(T, Src0RM); 31771d235425dab1f3dd059973fc53f1b1d5879469e3John Porto _mov(Dest, T); 31781d235425dab1f3dd059973fc53f1b1d5879469e3John Porto } else { 31791d235425dab1f3dd059973fc53f1b1d5879469e3John Porto Src0 = legalize(Src0); 31804a56686b5b56db6803f90ad53514bf2fa190d9f7John Porto if (llvm::isa<X86OperandMem>(Src0)) { 3181b9a847280e4486e566dabdd5b0d571309b4ad628Jim Stichnoth Variable *T = makeReg(DestTy); 31821d235425dab1f3dd059973fc53f1b1d5879469e3John Porto _movq(T, Src0); 31831d235425dab1f3dd059973fc53f1b1d5879469e3John Porto _movq(Dest, T); 31841d235425dab1f3dd059973fc53f1b1d5879469e3John Porto break; 31851d235425dab1f3dd059973fc53f1b1d5879469e3John Porto } 31861d235425dab1f3dd059973fc53f1b1d5879469e3John Porto // a.f64 = bitcast b.i64 ==> 31871d235425dab1f3dd059973fc53f1b1d5879469e3John Porto // t_lo.i32 = b_lo.i32 31881d235425dab1f3dd059973fc53f1b1d5879469e3John Porto // FakeDef(s.f64) 31891d235425dab1f3dd059973fc53f1b1d5879469e3John Porto // lo(s.f64) = t_lo.i32 31901d235425dab1f3dd059973fc53f1b1d5879469e3John Porto // t_hi.i32 = b_hi.i32 31911d235425dab1f3dd059973fc53f1b1d5879469e3John Porto // hi(s.f64) = t_hi.i32 31921d235425dab1f3dd059973fc53f1b1d5879469e3John Porto // a.f64 = s.f64 3193e343e0660aac84a8c6369f67fde19b401ca00104Jim Stichnoth Variable *Spill = Func->makeVariable(IceType_f64); 3194e343e0660aac84a8c6369f67fde19b401ca00104Jim Stichnoth Spill->setLinkedTo(Dest); 319511c9a325399b282cb4ea7d1d24d42fceeec2a09aAndrew Scull Spill->setMustNotHaveReg(); 31967e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto 31971d235425dab1f3dd059973fc53f1b1d5879469e3John Porto Variable *T_Lo = nullptr, *T_Hi = nullptr; 319854f3d518805b04b3f2958b7906a7d07dedb4e1aeJim Stichnoth auto *SpillLo = Traits::VariableSplit::create( 31991d235425dab1f3dd059973fc53f1b1d5879469e3John Porto Func, Spill, Traits::VariableSplit::Low); 320054f3d518805b04b3f2958b7906a7d07dedb4e1aeJim Stichnoth auto *SpillHi = Traits::VariableSplit::create( 32011d235425dab1f3dd059973fc53f1b1d5879469e3John Porto Func, Spill, Traits::VariableSplit::High); 32021d235425dab1f3dd059973fc53f1b1d5879469e3John Porto _mov(T_Lo, loOperand(Src0)); 32031d235425dab1f3dd059973fc53f1b1d5879469e3John Porto // Technically, the Spill is defined after the _store happens, but 320457e126899b20c65ff3ea23a3b7d7a67ab30b99dcAndrew Scull // SpillLo is considered a "use" of Spill so define Spill before it is 320557e126899b20c65ff3ea23a3b7d7a67ab30b99dcAndrew Scull // used. 32061d937a8e35b7ae51ee64c32f8f34d51ba33498f4John Porto Context.insert<InstFakeDef>(Spill); 32071d235425dab1f3dd059973fc53f1b1d5879469e3John Porto _store(T_Lo, SpillLo); 32081d235425dab1f3dd059973fc53f1b1d5879469e3John Porto _mov(T_Hi, hiOperand(Src0)); 32091d235425dab1f3dd059973fc53f1b1d5879469e3John Porto _store(T_Hi, SpillHi); 32101d235425dab1f3dd059973fc53f1b1d5879469e3John Porto _movq(Dest, Spill); 32111d235425dab1f3dd059973fc53f1b1d5879469e3John Porto } 32127e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto } break; 32137e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto case IceType_v8i1: { 321426217e3333150e66fc96aca79c01105906797960David Sehr llvm::report_fatal_error("Helper call was expected"); 32157e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto } break; 32167e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto case IceType_v16i1: { 321726217e3333150e66fc96aca79c01105906797960David Sehr llvm::report_fatal_error("Helper call was expected"); 32187e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto } break; 32197e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto case IceType_v8i16: 32207e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto case IceType_v16i8: 32217e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto case IceType_v4i32: 32227e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto case IceType_v4f32: { 32238b8af8248849994d22b1c13c5e0d12b251338075Nicolas Capens if (Src0->getType() == IceType_i32) { 32248b8af8248849994d22b1c13c5e0d12b251338075Nicolas Capens // Bitcast requires equal type sizes, which isn't strictly the case 32258b8af8248849994d22b1c13c5e0d12b251338075Nicolas Capens // between scalars and vectors, but to emulate v4i8 vectors one has to 32268b8af8248849994d22b1c13c5e0d12b251338075Nicolas Capens // use v16i8 vectors. 3227d0e30307fdd0c0c4bdaeab8d18dc53fb6aa845eaNicolas Capens assert(getFlags().getApplicationBinaryInterface() != ABI_PNaCl && 3228d0e30307fdd0c0c4bdaeab8d18dc53fb6aa845eaNicolas Capens "PNaCl only supports real 128-bit vectors"); 32298b8af8248849994d22b1c13c5e0d12b251338075Nicolas Capens _movd(Dest, legalize(Src0, Legal_Reg | Legal_Mem)); 32308b8af8248849994d22b1c13c5e0d12b251338075Nicolas Capens } else { 32318b8af8248849994d22b1c13c5e0d12b251338075Nicolas Capens _movp(Dest, legalizeToReg(Src0)); 32328b8af8248849994d22b1c13c5e0d12b251338075Nicolas Capens } 32337e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto } break; 32347e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto } 32357e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto break; 32367e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto } 32377e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto } 32387e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto} 32397e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto 32404a56686b5b56db6803f90ad53514bf2fa190d9f7John Portotemplate <typename TraitsType> 32414a56686b5b56db6803f90ad53514bf2fa190d9f7John Portovoid TargetX86Base<TraitsType>::lowerExtractElement( 32428cfeb69e17190d3bfe22a8a1cbd7679a114d68cfJim Stichnoth const InstExtractElement *Instr) { 32438cfeb69e17190d3bfe22a8a1cbd7679a114d68cfJim Stichnoth Operand *SourceVectNotLegalized = Instr->getSrc(0); 32442d6c82679e9786a924b76023aa30e1f44d0f8cbbJim Stichnoth auto *ElementIndex = llvm::dyn_cast<ConstantInteger32>(Instr->getSrc(1)); 32457e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto // Only constant indices are allowed in PNaCl IR. 32467e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto assert(ElementIndex); 32477e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto 32487e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto unsigned Index = ElementIndex->getValue(); 32497e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto Type Ty = SourceVectNotLegalized->getType(); 32507e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto Type ElementTy = typeElementType(Ty); 32517e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto Type InVectorElementTy = Traits::getInVectorElementType(Ty); 32527e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto 32537e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto // TODO(wala): Determine the best lowering sequences for each type. 32547e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto bool CanUsePextr = Ty == IceType_v8i16 || Ty == IceType_v8i1 || 32555bff61c44841990680781892036adb17b3cff0c4Jim Stichnoth (InstructionSet >= Traits::SSE4_1 && Ty != IceType_v4f32); 32565bff61c44841990680781892036adb17b3cff0c4Jim Stichnoth Variable *ExtractedElementR = 32575bff61c44841990680781892036adb17b3cff0c4Jim Stichnoth makeReg(CanUsePextr ? IceType_i32 : InVectorElementTy); 32585bff61c44841990680781892036adb17b3cff0c4Jim Stichnoth if (CanUsePextr) { 32595bff61c44841990680781892036adb17b3cff0c4Jim Stichnoth // Use pextrb, pextrw, or pextrd. The "b" and "w" versions clear the upper 32605bff61c44841990680781892036adb17b3cff0c4Jim Stichnoth // bits of the destination register, so we represent this by always 32615bff61c44841990680781892036adb17b3cff0c4Jim Stichnoth // extracting into an i32 register. The _mov into Dest below will do 32625bff61c44841990680781892036adb17b3cff0c4Jim Stichnoth // truncation as necessary. 32637e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto Constant *Mask = Ctx->getConstantInt32(Index); 326497f460dcd848dcef3455ed553c1b9e107d0e6ae9Andrew Scull Variable *SourceVectR = legalizeToReg(SourceVectNotLegalized); 32657e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto _pextr(ExtractedElementR, SourceVectR, Mask); 32667e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto } else if (Ty == IceType_v4i32 || Ty == IceType_v4f32 || Ty == IceType_v4i1) { 32677e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto // Use pshufd and movd/movss. 32687e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto Variable *T = nullptr; 32697e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto if (Index) { 327057e126899b20c65ff3ea23a3b7d7a67ab30b99dcAndrew Scull // The shuffle only needs to occur if the element to be extracted is not 327157e126899b20c65ff3ea23a3b7d7a67ab30b99dcAndrew Scull // at the lowest index. 32727e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto Constant *Mask = Ctx->getConstantInt32(Index); 32737e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto T = makeReg(Ty); 32747e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto _pshufd(T, legalize(SourceVectNotLegalized, Legal_Reg | Legal_Mem), Mask); 32757e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto } else { 327697f460dcd848dcef3455ed553c1b9e107d0e6ae9Andrew Scull T = legalizeToReg(SourceVectNotLegalized); 32777e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto } 32787e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto 32797e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto if (InVectorElementTy == IceType_i32) { 32807e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto _movd(ExtractedElementR, T); 32817e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto } else { // Ty == IceType_f32 328257e126899b20c65ff3ea23a3b7d7a67ab30b99dcAndrew Scull // TODO(wala): _movss is only used here because _mov does not allow a 328357e126899b20c65ff3ea23a3b7d7a67ab30b99dcAndrew Scull // vector source and a scalar destination. _mov should be able to be 328457e126899b20c65ff3ea23a3b7d7a67ab30b99dcAndrew Scull // used here. 328557e126899b20c65ff3ea23a3b7d7a67ab30b99dcAndrew Scull // _movss is a binary instruction, so the FakeDef is needed to keep the 328657e126899b20c65ff3ea23a3b7d7a67ab30b99dcAndrew Scull // live range analysis consistent. 32871d937a8e35b7ae51ee64c32f8f34d51ba33498f4John Porto Context.insert<InstFakeDef>(ExtractedElementR); 32887e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto _movss(ExtractedElementR, T); 32897e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto } 32907e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto } else { 32917e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto assert(Ty == IceType_v16i8 || Ty == IceType_v16i1); 32927e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto // Spill the value to a stack slot and do the extraction in memory. 32937e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto // 329457e126899b20c65ff3ea23a3b7d7a67ab30b99dcAndrew Scull // TODO(wala): use legalize(SourceVectNotLegalized, Legal_Mem) when support 329557e126899b20c65ff3ea23a3b7d7a67ab30b99dcAndrew Scull // for legalizing to mem is implemented. 32965aeed955e17bac8cc44cc6d2b6ff7513cc714c2fJohn Porto Variable *Slot = Func->makeVariable(Ty); 329711c9a325399b282cb4ea7d1d24d42fceeec2a09aAndrew Scull Slot->setMustNotHaveReg(); 329897f460dcd848dcef3455ed553c1b9e107d0e6ae9Andrew Scull _movp(Slot, legalizeToReg(SourceVectNotLegalized)); 32997e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto 33007e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto // Compute the location of the element in memory. 33017e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto unsigned Offset = Index * typeWidthInBytes(InVectorElementTy); 33024a56686b5b56db6803f90ad53514bf2fa190d9f7John Porto X86OperandMem *Loc = 33037e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto getMemoryOperandForStackSlot(InVectorElementTy, Slot, Offset); 33047e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto _mov(ExtractedElementR, Loc); 33057e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto } 33067e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto 33077e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto if (ElementTy == IceType_i1) { 33087e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto // Truncate extracted integers to i1s if necessary. 33097e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto Variable *T = makeReg(IceType_i1); 33107e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto InstCast *Cast = 33117e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto InstCast::create(Func, InstCast::Trunc, T, ExtractedElementR); 33127e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto lowerCast(Cast); 33137e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto ExtractedElementR = T; 33147e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto } 33157e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto 33167e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto // Copy the element to the destination. 33178cfeb69e17190d3bfe22a8a1cbd7679a114d68cfJim Stichnoth Variable *Dest = Instr->getDest(); 33187e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto _mov(Dest, ExtractedElementR); 33197e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto} 33207e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto 33214a56686b5b56db6803f90ad53514bf2fa190d9f7John Portotemplate <typename TraitsType> 33224a56686b5b56db6803f90ad53514bf2fa190d9f7John Portovoid TargetX86Base<TraitsType>::lowerFcmp(const InstFcmp *Fcmp) { 3323e398428c3ef8ed2339c0f334dd30a7d3f0ed4b6eDavid Sehr Variable *Dest = Fcmp->getDest(); 33247e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto 33257e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto if (isVectorType(Dest->getType())) { 3326e398428c3ef8ed2339c0f334dd30a7d3f0ed4b6eDavid Sehr lowerFcmpVector(Fcmp); 3327e398428c3ef8ed2339c0f334dd30a7d3f0ed4b6eDavid Sehr } else { 3328e398428c3ef8ed2339c0f334dd30a7d3f0ed4b6eDavid Sehr constexpr Inst *Consumer = nullptr; 3329e398428c3ef8ed2339c0f334dd30a7d3f0ed4b6eDavid Sehr lowerFcmpAndConsumer(Fcmp, Consumer); 3330e398428c3ef8ed2339c0f334dd30a7d3f0ed4b6eDavid Sehr } 3331e398428c3ef8ed2339c0f334dd30a7d3f0ed4b6eDavid Sehr} 33327e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto 33334a56686b5b56db6803f90ad53514bf2fa190d9f7John Portotemplate <typename TraitsType> 33344a56686b5b56db6803f90ad53514bf2fa190d9f7John Portovoid TargetX86Base<TraitsType>::lowerFcmpAndConsumer(const InstFcmp *Fcmp, 33354a56686b5b56db6803f90ad53514bf2fa190d9f7John Porto const Inst *Consumer) { 3336e398428c3ef8ed2339c0f334dd30a7d3f0ed4b6eDavid Sehr Operand *Src0 = Fcmp->getSrc(0); 3337e398428c3ef8ed2339c0f334dd30a7d3f0ed4b6eDavid Sehr Operand *Src1 = Fcmp->getSrc(1); 3338e398428c3ef8ed2339c0f334dd30a7d3f0ed4b6eDavid Sehr Variable *Dest = Fcmp->getDest(); 3339e398428c3ef8ed2339c0f334dd30a7d3f0ed4b6eDavid Sehr 3340e398428c3ef8ed2339c0f334dd30a7d3f0ed4b6eDavid Sehr if (Consumer != nullptr) { 3341e398428c3ef8ed2339c0f334dd30a7d3f0ed4b6eDavid Sehr if (auto *Select = llvm::dyn_cast<InstSelect>(Consumer)) { 3342e398428c3ef8ed2339c0f334dd30a7d3f0ed4b6eDavid Sehr if (lowerOptimizeFcmpSelect(Fcmp, Select)) 3343e398428c3ef8ed2339c0f334dd30a7d3f0ed4b6eDavid Sehr return; 33447e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto } 33457e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto } 33467e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto 334747b6ba6db7c8b8dd36ac54dbd19ae7b4a5b77424Nicolas Capens if (isVectorType(Dest->getType())) { 334847b6ba6db7c8b8dd36ac54dbd19ae7b4a5b77424Nicolas Capens lowerFcmp(Fcmp); 334947b6ba6db7c8b8dd36ac54dbd19ae7b4a5b77424Nicolas Capens if (Consumer != nullptr) 335047b6ba6db7c8b8dd36ac54dbd19ae7b4a5b77424Nicolas Capens lowerSelectVector(llvm::cast<InstSelect>(Consumer)); 335147b6ba6db7c8b8dd36ac54dbd19ae7b4a5b77424Nicolas Capens return; 335247b6ba6db7c8b8dd36ac54dbd19ae7b4a5b77424Nicolas Capens } 335347b6ba6db7c8b8dd36ac54dbd19ae7b4a5b77424Nicolas Capens 33547e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto // Lowering a = fcmp cond, b, c 33557e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto // ucomiss b, c /* only if C1 != Br_None */ 33567e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto // /* but swap b,c order if SwapOperands==true */ 33577e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto // mov a, <default> 33587e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto // j<C1> label /* only if C1 != Br_None */ 33597e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto // j<C2> label /* only if C2 != Br_None */ 33607e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto // FakeUse(a) /* only if C1 != Br_None */ 33617e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto // mov a, !<default> /* only if C1 != Br_None */ 33627e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto // label: /* only if C1 != Br_None */ 33637e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto // 33647e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto // setcc lowering when C1 != Br_None && C2 == Br_None: 33657e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto // ucomiss b, c /* but swap b,c order if SwapOperands==true */ 33667e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto // setcc a, C1 3367e398428c3ef8ed2339c0f334dd30a7d3f0ed4b6eDavid Sehr InstFcmp::FCond Condition = Fcmp->getCondition(); 33682d6c82679e9786a924b76023aa30e1f44d0f8cbbJim Stichnoth assert(Condition < Traits::TableFcmpSize); 33692d6c82679e9786a924b76023aa30e1f44d0f8cbbJim Stichnoth if (Traits::TableFcmp[Condition].SwapScalarOperands) 33707e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto std::swap(Src0, Src1); 33712d6c82679e9786a924b76023aa30e1f44d0f8cbbJim Stichnoth const bool HasC1 = (Traits::TableFcmp[Condition].C1 != Traits::Cond::Br_None); 33722d6c82679e9786a924b76023aa30e1f44d0f8cbbJim Stichnoth const bool HasC2 = (Traits::TableFcmp[Condition].C2 != Traits::Cond::Br_None); 33737e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto if (HasC1) { 33747e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto Src0 = legalize(Src0); 33757e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto Operand *Src1RM = legalize(Src1, Legal_Reg | Legal_Mem); 33767e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto Variable *T = nullptr; 33777e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto _mov(T, Src0); 33787e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto _ucomiss(T, Src1RM); 33797e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto if (!HasC2) { 33802d6c82679e9786a924b76023aa30e1f44d0f8cbbJim Stichnoth assert(Traits::TableFcmp[Condition].Default); 33812d6c82679e9786a924b76023aa30e1f44d0f8cbbJim Stichnoth setccOrConsumer(Traits::TableFcmp[Condition].C1, Dest, Consumer); 33827e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto return; 33837e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto } 33847e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto } 33852d6c82679e9786a924b76023aa30e1f44d0f8cbbJim Stichnoth int32_t IntDefault = Traits::TableFcmp[Condition].Default; 3386e398428c3ef8ed2339c0f334dd30a7d3f0ed4b6eDavid Sehr if (Consumer == nullptr) { 3387daf096cd1b8b9ec3d29c8d7823c3d60cd51c65c9David Sehr Constant *Default = Ctx->getConstantInt(Dest->getType(), IntDefault); 3388daf096cd1b8b9ec3d29c8d7823c3d60cd51c65c9David Sehr _mov(Dest, Default); 3389daf096cd1b8b9ec3d29c8d7823c3d60cd51c65c9David Sehr if (HasC1) { 33904a56686b5b56db6803f90ad53514bf2fa190d9f7John Porto InstX86Label *Label = InstX86Label::create(Func, this); 33912d6c82679e9786a924b76023aa30e1f44d0f8cbbJim Stichnoth _br(Traits::TableFcmp[Condition].C1, Label); 3392daf096cd1b8b9ec3d29c8d7823c3d60cd51c65c9David Sehr if (HasC2) { 33932d6c82679e9786a924b76023aa30e1f44d0f8cbbJim Stichnoth _br(Traits::TableFcmp[Condition].C2, Label); 3394daf096cd1b8b9ec3d29c8d7823c3d60cd51c65c9David Sehr } 3395daf096cd1b8b9ec3d29c8d7823c3d60cd51c65c9David Sehr Constant *NonDefault = Ctx->getConstantInt(Dest->getType(), !IntDefault); 3396e398428c3ef8ed2339c0f334dd30a7d3f0ed4b6eDavid Sehr _redefined(_mov(Dest, NonDefault)); 3397daf096cd1b8b9ec3d29c8d7823c3d60cd51c65c9David Sehr Context.insert(Label); 33987e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto } 3399e398428c3ef8ed2339c0f334dd30a7d3f0ed4b6eDavid Sehr return; 3400e398428c3ef8ed2339c0f334dd30a7d3f0ed4b6eDavid Sehr } 3401e398428c3ef8ed2339c0f334dd30a7d3f0ed4b6eDavid Sehr if (const auto *Br = llvm::dyn_cast<InstBr>(Consumer)) { 3402daf096cd1b8b9ec3d29c8d7823c3d60cd51c65c9David Sehr CfgNode *TrueSucc = Br->getTargetTrue(); 3403daf096cd1b8b9ec3d29c8d7823c3d60cd51c65c9David Sehr CfgNode *FalseSucc = Br->getTargetFalse(); 3404daf096cd1b8b9ec3d29c8d7823c3d60cd51c65c9David Sehr if (IntDefault != 0) 3405daf096cd1b8b9ec3d29c8d7823c3d60cd51c65c9David Sehr std::swap(TrueSucc, FalseSucc); 3406daf096cd1b8b9ec3d29c8d7823c3d60cd51c65c9David Sehr if (HasC1) { 34072d6c82679e9786a924b76023aa30e1f44d0f8cbbJim Stichnoth _br(Traits::TableFcmp[Condition].C1, FalseSucc); 3408daf096cd1b8b9ec3d29c8d7823c3d60cd51c65c9David Sehr if (HasC2) { 34092d6c82679e9786a924b76023aa30e1f44d0f8cbbJim Stichnoth _br(Traits::TableFcmp[Condition].C2, FalseSucc); 3410daf096cd1b8b9ec3d29c8d7823c3d60cd51c65c9David Sehr } 3411daf096cd1b8b9ec3d29c8d7823c3d60cd51c65c9David Sehr _br(TrueSucc); 3412daf096cd1b8b9ec3d29c8d7823c3d60cd51c65c9David Sehr return; 3413daf096cd1b8b9ec3d29c8d7823c3d60cd51c65c9David Sehr } 3414daf096cd1b8b9ec3d29c8d7823c3d60cd51c65c9David Sehr _br(FalseSucc); 3415e398428c3ef8ed2339c0f334dd30a7d3f0ed4b6eDavid Sehr return; 34167e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto } 3417e398428c3ef8ed2339c0f334dd30a7d3f0ed4b6eDavid Sehr if (auto *Select = llvm::dyn_cast<InstSelect>(Consumer)) { 3418e398428c3ef8ed2339c0f334dd30a7d3f0ed4b6eDavid Sehr Operand *SrcT = Select->getTrueOperand(); 3419e398428c3ef8ed2339c0f334dd30a7d3f0ed4b6eDavid Sehr Operand *SrcF = Select->getFalseOperand(); 3420e398428c3ef8ed2339c0f334dd30a7d3f0ed4b6eDavid Sehr Variable *SelectDest = Select->getDest(); 3421e398428c3ef8ed2339c0f334dd30a7d3f0ed4b6eDavid Sehr if (IntDefault != 0) 3422e398428c3ef8ed2339c0f334dd30a7d3f0ed4b6eDavid Sehr std::swap(SrcT, SrcF); 3423e398428c3ef8ed2339c0f334dd30a7d3f0ed4b6eDavid Sehr lowerMove(SelectDest, SrcF, false); 3424e398428c3ef8ed2339c0f334dd30a7d3f0ed4b6eDavid Sehr if (HasC1) { 34254a56686b5b56db6803f90ad53514bf2fa190d9f7John Porto InstX86Label *Label = InstX86Label::create(Func, this); 34262d6c82679e9786a924b76023aa30e1f44d0f8cbbJim Stichnoth _br(Traits::TableFcmp[Condition].C1, Label); 3427e398428c3ef8ed2339c0f334dd30a7d3f0ed4b6eDavid Sehr if (HasC2) { 34282d6c82679e9786a924b76023aa30e1f44d0f8cbbJim Stichnoth _br(Traits::TableFcmp[Condition].C2, Label); 3429e398428c3ef8ed2339c0f334dd30a7d3f0ed4b6eDavid Sehr } 3430e398428c3ef8ed2339c0f334dd30a7d3f0ed4b6eDavid Sehr static constexpr bool IsRedefinition = true; 3431e398428c3ef8ed2339c0f334dd30a7d3f0ed4b6eDavid Sehr lowerMove(SelectDest, SrcT, IsRedefinition); 3432e398428c3ef8ed2339c0f334dd30a7d3f0ed4b6eDavid Sehr Context.insert(Label); 3433e398428c3ef8ed2339c0f334dd30a7d3f0ed4b6eDavid Sehr } 3434e398428c3ef8ed2339c0f334dd30a7d3f0ed4b6eDavid Sehr return; 3435e398428c3ef8ed2339c0f334dd30a7d3f0ed4b6eDavid Sehr } 3436e398428c3ef8ed2339c0f334dd30a7d3f0ed4b6eDavid Sehr llvm::report_fatal_error("Unexpected consumer type"); 34375c87542ab707c91b1cc0d3d32ff4073e27f9bdf7David Sehr} 34385c87542ab707c91b1cc0d3d32ff4073e27f9bdf7David Sehr 34394a56686b5b56db6803f90ad53514bf2fa190d9f7John Portotemplate <typename TraitsType> 34404a56686b5b56db6803f90ad53514bf2fa190d9f7John Portovoid TargetX86Base<TraitsType>::lowerFcmpVector(const InstFcmp *Fcmp) { 3441e398428c3ef8ed2339c0f334dd30a7d3f0ed4b6eDavid Sehr Operand *Src0 = Fcmp->getSrc(0); 3442e398428c3ef8ed2339c0f334dd30a7d3f0ed4b6eDavid Sehr Operand *Src1 = Fcmp->getSrc(1); 3443e398428c3ef8ed2339c0f334dd30a7d3f0ed4b6eDavid Sehr Variable *Dest = Fcmp->getDest(); 3444d981025a26d962d50f7fead809b44d16be9051f3David Sehr 3445e398428c3ef8ed2339c0f334dd30a7d3f0ed4b6eDavid Sehr if (!isVectorType(Dest->getType())) 3446e398428c3ef8ed2339c0f334dd30a7d3f0ed4b6eDavid Sehr llvm::report_fatal_error("Expected vector compare"); 34477e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto 3448e398428c3ef8ed2339c0f334dd30a7d3f0ed4b6eDavid Sehr InstFcmp::FCond Condition = Fcmp->getCondition(); 34492d6c82679e9786a924b76023aa30e1f44d0f8cbbJim Stichnoth assert(Condition < Traits::TableFcmpSize); 34507e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto 34512d6c82679e9786a924b76023aa30e1f44d0f8cbbJim Stichnoth if (Traits::TableFcmp[Condition].SwapVectorOperands) 3452e398428c3ef8ed2339c0f334dd30a7d3f0ed4b6eDavid Sehr std::swap(Src0, Src1); 34537e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto 3454e398428c3ef8ed2339c0f334dd30a7d3f0ed4b6eDavid Sehr Variable *T = nullptr; 3455e398428c3ef8ed2339c0f334dd30a7d3f0ed4b6eDavid Sehr 3456e398428c3ef8ed2339c0f334dd30a7d3f0ed4b6eDavid Sehr if (Condition == InstFcmp::True) { 3457e398428c3ef8ed2339c0f334dd30a7d3f0ed4b6eDavid Sehr // makeVectorOfOnes() requires an integer vector type. 3458e398428c3ef8ed2339c0f334dd30a7d3f0ed4b6eDavid Sehr T = makeVectorOfMinusOnes(IceType_v4i32); 3459e398428c3ef8ed2339c0f334dd30a7d3f0ed4b6eDavid Sehr } else if (Condition == InstFcmp::False) { 3460e398428c3ef8ed2339c0f334dd30a7d3f0ed4b6eDavid Sehr T = makeVectorOfZeros(Dest->getType()); 3461e398428c3ef8ed2339c0f334dd30a7d3f0ed4b6eDavid Sehr } else { 34627e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto Operand *Src0RM = legalize(Src0, Legal_Reg | Legal_Mem); 34637e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto Operand *Src1RM = legalize(Src1, Legal_Reg | Legal_Mem); 34644a56686b5b56db6803f90ad53514bf2fa190d9f7John Porto if (llvm::isa<X86OperandMem>(Src1RM)) 3465e398428c3ef8ed2339c0f334dd30a7d3f0ed4b6eDavid Sehr Src1RM = legalizeToReg(Src1RM); 34667e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto 34677e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto switch (Condition) { 3468e398428c3ef8ed2339c0f334dd30a7d3f0ed4b6eDavid Sehr default: { 34692d6c82679e9786a924b76023aa30e1f44d0f8cbbJim Stichnoth const CmppsCond Predicate = Traits::TableFcmp[Condition].Predicate; 3470e398428c3ef8ed2339c0f334dd30a7d3f0ed4b6eDavid Sehr assert(Predicate != Traits::Cond::Cmpps_Invalid); 3471e398428c3ef8ed2339c0f334dd30a7d3f0ed4b6eDavid Sehr T = makeReg(Src0RM->getType()); 34727e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto _movp(T, Src0RM); 3473e398428c3ef8ed2339c0f334dd30a7d3f0ed4b6eDavid Sehr _cmpps(T, Src1RM, Predicate); 34747e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto } break; 3475e398428c3ef8ed2339c0f334dd30a7d3f0ed4b6eDavid Sehr case InstFcmp::One: { 3476e398428c3ef8ed2339c0f334dd30a7d3f0ed4b6eDavid Sehr // Check both unequal and ordered. 3477e398428c3ef8ed2339c0f334dd30a7d3f0ed4b6eDavid Sehr T = makeReg(Src0RM->getType()); 3478e398428c3ef8ed2339c0f334dd30a7d3f0ed4b6eDavid Sehr Variable *T2 = makeReg(Src0RM->getType()); 34797e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto _movp(T, Src0RM); 3480e398428c3ef8ed2339c0f334dd30a7d3f0ed4b6eDavid Sehr _cmpps(T, Src1RM, Traits::Cond::Cmpps_neq); 3481e398428c3ef8ed2339c0f334dd30a7d3f0ed4b6eDavid Sehr _movp(T2, Src0RM); 3482e398428c3ef8ed2339c0f334dd30a7d3f0ed4b6eDavid Sehr _cmpps(T2, Src1RM, Traits::Cond::Cmpps_ord); 3483e398428c3ef8ed2339c0f334dd30a7d3f0ed4b6eDavid Sehr _pand(T, T2); 34847e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto } break; 3485e398428c3ef8ed2339c0f334dd30a7d3f0ed4b6eDavid Sehr case InstFcmp::Ueq: { 3486e398428c3ef8ed2339c0f334dd30a7d3f0ed4b6eDavid Sehr // Check both equal or unordered. 3487e398428c3ef8ed2339c0f334dd30a7d3f0ed4b6eDavid Sehr T = makeReg(Src0RM->getType()); 3488e398428c3ef8ed2339c0f334dd30a7d3f0ed4b6eDavid Sehr Variable *T2 = makeReg(Src0RM->getType()); 34897e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto _movp(T, Src0RM); 3490e398428c3ef8ed2339c0f334dd30a7d3f0ed4b6eDavid Sehr _cmpps(T, Src1RM, Traits::Cond::Cmpps_eq); 3491e398428c3ef8ed2339c0f334dd30a7d3f0ed4b6eDavid Sehr _movp(T2, Src0RM); 3492e398428c3ef8ed2339c0f334dd30a7d3f0ed4b6eDavid Sehr _cmpps(T2, Src1RM, Traits::Cond::Cmpps_unord); 3493e398428c3ef8ed2339c0f334dd30a7d3f0ed4b6eDavid Sehr _por(T, T2); 34947e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto } break; 34957e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto } 34967e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto } 34977e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto 3498e398428c3ef8ed2339c0f334dd30a7d3f0ed4b6eDavid Sehr assert(T != nullptr); 3499e398428c3ef8ed2339c0f334dd30a7d3f0ed4b6eDavid Sehr _movp(Dest, T); 3500e398428c3ef8ed2339c0f334dd30a7d3f0ed4b6eDavid Sehr eliminateNextVectorSextInstruction(Dest); 3501e398428c3ef8ed2339c0f334dd30a7d3f0ed4b6eDavid Sehr} 3502e398428c3ef8ed2339c0f334dd30a7d3f0ed4b6eDavid Sehr 3503e398428c3ef8ed2339c0f334dd30a7d3f0ed4b6eDavid Sehrinline bool isZero(const Operand *Opnd) { 3504e398428c3ef8ed2339c0f334dd30a7d3f0ed4b6eDavid Sehr if (auto *C64 = llvm::dyn_cast<ConstantInteger64>(Opnd)) 3505e398428c3ef8ed2339c0f334dd30a7d3f0ed4b6eDavid Sehr return C64->getValue() == 0; 3506e398428c3ef8ed2339c0f334dd30a7d3f0ed4b6eDavid Sehr if (auto *C32 = llvm::dyn_cast<ConstantInteger32>(Opnd)) 3507e398428c3ef8ed2339c0f334dd30a7d3f0ed4b6eDavid Sehr return C32->getValue() == 0; 3508e398428c3ef8ed2339c0f334dd30a7d3f0ed4b6eDavid Sehr return false; 3509e398428c3ef8ed2339c0f334dd30a7d3f0ed4b6eDavid Sehr} 3510e398428c3ef8ed2339c0f334dd30a7d3f0ed4b6eDavid Sehr 35114a56686b5b56db6803f90ad53514bf2fa190d9f7John Portotemplate <typename TraitsType> 35124a56686b5b56db6803f90ad53514bf2fa190d9f7John Portovoid TargetX86Base<TraitsType>::lowerIcmpAndConsumer(const InstIcmp *Icmp, 35134a56686b5b56db6803f90ad53514bf2fa190d9f7John Porto const Inst *Consumer) { 3514e398428c3ef8ed2339c0f334dd30a7d3f0ed4b6eDavid Sehr Operand *Src0 = legalize(Icmp->getSrc(0)); 3515e398428c3ef8ed2339c0f334dd30a7d3f0ed4b6eDavid Sehr Operand *Src1 = legalize(Icmp->getSrc(1)); 3516e398428c3ef8ed2339c0f334dd30a7d3f0ed4b6eDavid Sehr Variable *Dest = Icmp->getDest(); 3517e398428c3ef8ed2339c0f334dd30a7d3f0ed4b6eDavid Sehr 351847b6ba6db7c8b8dd36ac54dbd19ae7b4a5b77424Nicolas Capens if (isVectorType(Dest->getType())) { 351947b6ba6db7c8b8dd36ac54dbd19ae7b4a5b77424Nicolas Capens lowerIcmp(Icmp); 352047b6ba6db7c8b8dd36ac54dbd19ae7b4a5b77424Nicolas Capens if (Consumer != nullptr) 352147b6ba6db7c8b8dd36ac54dbd19ae7b4a5b77424Nicolas Capens lowerSelectVector(llvm::cast<InstSelect>(Consumer)); 352247b6ba6db7c8b8dd36ac54dbd19ae7b4a5b77424Nicolas Capens return; 352347b6ba6db7c8b8dd36ac54dbd19ae7b4a5b77424Nicolas Capens } 3524e398428c3ef8ed2339c0f334dd30a7d3f0ed4b6eDavid Sehr 35251d235425dab1f3dd059973fc53f1b1d5879469e3John Porto if (!Traits::Is64Bit && Src0->getType() == IceType_i64) { 3526e398428c3ef8ed2339c0f334dd30a7d3f0ed4b6eDavid Sehr lowerIcmp64(Icmp, Consumer); 35277e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto return; 35287e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto } 35297e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto 35307e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto // cmp b, c 35315c87542ab707c91b1cc0d3d32ff4073e27f9bdf7David Sehr if (isZero(Src1)) { 3532d981025a26d962d50f7fead809b44d16be9051f3David Sehr switch (Icmp->getCondition()) { 35335c87542ab707c91b1cc0d3d32ff4073e27f9bdf7David Sehr default: 35345c87542ab707c91b1cc0d3d32ff4073e27f9bdf7David Sehr break; 35355c87542ab707c91b1cc0d3d32ff4073e27f9bdf7David Sehr case InstIcmp::Uge: 3536e398428c3ef8ed2339c0f334dd30a7d3f0ed4b6eDavid Sehr movOrConsumer(true, Dest, Consumer); 35375c87542ab707c91b1cc0d3d32ff4073e27f9bdf7David Sehr return; 35385c87542ab707c91b1cc0d3d32ff4073e27f9bdf7David Sehr case InstIcmp::Ult: 3539e398428c3ef8ed2339c0f334dd30a7d3f0ed4b6eDavid Sehr movOrConsumer(false, Dest, Consumer); 35405c87542ab707c91b1cc0d3d32ff4073e27f9bdf7David Sehr return; 35415c87542ab707c91b1cc0d3d32ff4073e27f9bdf7David Sehr } 35425c87542ab707c91b1cc0d3d32ff4073e27f9bdf7David Sehr } 35437e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto Operand *Src0RM = legalizeSrc0ForCmp(Src0, Src1); 35447e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto _cmp(Src0RM, Src1); 3545e398428c3ef8ed2339c0f334dd30a7d3f0ed4b6eDavid Sehr setccOrConsumer(Traits::getIcmp32Mapping(Icmp->getCondition()), Dest, 3546e398428c3ef8ed2339c0f334dd30a7d3f0ed4b6eDavid Sehr Consumer); 3547e398428c3ef8ed2339c0f334dd30a7d3f0ed4b6eDavid Sehr} 3548e398428c3ef8ed2339c0f334dd30a7d3f0ed4b6eDavid Sehr 35494a56686b5b56db6803f90ad53514bf2fa190d9f7John Portotemplate <typename TraitsType> 35504a56686b5b56db6803f90ad53514bf2fa190d9f7John Portovoid TargetX86Base<TraitsType>::lowerIcmpVector(const InstIcmp *Icmp) { 3551e398428c3ef8ed2339c0f334dd30a7d3f0ed4b6eDavid Sehr Operand *Src0 = legalize(Icmp->getSrc(0)); 3552e398428c3ef8ed2339c0f334dd30a7d3f0ed4b6eDavid Sehr Operand *Src1 = legalize(Icmp->getSrc(1)); 3553e398428c3ef8ed2339c0f334dd30a7d3f0ed4b6eDavid Sehr Variable *Dest = Icmp->getDest(); 3554e398428c3ef8ed2339c0f334dd30a7d3f0ed4b6eDavid Sehr 3555e398428c3ef8ed2339c0f334dd30a7d3f0ed4b6eDavid Sehr if (!isVectorType(Dest->getType())) 3556e398428c3ef8ed2339c0f334dd30a7d3f0ed4b6eDavid Sehr llvm::report_fatal_error("Expected a vector compare"); 3557e398428c3ef8ed2339c0f334dd30a7d3f0ed4b6eDavid Sehr 3558e398428c3ef8ed2339c0f334dd30a7d3f0ed4b6eDavid Sehr Type Ty = Src0->getType(); 3559e398428c3ef8ed2339c0f334dd30a7d3f0ed4b6eDavid Sehr // Promote i1 vectors to 128 bit integer vector types. 3560e398428c3ef8ed2339c0f334dd30a7d3f0ed4b6eDavid Sehr if (typeElementType(Ty) == IceType_i1) { 3561e398428c3ef8ed2339c0f334dd30a7d3f0ed4b6eDavid Sehr Type NewTy = IceType_NUM; 3562e398428c3ef8ed2339c0f334dd30a7d3f0ed4b6eDavid Sehr switch (Ty) { 3563e398428c3ef8ed2339c0f334dd30a7d3f0ed4b6eDavid Sehr default: 3564e398428c3ef8ed2339c0f334dd30a7d3f0ed4b6eDavid Sehr llvm::report_fatal_error("unexpected type"); 3565e398428c3ef8ed2339c0f334dd30a7d3f0ed4b6eDavid Sehr break; 3566e398428c3ef8ed2339c0f334dd30a7d3f0ed4b6eDavid Sehr case IceType_v4i1: 3567e398428c3ef8ed2339c0f334dd30a7d3f0ed4b6eDavid Sehr NewTy = IceType_v4i32; 3568e398428c3ef8ed2339c0f334dd30a7d3f0ed4b6eDavid Sehr break; 3569e398428c3ef8ed2339c0f334dd30a7d3f0ed4b6eDavid Sehr case IceType_v8i1: 3570e398428c3ef8ed2339c0f334dd30a7d3f0ed4b6eDavid Sehr NewTy = IceType_v8i16; 3571e398428c3ef8ed2339c0f334dd30a7d3f0ed4b6eDavid Sehr break; 3572e398428c3ef8ed2339c0f334dd30a7d3f0ed4b6eDavid Sehr case IceType_v16i1: 3573e398428c3ef8ed2339c0f334dd30a7d3f0ed4b6eDavid Sehr NewTy = IceType_v16i8; 3574e398428c3ef8ed2339c0f334dd30a7d3f0ed4b6eDavid Sehr break; 3575e398428c3ef8ed2339c0f334dd30a7d3f0ed4b6eDavid Sehr } 3576e398428c3ef8ed2339c0f334dd30a7d3f0ed4b6eDavid Sehr Variable *NewSrc0 = Func->makeVariable(NewTy); 3577e398428c3ef8ed2339c0f334dd30a7d3f0ed4b6eDavid Sehr Variable *NewSrc1 = Func->makeVariable(NewTy); 3578e398428c3ef8ed2339c0f334dd30a7d3f0ed4b6eDavid Sehr lowerCast(InstCast::create(Func, InstCast::Sext, NewSrc0, Src0)); 3579e398428c3ef8ed2339c0f334dd30a7d3f0ed4b6eDavid Sehr lowerCast(InstCast::create(Func, InstCast::Sext, NewSrc1, Src1)); 3580e398428c3ef8ed2339c0f334dd30a7d3f0ed4b6eDavid Sehr Src0 = NewSrc0; 3581e398428c3ef8ed2339c0f334dd30a7d3f0ed4b6eDavid Sehr Src1 = NewSrc1; 3582e398428c3ef8ed2339c0f334dd30a7d3f0ed4b6eDavid Sehr Ty = NewTy; 3583e398428c3ef8ed2339c0f334dd30a7d3f0ed4b6eDavid Sehr } 3584e398428c3ef8ed2339c0f334dd30a7d3f0ed4b6eDavid Sehr 3585e398428c3ef8ed2339c0f334dd30a7d3f0ed4b6eDavid Sehr InstIcmp::ICond Condition = Icmp->getCondition(); 3586e398428c3ef8ed2339c0f334dd30a7d3f0ed4b6eDavid Sehr 3587e398428c3ef8ed2339c0f334dd30a7d3f0ed4b6eDavid Sehr Operand *Src0RM = legalize(Src0, Legal_Reg | Legal_Mem); 3588e398428c3ef8ed2339c0f334dd30a7d3f0ed4b6eDavid Sehr Operand *Src1RM = legalize(Src1, Legal_Reg | Legal_Mem); 3589e398428c3ef8ed2339c0f334dd30a7d3f0ed4b6eDavid Sehr 3590e398428c3ef8ed2339c0f334dd30a7d3f0ed4b6eDavid Sehr // SSE2 only has signed comparison operations. Transform unsigned inputs in 3591e398428c3ef8ed2339c0f334dd30a7d3f0ed4b6eDavid Sehr // a manner that allows for the use of signed comparison operations by 3592e398428c3ef8ed2339c0f334dd30a7d3f0ed4b6eDavid Sehr // flipping the high order bits. 3593e398428c3ef8ed2339c0f334dd30a7d3f0ed4b6eDavid Sehr if (Condition == InstIcmp::Ugt || Condition == InstIcmp::Uge || 3594e398428c3ef8ed2339c0f334dd30a7d3f0ed4b6eDavid Sehr Condition == InstIcmp::Ult || Condition == InstIcmp::Ule) { 3595e398428c3ef8ed2339c0f334dd30a7d3f0ed4b6eDavid Sehr Variable *T0 = makeReg(Ty); 3596e398428c3ef8ed2339c0f334dd30a7d3f0ed4b6eDavid Sehr Variable *T1 = makeReg(Ty); 3597e398428c3ef8ed2339c0f334dd30a7d3f0ed4b6eDavid Sehr Variable *HighOrderBits = makeVectorOfHighOrderBits(Ty); 3598e398428c3ef8ed2339c0f334dd30a7d3f0ed4b6eDavid Sehr _movp(T0, Src0RM); 3599e398428c3ef8ed2339c0f334dd30a7d3f0ed4b6eDavid Sehr _pxor(T0, HighOrderBits); 3600e398428c3ef8ed2339c0f334dd30a7d3f0ed4b6eDavid Sehr _movp(T1, Src1RM); 3601e398428c3ef8ed2339c0f334dd30a7d3f0ed4b6eDavid Sehr _pxor(T1, HighOrderBits); 3602e398428c3ef8ed2339c0f334dd30a7d3f0ed4b6eDavid Sehr Src0RM = T0; 3603e398428c3ef8ed2339c0f334dd30a7d3f0ed4b6eDavid Sehr Src1RM = T1; 3604e398428c3ef8ed2339c0f334dd30a7d3f0ed4b6eDavid Sehr } 3605e398428c3ef8ed2339c0f334dd30a7d3f0ed4b6eDavid Sehr 3606e398428c3ef8ed2339c0f334dd30a7d3f0ed4b6eDavid Sehr Variable *T = makeReg(Ty); 3607e398428c3ef8ed2339c0f334dd30a7d3f0ed4b6eDavid Sehr switch (Condition) { 3608e398428c3ef8ed2339c0f334dd30a7d3f0ed4b6eDavid Sehr default: 3609e398428c3ef8ed2339c0f334dd30a7d3f0ed4b6eDavid Sehr llvm_unreachable("unexpected condition"); 3610e398428c3ef8ed2339c0f334dd30a7d3f0ed4b6eDavid Sehr break; 3611e398428c3ef8ed2339c0f334dd30a7d3f0ed4b6eDavid Sehr case InstIcmp::Eq: { 36124a56686b5b56db6803f90ad53514bf2fa190d9f7John Porto if (llvm::isa<X86OperandMem>(Src1RM)) 3613e398428c3ef8ed2339c0f334dd30a7d3f0ed4b6eDavid Sehr Src1RM = legalizeToReg(Src1RM); 3614e398428c3ef8ed2339c0f334dd30a7d3f0ed4b6eDavid Sehr _movp(T, Src0RM); 3615e398428c3ef8ed2339c0f334dd30a7d3f0ed4b6eDavid Sehr _pcmpeq(T, Src1RM); 3616e398428c3ef8ed2339c0f334dd30a7d3f0ed4b6eDavid Sehr } break; 3617e398428c3ef8ed2339c0f334dd30a7d3f0ed4b6eDavid Sehr case InstIcmp::Ne: { 36184a56686b5b56db6803f90ad53514bf2fa190d9f7John Porto if (llvm::isa<X86OperandMem>(Src1RM)) 3619e398428c3ef8ed2339c0f334dd30a7d3f0ed4b6eDavid Sehr Src1RM = legalizeToReg(Src1RM); 3620e398428c3ef8ed2339c0f334dd30a7d3f0ed4b6eDavid Sehr _movp(T, Src0RM); 3621e398428c3ef8ed2339c0f334dd30a7d3f0ed4b6eDavid Sehr _pcmpeq(T, Src1RM); 3622e398428c3ef8ed2339c0f334dd30a7d3f0ed4b6eDavid Sehr Variable *MinusOne = makeVectorOfMinusOnes(Ty); 3623e398428c3ef8ed2339c0f334dd30a7d3f0ed4b6eDavid Sehr _pxor(T, MinusOne); 3624e398428c3ef8ed2339c0f334dd30a7d3f0ed4b6eDavid Sehr } break; 3625e398428c3ef8ed2339c0f334dd30a7d3f0ed4b6eDavid Sehr case InstIcmp::Ugt: 3626e398428c3ef8ed2339c0f334dd30a7d3f0ed4b6eDavid Sehr case InstIcmp::Sgt: { 36274a56686b5b56db6803f90ad53514bf2fa190d9f7John Porto if (llvm::isa<X86OperandMem>(Src1RM)) 3628e398428c3ef8ed2339c0f334dd30a7d3f0ed4b6eDavid Sehr Src1RM = legalizeToReg(Src1RM); 3629e398428c3ef8ed2339c0f334dd30a7d3f0ed4b6eDavid Sehr _movp(T, Src0RM); 3630e398428c3ef8ed2339c0f334dd30a7d3f0ed4b6eDavid Sehr _pcmpgt(T, Src1RM); 3631e398428c3ef8ed2339c0f334dd30a7d3f0ed4b6eDavid Sehr } break; 3632e398428c3ef8ed2339c0f334dd30a7d3f0ed4b6eDavid Sehr case InstIcmp::Uge: 3633e398428c3ef8ed2339c0f334dd30a7d3f0ed4b6eDavid Sehr case InstIcmp::Sge: { 3634e398428c3ef8ed2339c0f334dd30a7d3f0ed4b6eDavid Sehr // !(Src1RM > Src0RM) 36354a56686b5b56db6803f90ad53514bf2fa190d9f7John Porto if (llvm::isa<X86OperandMem>(Src0RM)) 3636e398428c3ef8ed2339c0f334dd30a7d3f0ed4b6eDavid Sehr Src0RM = legalizeToReg(Src0RM); 3637e398428c3ef8ed2339c0f334dd30a7d3f0ed4b6eDavid Sehr _movp(T, Src1RM); 3638e398428c3ef8ed2339c0f334dd30a7d3f0ed4b6eDavid Sehr _pcmpgt(T, Src0RM); 3639e398428c3ef8ed2339c0f334dd30a7d3f0ed4b6eDavid Sehr Variable *MinusOne = makeVectorOfMinusOnes(Ty); 3640e398428c3ef8ed2339c0f334dd30a7d3f0ed4b6eDavid Sehr _pxor(T, MinusOne); 3641e398428c3ef8ed2339c0f334dd30a7d3f0ed4b6eDavid Sehr } break; 3642e398428c3ef8ed2339c0f334dd30a7d3f0ed4b6eDavid Sehr case InstIcmp::Ult: 3643e398428c3ef8ed2339c0f334dd30a7d3f0ed4b6eDavid Sehr case InstIcmp::Slt: { 36444a56686b5b56db6803f90ad53514bf2fa190d9f7John Porto if (llvm::isa<X86OperandMem>(Src0RM)) 3645e398428c3ef8ed2339c0f334dd30a7d3f0ed4b6eDavid Sehr Src0RM = legalizeToReg(Src0RM); 3646e398428c3ef8ed2339c0f334dd30a7d3f0ed4b6eDavid Sehr _movp(T, Src1RM); 3647e398428c3ef8ed2339c0f334dd30a7d3f0ed4b6eDavid Sehr _pcmpgt(T, Src0RM); 3648e398428c3ef8ed2339c0f334dd30a7d3f0ed4b6eDavid Sehr } break; 3649e398428c3ef8ed2339c0f334dd30a7d3f0ed4b6eDavid Sehr case InstIcmp::Ule: 3650e398428c3ef8ed2339c0f334dd30a7d3f0ed4b6eDavid Sehr case InstIcmp::Sle: { 3651e398428c3ef8ed2339c0f334dd30a7d3f0ed4b6eDavid Sehr // !(Src0RM > Src1RM) 36524a56686b5b56db6803f90ad53514bf2fa190d9f7John Porto if (llvm::isa<X86OperandMem>(Src1RM)) 3653e398428c3ef8ed2339c0f334dd30a7d3f0ed4b6eDavid Sehr Src1RM = legalizeToReg(Src1RM); 3654e398428c3ef8ed2339c0f334dd30a7d3f0ed4b6eDavid Sehr _movp(T, Src0RM); 3655e398428c3ef8ed2339c0f334dd30a7d3f0ed4b6eDavid Sehr _pcmpgt(T, Src1RM); 3656e398428c3ef8ed2339c0f334dd30a7d3f0ed4b6eDavid Sehr Variable *MinusOne = makeVectorOfMinusOnes(Ty); 3657e398428c3ef8ed2339c0f334dd30a7d3f0ed4b6eDavid Sehr _pxor(T, MinusOne); 3658e398428c3ef8ed2339c0f334dd30a7d3f0ed4b6eDavid Sehr } break; 3659e398428c3ef8ed2339c0f334dd30a7d3f0ed4b6eDavid Sehr } 3660e398428c3ef8ed2339c0f334dd30a7d3f0ed4b6eDavid Sehr 3661e398428c3ef8ed2339c0f334dd30a7d3f0ed4b6eDavid Sehr _movp(Dest, T); 3662e398428c3ef8ed2339c0f334dd30a7d3f0ed4b6eDavid Sehr eliminateNextVectorSextInstruction(Dest); 36637e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto} 36647e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto 36654a56686b5b56db6803f90ad53514bf2fa190d9f7John Portotemplate <typename TraitsType> 36661d235425dab1f3dd059973fc53f1b1d5879469e3John Portotemplate <typename T> 36671d235425dab1f3dd059973fc53f1b1d5879469e3John Portotypename std::enable_if<!T::Is64Bit, void>::type 36684a56686b5b56db6803f90ad53514bf2fa190d9f7John PortoTargetX86Base<TraitsType>::lowerIcmp64(const InstIcmp *Icmp, 36694a56686b5b56db6803f90ad53514bf2fa190d9f7John Porto const Inst *Consumer) { 36701d235425dab1f3dd059973fc53f1b1d5879469e3John Porto // a=icmp cond, b, c ==> cmp b,c; a=1; br cond,L1; FakeUse(a); a=0; L1: 3671d981025a26d962d50f7fead809b44d16be9051f3David Sehr Operand *Src0 = legalize(Icmp->getSrc(0)); 3672d981025a26d962d50f7fead809b44d16be9051f3David Sehr Operand *Src1 = legalize(Icmp->getSrc(1)); 3673d981025a26d962d50f7fead809b44d16be9051f3David Sehr Variable *Dest = Icmp->getDest(); 3674d981025a26d962d50f7fead809b44d16be9051f3David Sehr InstIcmp::ICond Condition = Icmp->getCondition(); 36752d6c82679e9786a924b76023aa30e1f44d0f8cbbJim Stichnoth assert(Condition < Traits::TableIcmp64Size); 36765c87542ab707c91b1cc0d3d32ff4073e27f9bdf7David Sehr Operand *Src0LoRM = nullptr; 36775c87542ab707c91b1cc0d3d32ff4073e27f9bdf7David Sehr Operand *Src0HiRM = nullptr; 36785c87542ab707c91b1cc0d3d32ff4073e27f9bdf7David Sehr // Legalize the portions of Src0 that are going to be needed. 36795c87542ab707c91b1cc0d3d32ff4073e27f9bdf7David Sehr if (isZero(Src1)) { 36805c87542ab707c91b1cc0d3d32ff4073e27f9bdf7David Sehr switch (Condition) { 36815c87542ab707c91b1cc0d3d32ff4073e27f9bdf7David Sehr default: 36825c87542ab707c91b1cc0d3d32ff4073e27f9bdf7David Sehr llvm_unreachable("unexpected condition"); 36835c87542ab707c91b1cc0d3d32ff4073e27f9bdf7David Sehr break; 36845c87542ab707c91b1cc0d3d32ff4073e27f9bdf7David Sehr // These two are not optimized, so we fall through to the general case, 36855c87542ab707c91b1cc0d3d32ff4073e27f9bdf7David Sehr // which needs the upper and lower halves legalized. 36865c87542ab707c91b1cc0d3d32ff4073e27f9bdf7David Sehr case InstIcmp::Sgt: 36875c87542ab707c91b1cc0d3d32ff4073e27f9bdf7David Sehr case InstIcmp::Sle: 36881fb030c668841de8f5393e2d2c9e2ad1296af3dbJim Stichnoth // These four compare after performing an "or" of the high and low half, so 36891fb030c668841de8f5393e2d2c9e2ad1296af3dbJim Stichnoth // they need the upper and lower halves legalized. 36905c87542ab707c91b1cc0d3d32ff4073e27f9bdf7David Sehr case InstIcmp::Eq: 36915c87542ab707c91b1cc0d3d32ff4073e27f9bdf7David Sehr case InstIcmp::Ule: 36925c87542ab707c91b1cc0d3d32ff4073e27f9bdf7David Sehr case InstIcmp::Ne: 36935c87542ab707c91b1cc0d3d32ff4073e27f9bdf7David Sehr case InstIcmp::Ugt: 36945c87542ab707c91b1cc0d3d32ff4073e27f9bdf7David Sehr Src0LoRM = legalize(loOperand(Src0), Legal_Reg | Legal_Mem); 36955c87542ab707c91b1cc0d3d32ff4073e27f9bdf7David Sehr // These two test only the high half's sign bit, so they need only 36965c87542ab707c91b1cc0d3d32ff4073e27f9bdf7David Sehr // the upper half legalized. 36975c87542ab707c91b1cc0d3d32ff4073e27f9bdf7David Sehr case InstIcmp::Sge: 36985c87542ab707c91b1cc0d3d32ff4073e27f9bdf7David Sehr case InstIcmp::Slt: 36995c87542ab707c91b1cc0d3d32ff4073e27f9bdf7David Sehr Src0HiRM = legalize(hiOperand(Src0), Legal_Reg | Legal_Mem); 37005c87542ab707c91b1cc0d3d32ff4073e27f9bdf7David Sehr break; 37015c87542ab707c91b1cc0d3d32ff4073e27f9bdf7David Sehr 37025c87542ab707c91b1cc0d3d32ff4073e27f9bdf7David Sehr // These two move constants and hence need no legalization. 37035c87542ab707c91b1cc0d3d32ff4073e27f9bdf7David Sehr case InstIcmp::Uge: 37045c87542ab707c91b1cc0d3d32ff4073e27f9bdf7David Sehr case InstIcmp::Ult: 37055c87542ab707c91b1cc0d3d32ff4073e27f9bdf7David Sehr break; 37065c87542ab707c91b1cc0d3d32ff4073e27f9bdf7David Sehr } 37075c87542ab707c91b1cc0d3d32ff4073e27f9bdf7David Sehr } else { 37085c87542ab707c91b1cc0d3d32ff4073e27f9bdf7David Sehr Src0LoRM = legalize(loOperand(Src0), Legal_Reg | Legal_Mem); 37095c87542ab707c91b1cc0d3d32ff4073e27f9bdf7David Sehr Src0HiRM = legalize(hiOperand(Src0), Legal_Reg | Legal_Mem); 37105c87542ab707c91b1cc0d3d32ff4073e27f9bdf7David Sehr } 37115c87542ab707c91b1cc0d3d32ff4073e27f9bdf7David Sehr // Optimize comparisons with zero. 37125c87542ab707c91b1cc0d3d32ff4073e27f9bdf7David Sehr if (isZero(Src1)) { 37135c87542ab707c91b1cc0d3d32ff4073e27f9bdf7David Sehr Constant *SignMask = Ctx->getConstantInt32(0x80000000); 37145c87542ab707c91b1cc0d3d32ff4073e27f9bdf7David Sehr Variable *Temp = nullptr; 37155c87542ab707c91b1cc0d3d32ff4073e27f9bdf7David Sehr switch (Condition) { 37165c87542ab707c91b1cc0d3d32ff4073e27f9bdf7David Sehr default: 37175c87542ab707c91b1cc0d3d32ff4073e27f9bdf7David Sehr llvm_unreachable("unexpected condition"); 37185c87542ab707c91b1cc0d3d32ff4073e27f9bdf7David Sehr break; 37195c87542ab707c91b1cc0d3d32ff4073e27f9bdf7David Sehr case InstIcmp::Eq: 37205c87542ab707c91b1cc0d3d32ff4073e27f9bdf7David Sehr case InstIcmp::Ule: 3721aa0b1a176f6c3306be3e95374d72e735bab2bbabDavid Sehr // Mov Src0HiRM first, because it was legalized most recently, and will 3722aa0b1a176f6c3306be3e95374d72e735bab2bbabDavid Sehr // sometimes avoid a move before the OR. 3723aa0b1a176f6c3306be3e95374d72e735bab2bbabDavid Sehr _mov(Temp, Src0HiRM); 3724aa0b1a176f6c3306be3e95374d72e735bab2bbabDavid Sehr _or(Temp, Src0LoRM); 37251d937a8e35b7ae51ee64c32f8f34d51ba33498f4John Porto Context.insert<InstFakeUse>(Temp); 3726e398428c3ef8ed2339c0f334dd30a7d3f0ed4b6eDavid Sehr setccOrConsumer(Traits::Cond::Br_e, Dest, Consumer); 37275c87542ab707c91b1cc0d3d32ff4073e27f9bdf7David Sehr return; 37285c87542ab707c91b1cc0d3d32ff4073e27f9bdf7David Sehr case InstIcmp::Ne: 37295c87542ab707c91b1cc0d3d32ff4073e27f9bdf7David Sehr case InstIcmp::Ugt: 3730aa0b1a176f6c3306be3e95374d72e735bab2bbabDavid Sehr // Mov Src0HiRM first, because it was legalized most recently, and will 3731aa0b1a176f6c3306be3e95374d72e735bab2bbabDavid Sehr // sometimes avoid a move before the OR. 3732aa0b1a176f6c3306be3e95374d72e735bab2bbabDavid Sehr _mov(Temp, Src0HiRM); 3733aa0b1a176f6c3306be3e95374d72e735bab2bbabDavid Sehr _or(Temp, Src0LoRM); 37341d937a8e35b7ae51ee64c32f8f34d51ba33498f4John Porto Context.insert<InstFakeUse>(Temp); 3735e398428c3ef8ed2339c0f334dd30a7d3f0ed4b6eDavid Sehr setccOrConsumer(Traits::Cond::Br_ne, Dest, Consumer); 37365c87542ab707c91b1cc0d3d32ff4073e27f9bdf7David Sehr return; 37375c87542ab707c91b1cc0d3d32ff4073e27f9bdf7David Sehr case InstIcmp::Uge: 3738e398428c3ef8ed2339c0f334dd30a7d3f0ed4b6eDavid Sehr movOrConsumer(true, Dest, Consumer); 37395c87542ab707c91b1cc0d3d32ff4073e27f9bdf7David Sehr return; 37405c87542ab707c91b1cc0d3d32ff4073e27f9bdf7David Sehr case InstIcmp::Ult: 3741e398428c3ef8ed2339c0f334dd30a7d3f0ed4b6eDavid Sehr movOrConsumer(false, Dest, Consumer); 37425c87542ab707c91b1cc0d3d32ff4073e27f9bdf7David Sehr return; 37435c87542ab707c91b1cc0d3d32ff4073e27f9bdf7David Sehr case InstIcmp::Sgt: 37445c87542ab707c91b1cc0d3d32ff4073e27f9bdf7David Sehr break; 37455c87542ab707c91b1cc0d3d32ff4073e27f9bdf7David Sehr case InstIcmp::Sge: 37465c87542ab707c91b1cc0d3d32ff4073e27f9bdf7David Sehr _test(Src0HiRM, SignMask); 3747e398428c3ef8ed2339c0f334dd30a7d3f0ed4b6eDavid Sehr setccOrConsumer(Traits::Cond::Br_e, Dest, Consumer); 37485c87542ab707c91b1cc0d3d32ff4073e27f9bdf7David Sehr return; 37495c87542ab707c91b1cc0d3d32ff4073e27f9bdf7David Sehr case InstIcmp::Slt: 37505c87542ab707c91b1cc0d3d32ff4073e27f9bdf7David Sehr _test(Src0HiRM, SignMask); 3751e398428c3ef8ed2339c0f334dd30a7d3f0ed4b6eDavid Sehr setccOrConsumer(Traits::Cond::Br_ne, Dest, Consumer); 37525c87542ab707c91b1cc0d3d32ff4073e27f9bdf7David Sehr return; 37535c87542ab707c91b1cc0d3d32ff4073e27f9bdf7David Sehr case InstIcmp::Sle: 37545c87542ab707c91b1cc0d3d32ff4073e27f9bdf7David Sehr break; 37555c87542ab707c91b1cc0d3d32ff4073e27f9bdf7David Sehr } 37565c87542ab707c91b1cc0d3d32ff4073e27f9bdf7David Sehr } 37575c87542ab707c91b1cc0d3d32ff4073e27f9bdf7David Sehr // Handle general compares. 37585c87542ab707c91b1cc0d3d32ff4073e27f9bdf7David Sehr Operand *Src1LoRI = legalize(loOperand(Src1), Legal_Reg | Legal_Imm); 37595c87542ab707c91b1cc0d3d32ff4073e27f9bdf7David Sehr Operand *Src1HiRI = legalize(hiOperand(Src1), Legal_Reg | Legal_Imm); 3760e398428c3ef8ed2339c0f334dd30a7d3f0ed4b6eDavid Sehr if (Consumer == nullptr) { 3761d981025a26d962d50f7fead809b44d16be9051f3David Sehr Constant *Zero = Ctx->getConstantInt(Dest->getType(), 0); 3762d981025a26d962d50f7fead809b44d16be9051f3David Sehr Constant *One = Ctx->getConstantInt(Dest->getType(), 1); 37634a56686b5b56db6803f90ad53514bf2fa190d9f7John Porto InstX86Label *LabelFalse = InstX86Label::create(Func, this); 37644a56686b5b56db6803f90ad53514bf2fa190d9f7John Porto InstX86Label *LabelTrue = InstX86Label::create(Func, this); 3765d981025a26d962d50f7fead809b44d16be9051f3David Sehr _mov(Dest, One); 3766d981025a26d962d50f7fead809b44d16be9051f3David Sehr _cmp(Src0HiRM, Src1HiRI); 37672d6c82679e9786a924b76023aa30e1f44d0f8cbbJim Stichnoth if (Traits::TableIcmp64[Condition].C1 != Traits::Cond::Br_None) 37682d6c82679e9786a924b76023aa30e1f44d0f8cbbJim Stichnoth _br(Traits::TableIcmp64[Condition].C1, LabelTrue); 37692d6c82679e9786a924b76023aa30e1f44d0f8cbbJim Stichnoth if (Traits::TableIcmp64[Condition].C2 != Traits::Cond::Br_None) 37702d6c82679e9786a924b76023aa30e1f44d0f8cbbJim Stichnoth _br(Traits::TableIcmp64[Condition].C2, LabelFalse); 3771d981025a26d962d50f7fead809b44d16be9051f3David Sehr _cmp(Src0LoRM, Src1LoRI); 37722d6c82679e9786a924b76023aa30e1f44d0f8cbbJim Stichnoth _br(Traits::TableIcmp64[Condition].C3, LabelTrue); 3773d981025a26d962d50f7fead809b44d16be9051f3David Sehr Context.insert(LabelFalse); 3774e398428c3ef8ed2339c0f334dd30a7d3f0ed4b6eDavid Sehr _redefined(_mov(Dest, Zero)); 3775d981025a26d962d50f7fead809b44d16be9051f3David Sehr Context.insert(LabelTrue); 3776e398428c3ef8ed2339c0f334dd30a7d3f0ed4b6eDavid Sehr return; 3777e398428c3ef8ed2339c0f334dd30a7d3f0ed4b6eDavid Sehr } 3778e398428c3ef8ed2339c0f334dd30a7d3f0ed4b6eDavid Sehr if (const auto *Br = llvm::dyn_cast<InstBr>(Consumer)) { 3779d981025a26d962d50f7fead809b44d16be9051f3David Sehr _cmp(Src0HiRM, Src1HiRI); 37802d6c82679e9786a924b76023aa30e1f44d0f8cbbJim Stichnoth if (Traits::TableIcmp64[Condition].C1 != Traits::Cond::Br_None) 37812d6c82679e9786a924b76023aa30e1f44d0f8cbbJim Stichnoth _br(Traits::TableIcmp64[Condition].C1, Br->getTargetTrue()); 37822d6c82679e9786a924b76023aa30e1f44d0f8cbbJim Stichnoth if (Traits::TableIcmp64[Condition].C2 != Traits::Cond::Br_None) 37832d6c82679e9786a924b76023aa30e1f44d0f8cbbJim Stichnoth _br(Traits::TableIcmp64[Condition].C2, Br->getTargetFalse()); 3784d981025a26d962d50f7fead809b44d16be9051f3David Sehr _cmp(Src0LoRM, Src1LoRI); 37852d6c82679e9786a924b76023aa30e1f44d0f8cbbJim Stichnoth _br(Traits::TableIcmp64[Condition].C3, Br->getTargetTrue(), 3786d981025a26d962d50f7fead809b44d16be9051f3David Sehr Br->getTargetFalse()); 3787e398428c3ef8ed2339c0f334dd30a7d3f0ed4b6eDavid Sehr return; 3788d981025a26d962d50f7fead809b44d16be9051f3David Sehr } 3789e398428c3ef8ed2339c0f334dd30a7d3f0ed4b6eDavid Sehr if (auto *Select = llvm::dyn_cast<InstSelect>(Consumer)) { 3790e398428c3ef8ed2339c0f334dd30a7d3f0ed4b6eDavid Sehr Operand *SrcT = Select->getTrueOperand(); 3791e398428c3ef8ed2339c0f334dd30a7d3f0ed4b6eDavid Sehr Operand *SrcF = Select->getFalseOperand(); 3792e398428c3ef8ed2339c0f334dd30a7d3f0ed4b6eDavid Sehr Variable *SelectDest = Select->getDest(); 37934a56686b5b56db6803f90ad53514bf2fa190d9f7John Porto InstX86Label *LabelFalse = InstX86Label::create(Func, this); 37944a56686b5b56db6803f90ad53514bf2fa190d9f7John Porto InstX86Label *LabelTrue = InstX86Label::create(Func, this); 3795e398428c3ef8ed2339c0f334dd30a7d3f0ed4b6eDavid Sehr lowerMove(SelectDest, SrcT, false); 3796e398428c3ef8ed2339c0f334dd30a7d3f0ed4b6eDavid Sehr _cmp(Src0HiRM, Src1HiRI); 37972d6c82679e9786a924b76023aa30e1f44d0f8cbbJim Stichnoth if (Traits::TableIcmp64[Condition].C1 != Traits::Cond::Br_None) 37982d6c82679e9786a924b76023aa30e1f44d0f8cbbJim Stichnoth _br(Traits::TableIcmp64[Condition].C1, LabelTrue); 37992d6c82679e9786a924b76023aa30e1f44d0f8cbbJim Stichnoth if (Traits::TableIcmp64[Condition].C2 != Traits::Cond::Br_None) 38002d6c82679e9786a924b76023aa30e1f44d0f8cbbJim Stichnoth _br(Traits::TableIcmp64[Condition].C2, LabelFalse); 3801e398428c3ef8ed2339c0f334dd30a7d3f0ed4b6eDavid Sehr _cmp(Src0LoRM, Src1LoRI); 38022d6c82679e9786a924b76023aa30e1f44d0f8cbbJim Stichnoth _br(Traits::TableIcmp64[Condition].C3, LabelTrue); 3803e398428c3ef8ed2339c0f334dd30a7d3f0ed4b6eDavid Sehr Context.insert(LabelFalse); 3804e398428c3ef8ed2339c0f334dd30a7d3f0ed4b6eDavid Sehr static constexpr bool IsRedefinition = true; 3805e398428c3ef8ed2339c0f334dd30a7d3f0ed4b6eDavid Sehr lowerMove(SelectDest, SrcF, IsRedefinition); 3806e398428c3ef8ed2339c0f334dd30a7d3f0ed4b6eDavid Sehr Context.insert(LabelTrue); 3807e398428c3ef8ed2339c0f334dd30a7d3f0ed4b6eDavid Sehr return; 3808e398428c3ef8ed2339c0f334dd30a7d3f0ed4b6eDavid Sehr } 3809e398428c3ef8ed2339c0f334dd30a7d3f0ed4b6eDavid Sehr llvm::report_fatal_error("Unexpected consumer type"); 3810d981025a26d962d50f7fead809b44d16be9051f3David Sehr} 3811d981025a26d962d50f7fead809b44d16be9051f3David Sehr 38124a56686b5b56db6803f90ad53514bf2fa190d9f7John Portotemplate <typename TraitsType> 38134a56686b5b56db6803f90ad53514bf2fa190d9f7John Portovoid TargetX86Base<TraitsType>::setccOrConsumer(BrCond Condition, 38144a56686b5b56db6803f90ad53514bf2fa190d9f7John Porto Variable *Dest, 38154a56686b5b56db6803f90ad53514bf2fa190d9f7John Porto const Inst *Consumer) { 3816e398428c3ef8ed2339c0f334dd30a7d3f0ed4b6eDavid Sehr if (Consumer == nullptr) { 3817d981025a26d962d50f7fead809b44d16be9051f3David Sehr _setcc(Dest, Condition); 3818e398428c3ef8ed2339c0f334dd30a7d3f0ed4b6eDavid Sehr return; 3819e398428c3ef8ed2339c0f334dd30a7d3f0ed4b6eDavid Sehr } 3820e398428c3ef8ed2339c0f334dd30a7d3f0ed4b6eDavid Sehr if (const auto *Br = llvm::dyn_cast<InstBr>(Consumer)) { 3821d981025a26d962d50f7fead809b44d16be9051f3David Sehr _br(Condition, Br->getTargetTrue(), Br->getTargetFalse()); 3822e398428c3ef8ed2339c0f334dd30a7d3f0ed4b6eDavid Sehr return; 3823d981025a26d962d50f7fead809b44d16be9051f3David Sehr } 3824e398428c3ef8ed2339c0f334dd30a7d3f0ed4b6eDavid Sehr if (const auto *Select = llvm::dyn_cast<InstSelect>(Consumer)) { 3825e398428c3ef8ed2339c0f334dd30a7d3f0ed4b6eDavid Sehr Operand *SrcT = Select->getTrueOperand(); 3826e398428c3ef8ed2339c0f334dd30a7d3f0ed4b6eDavid Sehr Operand *SrcF = Select->getFalseOperand(); 3827e398428c3ef8ed2339c0f334dd30a7d3f0ed4b6eDavid Sehr Variable *SelectDest = Select->getDest(); 3828e398428c3ef8ed2339c0f334dd30a7d3f0ed4b6eDavid Sehr lowerSelectMove(SelectDest, Condition, SrcT, SrcF); 3829e398428c3ef8ed2339c0f334dd30a7d3f0ed4b6eDavid Sehr return; 3830e398428c3ef8ed2339c0f334dd30a7d3f0ed4b6eDavid Sehr } 3831e398428c3ef8ed2339c0f334dd30a7d3f0ed4b6eDavid Sehr llvm::report_fatal_error("Unexpected consumer type"); 3832d981025a26d962d50f7fead809b44d16be9051f3David Sehr} 3833d981025a26d962d50f7fead809b44d16be9051f3David Sehr 38344a56686b5b56db6803f90ad53514bf2fa190d9f7John Portotemplate <typename TraitsType> 38354a56686b5b56db6803f90ad53514bf2fa190d9f7John Portovoid TargetX86Base<TraitsType>::movOrConsumer(bool IcmpResult, Variable *Dest, 38364a56686b5b56db6803f90ad53514bf2fa190d9f7John Porto const Inst *Consumer) { 3837e398428c3ef8ed2339c0f334dd30a7d3f0ed4b6eDavid Sehr if (Consumer == nullptr) { 3838d981025a26d962d50f7fead809b44d16be9051f3David Sehr _mov(Dest, Ctx->getConstantInt(Dest->getType(), (IcmpResult ? 1 : 0))); 3839e398428c3ef8ed2339c0f334dd30a7d3f0ed4b6eDavid Sehr return; 3840e398428c3ef8ed2339c0f334dd30a7d3f0ed4b6eDavid Sehr } 3841e398428c3ef8ed2339c0f334dd30a7d3f0ed4b6eDavid Sehr if (const auto *Br = llvm::dyn_cast<InstBr>(Consumer)) { 3842d981025a26d962d50f7fead809b44d16be9051f3David Sehr // TODO(sehr,stichnot): This could be done with a single unconditional 3843d981025a26d962d50f7fead809b44d16be9051f3David Sehr // branch instruction, but subzero doesn't know how to handle the resulting 3844d981025a26d962d50f7fead809b44d16be9051f3David Sehr // control flow graph changes now. Make it do so to eliminate mov and cmp. 3845d981025a26d962d50f7fead809b44d16be9051f3David Sehr _mov(Dest, Ctx->getConstantInt(Dest->getType(), (IcmpResult ? 1 : 0))); 3846d981025a26d962d50f7fead809b44d16be9051f3David Sehr _cmp(Dest, Ctx->getConstantInt(Dest->getType(), 0)); 3847d981025a26d962d50f7fead809b44d16be9051f3David Sehr _br(Traits::Cond::Br_ne, Br->getTargetTrue(), Br->getTargetFalse()); 3848e398428c3ef8ed2339c0f334dd30a7d3f0ed4b6eDavid Sehr return; 3849e398428c3ef8ed2339c0f334dd30a7d3f0ed4b6eDavid Sehr } 3850e398428c3ef8ed2339c0f334dd30a7d3f0ed4b6eDavid Sehr if (const auto *Select = llvm::dyn_cast<InstSelect>(Consumer)) { 3851e398428c3ef8ed2339c0f334dd30a7d3f0ed4b6eDavid Sehr Operand *Src = nullptr; 3852e398428c3ef8ed2339c0f334dd30a7d3f0ed4b6eDavid Sehr if (IcmpResult) { 3853e398428c3ef8ed2339c0f334dd30a7d3f0ed4b6eDavid Sehr Src = legalize(Select->getTrueOperand(), Legal_Reg | Legal_Imm); 3854e398428c3ef8ed2339c0f334dd30a7d3f0ed4b6eDavid Sehr } else { 3855e398428c3ef8ed2339c0f334dd30a7d3f0ed4b6eDavid Sehr Src = legalize(Select->getFalseOperand(), Legal_Reg | Legal_Imm); 3856e398428c3ef8ed2339c0f334dd30a7d3f0ed4b6eDavid Sehr } 3857e398428c3ef8ed2339c0f334dd30a7d3f0ed4b6eDavid Sehr Variable *SelectDest = Select->getDest(); 3858e398428c3ef8ed2339c0f334dd30a7d3f0ed4b6eDavid Sehr lowerMove(SelectDest, Src, false); 3859e398428c3ef8ed2339c0f334dd30a7d3f0ed4b6eDavid Sehr return; 3860d981025a26d962d50f7fead809b44d16be9051f3David Sehr } 3861e398428c3ef8ed2339c0f334dd30a7d3f0ed4b6eDavid Sehr llvm::report_fatal_error("Unexpected consumer type"); 38621d235425dab1f3dd059973fc53f1b1d5879469e3John Porto} 38631d235425dab1f3dd059973fc53f1b1d5879469e3John Porto 38644a56686b5b56db6803f90ad53514bf2fa190d9f7John Portotemplate <typename TraitsType> 38654a56686b5b56db6803f90ad53514bf2fa190d9f7John Portovoid TargetX86Base<TraitsType>::lowerArithAndConsumer( 38664a56686b5b56db6803f90ad53514bf2fa190d9f7John Porto const InstArithmetic *Arith, const Inst *Consumer) { 3867daf096cd1b8b9ec3d29c8d7823c3d60cd51c65c9David Sehr Variable *T = nullptr; 3868daf096cd1b8b9ec3d29c8d7823c3d60cd51c65c9David Sehr Operand *Src0 = legalize(Arith->getSrc(0)); 3869daf096cd1b8b9ec3d29c8d7823c3d60cd51c65c9David Sehr Operand *Src1 = legalize(Arith->getSrc(1)); 3870daf096cd1b8b9ec3d29c8d7823c3d60cd51c65c9David Sehr Variable *Dest = Arith->getDest(); 3871daf096cd1b8b9ec3d29c8d7823c3d60cd51c65c9David Sehr switch (Arith->getOp()) { 3872daf096cd1b8b9ec3d29c8d7823c3d60cd51c65c9David Sehr default: 3873daf096cd1b8b9ec3d29c8d7823c3d60cd51c65c9David Sehr llvm_unreachable("arithmetic operator not AND or OR"); 3874daf096cd1b8b9ec3d29c8d7823c3d60cd51c65c9David Sehr break; 3875daf096cd1b8b9ec3d29c8d7823c3d60cd51c65c9David Sehr case InstArithmetic::And: 3876daf096cd1b8b9ec3d29c8d7823c3d60cd51c65c9David Sehr _mov(T, Src0); 3877daf096cd1b8b9ec3d29c8d7823c3d60cd51c65c9David Sehr // Test cannot have an address in the second position. Since T is 3878daf096cd1b8b9ec3d29c8d7823c3d60cd51c65c9David Sehr // guaranteed to be a register and Src1 could be a memory load, ensure 3879daf096cd1b8b9ec3d29c8d7823c3d60cd51c65c9David Sehr // that the second argument is a register. 3880daf096cd1b8b9ec3d29c8d7823c3d60cd51c65c9David Sehr if (llvm::isa<Constant>(Src1)) 3881daf096cd1b8b9ec3d29c8d7823c3d60cd51c65c9David Sehr _test(T, Src1); 3882daf096cd1b8b9ec3d29c8d7823c3d60cd51c65c9David Sehr else 3883daf096cd1b8b9ec3d29c8d7823c3d60cd51c65c9David Sehr _test(Src1, T); 3884daf096cd1b8b9ec3d29c8d7823c3d60cd51c65c9David Sehr break; 3885daf096cd1b8b9ec3d29c8d7823c3d60cd51c65c9David Sehr case InstArithmetic::Or: 3886daf096cd1b8b9ec3d29c8d7823c3d60cd51c65c9David Sehr _mov(T, Src0); 3887daf096cd1b8b9ec3d29c8d7823c3d60cd51c65c9David Sehr _or(T, Src1); 3888daf096cd1b8b9ec3d29c8d7823c3d60cd51c65c9David Sehr break; 3889daf096cd1b8b9ec3d29c8d7823c3d60cd51c65c9David Sehr } 3890e398428c3ef8ed2339c0f334dd30a7d3f0ed4b6eDavid Sehr 3891e398428c3ef8ed2339c0f334dd30a7d3f0ed4b6eDavid Sehr if (Consumer == nullptr) { 3892e398428c3ef8ed2339c0f334dd30a7d3f0ed4b6eDavid Sehr llvm::report_fatal_error("Expected a consumer instruction"); 3893e398428c3ef8ed2339c0f334dd30a7d3f0ed4b6eDavid Sehr } 3894e398428c3ef8ed2339c0f334dd30a7d3f0ed4b6eDavid Sehr if (const auto *Br = llvm::dyn_cast<InstBr>(Consumer)) { 38951d937a8e35b7ae51ee64c32f8f34d51ba33498f4John Porto Context.insert<InstFakeUse>(T); 38961d937a8e35b7ae51ee64c32f8f34d51ba33498f4John Porto Context.insert<InstFakeDef>(Dest); 3897e398428c3ef8ed2339c0f334dd30a7d3f0ed4b6eDavid Sehr _br(Traits::Cond::Br_ne, Br->getTargetTrue(), Br->getTargetFalse()); 3898e398428c3ef8ed2339c0f334dd30a7d3f0ed4b6eDavid Sehr return; 3899e398428c3ef8ed2339c0f334dd30a7d3f0ed4b6eDavid Sehr } 3900e398428c3ef8ed2339c0f334dd30a7d3f0ed4b6eDavid Sehr llvm::report_fatal_error("Unexpected consumer type"); 3901daf096cd1b8b9ec3d29c8d7823c3d60cd51c65c9David Sehr} 3902daf096cd1b8b9ec3d29c8d7823c3d60cd51c65c9David Sehr 39034a56686b5b56db6803f90ad53514bf2fa190d9f7John Portotemplate <typename TraitsType> 39044a56686b5b56db6803f90ad53514bf2fa190d9f7John Portovoid TargetX86Base<TraitsType>::lowerInsertElement( 39058cfeb69e17190d3bfe22a8a1cbd7679a114d68cfJim Stichnoth const InstInsertElement *Instr) { 39068cfeb69e17190d3bfe22a8a1cbd7679a114d68cfJim Stichnoth Operand *SourceVectNotLegalized = Instr->getSrc(0); 39078cfeb69e17190d3bfe22a8a1cbd7679a114d68cfJim Stichnoth Operand *ElementToInsertNotLegalized = Instr->getSrc(1); 39082d6c82679e9786a924b76023aa30e1f44d0f8cbbJim Stichnoth auto *ElementIndex = llvm::dyn_cast<ConstantInteger32>(Instr->getSrc(2)); 39097e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto // Only constant indices are allowed in PNaCl IR. 39107e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto assert(ElementIndex); 39117e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto unsigned Index = ElementIndex->getValue(); 39127e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto assert(Index < typeNumElements(SourceVectNotLegalized->getType())); 39137e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto 39147e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto Type Ty = SourceVectNotLegalized->getType(); 39157e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto Type ElementTy = typeElementType(Ty); 39167e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto Type InVectorElementTy = Traits::getInVectorElementType(Ty); 39177e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto 39187e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto if (ElementTy == IceType_i1) { 391957e126899b20c65ff3ea23a3b7d7a67ab30b99dcAndrew Scull // Expand the element to the appropriate size for it to be inserted in the 392057e126899b20c65ff3ea23a3b7d7a67ab30b99dcAndrew Scull // vector. 39215aeed955e17bac8cc44cc6d2b6ff7513cc714c2fJohn Porto Variable *Expanded = Func->makeVariable(InVectorElementTy); 392254f3d518805b04b3f2958b7906a7d07dedb4e1aeJim Stichnoth auto *Cast = InstCast::create(Func, InstCast::Zext, Expanded, 392354f3d518805b04b3f2958b7906a7d07dedb4e1aeJim Stichnoth ElementToInsertNotLegalized); 39247e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto lowerCast(Cast); 39257e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto ElementToInsertNotLegalized = Expanded; 39267e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto } 39277e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto 39287e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto if (Ty == IceType_v8i16 || Ty == IceType_v8i1 || 39295d0acff3a2fa421923392aadb4df2742064b6248John Porto InstructionSet >= Traits::SSE4_1) { 39307e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto // Use insertps, pinsrb, pinsrw, or pinsrd. 39317e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto Operand *ElementRM = 39327e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto legalize(ElementToInsertNotLegalized, Legal_Reg | Legal_Mem); 39337e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto Operand *SourceVectRM = 39347e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto legalize(SourceVectNotLegalized, Legal_Reg | Legal_Mem); 39357e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto Variable *T = makeReg(Ty); 39367e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto _movp(T, SourceVectRM); 3937c59288b334b91f4c0b2edf0de7415c68c760aa12Jim Stichnoth if (Ty == IceType_v4f32) { 39387e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto _insertps(T, ElementRM, Ctx->getConstantInt32(Index << 4)); 3939c59288b334b91f4c0b2edf0de7415c68c760aa12Jim Stichnoth } else { 3940c59288b334b91f4c0b2edf0de7415c68c760aa12Jim Stichnoth // For the pinsrb and pinsrw instructions, when the source operand is a 3941c59288b334b91f4c0b2edf0de7415c68c760aa12Jim Stichnoth // register, it must be a full r32 register like eax, and not ax/al/ah. 39424a56686b5b56db6803f90ad53514bf2fa190d9f7John Porto // For filetype=asm, InstX86Pinsr<TraitsType>::emit() compensates for 39434a56686b5b56db6803f90ad53514bf2fa190d9f7John Porto // the use 3944c59288b334b91f4c0b2edf0de7415c68c760aa12Jim Stichnoth // of r16 and r8 by converting them through getBaseReg(), while emitIAS() 3945c59288b334b91f4c0b2edf0de7415c68c760aa12Jim Stichnoth // validates that the original and base register encodings are the same. 3946c59288b334b91f4c0b2edf0de7415c68c760aa12Jim Stichnoth if (ElementRM->getType() == IceType_i8 && 3947c59288b334b91f4c0b2edf0de7415c68c760aa12Jim Stichnoth llvm::isa<Variable>(ElementRM)) { 3948c59288b334b91f4c0b2edf0de7415c68c760aa12Jim Stichnoth // Don't use ah/bh/ch/dh for pinsrb. 3949c59288b334b91f4c0b2edf0de7415c68c760aa12Jim Stichnoth ElementRM = copyToReg8(ElementRM); 3950c59288b334b91f4c0b2edf0de7415c68c760aa12Jim Stichnoth } 39517e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto _pinsr(T, ElementRM, Ctx->getConstantInt32(Index)); 3952c59288b334b91f4c0b2edf0de7415c68c760aa12Jim Stichnoth } 39538cfeb69e17190d3bfe22a8a1cbd7679a114d68cfJim Stichnoth _movp(Instr->getDest(), T); 39547e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto } else if (Ty == IceType_v4i32 || Ty == IceType_v4f32 || Ty == IceType_v4i1) { 39557e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto // Use shufps or movss. 39567e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto Variable *ElementR = nullptr; 39577e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto Operand *SourceVectRM = 39587e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto legalize(SourceVectNotLegalized, Legal_Reg | Legal_Mem); 39597e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto 39607e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto if (InVectorElementTy == IceType_f32) { 39617e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto // ElementR will be in an XMM register since it is floating point. 396297f460dcd848dcef3455ed553c1b9e107d0e6ae9Andrew Scull ElementR = legalizeToReg(ElementToInsertNotLegalized); 39637e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto } else { 39647e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto // Copy an integer to an XMM register. 39657e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto Operand *T = legalize(ElementToInsertNotLegalized, Legal_Reg | Legal_Mem); 39667e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto ElementR = makeReg(Ty); 39677e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto _movd(ElementR, T); 39687e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto } 39697e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto 39707e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto if (Index == 0) { 39717e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto Variable *T = makeReg(Ty); 39727e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto _movp(T, SourceVectRM); 39737e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto _movss(T, ElementR); 39748cfeb69e17190d3bfe22a8a1cbd7679a114d68cfJim Stichnoth _movp(Instr->getDest(), T); 39757e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto return; 39767e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto } 39777e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto 397857e126899b20c65ff3ea23a3b7d7a67ab30b99dcAndrew Scull // shufps treats the source and destination operands as vectors of four 397957e126899b20c65ff3ea23a3b7d7a67ab30b99dcAndrew Scull // doublewords. The destination's two high doublewords are selected from 398057e126899b20c65ff3ea23a3b7d7a67ab30b99dcAndrew Scull // the source operand and the two low doublewords are selected from the 398157e126899b20c65ff3ea23a3b7d7a67ab30b99dcAndrew Scull // (original value of) the destination operand. An insertelement operation 398257e126899b20c65ff3ea23a3b7d7a67ab30b99dcAndrew Scull // can be effected with a sequence of two shufps operations with 398357e126899b20c65ff3ea23a3b7d7a67ab30b99dcAndrew Scull // appropriate masks. In all cases below, Element[0] is being inserted into 398457e126899b20c65ff3ea23a3b7d7a67ab30b99dcAndrew Scull // SourceVectOperand. Indices are ordered from left to right. 39857e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto // 39867e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto // insertelement into index 1 (result is stored in ElementR): 39877e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto // ElementR := ElementR[0, 0] SourceVectRM[0, 0] 39887e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto // ElementR := ElementR[3, 0] SourceVectRM[2, 3] 39897e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto // 39907e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto // insertelement into index 2 (result is stored in T): 39917e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto // T := SourceVectRM 39927e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto // ElementR := ElementR[0, 0] T[0, 3] 39937e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto // T := T[0, 1] ElementR[0, 3] 39947e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto // 39957e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto // insertelement into index 3 (result is stored in T): 39967e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto // T := SourceVectRM 39977e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto // ElementR := ElementR[0, 0] T[0, 2] 39987e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto // T := T[0, 1] ElementR[3, 0] 39997e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto const unsigned char Mask1[3] = {0, 192, 128}; 40007e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto const unsigned char Mask2[3] = {227, 196, 52}; 40017e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto 40027e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto Constant *Mask1Constant = Ctx->getConstantInt32(Mask1[Index - 1]); 40037e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto Constant *Mask2Constant = Ctx->getConstantInt32(Mask2[Index - 1]); 40047e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto 40057e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto if (Index == 1) { 40067e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto _shufps(ElementR, SourceVectRM, Mask1Constant); 40077e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto _shufps(ElementR, SourceVectRM, Mask2Constant); 40088cfeb69e17190d3bfe22a8a1cbd7679a114d68cfJim Stichnoth _movp(Instr->getDest(), ElementR); 40097e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto } else { 40107e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto Variable *T = makeReg(Ty); 40117e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto _movp(T, SourceVectRM); 40127e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto _shufps(ElementR, T, Mask1Constant); 40137e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto _shufps(T, ElementR, Mask2Constant); 40148cfeb69e17190d3bfe22a8a1cbd7679a114d68cfJim Stichnoth _movp(Instr->getDest(), T); 40157e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto } 40167e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto } else { 40177e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto assert(Ty == IceType_v16i8 || Ty == IceType_v16i1); 401857e126899b20c65ff3ea23a3b7d7a67ab30b99dcAndrew Scull // Spill the value to a stack slot and perform the insertion in memory. 40197e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto // 402057e126899b20c65ff3ea23a3b7d7a67ab30b99dcAndrew Scull // TODO(wala): use legalize(SourceVectNotLegalized, Legal_Mem) when support 402157e126899b20c65ff3ea23a3b7d7a67ab30b99dcAndrew Scull // for legalizing to mem is implemented. 40225aeed955e17bac8cc44cc6d2b6ff7513cc714c2fJohn Porto Variable *Slot = Func->makeVariable(Ty); 402311c9a325399b282cb4ea7d1d24d42fceeec2a09aAndrew Scull Slot->setMustNotHaveReg(); 402497f460dcd848dcef3455ed553c1b9e107d0e6ae9Andrew Scull _movp(Slot, legalizeToReg(SourceVectNotLegalized)); 40257e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto 40267e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto // Compute the location of the position to insert in memory. 40277e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto unsigned Offset = Index * typeWidthInBytes(InVectorElementTy); 40284a56686b5b56db6803f90ad53514bf2fa190d9f7John Porto X86OperandMem *Loc = 40297e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto getMemoryOperandForStackSlot(InVectorElementTy, Slot, Offset); 403097f460dcd848dcef3455ed553c1b9e107d0e6ae9Andrew Scull _store(legalizeToReg(ElementToInsertNotLegalized), Loc); 40317e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto 40327e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto Variable *T = makeReg(Ty); 40337e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto _movp(T, Slot); 40348cfeb69e17190d3bfe22a8a1cbd7679a114d68cfJim Stichnoth _movp(Instr->getDest(), T); 40357e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto } 40367e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto} 40377e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto 40384a56686b5b56db6803f90ad53514bf2fa190d9f7John Portotemplate <typename TraitsType> 40394a56686b5b56db6803f90ad53514bf2fa190d9f7John Portovoid TargetX86Base<TraitsType>::lowerIntrinsicCall( 40407e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto const InstIntrinsicCall *Instr) { 40417e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto switch (Intrinsics::IntrinsicID ID = Instr->getIntrinsicInfo().ID) { 40427e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto case Intrinsics::AtomicCmpxchg: { 40437e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto if (!Intrinsics::isMemoryOrderValid( 40447e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto ID, getConstantMemoryOrder(Instr->getArg(3)), 40457e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto getConstantMemoryOrder(Instr->getArg(4)))) { 40467e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto Func->setError("Unexpected memory ordering for AtomicCmpxchg"); 40477e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto return; 40487e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto } 40497e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto Variable *DestPrev = Instr->getDest(); 4050fbdd244013aad0808220386eb6bd17eb0635dd4cJan Voung Operand *PtrToMem = legalize(Instr->getArg(0)); 4051fbdd244013aad0808220386eb6bd17eb0635dd4cJan Voung Operand *Expected = legalize(Instr->getArg(1)); 4052fbdd244013aad0808220386eb6bd17eb0635dd4cJan Voung Operand *Desired = legalize(Instr->getArg(2)); 40537e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto if (tryOptimizedCmpxchgCmpBr(DestPrev, PtrToMem, Expected, Desired)) 40547e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto return; 40557e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto lowerAtomicCmpxchg(DestPrev, PtrToMem, Expected, Desired); 40567e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto return; 40577e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto } 40587e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto case Intrinsics::AtomicFence: 40597e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto if (!Intrinsics::isMemoryOrderValid( 40607e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto ID, getConstantMemoryOrder(Instr->getArg(0)))) { 40617e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto Func->setError("Unexpected memory ordering for AtomicFence"); 40627e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto return; 40637e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto } 40647e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto _mfence(); 40657e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto return; 40667e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto case Intrinsics::AtomicFenceAll: 406757e126899b20c65ff3ea23a3b7d7a67ab30b99dcAndrew Scull // NOTE: FenceAll should prevent and load/store from being moved across the 406857e126899b20c65ff3ea23a3b7d7a67ab30b99dcAndrew Scull // fence (both atomic and non-atomic). The InstX8632Mfence instruction is 406957e126899b20c65ff3ea23a3b7d7a67ab30b99dcAndrew Scull // currently marked coarsely as "HasSideEffects". 40707e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto _mfence(); 40717e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto return; 40727e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto case Intrinsics::AtomicIsLockFree: { 40737e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto // X86 is always lock free for 8/16/32/64 bit accesses. 407457e126899b20c65ff3ea23a3b7d7a67ab30b99dcAndrew Scull // TODO(jvoung): Since the result is constant when given a constant byte 407557e126899b20c65ff3ea23a3b7d7a67ab30b99dcAndrew Scull // size, this opens up DCE opportunities. 40767e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto Operand *ByteSize = Instr->getArg(0); 40777e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto Variable *Dest = Instr->getDest(); 407854f3d518805b04b3f2958b7906a7d07dedb4e1aeJim Stichnoth if (auto *CI = llvm::dyn_cast<ConstantInteger32>(ByteSize)) { 40797e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto Constant *Result; 40807e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto switch (CI->getValue()) { 40817e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto default: 408257e126899b20c65ff3ea23a3b7d7a67ab30b99dcAndrew Scull // Some x86-64 processors support the cmpxchg16b instruction, which can 408357e126899b20c65ff3ea23a3b7d7a67ab30b99dcAndrew Scull // make 16-byte operations lock free (when used with the LOCK prefix). 408457e126899b20c65ff3ea23a3b7d7a67ab30b99dcAndrew Scull // However, that's not supported in 32-bit mode, so just return 0 even 408557e126899b20c65ff3ea23a3b7d7a67ab30b99dcAndrew Scull // for large sizes. 40867e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto Result = Ctx->getConstantZero(IceType_i32); 40877e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto break; 40887e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto case 1: 40897e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto case 2: 40907e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto case 4: 40917e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto case 8: 40927e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto Result = Ctx->getConstantInt32(1); 40937e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto break; 40947e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto } 40957e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto _mov(Dest, Result); 40967e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto return; 40977e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto } 40987e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto // The PNaCl ABI requires the byte size to be a compile-time constant. 40997e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto Func->setError("AtomicIsLockFree byte size should be compile-time const"); 41007e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto return; 41017e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto } 41027e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto case Intrinsics::AtomicLoad: { 410357e126899b20c65ff3ea23a3b7d7a67ab30b99dcAndrew Scull // We require the memory address to be naturally aligned. Given that is the 410457e126899b20c65ff3ea23a3b7d7a67ab30b99dcAndrew Scull // case, then normal loads are atomic. 41057e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto if (!Intrinsics::isMemoryOrderValid( 41067e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto ID, getConstantMemoryOrder(Instr->getArg(1)))) { 41077e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto Func->setError("Unexpected memory ordering for AtomicLoad"); 41087e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto return; 41097e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto } 41107e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto Variable *Dest = Instr->getDest(); 41116d47bcdcf16ce8eddc737d4f26baad9653604048Andrew Scull if (!Traits::Is64Bit) { 41126d47bcdcf16ce8eddc737d4f26baad9653604048Andrew Scull if (auto *Dest64On32 = llvm::dyn_cast<Variable64On32>(Dest)) { 41136d47bcdcf16ce8eddc737d4f26baad9653604048Andrew Scull // Follow what GCC does and use a movq instead of what lowerLoad() 41146d47bcdcf16ce8eddc737d4f26baad9653604048Andrew Scull // normally does (split the load into two). Thus, this skips 41156d47bcdcf16ce8eddc737d4f26baad9653604048Andrew Scull // load/arithmetic op folding. Load/arithmetic folding can't happen 41166d47bcdcf16ce8eddc737d4f26baad9653604048Andrew Scull // anyway, since this is x86-32 and integer arithmetic only happens on 41176d47bcdcf16ce8eddc737d4f26baad9653604048Andrew Scull // 32-bit quantities. 41186d47bcdcf16ce8eddc737d4f26baad9653604048Andrew Scull Variable *T = makeReg(IceType_f64); 41194a56686b5b56db6803f90ad53514bf2fa190d9f7John Porto X86OperandMem *Addr = formMemoryOperand(Instr->getArg(0), IceType_f64); 41206d47bcdcf16ce8eddc737d4f26baad9653604048Andrew Scull _movq(T, Addr); 41216d47bcdcf16ce8eddc737d4f26baad9653604048Andrew Scull // Then cast the bits back out of the XMM register to the i64 Dest. 412254f3d518805b04b3f2958b7906a7d07dedb4e1aeJim Stichnoth auto *Cast = InstCast::create(Func, InstCast::Bitcast, Dest, T); 41236d47bcdcf16ce8eddc737d4f26baad9653604048Andrew Scull lowerCast(Cast); 41246d47bcdcf16ce8eddc737d4f26baad9653604048Andrew Scull // Make sure that the atomic load isn't elided when unused. 41251d937a8e35b7ae51ee64c32f8f34d51ba33498f4John Porto Context.insert<InstFakeUse>(Dest64On32->getLo()); 41261d937a8e35b7ae51ee64c32f8f34d51ba33498f4John Porto Context.insert<InstFakeUse>(Dest64On32->getHi()); 41276d47bcdcf16ce8eddc737d4f26baad9653604048Andrew Scull return; 41286d47bcdcf16ce8eddc737d4f26baad9653604048Andrew Scull } 41297e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto } 413054f3d518805b04b3f2958b7906a7d07dedb4e1aeJim Stichnoth auto *Load = InstLoad::create(Func, Dest, Instr->getArg(0)); 41317e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto lowerLoad(Load); 41327e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto // Make sure the atomic load isn't elided when unused, by adding a FakeUse. 413357e126899b20c65ff3ea23a3b7d7a67ab30b99dcAndrew Scull // Since lowerLoad may fuse the load w/ an arithmetic instruction, insert 413457e126899b20c65ff3ea23a3b7d7a67ab30b99dcAndrew Scull // the FakeUse on the last-inserted instruction's dest. 41351d937a8e35b7ae51ee64c32f8f34d51ba33498f4John Porto Context.insert<InstFakeUse>(Context.getLastInserted()->getDest()); 41367e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto return; 41377e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto } 41387e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto case Intrinsics::AtomicRMW: 41397e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto if (!Intrinsics::isMemoryOrderValid( 41407e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto ID, getConstantMemoryOrder(Instr->getArg(3)))) { 41417e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto Func->setError("Unexpected memory ordering for AtomicRMW"); 41427e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto return; 41437e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto } 414420b71f5890ee8651983b126c5978594a01e0af96Jim Stichnoth lowerAtomicRMW( 414520b71f5890ee8651983b126c5978594a01e0af96Jim Stichnoth Instr->getDest(), 414620b71f5890ee8651983b126c5978594a01e0af96Jim Stichnoth static_cast<uint32_t>( 414720b71f5890ee8651983b126c5978594a01e0af96Jim Stichnoth llvm::cast<ConstantInteger32>(Instr->getArg(0))->getValue()), 414820b71f5890ee8651983b126c5978594a01e0af96Jim Stichnoth Instr->getArg(1), Instr->getArg(2)); 41497e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto return; 41507e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto case Intrinsics::AtomicStore: { 41517e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto if (!Intrinsics::isMemoryOrderValid( 41527e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto ID, getConstantMemoryOrder(Instr->getArg(2)))) { 41537e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto Func->setError("Unexpected memory ordering for AtomicStore"); 41547e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto return; 41557e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto } 415657e126899b20c65ff3ea23a3b7d7a67ab30b99dcAndrew Scull // We require the memory address to be naturally aligned. Given that is the 415757e126899b20c65ff3ea23a3b7d7a67ab30b99dcAndrew Scull // case, then normal stores are atomic. Add a fence after the store to make 415857e126899b20c65ff3ea23a3b7d7a67ab30b99dcAndrew Scull // it visible. 41597e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto Operand *Value = Instr->getArg(0); 41607e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto Operand *Ptr = Instr->getArg(1); 41611d235425dab1f3dd059973fc53f1b1d5879469e3John Porto if (!Traits::Is64Bit && Value->getType() == IceType_i64) { 416257e126899b20c65ff3ea23a3b7d7a67ab30b99dcAndrew Scull // Use a movq instead of what lowerStore() normally does (split the store 416357e126899b20c65ff3ea23a3b7d7a67ab30b99dcAndrew Scull // into two), following what GCC does. Cast the bits from int -> to an 416457e126899b20c65ff3ea23a3b7d7a67ab30b99dcAndrew Scull // xmm register first. 41657e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto Variable *T = makeReg(IceType_f64); 416654f3d518805b04b3f2958b7906a7d07dedb4e1aeJim Stichnoth auto *Cast = InstCast::create(Func, InstCast::Bitcast, T, Value); 41677e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto lowerCast(Cast); 41687e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto // Then store XMM w/ a movq. 41694a56686b5b56db6803f90ad53514bf2fa190d9f7John Porto X86OperandMem *Addr = formMemoryOperand(Ptr, IceType_f64); 41707e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto _storeq(T, Addr); 41717e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto _mfence(); 41727e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto return; 41737e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto } 417454f3d518805b04b3f2958b7906a7d07dedb4e1aeJim Stichnoth auto *Store = InstStore::create(Func, Value, Ptr); 41757e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto lowerStore(Store); 41767e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto _mfence(); 41777e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto return; 41787e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto } 41797e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto case Intrinsics::Bswap: { 41807e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto Variable *Dest = Instr->getDest(); 41817e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto Operand *Val = Instr->getArg(0); 418257e126899b20c65ff3ea23a3b7d7a67ab30b99dcAndrew Scull // In 32-bit mode, bswap only works on 32-bit arguments, and the argument 418357e126899b20c65ff3ea23a3b7d7a67ab30b99dcAndrew Scull // must be a register. Use rotate left for 16-bit bswap. 41841d235425dab1f3dd059973fc53f1b1d5879469e3John Porto if (!Traits::Is64Bit && Val->getType() == IceType_i64) { 4185fbdd244013aad0808220386eb6bd17eb0635dd4cJan Voung Val = legalizeUndef(Val); 418697f460dcd848dcef3455ed553c1b9e107d0e6ae9Andrew Scull Variable *T_Lo = legalizeToReg(loOperand(Val)); 418797f460dcd848dcef3455ed553c1b9e107d0e6ae9Andrew Scull Variable *T_Hi = legalizeToReg(hiOperand(Val)); 418854f3d518805b04b3f2958b7906a7d07dedb4e1aeJim Stichnoth auto *DestLo = llvm::cast<Variable>(loOperand(Dest)); 418954f3d518805b04b3f2958b7906a7d07dedb4e1aeJim Stichnoth auto *DestHi = llvm::cast<Variable>(hiOperand(Dest)); 41907e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto _bswap(T_Lo); 41917e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto _bswap(T_Hi); 41927e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto _mov(DestLo, T_Hi); 41937e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto _mov(DestHi, T_Lo); 41941d235425dab1f3dd059973fc53f1b1d5879469e3John Porto } else if ((Traits::Is64Bit && Val->getType() == IceType_i64) || 41951d235425dab1f3dd059973fc53f1b1d5879469e3John Porto Val->getType() == IceType_i32) { 419697f460dcd848dcef3455ed553c1b9e107d0e6ae9Andrew Scull Variable *T = legalizeToReg(Val); 41977e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto _bswap(T); 41987e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto _mov(Dest, T); 41997e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto } else { 42007e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto assert(Val->getType() == IceType_i16); 42017e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto Constant *Eight = Ctx->getConstantInt16(8); 42027e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto Variable *T = nullptr; 4203fbdd244013aad0808220386eb6bd17eb0635dd4cJan Voung Val = legalize(Val); 42047e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto _mov(T, Val); 42057e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto _rol(T, Eight); 42067e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto _mov(Dest, T); 42077e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto } 42087e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto return; 42097e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto } 42107e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto case Intrinsics::Ctpop: { 42117e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto Variable *Dest = Instr->getDest(); 42121d235425dab1f3dd059973fc53f1b1d5879469e3John Porto Variable *T = nullptr; 42137e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto Operand *Val = Instr->getArg(0); 42141d235425dab1f3dd059973fc53f1b1d5879469e3John Porto Type ValTy = Val->getType(); 42151d235425dab1f3dd059973fc53f1b1d5879469e3John Porto assert(ValTy == IceType_i32 || ValTy == IceType_i64); 42161d235425dab1f3dd059973fc53f1b1d5879469e3John Porto 42171d235425dab1f3dd059973fc53f1b1d5879469e3John Porto if (!Traits::Is64Bit) { 42181d235425dab1f3dd059973fc53f1b1d5879469e3John Porto T = Dest; 42191d235425dab1f3dd059973fc53f1b1d5879469e3John Porto } else { 42201d235425dab1f3dd059973fc53f1b1d5879469e3John Porto T = makeReg(IceType_i64); 42211d235425dab1f3dd059973fc53f1b1d5879469e3John Porto if (ValTy == IceType_i32) { 42221d235425dab1f3dd059973fc53f1b1d5879469e3John Porto // in x86-64, __popcountsi2 is not defined, so we cheat a bit by 42231d235425dab1f3dd059973fc53f1b1d5879469e3John Porto // converting it to a 64-bit value, and using ctpop_i64. _movzx should 42241d235425dab1f3dd059973fc53f1b1d5879469e3John Porto // ensure we will not have any bits set on Val's upper 32 bits. 42251d235425dab1f3dd059973fc53f1b1d5879469e3John Porto Variable *V = makeReg(IceType_i64); 4226373913fa0a4c7c4dd23ada9e6b4e1780fdea0096Jim Stichnoth Operand *ValRM = legalize(Val, Legal_Reg | Legal_Mem); 4227373913fa0a4c7c4dd23ada9e6b4e1780fdea0096Jim Stichnoth _movzx(V, ValRM); 42281d235425dab1f3dd059973fc53f1b1d5879469e3John Porto Val = V; 42291d235425dab1f3dd059973fc53f1b1d5879469e3John Porto } 42301d235425dab1f3dd059973fc53f1b1d5879469e3John Porto ValTy = IceType_i64; 42311d235425dab1f3dd059973fc53f1b1d5879469e3John Porto } 42321d235425dab1f3dd059973fc53f1b1d5879469e3John Porto 423320070e84c21b9aa15a83e53b5d2ffa692c4e081aKarl Schimpf InstCall *Call = 423420070e84c21b9aa15a83e53b5d2ffa692c4e081aKarl Schimpf makeHelperCall(ValTy == IceType_i32 ? RuntimeHelper::H_call_ctpop_i32 423520070e84c21b9aa15a83e53b5d2ffa692c4e081aKarl Schimpf : RuntimeHelper::H_call_ctpop_i64, 423620070e84c21b9aa15a83e53b5d2ffa692c4e081aKarl Schimpf T, 1); 42377e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto Call->addArg(Val); 42387e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto lowerCall(Call); 42397e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto // The popcount helpers always return 32-bit values, while the intrinsic's 42407e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto // signature matches the native POPCNT instruction and fills a 64-bit reg 42417e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto // (in 64-bit mode). Thus, clear the upper bits of the dest just in case 42427e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto // the user doesn't do that in the IR. If the user does that in the IR, 42437e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto // then this zero'ing instruction is dead and gets optimized out. 42441d235425dab1f3dd059973fc53f1b1d5879469e3John Porto if (!Traits::Is64Bit) { 42451d235425dab1f3dd059973fc53f1b1d5879469e3John Porto assert(T == Dest); 42461d235425dab1f3dd059973fc53f1b1d5879469e3John Porto if (Val->getType() == IceType_i64) { 424754f3d518805b04b3f2958b7906a7d07dedb4e1aeJim Stichnoth auto *DestHi = llvm::cast<Variable>(hiOperand(Dest)); 42481d235425dab1f3dd059973fc53f1b1d5879469e3John Porto Constant *Zero = Ctx->getConstantZero(IceType_i32); 42491d235425dab1f3dd059973fc53f1b1d5879469e3John Porto _mov(DestHi, Zero); 42501d235425dab1f3dd059973fc53f1b1d5879469e3John Porto } 42511d235425dab1f3dd059973fc53f1b1d5879469e3John Porto } else { 42521d235425dab1f3dd059973fc53f1b1d5879469e3John Porto assert(Val->getType() == IceType_i64); 42531d235425dab1f3dd059973fc53f1b1d5879469e3John Porto // T is 64 bit. It needs to be copied to dest. We need to: 42541d235425dab1f3dd059973fc53f1b1d5879469e3John Porto // 42551d235425dab1f3dd059973fc53f1b1d5879469e3John Porto // T_1.32 = trunc T.64 to i32 42561d235425dab1f3dd059973fc53f1b1d5879469e3John Porto // T_2.64 = zext T_1.32 to i64 42571d235425dab1f3dd059973fc53f1b1d5879469e3John Porto // Dest.<<right_size>> = T_2.<<right_size>> 42581d235425dab1f3dd059973fc53f1b1d5879469e3John Porto // 42591d235425dab1f3dd059973fc53f1b1d5879469e3John Porto // which ensures the upper 32 bits will always be cleared. Just doing a 42601d235425dab1f3dd059973fc53f1b1d5879469e3John Porto // 42611d235425dab1f3dd059973fc53f1b1d5879469e3John Porto // mov Dest.32 = trunc T.32 to i32 42621d235425dab1f3dd059973fc53f1b1d5879469e3John Porto // 42631d235425dab1f3dd059973fc53f1b1d5879469e3John Porto // is dangerous because there's a chance the compiler will optimize this 42641d235425dab1f3dd059973fc53f1b1d5879469e3John Porto // copy out. To use _movzx we need two new registers (one 32-, and 42651d235425dab1f3dd059973fc53f1b1d5879469e3John Porto // another 64-bit wide.) 42661d235425dab1f3dd059973fc53f1b1d5879469e3John Porto Variable *T_1 = makeReg(IceType_i32); 42671d235425dab1f3dd059973fc53f1b1d5879469e3John Porto _mov(T_1, T); 42681d235425dab1f3dd059973fc53f1b1d5879469e3John Porto Variable *T_2 = makeReg(IceType_i64); 42691d235425dab1f3dd059973fc53f1b1d5879469e3John Porto _movzx(T_2, T_1); 42701d235425dab1f3dd059973fc53f1b1d5879469e3John Porto _mov(Dest, T_2); 42717e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto } 42727e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto return; 42737e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto } 42747e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto case Intrinsics::Ctlz: { 427557e126899b20c65ff3ea23a3b7d7a67ab30b99dcAndrew Scull // The "is zero undef" parameter is ignored and we always return a 427657e126899b20c65ff3ea23a3b7d7a67ab30b99dcAndrew Scull // well-defined value. 42777e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto Operand *Val = legalize(Instr->getArg(0)); 42787e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto Operand *FirstVal; 42797e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto Operand *SecondVal = nullptr; 42801d235425dab1f3dd059973fc53f1b1d5879469e3John Porto if (!Traits::Is64Bit && Val->getType() == IceType_i64) { 42817e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto FirstVal = loOperand(Val); 42827e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto SecondVal = hiOperand(Val); 42837e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto } else { 42847e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto FirstVal = Val; 42857e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto } 42865bff61c44841990680781892036adb17b3cff0c4Jim Stichnoth constexpr bool IsCttz = false; 42877e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto lowerCountZeros(IsCttz, Val->getType(), Instr->getDest(), FirstVal, 42887e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto SecondVal); 42897e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto return; 42907e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto } 42917e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto case Intrinsics::Cttz: { 429257e126899b20c65ff3ea23a3b7d7a67ab30b99dcAndrew Scull // The "is zero undef" parameter is ignored and we always return a 429357e126899b20c65ff3ea23a3b7d7a67ab30b99dcAndrew Scull // well-defined value. 42947e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto Operand *Val = legalize(Instr->getArg(0)); 42957e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto Operand *FirstVal; 42967e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto Operand *SecondVal = nullptr; 42971d235425dab1f3dd059973fc53f1b1d5879469e3John Porto if (!Traits::Is64Bit && Val->getType() == IceType_i64) { 42987e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto FirstVal = hiOperand(Val); 42997e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto SecondVal = loOperand(Val); 43007e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto } else { 43017e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto FirstVal = Val; 43027e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto } 43035bff61c44841990680781892036adb17b3cff0c4Jim Stichnoth constexpr bool IsCttz = true; 43047e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto lowerCountZeros(IsCttz, Val->getType(), Instr->getDest(), FirstVal, 43057e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto SecondVal); 43067e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto return; 43077e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto } 43087e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto case Intrinsics::Fabs: { 43097e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto Operand *Src = legalize(Instr->getArg(0)); 43107e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto Type Ty = Src->getType(); 43117e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto Variable *Dest = Instr->getDest(); 43127e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto Variable *T = makeVectorOfFabsMask(Ty); 431357e126899b20c65ff3ea23a3b7d7a67ab30b99dcAndrew Scull // The pand instruction operates on an m128 memory operand, so if Src is an 431457e126899b20c65ff3ea23a3b7d7a67ab30b99dcAndrew Scull // f32 or f64, we need to make sure it's in a register. 43157e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto if (isVectorType(Ty)) { 43164a56686b5b56db6803f90ad53514bf2fa190d9f7John Porto if (llvm::isa<X86OperandMem>(Src)) 431797f460dcd848dcef3455ed553c1b9e107d0e6ae9Andrew Scull Src = legalizeToReg(Src); 43187e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto } else { 431997f460dcd848dcef3455ed553c1b9e107d0e6ae9Andrew Scull Src = legalizeToReg(Src); 43207e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto } 43217e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto _pand(T, Src); 43227e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto if (isVectorType(Ty)) 43237e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto _movp(Dest, T); 43247e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto else 43257e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto _mov(Dest, T); 43267e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto return; 43277e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto } 43287e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto case Intrinsics::Longjmp: { 432920070e84c21b9aa15a83e53b5d2ffa692c4e081aKarl Schimpf InstCall *Call = makeHelperCall(RuntimeHelper::H_call_longjmp, nullptr, 2); 43307e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto Call->addArg(Instr->getArg(0)); 43317e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto Call->addArg(Instr->getArg(1)); 43327e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto lowerCall(Call); 43337e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto return; 43347e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto } 43357e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto case Intrinsics::Memcpy: { 43369df4a379ebdb2f2200ec7f65017bd9fc8544794bAndrew Scull lowerMemcpy(Instr->getArg(0), Instr->getArg(1), Instr->getArg(2)); 43377e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto return; 43387e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto } 43397e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto case Intrinsics::Memmove: { 4340cfa628b5f7612d202de234980b21bd6d59a11998Andrew Scull lowerMemmove(Instr->getArg(0), Instr->getArg(1), Instr->getArg(2)); 43417e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto return; 43427e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto } 43437e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto case Intrinsics::Memset: { 4344713dbdecbdd74f77cbb1cae43b19bdd6f6b3144fAndrew Scull lowerMemset(Instr->getArg(0), Instr->getArg(1), Instr->getArg(2)); 43457e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto return; 43467e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto } 43477e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto case Intrinsics::NaClReadTP: { 434856958cb33d3c1d045f2844408d825442d523f59fJohn Porto if (NeedSandboxing) { 43494a56686b5b56db6803f90ad53514bf2fa190d9f7John Porto Operand *Src = 43504a56686b5b56db6803f90ad53514bf2fa190d9f7John Porto dispatchToConcrete(&ConcreteTarget::createNaClReadTPSrcOperand); 43517e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto Variable *Dest = Instr->getDest(); 43527e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto Variable *T = nullptr; 43537e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto _mov(T, Src); 43547e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto _mov(Dest, T); 43557e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto } else { 435620070e84c21b9aa15a83e53b5d2ffa692c4e081aKarl Schimpf InstCall *Call = 435720070e84c21b9aa15a83e53b5d2ffa692c4e081aKarl Schimpf makeHelperCall(RuntimeHelper::H_call_read_tp, Instr->getDest(), 0); 43587e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto lowerCall(Call); 43597e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto } 43607e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto return; 43617e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto } 43627e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto case Intrinsics::Setjmp: { 436320070e84c21b9aa15a83e53b5d2ffa692c4e081aKarl Schimpf InstCall *Call = 436420070e84c21b9aa15a83e53b5d2ffa692c4e081aKarl Schimpf makeHelperCall(RuntimeHelper::H_call_setjmp, Instr->getDest(), 1); 43657e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto Call->addArg(Instr->getArg(0)); 43667e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto lowerCall(Call); 43677e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto return; 43687e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto } 43697e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto case Intrinsics::Sqrt: { 4370956cfd60e8ee9efcc374cd19513bbf35a20e52eeNicolas Capens assert(isScalarFloatingType(Instr->getDest()->getType()) || 4371956cfd60e8ee9efcc374cd19513bbf35a20e52eeNicolas Capens getFlags().getApplicationBinaryInterface() != ::Ice::ABI_PNaCl); 43727e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto Operand *Src = legalize(Instr->getArg(0)); 43737e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto Variable *Dest = Instr->getDest(); 43747e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto Variable *T = makeReg(Dest->getType()); 4375956cfd60e8ee9efcc374cd19513bbf35a20e52eeNicolas Capens _sqrt(T, Src); 43767e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto _mov(Dest, T); 43777e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto return; 43787e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto } 43797e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto case Intrinsics::Stacksave: { 438056958cb33d3c1d045f2844408d825442d523f59fJohn Porto if (!Traits::Is64Bit || !NeedSandboxing) { 438156958cb33d3c1d045f2844408d825442d523f59fJohn Porto Variable *esp = Func->getTarget()->getPhysicalRegister(getStackReg(), 438256958cb33d3c1d045f2844408d825442d523f59fJohn Porto Traits::WordType); 438356958cb33d3c1d045f2844408d825442d523f59fJohn Porto Variable *Dest = Instr->getDest(); 438456958cb33d3c1d045f2844408d825442d523f59fJohn Porto _mov(Dest, esp); 438556958cb33d3c1d045f2844408d825442d523f59fJohn Porto return; 438656958cb33d3c1d045f2844408d825442d523f59fJohn Porto } 438756958cb33d3c1d045f2844408d825442d523f59fJohn Porto Variable *esp = Func->getTarget()->getPhysicalRegister( 438856958cb33d3c1d045f2844408d825442d523f59fJohn Porto Traits::RegisterSet::Reg_esp, IceType_i32); 43897e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto Variable *Dest = Instr->getDest(); 43907e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto _mov(Dest, esp); 439156958cb33d3c1d045f2844408d825442d523f59fJohn Porto 43927e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto return; 43937e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto } 43947e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto case Intrinsics::Stackrestore: { 4395008f4ce5417c97891ea02b54b691b39aa2e2a0afJohn Porto Operand *Src = Instr->getArg(0); 439656958cb33d3c1d045f2844408d825442d523f59fJohn Porto _mov_sp(Src); 43977e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto return; 43987e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto } 439956958cb33d3c1d045f2844408d825442d523f59fJohn Porto 44007e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto case Intrinsics::Trap: 44017e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto _ud2(); 44027e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto return; 4403acfb3df0bdd163dc2c5f94f450be2e3c7ec0fff1Nicolas Capens case Intrinsics::LoadSubVector: { 4404ef18fc51dd5c1b2640c291a363cf1e4712142eaaNicolas Capens assert(llvm::isa<ConstantInteger32>(Instr->getArg(1)) && 4405ef18fc51dd5c1b2640c291a363cf1e4712142eaaNicolas Capens "LoadSubVector second argument must be a constant"); 4406acfb3df0bdd163dc2c5f94f450be2e3c7ec0fff1Nicolas Capens Variable *Dest = Instr->getDest(); 4407acfb3df0bdd163dc2c5f94f450be2e3c7ec0fff1Nicolas Capens Type Ty = Dest->getType(); 4408ef18fc51dd5c1b2640c291a363cf1e4712142eaaNicolas Capens auto *SubVectorSize = llvm::cast<ConstantInteger32>(Instr->getArg(1)); 4409ef18fc51dd5c1b2640c291a363cf1e4712142eaaNicolas Capens Operand *Addr = Instr->getArg(0); 4410acfb3df0bdd163dc2c5f94f450be2e3c7ec0fff1Nicolas Capens X86OperandMem *Src = formMemoryOperand(Addr, Ty); 4411acfb3df0bdd163dc2c5f94f450be2e3c7ec0fff1Nicolas Capens doMockBoundsCheck(Src); 4412acfb3df0bdd163dc2c5f94f450be2e3c7ec0fff1Nicolas Capens 4413acfb3df0bdd163dc2c5f94f450be2e3c7ec0fff1Nicolas Capens if (Dest->isRematerializable()) { 4414acfb3df0bdd163dc2c5f94f450be2e3c7ec0fff1Nicolas Capens Context.insert<InstFakeDef>(Dest); 4415acfb3df0bdd163dc2c5f94f450be2e3c7ec0fff1Nicolas Capens return; 4416acfb3df0bdd163dc2c5f94f450be2e3c7ec0fff1Nicolas Capens } 4417acfb3df0bdd163dc2c5f94f450be2e3c7ec0fff1Nicolas Capens 4418a29da906bf910a32de3a81ef43d6ec43f138f2abNicolas Capens auto *T = makeReg(Ty); 4419acfb3df0bdd163dc2c5f94f450be2e3c7ec0fff1Nicolas Capens switch (SubVectorSize->getValue()) { 4420acfb3df0bdd163dc2c5f94f450be2e3c7ec0fff1Nicolas Capens case 4: 4421a29da906bf910a32de3a81ef43d6ec43f138f2abNicolas Capens _movd(T, Src); 4422acfb3df0bdd163dc2c5f94f450be2e3c7ec0fff1Nicolas Capens break; 4423acfb3df0bdd163dc2c5f94f450be2e3c7ec0fff1Nicolas Capens case 8: 4424a29da906bf910a32de3a81ef43d6ec43f138f2abNicolas Capens _movq(T, Src); 4425acfb3df0bdd163dc2c5f94f450be2e3c7ec0fff1Nicolas Capens break; 4426acfb3df0bdd163dc2c5f94f450be2e3c7ec0fff1Nicolas Capens default: 4427acfb3df0bdd163dc2c5f94f450be2e3c7ec0fff1Nicolas Capens Func->setError("Unexpected size for LoadSubVector"); 4428acfb3df0bdd163dc2c5f94f450be2e3c7ec0fff1Nicolas Capens return; 4429acfb3df0bdd163dc2c5f94f450be2e3c7ec0fff1Nicolas Capens } 4430a29da906bf910a32de3a81ef43d6ec43f138f2abNicolas Capens _movp(Dest, T); 4431acfb3df0bdd163dc2c5f94f450be2e3c7ec0fff1Nicolas Capens return; 4432acfb3df0bdd163dc2c5f94f450be2e3c7ec0fff1Nicolas Capens } 4433acfb3df0bdd163dc2c5f94f450be2e3c7ec0fff1Nicolas Capens case Intrinsics::StoreSubVector: { 4434ef18fc51dd5c1b2640c291a363cf1e4712142eaaNicolas Capens assert(llvm::isa<ConstantInteger32>(Instr->getArg(2)) && 4435ef18fc51dd5c1b2640c291a363cf1e4712142eaaNicolas Capens "StoreSubVector third argument must be a constant"); 4436ef18fc51dd5c1b2640c291a363cf1e4712142eaaNicolas Capens auto *SubVectorSize = llvm::cast<ConstantInteger32>(Instr->getArg(2)); 4437ef18fc51dd5c1b2640c291a363cf1e4712142eaaNicolas Capens Operand *Value = Instr->getArg(0); 4438ef18fc51dd5c1b2640c291a363cf1e4712142eaaNicolas Capens Operand *Addr = Instr->getArg(1); 4439acfb3df0bdd163dc2c5f94f450be2e3c7ec0fff1Nicolas Capens X86OperandMem *NewAddr = formMemoryOperand(Addr, Value->getType()); 4440acfb3df0bdd163dc2c5f94f450be2e3c7ec0fff1Nicolas Capens doMockBoundsCheck(NewAddr); 4441acfb3df0bdd163dc2c5f94f450be2e3c7ec0fff1Nicolas Capens 4442acfb3df0bdd163dc2c5f94f450be2e3c7ec0fff1Nicolas Capens Value = legalizeToReg(Value); 4443acfb3df0bdd163dc2c5f94f450be2e3c7ec0fff1Nicolas Capens 4444acfb3df0bdd163dc2c5f94f450be2e3c7ec0fff1Nicolas Capens switch (SubVectorSize->getValue()) { 4445acfb3df0bdd163dc2c5f94f450be2e3c7ec0fff1Nicolas Capens case 4: 4446acfb3df0bdd163dc2c5f94f450be2e3c7ec0fff1Nicolas Capens _stored(Value, NewAddr); 4447acfb3df0bdd163dc2c5f94f450be2e3c7ec0fff1Nicolas Capens break; 4448acfb3df0bdd163dc2c5f94f450be2e3c7ec0fff1Nicolas Capens case 8: 4449acfb3df0bdd163dc2c5f94f450be2e3c7ec0fff1Nicolas Capens _storeq(Value, NewAddr); 4450acfb3df0bdd163dc2c5f94f450be2e3c7ec0fff1Nicolas Capens break; 4451acfb3df0bdd163dc2c5f94f450be2e3c7ec0fff1Nicolas Capens default: 4452acfb3df0bdd163dc2c5f94f450be2e3c7ec0fff1Nicolas Capens Func->setError("Unexpected size for StoreSubVector"); 4453acfb3df0bdd163dc2c5f94f450be2e3c7ec0fff1Nicolas Capens return; 4454acfb3df0bdd163dc2c5f94f450be2e3c7ec0fff1Nicolas Capens } 4455acfb3df0bdd163dc2c5f94f450be2e3c7ec0fff1Nicolas Capens return; 4456acfb3df0bdd163dc2c5f94f450be2e3c7ec0fff1Nicolas Capens } 4457ef8210d9cf35b11d6ce8210a43e127466098826bNicolas Capens case Intrinsics::VectorPackSigned: { 4458ef8210d9cf35b11d6ce8210a43e127466098826bNicolas Capens Operand *Src0 = Instr->getArg(0); 4459ef8210d9cf35b11d6ce8210a43e127466098826bNicolas Capens Operand *Src1 = Instr->getArg(1); 4460ef8210d9cf35b11d6ce8210a43e127466098826bNicolas Capens Variable *Dest = Instr->getDest(); 446161593fb9d46f1b3d98878d4b90dcb07009b93fbcNicolas Capens auto *T = makeReg(Src0->getType()); 4462ef8210d9cf35b11d6ce8210a43e127466098826bNicolas Capens auto *Src0RM = legalize(Src0, Legal_Reg | Legal_Mem); 4463ef8210d9cf35b11d6ce8210a43e127466098826bNicolas Capens auto *Src1RM = legalize(Src1, Legal_Reg | Legal_Mem); 4464ef8210d9cf35b11d6ce8210a43e127466098826bNicolas Capens _movp(T, Src0RM); 4465ef8210d9cf35b11d6ce8210a43e127466098826bNicolas Capens _packss(T, Src1RM); 4466ef8210d9cf35b11d6ce8210a43e127466098826bNicolas Capens _movp(Dest, T); 4467ef8210d9cf35b11d6ce8210a43e127466098826bNicolas Capens return; 4468ef8210d9cf35b11d6ce8210a43e127466098826bNicolas Capens } 4469ef8210d9cf35b11d6ce8210a43e127466098826bNicolas Capens case Intrinsics::VectorPackUnsigned: { 4470ef8210d9cf35b11d6ce8210a43e127466098826bNicolas Capens Operand *Src0 = Instr->getArg(0); 4471ef8210d9cf35b11d6ce8210a43e127466098826bNicolas Capens Operand *Src1 = Instr->getArg(1); 4472ef8210d9cf35b11d6ce8210a43e127466098826bNicolas Capens Variable *Dest = Instr->getDest(); 447361593fb9d46f1b3d98878d4b90dcb07009b93fbcNicolas Capens auto *T = makeReg(Src0->getType()); 4474ef8210d9cf35b11d6ce8210a43e127466098826bNicolas Capens auto *Src0RM = legalize(Src0, Legal_Reg | Legal_Mem); 4475ef8210d9cf35b11d6ce8210a43e127466098826bNicolas Capens auto *Src1RM = legalize(Src1, Legal_Reg | Legal_Mem); 4476ef8210d9cf35b11d6ce8210a43e127466098826bNicolas Capens _movp(T, Src0RM); 4477ef8210d9cf35b11d6ce8210a43e127466098826bNicolas Capens _packus(T, Src1RM); 4478ef8210d9cf35b11d6ce8210a43e127466098826bNicolas Capens _movp(Dest, T); 4479ef8210d9cf35b11d6ce8210a43e127466098826bNicolas Capens return; 4480ef8210d9cf35b11d6ce8210a43e127466098826bNicolas Capens } 4481e3cabdafc42cb8120ebb268378e9facd9a8107bdNicolas Capens case Intrinsics::SignMask: { 4482e3cabdafc42cb8120ebb268378e9facd9a8107bdNicolas Capens Operand *SrcReg = legalizeToReg(Instr->getArg(0)); 4483e3cabdafc42cb8120ebb268378e9facd9a8107bdNicolas Capens Variable *Dest = Instr->getDest(); 4484e3cabdafc42cb8120ebb268378e9facd9a8107bdNicolas Capens Variable *T = makeReg(IceType_i32); 4485e3cabdafc42cb8120ebb268378e9facd9a8107bdNicolas Capens if (SrcReg->getType() == IceType_v4f32 || 4486e3cabdafc42cb8120ebb268378e9facd9a8107bdNicolas Capens SrcReg->getType() == IceType_v4i32 || 4487e3cabdafc42cb8120ebb268378e9facd9a8107bdNicolas Capens SrcReg->getType() == IceType_v16i8) { 4488e3cabdafc42cb8120ebb268378e9facd9a8107bdNicolas Capens _movmsk(T, SrcReg); 4489e3cabdafc42cb8120ebb268378e9facd9a8107bdNicolas Capens } else { 4490e3cabdafc42cb8120ebb268378e9facd9a8107bdNicolas Capens // TODO(capn): We could implement v8i16 sign mask using packsswb/pmovmskb 4491e3cabdafc42cb8120ebb268378e9facd9a8107bdNicolas Capens llvm::report_fatal_error("Invalid type for SignMask intrinsic"); 4492e3cabdafc42cb8120ebb268378e9facd9a8107bdNicolas Capens } 4493e3cabdafc42cb8120ebb268378e9facd9a8107bdNicolas Capens _mov(Dest, T); 4494e3cabdafc42cb8120ebb268378e9facd9a8107bdNicolas Capens return; 4495e3cabdafc42cb8120ebb268378e9facd9a8107bdNicolas Capens } 449613cde0f5d3af36944075ef510c3b05aeacf959b3Nicolas Capens case Intrinsics::MultiplyHighSigned: { 449713cde0f5d3af36944075ef510c3b05aeacf959b3Nicolas Capens Operand *Src0 = Instr->getArg(0); 449813cde0f5d3af36944075ef510c3b05aeacf959b3Nicolas Capens Operand *Src1 = Instr->getArg(1); 449913cde0f5d3af36944075ef510c3b05aeacf959b3Nicolas Capens Variable *Dest = Instr->getDest(); 450013cde0f5d3af36944075ef510c3b05aeacf959b3Nicolas Capens auto *T = makeReg(Dest->getType()); 450113cde0f5d3af36944075ef510c3b05aeacf959b3Nicolas Capens auto *Src0RM = legalize(Src0, Legal_Reg | Legal_Mem); 450213cde0f5d3af36944075ef510c3b05aeacf959b3Nicolas Capens auto *Src1RM = legalize(Src1, Legal_Reg | Legal_Mem); 450313cde0f5d3af36944075ef510c3b05aeacf959b3Nicolas Capens _movp(T, Src0RM); 450413cde0f5d3af36944075ef510c3b05aeacf959b3Nicolas Capens _pmulhw(T, Src1RM); 450513cde0f5d3af36944075ef510c3b05aeacf959b3Nicolas Capens _movp(Dest, T); 450613cde0f5d3af36944075ef510c3b05aeacf959b3Nicolas Capens return; 450713cde0f5d3af36944075ef510c3b05aeacf959b3Nicolas Capens } 450813cde0f5d3af36944075ef510c3b05aeacf959b3Nicolas Capens case Intrinsics::MultiplyHighUnsigned: { 450913cde0f5d3af36944075ef510c3b05aeacf959b3Nicolas Capens Operand *Src0 = Instr->getArg(0); 451013cde0f5d3af36944075ef510c3b05aeacf959b3Nicolas Capens Operand *Src1 = Instr->getArg(1); 451113cde0f5d3af36944075ef510c3b05aeacf959b3Nicolas Capens Variable *Dest = Instr->getDest(); 451213cde0f5d3af36944075ef510c3b05aeacf959b3Nicolas Capens auto *T = makeReg(Dest->getType()); 451313cde0f5d3af36944075ef510c3b05aeacf959b3Nicolas Capens auto *Src0RM = legalize(Src0, Legal_Reg | Legal_Mem); 451413cde0f5d3af36944075ef510c3b05aeacf959b3Nicolas Capens auto *Src1RM = legalize(Src1, Legal_Reg | Legal_Mem); 451513cde0f5d3af36944075ef510c3b05aeacf959b3Nicolas Capens _movp(T, Src0RM); 451613cde0f5d3af36944075ef510c3b05aeacf959b3Nicolas Capens _pmulhuw(T, Src1RM); 451713cde0f5d3af36944075ef510c3b05aeacf959b3Nicolas Capens _movp(Dest, T); 451813cde0f5d3af36944075ef510c3b05aeacf959b3Nicolas Capens return; 451913cde0f5d3af36944075ef510c3b05aeacf959b3Nicolas Capens } 452013cde0f5d3af36944075ef510c3b05aeacf959b3Nicolas Capens case Intrinsics::MultiplyAddPairs: { 452113cde0f5d3af36944075ef510c3b05aeacf959b3Nicolas Capens Operand *Src0 = Instr->getArg(0); 452213cde0f5d3af36944075ef510c3b05aeacf959b3Nicolas Capens Operand *Src1 = Instr->getArg(1); 452313cde0f5d3af36944075ef510c3b05aeacf959b3Nicolas Capens Variable *Dest = Instr->getDest(); 452413cde0f5d3af36944075ef510c3b05aeacf959b3Nicolas Capens auto *T = makeReg(Dest->getType()); 452513cde0f5d3af36944075ef510c3b05aeacf959b3Nicolas Capens auto *Src0RM = legalize(Src0, Legal_Reg | Legal_Mem); 452613cde0f5d3af36944075ef510c3b05aeacf959b3Nicolas Capens auto *Src1RM = legalize(Src1, Legal_Reg | Legal_Mem); 452713cde0f5d3af36944075ef510c3b05aeacf959b3Nicolas Capens _movp(T, Src0RM); 452813cde0f5d3af36944075ef510c3b05aeacf959b3Nicolas Capens _pmaddwd(T, Src1RM); 452913cde0f5d3af36944075ef510c3b05aeacf959b3Nicolas Capens _movp(Dest, T); 453013cde0f5d3af36944075ef510c3b05aeacf959b3Nicolas Capens return; 453113cde0f5d3af36944075ef510c3b05aeacf959b3Nicolas Capens } 453267a49b5b3606815c5636d9a061a372c367f28685Nicolas Capens case Intrinsics::AddSaturateSigned: { 453367a49b5b3606815c5636d9a061a372c367f28685Nicolas Capens Operand *Src0 = Instr->getArg(0); 453467a49b5b3606815c5636d9a061a372c367f28685Nicolas Capens Operand *Src1 = Instr->getArg(1); 453567a49b5b3606815c5636d9a061a372c367f28685Nicolas Capens Variable *Dest = Instr->getDest(); 453667a49b5b3606815c5636d9a061a372c367f28685Nicolas Capens auto *T = makeReg(Dest->getType()); 453767a49b5b3606815c5636d9a061a372c367f28685Nicolas Capens auto *Src0RM = legalize(Src0, Legal_Reg | Legal_Mem); 453867a49b5b3606815c5636d9a061a372c367f28685Nicolas Capens auto *Src1RM = legalize(Src1, Legal_Reg | Legal_Mem); 453967a49b5b3606815c5636d9a061a372c367f28685Nicolas Capens _movp(T, Src0RM); 454067a49b5b3606815c5636d9a061a372c367f28685Nicolas Capens _padds(T, Src1RM); 454167a49b5b3606815c5636d9a061a372c367f28685Nicolas Capens _movp(Dest, T); 454267a49b5b3606815c5636d9a061a372c367f28685Nicolas Capens return; 454367a49b5b3606815c5636d9a061a372c367f28685Nicolas Capens } 454467a49b5b3606815c5636d9a061a372c367f28685Nicolas Capens case Intrinsics::SubtractSaturateSigned: { 454567a49b5b3606815c5636d9a061a372c367f28685Nicolas Capens Operand *Src0 = Instr->getArg(0); 454667a49b5b3606815c5636d9a061a372c367f28685Nicolas Capens Operand *Src1 = Instr->getArg(1); 454767a49b5b3606815c5636d9a061a372c367f28685Nicolas Capens Variable *Dest = Instr->getDest(); 454867a49b5b3606815c5636d9a061a372c367f28685Nicolas Capens auto *T = makeReg(Dest->getType()); 454967a49b5b3606815c5636d9a061a372c367f28685Nicolas Capens auto *Src0RM = legalize(Src0, Legal_Reg | Legal_Mem); 455067a49b5b3606815c5636d9a061a372c367f28685Nicolas Capens auto *Src1RM = legalize(Src1, Legal_Reg | Legal_Mem); 455167a49b5b3606815c5636d9a061a372c367f28685Nicolas Capens _movp(T, Src0RM); 455267a49b5b3606815c5636d9a061a372c367f28685Nicolas Capens _psubs(T, Src1RM); 455367a49b5b3606815c5636d9a061a372c367f28685Nicolas Capens _movp(Dest, T); 455467a49b5b3606815c5636d9a061a372c367f28685Nicolas Capens return; 455567a49b5b3606815c5636d9a061a372c367f28685Nicolas Capens } 455667a49b5b3606815c5636d9a061a372c367f28685Nicolas Capens case Intrinsics::AddSaturateUnsigned: { 455767a49b5b3606815c5636d9a061a372c367f28685Nicolas Capens Operand *Src0 = Instr->getArg(0); 455867a49b5b3606815c5636d9a061a372c367f28685Nicolas Capens Operand *Src1 = Instr->getArg(1); 455967a49b5b3606815c5636d9a061a372c367f28685Nicolas Capens Variable *Dest = Instr->getDest(); 456067a49b5b3606815c5636d9a061a372c367f28685Nicolas Capens auto *T = makeReg(Dest->getType()); 456167a49b5b3606815c5636d9a061a372c367f28685Nicolas Capens auto *Src0RM = legalize(Src0, Legal_Reg | Legal_Mem); 456267a49b5b3606815c5636d9a061a372c367f28685Nicolas Capens auto *Src1RM = legalize(Src1, Legal_Reg | Legal_Mem); 456367a49b5b3606815c5636d9a061a372c367f28685Nicolas Capens _movp(T, Src0RM); 456467a49b5b3606815c5636d9a061a372c367f28685Nicolas Capens _paddus(T, Src1RM); 456567a49b5b3606815c5636d9a061a372c367f28685Nicolas Capens _movp(Dest, T); 456667a49b5b3606815c5636d9a061a372c367f28685Nicolas Capens return; 456767a49b5b3606815c5636d9a061a372c367f28685Nicolas Capens } 456867a49b5b3606815c5636d9a061a372c367f28685Nicolas Capens case Intrinsics::SubtractSaturateUnsigned: { 456967a49b5b3606815c5636d9a061a372c367f28685Nicolas Capens Operand *Src0 = Instr->getArg(0); 457067a49b5b3606815c5636d9a061a372c367f28685Nicolas Capens Operand *Src1 = Instr->getArg(1); 457167a49b5b3606815c5636d9a061a372c367f28685Nicolas Capens Variable *Dest = Instr->getDest(); 457267a49b5b3606815c5636d9a061a372c367f28685Nicolas Capens auto *T = makeReg(Dest->getType()); 457367a49b5b3606815c5636d9a061a372c367f28685Nicolas Capens auto *Src0RM = legalize(Src0, Legal_Reg | Legal_Mem); 457467a49b5b3606815c5636d9a061a372c367f28685Nicolas Capens auto *Src1RM = legalize(Src1, Legal_Reg | Legal_Mem); 457567a49b5b3606815c5636d9a061a372c367f28685Nicolas Capens _movp(T, Src0RM); 457667a49b5b3606815c5636d9a061a372c367f28685Nicolas Capens _psubus(T, Src1RM); 457767a49b5b3606815c5636d9a061a372c367f28685Nicolas Capens _movp(Dest, T); 457867a49b5b3606815c5636d9a061a372c367f28685Nicolas Capens return; 457967a49b5b3606815c5636d9a061a372c367f28685Nicolas Capens } 4580dbf81e0c0e364173176159e0e2548e9948b197a4Nicolas Capens case Intrinsics::Nearbyint: { 4581dbf81e0c0e364173176159e0e2548e9948b197a4Nicolas Capens Operand *Src = Instr->getArg(0); 4582dbf81e0c0e364173176159e0e2548e9948b197a4Nicolas Capens Variable *Dest = Instr->getDest(); 4583dbf81e0c0e364173176159e0e2548e9948b197a4Nicolas Capens Type DestTy = Dest->getType(); 4584dbf81e0c0e364173176159e0e2548e9948b197a4Nicolas Capens if (isVectorType(DestTy)) { 4585dbf81e0c0e364173176159e0e2548e9948b197a4Nicolas Capens assert(DestTy == IceType_v4i32); 4586dbf81e0c0e364173176159e0e2548e9948b197a4Nicolas Capens assert(Src->getType() == IceType_v4f32); 4587dbf81e0c0e364173176159e0e2548e9948b197a4Nicolas Capens Operand *Src0R = legalizeToReg(Src); 4588dbf81e0c0e364173176159e0e2548e9948b197a4Nicolas Capens Variable *T = makeReg(DestTy); 4589dbf81e0c0e364173176159e0e2548e9948b197a4Nicolas Capens _cvt(T, Src0R, Traits::Insts::Cvt::Ps2dq); 4590dbf81e0c0e364173176159e0e2548e9948b197a4Nicolas Capens _movp(Dest, T); 4591dbf81e0c0e364173176159e0e2548e9948b197a4Nicolas Capens } else if (!Traits::Is64Bit && DestTy == IceType_i64) { 4592dbf81e0c0e364173176159e0e2548e9948b197a4Nicolas Capens llvm::report_fatal_error("Helper call was expected"); 4593dbf81e0c0e364173176159e0e2548e9948b197a4Nicolas Capens } else { 4594dbf81e0c0e364173176159e0e2548e9948b197a4Nicolas Capens Operand *Src0RM = legalize(Src, Legal_Reg | Legal_Mem); 4595dbf81e0c0e364173176159e0e2548e9948b197a4Nicolas Capens // t1.i32 = cvt Src0RM; t2.dest_type = t1; Dest = t2.dest_type 4596dbf81e0c0e364173176159e0e2548e9948b197a4Nicolas Capens Variable *T_1 = nullptr; 4597dbf81e0c0e364173176159e0e2548e9948b197a4Nicolas Capens if (Traits::Is64Bit && DestTy == IceType_i64) { 4598dbf81e0c0e364173176159e0e2548e9948b197a4Nicolas Capens T_1 = makeReg(IceType_i64); 4599dbf81e0c0e364173176159e0e2548e9948b197a4Nicolas Capens } else { 4600dbf81e0c0e364173176159e0e2548e9948b197a4Nicolas Capens assert(DestTy != IceType_i64); 4601dbf81e0c0e364173176159e0e2548e9948b197a4Nicolas Capens T_1 = makeReg(IceType_i32); 4602dbf81e0c0e364173176159e0e2548e9948b197a4Nicolas Capens } 4603dbf81e0c0e364173176159e0e2548e9948b197a4Nicolas Capens // cvt() requires its integer argument to be a GPR. 4604dbf81e0c0e364173176159e0e2548e9948b197a4Nicolas Capens Variable *T_2 = makeReg(DestTy); 4605dbf81e0c0e364173176159e0e2548e9948b197a4Nicolas Capens if (isByteSizedType(DestTy)) { 4606dbf81e0c0e364173176159e0e2548e9948b197a4Nicolas Capens assert(T_1->getType() == IceType_i32); 4607dbf81e0c0e364173176159e0e2548e9948b197a4Nicolas Capens T_1->setRegClass(RCX86_Is32To8); 4608dbf81e0c0e364173176159e0e2548e9948b197a4Nicolas Capens T_2->setRegClass(RCX86_IsTrunc8Rcvr); 4609dbf81e0c0e364173176159e0e2548e9948b197a4Nicolas Capens } 4610dbf81e0c0e364173176159e0e2548e9948b197a4Nicolas Capens _cvt(T_1, Src0RM, Traits::Insts::Cvt::Ss2si); 4611dbf81e0c0e364173176159e0e2548e9948b197a4Nicolas Capens _mov(T_2, T_1); // T_1 and T_2 may have different integer types 4612dbf81e0c0e364173176159e0e2548e9948b197a4Nicolas Capens if (DestTy == IceType_i1) 4613dbf81e0c0e364173176159e0e2548e9948b197a4Nicolas Capens _and(T_2, Ctx->getConstantInt1(1)); 4614dbf81e0c0e364173176159e0e2548e9948b197a4Nicolas Capens _mov(Dest, T_2); 4615dbf81e0c0e364173176159e0e2548e9948b197a4Nicolas Capens } 4616dbf81e0c0e364173176159e0e2548e9948b197a4Nicolas Capens return; 4617dbf81e0c0e364173176159e0e2548e9948b197a4Nicolas Capens } 4618f0d12c30600aad72ee5184cb47ddf2b67e7ccbeeNicolas Capens case Intrinsics::Round: { 4619dbf81e0c0e364173176159e0e2548e9948b197a4Nicolas Capens assert(InstructionSet >= Traits::SSE4_1); 4620f0d12c30600aad72ee5184cb47ddf2b67e7ccbeeNicolas Capens Variable *Dest = Instr->getDest(); 4621f0d12c30600aad72ee5184cb47ddf2b67e7ccbeeNicolas Capens Operand *Src = Instr->getArg(0); 4622f0d12c30600aad72ee5184cb47ddf2b67e7ccbeeNicolas Capens Operand *Mode = Instr->getArg(1); 4623f0d12c30600aad72ee5184cb47ddf2b67e7ccbeeNicolas Capens assert(llvm::isa<ConstantInteger32>(Mode) && 4624f0d12c30600aad72ee5184cb47ddf2b67e7ccbeeNicolas Capens "Round last argument must be a constant"); 4625f0d12c30600aad72ee5184cb47ddf2b67e7ccbeeNicolas Capens auto *SrcRM = legalize(Src, Legal_Reg | Legal_Mem); 4626f0d12c30600aad72ee5184cb47ddf2b67e7ccbeeNicolas Capens int32_t Imm = llvm::cast<ConstantInteger32>(Mode)->getValue(); 4627f0d12c30600aad72ee5184cb47ddf2b67e7ccbeeNicolas Capens (void)Imm; 4628f0d12c30600aad72ee5184cb47ddf2b67e7ccbeeNicolas Capens assert(Imm >= 0 && Imm < 4 && "Invalid rounding mode"); 4629f0d12c30600aad72ee5184cb47ddf2b67e7ccbeeNicolas Capens auto *T = makeReg(Dest->getType()); 4630f0d12c30600aad72ee5184cb47ddf2b67e7ccbeeNicolas Capens _round(T, SrcRM, Mode); 4631f0d12c30600aad72ee5184cb47ddf2b67e7ccbeeNicolas Capens _movp(Dest, T); 4632f0d12c30600aad72ee5184cb47ddf2b67e7ccbeeNicolas Capens return; 4633f0d12c30600aad72ee5184cb47ddf2b67e7ccbeeNicolas Capens } 46347145e69306813c75e1d9260ed468c266f9e038f8Jim Stichnoth default: // UnknownIntrinsic 46357145e69306813c75e1d9260ed468c266f9e038f8Jim Stichnoth Func->setError("Unexpected intrinsic"); 46367e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto return; 46377e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto } 46387e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto return; 46397e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto} 46407e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto 46414a56686b5b56db6803f90ad53514bf2fa190d9f7John Portotemplate <typename TraitsType> 46424a56686b5b56db6803f90ad53514bf2fa190d9f7John Portovoid TargetX86Base<TraitsType>::lowerAtomicCmpxchg(Variable *DestPrev, 46434a56686b5b56db6803f90ad53514bf2fa190d9f7John Porto Operand *Ptr, 46444a56686b5b56db6803f90ad53514bf2fa190d9f7John Porto Operand *Expected, 46454a56686b5b56db6803f90ad53514bf2fa190d9f7John Porto Operand *Desired) { 46465bff61c44841990680781892036adb17b3cff0c4Jim Stichnoth Type Ty = Expected->getType(); 46475bff61c44841990680781892036adb17b3cff0c4Jim Stichnoth if (!Traits::Is64Bit && Ty == IceType_i64) { 46487e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto // Reserve the pre-colored registers first, before adding any more 46497e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto // infinite-weight variables from formMemoryOperand's legalization. 46505d0acff3a2fa421923392aadb4df2742064b6248John Porto Variable *T_edx = makeReg(IceType_i32, Traits::RegisterSet::Reg_edx); 46515d0acff3a2fa421923392aadb4df2742064b6248John Porto Variable *T_eax = makeReg(IceType_i32, Traits::RegisterSet::Reg_eax); 46525d0acff3a2fa421923392aadb4df2742064b6248John Porto Variable *T_ecx = makeReg(IceType_i32, Traits::RegisterSet::Reg_ecx); 46535d0acff3a2fa421923392aadb4df2742064b6248John Porto Variable *T_ebx = makeReg(IceType_i32, Traits::RegisterSet::Reg_ebx); 46547e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto _mov(T_eax, loOperand(Expected)); 46557e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto _mov(T_edx, hiOperand(Expected)); 46567e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto _mov(T_ebx, loOperand(Desired)); 46577e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto _mov(T_ecx, hiOperand(Desired)); 46584a56686b5b56db6803f90ad53514bf2fa190d9f7John Porto X86OperandMem *Addr = formMemoryOperand(Ptr, Ty); 46595bff61c44841990680781892036adb17b3cff0c4Jim Stichnoth constexpr bool Locked = true; 46607e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto _cmpxchg8b(Addr, T_edx, T_eax, T_ecx, T_ebx, Locked); 466154f3d518805b04b3f2958b7906a7d07dedb4e1aeJim Stichnoth auto *DestLo = llvm::cast<Variable>(loOperand(DestPrev)); 466254f3d518805b04b3f2958b7906a7d07dedb4e1aeJim Stichnoth auto *DestHi = llvm::cast<Variable>(hiOperand(DestPrev)); 46637e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto _mov(DestLo, T_eax); 46647e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto _mov(DestHi, T_edx); 46657e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto return; 46667e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto } 46678aa396610b7baf728631a43ea16ad3d13e38397aJim Stichnoth RegNumT Eax; 46685bff61c44841990680781892036adb17b3cff0c4Jim Stichnoth switch (Ty) { 46695bff61c44841990680781892036adb17b3cff0c4Jim Stichnoth default: 46703c275ce1e832ba9ccfb730c4235db786cf080465John Porto llvm::report_fatal_error("Bad type for cmpxchg"); 46713c275ce1e832ba9ccfb730c4235db786cf080465John Porto case IceType_i64: 46723c275ce1e832ba9ccfb730c4235db786cf080465John Porto Eax = Traits::getRaxOrDie(); 46733c275ce1e832ba9ccfb730c4235db786cf080465John Porto break; 46745bff61c44841990680781892036adb17b3cff0c4Jim Stichnoth case IceType_i32: 46755bff61c44841990680781892036adb17b3cff0c4Jim Stichnoth Eax = Traits::RegisterSet::Reg_eax; 46765bff61c44841990680781892036adb17b3cff0c4Jim Stichnoth break; 46775bff61c44841990680781892036adb17b3cff0c4Jim Stichnoth case IceType_i16: 46785bff61c44841990680781892036adb17b3cff0c4Jim Stichnoth Eax = Traits::RegisterSet::Reg_ax; 46795bff61c44841990680781892036adb17b3cff0c4Jim Stichnoth break; 46805bff61c44841990680781892036adb17b3cff0c4Jim Stichnoth case IceType_i8: 46815bff61c44841990680781892036adb17b3cff0c4Jim Stichnoth Eax = Traits::RegisterSet::Reg_al; 46825bff61c44841990680781892036adb17b3cff0c4Jim Stichnoth break; 46835bff61c44841990680781892036adb17b3cff0c4Jim Stichnoth } 46845bff61c44841990680781892036adb17b3cff0c4Jim Stichnoth Variable *T_eax = makeReg(Ty, Eax); 46857e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto _mov(T_eax, Expected); 46864a56686b5b56db6803f90ad53514bf2fa190d9f7John Porto X86OperandMem *Addr = formMemoryOperand(Ptr, Ty); 468797f460dcd848dcef3455ed553c1b9e107d0e6ae9Andrew Scull Variable *DesiredReg = legalizeToReg(Desired); 46885bff61c44841990680781892036adb17b3cff0c4Jim Stichnoth constexpr bool Locked = true; 46897e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto _cmpxchg(Addr, T_eax, DesiredReg, Locked); 46907e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto _mov(DestPrev, T_eax); 46917e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto} 46927e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto 46934a56686b5b56db6803f90ad53514bf2fa190d9f7John Portotemplate <typename TraitsType> 46944a56686b5b56db6803f90ad53514bf2fa190d9f7John Portobool TargetX86Base<TraitsType>::tryOptimizedCmpxchgCmpBr(Variable *Dest, 46954a56686b5b56db6803f90ad53514bf2fa190d9f7John Porto Operand *PtrToMem, 46964a56686b5b56db6803f90ad53514bf2fa190d9f7John Porto Operand *Expected, 46974a56686b5b56db6803f90ad53514bf2fa190d9f7John Porto Operand *Desired) { 4698dd6dcfaf765dc93ae64ec45d623106f4b3a3c13aJim Stichnoth if (Func->getOptLevel() == Opt_m1) 46997e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto return false; 47007e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto // Peek ahead a few instructions and see how Dest is used. 47017e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto // It's very common to have: 47027e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto // 47037e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto // %x = call i32 @llvm.nacl.atomic.cmpxchg.i32(i32* ptr, i32 %expected, ...) 47047e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto // [%y_phi = ...] // list of phi stores 47057e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto // %p = icmp eq i32 %x, %expected 47067e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto // br i1 %p, label %l1, label %l2 47077e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto // 47087e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto // which we can optimize into: 47097e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto // 47107e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto // %x = <cmpxchg code> 47117e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto // [%y_phi = ...] // list of phi stores 47127e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto // br eq, %l1, %l2 47137e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto InstList::iterator I = Context.getCur(); 47147e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto // I is currently the InstIntrinsicCall. Peek past that. 47157e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto // This assumes that the atomic cmpxchg has not been lowered yet, 47167e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto // so that the instructions seen in the scan from "Cur" is simple. 47177e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto assert(llvm::isa<InstIntrinsicCall>(*I)); 47187e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto Inst *NextInst = Context.getNextInst(I); 47197e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto if (!NextInst) 47207e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto return false; 47217e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto // There might be phi assignments right before the compare+branch, since this 47227e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto // could be a backward branch for a loop. This placement of assignments is 47237e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto // determined by placePhiStores(). 4724e82b560e649f8a68bcb252b9b002708e74d962d3John Porto CfgVector<InstAssign *> PhiAssigns; 472554f3d518805b04b3f2958b7906a7d07dedb4e1aeJim Stichnoth while (auto *PhiAssign = llvm::dyn_cast<InstAssign>(NextInst)) { 47267e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto if (PhiAssign->getDest() == Dest) 47277e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto return false; 47287e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto PhiAssigns.push_back(PhiAssign); 47297e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto NextInst = Context.getNextInst(I); 47307e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto if (!NextInst) 47317e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto return false; 47327e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto } 473354f3d518805b04b3f2958b7906a7d07dedb4e1aeJim Stichnoth if (auto *NextCmp = llvm::dyn_cast<InstIcmp>(NextInst)) { 47347e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto if (!(NextCmp->getCondition() == InstIcmp::Eq && 47357e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto ((NextCmp->getSrc(0) == Dest && NextCmp->getSrc(1) == Expected) || 47367e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto (NextCmp->getSrc(1) == Dest && NextCmp->getSrc(0) == Expected)))) { 47377e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto return false; 47387e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto } 47397e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto NextInst = Context.getNextInst(I); 47407e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto if (!NextInst) 47417e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto return false; 474254f3d518805b04b3f2958b7906a7d07dedb4e1aeJim Stichnoth if (auto *NextBr = llvm::dyn_cast<InstBr>(NextInst)) { 47437e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto if (!NextBr->isUnconditional() && 47447e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto NextCmp->getDest() == NextBr->getCondition() && 47457e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto NextBr->isLastUse(NextCmp->getDest())) { 47467e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto lowerAtomicCmpxchg(Dest, PtrToMem, Expected, Desired); 47477e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto for (size_t i = 0; i < PhiAssigns.size(); ++i) { 47487e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto // Lower the phi assignments now, before the branch (same placement 47497e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto // as before). 47507e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto InstAssign *PhiAssign = PhiAssigns[i]; 47517e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto PhiAssign->setDeleted(); 47527e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto lowerAssign(PhiAssign); 47537e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto Context.advanceNext(); 47547e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto } 47555d0acff3a2fa421923392aadb4df2742064b6248John Porto _br(Traits::Cond::Br_e, NextBr->getTargetTrue(), 47565d0acff3a2fa421923392aadb4df2742064b6248John Porto NextBr->getTargetFalse()); 47577e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto // Skip over the old compare and branch, by deleting them. 47587e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto NextCmp->setDeleted(); 47597e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto NextBr->setDeleted(); 47607e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto Context.advanceNext(); 47617e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto Context.advanceNext(); 47627e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto return true; 47637e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto } 47647e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto } 47657e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto } 47667e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto return false; 47677e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto} 47687e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto 47694a56686b5b56db6803f90ad53514bf2fa190d9f7John Portotemplate <typename TraitsType> 47704a56686b5b56db6803f90ad53514bf2fa190d9f7John Portovoid TargetX86Base<TraitsType>::lowerAtomicRMW(Variable *Dest, 47714a56686b5b56db6803f90ad53514bf2fa190d9f7John Porto uint32_t Operation, Operand *Ptr, 47724a56686b5b56db6803f90ad53514bf2fa190d9f7John Porto Operand *Val) { 47737e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto bool NeedsCmpxchg = false; 47747e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto LowerBinOp Op_Lo = nullptr; 47757e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto LowerBinOp Op_Hi = nullptr; 47767e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto switch (Operation) { 47777e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto default: 47787e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto Func->setError("Unknown AtomicRMW operation"); 47797e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto return; 47807e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto case Intrinsics::AtomicAdd: { 47811d235425dab1f3dd059973fc53f1b1d5879469e3John Porto if (!Traits::Is64Bit && Dest->getType() == IceType_i64) { 47827e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto // All the fall-through paths must set this to true, but use this 47837e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto // for asserting. 47847e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto NeedsCmpxchg = true; 47854a56686b5b56db6803f90ad53514bf2fa190d9f7John Porto Op_Lo = &TargetX86Base<TraitsType>::_add; 47864a56686b5b56db6803f90ad53514bf2fa190d9f7John Porto Op_Hi = &TargetX86Base<TraitsType>::_adc; 47877e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto break; 47887e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto } 47894a56686b5b56db6803f90ad53514bf2fa190d9f7John Porto X86OperandMem *Addr = formMemoryOperand(Ptr, Dest->getType()); 47905bff61c44841990680781892036adb17b3cff0c4Jim Stichnoth constexpr bool Locked = true; 47917e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto Variable *T = nullptr; 47927e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto _mov(T, Val); 47937e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto _xadd(Addr, T, Locked); 47947e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto _mov(Dest, T); 47957e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto return; 47967e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto } 47977e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto case Intrinsics::AtomicSub: { 47981d235425dab1f3dd059973fc53f1b1d5879469e3John Porto if (!Traits::Is64Bit && Dest->getType() == IceType_i64) { 47997e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto NeedsCmpxchg = true; 48004a56686b5b56db6803f90ad53514bf2fa190d9f7John Porto Op_Lo = &TargetX86Base<TraitsType>::_sub; 48014a56686b5b56db6803f90ad53514bf2fa190d9f7John Porto Op_Hi = &TargetX86Base<TraitsType>::_sbb; 48027e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto break; 48037e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto } 48044a56686b5b56db6803f90ad53514bf2fa190d9f7John Porto X86OperandMem *Addr = formMemoryOperand(Ptr, Dest->getType()); 48055bff61c44841990680781892036adb17b3cff0c4Jim Stichnoth constexpr bool Locked = true; 48067e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto Variable *T = nullptr; 48077e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto _mov(T, Val); 48087e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto _neg(T); 48097e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto _xadd(Addr, T, Locked); 48107e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto _mov(Dest, T); 48117e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto return; 48127e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto } 48137e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto case Intrinsics::AtomicOr: 48147e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto // TODO(jvoung): If Dest is null or dead, then some of these 48157e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto // operations do not need an "exchange", but just a locked op. 48167e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto // That appears to be "worth" it for sub, or, and, and xor. 48177e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto // xadd is probably fine vs lock add for add, and xchg is fine 48187e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto // vs an atomic store. 48197e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto NeedsCmpxchg = true; 48204a56686b5b56db6803f90ad53514bf2fa190d9f7John Porto Op_Lo = &TargetX86Base<TraitsType>::_or; 48214a56686b5b56db6803f90ad53514bf2fa190d9f7John Porto Op_Hi = &TargetX86Base<TraitsType>::_or; 48227e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto break; 48237e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto case Intrinsics::AtomicAnd: 48247e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto NeedsCmpxchg = true; 48254a56686b5b56db6803f90ad53514bf2fa190d9f7John Porto Op_Lo = &TargetX86Base<TraitsType>::_and; 48264a56686b5b56db6803f90ad53514bf2fa190d9f7John Porto Op_Hi = &TargetX86Base<TraitsType>::_and; 48277e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto break; 48287e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto case Intrinsics::AtomicXor: 48297e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto NeedsCmpxchg = true; 48304a56686b5b56db6803f90ad53514bf2fa190d9f7John Porto Op_Lo = &TargetX86Base<TraitsType>::_xor; 48314a56686b5b56db6803f90ad53514bf2fa190d9f7John Porto Op_Hi = &TargetX86Base<TraitsType>::_xor; 48327e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto break; 48337e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto case Intrinsics::AtomicExchange: 48341d235425dab1f3dd059973fc53f1b1d5879469e3John Porto if (!Traits::Is64Bit && Dest->getType() == IceType_i64) { 48357e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto NeedsCmpxchg = true; 48367e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto // NeedsCmpxchg, but no real Op_Lo/Op_Hi need to be done. The values 48377e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto // just need to be moved to the ecx and ebx registers. 48387e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto Op_Lo = nullptr; 48397e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto Op_Hi = nullptr; 48407e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto break; 48417e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto } 48424a56686b5b56db6803f90ad53514bf2fa190d9f7John Porto X86OperandMem *Addr = formMemoryOperand(Ptr, Dest->getType()); 48437e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto Variable *T = nullptr; 48447e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto _mov(T, Val); 48457e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto _xchg(Addr, T); 48467e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto _mov(Dest, T); 48477e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto return; 48487e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto } 48497e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto // Otherwise, we need a cmpxchg loop. 48507e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto (void)NeedsCmpxchg; 48517e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto assert(NeedsCmpxchg); 48527e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto expandAtomicRMWAsCmpxchg(Op_Lo, Op_Hi, Dest, Ptr, Val); 48537e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto} 48547e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto 48554a56686b5b56db6803f90ad53514bf2fa190d9f7John Portotemplate <typename TraitsType> 48564a56686b5b56db6803f90ad53514bf2fa190d9f7John Portovoid TargetX86Base<TraitsType>::expandAtomicRMWAsCmpxchg(LowerBinOp Op_Lo, 48574a56686b5b56db6803f90ad53514bf2fa190d9f7John Porto LowerBinOp Op_Hi, 48584a56686b5b56db6803f90ad53514bf2fa190d9f7John Porto Variable *Dest, 48594a56686b5b56db6803f90ad53514bf2fa190d9f7John Porto Operand *Ptr, 48604a56686b5b56db6803f90ad53514bf2fa190d9f7John Porto Operand *Val) { 48617e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto // Expand a more complex RMW operation as a cmpxchg loop: 48627e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto // For 64-bit: 48637e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto // mov eax, [ptr] 48647e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto // mov edx, [ptr + 4] 48657e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto // .LABEL: 48667e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto // mov ebx, eax 48677e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto // <Op_Lo> ebx, <desired_adj_lo> 48687e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto // mov ecx, edx 48697e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto // <Op_Hi> ecx, <desired_adj_hi> 48707e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto // lock cmpxchg8b [ptr] 48717e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto // jne .LABEL 48727e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto // mov <dest_lo>, eax 48737e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto // mov <dest_lo>, edx 48747e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto // 48757e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto // For 32-bit: 48767e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto // mov eax, [ptr] 48777e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto // .LABEL: 48787e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto // mov <reg>, eax 48797e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto // op <reg>, [desired_adj] 48807e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto // lock cmpxchg [ptr], <reg> 48817e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto // jne .LABEL 48827e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto // mov <dest>, eax 48837e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto // 48847e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto // If Op_{Lo,Hi} are nullptr, then just copy the value. 48857e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto Val = legalize(Val); 48867e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto Type Ty = Val->getType(); 48871d235425dab1f3dd059973fc53f1b1d5879469e3John Porto if (!Traits::Is64Bit && Ty == IceType_i64) { 48885d0acff3a2fa421923392aadb4df2742064b6248John Porto Variable *T_edx = makeReg(IceType_i32, Traits::RegisterSet::Reg_edx); 48895d0acff3a2fa421923392aadb4df2742064b6248John Porto Variable *T_eax = makeReg(IceType_i32, Traits::RegisterSet::Reg_eax); 48904a56686b5b56db6803f90ad53514bf2fa190d9f7John Porto X86OperandMem *Addr = formMemoryOperand(Ptr, Ty); 48917e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto _mov(T_eax, loOperand(Addr)); 48927e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto _mov(T_edx, hiOperand(Addr)); 48935d0acff3a2fa421923392aadb4df2742064b6248John Porto Variable *T_ecx = makeReg(IceType_i32, Traits::RegisterSet::Reg_ecx); 48945d0acff3a2fa421923392aadb4df2742064b6248John Porto Variable *T_ebx = makeReg(IceType_i32, Traits::RegisterSet::Reg_ebx); 48954a56686b5b56db6803f90ad53514bf2fa190d9f7John Porto InstX86Label *Label = InstX86Label::create(Func, this); 48967e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto const bool IsXchg8b = Op_Lo == nullptr && Op_Hi == nullptr; 48977e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto if (!IsXchg8b) { 48987e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto Context.insert(Label); 48997e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto _mov(T_ebx, T_eax); 49007e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto (this->*Op_Lo)(T_ebx, loOperand(Val)); 49017e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto _mov(T_ecx, T_edx); 49027e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto (this->*Op_Hi)(T_ecx, hiOperand(Val)); 49037e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto } else { 49047e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto // This is for xchg, which doesn't need an actual Op_Lo/Op_Hi. 49057e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto // It just needs the Val loaded into ebx and ecx. 49067e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto // That can also be done before the loop. 49077e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto _mov(T_ebx, loOperand(Val)); 49087e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto _mov(T_ecx, hiOperand(Val)); 49097e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto Context.insert(Label); 49107e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto } 49115bff61c44841990680781892036adb17b3cff0c4Jim Stichnoth constexpr bool Locked = true; 49127e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto _cmpxchg8b(Addr, T_edx, T_eax, T_ecx, T_ebx, Locked); 49135d0acff3a2fa421923392aadb4df2742064b6248John Porto _br(Traits::Cond::Br_ne, Label); 49147e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto if (!IsXchg8b) { 49157e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto // If Val is a variable, model the extended live range of Val through 49167e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto // the end of the loop, since it will be re-used by the loop. 491754f3d518805b04b3f2958b7906a7d07dedb4e1aeJim Stichnoth if (auto *ValVar = llvm::dyn_cast<Variable>(Val)) { 491854f3d518805b04b3f2958b7906a7d07dedb4e1aeJim Stichnoth auto *ValLo = llvm::cast<Variable>(loOperand(ValVar)); 491954f3d518805b04b3f2958b7906a7d07dedb4e1aeJim Stichnoth auto *ValHi = llvm::cast<Variable>(hiOperand(ValVar)); 49201d937a8e35b7ae51ee64c32f8f34d51ba33498f4John Porto Context.insert<InstFakeUse>(ValLo); 49211d937a8e35b7ae51ee64c32f8f34d51ba33498f4John Porto Context.insert<InstFakeUse>(ValHi); 49227e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto } 49237e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto } else { 49247e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto // For xchg, the loop is slightly smaller and ebx/ecx are used. 49251d937a8e35b7ae51ee64c32f8f34d51ba33498f4John Porto Context.insert<InstFakeUse>(T_ebx); 49261d937a8e35b7ae51ee64c32f8f34d51ba33498f4John Porto Context.insert<InstFakeUse>(T_ecx); 49277e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto } 49287e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto // The address base (if any) is also reused in the loop. 49297e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto if (Variable *Base = Addr->getBase()) 49301d937a8e35b7ae51ee64c32f8f34d51ba33498f4John Porto Context.insert<InstFakeUse>(Base); 493154f3d518805b04b3f2958b7906a7d07dedb4e1aeJim Stichnoth auto *DestLo = llvm::cast<Variable>(loOperand(Dest)); 493254f3d518805b04b3f2958b7906a7d07dedb4e1aeJim Stichnoth auto *DestHi = llvm::cast<Variable>(hiOperand(Dest)); 49337e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto _mov(DestLo, T_eax); 49347e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto _mov(DestHi, T_edx); 49357e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto return; 49367e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto } 49374a56686b5b56db6803f90ad53514bf2fa190d9f7John Porto X86OperandMem *Addr = formMemoryOperand(Ptr, Ty); 49388aa396610b7baf728631a43ea16ad3d13e38397aJim Stichnoth RegNumT Eax; 49395bff61c44841990680781892036adb17b3cff0c4Jim Stichnoth switch (Ty) { 49405bff61c44841990680781892036adb17b3cff0c4Jim Stichnoth default: 49413c275ce1e832ba9ccfb730c4235db786cf080465John Porto llvm::report_fatal_error("Bad type for atomicRMW"); 49423c275ce1e832ba9ccfb730c4235db786cf080465John Porto case IceType_i64: 49433c275ce1e832ba9ccfb730c4235db786cf080465John Porto Eax = Traits::getRaxOrDie(); 49443c275ce1e832ba9ccfb730c4235db786cf080465John Porto break; 49455bff61c44841990680781892036adb17b3cff0c4Jim Stichnoth case IceType_i32: 49465bff61c44841990680781892036adb17b3cff0c4Jim Stichnoth Eax = Traits::RegisterSet::Reg_eax; 49475bff61c44841990680781892036adb17b3cff0c4Jim Stichnoth break; 49485bff61c44841990680781892036adb17b3cff0c4Jim Stichnoth case IceType_i16: 49495bff61c44841990680781892036adb17b3cff0c4Jim Stichnoth Eax = Traits::RegisterSet::Reg_ax; 49505bff61c44841990680781892036adb17b3cff0c4Jim Stichnoth break; 49515bff61c44841990680781892036adb17b3cff0c4Jim Stichnoth case IceType_i8: 49525bff61c44841990680781892036adb17b3cff0c4Jim Stichnoth Eax = Traits::RegisterSet::Reg_al; 49535bff61c44841990680781892036adb17b3cff0c4Jim Stichnoth break; 49545bff61c44841990680781892036adb17b3cff0c4Jim Stichnoth } 49555bff61c44841990680781892036adb17b3cff0c4Jim Stichnoth Variable *T_eax = makeReg(Ty, Eax); 49567e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto _mov(T_eax, Addr); 49574a56686b5b56db6803f90ad53514bf2fa190d9f7John Porto auto *Label = Context.insert<InstX86Label>(this); 49587e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto // We want to pick a different register for T than Eax, so don't use 49597e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto // _mov(T == nullptr, T_eax). 49607e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto Variable *T = makeReg(Ty); 49617e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto _mov(T, T_eax); 49627e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto (this->*Op_Lo)(T, Val); 49635bff61c44841990680781892036adb17b3cff0c4Jim Stichnoth constexpr bool Locked = true; 49647e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto _cmpxchg(Addr, T_eax, T, Locked); 49655d0acff3a2fa421923392aadb4df2742064b6248John Porto _br(Traits::Cond::Br_ne, Label); 49667e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto // If Val is a variable, model the extended live range of Val through 49677e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto // the end of the loop, since it will be re-used by the loop. 496854f3d518805b04b3f2958b7906a7d07dedb4e1aeJim Stichnoth if (auto *ValVar = llvm::dyn_cast<Variable>(Val)) { 49691d937a8e35b7ae51ee64c32f8f34d51ba33498f4John Porto Context.insert<InstFakeUse>(ValVar); 49707e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto } 49717e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto // The address base (if any) is also reused in the loop. 49727e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto if (Variable *Base = Addr->getBase()) 49731d937a8e35b7ae51ee64c32f8f34d51ba33498f4John Porto Context.insert<InstFakeUse>(Base); 49747e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto _mov(Dest, T_eax); 49757e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto} 49767e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto 49779612d32c7e5eb2cb403686ef31172d42e075e460Andrew Scull/// Lowers count {trailing, leading} zeros intrinsic. 49789612d32c7e5eb2cb403686ef31172d42e075e460Andrew Scull/// 49799612d32c7e5eb2cb403686ef31172d42e075e460Andrew Scull/// We could do constant folding here, but that should have 49809612d32c7e5eb2cb403686ef31172d42e075e460Andrew Scull/// been done by the front-end/middle-end optimizations. 49814a56686b5b56db6803f90ad53514bf2fa190d9f7John Portotemplate <typename TraitsType> 49824a56686b5b56db6803f90ad53514bf2fa190d9f7John Portovoid TargetX86Base<TraitsType>::lowerCountZeros(bool Cttz, Type Ty, 49834a56686b5b56db6803f90ad53514bf2fa190d9f7John Porto Variable *Dest, 49844a56686b5b56db6803f90ad53514bf2fa190d9f7John Porto Operand *FirstVal, 49854a56686b5b56db6803f90ad53514bf2fa190d9f7John Porto Operand *SecondVal) { 49867e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto // TODO(jvoung): Determine if the user CPU supports LZCNT (BMI). 49877e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto // Then the instructions will handle the Val == 0 case much more simply 49887e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto // and won't require conversion from bit position to number of zeros. 49897e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto // 49907e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto // Otherwise: 49917e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto // bsr IF_NOT_ZERO, Val 499234d276a31732a9bbf514dbc9a52c7488e15c5bbaJohn Porto // mov T_DEST, ((Ty == i32) ? 63 : 127) 49937e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto // cmovne T_DEST, IF_NOT_ZERO 499434d276a31732a9bbf514dbc9a52c7488e15c5bbaJohn Porto // xor T_DEST, ((Ty == i32) ? 31 : 63) 49957e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto // mov DEST, T_DEST 49967e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto // 49977e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto // NOTE: T_DEST must be a register because cmov requires its dest to be a 49987e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto // register. Also, bsf and bsr require their dest to be a register. 49997e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto // 500034d276a31732a9bbf514dbc9a52c7488e15c5bbaJohn Porto // The xor DEST, C(31|63) converts a bit position to # of leading zeroes. 50017e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto // E.g., for 000... 00001100, bsr will say that the most significant bit 50027e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto // set is at position 3, while the number of leading zeros is 28. Xor is 500334d276a31732a9bbf514dbc9a52c7488e15c5bbaJohn Porto // like (M - N) for N <= M, and converts 63 to 32, and 127 to 64 (for the 500434d276a31732a9bbf514dbc9a52c7488e15c5bbaJohn Porto // all-zeros case). 50057e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto // 500634d276a31732a9bbf514dbc9a52c7488e15c5bbaJohn Porto // X8632 only: Similar for 64-bit, but start w/ speculating that the upper 32 500734d276a31732a9bbf514dbc9a52c7488e15c5bbaJohn Porto // bits are all zero, and compute the result for that case (checking the 500834d276a31732a9bbf514dbc9a52c7488e15c5bbaJohn Porto // lower 32 bits). Then actually compute the result for the upper bits and 50097e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto // cmov in the result from the lower computation if the earlier speculation 50107e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto // was correct. 50117e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto // 50127e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto // Cttz, is similar, but uses bsf instead, and doesn't require the xor 50137e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto // bit position conversion, and the speculation is reversed. 501434d276a31732a9bbf514dbc9a52c7488e15c5bbaJohn Porto 501534d276a31732a9bbf514dbc9a52c7488e15c5bbaJohn Porto // TODO(jpp): refactor this method. 50167e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto assert(Ty == IceType_i32 || Ty == IceType_i64); 50173c275ce1e832ba9ccfb730c4235db786cf080465John Porto const Type DestTy = Traits::Is64Bit ? Dest->getType() : IceType_i32; 50183c275ce1e832ba9ccfb730c4235db786cf080465John Porto Variable *T = makeReg(DestTy); 50197e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto Operand *FirstValRM = legalize(FirstVal, Legal_Mem | Legal_Reg); 50207e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto if (Cttz) { 50217e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto _bsf(T, FirstValRM); 50227e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto } else { 50237e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto _bsr(T, FirstValRM); 50247e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto } 50253c275ce1e832ba9ccfb730c4235db786cf080465John Porto Variable *T_Dest = makeReg(DestTy); 50263c275ce1e832ba9ccfb730c4235db786cf080465John Porto Constant *_31 = Ctx->getConstantInt32(31); 50273c275ce1e832ba9ccfb730c4235db786cf080465John Porto Constant *_32 = Ctx->getConstantInt(DestTy, 32); 502834d276a31732a9bbf514dbc9a52c7488e15c5bbaJohn Porto Constant *_63 = Ctx->getConstantInt(DestTy, 63); 502934d276a31732a9bbf514dbc9a52c7488e15c5bbaJohn Porto Constant *_64 = Ctx->getConstantInt(DestTy, 64); 50307e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto if (Cttz) { 503134d276a31732a9bbf514dbc9a52c7488e15c5bbaJohn Porto if (DestTy == IceType_i64) { 503234d276a31732a9bbf514dbc9a52c7488e15c5bbaJohn Porto _mov(T_Dest, _64); 503334d276a31732a9bbf514dbc9a52c7488e15c5bbaJohn Porto } else { 503434d276a31732a9bbf514dbc9a52c7488e15c5bbaJohn Porto _mov(T_Dest, _32); 503534d276a31732a9bbf514dbc9a52c7488e15c5bbaJohn Porto } 50367e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto } else { 503734d276a31732a9bbf514dbc9a52c7488e15c5bbaJohn Porto Constant *_127 = Ctx->getConstantInt(DestTy, 127); 503834d276a31732a9bbf514dbc9a52c7488e15c5bbaJohn Porto if (DestTy == IceType_i64) { 503934d276a31732a9bbf514dbc9a52c7488e15c5bbaJohn Porto _mov(T_Dest, _127); 504034d276a31732a9bbf514dbc9a52c7488e15c5bbaJohn Porto } else { 504134d276a31732a9bbf514dbc9a52c7488e15c5bbaJohn Porto _mov(T_Dest, _63); 504234d276a31732a9bbf514dbc9a52c7488e15c5bbaJohn Porto } 50437e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto } 50445d0acff3a2fa421923392aadb4df2742064b6248John Porto _cmov(T_Dest, T, Traits::Cond::Br_ne); 50457e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto if (!Cttz) { 504634d276a31732a9bbf514dbc9a52c7488e15c5bbaJohn Porto if (DestTy == IceType_i64) { 504734d276a31732a9bbf514dbc9a52c7488e15c5bbaJohn Porto // Even though there's a _63 available at this point, that constant might 504834d276a31732a9bbf514dbc9a52c7488e15c5bbaJohn Porto // not be an i32, which will cause the xor emission to fail. 504934d276a31732a9bbf514dbc9a52c7488e15c5bbaJohn Porto Constant *_63 = Ctx->getConstantInt32(63); 505034d276a31732a9bbf514dbc9a52c7488e15c5bbaJohn Porto _xor(T_Dest, _63); 505134d276a31732a9bbf514dbc9a52c7488e15c5bbaJohn Porto } else { 505234d276a31732a9bbf514dbc9a52c7488e15c5bbaJohn Porto _xor(T_Dest, _31); 505334d276a31732a9bbf514dbc9a52c7488e15c5bbaJohn Porto } 50547e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto } 50551d235425dab1f3dd059973fc53f1b1d5879469e3John Porto if (Traits::Is64Bit || Ty == IceType_i32) { 50567e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto _mov(Dest, T_Dest); 50577e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto return; 50587e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto } 50593c275ce1e832ba9ccfb730c4235db786cf080465John Porto _add(T_Dest, _32); 506054f3d518805b04b3f2958b7906a7d07dedb4e1aeJim Stichnoth auto *DestLo = llvm::cast<Variable>(loOperand(Dest)); 506154f3d518805b04b3f2958b7906a7d07dedb4e1aeJim Stichnoth auto *DestHi = llvm::cast<Variable>(hiOperand(Dest)); 50627e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto // Will be using "test" on this, so we need a registerized variable. 506397f460dcd848dcef3455ed553c1b9e107d0e6ae9Andrew Scull Variable *SecondVar = legalizeToReg(SecondVal); 50647e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto Variable *T_Dest2 = makeReg(IceType_i32); 50657e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto if (Cttz) { 50667e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto _bsf(T_Dest2, SecondVar); 50677e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto } else { 50687e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto _bsr(T_Dest2, SecondVar); 50693c275ce1e832ba9ccfb730c4235db786cf080465John Porto _xor(T_Dest2, _31); 50707e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto } 50717e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto _test(SecondVar, SecondVar); 50725d0acff3a2fa421923392aadb4df2742064b6248John Porto _cmov(T_Dest2, T_Dest, Traits::Cond::Br_e); 50737e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto _mov(DestLo, T_Dest2); 50747e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto _mov(DestHi, Ctx->getConstantZero(IceType_i32)); 50757e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto} 50767e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto 50774a56686b5b56db6803f90ad53514bf2fa190d9f7John Portotemplate <typename TraitsType> 50784a56686b5b56db6803f90ad53514bf2fa190d9f7John Portovoid TargetX86Base<TraitsType>::typedLoad(Type Ty, Variable *Dest, 50794a56686b5b56db6803f90ad53514bf2fa190d9f7John Porto Variable *Base, Constant *Offset) { 50808ff4b2819944bc4f02fb29204a1fa5ba7dea5682Jim Stichnoth // If Offset is a ConstantRelocatable in Non-SFI mode, we will need to 50818ff4b2819944bc4f02fb29204a1fa5ba7dea5682Jim Stichnoth // legalize Mem properly. 50828ff4b2819944bc4f02fb29204a1fa5ba7dea5682Jim Stichnoth if (Offset) 50838ff4b2819944bc4f02fb29204a1fa5ba7dea5682Jim Stichnoth assert(!llvm::isa<ConstantRelocatable>(Offset)); 50848ff4b2819944bc4f02fb29204a1fa5ba7dea5682Jim Stichnoth 50854a56686b5b56db6803f90ad53514bf2fa190d9f7John Porto auto *Mem = X86OperandMem::create(Func, Ty, Base, Offset); 5086cfa628b5f7612d202de234980b21bd6d59a11998Andrew Scull 5087cfa628b5f7612d202de234980b21bd6d59a11998Andrew Scull if (isVectorType(Ty)) 5088cfa628b5f7612d202de234980b21bd6d59a11998Andrew Scull _movp(Dest, Mem); 5089cfa628b5f7612d202de234980b21bd6d59a11998Andrew Scull else if (Ty == IceType_f64) 5090cfa628b5f7612d202de234980b21bd6d59a11998Andrew Scull _movq(Dest, Mem); 5091cfa628b5f7612d202de234980b21bd6d59a11998Andrew Scull else 5092cfa628b5f7612d202de234980b21bd6d59a11998Andrew Scull _mov(Dest, Mem); 5093cfa628b5f7612d202de234980b21bd6d59a11998Andrew Scull} 5094cfa628b5f7612d202de234980b21bd6d59a11998Andrew Scull 50954a56686b5b56db6803f90ad53514bf2fa190d9f7John Portotemplate <typename TraitsType> 50964a56686b5b56db6803f90ad53514bf2fa190d9f7John Portovoid TargetX86Base<TraitsType>::typedStore(Type Ty, Variable *Value, 50974a56686b5b56db6803f90ad53514bf2fa190d9f7John Porto Variable *Base, Constant *Offset) { 50988ff4b2819944bc4f02fb29204a1fa5ba7dea5682Jim Stichnoth // If Offset is a ConstantRelocatable in Non-SFI mode, we will need to 50998ff4b2819944bc4f02fb29204a1fa5ba7dea5682Jim Stichnoth // legalize Mem properly. 51008ff4b2819944bc4f02fb29204a1fa5ba7dea5682Jim Stichnoth if (Offset) 51018ff4b2819944bc4f02fb29204a1fa5ba7dea5682Jim Stichnoth assert(!llvm::isa<ConstantRelocatable>(Offset)); 51028ff4b2819944bc4f02fb29204a1fa5ba7dea5682Jim Stichnoth 51034a56686b5b56db6803f90ad53514bf2fa190d9f7John Porto auto *Mem = X86OperandMem::create(Func, Ty, Base, Offset); 5104cfa628b5f7612d202de234980b21bd6d59a11998Andrew Scull 5105cfa628b5f7612d202de234980b21bd6d59a11998Andrew Scull if (isVectorType(Ty)) 5106cfa628b5f7612d202de234980b21bd6d59a11998Andrew Scull _storep(Value, Mem); 5107cfa628b5f7612d202de234980b21bd6d59a11998Andrew Scull else if (Ty == IceType_f64) 5108cfa628b5f7612d202de234980b21bd6d59a11998Andrew Scull _storeq(Value, Mem); 5109cfa628b5f7612d202de234980b21bd6d59a11998Andrew Scull else 5110cfa628b5f7612d202de234980b21bd6d59a11998Andrew Scull _store(Value, Mem); 5111cfa628b5f7612d202de234980b21bd6d59a11998Andrew Scull} 5112cfa628b5f7612d202de234980b21bd6d59a11998Andrew Scull 51134a56686b5b56db6803f90ad53514bf2fa190d9f7John Portotemplate <typename TraitsType> 51144a56686b5b56db6803f90ad53514bf2fa190d9f7John Portovoid TargetX86Base<TraitsType>::copyMemory(Type Ty, Variable *Dest, 51154a56686b5b56db6803f90ad53514bf2fa190d9f7John Porto Variable *Src, int32_t OffsetAmt) { 5116cfa628b5f7612d202de234980b21bd6d59a11998Andrew Scull Constant *Offset = OffsetAmt ? Ctx->getConstantInt32(OffsetAmt) : nullptr; 5117cfa628b5f7612d202de234980b21bd6d59a11998Andrew Scull // TODO(ascull): this or add nullptr test to _movp, _movq 5118cfa628b5f7612d202de234980b21bd6d59a11998Andrew Scull Variable *Data = makeReg(Ty); 5119cfa628b5f7612d202de234980b21bd6d59a11998Andrew Scull 5120cfa628b5f7612d202de234980b21bd6d59a11998Andrew Scull typedLoad(Ty, Data, Src, Offset); 5121cfa628b5f7612d202de234980b21bd6d59a11998Andrew Scull typedStore(Ty, Data, Dest, Offset); 5122cfa628b5f7612d202de234980b21bd6d59a11998Andrew Scull} 5123cfa628b5f7612d202de234980b21bd6d59a11998Andrew Scull 51244a56686b5b56db6803f90ad53514bf2fa190d9f7John Portotemplate <typename TraitsType> 51254a56686b5b56db6803f90ad53514bf2fa190d9f7John Portovoid TargetX86Base<TraitsType>::lowerMemcpy(Operand *Dest, Operand *Src, 51264a56686b5b56db6803f90ad53514bf2fa190d9f7John Porto Operand *Count) { 51279df4a379ebdb2f2200ec7f65017bd9fc8544794bAndrew Scull // There is a load and store for each chunk in the unroll 51289df4a379ebdb2f2200ec7f65017bd9fc8544794bAndrew Scull constexpr uint32_t BytesPerStorep = 16; 51299df4a379ebdb2f2200ec7f65017bd9fc8544794bAndrew Scull 51309df4a379ebdb2f2200ec7f65017bd9fc8544794bAndrew Scull // Check if the operands are constants 51319df4a379ebdb2f2200ec7f65017bd9fc8544794bAndrew Scull const auto *CountConst = llvm::dyn_cast<const ConstantInteger32>(Count); 51329df4a379ebdb2f2200ec7f65017bd9fc8544794bAndrew Scull const bool IsCountConst = CountConst != nullptr; 51339df4a379ebdb2f2200ec7f65017bd9fc8544794bAndrew Scull const uint32_t CountValue = IsCountConst ? CountConst->getValue() : 0; 51349df4a379ebdb2f2200ec7f65017bd9fc8544794bAndrew Scull 5135cfa628b5f7612d202de234980b21bd6d59a11998Andrew Scull if (shouldOptimizeMemIntrins() && IsCountConst && 5136cfa628b5f7612d202de234980b21bd6d59a11998Andrew Scull CountValue <= BytesPerStorep * Traits::MEMCPY_UNROLL_LIMIT) { 51379df4a379ebdb2f2200ec7f65017bd9fc8544794bAndrew Scull // Unlikely, but nothing to do if it does happen 51389df4a379ebdb2f2200ec7f65017bd9fc8544794bAndrew Scull if (CountValue == 0) 51399df4a379ebdb2f2200ec7f65017bd9fc8544794bAndrew Scull return; 51409df4a379ebdb2f2200ec7f65017bd9fc8544794bAndrew Scull 51419df4a379ebdb2f2200ec7f65017bd9fc8544794bAndrew Scull Variable *SrcBase = legalizeToReg(Src); 51429df4a379ebdb2f2200ec7f65017bd9fc8544794bAndrew Scull Variable *DestBase = legalizeToReg(Dest); 51439df4a379ebdb2f2200ec7f65017bd9fc8544794bAndrew Scull 5144cfa628b5f7612d202de234980b21bd6d59a11998Andrew Scull // Find the largest type that can be used and use it as much as possible in 5145cfa628b5f7612d202de234980b21bd6d59a11998Andrew Scull // reverse order. Then handle any remainder with overlapping copies. Since 5146cfa628b5f7612d202de234980b21bd6d59a11998Andrew Scull // the remainder will be at the end, there will be reduced pressure on the 5147cfa628b5f7612d202de234980b21bd6d59a11998Andrew Scull // memory unit as the accesses to the same memory are far apart. 5148cfa628b5f7612d202de234980b21bd6d59a11998Andrew Scull Type Ty = largestTypeInSize(CountValue); 5149cfa628b5f7612d202de234980b21bd6d59a11998Andrew Scull uint32_t TyWidth = typeWidthInBytes(Ty); 5150cfa628b5f7612d202de234980b21bd6d59a11998Andrew Scull 5151cfa628b5f7612d202de234980b21bd6d59a11998Andrew Scull uint32_t RemainingBytes = CountValue; 5152cfa628b5f7612d202de234980b21bd6d59a11998Andrew Scull int32_t Offset = (CountValue & ~(TyWidth - 1)) - TyWidth; 5153cfa628b5f7612d202de234980b21bd6d59a11998Andrew Scull while (RemainingBytes >= TyWidth) { 5154cfa628b5f7612d202de234980b21bd6d59a11998Andrew Scull copyMemory(Ty, DestBase, SrcBase, Offset); 5155cfa628b5f7612d202de234980b21bd6d59a11998Andrew Scull RemainingBytes -= TyWidth; 5156cfa628b5f7612d202de234980b21bd6d59a11998Andrew Scull Offset -= TyWidth; 5157cfa628b5f7612d202de234980b21bd6d59a11998Andrew Scull } 51589df4a379ebdb2f2200ec7f65017bd9fc8544794bAndrew Scull 5159cfa628b5f7612d202de234980b21bd6d59a11998Andrew Scull if (RemainingBytes == 0) 5160cfa628b5f7612d202de234980b21bd6d59a11998Andrew Scull return; 51619df4a379ebdb2f2200ec7f65017bd9fc8544794bAndrew Scull 5162cfa628b5f7612d202de234980b21bd6d59a11998Andrew Scull // Lower the remaining bytes. Adjust to larger types in order to make use 5163cfa628b5f7612d202de234980b21bd6d59a11998Andrew Scull // of overlaps in the copies. 5164cfa628b5f7612d202de234980b21bd6d59a11998Andrew Scull Type LeftOverTy = firstTypeThatFitsSize(RemainingBytes); 5165cfa628b5f7612d202de234980b21bd6d59a11998Andrew Scull Offset = CountValue - typeWidthInBytes(LeftOverTy); 5166cfa628b5f7612d202de234980b21bd6d59a11998Andrew Scull copyMemory(LeftOverTy, DestBase, SrcBase, Offset); 5167cfa628b5f7612d202de234980b21bd6d59a11998Andrew Scull return; 5168cfa628b5f7612d202de234980b21bd6d59a11998Andrew Scull } 51699df4a379ebdb2f2200ec7f65017bd9fc8544794bAndrew Scull 5170cfa628b5f7612d202de234980b21bd6d59a11998Andrew Scull // Fall back on a function call 517120070e84c21b9aa15a83e53b5d2ffa692c4e081aKarl Schimpf InstCall *Call = makeHelperCall(RuntimeHelper::H_call_memcpy, nullptr, 3); 5172cfa628b5f7612d202de234980b21bd6d59a11998Andrew Scull Call->addArg(Dest); 5173cfa628b5f7612d202de234980b21bd6d59a11998Andrew Scull Call->addArg(Src); 5174cfa628b5f7612d202de234980b21bd6d59a11998Andrew Scull Call->addArg(Count); 5175cfa628b5f7612d202de234980b21bd6d59a11998Andrew Scull lowerCall(Call); 5176cfa628b5f7612d202de234980b21bd6d59a11998Andrew Scull} 51779df4a379ebdb2f2200ec7f65017bd9fc8544794bAndrew Scull 51784a56686b5b56db6803f90ad53514bf2fa190d9f7John Portotemplate <typename TraitsType> 51794a56686b5b56db6803f90ad53514bf2fa190d9f7John Portovoid TargetX86Base<TraitsType>::lowerMemmove(Operand *Dest, Operand *Src, 51804a56686b5b56db6803f90ad53514bf2fa190d9f7John Porto Operand *Count) { 5181cfa628b5f7612d202de234980b21bd6d59a11998Andrew Scull // There is a load and store for each chunk in the unroll 5182cfa628b5f7612d202de234980b21bd6d59a11998Andrew Scull constexpr uint32_t BytesPerStorep = 16; 51839df4a379ebdb2f2200ec7f65017bd9fc8544794bAndrew Scull 5184cfa628b5f7612d202de234980b21bd6d59a11998Andrew Scull // Check if the operands are constants 5185cfa628b5f7612d202de234980b21bd6d59a11998Andrew Scull const auto *CountConst = llvm::dyn_cast<const ConstantInteger32>(Count); 5186cfa628b5f7612d202de234980b21bd6d59a11998Andrew Scull const bool IsCountConst = CountConst != nullptr; 5187cfa628b5f7612d202de234980b21bd6d59a11998Andrew Scull const uint32_t CountValue = IsCountConst ? CountConst->getValue() : 0; 5188cfa628b5f7612d202de234980b21bd6d59a11998Andrew Scull 5189cfa628b5f7612d202de234980b21bd6d59a11998Andrew Scull if (shouldOptimizeMemIntrins() && IsCountConst && 5190cfa628b5f7612d202de234980b21bd6d59a11998Andrew Scull CountValue <= BytesPerStorep * Traits::MEMMOVE_UNROLL_LIMIT) { 5191cfa628b5f7612d202de234980b21bd6d59a11998Andrew Scull // Unlikely, but nothing to do if it does happen 5192cfa628b5f7612d202de234980b21bd6d59a11998Andrew Scull if (CountValue == 0) 51939df4a379ebdb2f2200ec7f65017bd9fc8544794bAndrew Scull return; 5194cfa628b5f7612d202de234980b21bd6d59a11998Andrew Scull 5195cfa628b5f7612d202de234980b21bd6d59a11998Andrew Scull Variable *SrcBase = legalizeToReg(Src); 5196cfa628b5f7612d202de234980b21bd6d59a11998Andrew Scull Variable *DestBase = legalizeToReg(Dest); 5197cfa628b5f7612d202de234980b21bd6d59a11998Andrew Scull 5198cfa628b5f7612d202de234980b21bd6d59a11998Andrew Scull std::tuple<Type, Constant *, Variable *> 5199cfa628b5f7612d202de234980b21bd6d59a11998Andrew Scull Moves[Traits::MEMMOVE_UNROLL_LIMIT]; 5200cfa628b5f7612d202de234980b21bd6d59a11998Andrew Scull Constant *Offset; 5201cfa628b5f7612d202de234980b21bd6d59a11998Andrew Scull Variable *Reg; 5202cfa628b5f7612d202de234980b21bd6d59a11998Andrew Scull 5203cfa628b5f7612d202de234980b21bd6d59a11998Andrew Scull // Copy the data into registers as the source and destination could overlap 520457e126899b20c65ff3ea23a3b7d7a67ab30b99dcAndrew Scull // so make sure not to clobber the memory. This also means overlapping 520557e126899b20c65ff3ea23a3b7d7a67ab30b99dcAndrew Scull // moves can be used as we are taking a safe snapshot of the memory. 5206cfa628b5f7612d202de234980b21bd6d59a11998Andrew Scull Type Ty = largestTypeInSize(CountValue); 5207cfa628b5f7612d202de234980b21bd6d59a11998Andrew Scull uint32_t TyWidth = typeWidthInBytes(Ty); 5208cfa628b5f7612d202de234980b21bd6d59a11998Andrew Scull 5209cfa628b5f7612d202de234980b21bd6d59a11998Andrew Scull uint32_t RemainingBytes = CountValue; 5210cfa628b5f7612d202de234980b21bd6d59a11998Andrew Scull int32_t OffsetAmt = (CountValue & ~(TyWidth - 1)) - TyWidth; 5211cfa628b5f7612d202de234980b21bd6d59a11998Andrew Scull size_t N = 0; 5212cfa628b5f7612d202de234980b21bd6d59a11998Andrew Scull while (RemainingBytes >= TyWidth) { 5213cfa628b5f7612d202de234980b21bd6d59a11998Andrew Scull assert(N <= Traits::MEMMOVE_UNROLL_LIMIT); 5214cfa628b5f7612d202de234980b21bd6d59a11998Andrew Scull Offset = Ctx->getConstantInt32(OffsetAmt); 5215cfa628b5f7612d202de234980b21bd6d59a11998Andrew Scull Reg = makeReg(Ty); 5216cfa628b5f7612d202de234980b21bd6d59a11998Andrew Scull typedLoad(Ty, Reg, SrcBase, Offset); 5217cfa628b5f7612d202de234980b21bd6d59a11998Andrew Scull RemainingBytes -= TyWidth; 5218cfa628b5f7612d202de234980b21bd6d59a11998Andrew Scull OffsetAmt -= TyWidth; 5219cfa628b5f7612d202de234980b21bd6d59a11998Andrew Scull Moves[N++] = std::make_tuple(Ty, Offset, Reg); 52209df4a379ebdb2f2200ec7f65017bd9fc8544794bAndrew Scull } 52219df4a379ebdb2f2200ec7f65017bd9fc8544794bAndrew Scull 5222cfa628b5f7612d202de234980b21bd6d59a11998Andrew Scull if (RemainingBytes != 0) { 5223cfa628b5f7612d202de234980b21bd6d59a11998Andrew Scull // Lower the remaining bytes. Adjust to larger types in order to make use 5224cfa628b5f7612d202de234980b21bd6d59a11998Andrew Scull // of overlaps in the copies. 5225cfa628b5f7612d202de234980b21bd6d59a11998Andrew Scull assert(N <= Traits::MEMMOVE_UNROLL_LIMIT); 5226cfa628b5f7612d202de234980b21bd6d59a11998Andrew Scull Ty = firstTypeThatFitsSize(RemainingBytes); 5227cfa628b5f7612d202de234980b21bd6d59a11998Andrew Scull Offset = Ctx->getConstantInt32(CountValue - typeWidthInBytes(Ty)); 5228cfa628b5f7612d202de234980b21bd6d59a11998Andrew Scull Reg = makeReg(Ty); 5229cfa628b5f7612d202de234980b21bd6d59a11998Andrew Scull typedLoad(Ty, Reg, SrcBase, Offset); 5230cfa628b5f7612d202de234980b21bd6d59a11998Andrew Scull Moves[N++] = std::make_tuple(Ty, Offset, Reg); 52319df4a379ebdb2f2200ec7f65017bd9fc8544794bAndrew Scull } 52329df4a379ebdb2f2200ec7f65017bd9fc8544794bAndrew Scull 5233cfa628b5f7612d202de234980b21bd6d59a11998Andrew Scull // Copy the data out into the destination memory 5234cfa628b5f7612d202de234980b21bd6d59a11998Andrew Scull for (size_t i = 0; i < N; ++i) { 5235cfa628b5f7612d202de234980b21bd6d59a11998Andrew Scull std::tie(Ty, Offset, Reg) = Moves[i]; 5236cfa628b5f7612d202de234980b21bd6d59a11998Andrew Scull typedStore(Ty, Reg, DestBase, Offset); 52379df4a379ebdb2f2200ec7f65017bd9fc8544794bAndrew Scull } 52389df4a379ebdb2f2200ec7f65017bd9fc8544794bAndrew Scull 52399df4a379ebdb2f2200ec7f65017bd9fc8544794bAndrew Scull return; 52409df4a379ebdb2f2200ec7f65017bd9fc8544794bAndrew Scull } 52419df4a379ebdb2f2200ec7f65017bd9fc8544794bAndrew Scull 52429df4a379ebdb2f2200ec7f65017bd9fc8544794bAndrew Scull // Fall back on a function call 524320070e84c21b9aa15a83e53b5d2ffa692c4e081aKarl Schimpf InstCall *Call = makeHelperCall(RuntimeHelper::H_call_memmove, nullptr, 3); 52449df4a379ebdb2f2200ec7f65017bd9fc8544794bAndrew Scull Call->addArg(Dest); 52459df4a379ebdb2f2200ec7f65017bd9fc8544794bAndrew Scull Call->addArg(Src); 52469df4a379ebdb2f2200ec7f65017bd9fc8544794bAndrew Scull Call->addArg(Count); 52479df4a379ebdb2f2200ec7f65017bd9fc8544794bAndrew Scull lowerCall(Call); 52489df4a379ebdb2f2200ec7f65017bd9fc8544794bAndrew Scull} 52499df4a379ebdb2f2200ec7f65017bd9fc8544794bAndrew Scull 52504a56686b5b56db6803f90ad53514bf2fa190d9f7John Portotemplate <typename TraitsType> 52514a56686b5b56db6803f90ad53514bf2fa190d9f7John Portovoid TargetX86Base<TraitsType>::lowerMemset(Operand *Dest, Operand *Val, 52524a56686b5b56db6803f90ad53514bf2fa190d9f7John Porto Operand *Count) { 52539df4a379ebdb2f2200ec7f65017bd9fc8544794bAndrew Scull constexpr uint32_t BytesPerStorep = 16; 52549df4a379ebdb2f2200ec7f65017bd9fc8544794bAndrew Scull constexpr uint32_t BytesPerStoreq = 8; 52559df4a379ebdb2f2200ec7f65017bd9fc8544794bAndrew Scull constexpr uint32_t BytesPerStorei32 = 4; 5256713dbdecbdd74f77cbb1cae43b19bdd6f6b3144fAndrew Scull assert(Val->getType() == IceType_i8); 5257713dbdecbdd74f77cbb1cae43b19bdd6f6b3144fAndrew Scull 5258713dbdecbdd74f77cbb1cae43b19bdd6f6b3144fAndrew Scull // Check if the operands are constants 5259713dbdecbdd74f77cbb1cae43b19bdd6f6b3144fAndrew Scull const auto *CountConst = llvm::dyn_cast<const ConstantInteger32>(Count); 5260713dbdecbdd74f77cbb1cae43b19bdd6f6b3144fAndrew Scull const auto *ValConst = llvm::dyn_cast<const ConstantInteger32>(Val); 5261713dbdecbdd74f77cbb1cae43b19bdd6f6b3144fAndrew Scull const bool IsCountConst = CountConst != nullptr; 5262713dbdecbdd74f77cbb1cae43b19bdd6f6b3144fAndrew Scull const bool IsValConst = ValConst != nullptr; 5263713dbdecbdd74f77cbb1cae43b19bdd6f6b3144fAndrew Scull const uint32_t CountValue = IsCountConst ? CountConst->getValue() : 0; 5264713dbdecbdd74f77cbb1cae43b19bdd6f6b3144fAndrew Scull const uint32_t ValValue = IsValConst ? ValConst->getValue() : 0; 5265713dbdecbdd74f77cbb1cae43b19bdd6f6b3144fAndrew Scull 5266713dbdecbdd74f77cbb1cae43b19bdd6f6b3144fAndrew Scull // Unlikely, but nothing to do if it does happen 5267713dbdecbdd74f77cbb1cae43b19bdd6f6b3144fAndrew Scull if (IsCountConst && CountValue == 0) 5268713dbdecbdd74f77cbb1cae43b19bdd6f6b3144fAndrew Scull return; 5269713dbdecbdd74f77cbb1cae43b19bdd6f6b3144fAndrew Scull 5270713dbdecbdd74f77cbb1cae43b19bdd6f6b3144fAndrew Scull // TODO(ascull): if the count is constant but val is not it would be possible 5271713dbdecbdd74f77cbb1cae43b19bdd6f6b3144fAndrew Scull // to inline by spreading the value across 4 bytes and accessing subregs e.g. 5272713dbdecbdd74f77cbb1cae43b19bdd6f6b3144fAndrew Scull // eax, ax and al. 5273cfa628b5f7612d202de234980b21bd6d59a11998Andrew Scull if (shouldOptimizeMemIntrins() && IsCountConst && IsValConst) { 52749df4a379ebdb2f2200ec7f65017bd9fc8544794bAndrew Scull Variable *Base = nullptr; 5275cfa628b5f7612d202de234980b21bd6d59a11998Andrew Scull Variable *VecReg = nullptr; 527659ce615334de49a062aa40f34232890458b8770dJim Stichnoth const uint32_t MaskValue = (ValValue & 0xff); 52779df4a379ebdb2f2200ec7f65017bd9fc8544794bAndrew Scull const uint32_t SpreadValue = 527859ce615334de49a062aa40f34232890458b8770dJim Stichnoth (MaskValue << 24) | (MaskValue << 16) | (MaskValue << 8) | MaskValue; 52799df4a379ebdb2f2200ec7f65017bd9fc8544794bAndrew Scull 52809df4a379ebdb2f2200ec7f65017bd9fc8544794bAndrew Scull auto lowerSet = [this, &Base, SpreadValue, &VecReg](Type Ty, 5281992f91ddfeb7e68bdb51498194c2af7e9017aeb1Jim Stichnoth uint32_t OffsetAmt) { 52829df4a379ebdb2f2200ec7f65017bd9fc8544794bAndrew Scull assert(Base != nullptr); 52839df4a379ebdb2f2200ec7f65017bd9fc8544794bAndrew Scull Constant *Offset = OffsetAmt ? Ctx->getConstantInt32(OffsetAmt) : nullptr; 52849df4a379ebdb2f2200ec7f65017bd9fc8544794bAndrew Scull 52859df4a379ebdb2f2200ec7f65017bd9fc8544794bAndrew Scull // TODO(ascull): is 64-bit better with vector or scalar movq? 52864a56686b5b56db6803f90ad53514bf2fa190d9f7John Porto auto *Mem = X86OperandMem::create(Func, Ty, Base, Offset); 52879df4a379ebdb2f2200ec7f65017bd9fc8544794bAndrew Scull if (isVectorType(Ty)) { 5288713dbdecbdd74f77cbb1cae43b19bdd6f6b3144fAndrew Scull assert(VecReg != nullptr); 5289713dbdecbdd74f77cbb1cae43b19bdd6f6b3144fAndrew Scull _storep(VecReg, Mem); 5290cfa628b5f7612d202de234980b21bd6d59a11998Andrew Scull } else if (Ty == IceType_f64) { 5291713dbdecbdd74f77cbb1cae43b19bdd6f6b3144fAndrew Scull assert(VecReg != nullptr); 5292713dbdecbdd74f77cbb1cae43b19bdd6f6b3144fAndrew Scull _storeq(VecReg, Mem); 52939df4a379ebdb2f2200ec7f65017bd9fc8544794bAndrew Scull } else { 52943c275ce1e832ba9ccfb730c4235db786cf080465John Porto assert(Ty != IceType_i64); 52959df4a379ebdb2f2200ec7f65017bd9fc8544794bAndrew Scull _store(Ctx->getConstantInt(Ty, SpreadValue), Mem); 5296713dbdecbdd74f77cbb1cae43b19bdd6f6b3144fAndrew Scull } 5297713dbdecbdd74f77cbb1cae43b19bdd6f6b3144fAndrew Scull }; 5298713dbdecbdd74f77cbb1cae43b19bdd6f6b3144fAndrew Scull 5299cfa628b5f7612d202de234980b21bd6d59a11998Andrew Scull // Find the largest type that can be used and use it as much as possible in 5300cfa628b5f7612d202de234980b21bd6d59a11998Andrew Scull // reverse order. Then handle any remainder with overlapping copies. Since 5301cfa628b5f7612d202de234980b21bd6d59a11998Andrew Scull // the remainder will be at the end, there will be reduces pressure on the 5302cfa628b5f7612d202de234980b21bd6d59a11998Andrew Scull // memory unit as the access to the same memory are far apart. 5303cfa628b5f7612d202de234980b21bd6d59a11998Andrew Scull Type Ty; 53049df4a379ebdb2f2200ec7f65017bd9fc8544794bAndrew Scull if (ValValue == 0 && CountValue >= BytesPerStoreq && 530535e160025c5a7005f97c5ad55120999cf83a4ce1Jim Stichnoth CountValue <= BytesPerStorep * Traits::MEMSET_UNROLL_LIMIT) { 5306cfa628b5f7612d202de234980b21bd6d59a11998Andrew Scull // When the value is zero it can be loaded into a vector register cheaply 5307cfa628b5f7612d202de234980b21bd6d59a11998Andrew Scull // using the xor trick. 53089df4a379ebdb2f2200ec7f65017bd9fc8544794bAndrew Scull Base = legalizeToReg(Dest); 53099df4a379ebdb2f2200ec7f65017bd9fc8544794bAndrew Scull VecReg = makeVectorOfZeros(IceType_v16i8); 5310cfa628b5f7612d202de234980b21bd6d59a11998Andrew Scull Ty = largestTypeInSize(CountValue); 531135e160025c5a7005f97c5ad55120999cf83a4ce1Jim Stichnoth } else if (CountValue <= BytesPerStorei32 * Traits::MEMSET_UNROLL_LIMIT) { 5312cfa628b5f7612d202de234980b21bd6d59a11998Andrew Scull // When the value is non-zero or the count is small we can't use vector 5313cfa628b5f7612d202de234980b21bd6d59a11998Andrew Scull // instructions so are limited to 32-bit stores. 5314cfa628b5f7612d202de234980b21bd6d59a11998Andrew Scull Base = legalizeToReg(Dest); 5315cfa628b5f7612d202de234980b21bd6d59a11998Andrew Scull constexpr uint32_t MaxSize = 4; 5316cfa628b5f7612d202de234980b21bd6d59a11998Andrew Scull Ty = largestTypeInSize(CountValue, MaxSize); 5317cfa628b5f7612d202de234980b21bd6d59a11998Andrew Scull } 5318713dbdecbdd74f77cbb1cae43b19bdd6f6b3144fAndrew Scull 5319cfa628b5f7612d202de234980b21bd6d59a11998Andrew Scull if (Base) { 5320cfa628b5f7612d202de234980b21bd6d59a11998Andrew Scull uint32_t TyWidth = typeWidthInBytes(Ty); 5321cfa628b5f7612d202de234980b21bd6d59a11998Andrew Scull 5322cfa628b5f7612d202de234980b21bd6d59a11998Andrew Scull uint32_t RemainingBytes = CountValue; 5323cfa628b5f7612d202de234980b21bd6d59a11998Andrew Scull uint32_t Offset = (CountValue & ~(TyWidth - 1)) - TyWidth; 5324cfa628b5f7612d202de234980b21bd6d59a11998Andrew Scull while (RemainingBytes >= TyWidth) { 5325cfa628b5f7612d202de234980b21bd6d59a11998Andrew Scull lowerSet(Ty, Offset); 5326cfa628b5f7612d202de234980b21bd6d59a11998Andrew Scull RemainingBytes -= TyWidth; 5327cfa628b5f7612d202de234980b21bd6d59a11998Andrew Scull Offset -= TyWidth; 5328713dbdecbdd74f77cbb1cae43b19bdd6f6b3144fAndrew Scull } 5329713dbdecbdd74f77cbb1cae43b19bdd6f6b3144fAndrew Scull 5330cfa628b5f7612d202de234980b21bd6d59a11998Andrew Scull if (RemainingBytes == 0) 53319df4a379ebdb2f2200ec7f65017bd9fc8544794bAndrew Scull return; 53329df4a379ebdb2f2200ec7f65017bd9fc8544794bAndrew Scull 5333cfa628b5f7612d202de234980b21bd6d59a11998Andrew Scull // Lower the remaining bytes. Adjust to larger types in order to make use 5334cfa628b5f7612d202de234980b21bd6d59a11998Andrew Scull // of overlaps in the copies. 5335cfa628b5f7612d202de234980b21bd6d59a11998Andrew Scull Type LeftOverTy = firstTypeThatFitsSize(RemainingBytes); 5336cfa628b5f7612d202de234980b21bd6d59a11998Andrew Scull Offset = CountValue - typeWidthInBytes(LeftOverTy); 5337cfa628b5f7612d202de234980b21bd6d59a11998Andrew Scull lowerSet(LeftOverTy, Offset); 5338713dbdecbdd74f77cbb1cae43b19bdd6f6b3144fAndrew Scull return; 5339713dbdecbdd74f77cbb1cae43b19bdd6f6b3144fAndrew Scull } 5340713dbdecbdd74f77cbb1cae43b19bdd6f6b3144fAndrew Scull } 5341713dbdecbdd74f77cbb1cae43b19bdd6f6b3144fAndrew Scull 5342713dbdecbdd74f77cbb1cae43b19bdd6f6b3144fAndrew Scull // Fall back on calling the memset function. The value operand needs to be 5343713dbdecbdd74f77cbb1cae43b19bdd6f6b3144fAndrew Scull // extended to a stack slot size because the PNaCl ABI requires arguments to 5344713dbdecbdd74f77cbb1cae43b19bdd6f6b3144fAndrew Scull // be at least 32 bits wide. 5345713dbdecbdd74f77cbb1cae43b19bdd6f6b3144fAndrew Scull Operand *ValExt; 5346713dbdecbdd74f77cbb1cae43b19bdd6f6b3144fAndrew Scull if (IsValConst) { 5347713dbdecbdd74f77cbb1cae43b19bdd6f6b3144fAndrew Scull ValExt = Ctx->getConstantInt(stackSlotType(), ValValue); 5348713dbdecbdd74f77cbb1cae43b19bdd6f6b3144fAndrew Scull } else { 5349713dbdecbdd74f77cbb1cae43b19bdd6f6b3144fAndrew Scull Variable *ValExtVar = Func->makeVariable(stackSlotType()); 5350713dbdecbdd74f77cbb1cae43b19bdd6f6b3144fAndrew Scull lowerCast(InstCast::create(Func, InstCast::Zext, ValExtVar, Val)); 5351713dbdecbdd74f77cbb1cae43b19bdd6f6b3144fAndrew Scull ValExt = ValExtVar; 5352713dbdecbdd74f77cbb1cae43b19bdd6f6b3144fAndrew Scull } 535320070e84c21b9aa15a83e53b5d2ffa692c4e081aKarl Schimpf InstCall *Call = makeHelperCall(RuntimeHelper::H_call_memset, nullptr, 3); 5354713dbdecbdd74f77cbb1cae43b19bdd6f6b3144fAndrew Scull Call->addArg(Dest); 5355713dbdecbdd74f77cbb1cae43b19bdd6f6b3144fAndrew Scull Call->addArg(ValExt); 5356713dbdecbdd74f77cbb1cae43b19bdd6f6b3144fAndrew Scull Call->addArg(Count); 5357713dbdecbdd74f77cbb1cae43b19bdd6f6b3144fAndrew Scull lowerCall(Call); 5358713dbdecbdd74f77cbb1cae43b19bdd6f6b3144fAndrew Scull} 5359713dbdecbdd74f77cbb1cae43b19bdd6f6b3144fAndrew Scull 5360ac2388c385afda17acb3fc242b17b9232ec2262aJohn Portoclass AddressOptimizer { 5361ac2388c385afda17acb3fc242b17b9232ec2262aJohn Porto AddressOptimizer() = delete; 5362ac2388c385afda17acb3fc242b17b9232ec2262aJohn Porto AddressOptimizer(const AddressOptimizer &) = delete; 5363ac2388c385afda17acb3fc242b17b9232ec2262aJohn Porto AddressOptimizer &operator=(const AddressOptimizer &) = delete; 5364ac2388c385afda17acb3fc242b17b9232ec2262aJohn Porto 5365ac2388c385afda17acb3fc242b17b9232ec2262aJohn Portopublic: 5366ac2388c385afda17acb3fc242b17b9232ec2262aJohn Porto explicit AddressOptimizer(const Cfg *Func) 5367ac2388c385afda17acb3fc242b17b9232ec2262aJohn Porto : Func(Func), VMetadata(Func->getVMetadata()) {} 5368ac2388c385afda17acb3fc242b17b9232ec2262aJohn Porto 5369ac2388c385afda17acb3fc242b17b9232ec2262aJohn Porto inline void dumpAddressOpt(const ConstantRelocatable *const Relocatable, 5370ac2388c385afda17acb3fc242b17b9232ec2262aJohn Porto int32_t Offset, const Variable *Base, 5371ac2388c385afda17acb3fc242b17b9232ec2262aJohn Porto const Variable *Index, uint16_t Shift, 5372ac2388c385afda17acb3fc242b17b9232ec2262aJohn Porto const Inst *Reason) const; 5373ac2388c385afda17acb3fc242b17b9232ec2262aJohn Porto 5374ac2388c385afda17acb3fc242b17b9232ec2262aJohn Porto inline const Inst *matchAssign(Variable **Var, 5375ac2388c385afda17acb3fc242b17b9232ec2262aJohn Porto ConstantRelocatable **Relocatable, 5376ac2388c385afda17acb3fc242b17b9232ec2262aJohn Porto int32_t *Offset); 5377ac2388c385afda17acb3fc242b17b9232ec2262aJohn Porto 5378ac2388c385afda17acb3fc242b17b9232ec2262aJohn Porto inline const Inst *matchCombinedBaseIndex(Variable **Base, Variable **Index, 5379ac2388c385afda17acb3fc242b17b9232ec2262aJohn Porto uint16_t *Shift); 5380ac2388c385afda17acb3fc242b17b9232ec2262aJohn Porto 5381ac2388c385afda17acb3fc242b17b9232ec2262aJohn Porto inline const Inst *matchShiftedIndex(Variable **Index, uint16_t *Shift); 5382ac2388c385afda17acb3fc242b17b9232ec2262aJohn Porto 53832e4b960b20e6ed2ad9453657fe8d34d0e7fd933cManasij Mukherjee inline const Inst *matchOffsetIndexOrBase(Variable **IndexOrBase, 53842e4b960b20e6ed2ad9453657fe8d34d0e7fd933cManasij Mukherjee const uint16_t Shift, 53852e4b960b20e6ed2ad9453657fe8d34d0e7fd933cManasij Mukherjee ConstantRelocatable **Relocatable, 53862e4b960b20e6ed2ad9453657fe8d34d0e7fd933cManasij Mukherjee int32_t *Offset); 5387ac2388c385afda17acb3fc242b17b9232ec2262aJohn Porto 5388ac2388c385afda17acb3fc242b17b9232ec2262aJohn Portoprivate: 5389ac2388c385afda17acb3fc242b17b9232ec2262aJohn Porto const Cfg *const Func; 5390ac2388c385afda17acb3fc242b17b9232ec2262aJohn Porto const VariablesMetadata *const VMetadata; 5391ac2388c385afda17acb3fc242b17b9232ec2262aJohn Porto 53928cfeb69e17190d3bfe22a8a1cbd7679a114d68cfJim Stichnoth static bool isAdd(const Inst *Instr) { 53938cfeb69e17190d3bfe22a8a1cbd7679a114d68cfJim Stichnoth if (auto *Arith = llvm::dyn_cast_or_null<const InstArithmetic>(Instr)) { 5394ac2388c385afda17acb3fc242b17b9232ec2262aJohn Porto return (Arith->getOp() == InstArithmetic::Add); 5395ac2388c385afda17acb3fc242b17b9232ec2262aJohn Porto } 5396ac2388c385afda17acb3fc242b17b9232ec2262aJohn Porto return false; 53977e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto } 5398ac2388c385afda17acb3fc242b17b9232ec2262aJohn Porto}; 53997e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto 5400ac2388c385afda17acb3fc242b17b9232ec2262aJohn Portovoid AddressOptimizer::dumpAddressOpt( 5401ac2388c385afda17acb3fc242b17b9232ec2262aJohn Porto const ConstantRelocatable *const Relocatable, int32_t Offset, 5402ac2388c385afda17acb3fc242b17b9232ec2262aJohn Porto const Variable *Base, const Variable *Index, uint16_t Shift, 5403ac2388c385afda17acb3fc242b17b9232ec2262aJohn Porto const Inst *Reason) const { 540420b71f5890ee8651983b126c5978594a01e0af96Jim Stichnoth if (!BuildDefs::dump()) 54057e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto return; 54067e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto if (!Func->isVerbose(IceV_AddrOpt)) 54077e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto return; 54087e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto OstreamLocker L(Func->getContext()); 54097e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto Ostream &Str = Func->getContext()->getStrDump(); 54107e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto Str << "Instruction: "; 54117e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto Reason->dumpDecorated(Func); 54127e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto Str << " results in Base="; 54137e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto if (Base) 54147e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto Base->dump(Func); 54157e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto else 54167e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto Str << "<null>"; 54177e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto Str << ", Index="; 54187e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto if (Index) 54197e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto Index->dump(Func); 54207e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto else 54217e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto Str << "<null>"; 5422aa0b1a176f6c3306be3e95374d72e735bab2bbabDavid Sehr Str << ", Shift=" << Shift << ", Offset=" << Offset 5423aa0b1a176f6c3306be3e95374d72e735bab2bbabDavid Sehr << ", Relocatable=" << Relocatable << "\n"; 54247e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto} 54257e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto 5426ac2388c385afda17acb3fc242b17b9232ec2262aJohn Portoconst Inst *AddressOptimizer::matchAssign(Variable **Var, 5427ac2388c385afda17acb3fc242b17b9232ec2262aJohn Porto ConstantRelocatable **Relocatable, 5428ac2388c385afda17acb3fc242b17b9232ec2262aJohn Porto int32_t *Offset) { 542957e126899b20c65ff3ea23a3b7d7a67ab30b99dcAndrew Scull // Var originates from Var=SrcVar ==> set Var:=SrcVar 5430ac2388c385afda17acb3fc242b17b9232ec2262aJohn Porto if (*Var == nullptr) 5431ac2388c385afda17acb3fc242b17b9232ec2262aJohn Porto return nullptr; 5432ac2388c385afda17acb3fc242b17b9232ec2262aJohn Porto if (const Inst *VarAssign = VMetadata->getSingleDefinition(*Var)) { 5433ac2388c385afda17acb3fc242b17b9232ec2262aJohn Porto assert(!VMetadata->isMultiDef(*Var)); 54347e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto if (llvm::isa<InstAssign>(VarAssign)) { 54357e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto Operand *SrcOp = VarAssign->getSrc(0); 54367e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto assert(SrcOp); 5437aa0b1a176f6c3306be3e95374d72e735bab2bbabDavid Sehr if (auto *SrcVar = llvm::dyn_cast<Variable>(SrcOp)) { 54387e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto if (!VMetadata->isMultiDef(SrcVar) && 54397e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto // TODO: ensure SrcVar stays single-BB 54407e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto true) { 5441ac2388c385afda17acb3fc242b17b9232ec2262aJohn Porto *Var = SrcVar; 5442ac2388c385afda17acb3fc242b17b9232ec2262aJohn Porto return VarAssign; 54437e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto } 5444aa0b1a176f6c3306be3e95374d72e735bab2bbabDavid Sehr } else if (auto *Const = llvm::dyn_cast<ConstantInteger32>(SrcOp)) { 5445aa0b1a176f6c3306be3e95374d72e735bab2bbabDavid Sehr int32_t MoreOffset = Const->getValue(); 5446ac2388c385afda17acb3fc242b17b9232ec2262aJohn Porto if (Utils::WouldOverflowAdd(*Offset, MoreOffset)) 5447ac2388c385afda17acb3fc242b17b9232ec2262aJohn Porto return nullptr; 5448ac2388c385afda17acb3fc242b17b9232ec2262aJohn Porto *Var = nullptr; 54492220990ebb0419cca8938bd023a13da4eff47aa1Nicolas Capens *Offset += MoreOffset; 5450ac2388c385afda17acb3fc242b17b9232ec2262aJohn Porto return VarAssign; 5451aa0b1a176f6c3306be3e95374d72e735bab2bbabDavid Sehr } else if (auto *AddReloc = llvm::dyn_cast<ConstantRelocatable>(SrcOp)) { 5452ac2388c385afda17acb3fc242b17b9232ec2262aJohn Porto if (*Relocatable == nullptr) { 5453ac2388c385afda17acb3fc242b17b9232ec2262aJohn Porto // It is always safe to fold a relocatable through assignment -- the 5454ac2388c385afda17acb3fc242b17b9232ec2262aJohn Porto // assignment frees a slot in the address operand that can be used to 5455ac2388c385afda17acb3fc242b17b9232ec2262aJohn Porto // hold the Sandbox Pointer -- if any. 5456ac2388c385afda17acb3fc242b17b9232ec2262aJohn Porto *Var = nullptr; 5457ac2388c385afda17acb3fc242b17b9232ec2262aJohn Porto *Relocatable = AddReloc; 5458ac2388c385afda17acb3fc242b17b9232ec2262aJohn Porto return VarAssign; 5459aa0b1a176f6c3306be3e95374d72e735bab2bbabDavid Sehr } 54607e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto } 54617e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto } 54627e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto } 5463ac2388c385afda17acb3fc242b17b9232ec2262aJohn Porto return nullptr; 54647e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto} 54657e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto 5466ac2388c385afda17acb3fc242b17b9232ec2262aJohn Portoconst Inst *AddressOptimizer::matchCombinedBaseIndex(Variable **Base, 5467ac2388c385afda17acb3fc242b17b9232ec2262aJohn Porto Variable **Index, 5468ac2388c385afda17acb3fc242b17b9232ec2262aJohn Porto uint16_t *Shift) { 54697e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto // Index==nullptr && Base is Base=Var1+Var2 ==> 54707e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto // set Base=Var1, Index=Var2, Shift=0 5471ac2388c385afda17acb3fc242b17b9232ec2262aJohn Porto if (*Base == nullptr) 5472ac2388c385afda17acb3fc242b17b9232ec2262aJohn Porto return nullptr; 5473ac2388c385afda17acb3fc242b17b9232ec2262aJohn Porto if (*Index != nullptr) 5474ac2388c385afda17acb3fc242b17b9232ec2262aJohn Porto return nullptr; 5475ac2388c385afda17acb3fc242b17b9232ec2262aJohn Porto auto *BaseInst = VMetadata->getSingleDefinition(*Base); 54767e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto if (BaseInst == nullptr) 5477ac2388c385afda17acb3fc242b17b9232ec2262aJohn Porto return nullptr; 5478ac2388c385afda17acb3fc242b17b9232ec2262aJohn Porto assert(!VMetadata->isMultiDef(*Base)); 54797e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto if (BaseInst->getSrcSize() < 2) 5480ac2388c385afda17acb3fc242b17b9232ec2262aJohn Porto return nullptr; 5481aa0b1a176f6c3306be3e95374d72e735bab2bbabDavid Sehr if (auto *Var1 = llvm::dyn_cast<Variable>(BaseInst->getSrc(0))) { 54827e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto if (VMetadata->isMultiDef(Var1)) 5483ac2388c385afda17acb3fc242b17b9232ec2262aJohn Porto return nullptr; 5484aa0b1a176f6c3306be3e95374d72e735bab2bbabDavid Sehr if (auto *Var2 = llvm::dyn_cast<Variable>(BaseInst->getSrc(1))) { 54857e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto if (VMetadata->isMultiDef(Var2)) 5486ac2388c385afda17acb3fc242b17b9232ec2262aJohn Porto return nullptr; 54877e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto if (isAdd(BaseInst) && 54887e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto // TODO: ensure Var1 and Var2 stay single-BB 54897e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto true) { 5490ac2388c385afda17acb3fc242b17b9232ec2262aJohn Porto *Base = Var1; 5491ac2388c385afda17acb3fc242b17b9232ec2262aJohn Porto *Index = Var2; 5492ac2388c385afda17acb3fc242b17b9232ec2262aJohn Porto *Shift = 0; // should already have been 0 5493ac2388c385afda17acb3fc242b17b9232ec2262aJohn Porto return BaseInst; 54947e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto } 54957e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto } 54967e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto } 5497ac2388c385afda17acb3fc242b17b9232ec2262aJohn Porto return nullptr; 54987e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto} 54997e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto 5500ac2388c385afda17acb3fc242b17b9232ec2262aJohn Portoconst Inst *AddressOptimizer::matchShiftedIndex(Variable **Index, 5501ac2388c385afda17acb3fc242b17b9232ec2262aJohn Porto uint16_t *Shift) { 55027e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto // Index is Index=Var*Const && log2(Const)+Shift<=3 ==> 55037e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto // Index=Var, Shift+=log2(Const) 5504ac2388c385afda17acb3fc242b17b9232ec2262aJohn Porto if (*Index == nullptr) 5505ac2388c385afda17acb3fc242b17b9232ec2262aJohn Porto return nullptr; 5506ac2388c385afda17acb3fc242b17b9232ec2262aJohn Porto auto *IndexInst = VMetadata->getSingleDefinition(*Index); 55077e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto if (IndexInst == nullptr) 5508ac2388c385afda17acb3fc242b17b9232ec2262aJohn Porto return nullptr; 5509ac2388c385afda17acb3fc242b17b9232ec2262aJohn Porto assert(!VMetadata->isMultiDef(*Index)); 5510aaf04f586f4e94a04e20f47c3d7268a607b8a1e0Nicolas Capens 5511aaf04f586f4e94a04e20f47c3d7268a607b8a1e0Nicolas Capens // When using an unsigned 32-bit array index on x64, it gets zero-extended 5512aaf04f586f4e94a04e20f47c3d7268a607b8a1e0Nicolas Capens // before the shift & add. The explicit zero extension can be eliminated 5513aaf04f586f4e94a04e20f47c3d7268a607b8a1e0Nicolas Capens // because x86 32-bit operations automatically get zero-extended into the 5514aaf04f586f4e94a04e20f47c3d7268a607b8a1e0Nicolas Capens // corresponding 64-bit register. 5515aaf04f586f4e94a04e20f47c3d7268a607b8a1e0Nicolas Capens if (auto *CastInst = llvm::dyn_cast<InstCast>(IndexInst)) { 5516aaf04f586f4e94a04e20f47c3d7268a607b8a1e0Nicolas Capens if (CastInst->getCastKind() == InstCast::Zext) { 5517aaf04f586f4e94a04e20f47c3d7268a607b8a1e0Nicolas Capens if (auto *Var = llvm::dyn_cast<Variable>(CastInst->getSrc(0))) { 5518aaf04f586f4e94a04e20f47c3d7268a607b8a1e0Nicolas Capens if (Var->getType() == IceType_i32 && 5519aaf04f586f4e94a04e20f47c3d7268a607b8a1e0Nicolas Capens CastInst->getDest()->getType() == IceType_i64) { 5520aaf04f586f4e94a04e20f47c3d7268a607b8a1e0Nicolas Capens IndexInst = VMetadata->getSingleDefinition(Var); 5521aaf04f586f4e94a04e20f47c3d7268a607b8a1e0Nicolas Capens } 5522aaf04f586f4e94a04e20f47c3d7268a607b8a1e0Nicolas Capens } 5523aaf04f586f4e94a04e20f47c3d7268a607b8a1e0Nicolas Capens } 5524aaf04f586f4e94a04e20f47c3d7268a607b8a1e0Nicolas Capens } 5525aaf04f586f4e94a04e20f47c3d7268a607b8a1e0Nicolas Capens 55267e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto if (IndexInst->getSrcSize() < 2) 5527ac2388c385afda17acb3fc242b17b9232ec2262aJohn Porto return nullptr; 5528aa0b1a176f6c3306be3e95374d72e735bab2bbabDavid Sehr if (auto *ArithInst = llvm::dyn_cast<InstArithmetic>(IndexInst)) { 5529aa0b1a176f6c3306be3e95374d72e735bab2bbabDavid Sehr if (auto *Var = llvm::dyn_cast<Variable>(ArithInst->getSrc(0))) { 5530aa0b1a176f6c3306be3e95374d72e735bab2bbabDavid Sehr if (auto *Const = 55317e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto llvm::dyn_cast<ConstantInteger32>(ArithInst->getSrc(1))) { 5532aa0b1a176f6c3306be3e95374d72e735bab2bbabDavid Sehr if (VMetadata->isMultiDef(Var) || Const->getType() != IceType_i32) 5533ac2388c385afda17acb3fc242b17b9232ec2262aJohn Porto return nullptr; 5534aa0b1a176f6c3306be3e95374d72e735bab2bbabDavid Sehr switch (ArithInst->getOp()) { 5535aa0b1a176f6c3306be3e95374d72e735bab2bbabDavid Sehr default: 5536ac2388c385afda17acb3fc242b17b9232ec2262aJohn Porto return nullptr; 5537aa0b1a176f6c3306be3e95374d72e735bab2bbabDavid Sehr case InstArithmetic::Mul: { 5538aa0b1a176f6c3306be3e95374d72e735bab2bbabDavid Sehr uint32_t Mult = Const->getValue(); 55397e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto uint32_t LogMult; 55407e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto switch (Mult) { 55417e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto case 1: 55427e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto LogMult = 0; 55437e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto break; 55447e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto case 2: 55457e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto LogMult = 1; 55467e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto break; 55477e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto case 4: 55487e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto LogMult = 2; 55497e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto break; 55507e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto case 8: 55517e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto LogMult = 3; 55527e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto break; 55537e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto default: 5554ac2388c385afda17acb3fc242b17b9232ec2262aJohn Porto return nullptr; 55557e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto } 5556ac2388c385afda17acb3fc242b17b9232ec2262aJohn Porto if (*Shift + LogMult <= 3) { 5557ac2388c385afda17acb3fc242b17b9232ec2262aJohn Porto *Index = Var; 5558ac2388c385afda17acb3fc242b17b9232ec2262aJohn Porto *Shift += LogMult; 5559ac2388c385afda17acb3fc242b17b9232ec2262aJohn Porto return IndexInst; 55607e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto } 55617e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto } 5562aa0b1a176f6c3306be3e95374d72e735bab2bbabDavid Sehr case InstArithmetic::Shl: { 5563aa0b1a176f6c3306be3e95374d72e735bab2bbabDavid Sehr uint32_t ShiftAmount = Const->getValue(); 5564aa0b1a176f6c3306be3e95374d72e735bab2bbabDavid Sehr switch (ShiftAmount) { 5565aa0b1a176f6c3306be3e95374d72e735bab2bbabDavid Sehr case 0: 5566aa0b1a176f6c3306be3e95374d72e735bab2bbabDavid Sehr case 1: 5567aa0b1a176f6c3306be3e95374d72e735bab2bbabDavid Sehr case 2: 5568aa0b1a176f6c3306be3e95374d72e735bab2bbabDavid Sehr case 3: 5569aa0b1a176f6c3306be3e95374d72e735bab2bbabDavid Sehr break; 5570aa0b1a176f6c3306be3e95374d72e735bab2bbabDavid Sehr default: 5571ac2388c385afda17acb3fc242b17b9232ec2262aJohn Porto return nullptr; 5572aa0b1a176f6c3306be3e95374d72e735bab2bbabDavid Sehr } 5573ac2388c385afda17acb3fc242b17b9232ec2262aJohn Porto if (*Shift + ShiftAmount <= 3) { 5574ac2388c385afda17acb3fc242b17b9232ec2262aJohn Porto *Index = Var; 5575ac2388c385afda17acb3fc242b17b9232ec2262aJohn Porto *Shift += ShiftAmount; 5576ac2388c385afda17acb3fc242b17b9232ec2262aJohn Porto return IndexInst; 5577aa0b1a176f6c3306be3e95374d72e735bab2bbabDavid Sehr } 5578aa0b1a176f6c3306be3e95374d72e735bab2bbabDavid Sehr } 5579aa0b1a176f6c3306be3e95374d72e735bab2bbabDavid Sehr } 55807e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto } 55817e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto } 55827e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto } 5583ac2388c385afda17acb3fc242b17b9232ec2262aJohn Porto return nullptr; 55847e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto} 55857e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto 55862e4b960b20e6ed2ad9453657fe8d34d0e7fd933cManasij Mukherjeeconst Inst *AddressOptimizer::matchOffsetIndexOrBase( 55872e4b960b20e6ed2ad9453657fe8d34d0e7fd933cManasij Mukherjee Variable **IndexOrBase, const uint16_t Shift, 55882e4b960b20e6ed2ad9453657fe8d34d0e7fd933cManasij Mukherjee ConstantRelocatable **Relocatable, int32_t *Offset) { 55897e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto // Base is Base=Var+Const || Base is Base=Const+Var ==> 55907e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto // set Base=Var, Offset+=Const 55917e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto // Base is Base=Var-Const ==> 55927e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto // set Base=Var, Offset-=Const 55932e4b960b20e6ed2ad9453657fe8d34d0e7fd933cManasij Mukherjee // Index is Index=Var+Const ==> 55942e4b960b20e6ed2ad9453657fe8d34d0e7fd933cManasij Mukherjee // set Index=Var, Offset+=(Const<<Shift) 55952e4b960b20e6ed2ad9453657fe8d34d0e7fd933cManasij Mukherjee // Index is Index=Const+Var ==> 55962e4b960b20e6ed2ad9453657fe8d34d0e7fd933cManasij Mukherjee // set Index=Var, Offset+=(Const<<Shift) 55972e4b960b20e6ed2ad9453657fe8d34d0e7fd933cManasij Mukherjee // Index is Index=Var-Const ==> 55982e4b960b20e6ed2ad9453657fe8d34d0e7fd933cManasij Mukherjee // set Index=Var, Offset-=(Const<<Shift) 55999dd397d322848cbc9c54e80acb6dc0a2b30aaeb7Manasij Mukherjee // Treat Index=Var Or Const as Index=Var + Const 56009dd397d322848cbc9c54e80acb6dc0a2b30aaeb7Manasij Mukherjee // when Var = Var' << N and log2(Const) <= N 56019dd397d322848cbc9c54e80acb6dc0a2b30aaeb7Manasij Mukherjee // or when Var = (2^M) * (2^N) and log2(Const) <= (M+N) 56022e4b960b20e6ed2ad9453657fe8d34d0e7fd933cManasij Mukherjee 56032e4b960b20e6ed2ad9453657fe8d34d0e7fd933cManasij Mukherjee if (*IndexOrBase == nullptr) { 5604ac2388c385afda17acb3fc242b17b9232ec2262aJohn Porto return nullptr; 5605aa0b1a176f6c3306be3e95374d72e735bab2bbabDavid Sehr } 56062e4b960b20e6ed2ad9453657fe8d34d0e7fd933cManasij Mukherjee const Inst *Definition = VMetadata->getSingleDefinition(*IndexOrBase); 56072e4b960b20e6ed2ad9453657fe8d34d0e7fd933cManasij Mukherjee if (Definition == nullptr) { 5608ac2388c385afda17acb3fc242b17b9232ec2262aJohn Porto return nullptr; 5609aa0b1a176f6c3306be3e95374d72e735bab2bbabDavid Sehr } 56102e4b960b20e6ed2ad9453657fe8d34d0e7fd933cManasij Mukherjee assert(!VMetadata->isMultiDef(*IndexOrBase)); 56112e4b960b20e6ed2ad9453657fe8d34d0e7fd933cManasij Mukherjee if (auto *ArithInst = llvm::dyn_cast<const InstArithmetic>(Definition)) { 56129dd397d322848cbc9c54e80acb6dc0a2b30aaeb7Manasij Mukherjee switch (ArithInst->getOp()) { 56139dd397d322848cbc9c54e80acb6dc0a2b30aaeb7Manasij Mukherjee case InstArithmetic::Add: 56149dd397d322848cbc9c54e80acb6dc0a2b30aaeb7Manasij Mukherjee case InstArithmetic::Sub: 56159dd397d322848cbc9c54e80acb6dc0a2b30aaeb7Manasij Mukherjee case InstArithmetic::Or: 56169dd397d322848cbc9c54e80acb6dc0a2b30aaeb7Manasij Mukherjee break; 56179dd397d322848cbc9c54e80acb6dc0a2b30aaeb7Manasij Mukherjee default: 5618ac2388c385afda17acb3fc242b17b9232ec2262aJohn Porto return nullptr; 56199dd397d322848cbc9c54e80acb6dc0a2b30aaeb7Manasij Mukherjee } 56209dd397d322848cbc9c54e80acb6dc0a2b30aaeb7Manasij Mukherjee 5621aa0b1a176f6c3306be3e95374d72e735bab2bbabDavid Sehr Operand *Src0 = ArithInst->getSrc(0); 5622aa0b1a176f6c3306be3e95374d72e735bab2bbabDavid Sehr Operand *Src1 = ArithInst->getSrc(1); 5623aa0b1a176f6c3306be3e95374d72e735bab2bbabDavid Sehr auto *Var0 = llvm::dyn_cast<Variable>(Src0); 5624aa0b1a176f6c3306be3e95374d72e735bab2bbabDavid Sehr auto *Var1 = llvm::dyn_cast<Variable>(Src1); 5625aa0b1a176f6c3306be3e95374d72e735bab2bbabDavid Sehr auto *Const0 = llvm::dyn_cast<ConstantInteger32>(Src0); 5626aa0b1a176f6c3306be3e95374d72e735bab2bbabDavid Sehr auto *Const1 = llvm::dyn_cast<ConstantInteger32>(Src1); 5627aa0b1a176f6c3306be3e95374d72e735bab2bbabDavid Sehr auto *Reloc0 = llvm::dyn_cast<ConstantRelocatable>(Src0); 5628aa0b1a176f6c3306be3e95374d72e735bab2bbabDavid Sehr auto *Reloc1 = llvm::dyn_cast<ConstantRelocatable>(Src1); 56299dd397d322848cbc9c54e80acb6dc0a2b30aaeb7Manasij Mukherjee 56309dd397d322848cbc9c54e80acb6dc0a2b30aaeb7Manasij Mukherjee bool IsAdd = false; 56319dd397d322848cbc9c54e80acb6dc0a2b30aaeb7Manasij Mukherjee if (ArithInst->getOp() == InstArithmetic::Or) { 56329dd397d322848cbc9c54e80acb6dc0a2b30aaeb7Manasij Mukherjee Variable *Var = nullptr; 56339dd397d322848cbc9c54e80acb6dc0a2b30aaeb7Manasij Mukherjee ConstantInteger32 *Const = nullptr; 56349dd397d322848cbc9c54e80acb6dc0a2b30aaeb7Manasij Mukherjee if (Var0 && Const1) { 56359dd397d322848cbc9c54e80acb6dc0a2b30aaeb7Manasij Mukherjee Var = Var0; 56369dd397d322848cbc9c54e80acb6dc0a2b30aaeb7Manasij Mukherjee Const = Const1; 56379dd397d322848cbc9c54e80acb6dc0a2b30aaeb7Manasij Mukherjee } else if (Const0 && Var1) { 56389dd397d322848cbc9c54e80acb6dc0a2b30aaeb7Manasij Mukherjee Var = Var1; 56399dd397d322848cbc9c54e80acb6dc0a2b30aaeb7Manasij Mukherjee Const = Const0; 56409dd397d322848cbc9c54e80acb6dc0a2b30aaeb7Manasij Mukherjee } else { 56419dd397d322848cbc9c54e80acb6dc0a2b30aaeb7Manasij Mukherjee return nullptr; 56429dd397d322848cbc9c54e80acb6dc0a2b30aaeb7Manasij Mukherjee } 56439dd397d322848cbc9c54e80acb6dc0a2b30aaeb7Manasij Mukherjee auto *VarDef = 56449dd397d322848cbc9c54e80acb6dc0a2b30aaeb7Manasij Mukherjee llvm::dyn_cast<InstArithmetic>(VMetadata->getSingleDefinition(Var)); 56459dd397d322848cbc9c54e80acb6dc0a2b30aaeb7Manasij Mukherjee if (VarDef == nullptr) 56469dd397d322848cbc9c54e80acb6dc0a2b30aaeb7Manasij Mukherjee return nullptr; 56479dd397d322848cbc9c54e80acb6dc0a2b30aaeb7Manasij Mukherjee 56489dd397d322848cbc9c54e80acb6dc0a2b30aaeb7Manasij Mukherjee SizeT ZeroesAvailable = 0; 56499dd397d322848cbc9c54e80acb6dc0a2b30aaeb7Manasij Mukherjee if (VarDef->getOp() == InstArithmetic::Shl) { 56509dd397d322848cbc9c54e80acb6dc0a2b30aaeb7Manasij Mukherjee if (auto *ConstInt = 56519dd397d322848cbc9c54e80acb6dc0a2b30aaeb7Manasij Mukherjee llvm::dyn_cast<ConstantInteger32>(VarDef->getSrc(1))) { 56529dd397d322848cbc9c54e80acb6dc0a2b30aaeb7Manasij Mukherjee ZeroesAvailable = ConstInt->getValue(); 56539dd397d322848cbc9c54e80acb6dc0a2b30aaeb7Manasij Mukherjee } 56549dd397d322848cbc9c54e80acb6dc0a2b30aaeb7Manasij Mukherjee } else if (VarDef->getOp() == InstArithmetic::Mul) { 56559dd397d322848cbc9c54e80acb6dc0a2b30aaeb7Manasij Mukherjee SizeT PowerOfTwo = 0; 5656d615c8615ad5112f0e4359d08c51032cac7362b5Jim Stichnoth if (auto *MultConst = 5657d615c8615ad5112f0e4359d08c51032cac7362b5Jim Stichnoth llvm::dyn_cast<ConstantInteger32>(VarDef->getSrc(0))) { 5658d615c8615ad5112f0e4359d08c51032cac7362b5Jim Stichnoth if (llvm::isPowerOf2_32(MultConst->getValue())) { 5659d615c8615ad5112f0e4359d08c51032cac7362b5Jim Stichnoth PowerOfTwo += MultConst->getValue(); 5660d615c8615ad5112f0e4359d08c51032cac7362b5Jim Stichnoth } 56619dd397d322848cbc9c54e80acb6dc0a2b30aaeb7Manasij Mukherjee } 5662d615c8615ad5112f0e4359d08c51032cac7362b5Jim Stichnoth if (auto *MultConst = 5663d615c8615ad5112f0e4359d08c51032cac7362b5Jim Stichnoth llvm::dyn_cast<ConstantInteger32>(VarDef->getSrc(1))) { 5664d615c8615ad5112f0e4359d08c51032cac7362b5Jim Stichnoth if (llvm::isPowerOf2_32(MultConst->getValue())) { 5665d615c8615ad5112f0e4359d08c51032cac7362b5Jim Stichnoth PowerOfTwo += MultConst->getValue(); 5666d615c8615ad5112f0e4359d08c51032cac7362b5Jim Stichnoth } 56679dd397d322848cbc9c54e80acb6dc0a2b30aaeb7Manasij Mukherjee } 56689dd397d322848cbc9c54e80acb6dc0a2b30aaeb7Manasij Mukherjee ZeroesAvailable = llvm::Log2_32(PowerOfTwo) + 1; 56699dd397d322848cbc9c54e80acb6dc0a2b30aaeb7Manasij Mukherjee } 56709dd397d322848cbc9c54e80acb6dc0a2b30aaeb7Manasij Mukherjee SizeT ZeroesNeeded = llvm::Log2_32(Const->getValue()) + 1; 56719dd397d322848cbc9c54e80acb6dc0a2b30aaeb7Manasij Mukherjee if (ZeroesNeeded == 0 || ZeroesNeeded > ZeroesAvailable) 56729dd397d322848cbc9c54e80acb6dc0a2b30aaeb7Manasij Mukherjee return nullptr; 56739dd397d322848cbc9c54e80acb6dc0a2b30aaeb7Manasij Mukherjee IsAdd = true; // treat it as an add if the above conditions hold 56749dd397d322848cbc9c54e80acb6dc0a2b30aaeb7Manasij Mukherjee } else { 56759dd397d322848cbc9c54e80acb6dc0a2b30aaeb7Manasij Mukherjee IsAdd = ArithInst->getOp() == InstArithmetic::Add; 56769dd397d322848cbc9c54e80acb6dc0a2b30aaeb7Manasij Mukherjee } 56779dd397d322848cbc9c54e80acb6dc0a2b30aaeb7Manasij Mukherjee 56782e4b960b20e6ed2ad9453657fe8d34d0e7fd933cManasij Mukherjee Variable *NewIndexOrBase = nullptr; 56792e4b960b20e6ed2ad9453657fe8d34d0e7fd933cManasij Mukherjee int32_t NewOffset = 0; 5680ac2388c385afda17acb3fc242b17b9232ec2262aJohn Porto ConstantRelocatable *NewRelocatable = *Relocatable; 5681aa0b1a176f6c3306be3e95374d72e735bab2bbabDavid Sehr if (Var0 && Var1) 5682aa0b1a176f6c3306be3e95374d72e735bab2bbabDavid Sehr // TODO(sehr): merge base/index splitting into here. 5683ac2388c385afda17acb3fc242b17b9232ec2262aJohn Porto return nullptr; 5684aa0b1a176f6c3306be3e95374d72e735bab2bbabDavid Sehr if (!IsAdd && Var1) 5685ac2388c385afda17acb3fc242b17b9232ec2262aJohn Porto return nullptr; 5686aa0b1a176f6c3306be3e95374d72e735bab2bbabDavid Sehr if (Var0) 56872e4b960b20e6ed2ad9453657fe8d34d0e7fd933cManasij Mukherjee NewIndexOrBase = Var0; 5688aa0b1a176f6c3306be3e95374d72e735bab2bbabDavid Sehr else if (Var1) 56892e4b960b20e6ed2ad9453657fe8d34d0e7fd933cManasij Mukherjee NewIndexOrBase = Var1; 5690aa0b1a176f6c3306be3e95374d72e735bab2bbabDavid Sehr // Don't know how to add/subtract two relocatables. 5691ac2388c385afda17acb3fc242b17b9232ec2262aJohn Porto if ((*Relocatable && (Reloc0 || Reloc1)) || (Reloc0 && Reloc1)) 5692ac2388c385afda17acb3fc242b17b9232ec2262aJohn Porto return nullptr; 5693aa0b1a176f6c3306be3e95374d72e735bab2bbabDavid Sehr // Don't know how to subtract a relocatable. 5694aa0b1a176f6c3306be3e95374d72e735bab2bbabDavid Sehr if (!IsAdd && Reloc1) 5695ac2388c385afda17acb3fc242b17b9232ec2262aJohn Porto return nullptr; 5696aa0b1a176f6c3306be3e95374d72e735bab2bbabDavid Sehr // Incorporate ConstantRelocatables. 5697aa0b1a176f6c3306be3e95374d72e735bab2bbabDavid Sehr if (Reloc0) 5698aa0b1a176f6c3306be3e95374d72e735bab2bbabDavid Sehr NewRelocatable = Reloc0; 5699aa0b1a176f6c3306be3e95374d72e735bab2bbabDavid Sehr else if (Reloc1) 5700aa0b1a176f6c3306be3e95374d72e735bab2bbabDavid Sehr NewRelocatable = Reloc1; 5701aa0b1a176f6c3306be3e95374d72e735bab2bbabDavid Sehr // Compute the updated constant offset. 5702aa0b1a176f6c3306be3e95374d72e735bab2bbabDavid Sehr if (Const0) { 570356958cb33d3c1d045f2844408d825442d523f59fJohn Porto const int32_t MoreOffset = 570456958cb33d3c1d045f2844408d825442d523f59fJohn Porto IsAdd ? Const0->getValue() : -Const0->getValue(); 57052e4b960b20e6ed2ad9453657fe8d34d0e7fd933cManasij Mukherjee if (Utils::WouldOverflowAdd(*Offset + NewOffset, MoreOffset)) 5706ac2388c385afda17acb3fc242b17b9232ec2262aJohn Porto return nullptr; 5707aa0b1a176f6c3306be3e95374d72e735bab2bbabDavid Sehr NewOffset += MoreOffset; 5708aa0b1a176f6c3306be3e95374d72e735bab2bbabDavid Sehr } 5709aa0b1a176f6c3306be3e95374d72e735bab2bbabDavid Sehr if (Const1) { 571056958cb33d3c1d045f2844408d825442d523f59fJohn Porto const int32_t MoreOffset = 571156958cb33d3c1d045f2844408d825442d523f59fJohn Porto IsAdd ? Const1->getValue() : -Const1->getValue(); 57122e4b960b20e6ed2ad9453657fe8d34d0e7fd933cManasij Mukherjee if (Utils::WouldOverflowAdd(*Offset + NewOffset, MoreOffset)) 5713ac2388c385afda17acb3fc242b17b9232ec2262aJohn Porto return nullptr; 5714aa0b1a176f6c3306be3e95374d72e735bab2bbabDavid Sehr NewOffset += MoreOffset; 5715aa0b1a176f6c3306be3e95374d72e735bab2bbabDavid Sehr } 57162e4b960b20e6ed2ad9453657fe8d34d0e7fd933cManasij Mukherjee if (Utils::WouldOverflowAdd(*Offset, NewOffset << Shift)) 57179dd397d322848cbc9c54e80acb6dc0a2b30aaeb7Manasij Mukherjee return nullptr; 57182e4b960b20e6ed2ad9453657fe8d34d0e7fd933cManasij Mukherjee *IndexOrBase = NewIndexOrBase; 57192e4b960b20e6ed2ad9453657fe8d34d0e7fd933cManasij Mukherjee *Offset += (NewOffset << Shift); 57202e4b960b20e6ed2ad9453657fe8d34d0e7fd933cManasij Mukherjee // Shift is always zero if this is called with the base 5721ac2388c385afda17acb3fc242b17b9232ec2262aJohn Porto *Relocatable = NewRelocatable; 57222e4b960b20e6ed2ad9453657fe8d34d0e7fd933cManasij Mukherjee return Definition; 57237e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto } 5724ac2388c385afda17acb3fc242b17b9232ec2262aJohn Porto return nullptr; 57257e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto} 57267e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto 5727ac2388c385afda17acb3fc242b17b9232ec2262aJohn Portotemplate <typename TypeTraits> 5728ac2388c385afda17acb3fc242b17b9232ec2262aJohn Portotypename TargetX86Base<TypeTraits>::X86OperandMem * 5729ac2388c385afda17acb3fc242b17b9232ec2262aJohn PortoTargetX86Base<TypeTraits>::computeAddressOpt(const Inst *Instr, Type MemType, 5730ac2388c385afda17acb3fc242b17b9232ec2262aJohn Porto Operand *Addr) { 57317e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto Func->resetCurrentNode(); 57327e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto if (Func->isVerbose(IceV_AddrOpt)) { 57337e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto OstreamLocker L(Func->getContext()); 57347e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto Ostream &Str = Func->getContext()->getStrDump(); 57357e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto Str << "\nStarting computeAddressOpt for instruction:\n "; 57367e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto Instr->dumpDecorated(Func); 57377e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto } 5738ac2388c385afda17acb3fc242b17b9232ec2262aJohn Porto 5739ac2388c385afda17acb3fc242b17b9232ec2262aJohn Porto OptAddr NewAddr; 5740ac2388c385afda17acb3fc242b17b9232ec2262aJohn Porto NewAddr.Base = llvm::dyn_cast<Variable>(Addr); 5741ac2388c385afda17acb3fc242b17b9232ec2262aJohn Porto if (NewAddr.Base == nullptr) 5742ac2388c385afda17acb3fc242b17b9232ec2262aJohn Porto return nullptr; 5743ac2388c385afda17acb3fc242b17b9232ec2262aJohn Porto 574457e126899b20c65ff3ea23a3b7d7a67ab30b99dcAndrew Scull // If the Base has more than one use or is live across multiple blocks, then 574557e126899b20c65ff3ea23a3b7d7a67ab30b99dcAndrew Scull // don't go further. Alternatively (?), never consider a transformation that 574657e126899b20c65ff3ea23a3b7d7a67ab30b99dcAndrew Scull // would change a variable that is currently *not* live across basic block 574757e126899b20c65ff3ea23a3b7d7a67ab30b99dcAndrew Scull // boundaries into one that *is*. 5748f47d520c1a91bb029d73a5a5f7c0e416ed763f23Manasij Mukherjee if (!getFlags().getLoopInvariantCodeMotion()) { 5749f47d520c1a91bb029d73a5a5f7c0e416ed763f23Manasij Mukherjee // Need multi block address opt when licm is enabled. 5750f47d520c1a91bb029d73a5a5f7c0e416ed763f23Manasij Mukherjee // Might make sense to restrict to current node and loop header. 5751f47d520c1a91bb029d73a5a5f7c0e416ed763f23Manasij Mukherjee if (Func->getVMetadata()->isMultiBlock( 5752f47d520c1a91bb029d73a5a5f7c0e416ed763f23Manasij Mukherjee NewAddr.Base) /* || Base->getUseCount() > 1*/) 5753f47d520c1a91bb029d73a5a5f7c0e416ed763f23Manasij Mukherjee return nullptr; 5754f47d520c1a91bb029d73a5a5f7c0e416ed763f23Manasij Mukherjee } 5755ac2388c385afda17acb3fc242b17b9232ec2262aJohn Porto AddressOptimizer AddrOpt(Func); 5756d46999474d2b4a388e1d8a7c71f06cd4cec51bfcKarl Schimpf const bool MockBounds = getFlags().getMockBoundsCheck(); 5757aa0b1a176f6c3306be3e95374d72e735bab2bbabDavid Sehr const Inst *Reason = nullptr; 5758ac2388c385afda17acb3fc242b17b9232ec2262aJohn Porto bool AddressWasOptimized = false; 5759ac2388c385afda17acb3fc242b17b9232ec2262aJohn Porto // The following unnamed struct identifies the address mode formation steps 5760ac2388c385afda17acb3fc242b17b9232ec2262aJohn Porto // that could potentially create an invalid memory operand (i.e., no free 5761ac2388c385afda17acb3fc242b17b9232ec2262aJohn Porto // slots for RebasePtr.) We add all those variables to this struct so that we 5762ac2388c385afda17acb3fc242b17b9232ec2262aJohn Porto // can use memset() to reset all members to false. 5763ac2388c385afda17acb3fc242b17b9232ec2262aJohn Porto struct { 5764ac2388c385afda17acb3fc242b17b9232ec2262aJohn Porto bool AssignBase = false; 5765ac2388c385afda17acb3fc242b17b9232ec2262aJohn Porto bool AssignIndex = false; 5766ac2388c385afda17acb3fc242b17b9232ec2262aJohn Porto bool OffsetFromBase = false; 5767ac2388c385afda17acb3fc242b17b9232ec2262aJohn Porto bool OffsetFromIndex = false; 5768ac2388c385afda17acb3fc242b17b9232ec2262aJohn Porto bool CombinedBaseIndex = false; 5769ac2388c385afda17acb3fc242b17b9232ec2262aJohn Porto } Skip; 5770ac2388c385afda17acb3fc242b17b9232ec2262aJohn Porto // This points to the boolean in Skip that represents the last folding 5771ac2388c385afda17acb3fc242b17b9232ec2262aJohn Porto // performed. This is used to disable a pattern match that generated an 5772ac2388c385afda17acb3fc242b17b9232ec2262aJohn Porto // invalid address. Without this, the algorithm would never finish. 5773ac2388c385afda17acb3fc242b17b9232ec2262aJohn Porto bool *SkipLastFolding = nullptr; 5774ac2388c385afda17acb3fc242b17b9232ec2262aJohn Porto // NewAddrCheckpoint is used to rollback the address being formed in case an 5775ac2388c385afda17acb3fc242b17b9232ec2262aJohn Porto // invalid address is formed. 5776ac2388c385afda17acb3fc242b17b9232ec2262aJohn Porto OptAddr NewAddrCheckpoint; 5777ac2388c385afda17acb3fc242b17b9232ec2262aJohn Porto Reason = Instr; 5778aa0b1a176f6c3306be3e95374d72e735bab2bbabDavid Sehr do { 5779ac2388c385afda17acb3fc242b17b9232ec2262aJohn Porto if (SandboxingType != ST_None) { 5780ac2388c385afda17acb3fc242b17b9232ec2262aJohn Porto // When sandboxing, we defer the sandboxing of NewAddr to the Concrete 5781ac2388c385afda17acb3fc242b17b9232ec2262aJohn Porto // Target. If our optimization was overly aggressive, then we simply undo 5782ac2388c385afda17acb3fc242b17b9232ec2262aJohn Porto // what the previous iteration did, and set the previous pattern's skip 5783ac2388c385afda17acb3fc242b17b9232ec2262aJohn Porto // bit to true. 5784ac2388c385afda17acb3fc242b17b9232ec2262aJohn Porto if (!legalizeOptAddrForSandbox(&NewAddr)) { 5785ac2388c385afda17acb3fc242b17b9232ec2262aJohn Porto *SkipLastFolding = true; 5786ac2388c385afda17acb3fc242b17b9232ec2262aJohn Porto SkipLastFolding = nullptr; 5787ac2388c385afda17acb3fc242b17b9232ec2262aJohn Porto NewAddr = NewAddrCheckpoint; 5788ac2388c385afda17acb3fc242b17b9232ec2262aJohn Porto Reason = nullptr; 5789ac2388c385afda17acb3fc242b17b9232ec2262aJohn Porto } 5790ac2388c385afda17acb3fc242b17b9232ec2262aJohn Porto } 5791ac2388c385afda17acb3fc242b17b9232ec2262aJohn Porto 5792aa0b1a176f6c3306be3e95374d72e735bab2bbabDavid Sehr if (Reason) { 5793ac2388c385afda17acb3fc242b17b9232ec2262aJohn Porto AddrOpt.dumpAddressOpt(NewAddr.Relocatable, NewAddr.Offset, NewAddr.Base, 5794ac2388c385afda17acb3fc242b17b9232ec2262aJohn Porto NewAddr.Index, NewAddr.Shift, Reason); 5795aa0b1a176f6c3306be3e95374d72e735bab2bbabDavid Sehr AddressWasOptimized = true; 5796aa0b1a176f6c3306be3e95374d72e735bab2bbabDavid Sehr Reason = nullptr; 5797ac2388c385afda17acb3fc242b17b9232ec2262aJohn Porto SkipLastFolding = nullptr; 5798ac2388c385afda17acb3fc242b17b9232ec2262aJohn Porto memset(&Skip, 0, sizeof(Skip)); 57997e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto } 5800ac2388c385afda17acb3fc242b17b9232ec2262aJohn Porto 5801ac2388c385afda17acb3fc242b17b9232ec2262aJohn Porto NewAddrCheckpoint = NewAddr; 5802ac2388c385afda17acb3fc242b17b9232ec2262aJohn Porto 5803aa0b1a176f6c3306be3e95374d72e735bab2bbabDavid Sehr // Update Base and Index to follow through assignments to definitions. 5804ac2388c385afda17acb3fc242b17b9232ec2262aJohn Porto if (!Skip.AssignBase && 5805ac2388c385afda17acb3fc242b17b9232ec2262aJohn Porto (Reason = AddrOpt.matchAssign(&NewAddr.Base, &NewAddr.Relocatable, 5806ac2388c385afda17acb3fc242b17b9232ec2262aJohn Porto &NewAddr.Offset))) { 5807ac2388c385afda17acb3fc242b17b9232ec2262aJohn Porto SkipLastFolding = &Skip.AssignBase; 5808aa0b1a176f6c3306be3e95374d72e735bab2bbabDavid Sehr // Assignments of Base from a Relocatable or ConstantInt32 can result 5809aa0b1a176f6c3306be3e95374d72e735bab2bbabDavid Sehr // in Base becoming nullptr. To avoid code duplication in this loop we 5810aa0b1a176f6c3306be3e95374d72e735bab2bbabDavid Sehr // prefer that Base be non-nullptr if possible. 5811ac2388c385afda17acb3fc242b17b9232ec2262aJohn Porto if ((NewAddr.Base == nullptr) && (NewAddr.Index != nullptr) && 5812ac2388c385afda17acb3fc242b17b9232ec2262aJohn Porto NewAddr.Shift == 0) { 5813ac2388c385afda17acb3fc242b17b9232ec2262aJohn Porto std::swap(NewAddr.Base, NewAddr.Index); 5814ac2388c385afda17acb3fc242b17b9232ec2262aJohn Porto } 5815aa0b1a176f6c3306be3e95374d72e735bab2bbabDavid Sehr continue; 5816aa0b1a176f6c3306be3e95374d72e735bab2bbabDavid Sehr } 5817ac2388c385afda17acb3fc242b17b9232ec2262aJohn Porto if (!Skip.AssignBase && 5818ac2388c385afda17acb3fc242b17b9232ec2262aJohn Porto (Reason = AddrOpt.matchAssign(&NewAddr.Index, &NewAddr.Relocatable, 5819ac2388c385afda17acb3fc242b17b9232ec2262aJohn Porto &NewAddr.Offset))) { 5820ac2388c385afda17acb3fc242b17b9232ec2262aJohn Porto SkipLastFolding = &Skip.AssignIndex; 5821aa0b1a176f6c3306be3e95374d72e735bab2bbabDavid Sehr continue; 5822ac2388c385afda17acb3fc242b17b9232ec2262aJohn Porto } 58237e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto 5824aa0b1a176f6c3306be3e95374d72e735bab2bbabDavid Sehr if (!MockBounds) { 5825aa0b1a176f6c3306be3e95374d72e735bab2bbabDavid Sehr // Transition from: 5826aa0b1a176f6c3306be3e95374d72e735bab2bbabDavid Sehr // <Relocatable + Offset>(Base) to 5827aa0b1a176f6c3306be3e95374d72e735bab2bbabDavid Sehr // <Relocatable + Offset>(Base, Index) 5828ac2388c385afda17acb3fc242b17b9232ec2262aJohn Porto if (!Skip.CombinedBaseIndex && 5829ac2388c385afda17acb3fc242b17b9232ec2262aJohn Porto (Reason = AddrOpt.matchCombinedBaseIndex( 5830ac2388c385afda17acb3fc242b17b9232ec2262aJohn Porto &NewAddr.Base, &NewAddr.Index, &NewAddr.Shift))) { 5831ac2388c385afda17acb3fc242b17b9232ec2262aJohn Porto SkipLastFolding = &Skip.CombinedBaseIndex; 5832aa0b1a176f6c3306be3e95374d72e735bab2bbabDavid Sehr continue; 5833ac2388c385afda17acb3fc242b17b9232ec2262aJohn Porto } 5834ac2388c385afda17acb3fc242b17b9232ec2262aJohn Porto 5835aa0b1a176f6c3306be3e95374d72e735bab2bbabDavid Sehr // Recognize multiply/shift and update Shift amount. 5836aa0b1a176f6c3306be3e95374d72e735bab2bbabDavid Sehr // Index becomes Index=Var<<Const && Const+Shift<=3 ==> 5837aa0b1a176f6c3306be3e95374d72e735bab2bbabDavid Sehr // Index=Var, Shift+=Const 5838aa0b1a176f6c3306be3e95374d72e735bab2bbabDavid Sehr // Index becomes Index=Const*Var && log2(Const)+Shift<=3 ==> 5839aa0b1a176f6c3306be3e95374d72e735bab2bbabDavid Sehr // Index=Var, Shift+=log2(Const) 5840ac2388c385afda17acb3fc242b17b9232ec2262aJohn Porto if ((Reason = 5841ac2388c385afda17acb3fc242b17b9232ec2262aJohn Porto AddrOpt.matchShiftedIndex(&NewAddr.Index, &NewAddr.Shift))) { 5842aa0b1a176f6c3306be3e95374d72e735bab2bbabDavid Sehr continue; 5843ac2388c385afda17acb3fc242b17b9232ec2262aJohn Porto } 5844ac2388c385afda17acb3fc242b17b9232ec2262aJohn Porto 5845aa0b1a176f6c3306be3e95374d72e735bab2bbabDavid Sehr // If Shift is zero, the choice of Base and Index was purely arbitrary. 5846aa0b1a176f6c3306be3e95374d72e735bab2bbabDavid Sehr // Recognize multiply/shift and set Shift amount. 5847aa0b1a176f6c3306be3e95374d72e735bab2bbabDavid Sehr // Shift==0 && Base is Base=Var*Const && log2(Const)+Shift<=3 ==> 5848aa0b1a176f6c3306be3e95374d72e735bab2bbabDavid Sehr // swap(Index,Base) 5849aa0b1a176f6c3306be3e95374d72e735bab2bbabDavid Sehr // Similar for Base=Const*Var and Base=Var<<Const 5850ac2388c385afda17acb3fc242b17b9232ec2262aJohn Porto if (NewAddr.Shift == 0 && 5851ac2388c385afda17acb3fc242b17b9232ec2262aJohn Porto (Reason = AddrOpt.matchShiftedIndex(&NewAddr.Base, &NewAddr.Shift))) { 5852ac2388c385afda17acb3fc242b17b9232ec2262aJohn Porto std::swap(NewAddr.Base, NewAddr.Index); 5853aa0b1a176f6c3306be3e95374d72e735bab2bbabDavid Sehr continue; 5854aa0b1a176f6c3306be3e95374d72e735bab2bbabDavid Sehr } 5855aa0b1a176f6c3306be3e95374d72e735bab2bbabDavid Sehr } 5856ac2388c385afda17acb3fc242b17b9232ec2262aJohn Porto 5857aa0b1a176f6c3306be3e95374d72e735bab2bbabDavid Sehr // Update Offset to reflect additions/subtractions with constants and 5858aa0b1a176f6c3306be3e95374d72e735bab2bbabDavid Sehr // relocatables. 5859aa0b1a176f6c3306be3e95374d72e735bab2bbabDavid Sehr // TODO: consider overflow issues with respect to Offset. 58602e4b960b20e6ed2ad9453657fe8d34d0e7fd933cManasij Mukherjee if (!Skip.OffsetFromBase && (Reason = AddrOpt.matchOffsetIndexOrBase( 58612e4b960b20e6ed2ad9453657fe8d34d0e7fd933cManasij Mukherjee &NewAddr.Base, /*Shift =*/0, 58622e4b960b20e6ed2ad9453657fe8d34d0e7fd933cManasij Mukherjee &NewAddr.Relocatable, &NewAddr.Offset))) { 5863ac2388c385afda17acb3fc242b17b9232ec2262aJohn Porto SkipLastFolding = &Skip.OffsetFromBase; 586469e9290c092e551a130e6c3a1477a6e97c54212dDavid Sehr continue; 5865ac2388c385afda17acb3fc242b17b9232ec2262aJohn Porto } 58662e4b960b20e6ed2ad9453657fe8d34d0e7fd933cManasij Mukherjee if (!Skip.OffsetFromIndex && (Reason = AddrOpt.matchOffsetIndexOrBase( 58672e4b960b20e6ed2ad9453657fe8d34d0e7fd933cManasij Mukherjee &NewAddr.Index, NewAddr.Shift, 58682e4b960b20e6ed2ad9453657fe8d34d0e7fd933cManasij Mukherjee &NewAddr.Relocatable, &NewAddr.Offset))) { 5869ac2388c385afda17acb3fc242b17b9232ec2262aJohn Porto SkipLastFolding = &Skip.OffsetFromIndex; 5870aa0b1a176f6c3306be3e95374d72e735bab2bbabDavid Sehr continue; 5871ac2388c385afda17acb3fc242b17b9232ec2262aJohn Porto } 5872ac2388c385afda17acb3fc242b17b9232ec2262aJohn Porto 5873aa0b1a176f6c3306be3e95374d72e735bab2bbabDavid Sehr break; 5874aa0b1a176f6c3306be3e95374d72e735bab2bbabDavid Sehr } while (Reason); 5875ac2388c385afda17acb3fc242b17b9232ec2262aJohn Porto 5876ac2388c385afda17acb3fc242b17b9232ec2262aJohn Porto if (!AddressWasOptimized) { 5877ac2388c385afda17acb3fc242b17b9232ec2262aJohn Porto return nullptr; 5878ac2388c385afda17acb3fc242b17b9232ec2262aJohn Porto } 5879ac2388c385afda17acb3fc242b17b9232ec2262aJohn Porto 5880ac2388c385afda17acb3fc242b17b9232ec2262aJohn Porto // Undo any addition of RebasePtr. It will be added back when the mem 5881ac2388c385afda17acb3fc242b17b9232ec2262aJohn Porto // operand is sandboxed. 5882ac2388c385afda17acb3fc242b17b9232ec2262aJohn Porto if (NewAddr.Base == RebasePtr) { 5883ac2388c385afda17acb3fc242b17b9232ec2262aJohn Porto NewAddr.Base = nullptr; 5884ac2388c385afda17acb3fc242b17b9232ec2262aJohn Porto } 5885ac2388c385afda17acb3fc242b17b9232ec2262aJohn Porto 5886ac2388c385afda17acb3fc242b17b9232ec2262aJohn Porto if (NewAddr.Index == RebasePtr) { 5887ac2388c385afda17acb3fc242b17b9232ec2262aJohn Porto NewAddr.Index = nullptr; 5888ac2388c385afda17acb3fc242b17b9232ec2262aJohn Porto NewAddr.Shift = 0; 5889ac2388c385afda17acb3fc242b17b9232ec2262aJohn Porto } 5890ac2388c385afda17acb3fc242b17b9232ec2262aJohn Porto 5891ac2388c385afda17acb3fc242b17b9232ec2262aJohn Porto Constant *OffsetOp = nullptr; 5892ac2388c385afda17acb3fc242b17b9232ec2262aJohn Porto if (NewAddr.Relocatable == nullptr) { 5893ac2388c385afda17acb3fc242b17b9232ec2262aJohn Porto OffsetOp = Ctx->getConstantInt32(NewAddr.Offset); 5894ac2388c385afda17acb3fc242b17b9232ec2262aJohn Porto } else { 5895ac2388c385afda17acb3fc242b17b9232ec2262aJohn Porto OffsetOp = 5896ac2388c385afda17acb3fc242b17b9232ec2262aJohn Porto Ctx->getConstantSym(NewAddr.Relocatable->getOffset() + NewAddr.Offset, 589798ba00666271be1bdcd45b72b3dec04419efe61bJim Stichnoth NewAddr.Relocatable->getName()); 5898ac2388c385afda17acb3fc242b17b9232ec2262aJohn Porto } 5899ac2388c385afda17acb3fc242b17b9232ec2262aJohn Porto // Vanilla ICE load instructions should not use the segment registers, and 5900ac2388c385afda17acb3fc242b17b9232ec2262aJohn Porto // computeAddressOpt only works at the level of Variables and Constants, not 5901ac2388c385afda17acb3fc242b17b9232ec2262aJohn Porto // other X86OperandMem, so there should be no mention of segment 5902ac2388c385afda17acb3fc242b17b9232ec2262aJohn Porto // registers there either. 5903ac2388c385afda17acb3fc242b17b9232ec2262aJohn Porto static constexpr auto SegmentReg = 5904ac2388c385afda17acb3fc242b17b9232ec2262aJohn Porto X86OperandMem::SegmentRegisters::DefaultSegment; 5905ac2388c385afda17acb3fc242b17b9232ec2262aJohn Porto 5906ac2388c385afda17acb3fc242b17b9232ec2262aJohn Porto return X86OperandMem::create(Func, MemType, NewAddr.Base, OffsetOp, 5907ac2388c385afda17acb3fc242b17b9232ec2262aJohn Porto NewAddr.Index, NewAddr.Shift, SegmentReg); 59087e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto} 59097e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto 5910ad2989b6e497f6237a3ae66ae3cfbcceaa99d4f5Jim Stichnoth/// Add a mock bounds check on the memory address before using it as a load or 5911ad2989b6e497f6237a3ae66ae3cfbcceaa99d4f5Jim Stichnoth/// store operand. The basic idea is that given a memory operand [reg], we 5912ad2989b6e497f6237a3ae66ae3cfbcceaa99d4f5Jim Stichnoth/// would first add bounds-check code something like: 5913ad2989b6e497f6237a3ae66ae3cfbcceaa99d4f5Jim Stichnoth/// 5914ad2989b6e497f6237a3ae66ae3cfbcceaa99d4f5Jim Stichnoth/// cmp reg, <lb> 5915ad2989b6e497f6237a3ae66ae3cfbcceaa99d4f5Jim Stichnoth/// jl out_of_line_error 5916ad2989b6e497f6237a3ae66ae3cfbcceaa99d4f5Jim Stichnoth/// cmp reg, <ub> 5917ad2989b6e497f6237a3ae66ae3cfbcceaa99d4f5Jim Stichnoth/// jg out_of_line_error 5918ad2989b6e497f6237a3ae66ae3cfbcceaa99d4f5Jim Stichnoth/// 5919ad2989b6e497f6237a3ae66ae3cfbcceaa99d4f5Jim Stichnoth/// In reality, the specific code will depend on how <lb> and <ub> are 5920ad2989b6e497f6237a3ae66ae3cfbcceaa99d4f5Jim Stichnoth/// represented, e.g. an immediate, a global, or a function argument. 5921ad2989b6e497f6237a3ae66ae3cfbcceaa99d4f5Jim Stichnoth/// 5922ad2989b6e497f6237a3ae66ae3cfbcceaa99d4f5Jim Stichnoth/// As such, we need to enforce that the memory operand does not have the form 5923ad2989b6e497f6237a3ae66ae3cfbcceaa99d4f5Jim Stichnoth/// [reg1+reg2], because then there is no simple cmp instruction that would 5924ad2989b6e497f6237a3ae66ae3cfbcceaa99d4f5Jim Stichnoth/// suffice. However, we consider [reg+offset] to be OK because the offset is 5925ad2989b6e497f6237a3ae66ae3cfbcceaa99d4f5Jim Stichnoth/// usually small, and so <ub> could have a safety buffer built in and then we 5926ad2989b6e497f6237a3ae66ae3cfbcceaa99d4f5Jim Stichnoth/// could instead branch to a custom out_of_line_error that does the precise 5927ad2989b6e497f6237a3ae66ae3cfbcceaa99d4f5Jim Stichnoth/// check and jumps back if it turns out OK. 5928ad2989b6e497f6237a3ae66ae3cfbcceaa99d4f5Jim Stichnoth/// 5929ad2989b6e497f6237a3ae66ae3cfbcceaa99d4f5Jim Stichnoth/// For the purpose of mocking the bounds check, we'll do something like this: 5930ad2989b6e497f6237a3ae66ae3cfbcceaa99d4f5Jim Stichnoth/// 5931ad2989b6e497f6237a3ae66ae3cfbcceaa99d4f5Jim Stichnoth/// cmp reg, 0 5932ad2989b6e497f6237a3ae66ae3cfbcceaa99d4f5Jim Stichnoth/// je label 5933ad2989b6e497f6237a3ae66ae3cfbcceaa99d4f5Jim Stichnoth/// cmp reg, 1 5934ad2989b6e497f6237a3ae66ae3cfbcceaa99d4f5Jim Stichnoth/// je label 5935ad2989b6e497f6237a3ae66ae3cfbcceaa99d4f5Jim Stichnoth/// label: 5936ad2989b6e497f6237a3ae66ae3cfbcceaa99d4f5Jim Stichnoth/// 5937ad2989b6e497f6237a3ae66ae3cfbcceaa99d4f5Jim Stichnoth/// Also note that we don't need to add a bounds check to a dereference of a 5938ad2989b6e497f6237a3ae66ae3cfbcceaa99d4f5Jim Stichnoth/// simple global variable address. 59394a56686b5b56db6803f90ad53514bf2fa190d9f7John Portotemplate <typename TraitsType> 59404a56686b5b56db6803f90ad53514bf2fa190d9f7John Portovoid TargetX86Base<TraitsType>::doMockBoundsCheck(Operand *Opnd) { 5941d46999474d2b4a388e1d8a7c71f06cd4cec51bfcKarl Schimpf if (!getFlags().getMockBoundsCheck()) 5942ad2989b6e497f6237a3ae66ae3cfbcceaa99d4f5Jim Stichnoth return; 59434a56686b5b56db6803f90ad53514bf2fa190d9f7John Porto if (auto *Mem = llvm::dyn_cast<X86OperandMem>(Opnd)) { 5944ad2989b6e497f6237a3ae66ae3cfbcceaa99d4f5Jim Stichnoth if (Mem->getIndex()) { 5945ad2989b6e497f6237a3ae66ae3cfbcceaa99d4f5Jim Stichnoth llvm::report_fatal_error("doMockBoundsCheck: Opnd contains index reg"); 5946ad2989b6e497f6237a3ae66ae3cfbcceaa99d4f5Jim Stichnoth } 5947ad2989b6e497f6237a3ae66ae3cfbcceaa99d4f5Jim Stichnoth Opnd = Mem->getBase(); 5948ad2989b6e497f6237a3ae66ae3cfbcceaa99d4f5Jim Stichnoth } 5949ad2989b6e497f6237a3ae66ae3cfbcceaa99d4f5Jim Stichnoth // At this point Opnd could be nullptr, or Variable, or Constant, or perhaps 5950ad2989b6e497f6237a3ae66ae3cfbcceaa99d4f5Jim Stichnoth // something else. We only care if it is Variable. 5951ad2989b6e497f6237a3ae66ae3cfbcceaa99d4f5Jim Stichnoth auto *Var = llvm::dyn_cast_or_null<Variable>(Opnd); 5952ad2989b6e497f6237a3ae66ae3cfbcceaa99d4f5Jim Stichnoth if (Var == nullptr) 5953ad2989b6e497f6237a3ae66ae3cfbcceaa99d4f5Jim Stichnoth return; 5954ad2989b6e497f6237a3ae66ae3cfbcceaa99d4f5Jim Stichnoth // We use lowerStore() to copy out-args onto the stack. This creates a memory 5955ad2989b6e497f6237a3ae66ae3cfbcceaa99d4f5Jim Stichnoth // operand with the stack pointer as the base register. Don't do bounds 5956ad2989b6e497f6237a3ae66ae3cfbcceaa99d4f5Jim Stichnoth // checks on that. 59578aa396610b7baf728631a43ea16ad3d13e38397aJim Stichnoth if (Var->getRegNum() == getStackReg()) 5958ad2989b6e497f6237a3ae66ae3cfbcceaa99d4f5Jim Stichnoth return; 5959ad2989b6e497f6237a3ae66ae3cfbcceaa99d4f5Jim Stichnoth 59604a56686b5b56db6803f90ad53514bf2fa190d9f7John Porto auto *Label = InstX86Label::create(Func, this); 5961ad2989b6e497f6237a3ae66ae3cfbcceaa99d4f5Jim Stichnoth _cmp(Opnd, Ctx->getConstantZero(IceType_i32)); 5962ad2989b6e497f6237a3ae66ae3cfbcceaa99d4f5Jim Stichnoth _br(Traits::Cond::Br_e, Label); 5963ad2989b6e497f6237a3ae66ae3cfbcceaa99d4f5Jim Stichnoth _cmp(Opnd, Ctx->getConstantInt32(1)); 5964ad2989b6e497f6237a3ae66ae3cfbcceaa99d4f5Jim Stichnoth _br(Traits::Cond::Br_e, Label); 5965ad2989b6e497f6237a3ae66ae3cfbcceaa99d4f5Jim Stichnoth Context.insert(Label); 5966ad2989b6e497f6237a3ae66ae3cfbcceaa99d4f5Jim Stichnoth} 5967ad2989b6e497f6237a3ae66ae3cfbcceaa99d4f5Jim Stichnoth 59684a56686b5b56db6803f90ad53514bf2fa190d9f7John Portotemplate <typename TraitsType> 59694a56686b5b56db6803f90ad53514bf2fa190d9f7John Portovoid TargetX86Base<TraitsType>::lowerLoad(const InstLoad *Load) { 5970921856d4e4bd4f0deedc7324d5f6a4ca0be3439fJohn Porto // A Load instruction can be treated the same as an Assign instruction, after 59718ff4b2819944bc4f02fb29204a1fa5ba7dea5682Jim Stichnoth // the source operand is transformed into an X86OperandMem operand. Note that 59728ff4b2819944bc4f02fb29204a1fa5ba7dea5682Jim Stichnoth // the address mode optimization already creates an X86OperandMem operand, so 59738ff4b2819944bc4f02fb29204a1fa5ba7dea5682Jim Stichnoth // it doesn't need another level of transformation. 59747e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto Variable *DestLoad = Load->getDest(); 59757e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto Type Ty = DestLoad->getType(); 59767e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto Operand *Src0 = formMemoryOperand(Load->getSourceAddress(), Ty); 5977ad2989b6e497f6237a3ae66ae3cfbcceaa99d4f5Jim Stichnoth doMockBoundsCheck(Src0); 597854f3d518805b04b3f2958b7906a7d07dedb4e1aeJim Stichnoth auto *Assign = InstAssign::create(Func, DestLoad, Src0); 59797e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto lowerAssign(Assign); 59807e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto} 59817e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto 59824a56686b5b56db6803f90ad53514bf2fa190d9f7John Portotemplate <typename TraitsType> 59830c70417672c2da35b06d121fc49e8bb6967e102eManasij Mukherjeevoid TargetX86Base<TraitsType>::doAddressOptOther() { 59840c70417672c2da35b06d121fc49e8bb6967e102eManasij Mukherjee // Inverts some Icmp instructions which helps doAddressOptLoad later. 59850c70417672c2da35b06d121fc49e8bb6967e102eManasij Mukherjee // TODO(manasijm): Refactor to unify the conditions for Var0 and Var1 5986efdf412032f7622a0663696896708d75b82e92f6Jim Stichnoth Inst *Instr = iteratorToInst(Context.getCur()); 59870c70417672c2da35b06d121fc49e8bb6967e102eManasij Mukherjee auto *VMetadata = Func->getVMetadata(); 59880c70417672c2da35b06d121fc49e8bb6967e102eManasij Mukherjee if (auto *Icmp = llvm::dyn_cast<InstIcmp>(Instr)) { 59890c70417672c2da35b06d121fc49e8bb6967e102eManasij Mukherjee if (llvm::isa<Constant>(Icmp->getSrc(0)) || 59900c70417672c2da35b06d121fc49e8bb6967e102eManasij Mukherjee llvm::isa<Constant>(Icmp->getSrc(1))) 59910c70417672c2da35b06d121fc49e8bb6967e102eManasij Mukherjee return; 59920c70417672c2da35b06d121fc49e8bb6967e102eManasij Mukherjee auto *Var0 = llvm::dyn_cast<Variable>(Icmp->getSrc(0)); 59930c70417672c2da35b06d121fc49e8bb6967e102eManasij Mukherjee if (Var0 == nullptr) 59940c70417672c2da35b06d121fc49e8bb6967e102eManasij Mukherjee return; 59950c70417672c2da35b06d121fc49e8bb6967e102eManasij Mukherjee if (!VMetadata->isTracked(Var0)) 59960c70417672c2da35b06d121fc49e8bb6967e102eManasij Mukherjee return; 59970c70417672c2da35b06d121fc49e8bb6967e102eManasij Mukherjee auto *Op0Def = VMetadata->getFirstDefinitionSingleBlock(Var0); 59980c70417672c2da35b06d121fc49e8bb6967e102eManasij Mukherjee if (Op0Def == nullptr || !llvm::isa<InstLoad>(Op0Def)) 59990c70417672c2da35b06d121fc49e8bb6967e102eManasij Mukherjee return; 60000c70417672c2da35b06d121fc49e8bb6967e102eManasij Mukherjee if (VMetadata->getLocalUseNode(Var0) != Context.getNode()) 60010c70417672c2da35b06d121fc49e8bb6967e102eManasij Mukherjee return; 60020c70417672c2da35b06d121fc49e8bb6967e102eManasij Mukherjee 60030c70417672c2da35b06d121fc49e8bb6967e102eManasij Mukherjee auto *Var1 = llvm::dyn_cast<Variable>(Icmp->getSrc(1)); 60040c70417672c2da35b06d121fc49e8bb6967e102eManasij Mukherjee if (Var1 != nullptr && VMetadata->isTracked(Var1)) { 60050c70417672c2da35b06d121fc49e8bb6967e102eManasij Mukherjee auto *Op1Def = VMetadata->getFirstDefinitionSingleBlock(Var1); 60060c70417672c2da35b06d121fc49e8bb6967e102eManasij Mukherjee if (Op1Def != nullptr && !VMetadata->isMultiBlock(Var1) && 60070c70417672c2da35b06d121fc49e8bb6967e102eManasij Mukherjee llvm::isa<InstLoad>(Op1Def)) { 60080c70417672c2da35b06d121fc49e8bb6967e102eManasij Mukherjee return; // Both are loads 60090c70417672c2da35b06d121fc49e8bb6967e102eManasij Mukherjee } 60100c70417672c2da35b06d121fc49e8bb6967e102eManasij Mukherjee } 60110c70417672c2da35b06d121fc49e8bb6967e102eManasij Mukherjee Icmp->reverseConditionAndOperands(); 60120c70417672c2da35b06d121fc49e8bb6967e102eManasij Mukherjee } 60130c70417672c2da35b06d121fc49e8bb6967e102eManasij Mukherjee} 60140c70417672c2da35b06d121fc49e8bb6967e102eManasij Mukherjee 60150c70417672c2da35b06d121fc49e8bb6967e102eManasij Mukherjeetemplate <typename TraitsType> 60164a56686b5b56db6803f90ad53514bf2fa190d9f7John Portovoid TargetX86Base<TraitsType>::doAddressOptLoad() { 6017efdf412032f7622a0663696896708d75b82e92f6Jim Stichnoth Inst *Instr = iteratorToInst(Context.getCur()); 60188cfeb69e17190d3bfe22a8a1cbd7679a114d68cfJim Stichnoth Operand *Addr = Instr->getSrc(0); 60198cfeb69e17190d3bfe22a8a1cbd7679a114d68cfJim Stichnoth Variable *Dest = Instr->getDest(); 60208cfeb69e17190d3bfe22a8a1cbd7679a114d68cfJim Stichnoth if (auto *OptAddr = computeAddressOpt(Instr, Dest->getType(), Addr)) { 60218cfeb69e17190d3bfe22a8a1cbd7679a114d68cfJim Stichnoth Instr->setDeleted(); 6022ac2388c385afda17acb3fc242b17b9232ec2262aJohn Porto Context.insert<InstLoad>(Dest, OptAddr); 60237e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto } 60247e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto} 60257e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto 60264a56686b5b56db6803f90ad53514bf2fa190d9f7John Portotemplate <typename TraitsType> 6027e986b318b8295be4e361fe55c21bef6939151e49Nicolas Capensvoid TargetX86Base<TraitsType>::doAddressOptLoadSubVector() { 6028e986b318b8295be4e361fe55c21bef6939151e49Nicolas Capens auto *Intrinsic = llvm::cast<InstIntrinsicCall>(Context.getCur()); 6029e986b318b8295be4e361fe55c21bef6939151e49Nicolas Capens Operand *Addr = Intrinsic->getArg(0); 6030e986b318b8295be4e361fe55c21bef6939151e49Nicolas Capens Variable *Dest = Intrinsic->getDest(); 6031e986b318b8295be4e361fe55c21bef6939151e49Nicolas Capens if (auto *OptAddr = computeAddressOpt(Intrinsic, Dest->getType(), Addr)) { 6032e986b318b8295be4e361fe55c21bef6939151e49Nicolas Capens Intrinsic->setDeleted(); 6033e986b318b8295be4e361fe55c21bef6939151e49Nicolas Capens const Ice::Intrinsics::IntrinsicInfo Info = { 6034e986b318b8295be4e361fe55c21bef6939151e49Nicolas Capens Ice::Intrinsics::LoadSubVector, Ice::Intrinsics::SideEffects_F, 6035e986b318b8295be4e361fe55c21bef6939151e49Nicolas Capens Ice::Intrinsics::ReturnsTwice_F, Ice::Intrinsics::MemoryWrite_F}; 6036e986b318b8295be4e361fe55c21bef6939151e49Nicolas Capens auto Target = Ctx->getConstantUndef(Ice::IceType_i32); 6037e986b318b8295be4e361fe55c21bef6939151e49Nicolas Capens auto *NewLoad = Context.insert<InstIntrinsicCall>(2, Dest, Target, Info); 6038e986b318b8295be4e361fe55c21bef6939151e49Nicolas Capens NewLoad->addArg(OptAddr); 6039e986b318b8295be4e361fe55c21bef6939151e49Nicolas Capens NewLoad->addArg(Intrinsic->getArg(1)); 6040e986b318b8295be4e361fe55c21bef6939151e49Nicolas Capens } 6041e986b318b8295be4e361fe55c21bef6939151e49Nicolas Capens} 6042e986b318b8295be4e361fe55c21bef6939151e49Nicolas Capens 6043e986b318b8295be4e361fe55c21bef6939151e49Nicolas Capenstemplate <typename TraitsType> 60444a56686b5b56db6803f90ad53514bf2fa190d9f7John Portovoid TargetX86Base<TraitsType>::randomlyInsertNop(float Probability, 60454a56686b5b56db6803f90ad53514bf2fa190d9f7John Porto RandomNumberGenerator &RNG) { 6046aee5fa8dd6aa948160a290c8237d7ae4875811fbQining Lu RandomNumberGeneratorWrapper RNGW(RNG); 6047aee5fa8dd6aa948160a290c8237d7ae4875811fbQining Lu if (RNGW.getTrueWithProbability(Probability)) { 6048aee5fa8dd6aa948160a290c8237d7ae4875811fbQining Lu _nop(RNGW(Traits::X86_NUM_NOP_VARIANTS)); 60497e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto } 60507e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto} 60517e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto 60524a56686b5b56db6803f90ad53514bf2fa190d9f7John Portotemplate <typename TraitsType> 60538cfeb69e17190d3bfe22a8a1cbd7679a114d68cfJim Stichnothvoid TargetX86Base<TraitsType>::lowerPhi(const InstPhi * /*Instr*/) { 60547e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto Func->setError("Phi found in regular instruction list"); 60557e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto} 60567e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto 60574a56686b5b56db6803f90ad53514bf2fa190d9f7John Portotemplate <typename TraitsType> 60588cfeb69e17190d3bfe22a8a1cbd7679a114d68cfJim Stichnothvoid TargetX86Base<TraitsType>::lowerRet(const InstRet *Instr) { 60590c68bef895ea8987922922ede6f1918763eaa00bDavid Sehr Variable *Reg = nullptr; 60608cfeb69e17190d3bfe22a8a1cbd7679a114d68cfJim Stichnoth if (Instr->hasRetValue()) { 60618cfeb69e17190d3bfe22a8a1cbd7679a114d68cfJim Stichnoth Operand *RetValue = legalize(Instr->getRetValue()); 60620c68bef895ea8987922922ede6f1918763eaa00bDavid Sehr const Type ReturnType = RetValue->getType(); 60630c68bef895ea8987922922ede6f1918763eaa00bDavid Sehr assert(isVectorType(ReturnType) || isScalarFloatingType(ReturnType) || 60640c68bef895ea8987922922ede6f1918763eaa00bDavid Sehr (ReturnType == IceType_i32) || (ReturnType == IceType_i64)); 60650c68bef895ea8987922922ede6f1918763eaa00bDavid Sehr Reg = moveReturnValueToRegister(RetValue, ReturnType); 60660c68bef895ea8987922922ede6f1918763eaa00bDavid Sehr } 60670c68bef895ea8987922922ede6f1918763eaa00bDavid Sehr // Add a ret instruction even if sandboxing is enabled, because addEpilog 60680c68bef895ea8987922922ede6f1918763eaa00bDavid Sehr // explicitly looks for a ret instruction as a marker for where to insert the 60690c68bef895ea8987922922ede6f1918763eaa00bDavid Sehr // frame removal instructions. 60700c68bef895ea8987922922ede6f1918763eaa00bDavid Sehr _ret(Reg); 60710c68bef895ea8987922922ede6f1918763eaa00bDavid Sehr // Add a fake use of esp to make sure esp stays alive for the entire 60720c68bef895ea8987922922ede6f1918763eaa00bDavid Sehr // function. Otherwise post-call esp adjustments get dead-code eliminated. 60730c68bef895ea8987922922ede6f1918763eaa00bDavid Sehr keepEspLiveAtExit(); 60740c68bef895ea8987922922ede6f1918763eaa00bDavid Sehr} 60750c68bef895ea8987922922ede6f1918763eaa00bDavid Sehr 6076ae15f0fd85df9f13a251222d048761a9aa1dcf2aJohn Portoinline uint32_t makePshufdMask(SizeT Index0, SizeT Index1, SizeT Index2, 6077ae15f0fd85df9f13a251222d048761a9aa1dcf2aJohn Porto SizeT Index3) { 6078ae15f0fd85df9f13a251222d048761a9aa1dcf2aJohn Porto const SizeT Mask = (Index0 & 0x3) | ((Index1 & 0x3) << 2) | 6079ae15f0fd85df9f13a251222d048761a9aa1dcf2aJohn Porto ((Index2 & 0x3) << 4) | ((Index3 & 0x3) << 6); 6080ae15f0fd85df9f13a251222d048761a9aa1dcf2aJohn Porto assert(Mask < 256); 6081ae15f0fd85df9f13a251222d048761a9aa1dcf2aJohn Porto return Mask; 6082ae15f0fd85df9f13a251222d048761a9aa1dcf2aJohn Porto} 6083ae15f0fd85df9f13a251222d048761a9aa1dcf2aJohn Porto 6084ae15f0fd85df9f13a251222d048761a9aa1dcf2aJohn Portotemplate <typename TraitsType> 6085ae15f0fd85df9f13a251222d048761a9aa1dcf2aJohn PortoVariable *TargetX86Base<TraitsType>::lowerShuffleVector_AllFromSameSrc( 6086579b1b3a84da15d233c9ab4e3d3dc35cff4edf86Nicolas Capens Operand *Src, SizeT Index0, SizeT Index1, SizeT Index2, SizeT Index3) { 6087ae15f0fd85df9f13a251222d048761a9aa1dcf2aJohn Porto constexpr SizeT SrcBit = 1 << 2; 6088ae15f0fd85df9f13a251222d048761a9aa1dcf2aJohn Porto assert((Index0 & SrcBit) == (Index1 & SrcBit)); 6089ae15f0fd85df9f13a251222d048761a9aa1dcf2aJohn Porto assert((Index0 & SrcBit) == (Index2 & SrcBit)); 6090ae15f0fd85df9f13a251222d048761a9aa1dcf2aJohn Porto assert((Index0 & SrcBit) == (Index3 & SrcBit)); 6091ae15f0fd85df9f13a251222d048761a9aa1dcf2aJohn Porto (void)SrcBit; 6092ae15f0fd85df9f13a251222d048761a9aa1dcf2aJohn Porto 6093ae15f0fd85df9f13a251222d048761a9aa1dcf2aJohn Porto const Type SrcTy = Src->getType(); 6094ae15f0fd85df9f13a251222d048761a9aa1dcf2aJohn Porto auto *T = makeReg(SrcTy); 6095ae15f0fd85df9f13a251222d048761a9aa1dcf2aJohn Porto auto *SrcRM = legalize(Src, Legal_Reg | Legal_Mem); 6096ae15f0fd85df9f13a251222d048761a9aa1dcf2aJohn Porto auto *Mask = 6097ae15f0fd85df9f13a251222d048761a9aa1dcf2aJohn Porto Ctx->getConstantInt32(makePshufdMask(Index0, Index1, Index2, Index3)); 6098ae15f0fd85df9f13a251222d048761a9aa1dcf2aJohn Porto _pshufd(T, SrcRM, Mask); 6099ae15f0fd85df9f13a251222d048761a9aa1dcf2aJohn Porto return T; 6100ae15f0fd85df9f13a251222d048761a9aa1dcf2aJohn Porto} 6101ae15f0fd85df9f13a251222d048761a9aa1dcf2aJohn Porto 6102ae15f0fd85df9f13a251222d048761a9aa1dcf2aJohn Portotemplate <typename TraitsType> 6103ae15f0fd85df9f13a251222d048761a9aa1dcf2aJohn PortoVariable *TargetX86Base<TraitsType>::lowerShuffleVector_TwoFromSameSrc( 6104579b1b3a84da15d233c9ab4e3d3dc35cff4edf86Nicolas Capens Operand *Src0, SizeT Index0, SizeT Index1, Operand *Src1, SizeT Index2, 6105ae15f0fd85df9f13a251222d048761a9aa1dcf2aJohn Porto SizeT Index3) { 6106ae15f0fd85df9f13a251222d048761a9aa1dcf2aJohn Porto constexpr SizeT SrcBit = 1 << 2; 6107ae15f0fd85df9f13a251222d048761a9aa1dcf2aJohn Porto assert((Index0 & SrcBit) == (Index1 & SrcBit) || (Index1 == IGNORE_INDEX)); 6108ae15f0fd85df9f13a251222d048761a9aa1dcf2aJohn Porto assert((Index2 & SrcBit) == (Index3 & SrcBit) || (Index3 == IGNORE_INDEX)); 6109ae15f0fd85df9f13a251222d048761a9aa1dcf2aJohn Porto (void)SrcBit; 6110ae15f0fd85df9f13a251222d048761a9aa1dcf2aJohn Porto 6111ae15f0fd85df9f13a251222d048761a9aa1dcf2aJohn Porto const Type SrcTy = Src0->getType(); 6112ae15f0fd85df9f13a251222d048761a9aa1dcf2aJohn Porto assert(Src1->getType() == SrcTy); 6113ae15f0fd85df9f13a251222d048761a9aa1dcf2aJohn Porto auto *T = makeReg(SrcTy); 6114ae15f0fd85df9f13a251222d048761a9aa1dcf2aJohn Porto auto *Src0R = legalizeToReg(Src0); 6115ae15f0fd85df9f13a251222d048761a9aa1dcf2aJohn Porto auto *Src1RM = legalize(Src1, Legal_Reg | Legal_Mem); 6116ae15f0fd85df9f13a251222d048761a9aa1dcf2aJohn Porto auto *Mask = 6117ae15f0fd85df9f13a251222d048761a9aa1dcf2aJohn Porto Ctx->getConstantInt32(makePshufdMask(Index0, Index1, Index2, Index3)); 6118ae15f0fd85df9f13a251222d048761a9aa1dcf2aJohn Porto _movp(T, Src0R); 6119ae15f0fd85df9f13a251222d048761a9aa1dcf2aJohn Porto _shufps(T, Src1RM, Mask); 6120ae15f0fd85df9f13a251222d048761a9aa1dcf2aJohn Porto return T; 6121ae15f0fd85df9f13a251222d048761a9aa1dcf2aJohn Porto} 6122ae15f0fd85df9f13a251222d048761a9aa1dcf2aJohn Porto 6123ae15f0fd85df9f13a251222d048761a9aa1dcf2aJohn Portotemplate <typename TraitsType> 6124ae15f0fd85df9f13a251222d048761a9aa1dcf2aJohn PortoVariable *TargetX86Base<TraitsType>::lowerShuffleVector_UnifyFromDifferentSrcs( 6125579b1b3a84da15d233c9ab4e3d3dc35cff4edf86Nicolas Capens Operand *Src0, SizeT Index0, Operand *Src1, SizeT Index1) { 6126ae15f0fd85df9f13a251222d048761a9aa1dcf2aJohn Porto return lowerShuffleVector_TwoFromSameSrc(Src0, Index0, IGNORE_INDEX, Src1, 6127ae15f0fd85df9f13a251222d048761a9aa1dcf2aJohn Porto Index1, IGNORE_INDEX); 6128ae15f0fd85df9f13a251222d048761a9aa1dcf2aJohn Porto} 6129ae15f0fd85df9f13a251222d048761a9aa1dcf2aJohn Porto 6130ae15f0fd85df9f13a251222d048761a9aa1dcf2aJohn Portoinline SizeT makeSrcSwitchMask(SizeT Index0, SizeT Index1, SizeT Index2, 6131ae15f0fd85df9f13a251222d048761a9aa1dcf2aJohn Porto SizeT Index3) { 6132ae15f0fd85df9f13a251222d048761a9aa1dcf2aJohn Porto constexpr SizeT SrcBit = 1 << 2; 6133ae15f0fd85df9f13a251222d048761a9aa1dcf2aJohn Porto const SizeT Index0Bits = ((Index0 & SrcBit) == 0) ? 0 : (1 << 0); 6134ae15f0fd85df9f13a251222d048761a9aa1dcf2aJohn Porto const SizeT Index1Bits = ((Index1 & SrcBit) == 0) ? 0 : (1 << 1); 6135ae15f0fd85df9f13a251222d048761a9aa1dcf2aJohn Porto const SizeT Index2Bits = ((Index2 & SrcBit) == 0) ? 0 : (1 << 2); 6136ae15f0fd85df9f13a251222d048761a9aa1dcf2aJohn Porto const SizeT Index3Bits = ((Index3 & SrcBit) == 0) ? 0 : (1 << 3); 6137ae15f0fd85df9f13a251222d048761a9aa1dcf2aJohn Porto return Index0Bits | Index1Bits | Index2Bits | Index3Bits; 6138ae15f0fd85df9f13a251222d048761a9aa1dcf2aJohn Porto} 6139ae15f0fd85df9f13a251222d048761a9aa1dcf2aJohn Porto 61400c68bef895ea8987922922ede6f1918763eaa00bDavid Sehrtemplate <typename TraitsType> 6141de29f1201e8b77e58f5500ef4cfba51d547368b7John PortoGlobalString TargetX86Base<TraitsType>::lowerShuffleVector_NewMaskName() { 6142de29f1201e8b77e58f5500ef4cfba51d547368b7John Porto GlobalString FuncName = Func->getFunctionName(); 6143de29f1201e8b77e58f5500ef4cfba51d547368b7John Porto const SizeT Id = PshufbMaskCount++; 6144de29f1201e8b77e58f5500ef4cfba51d547368b7John Porto if (!BuildDefs::dump() || !FuncName.hasStdString()) { 6145de29f1201e8b77e58f5500ef4cfba51d547368b7John Porto return GlobalString::createWithString( 6146de29f1201e8b77e58f5500ef4cfba51d547368b7John Porto Ctx, 6147de29f1201e8b77e58f5500ef4cfba51d547368b7John Porto "$PS" + std::to_string(FuncName.getID()) + "_" + std::to_string(Id)); 6148de29f1201e8b77e58f5500ef4cfba51d547368b7John Porto } 6149de29f1201e8b77e58f5500ef4cfba51d547368b7John Porto return GlobalString::createWithString( 6150de29f1201e8b77e58f5500ef4cfba51d547368b7John Porto Ctx, "Pshufb$" + Func->getFunctionName() + "$" + std::to_string(Id)); 6151de29f1201e8b77e58f5500ef4cfba51d547368b7John Porto} 6152de29f1201e8b77e58f5500ef4cfba51d547368b7John Porto 6153de29f1201e8b77e58f5500ef4cfba51d547368b7John Portotemplate <typename TraitsType> 6154de29f1201e8b77e58f5500ef4cfba51d547368b7John PortoConstantRelocatable * 6155de29f1201e8b77e58f5500ef4cfba51d547368b7John PortoTargetX86Base<TraitsType>::lowerShuffleVector_CreatePshufbMask( 6156de29f1201e8b77e58f5500ef4cfba51d547368b7John Porto int8_t Idx0, int8_t Idx1, int8_t Idx2, int8_t Idx3, int8_t Idx4, 6157de29f1201e8b77e58f5500ef4cfba51d547368b7John Porto int8_t Idx5, int8_t Idx6, int8_t Idx7, int8_t Idx8, int8_t Idx9, 6158de29f1201e8b77e58f5500ef4cfba51d547368b7John Porto int8_t Idx10, int8_t Idx11, int8_t Idx12, int8_t Idx13, int8_t Idx14, 6159de29f1201e8b77e58f5500ef4cfba51d547368b7John Porto int8_t Idx15) { 6160de29f1201e8b77e58f5500ef4cfba51d547368b7John Porto static constexpr uint8_t NumElements = 16; 6161de29f1201e8b77e58f5500ef4cfba51d547368b7John Porto const char Initializer[NumElements] = { 6162de29f1201e8b77e58f5500ef4cfba51d547368b7John Porto Idx0, Idx1, Idx2, Idx3, Idx4, Idx5, Idx6, Idx7, 6163de29f1201e8b77e58f5500ef4cfba51d547368b7John Porto Idx8, Idx9, Idx10, Idx11, Idx12, Idx13, Idx14, Idx15, 6164de29f1201e8b77e58f5500ef4cfba51d547368b7John Porto }; 6165de29f1201e8b77e58f5500ef4cfba51d547368b7John Porto 6166de29f1201e8b77e58f5500ef4cfba51d547368b7John Porto static constexpr Type V4VectorType = IceType_v4i32; 6167de29f1201e8b77e58f5500ef4cfba51d547368b7John Porto const uint32_t MaskAlignment = typeWidthInBytesOnStack(V4VectorType); 6168de29f1201e8b77e58f5500ef4cfba51d547368b7John Porto auto *Mask = VariableDeclaration::create(Func->getGlobalPool()); 6169de29f1201e8b77e58f5500ef4cfba51d547368b7John Porto GlobalString MaskName = lowerShuffleVector_NewMaskName(); 6170de29f1201e8b77e58f5500ef4cfba51d547368b7John Porto Mask->setIsConstant(true); 6171de29f1201e8b77e58f5500ef4cfba51d547368b7John Porto Mask->addInitializer(VariableDeclaration::DataInitializer::create( 6172de29f1201e8b77e58f5500ef4cfba51d547368b7John Porto Func->getGlobalPool(), Initializer, NumElements)); 6173de29f1201e8b77e58f5500ef4cfba51d547368b7John Porto Mask->setName(MaskName); 6174de29f1201e8b77e58f5500ef4cfba51d547368b7John Porto // Mask needs to be 16-byte aligned, or pshufb will seg fault. 6175de29f1201e8b77e58f5500ef4cfba51d547368b7John Porto Mask->setAlignment(MaskAlignment); 6176de29f1201e8b77e58f5500ef4cfba51d547368b7John Porto Func->addGlobal(Mask); 6177de29f1201e8b77e58f5500ef4cfba51d547368b7John Porto 6178de29f1201e8b77e58f5500ef4cfba51d547368b7John Porto constexpr RelocOffsetT Offset = 0; 6179de29f1201e8b77e58f5500ef4cfba51d547368b7John Porto return llvm::cast<ConstantRelocatable>(Ctx->getConstantSym(Offset, MaskName)); 6180de29f1201e8b77e58f5500ef4cfba51d547368b7John Porto} 6181de29f1201e8b77e58f5500ef4cfba51d547368b7John Porto 6182de29f1201e8b77e58f5500ef4cfba51d547368b7John Portotemplate <typename TraitsType> 6183de29f1201e8b77e58f5500ef4cfba51d547368b7John Portovoid TargetX86Base<TraitsType>::lowerShuffleVector_UsingPshufb( 6184de29f1201e8b77e58f5500ef4cfba51d547368b7John Porto Variable *Dest, Operand *Src0, Operand *Src1, int8_t Idx0, int8_t Idx1, 6185de29f1201e8b77e58f5500ef4cfba51d547368b7John Porto int8_t Idx2, int8_t Idx3, int8_t Idx4, int8_t Idx5, int8_t Idx6, 6186de29f1201e8b77e58f5500ef4cfba51d547368b7John Porto int8_t Idx7, int8_t Idx8, int8_t Idx9, int8_t Idx10, int8_t Idx11, 6187de29f1201e8b77e58f5500ef4cfba51d547368b7John Porto int8_t Idx12, int8_t Idx13, int8_t Idx14, int8_t Idx15) { 6188de29f1201e8b77e58f5500ef4cfba51d547368b7John Porto const Type DestTy = Dest->getType(); 6189de29f1201e8b77e58f5500ef4cfba51d547368b7John Porto static constexpr bool NotRebased = false; 6190de29f1201e8b77e58f5500ef4cfba51d547368b7John Porto static constexpr Variable *NoBase = nullptr; 6191de29f1201e8b77e58f5500ef4cfba51d547368b7John Porto // We use void for the memory operand instead of DestTy because using the 6192de29f1201e8b77e58f5500ef4cfba51d547368b7John Porto // latter causes a validation failure: the X86 Inst layer complains that 6193de29f1201e8b77e58f5500ef4cfba51d547368b7John Porto // vector mem operands could be under aligned. Thus, using void we avoid the 6194de29f1201e8b77e58f5500ef4cfba51d547368b7John Porto // validation error. Note that the mask global declaration is aligned, so it 6195de29f1201e8b77e58f5500ef4cfba51d547368b7John Porto // can be used as an XMM mem operand. 6196de29f1201e8b77e58f5500ef4cfba51d547368b7John Porto static constexpr Type MaskType = IceType_void; 6197de29f1201e8b77e58f5500ef4cfba51d547368b7John Porto#define IDX_IN_SRC(N, S) \ 6198de29f1201e8b77e58f5500ef4cfba51d547368b7John Porto ((((N) & (1 << 4)) == (S << 4)) ? ((N)&0xf) : CLEAR_ALL_BITS) 6199de29f1201e8b77e58f5500ef4cfba51d547368b7John Porto auto *Mask0M = X86OperandMem::create( 6200de29f1201e8b77e58f5500ef4cfba51d547368b7John Porto Func, MaskType, NoBase, 6201de29f1201e8b77e58f5500ef4cfba51d547368b7John Porto lowerShuffleVector_CreatePshufbMask( 6202de29f1201e8b77e58f5500ef4cfba51d547368b7John Porto IDX_IN_SRC(Idx0, 0), IDX_IN_SRC(Idx1, 0), IDX_IN_SRC(Idx2, 0), 6203de29f1201e8b77e58f5500ef4cfba51d547368b7John Porto IDX_IN_SRC(Idx3, 0), IDX_IN_SRC(Idx4, 0), IDX_IN_SRC(Idx5, 0), 6204de29f1201e8b77e58f5500ef4cfba51d547368b7John Porto IDX_IN_SRC(Idx6, 0), IDX_IN_SRC(Idx7, 0), IDX_IN_SRC(Idx8, 0), 6205de29f1201e8b77e58f5500ef4cfba51d547368b7John Porto IDX_IN_SRC(Idx9, 0), IDX_IN_SRC(Idx10, 0), IDX_IN_SRC(Idx11, 0), 6206de29f1201e8b77e58f5500ef4cfba51d547368b7John Porto IDX_IN_SRC(Idx12, 0), IDX_IN_SRC(Idx13, 0), IDX_IN_SRC(Idx14, 0), 6207de29f1201e8b77e58f5500ef4cfba51d547368b7John Porto IDX_IN_SRC(Idx15, 0)), 6208de29f1201e8b77e58f5500ef4cfba51d547368b7John Porto NotRebased); 620936bcf2d917adf63b67fd26dd24b2d50e92c53f4eNicolas Capens 6210de29f1201e8b77e58f5500ef4cfba51d547368b7John Porto auto *T0 = makeReg(DestTy); 6211de29f1201e8b77e58f5500ef4cfba51d547368b7John Porto auto *Src0RM = legalize(Src0, Legal_Reg | Legal_Mem); 6212de29f1201e8b77e58f5500ef4cfba51d547368b7John Porto _movp(T0, Src0RM); 6213de29f1201e8b77e58f5500ef4cfba51d547368b7John Porto 6214de29f1201e8b77e58f5500ef4cfba51d547368b7John Porto _pshufb(T0, Mask0M); 621536bcf2d917adf63b67fd26dd24b2d50e92c53f4eNicolas Capens 6216c48bb8b02c98ae49438e43aa1143a958784822a5Nicolas Capens if (Idx0 >= 16 || Idx1 >= 16 || Idx2 >= 16 || Idx3 >= 16 || Idx4 >= 16 || 6217c48bb8b02c98ae49438e43aa1143a958784822a5Nicolas Capens Idx5 >= 16 || Idx6 >= 16 || Idx7 >= 16 || Idx8 >= 16 || Idx9 >= 16 || 6218c48bb8b02c98ae49438e43aa1143a958784822a5Nicolas Capens Idx10 >= 16 || Idx11 >= 16 || Idx12 >= 16 || Idx13 >= 16 || Idx14 >= 16 || 6219c48bb8b02c98ae49438e43aa1143a958784822a5Nicolas Capens Idx15 >= 16) { 622036bcf2d917adf63b67fd26dd24b2d50e92c53f4eNicolas Capens auto *Mask1M = X86OperandMem::create( 622136bcf2d917adf63b67fd26dd24b2d50e92c53f4eNicolas Capens Func, MaskType, NoBase, 622236bcf2d917adf63b67fd26dd24b2d50e92c53f4eNicolas Capens lowerShuffleVector_CreatePshufbMask( 622336bcf2d917adf63b67fd26dd24b2d50e92c53f4eNicolas Capens IDX_IN_SRC(Idx0, 1), IDX_IN_SRC(Idx1, 1), IDX_IN_SRC(Idx2, 1), 622436bcf2d917adf63b67fd26dd24b2d50e92c53f4eNicolas Capens IDX_IN_SRC(Idx3, 1), IDX_IN_SRC(Idx4, 1), IDX_IN_SRC(Idx5, 1), 622536bcf2d917adf63b67fd26dd24b2d50e92c53f4eNicolas Capens IDX_IN_SRC(Idx6, 1), IDX_IN_SRC(Idx7, 1), IDX_IN_SRC(Idx8, 1), 622636bcf2d917adf63b67fd26dd24b2d50e92c53f4eNicolas Capens IDX_IN_SRC(Idx9, 1), IDX_IN_SRC(Idx10, 1), IDX_IN_SRC(Idx11, 1), 622736bcf2d917adf63b67fd26dd24b2d50e92c53f4eNicolas Capens IDX_IN_SRC(Idx12, 1), IDX_IN_SRC(Idx13, 1), IDX_IN_SRC(Idx14, 1), 622836bcf2d917adf63b67fd26dd24b2d50e92c53f4eNicolas Capens IDX_IN_SRC(Idx15, 1)), 622936bcf2d917adf63b67fd26dd24b2d50e92c53f4eNicolas Capens NotRebased); 623036bcf2d917adf63b67fd26dd24b2d50e92c53f4eNicolas Capens#undef IDX_IN_SRC 623136bcf2d917adf63b67fd26dd24b2d50e92c53f4eNicolas Capens auto *T1 = makeReg(DestTy); 623236bcf2d917adf63b67fd26dd24b2d50e92c53f4eNicolas Capens auto *Src1RM = legalize(Src1, Legal_Reg | Legal_Mem); 623336bcf2d917adf63b67fd26dd24b2d50e92c53f4eNicolas Capens _movp(T1, Src1RM); 623436bcf2d917adf63b67fd26dd24b2d50e92c53f4eNicolas Capens _pshufb(T1, Mask1M); 623536bcf2d917adf63b67fd26dd24b2d50e92c53f4eNicolas Capens _por(T0, T1); 623636bcf2d917adf63b67fd26dd24b2d50e92c53f4eNicolas Capens } 623736bcf2d917adf63b67fd26dd24b2d50e92c53f4eNicolas Capens 623836bcf2d917adf63b67fd26dd24b2d50e92c53f4eNicolas Capens _movp(Dest, T0); 6239de29f1201e8b77e58f5500ef4cfba51d547368b7John Porto} 6240de29f1201e8b77e58f5500ef4cfba51d547368b7John Porto 6241de29f1201e8b77e58f5500ef4cfba51d547368b7John Portotemplate <typename TraitsType> 6242a47c11c7f17022050043d69c0802241e0747a056John Portovoid TargetX86Base<TraitsType>::lowerShuffleVector( 6243a47c11c7f17022050043d69c0802241e0747a056John Porto const InstShuffleVector *Instr) { 6244a47c11c7f17022050043d69c0802241e0747a056John Porto auto *Dest = Instr->getDest(); 6245a47c11c7f17022050043d69c0802241e0747a056John Porto const Type DestTy = Dest->getType(); 6246579b1b3a84da15d233c9ab4e3d3dc35cff4edf86Nicolas Capens auto *Src0 = Instr->getSrc(0); 6247579b1b3a84da15d233c9ab4e3d3dc35cff4edf86Nicolas Capens auto *Src1 = Instr->getSrc(1); 6248ae15f0fd85df9f13a251222d048761a9aa1dcf2aJohn Porto const SizeT NumElements = typeNumElements(DestTy); 6249a47c11c7f17022050043d69c0802241e0747a056John Porto 6250a47c11c7f17022050043d69c0802241e0747a056John Porto auto *T = makeReg(DestTy); 6251a47c11c7f17022050043d69c0802241e0747a056John Porto 6252a47c11c7f17022050043d69c0802241e0747a056John Porto switch (DestTy) { 6253a47c11c7f17022050043d69c0802241e0747a056John Porto default: 6254de29f1201e8b77e58f5500ef4cfba51d547368b7John Porto llvm::report_fatal_error("Unexpected vector type."); 6255de29f1201e8b77e58f5500ef4cfba51d547368b7John Porto case IceType_v16i1: 6256de29f1201e8b77e58f5500ef4cfba51d547368b7John Porto case IceType_v16i8: { 6257de29f1201e8b77e58f5500ef4cfba51d547368b7John Porto static constexpr SizeT ExpectedNumElements = 16; 6258de29f1201e8b77e58f5500ef4cfba51d547368b7John Porto assert(ExpectedNumElements == Instr->getNumIndexes()); 6259de29f1201e8b77e58f5500ef4cfba51d547368b7John Porto (void)ExpectedNumElements; 62601448d95c891c926f480e4b898c6cb5716983e582Nicolas Capens 62611448d95c891c926f480e4b898c6cb5716983e582Nicolas Capens if (Instr->indexesAre(0, 0, 1, 1, 2, 2, 3, 3, 4, 4, 5, 5, 6, 6, 7, 7)) { 62621448d95c891c926f480e4b898c6cb5716983e582Nicolas Capens auto *T = makeReg(DestTy); 62631448d95c891c926f480e4b898c6cb5716983e582Nicolas Capens auto *Src0RM = legalize(Src0, Legal_Reg | Legal_Mem); 62641448d95c891c926f480e4b898c6cb5716983e582Nicolas Capens _movp(T, Src0RM); 62651448d95c891c926f480e4b898c6cb5716983e582Nicolas Capens _punpckl(T, Src0RM); 62661448d95c891c926f480e4b898c6cb5716983e582Nicolas Capens _movp(Dest, T); 62671448d95c891c926f480e4b898c6cb5716983e582Nicolas Capens return; 62681448d95c891c926f480e4b898c6cb5716983e582Nicolas Capens } 62691448d95c891c926f480e4b898c6cb5716983e582Nicolas Capens 62701448d95c891c926f480e4b898c6cb5716983e582Nicolas Capens if (Instr->indexesAre(0, 16, 1, 17, 2, 18, 3, 19, 4, 20, 5, 21, 6, 22, 7, 62711448d95c891c926f480e4b898c6cb5716983e582Nicolas Capens 23)) { 62721448d95c891c926f480e4b898c6cb5716983e582Nicolas Capens auto *T = makeReg(DestTy); 62731448d95c891c926f480e4b898c6cb5716983e582Nicolas Capens auto *Src0RM = legalize(Src0, Legal_Reg | Legal_Mem); 62741448d95c891c926f480e4b898c6cb5716983e582Nicolas Capens auto *Src1RM = legalize(Src1, Legal_Reg | Legal_Mem); 62751448d95c891c926f480e4b898c6cb5716983e582Nicolas Capens _movp(T, Src0RM); 62761448d95c891c926f480e4b898c6cb5716983e582Nicolas Capens _punpckl(T, Src1RM); 62771448d95c891c926f480e4b898c6cb5716983e582Nicolas Capens _movp(Dest, T); 62781448d95c891c926f480e4b898c6cb5716983e582Nicolas Capens return; 62791448d95c891c926f480e4b898c6cb5716983e582Nicolas Capens } 62801448d95c891c926f480e4b898c6cb5716983e582Nicolas Capens 62811448d95c891c926f480e4b898c6cb5716983e582Nicolas Capens if (Instr->indexesAre(8, 8, 9, 9, 10, 10, 11, 11, 12, 12, 13, 13, 14, 14, 62821448d95c891c926f480e4b898c6cb5716983e582Nicolas Capens 15, 15)) { 62831448d95c891c926f480e4b898c6cb5716983e582Nicolas Capens auto *T = makeReg(DestTy); 62841448d95c891c926f480e4b898c6cb5716983e582Nicolas Capens auto *Src0RM = legalize(Src0, Legal_Reg | Legal_Mem); 62851448d95c891c926f480e4b898c6cb5716983e582Nicolas Capens _movp(T, Src0RM); 62861448d95c891c926f480e4b898c6cb5716983e582Nicolas Capens _punpckh(T, Src0RM); 62871448d95c891c926f480e4b898c6cb5716983e582Nicolas Capens _movp(Dest, T); 62881448d95c891c926f480e4b898c6cb5716983e582Nicolas Capens return; 62891448d95c891c926f480e4b898c6cb5716983e582Nicolas Capens } 62901448d95c891c926f480e4b898c6cb5716983e582Nicolas Capens 62911448d95c891c926f480e4b898c6cb5716983e582Nicolas Capens if (Instr->indexesAre(8, 24, 9, 25, 10, 26, 11, 27, 12, 28, 13, 29, 14, 30, 62921448d95c891c926f480e4b898c6cb5716983e582Nicolas Capens 15, 31)) { 62931448d95c891c926f480e4b898c6cb5716983e582Nicolas Capens auto *T = makeReg(DestTy); 62941448d95c891c926f480e4b898c6cb5716983e582Nicolas Capens auto *Src0RM = legalize(Src0, Legal_Reg | Legal_Mem); 62951448d95c891c926f480e4b898c6cb5716983e582Nicolas Capens auto *Src1RM = legalize(Src1, Legal_Reg | Legal_Mem); 62961448d95c891c926f480e4b898c6cb5716983e582Nicolas Capens _movp(T, Src0RM); 62971448d95c891c926f480e4b898c6cb5716983e582Nicolas Capens _punpckh(T, Src1RM); 62981448d95c891c926f480e4b898c6cb5716983e582Nicolas Capens _movp(Dest, T); 62991448d95c891c926f480e4b898c6cb5716983e582Nicolas Capens return; 63001448d95c891c926f480e4b898c6cb5716983e582Nicolas Capens } 63011448d95c891c926f480e4b898c6cb5716983e582Nicolas Capens 63021448d95c891c926f480e4b898c6cb5716983e582Nicolas Capens if (InstructionSet < Traits::SSE4_1) { 63031448d95c891c926f480e4b898c6cb5716983e582Nicolas Capens // TODO(jpp): figure out how to lower with sse2. 63041448d95c891c926f480e4b898c6cb5716983e582Nicolas Capens break; 63051448d95c891c926f480e4b898c6cb5716983e582Nicolas Capens } 63061448d95c891c926f480e4b898c6cb5716983e582Nicolas Capens 6307de29f1201e8b77e58f5500ef4cfba51d547368b7John Porto const SizeT Index0 = Instr->getIndex(0)->getValue(); 6308de29f1201e8b77e58f5500ef4cfba51d547368b7John Porto const SizeT Index1 = Instr->getIndex(1)->getValue(); 6309de29f1201e8b77e58f5500ef4cfba51d547368b7John Porto const SizeT Index2 = Instr->getIndex(2)->getValue(); 6310de29f1201e8b77e58f5500ef4cfba51d547368b7John Porto const SizeT Index3 = Instr->getIndex(3)->getValue(); 6311de29f1201e8b77e58f5500ef4cfba51d547368b7John Porto const SizeT Index4 = Instr->getIndex(4)->getValue(); 6312de29f1201e8b77e58f5500ef4cfba51d547368b7John Porto const SizeT Index5 = Instr->getIndex(5)->getValue(); 6313de29f1201e8b77e58f5500ef4cfba51d547368b7John Porto const SizeT Index6 = Instr->getIndex(6)->getValue(); 6314de29f1201e8b77e58f5500ef4cfba51d547368b7John Porto const SizeT Index7 = Instr->getIndex(7)->getValue(); 6315de29f1201e8b77e58f5500ef4cfba51d547368b7John Porto const SizeT Index8 = Instr->getIndex(8)->getValue(); 6316de29f1201e8b77e58f5500ef4cfba51d547368b7John Porto const SizeT Index9 = Instr->getIndex(9)->getValue(); 6317de29f1201e8b77e58f5500ef4cfba51d547368b7John Porto const SizeT Index10 = Instr->getIndex(10)->getValue(); 6318de29f1201e8b77e58f5500ef4cfba51d547368b7John Porto const SizeT Index11 = Instr->getIndex(11)->getValue(); 6319de29f1201e8b77e58f5500ef4cfba51d547368b7John Porto const SizeT Index12 = Instr->getIndex(12)->getValue(); 6320de29f1201e8b77e58f5500ef4cfba51d547368b7John Porto const SizeT Index13 = Instr->getIndex(13)->getValue(); 6321de29f1201e8b77e58f5500ef4cfba51d547368b7John Porto const SizeT Index14 = Instr->getIndex(14)->getValue(); 6322de29f1201e8b77e58f5500ef4cfba51d547368b7John Porto const SizeT Index15 = Instr->getIndex(15)->getValue(); 632371c6937378c27d04edd59399817873dc090025a7Nicolas Capens 6324de29f1201e8b77e58f5500ef4cfba51d547368b7John Porto lowerShuffleVector_UsingPshufb(Dest, Src0, Src1, Index0, Index1, Index2, 6325de29f1201e8b77e58f5500ef4cfba51d547368b7John Porto Index3, Index4, Index5, Index6, Index7, 6326de29f1201e8b77e58f5500ef4cfba51d547368b7John Porto Index8, Index9, Index10, Index11, Index12, 6327de29f1201e8b77e58f5500ef4cfba51d547368b7John Porto Index13, Index14, Index15); 6328de29f1201e8b77e58f5500ef4cfba51d547368b7John Porto return; 6329de29f1201e8b77e58f5500ef4cfba51d547368b7John Porto } 6330de29f1201e8b77e58f5500ef4cfba51d547368b7John Porto case IceType_v8i1: 6331de29f1201e8b77e58f5500ef4cfba51d547368b7John Porto case IceType_v8i16: { 6332de29f1201e8b77e58f5500ef4cfba51d547368b7John Porto static constexpr SizeT ExpectedNumElements = 8; 6333de29f1201e8b77e58f5500ef4cfba51d547368b7John Porto assert(ExpectedNumElements == Instr->getNumIndexes()); 6334de29f1201e8b77e58f5500ef4cfba51d547368b7John Porto (void)ExpectedNumElements; 633571c6937378c27d04edd59399817873dc090025a7Nicolas Capens 63361448d95c891c926f480e4b898c6cb5716983e582Nicolas Capens if (Instr->indexesAre(0, 0, 1, 1, 2, 2, 3, 3)) { 633771c6937378c27d04edd59399817873dc090025a7Nicolas Capens auto *T = makeReg(DestTy); 633871c6937378c27d04edd59399817873dc090025a7Nicolas Capens auto *Src0RM = legalize(Src0, Legal_Reg | Legal_Mem); 633971c6937378c27d04edd59399817873dc090025a7Nicolas Capens _movp(T, Src0RM); 6340f52cea4bb987e8c43a881f243dbb8f41f31797d5Nicolas Capens _punpckl(T, Src0RM); 634171c6937378c27d04edd59399817873dc090025a7Nicolas Capens _movp(Dest, T); 634271c6937378c27d04edd59399817873dc090025a7Nicolas Capens return; 634371c6937378c27d04edd59399817873dc090025a7Nicolas Capens } 634471c6937378c27d04edd59399817873dc090025a7Nicolas Capens 63451448d95c891c926f480e4b898c6cb5716983e582Nicolas Capens if (Instr->indexesAre(0, 8, 1, 9, 2, 10, 3, 11)) { 63461448d95c891c926f480e4b898c6cb5716983e582Nicolas Capens auto *T = makeReg(DestTy); 63471448d95c891c926f480e4b898c6cb5716983e582Nicolas Capens auto *Src0RM = legalize(Src0, Legal_Reg | Legal_Mem); 6348a3688eaffa0b11e21b7dc48bbba514fc3f999198Nicolas Capens auto *Src1RM = legalize(Src1, Legal_Reg | Legal_Mem); 63491448d95c891c926f480e4b898c6cb5716983e582Nicolas Capens _movp(T, Src0RM); 6350a3688eaffa0b11e21b7dc48bbba514fc3f999198Nicolas Capens _punpckl(T, Src1RM); 63511448d95c891c926f480e4b898c6cb5716983e582Nicolas Capens _movp(Dest, T); 63521448d95c891c926f480e4b898c6cb5716983e582Nicolas Capens return; 63531448d95c891c926f480e4b898c6cb5716983e582Nicolas Capens } 63541448d95c891c926f480e4b898c6cb5716983e582Nicolas Capens 63551448d95c891c926f480e4b898c6cb5716983e582Nicolas Capens if (Instr->indexesAre(4, 4, 5, 5, 6, 6, 7, 7)) { 63561448d95c891c926f480e4b898c6cb5716983e582Nicolas Capens auto *T = makeReg(DestTy); 63571448d95c891c926f480e4b898c6cb5716983e582Nicolas Capens auto *Src0RM = legalize(Src0, Legal_Reg | Legal_Mem); 63581448d95c891c926f480e4b898c6cb5716983e582Nicolas Capens _movp(T, Src0RM); 63591448d95c891c926f480e4b898c6cb5716983e582Nicolas Capens _punpckh(T, Src0RM); 63601448d95c891c926f480e4b898c6cb5716983e582Nicolas Capens _movp(Dest, T); 63611448d95c891c926f480e4b898c6cb5716983e582Nicolas Capens return; 63621448d95c891c926f480e4b898c6cb5716983e582Nicolas Capens } 63631448d95c891c926f480e4b898c6cb5716983e582Nicolas Capens 63641448d95c891c926f480e4b898c6cb5716983e582Nicolas Capens if (Instr->indexesAre(4, 12, 5, 13, 6, 14, 7, 15)) { 63651448d95c891c926f480e4b898c6cb5716983e582Nicolas Capens auto *T = makeReg(DestTy); 63661448d95c891c926f480e4b898c6cb5716983e582Nicolas Capens auto *Src0RM = legalize(Src0, Legal_Reg | Legal_Mem); 63671448d95c891c926f480e4b898c6cb5716983e582Nicolas Capens auto *Src1RM = legalize(Src1, Legal_Reg | Legal_Mem); 63681448d95c891c926f480e4b898c6cb5716983e582Nicolas Capens _movp(T, Src0RM); 63691448d95c891c926f480e4b898c6cb5716983e582Nicolas Capens _punpckh(T, Src1RM); 63701448d95c891c926f480e4b898c6cb5716983e582Nicolas Capens _movp(Dest, T); 63711448d95c891c926f480e4b898c6cb5716983e582Nicolas Capens return; 63721448d95c891c926f480e4b898c6cb5716983e582Nicolas Capens } 63731448d95c891c926f480e4b898c6cb5716983e582Nicolas Capens 637471c6937378c27d04edd59399817873dc090025a7Nicolas Capens if (InstructionSet < Traits::SSE4_1) { 637571c6937378c27d04edd59399817873dc090025a7Nicolas Capens // TODO(jpp): figure out how to lower with sse2. 637671c6937378c27d04edd59399817873dc090025a7Nicolas Capens break; 637771c6937378c27d04edd59399817873dc090025a7Nicolas Capens } 637871c6937378c27d04edd59399817873dc090025a7Nicolas Capens 63791448d95c891c926f480e4b898c6cb5716983e582Nicolas Capens const SizeT Index0 = Instr->getIndex(0)->getValue(); 63801448d95c891c926f480e4b898c6cb5716983e582Nicolas Capens const SizeT Index1 = Instr->getIndex(1)->getValue(); 63811448d95c891c926f480e4b898c6cb5716983e582Nicolas Capens const SizeT Index2 = Instr->getIndex(2)->getValue(); 63821448d95c891c926f480e4b898c6cb5716983e582Nicolas Capens const SizeT Index3 = Instr->getIndex(3)->getValue(); 63831448d95c891c926f480e4b898c6cb5716983e582Nicolas Capens const SizeT Index4 = Instr->getIndex(4)->getValue(); 63841448d95c891c926f480e4b898c6cb5716983e582Nicolas Capens const SizeT Index5 = Instr->getIndex(5)->getValue(); 63851448d95c891c926f480e4b898c6cb5716983e582Nicolas Capens const SizeT Index6 = Instr->getIndex(6)->getValue(); 63861448d95c891c926f480e4b898c6cb5716983e582Nicolas Capens const SizeT Index7 = Instr->getIndex(7)->getValue(); 63871448d95c891c926f480e4b898c6cb5716983e582Nicolas Capens 6388de29f1201e8b77e58f5500ef4cfba51d547368b7John Porto#define TO_BYTE_INDEX(I) ((I) << 1) 6389de29f1201e8b77e58f5500ef4cfba51d547368b7John Porto lowerShuffleVector_UsingPshufb( 6390de29f1201e8b77e58f5500ef4cfba51d547368b7John Porto Dest, Src0, Src1, TO_BYTE_INDEX(Index0), TO_BYTE_INDEX(Index0) + 1, 6391de29f1201e8b77e58f5500ef4cfba51d547368b7John Porto TO_BYTE_INDEX(Index1), TO_BYTE_INDEX(Index1) + 1, TO_BYTE_INDEX(Index2), 6392de29f1201e8b77e58f5500ef4cfba51d547368b7John Porto TO_BYTE_INDEX(Index2) + 1, TO_BYTE_INDEX(Index3), 6393de29f1201e8b77e58f5500ef4cfba51d547368b7John Porto TO_BYTE_INDEX(Index3) + 1, TO_BYTE_INDEX(Index4), 6394de29f1201e8b77e58f5500ef4cfba51d547368b7John Porto TO_BYTE_INDEX(Index4) + 1, TO_BYTE_INDEX(Index5), 6395de29f1201e8b77e58f5500ef4cfba51d547368b7John Porto TO_BYTE_INDEX(Index5) + 1, TO_BYTE_INDEX(Index6), 6396de29f1201e8b77e58f5500ef4cfba51d547368b7John Porto TO_BYTE_INDEX(Index6) + 1, TO_BYTE_INDEX(Index7), 6397de29f1201e8b77e58f5500ef4cfba51d547368b7John Porto TO_BYTE_INDEX(Index7) + 1); 6398de29f1201e8b77e58f5500ef4cfba51d547368b7John Porto#undef TO_BYTE_INDEX 6399de29f1201e8b77e58f5500ef4cfba51d547368b7John Porto return; 6400de29f1201e8b77e58f5500ef4cfba51d547368b7John Porto } 6401ae15f0fd85df9f13a251222d048761a9aa1dcf2aJohn Porto case IceType_v4i1: 6402ae15f0fd85df9f13a251222d048761a9aa1dcf2aJohn Porto case IceType_v4i32: 6403ae15f0fd85df9f13a251222d048761a9aa1dcf2aJohn Porto case IceType_v4f32: { 6404ae15f0fd85df9f13a251222d048761a9aa1dcf2aJohn Porto static constexpr SizeT ExpectedNumElements = 4; 6405ae15f0fd85df9f13a251222d048761a9aa1dcf2aJohn Porto assert(ExpectedNumElements == Instr->getNumIndexes()); 6406ae15f0fd85df9f13a251222d048761a9aa1dcf2aJohn Porto const SizeT Index0 = Instr->getIndex(0)->getValue(); 6407ae15f0fd85df9f13a251222d048761a9aa1dcf2aJohn Porto const SizeT Index1 = Instr->getIndex(1)->getValue(); 6408ae15f0fd85df9f13a251222d048761a9aa1dcf2aJohn Porto const SizeT Index2 = Instr->getIndex(2)->getValue(); 6409ae15f0fd85df9f13a251222d048761a9aa1dcf2aJohn Porto const SizeT Index3 = Instr->getIndex(3)->getValue(); 6410ae15f0fd85df9f13a251222d048761a9aa1dcf2aJohn Porto Variable *T = nullptr; 6411ae15f0fd85df9f13a251222d048761a9aa1dcf2aJohn Porto switch (makeSrcSwitchMask(Index0, Index1, Index2, Index3)) { 6412ae15f0fd85df9f13a251222d048761a9aa1dcf2aJohn Porto#define CASE_SRCS_IN(S0, S1, S2, S3) \ 6413ae15f0fd85df9f13a251222d048761a9aa1dcf2aJohn Porto case (((S0) << 0) | ((S1) << 1) | ((S2) << 2) | ((S3) << 3)) 6414ae15f0fd85df9f13a251222d048761a9aa1dcf2aJohn Porto CASE_SRCS_IN(0, 0, 0, 0) : { 6415ae15f0fd85df9f13a251222d048761a9aa1dcf2aJohn Porto T = lowerShuffleVector_AllFromSameSrc(Src0, Index0, Index1, Index2, 6416ae15f0fd85df9f13a251222d048761a9aa1dcf2aJohn Porto Index3); 6417ae15f0fd85df9f13a251222d048761a9aa1dcf2aJohn Porto } 6418ae15f0fd85df9f13a251222d048761a9aa1dcf2aJohn Porto break; 6419ae15f0fd85df9f13a251222d048761a9aa1dcf2aJohn Porto CASE_SRCS_IN(0, 0, 0, 1) : { 6420ae15f0fd85df9f13a251222d048761a9aa1dcf2aJohn Porto assert(false && "Following code is untested but likely correct; test " 6421ae15f0fd85df9f13a251222d048761a9aa1dcf2aJohn Porto "and remove assert."); 6422ae15f0fd85df9f13a251222d048761a9aa1dcf2aJohn Porto auto *Unified = lowerShuffleVector_UnifyFromDifferentSrcs(Src0, Index2, 6423ae15f0fd85df9f13a251222d048761a9aa1dcf2aJohn Porto Src1, Index3); 6424ae15f0fd85df9f13a251222d048761a9aa1dcf2aJohn Porto T = lowerShuffleVector_TwoFromSameSrc(Src0, Index0, Index1, Unified, 6425ae15f0fd85df9f13a251222d048761a9aa1dcf2aJohn Porto UNIFIED_INDEX_0, UNIFIED_INDEX_1); 6426ae15f0fd85df9f13a251222d048761a9aa1dcf2aJohn Porto } 6427ae15f0fd85df9f13a251222d048761a9aa1dcf2aJohn Porto break; 6428ae15f0fd85df9f13a251222d048761a9aa1dcf2aJohn Porto CASE_SRCS_IN(0, 0, 1, 0) : { 6429ae15f0fd85df9f13a251222d048761a9aa1dcf2aJohn Porto auto *Unified = lowerShuffleVector_UnifyFromDifferentSrcs(Src1, Index2, 6430ae15f0fd85df9f13a251222d048761a9aa1dcf2aJohn Porto Src0, Index3); 6431ae15f0fd85df9f13a251222d048761a9aa1dcf2aJohn Porto T = lowerShuffleVector_TwoFromSameSrc(Src0, Index0, Index1, Unified, 6432ae15f0fd85df9f13a251222d048761a9aa1dcf2aJohn Porto UNIFIED_INDEX_0, UNIFIED_INDEX_1); 6433ae15f0fd85df9f13a251222d048761a9aa1dcf2aJohn Porto } 6434ae15f0fd85df9f13a251222d048761a9aa1dcf2aJohn Porto break; 6435ae15f0fd85df9f13a251222d048761a9aa1dcf2aJohn Porto CASE_SRCS_IN(0, 0, 1, 1) : { 6436ae15f0fd85df9f13a251222d048761a9aa1dcf2aJohn Porto T = lowerShuffleVector_TwoFromSameSrc(Src0, Index0, Index1, Src1, 6437ae15f0fd85df9f13a251222d048761a9aa1dcf2aJohn Porto Index2, Index3); 6438ae15f0fd85df9f13a251222d048761a9aa1dcf2aJohn Porto } 6439ae15f0fd85df9f13a251222d048761a9aa1dcf2aJohn Porto break; 6440ae15f0fd85df9f13a251222d048761a9aa1dcf2aJohn Porto CASE_SRCS_IN(0, 1, 0, 0) : { 6441ae15f0fd85df9f13a251222d048761a9aa1dcf2aJohn Porto auto *Unified = lowerShuffleVector_UnifyFromDifferentSrcs(Src0, Index0, 6442ae15f0fd85df9f13a251222d048761a9aa1dcf2aJohn Porto Src1, Index1); 6443ae15f0fd85df9f13a251222d048761a9aa1dcf2aJohn Porto T = lowerShuffleVector_TwoFromSameSrc( 6444ae15f0fd85df9f13a251222d048761a9aa1dcf2aJohn Porto Unified, UNIFIED_INDEX_0, UNIFIED_INDEX_1, Src0, Index2, Index3); 6445ae15f0fd85df9f13a251222d048761a9aa1dcf2aJohn Porto } 6446ae15f0fd85df9f13a251222d048761a9aa1dcf2aJohn Porto break; 6447ae15f0fd85df9f13a251222d048761a9aa1dcf2aJohn Porto CASE_SRCS_IN(0, 1, 0, 1) : { 6448ae15f0fd85df9f13a251222d048761a9aa1dcf2aJohn Porto if (Index0 == 0 && (Index1 - ExpectedNumElements) == 0 && Index2 == 1 && 6449ae15f0fd85df9f13a251222d048761a9aa1dcf2aJohn Porto (Index3 - ExpectedNumElements) == 1) { 6450ae15f0fd85df9f13a251222d048761a9aa1dcf2aJohn Porto auto *Src1RM = legalize(Src1, Legal_Reg | Legal_Mem); 6451ae15f0fd85df9f13a251222d048761a9aa1dcf2aJohn Porto auto *Src0R = legalizeToReg(Src0); 6452ae15f0fd85df9f13a251222d048761a9aa1dcf2aJohn Porto T = makeReg(DestTy); 6453ae15f0fd85df9f13a251222d048761a9aa1dcf2aJohn Porto _movp(T, Src0R); 6454ae15f0fd85df9f13a251222d048761a9aa1dcf2aJohn Porto _punpckl(T, Src1RM); 6455ae15f0fd85df9f13a251222d048761a9aa1dcf2aJohn Porto } else if (Index0 == Index2 && Index1 == Index3) { 6456ae15f0fd85df9f13a251222d048761a9aa1dcf2aJohn Porto assert(false && "Following code is untested but likely correct; test " 6457ae15f0fd85df9f13a251222d048761a9aa1dcf2aJohn Porto "and remove assert."); 6458ae15f0fd85df9f13a251222d048761a9aa1dcf2aJohn Porto auto *Unified = lowerShuffleVector_UnifyFromDifferentSrcs( 6459ae15f0fd85df9f13a251222d048761a9aa1dcf2aJohn Porto Src0, Index0, Src1, Index1); 6460ae15f0fd85df9f13a251222d048761a9aa1dcf2aJohn Porto T = lowerShuffleVector_AllFromSameSrc( 6461ae15f0fd85df9f13a251222d048761a9aa1dcf2aJohn Porto Unified, UNIFIED_INDEX_0, UNIFIED_INDEX_1, UNIFIED_INDEX_0, 6462ae15f0fd85df9f13a251222d048761a9aa1dcf2aJohn Porto UNIFIED_INDEX_1); 6463ae15f0fd85df9f13a251222d048761a9aa1dcf2aJohn Porto } else { 6464ae15f0fd85df9f13a251222d048761a9aa1dcf2aJohn Porto auto *Unified0 = lowerShuffleVector_UnifyFromDifferentSrcs( 6465ae15f0fd85df9f13a251222d048761a9aa1dcf2aJohn Porto Src0, Index0, Src1, Index1); 6466ae15f0fd85df9f13a251222d048761a9aa1dcf2aJohn Porto auto *Unified1 = lowerShuffleVector_UnifyFromDifferentSrcs( 6467ae15f0fd85df9f13a251222d048761a9aa1dcf2aJohn Porto Src0, Index2, Src1, Index3); 6468ae15f0fd85df9f13a251222d048761a9aa1dcf2aJohn Porto T = lowerShuffleVector_TwoFromSameSrc( 6469ae15f0fd85df9f13a251222d048761a9aa1dcf2aJohn Porto Unified0, UNIFIED_INDEX_0, UNIFIED_INDEX_1, Unified1, 6470ae15f0fd85df9f13a251222d048761a9aa1dcf2aJohn Porto UNIFIED_INDEX_0, UNIFIED_INDEX_1); 6471ae15f0fd85df9f13a251222d048761a9aa1dcf2aJohn Porto } 6472ae15f0fd85df9f13a251222d048761a9aa1dcf2aJohn Porto } 6473ae15f0fd85df9f13a251222d048761a9aa1dcf2aJohn Porto break; 6474ae15f0fd85df9f13a251222d048761a9aa1dcf2aJohn Porto CASE_SRCS_IN(0, 1, 1, 0) : { 6475ae15f0fd85df9f13a251222d048761a9aa1dcf2aJohn Porto if (Index0 == Index3 && Index1 == Index2) { 6476ae15f0fd85df9f13a251222d048761a9aa1dcf2aJohn Porto auto *Unified = lowerShuffleVector_UnifyFromDifferentSrcs( 6477ae15f0fd85df9f13a251222d048761a9aa1dcf2aJohn Porto Src0, Index0, Src1, Index1); 6478ae15f0fd85df9f13a251222d048761a9aa1dcf2aJohn Porto T = lowerShuffleVector_AllFromSameSrc( 6479ae15f0fd85df9f13a251222d048761a9aa1dcf2aJohn Porto Unified, UNIFIED_INDEX_0, UNIFIED_INDEX_1, UNIFIED_INDEX_1, 6480ae15f0fd85df9f13a251222d048761a9aa1dcf2aJohn Porto UNIFIED_INDEX_0); 6481ae15f0fd85df9f13a251222d048761a9aa1dcf2aJohn Porto } else { 6482ae15f0fd85df9f13a251222d048761a9aa1dcf2aJohn Porto auto *Unified0 = lowerShuffleVector_UnifyFromDifferentSrcs( 6483ae15f0fd85df9f13a251222d048761a9aa1dcf2aJohn Porto Src0, Index0, Src1, Index1); 6484ae15f0fd85df9f13a251222d048761a9aa1dcf2aJohn Porto auto *Unified1 = lowerShuffleVector_UnifyFromDifferentSrcs( 6485ae15f0fd85df9f13a251222d048761a9aa1dcf2aJohn Porto Src1, Index2, Src0, Index3); 6486ae15f0fd85df9f13a251222d048761a9aa1dcf2aJohn Porto T = lowerShuffleVector_TwoFromSameSrc( 6487ae15f0fd85df9f13a251222d048761a9aa1dcf2aJohn Porto Unified0, UNIFIED_INDEX_0, UNIFIED_INDEX_1, Unified1, 6488ae15f0fd85df9f13a251222d048761a9aa1dcf2aJohn Porto UNIFIED_INDEX_0, UNIFIED_INDEX_1); 6489ae15f0fd85df9f13a251222d048761a9aa1dcf2aJohn Porto } 6490ae15f0fd85df9f13a251222d048761a9aa1dcf2aJohn Porto } 6491ae15f0fd85df9f13a251222d048761a9aa1dcf2aJohn Porto break; 6492ae15f0fd85df9f13a251222d048761a9aa1dcf2aJohn Porto CASE_SRCS_IN(0, 1, 1, 1) : { 6493ae15f0fd85df9f13a251222d048761a9aa1dcf2aJohn Porto assert(false && "Following code is untested but likely correct; test " 6494ae15f0fd85df9f13a251222d048761a9aa1dcf2aJohn Porto "and remove assert."); 6495ae15f0fd85df9f13a251222d048761a9aa1dcf2aJohn Porto auto *Unified = lowerShuffleVector_UnifyFromDifferentSrcs(Src0, Index0, 6496ae15f0fd85df9f13a251222d048761a9aa1dcf2aJohn Porto Src1, Index1); 6497ae15f0fd85df9f13a251222d048761a9aa1dcf2aJohn Porto T = lowerShuffleVector_TwoFromSameSrc( 6498ae15f0fd85df9f13a251222d048761a9aa1dcf2aJohn Porto Unified, UNIFIED_INDEX_0, UNIFIED_INDEX_1, Src1, Index2, Index3); 6499ae15f0fd85df9f13a251222d048761a9aa1dcf2aJohn Porto } 6500ae15f0fd85df9f13a251222d048761a9aa1dcf2aJohn Porto break; 6501ae15f0fd85df9f13a251222d048761a9aa1dcf2aJohn Porto CASE_SRCS_IN(1, 0, 0, 0) : { 6502ae15f0fd85df9f13a251222d048761a9aa1dcf2aJohn Porto auto *Unified = lowerShuffleVector_UnifyFromDifferentSrcs(Src1, Index0, 6503ae15f0fd85df9f13a251222d048761a9aa1dcf2aJohn Porto Src0, Index1); 6504ae15f0fd85df9f13a251222d048761a9aa1dcf2aJohn Porto T = lowerShuffleVector_TwoFromSameSrc( 6505ae15f0fd85df9f13a251222d048761a9aa1dcf2aJohn Porto Unified, UNIFIED_INDEX_0, UNIFIED_INDEX_1, Src0, Index2, Index3); 6506ae15f0fd85df9f13a251222d048761a9aa1dcf2aJohn Porto } 6507ae15f0fd85df9f13a251222d048761a9aa1dcf2aJohn Porto break; 6508ae15f0fd85df9f13a251222d048761a9aa1dcf2aJohn Porto CASE_SRCS_IN(1, 0, 0, 1) : { 6509ae15f0fd85df9f13a251222d048761a9aa1dcf2aJohn Porto if (Index0 == Index3 && Index1 == Index2) { 6510ae15f0fd85df9f13a251222d048761a9aa1dcf2aJohn Porto assert(false && "Following code is untested but likely correct; test " 6511ae15f0fd85df9f13a251222d048761a9aa1dcf2aJohn Porto "and remove assert."); 6512ae15f0fd85df9f13a251222d048761a9aa1dcf2aJohn Porto auto *Unified = lowerShuffleVector_UnifyFromDifferentSrcs( 6513ae15f0fd85df9f13a251222d048761a9aa1dcf2aJohn Porto Src1, Index0, Src0, Index1); 6514ae15f0fd85df9f13a251222d048761a9aa1dcf2aJohn Porto T = lowerShuffleVector_AllFromSameSrc( 6515ae15f0fd85df9f13a251222d048761a9aa1dcf2aJohn Porto Unified, UNIFIED_INDEX_0, UNIFIED_INDEX_1, UNIFIED_INDEX_1, 6516ae15f0fd85df9f13a251222d048761a9aa1dcf2aJohn Porto UNIFIED_INDEX_0); 6517ae15f0fd85df9f13a251222d048761a9aa1dcf2aJohn Porto } else { 6518ae15f0fd85df9f13a251222d048761a9aa1dcf2aJohn Porto assert(false && "Following code is untested but likely correct; test " 6519ae15f0fd85df9f13a251222d048761a9aa1dcf2aJohn Porto "and remove assert."); 6520ae15f0fd85df9f13a251222d048761a9aa1dcf2aJohn Porto auto *Unified0 = lowerShuffleVector_UnifyFromDifferentSrcs( 6521ae15f0fd85df9f13a251222d048761a9aa1dcf2aJohn Porto Src1, Index0, Src0, Index1); 6522ae15f0fd85df9f13a251222d048761a9aa1dcf2aJohn Porto auto *Unified1 = lowerShuffleVector_UnifyFromDifferentSrcs( 6523ae15f0fd85df9f13a251222d048761a9aa1dcf2aJohn Porto Src0, Index2, Src1, Index3); 6524ae15f0fd85df9f13a251222d048761a9aa1dcf2aJohn Porto T = lowerShuffleVector_TwoFromSameSrc( 6525ae15f0fd85df9f13a251222d048761a9aa1dcf2aJohn Porto Unified0, UNIFIED_INDEX_0, UNIFIED_INDEX_1, Unified1, 6526ae15f0fd85df9f13a251222d048761a9aa1dcf2aJohn Porto UNIFIED_INDEX_0, UNIFIED_INDEX_1); 6527ae15f0fd85df9f13a251222d048761a9aa1dcf2aJohn Porto } 6528ae15f0fd85df9f13a251222d048761a9aa1dcf2aJohn Porto } 6529ae15f0fd85df9f13a251222d048761a9aa1dcf2aJohn Porto break; 6530ae15f0fd85df9f13a251222d048761a9aa1dcf2aJohn Porto CASE_SRCS_IN(1, 0, 1, 0) : { 6531ae15f0fd85df9f13a251222d048761a9aa1dcf2aJohn Porto if ((Index0 - ExpectedNumElements) == 0 && Index1 == 0 && 6532ae15f0fd85df9f13a251222d048761a9aa1dcf2aJohn Porto (Index2 - ExpectedNumElements) == 1 && Index3 == 1) { 6533ae15f0fd85df9f13a251222d048761a9aa1dcf2aJohn Porto auto *Src1RM = legalize(Src0, Legal_Reg | Legal_Mem); 6534ae15f0fd85df9f13a251222d048761a9aa1dcf2aJohn Porto auto *Src0R = legalizeToReg(Src1); 6535ae15f0fd85df9f13a251222d048761a9aa1dcf2aJohn Porto T = makeReg(DestTy); 6536ae15f0fd85df9f13a251222d048761a9aa1dcf2aJohn Porto _movp(T, Src0R); 6537ae15f0fd85df9f13a251222d048761a9aa1dcf2aJohn Porto _punpckl(T, Src1RM); 6538ae15f0fd85df9f13a251222d048761a9aa1dcf2aJohn Porto } else if (Index0 == Index2 && Index1 == Index3) { 6539ae15f0fd85df9f13a251222d048761a9aa1dcf2aJohn Porto auto *Unified = lowerShuffleVector_UnifyFromDifferentSrcs( 6540ae15f0fd85df9f13a251222d048761a9aa1dcf2aJohn Porto Src1, Index0, Src0, Index1); 6541ae15f0fd85df9f13a251222d048761a9aa1dcf2aJohn Porto T = lowerShuffleVector_AllFromSameSrc( 6542ae15f0fd85df9f13a251222d048761a9aa1dcf2aJohn Porto Unified, UNIFIED_INDEX_0, UNIFIED_INDEX_1, UNIFIED_INDEX_0, 6543ae15f0fd85df9f13a251222d048761a9aa1dcf2aJohn Porto UNIFIED_INDEX_1); 6544ae15f0fd85df9f13a251222d048761a9aa1dcf2aJohn Porto } else { 6545ae15f0fd85df9f13a251222d048761a9aa1dcf2aJohn Porto auto *Unified0 = lowerShuffleVector_UnifyFromDifferentSrcs( 6546ae15f0fd85df9f13a251222d048761a9aa1dcf2aJohn Porto Src1, Index0, Src0, Index1); 6547ae15f0fd85df9f13a251222d048761a9aa1dcf2aJohn Porto auto *Unified1 = lowerShuffleVector_UnifyFromDifferentSrcs( 6548ae15f0fd85df9f13a251222d048761a9aa1dcf2aJohn Porto Src1, Index2, Src0, Index3); 6549ae15f0fd85df9f13a251222d048761a9aa1dcf2aJohn Porto T = lowerShuffleVector_TwoFromSameSrc( 6550ae15f0fd85df9f13a251222d048761a9aa1dcf2aJohn Porto Unified0, UNIFIED_INDEX_0, UNIFIED_INDEX_1, Unified1, 6551ae15f0fd85df9f13a251222d048761a9aa1dcf2aJohn Porto UNIFIED_INDEX_0, UNIFIED_INDEX_1); 6552ae15f0fd85df9f13a251222d048761a9aa1dcf2aJohn Porto } 6553ae15f0fd85df9f13a251222d048761a9aa1dcf2aJohn Porto } 6554ae15f0fd85df9f13a251222d048761a9aa1dcf2aJohn Porto break; 6555ae15f0fd85df9f13a251222d048761a9aa1dcf2aJohn Porto CASE_SRCS_IN(1, 0, 1, 1) : { 6556ae15f0fd85df9f13a251222d048761a9aa1dcf2aJohn Porto assert(false && "Following code is untested but likely correct; test " 6557ae15f0fd85df9f13a251222d048761a9aa1dcf2aJohn Porto "and remove assert."); 6558ae15f0fd85df9f13a251222d048761a9aa1dcf2aJohn Porto auto *Unified = lowerShuffleVector_UnifyFromDifferentSrcs(Src1, Index0, 6559ae15f0fd85df9f13a251222d048761a9aa1dcf2aJohn Porto Src0, Index1); 6560ae15f0fd85df9f13a251222d048761a9aa1dcf2aJohn Porto T = lowerShuffleVector_TwoFromSameSrc( 6561ae15f0fd85df9f13a251222d048761a9aa1dcf2aJohn Porto Unified, UNIFIED_INDEX_0, UNIFIED_INDEX_1, Src1, Index2, Index3); 6562ae15f0fd85df9f13a251222d048761a9aa1dcf2aJohn Porto } 6563ae15f0fd85df9f13a251222d048761a9aa1dcf2aJohn Porto break; 6564ae15f0fd85df9f13a251222d048761a9aa1dcf2aJohn Porto CASE_SRCS_IN(1, 1, 0, 0) : { 6565ae15f0fd85df9f13a251222d048761a9aa1dcf2aJohn Porto T = lowerShuffleVector_TwoFromSameSrc(Src1, Index0, Index1, Src0, 6566ae15f0fd85df9f13a251222d048761a9aa1dcf2aJohn Porto Index2, Index3); 6567ae15f0fd85df9f13a251222d048761a9aa1dcf2aJohn Porto } 6568ae15f0fd85df9f13a251222d048761a9aa1dcf2aJohn Porto break; 6569ae15f0fd85df9f13a251222d048761a9aa1dcf2aJohn Porto CASE_SRCS_IN(1, 1, 0, 1) : { 6570ae15f0fd85df9f13a251222d048761a9aa1dcf2aJohn Porto assert(false && "Following code is untested but likely correct; test " 6571ae15f0fd85df9f13a251222d048761a9aa1dcf2aJohn Porto "and remove assert."); 6572ae15f0fd85df9f13a251222d048761a9aa1dcf2aJohn Porto auto *Unified = lowerShuffleVector_UnifyFromDifferentSrcs(Src0, Index2, 6573ae15f0fd85df9f13a251222d048761a9aa1dcf2aJohn Porto Src1, Index3); 6574ae15f0fd85df9f13a251222d048761a9aa1dcf2aJohn Porto T = lowerShuffleVector_TwoFromSameSrc(Src1, Index0, Index1, Unified, 6575ae15f0fd85df9f13a251222d048761a9aa1dcf2aJohn Porto UNIFIED_INDEX_0, UNIFIED_INDEX_1); 6576ae15f0fd85df9f13a251222d048761a9aa1dcf2aJohn Porto } 6577ae15f0fd85df9f13a251222d048761a9aa1dcf2aJohn Porto break; 6578ae15f0fd85df9f13a251222d048761a9aa1dcf2aJohn Porto CASE_SRCS_IN(1, 1, 1, 0) : { 6579ae15f0fd85df9f13a251222d048761a9aa1dcf2aJohn Porto auto *Unified = lowerShuffleVector_UnifyFromDifferentSrcs(Src1, Index2, 6580ae15f0fd85df9f13a251222d048761a9aa1dcf2aJohn Porto Src0, Index3); 6581ae15f0fd85df9f13a251222d048761a9aa1dcf2aJohn Porto T = lowerShuffleVector_TwoFromSameSrc(Src1, Index0, Index1, Unified, 6582ae15f0fd85df9f13a251222d048761a9aa1dcf2aJohn Porto UNIFIED_INDEX_0, UNIFIED_INDEX_1); 6583ae15f0fd85df9f13a251222d048761a9aa1dcf2aJohn Porto } 6584ae15f0fd85df9f13a251222d048761a9aa1dcf2aJohn Porto break; 6585ae15f0fd85df9f13a251222d048761a9aa1dcf2aJohn Porto CASE_SRCS_IN(1, 1, 1, 1) : { 6586ae15f0fd85df9f13a251222d048761a9aa1dcf2aJohn Porto assert(false && "Following code is untested but likely correct; test " 6587ae15f0fd85df9f13a251222d048761a9aa1dcf2aJohn Porto "and remove assert."); 6588ae15f0fd85df9f13a251222d048761a9aa1dcf2aJohn Porto T = lowerShuffleVector_AllFromSameSrc(Src1, Index0, Index1, Index2, 6589ae15f0fd85df9f13a251222d048761a9aa1dcf2aJohn Porto Index3); 6590ae15f0fd85df9f13a251222d048761a9aa1dcf2aJohn Porto } 6591ae15f0fd85df9f13a251222d048761a9aa1dcf2aJohn Porto break; 6592ae15f0fd85df9f13a251222d048761a9aa1dcf2aJohn Porto#undef CASE_SRCS_IN 6593ae15f0fd85df9f13a251222d048761a9aa1dcf2aJohn Porto } 6594ae15f0fd85df9f13a251222d048761a9aa1dcf2aJohn Porto 6595ae15f0fd85df9f13a251222d048761a9aa1dcf2aJohn Porto assert(T != nullptr); 6596ae15f0fd85df9f13a251222d048761a9aa1dcf2aJohn Porto assert(T->getType() == DestTy); 6597ae15f0fd85df9f13a251222d048761a9aa1dcf2aJohn Porto _movp(Dest, T); 6598ae15f0fd85df9f13a251222d048761a9aa1dcf2aJohn Porto return; 6599ae15f0fd85df9f13a251222d048761a9aa1dcf2aJohn Porto } break; 6600a47c11c7f17022050043d69c0802241e0747a056John Porto } 6601a47c11c7f17022050043d69c0802241e0747a056John Porto 6602a47c11c7f17022050043d69c0802241e0747a056John Porto // Unoptimized shuffle. Perform a series of inserts and extracts. 6603a47c11c7f17022050043d69c0802241e0747a056John Porto Context.insert<InstFakeDef>(T); 6604a47c11c7f17022050043d69c0802241e0747a056John Porto const Type ElementType = typeElementType(DestTy); 6605a47c11c7f17022050043d69c0802241e0747a056John Porto for (SizeT I = 0; I < Instr->getNumIndexes(); ++I) { 6606a47c11c7f17022050043d69c0802241e0747a056John Porto auto *Index = Instr->getIndex(I); 6607a47c11c7f17022050043d69c0802241e0747a056John Porto const SizeT Elem = Index->getValue(); 6608a47c11c7f17022050043d69c0802241e0747a056John Porto auto *ExtElmt = makeReg(ElementType); 6609a47c11c7f17022050043d69c0802241e0747a056John Porto if (Elem < NumElements) { 6610a47c11c7f17022050043d69c0802241e0747a056John Porto lowerExtractElement( 6611a47c11c7f17022050043d69c0802241e0747a056John Porto InstExtractElement::create(Func, ExtElmt, Src0, Index)); 6612a47c11c7f17022050043d69c0802241e0747a056John Porto } else { 6613a47c11c7f17022050043d69c0802241e0747a056John Porto lowerExtractElement(InstExtractElement::create( 6614a47c11c7f17022050043d69c0802241e0747a056John Porto Func, ExtElmt, Src1, 6615a47c11c7f17022050043d69c0802241e0747a056John Porto Ctx->getConstantInt32(Index->getValue() - NumElements))); 6616a47c11c7f17022050043d69c0802241e0747a056John Porto } 6617a47c11c7f17022050043d69c0802241e0747a056John Porto auto *NewT = makeReg(DestTy); 6618a47c11c7f17022050043d69c0802241e0747a056John Porto lowerInsertElement(InstInsertElement::create(Func, NewT, T, ExtElmt, 6619a47c11c7f17022050043d69c0802241e0747a056John Porto Ctx->getConstantInt32(I))); 6620a47c11c7f17022050043d69c0802241e0747a056John Porto T = NewT; 6621a47c11c7f17022050043d69c0802241e0747a056John Porto } 6622a47c11c7f17022050043d69c0802241e0747a056John Porto _movp(Dest, T); 6623a47c11c7f17022050043d69c0802241e0747a056John Porto} 6624a47c11c7f17022050043d69c0802241e0747a056John Porto 6625a47c11c7f17022050043d69c0802241e0747a056John Portotemplate <typename TraitsType> 66264a56686b5b56db6803f90ad53514bf2fa190d9f7John Portovoid TargetX86Base<TraitsType>::lowerSelect(const InstSelect *Select) { 6627e398428c3ef8ed2339c0f334dd30a7d3f0ed4b6eDavid Sehr Variable *Dest = Select->getDest(); 66287e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto 6629e398428c3ef8ed2339c0f334dd30a7d3f0ed4b6eDavid Sehr Operand *Condition = Select->getCondition(); 66307e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto // Handle folding opportunities. 6631e398428c3ef8ed2339c0f334dd30a7d3f0ed4b6eDavid Sehr if (const Inst *Producer = FoldingInfo.getProducerFor(Condition)) { 66327e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto assert(Producer->isDeleted()); 6633caeaa27b18e94cbad635dbe21da87e816a35c7daJim Stichnoth switch (BoolFolding<Traits>::getProducerKind(Producer)) { 66347e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto default: 66357e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto break; 6636caeaa27b18e94cbad635dbe21da87e816a35c7daJim Stichnoth case BoolFolding<Traits>::PK_Icmp32: 6637caeaa27b18e94cbad635dbe21da87e816a35c7daJim Stichnoth case BoolFolding<Traits>::PK_Icmp64: { 66382d6c82679e9786a924b76023aa30e1f44d0f8cbbJim Stichnoth lowerIcmpAndConsumer(llvm::cast<InstIcmp>(Producer), Select); 6639e398428c3ef8ed2339c0f334dd30a7d3f0ed4b6eDavid Sehr return; 6640e398428c3ef8ed2339c0f334dd30a7d3f0ed4b6eDavid Sehr } 6641caeaa27b18e94cbad635dbe21da87e816a35c7daJim Stichnoth case BoolFolding<Traits>::PK_Fcmp: { 66422d6c82679e9786a924b76023aa30e1f44d0f8cbbJim Stichnoth lowerFcmpAndConsumer(llvm::cast<InstFcmp>(Producer), Select); 6643e398428c3ef8ed2339c0f334dd30a7d3f0ed4b6eDavid Sehr return; 6644e398428c3ef8ed2339c0f334dd30a7d3f0ed4b6eDavid Sehr } 66457e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto } 66467e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto } 66477e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto 664847b6ba6db7c8b8dd36ac54dbd19ae7b4a5b77424Nicolas Capens if (isVectorType(Dest->getType())) { 664947b6ba6db7c8b8dd36ac54dbd19ae7b4a5b77424Nicolas Capens lowerSelectVector(Select); 665047b6ba6db7c8b8dd36ac54dbd19ae7b4a5b77424Nicolas Capens return; 665147b6ba6db7c8b8dd36ac54dbd19ae7b4a5b77424Nicolas Capens } 665247b6ba6db7c8b8dd36ac54dbd19ae7b4a5b77424Nicolas Capens 6653e398428c3ef8ed2339c0f334dd30a7d3f0ed4b6eDavid Sehr Operand *CmpResult = legalize(Condition, Legal_Reg | Legal_Mem); 6654e398428c3ef8ed2339c0f334dd30a7d3f0ed4b6eDavid Sehr Operand *Zero = Ctx->getConstantZero(IceType_i32); 6655e398428c3ef8ed2339c0f334dd30a7d3f0ed4b6eDavid Sehr _cmp(CmpResult, Zero); 6656e398428c3ef8ed2339c0f334dd30a7d3f0ed4b6eDavid Sehr Operand *SrcT = Select->getTrueOperand(); 6657e398428c3ef8ed2339c0f334dd30a7d3f0ed4b6eDavid Sehr Operand *SrcF = Select->getFalseOperand(); 66584a56686b5b56db6803f90ad53514bf2fa190d9f7John Porto const BrCond Cond = Traits::Cond::Br_ne; 6659e398428c3ef8ed2339c0f334dd30a7d3f0ed4b6eDavid Sehr lowerSelectMove(Dest, Cond, SrcT, SrcF); 6660e398428c3ef8ed2339c0f334dd30a7d3f0ed4b6eDavid Sehr} 6661e398428c3ef8ed2339c0f334dd30a7d3f0ed4b6eDavid Sehr 66624a56686b5b56db6803f90ad53514bf2fa190d9f7John Portotemplate <typename TraitsType> 66634a56686b5b56db6803f90ad53514bf2fa190d9f7John Portovoid TargetX86Base<TraitsType>::lowerSelectMove(Variable *Dest, BrCond Cond, 66644a56686b5b56db6803f90ad53514bf2fa190d9f7John Porto Operand *SrcT, Operand *SrcF) { 6665e398428c3ef8ed2339c0f334dd30a7d3f0ed4b6eDavid Sehr Type DestTy = Dest->getType(); 66667e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto if (typeWidthInBytes(DestTy) == 1 || isFloatingType(DestTy)) { 666757e126899b20c65ff3ea23a3b7d7a67ab30b99dcAndrew Scull // The cmov instruction doesn't allow 8-bit or FP operands, so we need 666857e126899b20c65ff3ea23a3b7d7a67ab30b99dcAndrew Scull // explicit control flow. 66697e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto // d=cmp e,f; a=d?b:c ==> cmp e,f; a=b; jne L1; a=c; L1: 66704a56686b5b56db6803f90ad53514bf2fa190d9f7John Porto auto *Label = InstX86Label::create(Func, this); 66717e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto SrcT = legalize(SrcT, Legal_Reg | Legal_Imm); 66727e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto _mov(Dest, SrcT); 66737e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto _br(Cond, Label); 66747e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto SrcF = legalize(SrcF, Legal_Reg | Legal_Imm); 6675e398428c3ef8ed2339c0f334dd30a7d3f0ed4b6eDavid Sehr _redefined(_mov(Dest, SrcF)); 66767e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto Context.insert(Label); 66777e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto return; 66787e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto } 66797e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto // mov t, SrcF; cmov_cond t, SrcT; mov dest, t 668057e126899b20c65ff3ea23a3b7d7a67ab30b99dcAndrew Scull // But if SrcT is immediate, we might be able to do better, as the cmov 668157e126899b20c65ff3ea23a3b7d7a67ab30b99dcAndrew Scull // instruction doesn't allow an immediate operand: 66827e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto // mov t, SrcT; cmov_!cond t, SrcF; mov dest, t 66837e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto if (llvm::isa<Constant>(SrcT) && !llvm::isa<Constant>(SrcF)) { 66847e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto std::swap(SrcT, SrcF); 66854a56686b5b56db6803f90ad53514bf2fa190d9f7John Porto Cond = InstImpl<TraitsType>::InstX86Base::getOppositeCondition(Cond); 66867e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto } 66871d235425dab1f3dd059973fc53f1b1d5879469e3John Porto if (!Traits::Is64Bit && DestTy == IceType_i64) { 6688fbdd244013aad0808220386eb6bd17eb0635dd4cJan Voung SrcT = legalizeUndef(SrcT); 6689fbdd244013aad0808220386eb6bd17eb0635dd4cJan Voung SrcF = legalizeUndef(SrcF); 66907e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto // Set the low portion. 66912d6c82679e9786a924b76023aa30e1f44d0f8cbbJim Stichnoth auto *DestLo = llvm::cast<Variable>(loOperand(Dest)); 6692e398428c3ef8ed2339c0f334dd30a7d3f0ed4b6eDavid Sehr lowerSelectIntMove(DestLo, Cond, loOperand(SrcT), loOperand(SrcF)); 66937e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto // Set the high portion. 66942d6c82679e9786a924b76023aa30e1f44d0f8cbbJim Stichnoth auto *DestHi = llvm::cast<Variable>(hiOperand(Dest)); 6695e398428c3ef8ed2339c0f334dd30a7d3f0ed4b6eDavid Sehr lowerSelectIntMove(DestHi, Cond, hiOperand(SrcT), hiOperand(SrcF)); 66967e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto return; 66977e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto } 66987e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto 66991d235425dab1f3dd059973fc53f1b1d5879469e3John Porto assert(DestTy == IceType_i16 || DestTy == IceType_i32 || 67001d235425dab1f3dd059973fc53f1b1d5879469e3John Porto (Traits::Is64Bit && DestTy == IceType_i64)); 6701e398428c3ef8ed2339c0f334dd30a7d3f0ed4b6eDavid Sehr lowerSelectIntMove(Dest, Cond, SrcT, SrcF); 6702e398428c3ef8ed2339c0f334dd30a7d3f0ed4b6eDavid Sehr} 6703e398428c3ef8ed2339c0f334dd30a7d3f0ed4b6eDavid Sehr 67044a56686b5b56db6803f90ad53514bf2fa190d9f7John Portotemplate <typename TraitsType> 67054a56686b5b56db6803f90ad53514bf2fa190d9f7John Portovoid TargetX86Base<TraitsType>::lowerSelectIntMove(Variable *Dest, BrCond Cond, 67064a56686b5b56db6803f90ad53514bf2fa190d9f7John Porto Operand *SrcT, 67074a56686b5b56db6803f90ad53514bf2fa190d9f7John Porto Operand *SrcF) { 67087e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto Variable *T = nullptr; 67097e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto SrcF = legalize(SrcF); 67107e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto _mov(T, SrcF); 67117e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto SrcT = legalize(SrcT, Legal_Reg | Legal_Mem); 67127e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto _cmov(T, SrcT, Cond); 67137e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto _mov(Dest, T); 67147e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto} 67157e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto 67164a56686b5b56db6803f90ad53514bf2fa190d9f7John Portotemplate <typename TraitsType> 67174a56686b5b56db6803f90ad53514bf2fa190d9f7John Portovoid TargetX86Base<TraitsType>::lowerMove(Variable *Dest, Operand *Src, 67184a56686b5b56db6803f90ad53514bf2fa190d9f7John Porto bool IsRedefinition) { 6719e398428c3ef8ed2339c0f334dd30a7d3f0ed4b6eDavid Sehr assert(Dest->getType() == Src->getType()); 6720e398428c3ef8ed2339c0f334dd30a7d3f0ed4b6eDavid Sehr assert(!Dest->isRematerializable()); 6721e398428c3ef8ed2339c0f334dd30a7d3f0ed4b6eDavid Sehr if (!Traits::Is64Bit && Dest->getType() == IceType_i64) { 6722e398428c3ef8ed2339c0f334dd30a7d3f0ed4b6eDavid Sehr Src = legalize(Src); 6723e398428c3ef8ed2339c0f334dd30a7d3f0ed4b6eDavid Sehr Operand *SrcLo = loOperand(Src); 6724e398428c3ef8ed2339c0f334dd30a7d3f0ed4b6eDavid Sehr Operand *SrcHi = hiOperand(Src); 67252d6c82679e9786a924b76023aa30e1f44d0f8cbbJim Stichnoth auto *DestLo = llvm::cast<Variable>(loOperand(Dest)); 67262d6c82679e9786a924b76023aa30e1f44d0f8cbbJim Stichnoth auto *DestHi = llvm::cast<Variable>(hiOperand(Dest)); 6727e398428c3ef8ed2339c0f334dd30a7d3f0ed4b6eDavid Sehr Variable *T_Lo = nullptr, *T_Hi = nullptr; 6728e398428c3ef8ed2339c0f334dd30a7d3f0ed4b6eDavid Sehr _mov(T_Lo, SrcLo); 6729e398428c3ef8ed2339c0f334dd30a7d3f0ed4b6eDavid Sehr _redefined(_mov(DestLo, T_Lo), IsRedefinition); 6730e398428c3ef8ed2339c0f334dd30a7d3f0ed4b6eDavid Sehr _mov(T_Hi, SrcHi); 6731e398428c3ef8ed2339c0f334dd30a7d3f0ed4b6eDavid Sehr _redefined(_mov(DestHi, T_Hi), IsRedefinition); 6732e398428c3ef8ed2339c0f334dd30a7d3f0ed4b6eDavid Sehr } else { 6733e398428c3ef8ed2339c0f334dd30a7d3f0ed4b6eDavid Sehr Operand *SrcLegal; 6734e398428c3ef8ed2339c0f334dd30a7d3f0ed4b6eDavid Sehr if (Dest->hasReg()) { 6735e398428c3ef8ed2339c0f334dd30a7d3f0ed4b6eDavid Sehr // If Dest already has a physical register, then only basic legalization 6736e398428c3ef8ed2339c0f334dd30a7d3f0ed4b6eDavid Sehr // is needed, as the source operand can be a register, immediate, or 6737e398428c3ef8ed2339c0f334dd30a7d3f0ed4b6eDavid Sehr // memory. 6738e398428c3ef8ed2339c0f334dd30a7d3f0ed4b6eDavid Sehr SrcLegal = legalize(Src, Legal_Reg, Dest->getRegNum()); 6739e398428c3ef8ed2339c0f334dd30a7d3f0ed4b6eDavid Sehr } else { 6740e398428c3ef8ed2339c0f334dd30a7d3f0ed4b6eDavid Sehr // If Dest could be a stack operand, then RI must be a physical register 6741e398428c3ef8ed2339c0f334dd30a7d3f0ed4b6eDavid Sehr // or a scalar integer immediate. 6742e398428c3ef8ed2339c0f334dd30a7d3f0ed4b6eDavid Sehr SrcLegal = legalize(Src, Legal_Reg | Legal_Imm); 6743e398428c3ef8ed2339c0f334dd30a7d3f0ed4b6eDavid Sehr } 6744e398428c3ef8ed2339c0f334dd30a7d3f0ed4b6eDavid Sehr if (isVectorType(Dest->getType())) { 6745e398428c3ef8ed2339c0f334dd30a7d3f0ed4b6eDavid Sehr _redefined(_movp(Dest, SrcLegal), IsRedefinition); 6746e398428c3ef8ed2339c0f334dd30a7d3f0ed4b6eDavid Sehr } else { 6747e398428c3ef8ed2339c0f334dd30a7d3f0ed4b6eDavid Sehr _redefined(_mov(Dest, SrcLegal), IsRedefinition); 6748e398428c3ef8ed2339c0f334dd30a7d3f0ed4b6eDavid Sehr } 6749e398428c3ef8ed2339c0f334dd30a7d3f0ed4b6eDavid Sehr } 6750e398428c3ef8ed2339c0f334dd30a7d3f0ed4b6eDavid Sehr} 6751e398428c3ef8ed2339c0f334dd30a7d3f0ed4b6eDavid Sehr 67524a56686b5b56db6803f90ad53514bf2fa190d9f7John Portotemplate <typename TraitsType> 67534a56686b5b56db6803f90ad53514bf2fa190d9f7John Portobool TargetX86Base<TraitsType>::lowerOptimizeFcmpSelect( 67544a56686b5b56db6803f90ad53514bf2fa190d9f7John Porto const InstFcmp *Fcmp, const InstSelect *Select) { 6755e398428c3ef8ed2339c0f334dd30a7d3f0ed4b6eDavid Sehr Operand *CmpSrc0 = Fcmp->getSrc(0); 6756e398428c3ef8ed2339c0f334dd30a7d3f0ed4b6eDavid Sehr Operand *CmpSrc1 = Fcmp->getSrc(1); 6757e398428c3ef8ed2339c0f334dd30a7d3f0ed4b6eDavid Sehr Operand *SelectSrcT = Select->getTrueOperand(); 6758e398428c3ef8ed2339c0f334dd30a7d3f0ed4b6eDavid Sehr Operand *SelectSrcF = Select->getFalseOperand(); 675947b6ba6db7c8b8dd36ac54dbd19ae7b4a5b77424Nicolas Capens Variable *SelectDest = Select->getDest(); 6760e398428c3ef8ed2339c0f334dd30a7d3f0ed4b6eDavid Sehr 676147b6ba6db7c8b8dd36ac54dbd19ae7b4a5b77424Nicolas Capens // TODO(capn): also handle swapped compare/select operand order. 676247b6ba6db7c8b8dd36ac54dbd19ae7b4a5b77424Nicolas Capens if (CmpSrc0 != SelectSrcT || CmpSrc1 != SelectSrcF) 6763e398428c3ef8ed2339c0f334dd30a7d3f0ed4b6eDavid Sehr return false; 6764e398428c3ef8ed2339c0f334dd30a7d3f0ed4b6eDavid Sehr 676547b6ba6db7c8b8dd36ac54dbd19ae7b4a5b77424Nicolas Capens // TODO(sehr, stichnot): fcmp/select patterns (e.g., minsd/maxss) go here. 6766e398428c3ef8ed2339c0f334dd30a7d3f0ed4b6eDavid Sehr InstFcmp::FCond Condition = Fcmp->getCondition(); 6767e398428c3ef8ed2339c0f334dd30a7d3f0ed4b6eDavid Sehr switch (Condition) { 6768e398428c3ef8ed2339c0f334dd30a7d3f0ed4b6eDavid Sehr default: 6769e398428c3ef8ed2339c0f334dd30a7d3f0ed4b6eDavid Sehr return false; 6770e398428c3ef8ed2339c0f334dd30a7d3f0ed4b6eDavid Sehr case InstFcmp::True: 677147b6ba6db7c8b8dd36ac54dbd19ae7b4a5b77424Nicolas Capens break; 6772e398428c3ef8ed2339c0f334dd30a7d3f0ed4b6eDavid Sehr case InstFcmp::False: 6773e398428c3ef8ed2339c0f334dd30a7d3f0ed4b6eDavid Sehr break; 677447b6ba6db7c8b8dd36ac54dbd19ae7b4a5b77424Nicolas Capens case InstFcmp::Ogt: { 677547b6ba6db7c8b8dd36ac54dbd19ae7b4a5b77424Nicolas Capens Variable *T = makeReg(SelectDest->getType()); 677647b6ba6db7c8b8dd36ac54dbd19ae7b4a5b77424Nicolas Capens if (isScalarFloatingType(SelectSrcT->getType())) { 677747b6ba6db7c8b8dd36ac54dbd19ae7b4a5b77424Nicolas Capens _mov(T, legalize(SelectSrcT, Legal_Reg | Legal_Mem)); 677847b6ba6db7c8b8dd36ac54dbd19ae7b4a5b77424Nicolas Capens _maxss(T, legalize(SelectSrcF, Legal_Reg | Legal_Mem)); 677947b6ba6db7c8b8dd36ac54dbd19ae7b4a5b77424Nicolas Capens _mov(SelectDest, T); 678047b6ba6db7c8b8dd36ac54dbd19ae7b4a5b77424Nicolas Capens } else { 678147b6ba6db7c8b8dd36ac54dbd19ae7b4a5b77424Nicolas Capens _movp(T, legalize(SelectSrcT, Legal_Reg | Legal_Mem)); 678247b6ba6db7c8b8dd36ac54dbd19ae7b4a5b77424Nicolas Capens _maxps(T, legalize(SelectSrcF, Legal_Reg | Legal_Mem)); 678347b6ba6db7c8b8dd36ac54dbd19ae7b4a5b77424Nicolas Capens _movp(SelectDest, T); 678447b6ba6db7c8b8dd36ac54dbd19ae7b4a5b77424Nicolas Capens } 678547b6ba6db7c8b8dd36ac54dbd19ae7b4a5b77424Nicolas Capens return true; 678647b6ba6db7c8b8dd36ac54dbd19ae7b4a5b77424Nicolas Capens } break; 678747b6ba6db7c8b8dd36ac54dbd19ae7b4a5b77424Nicolas Capens case InstFcmp::Olt: { 678847b6ba6db7c8b8dd36ac54dbd19ae7b4a5b77424Nicolas Capens Variable *T = makeReg(SelectSrcT->getType()); 678947b6ba6db7c8b8dd36ac54dbd19ae7b4a5b77424Nicolas Capens if (isScalarFloatingType(SelectSrcT->getType())) { 679047b6ba6db7c8b8dd36ac54dbd19ae7b4a5b77424Nicolas Capens _mov(T, legalize(SelectSrcT, Legal_Reg | Legal_Mem)); 679147b6ba6db7c8b8dd36ac54dbd19ae7b4a5b77424Nicolas Capens _minss(T, legalize(SelectSrcF, Legal_Reg | Legal_Mem)); 679247b6ba6db7c8b8dd36ac54dbd19ae7b4a5b77424Nicolas Capens _mov(SelectDest, T); 679347b6ba6db7c8b8dd36ac54dbd19ae7b4a5b77424Nicolas Capens } else { 679447b6ba6db7c8b8dd36ac54dbd19ae7b4a5b77424Nicolas Capens _movp(T, legalize(SelectSrcT, Legal_Reg | Legal_Mem)); 679547b6ba6db7c8b8dd36ac54dbd19ae7b4a5b77424Nicolas Capens _minps(T, legalize(SelectSrcF, Legal_Reg | Legal_Mem)); 679647b6ba6db7c8b8dd36ac54dbd19ae7b4a5b77424Nicolas Capens _movp(SelectDest, T); 679747b6ba6db7c8b8dd36ac54dbd19ae7b4a5b77424Nicolas Capens } 679847b6ba6db7c8b8dd36ac54dbd19ae7b4a5b77424Nicolas Capens return true; 679947b6ba6db7c8b8dd36ac54dbd19ae7b4a5b77424Nicolas Capens } break; 6800e398428c3ef8ed2339c0f334dd30a7d3f0ed4b6eDavid Sehr } 6801e398428c3ef8ed2339c0f334dd30a7d3f0ed4b6eDavid Sehr return false; 6802e398428c3ef8ed2339c0f334dd30a7d3f0ed4b6eDavid Sehr} 6803e398428c3ef8ed2339c0f334dd30a7d3f0ed4b6eDavid Sehr 68044a56686b5b56db6803f90ad53514bf2fa190d9f7John Portotemplate <typename TraitsType> 68054a56686b5b56db6803f90ad53514bf2fa190d9f7John Portovoid TargetX86Base<TraitsType>::lowerIcmp(const InstIcmp *Icmp) { 6806e398428c3ef8ed2339c0f334dd30a7d3f0ed4b6eDavid Sehr Variable *Dest = Icmp->getDest(); 6807e398428c3ef8ed2339c0f334dd30a7d3f0ed4b6eDavid Sehr if (isVectorType(Dest->getType())) { 6808e398428c3ef8ed2339c0f334dd30a7d3f0ed4b6eDavid Sehr lowerIcmpVector(Icmp); 6809e398428c3ef8ed2339c0f334dd30a7d3f0ed4b6eDavid Sehr } else { 6810e398428c3ef8ed2339c0f334dd30a7d3f0ed4b6eDavid Sehr constexpr Inst *Consumer = nullptr; 6811e398428c3ef8ed2339c0f334dd30a7d3f0ed4b6eDavid Sehr lowerIcmpAndConsumer(Icmp, Consumer); 6812e398428c3ef8ed2339c0f334dd30a7d3f0ed4b6eDavid Sehr } 6813e398428c3ef8ed2339c0f334dd30a7d3f0ed4b6eDavid Sehr} 6814e398428c3ef8ed2339c0f334dd30a7d3f0ed4b6eDavid Sehr 68154a56686b5b56db6803f90ad53514bf2fa190d9f7John Portotemplate <typename TraitsType> 68168cfeb69e17190d3bfe22a8a1cbd7679a114d68cfJim Stichnothvoid TargetX86Base<TraitsType>::lowerSelectVector(const InstSelect *Instr) { 68178cfeb69e17190d3bfe22a8a1cbd7679a114d68cfJim Stichnoth Variable *Dest = Instr->getDest(); 6818e398428c3ef8ed2339c0f334dd30a7d3f0ed4b6eDavid Sehr Type DestTy = Dest->getType(); 68198cfeb69e17190d3bfe22a8a1cbd7679a114d68cfJim Stichnoth Operand *SrcT = Instr->getTrueOperand(); 68208cfeb69e17190d3bfe22a8a1cbd7679a114d68cfJim Stichnoth Operand *SrcF = Instr->getFalseOperand(); 68218cfeb69e17190d3bfe22a8a1cbd7679a114d68cfJim Stichnoth Operand *Condition = Instr->getCondition(); 6822e398428c3ef8ed2339c0f334dd30a7d3f0ed4b6eDavid Sehr 6823e398428c3ef8ed2339c0f334dd30a7d3f0ed4b6eDavid Sehr if (!isVectorType(DestTy)) 6824e398428c3ef8ed2339c0f334dd30a7d3f0ed4b6eDavid Sehr llvm::report_fatal_error("Expected a vector select"); 6825e398428c3ef8ed2339c0f334dd30a7d3f0ed4b6eDavid Sehr 6826e398428c3ef8ed2339c0f334dd30a7d3f0ed4b6eDavid Sehr Type SrcTy = SrcT->getType(); 6827e398428c3ef8ed2339c0f334dd30a7d3f0ed4b6eDavid Sehr Variable *T = makeReg(SrcTy); 6828e398428c3ef8ed2339c0f334dd30a7d3f0ed4b6eDavid Sehr Operand *SrcTRM = legalize(SrcT, Legal_Reg | Legal_Mem); 6829e398428c3ef8ed2339c0f334dd30a7d3f0ed4b6eDavid Sehr Operand *SrcFRM = legalize(SrcF, Legal_Reg | Legal_Mem); 683047b6ba6db7c8b8dd36ac54dbd19ae7b4a5b77424Nicolas Capens 6831e398428c3ef8ed2339c0f334dd30a7d3f0ed4b6eDavid Sehr if (InstructionSet >= Traits::SSE4_1) { 6832e398428c3ef8ed2339c0f334dd30a7d3f0ed4b6eDavid Sehr // TODO(wala): If the condition operand is a constant, use blendps or 6833e398428c3ef8ed2339c0f334dd30a7d3f0ed4b6eDavid Sehr // pblendw. 6834e398428c3ef8ed2339c0f334dd30a7d3f0ed4b6eDavid Sehr // 6835e398428c3ef8ed2339c0f334dd30a7d3f0ed4b6eDavid Sehr // Use blendvps or pblendvb to implement select. 6836e398428c3ef8ed2339c0f334dd30a7d3f0ed4b6eDavid Sehr if (SrcTy == IceType_v4i1 || SrcTy == IceType_v4i32 || 6837e398428c3ef8ed2339c0f334dd30a7d3f0ed4b6eDavid Sehr SrcTy == IceType_v4f32) { 6838e398428c3ef8ed2339c0f334dd30a7d3f0ed4b6eDavid Sehr Operand *ConditionRM = legalize(Condition, Legal_Reg | Legal_Mem); 6839e398428c3ef8ed2339c0f334dd30a7d3f0ed4b6eDavid Sehr Variable *xmm0 = makeReg(IceType_v4i32, Traits::RegisterSet::Reg_xmm0); 6840e398428c3ef8ed2339c0f334dd30a7d3f0ed4b6eDavid Sehr _movp(xmm0, ConditionRM); 6841e398428c3ef8ed2339c0f334dd30a7d3f0ed4b6eDavid Sehr _psll(xmm0, Ctx->getConstantInt8(31)); 6842e398428c3ef8ed2339c0f334dd30a7d3f0ed4b6eDavid Sehr _movp(T, SrcFRM); 6843e398428c3ef8ed2339c0f334dd30a7d3f0ed4b6eDavid Sehr _blendvps(T, SrcTRM, xmm0); 6844e398428c3ef8ed2339c0f334dd30a7d3f0ed4b6eDavid Sehr _movp(Dest, T); 6845e398428c3ef8ed2339c0f334dd30a7d3f0ed4b6eDavid Sehr } else { 6846e398428c3ef8ed2339c0f334dd30a7d3f0ed4b6eDavid Sehr assert(typeNumElements(SrcTy) == 8 || typeNumElements(SrcTy) == 16); 6847e398428c3ef8ed2339c0f334dd30a7d3f0ed4b6eDavid Sehr Type SignExtTy = 6848e398428c3ef8ed2339c0f334dd30a7d3f0ed4b6eDavid Sehr Condition->getType() == IceType_v8i1 ? IceType_v8i16 : IceType_v16i8; 6849e398428c3ef8ed2339c0f334dd30a7d3f0ed4b6eDavid Sehr Variable *xmm0 = makeReg(SignExtTy, Traits::RegisterSet::Reg_xmm0); 6850e398428c3ef8ed2339c0f334dd30a7d3f0ed4b6eDavid Sehr lowerCast(InstCast::create(Func, InstCast::Sext, xmm0, Condition)); 6851e398428c3ef8ed2339c0f334dd30a7d3f0ed4b6eDavid Sehr _movp(T, SrcFRM); 6852e398428c3ef8ed2339c0f334dd30a7d3f0ed4b6eDavid Sehr _pblendvb(T, SrcTRM, xmm0); 6853e398428c3ef8ed2339c0f334dd30a7d3f0ed4b6eDavid Sehr _movp(Dest, T); 6854e398428c3ef8ed2339c0f334dd30a7d3f0ed4b6eDavid Sehr } 6855e398428c3ef8ed2339c0f334dd30a7d3f0ed4b6eDavid Sehr return; 6856e398428c3ef8ed2339c0f334dd30a7d3f0ed4b6eDavid Sehr } 6857e398428c3ef8ed2339c0f334dd30a7d3f0ed4b6eDavid Sehr // Lower select without Traits::SSE4.1: 6858e398428c3ef8ed2339c0f334dd30a7d3f0ed4b6eDavid Sehr // a=d?b:c ==> 6859e398428c3ef8ed2339c0f334dd30a7d3f0ed4b6eDavid Sehr // if elementtype(d) != i1: 6860e398428c3ef8ed2339c0f334dd30a7d3f0ed4b6eDavid Sehr // d=sext(d); 6861e398428c3ef8ed2339c0f334dd30a7d3f0ed4b6eDavid Sehr // a=(b&d)|(c&~d); 6862e398428c3ef8ed2339c0f334dd30a7d3f0ed4b6eDavid Sehr Variable *T2 = makeReg(SrcTy); 6863e398428c3ef8ed2339c0f334dd30a7d3f0ed4b6eDavid Sehr // Sign extend the condition operand if applicable. 6864e398428c3ef8ed2339c0f334dd30a7d3f0ed4b6eDavid Sehr if (SrcTy == IceType_v4f32) { 6865e398428c3ef8ed2339c0f334dd30a7d3f0ed4b6eDavid Sehr // The sext operation takes only integer arguments. 6866e398428c3ef8ed2339c0f334dd30a7d3f0ed4b6eDavid Sehr Variable *T3 = Func->makeVariable(IceType_v4i32); 6867e398428c3ef8ed2339c0f334dd30a7d3f0ed4b6eDavid Sehr lowerCast(InstCast::create(Func, InstCast::Sext, T3, Condition)); 6868e398428c3ef8ed2339c0f334dd30a7d3f0ed4b6eDavid Sehr _movp(T, T3); 6869e398428c3ef8ed2339c0f334dd30a7d3f0ed4b6eDavid Sehr } else if (typeElementType(SrcTy) != IceType_i1) { 6870e398428c3ef8ed2339c0f334dd30a7d3f0ed4b6eDavid Sehr lowerCast(InstCast::create(Func, InstCast::Sext, T, Condition)); 6871e398428c3ef8ed2339c0f334dd30a7d3f0ed4b6eDavid Sehr } else { 6872e398428c3ef8ed2339c0f334dd30a7d3f0ed4b6eDavid Sehr Operand *ConditionRM = legalize(Condition, Legal_Reg | Legal_Mem); 6873e398428c3ef8ed2339c0f334dd30a7d3f0ed4b6eDavid Sehr _movp(T, ConditionRM); 6874e398428c3ef8ed2339c0f334dd30a7d3f0ed4b6eDavid Sehr } 6875e398428c3ef8ed2339c0f334dd30a7d3f0ed4b6eDavid Sehr _movp(T2, T); 6876e398428c3ef8ed2339c0f334dd30a7d3f0ed4b6eDavid Sehr _pand(T, SrcTRM); 6877e398428c3ef8ed2339c0f334dd30a7d3f0ed4b6eDavid Sehr _pandn(T2, SrcFRM); 6878e398428c3ef8ed2339c0f334dd30a7d3f0ed4b6eDavid Sehr _por(T, T2); 6879e398428c3ef8ed2339c0f334dd30a7d3f0ed4b6eDavid Sehr _movp(Dest, T); 6880e398428c3ef8ed2339c0f334dd30a7d3f0ed4b6eDavid Sehr 6881e398428c3ef8ed2339c0f334dd30a7d3f0ed4b6eDavid Sehr return; 6882e398428c3ef8ed2339c0f334dd30a7d3f0ed4b6eDavid Sehr} 6883e398428c3ef8ed2339c0f334dd30a7d3f0ed4b6eDavid Sehr 68844a56686b5b56db6803f90ad53514bf2fa190d9f7John Portotemplate <typename TraitsType> 68858cfeb69e17190d3bfe22a8a1cbd7679a114d68cfJim Stichnothvoid TargetX86Base<TraitsType>::lowerStore(const InstStore *Instr) { 68868cfeb69e17190d3bfe22a8a1cbd7679a114d68cfJim Stichnoth Operand *Value = Instr->getData(); 68878cfeb69e17190d3bfe22a8a1cbd7679a114d68cfJim Stichnoth Operand *Addr = Instr->getAddr(); 68884a56686b5b56db6803f90ad53514bf2fa190d9f7John Porto X86OperandMem *NewAddr = formMemoryOperand(Addr, Value->getType()); 6889ad2989b6e497f6237a3ae66ae3cfbcceaa99d4f5Jim Stichnoth doMockBoundsCheck(NewAddr); 68907e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto Type Ty = NewAddr->getType(); 68917e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto 68921d235425dab1f3dd059973fc53f1b1d5879469e3John Porto if (!Traits::Is64Bit && Ty == IceType_i64) { 6893fbdd244013aad0808220386eb6bd17eb0635dd4cJan Voung Value = legalizeUndef(Value); 68947e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto Operand *ValueHi = legalize(hiOperand(Value), Legal_Reg | Legal_Imm); 68954a56686b5b56db6803f90ad53514bf2fa190d9f7John Porto _store(ValueHi, llvm::cast<X86OperandMem>(hiOperand(NewAddr))); 6896b40595a17b83cca5d11f8d056a4ac5a4d8102a84Jim Stichnoth Operand *ValueLo = legalize(loOperand(Value), Legal_Reg | Legal_Imm); 68974a56686b5b56db6803f90ad53514bf2fa190d9f7John Porto _store(ValueLo, llvm::cast<X86OperandMem>(loOperand(NewAddr))); 68987e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto } else if (isVectorType(Ty)) { 689997f460dcd848dcef3455ed553c1b9e107d0e6ae9Andrew Scull _storep(legalizeToReg(Value), NewAddr); 69007e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto } else { 69017e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto Value = legalize(Value, Legal_Reg | Legal_Imm); 69027e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto _store(Value, NewAddr); 69037e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto } 69047e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto} 69057e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto 69064a56686b5b56db6803f90ad53514bf2fa190d9f7John Portotemplate <typename TraitsType> 69074a56686b5b56db6803f90ad53514bf2fa190d9f7John Portovoid TargetX86Base<TraitsType>::doAddressOptStore() { 69088cfeb69e17190d3bfe22a8a1cbd7679a114d68cfJim Stichnoth auto *Instr = llvm::cast<InstStore>(Context.getCur()); 69098cfeb69e17190d3bfe22a8a1cbd7679a114d68cfJim Stichnoth Operand *Addr = Instr->getAddr(); 69108cfeb69e17190d3bfe22a8a1cbd7679a114d68cfJim Stichnoth Operand *Data = Instr->getData(); 69118cfeb69e17190d3bfe22a8a1cbd7679a114d68cfJim Stichnoth if (auto *OptAddr = computeAddressOpt(Instr, Data->getType(), Addr)) { 69128cfeb69e17190d3bfe22a8a1cbd7679a114d68cfJim Stichnoth Instr->setDeleted(); 6913ac2388c385afda17acb3fc242b17b9232ec2262aJohn Porto auto *NewStore = Context.insert<InstStore>(Data, OptAddr); 69148cfeb69e17190d3bfe22a8a1cbd7679a114d68cfJim Stichnoth if (Instr->getDest()) 69158cfeb69e17190d3bfe22a8a1cbd7679a114d68cfJim Stichnoth NewStore->setRmwBeacon(Instr->getRmwBeacon()); 69167e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto } 69177e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto} 69187e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto 69194a56686b5b56db6803f90ad53514bf2fa190d9f7John Portotemplate <typename TraitsType> 6920e986b318b8295be4e361fe55c21bef6939151e49Nicolas Capensvoid TargetX86Base<TraitsType>::doAddressOptStoreSubVector() { 6921e986b318b8295be4e361fe55c21bef6939151e49Nicolas Capens auto *Intrinsic = llvm::cast<InstIntrinsicCall>(Context.getCur()); 6922e986b318b8295be4e361fe55c21bef6939151e49Nicolas Capens Operand *Addr = Intrinsic->getArg(1); 6923e986b318b8295be4e361fe55c21bef6939151e49Nicolas Capens Operand *Data = Intrinsic->getArg(0); 6924e986b318b8295be4e361fe55c21bef6939151e49Nicolas Capens if (auto *OptAddr = computeAddressOpt(Intrinsic, Data->getType(), Addr)) { 6925e986b318b8295be4e361fe55c21bef6939151e49Nicolas Capens Intrinsic->setDeleted(); 6926e986b318b8295be4e361fe55c21bef6939151e49Nicolas Capens const Ice::Intrinsics::IntrinsicInfo Info = { 6927e986b318b8295be4e361fe55c21bef6939151e49Nicolas Capens Ice::Intrinsics::StoreSubVector, Ice::Intrinsics::SideEffects_T, 6928e986b318b8295be4e361fe55c21bef6939151e49Nicolas Capens Ice::Intrinsics::ReturnsTwice_F, Ice::Intrinsics::MemoryWrite_T}; 6929e986b318b8295be4e361fe55c21bef6939151e49Nicolas Capens auto Target = Ctx->getConstantUndef(Ice::IceType_i32); 6930e986b318b8295be4e361fe55c21bef6939151e49Nicolas Capens auto *NewStore = 6931e986b318b8295be4e361fe55c21bef6939151e49Nicolas Capens Context.insert<InstIntrinsicCall>(3, nullptr, Target, Info); 6932e986b318b8295be4e361fe55c21bef6939151e49Nicolas Capens NewStore->addArg(Data); 6933e986b318b8295be4e361fe55c21bef6939151e49Nicolas Capens NewStore->addArg(OptAddr); 6934e986b318b8295be4e361fe55c21bef6939151e49Nicolas Capens NewStore->addArg(Intrinsic->getArg(2)); 6935e986b318b8295be4e361fe55c21bef6939151e49Nicolas Capens } 6936e986b318b8295be4e361fe55c21bef6939151e49Nicolas Capens} 6937e986b318b8295be4e361fe55c21bef6939151e49Nicolas Capens 6938e986b318b8295be4e361fe55c21bef6939151e49Nicolas Capenstemplate <typename TraitsType> 69394a56686b5b56db6803f90ad53514bf2fa190d9f7John PortoOperand *TargetX86Base<TraitsType>::lowerCmpRange(Operand *Comparison, 69404a56686b5b56db6803f90ad53514bf2fa190d9f7John Porto uint64_t Min, uint64_t Max) { 694187f80c128a9e2bc95829cd1aac2bbc3c44b44ac1Andrew Scull // TODO(ascull): 64-bit should not reach here but only because it is not 694287f80c128a9e2bc95829cd1aac2bbc3c44b44ac1Andrew Scull // implemented yet. This should be able to handle the 64-bit case. 69431d235425dab1f3dd059973fc53f1b1d5879469e3John Porto assert(Traits::Is64Bit || Comparison->getType() != IceType_i64); 694487f80c128a9e2bc95829cd1aac2bbc3c44b44ac1Andrew Scull // Subtracting 0 is a nop so don't do it 694587f80c128a9e2bc95829cd1aac2bbc3c44b44ac1Andrew Scull if (Min != 0) { 694687f80c128a9e2bc95829cd1aac2bbc3c44b44ac1Andrew Scull // Avoid clobbering the comparison by copying it 694787f80c128a9e2bc95829cd1aac2bbc3c44b44ac1Andrew Scull Variable *T = nullptr; 694887f80c128a9e2bc95829cd1aac2bbc3c44b44ac1Andrew Scull _mov(T, Comparison); 694987f80c128a9e2bc95829cd1aac2bbc3c44b44ac1Andrew Scull _sub(T, Ctx->getConstantInt32(Min)); 695087f80c128a9e2bc95829cd1aac2bbc3c44b44ac1Andrew Scull Comparison = T; 695187f80c128a9e2bc95829cd1aac2bbc3c44b44ac1Andrew Scull } 695287f80c128a9e2bc95829cd1aac2bbc3c44b44ac1Andrew Scull 695387f80c128a9e2bc95829cd1aac2bbc3c44b44ac1Andrew Scull _cmp(Comparison, Ctx->getConstantInt32(Max - Min)); 695487f80c128a9e2bc95829cd1aac2bbc3c44b44ac1Andrew Scull 695587f80c128a9e2bc95829cd1aac2bbc3c44b44ac1Andrew Scull return Comparison; 695687f80c128a9e2bc95829cd1aac2bbc3c44b44ac1Andrew Scull} 695787f80c128a9e2bc95829cd1aac2bbc3c44b44ac1Andrew Scull 69584a56686b5b56db6803f90ad53514bf2fa190d9f7John Portotemplate <typename TraitsType> 69594a56686b5b56db6803f90ad53514bf2fa190d9f7John Portovoid TargetX86Base<TraitsType>::lowerCaseCluster(const CaseCluster &Case, 69604a56686b5b56db6803f90ad53514bf2fa190d9f7John Porto Operand *Comparison, 69614a56686b5b56db6803f90ad53514bf2fa190d9f7John Porto bool DoneCmp, 69624a56686b5b56db6803f90ad53514bf2fa190d9f7John Porto CfgNode *DefaultTarget) { 696387f80c128a9e2bc95829cd1aac2bbc3c44b44ac1Andrew Scull switch (Case.getKind()) { 696487f80c128a9e2bc95829cd1aac2bbc3c44b44ac1Andrew Scull case CaseCluster::JumpTable: { 69654a56686b5b56db6803f90ad53514bf2fa190d9f7John Porto InstX86Label *SkipJumpTable; 696687f80c128a9e2bc95829cd1aac2bbc3c44b44ac1Andrew Scull 696787f80c128a9e2bc95829cd1aac2bbc3c44b44ac1Andrew Scull Operand *RangeIndex = 696887f80c128a9e2bc95829cd1aac2bbc3c44b44ac1Andrew Scull lowerCmpRange(Comparison, Case.getLow(), Case.getHigh()); 696986df4e9e6d183f07638440afd2c225b485c03917Andrew Scull if (DefaultTarget == nullptr) { 697087f80c128a9e2bc95829cd1aac2bbc3c44b44ac1Andrew Scull // Skip over jump table logic if comparison not in range and no default 69714a56686b5b56db6803f90ad53514bf2fa190d9f7John Porto SkipJumpTable = InstX86Label::create(Func, this); 697287f80c128a9e2bc95829cd1aac2bbc3c44b44ac1Andrew Scull _br(Traits::Cond::Br_a, SkipJumpTable); 697386df4e9e6d183f07638440afd2c225b485c03917Andrew Scull } else { 697486df4e9e6d183f07638440afd2c225b485c03917Andrew Scull _br(Traits::Cond::Br_a, DefaultTarget); 697587f80c128a9e2bc95829cd1aac2bbc3c44b44ac1Andrew Scull } 697687f80c128a9e2bc95829cd1aac2bbc3c44b44ac1Andrew Scull 697787f80c128a9e2bc95829cd1aac2bbc3c44b44ac1Andrew Scull InstJumpTable *JumpTable = Case.getJumpTable(); 697887f80c128a9e2bc95829cd1aac2bbc3c44b44ac1Andrew Scull Context.insert(JumpTable); 697987f80c128a9e2bc95829cd1aac2bbc3c44b44ac1Andrew Scull 698087f80c128a9e2bc95829cd1aac2bbc3c44b44ac1Andrew Scull // Make sure the index is a register of the same width as the base 698187f80c128a9e2bc95829cd1aac2bbc3c44b44ac1Andrew Scull Variable *Index; 698256958cb33d3c1d045f2844408d825442d523f59fJohn Porto const Type PointerType = getPointerType(); 698356958cb33d3c1d045f2844408d825442d523f59fJohn Porto if (RangeIndex->getType() != PointerType) { 698456958cb33d3c1d045f2844408d825442d523f59fJohn Porto Index = makeReg(PointerType); 6985e641e92cb728bb01453d926fa4e69ba8b9ae53bbJim Stichnoth if (RangeIndex->getType() == IceType_i64) { 6986e641e92cb728bb01453d926fa4e69ba8b9ae53bbJim Stichnoth assert(Traits::Is64Bit); 6987e641e92cb728bb01453d926fa4e69ba8b9ae53bbJim Stichnoth _mov(Index, RangeIndex); // trunc 6988e641e92cb728bb01453d926fa4e69ba8b9ae53bbJim Stichnoth } else { 6989373913fa0a4c7c4dd23ada9e6b4e1780fdea0096Jim Stichnoth Operand *RangeIndexRM = legalize(RangeIndex, Legal_Reg | Legal_Mem); 6990373913fa0a4c7c4dd23ada9e6b4e1780fdea0096Jim Stichnoth _movzx(Index, RangeIndexRM); 6991e641e92cb728bb01453d926fa4e69ba8b9ae53bbJim Stichnoth } 699287f80c128a9e2bc95829cd1aac2bbc3c44b44ac1Andrew Scull } else { 699397f460dcd848dcef3455ed553c1b9e107d0e6ae9Andrew Scull Index = legalizeToReg(RangeIndex); 699487f80c128a9e2bc95829cd1aac2bbc3c44b44ac1Andrew Scull } 699587f80c128a9e2bc95829cd1aac2bbc3c44b44ac1Andrew Scull 699687f80c128a9e2bc95829cd1aac2bbc3c44b44ac1Andrew Scull constexpr RelocOffsetT RelocOffset = 0; 6997ac2388c385afda17acb3fc242b17b9232ec2262aJohn Porto constexpr Variable *NoBase = nullptr; 699883425dec5ecae21e092a9a440845ce99a13ded69Nicolas Capens constexpr Constant *NoOffset = nullptr; 6999030772114216c1a57c749050bb58d07de8ceaa7cJohn Porto auto JTName = GlobalString::createWithString(Ctx, JumpTable->getName()); 7000467ffe51bebcb3ae3e3ce745c38fc20e8837c31cJim Stichnoth Constant *Offset = Ctx->getConstantSym(RelocOffset, JTName); 700156958cb33d3c1d045f2844408d825442d523f59fJohn Porto uint16_t Shift = typeWidthInBytesLog2(PointerType); 70028ff4b2819944bc4f02fb29204a1fa5ba7dea5682Jim Stichnoth constexpr auto Segment = X86OperandMem::SegmentRegisters::DefaultSegment; 700356958cb33d3c1d045f2844408d825442d523f59fJohn Porto 700487f80c128a9e2bc95829cd1aac2bbc3c44b44ac1Andrew Scull Variable *Target = nullptr; 700556958cb33d3c1d045f2844408d825442d523f59fJohn Porto if (Traits::Is64Bit && NeedSandboxing) { 700656958cb33d3c1d045f2844408d825442d523f59fJohn Porto assert(Index != nullptr && Index->getType() == IceType_i32); 700756958cb33d3c1d045f2844408d825442d523f59fJohn Porto } 700883425dec5ecae21e092a9a440845ce99a13ded69Nicolas Capens 700983425dec5ecae21e092a9a440845ce99a13ded69Nicolas Capens if (PointerType == IceType_i32) { 701083425dec5ecae21e092a9a440845ce99a13ded69Nicolas Capens _mov(Target, X86OperandMem::create(Func, PointerType, NoBase, Offset, 701183425dec5ecae21e092a9a440845ce99a13ded69Nicolas Capens Index, Shift, Segment)); 701283425dec5ecae21e092a9a440845ce99a13ded69Nicolas Capens } else { 701383425dec5ecae21e092a9a440845ce99a13ded69Nicolas Capens auto *Base = makeReg(IceType_i64); 701483425dec5ecae21e092a9a440845ce99a13ded69Nicolas Capens _lea(Base, X86OperandMem::create(Func, IceType_void, NoBase, Offset)); 701583425dec5ecae21e092a9a440845ce99a13ded69Nicolas Capens _mov(Target, X86OperandMem::create(Func, PointerType, Base, NoOffset, 701683425dec5ecae21e092a9a440845ce99a13ded69Nicolas Capens Index, Shift, Segment)); 701783425dec5ecae21e092a9a440845ce99a13ded69Nicolas Capens } 701856958cb33d3c1d045f2844408d825442d523f59fJohn Porto 701986df4e9e6d183f07638440afd2c225b485c03917Andrew Scull lowerIndirectJump(Target); 702087f80c128a9e2bc95829cd1aac2bbc3c44b44ac1Andrew Scull 702186df4e9e6d183f07638440afd2c225b485c03917Andrew Scull if (DefaultTarget == nullptr) 702287f80c128a9e2bc95829cd1aac2bbc3c44b44ac1Andrew Scull Context.insert(SkipJumpTable); 702387f80c128a9e2bc95829cd1aac2bbc3c44b44ac1Andrew Scull return; 702487f80c128a9e2bc95829cd1aac2bbc3c44b44ac1Andrew Scull } 702587f80c128a9e2bc95829cd1aac2bbc3c44b44ac1Andrew Scull case CaseCluster::Range: { 702686df4e9e6d183f07638440afd2c225b485c03917Andrew Scull if (Case.isUnitRange()) { 702787f80c128a9e2bc95829cd1aac2bbc3c44b44ac1Andrew Scull // Single item 702886df4e9e6d183f07638440afd2c225b485c03917Andrew Scull if (!DoneCmp) { 702986df4e9e6d183f07638440afd2c225b485c03917Andrew Scull Constant *Value = Ctx->getConstantInt32(Case.getLow()); 703087f80c128a9e2bc95829cd1aac2bbc3c44b44ac1Andrew Scull _cmp(Comparison, Value); 703186df4e9e6d183f07638440afd2c225b485c03917Andrew Scull } 703286df4e9e6d183f07638440afd2c225b485c03917Andrew Scull _br(Traits::Cond::Br_e, Case.getTarget()); 703386df4e9e6d183f07638440afd2c225b485c03917Andrew Scull } else if (DoneCmp && Case.isPairRange()) { 703486df4e9e6d183f07638440afd2c225b485c03917Andrew Scull // Range of two items with first item aleady compared against 703586df4e9e6d183f07638440afd2c225b485c03917Andrew Scull _br(Traits::Cond::Br_e, Case.getTarget()); 703686df4e9e6d183f07638440afd2c225b485c03917Andrew Scull Constant *Value = Ctx->getConstantInt32(Case.getHigh()); 703786df4e9e6d183f07638440afd2c225b485c03917Andrew Scull _cmp(Comparison, Value); 703886df4e9e6d183f07638440afd2c225b485c03917Andrew Scull _br(Traits::Cond::Br_e, Case.getTarget()); 703987f80c128a9e2bc95829cd1aac2bbc3c44b44ac1Andrew Scull } else { 704087f80c128a9e2bc95829cd1aac2bbc3c44b44ac1Andrew Scull // Range 704187f80c128a9e2bc95829cd1aac2bbc3c44b44ac1Andrew Scull lowerCmpRange(Comparison, Case.getLow(), Case.getHigh()); 704286df4e9e6d183f07638440afd2c225b485c03917Andrew Scull _br(Traits::Cond::Br_be, Case.getTarget()); 704387f80c128a9e2bc95829cd1aac2bbc3c44b44ac1Andrew Scull } 704486df4e9e6d183f07638440afd2c225b485c03917Andrew Scull if (DefaultTarget != nullptr) 704586df4e9e6d183f07638440afd2c225b485c03917Andrew Scull _br(DefaultTarget); 704687f80c128a9e2bc95829cd1aac2bbc3c44b44ac1Andrew Scull return; 704787f80c128a9e2bc95829cd1aac2bbc3c44b44ac1Andrew Scull } 704887f80c128a9e2bc95829cd1aac2bbc3c44b44ac1Andrew Scull } 704987f80c128a9e2bc95829cd1aac2bbc3c44b44ac1Andrew Scull} 705087f80c128a9e2bc95829cd1aac2bbc3c44b44ac1Andrew Scull 70514a56686b5b56db6803f90ad53514bf2fa190d9f7John Portotemplate <typename TraitsType> 70528cfeb69e17190d3bfe22a8a1cbd7679a114d68cfJim Stichnothvoid TargetX86Base<TraitsType>::lowerSwitch(const InstSwitch *Instr) { 705387f80c128a9e2bc95829cd1aac2bbc3c44b44ac1Andrew Scull // Group cases together and navigate through them with a binary search 70548cfeb69e17190d3bfe22a8a1cbd7679a114d68cfJim Stichnoth CaseClusterArray CaseClusters = CaseCluster::clusterizeSwitch(Func, Instr); 70558cfeb69e17190d3bfe22a8a1cbd7679a114d68cfJim Stichnoth Operand *Src0 = Instr->getComparison(); 70568cfeb69e17190d3bfe22a8a1cbd7679a114d68cfJim Stichnoth CfgNode *DefaultTarget = Instr->getLabelDefault(); 705787f80c128a9e2bc95829cd1aac2bbc3c44b44ac1Andrew Scull 705887f80c128a9e2bc95829cd1aac2bbc3c44b44ac1Andrew Scull assert(CaseClusters.size() != 0); // Should always be at least one 705987f80c128a9e2bc95829cd1aac2bbc3c44b44ac1Andrew Scull 70601d235425dab1f3dd059973fc53f1b1d5879469e3John Porto if (!Traits::Is64Bit && Src0->getType() == IceType_i64) { 706187f80c128a9e2bc95829cd1aac2bbc3c44b44ac1Andrew Scull Src0 = legalize(Src0); // get Base/Index into physical registers 70627e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto Operand *Src0Lo = loOperand(Src0); 70637e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto Operand *Src0Hi = hiOperand(Src0); 706487f80c128a9e2bc95829cd1aac2bbc3c44b44ac1Andrew Scull if (CaseClusters.back().getHigh() > UINT32_MAX) { 706587f80c128a9e2bc95829cd1aac2bbc3c44b44ac1Andrew Scull // TODO(ascull): handle 64-bit case properly (currently naive version) 706687f80c128a9e2bc95829cd1aac2bbc3c44b44ac1Andrew Scull // This might be handled by a higher level lowering of switches. 70678cfeb69e17190d3bfe22a8a1cbd7679a114d68cfJim Stichnoth SizeT NumCases = Instr->getNumCases(); 706887f80c128a9e2bc95829cd1aac2bbc3c44b44ac1Andrew Scull if (NumCases >= 2) { 706997f460dcd848dcef3455ed553c1b9e107d0e6ae9Andrew Scull Src0Lo = legalizeToReg(Src0Lo); 707097f460dcd848dcef3455ed553c1b9e107d0e6ae9Andrew Scull Src0Hi = legalizeToReg(Src0Hi); 707187f80c128a9e2bc95829cd1aac2bbc3c44b44ac1Andrew Scull } else { 707287f80c128a9e2bc95829cd1aac2bbc3c44b44ac1Andrew Scull Src0Lo = legalize(Src0Lo, Legal_Reg | Legal_Mem); 707387f80c128a9e2bc95829cd1aac2bbc3c44b44ac1Andrew Scull Src0Hi = legalize(Src0Hi, Legal_Reg | Legal_Mem); 707487f80c128a9e2bc95829cd1aac2bbc3c44b44ac1Andrew Scull } 707587f80c128a9e2bc95829cd1aac2bbc3c44b44ac1Andrew Scull for (SizeT I = 0; I < NumCases; ++I) { 70768cfeb69e17190d3bfe22a8a1cbd7679a114d68cfJim Stichnoth Constant *ValueLo = Ctx->getConstantInt32(Instr->getValue(I)); 70778cfeb69e17190d3bfe22a8a1cbd7679a114d68cfJim Stichnoth Constant *ValueHi = Ctx->getConstantInt32(Instr->getValue(I) >> 32); 70784a56686b5b56db6803f90ad53514bf2fa190d9f7John Porto InstX86Label *Label = InstX86Label::create(Func, this); 707987f80c128a9e2bc95829cd1aac2bbc3c44b44ac1Andrew Scull _cmp(Src0Lo, ValueLo); 708087f80c128a9e2bc95829cd1aac2bbc3c44b44ac1Andrew Scull _br(Traits::Cond::Br_ne, Label); 708187f80c128a9e2bc95829cd1aac2bbc3c44b44ac1Andrew Scull _cmp(Src0Hi, ValueHi); 70828cfeb69e17190d3bfe22a8a1cbd7679a114d68cfJim Stichnoth _br(Traits::Cond::Br_e, Instr->getLabel(I)); 708387f80c128a9e2bc95829cd1aac2bbc3c44b44ac1Andrew Scull Context.insert(Label); 708487f80c128a9e2bc95829cd1aac2bbc3c44b44ac1Andrew Scull } 70858cfeb69e17190d3bfe22a8a1cbd7679a114d68cfJim Stichnoth _br(Instr->getLabelDefault()); 708687f80c128a9e2bc95829cd1aac2bbc3c44b44ac1Andrew Scull return; 70877e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto } else { 708887f80c128a9e2bc95829cd1aac2bbc3c44b44ac1Andrew Scull // All the values are 32-bit so just check the operand is too and then 708987f80c128a9e2bc95829cd1aac2bbc3c44b44ac1Andrew Scull // fall through to the 32-bit implementation. This is a common case. 70907e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto Src0Hi = legalize(Src0Hi, Legal_Reg | Legal_Mem); 709187f80c128a9e2bc95829cd1aac2bbc3c44b44ac1Andrew Scull Constant *Zero = Ctx->getConstantInt32(0); 709287f80c128a9e2bc95829cd1aac2bbc3c44b44ac1Andrew Scull _cmp(Src0Hi, Zero); 709386df4e9e6d183f07638440afd2c225b485c03917Andrew Scull _br(Traits::Cond::Br_ne, DefaultTarget); 709487f80c128a9e2bc95829cd1aac2bbc3c44b44ac1Andrew Scull Src0 = Src0Lo; 70957e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto } 709687f80c128a9e2bc95829cd1aac2bbc3c44b44ac1Andrew Scull } 709787f80c128a9e2bc95829cd1aac2bbc3c44b44ac1Andrew Scull 709887f80c128a9e2bc95829cd1aac2bbc3c44b44ac1Andrew Scull // 32-bit lowering 709987f80c128a9e2bc95829cd1aac2bbc3c44b44ac1Andrew Scull 710087f80c128a9e2bc95829cd1aac2bbc3c44b44ac1Andrew Scull if (CaseClusters.size() == 1) { 710187f80c128a9e2bc95829cd1aac2bbc3c44b44ac1Andrew Scull // Jump straight to default if needed. Currently a common case as jump 710287f80c128a9e2bc95829cd1aac2bbc3c44b44ac1Andrew Scull // tables occur on their own. 710387f80c128a9e2bc95829cd1aac2bbc3c44b44ac1Andrew Scull constexpr bool DoneCmp = false; 710486df4e9e6d183f07638440afd2c225b485c03917Andrew Scull lowerCaseCluster(CaseClusters.front(), Src0, DoneCmp, DefaultTarget); 710587f80c128a9e2bc95829cd1aac2bbc3c44b44ac1Andrew Scull return; 710687f80c128a9e2bc95829cd1aac2bbc3c44b44ac1Andrew Scull } 710787f80c128a9e2bc95829cd1aac2bbc3c44b44ac1Andrew Scull 710887f80c128a9e2bc95829cd1aac2bbc3c44b44ac1Andrew Scull // Going to be using multiple times so get it in a register early 710997f460dcd848dcef3455ed553c1b9e107d0e6ae9Andrew Scull Variable *Comparison = legalizeToReg(Src0); 711087f80c128a9e2bc95829cd1aac2bbc3c44b44ac1Andrew Scull 711187f80c128a9e2bc95829cd1aac2bbc3c44b44ac1Andrew Scull // A span is over the clusters 711287f80c128a9e2bc95829cd1aac2bbc3c44b44ac1Andrew Scull struct SearchSpan { 71134a56686b5b56db6803f90ad53514bf2fa190d9f7John Porto SearchSpan(SizeT Begin, SizeT Size, InstX86Label *Label) 711487f80c128a9e2bc95829cd1aac2bbc3c44b44ac1Andrew Scull : Begin(Begin), Size(Size), Label(Label) {} 711587f80c128a9e2bc95829cd1aac2bbc3c44b44ac1Andrew Scull 711687f80c128a9e2bc95829cd1aac2bbc3c44b44ac1Andrew Scull SizeT Begin; 711787f80c128a9e2bc95829cd1aac2bbc3c44b44ac1Andrew Scull SizeT Size; 71184a56686b5b56db6803f90ad53514bf2fa190d9f7John Porto InstX86Label *Label; 711987f80c128a9e2bc95829cd1aac2bbc3c44b44ac1Andrew Scull }; 71208447bbaebb4f0af42dc23b23d9171e4f4a6e04f0Andrew Scull // The stack will only grow to the height of the tree so 12 should be plenty 71218447bbaebb4f0af42dc23b23d9171e4f4a6e04f0Andrew Scull std::stack<SearchSpan, llvm::SmallVector<SearchSpan, 12>> SearchSpanStack; 712287f80c128a9e2bc95829cd1aac2bbc3c44b44ac1Andrew Scull SearchSpanStack.emplace(0, CaseClusters.size(), nullptr); 712387f80c128a9e2bc95829cd1aac2bbc3c44b44ac1Andrew Scull bool DoneCmp = false; 712487f80c128a9e2bc95829cd1aac2bbc3c44b44ac1Andrew Scull 712587f80c128a9e2bc95829cd1aac2bbc3c44b44ac1Andrew Scull while (!SearchSpanStack.empty()) { 712687f80c128a9e2bc95829cd1aac2bbc3c44b44ac1Andrew Scull SearchSpan Span = SearchSpanStack.top(); 712787f80c128a9e2bc95829cd1aac2bbc3c44b44ac1Andrew Scull SearchSpanStack.pop(); 712887f80c128a9e2bc95829cd1aac2bbc3c44b44ac1Andrew Scull 712987f80c128a9e2bc95829cd1aac2bbc3c44b44ac1Andrew Scull if (Span.Label != nullptr) 713087f80c128a9e2bc95829cd1aac2bbc3c44b44ac1Andrew Scull Context.insert(Span.Label); 713187f80c128a9e2bc95829cd1aac2bbc3c44b44ac1Andrew Scull 713287f80c128a9e2bc95829cd1aac2bbc3c44b44ac1Andrew Scull switch (Span.Size) { 713387f80c128a9e2bc95829cd1aac2bbc3c44b44ac1Andrew Scull case 0: 713487f80c128a9e2bc95829cd1aac2bbc3c44b44ac1Andrew Scull llvm::report_fatal_error("Invalid SearchSpan size"); 713587f80c128a9e2bc95829cd1aac2bbc3c44b44ac1Andrew Scull break; 713687f80c128a9e2bc95829cd1aac2bbc3c44b44ac1Andrew Scull 713787f80c128a9e2bc95829cd1aac2bbc3c44b44ac1Andrew Scull case 1: 713887f80c128a9e2bc95829cd1aac2bbc3c44b44ac1Andrew Scull lowerCaseCluster(CaseClusters[Span.Begin], Comparison, DoneCmp, 713986df4e9e6d183f07638440afd2c225b485c03917Andrew Scull SearchSpanStack.empty() ? nullptr : DefaultTarget); 714087f80c128a9e2bc95829cd1aac2bbc3c44b44ac1Andrew Scull DoneCmp = false; 714187f80c128a9e2bc95829cd1aac2bbc3c44b44ac1Andrew Scull break; 714287f80c128a9e2bc95829cd1aac2bbc3c44b44ac1Andrew Scull 714386df4e9e6d183f07638440afd2c225b485c03917Andrew Scull case 2: { 714486df4e9e6d183f07638440afd2c225b485c03917Andrew Scull const CaseCluster *CaseA = &CaseClusters[Span.Begin]; 714586df4e9e6d183f07638440afd2c225b485c03917Andrew Scull const CaseCluster *CaseB = &CaseClusters[Span.Begin + 1]; 714686df4e9e6d183f07638440afd2c225b485c03917Andrew Scull 714786df4e9e6d183f07638440afd2c225b485c03917Andrew Scull // Placing a range last may allow register clobbering during the range 714886df4e9e6d183f07638440afd2c225b485c03917Andrew Scull // test. That means there is no need to clone the register. If it is a 714986df4e9e6d183f07638440afd2c225b485c03917Andrew Scull // unit range the comparison may have already been done in the binary 715086df4e9e6d183f07638440afd2c225b485c03917Andrew Scull // search (DoneCmp) and so it should be placed first. If this is a range 715186df4e9e6d183f07638440afd2c225b485c03917Andrew Scull // of two items and the comparison with the low value has already been 715286df4e9e6d183f07638440afd2c225b485c03917Andrew Scull // done, comparing with the other element is cheaper than a range test. 715386df4e9e6d183f07638440afd2c225b485c03917Andrew Scull // If the low end of the range is zero then there is no subtraction and 715486df4e9e6d183f07638440afd2c225b485c03917Andrew Scull // nothing to be gained. 715586df4e9e6d183f07638440afd2c225b485c03917Andrew Scull if (!CaseA->isUnitRange() && 715686df4e9e6d183f07638440afd2c225b485c03917Andrew Scull !(CaseA->getLow() == 0 || (DoneCmp && CaseA->isPairRange()))) { 715786df4e9e6d183f07638440afd2c225b485c03917Andrew Scull std::swap(CaseA, CaseB); 715886df4e9e6d183f07638440afd2c225b485c03917Andrew Scull DoneCmp = false; 715986df4e9e6d183f07638440afd2c225b485c03917Andrew Scull } 716086df4e9e6d183f07638440afd2c225b485c03917Andrew Scull 716186df4e9e6d183f07638440afd2c225b485c03917Andrew Scull lowerCaseCluster(*CaseA, Comparison, DoneCmp); 716287f80c128a9e2bc95829cd1aac2bbc3c44b44ac1Andrew Scull DoneCmp = false; 716386df4e9e6d183f07638440afd2c225b485c03917Andrew Scull lowerCaseCluster(*CaseB, Comparison, DoneCmp, 716486df4e9e6d183f07638440afd2c225b485c03917Andrew Scull SearchSpanStack.empty() ? nullptr : DefaultTarget); 716586df4e9e6d183f07638440afd2c225b485c03917Andrew Scull } break; 716687f80c128a9e2bc95829cd1aac2bbc3c44b44ac1Andrew Scull 716787f80c128a9e2bc95829cd1aac2bbc3c44b44ac1Andrew Scull default: 716887f80c128a9e2bc95829cd1aac2bbc3c44b44ac1Andrew Scull // Pick the middle item and branch b or ae 716987f80c128a9e2bc95829cd1aac2bbc3c44b44ac1Andrew Scull SizeT PivotIndex = Span.Begin + (Span.Size / 2); 717087f80c128a9e2bc95829cd1aac2bbc3c44b44ac1Andrew Scull const CaseCluster &Pivot = CaseClusters[PivotIndex]; 717187f80c128a9e2bc95829cd1aac2bbc3c44b44ac1Andrew Scull Constant *Value = Ctx->getConstantInt32(Pivot.getLow()); 71724a56686b5b56db6803f90ad53514bf2fa190d9f7John Porto InstX86Label *Label = InstX86Label::create(Func, this); 717387f80c128a9e2bc95829cd1aac2bbc3c44b44ac1Andrew Scull _cmp(Comparison, Value); 717486df4e9e6d183f07638440afd2c225b485c03917Andrew Scull // TODO(ascull): does it alway have to be far? 71754a56686b5b56db6803f90ad53514bf2fa190d9f7John Porto _br(Traits::Cond::Br_b, Label, InstX86Br::Far); 717687f80c128a9e2bc95829cd1aac2bbc3c44b44ac1Andrew Scull // Lower the left and (pivot+right) sides, falling through to the right 717787f80c128a9e2bc95829cd1aac2bbc3c44b44ac1Andrew Scull SearchSpanStack.emplace(Span.Begin, Span.Size / 2, Label); 717887f80c128a9e2bc95829cd1aac2bbc3c44b44ac1Andrew Scull SearchSpanStack.emplace(PivotIndex, Span.Size - (Span.Size / 2), nullptr); 717987f80c128a9e2bc95829cd1aac2bbc3c44b44ac1Andrew Scull DoneCmp = true; 718087f80c128a9e2bc95829cd1aac2bbc3c44b44ac1Andrew Scull break; 71817e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto } 71827e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto } 71837e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto 718486df4e9e6d183f07638440afd2c225b485c03917Andrew Scull _br(DefaultTarget); 71857e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto} 71867e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto 71879612d32c7e5eb2cb403686ef31172d42e075e460Andrew Scull/// The following pattern occurs often in lowered C and C++ code: 71889612d32c7e5eb2cb403686ef31172d42e075e460Andrew Scull/// 71899612d32c7e5eb2cb403686ef31172d42e075e460Andrew Scull/// %cmp = fcmp/icmp pred <n x ty> %src0, %src1 71909612d32c7e5eb2cb403686ef31172d42e075e460Andrew Scull/// %cmp.ext = sext <n x i1> %cmp to <n x ty> 71919612d32c7e5eb2cb403686ef31172d42e075e460Andrew Scull/// 71929612d32c7e5eb2cb403686ef31172d42e075e460Andrew Scull/// We can eliminate the sext operation by copying the result of pcmpeqd, 719357e126899b20c65ff3ea23a3b7d7a67ab30b99dcAndrew Scull/// pcmpgtd, or cmpps (which produce sign extended results) to the result of the 719457e126899b20c65ff3ea23a3b7d7a67ab30b99dcAndrew Scull/// sext operation. 71954a56686b5b56db6803f90ad53514bf2fa190d9f7John Portotemplate <typename TraitsType> 71964a56686b5b56db6803f90ad53514bf2fa190d9f7John Portovoid TargetX86Base<TraitsType>::eliminateNextVectorSextInstruction( 71977e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto Variable *SignExtendedResult) { 719854f3d518805b04b3f2958b7906a7d07dedb4e1aeJim Stichnoth if (auto *NextCast = 71997e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto llvm::dyn_cast_or_null<InstCast>(Context.getNextInst())) { 72007e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto if (NextCast->getCastKind() == InstCast::Sext && 72017e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto NextCast->getSrc(0) == SignExtendedResult) { 72027e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto NextCast->setDeleted(); 720397f460dcd848dcef3455ed553c1b9e107d0e6ae9Andrew Scull _movp(NextCast->getDest(), legalizeToReg(SignExtendedResult)); 72047e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto // Skip over the instruction. 72057e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto Context.advanceNext(); 72067e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto } 72077e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto } 72087e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto} 72097e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto 72104a56686b5b56db6803f90ad53514bf2fa190d9f7John Portotemplate <typename TraitsType> 72114a56686b5b56db6803f90ad53514bf2fa190d9f7John Portovoid TargetX86Base<TraitsType>::lowerUnreachable( 72128cfeb69e17190d3bfe22a8a1cbd7679a114d68cfJim Stichnoth const InstUnreachable * /*Instr*/) { 72137e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto _ud2(); 721421fd103c847f7968e8df731d7bf4fa5733bd0f3fDavid Sehr // Add a fake use of esp to make sure esp adjustments after the unreachable 721521fd103c847f7968e8df731d7bf4fa5733bd0f3fDavid Sehr // do not get dead-code eliminated. 721621fd103c847f7968e8df731d7bf4fa5733bd0f3fDavid Sehr keepEspLiveAtExit(); 72177e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto} 72187e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto 72194a56686b5b56db6803f90ad53514bf2fa190d9f7John Portotemplate <typename TraitsType> 722067c7c4161ad5bcf234357348a421e313932a8ee3Eric Holkvoid TargetX86Base<TraitsType>::lowerBreakpoint( 722167c7c4161ad5bcf234357348a421e313932a8ee3Eric Holk const InstBreakpoint * /*Instr*/) { 722267c7c4161ad5bcf234357348a421e313932a8ee3Eric Holk _int3(); 722367c7c4161ad5bcf234357348a421e313932a8ee3Eric Holk} 722467c7c4161ad5bcf234357348a421e313932a8ee3Eric Holk 722567c7c4161ad5bcf234357348a421e313932a8ee3Eric Holktemplate <typename TraitsType> 72264a56686b5b56db6803f90ad53514bf2fa190d9f7John Portovoid TargetX86Base<TraitsType>::lowerRMW(const InstX86FakeRMW *RMW) { 722757e126899b20c65ff3ea23a3b7d7a67ab30b99dcAndrew Scull // If the beacon variable's live range does not end in this instruction, then 722857e126899b20c65ff3ea23a3b7d7a67ab30b99dcAndrew Scull // it must end in the modified Store instruction that follows. This means 722957e126899b20c65ff3ea23a3b7d7a67ab30b99dcAndrew Scull // that the original Store instruction is still there, either because the 723057e126899b20c65ff3ea23a3b7d7a67ab30b99dcAndrew Scull // value being stored is used beyond the Store instruction, or because dead 723157e126899b20c65ff3ea23a3b7d7a67ab30b99dcAndrew Scull // code elimination did not happen. In either case, we cancel RMW lowering 723257e126899b20c65ff3ea23a3b7d7a67ab30b99dcAndrew Scull // (and the caller deletes the RMW instruction). 72337e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto if (!RMW->isLastUse(RMW->getBeacon())) 72347e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto return; 72357e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto Operand *Src = RMW->getData(); 72367e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto Type Ty = Src->getType(); 72374a56686b5b56db6803f90ad53514bf2fa190d9f7John Porto X86OperandMem *Addr = formMemoryOperand(RMW->getAddr(), Ty); 7238ad2989b6e497f6237a3ae66ae3cfbcceaa99d4f5Jim Stichnoth doMockBoundsCheck(Addr); 72391d235425dab1f3dd059973fc53f1b1d5879469e3John Porto if (!Traits::Is64Bit && Ty == IceType_i64) { 7240fbdd244013aad0808220386eb6bd17eb0635dd4cJan Voung Src = legalizeUndef(Src); 72417e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto Operand *SrcLo = legalize(loOperand(Src), Legal_Reg | Legal_Imm); 72427e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto Operand *SrcHi = legalize(hiOperand(Src), Legal_Reg | Legal_Imm); 72432d6c82679e9786a924b76023aa30e1f44d0f8cbbJim Stichnoth auto *AddrLo = llvm::cast<X86OperandMem>(loOperand(Addr)); 72442d6c82679e9786a924b76023aa30e1f44d0f8cbbJim Stichnoth auto *AddrHi = llvm::cast<X86OperandMem>(hiOperand(Addr)); 72457e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto switch (RMW->getOp()) { 72467e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto default: 72477e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto // TODO(stichnot): Implement other arithmetic operators. 72487e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto break; 72497e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto case InstArithmetic::Add: 72507e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto _add_rmw(AddrLo, SrcLo); 72517e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto _adc_rmw(AddrHi, SrcHi); 72527e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto return; 72537e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto case InstArithmetic::Sub: 72547e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto _sub_rmw(AddrLo, SrcLo); 72557e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto _sbb_rmw(AddrHi, SrcHi); 72567e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto return; 72577e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto case InstArithmetic::And: 72587e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto _and_rmw(AddrLo, SrcLo); 72597e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto _and_rmw(AddrHi, SrcHi); 72607e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto return; 72617e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto case InstArithmetic::Or: 72627e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto _or_rmw(AddrLo, SrcLo); 72637e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto _or_rmw(AddrHi, SrcHi); 72647e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto return; 72657e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto case InstArithmetic::Xor: 72667e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto _xor_rmw(AddrLo, SrcLo); 72677e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto _xor_rmw(AddrHi, SrcHi); 72687e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto return; 72697e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto } 72707e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto } else { 72711d235425dab1f3dd059973fc53f1b1d5879469e3John Porto // x86-32: i8, i16, i32 72721d235425dab1f3dd059973fc53f1b1d5879469e3John Porto // x86-64: i8, i16, i32, i64 72737e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto switch (RMW->getOp()) { 72747e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto default: 72757e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto // TODO(stichnot): Implement other arithmetic operators. 72767e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto break; 72777e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto case InstArithmetic::Add: 72787e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto Src = legalize(Src, Legal_Reg | Legal_Imm); 72797e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto _add_rmw(Addr, Src); 72807e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto return; 72817e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto case InstArithmetic::Sub: 72827e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto Src = legalize(Src, Legal_Reg | Legal_Imm); 72837e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto _sub_rmw(Addr, Src); 72847e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto return; 72857e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto case InstArithmetic::And: 72867e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto Src = legalize(Src, Legal_Reg | Legal_Imm); 72877e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto _and_rmw(Addr, Src); 72887e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto return; 72897e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto case InstArithmetic::Or: 72907e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto Src = legalize(Src, Legal_Reg | Legal_Imm); 72917e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto _or_rmw(Addr, Src); 72927e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto return; 72937e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto case InstArithmetic::Xor: 72947e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto Src = legalize(Src, Legal_Reg | Legal_Imm); 72957e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto _xor_rmw(Addr, Src); 72967e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto return; 72977e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto } 72987e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto } 72997e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto llvm::report_fatal_error("Couldn't lower RMW instruction"); 73007e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto} 73017e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto 73024a56686b5b56db6803f90ad53514bf2fa190d9f7John Portotemplate <typename TraitsType> 73034a56686b5b56db6803f90ad53514bf2fa190d9f7John Portovoid TargetX86Base<TraitsType>::lowerOther(const Inst *Instr) { 73044a56686b5b56db6803f90ad53514bf2fa190d9f7John Porto if (const auto *RMW = llvm::dyn_cast<InstX86FakeRMW>(Instr)) { 73057e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto lowerRMW(RMW); 73067e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto } else { 73077e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto TargetLowering::lowerOther(Instr); 73087e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto } 73097e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto} 73107e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto 731157e126899b20c65ff3ea23a3b7d7a67ab30b99dcAndrew Scull/// Turn an i64 Phi instruction into a pair of i32 Phi instructions, to preserve 731257e126899b20c65ff3ea23a3b7d7a67ab30b99dcAndrew Scull/// integrity of liveness analysis. Undef values are also turned into zeroes, 73138ff4b2819944bc4f02fb29204a1fa5ba7dea5682Jim Stichnoth/// since loOperand() and hiOperand() don't expect Undef input. Also, in 7314ac2388c385afda17acb3fc242b17b9232ec2262aJohn Porto/// Non-SFI mode, add a FakeUse(RebasePtr) for every pooled constant operand. 73154a56686b5b56db6803f90ad53514bf2fa190d9f7John Portotemplate <typename TraitsType> void TargetX86Base<TraitsType>::prelowerPhis() { 7316d46999474d2b4a388e1d8a7c71f06cd4cec51bfcKarl Schimpf if (getFlags().getUseNonsfi()) { 7317ac2388c385afda17acb3fc242b17b9232ec2262aJohn Porto assert(RebasePtr); 73188ff4b2819944bc4f02fb29204a1fa5ba7dea5682Jim Stichnoth CfgNode *Node = Context.getNode(); 7319ac2388c385afda17acb3fc242b17b9232ec2262aJohn Porto uint32_t RebasePtrUseCount = 0; 73208ff4b2819944bc4f02fb29204a1fa5ba7dea5682Jim Stichnoth for (Inst &I : Node->getPhis()) { 73218ff4b2819944bc4f02fb29204a1fa5ba7dea5682Jim Stichnoth auto *Phi = llvm::dyn_cast<InstPhi>(&I); 73228ff4b2819944bc4f02fb29204a1fa5ba7dea5682Jim Stichnoth if (Phi->isDeleted()) 73238ff4b2819944bc4f02fb29204a1fa5ba7dea5682Jim Stichnoth continue; 73248ff4b2819944bc4f02fb29204a1fa5ba7dea5682Jim Stichnoth for (SizeT I = 0; I < Phi->getSrcSize(); ++I) { 73258ff4b2819944bc4f02fb29204a1fa5ba7dea5682Jim Stichnoth Operand *Src = Phi->getSrc(I); 73268ff4b2819944bc4f02fb29204a1fa5ba7dea5682Jim Stichnoth // TODO(stichnot): This over-counts for +0.0, and under-counts for other 73278ff4b2819944bc4f02fb29204a1fa5ba7dea5682Jim Stichnoth // kinds of pooling. 73288ff4b2819944bc4f02fb29204a1fa5ba7dea5682Jim Stichnoth if (llvm::isa<ConstantRelocatable>(Src) || 73298ff4b2819944bc4f02fb29204a1fa5ba7dea5682Jim Stichnoth llvm::isa<ConstantFloat>(Src) || llvm::isa<ConstantDouble>(Src)) { 7330ac2388c385afda17acb3fc242b17b9232ec2262aJohn Porto ++RebasePtrUseCount; 73318ff4b2819944bc4f02fb29204a1fa5ba7dea5682Jim Stichnoth } 73328ff4b2819944bc4f02fb29204a1fa5ba7dea5682Jim Stichnoth } 73338ff4b2819944bc4f02fb29204a1fa5ba7dea5682Jim Stichnoth } 7334ac2388c385afda17acb3fc242b17b9232ec2262aJohn Porto if (RebasePtrUseCount) { 7335ac2388c385afda17acb3fc242b17b9232ec2262aJohn Porto Node->getInsts().push_front(InstFakeUse::create(Func, RebasePtr)); 73368ff4b2819944bc4f02fb29204a1fa5ba7dea5682Jim Stichnoth } 73378ff4b2819944bc4f02fb29204a1fa5ba7dea5682Jim Stichnoth } 73381d235425dab1f3dd059973fc53f1b1d5879469e3John Porto if (Traits::Is64Bit) { 73391d235425dab1f3dd059973fc53f1b1d5879469e3John Porto // On x86-64 we don't need to prelower phis -- the architecture can handle 73401d235425dab1f3dd059973fc53f1b1d5879469e3John Porto // 64-bit integer natively. 73411d235425dab1f3dd059973fc53f1b1d5879469e3John Porto return; 73421d235425dab1f3dd059973fc53f1b1d5879469e3John Porto } 73431d235425dab1f3dd059973fc53f1b1d5879469e3John Porto 734457e126899b20c65ff3ea23a3b7d7a67ab30b99dcAndrew Scull // Pause constant blinding or pooling, blinding or pooling will be done later 734557e126899b20c65ff3ea23a3b7d7a67ab30b99dcAndrew Scull // during phi lowering assignments 73467e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto BoolFlagSaver B(RandomizationPoolingPaused, true); 73474a56686b5b56db6803f90ad53514bf2fa190d9f7John Porto PhiLowering::prelowerPhis32Bit<TargetX86Base<TraitsType>>( 734853483691eba6e23de63afe0579b436002d06d187Jan Voung this, Context.getNode(), Func); 73497e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto} 73507e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto 73514a56686b5b56db6803f90ad53514bf2fa190d9f7John Portotemplate <typename TraitsType> 73524a56686b5b56db6803f90ad53514bf2fa190d9f7John Portovoid TargetX86Base<TraitsType>::genTargetHelperCallFor(Inst *Instr) { 735326217e3333150e66fc96aca79c01105906797960David Sehr uint32_t StackArgumentsSize = 0; 735426217e3333150e66fc96aca79c01105906797960David Sehr if (auto *Arith = llvm::dyn_cast<InstArithmetic>(Instr)) { 735520070e84c21b9aa15a83e53b5d2ffa692c4e081aKarl Schimpf RuntimeHelper HelperID = RuntimeHelper::H_Num; 735626217e3333150e66fc96aca79c01105906797960David Sehr Variable *Dest = Arith->getDest(); 735726217e3333150e66fc96aca79c01105906797960David Sehr Type DestTy = Dest->getType(); 735826217e3333150e66fc96aca79c01105906797960David Sehr if (!Traits::Is64Bit && DestTy == IceType_i64) { 735926217e3333150e66fc96aca79c01105906797960David Sehr switch (Arith->getOp()) { 736026217e3333150e66fc96aca79c01105906797960David Sehr default: 736126217e3333150e66fc96aca79c01105906797960David Sehr return; 736226217e3333150e66fc96aca79c01105906797960David Sehr case InstArithmetic::Udiv: 736320070e84c21b9aa15a83e53b5d2ffa692c4e081aKarl Schimpf HelperID = RuntimeHelper::H_udiv_i64; 736426217e3333150e66fc96aca79c01105906797960David Sehr break; 736526217e3333150e66fc96aca79c01105906797960David Sehr case InstArithmetic::Sdiv: 736620070e84c21b9aa15a83e53b5d2ffa692c4e081aKarl Schimpf HelperID = RuntimeHelper::H_sdiv_i64; 736726217e3333150e66fc96aca79c01105906797960David Sehr break; 736826217e3333150e66fc96aca79c01105906797960David Sehr case InstArithmetic::Urem: 736920070e84c21b9aa15a83e53b5d2ffa692c4e081aKarl Schimpf HelperID = RuntimeHelper::H_urem_i64; 737026217e3333150e66fc96aca79c01105906797960David Sehr break; 737126217e3333150e66fc96aca79c01105906797960David Sehr case InstArithmetic::Srem: 737220070e84c21b9aa15a83e53b5d2ffa692c4e081aKarl Schimpf HelperID = RuntimeHelper::H_srem_i64; 737326217e3333150e66fc96aca79c01105906797960David Sehr break; 737426217e3333150e66fc96aca79c01105906797960David Sehr } 737526217e3333150e66fc96aca79c01105906797960David Sehr } else if (isVectorType(DestTy)) { 737626217e3333150e66fc96aca79c01105906797960David Sehr Variable *Dest = Arith->getDest(); 737726217e3333150e66fc96aca79c01105906797960David Sehr Operand *Src0 = Arith->getSrc(0); 737826217e3333150e66fc96aca79c01105906797960David Sehr Operand *Src1 = Arith->getSrc(1); 737926217e3333150e66fc96aca79c01105906797960David Sehr switch (Arith->getOp()) { 738026217e3333150e66fc96aca79c01105906797960David Sehr default: 738126217e3333150e66fc96aca79c01105906797960David Sehr return; 738226217e3333150e66fc96aca79c01105906797960David Sehr case InstArithmetic::Mul: 738326217e3333150e66fc96aca79c01105906797960David Sehr if (DestTy == IceType_v16i8) { 738426217e3333150e66fc96aca79c01105906797960David Sehr scalarizeArithmetic(Arith->getOp(), Dest, Src0, Src1); 738526217e3333150e66fc96aca79c01105906797960David Sehr Arith->setDeleted(); 738626217e3333150e66fc96aca79c01105906797960David Sehr } 738726217e3333150e66fc96aca79c01105906797960David Sehr return; 738826217e3333150e66fc96aca79c01105906797960David Sehr case InstArithmetic::Shl: 738926217e3333150e66fc96aca79c01105906797960David Sehr case InstArithmetic::Lshr: 739026217e3333150e66fc96aca79c01105906797960David Sehr case InstArithmetic::Ashr: 7391b09353904b93e8914d459a286b6b9badb8b1983bNicolas Capens if (llvm::isa<Constant>(Src1)) { 7392b09353904b93e8914d459a286b6b9badb8b1983bNicolas Capens return; 7393b09353904b93e8914d459a286b6b9badb8b1983bNicolas Capens } 739426217e3333150e66fc96aca79c01105906797960David Sehr case InstArithmetic::Udiv: 739526217e3333150e66fc96aca79c01105906797960David Sehr case InstArithmetic::Urem: 739626217e3333150e66fc96aca79c01105906797960David Sehr case InstArithmetic::Sdiv: 739726217e3333150e66fc96aca79c01105906797960David Sehr case InstArithmetic::Srem: 739826217e3333150e66fc96aca79c01105906797960David Sehr case InstArithmetic::Frem: 739926217e3333150e66fc96aca79c01105906797960David Sehr scalarizeArithmetic(Arith->getOp(), Dest, Src0, Src1); 740026217e3333150e66fc96aca79c01105906797960David Sehr Arith->setDeleted(); 740126217e3333150e66fc96aca79c01105906797960David Sehr return; 740226217e3333150e66fc96aca79c01105906797960David Sehr } 740326217e3333150e66fc96aca79c01105906797960David Sehr } else { 740426217e3333150e66fc96aca79c01105906797960David Sehr switch (Arith->getOp()) { 740526217e3333150e66fc96aca79c01105906797960David Sehr default: 740626217e3333150e66fc96aca79c01105906797960David Sehr return; 740726217e3333150e66fc96aca79c01105906797960David Sehr case InstArithmetic::Frem: 740826217e3333150e66fc96aca79c01105906797960David Sehr if (isFloat32Asserting32Or64(DestTy)) 740920070e84c21b9aa15a83e53b5d2ffa692c4e081aKarl Schimpf HelperID = RuntimeHelper::H_frem_f32; 741026217e3333150e66fc96aca79c01105906797960David Sehr else 741120070e84c21b9aa15a83e53b5d2ffa692c4e081aKarl Schimpf HelperID = RuntimeHelper::H_frem_f64; 741226217e3333150e66fc96aca79c01105906797960David Sehr } 741326217e3333150e66fc96aca79c01105906797960David Sehr } 741426217e3333150e66fc96aca79c01105906797960David Sehr constexpr SizeT MaxSrcs = 2; 741520070e84c21b9aa15a83e53b5d2ffa692c4e081aKarl Schimpf InstCall *Call = makeHelperCall(HelperID, Dest, MaxSrcs); 741626217e3333150e66fc96aca79c01105906797960David Sehr Call->addArg(Arith->getSrc(0)); 741726217e3333150e66fc96aca79c01105906797960David Sehr Call->addArg(Arith->getSrc(1)); 741826217e3333150e66fc96aca79c01105906797960David Sehr StackArgumentsSize = getCallStackArgumentsSizeBytes(Call); 741926217e3333150e66fc96aca79c01105906797960David Sehr Context.insert(Call); 742026217e3333150e66fc96aca79c01105906797960David Sehr Arith->setDeleted(); 742126217e3333150e66fc96aca79c01105906797960David Sehr } else if (auto *Cast = llvm::dyn_cast<InstCast>(Instr)) { 742226217e3333150e66fc96aca79c01105906797960David Sehr InstCast::OpKind CastKind = Cast->getCastKind(); 742326217e3333150e66fc96aca79c01105906797960David Sehr Operand *Src0 = Cast->getSrc(0); 742426217e3333150e66fc96aca79c01105906797960David Sehr const Type SrcType = Src0->getType(); 742526217e3333150e66fc96aca79c01105906797960David Sehr Variable *Dest = Cast->getDest(); 742626217e3333150e66fc96aca79c01105906797960David Sehr const Type DestTy = Dest->getType(); 742720070e84c21b9aa15a83e53b5d2ffa692c4e081aKarl Schimpf RuntimeHelper HelperID = RuntimeHelper::H_Num; 7428b19d39cc5fa8dc60c678c2507af02147184f168fDavid Sehr Variable *CallDest = Dest; 742926217e3333150e66fc96aca79c01105906797960David Sehr switch (CastKind) { 743026217e3333150e66fc96aca79c01105906797960David Sehr default: 743126217e3333150e66fc96aca79c01105906797960David Sehr return; 743226217e3333150e66fc96aca79c01105906797960David Sehr case InstCast::Fptosi: 743326217e3333150e66fc96aca79c01105906797960David Sehr if (!Traits::Is64Bit && DestTy == IceType_i64) { 743420070e84c21b9aa15a83e53b5d2ffa692c4e081aKarl Schimpf HelperID = isFloat32Asserting32Or64(SrcType) 743520070e84c21b9aa15a83e53b5d2ffa692c4e081aKarl Schimpf ? RuntimeHelper::H_fptosi_f32_i64 743620070e84c21b9aa15a83e53b5d2ffa692c4e081aKarl Schimpf : RuntimeHelper::H_fptosi_f64_i64; 743726217e3333150e66fc96aca79c01105906797960David Sehr } else { 743826217e3333150e66fc96aca79c01105906797960David Sehr return; 743926217e3333150e66fc96aca79c01105906797960David Sehr } 744026217e3333150e66fc96aca79c01105906797960David Sehr break; 744126217e3333150e66fc96aca79c01105906797960David Sehr case InstCast::Fptoui: 744226217e3333150e66fc96aca79c01105906797960David Sehr if (isVectorType(DestTy)) { 7443dbf81e0c0e364173176159e0e2548e9948b197a4Nicolas Capens assert(DestTy == IceType_v4i32); 7444dbf81e0c0e364173176159e0e2548e9948b197a4Nicolas Capens assert(SrcType == IceType_v4f32); 744520070e84c21b9aa15a83e53b5d2ffa692c4e081aKarl Schimpf HelperID = RuntimeHelper::H_fptoui_4xi32_f32; 744626217e3333150e66fc96aca79c01105906797960David Sehr } else if (DestTy == IceType_i64 || 744726217e3333150e66fc96aca79c01105906797960David Sehr (!Traits::Is64Bit && DestTy == IceType_i32)) { 744826217e3333150e66fc96aca79c01105906797960David Sehr if (Traits::Is64Bit) { 744920070e84c21b9aa15a83e53b5d2ffa692c4e081aKarl Schimpf HelperID = isFloat32Asserting32Or64(SrcType) 745020070e84c21b9aa15a83e53b5d2ffa692c4e081aKarl Schimpf ? RuntimeHelper::H_fptoui_f32_i64 745120070e84c21b9aa15a83e53b5d2ffa692c4e081aKarl Schimpf : RuntimeHelper::H_fptoui_f64_i64; 745226217e3333150e66fc96aca79c01105906797960David Sehr } else if (isInt32Asserting32Or64(DestTy)) { 745320070e84c21b9aa15a83e53b5d2ffa692c4e081aKarl Schimpf HelperID = isFloat32Asserting32Or64(SrcType) 745420070e84c21b9aa15a83e53b5d2ffa692c4e081aKarl Schimpf ? RuntimeHelper::H_fptoui_f32_i32 745520070e84c21b9aa15a83e53b5d2ffa692c4e081aKarl Schimpf : RuntimeHelper::H_fptoui_f64_i32; 745626217e3333150e66fc96aca79c01105906797960David Sehr } else { 745720070e84c21b9aa15a83e53b5d2ffa692c4e081aKarl Schimpf HelperID = isFloat32Asserting32Or64(SrcType) 745820070e84c21b9aa15a83e53b5d2ffa692c4e081aKarl Schimpf ? RuntimeHelper::H_fptoui_f32_i64 745920070e84c21b9aa15a83e53b5d2ffa692c4e081aKarl Schimpf : RuntimeHelper::H_fptoui_f64_i64; 746026217e3333150e66fc96aca79c01105906797960David Sehr } 746126217e3333150e66fc96aca79c01105906797960David Sehr } else { 746226217e3333150e66fc96aca79c01105906797960David Sehr return; 746326217e3333150e66fc96aca79c01105906797960David Sehr } 746426217e3333150e66fc96aca79c01105906797960David Sehr break; 746526217e3333150e66fc96aca79c01105906797960David Sehr case InstCast::Sitofp: 746626217e3333150e66fc96aca79c01105906797960David Sehr if (!Traits::Is64Bit && SrcType == IceType_i64) { 746720070e84c21b9aa15a83e53b5d2ffa692c4e081aKarl Schimpf HelperID = isFloat32Asserting32Or64(DestTy) 746820070e84c21b9aa15a83e53b5d2ffa692c4e081aKarl Schimpf ? RuntimeHelper::H_sitofp_i64_f32 746920070e84c21b9aa15a83e53b5d2ffa692c4e081aKarl Schimpf : RuntimeHelper::H_sitofp_i64_f64; 747026217e3333150e66fc96aca79c01105906797960David Sehr } else { 747126217e3333150e66fc96aca79c01105906797960David Sehr return; 747226217e3333150e66fc96aca79c01105906797960David Sehr } 747326217e3333150e66fc96aca79c01105906797960David Sehr break; 747426217e3333150e66fc96aca79c01105906797960David Sehr case InstCast::Uitofp: 747526217e3333150e66fc96aca79c01105906797960David Sehr if (isVectorType(SrcType)) { 7476dbf81e0c0e364173176159e0e2548e9948b197a4Nicolas Capens assert(DestTy == IceType_v4f32); 7477dbf81e0c0e364173176159e0e2548e9948b197a4Nicolas Capens assert(SrcType == IceType_v4i32); 747820070e84c21b9aa15a83e53b5d2ffa692c4e081aKarl Schimpf HelperID = RuntimeHelper::H_uitofp_4xi32_4xf32; 747926217e3333150e66fc96aca79c01105906797960David Sehr } else if (SrcType == IceType_i64 || 748026217e3333150e66fc96aca79c01105906797960David Sehr (!Traits::Is64Bit && SrcType == IceType_i32)) { 748126217e3333150e66fc96aca79c01105906797960David Sehr if (isInt32Asserting32Or64(SrcType)) { 748220070e84c21b9aa15a83e53b5d2ffa692c4e081aKarl Schimpf HelperID = isFloat32Asserting32Or64(DestTy) 748320070e84c21b9aa15a83e53b5d2ffa692c4e081aKarl Schimpf ? RuntimeHelper::H_uitofp_i32_f32 748420070e84c21b9aa15a83e53b5d2ffa692c4e081aKarl Schimpf : RuntimeHelper::H_uitofp_i32_f64; 748526217e3333150e66fc96aca79c01105906797960David Sehr } else { 748620070e84c21b9aa15a83e53b5d2ffa692c4e081aKarl Schimpf HelperID = isFloat32Asserting32Or64(DestTy) 748720070e84c21b9aa15a83e53b5d2ffa692c4e081aKarl Schimpf ? RuntimeHelper::H_uitofp_i64_f32 748820070e84c21b9aa15a83e53b5d2ffa692c4e081aKarl Schimpf : RuntimeHelper::H_uitofp_i64_f64; 748926217e3333150e66fc96aca79c01105906797960David Sehr } 749026217e3333150e66fc96aca79c01105906797960David Sehr } else { 749126217e3333150e66fc96aca79c01105906797960David Sehr return; 749226217e3333150e66fc96aca79c01105906797960David Sehr } 749326217e3333150e66fc96aca79c01105906797960David Sehr break; 749426217e3333150e66fc96aca79c01105906797960David Sehr case InstCast::Bitcast: { 749526217e3333150e66fc96aca79c01105906797960David Sehr if (DestTy == Src0->getType()) 749626217e3333150e66fc96aca79c01105906797960David Sehr return; 749726217e3333150e66fc96aca79c01105906797960David Sehr switch (DestTy) { 749826217e3333150e66fc96aca79c01105906797960David Sehr default: 749926217e3333150e66fc96aca79c01105906797960David Sehr return; 750026217e3333150e66fc96aca79c01105906797960David Sehr case IceType_i8: 750126217e3333150e66fc96aca79c01105906797960David Sehr assert(Src0->getType() == IceType_v8i1); 750220070e84c21b9aa15a83e53b5d2ffa692c4e081aKarl Schimpf HelperID = RuntimeHelper::H_bitcast_8xi1_i8; 7503b19d39cc5fa8dc60c678c2507af02147184f168fDavid Sehr CallDest = Func->makeVariable(IceType_i32); 750426217e3333150e66fc96aca79c01105906797960David Sehr break; 750526217e3333150e66fc96aca79c01105906797960David Sehr case IceType_i16: 750626217e3333150e66fc96aca79c01105906797960David Sehr assert(Src0->getType() == IceType_v16i1); 750720070e84c21b9aa15a83e53b5d2ffa692c4e081aKarl Schimpf HelperID = RuntimeHelper::H_bitcast_16xi1_i16; 7508b19d39cc5fa8dc60c678c2507af02147184f168fDavid Sehr CallDest = Func->makeVariable(IceType_i32); 750926217e3333150e66fc96aca79c01105906797960David Sehr break; 751026217e3333150e66fc96aca79c01105906797960David Sehr case IceType_v8i1: { 751126217e3333150e66fc96aca79c01105906797960David Sehr assert(Src0->getType() == IceType_i8); 751220070e84c21b9aa15a83e53b5d2ffa692c4e081aKarl Schimpf HelperID = RuntimeHelper::H_bitcast_i8_8xi1; 751326217e3333150e66fc96aca79c01105906797960David Sehr Variable *Src0AsI32 = Func->makeVariable(stackSlotType()); 751426217e3333150e66fc96aca79c01105906797960David Sehr // Arguments to functions are required to be at least 32 bits wide. 75151d937a8e35b7ae51ee64c32f8f34d51ba33498f4John Porto Context.insert<InstCast>(InstCast::Zext, Src0AsI32, Src0); 751626217e3333150e66fc96aca79c01105906797960David Sehr Src0 = Src0AsI32; 751726217e3333150e66fc96aca79c01105906797960David Sehr } break; 751826217e3333150e66fc96aca79c01105906797960David Sehr case IceType_v16i1: { 751926217e3333150e66fc96aca79c01105906797960David Sehr assert(Src0->getType() == IceType_i16); 752020070e84c21b9aa15a83e53b5d2ffa692c4e081aKarl Schimpf HelperID = RuntimeHelper::H_bitcast_i16_16xi1; 752126217e3333150e66fc96aca79c01105906797960David Sehr Variable *Src0AsI32 = Func->makeVariable(stackSlotType()); 752226217e3333150e66fc96aca79c01105906797960David Sehr // Arguments to functions are required to be at least 32 bits wide. 75231d937a8e35b7ae51ee64c32f8f34d51ba33498f4John Porto Context.insert<InstCast>(InstCast::Zext, Src0AsI32, Src0); 752426217e3333150e66fc96aca79c01105906797960David Sehr Src0 = Src0AsI32; 752526217e3333150e66fc96aca79c01105906797960David Sehr } break; 752626217e3333150e66fc96aca79c01105906797960David Sehr } 752726217e3333150e66fc96aca79c01105906797960David Sehr } break; 752826217e3333150e66fc96aca79c01105906797960David Sehr } 752926217e3333150e66fc96aca79c01105906797960David Sehr constexpr SizeT MaxSrcs = 1; 753020070e84c21b9aa15a83e53b5d2ffa692c4e081aKarl Schimpf InstCall *Call = makeHelperCall(HelperID, CallDest, MaxSrcs); 753126217e3333150e66fc96aca79c01105906797960David Sehr Call->addArg(Src0); 753226217e3333150e66fc96aca79c01105906797960David Sehr StackArgumentsSize = getCallStackArgumentsSizeBytes(Call); 753326217e3333150e66fc96aca79c01105906797960David Sehr Context.insert(Call); 7534b19d39cc5fa8dc60c678c2507af02147184f168fDavid Sehr // The PNaCl ABI disallows i8/i16 return types, so truncate the helper call 7535b19d39cc5fa8dc60c678c2507af02147184f168fDavid Sehr // result to the appropriate type as necessary. 7536b19d39cc5fa8dc60c678c2507af02147184f168fDavid Sehr if (CallDest->getType() != Dest->getType()) 7537b19d39cc5fa8dc60c678c2507af02147184f168fDavid Sehr Context.insert<InstCast>(InstCast::Trunc, Dest, CallDest); 753826217e3333150e66fc96aca79c01105906797960David Sehr Cast->setDeleted(); 753926217e3333150e66fc96aca79c01105906797960David Sehr } else if (auto *Intrinsic = llvm::dyn_cast<InstIntrinsicCall>(Instr)) { 7540e82b560e649f8a68bcb252b9b002708e74d962d3John Porto CfgVector<Type> ArgTypes; 754126217e3333150e66fc96aca79c01105906797960David Sehr Type ReturnType = IceType_void; 754226217e3333150e66fc96aca79c01105906797960David Sehr switch (Intrinsics::IntrinsicID ID = Intrinsic->getIntrinsicInfo().ID) { 754326217e3333150e66fc96aca79c01105906797960David Sehr default: 754426217e3333150e66fc96aca79c01105906797960David Sehr return; 754526217e3333150e66fc96aca79c01105906797960David Sehr case Intrinsics::Ctpop: { 754626217e3333150e66fc96aca79c01105906797960David Sehr Operand *Val = Intrinsic->getArg(0); 754726217e3333150e66fc96aca79c01105906797960David Sehr Type ValTy = Val->getType(); 754826217e3333150e66fc96aca79c01105906797960David Sehr if (ValTy == IceType_i64) 754926217e3333150e66fc96aca79c01105906797960David Sehr ArgTypes = {IceType_i64}; 755026217e3333150e66fc96aca79c01105906797960David Sehr else 755126217e3333150e66fc96aca79c01105906797960David Sehr ArgTypes = {IceType_i32}; 755226217e3333150e66fc96aca79c01105906797960David Sehr ReturnType = IceType_i32; 755326217e3333150e66fc96aca79c01105906797960David Sehr } break; 755426217e3333150e66fc96aca79c01105906797960David Sehr case Intrinsics::Longjmp: 755526217e3333150e66fc96aca79c01105906797960David Sehr ArgTypes = {IceType_i32, IceType_i32}; 755626217e3333150e66fc96aca79c01105906797960David Sehr ReturnType = IceType_void; 755726217e3333150e66fc96aca79c01105906797960David Sehr break; 755826217e3333150e66fc96aca79c01105906797960David Sehr case Intrinsics::Memcpy: 755926217e3333150e66fc96aca79c01105906797960David Sehr ArgTypes = {IceType_i32, IceType_i32, IceType_i32}; 756026217e3333150e66fc96aca79c01105906797960David Sehr ReturnType = IceType_void; 756126217e3333150e66fc96aca79c01105906797960David Sehr break; 756226217e3333150e66fc96aca79c01105906797960David Sehr case Intrinsics::Memmove: 756326217e3333150e66fc96aca79c01105906797960David Sehr ArgTypes = {IceType_i32, IceType_i32, IceType_i32}; 756426217e3333150e66fc96aca79c01105906797960David Sehr ReturnType = IceType_void; 756526217e3333150e66fc96aca79c01105906797960David Sehr break; 756626217e3333150e66fc96aca79c01105906797960David Sehr case Intrinsics::Memset: 756726217e3333150e66fc96aca79c01105906797960David Sehr ArgTypes = {IceType_i32, IceType_i32, IceType_i32}; 756826217e3333150e66fc96aca79c01105906797960David Sehr ReturnType = IceType_void; 756926217e3333150e66fc96aca79c01105906797960David Sehr break; 757026217e3333150e66fc96aca79c01105906797960David Sehr case Intrinsics::NaClReadTP: 757126217e3333150e66fc96aca79c01105906797960David Sehr ReturnType = IceType_i32; 757226217e3333150e66fc96aca79c01105906797960David Sehr break; 757326217e3333150e66fc96aca79c01105906797960David Sehr case Intrinsics::Setjmp: 757426217e3333150e66fc96aca79c01105906797960David Sehr ArgTypes = {IceType_i32}; 757526217e3333150e66fc96aca79c01105906797960David Sehr ReturnType = IceType_i32; 757626217e3333150e66fc96aca79c01105906797960David Sehr break; 757726217e3333150e66fc96aca79c01105906797960David Sehr } 757826217e3333150e66fc96aca79c01105906797960David Sehr StackArgumentsSize = getCallStackArgumentsSizeBytes(ArgTypes, ReturnType); 757926217e3333150e66fc96aca79c01105906797960David Sehr } else if (auto *Call = llvm::dyn_cast<InstCall>(Instr)) { 758026217e3333150e66fc96aca79c01105906797960David Sehr StackArgumentsSize = getCallStackArgumentsSizeBytes(Call); 758126217e3333150e66fc96aca79c01105906797960David Sehr } else if (auto *Ret = llvm::dyn_cast<InstRet>(Instr)) { 758226217e3333150e66fc96aca79c01105906797960David Sehr if (!Ret->hasRetValue()) 758326217e3333150e66fc96aca79c01105906797960David Sehr return; 758426217e3333150e66fc96aca79c01105906797960David Sehr Operand *RetValue = Ret->getRetValue(); 758526217e3333150e66fc96aca79c01105906797960David Sehr Type ReturnType = RetValue->getType(); 758626217e3333150e66fc96aca79c01105906797960David Sehr if (!isScalarFloatingType(ReturnType)) 758726217e3333150e66fc96aca79c01105906797960David Sehr return; 758826217e3333150e66fc96aca79c01105906797960David Sehr StackArgumentsSize = typeWidthInBytes(ReturnType); 758926217e3333150e66fc96aca79c01105906797960David Sehr } else { 759026217e3333150e66fc96aca79c01105906797960David Sehr return; 759126217e3333150e66fc96aca79c01105906797960David Sehr } 759226217e3333150e66fc96aca79c01105906797960David Sehr StackArgumentsSize = Traits::applyStackAlignment(StackArgumentsSize); 759326217e3333150e66fc96aca79c01105906797960David Sehr updateMaxOutArgsSizeBytes(StackArgumentsSize); 759426217e3333150e66fc96aca79c01105906797960David Sehr} 759526217e3333150e66fc96aca79c01105906797960David Sehr 75964a56686b5b56db6803f90ad53514bf2fa190d9f7John Portotemplate <typename TraitsType> 75974a56686b5b56db6803f90ad53514bf2fa190d9f7John Portouint32_t TargetX86Base<TraitsType>::getCallStackArgumentsSizeBytes( 7598e82b560e649f8a68bcb252b9b002708e74d962d3John Porto const CfgVector<Type> &ArgTypes, Type ReturnType) { 75994163b9fcd92c0e2604a0d5f52ad1eccc86cad266David Sehr uint32_t OutArgumentsSizeBytes = 0; 76004163b9fcd92c0e2604a0d5f52ad1eccc86cad266David Sehr uint32_t XmmArgCount = 0; 76014163b9fcd92c0e2604a0d5f52ad1eccc86cad266David Sehr uint32_t GprArgCount = 0; 760226217e3333150e66fc96aca79c01105906797960David Sehr for (Type Ty : ArgTypes) { 76034163b9fcd92c0e2604a0d5f52ad1eccc86cad266David Sehr // The PNaCl ABI requires the width of arguments to be at least 32 bits. 76044163b9fcd92c0e2604a0d5f52ad1eccc86cad266David Sehr assert(typeWidthInBytes(Ty) >= 4); 76054163b9fcd92c0e2604a0d5f52ad1eccc86cad266David Sehr if (isVectorType(Ty) && XmmArgCount < Traits::X86_MAX_XMM_ARGS) { 7606c577727f49d6f4ccdbf40e9fa527cf1c97f23a7aJim Stichnoth ++XmmArgCount; 7607c577727f49d6f4ccdbf40e9fa527cf1c97f23a7aJim Stichnoth } else if (isScalarFloatingType(Ty) && Traits::X86_PASS_SCALAR_FP_IN_XMM && 7608c577727f49d6f4ccdbf40e9fa527cf1c97f23a7aJim Stichnoth XmmArgCount < Traits::X86_MAX_XMM_ARGS) { 76094163b9fcd92c0e2604a0d5f52ad1eccc86cad266David Sehr ++XmmArgCount; 76104163b9fcd92c0e2604a0d5f52ad1eccc86cad266David Sehr } else if (isScalarIntegerType(Ty) && 76114163b9fcd92c0e2604a0d5f52ad1eccc86cad266David Sehr GprArgCount < Traits::X86_MAX_GPR_ARGS) { 76124163b9fcd92c0e2604a0d5f52ad1eccc86cad266David Sehr // The 64 bit ABI allows some integers to be passed in GPRs. 76134163b9fcd92c0e2604a0d5f52ad1eccc86cad266David Sehr ++GprArgCount; 76144163b9fcd92c0e2604a0d5f52ad1eccc86cad266David Sehr } else { 761526217e3333150e66fc96aca79c01105906797960David Sehr if (isVectorType(Ty)) { 76164163b9fcd92c0e2604a0d5f52ad1eccc86cad266David Sehr OutArgumentsSizeBytes = 76174163b9fcd92c0e2604a0d5f52ad1eccc86cad266David Sehr Traits::applyStackAlignment(OutArgumentsSizeBytes); 76184163b9fcd92c0e2604a0d5f52ad1eccc86cad266David Sehr } 761926217e3333150e66fc96aca79c01105906797960David Sehr OutArgumentsSizeBytes += typeWidthInBytesOnStack(Ty); 76204163b9fcd92c0e2604a0d5f52ad1eccc86cad266David Sehr } 76214163b9fcd92c0e2604a0d5f52ad1eccc86cad266David Sehr } 76224163b9fcd92c0e2604a0d5f52ad1eccc86cad266David Sehr if (Traits::Is64Bit) 76234163b9fcd92c0e2604a0d5f52ad1eccc86cad266David Sehr return OutArgumentsSizeBytes; 76244163b9fcd92c0e2604a0d5f52ad1eccc86cad266David Sehr // The 32 bit ABI requires floating point values to be returned on the x87 FP 76254163b9fcd92c0e2604a0d5f52ad1eccc86cad266David Sehr // stack. Ensure there is enough space for the fstp/movs for floating returns. 762626217e3333150e66fc96aca79c01105906797960David Sehr if (isScalarFloatingType(ReturnType)) { 76274163b9fcd92c0e2604a0d5f52ad1eccc86cad266David Sehr OutArgumentsSizeBytes = 76284163b9fcd92c0e2604a0d5f52ad1eccc86cad266David Sehr std::max(OutArgumentsSizeBytes, 762926217e3333150e66fc96aca79c01105906797960David Sehr static_cast<uint32_t>(typeWidthInBytesOnStack(ReturnType))); 76304163b9fcd92c0e2604a0d5f52ad1eccc86cad266David Sehr } 76314163b9fcd92c0e2604a0d5f52ad1eccc86cad266David Sehr return OutArgumentsSizeBytes; 76324163b9fcd92c0e2604a0d5f52ad1eccc86cad266David Sehr} 76334163b9fcd92c0e2604a0d5f52ad1eccc86cad266David Sehr 76344a56686b5b56db6803f90ad53514bf2fa190d9f7John Portotemplate <typename TraitsType> 76354a56686b5b56db6803f90ad53514bf2fa190d9f7John Portouint32_t TargetX86Base<TraitsType>::getCallStackArgumentsSizeBytes( 76364a56686b5b56db6803f90ad53514bf2fa190d9f7John Porto const InstCall *Instr) { 763726217e3333150e66fc96aca79c01105906797960David Sehr // Build a vector of the arguments' types. 7638e82b560e649f8a68bcb252b9b002708e74d962d3John Porto const SizeT NumArgs = Instr->getNumArgs(); 7639e82b560e649f8a68bcb252b9b002708e74d962d3John Porto CfgVector<Type> ArgTypes; 7640e82b560e649f8a68bcb252b9b002708e74d962d3John Porto ArgTypes.reserve(NumArgs); 7641e82b560e649f8a68bcb252b9b002708e74d962d3John Porto for (SizeT i = 0; i < NumArgs; ++i) { 764226217e3333150e66fc96aca79c01105906797960David Sehr Operand *Arg = Instr->getArg(i); 764326217e3333150e66fc96aca79c01105906797960David Sehr ArgTypes.emplace_back(Arg->getType()); 764426217e3333150e66fc96aca79c01105906797960David Sehr } 764526217e3333150e66fc96aca79c01105906797960David Sehr // Compute the return type (if any); 764626217e3333150e66fc96aca79c01105906797960David Sehr Type ReturnType = IceType_void; 764726217e3333150e66fc96aca79c01105906797960David Sehr Variable *Dest = Instr->getDest(); 764826217e3333150e66fc96aca79c01105906797960David Sehr if (Dest != nullptr) 764926217e3333150e66fc96aca79c01105906797960David Sehr ReturnType = Dest->getType(); 765026217e3333150e66fc96aca79c01105906797960David Sehr return getCallStackArgumentsSizeBytes(ArgTypes, ReturnType); 765126217e3333150e66fc96aca79c01105906797960David Sehr} 765226217e3333150e66fc96aca79c01105906797960David Sehr 76534a56686b5b56db6803f90ad53514bf2fa190d9f7John Portotemplate <typename TraitsType> 76544a56686b5b56db6803f90ad53514bf2fa190d9f7John PortoVariable *TargetX86Base<TraitsType>::makeZeroedRegister(Type Ty, 76558aa396610b7baf728631a43ea16ad3d13e38397aJim Stichnoth RegNumT RegNum) { 7656991656678c7c5d661093d9ca63d83e20afc9e468Jim Stichnoth Variable *Reg = makeReg(Ty, RegNum); 7657991656678c7c5d661093d9ca63d83e20afc9e468Jim Stichnoth switch (Ty) { 7658991656678c7c5d661093d9ca63d83e20afc9e468Jim Stichnoth case IceType_i1: 7659991656678c7c5d661093d9ca63d83e20afc9e468Jim Stichnoth case IceType_i8: 7660991656678c7c5d661093d9ca63d83e20afc9e468Jim Stichnoth case IceType_i16: 7661991656678c7c5d661093d9ca63d83e20afc9e468Jim Stichnoth case IceType_i32: 7662991656678c7c5d661093d9ca63d83e20afc9e468Jim Stichnoth case IceType_i64: 7663991656678c7c5d661093d9ca63d83e20afc9e468Jim Stichnoth // Conservatively do "mov reg, 0" to avoid modifying FLAGS. 7664991656678c7c5d661093d9ca63d83e20afc9e468Jim Stichnoth _mov(Reg, Ctx->getConstantZero(Ty)); 7665991656678c7c5d661093d9ca63d83e20afc9e468Jim Stichnoth break; 7666991656678c7c5d661093d9ca63d83e20afc9e468Jim Stichnoth case IceType_f32: 7667991656678c7c5d661093d9ca63d83e20afc9e468Jim Stichnoth case IceType_f64: 76681d937a8e35b7ae51ee64c32f8f34d51ba33498f4John Porto Context.insert<InstFakeDef>(Reg); 7669e398428c3ef8ed2339c0f334dd30a7d3f0ed4b6eDavid Sehr _xorps(Reg, Reg); 7670991656678c7c5d661093d9ca63d83e20afc9e468Jim Stichnoth break; 7671991656678c7c5d661093d9ca63d83e20afc9e468Jim Stichnoth default: 7672991656678c7c5d661093d9ca63d83e20afc9e468Jim Stichnoth // All vector types use the same pxor instruction. 7673991656678c7c5d661093d9ca63d83e20afc9e468Jim Stichnoth assert(isVectorType(Ty)); 76741d937a8e35b7ae51ee64c32f8f34d51ba33498f4John Porto Context.insert<InstFakeDef>(Reg); 7675991656678c7c5d661093d9ca63d83e20afc9e468Jim Stichnoth _pxor(Reg, Reg); 7676991656678c7c5d661093d9ca63d83e20afc9e468Jim Stichnoth break; 7677991656678c7c5d661093d9ca63d83e20afc9e468Jim Stichnoth } 7678991656678c7c5d661093d9ca63d83e20afc9e468Jim Stichnoth return Reg; 7679991656678c7c5d661093d9ca63d83e20afc9e468Jim Stichnoth} 7680991656678c7c5d661093d9ca63d83e20afc9e468Jim Stichnoth 768157e126899b20c65ff3ea23a3b7d7a67ab30b99dcAndrew Scull// There is no support for loading or emitting vector constants, so the vector 768257e126899b20c65ff3ea23a3b7d7a67ab30b99dcAndrew Scull// values returned from makeVectorOfZeros, makeVectorOfOnes, etc. are 768357e126899b20c65ff3ea23a3b7d7a67ab30b99dcAndrew Scull// initialized with register operations. 76847e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto// 768557e126899b20c65ff3ea23a3b7d7a67ab30b99dcAndrew Scull// TODO(wala): Add limited support for vector constants so that complex 768657e126899b20c65ff3ea23a3b7d7a67ab30b99dcAndrew Scull// initialization in registers is unnecessary. 76877e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto 76884a56686b5b56db6803f90ad53514bf2fa190d9f7John Portotemplate <typename TraitsType> 76894a56686b5b56db6803f90ad53514bf2fa190d9f7John PortoVariable *TargetX86Base<TraitsType>::makeVectorOfZeros(Type Ty, 76908aa396610b7baf728631a43ea16ad3d13e38397aJim Stichnoth RegNumT RegNum) { 7691991656678c7c5d661093d9ca63d83e20afc9e468Jim Stichnoth return makeZeroedRegister(Ty, RegNum); 76927e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto} 76937e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto 76944a56686b5b56db6803f90ad53514bf2fa190d9f7John Portotemplate <typename TraitsType> 76954a56686b5b56db6803f90ad53514bf2fa190d9f7John PortoVariable *TargetX86Base<TraitsType>::makeVectorOfMinusOnes(Type Ty, 76968aa396610b7baf728631a43ea16ad3d13e38397aJim Stichnoth RegNumT RegNum) { 76977e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto Variable *MinusOnes = makeReg(Ty, RegNum); 76987e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto // Insert a FakeDef so the live range of MinusOnes is not overestimated. 76991d937a8e35b7ae51ee64c32f8f34d51ba33498f4John Porto Context.insert<InstFakeDef>(MinusOnes); 7700b19d39cc5fa8dc60c678c2507af02147184f168fDavid Sehr if (Ty == IceType_f64) 7701b19d39cc5fa8dc60c678c2507af02147184f168fDavid Sehr // Making a vector of minus ones of type f64 is currently only used for the 7702b19d39cc5fa8dc60c678c2507af02147184f168fDavid Sehr // fabs intrinsic. To use the f64 type to create this mask with pcmpeqq 7703b19d39cc5fa8dc60c678c2507af02147184f168fDavid Sehr // requires SSE 4.1. Since we're just creating a mask, pcmpeqd does the 7704b19d39cc5fa8dc60c678c2507af02147184f168fDavid Sehr // same job and only requires SSE2. 7705b19d39cc5fa8dc60c678c2507af02147184f168fDavid Sehr _pcmpeq(MinusOnes, MinusOnes, IceType_f32); 7706b19d39cc5fa8dc60c678c2507af02147184f168fDavid Sehr else 7707b19d39cc5fa8dc60c678c2507af02147184f168fDavid Sehr _pcmpeq(MinusOnes, MinusOnes); 77087e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto return MinusOnes; 77097e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto} 77107e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto 77114a56686b5b56db6803f90ad53514bf2fa190d9f7John Portotemplate <typename TraitsType> 77128aa396610b7baf728631a43ea16ad3d13e38397aJim StichnothVariable *TargetX86Base<TraitsType>::makeVectorOfOnes(Type Ty, RegNumT RegNum) { 77137e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto Variable *Dest = makeVectorOfZeros(Ty, RegNum); 77147e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto Variable *MinusOne = makeVectorOfMinusOnes(Ty); 77157e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto _psub(Dest, MinusOne); 77167e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto return Dest; 77177e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto} 77187e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto 77194a56686b5b56db6803f90ad53514bf2fa190d9f7John Portotemplate <typename TraitsType> 77204a56686b5b56db6803f90ad53514bf2fa190d9f7John PortoVariable *TargetX86Base<TraitsType>::makeVectorOfHighOrderBits(Type Ty, 77218aa396610b7baf728631a43ea16ad3d13e38397aJim Stichnoth RegNumT RegNum) { 77227e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto assert(Ty == IceType_v4i32 || Ty == IceType_v4f32 || Ty == IceType_v8i16 || 77237e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto Ty == IceType_v16i8); 77247e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto if (Ty == IceType_v4f32 || Ty == IceType_v4i32 || Ty == IceType_v8i16) { 77257e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto Variable *Reg = makeVectorOfOnes(Ty, RegNum); 77267e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto SizeT Shift = 77277e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto typeWidthInBytes(typeElementType(Ty)) * Traits::X86_CHAR_BIT - 1; 77287e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto _psll(Reg, Ctx->getConstantInt8(Shift)); 77297e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto return Reg; 77307e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto } else { 77317e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto // SSE has no left shift operation for vectors of 8 bit integers. 77325bff61c44841990680781892036adb17b3cff0c4Jim Stichnoth constexpr uint32_t HIGH_ORDER_BITS_MASK = 0x80808080; 77337e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto Constant *ConstantMask = Ctx->getConstantInt32(HIGH_ORDER_BITS_MASK); 77347e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto Variable *Reg = makeReg(Ty, RegNum); 77357e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto _movd(Reg, legalize(ConstantMask, Legal_Reg | Legal_Mem)); 77367e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto _pshufd(Reg, Reg, Ctx->getConstantZero(IceType_i8)); 77377e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto return Reg; 77387e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto } 77397e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto} 77407e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto 774157e126899b20c65ff3ea23a3b7d7a67ab30b99dcAndrew Scull/// Construct a mask in a register that can be and'ed with a floating-point 774257e126899b20c65ff3ea23a3b7d7a67ab30b99dcAndrew Scull/// value to mask off its sign bit. The value will be <4 x 0x7fffffff> for f32 774357e126899b20c65ff3ea23a3b7d7a67ab30b99dcAndrew Scull/// and v4f32, and <2 x 0x7fffffffffffffff> for f64. Construct it as vector of 774457e126899b20c65ff3ea23a3b7d7a67ab30b99dcAndrew Scull/// ones logically right shifted one bit. 774557e126899b20c65ff3ea23a3b7d7a67ab30b99dcAndrew Scull// TODO(stichnot): Fix the wala 774657e126899b20c65ff3ea23a3b7d7a67ab30b99dcAndrew Scull// TODO: above, to represent vector constants in memory. 77474a56686b5b56db6803f90ad53514bf2fa190d9f7John Portotemplate <typename TraitsType> 77484a56686b5b56db6803f90ad53514bf2fa190d9f7John PortoVariable *TargetX86Base<TraitsType>::makeVectorOfFabsMask(Type Ty, 77498aa396610b7baf728631a43ea16ad3d13e38397aJim Stichnoth RegNumT RegNum) { 77507e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto Variable *Reg = makeVectorOfMinusOnes(Ty, RegNum); 77517e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto _psrl(Reg, Ctx->getConstantInt8(1)); 77527e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto return Reg; 77537e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto} 77547e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto 77554a56686b5b56db6803f90ad53514bf2fa190d9f7John Portotemplate <typename TraitsType> 77564a56686b5b56db6803f90ad53514bf2fa190d9f7John Portotypename TargetX86Base<TraitsType>::X86OperandMem * 77574a56686b5b56db6803f90ad53514bf2fa190d9f7John PortoTargetX86Base<TraitsType>::getMemoryOperandForStackSlot(Type Ty, Variable *Slot, 77584a56686b5b56db6803f90ad53514bf2fa190d9f7John Porto uint32_t Offset) { 77597e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto // Ensure that Loc is a stack slot. 776011c9a325399b282cb4ea7d1d24d42fceeec2a09aAndrew Scull assert(Slot->mustNotHaveReg()); 77615fa0a5f7f730f33e6987527a261de8d833d7585cReed Kotler assert(Slot->getRegNum().hasNoValue()); 77627e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto // Compute the location of Loc in memory. 776357e126899b20c65ff3ea23a3b7d7a67ab30b99dcAndrew Scull // TODO(wala,stichnot): lea should not 776457e126899b20c65ff3ea23a3b7d7a67ab30b99dcAndrew Scull // be required. The address of the stack slot is known at compile time 776557e126899b20c65ff3ea23a3b7d7a67ab30b99dcAndrew Scull // (although not until after addProlog()). 776632f9ccef8dee33110cc9ebdca2853d6a078752edNicolas Capens const Type PointerType = getPointerType(); 77677e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto Variable *Loc = makeReg(PointerType); 77687e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto _lea(Loc, Slot); 77697e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto Constant *ConstantOffset = Ctx->getConstantInt32(Offset); 77704a56686b5b56db6803f90ad53514bf2fa190d9f7John Porto return X86OperandMem::create(Func, Ty, Loc, ConstantOffset); 77717e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto} 77727e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto 7773c59288b334b91f4c0b2edf0de7415c68c760aa12Jim Stichnoth/// Lowering helper to copy a scalar integer source operand into some 8-bit GPR. 7774c59288b334b91f4c0b2edf0de7415c68c760aa12Jim Stichnoth/// Src is assumed to already be legalized. If the source operand is known to 7775c59288b334b91f4c0b2edf0de7415c68c760aa12Jim Stichnoth/// be a memory or immediate operand, a simple mov will suffice. But if the 7776c59288b334b91f4c0b2edf0de7415c68c760aa12Jim Stichnoth/// source operand can be a physical register, then it must first be copied into 7777c59288b334b91f4c0b2edf0de7415c68c760aa12Jim Stichnoth/// a physical register that is truncable to 8-bit, then truncated into a 7778c59288b334b91f4c0b2edf0de7415c68c760aa12Jim Stichnoth/// physical register that can receive a truncation, and finally copied into the 7779c59288b334b91f4c0b2edf0de7415c68c760aa12Jim Stichnoth/// result 8-bit register (which in general can be any 8-bit register). For 7780c59288b334b91f4c0b2edf0de7415c68c760aa12Jim Stichnoth/// example, moving %ebp into %ah may be accomplished as: 7781c59288b334b91f4c0b2edf0de7415c68c760aa12Jim Stichnoth/// movl %ebp, %edx 7782c59288b334b91f4c0b2edf0de7415c68c760aa12Jim Stichnoth/// mov_trunc %edx, %dl // this redundant assignment is ultimately elided 7783c59288b334b91f4c0b2edf0de7415c68c760aa12Jim Stichnoth/// movb %dl, %ah 7784c59288b334b91f4c0b2edf0de7415c68c760aa12Jim Stichnoth/// On the other hand, moving a memory or immediate operand into ah: 7785c59288b334b91f4c0b2edf0de7415c68c760aa12Jim Stichnoth/// movb 4(%ebp), %ah 7786c59288b334b91f4c0b2edf0de7415c68c760aa12Jim Stichnoth/// movb $my_imm, %ah 7787c59288b334b91f4c0b2edf0de7415c68c760aa12Jim Stichnoth/// 7788c59288b334b91f4c0b2edf0de7415c68c760aa12Jim Stichnoth/// Note #1. On a 64-bit target, the "movb 4(%ebp), %ah" is likely not 7789c59288b334b91f4c0b2edf0de7415c68c760aa12Jim Stichnoth/// encodable, so RegNum=Reg_ah should NOT be given as an argument. Instead, 77905fa0a5f7f730f33e6987527a261de8d833d7585cReed Kotler/// use RegNum=RegNumT() and then let the caller do a separate copy into 7791c59288b334b91f4c0b2edf0de7415c68c760aa12Jim Stichnoth/// Reg_ah. 7792c59288b334b91f4c0b2edf0de7415c68c760aa12Jim Stichnoth/// 7793c59288b334b91f4c0b2edf0de7415c68c760aa12Jim Stichnoth/// Note #2. ConstantRelocatable operands are also put through this process 7794c59288b334b91f4c0b2edf0de7415c68c760aa12Jim Stichnoth/// (not truncated directly) because our ELF emitter does R_386_32 relocations 7795c59288b334b91f4c0b2edf0de7415c68c760aa12Jim Stichnoth/// but not R_386_8 relocations. 7796c59288b334b91f4c0b2edf0de7415c68c760aa12Jim Stichnoth/// 7797c59288b334b91f4c0b2edf0de7415c68c760aa12Jim Stichnoth/// Note #3. If Src is a Variable, the result will be an infinite-weight i8 7798c59288b334b91f4c0b2edf0de7415c68c760aa12Jim Stichnoth/// Variable with the RCX86_IsTrunc8Rcvr register class. As such, this helper 7799c59288b334b91f4c0b2edf0de7415c68c760aa12Jim Stichnoth/// is a convenient way to prevent ah/bh/ch/dh from being an (invalid) argument 7800c59288b334b91f4c0b2edf0de7415c68c760aa12Jim Stichnoth/// to the pinsrb instruction. 78014a56686b5b56db6803f90ad53514bf2fa190d9f7John Portotemplate <typename TraitsType> 78028aa396610b7baf728631a43ea16ad3d13e38397aJim StichnothVariable *TargetX86Base<TraitsType>::copyToReg8(Operand *Src, RegNumT RegNum) { 7803c59288b334b91f4c0b2edf0de7415c68c760aa12Jim Stichnoth Type Ty = Src->getType(); 7804c59288b334b91f4c0b2edf0de7415c68c760aa12Jim Stichnoth assert(isScalarIntegerType(Ty)); 7805c59288b334b91f4c0b2edf0de7415c68c760aa12Jim Stichnoth assert(Ty != IceType_i1); 7806c59288b334b91f4c0b2edf0de7415c68c760aa12Jim Stichnoth Variable *Reg = makeReg(IceType_i8, RegNum); 7807c59288b334b91f4c0b2edf0de7415c68c760aa12Jim Stichnoth Reg->setRegClass(RCX86_IsTrunc8Rcvr); 7808c59288b334b91f4c0b2edf0de7415c68c760aa12Jim Stichnoth if (llvm::isa<Variable>(Src) || llvm::isa<ConstantRelocatable>(Src)) { 7809c59288b334b91f4c0b2edf0de7415c68c760aa12Jim Stichnoth Variable *SrcTruncable = makeReg(Ty); 7810c59288b334b91f4c0b2edf0de7415c68c760aa12Jim Stichnoth switch (Ty) { 7811c59288b334b91f4c0b2edf0de7415c68c760aa12Jim Stichnoth case IceType_i64: 7812c59288b334b91f4c0b2edf0de7415c68c760aa12Jim Stichnoth SrcTruncable->setRegClass(RCX86_Is64To8); 7813c59288b334b91f4c0b2edf0de7415c68c760aa12Jim Stichnoth break; 7814c59288b334b91f4c0b2edf0de7415c68c760aa12Jim Stichnoth case IceType_i32: 7815c59288b334b91f4c0b2edf0de7415c68c760aa12Jim Stichnoth SrcTruncable->setRegClass(RCX86_Is32To8); 7816c59288b334b91f4c0b2edf0de7415c68c760aa12Jim Stichnoth break; 7817c59288b334b91f4c0b2edf0de7415c68c760aa12Jim Stichnoth case IceType_i16: 7818c59288b334b91f4c0b2edf0de7415c68c760aa12Jim Stichnoth SrcTruncable->setRegClass(RCX86_Is16To8); 7819c59288b334b91f4c0b2edf0de7415c68c760aa12Jim Stichnoth break; 7820c59288b334b91f4c0b2edf0de7415c68c760aa12Jim Stichnoth default: 7821c59288b334b91f4c0b2edf0de7415c68c760aa12Jim Stichnoth // i8 - just use default register class 7822c59288b334b91f4c0b2edf0de7415c68c760aa12Jim Stichnoth break; 7823c59288b334b91f4c0b2edf0de7415c68c760aa12Jim Stichnoth } 7824c59288b334b91f4c0b2edf0de7415c68c760aa12Jim Stichnoth Variable *SrcRcvr = makeReg(IceType_i8); 7825c59288b334b91f4c0b2edf0de7415c68c760aa12Jim Stichnoth SrcRcvr->setRegClass(RCX86_IsTrunc8Rcvr); 7826c59288b334b91f4c0b2edf0de7415c68c760aa12Jim Stichnoth _mov(SrcTruncable, Src); 7827c59288b334b91f4c0b2edf0de7415c68c760aa12Jim Stichnoth _mov(SrcRcvr, SrcTruncable); 7828c59288b334b91f4c0b2edf0de7415c68c760aa12Jim Stichnoth Src = SrcRcvr; 7829c59288b334b91f4c0b2edf0de7415c68c760aa12Jim Stichnoth } 7830c59288b334b91f4c0b2edf0de7415c68c760aa12Jim Stichnoth _mov(Reg, Src); 7831c59288b334b91f4c0b2edf0de7415c68c760aa12Jim Stichnoth return Reg; 7832c59288b334b91f4c0b2edf0de7415c68c760aa12Jim Stichnoth} 7833c59288b334b91f4c0b2edf0de7415c68c760aa12Jim Stichnoth 78349612d32c7e5eb2cb403686ef31172d42e075e460Andrew Scull/// Helper for legalize() to emit the right code to lower an operand to a 78359612d32c7e5eb2cb403686ef31172d42e075e460Andrew Scull/// register of the appropriate type. 78364a56686b5b56db6803f90ad53514bf2fa190d9f7John Portotemplate <typename TraitsType> 78378aa396610b7baf728631a43ea16ad3d13e38397aJim StichnothVariable *TargetX86Base<TraitsType>::copyToReg(Operand *Src, RegNumT RegNum) { 78387e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto Type Ty = Src->getType(); 78397e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto Variable *Reg = makeReg(Ty, RegNum); 78407e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto if (isVectorType(Ty)) { 78417e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto _movp(Reg, Src); 78427e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto } else { 78437e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto _mov(Reg, Src); 78447e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto } 78457e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto return Reg; 78467e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto} 78477e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto 78484a56686b5b56db6803f90ad53514bf2fa190d9f7John Portotemplate <typename TraitsType> 78494a56686b5b56db6803f90ad53514bf2fa190d9f7John PortoOperand *TargetX86Base<TraitsType>::legalize(Operand *From, LegalMask Allowed, 78508aa396610b7baf728631a43ea16ad3d13e38397aJim Stichnoth RegNumT RegNum) { 7851d46999474d2b4a388e1d8a7c71f06cd4cec51bfcKarl Schimpf const bool UseNonsfi = getFlags().getUseNonsfi(); 78528ff4b2819944bc4f02fb29204a1fa5ba7dea5682Jim Stichnoth const Type Ty = From->getType(); 785357e126899b20c65ff3ea23a3b7d7a67ab30b99dcAndrew Scull // Assert that a physical register is allowed. To date, all calls to 785457e126899b20c65ff3ea23a3b7d7a67ab30b99dcAndrew Scull // legalize() allow a physical register. If a physical register needs to be 785557e126899b20c65ff3ea23a3b7d7a67ab30b99dcAndrew Scull // explicitly disallowed, then new code will need to be written to force a 785657e126899b20c65ff3ea23a3b7d7a67ab30b99dcAndrew Scull // spill. 78577e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto assert(Allowed & Legal_Reg); 785857e126899b20c65ff3ea23a3b7d7a67ab30b99dcAndrew Scull // If we're asking for a specific physical register, make sure we're not 785957e126899b20c65ff3ea23a3b7d7a67ab30b99dcAndrew Scull // allowing any other operand kinds. (This could be future work, e.g. allow 786057e126899b20c65ff3ea23a3b7d7a67ab30b99dcAndrew Scull // the shl shift amount to be either an immediate or in ecx.) 78615fa0a5f7f730f33e6987527a261de8d833d7585cReed Kotler assert(RegNum.hasNoValue() || Allowed == Legal_Reg); 78627e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto 7863318f4cdaa21eac5ef1d16731e51cd7adb3083d3bJim Stichnoth // Substitute with an available infinite-weight variable if possible. Only do 7864318f4cdaa21eac5ef1d16731e51cd7adb3083d3bJim Stichnoth // this when we are not asking for a specific register, and when the 7865318f4cdaa21eac5ef1d16731e51cd7adb3083d3bJim Stichnoth // substitution is not locked to a specific register, and when the types 7866318f4cdaa21eac5ef1d16731e51cd7adb3083d3bJim Stichnoth // match, in order to capture the vast majority of opportunities and avoid 7867318f4cdaa21eac5ef1d16731e51cd7adb3083d3bJim Stichnoth // corner cases in the lowering. 78685fa0a5f7f730f33e6987527a261de8d833d7585cReed Kotler if (RegNum.hasNoValue()) { 7869318f4cdaa21eac5ef1d16731e51cd7adb3083d3bJim Stichnoth if (Variable *Subst = getContext().availabilityGet(From)) { 7870318f4cdaa21eac5ef1d16731e51cd7adb3083d3bJim Stichnoth // At this point we know there is a potential substitution available. 7871318f4cdaa21eac5ef1d16731e51cd7adb3083d3bJim Stichnoth if (Subst->mustHaveReg() && !Subst->hasReg()) { 7872318f4cdaa21eac5ef1d16731e51cd7adb3083d3bJim Stichnoth // At this point we know the substitution will have a register. 7873318f4cdaa21eac5ef1d16731e51cd7adb3083d3bJim Stichnoth if (From->getType() == Subst->getType()) { 7874318f4cdaa21eac5ef1d16731e51cd7adb3083d3bJim Stichnoth // At this point we know the substitution's register is compatible. 7875318f4cdaa21eac5ef1d16731e51cd7adb3083d3bJim Stichnoth return Subst; 7876318f4cdaa21eac5ef1d16731e51cd7adb3083d3bJim Stichnoth } 7877318f4cdaa21eac5ef1d16731e51cd7adb3083d3bJim Stichnoth } 7878318f4cdaa21eac5ef1d16731e51cd7adb3083d3bJim Stichnoth } 7879318f4cdaa21eac5ef1d16731e51cd7adb3083d3bJim Stichnoth } 7880318f4cdaa21eac5ef1d16731e51cd7adb3083d3bJim Stichnoth 78814a56686b5b56db6803f90ad53514bf2fa190d9f7John Porto if (auto *Mem = llvm::dyn_cast<X86OperandMem>(From)) { 788257e126899b20c65ff3ea23a3b7d7a67ab30b99dcAndrew Scull // Before doing anything with a Mem operand, we need to ensure that the 788357e126899b20c65ff3ea23a3b7d7a67ab30b99dcAndrew Scull // Base and Index components are in physical registers. 78847e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto Variable *Base = Mem->getBase(); 78857e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto Variable *Index = Mem->getIndex(); 78868ff4b2819944bc4f02fb29204a1fa5ba7dea5682Jim Stichnoth Constant *Offset = Mem->getOffset(); 78877e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto Variable *RegBase = nullptr; 78887e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto Variable *RegIndex = nullptr; 788956958cb33d3c1d045f2844408d825442d523f59fJohn Porto uint16_t Shift = Mem->getShift(); 78907e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto if (Base) { 78914318a410939b5cee89736f6f15a3185e385b5dc7David Sehr RegBase = llvm::cast<Variable>( 78924318a410939b5cee89736f6f15a3185e385b5dc7David Sehr legalize(Base, Legal_Reg | Legal_Rematerializable)); 78937e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto } 78947e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto if (Index) { 789556958cb33d3c1d045f2844408d825442d523f59fJohn Porto // TODO(jpp): perhaps we should only allow Legal_Reg if 789656958cb33d3c1d045f2844408d825442d523f59fJohn Porto // Base->isRematerializable. 78974318a410939b5cee89736f6f15a3185e385b5dc7David Sehr RegIndex = llvm::cast<Variable>( 78984318a410939b5cee89736f6f15a3185e385b5dc7David Sehr legalize(Index, Legal_Reg | Legal_Rematerializable)); 78997e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto } 7900ac2388c385afda17acb3fc242b17b9232ec2262aJohn Porto 79017e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto if (Base != RegBase || Index != RegIndex) { 790256958cb33d3c1d045f2844408d825442d523f59fJohn Porto Mem = X86OperandMem::create(Func, Ty, RegBase, Offset, RegIndex, Shift, 7903ac2388c385afda17acb3fc242b17b9232ec2262aJohn Porto Mem->getSegmentRegister()); 79047e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto } 79057e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto 7906ac2388c385afda17acb3fc242b17b9232ec2262aJohn Porto // For all Memory Operands, we do randomization/pooling here. 79077e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto From = randomizeOrPoolImmediate(Mem); 79087e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto 79097e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto if (!(Allowed & Legal_Mem)) { 79107e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto From = copyToReg(From, RegNum); 79117e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto } 79127e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto return From; 79137e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto } 79148ff4b2819944bc4f02fb29204a1fa5ba7dea5682Jim Stichnoth 79157e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto if (auto *Const = llvm::dyn_cast<Constant>(From)) { 79167e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto if (llvm::isa<ConstantUndef>(Const)) { 7917fbdd244013aad0808220386eb6bd17eb0635dd4cJan Voung From = legalizeUndef(Const, RegNum); 79187e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto if (isVectorType(Ty)) 7919fbdd244013aad0808220386eb6bd17eb0635dd4cJan Voung return From; 7920fbdd244013aad0808220386eb6bd17eb0635dd4cJan Voung Const = llvm::cast<Constant>(From); 79217e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto } 79227e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto // There should be no constants of vector type (other than undef). 79237e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto assert(!isVectorType(Ty)); 79247e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto 79251d235425dab1f3dd059973fc53f1b1d5879469e3John Porto // If the operand is a 64 bit constant integer we need to legalize it to a 79261d235425dab1f3dd059973fc53f1b1d5879469e3John Porto // register in x86-64. 79271d235425dab1f3dd059973fc53f1b1d5879469e3John Porto if (Traits::Is64Bit) { 79289c2c093f37106ab952a439fa098d862101bb5854Jim Stichnoth if (auto *C64 = llvm::dyn_cast<ConstantInteger64>(Const)) { 79299c2c093f37106ab952a439fa098d862101bb5854Jim Stichnoth if (!Utils::IsInt(32, C64->getValue())) { 79309c2c093f37106ab952a439fa098d862101bb5854Jim Stichnoth if (RegNum.hasValue()) { 79319c2c093f37106ab952a439fa098d862101bb5854Jim Stichnoth assert(Traits::getGprForType(IceType_i64, RegNum) == RegNum); 79329c2c093f37106ab952a439fa098d862101bb5854Jim Stichnoth } 79339c2c093f37106ab952a439fa098d862101bb5854Jim Stichnoth return copyToReg(Const, RegNum); 7934008f4ce5417c97891ea02b54b691b39aa2e2a0afJohn Porto } 79351d235425dab1f3dd059973fc53f1b1d5879469e3John Porto } 79361d235425dab1f3dd059973fc53f1b1d5879469e3John Porto } 79371d235425dab1f3dd059973fc53f1b1d5879469e3John Porto 793857e126899b20c65ff3ea23a3b7d7a67ab30b99dcAndrew Scull // If the operand is an 32 bit constant integer, we should check whether we 793957e126899b20c65ff3ea23a3b7d7a67ab30b99dcAndrew Scull // need to randomize it or pool it. 794054f3d518805b04b3f2958b7906a7d07dedb4e1aeJim Stichnoth if (auto *C = llvm::dyn_cast<ConstantInteger32>(Const)) { 79417e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto Operand *NewConst = randomizeOrPoolImmediate(C, RegNum); 79427e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto if (NewConst != Const) { 79437e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto return NewConst; 79447e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto } 79457e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto } 79467e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto 79478ff4b2819944bc4f02fb29204a1fa5ba7dea5682Jim Stichnoth if (auto *CR = llvm::dyn_cast<ConstantRelocatable>(Const)) { 7948ac2388c385afda17acb3fc242b17b9232ec2262aJohn Porto // If the operand is a ConstantRelocatable, and Legal_AddrAbs is not 7949ac2388c385afda17acb3fc242b17b9232ec2262aJohn Porto // specified, and UseNonsfi is indicated, we need to add RebasePtr. 79508ff4b2819944bc4f02fb29204a1fa5ba7dea5682Jim Stichnoth if (UseNonsfi && !(Allowed & Legal_AddrAbs)) { 79518ff4b2819944bc4f02fb29204a1fa5ba7dea5682Jim Stichnoth assert(Ty == IceType_i32); 79528ff4b2819944bc4f02fb29204a1fa5ba7dea5682Jim Stichnoth Variable *NewVar = makeReg(Ty, RegNum); 7953ac2388c385afda17acb3fc242b17b9232ec2262aJohn Porto auto *Mem = Traits::X86OperandMem::create(Func, Ty, nullptr, CR); 7954ac2388c385afda17acb3fc242b17b9232ec2262aJohn Porto // LEAs are not automatically sandboxed, thus we explicitly invoke 7955ac2388c385afda17acb3fc242b17b9232ec2262aJohn Porto // _sandbox_mem_reference. 7956ac2388c385afda17acb3fc242b17b9232ec2262aJohn Porto _lea(NewVar, _sandbox_mem_reference(Mem)); 79578ff4b2819944bc4f02fb29204a1fa5ba7dea5682Jim Stichnoth From = NewVar; 79588ff4b2819944bc4f02fb29204a1fa5ba7dea5682Jim Stichnoth } 7959ac2388c385afda17acb3fc242b17b9232ec2262aJohn Porto } else if (isScalarFloatingType(Ty)) { 7960ac2388c385afda17acb3fc242b17b9232ec2262aJohn Porto // Convert a scalar floating point constant into an explicit memory 7961ac2388c385afda17acb3fc242b17b9232ec2262aJohn Porto // operand. 7962991656678c7c5d661093d9ca63d83e20afc9e468Jim Stichnoth if (auto *ConstFloat = llvm::dyn_cast<ConstantFloat>(Const)) { 7963ccea793fe4259ba9aa0b6bcd3f281a5c08ac2aa4John Porto if (Utils::isPositiveZero(ConstFloat->getValue())) 7964991656678c7c5d661093d9ca63d83e20afc9e468Jim Stichnoth return makeZeroedRegister(Ty, RegNum); 7965991656678c7c5d661093d9ca63d83e20afc9e468Jim Stichnoth } else if (auto *ConstDouble = llvm::dyn_cast<ConstantDouble>(Const)) { 7966ccea793fe4259ba9aa0b6bcd3f281a5c08ac2aa4John Porto if (Utils::isPositiveZero(ConstDouble->getValue())) 7967991656678c7c5d661093d9ca63d83e20afc9e468Jim Stichnoth return makeZeroedRegister(Ty, RegNum); 7968991656678c7c5d661093d9ca63d83e20afc9e468Jim Stichnoth } 7969ac2388c385afda17acb3fc242b17b9232ec2262aJohn Porto 7970467ffe51bebcb3ae3e3ce745c38fc20e8837c31cJim Stichnoth auto *CFrom = llvm::cast<Constant>(From); 7971467ffe51bebcb3ae3e3ce745c38fc20e8837c31cJim Stichnoth assert(CFrom->getShouldBePooled()); 7972467ffe51bebcb3ae3e3ce745c38fc20e8837c31cJim Stichnoth Constant *Offset = Ctx->getConstantSym(0, CFrom->getLabelName()); 7973ac2388c385afda17acb3fc242b17b9232ec2262aJohn Porto auto *Mem = X86OperandMem::create(Func, Ty, nullptr, Offset); 79748ff4b2819944bc4f02fb29204a1fa5ba7dea5682Jim Stichnoth From = Mem; 79757e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto } 7976ac2388c385afda17acb3fc242b17b9232ec2262aJohn Porto 79777e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto bool NeedsReg = false; 79787e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto if (!(Allowed & Legal_Imm) && !isScalarFloatingType(Ty)) 7979ac2388c385afda17acb3fc242b17b9232ec2262aJohn Porto // Immediate specifically not allowed. 79807e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto NeedsReg = true; 79817e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto if (!(Allowed & Legal_Mem) && isScalarFloatingType(Ty)) 79827e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto // On x86, FP constants are lowered to mem operands. 79837e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto NeedsReg = true; 79847e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto if (NeedsReg) { 79857e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto From = copyToReg(From, RegNum); 79867e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto } 79877e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto return From; 79887e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto } 79898ff4b2819944bc4f02fb29204a1fa5ba7dea5682Jim Stichnoth 79905bff61c44841990680781892036adb17b3cff0c4Jim Stichnoth if (auto *Var = llvm::dyn_cast<Variable>(From)) { 799157e126899b20c65ff3ea23a3b7d7a67ab30b99dcAndrew Scull // Check if the variable is guaranteed a physical register. This can happen 799257e126899b20c65ff3ea23a3b7d7a67ab30b99dcAndrew Scull // either when the variable is pre-colored or when it is assigned infinite 799357e126899b20c65ff3ea23a3b7d7a67ab30b99dcAndrew Scull // weight. 799411c9a325399b282cb4ea7d1d24d42fceeec2a09aAndrew Scull bool MustHaveRegister = (Var->hasReg() || Var->mustHaveReg()); 79954318a410939b5cee89736f6f15a3185e385b5dc7David Sehr bool MustRematerialize = 79964318a410939b5cee89736f6f15a3185e385b5dc7David Sehr (Var->isRematerializable() && !(Allowed & Legal_Rematerializable)); 79977e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto // We need a new physical register for the operand if: 79984318a410939b5cee89736f6f15a3185e385b5dc7David Sehr // - Mem is not allowed and Var isn't guaranteed a physical register, or 79994318a410939b5cee89736f6f15a3185e385b5dc7David Sehr // - RegNum is required and Var->getRegNum() doesn't match, or 80004318a410939b5cee89736f6f15a3185e385b5dc7David Sehr // - Var is a rematerializable variable and rematerializable pass-through is 80014318a410939b5cee89736f6f15a3185e385b5dc7David Sehr // not allowed (in which case we need an lea instruction). 80024318a410939b5cee89736f6f15a3185e385b5dc7David Sehr if (MustRematerialize) { 80034318a410939b5cee89736f6f15a3185e385b5dc7David Sehr assert(Ty == IceType_i32); 80044318a410939b5cee89736f6f15a3185e385b5dc7David Sehr Variable *NewVar = makeReg(Ty, RegNum); 80054318a410939b5cee89736f6f15a3185e385b5dc7David Sehr // Since Var is rematerializable, the offset will be added when the lea is 80064318a410939b5cee89736f6f15a3185e385b5dc7David Sehr // emitted. 80074318a410939b5cee89736f6f15a3185e385b5dc7David Sehr constexpr Constant *NoOffset = nullptr; 80084a56686b5b56db6803f90ad53514bf2fa190d9f7John Porto auto *Mem = X86OperandMem::create(Func, Ty, Var, NoOffset); 80094318a410939b5cee89736f6f15a3185e385b5dc7David Sehr _lea(NewVar, Mem); 80104318a410939b5cee89736f6f15a3185e385b5dc7David Sehr From = NewVar; 80114318a410939b5cee89736f6f15a3185e385b5dc7David Sehr } else if ((!(Allowed & Legal_Mem) && !MustHaveRegister) || 80125fa0a5f7f730f33e6987527a261de8d833d7585cReed Kotler (RegNum.hasValue() && RegNum != Var->getRegNum())) { 80137e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto From = copyToReg(From, RegNum); 80147e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto } 80157e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto return From; 80167e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto } 80178ff4b2819944bc4f02fb29204a1fa5ba7dea5682Jim Stichnoth 80188ff4b2819944bc4f02fb29204a1fa5ba7dea5682Jim Stichnoth llvm::report_fatal_error("Unhandled operand kind in legalize()"); 80197e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto return From; 80207e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto} 80217e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto 80229612d32c7e5eb2cb403686ef31172d42e075e460Andrew Scull/// Provide a trivial wrapper to legalize() for this common usage. 80234a56686b5b56db6803f90ad53514bf2fa190d9f7John Portotemplate <typename TraitsType> 80244a56686b5b56db6803f90ad53514bf2fa190d9f7John PortoVariable *TargetX86Base<TraitsType>::legalizeToReg(Operand *From, 80258aa396610b7baf728631a43ea16ad3d13e38397aJim Stichnoth RegNumT RegNum) { 80267e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto return llvm::cast<Variable>(legalize(From, Legal_Reg, RegNum)); 80277e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto} 80287e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto 8029fbdd244013aad0808220386eb6bd17eb0635dd4cJan Voung/// Legalize undef values to concrete values. 80304a56686b5b56db6803f90ad53514bf2fa190d9f7John Portotemplate <typename TraitsType> 80314a56686b5b56db6803f90ad53514bf2fa190d9f7John PortoOperand *TargetX86Base<TraitsType>::legalizeUndef(Operand *From, 80328aa396610b7baf728631a43ea16ad3d13e38397aJim Stichnoth RegNumT RegNum) { 8033fbdd244013aad0808220386eb6bd17eb0635dd4cJan Voung Type Ty = From->getType(); 8034fbdd244013aad0808220386eb6bd17eb0635dd4cJan Voung if (llvm::isa<ConstantUndef>(From)) { 8035fbdd244013aad0808220386eb6bd17eb0635dd4cJan Voung // Lower undefs to zero. Another option is to lower undefs to an 803657e126899b20c65ff3ea23a3b7d7a67ab30b99dcAndrew Scull // uninitialized register; however, using an uninitialized register results 803757e126899b20c65ff3ea23a3b7d7a67ab30b99dcAndrew Scull // in less predictable code. 8038fbdd244013aad0808220386eb6bd17eb0635dd4cJan Voung // 803957e126899b20c65ff3ea23a3b7d7a67ab30b99dcAndrew Scull // If in the future the implementation is changed to lower undef values to 804057e126899b20c65ff3ea23a3b7d7a67ab30b99dcAndrew Scull // uninitialized registers, a FakeDef will be needed: 80411d937a8e35b7ae51ee64c32f8f34d51ba33498f4John Porto // Context.insert<InstFakeDef>(Reg); 8042fbdd244013aad0808220386eb6bd17eb0635dd4cJan Voung // This is in order to ensure that the live range of Reg is not 804357e126899b20c65ff3ea23a3b7d7a67ab30b99dcAndrew Scull // overestimated. If the constant being lowered is a 64 bit value, then 804457e126899b20c65ff3ea23a3b7d7a67ab30b99dcAndrew Scull // the result should be split and the lo and hi components will need to go 804557e126899b20c65ff3ea23a3b7d7a67ab30b99dcAndrew Scull // in uninitialized registers. 8046fbdd244013aad0808220386eb6bd17eb0635dd4cJan Voung if (isVectorType(Ty)) 8047fbdd244013aad0808220386eb6bd17eb0635dd4cJan Voung return makeVectorOfZeros(Ty, RegNum); 8048fbdd244013aad0808220386eb6bd17eb0635dd4cJan Voung return Ctx->getConstantZero(Ty); 8049fbdd244013aad0808220386eb6bd17eb0635dd4cJan Voung } 8050fbdd244013aad0808220386eb6bd17eb0635dd4cJan Voung return From; 8051fbdd244013aad0808220386eb6bd17eb0635dd4cJan Voung} 8052fbdd244013aad0808220386eb6bd17eb0635dd4cJan Voung 805357e126899b20c65ff3ea23a3b7d7a67ab30b99dcAndrew Scull/// For the cmp instruction, if Src1 is an immediate, or known to be a physical 805457e126899b20c65ff3ea23a3b7d7a67ab30b99dcAndrew Scull/// register, we can allow Src0 to be a memory operand. Otherwise, Src0 must be 805557e126899b20c65ff3ea23a3b7d7a67ab30b99dcAndrew Scull/// copied into a physical register. (Actually, either Src0 or Src1 can be 805657e126899b20c65ff3ea23a3b7d7a67ab30b99dcAndrew Scull/// chosen for the physical register, but unfortunately we have to commit to one 805757e126899b20c65ff3ea23a3b7d7a67ab30b99dcAndrew Scull/// or the other before register allocation.) 80584a56686b5b56db6803f90ad53514bf2fa190d9f7John Portotemplate <typename TraitsType> 80594a56686b5b56db6803f90ad53514bf2fa190d9f7John PortoOperand *TargetX86Base<TraitsType>::legalizeSrc0ForCmp(Operand *Src0, 80604a56686b5b56db6803f90ad53514bf2fa190d9f7John Porto Operand *Src1) { 80617e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto bool IsSrc1ImmOrReg = false; 80627e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto if (llvm::isa<Constant>(Src1)) { 80637e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto IsSrc1ImmOrReg = true; 8064fbdd244013aad0808220386eb6bd17eb0635dd4cJan Voung } else if (auto *Var = llvm::dyn_cast<Variable>(Src1)) { 80657e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto if (Var->hasReg()) 80667e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto IsSrc1ImmOrReg = true; 80677e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto } 80687e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto return legalize(Src0, IsSrc1ImmOrReg ? (Legal_Reg | Legal_Mem) : Legal_Reg); 80697e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto} 80707e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto 80714a56686b5b56db6803f90ad53514bf2fa190d9f7John Portotemplate <typename TraitsType> 80724a56686b5b56db6803f90ad53514bf2fa190d9f7John Portotypename TargetX86Base<TraitsType>::X86OperandMem * 80734a56686b5b56db6803f90ad53514bf2fa190d9f7John PortoTargetX86Base<TraitsType>::formMemoryOperand(Operand *Opnd, Type Ty, 80744a56686b5b56db6803f90ad53514bf2fa190d9f7John Porto bool DoLegalize) { 80754a56686b5b56db6803f90ad53514bf2fa190d9f7John Porto auto *Mem = llvm::dyn_cast<X86OperandMem>(Opnd); 8076921856d4e4bd4f0deedc7324d5f6a4ca0be3439fJohn Porto // It may be the case that address mode optimization already creates an 80774a56686b5b56db6803f90ad53514bf2fa190d9f7John Porto // X86OperandMem, so in that case it wouldn't need another level of 8078921856d4e4bd4f0deedc7324d5f6a4ca0be3439fJohn Porto // transformation. 80797e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto if (!Mem) { 808054f3d518805b04b3f2958b7906a7d07dedb4e1aeJim Stichnoth auto *Base = llvm::dyn_cast<Variable>(Opnd); 808154f3d518805b04b3f2958b7906a7d07dedb4e1aeJim Stichnoth auto *Offset = llvm::dyn_cast<Constant>(Opnd); 80827e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto assert(Base || Offset); 80837e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto if (Offset) { 808457e126899b20c65ff3ea23a3b7d7a67ab30b99dcAndrew Scull // During memory operand building, we do not blind or pool the constant 808557e126899b20c65ff3ea23a3b7d7a67ab30b99dcAndrew Scull // offset, we will work on the whole memory operand later as one entity 808657e126899b20c65ff3ea23a3b7d7a67ab30b99dcAndrew Scull // later, this save one instruction. By turning blinding and pooling off, 808757e126899b20c65ff3ea23a3b7d7a67ab30b99dcAndrew Scull // we guarantee legalize(Offset) will return a Constant*. 80888ff4b2819944bc4f02fb29204a1fa5ba7dea5682Jim Stichnoth if (!llvm::isa<ConstantRelocatable>(Offset)) { 80897e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto BoolFlagSaver B(RandomizationPoolingPaused, true); 80907e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto 80917e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto Offset = llvm::cast<Constant>(legalize(Offset)); 80927e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto } 80937e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto 80947e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto assert(llvm::isa<ConstantInteger32>(Offset) || 80957e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto llvm::isa<ConstantRelocatable>(Offset)); 80967e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto } 809756958cb33d3c1d045f2844408d825442d523f59fJohn Porto // Not completely sure whether it's OK to leave IsRebased unset when 809856958cb33d3c1d045f2844408d825442d523f59fJohn Porto // creating the mem operand. If DoLegalize is true, it will definitely be 809956958cb33d3c1d045f2844408d825442d523f59fJohn Porto // applied during the legalize() call, but perhaps not during the 81008ff4b2819944bc4f02fb29204a1fa5ba7dea5682Jim Stichnoth // randomizeOrPoolImmediate() call. In any case, the emit routines will 81018ff4b2819944bc4f02fb29204a1fa5ba7dea5682Jim Stichnoth // assert that PIC legalization has been applied. 81024a56686b5b56db6803f90ad53514bf2fa190d9f7John Porto Mem = X86OperandMem::create(Func, Ty, Base, Offset); 81037e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto } 810457e126899b20c65ff3ea23a3b7d7a67ab30b99dcAndrew Scull // Do legalization, which contains randomization/pooling or do 810557e126899b20c65ff3ea23a3b7d7a67ab30b99dcAndrew Scull // randomization/pooling. 81064a56686b5b56db6803f90ad53514bf2fa190d9f7John Porto return llvm::cast<X86OperandMem>(DoLegalize ? legalize(Mem) 81074a56686b5b56db6803f90ad53514bf2fa190d9f7John Porto : randomizeOrPoolImmediate(Mem)); 81087e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto} 81097e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto 81104a56686b5b56db6803f90ad53514bf2fa190d9f7John Portotemplate <typename TraitsType> 81118aa396610b7baf728631a43ea16ad3d13e38397aJim StichnothVariable *TargetX86Base<TraitsType>::makeReg(Type Type, RegNumT RegNum) { 81127e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto // There aren't any 64-bit integer registers for x86-32. 81131d235425dab1f3dd059973fc53f1b1d5879469e3John Porto assert(Traits::Is64Bit || Type != IceType_i64); 81145aeed955e17bac8cc44cc6d2b6ff7513cc714c2fJohn Porto Variable *Reg = Func->makeVariable(Type); 81155fa0a5f7f730f33e6987527a261de8d833d7585cReed Kotler if (RegNum.hasValue()) 81167e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto Reg->setRegNum(RegNum); 81175fa0a5f7f730f33e6987527a261de8d833d7585cReed Kotler else 81185fa0a5f7f730f33e6987527a261de8d833d7585cReed Kotler Reg->setMustHaveReg(); 81197e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto return Reg; 81207e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto} 81217e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto 81220e137b2e9d2341c741bca8968a65646c1a3f547cNicolas Capensconst Type TypeForSize[] = {IceType_i8, IceType_i16, IceType_i32, IceType_f64, 81230e137b2e9d2341c741bca8968a65646c1a3f547cNicolas Capens IceType_v16i8}; 81240e137b2e9d2341c741bca8968a65646c1a3f547cNicolas Capens 81254a56686b5b56db6803f90ad53514bf2fa190d9f7John Portotemplate <typename TraitsType> 81264a56686b5b56db6803f90ad53514bf2fa190d9f7John PortoType TargetX86Base<TraitsType>::largestTypeInSize(uint32_t Size, 81274a56686b5b56db6803f90ad53514bf2fa190d9f7John Porto uint32_t MaxSize) { 8128cfa628b5f7612d202de234980b21bd6d59a11998Andrew Scull assert(Size != 0); 8129cfa628b5f7612d202de234980b21bd6d59a11998Andrew Scull uint32_t TyIndex = llvm::findLastSet(Size, llvm::ZB_Undefined); 8130cfa628b5f7612d202de234980b21bd6d59a11998Andrew Scull uint32_t MaxIndex = MaxSize == NoSizeLimit 8131cfa628b5f7612d202de234980b21bd6d59a11998Andrew Scull ? llvm::array_lengthof(TypeForSize) - 1 8132cfa628b5f7612d202de234980b21bd6d59a11998Andrew Scull : llvm::findLastSet(MaxSize, llvm::ZB_Undefined); 8133cfa628b5f7612d202de234980b21bd6d59a11998Andrew Scull return TypeForSize[std::min(TyIndex, MaxIndex)]; 8134cfa628b5f7612d202de234980b21bd6d59a11998Andrew Scull} 8135cfa628b5f7612d202de234980b21bd6d59a11998Andrew Scull 81364a56686b5b56db6803f90ad53514bf2fa190d9f7John Portotemplate <typename TraitsType> 81374a56686b5b56db6803f90ad53514bf2fa190d9f7John PortoType TargetX86Base<TraitsType>::firstTypeThatFitsSize(uint32_t Size, 81384a56686b5b56db6803f90ad53514bf2fa190d9f7John Porto uint32_t MaxSize) { 8139cfa628b5f7612d202de234980b21bd6d59a11998Andrew Scull assert(Size != 0); 8140cfa628b5f7612d202de234980b21bd6d59a11998Andrew Scull uint32_t TyIndex = llvm::findLastSet(Size, llvm::ZB_Undefined); 8141cfa628b5f7612d202de234980b21bd6d59a11998Andrew Scull if (!llvm::isPowerOf2_32(Size)) 8142cfa628b5f7612d202de234980b21bd6d59a11998Andrew Scull ++TyIndex; 8143cfa628b5f7612d202de234980b21bd6d59a11998Andrew Scull uint32_t MaxIndex = MaxSize == NoSizeLimit 8144cfa628b5f7612d202de234980b21bd6d59a11998Andrew Scull ? llvm::array_lengthof(TypeForSize) - 1 8145cfa628b5f7612d202de234980b21bd6d59a11998Andrew Scull : llvm::findLastSet(MaxSize, llvm::ZB_Undefined); 8146cfa628b5f7612d202de234980b21bd6d59a11998Andrew Scull return TypeForSize[std::min(TyIndex, MaxIndex)]; 8147cfa628b5f7612d202de234980b21bd6d59a11998Andrew Scull} 8148cfa628b5f7612d202de234980b21bd6d59a11998Andrew Scull 81494a56686b5b56db6803f90ad53514bf2fa190d9f7John Portotemplate <typename TraitsType> void TargetX86Base<TraitsType>::postLower() { 8150dd6dcfaf765dc93ae64ec45d623106f4b3a3c13aJim Stichnoth if (Func->getOptLevel() == Opt_m1) 81517e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto return; 8152230d4101fb3c591d044406eef27d0ce43ffab53eJim Stichnoth markRedefinitions(); 8153318f4cdaa21eac5ef1d16731e51cd7adb3083d3bJim Stichnoth Context.availabilityUpdate(); 81547e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto} 81557e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto 81564a56686b5b56db6803f90ad53514bf2fa190d9f7John Portotemplate <typename TraitsType> 81574a56686b5b56db6803f90ad53514bf2fa190d9f7John Portovoid TargetX86Base<TraitsType>::makeRandomRegisterPermutation( 81588aa396610b7baf728631a43ea16ad3d13e38397aJim Stichnoth llvm::SmallVectorImpl<RegNumT> &Permutation, 8159e82b560e649f8a68bcb252b9b002708e74d962d3John Porto const SmallBitVector &ExcludeRegisters, uint64_t Salt) const { 8160d46999474d2b4a388e1d8a7c71f06cd4cec51bfcKarl Schimpf Traits::makeRandomRegisterPermutation(Func, Permutation, ExcludeRegisters, 8161d46999474d2b4a388e1d8a7c71f06cd4cec51bfcKarl Schimpf Salt); 81627e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto} 81637e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto 81644a56686b5b56db6803f90ad53514bf2fa190d9f7John Portotemplate <typename TraitsType> 81654a56686b5b56db6803f90ad53514bf2fa190d9f7John Portovoid TargetX86Base<TraitsType>::emit(const ConstantInteger32 *C) const { 816620b71f5890ee8651983b126c5978594a01e0af96Jim Stichnoth if (!BuildDefs::dump()) 81677e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto return; 81687e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto Ostream &Str = Ctx->getStrEmit(); 81698ff4b2819944bc4f02fb29204a1fa5ba7dea5682Jim Stichnoth Str << "$" << C->getValue(); 81707e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto} 81717e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto 81724a56686b5b56db6803f90ad53514bf2fa190d9f7John Portotemplate <typename TraitsType> 81734a56686b5b56db6803f90ad53514bf2fa190d9f7John Portovoid TargetX86Base<TraitsType>::emit(const ConstantInteger64 *C) const { 81741d235425dab1f3dd059973fc53f1b1d5879469e3John Porto if (!Traits::Is64Bit) { 81751d235425dab1f3dd059973fc53f1b1d5879469e3John Porto llvm::report_fatal_error("Not expecting to emit 64-bit integers"); 81761d235425dab1f3dd059973fc53f1b1d5879469e3John Porto } else { 81771d235425dab1f3dd059973fc53f1b1d5879469e3John Porto if (!BuildDefs::dump()) 81781d235425dab1f3dd059973fc53f1b1d5879469e3John Porto return; 81791d235425dab1f3dd059973fc53f1b1d5879469e3John Porto Ostream &Str = Ctx->getStrEmit(); 81808ff4b2819944bc4f02fb29204a1fa5ba7dea5682Jim Stichnoth Str << "$" << C->getValue(); 81811d235425dab1f3dd059973fc53f1b1d5879469e3John Porto } 81827e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto} 81837e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto 81844a56686b5b56db6803f90ad53514bf2fa190d9f7John Portotemplate <typename TraitsType> 81854a56686b5b56db6803f90ad53514bf2fa190d9f7John Portovoid TargetX86Base<TraitsType>::emit(const ConstantFloat *C) const { 818620b71f5890ee8651983b126c5978594a01e0af96Jim Stichnoth if (!BuildDefs::dump()) 81877e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto return; 81887e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto Ostream &Str = Ctx->getStrEmit(); 8189467ffe51bebcb3ae3e3ce745c38fc20e8837c31cJim Stichnoth Str << C->getLabelName(); 81907e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto} 81917e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto 81924a56686b5b56db6803f90ad53514bf2fa190d9f7John Portotemplate <typename TraitsType> 81934a56686b5b56db6803f90ad53514bf2fa190d9f7John Portovoid TargetX86Base<TraitsType>::emit(const ConstantDouble *C) const { 819420b71f5890ee8651983b126c5978594a01e0af96Jim Stichnoth if (!BuildDefs::dump()) 81957e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto return; 81967e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto Ostream &Str = Ctx->getStrEmit(); 8197467ffe51bebcb3ae3e3ce745c38fc20e8837c31cJim Stichnoth Str << C->getLabelName(); 81987e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto} 81997e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto 82004a56686b5b56db6803f90ad53514bf2fa190d9f7John Portotemplate <typename TraitsType> 82014a56686b5b56db6803f90ad53514bf2fa190d9f7John Portovoid TargetX86Base<TraitsType>::emit(const ConstantUndef *) const { 82027e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto llvm::report_fatal_error("undef value encountered by emitter."); 82037e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto} 82047e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto 82058ff4b2819944bc4f02fb29204a1fa5ba7dea5682Jim Stichnothtemplate <class Machine> 82068ff4b2819944bc4f02fb29204a1fa5ba7dea5682Jim Stichnothvoid TargetX86Base<Machine>::emit(const ConstantRelocatable *C) const { 82078ff4b2819944bc4f02fb29204a1fa5ba7dea5682Jim Stichnoth if (!BuildDefs::dump()) 82088ff4b2819944bc4f02fb29204a1fa5ba7dea5682Jim Stichnoth return; 8209d46999474d2b4a388e1d8a7c71f06cd4cec51bfcKarl Schimpf assert(!getFlags().getUseNonsfi() || 8210467ffe51bebcb3ae3e3ce745c38fc20e8837c31cJim Stichnoth C->getName().toString() == GlobalOffsetTable); 82118ff4b2819944bc4f02fb29204a1fa5ba7dea5682Jim Stichnoth Ostream &Str = Ctx->getStrEmit(); 82128ff4b2819944bc4f02fb29204a1fa5ba7dea5682Jim Stichnoth Str << "$"; 82138ff4b2819944bc4f02fb29204a1fa5ba7dea5682Jim Stichnoth emitWithoutPrefix(C); 82148ff4b2819944bc4f02fb29204a1fa5ba7dea5682Jim Stichnoth} 82158ff4b2819944bc4f02fb29204a1fa5ba7dea5682Jim Stichnoth 82169612d32c7e5eb2cb403686ef31172d42e075e460Andrew Scull/// Randomize or pool an Immediate. 82174a56686b5b56db6803f90ad53514bf2fa190d9f7John Portotemplate <typename TraitsType> 82184a56686b5b56db6803f90ad53514bf2fa190d9f7John PortoOperand * 82194a56686b5b56db6803f90ad53514bf2fa190d9f7John PortoTargetX86Base<TraitsType>::randomizeOrPoolImmediate(Constant *Immediate, 82208aa396610b7baf728631a43ea16ad3d13e38397aJim Stichnoth RegNumT RegNum) { 82217e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto assert(llvm::isa<ConstantInteger32>(Immediate) || 82227e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto llvm::isa<ConstantRelocatable>(Immediate)); 8223d46999474d2b4a388e1d8a7c71f06cd4cec51bfcKarl Schimpf if (getFlags().getRandomizeAndPoolImmediatesOption() == RPI_None || 82247e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto RandomizationPoolingPaused == true) { 82257e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto // Immediates randomization/pooling off or paused 82267e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto return Immediate; 82277e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto } 822856958cb33d3c1d045f2844408d825442d523f59fJohn Porto 822956958cb33d3c1d045f2844408d825442d523f59fJohn Porto if (Traits::Is64Bit && NeedSandboxing) { 823056958cb33d3c1d045f2844408d825442d523f59fJohn Porto // Immediate randomization/pooling is currently disabled for x86-64 823156958cb33d3c1d045f2844408d825442d523f59fJohn Porto // sandboxing for it could generate invalid memory operands. 823256958cb33d3c1d045f2844408d825442d523f59fJohn Porto assert(false && 823356958cb33d3c1d045f2844408d825442d523f59fJohn Porto "Constant pooling/randomization is disabled for x8664 sandbox."); 823456958cb33d3c1d045f2844408d825442d523f59fJohn Porto return Immediate; 823556958cb33d3c1d045f2844408d825442d523f59fJohn Porto } 823656958cb33d3c1d045f2844408d825442d523f59fJohn Porto 8237467ffe51bebcb3ae3e3ce745c38fc20e8837c31cJim Stichnoth if (!Immediate->shouldBeRandomizedOrPooled()) { 823856958cb33d3c1d045f2844408d825442d523f59fJohn Porto // the constant Immediate is not eligible for blinding/pooling 823956958cb33d3c1d045f2844408d825442d523f59fJohn Porto return Immediate; 824056958cb33d3c1d045f2844408d825442d523f59fJohn Porto } 824156958cb33d3c1d045f2844408d825442d523f59fJohn Porto Ctx->statsUpdateRPImms(); 8242d46999474d2b4a388e1d8a7c71f06cd4cec51bfcKarl Schimpf switch (getFlags().getRandomizeAndPoolImmediatesOption()) { 824356958cb33d3c1d045f2844408d825442d523f59fJohn Porto default: 824456958cb33d3c1d045f2844408d825442d523f59fJohn Porto llvm::report_fatal_error("Unsupported -randomize-pool-immediates option"); 824556958cb33d3c1d045f2844408d825442d523f59fJohn Porto case RPI_Randomize: { 824656958cb33d3c1d045f2844408d825442d523f59fJohn Porto // blind the constant 824756958cb33d3c1d045f2844408d825442d523f59fJohn Porto // FROM: 824856958cb33d3c1d045f2844408d825442d523f59fJohn Porto // imm 824956958cb33d3c1d045f2844408d825442d523f59fJohn Porto // TO: 825056958cb33d3c1d045f2844408d825442d523f59fJohn Porto // insert: mov imm+cookie, Reg 825156958cb33d3c1d045f2844408d825442d523f59fJohn Porto // insert: lea -cookie[Reg], Reg 825256958cb33d3c1d045f2844408d825442d523f59fJohn Porto // => Reg 825356958cb33d3c1d045f2844408d825442d523f59fJohn Porto // If we have already assigned a phy register, we must come from 825456958cb33d3c1d045f2844408d825442d523f59fJohn Porto // advancedPhiLowering()=>lowerAssign(). In this case we should reuse the 825556958cb33d3c1d045f2844408d825442d523f59fJohn Porto // assigned register as this assignment is that start of its use-def 825656958cb33d3c1d045f2844408d825442d523f59fJohn Porto // chain. So we add RegNum argument here. Note we use 'lea' instruction 825756958cb33d3c1d045f2844408d825442d523f59fJohn Porto // instead of 'xor' to avoid affecting the flags. 825856958cb33d3c1d045f2844408d825442d523f59fJohn Porto Variable *Reg = makeReg(IceType_i32, RegNum); 825956958cb33d3c1d045f2844408d825442d523f59fJohn Porto auto *Integer = llvm::cast<ConstantInteger32>(Immediate); 826056958cb33d3c1d045f2844408d825442d523f59fJohn Porto uint32_t Value = Integer->getValue(); 826156958cb33d3c1d045f2844408d825442d523f59fJohn Porto uint32_t Cookie = Func->getConstantBlindingCookie(); 826256958cb33d3c1d045f2844408d825442d523f59fJohn Porto _mov(Reg, Ctx->getConstantInt(IceType_i32, Cookie + Value)); 826356958cb33d3c1d045f2844408d825442d523f59fJohn Porto Constant *Offset = Ctx->getConstantInt(IceType_i32, 0 - Cookie); 826456958cb33d3c1d045f2844408d825442d523f59fJohn Porto _lea(Reg, X86OperandMem::create(Func, IceType_i32, Reg, Offset)); 826556958cb33d3c1d045f2844408d825442d523f59fJohn Porto if (Immediate->getType() == IceType_i32) { 82667e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto return Reg; 82677e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto } 826856958cb33d3c1d045f2844408d825442d523f59fJohn Porto Variable *TruncReg = makeReg(Immediate->getType(), RegNum); 826956958cb33d3c1d045f2844408d825442d523f59fJohn Porto _mov(TruncReg, Reg); 827056958cb33d3c1d045f2844408d825442d523f59fJohn Porto return TruncReg; 827156958cb33d3c1d045f2844408d825442d523f59fJohn Porto } 827256958cb33d3c1d045f2844408d825442d523f59fJohn Porto case RPI_Pool: { 827356958cb33d3c1d045f2844408d825442d523f59fJohn Porto // pool the constant 827456958cb33d3c1d045f2844408d825442d523f59fJohn Porto // FROM: 827556958cb33d3c1d045f2844408d825442d523f59fJohn Porto // imm 827656958cb33d3c1d045f2844408d825442d523f59fJohn Porto // TO: 827756958cb33d3c1d045f2844408d825442d523f59fJohn Porto // insert: mov $label, Reg 827856958cb33d3c1d045f2844408d825442d523f59fJohn Porto // => Reg 8279d46999474d2b4a388e1d8a7c71f06cd4cec51bfcKarl Schimpf assert(getFlags().getRandomizeAndPoolImmediatesOption() == RPI_Pool); 8280467ffe51bebcb3ae3e3ce745c38fc20e8837c31cJim Stichnoth assert(Immediate->getShouldBePooled()); 828156958cb33d3c1d045f2844408d825442d523f59fJohn Porto // if we have already assigned a phy register, we must come from 828256958cb33d3c1d045f2844408d825442d523f59fJohn Porto // advancedPhiLowering()=>lowerAssign(). In this case we should reuse the 828356958cb33d3c1d045f2844408d825442d523f59fJohn Porto // assigned register as this assignment is that start of its use-def 828456958cb33d3c1d045f2844408d825442d523f59fJohn Porto // chain. So we add RegNum argument here. 828556958cb33d3c1d045f2844408d825442d523f59fJohn Porto Variable *Reg = makeReg(Immediate->getType(), RegNum); 828656958cb33d3c1d045f2844408d825442d523f59fJohn Porto constexpr RelocOffsetT Offset = 0; 8287467ffe51bebcb3ae3e3ce745c38fc20e8837c31cJim Stichnoth Constant *Symbol = Ctx->getConstantSym(Offset, Immediate->getLabelName()); 8288ac2388c385afda17acb3fc242b17b9232ec2262aJohn Porto constexpr Variable *NoBase = nullptr; 8289ac2388c385afda17acb3fc242b17b9232ec2262aJohn Porto X86OperandMem *MemOperand = 8290ac2388c385afda17acb3fc242b17b9232ec2262aJohn Porto X86OperandMem::create(Func, Immediate->getType(), NoBase, Symbol); 829156958cb33d3c1d045f2844408d825442d523f59fJohn Porto _mov(Reg, MemOperand); 829256958cb33d3c1d045f2844408d825442d523f59fJohn Porto return Reg; 829356958cb33d3c1d045f2844408d825442d523f59fJohn Porto } 82947e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto } 82957e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto} 82967e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto 82974a56686b5b56db6803f90ad53514bf2fa190d9f7John Portotemplate <typename TraitsType> 82984a56686b5b56db6803f90ad53514bf2fa190d9f7John Portotypename TargetX86Base<TraitsType>::X86OperandMem * 82994a56686b5b56db6803f90ad53514bf2fa190d9f7John PortoTargetX86Base<TraitsType>::randomizeOrPoolImmediate(X86OperandMem *MemOperand, 83008aa396610b7baf728631a43ea16ad3d13e38397aJim Stichnoth RegNumT RegNum) { 83017e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto assert(MemOperand); 8302d46999474d2b4a388e1d8a7c71f06cd4cec51bfcKarl Schimpf if (getFlags().getRandomizeAndPoolImmediatesOption() == RPI_None || 83037e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto RandomizationPoolingPaused == true) { 83047e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto // immediates randomization/pooling is turned off 83057e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto return MemOperand; 83067e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto } 83077e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto 830856958cb33d3c1d045f2844408d825442d523f59fJohn Porto if (Traits::Is64Bit && NeedSandboxing) { 830956958cb33d3c1d045f2844408d825442d523f59fJohn Porto // Immediate randomization/pooling is currently disabled for x86-64 831056958cb33d3c1d045f2844408d825442d523f59fJohn Porto // sandboxing for it could generate invalid memory operands. 831156958cb33d3c1d045f2844408d825442d523f59fJohn Porto assert(false && 831256958cb33d3c1d045f2844408d825442d523f59fJohn Porto "Constant pooling/randomization is disabled for x8664 sandbox."); 831356958cb33d3c1d045f2844408d825442d523f59fJohn Porto return MemOperand; 831456958cb33d3c1d045f2844408d825442d523f59fJohn Porto } 831556958cb33d3c1d045f2844408d825442d523f59fJohn Porto 831657e126899b20c65ff3ea23a3b7d7a67ab30b99dcAndrew Scull // If this memory operand is already a randomized one, we do not randomize it 831757e126899b20c65ff3ea23a3b7d7a67ab30b99dcAndrew Scull // again. 83187e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto if (MemOperand->getRandomized()) 83197e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto return MemOperand; 83207e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto 832156958cb33d3c1d045f2844408d825442d523f59fJohn Porto auto *C = llvm::dyn_cast_or_null<Constant>(MemOperand->getOffset()); 832256958cb33d3c1d045f2844408d825442d523f59fJohn Porto 832356958cb33d3c1d045f2844408d825442d523f59fJohn Porto if (C == nullptr) { 832456958cb33d3c1d045f2844408d825442d523f59fJohn Porto return MemOperand; 83257e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto } 83267e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto 8327467ffe51bebcb3ae3e3ce745c38fc20e8837c31cJim Stichnoth if (!C->shouldBeRandomizedOrPooled()) { 832856958cb33d3c1d045f2844408d825442d523f59fJohn Porto return MemOperand; 832956958cb33d3c1d045f2844408d825442d523f59fJohn Porto } 833056958cb33d3c1d045f2844408d825442d523f59fJohn Porto 833156958cb33d3c1d045f2844408d825442d523f59fJohn Porto // The offset of this mem operand should be blinded or pooled 833256958cb33d3c1d045f2844408d825442d523f59fJohn Porto Ctx->statsUpdateRPImms(); 8333d46999474d2b4a388e1d8a7c71f06cd4cec51bfcKarl Schimpf switch (getFlags().getRandomizeAndPoolImmediatesOption()) { 833456958cb33d3c1d045f2844408d825442d523f59fJohn Porto default: 833556958cb33d3c1d045f2844408d825442d523f59fJohn Porto llvm::report_fatal_error("Unsupported -randomize-pool-immediates option"); 833656958cb33d3c1d045f2844408d825442d523f59fJohn Porto case RPI_Randomize: { 833756958cb33d3c1d045f2844408d825442d523f59fJohn Porto // blind the constant offset 833856958cb33d3c1d045f2844408d825442d523f59fJohn Porto // FROM: 833956958cb33d3c1d045f2844408d825442d523f59fJohn Porto // offset[base, index, shift] 834056958cb33d3c1d045f2844408d825442d523f59fJohn Porto // TO: 834156958cb33d3c1d045f2844408d825442d523f59fJohn Porto // insert: lea offset+cookie[base], RegTemp 834256958cb33d3c1d045f2844408d825442d523f59fJohn Porto // => -cookie[RegTemp, index, shift] 834356958cb33d3c1d045f2844408d825442d523f59fJohn Porto uint32_t Value = 834456958cb33d3c1d045f2844408d825442d523f59fJohn Porto llvm::dyn_cast<ConstantInteger32>(MemOperand->getOffset())->getValue(); 834556958cb33d3c1d045f2844408d825442d523f59fJohn Porto uint32_t Cookie = Func->getConstantBlindingCookie(); 834656958cb33d3c1d045f2844408d825442d523f59fJohn Porto Constant *Mask1 = 834756958cb33d3c1d045f2844408d825442d523f59fJohn Porto Ctx->getConstantInt(MemOperand->getOffset()->getType(), Cookie + Value); 834856958cb33d3c1d045f2844408d825442d523f59fJohn Porto Constant *Mask2 = 834956958cb33d3c1d045f2844408d825442d523f59fJohn Porto Ctx->getConstantInt(MemOperand->getOffset()->getType(), 0 - Cookie); 835056958cb33d3c1d045f2844408d825442d523f59fJohn Porto 835156958cb33d3c1d045f2844408d825442d523f59fJohn Porto X86OperandMem *TempMemOperand = X86OperandMem::create( 835256958cb33d3c1d045f2844408d825442d523f59fJohn Porto Func, MemOperand->getType(), MemOperand->getBase(), Mask1); 835356958cb33d3c1d045f2844408d825442d523f59fJohn Porto // If we have already assigned a physical register, we must come from 835456958cb33d3c1d045f2844408d825442d523f59fJohn Porto // advancedPhiLowering()=>lowerAssign(). In this case we should reuse 835556958cb33d3c1d045f2844408d825442d523f59fJohn Porto // the assigned register as this assignment is that start of its 835656958cb33d3c1d045f2844408d825442d523f59fJohn Porto // use-def chain. So we add RegNum argument here. 835756958cb33d3c1d045f2844408d825442d523f59fJohn Porto Variable *RegTemp = makeReg(MemOperand->getOffset()->getType(), RegNum); 835856958cb33d3c1d045f2844408d825442d523f59fJohn Porto _lea(RegTemp, TempMemOperand); 835956958cb33d3c1d045f2844408d825442d523f59fJohn Porto 836056958cb33d3c1d045f2844408d825442d523f59fJohn Porto X86OperandMem *NewMemOperand = X86OperandMem::create( 836156958cb33d3c1d045f2844408d825442d523f59fJohn Porto Func, MemOperand->getType(), RegTemp, Mask2, MemOperand->getIndex(), 836256958cb33d3c1d045f2844408d825442d523f59fJohn Porto MemOperand->getShift(), MemOperand->getSegmentRegister(), 836356958cb33d3c1d045f2844408d825442d523f59fJohn Porto MemOperand->getIsRebased()); 836456958cb33d3c1d045f2844408d825442d523f59fJohn Porto 836556958cb33d3c1d045f2844408d825442d523f59fJohn Porto // Label this memory operand as randomized, so we won't randomize it 836656958cb33d3c1d045f2844408d825442d523f59fJohn Porto // again in case we call legalize() multiple times on this memory 836756958cb33d3c1d045f2844408d825442d523f59fJohn Porto // operand. 836856958cb33d3c1d045f2844408d825442d523f59fJohn Porto NewMemOperand->setRandomized(true); 836956958cb33d3c1d045f2844408d825442d523f59fJohn Porto return NewMemOperand; 837056958cb33d3c1d045f2844408d825442d523f59fJohn Porto } 837156958cb33d3c1d045f2844408d825442d523f59fJohn Porto case RPI_Pool: { 837256958cb33d3c1d045f2844408d825442d523f59fJohn Porto // pool the constant offset 837356958cb33d3c1d045f2844408d825442d523f59fJohn Porto // FROM: 837456958cb33d3c1d045f2844408d825442d523f59fJohn Porto // offset[base, index, shift] 837556958cb33d3c1d045f2844408d825442d523f59fJohn Porto // TO: 837656958cb33d3c1d045f2844408d825442d523f59fJohn Porto // insert: mov $label, RegTemp 837756958cb33d3c1d045f2844408d825442d523f59fJohn Porto // insert: lea [base, RegTemp], RegTemp 837856958cb33d3c1d045f2844408d825442d523f59fJohn Porto // =>[RegTemp, index, shift] 837956958cb33d3c1d045f2844408d825442d523f59fJohn Porto 838056958cb33d3c1d045f2844408d825442d523f59fJohn Porto // Memory operand should never exist as source operands in phi lowering 838156958cb33d3c1d045f2844408d825442d523f59fJohn Porto // assignments, so there is no need to reuse any registers here. For 838256958cb33d3c1d045f2844408d825442d523f59fJohn Porto // phi lowering, we should not ask for new physical registers in 838356958cb33d3c1d045f2844408d825442d523f59fJohn Porto // general. However, if we do meet Memory Operand during phi lowering, 838456958cb33d3c1d045f2844408d825442d523f59fJohn Porto // we should not blind or pool the immediates for now. 83855fa0a5f7f730f33e6987527a261de8d833d7585cReed Kotler if (RegNum.hasValue()) 838656958cb33d3c1d045f2844408d825442d523f59fJohn Porto return MemOperand; 838756958cb33d3c1d045f2844408d825442d523f59fJohn Porto Variable *RegTemp = makeReg(IceType_i32); 8388467ffe51bebcb3ae3e3ce745c38fc20e8837c31cJim Stichnoth assert(MemOperand->getOffset()->getShouldBePooled()); 838956958cb33d3c1d045f2844408d825442d523f59fJohn Porto constexpr RelocOffsetT SymOffset = 0; 8390467ffe51bebcb3ae3e3ce745c38fc20e8837c31cJim Stichnoth Constant *Symbol = 8391467ffe51bebcb3ae3e3ce745c38fc20e8837c31cJim Stichnoth Ctx->getConstantSym(SymOffset, MemOperand->getOffset()->getLabelName()); 8392ac2388c385afda17acb3fc242b17b9232ec2262aJohn Porto constexpr Variable *NoBase = nullptr; 839356958cb33d3c1d045f2844408d825442d523f59fJohn Porto X86OperandMem *SymbolOperand = X86OperandMem::create( 8394ac2388c385afda17acb3fc242b17b9232ec2262aJohn Porto Func, MemOperand->getOffset()->getType(), NoBase, Symbol); 839556958cb33d3c1d045f2844408d825442d523f59fJohn Porto _mov(RegTemp, SymbolOperand); 839656958cb33d3c1d045f2844408d825442d523f59fJohn Porto // If we have a base variable here, we should add the lea instruction 839756958cb33d3c1d045f2844408d825442d523f59fJohn Porto // to add the value of the base variable to RegTemp. If there is no 839856958cb33d3c1d045f2844408d825442d523f59fJohn Porto // base variable, we won't need this lea instruction. 839956958cb33d3c1d045f2844408d825442d523f59fJohn Porto if (MemOperand->getBase()) { 840056958cb33d3c1d045f2844408d825442d523f59fJohn Porto X86OperandMem *CalculateOperand = X86OperandMem::create( 840156958cb33d3c1d045f2844408d825442d523f59fJohn Porto Func, MemOperand->getType(), MemOperand->getBase(), nullptr, RegTemp, 840256958cb33d3c1d045f2844408d825442d523f59fJohn Porto 0, MemOperand->getSegmentRegister()); 840356958cb33d3c1d045f2844408d825442d523f59fJohn Porto _lea(RegTemp, CalculateOperand); 840456958cb33d3c1d045f2844408d825442d523f59fJohn Porto } 840556958cb33d3c1d045f2844408d825442d523f59fJohn Porto X86OperandMem *NewMemOperand = X86OperandMem::create( 840656958cb33d3c1d045f2844408d825442d523f59fJohn Porto Func, MemOperand->getType(), RegTemp, nullptr, MemOperand->getIndex(), 840756958cb33d3c1d045f2844408d825442d523f59fJohn Porto MemOperand->getShift(), MemOperand->getSegmentRegister()); 840856958cb33d3c1d045f2844408d825442d523f59fJohn Porto return NewMemOperand; 840956958cb33d3c1d045f2844408d825442d523f59fJohn Porto } 841056958cb33d3c1d045f2844408d825442d523f59fJohn Porto } 841156958cb33d3c1d045f2844408d825442d523f59fJohn Porto} 84126b80cf109e33aaf58de116b63a0562227d096429David Sehr 84136b80cf109e33aaf58de116b63a0562227d096429David Sehrtemplate <typename TraitsType> 84146b80cf109e33aaf58de116b63a0562227d096429David Sehrvoid TargetX86Base<TraitsType>::emitJumpTable( 8415030772114216c1a57c749050bb58d07de8ceaa7cJohn Porto const Cfg *, const InstJumpTable *JumpTable) const { 84166b80cf109e33aaf58de116b63a0562227d096429David Sehr if (!BuildDefs::dump()) 84176b80cf109e33aaf58de116b63a0562227d096429David Sehr return; 84186b80cf109e33aaf58de116b63a0562227d096429David Sehr Ostream &Str = Ctx->getStrEmit(); 8419d46999474d2b4a388e1d8a7c71f06cd4cec51bfcKarl Schimpf const bool UseNonsfi = getFlags().getUseNonsfi(); 8420467ffe51bebcb3ae3e3ce745c38fc20e8837c31cJim Stichnoth const char *Prefix = UseNonsfi ? ".data.rel.ro." : ".rodata."; 8421030772114216c1a57c749050bb58d07de8ceaa7cJohn Porto Str << "\t.section\t" << Prefix << JumpTable->getSectionName() 8422030772114216c1a57c749050bb58d07de8ceaa7cJohn Porto << ",\"a\",@progbits\n" 8423030772114216c1a57c749050bb58d07de8ceaa7cJohn Porto "\t.align\t" << typeWidthInBytes(getPointerType()) << "\n" 8424030772114216c1a57c749050bb58d07de8ceaa7cJohn Porto << JumpTable->getName() << ":"; 84256b80cf109e33aaf58de116b63a0562227d096429David Sehr 84266b80cf109e33aaf58de116b63a0562227d096429David Sehr // On X86 ILP32 pointers are 32-bit hence the use of .long 84276b80cf109e33aaf58de116b63a0562227d096429David Sehr for (SizeT I = 0; I < JumpTable->getNumTargets(); ++I) 84286b80cf109e33aaf58de116b63a0562227d096429David Sehr Str << "\n\t.long\t" << JumpTable->getTarget(I)->getAsmName(); 84296b80cf109e33aaf58de116b63a0562227d096429David Sehr Str << "\n"; 84306b80cf109e33aaf58de116b63a0562227d096429David Sehr} 84316b80cf109e33aaf58de116b63a0562227d096429David Sehr 84326b80cf109e33aaf58de116b63a0562227d096429David Sehrtemplate <typename TraitsType> 84336b80cf109e33aaf58de116b63a0562227d096429David Sehrtemplate <typename T> 84346b80cf109e33aaf58de116b63a0562227d096429David Sehrvoid TargetDataX86<TraitsType>::emitConstantPool(GlobalContext *Ctx) { 84356b80cf109e33aaf58de116b63a0562227d096429David Sehr if (!BuildDefs::dump()) 84366b80cf109e33aaf58de116b63a0562227d096429David Sehr return; 84376b80cf109e33aaf58de116b63a0562227d096429David Sehr Ostream &Str = Ctx->getStrEmit(); 84386b80cf109e33aaf58de116b63a0562227d096429David Sehr Type Ty = T::Ty; 84396b80cf109e33aaf58de116b63a0562227d096429David Sehr SizeT Align = typeAlignInBytes(Ty); 84406b80cf109e33aaf58de116b63a0562227d096429David Sehr ConstantList Pool = Ctx->getConstantPool(Ty); 84416b80cf109e33aaf58de116b63a0562227d096429David Sehr 84426b80cf109e33aaf58de116b63a0562227d096429David Sehr Str << "\t.section\t.rodata.cst" << Align << ",\"aM\",@progbits," << Align 84436b80cf109e33aaf58de116b63a0562227d096429David Sehr << "\n"; 84446b80cf109e33aaf58de116b63a0562227d096429David Sehr Str << "\t.align\t" << Align << "\n"; 84456b80cf109e33aaf58de116b63a0562227d096429David Sehr 84466b80cf109e33aaf58de116b63a0562227d096429David Sehr // If reorder-pooled-constants option is set to true, we need to shuffle the 84476b80cf109e33aaf58de116b63a0562227d096429David Sehr // constant pool before emitting it. 8448d46999474d2b4a388e1d8a7c71f06cd4cec51bfcKarl Schimpf if (getFlags().getReorderPooledConstants() && !Pool.empty()) { 84496b80cf109e33aaf58de116b63a0562227d096429David Sehr // Use the constant's kind value as the salt for creating random number 84506b80cf109e33aaf58de116b63a0562227d096429David Sehr // generator. 84516b80cf109e33aaf58de116b63a0562227d096429David Sehr Operand::OperandKind K = (*Pool.begin())->getKind(); 8452d46999474d2b4a388e1d8a7c71f06cd4cec51bfcKarl Schimpf RandomNumberGenerator RNG(getFlags().getRandomSeed(), 84536b80cf109e33aaf58de116b63a0562227d096429David Sehr RPE_PooledConstantReordering, K); 84546b80cf109e33aaf58de116b63a0562227d096429David Sehr RandomShuffle(Pool.begin(), Pool.end(), 84556b80cf109e33aaf58de116b63a0562227d096429David Sehr [&RNG](uint64_t N) { return (uint32_t)RNG.next(N); }); 84566b80cf109e33aaf58de116b63a0562227d096429David Sehr } 84576b80cf109e33aaf58de116b63a0562227d096429David Sehr 84586b80cf109e33aaf58de116b63a0562227d096429David Sehr for (Constant *C : Pool) { 84596b80cf109e33aaf58de116b63a0562227d096429David Sehr if (!C->getShouldBePooled()) 84606b80cf109e33aaf58de116b63a0562227d096429David Sehr continue; 84616b80cf109e33aaf58de116b63a0562227d096429David Sehr auto *Const = llvm::cast<typename T::IceType>(C); 84626b80cf109e33aaf58de116b63a0562227d096429David Sehr typename T::IceType::PrimType Value = Const->getValue(); 84636b80cf109e33aaf58de116b63a0562227d096429David Sehr // Use memcpy() to copy bits from Value into RawValue in a way that avoids 84646b80cf109e33aaf58de116b63a0562227d096429David Sehr // breaking strict-aliasing rules. 84656b80cf109e33aaf58de116b63a0562227d096429David Sehr typename T::PrimitiveIntType RawValue; 84666b80cf109e33aaf58de116b63a0562227d096429David Sehr memcpy(&RawValue, &Value, sizeof(Value)); 84676b80cf109e33aaf58de116b63a0562227d096429David Sehr char buf[30]; 84686b80cf109e33aaf58de116b63a0562227d096429David Sehr int CharsPrinted = 84696b80cf109e33aaf58de116b63a0562227d096429David Sehr snprintf(buf, llvm::array_lengthof(buf), T::PrintfString, RawValue); 84706b80cf109e33aaf58de116b63a0562227d096429David Sehr assert(CharsPrinted >= 0); 84716b80cf109e33aaf58de116b63a0562227d096429David Sehr assert((size_t)CharsPrinted < llvm::array_lengthof(buf)); 84726b80cf109e33aaf58de116b63a0562227d096429David Sehr (void)CharsPrinted; // avoid warnings if asserts are disabled 8473467ffe51bebcb3ae3e3ce745c38fc20e8837c31cJim Stichnoth Str << Const->getLabelName(); 84746b80cf109e33aaf58de116b63a0562227d096429David Sehr Str << ":\n\t" << T::AsmTag << "\t" << buf << "\t/* " << T::TypeName << " " 84756b80cf109e33aaf58de116b63a0562227d096429David Sehr << Value << " */\n"; 84766b80cf109e33aaf58de116b63a0562227d096429David Sehr } 84776b80cf109e33aaf58de116b63a0562227d096429David Sehr} 84786b80cf109e33aaf58de116b63a0562227d096429David Sehr 84796b80cf109e33aaf58de116b63a0562227d096429David Sehrtemplate <typename TraitsType> 84806b80cf109e33aaf58de116b63a0562227d096429David Sehrvoid TargetDataX86<TraitsType>::lowerConstants() { 8481d46999474d2b4a388e1d8a7c71f06cd4cec51bfcKarl Schimpf if (getFlags().getDisableTranslation()) 84826b80cf109e33aaf58de116b63a0562227d096429David Sehr return; 8483d46999474d2b4a388e1d8a7c71f06cd4cec51bfcKarl Schimpf switch (getFlags().getOutFileType()) { 84846b80cf109e33aaf58de116b63a0562227d096429David Sehr case FT_Elf: { 84856b80cf109e33aaf58de116b63a0562227d096429David Sehr ELFObjectWriter *Writer = Ctx->getObjectWriter(); 84866b80cf109e33aaf58de116b63a0562227d096429David Sehr 84876b80cf109e33aaf58de116b63a0562227d096429David Sehr Writer->writeConstantPool<ConstantInteger32>(IceType_i8); 84886b80cf109e33aaf58de116b63a0562227d096429David Sehr Writer->writeConstantPool<ConstantInteger32>(IceType_i16); 84896b80cf109e33aaf58de116b63a0562227d096429David Sehr Writer->writeConstantPool<ConstantInteger32>(IceType_i32); 84906b80cf109e33aaf58de116b63a0562227d096429David Sehr 84916b80cf109e33aaf58de116b63a0562227d096429David Sehr Writer->writeConstantPool<ConstantFloat>(IceType_f32); 84926b80cf109e33aaf58de116b63a0562227d096429David Sehr Writer->writeConstantPool<ConstantDouble>(IceType_f64); 84936b80cf109e33aaf58de116b63a0562227d096429David Sehr } break; 84946b80cf109e33aaf58de116b63a0562227d096429David Sehr case FT_Asm: 84956b80cf109e33aaf58de116b63a0562227d096429David Sehr case FT_Iasm: { 84966b80cf109e33aaf58de116b63a0562227d096429David Sehr OstreamLocker L(Ctx); 84976b80cf109e33aaf58de116b63a0562227d096429David Sehr 84986b80cf109e33aaf58de116b63a0562227d096429David Sehr emitConstantPool<PoolTypeConverter<uint8_t>>(Ctx); 84996b80cf109e33aaf58de116b63a0562227d096429David Sehr emitConstantPool<PoolTypeConverter<uint16_t>>(Ctx); 85006b80cf109e33aaf58de116b63a0562227d096429David Sehr emitConstantPool<PoolTypeConverter<uint32_t>>(Ctx); 85016b80cf109e33aaf58de116b63a0562227d096429David Sehr 85026b80cf109e33aaf58de116b63a0562227d096429David Sehr emitConstantPool<PoolTypeConverter<float>>(Ctx); 85036b80cf109e33aaf58de116b63a0562227d096429David Sehr emitConstantPool<PoolTypeConverter<double>>(Ctx); 85046b80cf109e33aaf58de116b63a0562227d096429David Sehr } break; 85056b80cf109e33aaf58de116b63a0562227d096429David Sehr } 85066b80cf109e33aaf58de116b63a0562227d096429David Sehr} 85076b80cf109e33aaf58de116b63a0562227d096429David Sehr 85086b80cf109e33aaf58de116b63a0562227d096429David Sehrtemplate <typename TraitsType> 85096b80cf109e33aaf58de116b63a0562227d096429David Sehrvoid TargetDataX86<TraitsType>::lowerJumpTables() { 8510d46999474d2b4a388e1d8a7c71f06cd4cec51bfcKarl Schimpf const bool IsPIC = getFlags().getUseNonsfi(); 8511d46999474d2b4a388e1d8a7c71f06cd4cec51bfcKarl Schimpf switch (getFlags().getOutFileType()) { 85126b80cf109e33aaf58de116b63a0562227d096429David Sehr case FT_Elf: { 85136b80cf109e33aaf58de116b63a0562227d096429David Sehr ELFObjectWriter *Writer = Ctx->getObjectWriter(); 851483425dec5ecae21e092a9a440845ce99a13ded69Nicolas Capens constexpr FixupKind FK_Abs64 = llvm::ELF::R_X86_64_64; 851583425dec5ecae21e092a9a440845ce99a13ded69Nicolas Capens const FixupKind RelocationKind = 851683425dec5ecae21e092a9a440845ce99a13ded69Nicolas Capens (getPointerType() == IceType_i32) ? Traits::FK_Abs : FK_Abs64; 85176b80cf109e33aaf58de116b63a0562227d096429David Sehr for (const JumpTableData &JT : Ctx->getJumpTables()) 851883425dec5ecae21e092a9a440845ce99a13ded69Nicolas Capens Writer->writeJumpTable(JT, RelocationKind, IsPIC); 85196b80cf109e33aaf58de116b63a0562227d096429David Sehr } break; 85206b80cf109e33aaf58de116b63a0562227d096429David Sehr case FT_Asm: 85216b80cf109e33aaf58de116b63a0562227d096429David Sehr // Already emitted from Cfg 85226b80cf109e33aaf58de116b63a0562227d096429David Sehr break; 85236b80cf109e33aaf58de116b63a0562227d096429David Sehr case FT_Iasm: { 85246b80cf109e33aaf58de116b63a0562227d096429David Sehr if (!BuildDefs::dump()) 85256b80cf109e33aaf58de116b63a0562227d096429David Sehr return; 85266b80cf109e33aaf58de116b63a0562227d096429David Sehr Ostream &Str = Ctx->getStrEmit(); 8527467ffe51bebcb3ae3e3ce745c38fc20e8837c31cJim Stichnoth const char *Prefix = IsPIC ? ".data.rel.ro." : ".rodata."; 85286b80cf109e33aaf58de116b63a0562227d096429David Sehr for (const JumpTableData &JT : Ctx->getJumpTables()) { 8529030772114216c1a57c749050bb58d07de8ceaa7cJohn Porto Str << "\t.section\t" << Prefix << JT.getSectionName() 8530030772114216c1a57c749050bb58d07de8ceaa7cJohn Porto << ",\"a\",@progbits\n" 8531030772114216c1a57c749050bb58d07de8ceaa7cJohn Porto "\t.align\t" << typeWidthInBytes(getPointerType()) << "\n" 8532030772114216c1a57c749050bb58d07de8ceaa7cJohn Porto << JT.getName().toString() << ":"; 85336b80cf109e33aaf58de116b63a0562227d096429David Sehr 85346b80cf109e33aaf58de116b63a0562227d096429David Sehr // On X8664 ILP32 pointers are 32-bit hence the use of .long 85356b80cf109e33aaf58de116b63a0562227d096429David Sehr for (intptr_t TargetOffset : JT.getTargetOffsets()) 85366b80cf109e33aaf58de116b63a0562227d096429David Sehr Str << "\n\t.long\t" << JT.getFunctionName() << "+" << TargetOffset; 85376b80cf109e33aaf58de116b63a0562227d096429David Sehr Str << "\n"; 85386b80cf109e33aaf58de116b63a0562227d096429David Sehr } 85396b80cf109e33aaf58de116b63a0562227d096429David Sehr } break; 85406b80cf109e33aaf58de116b63a0562227d096429David Sehr } 85416b80cf109e33aaf58de116b63a0562227d096429David Sehr} 85426b80cf109e33aaf58de116b63a0562227d096429David Sehr 85436b80cf109e33aaf58de116b63a0562227d096429David Sehrtemplate <typename TraitsType> 85446b80cf109e33aaf58de116b63a0562227d096429David Sehrvoid TargetDataX86<TraitsType>::lowerGlobals( 8545467ffe51bebcb3ae3e3ce745c38fc20e8837c31cJim Stichnoth const VariableDeclarationList &Vars, const std::string &SectionSuffix) { 8546d46999474d2b4a388e1d8a7c71f06cd4cec51bfcKarl Schimpf const bool IsPIC = getFlags().getUseNonsfi(); 8547d46999474d2b4a388e1d8a7c71f06cd4cec51bfcKarl Schimpf switch (getFlags().getOutFileType()) { 85486b80cf109e33aaf58de116b63a0562227d096429David Sehr case FT_Elf: { 85496b80cf109e33aaf58de116b63a0562227d096429David Sehr ELFObjectWriter *Writer = Ctx->getObjectWriter(); 85506b80cf109e33aaf58de116b63a0562227d096429David Sehr Writer->writeDataSection(Vars, Traits::FK_Abs, SectionSuffix, IsPIC); 85516b80cf109e33aaf58de116b63a0562227d096429David Sehr } break; 85526b80cf109e33aaf58de116b63a0562227d096429David Sehr case FT_Asm: 85536b80cf109e33aaf58de116b63a0562227d096429David Sehr case FT_Iasm: { 85546b80cf109e33aaf58de116b63a0562227d096429David Sehr OstreamLocker L(Ctx); 85556b80cf109e33aaf58de116b63a0562227d096429David Sehr for (const VariableDeclaration *Var : Vars) { 8556dd6dcfaf765dc93ae64ec45d623106f4b3a3c13aJim Stichnoth if (getFlags().matchTranslateOnly(Var->getName(), 0)) { 85576b80cf109e33aaf58de116b63a0562227d096429David Sehr emitGlobal(*Var, SectionSuffix); 85586b80cf109e33aaf58de116b63a0562227d096429David Sehr } 85596b80cf109e33aaf58de116b63a0562227d096429David Sehr } 85606b80cf109e33aaf58de116b63a0562227d096429David Sehr } break; 85616b80cf109e33aaf58de116b63a0562227d096429David Sehr } 85626b80cf109e33aaf58de116b63a0562227d096429David Sehr} 85634a56686b5b56db6803f90ad53514bf2fa190d9f7John Porto} // end of namespace X86NAMESPACE 85647e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto} // end of namespace Ice 85657e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto 85667e93c62d7e223b7fd9e6e0889e4b70b635589282John Porto#endif // SUBZERO_SRC_ICETARGETLOWERINGX86BASEIMPL_H 8567