Lines Matching defs:mce
129 static IRExpr* expr2vbits ( struct _MCEnv* mce, IRExpr* e );
130 static IRTemp findShadowTmpB ( struct _MCEnv* mce, IRTemp orig );
185 "typeOfIRExpr(mce->bb->tyenv, ...)" at various places in the
217 Shadow IRTemps are therefore allocated on demand. mce.tmpMap is a
235 both the table in mce->sb and to our auxiliary mapping. Note that
236 newTemp may cause mce->tmpMap to resize, hence previous results
237 from VG_(indexXA)(mce->tmpMap) are invalidated. */
238 static IRTemp newTemp ( MCEnv* mce, IRType ty, TempKind kind )
242 IRTemp tmp = newIRTemp(mce->sb->tyenv, ty);
246 newIx = VG_(addToXA)( mce->tmpMap, &ent );
254 static IRTemp findShadowTmpV ( MCEnv* mce, IRTemp orig )
259 ent = (TempMapEnt*)VG_(indexXA)( mce->tmpMap, (Word)orig );
263 = newTemp( mce, shadowTypeV(mce->sb->tyenv->types[orig]), VSh );
264 /* newTemp may cause mce->tmpMap to resize, hence previous results
266 ent = (TempMapEnt*)VG_(indexXA)( mce->tmpMap, (Word)orig );
284 static void newShadowTmpV ( MCEnv* mce, IRTemp orig )
289 ent = (TempMapEnt*)VG_(indexXA)( mce->tmpMap, (Word)orig );
293 = newTemp( mce, shadowTypeV(mce->sb->tyenv->types[orig]), VSh );
294 /* newTemp may cause mce->tmpMap to resize, hence previous results
296 ent = (TempMapEnt*)VG_(indexXA)( mce->tmpMap, (Word)orig );
318 static Bool isOriginalAtom ( MCEnv* mce, IRAtom* a1 )
323 TempMapEnt* ent = VG_(indexXA)( mce->tmpMap, a1->Iex.RdTmp.tmp );
331 static Bool isShadowAtom ( MCEnv* mce, IRAtom* a1 )
336 TempMapEnt* ent = VG_(indexXA)( mce->tmpMap, a1->Iex.RdTmp.tmp );
406 static inline void stmt ( HChar cat, MCEnv* mce, IRStmt* st ) {
407 if (mce->trace) {
412 addStmtToIRSB(mce->sb, st);
417 void assign ( HChar cat, MCEnv* mce, IRTemp tmp, IRExpr* expr ) {
418 stmt(cat, mce, IRStmt_WrTmp(tmp,expr));
441 static IRAtom* assignNew ( HChar cat, MCEnv* mce, IRType ty, IRExpr* e )
445 IRType tyE = typeOfIRExpr(mce->sb->tyenv, e);
456 t = newTemp(mce, ty, k);
457 assign(cat, mce, t, e);
483 static IRAtom* mkDifD8 ( MCEnv* mce, IRAtom* a1, IRAtom* a2 ) {
484 tl_assert(isShadowAtom(mce,a1));
485 tl_assert(isShadowAtom(mce,a2));
486 return assignNew('V', mce, Ity_I8, binop(Iop_And8, a1, a2));
489 static IRAtom* mkDifD16 ( MCEnv* mce, IRAtom* a1, IRAtom* a2 ) {
490 tl_assert(isShadowAtom(mce,a1));
491 tl_assert(isShadowAtom(mce,a2));
492 return assignNew('V', mce, Ity_I16, binop(Iop_And16, a1, a2));
495 static IRAtom* mkDifD32 ( MCEnv* mce, IRAtom* a1, IRAtom* a2 ) {
496 tl_assert(isShadowAtom(mce,a1));
497 tl_assert(isShadowAtom(mce,a2));
498 return assignNew('V', mce, Ity_I32, binop(Iop_And32, a1, a2));
501 static IRAtom* mkDifD64 ( MCEnv* mce, IRAtom* a1, IRAtom* a2 ) {
502 tl_assert(isShadowAtom(mce,a1));
503 tl_assert(isShadowAtom(mce,a2));
504 return assignNew('V', mce, Ity_I64, binop(Iop_And64, a1, a2));
507 static IRAtom* mkDifDV128 ( MCEnv* mce, IRAtom* a1, IRAtom* a2 ) {
508 tl_assert(isShadowAtom(mce,a1));
509 tl_assert(isShadowAtom(mce,a2));
510 return assignNew('V', mce, Ity_V128, binop(Iop_AndV128, a1, a2));
513 static IRAtom* mkDifDV256 ( MCEnv* mce, IRAtom* a1, IRAtom* a2 ) {
514 tl_assert(isShadowAtom(mce,a1));
515 tl_assert(isShadowAtom(mce,a2));
516 return assignNew('V', mce, Ity_V256, binop(Iop_AndV256, a1, a2));
521 static IRAtom* mkUifU8 ( MCEnv* mce, IRAtom* a1, IRAtom* a2 ) {
522 tl_assert(isShadowAtom(mce,a1));
523 tl_assert(isShadowAtom(mce,a2));
524 return assignNew('V', mce, Ity_I8, binop(Iop_Or8, a1, a2));
527 static IRAtom* mkUifU16 ( MCEnv* mce, IRAtom* a1, IRAtom* a2 ) {
528 tl_assert(isShadowAtom(mce,a1));
529 tl_assert(isShadowAtom(mce,a2));
530 return assignNew('V', mce, Ity_I16, binop(Iop_Or16, a1, a2));
533 static IRAtom* mkUifU32 ( MCEnv* mce, IRAtom* a1, IRAtom* a2 ) {
534 tl_assert(isShadowAtom(mce,a1));
535 tl_assert(isShadowAtom(mce,a2));
536 return assignNew('V', mce, Ity_I32, binop(Iop_Or32, a1, a2));
539 static IRAtom* mkUifU64 ( MCEnv* mce, IRAtom* a1, IRAtom* a2 ) {
540 tl_assert(isShadowAtom(mce,a1));
541 tl_assert(isShadowAtom(mce,a2));
542 return assignNew('V', mce, Ity_I64, binop(Iop_Or64, a1, a2));
545 static IRAtom* mkUifU128 ( MCEnv* mce, IRAtom* a1, IRAtom* a2 ) {
547 tl_assert(isShadowAtom(mce,a1));
548 tl_assert(isShadowAtom(mce,a2));
549 tmp1 = assignNew('V', mce, Ity_I64, unop(Iop_128to64, a1));
550 tmp2 = assignNew('V', mce, Ity_I64, unop(Iop_128HIto64, a1));
551 tmp3 = assignNew('V', mce, Ity_I64, unop(Iop_128to64, a2));
552 tmp4 = assignNew('V', mce, Ity_I64, unop(Iop_128HIto64, a2));
553 tmp5 = assignNew('V', mce, Ity_I64, binop(Iop_Or64, tmp1, tmp3));
554 tmp6 = assignNew('V', mce, Ity_I64, binop(Iop_Or64, tmp2, tmp4));
556 return assignNew('V', mce, Ity_I128, binop(Iop_64HLto128, tmp6, tmp5));
559 static IRAtom* mkUifUV128 ( MCEnv* mce, IRAtom* a1, IRAtom* a2 ) {
560 tl_assert(isShadowAtom(mce,a1));
561 tl_assert(isShadowAtom(mce,a2));
562 return assignNew('V', mce, Ity_V128, binop(Iop_OrV128, a1, a2));
565 static IRAtom* mkUifUV256 ( MCEnv* mce, IRAtom* a1, IRAtom* a2 ) {
566 tl_assert(isShadowAtom(mce,a1));
567 tl_assert(isShadowAtom(mce,a2));
568 return assignNew('V', mce, Ity_V256, binop(Iop_OrV256, a1, a2));
571 static IRAtom* mkUifU ( MCEnv* mce, IRType vty, IRAtom* a1, IRAtom* a2 ) {
573 case Ity_I8: return mkUifU8(mce, a1, a2);
574 case Ity_I16: return mkUifU16(mce, a1, a2);
575 case Ity_I32: return mkUifU32(mce, a1, a2);
576 case Ity_I64: return mkUifU64(mce, a1, a2);
577 case Ity_I128: return mkUifU128(mce, a1, a2);
578 case Ity_V128: return mkUifUV128(mce, a1, a2);
587 static IRAtom* mkLeft8 ( MCEnv* mce, IRAtom* a1 ) {
588 tl_assert(isShadowAtom(mce,a1));
589 return assignNew('V', mce, Ity_I8, unop(Iop_Left8, a1));
592 static IRAtom* mkLeft16 ( MCEnv* mce, IRAtom* a1 ) {
593 tl_assert(isShadowAtom(mce,a1));
594 return assignNew('V', mce, Ity_I16, unop(Iop_Left16, a1));
597 static IRAtom* mkLeft32 ( MCEnv* mce, IRAtom* a1 ) {
598 tl_assert(isShadowAtom(mce,a1));
599 return assignNew('V', mce, Ity_I32, unop(Iop_Left32, a1));
602 static IRAtom* mkLeft64 ( MCEnv* mce, IRAtom* a1 ) {
603 tl_assert(isShadowAtom(mce,a1));
604 return assignNew('V', mce, Ity_I64, unop(Iop_Left64, a1));
612 static IRAtom* mkImproveAND8 ( MCEnv* mce, IRAtom* data, IRAtom* vbits )
614 tl_assert(isOriginalAtom(mce, data));
615 tl_assert(isShadowAtom(mce, vbits));
617 return assignNew('V', mce, Ity_I8, binop(Iop_Or8, data, vbits));
620 static IRAtom* mkImproveAND16 ( MCEnv* mce, IRAtom* data, IRAtom* vbits )
622 tl_assert(isOriginalAtom(mce, data));
623 tl_assert(isShadowAtom(mce, vbits));
625 return assignNew('V', mce, Ity_I16, binop(Iop_Or16, data, vbits));
628 static IRAtom* mkImproveAND32 ( MCEnv* mce, IRAtom* data, IRAtom* vbits )
630 tl_assert(isOriginalAtom(mce, data));
631 tl_assert(isShadowAtom(mce, vbits));
633 return assignNew('V', mce, Ity_I32, binop(Iop_Or32, data, vbits));
636 static IRAtom* mkImproveAND64 ( MCEnv* mce, IRAtom* data, IRAtom* vbits )
638 tl_assert(isOriginalAtom(mce, data));
639 tl_assert(isShadowAtom(mce, vbits));
641 return assignNew('V', mce, Ity_I64, binop(Iop_Or64, data, vbits));
644 static IRAtom* mkImproveANDV128 ( MCEnv* mce, IRAtom* data, IRAtom* vbits )
646 tl_assert(isOriginalAtom(mce, data));
647 tl_assert(isShadowAtom(mce, vbits));
649 return assignNew('V', mce, Ity_V128, binop(Iop_OrV128, data, vbits));
652 static IRAtom* mkImproveANDV256 ( MCEnv* mce, IRAtom* data, IRAtom* vbits )
654 tl_assert(isOriginalAtom(mce, data));
655 tl_assert(isShadowAtom(mce, vbits));
657 return assignNew('V', mce, Ity_V256, binop(Iop_OrV256, data, vbits));
663 static IRAtom* mkImproveOR8 ( MCEnv* mce, IRAtom* data, IRAtom* vbits )
665 tl_assert(isOriginalAtom(mce, data));
666 tl_assert(isShadowAtom(mce, vbits));
669 'V', mce, Ity_I8,
671 assignNew('V', mce, Ity_I8, unop(Iop_Not8, data)),
675 static IRAtom* mkImproveOR16 ( MCEnv* mce, IRAtom* data, IRAtom* vbits )
677 tl_assert(isOriginalAtom(mce, data));
678 tl_assert(isShadowAtom(mce, vbits));
681 'V', mce, Ity_I16,
683 assignNew('V', mce, Ity_I16, unop(Iop_Not16, data)),
687 static IRAtom* mkImproveOR32 ( MCEnv* mce, IRAtom* data, IRAtom* vbits )
689 tl_assert(isOriginalAtom(mce, data));
690 tl_assert(isShadowAtom(mce, vbits));
693 'V', mce, Ity_I32,
695 assignNew('V', mce, Ity_I32, unop(Iop_Not32, data)),
699 static IRAtom* mkImproveOR64 ( MCEnv* mce, IRAtom* data, IRAtom* vbits )
701 tl_assert(isOriginalAtom(mce, data));
702 tl_assert(isShadowAtom(mce, vbits));
705 'V', mce, Ity_I64,
707 assignNew('V', mce, Ity_I64, unop(Iop_Not64, data)),
711 static IRAtom* mkImproveORV128 ( MCEnv* mce, IRAtom* data, IRAtom* vbits )
713 tl_assert(isOriginalAtom(mce, data));
714 tl_assert(isShadowAtom(mce, vbits));
717 'V', mce, Ity_V128,
719 assignNew('V', mce, Ity_V128, unop(Iop_NotV128, data)),
723 static IRAtom* mkImproveORV256 ( MCEnv* mce, IRAtom* data, IRAtom* vbits )
725 tl_assert(isOriginalAtom(mce, data));
726 tl_assert(isShadowAtom(mce, vbits));
729 'V', mce, Ity_V256,
731 assignNew('V', mce, Ity_V256, unop(Iop_NotV256, data)),
741 static IRAtom* mkPCastTo( MCEnv* mce, IRType dst_ty, IRAtom* vbits )
747 tl_assert(isShadowAtom(mce,vbits));
748 src_ty = typeOfIRExpr(mce->sb->tyenv, vbits);
752 return assignNew('V', mce, Ity_I32, unop(Iop_CmpwNEZ32, vbits));
755 return assignNew('V', mce, Ity_I64, unop(Iop_CmpwNEZ64, vbits));
759 IRAtom* tmp = assignNew('V', mce, Ity_I32, unop(Iop_CmpwNEZ32, vbits));
760 return assignNew('V', mce, Ity_I64, binop(Iop_32HLto64, tmp, tmp));
766 IRAtom* tmp = assignNew('V', mce, Ity_I64, unop(Iop_CmpwNEZ64, vbits));
767 return assignNew('V', mce, Ity_I32, unop(Iop_64to32, tmp));
778 tmp1 = assignNew('V', mce, Ity_I1, unop(Iop_CmpNEZ8, vbits));
781 tmp1 = assignNew('V', mce, Ity_I1, unop(Iop_CmpNEZ16, vbits));
784 tmp1 = assignNew('V', mce, Ity_I1, unop(Iop_CmpNEZ32, vbits));
787 tmp1 = assignNew('V', mce, Ity_I1, unop(Iop_CmpNEZ64, vbits));
792 IRAtom* tmp2 = assignNew('V', mce, Ity_I64, unop(Iop_128HIto64, vbits));
793 IRAtom* tmp3 = assignNew('V', mce, Ity_I64, unop(Iop_128to64, vbits));
794 IRAtom* tmp4 = assignNew('V', mce, Ity_I64, binop(Iop_Or64, tmp2, tmp3));
795 tmp1 = assignNew('V', mce, Ity_I1,
809 return assignNew('V', mce, Ity_I8, unop(Iop_1Sto8, tmp1));
811 return assignNew('V', mce, Ity_I16, unop(Iop_1Sto16, tmp1));
813 return assignNew('V', mce, Ity_I32, unop(Iop_1Sto32, tmp1));
815 return assignNew('V', mce, Ity_I64, unop(Iop_1Sto64, tmp1));
817 tmp1 = assignNew('V', mce, Ity_I64, unop(Iop_1Sto64, tmp1));
818 tmp1 = assignNew('V', mce, Ity_V128, binop(Iop_64HLtoV128, tmp1, tmp1));
821 tmp1 = assignNew('V', mce, Ity_I64, unop(Iop_1Sto64, tmp1));
822 tmp1 = assignNew('V', mce, Ity_I128, binop(Iop_64HLto128, tmp1, tmp1));
873 static IRAtom* expensiveCmpEQorNE ( MCEnv* mce,
882 tl_assert(isShadowAtom(mce,vxx));
883 tl_assert(isShadowAtom(mce,vyy));
884 tl_assert(isOriginalAtom(mce,xx));
885 tl_assert(isOriginalAtom(mce,yy));
913 = mkPCastTo(mce,ty,
914 assignNew('V', mce, ty, binop(opUIFU, vxx, vyy)));
918 'V', mce,ty,
920 assignNew('V', mce,ty, binop(opOR, vxx, vyy)),
922 'V', mce,ty,
924 assignNew('V', mce,ty, binop(opXOR, xx, yy))))));
927 = mkPCastTo( mce,ty,
928 assignNew('V', mce,Ity_I1, binop(opCMP, vec, top)));
931 = assignNew( 'V', mce,ty, binop(opDIFD, naive, improvement_term) );
934 = mkPCastTo( mce, Ity_I1, improved );
985 static IRAtom* doCmpORD ( MCEnv* mce,
1004 tl_assert(isShadowAtom(mce,xxhash));
1005 tl_assert(isShadowAtom(mce,yyhash));
1006 tl_assert(isOriginalAtom(mce,xx));
1007 tl_assert(isOriginalAtom(mce,yy));
1027 'V', mce,ty,
1030 mkPCastTo(mce,ty, xxhash),
1034 'V', mce,ty,
1038 'V', mce,ty,
1049 mkPCastTo( mce,ty,
1050 mkUifU(mce,ty, xxhash,yyhash)),
1061 static IRAtom* schemeE ( MCEnv* mce, IRExpr* e ); /* fwds */
1069 static void setHelperAnns ( MCEnv* mce, IRDirty* di ) {
1072 di->fxState[0].offset = mce->layout->offset_SP;
1073 di->fxState[0].size = mce->layout->sizeof_SP;
1077 di->fxState[1].offset = mce->layout->offset_IP;
1078 di->fxState[1].size = mce->layout->sizeof_IP;
1094 static void complainIfUndefined ( MCEnv* mce, IRAtom* atom, IRExpr *guard )
1115 tl_assert(isOriginalAtom(mce, atom));
1116 vatom = expr2vbits( mce, atom );
1117 tl_assert(isShadowAtom(mce, vatom));
1120 ty = typeOfIRExpr(mce->sb->tyenv, vatom);
1125 cond = mkPCastTo( mce, Ity_I1, vatom );
1132 origin = schemeE( mce, atom );
1133 if (mce->hWordTy == Ity_I64) {
1134 origin = assignNew( 'B', mce, Ity_I64, unop(Iop_32Uto64, origin) );
1230 IRAtom *g1 = assignNew('V', mce, Ity_I32, unop(Iop_1Uto32, di->guard));
1231 IRAtom *g2 = assignNew('V', mce, Ity_I32, unop(Iop_1Uto32, guard));
1232 IRAtom *e = assignNew('V', mce, Ity_I32, binop(Iop_And32, g1, g2));
1234 di->guard = assignNew('V', mce, Ity_I1, unop(Iop_32to1, e));
1237 setHelperAnns( mce, di );
1238 stmt( 'V', mce, IRStmt_Dirty(di));
1247 newShadowTmpV(mce, atom->Iex.RdTmp.tmp);
1248 assign('V', mce, findShadowTmpV(mce, atom->Iex.RdTmp.tmp),
1263 static Bool isAlwaysDefd ( MCEnv* mce, Int offset, Int size )
1271 for (i = 0; i < mce->layout->n_alwaysDefd; i++) {
1272 minoffD = mce->layout->alwaysDefd[i].offset;
1273 maxoffD = minoffD + mce->layout->alwaysDefd[i].size - 1;
1296 void do_shadow_PUT ( MCEnv* mce, Int offset,
1309 tl_assert(isOriginalAtom(mce, atom));
1310 vatom = expr2vbits( mce, atom );
1313 tl_assert(isShadowAtom(mce, vatom));
1316 ty = typeOfIRExpr(mce->sb->tyenv, vatom);
1319 if (isAlwaysDefd(mce, offset, sizeofIRType(ty))) {
1322 /* complainIfUndefined(mce, atom); */
1330 cond = assignNew('V', mce, Ity_I8, unop(Iop_1Uto8, guard));
1331 iffalse = assignNew('V', mce, ty,
1332 IRExpr_Get(offset + mce->layout->total_sizeB, ty));
1333 vatom = assignNew('V', mce, ty, IRExpr_Mux0X(cond, iffalse, vatom));
1335 stmt( 'V', mce, IRStmt_Put( offset + mce->layout->total_sizeB, vatom ));
1344 void do_shadow_PUTI ( MCEnv* mce, IRPutI *puti)
1360 tl_assert(isOriginalAtom(mce,atom));
1361 vatom = expr2vbits( mce, atom );
1367 tl_assert(isOriginalAtom(mce,ix));
1368 complainIfUndefined(mce, ix, NULL);
1369 if (isAlwaysDefd(mce, descr->base, arrSize)) {
1372 /* complainIfUndefined(mce, atom); */
1377 = mkIRRegArray( descr->base + mce->layout->total_sizeB,
1379 stmt( 'V', mce, IRStmt_PutI( mkIRPutI(new_descr, ix, bias, vatom) ));
1388 IRExpr* shadow_GET ( MCEnv* mce, Int offset, IRType ty )
1393 if (isAlwaysDefd(mce, offset, sizeofIRType(ty))) {
1400 return IRExpr_Get( offset + mce->layout->total_sizeB, tyS );
1409 IRExpr* shadow_GETI ( MCEnv* mce,
1416 tl_assert(isOriginalAtom(mce,ix));
1417 complainIfUndefined(mce, ix, NULL);
1418 if (isAlwaysDefd(mce, descr->base, arrSize)) {
1425 = mkIRRegArray( descr->base + mce->layout->total_sizeB,
1441 IRAtom* mkLazy2 ( MCEnv* mce, IRType finalVty, IRAtom* va1, IRAtom* va2 )
1444 IRType t1 = typeOfIRExpr(mce->sb->tyenv, va1);
1445 IRType t2 = typeOfIRExpr(mce->sb->tyenv, va2);
1446 tl_assert(isShadowAtom(mce,va1));
1447 tl_assert(isShadowAtom(mce,va2));
1456 at = mkUifU(mce, Ity_I64, va1, va2);
1457 at = mkPCastTo(mce, Ity_I64, at);
1464 at = mkUifU(mce, Ity_I64, va1, va2);
1465 at = mkPCastTo(mce, Ity_I32, at);
1480 at = mkPCastTo(mce, Ity_I32, va1);
1481 at = mkUifU(mce, Ity_I32, at, mkPCastTo(mce, Ity_I32, va2));
1482 at = mkPCastTo(mce, finalVty, at);
1489 IRAtom* mkLazy3 ( MCEnv* mce, IRType finalVty,
1493 IRType t1 = typeOfIRExpr(mce->sb->tyenv, va1);
1494 IRType t2 = typeOfIRExpr(mce->sb->tyenv, va2);
1495 IRType t3 = typeOfIRExpr(mce->sb->tyenv, va3);
1496 tl_assert(isShadowAtom(mce,va1));
1497 tl_assert(isShadowAtom(mce,va2));
1498 tl_assert(isShadowAtom(mce,va3));
1512 at = mkPCastTo(mce, Ity_I64, va1);
1514 at = mkUifU(mce, Ity_I64, at, va2);
1515 at = mkUifU(mce, Ity_I64, at, va3);
1517 at = mkPCastTo(mce, Ity_I64, at);
1525 at = mkPCastTo(mce, Ity_I64, va1);
1526 at = mkUifU(mce, Ity_I64, at, va2);
1527 at = mkUifU(mce, Ity_I64, at, va3);
1528 at = mkPCastTo(mce, Ity_I32, at);
1538 at = mkUifU(mce, Ity_I32, at, va2);
1539 at = mkUifU(mce, Ity_I32, at, va3);
1540 at = mkPCastTo(mce, Ity_I32, at);
1552 at = mkPCastTo(mce, Ity_I128, va1);
1554 at = mkUifU(mce, Ity_I128, at, va2);
1555 at = mkUifU(mce, Ity_I128, at, va3);
1557 at = mkPCastTo(mce, Ity_I128, at);
1575 at = mkPCastTo(mce, Ity_I32, va1);
1576 at = mkUifU(mce, Ity_I32, at, mkPCastTo(mce, Ity_I32, va2));
1577 at = mkUifU(mce, Ity_I32, at, mkPCastTo(mce, Ity_I32, va3));
1578 at = mkPCastTo(mce, finalVty, at);
1586 IRAtom* mkLazy4 ( MCEnv* mce, IRType finalVty,
1590 IRType t1 = typeOfIRExpr(mce->sb->tyenv, va1);
1591 IRType t2 = typeOfIRExpr(mce->sb->tyenv, va2);
1592 IRType t3 = typeOfIRExpr(mce->sb->tyenv, va3);
1593 IRType t4 = typeOfIRExpr(mce->sb->tyenv, va4);
1594 tl_assert(isShadowAtom(mce,va1));
1595 tl_assert(isShadowAtom(mce,va2));
1596 tl_assert(isShadowAtom(mce,va3));
1597 tl_assert(isShadowAtom(mce,va4));
1611 at = mkPCastTo(mce, Ity_I64, va1);
1613 at = mkUifU(mce, Ity_I64, at, va2);
1614 at = mkUifU(mce, Ity_I64, at, va3);
1615 at = mkUifU(mce, Ity_I64, at, va4);
1617 at = mkPCastTo(mce, Ity_I64, at);
1627 at = mkUifU(mce, Ity_I32, at, va2);
1628 at = mkUifU(mce, Ity_I32, at, va3);
1629 at = mkUifU(mce, Ity_I32, at, va4);
1630 at = mkPCastTo(mce, Ity_I32, at);
1658 IRAtom* mkLazyN ( MCEnv* mce,
1672 tl_assert(isOriginalAtom(mce, exprvec[i]));
1675 if (typeOfIRExpr(mce->sb->tyenv, exprvec[i]) != Ity_I64)
1684 tl_assert(isOriginalAtom(mce, exprvec[i]));
1694 here = mkPCastTo( mce, mergeTy, expr2vbits(mce, exprvec[i]) );
1696 ? mkUifU64(mce, here, curr)
1697 : mkUifU32(mce, here, curr);
1700 return mkPCastTo(mce, finalVtype, curr );
1710 IRAtom* expensiveAddSub ( MCEnv* mce,
1719 tl_assert(isShadowAtom(mce,qaa));
1720 tl_assert(isShadowAtom(mce,qbb));
1721 tl_assert(isOriginalAtom(mce,aa));
1722 tl_assert(isOriginalAtom(mce,bb));
1748 a_min = assignNew('V', mce,ty,
1750 assignNew('V', mce,ty, unop(opNOT, qaa))));
1753 b_min = assignNew('V', mce,ty,
1755 assignNew('V', mce,ty, unop(opNOT, qbb))));
1758 a_max = assignNew('V', mce,ty, binop(opOR, aa, qaa));
1761 b_max = assignNew('V', mce,ty, binop(opOR, bb, qbb));
1766 assignNew('V', mce,ty,
1768 assignNew('V', mce,ty, binop(opOR, qaa, qbb)),
1769 assignNew('V', mce,ty,
1771 assignNew('V', mce,ty, binop(opADD, a_min, b_min)),
1772 assignNew('V', mce,ty, binop(opADD, a_max, b_max))
1780 assignNew('V', mce,ty,
1782 assignNew('V', mce,ty, binop(opOR, qaa, qbb)),
1783 assignNew('V', mce,ty,
1785 assignNew('V', mce,ty, binop(opSUB, a_min, b_max)),
1786 assignNew('V', mce,ty, binop(opSUB, a_max, b_min))
1819 static IRAtom* scalarShift ( MCEnv* mce,
1825 tl_assert(isShadowAtom(mce,qaa));
1826 tl_assert(isShadowAtom(mce,qbb));
1827 tl_assert(isOriginalAtom(mce,aa));
1828 tl_assert(isOriginalAtom(mce,bb));
1833 'V', mce, ty,
1834 mkUifU( mce, ty,
1835 assignNew('V', mce, ty, binop(original_op, qaa, bb)),
1836 mkPCastTo(mce, ty, qbb)
1848 static IRAtom* mkPCast8x16 ( MCEnv* mce, IRAtom* at )
1850 return assignNew('V', mce, Ity_V128, unop(Iop_CmpNEZ8x16, at));
1853 static IRAtom* mkPCast16x8 ( MCEnv* mce, IRAtom* at )
1855 return assignNew('V', mce, Ity_V128, unop(Iop_CmpNEZ16x8, at));
1858 static IRAtom* mkPCast32x4 ( MCEnv* mce, IRAtom* at )
1860 return assignNew('V', mce, Ity_V128, unop(Iop_CmpNEZ32x4, at));
1863 static IRAtom* mkPCast64x2 ( MCEnv* mce, IRAtom* at )
1865 return assignNew('V', mce, Ity_V128, unop(Iop_CmpNEZ64x2, at));
1868 static IRAtom* mkPCast64x4 ( MCEnv* mce, IRAtom* at )
1870 return assignNew('V', mce, Ity_V256, unop(Iop_CmpNEZ64x4, at));
1873 static IRAtom* mkPCast32x8 ( MCEnv* mce, IRAtom* at )
1875 return assignNew('V', mce, Ity_V256, unop(Iop_CmpNEZ32x8, at));
1878 static IRAtom* mkPCast32x2 ( MCEnv* mce, IRAtom* at )
1880 return assignNew('V', mce, Ity_I64, unop(Iop_CmpNEZ32x2, at));
1883 static IRAtom* mkPCast16x4 ( MCEnv* mce, IRAtom* at )
1885 return assignNew('V', mce, Ity_I64, unop(Iop_CmpNEZ16x4, at));
1888 static IRAtom* mkPCast8x8 ( MCEnv* mce, IRAtom* at )
1890 return assignNew('V', mce, Ity_I64, unop(Iop_CmpNEZ8x8, at));
1893 static IRAtom* mkPCast16x2 ( MCEnv* mce, IRAtom* at )
1895 return assignNew('V', mce, Ity_I32, unop(Iop_CmpNEZ16x2, at));
1898 static IRAtom* mkPCast8x4 ( MCEnv* mce, IRAtom* at )
1900 return assignNew('V', mce, Ity_I32, unop(Iop_CmpNEZ8x4, at));
1941 IRAtom* binary32Fx4 ( MCEnv* mce, IRAtom* vatomX, IRAtom* vatomY )
1944 tl_assert(isShadowAtom(mce, vatomX));
1945 tl_assert(isShadowAtom(mce, vatomY));
1946 at = mkUifUV128(mce, vatomX, vatomY);
1947 at = assignNew('V', mce, Ity_V128, mkPCast32x4(mce, at));
1952 IRAtom* unary32Fx4 ( MCEnv* mce, IRAtom* vatomX )
1955 tl_assert(isShadowAtom(mce, vatomX));
1956 at = assignNew('V', mce, Ity_V128, mkPCast32x4(mce, vatomX));
1961 IRAtom* binary32F0x4 ( MCEnv* mce, IRAtom* vatomX, IRAtom* vatomY )
1964 tl_assert(isShadowAtom(mce, vatomX));
1965 tl_assert(isShadowAtom(mce, vatomY));
1966 at = mkUifUV128(mce, vatomX, vatomY);
1967 at = assignNew('V', mce, Ity_I32, unop(Iop_V128to32, at));
1968 at = mkPCastTo(mce, Ity_I32, at);
1969 at = assignNew('V', mce, Ity_V128, binop(Iop_SetV128lo32, vatomX, at));
1974 IRAtom* unary32F0x4 ( MCEnv* mce, IRAtom* vatomX )
1977 tl_assert(isShadowAtom(mce, vatomX));
1978 at = assignNew('V', mce, Ity_I32, unop(Iop_V128to32, vatomX));
1979 at = mkPCastTo(mce, Ity_I32, at);
1980 at = assignNew('V', mce, Ity_V128, binop(Iop_SetV128lo32, vatomX, at));
1987 IRAtom* binary64Fx2 ( MCEnv* mce, IRAtom* vatomX, IRAtom* vatomY )
1990 tl_assert(isShadowAtom(mce, vatomX));
1991 tl_assert(isShadowAtom(mce, vatomY));
1992 at = mkUifUV128(mce, vatomX, vatomY);
1993 at = assignNew('V', mce, Ity_V128, mkPCast64x2(mce, at));
1998 IRAtom* unary64Fx2 ( MCEnv* mce, IRAtom* vatomX )
2001 tl_assert(isShadowAtom(mce, vatomX));
2002 at = assignNew('V', mce, Ity_V128, mkPCast64x2(mce, vatomX));
2007 IRAtom* binary64F0x2 ( MCEnv* mce, IRAtom* vatomX, IRAtom* vatomY )
2010 tl_assert(isShadowAtom(mce, vatomX));
2011 tl_assert(isShadowAtom(mce, vatomY));
2012 at = mkUifUV128(mce, vatomX, vatomY);
2013 at = assignNew('V', mce, Ity_I64, unop(Iop_V128to64, at));
2014 at = mkPCastTo(mce, Ity_I64, at);
2015 at = assignNew('V', mce, Ity_V128, binop(Iop_SetV128lo64, vatomX, at));
2020 IRAtom* unary64F0x2 ( MCEnv* mce, IRAtom* vatomX )
2023 tl_assert(isShadowAtom(mce, vatomX));
2024 at = assignNew('V', mce, Ity_I64, unop(Iop_V128to64, vatomX));
2025 at = mkPCastTo(mce, Ity_I64, at);
2026 at = assignNew('V', mce, Ity_V128, binop(Iop_SetV128lo64, vatomX, at));
2033 IRAtom* binary32Fx2 ( MCEnv* mce, IRAtom* vatomX, IRAtom* vatomY )
2036 tl_assert(isShadowAtom(mce, vatomX));
2037 tl_assert(isShadowAtom(mce, vatomY));
2038 at = mkUifU64(mce, vatomX, vatomY);
2039 at = assignNew('V', mce, Ity_I64, mkPCast32x2(mce, at));
2044 IRAtom* unary32Fx2 ( MCEnv* mce, IRAtom* vatomX )
2047 tl_assert(isShadowAtom(mce, vatomX));
2048 at = assignNew('V', mce, Ity_I64, mkPCast32x2(mce, vatomX));
2055 IRAtom* binary64Fx4 ( MCEnv* mce, IRAtom* vatomX, IRAtom* vatomY )
2058 tl_assert(isShadowAtom(mce, vatomX));
2059 tl_assert(isShadowAtom(mce, vatomY));
2060 at = mkUifUV256(mce, vatomX, vatomY);
2061 at = assignNew('V', mce, Ity_V256, mkPCast64x4(mce, at));
2066 IRAtom* unary64Fx4 ( MCEnv* mce, IRAtom* vatomX )
2069 tl_assert(isShadowAtom(mce, vatomX));
2070 at = assignNew('V', mce, Ity_V256, mkPCast64x4(mce, vatomX));
2077 IRAtom* binary32Fx8 ( MCEnv* mce, IRAtom* vatomX, IRAtom* vatomY )
2080 tl_assert(isShadowAtom(mce, vatomX));
2081 tl_assert(isShadowAtom(mce, vatomY));
2082 at = mkUifUV256(mce, vatomX, vatomY);
2083 at = assignNew('V', mce, Ity_V256, mkPCast32x8(mce, at));
2088 IRAtom* unary32Fx8 ( MCEnv* mce, IRAtom* vatomX )
2091 tl_assert(isShadowAtom(mce, vatomX));
2092 at = assignNew('V', mce, Ity_V256, mkPCast32x8(mce, vatomX));
2184 IRAtom* vectorNarrowBinV128 ( MCEnv* mce, IROp narrow_op,
2199 tl_assert(isShadowAtom(mce,vatom1));
2200 tl_assert(isShadowAtom(mce,vatom2));
2201 at1 = assignNew('V', mce, Ity_V128, pcast(mce, vatom1));
2202 at2 = assignNew('V', mce, Ity_V128, pcast(mce, vatom2));
2203 at3 = assignNew('V', mce, Ity_V128, binop(vanilla_narrow, at1, at2));
2208 IRAtom* vectorNarrowBin64 ( MCEnv* mce, IROp narrow_op,
2220 tl_assert(isShadowAtom(mce,vatom1));
2221 tl_assert(isShadowAtom(mce,vatom2));
2222 at1 = assignNew('V', mce, Ity_I64, pcast(mce, vatom1));
2223 at2 = assignNew('V', mce, Ity_I64, pcast(mce, vatom2));
2224 at3 = assignNew('V', mce, Ity_I64, binop(vanilla_narrow, at1, at2));
2229 IRAtom* vectorNarrowUnV128 ( MCEnv* mce, IROp narrow_op,
2234 tl_assert(isShadowAtom(mce,vatom1));
2241 at1 = assignNew('V', mce, Ity_I64, unop(narrow_op, vatom1));
2261 at1 = assignNew('V', mce, Ity_V128, pcast(mce, vatom1));
2262 at2 = assignNew('V', mce, Ity_I64, unop(vanilla_narrow, at1));
2267 IRAtom* vectorWidenI64 ( MCEnv* mce, IROp longen_op,
2281 tl_assert(isShadowAtom(mce,vatom1));
2282 at1 = assignNew('V', mce, Ity_V128, unop(longen_op, vatom1));
2283 at2 = assignNew('V', mce, Ity_V128, pcast(mce, at1));
2295 IRAtom* binary8Ix16 ( MCEnv* mce, IRAtom* vatom1, IRAtom* vatom2 )
2298 at = mkUifUV128(mce, vatom1, vatom2);
2299 at = mkPCast8x16(mce, at);
2304 IRAtom* binary16Ix8 ( MCEnv* mce, IRAtom* vatom1, IRAtom* vatom2 )
2307 at = mkUifUV128(mce, vatom1, vatom2);
2308 at = mkPCast16x8(mce, at);
2313 IRAtom* binary32Ix4 ( MCEnv* mce, IRAtom* vatom1, IRAtom* vatom2 )
2316 at = mkUifUV128(mce, vatom1, vatom2);
2317 at = mkPCast32x4(mce, at);
2322 IRAtom* binary64Ix2 ( MCEnv* mce, IRAtom* vatom1, IRAtom* vatom2 )
2325 at = mkUifUV128(mce, vatom1, vatom2);
2326 at = mkPCast64x2(mce, at);
2333 IRAtom* binary8Ix8 ( MCEnv* mce, IRAtom* vatom1, IRAtom* vatom2 )
2336 at = mkUifU64(mce, vatom1, vatom2);
2337 at = mkPCast8x8(mce, at);
2342 IRAtom* binary16Ix4 ( MCEnv* mce, IRAtom* vatom1, IRAtom* vatom2 )
2345 at = mkUifU64(mce, vatom1, vatom2);
2346 at = mkPCast16x4(mce, at);
2351 IRAtom* binary32Ix2 ( MCEnv* mce, IRAtom* vatom1, IRAtom* vatom2 )
2354 at = mkUifU64(mce, vatom1, vatom2);
2355 at = mkPCast32x2(mce, at);
2360 IRAtom* binary64Ix1 ( MCEnv* mce, IRAtom* vatom1, IRAtom* vatom2 )
2363 at = mkUifU64(mce, vatom1, vatom2);
2364 at = mkPCastTo(mce, Ity_I64, at);
2371 IRAtom* binary8Ix4 ( MCEnv* mce, IRAtom* vatom1, IRAtom* vatom2 )
2374 at = mkUifU32(mce, vatom1, vatom2);
2375 at = mkPCast8x4(mce, at);
2380 IRAtom* binary16Ix2 ( MCEnv* mce, IRAtom* vatom1, IRAtom* vatom2 )
2383 at = mkUifU32(mce, vatom1, vatom2);
2384 at = mkPCast16x2(mce, at);
2394 IRAtom* expr2vbits_Qop ( MCEnv* mce,
2399 IRAtom* vatom1 = expr2vbits( mce, atom1 );
2400 IRAtom* vatom2 = expr2vbits( mce, atom2 );
2401 IRAtom* vatom3 = expr2vbits( mce, atom3 );
2402 IRAtom* vatom4 = expr2vbits( mce, atom4 );
2404 tl_assert(isOriginalAtom(mce,atom1));
2405 tl_assert(isOriginalAtom(mce,atom2));
2406 tl_assert(isOriginalAtom(mce,atom3));
2407 tl_assert(isOriginalAtom(mce,atom4));
2408 tl_assert(isShadowAtom(mce,vatom1));
2409 tl_assert(isShadowAtom(mce,vatom2));
2410 tl_assert(isShadowAtom(mce,vatom3));
2411 tl_assert(isShadowAtom(mce,vatom4));
2422 return mkLazy4(mce, Ity_I64, vatom1, vatom2, vatom3, vatom4);
2427 return mkLazy4(mce, Ity_I32, vatom1, vatom2, vatom3, vatom4);
2431 return assignNew('V', mce, Ity_V256,
2442 IRAtom* expr2vbits_Triop ( MCEnv* mce,
2446 IRAtom* vatom1 = expr2vbits( mce, atom1 );
2447 IRAtom* vatom2 = expr2vbits( mce, atom2 );
2448 IRAtom* vatom3 = expr2vbits( mce, atom3 );
2450 tl_assert(isOriginalAtom(mce,atom1));
2451 tl_assert(isOriginalAtom(mce,atom2));
2452 tl_assert(isOriginalAtom(mce,atom3));
2453 tl_assert(isShadowAtom(mce,vatom1));
2454 tl_assert(isShadowAtom(mce,vatom2));
2455 tl_assert(isShadowAtom(mce,vatom3));
2470 return mkLazy3(mce, Ity_I128, vatom1, vatom2, vatom3);
2491 return mkLazy3(mce, Ity_I64, vatom1, vatom2, vatom3);
2495 return mkLazy3(mce, Ity_I32, vatom1, vatom2, vatom3);
2501 return mkLazy3(mce, Ity_I32, vatom1, vatom2, vatom3);
2504 return mkLazy3(mce, Ity_I64, vatom1, vatom2, vatom3);
2507 return mkLazy3(mce, Ity_I128, vatom1, vatom2, vatom3);
2509 complainIfUndefined(mce, atom3, NULL);
2510 return assignNew('V', mce, Ity_V128, triop(op, vatom1, vatom2, atom3));
2512 complainIfUndefined(mce, atom3, NULL);
2513 return assignNew('V', mce, Ity_I64, triop(op, vatom1, vatom2, atom3));
2517 complainIfUndefined(mce, atom2, NULL);
2518 return assignNew('V', mce, Ity_I64, triop(op, vatom1, atom2, vatom3));
2527 IRAtom* expr2vbits_Binop ( MCEnv* mce,
2536 IRAtom* vatom1 = expr2vbits( mce, atom1 );
2537 IRAtom* vatom2 = expr2vbits( mce, atom2 );
2539 tl_assert(isOriginalAtom(mce,atom1));
2540 tl_assert(isOriginalAtom(mce,atom2));
2541 tl_assert(isShadowAtom(mce,vatom1));
2542 tl_assert(isShadowAtom(mce,vatom2));
2558 return binary16Ix2(mce, vatom1, vatom2);
2570 return binary8Ix4(mce, vatom1, vatom2);
2584 complainIfUndefined(mce, atom2, NULL);
2585 return assignNew('V', mce, Ity_I64, binop(op, vatom1, atom2));
2590 return vectorNarrowBin64(mce, op, vatom1, vatom2);
2610 return binary8Ix8(mce, vatom1, vatom2);
2633 return binary16Ix4(mce, vatom1, vatom2);
2653 return binary32Ix2(mce, vatom1, vatom2);
2662 return binary64Ix1(mce, vatom1, vatom2);
2667 complainIfUndefined(mce, atom2, NULL);
2668 return mkPCast8x8(mce, vatom1);
2673 complainIfUndefined(mce, atom2, NULL);
2674 return mkPCast16x4(mce, vatom1);
2679 complainIfUndefined(mce, atom2, NULL);
2680 return mkPCast32x2(mce, vatom1);
2685 complainIfUndefined(mce, atom2, NULL);
2686 return mkPCast32x2(mce, vatom1);
2694 return assignNew('V', mce, Ity_I64,
2696 mkPCast32x2(mce, vatom1),
2697 mkPCast32x2(mce, vatom2)));
2703 return assignNew('V', mce, Ity_I64,
2705 mkPCast16x4(mce, vatom1),
2706 mkPCast16x4(mce, vatom2)));
2712 return assignNew('V', mce, Ity_I64,
2714 mkPCast8x8(mce, vatom1),
2715 mkPCast8x8(mce, vatom2)));
2719 return mkPCast32x2(mce,
2720 assignNew('V', mce, Ity_I64,
2722 mkPCast32x2(mce, vatom1),
2723 mkPCast32x2(mce, vatom2))));
2726 return mkPCast16x4(mce,
2727 assignNew('V', mce, Ity_I64,
2728 binop(op, mkPCast16x4(mce, vatom1),
2729 mkPCast16x4(mce, vatom2))));
2732 return mkPCast8x8(mce,
2733 assignNew('V', mce, Ity_I64,
2734 binop(op, mkPCast8x8(mce, vatom1),
2735 mkPCast8x8(mce, vatom2))));
2741 return mkUifU64(mce,
2742 assignNew('V', mce, Ity_I64, binop(op, vatom1, atom2)),
2743 mkPCast8x8(mce,vatom2)
2750 return mkUifU64(mce,
2751 assignNew('V', mce, Ity_I64, binop(op, vatom1, atom2)),
2752 mkPCast16x4(mce,vatom2)
2759 return mkUifU64(mce,
2760 assignNew('V', mce, Ity_I64, binop(op, vatom1, atom2)),
2761 mkPCast32x2(mce,vatom2)
2779 return assignNew('V', mce, Ity_I64, binop(op, vatom1, vatom2));
2782 complainIfUndefined(mce, atom2, NULL);
2783 return assignNew('V', mce, Ity_I8, binop(op, vatom1, atom2));
2785 complainIfUndefined(mce, atom2, NULL);
2786 return assignNew('V', mce, Ity_I16, binop(op, vatom1, atom2));
2788 complainIfUndefined(mce, atom2, NULL);
2789 return assignNew('V', mce, Ity_I32, binop(op, vatom1, atom2));
2796 mce,
2797 assignNew('V', mce, Ity_I64, binop(op, vatom1, atom2)),
2798 mkPCast8x8(mce, vatom2)
2818 complainIfUndefined(mce, atom2, NULL);
2819 return assignNew('V', mce, Ity_V128, binop(op, vatom1, atom2));
2827 return mkUifUV128(mce,
2828 assignNew('V', mce, Ity_V128, binop(op, vatom1, atom2)),
2829 mkPCast8x16(mce,vatom2)
2837 return mkUifUV128(mce,
2838 assignNew('V', mce, Ity_V128, binop(op, vatom1, atom2)),
2839 mkPCast16x8(mce,vatom2)
2847 return mkUifUV128(mce,
2848 assignNew('V', mce, Ity_V128, binop(op, vatom1, atom2)),
2849 mkPCast32x4(mce,vatom2)
2856 return mkUifUV128(mce,
2857 assignNew('V', mce, Ity_V128, binop(op, vatom1, atom2)),
2858 mkPCast64x2(mce,vatom2)
2865 complainIfUndefined(mce, atom2, NULL);
2866 return mkPCast32x4(mce, vatom1);
2872 complainIfUndefined(mce, atom2, NULL);
2873 return mkPCast32x2(mce, vatom1);
2894 return binary8Ix16(mce, vatom1, vatom2);
2918 return binary16Ix8(mce, vatom1, vatom2);
2940 return binary32Ix4(mce, vatom1, vatom2);
2952 return binary64Ix2(mce, vatom1, vatom2);
2960 return vectorNarrowBinV128(mce, op, vatom1, vatom2);
2972 return binary64Fx2(mce, vatom1, vatom2);
2984 return binary64F0x2(mce, vatom1, vatom2);
3000 return binary32Fx4(mce, vatom1, vatom2);
3012 return binary32Fx2(mce, vatom1, vatom2);
3024 return binary32F0x4(mce, vatom1, vatom2);
3029 complainIfUndefined(mce, atom2, NULL);
3030 return mkPCast8x16(mce, vatom1);
3035 complainIfUndefined(mce, atom2, NULL);
3036 return mkPCast16x8(mce, vatom1);
3041 complainIfUndefined(mce, atom2, NULL);
3042 return mkPCast32x4(mce, vatom1);
3047 complainIfUndefined(mce, atom2, NULL);
3048 return mkPCast32x4(mce, vatom1);
3053 return vectorWidenI64(mce, Iop_Widen32Sto64x2,
3054 mkUifU64(mce, vatom1, vatom2));
3059 return vectorWidenI64(mce, Iop_Widen16Sto32x4,
3060 mkUifU64(mce, vatom1, vatom2));
3065 return vectorWidenI64(mce, Iop_Widen8Sto16x8,
3066 mkUifU64(mce, vatom1, vatom2));
3069 return mkPCast32x4(mce,
3070 assignNew('V', mce, Ity_V128, binop(op, mkPCast32x4(mce, vatom1),
3071 mkPCast32x4(mce, vatom2))));
3074 return mkPCast16x8(mce,
3075 assignNew('V', mce, Ity_V128, binop(op, mkPCast16x8(mce, vatom1),
3076 mkPCast16x8(mce, vatom2))));
3079 return mkPCast8x16(mce,
3080 assignNew('V', mce, Ity_V128, binop(op, mkPCast8x16(mce, vatom1),
3081 mkPCast8x16(mce, vatom2))));
3107 return assignNew('V', mce, Ity_V128, binop(op, vatom1, vatom2));
3110 complainIfUndefined(mce, atom2, NULL);
3111 return assignNew('V', mce, Ity_I8, binop(op, vatom1, atom2));
3113 complainIfUndefined(mce, atom2, NULL);
3114 return assignNew('V', mce, Ity_I16, binop(op, vatom1, atom2));
3116 complainIfUndefined(mce, atom2, NULL);
3117 return assignNew('V', mce, Ity_I32, binop(op, vatom1, atom2));
3119 complainIfUndefined(mce, atom2, NULL);
3120 return assignNew('V', mce, Ity_I64, binop(op, vatom1, atom2));
3127 mce,
3128 assignNew('V', mce, Ity_V128, binop(op, vatom1, atom2)),
3129 mkPCast8x16(mce, vatom2)
3133 mce,
3134 assignNew('V', mce, Ity_V128, binop(op, vatom1, atom2)),
3135 mkPCast32x4(mce, vatom2)
3148 at = binary16Ix8(mce,vatom1,vatom2);
3149 at = assignNew('V', mce, Ity_V128, binop(Iop_ShlN32x4, at, mkU8(16)));
3150 at = assignNew('V', mce, Ity_V128, binop(Iop_SarN32x4, at, mkU8(16)));
3158 at = binary8Ix16(mce,vatom1,vatom2);
3159 at = assignNew('V', mce, Ity_V128, binop(Iop_ShlN16x8, at, mkU8(8)));
3160 at = assignNew('V', mce, Ity_V128, binop(Iop_SarN16x8, at, mkU8(8)));
3170 return assignNew('V', mce, Ity_V128,
3178 complainIfUndefined(mce, atom2, NULL);
3179 return assignNew('V', mce, Ity_V128, binop(op, vatom1, atom2));
3183 return assignNew('V', mce, Ity_I128, binop(op, vatom1, vatom2));
3193 return binary64Fx4(mce, vatom1, vatom2);
3201 return binary32Fx8(mce, vatom1, vatom2);
3205 return assignNew('V', mce, Ity_V256, binop(op, vatom1, vatom2));
3211 return mkLazy2(mce, Ity_I64, vatom1, vatom2);
3215 return mkLazy2(mce, Ity_I32, vatom1, vatom2);
3229 return mkLazy2(mce, Ity_I64, vatom1, vatom2);
3235 return mkLazy2(mce, Ity_I64, vatom1, vatom2);
3241 return mkLazy2(mce, Ity_I128, vatom1, vatom2);
3246 return mkLazy2(mce, Ity_I64, vatom1, vatom2);
3251 return mkLazy2(mce, Ity_I32, vatom1, vatom2);
3255 return mkLazy2(mce, Ity_I128, vatom1, vatom2);
3260 return mkLazy2(mce, Ity_I32, vatom1, vatom2);
3264 return mkLazy2(mce, Ity_I32, vatom1, vatom2);
3270 return mkLazy2(mce, Ity_I64, vatom1, vatom2);
3274 return assignNew('V', mce, Ity_I128,
3282 return mkLazy2(mce, Ity_I32, vatom1, vatom2);
3286 return mkLazy2(mce, Ity_I64, vatom1, vatom2);
3290 return mkLazy2(mce, Ity_I16, vatom1, vatom2);
3294 return mkLazy2(mce, Ity_I64, vatom1, vatom2);
3298 return mkLazy2(mce, Ity_I128, vatom1, vatom2);
3305 return mkLazy2(mce, Ity_I32, vatom1, vatom2);
3311 return mkLazy2(mce, Ity_I64, vatom1, vatom2);
3315 return mkLazy2(mce, Ity_I128, vatom1, vatom2);
3318 return assignNew('V', mce, Ity_I32, binop(op, vatom1, vatom2));
3320 return assignNew('V', mce, Ity_I64, binop(op, vatom1, vatom2));
3325 IRAtom* vLo64 = mkLeft64(mce, mkUifU64(mce, vatom1,vatom2));
3326 IRAtom* vHi64 = mkPCastTo(mce, Ity_I64, vLo64);
3327 return assignNew('V', mce, Ity_I128,
3333 IRAtom* vLo32 = mkLeft32(mce, mkUifU32(mce, vatom1,vatom2));
3334 IRAtom* vHi32 = mkPCastTo(mce, Ity_I32, vLo32);
3335 return assignNew('V', mce, Ity_I64,
3341 IRAtom* vLo16 = mkLeft16(mce, mkUifU16(mce, vatom1,vatom2));
3342 IRAtom* vHi16 = mkPCastTo(mce, Ity_I16, vLo16);
3343 return assignNew('V', mce, Ity_I32,
3349 IRAtom* vLo8 = mkLeft8(mce, mkUifU8(mce, vatom1,vatom2));
3350 IRAtom* vHi8 = mkPCastTo(mce, Ity_I8, vLo8);
3351 return assignNew('V', mce, Ity_I16, binop(Iop_8HLto16, vHi8, vLo8));
3361 return mkLazy2(mce, Ity_I32, vatom1, vatom2);
3367 return mkLazy2(mce, Ity_I64, vatom1, vatom2);
3370 if (mce->bogusLiterals || mce->useLLVMworkarounds)
3371 return expensiveAddSub(mce,True,Ity_I32,
3376 if (mce->bogusLiterals)
3377 return expensiveAddSub(mce,False,Ity_I32,
3384 return mkLeft32(mce, mkUifU32(mce, vatom1,vatom2));
3390 return doCmpORD(mce, op, vatom1,vatom2, atom1,atom2);
3393 if (mce->bogusLiterals || mce->useLLVMworkarounds)
3394 return expensiveAddSub(mce,True,Ity_I64,
3399 if (mce->bogusLiterals)
3400 return expensiveAddSub(mce,False,Ity_I64,
3407 return mkLeft64(mce, mkUifU64(mce, vatom1,vatom2));
3412 return mkLeft16(mce, mkUifU16(mce, vatom1,vatom2));
3416 return mkLeft8(mce, mkUifU8(mce, vatom1,vatom2));
3420 if (mce->bogusLiterals)
3421 return expensiveCmpEQorNE(mce,Ity_I64, vatom1,vatom2, atom1,atom2 );
3427 return mkPCastTo(mce, Ity_I1, mkUifU64(mce, vatom1,vatom2));
3431 if (mce->bogusLiterals)
3432 return expensiveCmpEQorNE(mce,Ity_I32, vatom1,vatom2, atom1,atom2 );
3438 return mkPCastTo(mce, Ity_I1, mkUifU32(mce, vatom1,vatom2));
3441 return mkPCastTo(mce, Ity_I1, mkUifU16(mce, vatom1,vatom2));
3444 return mkPCastTo(mce, Ity_I1, mkUifU8(mce, vatom1,vatom2));
3452 return assignNew('V', mce, Ity_I1, definedOfType(Ity_I1));
3455 return scalarShift( mce, Ity_I64, op, vatom1,vatom2, atom1,atom2 );
3458 return scalarShift( mce, Ity_I32, op, vatom1,vatom2, atom1,atom2 );
3461 return scalarShift( mce, Ity_I16, op, vatom1,vatom2, atom1,atom2 );
3464 return scalarShift( mce, Ity_I8, op, vatom1,vatom2, atom1,atom2 );
3507 'V', mce,
3509 difd(mce, uifu(mce, vatom1, vatom2),
3510 difd(mce, improve(mce, atom1, vatom1),
3511 improve(mce, atom2, vatom2) ) ) );
3514 return mkUifU8(mce, vatom1, vatom2);
3516 return mkUifU16(mce, vatom1, vatom2);
3518 return mkUifU32(mce, vatom1, vatom2);
3520 return mkUifU64(mce, vatom1, vatom2);
3522 return mkUifUV128(mce, vatom1, vatom2);
3524 return mkUifUV256(mce, vatom1, vatom2);
3534 IRExpr* expr2vbits_Unop ( MCEnv* mce, IROp op, IRAtom* atom )
3536 IRAtom* vatom = expr2vbits( mce, atom );
3537 tl_assert(isOriginalAtom(mce,atom));
3541 return unary64Fx2(mce, vatom);
3544 return unary64F0x2(mce, vatom);
3549 return unary32Fx8(mce, vatom);
3552 return unary64Fx4(mce, vatom);
3569 return unary32Fx4(mce, vatom);
3578 return unary32Fx2(mce, vatom);
3583 return unary32F0x4(mce, vatom);
3597 return assignNew('V', mce, Ity_V128, unop(op, vatom));
3601 return assignNew('V', mce, Ity_I64, unop(Iop_128HIto64, vatom));
3604 return assignNew('V', mce, Ity_I64, unop(Iop_128to64, vatom));
3608 return mkPCastTo(mce, Ity_I128, vatom);
3615 return mkPCastTo(mce, Ity_I128, vatom);
3634 return mkPCastTo(mce, Ity_I64, vatom);
3637 return mkPCastTo(mce, Ity_I128, vatom);
3644 return mkPCastTo(mce, Ity_I32, vatom);
3669 return assignNew('V', mce, Ity_I64, unop(op, vatom));
3681 return assignNew('V', mce, Ity_I32, unop(op, vatom));
3688 return assignNew('V', mce, Ity_I16, unop(op, vatom));
3696 return assignNew('V', mce, Ity_I8, unop(op, vatom));
3699 return assignNew('V', mce, Ity_I1, unop(Iop_32to1, vatom));
3702 return assignNew('V', mce, Ity_I1, unop(Iop_64to1, vatom));
3724 return mkPCast8x8(mce, vatom);
3731 return mkPCast8x16(mce, vatom);
3737 return mkPCast16x4(mce, vatom);
3743 return mkPCast16x8(mce, vatom);
3751 return mkPCast32x2(mce, vatom);
3759 return mkPCast32x4(mce, vatom);
3762 return mkPCastTo(mce, Ity_I64, vatom);
3765 return mkPCast64x2(mce, vatom);
3779 return vectorNarrowUnV128(mce, op, vatom);
3787 return vectorWidenI64(mce, op, vatom);
3791 return mkPCastTo(mce, Ity_I64,
3792 assignNew('V', mce, Ity_I64, unop(op, mkPCast32x2(mce, vatom))));
3796 return mkPCast32x2(mce,
3797 assignNew('V', mce, Ity_I64, unop(op, mkPCast16x4(mce, vatom))));
3801 return mkPCast16x4(mce,
3802 assignNew('V', mce, Ity_I64, unop(op, mkPCast8x8(mce, vatom))));
3806 return mkPCast64x2(mce,
3807 assignNew('V', mce, Ity_V128, unop(op, mkPCast32x4(mce, vatom))));
3811 return mkPCast32x4(mce,
3812 assignNew('V', mce, Ity_V128, unop(op, mkPCast16x8(mce, vatom))));
3816 return mkPCast16x8(mce,
3817 assignNew('V', mce, Ity_V128, unop(op, mkPCast8x16(mce, vatom))));
3829 IRAtom* expr2vbits_Load_WRK ( MCEnv* mce,
3839 tl_assert(isOriginalAtom(mce,addr));
3844 complainIfUndefined( mce, addr, NULL );
3892 IRType tyAddr = mce->hWordTy;
3896 addrAct = assignNew('V', mce, tyAddr, binop(mkAdd, addr, eBias) );
3901 datavbits = newTemp(mce, ty, VSh);
3906 setHelperAnns( mce, di );
3907 stmt( 'V', mce, IRStmt_Dirty(di) );
3914 IRAtom* expr2vbits_Load ( MCEnv* mce,
3924 return expr2vbits_Load_WRK(mce, end, ty, addr, bias);
3928 v64lo = expr2vbits_Load_WRK(mce, end, Ity_I64, addr, bias+0);
3929 v64hi = expr2vbits_Load_WRK(mce, end, Ity_I64, addr, bias+8);
3931 v64hi = expr2vbits_Load_WRK(mce, end, Ity_I64, addr, bias+0);
3932 v64lo = expr2vbits_Load_WRK(mce, end, Ity_I64, addr, bias+8);
3934 return assignNew( 'V', mce,
3942 IRAtom* v64Q0 = expr2vbits_Load_WRK(mce, end, Ity_I64, addr, bias+0);
3943 IRAtom* v64Q1 = expr2vbits_Load_WRK(mce, end, Ity_I64, addr, bias+8);
3944 IRAtom* v64Q2 = expr2vbits_Load_WRK(mce, end, Ity_I64, addr, bias+16);
3945 IRAtom* v64Q3 = expr2vbits_Load_WRK(mce, end, Ity_I64, addr, bias+24);
3946 return assignNew( 'V', mce,
3964 IRAtom* expr2vbits_guarded_Load ( MCEnv* mce,
3971 cond = assignNew('V', mce, Ity_I8, unop(Iop_1Uto8, guard));
3972 iftrue = assignNew('V', mce, ty,
3973 expr2vbits_Load(mce, end, ty, addr, bias));
3974 iffalse = assignNew('V', mce, ty, definedOfType(ty));
3976 return assignNew('V', mce, ty, IRExpr_Mux0X(cond, iffalse, iftrue));
3980 return expr2vbits_Load(mce, end, ty, addr, bias);
3985 IRAtom* expr2vbits_Mux0X ( MCEnv* mce,
3995 tl_assert(isOriginalAtom(mce, cond));
3996 tl_assert(isOriginalAtom(mce, expr0));
3997 tl_assert(isOriginalAtom(mce, exprX));
3999 vbitsC = expr2vbits(mce, cond);
4000 vbits0 = expr2vbits(mce, expr0);
4001 vbitsX = expr2vbits(mce, exprX);
4002 ty = typeOfIRExpr(mce->sb->tyenv, vbits0);
4005 mkUifU(mce, ty, assignNew('V', mce, ty,
4007 mkPCastTo(mce, ty, vbitsC) );
4013 IRExpr* expr2vbits ( MCEnv* mce, IRExpr* e )
4018 return shadow_GET( mce, e->Iex.Get.offset, e->Iex.Get.ty );
4021 return shadow_GETI( mce, e->Iex.GetI.descr,
4025 return IRExpr_RdTmp( findShadowTmpV(mce, e->Iex.RdTmp.tmp) );
4028 return definedOfType(shadowTypeV(typeOfIRExpr(mce->sb->tyenv, e)));
4032 mce,
4040 mce,
4048 mce,
4054 return expr2vbits_Unop( mce, e->Iex.Unop.op, e->Iex.Unop.arg );
4057 return expr2vbits_Load( mce, e->Iex.Load.end,
4062 return mkLazyN( mce, e->Iex.CCall.args,
4067 return expr2vbits_Mux0X( mce, e->Iex.Mux0X.cond, e->Iex.Mux0X.expr0,
4085 IRExpr* zwidenToHostWord ( MCEnv* mce, IRAtom* vatom )
4090 tl_assert(isShadowAtom(mce,vatom));
4092 ty = typeOfIRExpr(mce->sb->tyenv, vatom);
4093 tyH = mce->hWordTy;
4100 return assignNew('V', mce, tyH, unop(Iop_16Uto32, vatom));
4102 return assignNew('V', mce, tyH, unop(Iop_8Uto32, vatom));
4110 return assignNew('V', mce, tyH, unop(Iop_32Uto64, vatom));
4112 return assignNew('V', mce, tyH, unop(Iop_32Uto64,
4113 assignNew('V', mce, Ity_I32, unop(Iop_16Uto32, vatom))));
4115 return assignNew('V', mce, tyH, unop(Iop_32Uto64,
4116 assignNew('V', mce, Ity_I32, unop(Iop_8Uto32, vatom))));
4137 void do_shadow_Store ( MCEnv* mce,
4149 tyAddr = mce->hWordTy;
4156 tl_assert(isOriginalAtom(mce, data));
4158 vdata = expr2vbits( mce, data );
4163 tl_assert(isOriginalAtom(mce,addr));
4164 tl_assert(isShadowAtom(mce,vdata));
4167 tl_assert(isOriginalAtom(mce, guard));
4168 tl_assert(typeOfIRExpr(mce->sb->tyenv, guard) == Ity_I1);
4171 ty = typeOfIRExpr(mce->sb->tyenv, vdata);
4193 complainIfUndefined( mce, addr, guard );
4256 addrQ0 = assignNew('V', mce, tyAddr, binop(mkAdd, addr, eBiasQ0) );
4257 vdataQ0 = assignNew('V', mce, Ity_I64, unop(Iop_V256to64_0, vdata));
4265 addrQ1 = assignNew('V', mce, tyAddr, binop(mkAdd, addr, eBiasQ1) );
4266 vdataQ1 = assignNew('V', mce, Ity_I64, unop(Iop_V256to64_1, vdata));
4274 addrQ2 = assignNew('V', mce, tyAddr, binop(mkAdd, addr, eBiasQ2) );
4275 vdataQ2 = assignNew('V', mce, Ity_I64, unop(Iop_V256to64_2, vdata));
4283 addrQ3 = assignNew('V', mce, tyAddr, binop(mkAdd, addr, eBiasQ3) );
4284 vdataQ3 = assignNew('V', mce, Ity_I64, unop(Iop_V256to64_3, vdata));
4294 setHelperAnns( mce, diQ0 );
4295 setHelperAnns( mce, diQ1 );
4296 setHelperAnns( mce, diQ2 );
4297 setHelperAnns( mce, diQ3 );
4298 stmt( 'V', mce, IRStmt_Dirty(diQ0) );
4299 stmt( 'V', mce, IRStmt_Dirty(diQ1) );
4300 stmt( 'V', mce, IRStmt_Dirty(diQ2) );
4301 stmt( 'V', mce, IRStmt_Dirty(diQ3) );
4325 addrLo64 = assignNew('V', mce, tyAddr, binop(mkAdd, addr, eBiasLo64) );
4326 vdataLo64 = assignNew('V', mce, Ity_I64, unop(Iop_V128to64, vdata));
4333 addrHi64 = assignNew('V', mce, tyAddr, binop(mkAdd, addr, eBiasHi64) );
4334 vdataHi64 = assignNew('V', mce, Ity_I64, unop(Iop_V128HIto64, vdata));
4342 setHelperAnns( mce, diLo64 );
4343 setHelperAnns( mce, diHi64 );
4344 stmt( 'V', mce, IRStmt_Dirty(diLo64) );
4345 stmt( 'V', mce, IRStmt_Dirty(diHi64) );
4358 addrAct = assignNew('V', mce, tyAddr, binop(mkAdd, addr, eBias));
4375 zwidenToHostWord( mce, vdata ))
4379 setHelperAnns( mce, di );
4380 stmt( 'V', mce, IRStmt_Dirty(di) );
4402 void do_shadow_Dirty ( MCEnv* mce, IRDirty* d )
4420 complainIfUndefined(mce, d->guard, NULL);
4431 here = mkPCastTo( mce, Ity_I32, expr2vbits(mce, d->args[i]) );
4432 curr = mkUifU32(mce, here, curr);
4448 if (isAlwaysDefd(mce, gOff, gSz)) {
4470 cond = assignNew('V', mce, Ity_I8, unop(Iop_1Uto8, d->guard));
4471 iftrue = assignNew('V', mce, tySrc, shadow_GET(mce, gOff, tySrc));
4472 iffalse = assignNew('V', mce, tySrc, definedOfType(tySrc));
4473 src = assignNew('V', mce, tySrc,
4476 here = mkPCastTo( mce, Ity_I32, src );
4477 curr = mkUifU32(mce, here, curr);
4494 complainIfUndefined(mce, d->mAddr, d->guard);
4496 tyAddr = typeOfIRExpr(mce->sb->tyenv, d->mAddr);
4498 tl_assert(tyAddr == mce->hWordTy); /* not really right */
4510 mce, Ity_I32,
4511 expr2vbits_guarded_Load ( mce, end, Ity_I32, d->mAddr,
4514 curr = mkUifU32(mce, here, curr);
4520 mce, Ity_I32,
4521 expr2vbits_guarded_Load ( mce, end, Ity_I16, d->mAddr,
4524 curr = mkUifU32(mce, here, curr);
4530 mce, Ity_I32,
4531 expr2vbits_guarded_Load ( mce, end, Ity_I8, d->mAddr,
4534 curr = mkUifU32(mce, here, curr);
4546 dst = findShadowTmpV(mce, d->tmp);
4547 tyDst = typeOfIRTemp(mce->sb->tyenv, d->tmp);
4548 assign( 'V', mce, dst, mkPCastTo( mce, tyDst, curr) );
4563 if (isAlwaysDefd(mce, gOff, gSz))
4576 do_shadow_PUT( mce, gOff,
4578 mkPCastTo( mce, tyDst, curr ), d->guard );
4591 do_shadow_Store( mce, end, d->mAddr, d->mSize - toDo,
4593 mkPCastTo( mce, Ity_I32, curr ),
4599 do_shadow_Store( mce, end, d->mAddr, d->mSize - toDo,
4601 mkPCastTo( mce, Ity_I16, curr ),
4607 do_shadow_Store( mce, end, d->mAddr, d->mSize - toDo,
4609 mkPCastTo( mce, Ity_I8, curr ),
4628 void do_AbiHint ( MCEnv* mce, IRExpr* base, Int len, IRExpr* nia )
4645 stmt( 'V', mce, IRStmt_Dirty(di) );
4652 static IRAtom* gen_load_b ( MCEnv* mce, Int szB,
4654 static IRAtom* gen_maxU32 ( MCEnv* mce, IRAtom* b1, IRAtom* b2 );
4655 static void gen_store_b ( MCEnv* mce, Int szB,
4659 static void do_shadow_CAS_single ( MCEnv* mce, IRCAS* cas );
4660 static void do_shadow_CAS_double ( MCEnv* mce, IRCAS* cas );
4687 MCEnv* mce,
4690 tl_assert(isOriginalAtom(mce, orig));
4691 tl_assert(isShadowAtom(mce, shadow));
4699 assign('V', mce, findShadowTmpV(mce,orig->Iex.RdTmp.tmp),
4703 assign('B', mce, findShadowTmpB(mce,orig->Iex.RdTmp.tmp),
4714 void do_shadow_CAS ( MCEnv* mce, IRCAS* cas )
4840 do_shadow_CAS_single( mce, cas );
4842 do_shadow_CAS_double( mce, cas );
4847 static void do_shadow_CAS_single ( MCEnv* mce, IRCAS* cas )
4863 elemTy = typeOfIRExpr(mce->sb->tyenv, cas->expdLo);
4873 tl_assert(isOriginalAtom(mce, cas->dataLo));
4875 = assignNew('V', mce, elemTy, expr2vbits(mce, cas->dataLo));
4876 tl_assert(isShadowAtom(mce, vdataLo));
4879 = assignNew('B', mce, Ity_I32, schemeE(mce, cas->dataLo));
4880 tl_assert(isShadowAtom(mce, bdataLo));
4884 tl_assert(isOriginalAtom(mce, cas->expdLo));
4886 = assignNew('V', mce, elemTy, expr2vbits(mce, cas->expdLo));
4887 tl_assert(isShadowAtom(mce, vexpdLo));
4890 = assignNew('B', mce, Ity_I32, schemeE(mce, cas->expdLo));
4891 tl_assert(isShadowAtom(mce, bexpdLo));
4899 'V', mce, elemTy,
4901 mce,
4904 bind_shadow_tmp_to_orig('V', mce, mkexpr(cas->oldLo), voldLo);
4907 = assignNew('B', mce, Ity_I32,
4908 gen_load_b(mce, elemSzB, cas->addr, 0/*addr bias*/));
4909 bind_shadow_tmp_to_orig('B', mce, mkexpr(cas->oldLo), boldLo);
4913 stmt( 'C', mce, IRStmt_CAS(cas) );
4920 = assignNew('C', mce, Ity_I1,
4925 do_shadow_Store( mce, cas->end, cas->addr, 0/*bias*/,
4929 gen_store_b( mce, elemSzB, cas->addr, 0/*offset*/,
4936 static void do_shadow_CAS_double ( MCEnv* mce, IRCAS* cas )
4956 elemTy = typeOfIRExpr(mce->sb->tyenv, cas->expdLo);
4979 tl_assert(isOriginalAtom(mce, cas->dataHi));
4980 tl_assert(isOriginalAtom(mce, cas->dataLo));
4982 = assignNew('V', mce, elemTy, expr2vbits(mce, cas->dataHi));
4984 = assignNew('V', mce, elemTy, expr2vbits(mce, cas->dataLo));
4985 tl_assert(isShadowAtom(mce, vdataHi));
4986 tl_assert(isShadowAtom(mce, vdataLo));
4989 = assignNew('B', mce, Ity_I32, schemeE(mce, cas->dataHi));
4991 = assignNew('B', mce, Ity_I32, schemeE(mce, cas->dataLo));
4992 tl_assert(isShadowAtom(mce, bdataHi));
4993 tl_assert(isShadowAtom(mce, bdataLo));
4997 tl_assert(isOriginalAtom(mce, cas->expdHi));
4998 tl_assert(isOriginalAtom(mce, cas->expdLo));
5000 = assignNew('V', mce, elemTy, expr2vbits(mce, cas->expdHi));
5002 = assignNew('V', mce, elemTy, expr2vbits(mce, cas->expdLo));
5003 tl_assert(isShadowAtom(mce, vexpdHi));
5004 tl_assert(isShadowAtom(mce, vexpdLo));
5007 = assignNew('B', mce, Ity_I32, schemeE(mce, cas->expdHi));
5009 = assignNew('B', mce, Ity_I32, schemeE(mce, cas->expdLo));
5010 tl_assert(isShadowAtom(mce, bexpdHi));
5011 tl_assert(isShadowAtom(mce, bexpdLo));
5027 'V', mce, elemTy,
5029 mce,
5034 'V', mce, elemTy,
5036 mce,
5039 bind_shadow_tmp_to_orig('V', mce, mkexpr(cas->oldHi), voldHi);
5040 bind_shadow_tmp_to_orig('V', mce, mkexpr(cas->oldLo), voldLo);
5043 = assignNew('B', mce, Ity_I32,
5044 gen_load_b(mce, elemSzB, cas->addr,
5047 = assignNew('B', mce, Ity_I32,
5048 gen_load_b(mce, elemSzB, cas->addr,
5050 bind_shadow_tmp_to_orig('B', mce, mkexpr(cas->oldHi), boldHi);
5051 bind_shadow_tmp_to_orig('B', mce, mkexpr(cas->oldLo), boldLo);
5055 stmt( 'C', mce, IRStmt_CAS(cas) );
5067 xHi = assignNew('C', mce, elemTy,
5069 xLo = assignNew('C', mce, elemTy,
5071 xHL = assignNew('C', mce, elemTy,
5074 = assignNew('C', mce, Ity_I1,
5079 do_shadow_Store( mce, cas->end, cas->addr, memOffsHi/*bias*/,
5082 do_shadow_Store( mce, cas->end, cas->addr, memOffsLo/*bias*/,
5086 gen_store_b( mce, elemSzB, cas->addr, memOffsHi/*offset*/,
5089 gen_store_b( mce, elemSzB, cas->addr, memOffsLo/*offset*/,
5098 static void do_shadow_LLSC ( MCEnv* mce,
5108 IRType resTy = typeOfIRTemp(mce->sb->tyenv, stResult);
5109 IRTemp resTmp = findShadowTmpV(mce, stResult);
5122 assign( 'V', mce, resTmp,
5124 mce, stEnd, resTy, stAddr, 0/*addr bias*/));
5128 IRType dataTy = typeOfIRExpr(mce->sb->tyenv,
5132 do_shadow_Store( mce, stEnd,
5153 assign( 'V', mce, resTmp, definedOfType(resTy) );
5162 static void schemeS ( MCEnv* mce, IRStmt* st );
5299 MCEnv mce;
5326 VG_(memset)(&mce, 0, sizeof(mce));
5327 mce.sb = sb_out;
5328 mce.trace = verboze;
5329 mce.layout = layout;
5330 mce.hWordTy = hWordTy;
5331 mce.bogusLiterals = False;
5342 mce.useLLVMworkarounds = False;
5344 mce.useLLVMworkarounds = True;
5347 mce.tmpMap = VG_(newXA)( VG_(malloc), "mc.MC_(instrument).1", VG_(free),
5354 VG_(addToXA)( mce.tmpMap, &ent );
5356 tl_assert( VG_(sizeXA)( mce.tmpMap ) == sb_in->tyenv->types_used );
5383 mce.bogusLiterals = bogus;
5387 tl_assert(mce.sb == sb_out);
5388 tl_assert(mce.sb != sb_in);
5397 stmt( 'C', &mce, sb_in->stmts[i] );
5428 IRTemp tmp_v = findShadowTmpV(&mce, tmp_o);
5430 assign( 'V', &mce, tmp_v, definedOfType( ty_v ) );
5432 IRTemp tmp_b = findShadowTmpB(&mce, tmp_o);
5434 assign( 'B', &mce, tmp_b, mkU32(0)/* UNKNOWN ORIGIN */);
5465 schemeS( &mce, st );
5473 assign( 'V', &mce, findShadowTmpV(&mce, st->Ist.WrTmp.tmp),
5474 expr2vbits( &mce, st->Ist.WrTmp.data) );
5478 do_shadow_PUT( &mce,
5485 do_shadow_PUTI( &mce, st->Ist.PutI.details);
5489 do_shadow_Store( &mce, st->Ist.Store.end,
5497 complainIfUndefined( &mce, st->Ist.Exit.guard, NULL );
5508 do_shadow_Dirty( &mce, st->Ist.Dirty.details );
5512 do_AbiHint( &mce, st->Ist.AbiHint.base,
5518 do_shadow_CAS( &mce, st->Ist.CAS.details );
5528 do_shadow_LLSC( &mce,
5556 stmt('C', &mce, st);
5568 complainIfUndefined( &mce, sb_in->next, NULL );
5581 tl_assert( VG_(sizeXA)( mce.tmpMap ) == mce.sb->tyenv->types_used );
5582 VG_(deleteXA)( mce.tmpMap );
5584 tl_assert(mce.sb == sb_out);
5741 static IRTemp findShadowTmpB ( MCEnv* mce, IRTemp orig )
5746 ent = (TempMapEnt*)VG_(indexXA)( mce->tmpMap, (Word)orig );
5750 = newTemp( mce, Ity_I32, BSh );
5751 /* newTemp may cause mce->tmpMap to resize, hence previous results
5753 ent = (TempMapEnt*)VG_(indexXA)( mce->tmpMap, (Word)orig );
5761 static IRAtom* gen_maxU32 ( MCEnv* mce, IRAtom* b1, IRAtom* b2 )
5763 return assignNew( 'B', mce, Ity_I32, binop(Iop_Max32U, b1, b2) );
5766 static IRAtom* gen_load_b ( MCEnv* mce, Int szB,
5773 IRType aTy = typeOfIRExpr( mce->sb->tyenv, baseaddr );
5779 ea = assignNew( 'B', mce, aTy, binop(opAdd, ea, off));
5781 bTmp = newTemp(mce, mce->hWordTy, BSh);
5812 stmt( 'B', mce, IRStmt_Dirty(di) );
5813 if (mce->hWordTy == Ity_I64) {
5815 IRTemp bTmp32 = newTemp(mce, Ity_I32, BSh);
5816 assign( 'B', mce, bTmp32, unop(Iop_64to32, mkexpr(bTmp)) );
5824 static IRAtom* gen_guarded_load_b ( MCEnv* mce, Int szB, IRAtom* baseaddr,
5830 cond = assignNew('B', mce, Ity_I8, unop(Iop_1Uto8, guard));
5831 iftrue = assignNew('B', mce, Ity_I32,
5832 gen_load_b(mce, szB, baseaddr, offset));
5835 return assignNew('B', mce, Ity_I32, IRExpr_Mux0X(cond, iffalse, iftrue));
5838 return gen_load_b(mce, szB, baseaddr, offset);
5843 static void gen_store_b ( MCEnv* mce, Int szB,
5850 IRType aTy = typeOfIRExpr( mce->sb->tyenv, baseaddr );
5854 tl_assert(isOriginalAtom(mce, guard));
5855 tl_assert(typeOfIRExpr(mce->sb->tyenv, guard) == Ity_I1);
5860 ea = assignNew( 'B', mce, aTy, binop(opAdd, ea, off));
5862 if (mce->hWordTy == Ity_I64)
5863 dataB = assignNew( 'B', mce, Ity_I64, unop(Iop_32Uto64, dataB));
5894 stmt( 'B', mce, IRStmt_Dirty(di) );
5897 static IRAtom* narrowTo32 ( MCEnv* mce, IRAtom* e ) {
5898 IRType eTy = typeOfIRExpr(mce->sb->tyenv, e);
5900 return assignNew( 'B', mce, Ity_I32, unop(Iop_64to32, e) );
5906 static IRAtom* zWidenFrom32 ( MCEnv* mce, IRType dstTy, IRAtom* e ) {
5907 IRType eTy = typeOfIRExpr(mce->sb->tyenv, e);
5910 return assignNew( 'B', mce, Ity_I64, unop(Iop_32Uto64, e) );
5915 static IRAtom* schemeE ( MCEnv* mce, IRExpr* e )
5933 descr_b = mkIRRegArray( descr->base + 2*mce->layout->total_sizeB,
5941 t1 = assignNew( 'B', mce, equivIntTy,
5944 t2 = narrowTo32( mce, t1 );
5945 t3 = schemeE( mce, e->Iex.GetI.ix );
5946 t4 = gen_maxU32( mce, t2, t3 );
5956 tl_assert(isOriginalAtom(mce, args[i]));
5967 here = schemeE( mce, args[i] );
5968 curr = gen_maxU32( mce, curr, here );
5979 tl_assert(mce->hWordTy == Ity_I32 || mce->hWordTy == Ity_I64);
5980 return gen_load_b( mce, dszB, e->Iex.Load.addr, 0 );
5983 IRAtom* b1 = schemeE( mce, e->Iex.Mux0X.cond );
5984 IRAtom* b2 = schemeE( mce, e->Iex.Mux0X.expr0 );
5985 IRAtom* b3 = schemeE( mce, e->Iex.Mux0X.exprX );
5986 return gen_maxU32( mce, b1, gen_maxU32( mce, b2, b3 ));
5989 IRAtom* b1 = schemeE( mce, e->Iex.Qop.details->arg1 );
5990 IRAtom* b2 = schemeE( mce, e->Iex.Qop.details->arg2 );
5991 IRAtom* b3 = schemeE( mce, e->Iex.Qop.details->arg3 );
5992 IRAtom* b4 = schemeE( mce, e->Iex.Qop.details->arg4 );
5993 return gen_maxU32( mce, gen_maxU32( mce, b1, b2 ),
5994 gen_maxU32( mce, b3, b4 ) );
5997 IRAtom* b1 = schemeE( mce, e->Iex.Triop.details->arg1 );
5998 IRAtom* b2 = schemeE( mce, e->Iex.Triop.details->arg2 );
5999 IRAtom* b3 = schemeE( mce, e->Iex.Triop.details->arg3 );
6000 return gen_maxU32( mce, b1, gen_maxU32( mce, b2, b3 ) );
6013 IRAtom* b1 = schemeE( mce, e->Iex.Binop.arg1 );
6014 IRAtom* b2 = schemeE( mce, e->Iex.Binop.arg2 );
6015 return gen_maxU32( mce, b1, b2 );
6022 IRAtom* b1 = schemeE( mce, e->Iex.Unop.arg );
6028 return mkexpr( findShadowTmpB( mce, e->Iex.RdTmp.tmp ));
6035 && b_offset <= mce->layout->total_sizeB -4);
6038 return IRExpr_Get( b_offset + 2*mce->layout->total_sizeB,
6051 static void do_origins_Dirty ( MCEnv* mce, IRDirty* d )
6059 curr = schemeE( mce, d->guard );
6069 here = schemeE( mce, d->args[i] );
6070 curr = gen_maxU32( mce, curr, here );
6086 if (isAlwaysDefd(mce, gOff, gSz)) {
6109 cond = assignNew( 'B', mce, Ity_I8, unop(Iop_1Uto8, d->guard));
6111 iftrue = assignNew( 'B', mce, Ity_I32,
6113 + 2*mce->layout->total_sizeB,
6115 here = assignNew( 'B', mce, Ity_I32,
6117 curr = gen_maxU32( mce, curr, here );
6133 here = schemeE( mce, d->mAddr );
6134 curr = gen_maxU32( mce, curr, here );
6145 here = gen_guarded_load_b( mce, 4, d->mAddr, d->mSize - toDo,
6147 curr = gen_maxU32( mce, curr, here );
6152 here = gen_guarded_load_b( mce, 2, d->mAddr, d->mSize - toDo,
6154 curr = gen_maxU32( mce, curr, here );
6159 here = gen_guarded_load_b( mce, 1, d->mAddr, d->mSize - toDo,
6161 curr = gen_maxU32( mce, curr, here );
6173 dst = findShadowTmpB(mce, d->tmp);
6174 assign( 'V', mce, dst, curr );
6189 if (isAlwaysDefd(mce, gOff, gSz))
6208 cond = assignNew('B', mce, Ity_I8,
6210 iffalse = assignNew('B', mce, Ity_I32,
6212 2*mce->layout->total_sizeB,
6214 curr = assignNew('V', mce, Ity_I32,
6217 stmt( 'B', mce, IRStmt_Put(b_offset
6218 + 2*mce->layout->total_sizeB,
6233 gen_store_b( mce, 4, d->mAddr, d->mSize - toDo, curr,
6239 gen_store_b( mce, 2, d->mAddr, d->mSize - toDo, curr,
6245 gen_store_b( mce, 1, d->mAddr, d->mSize - toDo, curr,
6254 static void do_origins_Store ( MCEnv* mce,
6266 dszB = sizeofIRType( typeOfIRExpr(mce->sb->tyenv, stData ) );
6267 dataB = schemeE( mce, stData );
6268 gen_store_b( mce, dszB, stAddr, 0/*offset*/, dataB,
6273 static void schemeS ( MCEnv* mce, IRStmt* st )
6301 = mkIRRegArray( descr->base + 2*mce->layout->total_sizeB,
6306 t1 = schemeE( mce, puti->data );
6307 t2 = schemeE( mce, puti->ix );
6308 t3 = gen_maxU32( mce, t1, t2 );
6309 t4 = zWidenFrom32( mce, equivIntTy, t3 );
6310 stmt( 'B', mce, IRStmt_PutI( mkIRPutI(descr_b, puti->ix,
6316 do_origins_Dirty( mce, st->Ist.Dirty.details );
6320 do_origins_Store( mce, st->Ist.Store.end,
6333 = typeOfIRTemp(mce->sb->tyenv, st->Ist.LLSC.result);
6338 assign( 'B', mce, findShadowTmpB(mce, st->Ist.LLSC.result),
6339 schemeE(mce, vanillaLoad));
6342 do_origins_Store( mce, st->Ist.LLSC.end,
6349 assign( 'B', mce, findShadowTmpB(mce, st->Ist.LLSC.result),
6359 sizeofIRType(typeOfIRExpr(mce->sb->tyenv, st->Ist.Put.data))
6363 stmt( 'B', mce, IRStmt_Put(b_offset + 2*mce->layout->total_sizeB,
6364 schemeE( mce, st->Ist.Put.data )) );
6370 assign( 'B', mce, findShadowTmpB(mce, st->Ist.WrTmp.tmp),
6371 schemeE(mce, st->Ist.WrTmp.data) );