rsCpuScript.cpp revision 838c2d0dc0b8b07d4aa8de4360765738c6d2770e
1/* 2 * Copyright (C) 2011-2012 The Android Open Source Project 3 * 4 * Licensed under the Apache License, Version 2.0 (the "License"); 5 * you may not use this file except in compliance with the License. 6 * You may obtain a copy of the License at 7 * 8 * http://www.apache.org/licenses/LICENSE-2.0 9 * 10 * Unless required by applicable law or agreed to in writing, software 11 * distributed under the License is distributed on an "AS IS" BASIS, 12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 * See the License for the specific language governing permissions and 14 * limitations under the License. 15 */ 16 17#include "rsCpuCore.h" 18#include "rsCpuScript.h" 19#include "rsCpuExecutable.h" 20 21#ifdef RS_COMPATIBILITY_LIB 22 #include <stdio.h> 23 #include <sys/stat.h> 24 #include <unistd.h> 25#else 26 #include "rsCppUtils.h" 27 28 #include <bcc/Config/Config.h> 29 #include <bcinfo/MetadataExtractor.h> 30 #include <cutils/properties.h> 31 32 #include <zlib.h> 33 #include <sys/file.h> 34 #include <sys/types.h> 35 #include <unistd.h> 36 37 #include <string> 38 #include <vector> 39#endif 40 41#include <set> 42#include <string> 43#include <dlfcn.h> 44#include <stdlib.h> 45#include <string.h> 46#include <iostream> 47#include <sstream> 48 49namespace { 50 51static const bool kDebugGlobalVariables = false; 52 53static bool allocationLODIsNull(const android::renderscript::Allocation *alloc) { 54 // Even if alloc != nullptr, mallocPtr could be null if 55 // IO_OUTPUT/IO_INPUT with no bound surface. 56 return alloc && alloc->mHal.drvState.lod[0].mallocPtr == nullptr; 57} 58 59#ifndef RS_COMPATIBILITY_LIB 60 61static bool is_force_recompile() { 62#ifdef RS_SERVER 63 return false; 64#else 65 char buf[PROPERTY_VALUE_MAX]; 66 67 // Re-compile if floating point precision has been overridden. 68 property_get("debug.rs.precision", buf, ""); 69 if (buf[0] != '\0') { 70 return true; 71 } 72 73 // Re-compile if debug.rs.forcerecompile is set. 74 property_get("debug.rs.forcerecompile", buf, "0"); 75 if ((::strcmp(buf, "1") == 0) || (::strcmp(buf, "true") == 0)) { 76 return true; 77 } else { 78 return false; 79 } 80#endif // RS_SERVER 81} 82 83static void setCompileArguments(std::vector<const char*>* args, 84 const std::string& bcFileName, 85 const char* cacheDir, const char* resName, 86 const char* core_lib, bool useRSDebugContext, 87 const char* bccPluginName, bool emitGlobalInfo, 88 int optLevel, bool emitGlobalInfoSkipConstant) { 89 rsAssert(cacheDir && resName && core_lib); 90 args->push_back(android::renderscript::RsdCpuScriptImpl::BCC_EXE_PATH); 91 args->push_back("-unroll-runtime"); 92 args->push_back("-scalarize-load-store"); 93 if (emitGlobalInfo) { 94 args->push_back("-rs-global-info"); 95 if (emitGlobalInfoSkipConstant) { 96 args->push_back("-rs-global-info-skip-constant"); 97 } 98 } 99 args->push_back("-o"); 100 args->push_back(resName); 101 args->push_back("-output_path"); 102 args->push_back(cacheDir); 103 args->push_back("-bclib"); 104 args->push_back(core_lib); 105 args->push_back("-mtriple"); 106 args->push_back(DEFAULT_TARGET_TRIPLE_STRING); 107 args->push_back("-O"); 108 109 switch (optLevel) { 110 case (0): 111 args->push_back("0"); 112 break; 113 case (3): 114 args->push_back("3"); 115 break; 116 default: 117 ALOGW("Expected optimization level of 0 or 3. Received %d", optLevel); 118 args->push_back("3"); 119 break; 120 } 121 122 // Enable workaround for A53 codegen by default. 123#if defined(__aarch64__) && !defined(DISABLE_A53_WORKAROUND) 124 args->push_back("-aarch64-fix-cortex-a53-835769"); 125#endif 126 127 // Execute the bcc compiler. 128 if (useRSDebugContext) { 129 args->push_back("-rs-debug-ctx"); 130 } else { 131 // Only load additional libraries for compiles that don't use 132 // the debug context. 133 if (bccPluginName && strlen(bccPluginName) > 0) { 134 args->push_back("-load"); 135 args->push_back(bccPluginName); 136 } 137 } 138 139 args->push_back("-fPIC"); 140 args->push_back("-embedRSInfo"); 141 142 args->push_back(bcFileName.c_str()); 143 args->push_back(nullptr); 144} 145 146static bool compileBitcode(const std::string &bcFileName, 147 const char *bitcode, 148 size_t bitcodeSize, 149 std::vector<const char *> &compileArguments) { 150 rsAssert(bitcode && bitcodeSize); 151 152 FILE *bcfile = fopen(bcFileName.c_str(), "w"); 153 if (!bcfile) { 154 ALOGE("Could not write to %s", bcFileName.c_str()); 155 return false; 156 } 157 size_t nwritten = fwrite(bitcode, 1, bitcodeSize, bcfile); 158 fclose(bcfile); 159 if (nwritten != bitcodeSize) { 160 ALOGE("Could not write %zu bytes to %s", bitcodeSize, 161 bcFileName.c_str()); 162 return false; 163 } 164 165 return android::renderscript::rsuExecuteCommand( 166 android::renderscript::RsdCpuScriptImpl::BCC_EXE_PATH, 167 compileArguments.size()-1, compileArguments.data()); 168} 169 170// The checksum is unnecessary under a few conditions, since the primary 171// use-case for it is debugging. If we are loading something from the 172// system partition (read-only), we know that it was precompiled as part of 173// application ahead of time (and thus the checksum is completely 174// unnecessary). The checksum is also unnecessary on release (non-debug) 175// builds, as the only way to get a shared object is to have compiled the 176// script once already. On a release build, there is no way to adjust the 177// other libraries/dependencies, and so the only reason to recompile would 178// be for a source APK change or an OTA. In either case, the APK would be 179// reinstalled, which would already clear the code_cache/ directory. 180bool isChecksumNeeded(const char *cacheDir) { 181 if ((::strcmp(SYSLIBPATH, cacheDir) == 0) || 182 (::strcmp(SYSLIBPATH_VENDOR, cacheDir) == 0)) 183 return false; 184 char buf[PROPERTY_VALUE_MAX]; 185 property_get("ro.debuggable", buf, ""); 186 return (buf[0] == '1'); 187} 188 189bool addFileToChecksum(const char *fileName, uint32_t &checksum) { 190 int FD = open(fileName, O_RDONLY); 191 if (FD == -1) { 192 ALOGE("Cannot open file \'%s\' to compute checksum", fileName); 193 return false; 194 } 195 196 char buf[256]; 197 while (true) { 198 ssize_t nread = read(FD, buf, sizeof(buf)); 199 if (nread < 0) { // bail out on failed read 200 ALOGE("Error while computing checksum for file \'%s\'", fileName); 201 return false; 202 } 203 204 checksum = adler32(checksum, (const unsigned char *) buf, nread); 205 if (static_cast<size_t>(nread) < sizeof(buf)) // EOF 206 break; 207 } 208 209 if (close(FD) != 0) { 210 ALOGE("Cannot close file \'%s\' after computing checksum", fileName); 211 return false; 212 } 213 return true; 214} 215 216#endif // !defined(RS_COMPATIBILITY_LIB) 217} // namespace 218 219namespace android { 220namespace renderscript { 221 222#ifndef RS_COMPATIBILITY_LIB 223 224uint32_t constructBuildChecksum(uint8_t const *bitcode, size_t bitcodeSize, 225 const char *commandLine, 226 const char** bccFiles, size_t numFiles) { 227 uint32_t checksum = adler32(0L, Z_NULL, 0); 228 229 // include checksum of bitcode 230 if (bitcode != nullptr && bitcodeSize > 0) { 231 checksum = adler32(checksum, bitcode, bitcodeSize); 232 } 233 234 // include checksum of command line arguments 235 checksum = adler32(checksum, (const unsigned char *) commandLine, 236 strlen(commandLine)); 237 238 // include checksum of bccFiles 239 for (size_t i = 0; i < numFiles; i++) { 240 const char* bccFile = bccFiles[i]; 241 if (bccFile[0] != 0 && !addFileToChecksum(bccFile, checksum)) { 242 // return empty checksum instead of something partial/corrupt 243 return 0; 244 } 245 } 246 247 return checksum; 248} 249 250#endif // !RS_COMPATIBILITY_LIB 251 252RsdCpuScriptImpl::RsdCpuScriptImpl(RsdCpuReferenceImpl *ctx, const Script *s) { 253 mCtx = ctx; 254 mScript = s; 255 256 mScriptSO = nullptr; 257 258 mRoot = nullptr; 259 mRootExpand = nullptr; 260 mInit = nullptr; 261 mFreeChildren = nullptr; 262 mScriptExec = nullptr; 263 264 mBoundAllocs = nullptr; 265 mIntrinsicData = nullptr; 266 mIsThreadable = true; 267 268 mBuildChecksum = 0; 269 mChecksumNeeded = false; 270} 271 272bool RsdCpuScriptImpl::storeRSInfoFromSO() { 273 // The shared object may have an invalid build checksum. 274 // Validate and fail early. 275 mScriptExec = ScriptExecutable::createFromSharedObject( 276 mCtx->getContext(), mScriptSO, 277 mChecksumNeeded ? mBuildChecksum : 0); 278 279 if (mScriptExec == nullptr) { 280 return false; 281 } 282 283 mRoot = (RootFunc_t) dlsym(mScriptSO, "root"); 284 if (mRoot) { 285 //ALOGE("Found root(): %p", mRoot); 286 } 287 mRootExpand = (RootFunc_t) dlsym(mScriptSO, "root.expand"); 288 if (mRootExpand) { 289 //ALOGE("Found root.expand(): %p", mRootExpand); 290 } 291 mInit = (InitOrDtorFunc_t) dlsym(mScriptSO, "init"); 292 if (mInit) { 293 //ALOGE("Found init(): %p", mInit); 294 } 295 mFreeChildren = (InitOrDtorFunc_t) dlsym(mScriptSO, ".rs.dtor"); 296 if (mFreeChildren) { 297 //ALOGE("Found .rs.dtor(): %p", mFreeChildren); 298 } 299 300 size_t varCount = mScriptExec->getExportedVariableCount(); 301 if (varCount > 0) { 302 mBoundAllocs = new Allocation *[varCount]; 303 memset(mBoundAllocs, 0, varCount * sizeof(*mBoundAllocs)); 304 } 305 306 mIsThreadable = mScriptExec->getThreadable(); 307 //ALOGE("Script isThreadable? %d", mIsThreadable); 308 309 if (kDebugGlobalVariables) { 310 mScriptExec->dumpGlobalInfo(); 311 } 312 313 return true; 314} 315 316bool RsdCpuScriptImpl::init(char const *resName, char const *cacheDir, 317 uint8_t const *bitcode, size_t bitcodeSize, 318 uint32_t flags, char const *bccPluginName) { 319 //ALOGE("rsdScriptCreate %p %p %p %p %i %i %p", rsc, resName, cacheDir, 320 // bitcode, bitcodeSize, flags, lookupFunc); 321 //ALOGE("rsdScriptInit %p %p", rsc, script); 322 323 mCtx->lockMutex(); 324#ifndef RS_COMPATIBILITY_LIB 325 bool useRSDebugContext = false; 326 327 bcinfo::MetadataExtractor bitcodeMetadata((const char *) bitcode, bitcodeSize); 328 if (!bitcodeMetadata.extract()) { 329 ALOGE("Could not extract metadata from bitcode"); 330 mCtx->unlockMutex(); 331 return false; 332 } 333 334 const char* core_lib = findCoreLib(bitcodeMetadata, (const char*)bitcode, bitcodeSize); 335 336 if (mCtx->getContext()->getContextType() == RS_CONTEXT_TYPE_DEBUG) { 337 useRSDebugContext = true; 338 } 339 340 int optLevel = mCtx->getContext()->getOptLevel(); 341 342 std::string bcFileName(cacheDir); 343 bcFileName.append("/"); 344 bcFileName.append(resName); 345 bcFileName.append(".bc"); 346 347 std::vector<const char*> compileArguments; 348 bool emitGlobalInfo = mCtx->getEmbedGlobalInfo(); 349 bool emitGlobalInfoSkipConstant = mCtx->getEmbedGlobalInfoSkipConstant(); 350 setCompileArguments(&compileArguments, bcFileName, cacheDir, resName, core_lib, 351 useRSDebugContext, bccPluginName, emitGlobalInfo, 352 optLevel, emitGlobalInfoSkipConstant); 353 354 mChecksumNeeded = isChecksumNeeded(cacheDir); 355 if (mChecksumNeeded) { 356 std::vector<const char *> bccFiles = { BCC_EXE_PATH, 357 core_lib, 358 }; 359 360 // The last argument of compileArguments is a nullptr, so remove 1 from 361 // the size. 362 std::unique_ptr<const char> compileCommandLine( 363 rsuJoinStrings(compileArguments.size()-1, compileArguments.data())); 364 365 mBuildChecksum = constructBuildChecksum(bitcode, bitcodeSize, 366 compileCommandLine.get(), 367 bccFiles.data(), bccFiles.size()); 368 369 if (mBuildChecksum == 0) { 370 // cannot compute checksum but verification is enabled 371 mCtx->unlockMutex(); 372 return false; 373 } 374 } 375 else { 376 // add a dummy/constant as a checksum if verification is disabled 377 mBuildChecksum = 0xabadcafe; 378 } 379 380 // Append build checksum to commandline 381 // Handle the terminal nullptr in compileArguments 382 compileArguments.pop_back(); 383 compileArguments.push_back("-build-checksum"); 384 std::stringstream ss; 385 ss << std::hex << mBuildChecksum; 386 compileArguments.push_back(ss.str().c_str()); 387 compileArguments.push_back(nullptr); 388 389 if (!is_force_recompile() && !useRSDebugContext) { 390 mScriptSO = SharedLibraryUtils::loadSharedLibrary(cacheDir, resName); 391 392 // Read RS info from the shared object to detect checksum mismatch 393 if (mScriptSO != nullptr && !storeRSInfoFromSO()) { 394 dlclose(mScriptSO); 395 mScriptSO = nullptr; 396 } 397 } 398 399 // If we can't, it's either not there or out of date. We compile the bit code and try loading 400 // again. 401 if (mScriptSO == nullptr) { 402 if (!compileBitcode(bcFileName, (const char*)bitcode, bitcodeSize, 403 compileArguments)) 404 { 405 ALOGE("bcc: FAILS to compile '%s'", resName); 406 mCtx->unlockMutex(); 407 return false; 408 } 409 410 if (!SharedLibraryUtils::createSharedLibrary(mCtx->getContext()->getDriverName(), 411 cacheDir, resName)) { 412 ALOGE("Linker: Failed to link object file '%s'", resName); 413 mCtx->unlockMutex(); 414 return false; 415 } 416 417 mScriptSO = SharedLibraryUtils::loadSharedLibrary(cacheDir, resName); 418 if (mScriptSO == nullptr) { 419 ALOGE("Unable to load '%s'", resName); 420 mCtx->unlockMutex(); 421 return false; 422 } 423 424 // Read RS symbol information from the .so. 425 if (!storeRSInfoFromSO()) { 426 goto error; 427 } 428 } 429 430 mBitcodeFilePath.setTo(bcFileName.c_str()); 431 432#else // RS_COMPATIBILITY_LIB is defined 433 const char *nativeLibDir = mCtx->getContext()->getNativeLibDir(); 434 mScriptSO = SharedLibraryUtils::loadSharedLibrary(cacheDir, resName, nativeLibDir); 435 436 if (!mScriptSO) { 437 goto error; 438 } 439 440 if (!storeRSInfoFromSO()) { 441 goto error; 442 } 443#endif 444 mCtx->unlockMutex(); 445 return true; 446 447error: 448 449 mCtx->unlockMutex(); 450 if (mScriptSO) { 451 dlclose(mScriptSO); 452 mScriptSO = nullptr; 453 } 454 return false; 455} 456 457#ifndef RS_COMPATIBILITY_LIB 458 459const char* RsdCpuScriptImpl::findCoreLib(const bcinfo::MetadataExtractor& ME, const char* bitcode, 460 size_t bitcodeSize) { 461 const char* defaultLib = SYSLIBPATH"/libclcore.bc"; 462 463 // If we're debugging, use the debug library. 464 if (mCtx->getContext()->getContextType() == RS_CONTEXT_TYPE_DEBUG) { 465 return SYSLIBPATH"/libclcore_debug.bc"; 466 } 467 468 if (ME.hasDebugInfo()) { 469 return SYSLIBPATH"/libclcore_g.bc"; 470 } 471 472 // If a callback has been registered to specify a library, use that. 473 RSSelectRTCallback selectRTCallback = mCtx->getSelectRTCallback(); 474 if (selectRTCallback != nullptr) { 475 return selectRTCallback((const char*)bitcode, bitcodeSize); 476 } 477 478 // Check for a platform specific library 479#if defined(ARCH_ARM_HAVE_NEON) && !defined(DISABLE_CLCORE_NEON) 480 enum bcinfo::RSFloatPrecision prec = ME.getRSFloatPrecision(); 481 if (prec == bcinfo::RS_FP_Relaxed) { 482 // NEON-capable ARMv7a devices can use an accelerated math library 483 // for all reduced precision scripts. 484 // ARMv8 does not use NEON, as ASIMD can be used with all precision 485 // levels. 486 return SYSLIBPATH"/libclcore_neon.bc"; 487 } else { 488 return defaultLib; 489 } 490#elif defined(__i386__) || defined(__x86_64__) 491 // x86 devices will use an optimized library. 492 return SYSLIBPATH"/libclcore_x86.bc"; 493#else 494 return defaultLib; 495#endif 496} 497 498#endif 499 500void RsdCpuScriptImpl::populateScript(Script *script) { 501 // Copy info over to runtime 502 script->mHal.info.exportedFunctionCount = mScriptExec->getExportedFunctionCount(); 503 script->mHal.info.exportedReduceCount = mScriptExec->getExportedReduceCount(); 504 script->mHal.info.exportedReduceNewCount = mScriptExec->getExportedReduceNewCount(); 505 script->mHal.info.exportedForEachCount = mScriptExec->getExportedForEachCount(); 506 script->mHal.info.exportedVariableCount = mScriptExec->getExportedVariableCount(); 507 script->mHal.info.exportedPragmaCount = mScriptExec->getPragmaCount();; 508 script->mHal.info.exportedPragmaKeyList = mScriptExec->getPragmaKeys(); 509 script->mHal.info.exportedPragmaValueList = mScriptExec->getPragmaValues(); 510 511 // Bug, need to stash in metadata 512 if (mRootExpand) { 513 script->mHal.info.root = mRootExpand; 514 } else { 515 script->mHal.info.root = mRoot; 516 } 517} 518 519// Set up the launch dimensions, and write the values of the launch 520// dimensions into the mtls start/end fields. 521// 522// Inputs: 523// baseDim - base shape of the input 524// sc - used to constrain the launch dimensions 525// 526// Returns: 527// True on success, false on failure to set up 528bool RsdCpuScriptImpl::setUpMtlsDimensions(MTLaunchStructCommon *mtls, 529 const RsLaunchDimensions &baseDim, 530 const RsScriptCall *sc) { 531 rsAssert(mtls); 532 533#define SET_UP_DIMENSION(DIM_FIELD, SC_FIELD) do { \ 534 if (!sc || (sc->SC_FIELD##End == 0)) { \ 535 mtls->end.DIM_FIELD = baseDim.DIM_FIELD; \ 536 } else { \ 537 mtls->start.DIM_FIELD = \ 538 rsMin(baseDim.DIM_FIELD, sc->SC_FIELD##Start); \ 539 mtls->end.DIM_FIELD = \ 540 rsMin(baseDim.DIM_FIELD, sc->SC_FIELD##End); \ 541 if (mtls->start.DIM_FIELD >= mtls->end.DIM_FIELD) { \ 542 mCtx->getContext()->setError(RS_ERROR_BAD_SCRIPT, \ 543 "Failed to launch kernel; Invalid " \ 544 #SC_FIELD "Start or " #SC_FIELD "End."); \ 545 return false; \ 546 } \ 547 }} while(0) 548 549 SET_UP_DIMENSION(x, x); 550 SET_UP_DIMENSION(y, y); 551 SET_UP_DIMENSION(z, z); 552 SET_UP_DIMENSION(array[0], array); 553 SET_UP_DIMENSION(array[1], array2); 554 SET_UP_DIMENSION(array[2], array3); 555 SET_UP_DIMENSION(array[3], array4); 556#undef SET_UP_DIMENSION 557 558 return true; 559} 560 561// Preliminary work to prepare a simple reduce-style kernel for launch. 562bool RsdCpuScriptImpl::reduceMtlsSetup(const Allocation *ain, 563 const Allocation *aout, 564 const RsScriptCall *sc, 565 MTLaunchStructReduce *mtls) { 566 rsAssert(ain && aout); 567 memset(mtls, 0, sizeof(MTLaunchStructReduce)); 568 mtls->dimPtr = &mtls->inputDim; 569 570 if (allocationLODIsNull(ain) || allocationLODIsNull(aout)) { 571 mCtx->getContext()->setError(RS_ERROR_BAD_SCRIPT, 572 "reduce called with a null allocation"); 573 return false; 574 } 575 576 // Set up the dimensions of the input. 577 const Type *inType = ain->getType(); 578 mtls->inputDim.x = inType->getDimX(); 579 rsAssert(inType->getDimY() == 0); 580 581 if (!setUpMtlsDimensions(mtls, mtls->inputDim, sc)) { 582 return false; 583 } 584 585 mtls->rs = mCtx; 586 // Currently not threaded. 587 mtls->isThreadable = false; 588 mtls->mSliceNum = -1; 589 590 // Set up input and output. 591 mtls->inBuf = static_cast<uint8_t *>(ain->getPointerUnchecked(0, 0)); 592 mtls->outBuf = static_cast<uint8_t *>(aout->getPointerUnchecked(0, 0)); 593 594 rsAssert(mtls->inBuf && mtls->outBuf); 595 596 return true; 597} 598 599// Preliminary work to prepare a general reduce-style kernel for launch. 600bool RsdCpuScriptImpl::reduceNewMtlsSetup(const Allocation ** ains, 601 uint32_t inLen, 602 const Allocation * aout, 603 const RsScriptCall *sc, 604 MTLaunchStructReduceNew *mtls) { 605 rsAssert(ains && (inLen >= 1) && aout); 606 memset(mtls, 0, sizeof(MTLaunchStructReduceNew)); 607 mtls->dimPtr = &mtls->redp.dim; 608 609 for (int index = inLen; --index >= 0;) { 610 if (allocationLODIsNull(ains[index])) { 611 mCtx->getContext()->setError(RS_ERROR_BAD_SCRIPT, 612 "reduce called with null in allocations"); 613 return false; 614 } 615 } 616 617 if (allocationLODIsNull(aout)) { 618 mCtx->getContext()->setError(RS_ERROR_BAD_SCRIPT, 619 "reduce called with null out allocation"); 620 return false; 621 } 622 623 const Allocation *ain0 = ains[0]; 624 const Type *inType = ain0->getType(); 625 626 mtls->redp.dim.x = inType->getDimX(); 627 mtls->redp.dim.y = inType->getDimY(); 628 mtls->redp.dim.z = inType->getDimZ(); 629 630 for (int Index = inLen; --Index >= 1;) { 631 if (!ain0->hasSameDims(ains[Index])) { 632 mCtx->getContext()->setError(RS_ERROR_BAD_SCRIPT, 633 "Failed to launch reduction kernel;" 634 "dimensions of input allocations do not match."); 635 return false; 636 } 637 } 638 639 if (!setUpMtlsDimensions(mtls, mtls->redp.dim, sc)) { 640 return false; 641 } 642 643 // The X & Y walkers always want 0-1 min even if dim is not present 644 mtls->end.x = rsMax((uint32_t)1, mtls->end.x); 645 mtls->end.y = rsMax((uint32_t)1, mtls->end.y); 646 647 mtls->rs = mCtx; 648 649 // Currently not threaded. 650 mtls->isThreadable = false; 651 mtls->mSliceNum = -1; 652 653 // Set up output, 654 mtls->redp.outLen = 1; 655 mtls->redp.outPtr[0] = (uint8_t *)aout->mHal.drvState.lod[0].mallocPtr; 656 mtls->redp.outStride[0] = aout->getType()->getElementSizeBytes(); 657 658 // Set up input. 659 memcpy(mtls->ains, ains, inLen * sizeof(ains[0])); 660 mtls->redp.inLen = inLen; 661 for (int index = inLen; --index >= 0;) { 662 mtls->redp.inPtr[index] = (const uint8_t*)ains[index]->mHal.drvState.lod[0].mallocPtr; 663 mtls->redp.inStride[index] = ains[index]->getType()->getElementSizeBytes(); 664 } 665 666 // All validation passed, ok to launch threads 667 return true; 668} 669 670// Preliminary work to prepare a forEach-style kernel for launch. 671bool RsdCpuScriptImpl::forEachMtlsSetup(const Allocation ** ains, 672 uint32_t inLen, 673 Allocation * aout, 674 const void * usr, uint32_t usrLen, 675 const RsScriptCall *sc, 676 MTLaunchStructForEach *mtls) { 677 if (ains == nullptr && inLen != 0) { 678 mCtx->getContext()->setError(RS_ERROR_BAD_SCRIPT, 679 "rsForEach called with none-zero inLen with null in allocations"); 680 return false; 681 } 682 683 memset(mtls, 0, sizeof(MTLaunchStructForEach)); 684 mtls->dimPtr = &mtls->fep.dim; 685 686 for (int index = inLen; --index >= 0;) { 687 if (allocationLODIsNull(ains[index])) { 688 mCtx->getContext()->setError(RS_ERROR_BAD_SCRIPT, 689 "rsForEach called with null in allocations"); 690 return false; 691 } 692 } 693 694 if (allocationLODIsNull(aout)) { 695 mCtx->getContext()->setError(RS_ERROR_BAD_SCRIPT, 696 "rsForEach called with null out allocations"); 697 return false; 698 } 699 700 if (inLen > 0) { 701 const Allocation *ain0 = ains[0]; 702 const Type *inType = ain0->getType(); 703 704 mtls->fep.dim.x = inType->getDimX(); 705 mtls->fep.dim.y = inType->getDimY(); 706 mtls->fep.dim.z = inType->getDimZ(); 707 708 for (int Index = inLen; --Index >= 1;) { 709 if (!ain0->hasSameDims(ains[Index])) { 710 mCtx->getContext()->setError(RS_ERROR_BAD_SCRIPT, 711 "Failed to launch kernel; dimensions of input" 712 "allocations do not match."); 713 return false; 714 } 715 } 716 } else if (aout != nullptr) { 717 const Type *outType = aout->getType(); 718 719 mtls->fep.dim.x = outType->getDimX(); 720 mtls->fep.dim.y = outType->getDimY(); 721 mtls->fep.dim.z = outType->getDimZ(); 722 723 } else if (sc != nullptr) { 724 mtls->fep.dim.x = sc->xEnd; 725 mtls->fep.dim.y = sc->yEnd; 726 mtls->fep.dim.z = 0; 727 } else { 728 mCtx->getContext()->setError(RS_ERROR_BAD_SCRIPT, 729 "rsForEach called with null allocations"); 730 return false; 731 } 732 733 if (inLen > 0 && aout != nullptr) { 734 if (!ains[0]->hasSameDims(aout)) { 735 mCtx->getContext()->setError(RS_ERROR_BAD_SCRIPT, 736 "Failed to launch kernel; dimensions of input and output allocations do not match."); 737 738 return false; 739 } 740 } 741 742 if (!setUpMtlsDimensions(mtls, mtls->fep.dim, sc)) { 743 return false; 744 } 745 746 // The X & Y walkers always want 0-1 min even if dim is not present 747 mtls->end.x = rsMax((uint32_t)1, mtls->end.x); 748 mtls->end.y = rsMax((uint32_t)1, mtls->end.y); 749 mtls->rs = mCtx; 750 if (ains) { 751 memcpy(mtls->ains, ains, inLen * sizeof(ains[0])); 752 } 753 mtls->aout[0] = aout; 754 mtls->fep.usr = usr; 755 mtls->fep.usrLen = usrLen; 756 mtls->mSliceSize = 1; 757 mtls->mSliceNum = 0; 758 759 mtls->isThreadable = mIsThreadable; 760 761 if (inLen > 0) { 762 mtls->fep.inLen = inLen; 763 for (int index = inLen; --index >= 0;) { 764 mtls->fep.inPtr[index] = (const uint8_t*)ains[index]->mHal.drvState.lod[0].mallocPtr; 765 mtls->fep.inStride[index] = ains[index]->getType()->getElementSizeBytes(); 766 } 767 } 768 769 if (aout != nullptr) { 770 mtls->fep.outPtr[0] = (uint8_t *)aout->mHal.drvState.lod[0].mallocPtr; 771 mtls->fep.outStride[0] = aout->getType()->getElementSizeBytes(); 772 } 773 774 // All validation passed, ok to launch threads 775 return true; 776} 777 778 779void RsdCpuScriptImpl::invokeForEach(uint32_t slot, 780 const Allocation ** ains, 781 uint32_t inLen, 782 Allocation * aout, 783 const void * usr, 784 uint32_t usrLen, 785 const RsScriptCall *sc) { 786 787 MTLaunchStructForEach mtls; 788 789 if (forEachMtlsSetup(ains, inLen, aout, usr, usrLen, sc, &mtls)) { 790 forEachKernelSetup(slot, &mtls); 791 792 RsdCpuScriptImpl * oldTLS = mCtx->setTLS(this); 793 mCtx->launchForEach(ains, inLen, aout, sc, &mtls); 794 mCtx->setTLS(oldTLS); 795 } 796} 797 798void RsdCpuScriptImpl::invokeReduce(uint32_t slot, 799 const Allocation *ain, 800 Allocation *aout, 801 const RsScriptCall *sc) { 802 MTLaunchStructReduce mtls; 803 804 if (reduceMtlsSetup(ain, aout, sc, &mtls)) { 805 reduceKernelSetup(slot, &mtls); 806 RsdCpuScriptImpl *oldTLS = mCtx->setTLS(this); 807 mCtx->launchReduce(ain, aout, &mtls); 808 mCtx->setTLS(oldTLS); 809 } 810} 811 812void RsdCpuScriptImpl::invokeReduceNew(uint32_t slot, 813 const Allocation ** ains, uint32_t inLen, 814 Allocation *aout, 815 const RsScriptCall *sc) { 816 MTLaunchStructReduceNew mtls; 817 818 if (reduceNewMtlsSetup(ains, inLen, aout, sc, &mtls)) { 819 reduceNewKernelSetup(slot, &mtls); 820 RsdCpuScriptImpl *oldTLS = mCtx->setTLS(this); 821 mCtx->launchReduceNew(ains, inLen, aout, &mtls); 822 mCtx->setTLS(oldTLS); 823 } 824} 825 826void RsdCpuScriptImpl::forEachKernelSetup(uint32_t slot, MTLaunchStructForEach *mtls) { 827 mtls->script = this; 828 mtls->fep.slot = slot; 829 mtls->kernel = mScriptExec->getForEachFunction(slot); 830 rsAssert(mtls->kernel != nullptr); 831} 832 833void RsdCpuScriptImpl::reduceKernelSetup(uint32_t slot, MTLaunchStructReduce *mtls) { 834 mtls->script = this; 835 mtls->kernel = mScriptExec->getReduceFunction(slot); 836 rsAssert(mtls->kernel != nullptr); 837} 838 839void RsdCpuScriptImpl::reduceNewKernelSetup(uint32_t slot, MTLaunchStructReduceNew *mtls) { 840 mtls->script = this; 841 mtls->redp.slot = slot; 842 843 const ReduceNewDescription *desc = mScriptExec->getReduceNewDescription(slot); 844 mtls->accumFunc = desc->accumFunc; 845 mtls->initFunc = desc->initFunc; // might legally be nullptr 846 mtls->outFunc = desc->outFunc; // might legally be nullptr 847 mtls->accumSize = desc->accumSize; 848 849 rsAssert(mtls->accumFunc != nullptr); 850} 851 852int RsdCpuScriptImpl::invokeRoot() { 853 RsdCpuScriptImpl * oldTLS = mCtx->setTLS(this); 854 int ret = mRoot(); 855 mCtx->setTLS(oldTLS); 856 return ret; 857} 858 859void RsdCpuScriptImpl::invokeInit() { 860 if (mInit) { 861 mInit(); 862 } 863} 864 865void RsdCpuScriptImpl::invokeFreeChildren() { 866 if (mFreeChildren) { 867 mFreeChildren(); 868 } 869} 870 871void RsdCpuScriptImpl::invokeFunction(uint32_t slot, const void *params, 872 size_t paramLength) { 873 //ALOGE("invoke %i %p %zu", slot, params, paramLength); 874 void * ap = nullptr; 875 876#if defined(__x86_64__) 877 // The invoked function could have input parameter of vector type for example float4 which 878 // requires void* params to be 16 bytes aligned when using SSE instructions for x86_64 platform. 879 // So try to align void* params before passing them into RS exported function. 880 881 if ((uint8_t)(uint64_t)params & 0x0F) { 882 if ((ap = (void*)memalign(16, paramLength)) != nullptr) { 883 memcpy(ap, params, paramLength); 884 } else { 885 ALOGE("x86_64: invokeFunction memalign error, still use params which" 886 " is not 16 bytes aligned."); 887 } 888 } 889#endif 890 891 RsdCpuScriptImpl * oldTLS = mCtx->setTLS(this); 892 reinterpret_cast<void (*)(const void *, uint32_t)>( 893 mScriptExec->getInvokeFunction(slot))(ap? (const void *) ap: params, paramLength); 894 895#if defined(__x86_64__) 896 free(ap); 897#endif 898 899 mCtx->setTLS(oldTLS); 900} 901 902void RsdCpuScriptImpl::setGlobalVar(uint32_t slot, const void *data, size_t dataLength) { 903 //rsAssert(!script->mFieldIsObject[slot]); 904 //ALOGE("setGlobalVar %i %p %zu", slot, data, dataLength); 905 906 //if (mIntrinsicID) { 907 //mIntrinsicFuncs.setVar(dc, script, drv->mIntrinsicData, slot, data, dataLength); 908 //return; 909 //} 910 911 int32_t *destPtr = reinterpret_cast<int32_t *>(mScriptExec->getFieldAddress(slot)); 912 if (!destPtr) { 913 //ALOGV("Calling setVar on slot = %i which is null", slot); 914 return; 915 } 916 917 memcpy(destPtr, data, dataLength); 918} 919 920void RsdCpuScriptImpl::getGlobalVar(uint32_t slot, void *data, size_t dataLength) { 921 //rsAssert(!script->mFieldIsObject[slot]); 922 //ALOGE("getGlobalVar %i %p %zu", slot, data, dataLength); 923 924 int32_t *srcPtr = reinterpret_cast<int32_t *>(mScriptExec->getFieldAddress(slot)); 925 if (!srcPtr) { 926 //ALOGV("Calling setVar on slot = %i which is null", slot); 927 return; 928 } 929 memcpy(data, srcPtr, dataLength); 930} 931 932 933void RsdCpuScriptImpl::setGlobalVarWithElemDims(uint32_t slot, const void *data, size_t dataLength, 934 const Element *elem, 935 const uint32_t *dims, size_t dimLength) { 936 int32_t *destPtr = reinterpret_cast<int32_t *>(mScriptExec->getFieldAddress(slot)); 937 if (!destPtr) { 938 //ALOGV("Calling setVar on slot = %i which is null", slot); 939 return; 940 } 941 942 // We want to look at dimension in terms of integer components, 943 // but dimLength is given in terms of bytes. 944 dimLength /= sizeof(int); 945 946 // Only a single dimension is currently supported. 947 rsAssert(dimLength == 1); 948 if (dimLength == 1) { 949 // First do the increment loop. 950 size_t stride = elem->getSizeBytes(); 951 const char *cVal = reinterpret_cast<const char *>(data); 952 for (uint32_t i = 0; i < dims[0]; i++) { 953 elem->incRefs(cVal); 954 cVal += stride; 955 } 956 957 // Decrement loop comes after (to prevent race conditions). 958 char *oldVal = reinterpret_cast<char *>(destPtr); 959 for (uint32_t i = 0; i < dims[0]; i++) { 960 elem->decRefs(oldVal); 961 oldVal += stride; 962 } 963 } 964 965 memcpy(destPtr, data, dataLength); 966} 967 968void RsdCpuScriptImpl::setGlobalBind(uint32_t slot, Allocation *data) { 969 970 //rsAssert(!script->mFieldIsObject[slot]); 971 //ALOGE("setGlobalBind %i %p", slot, data); 972 973 int32_t *destPtr = reinterpret_cast<int32_t *>(mScriptExec->getFieldAddress(slot)); 974 if (!destPtr) { 975 //ALOGV("Calling setVar on slot = %i which is null", slot); 976 return; 977 } 978 979 void *ptr = nullptr; 980 mBoundAllocs[slot] = data; 981 if (data) { 982 ptr = data->mHal.drvState.lod[0].mallocPtr; 983 } 984 memcpy(destPtr, &ptr, sizeof(void *)); 985} 986 987void RsdCpuScriptImpl::setGlobalObj(uint32_t slot, ObjectBase *data) { 988 989 //rsAssert(script->mFieldIsObject[slot]); 990 //ALOGE("setGlobalObj %i %p", slot, data); 991 992 int32_t *destPtr = reinterpret_cast<int32_t *>(mScriptExec->getFieldAddress(slot)); 993 if (!destPtr) { 994 //ALOGV("Calling setVar on slot = %i which is null", slot); 995 return; 996 } 997 998 rsrSetObject(mCtx->getContext(), (rs_object_base *)destPtr, data); 999} 1000 1001const char* RsdCpuScriptImpl::getFieldName(uint32_t slot) const { 1002 return mScriptExec->getFieldName(slot); 1003} 1004 1005RsdCpuScriptImpl::~RsdCpuScriptImpl() { 1006 delete mScriptExec; 1007 delete[] mBoundAllocs; 1008 if (mScriptSO) { 1009 dlclose(mScriptSO); 1010 } 1011} 1012 1013Allocation * RsdCpuScriptImpl::getAllocationForPointer(const void *ptr) const { 1014 if (!ptr) { 1015 return nullptr; 1016 } 1017 1018 for (uint32_t ct=0; ct < mScript->mHal.info.exportedVariableCount; ct++) { 1019 Allocation *a = mBoundAllocs[ct]; 1020 if (!a) continue; 1021 if (a->mHal.drvState.lod[0].mallocPtr == ptr) { 1022 return a; 1023 } 1024 } 1025 ALOGE("rsGetAllocation, failed to find %p", ptr); 1026 return nullptr; 1027} 1028 1029int RsdCpuScriptImpl::getGlobalEntries() const { 1030 return mScriptExec->getGlobalEntries(); 1031} 1032 1033const char * RsdCpuScriptImpl::getGlobalName(int i) const { 1034 return mScriptExec->getGlobalName(i); 1035} 1036 1037const void * RsdCpuScriptImpl::getGlobalAddress(int i) const { 1038 return mScriptExec->getGlobalAddress(i); 1039} 1040 1041size_t RsdCpuScriptImpl::getGlobalSize(int i) const { 1042 return mScriptExec->getGlobalSize(i); 1043} 1044 1045uint32_t RsdCpuScriptImpl::getGlobalProperties(int i) const { 1046 return mScriptExec->getGlobalProperties(i); 1047} 1048 1049void RsdCpuScriptImpl::preLaunch(uint32_t slot, const Allocation ** ains, 1050 uint32_t inLen, Allocation * aout, 1051 const void * usr, uint32_t usrLen, 1052 const RsScriptCall *sc) {} 1053 1054void RsdCpuScriptImpl::postLaunch(uint32_t slot, const Allocation ** ains, 1055 uint32_t inLen, Allocation * aout, 1056 const void * usr, uint32_t usrLen, 1057 const RsScriptCall *sc) {} 1058 1059 1060} 1061} 1062