rsCpuScript.cpp revision 10adb0c2029f112b5738228617d5645f6ecea0c5 
1/*
2 * Copyright (C) 2011-2012 The Android Open Source Project
3 *
4 * Licensed under the Apache License, Version 2.0 (the "License");
5 * you may not use this file except in compliance with the License.
6 * You may obtain a copy of the License at
7 *
8 *      http://www.apache.org/licenses/LICENSE-2.0
9 *
10 * Unless required by applicable law or agreed to in writing, software
11 * distributed under the License is distributed on an "AS IS" BASIS,
12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 * See the License for the specific language governing permissions and
14 * limitations under the License.
15 */
16
17#include "rsCpuCore.h"
18#include "rsCpuScript.h"
19#include "rsCpuExecutable.h"
20
21#ifdef RS_COMPATIBILITY_LIB
22    #include <stdio.h>
23    #include <sys/stat.h>
24    #include <unistd.h>
25#else
26    #include "rsCppUtils.h"
27
28    #include <bcc/Config/Config.h>
29    #include <bcinfo/MetadataExtractor.h>
30    #include <cutils/properties.h>
31
32    #include <zlib.h>
33    #include <sys/file.h>
34    #include <sys/types.h>
35    #include <unistd.h>
36
37    #include <string>
38    #include <vector>
39#endif
40
41#include <set>
42#include <string>
43#include <dlfcn.h>
44#include <stdlib.h>
45#include <string.h>
46#include <iostream>
47#include <sstream>
48
49namespace {
50
51static const bool kDebugGlobalVariables = false;
52
53static bool allocationLODIsNull(const android::renderscript::Allocation *alloc) {
54  // Even if alloc != nullptr, mallocPtr could be null if
55  // IO_OUTPUT/IO_INPUT with no bound surface.
56  return alloc && alloc->mHal.drvState.lod[0].mallocPtr == nullptr;
57}
58
59#ifndef RS_COMPATIBILITY_LIB
60
61static bool is_force_recompile() {
62#ifdef RS_SERVER
63  return false;
64#else
65  char buf[PROPERTY_VALUE_MAX];
66
67  // Re-compile if floating point precision has been overridden.
68  property_get("debug.rs.precision", buf, "");
69  if (buf[0] != '\0') {
70    return true;
71  }
72
73  // Re-compile if debug.rs.forcerecompile is set.
74  property_get("debug.rs.forcerecompile", buf, "0");
75  if ((::strcmp(buf, "1") == 0) || (::strcmp(buf, "true") == 0)) {
76    return true;
77  } else {
78    return false;
79  }
80#endif  // RS_SERVER
81}
82
83static void setCompileArguments(std::vector<const char*>* args,
84                                const std::string& bcFileName,
85                                const char* cacheDir, const char* resName,
86                                const char* core_lib, bool useRSDebugContext,
87                                const char* bccPluginName, bool emitGlobalInfo,
88                                int optLevel, bool emitGlobalInfoSkipConstant) {
89    rsAssert(cacheDir && resName && core_lib);
90    args->push_back(android::renderscript::RsdCpuScriptImpl::BCC_EXE_PATH);
91    args->push_back("-unroll-runtime");
92    args->push_back("-scalarize-load-store");
93    if (emitGlobalInfo) {
94        args->push_back("-rs-global-info");
95        if (emitGlobalInfoSkipConstant) {
96            args->push_back("-rs-global-info-skip-constant");
97        }
98    }
99    args->push_back("-o");
100    args->push_back(resName);
101    args->push_back("-output_path");
102    args->push_back(cacheDir);
103    args->push_back("-bclib");
104    args->push_back(core_lib);
105    args->push_back("-mtriple");
106    args->push_back(DEFAULT_TARGET_TRIPLE_STRING);
107    args->push_back("-O");
108
109    switch (optLevel) {
110    case (0):
111        args->push_back("0");
112        break;
113    case (3):
114        args->push_back("3");
115        break;
116    default:
117        ALOGW("Expected optimization level of 0 or 3. Received %d", optLevel);
118        args->push_back("3");
119        break;
120    }
121
122    // Enable workaround for A53 codegen by default.
123#if defined(__aarch64__) && !defined(DISABLE_A53_WORKAROUND)
124    args->push_back("-aarch64-fix-cortex-a53-835769");
125#endif
126
127    // Execute the bcc compiler.
128    if (useRSDebugContext) {
129        args->push_back("-rs-debug-ctx");
130    } else {
131        // Only load additional libraries for compiles that don't use
132        // the debug context.
133        if (bccPluginName && strlen(bccPluginName) > 0) {
134            args->push_back("-load");
135            args->push_back(bccPluginName);
136        }
137    }
138
139    args->push_back("-fPIC");
140    args->push_back("-embedRSInfo");
141
142    args->push_back(bcFileName.c_str());
143    args->push_back(nullptr);
144}
145
146static bool compileBitcode(const std::string &bcFileName,
147                           const char *bitcode,
148                           size_t bitcodeSize,
149                           std::vector<const char *> &compileArguments) {
150    rsAssert(bitcode && bitcodeSize);
151
152    FILE *bcfile = fopen(bcFileName.c_str(), "w");
153    if (!bcfile) {
154        ALOGE("Could not write to %s", bcFileName.c_str());
155        return false;
156    }
157    size_t nwritten = fwrite(bitcode, 1, bitcodeSize, bcfile);
158    fclose(bcfile);
159    if (nwritten != bitcodeSize) {
160        ALOGE("Could not write %zu bytes to %s", bitcodeSize,
161              bcFileName.c_str());
162        return false;
163    }
164
165    return android::renderscript::rsuExecuteCommand(
166                   android::renderscript::RsdCpuScriptImpl::BCC_EXE_PATH,
167                   compileArguments.size()-1, compileArguments.data());
168}
169
170// The checksum is unnecessary under a few conditions, since the primary
171// use-case for it is debugging. If we are loading something from the
172// system partition (read-only), we know that it was precompiled as part of
173// application ahead of time (and thus the checksum is completely
174// unnecessary). The checksum is also unnecessary on release (non-debug)
175// builds, as the only way to get a shared object is to have compiled the
176// script once already. On a release build, there is no way to adjust the
177// other libraries/dependencies, and so the only reason to recompile would
178// be for a source APK change or an OTA. In either case, the APK would be
179// reinstalled, which would already clear the code_cache/ directory.
180bool isChecksumNeeded(const char *cacheDir) {
181    if ((::strcmp(SYSLIBPATH, cacheDir) == 0) ||
182        (::strcmp(SYSLIBPATH_VENDOR, cacheDir) == 0))
183        return false;
184    char buf[PROPERTY_VALUE_MAX];
185    property_get("ro.debuggable", buf, "");
186    return (buf[0] == '1');
187}
188
189bool addFileToChecksum(const char *fileName, uint32_t &checksum) {
190    int FD = open(fileName, O_RDONLY);
191    if (FD == -1) {
192        ALOGE("Cannot open file \'%s\' to compute checksum", fileName);
193        return false;
194    }
195
196    char buf[256];
197    while (true) {
198        ssize_t nread = read(FD, buf, sizeof(buf));
199        if (nread < 0) { // bail out on failed read
200            ALOGE("Error while computing checksum for file \'%s\'", fileName);
201            return false;
202        }
203
204        checksum = adler32(checksum, (const unsigned char *) buf, nread);
205        if (static_cast<size_t>(nread) < sizeof(buf)) // EOF
206            break;
207    }
208
209    if (close(FD) != 0) {
210        ALOGE("Cannot close file \'%s\' after computing checksum", fileName);
211        return false;
212    }
213    return true;
214}
215
216#endif  // !defined(RS_COMPATIBILITY_LIB)
217}  // namespace
218
219namespace android {
220namespace renderscript {
221
222#ifndef RS_COMPATIBILITY_LIB
223
224uint32_t constructBuildChecksum(uint8_t const *bitcode, size_t bitcodeSize,
225                                const char *commandLine,
226                                const char** bccFiles, size_t numFiles) {
227    uint32_t checksum = adler32(0L, Z_NULL, 0);
228
229    // include checksum of bitcode
230    if (bitcode != nullptr && bitcodeSize > 0) {
231        checksum = adler32(checksum, bitcode, bitcodeSize);
232    }
233
234    // include checksum of command line arguments
235    checksum = adler32(checksum, (const unsigned char *) commandLine,
236                       strlen(commandLine));
237
238    // include checksum of bccFiles
239    for (size_t i = 0; i < numFiles; i++) {
240        const char* bccFile = bccFiles[i];
241        if (bccFile[0] != 0 && !addFileToChecksum(bccFile, checksum)) {
242            // return empty checksum instead of something partial/corrupt
243            return 0;
244        }
245    }
246
247    return checksum;
248}
249
250#endif  // !RS_COMPATIBILITY_LIB
251
252RsdCpuScriptImpl::RsdCpuScriptImpl(RsdCpuReferenceImpl *ctx, const Script *s) {
253    mCtx = ctx;
254    mScript = s;
255
256    mScriptSO = nullptr;
257
258    mRoot = nullptr;
259    mRootExpand = nullptr;
260    mInit = nullptr;
261    mFreeChildren = nullptr;
262    mScriptExec = nullptr;
263
264    mBoundAllocs = nullptr;
265    mIntrinsicData = nullptr;
266    mIsThreadable = true;
267
268    mBuildChecksum = 0;
269    mChecksumNeeded = false;
270}
271
272bool RsdCpuScriptImpl::storeRSInfoFromSO() {
273    // The shared object may have an invalid build checksum.
274    // Validate and fail early.
275    mScriptExec = ScriptExecutable::createFromSharedObject(
276            mCtx->getContext(), mScriptSO,
277            mChecksumNeeded ? mBuildChecksum : 0);
278
279    if (mScriptExec == nullptr) {
280        return false;
281    }
282
283    mRoot = (RootFunc_t) dlsym(mScriptSO, "root");
284    if (mRoot) {
285        //ALOGE("Found root(): %p", mRoot);
286    }
287    mRootExpand = (RootFunc_t) dlsym(mScriptSO, "root.expand");
288    if (mRootExpand) {
289        //ALOGE("Found root.expand(): %p", mRootExpand);
290    }
291    mInit = (InitOrDtorFunc_t) dlsym(mScriptSO, "init");
292    if (mInit) {
293        //ALOGE("Found init(): %p", mInit);
294    }
295    mFreeChildren = (InitOrDtorFunc_t) dlsym(mScriptSO, ".rs.dtor");
296    if (mFreeChildren) {
297        //ALOGE("Found .rs.dtor(): %p", mFreeChildren);
298    }
299
300    size_t varCount = mScriptExec->getExportedVariableCount();
301    if (varCount > 0) {
302        mBoundAllocs = new Allocation *[varCount];
303        memset(mBoundAllocs, 0, varCount * sizeof(*mBoundAllocs));
304    }
305
306    mIsThreadable = mScriptExec->getThreadable();
307    //ALOGE("Script isThreadable? %d", mIsThreadable);
308
309    if (kDebugGlobalVariables) {
310        mScriptExec->dumpGlobalInfo();
311    }
312
313    return true;
314}
315
316bool RsdCpuScriptImpl::init(char const *resName, char const *cacheDir,
317                            uint8_t const *bitcode, size_t bitcodeSize,
318                            uint32_t flags, char const *bccPluginName) {
319    //ALOGE("rsdScriptCreate %p %p %p %p %i %i %p", rsc, resName, cacheDir,
320    // bitcode, bitcodeSize, flags, lookupFunc);
321    //ALOGE("rsdScriptInit %p %p", rsc, script);
322
323    mCtx->lockMutex();
324#ifndef RS_COMPATIBILITY_LIB
325    bool useRSDebugContext = false;
326
327    bcinfo::MetadataExtractor bitcodeMetadata((const char *) bitcode, bitcodeSize);
328    if (!bitcodeMetadata.extract()) {
329        ALOGE("Could not extract metadata from bitcode");
330        mCtx->unlockMutex();
331        return false;
332    }
333
334    const char* core_lib = findCoreLib(bitcodeMetadata, (const char*)bitcode, bitcodeSize);
335
336    if (mCtx->getContext()->getContextType() == RS_CONTEXT_TYPE_DEBUG) {
337        useRSDebugContext = true;
338    }
339
340    int optLevel = mCtx->getContext()->getOptLevel();
341
342    std::string bcFileName(cacheDir);
343    bcFileName.append("/");
344    bcFileName.append(resName);
345    bcFileName.append(".bc");
346
347    std::vector<const char*> compileArguments;
348    bool emitGlobalInfo = mCtx->getEmbedGlobalInfo();
349    bool emitGlobalInfoSkipConstant = mCtx->getEmbedGlobalInfoSkipConstant();
350    setCompileArguments(&compileArguments, bcFileName, cacheDir, resName, core_lib,
351                        useRSDebugContext, bccPluginName, emitGlobalInfo,
352                        optLevel, emitGlobalInfoSkipConstant);
353
354    mChecksumNeeded = isChecksumNeeded(cacheDir);
355    if (mChecksumNeeded) {
356        std::vector<const char *> bccFiles = { BCC_EXE_PATH,
357                                               core_lib,
358                                             };
359
360        // The last argument of compileArguments is a nullptr, so remove 1 from
361        // the size.
362        std::unique_ptr<const char> compileCommandLine(
363            rsuJoinStrings(compileArguments.size()-1, compileArguments.data()));
364
365        mBuildChecksum = constructBuildChecksum(bitcode, bitcodeSize,
366                                                compileCommandLine.get(),
367                                                bccFiles.data(), bccFiles.size());
368
369        if (mBuildChecksum == 0) {
370            // cannot compute checksum but verification is enabled
371            mCtx->unlockMutex();
372            return false;
373        }
374    }
375    else {
376        // add a dummy/constant as a checksum if verification is disabled
377        mBuildChecksum = 0xabadcafe;
378    }
379
380    // Append build checksum to commandline
381    // Handle the terminal nullptr in compileArguments
382    compileArguments.pop_back();
383    compileArguments.push_back("-build-checksum");
384    std::stringstream ss;
385    ss << std::hex << mBuildChecksum;
386    compileArguments.push_back(ss.str().c_str());
387    compileArguments.push_back(nullptr);
388
389    if (!is_force_recompile() && !useRSDebugContext) {
390        mScriptSO = SharedLibraryUtils::loadSharedLibrary(cacheDir, resName);
391
392        // Read RS info from the shared object to detect checksum mismatch
393        if (mScriptSO != nullptr && !storeRSInfoFromSO()) {
394            dlclose(mScriptSO);
395            mScriptSO = nullptr;
396        }
397    }
398
399    // If we can't, it's either not there or out of date.  We compile the bit code and try loading
400    // again.
401    if (mScriptSO == nullptr) {
402        if (!compileBitcode(bcFileName, (const char*)bitcode, bitcodeSize,
403                            compileArguments))
404        {
405            ALOGE("bcc: FAILS to compile '%s'", resName);
406            mCtx->unlockMutex();
407            return false;
408        }
409
410        if (!SharedLibraryUtils::createSharedLibrary(mCtx->getContext()->getDriverName(),
411                                                     cacheDir, resName)) {
412            ALOGE("Linker: Failed to link object file '%s'", resName);
413            mCtx->unlockMutex();
414            return false;
415        }
416
417        mScriptSO = SharedLibraryUtils::loadSharedLibrary(cacheDir, resName);
418        if (mScriptSO == nullptr) {
419            ALOGE("Unable to load '%s'", resName);
420            mCtx->unlockMutex();
421            return false;
422        }
423
424        // Read RS symbol information from the .so.
425        if (!storeRSInfoFromSO()) {
426            goto error;
427        }
428    }
429
430    mBitcodeFilePath.setTo(bcFileName.c_str());
431
432#else  // RS_COMPATIBILITY_LIB is defined
433    const char *nativeLibDir = mCtx->getContext()->getNativeLibDir();
434    mScriptSO = SharedLibraryUtils::loadSharedLibrary(cacheDir, resName, nativeLibDir);
435
436    if (!mScriptSO) {
437        goto error;
438    }
439
440    if (!storeRSInfoFromSO()) {
441        goto error;
442    }
443#endif
444    mCtx->unlockMutex();
445    return true;
446
447error:
448
449    mCtx->unlockMutex();
450    if (mScriptSO) {
451        dlclose(mScriptSO);
452        mScriptSO = nullptr;
453    }
454    return false;
455}
456
457#ifndef RS_COMPATIBILITY_LIB
458
459const char* RsdCpuScriptImpl::findCoreLib(const bcinfo::MetadataExtractor& ME, const char* bitcode,
460                                          size_t bitcodeSize) {
461    const char* defaultLib = SYSLIBPATH"/libclcore.bc";
462
463    // If we're debugging, use the debug library.
464    if (mCtx->getContext()->getContextType() == RS_CONTEXT_TYPE_DEBUG) {
465        return SYSLIBPATH"/libclcore_debug.bc";
466    }
467
468    if (ME.hasDebugInfo()) {
469        return SYSLIBPATH"/libclcore_g.bc";
470    }
471
472    // If a callback has been registered to specify a library, use that.
473    RSSelectRTCallback selectRTCallback = mCtx->getSelectRTCallback();
474    if (selectRTCallback != nullptr) {
475        return selectRTCallback((const char*)bitcode, bitcodeSize);
476    }
477
478    // Check for a platform specific library
479#if defined(ARCH_ARM_HAVE_NEON) && !defined(DISABLE_CLCORE_NEON)
480    enum bcinfo::RSFloatPrecision prec = ME.getRSFloatPrecision();
481    if (prec == bcinfo::RS_FP_Relaxed) {
482        // NEON-capable ARMv7a devices can use an accelerated math library
483        // for all reduced precision scripts.
484        // ARMv8 does not use NEON, as ASIMD can be used with all precision
485        // levels.
486        return SYSLIBPATH"/libclcore_neon.bc";
487    } else {
488        return defaultLib;
489    }
490#elif defined(__i386__) || defined(__x86_64__)
491    // x86 devices will use an optimized library.
492    return SYSLIBPATH"/libclcore_x86.bc";
493#else
494    return defaultLib;
495#endif
496}
497
498#endif
499
500void RsdCpuScriptImpl::populateScript(Script *script) {
501    // Copy info over to runtime
502    script->mHal.info.exportedFunctionCount = mScriptExec->getExportedFunctionCount();
503    script->mHal.info.exportedReduceCount = mScriptExec->getExportedReduceCount();
504    script->mHal.info.exportedReduceNewCount = mScriptExec->getExportedReduceNewCount();
505    script->mHal.info.exportedForEachCount = mScriptExec->getExportedForEachCount();
506    script->mHal.info.exportedVariableCount = mScriptExec->getExportedVariableCount();
507    script->mHal.info.exportedPragmaCount = mScriptExec->getPragmaCount();;
508    script->mHal.info.exportedPragmaKeyList = mScriptExec->getPragmaKeys();
509    script->mHal.info.exportedPragmaValueList = mScriptExec->getPragmaValues();
510
511    // Bug, need to stash in metadata
512    if (mRootExpand) {
513        script->mHal.info.root = mRootExpand;
514    } else {
515        script->mHal.info.root = mRoot;
516    }
517}
518
519// Set up the launch dimensions, and write the values of the launch
520// dimensions into the mtls start/end fields.
521//
522// Inputs:
523//    baseDim - base shape of the input
524//         sc - used to constrain the launch dimensions
525//
526// Returns:
527//   True on success, false on failure to set up
528bool RsdCpuScriptImpl::setUpMtlsDimensions(MTLaunchStructCommon *mtls,
529                                           const RsLaunchDimensions &baseDim,
530                                           const RsScriptCall *sc) {
531    rsAssert(mtls);
532
533#define SET_UP_DIMENSION(DIM_FIELD, SC_FIELD) do {            \
534    if (!sc || (sc->SC_FIELD##End == 0)) {                    \
535        mtls->end.DIM_FIELD = baseDim.DIM_FIELD;              \
536    } else {                                                  \
537        mtls->start.DIM_FIELD =                               \
538            rsMin(baseDim.DIM_FIELD, sc->SC_FIELD##Start);    \
539        mtls->end.DIM_FIELD =                                 \
540            rsMin(baseDim.DIM_FIELD, sc->SC_FIELD##End);      \
541        if (mtls->start.DIM_FIELD >= mtls->end.DIM_FIELD) {   \
542            mCtx->getContext()->setError(RS_ERROR_BAD_SCRIPT, \
543                "Failed to launch kernel; Invalid "           \
544                #SC_FIELD "Start or " #SC_FIELD "End.");      \
545            return false;                                     \
546        }                                                     \
547    }} while(0)
548
549    SET_UP_DIMENSION(x, x);
550    SET_UP_DIMENSION(y, y);
551    SET_UP_DIMENSION(z, z);
552    SET_UP_DIMENSION(array[0], array);
553    SET_UP_DIMENSION(array[1], array2);
554    SET_UP_DIMENSION(array[2], array3);
555    SET_UP_DIMENSION(array[3], array4);
556#undef SET_UP_DIMENSION
557
558    return true;
559}
560
561// Preliminary work to prepare a simple reduce-style kernel for launch.
562bool RsdCpuScriptImpl::reduceMtlsSetup(const Allocation *ain,
563                                       const Allocation *aout,
564                                       const RsScriptCall *sc,
565                                       MTLaunchStructReduce *mtls) {
566    rsAssert(ain && aout);
567    memset(mtls, 0, sizeof(MTLaunchStructReduce));
568    mtls->dimPtr = &mtls->inputDim;
569
570    if (allocationLODIsNull(ain) || allocationLODIsNull(aout)) {
571        mCtx->getContext()->setError(RS_ERROR_BAD_SCRIPT,
572                                     "reduce called with a null allocation");
573        return false;
574    }
575
576    // Set up the dimensions of the input.
577    const Type *inType = ain->getType();
578    mtls->inputDim.x = inType->getDimX();
579    rsAssert(inType->getDimY() == 0);
580
581    if (!setUpMtlsDimensions(mtls, mtls->inputDim, sc)) {
582        return false;
583    }
584
585    mtls->rs = mCtx;
586    // Currently not threaded.
587    mtls->isThreadable = false;
588    mtls->mSliceNum = -1;
589
590    // Set up input and output.
591    mtls->inBuf = static_cast<uint8_t *>(ain->getPointerUnchecked(0, 0));
592    mtls->outBuf = static_cast<uint8_t *>(aout->getPointerUnchecked(0, 0));
593
594    rsAssert(mtls->inBuf && mtls->outBuf);
595
596    return true;
597}
598
599// Preliminary work to prepare a general reduce-style kernel for launch.
600bool RsdCpuScriptImpl::reduceNewMtlsSetup(const Allocation ** ains,
601                                          uint32_t inLen,
602                                          const Allocation * aout,
603                                          const RsScriptCall *sc,
604                                          MTLaunchStructReduceNew *mtls) {
605    rsAssert(ains && (inLen >= 1) && aout);
606    memset(mtls, 0, sizeof(MTLaunchStructReduceNew));
607    mtls->dimPtr = &mtls->redp.dim;
608
609    for (int index = inLen; --index >= 0;) {
610        if (allocationLODIsNull(ains[index])) {
611            mCtx->getContext()->setError(RS_ERROR_BAD_SCRIPT,
612                                         "reduce called with null in allocations");
613            return false;
614        }
615    }
616
617    if (allocationLODIsNull(aout)) {
618        mCtx->getContext()->setError(RS_ERROR_BAD_SCRIPT,
619                                     "reduce called with null out allocation");
620        return false;
621    }
622
623    const Allocation *ain0   = ains[0];
624    const Type       *inType = ain0->getType();
625
626    mtls->redp.dim.x = inType->getDimX();
627    mtls->redp.dim.y = inType->getDimY();
628    mtls->redp.dim.z = inType->getDimZ();
629
630    for (int Index = inLen; --Index >= 1;) {
631        if (!ain0->hasSameDims(ains[Index])) {
632            mCtx->getContext()->setError(RS_ERROR_BAD_SCRIPT,
633                                         "Failed to launch reduction kernel;"
634                                         "dimensions of input allocations do not match.");
635            return false;
636        }
637    }
638
639    if (!setUpMtlsDimensions(mtls, mtls->redp.dim, sc)) {
640        return false;
641    }
642
643    // The X & Y walkers always want 0-1 min even if dim is not present
644    mtls->end.x = rsMax((uint32_t)1, mtls->end.x);
645    mtls->end.y = rsMax((uint32_t)1, mtls->end.y);
646
647    mtls->rs = mCtx;
648
649    mtls->mSliceNum    = 0;
650    mtls->mSliceSize   = 1;
651    mtls->isThreadable = mIsThreadable;
652
653    // Set up output,
654    mtls->redp.outLen = 1;
655    mtls->redp.outPtr[0] = (uint8_t *)aout->mHal.drvState.lod[0].mallocPtr;
656    mtls->redp.outStride[0] = aout->getType()->getElementSizeBytes();
657
658    // Set up input.
659    memcpy(mtls->ains, ains, inLen * sizeof(ains[0]));
660    mtls->redp.inLen = inLen;
661    for (int index = inLen; --index >= 0;) {
662        mtls->redp.inPtr[index] = (const uint8_t*)ains[index]->mHal.drvState.lod[0].mallocPtr;
663        mtls->redp.inStride[index] = ains[index]->getType()->getElementSizeBytes();
664    }
665
666    // All validation passed, ok to launch threads
667    return true;
668}
669
670// Preliminary work to prepare a forEach-style kernel for launch.
671bool RsdCpuScriptImpl::forEachMtlsSetup(const Allocation ** ains,
672                                        uint32_t inLen,
673                                        Allocation * aout,
674                                        const void * usr, uint32_t usrLen,
675                                        const RsScriptCall *sc,
676                                        MTLaunchStructForEach *mtls) {
677    if (ains == nullptr && inLen != 0) {
678        mCtx->getContext()->setError(RS_ERROR_BAD_SCRIPT,
679          "rsForEach called with none-zero inLen with null in allocations");
680        return false;
681    }
682
683    memset(mtls, 0, sizeof(MTLaunchStructForEach));
684    mtls->dimPtr = &mtls->fep.dim;
685
686    for (int index = inLen; --index >= 0;) {
687        if (allocationLODIsNull(ains[index])) {
688            mCtx->getContext()->setError(RS_ERROR_BAD_SCRIPT,
689                                         "rsForEach called with null in allocations");
690            return false;
691        }
692    }
693
694    if (allocationLODIsNull(aout)) {
695        mCtx->getContext()->setError(RS_ERROR_BAD_SCRIPT,
696                                     "rsForEach called with null out allocations");
697        return false;
698    }
699
700    if (inLen > 0) {
701        const Allocation *ain0   = ains[0];
702        const Type       *inType = ain0->getType();
703
704        mtls->fep.dim.x = inType->getDimX();
705        mtls->fep.dim.y = inType->getDimY();
706        mtls->fep.dim.z = inType->getDimZ();
707
708        for (int Index = inLen; --Index >= 1;) {
709            if (!ain0->hasSameDims(ains[Index])) {
710                mCtx->getContext()->setError(RS_ERROR_BAD_SCRIPT,
711                  "Failed to launch kernel; dimensions of input"
712                  "allocations do not match.");
713                return false;
714            }
715        }
716    } else if (aout != nullptr) {
717        const Type *outType = aout->getType();
718
719        mtls->fep.dim.x = outType->getDimX();
720        mtls->fep.dim.y = outType->getDimY();
721        mtls->fep.dim.z = outType->getDimZ();
722
723    } else if (sc != nullptr) {
724        mtls->fep.dim.x = sc->xEnd;
725        mtls->fep.dim.y = sc->yEnd;
726        mtls->fep.dim.z = 0;
727    } else {
728        mCtx->getContext()->setError(RS_ERROR_BAD_SCRIPT,
729                                     "rsForEach called with null allocations");
730        return false;
731    }
732
733    if (inLen > 0 && aout != nullptr) {
734        if (!ains[0]->hasSameDims(aout)) {
735            mCtx->getContext()->setError(RS_ERROR_BAD_SCRIPT,
736              "Failed to launch kernel; dimensions of input and output allocations do not match.");
737
738            return false;
739        }
740    }
741
742    if (!setUpMtlsDimensions(mtls, mtls->fep.dim, sc)) {
743        return false;
744    }
745
746    // The X & Y walkers always want 0-1 min even if dim is not present
747    mtls->end.x    = rsMax((uint32_t)1, mtls->end.x);
748    mtls->end.y    = rsMax((uint32_t)1, mtls->end.y);
749    mtls->rs       = mCtx;
750    if (ains) {
751        memcpy(mtls->ains, ains, inLen * sizeof(ains[0]));
752    }
753    mtls->aout[0]    = aout;
754    mtls->fep.usr    = usr;
755    mtls->fep.usrLen = usrLen;
756    mtls->mSliceSize = 1;
757    mtls->mSliceNum  = 0;
758
759    mtls->isThreadable  = mIsThreadable;
760
761    if (inLen > 0) {
762        mtls->fep.inLen = inLen;
763        for (int index = inLen; --index >= 0;) {
764            mtls->fep.inPtr[index] = (const uint8_t*)ains[index]->mHal.drvState.lod[0].mallocPtr;
765            mtls->fep.inStride[index] = ains[index]->getType()->getElementSizeBytes();
766        }
767    }
768
769    if (aout != nullptr) {
770        mtls->fep.outPtr[0] = (uint8_t *)aout->mHal.drvState.lod[0].mallocPtr;
771        mtls->fep.outStride[0] = aout->getType()->getElementSizeBytes();
772    }
773
774    // All validation passed, ok to launch threads
775    return true;
776}
777
778
779void RsdCpuScriptImpl::invokeForEach(uint32_t slot,
780                                     const Allocation ** ains,
781                                     uint32_t inLen,
782                                     Allocation * aout,
783                                     const void * usr,
784                                     uint32_t usrLen,
785                                     const RsScriptCall *sc) {
786
787    MTLaunchStructForEach mtls;
788
789    if (forEachMtlsSetup(ains, inLen, aout, usr, usrLen, sc, &mtls)) {
790        forEachKernelSetup(slot, &mtls);
791
792        RsdCpuScriptImpl * oldTLS = mCtx->setTLS(this);
793        mCtx->launchForEach(ains, inLen, aout, sc, &mtls);
794        mCtx->setTLS(oldTLS);
795    }
796}
797
798void RsdCpuScriptImpl::invokeReduce(uint32_t slot,
799                                    const Allocation *ain,
800                                    Allocation *aout,
801                                    const RsScriptCall *sc) {
802    MTLaunchStructReduce mtls;
803
804    if (reduceMtlsSetup(ain, aout, sc, &mtls)) {
805        reduceKernelSetup(slot, &mtls);
806        RsdCpuScriptImpl *oldTLS = mCtx->setTLS(this);
807        mCtx->launchReduce(ain, aout, &mtls);
808        mCtx->setTLS(oldTLS);
809    }
810}
811
812void RsdCpuScriptImpl::invokeReduceNew(uint32_t slot,
813                                       const Allocation ** ains, uint32_t inLen,
814                                       Allocation *aout,
815                                       const RsScriptCall *sc) {
816  MTLaunchStructReduceNew mtls;
817
818  if (reduceNewMtlsSetup(ains, inLen, aout, sc, &mtls)) {
819    reduceNewKernelSetup(slot, &mtls);
820    RsdCpuScriptImpl *oldTLS = mCtx->setTLS(this);
821    mCtx->launchReduceNew(ains, inLen, aout, &mtls);
822    mCtx->setTLS(oldTLS);
823  }
824}
825
826void RsdCpuScriptImpl::forEachKernelSetup(uint32_t slot, MTLaunchStructForEach *mtls) {
827    mtls->script = this;
828    mtls->fep.slot = slot;
829    mtls->kernel = mScriptExec->getForEachFunction(slot);
830    rsAssert(mtls->kernel != nullptr);
831}
832
833void RsdCpuScriptImpl::reduceKernelSetup(uint32_t slot, MTLaunchStructReduce *mtls) {
834    mtls->script = this;
835    mtls->kernel = mScriptExec->getReduceFunction(slot);
836    rsAssert(mtls->kernel != nullptr);
837}
838
839void RsdCpuScriptImpl::reduceNewKernelSetup(uint32_t slot, MTLaunchStructReduceNew *mtls) {
840    mtls->script = this;
841    mtls->redp.slot = slot;
842
843    const ReduceNewDescription *desc = mScriptExec->getReduceNewDescription(slot);
844    mtls->accumFunc = desc->accumFunc;
845    mtls->initFunc  = desc->initFunc;   // might legally be nullptr
846    mtls->combFunc  = desc->combFunc;   // might legally be nullptr
847    mtls->outFunc   = desc->outFunc;    // might legally be nullptr
848    mtls->accumSize = desc->accumSize;
849
850    rsAssert(mtls->accumFunc != nullptr);
851}
852
853int RsdCpuScriptImpl::invokeRoot() {
854    RsdCpuScriptImpl * oldTLS = mCtx->setTLS(this);
855    int ret = mRoot();
856    mCtx->setTLS(oldTLS);
857    return ret;
858}
859
860void RsdCpuScriptImpl::invokeInit() {
861    if (mInit) {
862        mInit();
863    }
864}
865
866void RsdCpuScriptImpl::invokeFreeChildren() {
867    if (mFreeChildren) {
868        mFreeChildren();
869    }
870}
871
872void RsdCpuScriptImpl::invokeFunction(uint32_t slot, const void *params,
873                                      size_t paramLength) {
874    //ALOGE("invoke %i %p %zu", slot, params, paramLength);
875    void * ap = nullptr;
876
877#if defined(__x86_64__)
878    // The invoked function could have input parameter of vector type for example float4 which
879    // requires void* params to be 16 bytes aligned when using SSE instructions for x86_64 platform.
880    // So try to align void* params before passing them into RS exported function.
881
882    if ((uint8_t)(uint64_t)params & 0x0F) {
883        if ((ap = (void*)memalign(16, paramLength)) != nullptr) {
884            memcpy(ap, params, paramLength);
885        } else {
886            ALOGE("x86_64: invokeFunction memalign error, still use params which"
887                  " is not 16 bytes aligned.");
888        }
889    }
890#endif
891
892    RsdCpuScriptImpl * oldTLS = mCtx->setTLS(this);
893    reinterpret_cast<void (*)(const void *, uint32_t)>(
894        mScriptExec->getInvokeFunction(slot))(ap? (const void *) ap: params, paramLength);
895
896#if defined(__x86_64__)
897    free(ap);
898#endif
899
900    mCtx->setTLS(oldTLS);
901}
902
903void RsdCpuScriptImpl::setGlobalVar(uint32_t slot, const void *data, size_t dataLength) {
904    //rsAssert(!script->mFieldIsObject[slot]);
905    //ALOGE("setGlobalVar %i %p %zu", slot, data, dataLength);
906
907    //if (mIntrinsicID) {
908        //mIntrinsicFuncs.setVar(dc, script, drv->mIntrinsicData, slot, data, dataLength);
909        //return;
910    //}
911
912    int32_t *destPtr = reinterpret_cast<int32_t *>(mScriptExec->getFieldAddress(slot));
913    if (!destPtr) {
914        //ALOGV("Calling setVar on slot = %i which is null", slot);
915        return;
916    }
917
918    memcpy(destPtr, data, dataLength);
919}
920
921void RsdCpuScriptImpl::getGlobalVar(uint32_t slot, void *data, size_t dataLength) {
922    //rsAssert(!script->mFieldIsObject[slot]);
923    //ALOGE("getGlobalVar %i %p %zu", slot, data, dataLength);
924
925    int32_t *srcPtr = reinterpret_cast<int32_t *>(mScriptExec->getFieldAddress(slot));
926    if (!srcPtr) {
927        //ALOGV("Calling setVar on slot = %i which is null", slot);
928        return;
929    }
930    memcpy(data, srcPtr, dataLength);
931}
932
933
934void RsdCpuScriptImpl::setGlobalVarWithElemDims(uint32_t slot, const void *data, size_t dataLength,
935                                                const Element *elem,
936                                                const uint32_t *dims, size_t dimLength) {
937    int32_t *destPtr = reinterpret_cast<int32_t *>(mScriptExec->getFieldAddress(slot));
938    if (!destPtr) {
939        //ALOGV("Calling setVar on slot = %i which is null", slot);
940        return;
941    }
942
943    // We want to look at dimension in terms of integer components,
944    // but dimLength is given in terms of bytes.
945    dimLength /= sizeof(int);
946
947    // Only a single dimension is currently supported.
948    rsAssert(dimLength == 1);
949    if (dimLength == 1) {
950        // First do the increment loop.
951        size_t stride = elem->getSizeBytes();
952        const char *cVal = reinterpret_cast<const char *>(data);
953        for (uint32_t i = 0; i < dims[0]; i++) {
954            elem->incRefs(cVal);
955            cVal += stride;
956        }
957
958        // Decrement loop comes after (to prevent race conditions).
959        char *oldVal = reinterpret_cast<char *>(destPtr);
960        for (uint32_t i = 0; i < dims[0]; i++) {
961            elem->decRefs(oldVal);
962            oldVal += stride;
963        }
964    }
965
966    memcpy(destPtr, data, dataLength);
967}
968
969void RsdCpuScriptImpl::setGlobalBind(uint32_t slot, Allocation *data) {
970
971    //rsAssert(!script->mFieldIsObject[slot]);
972    //ALOGE("setGlobalBind %i %p", slot, data);
973
974    int32_t *destPtr = reinterpret_cast<int32_t *>(mScriptExec->getFieldAddress(slot));
975    if (!destPtr) {
976        //ALOGV("Calling setVar on slot = %i which is null", slot);
977        return;
978    }
979
980    void *ptr = nullptr;
981    mBoundAllocs[slot] = data;
982    if (data) {
983        ptr = data->mHal.drvState.lod[0].mallocPtr;
984    }
985    memcpy(destPtr, &ptr, sizeof(void *));
986}
987
988void RsdCpuScriptImpl::setGlobalObj(uint32_t slot, ObjectBase *data) {
989
990    //rsAssert(script->mFieldIsObject[slot]);
991    //ALOGE("setGlobalObj %i %p", slot, data);
992
993    int32_t *destPtr = reinterpret_cast<int32_t *>(mScriptExec->getFieldAddress(slot));
994    if (!destPtr) {
995        //ALOGV("Calling setVar on slot = %i which is null", slot);
996        return;
997    }
998
999    rsrSetObject(mCtx->getContext(), (rs_object_base *)destPtr, data);
1000}
1001
1002const char* RsdCpuScriptImpl::getFieldName(uint32_t slot) const {
1003    return mScriptExec->getFieldName(slot);
1004}
1005
1006RsdCpuScriptImpl::~RsdCpuScriptImpl() {
1007    delete mScriptExec;
1008    delete[] mBoundAllocs;
1009    if (mScriptSO) {
1010        dlclose(mScriptSO);
1011    }
1012}
1013
1014Allocation * RsdCpuScriptImpl::getAllocationForPointer(const void *ptr) const {
1015    if (!ptr) {
1016        return nullptr;
1017    }
1018
1019    for (uint32_t ct=0; ct < mScript->mHal.info.exportedVariableCount; ct++) {
1020        Allocation *a = mBoundAllocs[ct];
1021        if (!a) continue;
1022        if (a->mHal.drvState.lod[0].mallocPtr == ptr) {
1023            return a;
1024        }
1025    }
1026    ALOGE("rsGetAllocation, failed to find %p", ptr);
1027    return nullptr;
1028}
1029
1030int RsdCpuScriptImpl::getGlobalEntries() const {
1031    return mScriptExec->getGlobalEntries();
1032}
1033
1034const char * RsdCpuScriptImpl::getGlobalName(int i) const {
1035    return mScriptExec->getGlobalName(i);
1036}
1037
1038const void * RsdCpuScriptImpl::getGlobalAddress(int i) const {
1039    return mScriptExec->getGlobalAddress(i);
1040}
1041
1042size_t RsdCpuScriptImpl::getGlobalSize(int i) const {
1043    return mScriptExec->getGlobalSize(i);
1044}
1045
1046uint32_t RsdCpuScriptImpl::getGlobalProperties(int i) const {
1047    return mScriptExec->getGlobalProperties(i);
1048}
1049
1050void RsdCpuScriptImpl::preLaunch(uint32_t slot, const Allocation ** ains,
1051                                 uint32_t inLen, Allocation * aout,
1052                                 const void * usr, uint32_t usrLen,
1053                                 const RsScriptCall *sc) {}
1054
1055void RsdCpuScriptImpl::postLaunch(uint32_t slot, const Allocation ** ains,
1056                                  uint32_t inLen, Allocation * aout,
1057                                  const void * usr, uint32_t usrLen,
1058                                  const RsScriptCall *sc) {}
1059
1060
1061}
1062}
1063