rsCpuScript.cpp revision b043df0676fef226336deb3a00ead2f31e02343f 
1/*
2 * Copyright (C) 2011-2012 The Android Open Source Project
3 *
4 * Licensed under the Apache License, Version 2.0 (the "License");
5 * you may not use this file except in compliance with the License.
6 * You may obtain a copy of the License at
7 *
8 *      http://www.apache.org/licenses/LICENSE-2.0
9 *
10 * Unless required by applicable law or agreed to in writing, software
11 * distributed under the License is distributed on an "AS IS" BASIS,
12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 * See the License for the specific language governing permissions and
14 * limitations under the License.
15 */
16
17#include "rsCpuCore.h"
18#include "rsCpuScript.h"
19#include "rsCpuExecutable.h"
20
21#ifdef RS_COMPATIBILITY_LIB
22    #include <stdio.h>
23    #include <sys/stat.h>
24    #include <unistd.h>
25#else
26    #include "rsCppUtils.h"
27
28    #include <bcc/Config/Config.h>
29    #include <bcinfo/MetadataExtractor.h>
30    #include <cutils/properties.h>
31
32    #include <zlib.h>
33    #include <sys/file.h>
34    #include <sys/types.h>
35    #include <unistd.h>
36
37    #include <string>
38    #include <vector>
39#endif
40
41#include <set>
42#include <string>
43#include <dlfcn.h>
44#include <stdlib.h>
45#include <string.h>
46#include <iostream>
47#include <sstream>
48
49#ifdef __LP64__
50#define SYSLIBPATH "/system/lib64"
51#else
52#define SYSLIBPATH "/system/lib"
53#endif
54
55namespace {
56
57static const bool kDebugGlobalVariables = false;
58
59#ifndef RS_COMPATIBILITY_LIB
60
61static bool is_force_recompile() {
62#ifdef RS_SERVER
63  return false;
64#else
65  char buf[PROPERTY_VALUE_MAX];
66
67  // Re-compile if floating point precision has been overridden.
68  property_get("debug.rs.precision", buf, "");
69  if (buf[0] != '\0') {
70    return true;
71  }
72
73  // Re-compile if debug.rs.forcerecompile is set.
74  property_get("debug.rs.forcerecompile", buf, "0");
75  if ((::strcmp(buf, "1") == 0) || (::strcmp(buf, "true") == 0)) {
76    return true;
77  } else {
78    return false;
79  }
80#endif  // RS_SERVER
81}
82
83static void setCompileArguments(std::vector<const char*>* args,
84                                const std::string& bcFileName,
85                                const char* cacheDir, const char* resName,
86                                const char* core_lib, bool useRSDebugContext,
87                                const char* bccPluginName, bool emitGlobalInfo,
88                                bool emitGlobalInfoSkipConstant) {
89    rsAssert(cacheDir && resName && core_lib);
90    args->push_back(android::renderscript::RsdCpuScriptImpl::BCC_EXE_PATH);
91    args->push_back("-unroll-runtime");
92    args->push_back("-scalarize-load-store");
93    if (emitGlobalInfo) {
94        args->push_back("-rs-global-info");
95        if (emitGlobalInfoSkipConstant) {
96            args->push_back("-rs-global-info-skip-constant");
97        }
98    }
99    args->push_back("-o");
100    args->push_back(resName);
101    args->push_back("-output_path");
102    args->push_back(cacheDir);
103    args->push_back("-bclib");
104    args->push_back(core_lib);
105    args->push_back("-mtriple");
106    args->push_back(DEFAULT_TARGET_TRIPLE_STRING);
107
108    // Enable workaround for A53 codegen by default.
109#if defined(__aarch64__) && !defined(DISABLE_A53_WORKAROUND)
110    args->push_back("-aarch64-fix-cortex-a53-835769");
111#endif
112
113    // Execute the bcc compiler.
114    if (useRSDebugContext) {
115        args->push_back("-rs-debug-ctx");
116    } else {
117        // Only load additional libraries for compiles that don't use
118        // the debug context.
119        if (bccPluginName && strlen(bccPluginName) > 0) {
120            args->push_back("-load");
121            args->push_back(bccPluginName);
122        }
123    }
124
125    args->push_back("-fPIC");
126    args->push_back("-embedRSInfo");
127
128    args->push_back(bcFileName.c_str());
129    args->push_back(nullptr);
130}
131
132static bool compileBitcode(const std::string &bcFileName,
133                           const char *bitcode,
134                           size_t bitcodeSize,
135                           std::vector<const char *> &compileArguments) {
136    rsAssert(bitcode && bitcodeSize);
137
138    FILE *bcfile = fopen(bcFileName.c_str(), "w");
139    if (!bcfile) {
140        ALOGE("Could not write to %s", bcFileName.c_str());
141        return false;
142    }
143    size_t nwritten = fwrite(bitcode, 1, bitcodeSize, bcfile);
144    fclose(bcfile);
145    if (nwritten != bitcodeSize) {
146        ALOGE("Could not write %zu bytes to %s", bitcodeSize,
147              bcFileName.c_str());
148        return false;
149    }
150
151    return android::renderscript::rsuExecuteCommand(
152                   android::renderscript::RsdCpuScriptImpl::BCC_EXE_PATH,
153                   compileArguments.size()-1, compileArguments.data());
154}
155
156bool isChecksumNeeded() {
157    char buf[PROPERTY_VALUE_MAX];
158    property_get("ro.debuggable", buf, "");
159    return (buf[0] == '1');
160}
161
162bool addFileToChecksum(const char *fileName, uint32_t &checksum) {
163    int FD = open(fileName, O_RDONLY);
164    if (FD == -1) {
165        ALOGE("Cannot open file \'%s\' to compute checksum", fileName);
166        return false;
167    }
168
169    char buf[256];
170    while (true) {
171        ssize_t nread = read(FD, buf, sizeof(buf));
172        if (nread < 0) { // bail out on failed read
173            ALOGE("Error while computing checksum for file \'%s\'", fileName);
174            return false;
175        }
176
177        checksum = adler32(checksum, (const unsigned char *) buf, nread);
178        if (static_cast<size_t>(nread) < sizeof(buf)) // EOF
179            break;
180    }
181
182    if (close(FD) != 0) {
183        ALOGE("Cannot close file \'%s\' after computing checksum", fileName);
184        return false;
185    }
186    return true;
187}
188
189#endif  // !defined(RS_COMPATIBILITY_LIB)
190}  // namespace
191
192namespace android {
193namespace renderscript {
194
195#ifndef RS_COMPATIBILITY_LIB
196
197uint32_t constructBuildChecksum(uint8_t const *bitcode, size_t bitcodeSize,
198                                const char *commandLine,
199                                const char** bccFiles, size_t numFiles) {
200    uint32_t checksum = adler32(0L, Z_NULL, 0);
201
202    // include checksum of bitcode
203    if (bitcode != nullptr && bitcodeSize > 0) {
204        checksum = adler32(checksum, bitcode, bitcodeSize);
205    }
206
207    // include checksum of command line arguments
208    checksum = adler32(checksum, (const unsigned char *) commandLine,
209                       strlen(commandLine));
210
211    // include checksum of bccFiles
212    for (size_t i = 0; i < numFiles; i++) {
213        const char* bccFile = bccFiles[i];
214        if (bccFile[0] != 0 && !addFileToChecksum(bccFile, checksum)) {
215            // return empty checksum instead of something partial/corrupt
216            return 0;
217        }
218    }
219
220    return checksum;
221}
222
223#endif  // !RS_COMPATIBILITY_LIB
224
225RsdCpuScriptImpl::RsdCpuScriptImpl(RsdCpuReferenceImpl *ctx, const Script *s) {
226    mCtx = ctx;
227    mScript = s;
228
229    mScriptSO = nullptr;
230
231    mRoot = nullptr;
232    mRootExpand = nullptr;
233    mInit = nullptr;
234    mFreeChildren = nullptr;
235    mScriptExec = nullptr;
236
237    mBoundAllocs = nullptr;
238    mIntrinsicData = nullptr;
239    mIsThreadable = true;
240
241    mBuildChecksum = 0;
242    mChecksumNeeded = false;
243}
244
245bool RsdCpuScriptImpl::storeRSInfoFromSO() {
246    // The shared object may have an invalid build checksum.
247    // Validate and fail early.
248    mScriptExec = ScriptExecutable::createFromSharedObject(
249            mCtx->getContext(), mScriptSO,
250            mChecksumNeeded ? mBuildChecksum : 0);
251
252    if (mScriptExec == nullptr) {
253        return false;
254    }
255
256    mRoot = (RootFunc_t) dlsym(mScriptSO, "root");
257    if (mRoot) {
258        //ALOGE("Found root(): %p", mRoot);
259    }
260    mRootExpand = (RootFunc_t) dlsym(mScriptSO, "root.expand");
261    if (mRootExpand) {
262        //ALOGE("Found root.expand(): %p", mRootExpand);
263    }
264    mInit = (InvokeFunc_t) dlsym(mScriptSO, "init");
265    if (mInit) {
266        //ALOGE("Found init(): %p", mInit);
267    }
268    mFreeChildren = (InvokeFunc_t) dlsym(mScriptSO, ".rs.dtor");
269    if (mFreeChildren) {
270        //ALOGE("Found .rs.dtor(): %p", mFreeChildren);
271    }
272
273    size_t varCount = mScriptExec->getExportedVariableCount();
274    if (varCount > 0) {
275        mBoundAllocs = new Allocation *[varCount];
276        memset(mBoundAllocs, 0, varCount * sizeof(*mBoundAllocs));
277    }
278
279    mIsThreadable = mScriptExec->getThreadable();
280    //ALOGE("Script isThreadable? %d", mIsThreadable);
281
282    if (kDebugGlobalVariables) {
283        mScriptExec->dumpGlobalInfo();
284    }
285
286    return true;
287}
288
289bool RsdCpuScriptImpl::init(char const *resName, char const *cacheDir,
290                            uint8_t const *bitcode, size_t bitcodeSize,
291                            uint32_t flags, char const *bccPluginName) {
292    //ALOGE("rsdScriptCreate %p %p %p %p %i %i %p", rsc, resName, cacheDir,
293    // bitcode, bitcodeSize, flags, lookupFunc);
294    //ALOGE("rsdScriptInit %p %p", rsc, script);
295
296    mCtx->lockMutex();
297#ifndef RS_COMPATIBILITY_LIB
298    bool useRSDebugContext = false;
299
300    bcinfo::MetadataExtractor bitcodeMetadata((const char *) bitcode, bitcodeSize);
301    if (!bitcodeMetadata.extract()) {
302        ALOGE("Could not extract metadata from bitcode");
303        mCtx->unlockMutex();
304        return false;
305    }
306
307    const char* core_lib = findCoreLib(bitcodeMetadata, (const char*)bitcode, bitcodeSize);
308
309    if (mCtx->getContext()->getContextType() == RS_CONTEXT_TYPE_DEBUG) {
310        useRSDebugContext = true;
311    }
312
313    std::string bcFileName(cacheDir);
314    bcFileName.append("/");
315    bcFileName.append(resName);
316    bcFileName.append(".bc");
317
318    std::vector<const char*> compileArguments;
319    bool emitGlobalInfo = mCtx->getEmbedGlobalInfo();
320    bool emitGlobalInfoSkipConstant = mCtx->getEmbedGlobalInfoSkipConstant();
321    setCompileArguments(&compileArguments, bcFileName, cacheDir, resName, core_lib,
322                        useRSDebugContext, bccPluginName, emitGlobalInfo,
323                        emitGlobalInfoSkipConstant);
324
325    mChecksumNeeded = isChecksumNeeded();
326    if (mChecksumNeeded) {
327        std::vector<const char *> bccFiles = { BCC_EXE_PATH,
328                                               core_lib,
329                                             };
330
331        // The last argument of compileArguments is a nullptr, so remove 1 from
332        // the size.
333        std::unique_ptr<const char> compileCommandLine(
334            rsuJoinStrings(compileArguments.size()-1, compileArguments.data()));
335
336        mBuildChecksum = constructBuildChecksum(bitcode, bitcodeSize,
337                                                compileCommandLine.get(),
338                                                bccFiles.data(), bccFiles.size());
339
340        if (mBuildChecksum == 0) {
341            // cannot compute checksum but verification is enabled
342            mCtx->unlockMutex();
343            return false;
344        }
345    }
346    else {
347        // add a dummy/constant as a checksum if verification is disabled
348        mBuildChecksum = 0xabadcafe;
349    }
350
351    // Append build checksum to commandline
352    // Handle the terminal nullptr in compileArguments
353    compileArguments.pop_back();
354    compileArguments.push_back("-build-checksum");
355    std::stringstream ss;
356    ss << std::hex << mBuildChecksum;
357    compileArguments.push_back(ss.str().c_str());
358    compileArguments.push_back(nullptr);
359
360    if (!is_force_recompile() && !useRSDebugContext) {
361        mScriptSO = SharedLibraryUtils::loadSharedLibrary(cacheDir, resName);
362
363        // Read RS info from the shared object to detect checksum mismatch
364        if (mScriptSO != nullptr && !storeRSInfoFromSO()) {
365            dlclose(mScriptSO);
366            mScriptSO = nullptr;
367        }
368    }
369
370    // If we can't, it's either not there or out of date.  We compile the bit code and try loading
371    // again.
372    if (mScriptSO == nullptr) {
373        if (!compileBitcode(bcFileName, (const char*)bitcode, bitcodeSize,
374                            compileArguments))
375        {
376            ALOGE("bcc: FAILS to compile '%s'", resName);
377            mCtx->unlockMutex();
378            return false;
379        }
380
381        if (!SharedLibraryUtils::createSharedLibrary(mCtx->getContext()->getDriverName(),
382                                                     cacheDir, resName)) {
383            ALOGE("Linker: Failed to link object file '%s'", resName);
384            mCtx->unlockMutex();
385            return false;
386        }
387
388        mScriptSO = SharedLibraryUtils::loadSharedLibrary(cacheDir, resName);
389        if (mScriptSO == nullptr) {
390            ALOGE("Unable to load '%s'", resName);
391            mCtx->unlockMutex();
392            return false;
393        }
394
395        // Read RS symbol information from the .so.
396        if (!storeRSInfoFromSO()) {
397            goto error;
398        }
399    }
400
401    mBitcodeFilePath.setTo(bcFileName.c_str());
402
403#else  // RS_COMPATIBILITY_LIB is defined
404    const char *nativeLibDir = mCtx->getContext()->getNativeLibDir();
405    mScriptSO = SharedLibraryUtils::loadSharedLibrary(cacheDir, resName, nativeLibDir);
406
407    if (!mScriptSO) {
408        goto error;
409    }
410
411    if (!storeRSInfoFromSO()) {
412        goto error;
413    }
414#endif
415    mCtx->unlockMutex();
416    return true;
417
418error:
419
420    mCtx->unlockMutex();
421    if (mScriptSO) {
422        dlclose(mScriptSO);
423        mScriptSO = nullptr;
424    }
425    return false;
426}
427
428#ifndef RS_COMPATIBILITY_LIB
429
430const char* RsdCpuScriptImpl::findCoreLib(const bcinfo::MetadataExtractor& ME, const char* bitcode,
431                                          size_t bitcodeSize) {
432    const char* defaultLib = SYSLIBPATH"/libclcore.bc";
433
434    // If we're debugging, use the debug library.
435    if (mCtx->getContext()->getContextType() == RS_CONTEXT_TYPE_DEBUG) {
436        return SYSLIBPATH"/libclcore_debug.bc";
437    }
438
439    // If a callback has been registered to specify a library, use that.
440    RSSelectRTCallback selectRTCallback = mCtx->getSelectRTCallback();
441    if (selectRTCallback != nullptr) {
442        return selectRTCallback((const char*)bitcode, bitcodeSize);
443    }
444
445    // Check for a platform specific library
446#if defined(ARCH_ARM_HAVE_NEON) && !defined(DISABLE_CLCORE_NEON)
447    enum bcinfo::RSFloatPrecision prec = ME.getRSFloatPrecision();
448    if (prec == bcinfo::RS_FP_Relaxed) {
449        // NEON-capable ARMv7a devices can use an accelerated math library
450        // for all reduced precision scripts.
451        // ARMv8 does not use NEON, as ASIMD can be used with all precision
452        // levels.
453        return SYSLIBPATH"/libclcore_neon.bc";
454    } else {
455        return defaultLib;
456    }
457#elif defined(__i386__) || defined(__x86_64__)
458    // x86 devices will use an optimized library.
459    return SYSLIBPATH"/libclcore_x86.bc";
460#else
461    return defaultLib;
462#endif
463}
464
465#endif
466
467void RsdCpuScriptImpl::populateScript(Script *script) {
468    // Copy info over to runtime
469    script->mHal.info.exportedFunctionCount = mScriptExec->getExportedFunctionCount();
470    script->mHal.info.exportedVariableCount = mScriptExec->getExportedVariableCount();
471    script->mHal.info.exportedPragmaCount = mScriptExec->getPragmaCount();;
472    script->mHal.info.exportedPragmaKeyList = mScriptExec->getPragmaKeys();
473    script->mHal.info.exportedPragmaValueList = mScriptExec->getPragmaValues();
474
475    // Bug, need to stash in metadata
476    if (mRootExpand) {
477        script->mHal.info.root = mRootExpand;
478    } else {
479        script->mHal.info.root = mRoot;
480    }
481}
482
483
484bool RsdCpuScriptImpl::forEachMtlsSetup(const Allocation ** ains,
485                                        uint32_t inLen,
486                                        Allocation * aout,
487                                        const void * usr, uint32_t usrLen,
488                                        const RsScriptCall *sc,
489                                        MTLaunchStruct *mtls) {
490
491    memset(mtls, 0, sizeof(MTLaunchStruct));
492
493    for (int index = inLen; --index >= 0;) {
494        const Allocation* ain = ains[index];
495
496        // possible for this to occur if IO_OUTPUT/IO_INPUT with no bound surface
497        if (ain != nullptr &&
498            (const uint8_t *)ain->mHal.drvState.lod[0].mallocPtr == nullptr) {
499
500            mCtx->getContext()->setError(RS_ERROR_BAD_SCRIPT,
501                                         "rsForEach called with null in allocations");
502            return false;
503        }
504    }
505
506    if (aout &&
507        (const uint8_t *)aout->mHal.drvState.lod[0].mallocPtr == nullptr) {
508
509        mCtx->getContext()->setError(RS_ERROR_BAD_SCRIPT,
510                                     "rsForEach called with null out allocations");
511        return false;
512    }
513
514    if (inLen > 0) {
515        const Allocation *ain0   = ains[0];
516        const Type       *inType = ain0->getType();
517
518        mtls->fep.dim.x = inType->getDimX();
519        mtls->fep.dim.y = inType->getDimY();
520        mtls->fep.dim.z = inType->getDimZ();
521
522        for (int Index = inLen; --Index >= 1;) {
523            if (!ain0->hasSameDims(ains[Index])) {
524                mCtx->getContext()->setError(RS_ERROR_BAD_SCRIPT,
525                  "Failed to launch kernel; dimensions of input and output"
526                  "allocations do not match.");
527
528                return false;
529            }
530        }
531
532    } else if (aout != nullptr) {
533        const Type *outType = aout->getType();
534
535        mtls->fep.dim.x = outType->getDimX();
536        mtls->fep.dim.y = outType->getDimY();
537        mtls->fep.dim.z = outType->getDimZ();
538
539    } else if (sc != nullptr) {
540        mtls->fep.dim.x = sc->xEnd;
541        mtls->fep.dim.y = sc->yEnd;
542        mtls->fep.dim.z = 0;
543    } else {
544        mCtx->getContext()->setError(RS_ERROR_BAD_SCRIPT,
545                                     "rsForEach called with null allocations");
546        return false;
547    }
548
549    if (inLen > 0 && aout != nullptr) {
550        if (!ains[0]->hasSameDims(aout)) {
551            mCtx->getContext()->setError(RS_ERROR_BAD_SCRIPT,
552              "Failed to launch kernel; dimensions of input and output allocations do not match.");
553
554            return false;
555        }
556    }
557
558    if (!sc || (sc->xEnd == 0)) {
559        mtls->end.x = mtls->fep.dim.x;
560    } else {
561        mtls->start.x = rsMin(mtls->fep.dim.x, sc->xStart);
562        mtls->end.x = rsMin(mtls->fep.dim.x, sc->xEnd);
563        if (mtls->start.x >= mtls->end.x) {
564            mCtx->getContext()->setError(RS_ERROR_BAD_SCRIPT,
565              "Failed to launch kernel; Invalid xStart or xEnd.");
566            return false;
567        }
568    }
569
570    if (!sc || (sc->yEnd == 0)) {
571        mtls->end.y = mtls->fep.dim.y;
572    } else {
573        mtls->start.y = rsMin(mtls->fep.dim.y, sc->yStart);
574        mtls->end.y = rsMin(mtls->fep.dim.y, sc->yEnd);
575        if (mtls->start.y >= mtls->end.y) {
576            mCtx->getContext()->setError(RS_ERROR_BAD_SCRIPT,
577              "Failed to launch kernel; Invalid yStart or yEnd.");
578            return false;
579        }
580    }
581
582    if (!sc || (sc->zEnd == 0)) {
583        mtls->end.z = mtls->fep.dim.z;
584    } else {
585        mtls->start.z = rsMin(mtls->fep.dim.z, sc->zStart);
586        mtls->end.z = rsMin(mtls->fep.dim.z, sc->zEnd);
587        if (mtls->start.z >= mtls->end.z) {
588            mCtx->getContext()->setError(RS_ERROR_BAD_SCRIPT,
589              "Failed to launch kernel; Invalid zStart or zEnd.");
590            return false;
591        }
592    }
593
594    if (!sc || (sc->arrayEnd == 0)) {
595        mtls->end.array[0] = mtls->fep.dim.array[0];
596    } else {
597        mtls->start.array[0] = rsMin(mtls->fep.dim.array[0], sc->arrayStart);
598        mtls->end.array[0] = rsMin(mtls->fep.dim.array[0], sc->arrayEnd);
599        if (mtls->start.array[0] >= mtls->end.array[0]) {
600            mCtx->getContext()->setError(RS_ERROR_BAD_SCRIPT,
601              "Failed to launch kernel; Invalid arrayStart or arrayEnd.");
602            return false;
603        }
604    }
605
606    if (!sc || (sc->array2End == 0)) {
607        mtls->end.array[1] = mtls->fep.dim.array[1];
608    } else {
609        mtls->start.array[1] = rsMin(mtls->fep.dim.array[1], sc->array2Start);
610        mtls->end.array[1] = rsMin(mtls->fep.dim.array[1], sc->array2End);
611        if (mtls->start.array[1] >= mtls->end.array[1]) {
612            mCtx->getContext()->setError(RS_ERROR_BAD_SCRIPT,
613              "Failed to launch kernel; Invalid array2Start or array2End.");
614            return false;
615        }
616    }
617
618    if (!sc || (sc->array3End == 0)) {
619        mtls->end.array[2] = mtls->fep.dim.array[2];
620    } else {
621        mtls->start.array[2] = rsMin(mtls->fep.dim.array[2], sc->array3Start);
622        mtls->end.array[2] = rsMin(mtls->fep.dim.array[2], sc->array3End);
623        if (mtls->start.array[2] >= mtls->end.array[2]) {
624            mCtx->getContext()->setError(RS_ERROR_BAD_SCRIPT,
625              "Failed to launch kernel; Invalid array3Start or array3End.");
626            return false;
627        }
628    }
629
630    if (!sc || (sc->array4End == 0)) {
631        mtls->end.array[3] = mtls->fep.dim.array[3];
632    } else {
633        mtls->start.array[3] = rsMin(mtls->fep.dim.array[3], sc->array4Start);
634        mtls->end.array[3] = rsMin(mtls->fep.dim.array[3], sc->array4End);
635        if (mtls->start.array[3] >= mtls->end.array[3]) {
636            mCtx->getContext()->setError(RS_ERROR_BAD_SCRIPT,
637              "Failed to launch kernel; Invalid array4Start or array4End.");
638            return false;
639        }
640    }
641
642
643    // The X & Y walkers always want 0-1 min even if dim is not present
644    mtls->end.x    = rsMax((uint32_t)1, mtls->end.x);
645    mtls->end.y    = rsMax((uint32_t)1, mtls->end.y);
646
647    mtls->rsc        = mCtx;
648    if (ains) {
649        memcpy(mtls->ains, ains, inLen * sizeof(ains[0]));
650    }
651    mtls->aout[0]    = aout;
652    mtls->fep.usr    = usr;
653    mtls->fep.usrLen = usrLen;
654    mtls->mSliceSize = 1;
655    mtls->mSliceNum  = 0;
656
657    mtls->isThreadable  = mIsThreadable;
658
659    if (inLen > 0) {
660        mtls->fep.inLen = inLen;
661        for (int index = inLen; --index >= 0;) {
662            mtls->fep.inPtr[index] = (const uint8_t*)ains[index]->mHal.drvState.lod[0].mallocPtr;
663            mtls->fep.inStride[index] = ains[index]->getType()->getElementSizeBytes();
664        }
665    }
666
667    if (aout != nullptr) {
668        mtls->fep.outPtr[0] = (uint8_t *)aout->mHal.drvState.lod[0].mallocPtr;
669        mtls->fep.outStride[0] = aout->getType()->getElementSizeBytes();
670    }
671
672    // All validation passed, ok to launch threads
673    return true;
674}
675
676
677void RsdCpuScriptImpl::invokeForEach(uint32_t slot,
678                                     const Allocation ** ains,
679                                     uint32_t inLen,
680                                     Allocation * aout,
681                                     const void * usr,
682                                     uint32_t usrLen,
683                                     const RsScriptCall *sc) {
684
685    MTLaunchStruct mtls;
686
687    if (forEachMtlsSetup(ains, inLen, aout, usr, usrLen, sc, &mtls)) {
688        forEachKernelSetup(slot, &mtls);
689
690        RsdCpuScriptImpl * oldTLS = mCtx->setTLS(this);
691        mCtx->launchThreads(ains, inLen, aout, sc, &mtls);
692        mCtx->setTLS(oldTLS);
693    }
694}
695
696void RsdCpuScriptImpl::forEachKernelSetup(uint32_t slot, MTLaunchStruct *mtls) {
697    mtls->script = this;
698    mtls->fep.slot = slot;
699    mtls->kernel = mScriptExec->getForEachFunction(slot);
700    rsAssert(mtls->kernel != nullptr);
701    mtls->sig = mScriptExec->getForEachSignature(slot);
702}
703
704int RsdCpuScriptImpl::invokeRoot() {
705    RsdCpuScriptImpl * oldTLS = mCtx->setTLS(this);
706    int ret = mRoot();
707    mCtx->setTLS(oldTLS);
708    return ret;
709}
710
711void RsdCpuScriptImpl::invokeInit() {
712    if (mInit) {
713        mInit();
714    }
715}
716
717void RsdCpuScriptImpl::invokeFreeChildren() {
718    if (mFreeChildren) {
719        mFreeChildren();
720    }
721}
722
723void RsdCpuScriptImpl::invokeFunction(uint32_t slot, const void *params,
724                                      size_t paramLength) {
725    //ALOGE("invoke %i %p %zu", slot, params, paramLength);
726    void * ap = nullptr;
727
728#if defined(__x86_64__)
729    // The invoked function could have input parameter of vector type for example float4 which
730    // requires void* params to be 16 bytes aligned when using SSE instructions for x86_64 platform.
731    // So try to align void* params before passing them into RS exported function.
732
733    if ((uint8_t)(uint64_t)params & 0x0F) {
734        if ((ap = (void*)memalign(16, paramLength)) != nullptr) {
735            memcpy(ap, params, paramLength);
736        } else {
737            ALOGE("x86_64: invokeFunction memalign error, still use params which"
738                  " is not 16 bytes aligned.");
739        }
740    }
741#endif
742
743    RsdCpuScriptImpl * oldTLS = mCtx->setTLS(this);
744    reinterpret_cast<void (*)(const void *, uint32_t)>(
745        mScriptExec->getInvokeFunction(slot))(ap? (const void *) ap: params, paramLength);
746
747    mCtx->setTLS(oldTLS);
748}
749
750void RsdCpuScriptImpl::setGlobalVar(uint32_t slot, const void *data, size_t dataLength) {
751    //rsAssert(!script->mFieldIsObject[slot]);
752    //ALOGE("setGlobalVar %i %p %zu", slot, data, dataLength);
753
754    //if (mIntrinsicID) {
755        //mIntrinsicFuncs.setVar(dc, script, drv->mIntrinsicData, slot, data, dataLength);
756        //return;
757    //}
758
759    int32_t *destPtr = reinterpret_cast<int32_t *>(mScriptExec->getFieldAddress(slot));
760    if (!destPtr) {
761        //ALOGV("Calling setVar on slot = %i which is null", slot);
762        return;
763    }
764
765    memcpy(destPtr, data, dataLength);
766}
767
768void RsdCpuScriptImpl::getGlobalVar(uint32_t slot, void *data, size_t dataLength) {
769    //rsAssert(!script->mFieldIsObject[slot]);
770    //ALOGE("getGlobalVar %i %p %zu", slot, data, dataLength);
771
772    int32_t *srcPtr = reinterpret_cast<int32_t *>(mScriptExec->getFieldAddress(slot));
773    if (!srcPtr) {
774        //ALOGV("Calling setVar on slot = %i which is null", slot);
775        return;
776    }
777    memcpy(data, srcPtr, dataLength);
778}
779
780
781void RsdCpuScriptImpl::setGlobalVarWithElemDims(uint32_t slot, const void *data, size_t dataLength,
782                                                const Element *elem,
783                                                const uint32_t *dims, size_t dimLength) {
784    int32_t *destPtr = reinterpret_cast<int32_t *>(mScriptExec->getFieldAddress(slot));
785    if (!destPtr) {
786        //ALOGV("Calling setVar on slot = %i which is null", slot);
787        return;
788    }
789
790    // We want to look at dimension in terms of integer components,
791    // but dimLength is given in terms of bytes.
792    dimLength /= sizeof(int);
793
794    // Only a single dimension is currently supported.
795    rsAssert(dimLength == 1);
796    if (dimLength == 1) {
797        // First do the increment loop.
798        size_t stride = elem->getSizeBytes();
799        const char *cVal = reinterpret_cast<const char *>(data);
800        for (uint32_t i = 0; i < dims[0]; i++) {
801            elem->incRefs(cVal);
802            cVal += stride;
803        }
804
805        // Decrement loop comes after (to prevent race conditions).
806        char *oldVal = reinterpret_cast<char *>(destPtr);
807        for (uint32_t i = 0; i < dims[0]; i++) {
808            elem->decRefs(oldVal);
809            oldVal += stride;
810        }
811    }
812
813    memcpy(destPtr, data, dataLength);
814}
815
816void RsdCpuScriptImpl::setGlobalBind(uint32_t slot, Allocation *data) {
817
818    //rsAssert(!script->mFieldIsObject[slot]);
819    //ALOGE("setGlobalBind %i %p", slot, data);
820
821    int32_t *destPtr = reinterpret_cast<int32_t *>(mScriptExec->getFieldAddress(slot));
822    if (!destPtr) {
823        //ALOGV("Calling setVar on slot = %i which is null", slot);
824        return;
825    }
826
827    void *ptr = nullptr;
828    mBoundAllocs[slot] = data;
829    if (data) {
830        ptr = data->mHal.drvState.lod[0].mallocPtr;
831    }
832    memcpy(destPtr, &ptr, sizeof(void *));
833}
834
835void RsdCpuScriptImpl::setGlobalObj(uint32_t slot, ObjectBase *data) {
836
837    //rsAssert(script->mFieldIsObject[slot]);
838    //ALOGE("setGlobalObj %i %p", slot, data);
839
840    int32_t *destPtr = reinterpret_cast<int32_t *>(mScriptExec->getFieldAddress(slot));
841    if (!destPtr) {
842        //ALOGV("Calling setVar on slot = %i which is null", slot);
843        return;
844    }
845
846    rsrSetObject(mCtx->getContext(), (rs_object_base *)destPtr, data);
847}
848
849const char* RsdCpuScriptImpl::getFieldName(uint32_t slot) const {
850    return mScriptExec->getFieldName(slot);
851}
852
853RsdCpuScriptImpl::~RsdCpuScriptImpl() {
854    if (mScriptExec != nullptr) {
855        delete mScriptExec;
856    }
857    if (mBoundAllocs) delete[] mBoundAllocs;
858    if (mScriptSO) {
859        dlclose(mScriptSO);
860    }
861}
862
863Allocation * RsdCpuScriptImpl::getAllocationForPointer(const void *ptr) const {
864    if (!ptr) {
865        return nullptr;
866    }
867
868    for (uint32_t ct=0; ct < mScript->mHal.info.exportedVariableCount; ct++) {
869        Allocation *a = mBoundAllocs[ct];
870        if (!a) continue;
871        if (a->mHal.drvState.lod[0].mallocPtr == ptr) {
872            return a;
873        }
874    }
875    ALOGE("rsGetAllocation, failed to find %p", ptr);
876    return nullptr;
877}
878
879int RsdCpuScriptImpl::getGlobalEntries() const {
880    return mScriptExec->getGlobalEntries();
881}
882
883const char * RsdCpuScriptImpl::getGlobalName(int i) const {
884    return mScriptExec->getGlobalName(i);
885}
886
887const void * RsdCpuScriptImpl::getGlobalAddress(int i) const {
888    return mScriptExec->getGlobalAddress(i);
889}
890
891size_t RsdCpuScriptImpl::getGlobalSize(int i) const {
892    return mScriptExec->getGlobalSize(i);
893}
894
895uint32_t RsdCpuScriptImpl::getGlobalProperties(int i) const {
896    return mScriptExec->getGlobalProperties(i);
897}
898
899void RsdCpuScriptImpl::preLaunch(uint32_t slot, const Allocation ** ains,
900                                 uint32_t inLen, Allocation * aout,
901                                 const void * usr, uint32_t usrLen,
902                                 const RsScriptCall *sc) {}
903
904void RsdCpuScriptImpl::postLaunch(uint32_t slot, const Allocation ** ains,
905                                  uint32_t inLen, Allocation * aout,
906                                  const void * usr, uint32_t usrLen,
907                                  const RsScriptCall *sc) {}
908
909
910}
911}
912