rsCpuScript.cpp revision a9139c724f8312b3634d213599f2d6b3b2505db2 
1/*
2 * Copyright (C) 2011-2012 The Android Open Source Project
3 *
4 * Licensed under the Apache License, Version 2.0 (the "License");
5 * you may not use this file except in compliance with the License.
6 * You may obtain a copy of the License at
7 *
8 *      http://www.apache.org/licenses/LICENSE-2.0
9 *
10 * Unless required by applicable law or agreed to in writing, software
11 * distributed under the License is distributed on an "AS IS" BASIS,
12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 * See the License for the specific language governing permissions and
14 * limitations under the License.
15 */
16
17#include "rsCpuCore.h"
18#include "rsCpuScript.h"
19#include "rsCpuExecutable.h"
20
21#ifdef RS_COMPATIBILITY_LIB
22    #include <stdio.h>
23    #include <sys/stat.h>
24    #include <unistd.h>
25#else
26    #include "rsCppUtils.h"
27
28    #include <bcc/BCCContext.h>
29    #include <bcc/Config/Config.h>
30    #include <bcc/Renderscript/RSCompilerDriver.h>
31    #include <bcinfo/MetadataExtractor.h>
32    #include <cutils/properties.h>
33
34    #include <zlib.h>
35    #include <sys/file.h>
36    #include <sys/types.h>
37    #include <unistd.h>
38
39    #include <string>
40    #include <vector>
41#endif
42
43#include <set>
44#include <string>
45#include <dlfcn.h>
46#include <stdlib.h>
47#include <string.h>
48#include <iostream>
49
50#ifdef __LP64__
51#define SYSLIBPATH "/system/lib64"
52#else
53#define SYSLIBPATH "/system/lib"
54#endif
55
56namespace {
57#ifndef RS_COMPATIBILITY_LIB
58
59static bool is_force_recompile() {
60#ifdef RS_SERVER
61  return false;
62#else
63  char buf[PROPERTY_VALUE_MAX];
64
65  // Re-compile if floating point precision has been overridden.
66  property_get("debug.rs.precision", buf, "");
67  if (buf[0] != '\0') {
68    return true;
69  }
70
71  // Re-compile if debug.rs.forcerecompile is set.
72  property_get("debug.rs.forcerecompile", buf, "0");
73  if ((::strcmp(buf, "1") == 0) || (::strcmp(buf, "true") == 0)) {
74    return true;
75  } else {
76    return false;
77  }
78#endif  // RS_SERVER
79}
80
81static void setCompileArguments(std::vector<const char*>* args,
82                                const std::string& bcFileName,
83                                const char* cacheDir, const char* resName,
84                                const char* core_lib, bool useRSDebugContext,
85                                const char* bccPluginName) {
86    rsAssert(cacheDir && resName && core_lib);
87    args->push_back(android::renderscript::RsdCpuScriptImpl::BCC_EXE_PATH);
88    args->push_back("-unroll-runtime");
89    args->push_back("-scalarize-load-store");
90    args->push_back("-o");
91    args->push_back(resName);
92    args->push_back("-output_path");
93    args->push_back(cacheDir);
94    args->push_back("-bclib");
95    args->push_back(core_lib);
96    args->push_back("-mtriple");
97    args->push_back(DEFAULT_TARGET_TRIPLE_STRING);
98
99    // Enable workaround for A53 codegen by default.
100#if defined(__aarch64__) && !defined(DISABLE_A53_WORKAROUND)
101    args->push_back("-aarch64-fix-cortex-a53-835769");
102#endif
103
104    // Execute the bcc compiler.
105    if (useRSDebugContext) {
106        args->push_back("-rs-debug-ctx");
107    } else {
108        // Only load additional libraries for compiles that don't use
109        // the debug context.
110        if (bccPluginName && strlen(bccPluginName) > 0) {
111            args->push_back("-load");
112            args->push_back(bccPluginName);
113        }
114    }
115
116    args->push_back("-fPIC");
117    args->push_back("-embedRSInfo");
118
119    args->push_back(bcFileName.c_str());
120    args->push_back(nullptr);
121}
122
123static bool compileBitcode(const std::string &bcFileName,
124                           const char *bitcode,
125                           size_t bitcodeSize,
126                           std::vector<const char *> &compileArguments) {
127    rsAssert(bitcode && bitcodeSize);
128
129    FILE *bcfile = fopen(bcFileName.c_str(), "w");
130    if (!bcfile) {
131        ALOGE("Could not write to %s", bcFileName.c_str());
132        return false;
133    }
134    size_t nwritten = fwrite(bitcode, 1, bitcodeSize, bcfile);
135    fclose(bcfile);
136    if (nwritten != bitcodeSize) {
137        ALOGE("Could not write %zu bytes to %s", bitcodeSize,
138              bcFileName.c_str());
139        return false;
140    }
141
142    return android::renderscript::rsuExecuteCommand(
143                   android::renderscript::RsdCpuScriptImpl::BCC_EXE_PATH,
144                   compileArguments.size()-1, compileArguments.data());
145}
146
147bool isChecksumNeeded() {
148    char buf[PROPERTY_VALUE_MAX];
149    property_get("ro.debuggable", buf, "");
150    return (buf[0] == '1');
151}
152
153char *constructBuildChecksum(uint8_t const *bitcode, size_t bitcodeSize,
154                             const char *commandLine,
155                             const std::vector<const char *> &bccFiles) {
156    uint32_t checksum = adler32(0L, Z_NULL, 0);
157
158    // include checksum of bitcode
159    checksum = adler32(checksum, bitcode, bitcodeSize);
160
161    // include checksum of command line arguments
162    checksum = adler32(checksum, (const unsigned char *) commandLine,
163                       strlen(commandLine));
164
165    // include checksum of bccFiles
166    for (auto bccFile : bccFiles) {
167        if (!android::renderscript::addFileToChecksum(bccFile, checksum)) {
168            // return empty checksum instead of something partial/corrupt
169            return nullptr;
170        }
171    }
172
173    char *checksumStr = new char[9]();
174    sprintf(checksumStr, "%08x", checksum);
175    return checksumStr;
176}
177
178#endif  // !defined(RS_COMPATIBILITY_LIB)
179}  // namespace
180
181namespace android {
182namespace renderscript {
183
184#ifndef RS_COMPATIBILITY_LIB
185bool addFileToChecksum(const char *fileName, uint32_t &checksum) {
186    int FD = open(fileName, O_RDONLY);
187    if (FD == -1) {
188        ALOGE("Cannot open file \'%s\' to compute checksum", fileName);
189        return false;
190    }
191
192    char buf[256];
193    while (true) {
194        ssize_t nread = read(FD, buf, sizeof(buf));
195        if (nread < 0) { // bail out on failed read
196            ALOGE("Error while computing checksum for file \'%s\'", fileName);
197            return false;
198        }
199
200        checksum = adler32(checksum, (const unsigned char *) buf, nread);
201        if (static_cast<size_t>(nread) < sizeof(buf)) // EOF
202            break;
203    }
204
205    if (TEMP_FAILURE_RETRY(close(FD)) != 0) {
206        ALOGE("Cannot close file \'%s\' after computing checksum", fileName);
207        return false;
208    }
209    return true;
210}
211#endif  // !RS_COMPATIBILITY_LIB
212
213RsdCpuScriptImpl::RsdCpuScriptImpl(RsdCpuReferenceImpl *ctx, const Script *s) {
214    mCtx = ctx;
215    mScript = s;
216
217    mScriptSO = nullptr;
218
219#ifndef RS_COMPATIBILITY_LIB
220    mCompilerDriver = nullptr;
221#endif
222
223
224    mRoot = nullptr;
225    mRootExpand = nullptr;
226    mInit = nullptr;
227    mFreeChildren = nullptr;
228    mScriptExec = nullptr;
229
230    mBoundAllocs = nullptr;
231    mIntrinsicData = nullptr;
232    mIsThreadable = true;
233
234    mBuildChecksum = nullptr;
235    mChecksumNeeded = false;
236}
237
238bool RsdCpuScriptImpl::storeRSInfoFromSO() {
239    // The shared object may have an invalid build checksum.
240    // Validate and fail early.
241    mScriptExec = ScriptExecutable::createFromSharedObject(
242            mCtx->getContext(), mScriptSO);
243
244    if (mScriptExec == nullptr) {
245        return false;
246    }
247
248    if (mChecksumNeeded && !mScriptExec->isChecksumValid(mBuildChecksum)) {
249        ALOGE("Found invalid checksum.  Expected %s, got %s\n",
250                  mBuildChecksum, mScriptExec->getBuildChecksum());
251        delete mScriptExec;
252        mScriptExec = nullptr;
253        return false;
254    }
255
256    mRoot = (RootFunc_t) dlsym(mScriptSO, "root");
257    if (mRoot) {
258        //ALOGE("Found root(): %p", mRoot);
259    }
260    mRootExpand = (RootFunc_t) dlsym(mScriptSO, "root.expand");
261    if (mRootExpand) {
262        //ALOGE("Found root.expand(): %p", mRootExpand);
263    }
264    mInit = (InvokeFunc_t) dlsym(mScriptSO, "init");
265    if (mInit) {
266        //ALOGE("Found init(): %p", mInit);
267    }
268    mFreeChildren = (InvokeFunc_t) dlsym(mScriptSO, ".rs.dtor");
269    if (mFreeChildren) {
270        //ALOGE("Found .rs.dtor(): %p", mFreeChildren);
271    }
272
273    size_t varCount = mScriptExec->getExportedVariableCount();
274    if (varCount > 0) {
275        mBoundAllocs = new Allocation *[varCount];
276        memset(mBoundAllocs, 0, varCount * sizeof(*mBoundAllocs));
277    }
278
279    mIsThreadable = mScriptExec->getThreadable();
280    //ALOGE("Script isThreadable? %d", mIsThreadable);
281
282    return true;
283}
284
285bool RsdCpuScriptImpl::init(char const *resName, char const *cacheDir,
286                            uint8_t const *bitcode, size_t bitcodeSize,
287                            uint32_t flags, char const *bccPluginName) {
288    //ALOGE("rsdScriptCreate %p %p %p %p %i %i %p", rsc, resName, cacheDir,
289    // bitcode, bitcodeSize, flags, lookupFunc);
290    //ALOGE("rsdScriptInit %p %p", rsc, script);
291
292    mCtx->lockMutex();
293#ifndef RS_COMPATIBILITY_LIB
294    bool useRSDebugContext = false;
295
296    mCompilerDriver = nullptr;
297
298    mCompilerDriver = new bcc::RSCompilerDriver();
299    if (mCompilerDriver == nullptr) {
300        ALOGE("bcc: FAILS to create compiler driver (out of memory)");
301        mCtx->unlockMutex();
302        return false;
303    }
304
305    // Run any compiler setup functions we have been provided with.
306    RSSetupCompilerCallback setupCompilerCallback =
307            mCtx->getSetupCompilerCallback();
308    if (setupCompilerCallback != nullptr) {
309        setupCompilerCallback(mCompilerDriver);
310    }
311
312    bcinfo::MetadataExtractor bitcodeMetadata((const char *) bitcode, bitcodeSize);
313    if (!bitcodeMetadata.extract()) {
314        ALOGE("Could not extract metadata from bitcode");
315        mCtx->unlockMutex();
316        return false;
317    }
318
319    const char* core_lib = findCoreLib(bitcodeMetadata, (const char*)bitcode, bitcodeSize);
320
321    if (mCtx->getContext()->getContextType() == RS_CONTEXT_TYPE_DEBUG) {
322        mCompilerDriver->setDebugContext(true);
323        useRSDebugContext = true;
324    }
325
326    std::string bcFileName(cacheDir);
327    bcFileName.append("/");
328    bcFileName.append(resName);
329    bcFileName.append(".bc");
330
331    std::vector<const char*> compileArguments;
332    setCompileArguments(&compileArguments, bcFileName, cacheDir, resName, core_lib,
333                        useRSDebugContext, bccPluginName);
334
335    mChecksumNeeded = isChecksumNeeded();
336    if (mChecksumNeeded) {
337        std::vector<const char *> bccFiles = { BCC_EXE_PATH,
338                                               core_lib,
339                                             };
340
341        // The last argument of compileArguments is a nullptr, so remove 1 from
342        // the size.
343        std::unique_ptr<const char> compileCommandLine(
344            rsuJoinStrings(compileArguments.size()-1, compileArguments.data()));
345
346        mBuildChecksum = constructBuildChecksum(bitcode, bitcodeSize,
347                                                compileCommandLine.get(),
348                                                bccFiles);
349
350        if (mBuildChecksum == nullptr) {
351            // cannot compute checksum but verification is enabled
352            mCtx->unlockMutex();
353            return false;
354        }
355    }
356    else {
357        // add a dummy/constant as a checksum if verification is disabled
358        mBuildChecksum = new char[9]();
359        strcpy(const_cast<char *>(mBuildChecksum), "abadcafe");
360    }
361
362    // Append build checksum to commandline
363    // Handle the terminal nullptr in compileArguments
364    compileArguments.pop_back();
365    compileArguments.push_back("-build-checksum");
366    compileArguments.push_back(mBuildChecksum);
367    compileArguments.push_back(nullptr);
368
369    if (!is_force_recompile() && !useRSDebugContext) {
370        mScriptSO = SharedLibraryUtils::loadSharedLibrary(cacheDir, resName);
371
372        // Read RS info from the shared object to detect checksum mismatch
373        if (mScriptSO != nullptr && !storeRSInfoFromSO()) {
374            dlclose(mScriptSO);
375            mScriptSO = nullptr;
376        }
377    }
378
379    // If we can't, it's either not there or out of date.  We compile the bit code and try loading
380    // again.
381    if (mScriptSO == nullptr) {
382        if (!compileBitcode(bcFileName, (const char*)bitcode, bitcodeSize,
383                            compileArguments))
384        {
385            ALOGE("bcc: FAILS to compile '%s'", resName);
386            mCtx->unlockMutex();
387            return false;
388        }
389
390        if (!SharedLibraryUtils::createSharedLibrary(cacheDir, resName)) {
391            ALOGE("Linker: Failed to link object file '%s'", resName);
392            mCtx->unlockMutex();
393            return false;
394        }
395
396        mScriptSO = SharedLibraryUtils::loadSharedLibrary(cacheDir, resName);
397        if (mScriptSO == nullptr) {
398            ALOGE("Unable to load '%s'", resName);
399            mCtx->unlockMutex();
400            return false;
401        }
402
403        // Read RS symbol information from the .so.
404        if (!storeRSInfoFromSO()) {
405            goto error;
406        }
407    }
408
409    mBitcodeFilePath.setTo(bcFileName.c_str());
410
411#else  // RS_COMPATIBILITY_LIB is defined
412    const char *nativeLibDir = mCtx->getContext()->getNativeLibDir();
413    mScriptSO = SharedLibraryUtils::loadSharedLibrary(cacheDir, resName, nativeLibDir);
414
415    if (!mScriptSO) {
416        goto error;
417    }
418
419    if (!storeRSInfoFromSO()) {
420        goto error;
421    }
422#endif
423    mCtx->unlockMutex();
424    return true;
425
426error:
427
428    mCtx->unlockMutex();
429    if (mScriptSO) {
430        dlclose(mScriptSO);
431        mScriptSO = nullptr;
432    }
433    return false;
434}
435
436#ifndef RS_COMPATIBILITY_LIB
437
438const char* RsdCpuScriptImpl::findCoreLib(const bcinfo::MetadataExtractor& ME, const char* bitcode,
439                                          size_t bitcodeSize) {
440    const char* defaultLib = SYSLIBPATH"/libclcore.bc";
441
442    // If we're debugging, use the debug library.
443    if (mCtx->getContext()->getContextType() == RS_CONTEXT_TYPE_DEBUG) {
444        return SYSLIBPATH"/libclcore_debug.bc";
445    }
446
447    // If a callback has been registered to specify a library, use that.
448    RSSelectRTCallback selectRTCallback = mCtx->getSelectRTCallback();
449    if (selectRTCallback != nullptr) {
450        return selectRTCallback((const char*)bitcode, bitcodeSize);
451    }
452
453    // Check for a platform specific library
454#if defined(ARCH_ARM_HAVE_NEON) && !defined(DISABLE_CLCORE_NEON)
455    enum bcinfo::RSFloatPrecision prec = ME.getRSFloatPrecision();
456    if (prec == bcinfo::RS_FP_Relaxed) {
457        // NEON-capable ARMv7a devices can use an accelerated math library
458        // for all reduced precision scripts.
459        // ARMv8 does not use NEON, as ASIMD can be used with all precision
460        // levels.
461        return SYSLIBPATH"/libclcore_neon.bc";
462    } else {
463        return defaultLib;
464    }
465#elif defined(__i386__) || defined(__x86_64__)
466    // x86 devices will use an optimized library.
467    return SYSLIBPATH"/libclcore_x86.bc";
468#else
469    return defaultLib;
470#endif
471}
472
473#endif
474
475void RsdCpuScriptImpl::populateScript(Script *script) {
476    // Copy info over to runtime
477    script->mHal.info.exportedFunctionCount = mScriptExec->getExportedFunctionCount();
478    script->mHal.info.exportedVariableCount = mScriptExec->getExportedVariableCount();
479    script->mHal.info.exportedPragmaCount = mScriptExec->getPragmaCount();;
480    script->mHal.info.exportedPragmaKeyList = mScriptExec->getPragmaKeys();
481    script->mHal.info.exportedPragmaValueList = mScriptExec->getPragmaValues();
482
483    // Bug, need to stash in metadata
484    if (mRootExpand) {
485        script->mHal.info.root = mRootExpand;
486    } else {
487        script->mHal.info.root = mRoot;
488    }
489}
490
491
492typedef void (*rs_t)(const void *, void *, const void *, uint32_t, uint32_t, uint32_t, uint32_t);
493
494bool RsdCpuScriptImpl::forEachMtlsSetup(const Allocation ** ains,
495                                        uint32_t inLen,
496                                        Allocation * aout,
497                                        const void * usr, uint32_t usrLen,
498                                        const RsScriptCall *sc,
499                                        MTLaunchStruct *mtls) {
500
501    memset(mtls, 0, sizeof(MTLaunchStruct));
502
503    for (int index = inLen; --index >= 0;) {
504        const Allocation* ain = ains[index];
505
506        // possible for this to occur if IO_OUTPUT/IO_INPUT with no bound surface
507        if (ain != nullptr &&
508            (const uint8_t *)ain->mHal.drvState.lod[0].mallocPtr == nullptr) {
509
510            mCtx->getContext()->setError(RS_ERROR_BAD_SCRIPT,
511                                         "rsForEach called with null in allocations");
512            return false;
513        }
514    }
515
516    if (aout &&
517        (const uint8_t *)aout->mHal.drvState.lod[0].mallocPtr == nullptr) {
518
519        mCtx->getContext()->setError(RS_ERROR_BAD_SCRIPT,
520                                     "rsForEach called with null out allocations");
521        return false;
522    }
523
524    if (inLen > 0) {
525        const Allocation *ain0   = ains[0];
526        const Type       *inType = ain0->getType();
527
528        mtls->fep.dim.x = inType->getDimX();
529        mtls->fep.dim.y = inType->getDimY();
530        mtls->fep.dim.z = inType->getDimZ();
531
532        for (int Index = inLen; --Index >= 1;) {
533            if (!ain0->hasSameDims(ains[Index])) {
534                mCtx->getContext()->setError(RS_ERROR_BAD_SCRIPT,
535                  "Failed to launch kernel; dimensions of input and output"
536                  "allocations do not match.");
537
538                return false;
539            }
540        }
541
542    } else if (aout != nullptr) {
543        const Type *outType = aout->getType();
544
545        mtls->fep.dim.x = outType->getDimX();
546        mtls->fep.dim.y = outType->getDimY();
547        mtls->fep.dim.z = outType->getDimZ();
548
549    } else if (sc != nullptr) {
550        mtls->fep.dim.x = sc->xEnd;
551        mtls->fep.dim.y = sc->yEnd;
552        mtls->fep.dim.z = 0;
553    } else {
554        mCtx->getContext()->setError(RS_ERROR_BAD_SCRIPT,
555                                     "rsForEach called with null allocations");
556        return false;
557    }
558
559    if (inLen > 0 && aout != nullptr) {
560        if (!ains[0]->hasSameDims(aout)) {
561            mCtx->getContext()->setError(RS_ERROR_BAD_SCRIPT,
562              "Failed to launch kernel; dimensions of input and output allocations do not match.");
563
564            return false;
565        }
566    }
567
568    if (!sc || (sc->xEnd == 0)) {
569        mtls->end.x = mtls->fep.dim.x;
570    } else {
571        mtls->start.x = rsMin(mtls->fep.dim.x, sc->xStart);
572        mtls->end.x = rsMin(mtls->fep.dim.x, sc->xEnd);
573        if (mtls->start.x >= mtls->end.x) return false;
574    }
575
576    if (!sc || (sc->yEnd == 0)) {
577        mtls->end.y = mtls->fep.dim.y;
578    } else {
579        mtls->start.y = rsMin(mtls->fep.dim.y, sc->yStart);
580        mtls->end.y = rsMin(mtls->fep.dim.y, sc->yEnd);
581        if (mtls->start.y >= mtls->end.y) return false;
582    }
583
584    if (!sc || (sc->zEnd == 0)) {
585        mtls->end.z = mtls->fep.dim.z;
586    } else {
587        mtls->start.z = rsMin(mtls->fep.dim.z, sc->zStart);
588        mtls->end.z = rsMin(mtls->fep.dim.z, sc->zEnd);
589        if (mtls->start.z >= mtls->end.z) return false;
590    }
591
592    if (!sc || (sc->arrayEnd == 0)) {
593        mtls->end.array[0] = mtls->fep.dim.array[0];
594    } else {
595        mtls->start.array[0] = rsMin(mtls->fep.dim.array[0], sc->arrayStart);
596        mtls->end.array[0] = rsMin(mtls->fep.dim.array[0], sc->arrayEnd);
597        if (mtls->start.array[0] >= mtls->end.array[0]) return false;
598    }
599
600    if (!sc || (sc->array2End == 0)) {
601        mtls->end.array[1] = mtls->fep.dim.array[1];
602    } else {
603        mtls->start.array[1] = rsMin(mtls->fep.dim.array[1], sc->array2Start);
604        mtls->end.array[1] = rsMin(mtls->fep.dim.array[1], sc->array2End);
605        if (mtls->start.array[1] >= mtls->end.array[1]) return false;
606    }
607
608    if (!sc || (sc->array3End == 0)) {
609        mtls->end.array[2] = mtls->fep.dim.array[2];
610    } else {
611        mtls->start.array[2] = rsMin(mtls->fep.dim.array[2], sc->array3Start);
612        mtls->end.array[2] = rsMin(mtls->fep.dim.array[2], sc->array3End);
613        if (mtls->start.array[2] >= mtls->end.array[2]) return false;
614    }
615
616    if (!sc || (sc->array4End == 0)) {
617        mtls->end.array[3] = mtls->fep.dim.array[3];
618    } else {
619        mtls->start.array[3] = rsMin(mtls->fep.dim.array[3], sc->array4Start);
620        mtls->end.array[3] = rsMin(mtls->fep.dim.array[3], sc->array4End);
621        if (mtls->start.array[3] >= mtls->end.array[3]) return false;
622    }
623
624
625    // The X & Y walkers always want 0-1 min even if dim is not present
626    mtls->end.x    = rsMax((uint32_t)1, mtls->end.x);
627    mtls->end.y    = rsMax((uint32_t)1, mtls->end.y);
628
629    mtls->rsc        = mCtx;
630    if (ains) {
631        memcpy(mtls->ains, ains, inLen * sizeof(ains[0]));
632    }
633    mtls->aout[0]    = aout;
634    mtls->fep.usr    = usr;
635    mtls->fep.usrLen = usrLen;
636    mtls->mSliceSize = 1;
637    mtls->mSliceNum  = 0;
638
639    mtls->isThreadable  = mIsThreadable;
640
641    if (inLen > 0) {
642        mtls->fep.inLen = inLen;
643        for (int index = inLen; --index >= 0;) {
644            mtls->fep.inPtr[index] = (const uint8_t*)ains[index]->mHal.drvState.lod[0].mallocPtr;
645            mtls->fep.inStride[index] = ains[index]->getType()->getElementSizeBytes();
646        }
647    }
648
649    if (aout != nullptr) {
650        mtls->fep.outPtr[0] = (uint8_t *)aout->mHal.drvState.lod[0].mallocPtr;
651        mtls->fep.outStride[0] = aout->getType()->getElementSizeBytes();
652    }
653
654    // All validation passed, ok to launch threads
655    return true;
656}
657
658
659void RsdCpuScriptImpl::invokeForEach(uint32_t slot,
660                                     const Allocation ** ains,
661                                     uint32_t inLen,
662                                     Allocation * aout,
663                                     const void * usr,
664                                     uint32_t usrLen,
665                                     const RsScriptCall *sc) {
666
667    MTLaunchStruct mtls;
668
669    if (forEachMtlsSetup(ains, inLen, aout, usr, usrLen, sc, &mtls)) {
670        forEachKernelSetup(slot, &mtls);
671
672        RsdCpuScriptImpl * oldTLS = mCtx->setTLS(this);
673        mCtx->launchThreads(ains, inLen, aout, sc, &mtls);
674        mCtx->setTLS(oldTLS);
675    }
676}
677
678void RsdCpuScriptImpl::forEachKernelSetup(uint32_t slot, MTLaunchStruct *mtls) {
679    mtls->script = this;
680    mtls->fep.slot = slot;
681    mtls->kernel = mScriptExec->getForEachFunction(slot);
682    rsAssert(mtls->kernel != nullptr);
683    mtls->sig = mScriptExec->getForEachSignature(slot);
684}
685
686int RsdCpuScriptImpl::invokeRoot() {
687    RsdCpuScriptImpl * oldTLS = mCtx->setTLS(this);
688    int ret = mRoot();
689    mCtx->setTLS(oldTLS);
690    return ret;
691}
692
693void RsdCpuScriptImpl::invokeInit() {
694    if (mInit) {
695        mInit();
696    }
697}
698
699void RsdCpuScriptImpl::invokeFreeChildren() {
700    if (mFreeChildren) {
701        mFreeChildren();
702    }
703}
704
705void RsdCpuScriptImpl::invokeFunction(uint32_t slot, const void *params,
706                                      size_t paramLength) {
707    //ALOGE("invoke %i %p %zu", slot, params, paramLength);
708    void * ap = nullptr;
709
710#if defined(__x86_64__)
711    // The invoked function could have input parameter of vector type for example float4 which
712    // requires void* params to be 16 bytes aligned when using SSE instructions for x86_64 platform.
713    // So try to align void* params before passing them into RS exported function.
714
715    if ((uint8_t)(uint64_t)params & 0x0F) {
716        if ((ap = (void*)memalign(16, paramLength)) != nullptr) {
717            memcpy(ap, params, paramLength);
718        } else {
719            ALOGE("x86_64: invokeFunction memalign error, still use params which"
720                  " is not 16 bytes aligned.");
721        }
722    }
723#endif
724
725    RsdCpuScriptImpl * oldTLS = mCtx->setTLS(this);
726    reinterpret_cast<void (*)(const void *, uint32_t)>(
727        mScriptExec->getInvokeFunction(slot))(ap? (const void *) ap: params, paramLength);
728
729    mCtx->setTLS(oldTLS);
730}
731
732void RsdCpuScriptImpl::setGlobalVar(uint32_t slot, const void *data, size_t dataLength) {
733    //rsAssert(!script->mFieldIsObject[slot]);
734    //ALOGE("setGlobalVar %i %p %zu", slot, data, dataLength);
735
736    //if (mIntrinsicID) {
737        //mIntrinsicFuncs.setVar(dc, script, drv->mIntrinsicData, slot, data, dataLength);
738        //return;
739    //}
740
741    int32_t *destPtr = reinterpret_cast<int32_t *>(mScriptExec->getFieldAddress(slot));
742    if (!destPtr) {
743        //ALOGV("Calling setVar on slot = %i which is null", slot);
744        return;
745    }
746
747    memcpy(destPtr, data, dataLength);
748}
749
750void RsdCpuScriptImpl::getGlobalVar(uint32_t slot, void *data, size_t dataLength) {
751    //rsAssert(!script->mFieldIsObject[slot]);
752    //ALOGE("getGlobalVar %i %p %zu", slot, data, dataLength);
753
754    int32_t *srcPtr = reinterpret_cast<int32_t *>(mScriptExec->getFieldAddress(slot));
755    if (!srcPtr) {
756        //ALOGV("Calling setVar on slot = %i which is null", slot);
757        return;
758    }
759    memcpy(data, srcPtr, dataLength);
760}
761
762
763void RsdCpuScriptImpl::setGlobalVarWithElemDims(uint32_t slot, const void *data, size_t dataLength,
764                                                const Element *elem,
765                                                const uint32_t *dims, size_t dimLength) {
766    int32_t *destPtr = reinterpret_cast<int32_t *>(mScriptExec->getFieldAddress(slot));
767    if (!destPtr) {
768        //ALOGV("Calling setVar on slot = %i which is null", slot);
769        return;
770    }
771
772    // We want to look at dimension in terms of integer components,
773    // but dimLength is given in terms of bytes.
774    dimLength /= sizeof(int);
775
776    // Only a single dimension is currently supported.
777    rsAssert(dimLength == 1);
778    if (dimLength == 1) {
779        // First do the increment loop.
780        size_t stride = elem->getSizeBytes();
781        const char *cVal = reinterpret_cast<const char *>(data);
782        for (uint32_t i = 0; i < dims[0]; i++) {
783            elem->incRefs(cVal);
784            cVal += stride;
785        }
786
787        // Decrement loop comes after (to prevent race conditions).
788        char *oldVal = reinterpret_cast<char *>(destPtr);
789        for (uint32_t i = 0; i < dims[0]; i++) {
790            elem->decRefs(oldVal);
791            oldVal += stride;
792        }
793    }
794
795    memcpy(destPtr, data, dataLength);
796}
797
798void RsdCpuScriptImpl::setGlobalBind(uint32_t slot, Allocation *data) {
799
800    //rsAssert(!script->mFieldIsObject[slot]);
801    //ALOGE("setGlobalBind %i %p", slot, data);
802
803    int32_t *destPtr = reinterpret_cast<int32_t *>(mScriptExec->getFieldAddress(slot));
804    if (!destPtr) {
805        //ALOGV("Calling setVar on slot = %i which is null", slot);
806        return;
807    }
808
809    void *ptr = nullptr;
810    mBoundAllocs[slot] = data;
811    if (data) {
812        ptr = data->mHal.drvState.lod[0].mallocPtr;
813    }
814    memcpy(destPtr, &ptr, sizeof(void *));
815}
816
817void RsdCpuScriptImpl::setGlobalObj(uint32_t slot, ObjectBase *data) {
818
819    //rsAssert(script->mFieldIsObject[slot]);
820    //ALOGE("setGlobalObj %i %p", slot, data);
821
822    int32_t *destPtr = reinterpret_cast<int32_t *>(mScriptExec->getFieldAddress(slot));
823    if (!destPtr) {
824        //ALOGV("Calling setVar on slot = %i which is null", slot);
825        return;
826    }
827
828    rsrSetObject(mCtx->getContext(), (rs_object_base *)destPtr, data);
829}
830
831const char* RsdCpuScriptImpl::getFieldName(uint32_t slot) const {
832    return mScriptExec->getFieldName(slot);
833}
834
835RsdCpuScriptImpl::~RsdCpuScriptImpl() {
836#ifndef RS_COMPATIBILITY_LIB
837    if (mCompilerDriver) {
838        delete mCompilerDriver;
839    }
840#endif
841
842    if (mScriptExec != nullptr) {
843        delete mScriptExec;
844    }
845    if (mBoundAllocs) delete[] mBoundAllocs;
846    if (mScriptSO) {
847        dlclose(mScriptSO);
848    }
849
850    delete[] mBuildChecksum;
851}
852
853Allocation * RsdCpuScriptImpl::getAllocationForPointer(const void *ptr) const {
854    if (!ptr) {
855        return nullptr;
856    }
857
858    for (uint32_t ct=0; ct < mScript->mHal.info.exportedVariableCount; ct++) {
859        Allocation *a = mBoundAllocs[ct];
860        if (!a) continue;
861        if (a->mHal.drvState.lod[0].mallocPtr == ptr) {
862            return a;
863        }
864    }
865    ALOGE("rsGetAllocation, failed to find %p", ptr);
866    return nullptr;
867}
868
869void RsdCpuScriptImpl::preLaunch(uint32_t slot, const Allocation ** ains,
870                                 uint32_t inLen, Allocation * aout,
871                                 const void * usr, uint32_t usrLen,
872                                 const RsScriptCall *sc) {}
873
874void RsdCpuScriptImpl::postLaunch(uint32_t slot, const Allocation ** ains,
875                                  uint32_t inLen, Allocation * aout,
876                                  const void * usr, uint32_t usrLen,
877                                  const RsScriptCall *sc) {}
878
879
880}
881}
882