rsCpuScript.cpp revision aa6757ffc1b23d771566439c3179fdbc1e5ba569 
1/*
2 * Copyright (C) 2011-2012 The Android Open Source Project
3 *
4 * Licensed under the Apache License, Version 2.0 (the "License");
5 * you may not use this file except in compliance with the License.
6 * You may obtain a copy of the License at
7 *
8 *      http://www.apache.org/licenses/LICENSE-2.0
9 *
10 * Unless required by applicable law or agreed to in writing, software
11 * distributed under the License is distributed on an "AS IS" BASIS,
12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 * See the License for the specific language governing permissions and
14 * limitations under the License.
15 */
16
17#include "rsCpuCore.h"
18#include "rsCpuScript.h"
19#include "rsCpuExecutable.h"
20
21#ifdef RS_COMPATIBILITY_LIB
22    #include <stdio.h>
23    #include <sys/stat.h>
24    #include <unistd.h>
25#else
26    #include <bcc/BCCContext.h>
27    #include <bcc/Config/Config.h>
28    #include <bcc/Renderscript/RSCompilerDriver.h>
29    #include <bcinfo/MetadataExtractor.h>
30    #include <cutils/properties.h>
31
32    #include <zlib.h>
33    #include <sys/file.h>
34    #include <sys/types.h>
35    #include <sys/wait.h>
36    #include <unistd.h>
37
38    #include <string>
39    #include <vector>
40#endif
41
42#include <set>
43#include <string>
44#include <dlfcn.h>
45#include <stdlib.h>
46#include <string.h>
47#include <iostream>
48
49#ifdef __LP64__
50#define SYSLIBPATH "/system/lib64"
51#else
52#define SYSLIBPATH "/system/lib"
53#endif
54
55namespace {
56#ifndef RS_COMPATIBILITY_LIB
57
58static bool is_force_recompile() {
59#ifdef RS_SERVER
60  return false;
61#else
62  char buf[PROPERTY_VALUE_MAX];
63
64  // Re-compile if floating point precision has been overridden.
65  property_get("debug.rs.precision", buf, "");
66  if (buf[0] != '\0') {
67    return true;
68  }
69
70  // Re-compile if debug.rs.forcerecompile is set.
71  property_get("debug.rs.forcerecompile", buf, "0");
72  if ((::strcmp(buf, "1") == 0) || (::strcmp(buf, "true") == 0)) {
73    return true;
74  } else {
75    return false;
76  }
77#endif  // RS_SERVER
78}
79
80static void setCompileArguments(std::vector<const char*>* args,
81                                const std::string& bcFileName,
82                                const char* cacheDir, const char* resName,
83                                const char* core_lib, bool useRSDebugContext,
84                                const char* bccPluginName) {
85    rsAssert(cacheDir && resName && core_lib);
86    args->push_back(android::renderscript::RsdCpuScriptImpl::BCC_EXE_PATH);
87    args->push_back("-unroll-runtime");
88    args->push_back("-scalarize-load-store");
89    args->push_back("-o");
90    args->push_back(resName);
91    args->push_back("-output_path");
92    args->push_back(cacheDir);
93    args->push_back("-bclib");
94    args->push_back(core_lib);
95    args->push_back("-mtriple");
96    args->push_back(DEFAULT_TARGET_TRIPLE_STRING);
97
98    // Enable workaround for A53 codegen by default.
99#if defined(__aarch64__) && !defined(DISABLE_A53_WORKAROUND)
100    args->push_back("-aarch64-fix-cortex-a53-835769");
101#endif
102
103    // Execute the bcc compiler.
104    if (useRSDebugContext) {
105        args->push_back("-rs-debug-ctx");
106    } else {
107        // Only load additional libraries for compiles that don't use
108        // the debug context.
109        if (bccPluginName && strlen(bccPluginName) > 0) {
110            args->push_back("-load");
111            args->push_back(bccPluginName);
112        }
113    }
114
115    args->push_back("-fPIC");
116    args->push_back("-embedRSInfo");
117
118    args->push_back(bcFileName.c_str());
119    args->push_back(nullptr);
120}
121
122static bool compileBitcode(const std::string &bcFileName,
123                           const char *bitcode,
124                           size_t bitcodeSize,
125                           const char **compileArguments,
126                           const char *compileCommandLine) {
127    rsAssert(bitcode && bitcodeSize);
128
129    FILE *bcfile = fopen(bcFileName.c_str(), "w");
130    if (!bcfile) {
131        ALOGE("Could not write to %s", bcFileName.c_str());
132        return false;
133    }
134    size_t nwritten = fwrite(bitcode, 1, bitcodeSize, bcfile);
135    fclose(bcfile);
136    if (nwritten != bitcodeSize) {
137        ALOGE("Could not write %zu bytes to %s", bitcodeSize,
138              bcFileName.c_str());
139        return false;
140    }
141
142    pid_t pid = fork();
143
144    switch (pid) {
145    case -1: {  // Error occurred (we attempt no recovery)
146        ALOGE("Couldn't fork for bcc compiler execution");
147        return false;
148    }
149    case 0: {  // Child process
150        ALOGV("Invoking BCC with: %s", compileCommandLine);
151        execv(android::renderscript::RsdCpuScriptImpl::BCC_EXE_PATH,
152              (char* const*)compileArguments);
153
154        ALOGE("execv() failed: %s", strerror(errno));
155        abort();
156        return false;
157    }
158    default: {  // Parent process (actual driver)
159        // Wait on child process to finish compiling the source.
160        int status = 0;
161        pid_t w = waitpid(pid, &status, 0);
162        if (w == -1) {
163            ALOGE("Could not wait for bcc compiler");
164            return false;
165        }
166
167        if (WIFEXITED(status) && WEXITSTATUS(status) == 0) {
168            return true;
169        }
170
171        ALOGE("bcc compiler terminated unexpectedly");
172        return false;
173    }
174    }
175}
176
177bool isChecksumNeeded() {
178    char buf[PROPERTY_VALUE_MAX];
179    property_get("ro.debuggable", buf, "");
180    return (buf[0] == '1');
181}
182
183bool addFileToChecksum(const char *fileName, uint32_t &checksum) {
184    int FD = open(fileName, O_RDONLY);
185    if (FD == -1) {
186        ALOGE("Cannot open file \'%s\' to compute checksum", fileName);
187        return false;
188    }
189
190    char buf[256];
191    while (true) {
192        ssize_t nread = read(FD, buf, sizeof(buf));
193        if (nread < 0) { // bail out on failed read
194            ALOGE("Error while computing checksum for file \'%s\'", fileName);
195            return false;
196        }
197
198        checksum = adler32(checksum, (const unsigned char *) buf, nread);
199        if (static_cast<size_t>(nread) < sizeof(buf)) // EOF
200            break;
201    }
202
203    if (close(FD) != 0) {
204        ALOGE("Cannot close file \'%s\' after computing checksum", fileName);
205        return false;
206    }
207    return true;
208}
209
210char *constructBuildChecksum(uint8_t const *bitcode, size_t bitcodeSize,
211                             const char *commandLine,
212                             const std::vector<const char *> &bccFiles) {
213    uint32_t checksum = adler32(0L, Z_NULL, 0);
214
215    // include checksum of bitcode
216    checksum = adler32(checksum, bitcode, bitcodeSize);
217
218    // include checksum of command line arguments
219    checksum = adler32(checksum, (const unsigned char *) commandLine,
220                       strlen(commandLine));
221
222    // include checksum of bccFiles
223    for (auto bccFile : bccFiles) {
224        if (!addFileToChecksum(bccFile, checksum)) {
225            // return empty checksum instead of something partial/corrupt
226            return nullptr;
227        }
228    }
229
230    char *checksumStr = new char[9]();
231    sprintf(checksumStr, "%08x", checksum);
232    return checksumStr;
233}
234
235#endif  // !defined(RS_COMPATIBILITY_LIB)
236}  // namespace
237
238namespace android {
239namespace renderscript {
240
241RsdCpuScriptImpl::RsdCpuScriptImpl(RsdCpuReferenceImpl *ctx, const Script *s) {
242    mCtx = ctx;
243    mScript = s;
244
245    mScriptSO = nullptr;
246
247#ifndef RS_COMPATIBILITY_LIB
248    mCompilerDriver = nullptr;
249#endif
250
251
252    mRoot = nullptr;
253    mRootExpand = nullptr;
254    mInit = nullptr;
255    mFreeChildren = nullptr;
256    mScriptExec = nullptr;
257
258    mBoundAllocs = nullptr;
259    mIntrinsicData = nullptr;
260    mIsThreadable = true;
261
262    mBuildChecksum = nullptr;
263    mChecksumNeeded = false;
264}
265
266bool RsdCpuScriptImpl::storeRSInfoFromSO() {
267    // The shared object may have an invalid build checksum.
268    // Validate and fail early.
269    mScriptExec = ScriptExecutable::createFromSharedObject(
270            mCtx->getContext(), mScriptSO);
271
272    if (mScriptExec == nullptr) {
273        return false;
274    }
275
276    if (mChecksumNeeded && !mScriptExec->isChecksumValid(mBuildChecksum)) {
277        ALOGE("Found invalid checksum.  Expected %s, got %s\n",
278                  mBuildChecksum, mScriptExec->getBuildChecksum());
279        delete mScriptExec;
280        mScriptExec = nullptr;
281        return false;
282    }
283
284    mRoot = (RootFunc_t) dlsym(mScriptSO, "root");
285    if (mRoot) {
286        //ALOGE("Found root(): %p", mRoot);
287    }
288    mRootExpand = (RootFunc_t) dlsym(mScriptSO, "root.expand");
289    if (mRootExpand) {
290        //ALOGE("Found root.expand(): %p", mRootExpand);
291    }
292    mInit = (InvokeFunc_t) dlsym(mScriptSO, "init");
293    if (mInit) {
294        //ALOGE("Found init(): %p", mInit);
295    }
296    mFreeChildren = (InvokeFunc_t) dlsym(mScriptSO, ".rs.dtor");
297    if (mFreeChildren) {
298        //ALOGE("Found .rs.dtor(): %p", mFreeChildren);
299    }
300
301    size_t varCount = mScriptExec->getExportedVariableCount();
302    if (varCount > 0) {
303        mBoundAllocs = new Allocation *[varCount];
304        memset(mBoundAllocs, 0, varCount * sizeof(*mBoundAllocs));
305    }
306
307    mIsThreadable = mScriptExec->getThreadable();
308    //ALOGE("Script isThreadable? %d", mIsThreadable);
309
310    return true;
311}
312
313bool RsdCpuScriptImpl::init(char const *resName, char const *cacheDir,
314                            uint8_t const *bitcode, size_t bitcodeSize,
315                            uint32_t flags, char const *bccPluginName) {
316    //ALOGE("rsdScriptCreate %p %p %p %p %i %i %p", rsc, resName, cacheDir,
317    // bitcode, bitcodeSize, flags, lookupFunc);
318    //ALOGE("rsdScriptInit %p %p", rsc, script);
319
320    mCtx->lockMutex();
321#ifndef RS_COMPATIBILITY_LIB
322    bool useRSDebugContext = false;
323
324    mCompilerDriver = nullptr;
325
326    mCompilerDriver = new bcc::RSCompilerDriver();
327    if (mCompilerDriver == nullptr) {
328        ALOGE("bcc: FAILS to create compiler driver (out of memory)");
329        mCtx->unlockMutex();
330        return false;
331    }
332
333    // Run any compiler setup functions we have been provided with.
334    RSSetupCompilerCallback setupCompilerCallback =
335            mCtx->getSetupCompilerCallback();
336    if (setupCompilerCallback != nullptr) {
337        setupCompilerCallback(mCompilerDriver);
338    }
339
340    bcinfo::MetadataExtractor bitcodeMetadata((const char *) bitcode, bitcodeSize);
341    if (!bitcodeMetadata.extract()) {
342        ALOGE("Could not extract metadata from bitcode");
343        mCtx->unlockMutex();
344        return false;
345    }
346
347    const char* core_lib = findCoreLib(bitcodeMetadata, (const char*)bitcode, bitcodeSize);
348
349    if (mCtx->getContext()->getContextType() == RS_CONTEXT_TYPE_DEBUG) {
350        mCompilerDriver->setDebugContext(true);
351        useRSDebugContext = true;
352    }
353
354    std::string bcFileName(cacheDir);
355    bcFileName.append("/");
356    bcFileName.append(resName);
357    bcFileName.append(".bc");
358
359    std::vector<const char*> compileArguments;
360    setCompileArguments(&compileArguments, bcFileName, cacheDir, resName, core_lib,
361                        useRSDebugContext, bccPluginName);
362
363    // The last argument of compileArguments is a nullptr, so remove 1 from the
364    // size.
365    std::unique_ptr<const char> compileCommandLine(
366        rsuJoinStrings(compileArguments.size() - 1, compileArguments.data()));
367
368    mChecksumNeeded = isChecksumNeeded();
369    if (mChecksumNeeded) {
370        std::vector<const char *> bccFiles = { BCC_EXE_PATH,
371                                               core_lib,
372                                             };
373        mBuildChecksum = constructBuildChecksum(bitcode, bitcodeSize,
374                                                compileCommandLine.get(),
375                                                bccFiles);
376
377        if (mBuildChecksum == nullptr) {
378            // cannot compute checksum but verification is enabled
379            mCtx->unlockMutex();
380            return false;
381        }
382    }
383    else {
384        // add a dummy/constant as a checksum if verification is disabled
385        mBuildChecksum = new char[9]();
386        strcpy(const_cast<char *>(mBuildChecksum), "abadcafe");
387    }
388
389    // Append build checksum to commandline
390    // Handle the terminal nullptr in compileArguments
391    compileArguments.pop_back();
392    compileArguments.push_back("-build-checksum");
393    compileArguments.push_back(mBuildChecksum);
394    compileArguments.push_back(nullptr);
395
396    // recompute compileCommandLine with the extra arguments
397    compileCommandLine.reset(
398        rsuJoinStrings(compileArguments.size() - 1, compileArguments.data()));
399
400    if (!is_force_recompile() && !useRSDebugContext) {
401        mScriptSO = SharedLibraryUtils::loadSharedLibrary(cacheDir, resName);
402
403        // Read RS info from the shared object to detect checksum mismatch
404        if (mScriptSO != nullptr && !storeRSInfoFromSO()) {
405            dlclose(mScriptSO);
406            mScriptSO = nullptr;
407        }
408    }
409
410    // If we can't, it's either not there or out of date.  We compile the bit code and try loading
411    // again.
412    if (mScriptSO == nullptr) {
413        if (!compileBitcode(bcFileName, (const char*)bitcode, bitcodeSize,
414                            compileArguments.data(), compileCommandLine.get()))
415        {
416            ALOGE("bcc: FAILS to compile '%s'", resName);
417            mCtx->unlockMutex();
418            return false;
419        }
420
421        if (!SharedLibraryUtils::createSharedLibrary(cacheDir, resName)) {
422            ALOGE("Linker: Failed to link object file '%s'", resName);
423            mCtx->unlockMutex();
424            return false;
425        }
426
427        mScriptSO = SharedLibraryUtils::loadSharedLibrary(cacheDir, resName);
428        if (mScriptSO == nullptr) {
429            ALOGE("Unable to load '%s'", resName);
430            mCtx->unlockMutex();
431            return false;
432        }
433
434        // Read RS symbol information from the .so.
435        if (!storeRSInfoFromSO()) {
436            goto error;
437        }
438    }
439
440    mBitcodeFilePath.setTo(bcFileName.c_str());
441
442#else  // RS_COMPATIBILITY_LIB is defined
443    const char *nativeLibDir = mCtx->getContext()->getNativeLibDir();
444    mScriptSO = SharedLibraryUtils::loadSharedLibrary(cacheDir, resName, nativeLibDir);
445
446    if (!mScriptSO) {
447        goto error;
448    }
449
450    if (!storeRSInfoFromSO()) {
451        goto error;
452    }
453#endif
454    mCtx->unlockMutex();
455    return true;
456
457error:
458
459    mCtx->unlockMutex();
460    if (mScriptSO) {
461        dlclose(mScriptSO);
462        mScriptSO = nullptr;
463    }
464    return false;
465}
466
467#ifndef RS_COMPATIBILITY_LIB
468
469const char* RsdCpuScriptImpl::findCoreLib(const bcinfo::MetadataExtractor& ME, const char* bitcode,
470                                          size_t bitcodeSize) {
471    const char* defaultLib = SYSLIBPATH"/libclcore.bc";
472
473    // If we're debugging, use the debug library.
474    if (mCtx->getContext()->getContextType() == RS_CONTEXT_TYPE_DEBUG) {
475        return SYSLIBPATH"/libclcore_debug.bc";
476    }
477
478    // If a callback has been registered to specify a library, use that.
479    RSSelectRTCallback selectRTCallback = mCtx->getSelectRTCallback();
480    if (selectRTCallback != nullptr) {
481        return selectRTCallback((const char*)bitcode, bitcodeSize);
482    }
483
484    // Check for a platform specific library
485#if defined(ARCH_ARM_HAVE_NEON) && !defined(DISABLE_CLCORE_NEON)
486    enum bcinfo::RSFloatPrecision prec = ME.getRSFloatPrecision();
487    if (prec == bcinfo::RS_FP_Relaxed) {
488        // NEON-capable ARMv7a devices can use an accelerated math library
489        // for all reduced precision scripts.
490        // ARMv8 does not use NEON, as ASIMD can be used with all precision
491        // levels.
492        return SYSLIBPATH"/libclcore_neon.bc";
493    } else {
494        return defaultLib;
495    }
496#elif defined(__i386__) || defined(__x86_64__)
497    // x86 devices will use an optimized library.
498    return SYSLIBPATH"/libclcore_x86.bc";
499#else
500    return defaultLib;
501#endif
502}
503
504#endif
505
506void RsdCpuScriptImpl::populateScript(Script *script) {
507    // Copy info over to runtime
508    script->mHal.info.exportedFunctionCount = mScriptExec->getExportedFunctionCount();
509    script->mHal.info.exportedVariableCount = mScriptExec->getExportedVariableCount();
510    script->mHal.info.exportedPragmaCount = mScriptExec->getPragmaCount();;
511    script->mHal.info.exportedPragmaKeyList = mScriptExec->getPragmaKeys();
512    script->mHal.info.exportedPragmaValueList = mScriptExec->getPragmaValues();
513
514    // Bug, need to stash in metadata
515    if (mRootExpand) {
516        script->mHal.info.root = mRootExpand;
517    } else {
518        script->mHal.info.root = mRoot;
519    }
520}
521
522
523typedef void (*rs_t)(const void *, void *, const void *, uint32_t, uint32_t, uint32_t, uint32_t);
524
525bool RsdCpuScriptImpl::forEachMtlsSetup(const Allocation ** ains,
526                                        uint32_t inLen,
527                                        Allocation * aout,
528                                        const void * usr, uint32_t usrLen,
529                                        const RsScriptCall *sc,
530                                        MTLaunchStruct *mtls) {
531
532    memset(mtls, 0, sizeof(MTLaunchStruct));
533
534    for (int index = inLen; --index >= 0;) {
535        const Allocation* ain = ains[index];
536
537        // possible for this to occur if IO_OUTPUT/IO_INPUT with no bound surface
538        if (ain != nullptr &&
539            (const uint8_t *)ain->mHal.drvState.lod[0].mallocPtr == nullptr) {
540
541            mCtx->getContext()->setError(RS_ERROR_BAD_SCRIPT,
542                                         "rsForEach called with null in allocations");
543            return false;
544        }
545    }
546
547    if (aout &&
548        (const uint8_t *)aout->mHal.drvState.lod[0].mallocPtr == nullptr) {
549
550        mCtx->getContext()->setError(RS_ERROR_BAD_SCRIPT,
551                                     "rsForEach called with null out allocations");
552        return false;
553    }
554
555    if (inLen > 0) {
556        const Allocation *ain0   = ains[0];
557        const Type       *inType = ain0->getType();
558
559        mtls->fep.dim.x = inType->getDimX();
560        mtls->fep.dim.y = inType->getDimY();
561        mtls->fep.dim.z = inType->getDimZ();
562
563        for (int Index = inLen; --Index >= 1;) {
564            if (!ain0->hasSameDims(ains[Index])) {
565                mCtx->getContext()->setError(RS_ERROR_BAD_SCRIPT,
566                  "Failed to launch kernel; dimensions of input and output"
567                  "allocations do not match.");
568
569                return false;
570            }
571        }
572
573    } else if (aout != nullptr) {
574        const Type *outType = aout->getType();
575
576        mtls->fep.dim.x = outType->getDimX();
577        mtls->fep.dim.y = outType->getDimY();
578        mtls->fep.dim.z = outType->getDimZ();
579
580    } else {
581        mCtx->getContext()->setError(RS_ERROR_BAD_SCRIPT,
582                                     "rsForEach called with null allocations");
583        return false;
584    }
585
586    if (inLen > 0 && aout != nullptr) {
587        if (!ains[0]->hasSameDims(aout)) {
588            mCtx->getContext()->setError(RS_ERROR_BAD_SCRIPT,
589              "Failed to launch kernel; dimensions of input and output allocations do not match.");
590
591            return false;
592        }
593    }
594
595    if (!sc || (sc->xEnd == 0)) {
596        mtls->end.x = mtls->fep.dim.x;
597    } else {
598        mtls->start.x = rsMin(mtls->fep.dim.x, sc->xStart);
599        mtls->end.x = rsMin(mtls->fep.dim.x, sc->xEnd);
600        if (mtls->start.x >= mtls->end.x) return false;
601    }
602
603    if (!sc || (sc->yEnd == 0)) {
604        mtls->end.y = mtls->fep.dim.y;
605    } else {
606        mtls->start.y = rsMin(mtls->fep.dim.y, sc->yStart);
607        mtls->end.y = rsMin(mtls->fep.dim.y, sc->yEnd);
608        if (mtls->start.y >= mtls->end.y) return false;
609    }
610
611    if (!sc || (sc->zEnd == 0)) {
612        mtls->end.z = mtls->fep.dim.z;
613    } else {
614        mtls->start.z = rsMin(mtls->fep.dim.z, sc->zStart);
615        mtls->end.z = rsMin(mtls->fep.dim.z, sc->zEnd);
616        if (mtls->start.z >= mtls->end.z) return false;
617    }
618
619    if (!sc || (sc->arrayEnd == 0)) {
620        mtls->end.array[0] = mtls->fep.dim.array[0];
621    } else {
622        mtls->start.array[0] = rsMin(mtls->fep.dim.array[0], sc->arrayStart);
623        mtls->end.array[0] = rsMin(mtls->fep.dim.array[0], sc->arrayEnd);
624        if (mtls->start.array[0] >= mtls->end.array[0]) return false;
625    }
626
627    if (!sc || (sc->array2End == 0)) {
628        mtls->end.array[1] = mtls->fep.dim.array[1];
629    } else {
630        mtls->start.array[1] = rsMin(mtls->fep.dim.array[1], sc->array2Start);
631        mtls->end.array[1] = rsMin(mtls->fep.dim.array[1], sc->array2End);
632        if (mtls->start.array[1] >= mtls->end.array[1]) return false;
633    }
634
635    if (!sc || (sc->array3End == 0)) {
636        mtls->end.array[2] = mtls->fep.dim.array[2];
637    } else {
638        mtls->start.array[2] = rsMin(mtls->fep.dim.array[2], sc->array3Start);
639        mtls->end.array[2] = rsMin(mtls->fep.dim.array[2], sc->array3End);
640        if (mtls->start.array[2] >= mtls->end.array[2]) return false;
641    }
642
643    if (!sc || (sc->array4End == 0)) {
644        mtls->end.array[3] = mtls->fep.dim.array[3];
645    } else {
646        mtls->start.array[3] = rsMin(mtls->fep.dim.array[3], sc->array4Start);
647        mtls->end.array[3] = rsMin(mtls->fep.dim.array[3], sc->array4End);
648        if (mtls->start.array[3] >= mtls->end.array[3]) return false;
649    }
650
651
652    // The X & Y walkers always want 0-1 min even if dim is not present
653    mtls->end.x    = rsMax((uint32_t)1, mtls->end.x);
654    mtls->end.y    = rsMax((uint32_t)1, mtls->end.y);
655
656    mtls->rsc        = mCtx;
657    if (ains) {
658        memcpy(mtls->ains, ains, inLen * sizeof(ains[0]));
659    }
660    mtls->aout[0]    = aout;
661    mtls->fep.usr    = usr;
662    mtls->fep.usrLen = usrLen;
663    mtls->mSliceSize = 1;
664    mtls->mSliceNum  = 0;
665
666    mtls->isThreadable  = mIsThreadable;
667
668    if (inLen > 0) {
669        mtls->fep.inLen = inLen;
670        for (int index = inLen; --index >= 0;) {
671            mtls->fep.inPtr[index] = (const uint8_t*)ains[index]->mHal.drvState.lod[0].mallocPtr;
672            mtls->fep.inStride[index] = ains[index]->getType()->getElementSizeBytes();
673        }
674    }
675
676    if (aout != nullptr) {
677        mtls->fep.outPtr[0] = (uint8_t *)aout->mHal.drvState.lod[0].mallocPtr;
678        mtls->fep.outStride[0] = aout->getType()->getElementSizeBytes();
679    }
680
681    // All validation passed, ok to launch threads
682    return true;
683}
684
685
686void RsdCpuScriptImpl::invokeForEach(uint32_t slot,
687                                     const Allocation ** ains,
688                                     uint32_t inLen,
689                                     Allocation * aout,
690                                     const void * usr,
691                                     uint32_t usrLen,
692                                     const RsScriptCall *sc) {
693
694    MTLaunchStruct mtls;
695
696    if (forEachMtlsSetup(ains, inLen, aout, usr, usrLen, sc, &mtls)) {
697        forEachKernelSetup(slot, &mtls);
698
699        RsdCpuScriptImpl * oldTLS = mCtx->setTLS(this);
700        mCtx->launchThreads(ains, inLen, aout, sc, &mtls);
701        mCtx->setTLS(oldTLS);
702    }
703}
704
705void RsdCpuScriptImpl::forEachKernelSetup(uint32_t slot, MTLaunchStruct *mtls) {
706    mtls->script = this;
707    mtls->fep.slot = slot;
708    mtls->kernel = mScriptExec->getForEachFunction(slot);
709    rsAssert(mtls->kernel != nullptr);
710    mtls->sig = mScriptExec->getForEachSignature(slot);
711}
712
713int RsdCpuScriptImpl::invokeRoot() {
714    RsdCpuScriptImpl * oldTLS = mCtx->setTLS(this);
715    int ret = mRoot();
716    mCtx->setTLS(oldTLS);
717    return ret;
718}
719
720void RsdCpuScriptImpl::invokeInit() {
721    if (mInit) {
722        mInit();
723    }
724}
725
726void RsdCpuScriptImpl::invokeFreeChildren() {
727    if (mFreeChildren) {
728        mFreeChildren();
729    }
730}
731
732void RsdCpuScriptImpl::invokeFunction(uint32_t slot, const void *params,
733                                      size_t paramLength) {
734    //ALOGE("invoke %i %p %zu", slot, params, paramLength);
735    void * ap = nullptr;
736
737#if defined(__x86_64__)
738    // The invoked function could have input parameter of vector type for example float4 which
739    // requires void* params to be 16 bytes aligned when using SSE instructions for x86_64 platform.
740    // So try to align void* params before passing them into RS exported function.
741
742    if ((uint8_t)(uint64_t)params & 0x0F) {
743        if ((ap = (void*)memalign(16, paramLength)) != nullptr) {
744            memcpy(ap, params, paramLength);
745        } else {
746            ALOGE("x86_64: invokeFunction memalign error, still use params which"
747                  " is not 16 bytes aligned.");
748        }
749    }
750#endif
751
752    RsdCpuScriptImpl * oldTLS = mCtx->setTLS(this);
753    reinterpret_cast<void (*)(const void *, uint32_t)>(
754        mScriptExec->getInvokeFunction(slot))(ap? (const void *) ap: params, paramLength);
755
756    mCtx->setTLS(oldTLS);
757}
758
759void RsdCpuScriptImpl::setGlobalVar(uint32_t slot, const void *data, size_t dataLength) {
760    //rsAssert(!script->mFieldIsObject[slot]);
761    //ALOGE("setGlobalVar %i %p %zu", slot, data, dataLength);
762
763    //if (mIntrinsicID) {
764        //mIntrinsicFuncs.setVar(dc, script, drv->mIntrinsicData, slot, data, dataLength);
765        //return;
766    //}
767
768    int32_t *destPtr = reinterpret_cast<int32_t *>(mScriptExec->getFieldAddress(slot));
769    if (!destPtr) {
770        //ALOGV("Calling setVar on slot = %i which is null", slot);
771        return;
772    }
773
774    memcpy(destPtr, data, dataLength);
775}
776
777void RsdCpuScriptImpl::getGlobalVar(uint32_t slot, void *data, size_t dataLength) {
778    //rsAssert(!script->mFieldIsObject[slot]);
779    //ALOGE("getGlobalVar %i %p %zu", slot, data, dataLength);
780
781    int32_t *srcPtr = reinterpret_cast<int32_t *>(mScriptExec->getFieldAddress(slot));
782    if (!srcPtr) {
783        //ALOGV("Calling setVar on slot = %i which is null", slot);
784        return;
785    }
786    memcpy(data, srcPtr, dataLength);
787}
788
789
790void RsdCpuScriptImpl::setGlobalVarWithElemDims(uint32_t slot, const void *data, size_t dataLength,
791                                                const Element *elem,
792                                                const uint32_t *dims, size_t dimLength) {
793    int32_t *destPtr = reinterpret_cast<int32_t *>(mScriptExec->getFieldAddress(slot));
794    if (!destPtr) {
795        //ALOGV("Calling setVar on slot = %i which is null", slot);
796        return;
797    }
798
799    // We want to look at dimension in terms of integer components,
800    // but dimLength is given in terms of bytes.
801    dimLength /= sizeof(int);
802
803    // Only a single dimension is currently supported.
804    rsAssert(dimLength == 1);
805    if (dimLength == 1) {
806        // First do the increment loop.
807        size_t stride = elem->getSizeBytes();
808        const char *cVal = reinterpret_cast<const char *>(data);
809        for (uint32_t i = 0; i < dims[0]; i++) {
810            elem->incRefs(cVal);
811            cVal += stride;
812        }
813
814        // Decrement loop comes after (to prevent race conditions).
815        char *oldVal = reinterpret_cast<char *>(destPtr);
816        for (uint32_t i = 0; i < dims[0]; i++) {
817            elem->decRefs(oldVal);
818            oldVal += stride;
819        }
820    }
821
822    memcpy(destPtr, data, dataLength);
823}
824
825void RsdCpuScriptImpl::setGlobalBind(uint32_t slot, Allocation *data) {
826
827    //rsAssert(!script->mFieldIsObject[slot]);
828    //ALOGE("setGlobalBind %i %p", slot, data);
829
830    int32_t *destPtr = reinterpret_cast<int32_t *>(mScriptExec->getFieldAddress(slot));
831    if (!destPtr) {
832        //ALOGV("Calling setVar on slot = %i which is null", slot);
833        return;
834    }
835
836    void *ptr = nullptr;
837    mBoundAllocs[slot] = data;
838    if (data) {
839        ptr = data->mHal.drvState.lod[0].mallocPtr;
840    }
841    memcpy(destPtr, &ptr, sizeof(void *));
842}
843
844void RsdCpuScriptImpl::setGlobalObj(uint32_t slot, ObjectBase *data) {
845
846    //rsAssert(script->mFieldIsObject[slot]);
847    //ALOGE("setGlobalObj %i %p", slot, data);
848
849    int32_t *destPtr = reinterpret_cast<int32_t *>(mScriptExec->getFieldAddress(slot));
850    if (!destPtr) {
851        //ALOGV("Calling setVar on slot = %i which is null", slot);
852        return;
853    }
854
855    rsrSetObject(mCtx->getContext(), (rs_object_base *)destPtr, data);
856}
857
858RsdCpuScriptImpl::~RsdCpuScriptImpl() {
859#ifndef RS_COMPATIBILITY_LIB
860    if (mCompilerDriver) {
861        delete mCompilerDriver;
862    }
863#endif
864
865    if (mScriptExec != nullptr) {
866        delete mScriptExec;
867    }
868    if (mBoundAllocs) delete[] mBoundAllocs;
869    if (mScriptSO) {
870        dlclose(mScriptSO);
871    }
872
873    delete[] mBuildChecksum;
874}
875
876Allocation * RsdCpuScriptImpl::getAllocationForPointer(const void *ptr) const {
877    if (!ptr) {
878        return nullptr;
879    }
880
881    for (uint32_t ct=0; ct < mScript->mHal.info.exportedVariableCount; ct++) {
882        Allocation *a = mBoundAllocs[ct];
883        if (!a) continue;
884        if (a->mHal.drvState.lod[0].mallocPtr == ptr) {
885            return a;
886        }
887    }
888    ALOGE("rsGetAllocation, failed to find %p", ptr);
889    return nullptr;
890}
891
892void RsdCpuScriptImpl::preLaunch(uint32_t slot, const Allocation ** ains,
893                                 uint32_t inLen, Allocation * aout,
894                                 const void * usr, uint32_t usrLen,
895                                 const RsScriptCall *sc) {}
896
897void RsdCpuScriptImpl::postLaunch(uint32_t slot, const Allocation ** ains,
898                                  uint32_t inLen, Allocation * aout,
899                                  const void * usr, uint32_t usrLen,
900                                  const RsScriptCall *sc) {}
901
902
903}
904}
905