1/*
2 * Copyright (C) 2011-2012 The Android Open Source Project
3 *
4 * Licensed under the Apache License, Version 2.0 (the "License");
5 * you may not use this file except in compliance with the License.
6 * You may obtain a copy of the License at
7 *
8 *      http://www.apache.org/licenses/LICENSE-2.0
9 *
10 * Unless required by applicable law or agreed to in writing, software
11 * distributed under the License is distributed on an "AS IS" BASIS,
12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 * See the License for the specific language governing permissions and
14 * limitations under the License.
15 */
16
17#include "rsCpuCore.h"
18#include "rsCpuScript.h"
19#include "rsCpuExecutable.h"
20
21#ifdef RS_COMPATIBILITY_LIB
22    #include <stdio.h>
23    #include <sys/stat.h>
24    #include <unistd.h>
25#else
26    #include "rsCppUtils.h"
27
28    #include <bcc/Config.h>
29    #include <bcinfo/MetadataExtractor.h>
30
31    #include <zlib.h>
32    #include <sys/file.h>
33    #include <sys/types.h>
34    #include <unistd.h>
35
36    #include <string>
37    #include <vector>
38#endif
39
40#include <set>
41#include <string>
42#include <dlfcn.h>
43#include <stdlib.h>
44#include <string.h>
45#include <iostream>
46#include <sstream>
47
48namespace {
49
50static const bool kDebugGlobalVariables = false;
51
52static bool allocationLODIsNull(const android::renderscript::Allocation *alloc) {
53  // Even if alloc != nullptr, mallocPtr could be null if
54  // IO_OUTPUT/IO_INPUT with no bound surface.
55  return alloc && alloc->mHal.drvState.lod[0].mallocPtr == nullptr;
56}
57
58#ifndef RS_COMPATIBILITY_LIB
59
60static void setCompileArguments(std::vector<const char*>* args,
61                                const std::string& bcFileName,
62                                const char* cacheDir, const char* resName,
63                                const char* core_lib, bool useRSDebugContext,
64                                const char* bccPluginName, bool emitGlobalInfo,
65                                int optLevel, bool emitGlobalInfoSkipConstant) {
66    rsAssert(cacheDir && resName && core_lib);
67    args->push_back(android::renderscript::RsdCpuScriptImpl::BCC_EXE_PATH);
68    args->push_back("-unroll-runtime");
69    args->push_back("-scalarize-load-store");
70    if (emitGlobalInfo) {
71        args->push_back("-rs-global-info");
72        if (emitGlobalInfoSkipConstant) {
73            args->push_back("-rs-global-info-skip-constant");
74        }
75    }
76    args->push_back("-o");
77    args->push_back(resName);
78    args->push_back("-output_path");
79    args->push_back(cacheDir);
80    args->push_back("-bclib");
81    args->push_back(core_lib);
82    args->push_back("-mtriple");
83    args->push_back(DEFAULT_TARGET_TRIPLE_STRING);
84    args->push_back("-O");
85
86    switch (optLevel) {
87    case 0:
88        args->push_back("0");
89        break;
90    case 3:
91        args->push_back("3");
92        break;
93    default:
94        ALOGW("Expected optimization level of 0 or 3. Received %d", optLevel);
95        args->push_back("3");
96        break;
97    }
98
99    // Enable workaround for A53 codegen by default.
100#if defined(__aarch64__) && !defined(DISABLE_A53_WORKAROUND)
101    args->push_back("-aarch64-fix-cortex-a53-835769");
102#endif
103
104    // Execute the bcc compiler.
105    if (useRSDebugContext) {
106        args->push_back("-rs-debug-ctx");
107    } else {
108        // Only load additional libraries for compiles that don't use
109        // the debug context.
110        if (bccPluginName && strlen(bccPluginName) > 0) {
111#ifdef __ANDROID__
112            // For Android, -plugin option must be used in order to load the
113            // vendor plugin from the sphal namespace.
114            args->push_back("-plugin");
115#else
116            args->push_back("-load");
117#endif
118            args->push_back(bccPluginName);
119        }
120    }
121
122    args->push_back("-fPIC");
123    args->push_back("-embedRSInfo");
124
125    args->push_back(bcFileName.c_str());
126    args->push_back(nullptr);
127}
128
129static bool compileBitcode(const std::string &bcFileName,
130                           const char *bitcode,
131                           size_t bitcodeSize,
132                           std::vector<const char *> &compileArguments) {
133    rsAssert(bitcode && bitcodeSize);
134
135    FILE *bcfile = fopen(bcFileName.c_str(), "w");
136    if (!bcfile) {
137        ALOGE("Could not write to %s", bcFileName.c_str());
138        return false;
139    }
140    size_t nwritten = fwrite(bitcode, 1, bitcodeSize, bcfile);
141    fclose(bcfile);
142    if (nwritten != bitcodeSize) {
143        ALOGE("Could not write %zu bytes to %s", bitcodeSize,
144              bcFileName.c_str());
145        return false;
146    }
147
148    return android::renderscript::rsuExecuteCommand(
149                   android::renderscript::RsdCpuScriptImpl::BCC_EXE_PATH,
150                   compileArguments.size()-1, compileArguments.data());
151}
152
153// The checksum is unnecessary under a few conditions, since the primary
154// use-case for it is debugging. If we are loading something from the
155// system partition (read-only), we know that it was precompiled as part of
156// application ahead of time (and thus the checksum is completely
157// unnecessary). The checksum is also unnecessary on release (non-debug)
158// builds, as the only way to get a shared object is to have compiled the
159// script once already. On a release build, there is no way to adjust the
160// other libraries/dependencies, and so the only reason to recompile would
161// be for a source APK change or an OTA. In either case, the APK would be
162// reinstalled, which would already clear the code_cache/ directory.
163bool isChecksumNeeded(const char *cacheDir) {
164    if ((::strcmp(SYSLIBPATH, cacheDir) == 0) ||
165        (::strcmp(SYSLIBPATH_VNDK, cacheDir) == 0) ||
166        (::strcmp(SYSLIBPATH_VENDOR, cacheDir) == 0))
167        return false;
168    char buf[PROP_VALUE_MAX];
169    android::renderscript::property_get("ro.debuggable", buf, "");
170    return (buf[0] == '1');
171}
172
173bool addFileToChecksum(const char *fileName, uint32_t &checksum) {
174    int FD = open(fileName, O_RDONLY);
175    if (FD == -1) {
176        ALOGE("Cannot open file \'%s\' to compute checksum", fileName);
177        return false;
178    }
179
180    char buf[256];
181    while (true) {
182        ssize_t nread = read(FD, buf, sizeof(buf));
183        if (nread < 0) { // bail out on failed read
184            ALOGE("Error while computing checksum for file \'%s\'", fileName);
185            return false;
186        }
187
188        checksum = adler32(checksum, (const unsigned char *) buf, nread);
189        if (static_cast<size_t>(nread) < sizeof(buf)) // EOF
190            break;
191    }
192
193    if (close(FD) != 0) {
194        ALOGE("Cannot close file \'%s\' after computing checksum", fileName);
195        return false;
196    }
197    return true;
198}
199
200#endif  // !defined(RS_COMPATIBILITY_LIB)
201}  // namespace
202
203namespace android {
204namespace renderscript {
205
206#ifndef RS_COMPATIBILITY_LIB
207
208uint32_t constructBuildChecksum(uint8_t const *bitcode, size_t bitcodeSize,
209                                const char *commandLine,
210                                const char** bccFiles, size_t numFiles) {
211    uint32_t checksum = adler32(0L, Z_NULL, 0);
212
213    // include checksum of bitcode
214    if (bitcode != nullptr && bitcodeSize > 0) {
215        checksum = adler32(checksum, bitcode, bitcodeSize);
216    }
217
218    // include checksum of command line arguments
219    checksum = adler32(checksum, (const unsigned char *) commandLine,
220                       strlen(commandLine));
221
222    // include checksum of bccFiles
223    for (size_t i = 0; i < numFiles; i++) {
224        const char* bccFile = bccFiles[i];
225        if (bccFile[0] != 0 && !addFileToChecksum(bccFile, checksum)) {
226            // return empty checksum instead of something partial/corrupt
227            return 0;
228        }
229    }
230
231    return checksum;
232}
233
234#endif  // !RS_COMPATIBILITY_LIB
235
236RsdCpuScriptImpl::RsdCpuScriptImpl(RsdCpuReferenceImpl *ctx, const Script *s) {
237    mCtx = ctx;
238    mScript = s;
239
240    mScriptSO = nullptr;
241
242    mRoot = nullptr;
243    mRootExpand = nullptr;
244    mInit = nullptr;
245    mFreeChildren = nullptr;
246    mScriptExec = nullptr;
247
248    mBoundAllocs = nullptr;
249    mIntrinsicData = nullptr;
250    mIsThreadable = true;
251
252    mBuildChecksum = 0;
253    mChecksumNeeded = false;
254}
255
256bool RsdCpuScriptImpl::storeRSInfoFromSO() {
257    // The shared object may have an invalid build checksum.
258    // Validate and fail early.
259    mScriptExec = ScriptExecutable::createFromSharedObject(
260            mScriptSO, mChecksumNeeded ? mBuildChecksum : 0);
261
262    if (mScriptExec == nullptr) {
263        return false;
264    }
265
266    mRoot = (RootFunc_t) dlsym(mScriptSO, "root");
267    if (mRoot) {
268        //ALOGE("Found root(): %p", mRoot);
269    }
270    mRootExpand = (RootFunc_t) dlsym(mScriptSO, "root.expand");
271    if (mRootExpand) {
272        //ALOGE("Found root.expand(): %p", mRootExpand);
273    }
274    mInit = (InitOrDtorFunc_t) dlsym(mScriptSO, "init");
275    if (mInit) {
276        //ALOGE("Found init(): %p", mInit);
277    }
278    mFreeChildren = (InitOrDtorFunc_t) dlsym(mScriptSO, ".rs.dtor");
279    if (mFreeChildren) {
280        //ALOGE("Found .rs.dtor(): %p", mFreeChildren);
281    }
282
283    size_t varCount = mScriptExec->getExportedVariableCount();
284    if (varCount > 0) {
285        mBoundAllocs = new Allocation *[varCount];
286        memset(mBoundAllocs, 0, varCount * sizeof(*mBoundAllocs));
287    }
288
289    mIsThreadable = mScriptExec->getThreadable();
290    //ALOGE("Script isThreadable? %d", mIsThreadable);
291
292    if (kDebugGlobalVariables) {
293        mScriptExec->dumpGlobalInfo();
294    }
295
296    return true;
297}
298
299bool RsdCpuScriptImpl::init(char const *resName, char const *cacheDir,
300                            uint8_t const *bitcode, size_t bitcodeSize,
301                            uint32_t flags, char const *bccPluginName) {
302    //ALOGE("rsdScriptCreate %p %p %p %p %i %i %p", rsc, resName, cacheDir,
303    // bitcode, bitcodeSize, flags, lookupFunc);
304    //ALOGE("rsdScriptInit %p %p", rsc, script);
305
306    mCtx->lockMutex();
307#ifndef RS_COMPATIBILITY_LIB
308    bool useRSDebugContext = false;
309
310    bcinfo::MetadataExtractor bitcodeMetadata((const char *) bitcode, bitcodeSize);
311    if (!bitcodeMetadata.extract()) {
312        ALOGE("Could not extract metadata from bitcode");
313        mCtx->unlockMutex();
314        return false;
315    }
316
317    const char* core_lib = findCoreLib(bitcodeMetadata, (const char*)bitcode, bitcodeSize);
318
319    if (mCtx->getContext()->getContextType() == RS_CONTEXT_TYPE_DEBUG) {
320        useRSDebugContext = true;
321    }
322
323    int optLevel = mCtx->getContext()->getOptLevel();
324
325    std::string bcFileName(cacheDir);
326    bcFileName.append("/");
327    bcFileName.append(resName);
328    bcFileName.append(".bc");
329
330    std::vector<const char*> compileArguments;
331    bool emitGlobalInfo = mCtx->getEmbedGlobalInfo();
332    bool emitGlobalInfoSkipConstant = mCtx->getEmbedGlobalInfoSkipConstant();
333    setCompileArguments(&compileArguments, bcFileName, cacheDir, resName, core_lib,
334                        useRSDebugContext, bccPluginName, emitGlobalInfo,
335                        optLevel, emitGlobalInfoSkipConstant);
336
337    mChecksumNeeded = isChecksumNeeded(cacheDir);
338    if (mChecksumNeeded) {
339        std::vector<const char *> bccFiles = { BCC_EXE_PATH,
340                                               core_lib,
341                                             };
342
343        // The last argument of compileArguments is a nullptr, so remove 1 from
344        // the size.
345        std::unique_ptr<const char> compileCommandLine(
346            rsuJoinStrings(compileArguments.size()-1, compileArguments.data()));
347
348        mBuildChecksum = constructBuildChecksum(bitcode, bitcodeSize,
349                                                compileCommandLine.get(),
350                                                bccFiles.data(), bccFiles.size());
351
352        if (mBuildChecksum == 0) {
353            // cannot compute checksum but verification is enabled
354            mCtx->unlockMutex();
355            return false;
356        }
357    }
358    else {
359        // add a dummy/constant as a checksum if verification is disabled
360        mBuildChecksum = 0xabadcafe;
361    }
362
363    // Append build checksum to commandline
364    // Handle the terminal nullptr in compileArguments
365    compileArguments.pop_back();
366    compileArguments.push_back("-build-checksum");
367    std::stringstream ss;
368    ss << std::hex << mBuildChecksum;
369    std::string checksumStr(ss.str());
370    compileArguments.push_back(checksumStr.c_str());
371    compileArguments.push_back(nullptr);
372
373    const bool reuse = !is_force_recompile() && !useRSDebugContext;
374    if (reuse) {
375        mScriptSO = SharedLibraryUtils::loadSharedLibrary(cacheDir, resName);
376
377        // Read RS info from the shared object to detect checksum mismatch
378        if (mScriptSO != nullptr && !storeRSInfoFromSO()) {
379            dlclose(mScriptSO);
380            mScriptSO = nullptr;
381        }
382    }
383
384    // If reuse is desired and we can't, it's either not there or out of date.
385    // We compile the bit code and try loading again.
386    if (mScriptSO == nullptr) {
387        if (!compileBitcode(bcFileName, (const char*)bitcode, bitcodeSize,
388                            compileArguments))
389        {
390            ALOGE("bcc: FAILS to compile '%s'", resName);
391            mCtx->unlockMutex();
392            return false;
393        }
394
395        std::string SOPath;
396
397        if (!SharedLibraryUtils::createSharedLibrary(
398                mCtx->getContext()->getDriverName(), cacheDir, resName, reuse,
399                &SOPath)) {
400            ALOGE("Linker: Failed to link object file '%s'", resName);
401            mCtx->unlockMutex();
402            return false;
403        }
404
405        if (reuse) {
406            mScriptSO = SharedLibraryUtils::loadSharedLibrary(cacheDir, resName);
407        } else {
408            mScriptSO = SharedLibraryUtils::loadAndDeleteSharedLibrary(SOPath.c_str());
409        }
410        if (mScriptSO == nullptr) {
411            ALOGE("Unable to load '%s'", resName);
412            mCtx->unlockMutex();
413            return false;
414        }
415
416        // Read RS symbol information from the .so.
417        if (!storeRSInfoFromSO()) {
418            goto error;
419        }
420    }
421
422    mBitcodeFilePath.assign(bcFileName.c_str());
423
424#else  // RS_COMPATIBILITY_LIB is defined
425    const char *nativeLibDir = mCtx->getContext()->getNativeLibDir();
426    mScriptSO = SharedLibraryUtils::loadSharedLibrary(cacheDir, resName, nativeLibDir);
427
428    if (!mScriptSO) {
429        goto error;
430    }
431
432    if (!storeRSInfoFromSO()) {
433        goto error;
434    }
435#endif
436    mCtx->unlockMutex();
437    return true;
438
439error:
440
441    mCtx->unlockMutex();
442    if (mScriptSO) {
443        dlclose(mScriptSO);
444        mScriptSO = nullptr;
445    }
446    return false;
447}
448
449#ifndef RS_COMPATIBILITY_LIB
450
451const char* RsdCpuScriptImpl::findCoreLib(const bcinfo::MetadataExtractor& ME, const char* bitcode,
452                                          size_t bitcodeSize) {
453    const char* defaultLib = SYSLIBPATH_BC"/libclcore.bc";
454
455    // If we're debugging, use the debug library.
456    if (mCtx->getContext()->getContextType() == RS_CONTEXT_TYPE_DEBUG) {
457        if (ME.hasDebugInfo()) {
458            return SYSLIBPATH_BC"/libclcore_debug_g.bc";
459        }
460        return SYSLIBPATH_BC"/libclcore_debug.bc";
461    }
462
463    if (ME.hasDebugInfo()) {
464        return SYSLIBPATH_BC"/libclcore_g.bc";
465    }
466
467    // If a callback has been registered to specify a library, use that.
468    RSSelectRTCallback selectRTCallback = mCtx->getSelectRTCallback();
469    if (selectRTCallback != nullptr) {
470        return selectRTCallback((const char*)bitcode, bitcodeSize);
471    }
472
473    // Check for a platform specific library
474#if defined(ARCH_ARM_HAVE_NEON) && !defined(DISABLE_CLCORE_NEON)
475    enum bcinfo::RSFloatPrecision prec = ME.getRSFloatPrecision();
476    if (prec == bcinfo::RS_FP_Relaxed) {
477        // NEON-capable ARMv7a devices can use an accelerated math library
478        // for all reduced precision scripts.
479        // ARMv8 does not use NEON, as ASIMD can be used with all precision
480        // levels.
481        return SYSLIBPATH_BC"/libclcore_neon.bc";
482    } else {
483        return defaultLib;
484    }
485#elif defined(__i386__) || defined(__x86_64__)
486    // x86 devices will use an optimized library.
487    return SYSLIBPATH_BC"/libclcore_x86.bc";
488#else
489    return defaultLib;
490#endif
491}
492
493#endif
494
495void RsdCpuScriptImpl::populateScript(Script *script) {
496    // Copy info over to runtime
497    script->mHal.info.exportedFunctionCount = mScriptExec->getExportedFunctionCount();
498    script->mHal.info.exportedReduceCount = mScriptExec->getExportedReduceCount();
499    script->mHal.info.exportedForEachCount = mScriptExec->getExportedForEachCount();
500    script->mHal.info.exportedVariableCount = mScriptExec->getExportedVariableCount();
501    script->mHal.info.exportedPragmaCount = mScriptExec->getPragmaCount();;
502    script->mHal.info.exportedPragmaKeyList = mScriptExec->getPragmaKeys();
503    script->mHal.info.exportedPragmaValueList = mScriptExec->getPragmaValues();
504
505    // Bug, need to stash in metadata
506    if (mRootExpand) {
507        script->mHal.info.root = mRootExpand;
508    } else {
509        script->mHal.info.root = mRoot;
510    }
511}
512
513// Set up the launch dimensions, and write the values of the launch
514// dimensions into the mtls start/end fields.
515//
516// Inputs:
517//    baseDim - base shape of the input
518//         sc - used to constrain the launch dimensions
519//
520// Returns:
521//   True on success, false on failure to set up
522bool RsdCpuScriptImpl::setUpMtlsDimensions(MTLaunchStructCommon *mtls,
523                                           const RsLaunchDimensions &baseDim,
524                                           const RsScriptCall *sc) {
525    rsAssert(mtls);
526
527#define SET_UP_DIMENSION(DIM_FIELD, SC_FIELD) do {            \
528    if (!sc || (sc->SC_FIELD##End == 0)) {                    \
529        mtls->end.DIM_FIELD = baseDim.DIM_FIELD;              \
530    } else {                                                  \
531        mtls->start.DIM_FIELD =                               \
532            rsMin(baseDim.DIM_FIELD, sc->SC_FIELD##Start);    \
533        mtls->end.DIM_FIELD =                                 \
534            rsMin(baseDim.DIM_FIELD, sc->SC_FIELD##End);      \
535        if (mtls->start.DIM_FIELD >= mtls->end.DIM_FIELD) {   \
536            mCtx->getContext()->setError(RS_ERROR_BAD_SCRIPT, \
537                "Failed to launch kernel; Invalid "           \
538                #SC_FIELD "Start or " #SC_FIELD "End.");      \
539            return false;                                     \
540        }                                                     \
541    }} while(0)
542
543    SET_UP_DIMENSION(x, x);
544    SET_UP_DIMENSION(y, y);
545    SET_UP_DIMENSION(z, z);
546    // Checks and setup of fields other than x, y, z are ignored, since those
547    // fields are not used in the runtime and are not visible in the Java API.
548#undef SET_UP_DIMENSION
549
550    return true;
551}
552
553// Preliminary work to prepare a general reduce-style kernel for launch.
554bool RsdCpuScriptImpl::reduceMtlsSetup(const Allocation ** ains,
555                                       uint32_t inLen,
556                                       const Allocation * aout,
557                                       const RsScriptCall *sc,
558                                       MTLaunchStructReduce *mtls) {
559    rsAssert(ains && (inLen >= 1) && aout);
560    memset(mtls, 0, sizeof(MTLaunchStructReduce));
561    mtls->dimPtr = &mtls->redp.dim;
562
563    for (int index = inLen; --index >= 0;) {
564        if (allocationLODIsNull(ains[index])) {
565            mCtx->getContext()->setError(RS_ERROR_BAD_SCRIPT,
566                                         "reduce called with null in allocations");
567            return false;
568        }
569    }
570
571    if (allocationLODIsNull(aout)) {
572        mCtx->getContext()->setError(RS_ERROR_BAD_SCRIPT,
573                                     "reduce called with null out allocation");
574        return false;
575    }
576
577    const Allocation *ain0   = ains[0];
578    const Type       *inType = ain0->getType();
579
580    mtls->redp.dim.x = inType->getDimX();
581    mtls->redp.dim.y = inType->getDimY();
582    mtls->redp.dim.z = inType->getDimZ();
583
584    for (int Index = inLen; --Index >= 1;) {
585        if (!ain0->hasSameDims(ains[Index])) {
586            mCtx->getContext()->setError(RS_ERROR_BAD_SCRIPT,
587                                         "Failed to launch reduction kernel;"
588                                         "dimensions of input allocations do not match.");
589            return false;
590        }
591    }
592
593    if (!setUpMtlsDimensions(mtls, mtls->redp.dim, sc)) {
594        return false;
595    }
596
597    // The X & Y walkers always want 0-1 min even if dim is not present
598    mtls->end.x = rsMax((uint32_t)1, mtls->end.x);
599    mtls->end.y = rsMax((uint32_t)1, mtls->end.y);
600
601    mtls->rs = mCtx;
602
603    mtls->mSliceNum    = 0;
604    mtls->mSliceSize   = 1;
605    mtls->isThreadable = mIsThreadable;
606
607    // Set up output,
608    mtls->redp.outLen = 1;
609    mtls->redp.outPtr[0] = (uint8_t *)aout->mHal.drvState.lod[0].mallocPtr;
610    mtls->redp.outStride[0] = aout->getType()->getElementSizeBytes();
611
612    // Set up input.
613    memcpy(mtls->ains, ains, inLen * sizeof(ains[0]));
614    mtls->redp.inLen = inLen;
615    for (int index = inLen; --index >= 0;) {
616        mtls->redp.inPtr[index] = (const uint8_t*)ains[index]->mHal.drvState.lod[0].mallocPtr;
617        mtls->redp.inStride[index] = ains[index]->getType()->getElementSizeBytes();
618    }
619
620    // All validation passed, ok to launch threads
621    return true;
622}
623
624// Preliminary work to prepare a forEach-style kernel for launch.
625bool RsdCpuScriptImpl::forEachMtlsSetup(const Allocation ** ains,
626                                        uint32_t inLen,
627                                        Allocation * aout,
628                                        const void * usr, uint32_t usrLen,
629                                        const RsScriptCall *sc,
630                                        MTLaunchStructForEach *mtls) {
631    if (ains == nullptr && inLen != 0) {
632        mCtx->getContext()->setError(RS_ERROR_BAD_SCRIPT,
633          "rsForEach called with none-zero inLen with null in allocations");
634        return false;
635    }
636
637    memset(mtls, 0, sizeof(MTLaunchStructForEach));
638    mtls->dimPtr = &mtls->fep.dim;
639
640    for (int index = inLen; --index >= 0;) {
641        if (allocationLODIsNull(ains[index])) {
642            mCtx->getContext()->setError(RS_ERROR_BAD_SCRIPT,
643                                         "rsForEach called with null in allocations");
644            return false;
645        }
646    }
647
648    if (allocationLODIsNull(aout)) {
649        mCtx->getContext()->setError(RS_ERROR_BAD_SCRIPT,
650                                     "rsForEach called with null out allocations");
651        return false;
652    }
653
654    // The only situation where ains[j] is null is when inLen==1 and j==0;
655    // and that can only happen for an old-style kernel in API level 11~13,
656    // where the input allocation cannot be skipped if the output allocation is specified.
657    if (inLen != 0)
658        rsAssert((inLen == 1) || (ains[0] != nullptr));
659
660    if (inLen > 0 && ains[0]) {
661        const Allocation *ain0   = ains[0];
662        const Type       *inType = ain0->getType();
663
664        mtls->fep.dim.x = inType->getDimX();
665        mtls->fep.dim.y = inType->getDimY();
666        mtls->fep.dim.z = inType->getDimZ();
667
668        for (int Index = inLen; --Index >= 1;) {
669            if (!ain0->hasSameDims(ains[Index])) {
670                mCtx->getContext()->setError(RS_ERROR_BAD_SCRIPT,
671                  "Failed to launch kernel; dimensions of input "
672                  "allocations do not match.");
673                return false;
674            }
675        }
676    } else if (aout != nullptr) {
677        const Type *outType = aout->getType();
678
679        mtls->fep.dim.x = outType->getDimX();
680        mtls->fep.dim.y = outType->getDimY();
681        mtls->fep.dim.z = outType->getDimZ();
682
683    } else if (sc != nullptr) {
684        mtls->fep.dim.x = sc->xEnd;
685        mtls->fep.dim.y = sc->yEnd;
686        mtls->fep.dim.z = 0;
687    } else {
688        mCtx->getContext()->setError(RS_ERROR_BAD_SCRIPT,
689                                     "rsForEach called with null allocations");
690        return false;
691    }
692
693    if (inLen > 0 && aout != nullptr) {
694        if (ains[0] && !ains[0]->hasSameDims(aout)) {
695            mCtx->getContext()->setError(RS_ERROR_BAD_SCRIPT,
696              "Failed to launch kernel; dimensions of input and output allocations do not match.");
697
698            return false;
699        }
700    }
701
702    if (!setUpMtlsDimensions(mtls, mtls->fep.dim, sc)) {
703        return false;
704    }
705
706    // The X & Y walkers always want 0-1 min even if dim is not present
707    mtls->end.x    = rsMax((uint32_t)1, mtls->end.x);
708    mtls->end.y    = rsMax((uint32_t)1, mtls->end.y);
709    mtls->rs       = mCtx;
710    if (ains) {
711        memcpy(mtls->ains, ains, inLen * sizeof(ains[0]));
712    }
713    mtls->aout[0]    = aout;
714    mtls->fep.usr    = usr;
715    mtls->fep.usrLen = usrLen;
716    mtls->mSliceSize = 1;
717    mtls->mSliceNum  = 0;
718
719    mtls->isThreadable  = mIsThreadable;
720
721    if (inLen > 0) {
722        mtls->fep.inLen = inLen;
723        for (int index = inLen; --index >= 0;) {
724            if (ains[index] == nullptr) {
725                // In old style kernels, the first and only input allocation could be null.
726                // Not allowed in newer styles.
727                rsAssert(inLen == 1 && index == 0);
728                continue;
729            }
730            mtls->fep.inPtr[index] = (const uint8_t*)ains[index]->mHal.drvState.lod[0].mallocPtr;
731            mtls->fep.inStride[index] = ains[index]->getType()->getElementSizeBytes();
732        }
733    }
734
735    if (aout != nullptr) {
736        mtls->fep.outPtr[0] = (uint8_t *)aout->mHal.drvState.lod[0].mallocPtr;
737        mtls->fep.outStride[0] = aout->getType()->getElementSizeBytes();
738    }
739
740    // All validation passed, ok to launch threads
741    return true;
742}
743
744
745void RsdCpuScriptImpl::invokeForEach(uint32_t slot,
746                                     const Allocation ** ains,
747                                     uint32_t inLen,
748                                     Allocation * aout,
749                                     const void * usr,
750                                     uint32_t usrLen,
751                                     const RsScriptCall *sc) {
752
753    MTLaunchStructForEach mtls;
754
755    if (forEachMtlsSetup(ains, inLen, aout, usr, usrLen, sc, &mtls)) {
756        forEachKernelSetup(slot, &mtls);
757
758        RsdCpuScriptImpl * oldTLS = mCtx->setTLS(this);
759        mCtx->launchForEach(ains, inLen, aout, sc, &mtls);
760        mCtx->setTLS(oldTLS);
761    }
762}
763
764void RsdCpuScriptImpl::invokeReduce(uint32_t slot,
765                                    const Allocation ** ains, uint32_t inLen,
766                                    Allocation *aout,
767                                    const RsScriptCall *sc) {
768  MTLaunchStructReduce mtls;
769
770  if (reduceMtlsSetup(ains, inLen, aout, sc, &mtls)) {
771    reduceKernelSetup(slot, &mtls);
772    RsdCpuScriptImpl *oldTLS = mCtx->setTLS(this);
773    mCtx->launchReduce(ains, inLen, aout, &mtls);
774    mCtx->setTLS(oldTLS);
775  }
776}
777
778void RsdCpuScriptImpl::forEachKernelSetup(uint32_t slot, MTLaunchStructForEach *mtls) {
779    mtls->script = this;
780    mtls->fep.slot = slot;
781    mtls->kernel = mScriptExec->getForEachFunction(slot);
782    rsAssert(mtls->kernel != nullptr);
783}
784
785void RsdCpuScriptImpl::reduceKernelSetup(uint32_t slot, MTLaunchStructReduce *mtls) {
786    mtls->script = this;
787    mtls->redp.slot = slot;
788
789    const ReduceDescription *desc = mScriptExec->getReduceDescription(slot);
790    mtls->accumFunc = desc->accumFunc;
791    mtls->initFunc  = desc->initFunc;   // might legally be nullptr
792    mtls->combFunc  = desc->combFunc;   // might legally be nullptr
793    mtls->outFunc   = desc->outFunc;    // might legally be nullptr
794    mtls->accumSize = desc->accumSize;
795
796    rsAssert(mtls->accumFunc != nullptr);
797}
798
799int RsdCpuScriptImpl::invokeRoot() {
800    RsdCpuScriptImpl * oldTLS = mCtx->setTLS(this);
801    int ret = mRoot();
802    mCtx->setTLS(oldTLS);
803    return ret;
804}
805
806void RsdCpuScriptImpl::invokeInit() {
807    if (mInit) {
808        mInit();
809    }
810}
811
812void RsdCpuScriptImpl::invokeFreeChildren() {
813    if (mFreeChildren) {
814        mFreeChildren();
815    }
816}
817
818void RsdCpuScriptImpl::invokeFunction(uint32_t slot, const void *params,
819                                      size_t paramLength) {
820    //ALOGE("invoke %i %p %zu", slot, params, paramLength);
821    void * ap = nullptr;
822
823#if defined(__x86_64__)
824    // The invoked function could have input parameter of vector type for example float4 which
825    // requires void* params to be 16 bytes aligned when using SSE instructions for x86_64 platform.
826    // So try to align void* params before passing them into RS exported function.
827
828    if ((uint8_t)(uint64_t)params & 0x0F) {
829        if ((ap = (void*)memalign(16, paramLength)) != nullptr) {
830            memcpy(ap, params, paramLength);
831        } else {
832            ALOGE("x86_64: invokeFunction memalign error, still use params which"
833                  " is not 16 bytes aligned.");
834        }
835    }
836#endif
837
838    RsdCpuScriptImpl * oldTLS = mCtx->setTLS(this);
839    reinterpret_cast<void (*)(const void *, uint32_t)>(
840        mScriptExec->getInvokeFunction(slot))(ap? (const void *) ap: params, paramLength);
841
842#if defined(__x86_64__)
843    free(ap);
844#endif
845
846    mCtx->setTLS(oldTLS);
847}
848
849void RsdCpuScriptImpl::setGlobalVar(uint32_t slot, const void *data, size_t dataLength) {
850    //rsAssert(!script->mFieldIsObject[slot]);
851    //ALOGE("setGlobalVar %i %p %zu", slot, data, dataLength);
852
853    //if (mIntrinsicID) {
854        //mIntrinsicFuncs.setVar(dc, script, drv->mIntrinsicData, slot, data, dataLength);
855        //return;
856    //}
857
858    int32_t *destPtr = reinterpret_cast<int32_t *>(mScriptExec->getFieldAddress(slot));
859    if (!destPtr) {
860        //ALOGV("Calling setVar on slot = %i which is null", slot);
861        return;
862    }
863
864    memcpy(destPtr, data, dataLength);
865}
866
867void RsdCpuScriptImpl::getGlobalVar(uint32_t slot, void *data, size_t dataLength) {
868    //rsAssert(!script->mFieldIsObject[slot]);
869    //ALOGE("getGlobalVar %i %p %zu", slot, data, dataLength);
870
871    int32_t *srcPtr = reinterpret_cast<int32_t *>(mScriptExec->getFieldAddress(slot));
872    if (!srcPtr) {
873        //ALOGV("Calling setVar on slot = %i which is null", slot);
874        return;
875    }
876    memcpy(data, srcPtr, dataLength);
877}
878
879
880void RsdCpuScriptImpl::setGlobalVarWithElemDims(uint32_t slot, const void *data, size_t dataLength,
881                                                const Element *elem,
882                                                const uint32_t *dims, size_t dimLength) {
883    int32_t *destPtr = reinterpret_cast<int32_t *>(mScriptExec->getFieldAddress(slot));
884    if (!destPtr) {
885        //ALOGV("Calling setVar on slot = %i which is null", slot);
886        return;
887    }
888
889    // We want to look at dimension in terms of integer components,
890    // but dimLength is given in terms of bytes.
891    dimLength /= sizeof(int);
892
893    // Only a single dimension is currently supported.
894    rsAssert(dimLength == 1);
895    if (dimLength == 1) {
896        // First do the increment loop.
897        size_t stride = elem->getSizeBytes();
898        const char *cVal = reinterpret_cast<const char *>(data);
899        for (uint32_t i = 0; i < dims[0]; i++) {
900            elem->incRefs(cVal);
901            cVal += stride;
902        }
903
904        // Decrement loop comes after (to prevent race conditions).
905        char *oldVal = reinterpret_cast<char *>(destPtr);
906        for (uint32_t i = 0; i < dims[0]; i++) {
907            elem->decRefs(oldVal);
908            oldVal += stride;
909        }
910    }
911
912    memcpy(destPtr, data, dataLength);
913}
914
915void RsdCpuScriptImpl::setGlobalBind(uint32_t slot, Allocation *data) {
916
917    //rsAssert(!script->mFieldIsObject[slot]);
918    //ALOGE("setGlobalBind %i %p", slot, data);
919
920    int32_t *destPtr = reinterpret_cast<int32_t *>(mScriptExec->getFieldAddress(slot));
921    if (!destPtr) {
922        //ALOGV("Calling setVar on slot = %i which is null", slot);
923        return;
924    }
925
926    void *ptr = nullptr;
927    mBoundAllocs[slot] = data;
928    if (data) {
929        ptr = data->mHal.drvState.lod[0].mallocPtr;
930    }
931    memcpy(destPtr, &ptr, sizeof(void *));
932}
933
934void RsdCpuScriptImpl::setGlobalObj(uint32_t slot, ObjectBase *data) {
935
936    //rsAssert(script->mFieldIsObject[slot]);
937    //ALOGE("setGlobalObj %i %p", slot, data);
938
939    int32_t *destPtr = reinterpret_cast<int32_t *>(mScriptExec->getFieldAddress(slot));
940    if (!destPtr) {
941        //ALOGV("Calling setVar on slot = %i which is null", slot);
942        return;
943    }
944
945    rsrSetObject(mCtx->getContext(), (rs_object_base *)destPtr, data);
946}
947
948const char* RsdCpuScriptImpl::getFieldName(uint32_t slot) const {
949    return mScriptExec->getFieldName(slot);
950}
951
952RsdCpuScriptImpl::~RsdCpuScriptImpl() {
953    delete mScriptExec;
954    delete[] mBoundAllocs;
955    if (mScriptSO) {
956        dlclose(mScriptSO);
957    }
958}
959
960Allocation * RsdCpuScriptImpl::getAllocationForPointer(const void *ptr) const {
961    if (!ptr) {
962        return nullptr;
963    }
964
965    for (uint32_t ct=0; ct < mScript->mHal.info.exportedVariableCount; ct++) {
966        Allocation *a = mBoundAllocs[ct];
967        if (!a) continue;
968        if (a->mHal.drvState.lod[0].mallocPtr == ptr) {
969            return a;
970        }
971    }
972    ALOGE("rsGetAllocation, failed to find %p", ptr);
973    return nullptr;
974}
975
976int RsdCpuScriptImpl::getGlobalEntries() const {
977    return mScriptExec->getGlobalEntries();
978}
979
980const char * RsdCpuScriptImpl::getGlobalName(int i) const {
981    return mScriptExec->getGlobalName(i);
982}
983
984const void * RsdCpuScriptImpl::getGlobalAddress(int i) const {
985    return mScriptExec->getGlobalAddress(i);
986}
987
988size_t RsdCpuScriptImpl::getGlobalSize(int i) const {
989    return mScriptExec->getGlobalSize(i);
990}
991
992uint32_t RsdCpuScriptImpl::getGlobalProperties(int i) const {
993    return mScriptExec->getGlobalProperties(i);
994}
995
996void RsdCpuScriptImpl::preLaunch(uint32_t slot, const Allocation ** ains,
997                                 uint32_t inLen, Allocation * aout,
998                                 const void * usr, uint32_t usrLen,
999                                 const RsScriptCall *sc) {}
1000
1001void RsdCpuScriptImpl::postLaunch(uint32_t slot, const Allocation ** ains,
1002                                  uint32_t inLen, Allocation * aout,
1003                                  const void * usr, uint32_t usrLen,
1004                                  const RsScriptCall *sc) {}
1005
1006
1007} // namespace renderscript
1008} // namespace android
1009