rsCpuScript.cpp revision f5029803ae6ce9d92d70b76e7a7cdd8d484f31ca 
1/*
2 * Copyright (C) 2011-2012 The Android Open Source Project
3 *
4 * Licensed under the Apache License, Version 2.0 (the "License");
5 * you may not use this file except in compliance with the License.
6 * You may obtain a copy of the License at
7 *
8 *      http://www.apache.org/licenses/LICENSE-2.0
9 *
10 * Unless required by applicable law or agreed to in writing, software
11 * distributed under the License is distributed on an "AS IS" BASIS,
12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 * See the License for the specific language governing permissions and
14 * limitations under the License.
15 */
16
17#include "rsCpuCore.h"
18#include "rsCpuScript.h"
19#include "rsCpuExecutable.h"
20
21#ifdef RS_COMPATIBILITY_LIB
22    #include <stdio.h>
23    #include <sys/stat.h>
24    #include <unistd.h>
25#else
26    #include "rsCppUtils.h"
27
28    #include <bcc/Config/Config.h>
29    #include <bcinfo/MetadataExtractor.h>
30    #include <cutils/properties.h>
31
32    #include <zlib.h>
33    #include <sys/file.h>
34    #include <sys/types.h>
35    #include <unistd.h>
36
37    #include <string>
38    #include <vector>
39#endif
40
41#include <set>
42#include <string>
43#include <dlfcn.h>
44#include <stdlib.h>
45#include <string.h>
46#include <iostream>
47#include <sstream>
48
49namespace {
50
51static const bool kDebugGlobalVariables = false;
52
53#ifndef RS_COMPATIBILITY_LIB
54
55static bool is_force_recompile() {
56#ifdef RS_SERVER
57  return false;
58#else
59  char buf[PROPERTY_VALUE_MAX];
60
61  // Re-compile if floating point precision has been overridden.
62  property_get("debug.rs.precision", buf, "");
63  if (buf[0] != '\0') {
64    return true;
65  }
66
67  // Re-compile if debug.rs.forcerecompile is set.
68  property_get("debug.rs.forcerecompile", buf, "0");
69  if ((::strcmp(buf, "1") == 0) || (::strcmp(buf, "true") == 0)) {
70    return true;
71  } else {
72    return false;
73  }
74#endif  // RS_SERVER
75}
76
77static void setCompileArguments(std::vector<const char*>* args,
78                                const std::string& bcFileName,
79                                const char* cacheDir, const char* resName,
80                                const char* core_lib, bool useRSDebugContext,
81                                const char* bccPluginName, bool emitGlobalInfo,
82                                int optLevel, bool emitGlobalInfoSkipConstant) {
83    rsAssert(cacheDir && resName && core_lib);
84    args->push_back(android::renderscript::RsdCpuScriptImpl::BCC_EXE_PATH);
85    args->push_back("-unroll-runtime");
86    args->push_back("-scalarize-load-store");
87    if (emitGlobalInfo) {
88        args->push_back("-rs-global-info");
89        if (emitGlobalInfoSkipConstant) {
90            args->push_back("-rs-global-info-skip-constant");
91        }
92    }
93    args->push_back("-o");
94    args->push_back(resName);
95    args->push_back("-output_path");
96    args->push_back(cacheDir);
97    args->push_back("-bclib");
98    args->push_back(core_lib);
99    args->push_back("-mtriple");
100    args->push_back(DEFAULT_TARGET_TRIPLE_STRING);
101    args->push_back("-O");
102
103    switch (optLevel) {
104    case (0):
105        args->push_back("0");
106        break;
107    case (3):
108        args->push_back("3");
109        break;
110    default:
111        ALOGW("Expected optimization level of 0 or 3. Received %d", optLevel);
112        args->push_back("3");
113        break;
114    }
115
116    // Enable workaround for A53 codegen by default.
117#if defined(__aarch64__) && !defined(DISABLE_A53_WORKAROUND)
118    args->push_back("-aarch64-fix-cortex-a53-835769");
119#endif
120
121    // Execute the bcc compiler.
122    if (useRSDebugContext) {
123        args->push_back("-rs-debug-ctx");
124    } else {
125        // Only load additional libraries for compiles that don't use
126        // the debug context.
127        if (bccPluginName && strlen(bccPluginName) > 0) {
128            args->push_back("-load");
129            args->push_back(bccPluginName);
130        }
131    }
132
133    args->push_back("-fPIC");
134    args->push_back("-embedRSInfo");
135
136    args->push_back(bcFileName.c_str());
137    args->push_back(nullptr);
138}
139
140static bool compileBitcode(const std::string &bcFileName,
141                           const char *bitcode,
142                           size_t bitcodeSize,
143                           std::vector<const char *> &compileArguments) {
144    rsAssert(bitcode && bitcodeSize);
145
146    FILE *bcfile = fopen(bcFileName.c_str(), "w");
147    if (!bcfile) {
148        ALOGE("Could not write to %s", bcFileName.c_str());
149        return false;
150    }
151    size_t nwritten = fwrite(bitcode, 1, bitcodeSize, bcfile);
152    fclose(bcfile);
153    if (nwritten != bitcodeSize) {
154        ALOGE("Could not write %zu bytes to %s", bitcodeSize,
155              bcFileName.c_str());
156        return false;
157    }
158
159    return android::renderscript::rsuExecuteCommand(
160                   android::renderscript::RsdCpuScriptImpl::BCC_EXE_PATH,
161                   compileArguments.size()-1, compileArguments.data());
162}
163
164// The checksum is unnecessary under a few conditions, since the primary
165// use-case for it is debugging. If we are loading something from the
166// system partition (read-only), we know that it was precompiled as part of
167// application ahead of time (and thus the checksum is completely
168// unnecessary). The checksum is also unnecessary on release (non-debug)
169// builds, as the only way to get a shared object is to have compiled the
170// script once already. On a release build, there is no way to adjust the
171// other libraries/dependencies, and so the only reason to recompile would
172// be for a source APK change or an OTA. In either case, the APK would be
173// reinstalled, which would already clear the code_cache/ directory.
174bool isChecksumNeeded(const char *cacheDir) {
175    if ((::strcmp(SYSLIBPATH, cacheDir) == 0) ||
176        (::strcmp(SYSLIBPATH_VENDOR, cacheDir) == 0))
177        return false;
178    char buf[PROPERTY_VALUE_MAX];
179    property_get("ro.debuggable", buf, "");
180    return (buf[0] == '1');
181}
182
183bool addFileToChecksum(const char *fileName, uint32_t &checksum) {
184    int FD = open(fileName, O_RDONLY);
185    if (FD == -1) {
186        ALOGE("Cannot open file \'%s\' to compute checksum", fileName);
187        return false;
188    }
189
190    char buf[256];
191    while (true) {
192        ssize_t nread = read(FD, buf, sizeof(buf));
193        if (nread < 0) { // bail out on failed read
194            ALOGE("Error while computing checksum for file \'%s\'", fileName);
195            return false;
196        }
197
198        checksum = adler32(checksum, (const unsigned char *) buf, nread);
199        if (static_cast<size_t>(nread) < sizeof(buf)) // EOF
200            break;
201    }
202
203    if (close(FD) != 0) {
204        ALOGE("Cannot close file \'%s\' after computing checksum", fileName);
205        return false;
206    }
207    return true;
208}
209
210#endif  // !defined(RS_COMPATIBILITY_LIB)
211}  // namespace
212
213namespace android {
214namespace renderscript {
215
216#ifndef RS_COMPATIBILITY_LIB
217
218uint32_t constructBuildChecksum(uint8_t const *bitcode, size_t bitcodeSize,
219                                const char *commandLine,
220                                const char** bccFiles, size_t numFiles) {
221    uint32_t checksum = adler32(0L, Z_NULL, 0);
222
223    // include checksum of bitcode
224    if (bitcode != nullptr && bitcodeSize > 0) {
225        checksum = adler32(checksum, bitcode, bitcodeSize);
226    }
227
228    // include checksum of command line arguments
229    checksum = adler32(checksum, (const unsigned char *) commandLine,
230                       strlen(commandLine));
231
232    // include checksum of bccFiles
233    for (size_t i = 0; i < numFiles; i++) {
234        const char* bccFile = bccFiles[i];
235        if (bccFile[0] != 0 && !addFileToChecksum(bccFile, checksum)) {
236            // return empty checksum instead of something partial/corrupt
237            return 0;
238        }
239    }
240
241    return checksum;
242}
243
244#endif  // !RS_COMPATIBILITY_LIB
245
246RsdCpuScriptImpl::RsdCpuScriptImpl(RsdCpuReferenceImpl *ctx, const Script *s) {
247    mCtx = ctx;
248    mScript = s;
249
250    mScriptSO = nullptr;
251
252    mRoot = nullptr;
253    mRootExpand = nullptr;
254    mInit = nullptr;
255    mFreeChildren = nullptr;
256    mScriptExec = nullptr;
257
258    mBoundAllocs = nullptr;
259    mIntrinsicData = nullptr;
260    mIsThreadable = true;
261
262    mBuildChecksum = 0;
263    mChecksumNeeded = false;
264}
265
266bool RsdCpuScriptImpl::storeRSInfoFromSO() {
267    // The shared object may have an invalid build checksum.
268    // Validate and fail early.
269    mScriptExec = ScriptExecutable::createFromSharedObject(
270            mCtx->getContext(), mScriptSO,
271            mChecksumNeeded ? mBuildChecksum : 0);
272
273    if (mScriptExec == nullptr) {
274        return false;
275    }
276
277    mRoot = (RootFunc_t) dlsym(mScriptSO, "root");
278    if (mRoot) {
279        //ALOGE("Found root(): %p", mRoot);
280    }
281    mRootExpand = (RootFunc_t) dlsym(mScriptSO, "root.expand");
282    if (mRootExpand) {
283        //ALOGE("Found root.expand(): %p", mRootExpand);
284    }
285    mInit = (InvokeFunc_t) dlsym(mScriptSO, "init");
286    if (mInit) {
287        //ALOGE("Found init(): %p", mInit);
288    }
289    mFreeChildren = (InvokeFunc_t) dlsym(mScriptSO, ".rs.dtor");
290    if (mFreeChildren) {
291        //ALOGE("Found .rs.dtor(): %p", mFreeChildren);
292    }
293
294    size_t varCount = mScriptExec->getExportedVariableCount();
295    if (varCount > 0) {
296        mBoundAllocs = new Allocation *[varCount];
297        memset(mBoundAllocs, 0, varCount * sizeof(*mBoundAllocs));
298    }
299
300    mIsThreadable = mScriptExec->getThreadable();
301    //ALOGE("Script isThreadable? %d", mIsThreadable);
302
303    if (kDebugGlobalVariables) {
304        mScriptExec->dumpGlobalInfo();
305    }
306
307    return true;
308}
309
310bool RsdCpuScriptImpl::init(char const *resName, char const *cacheDir,
311                            uint8_t const *bitcode, size_t bitcodeSize,
312                            uint32_t flags, char const *bccPluginName) {
313    //ALOGE("rsdScriptCreate %p %p %p %p %i %i %p", rsc, resName, cacheDir,
314    // bitcode, bitcodeSize, flags, lookupFunc);
315    //ALOGE("rsdScriptInit %p %p", rsc, script);
316
317    mCtx->lockMutex();
318#ifndef RS_COMPATIBILITY_LIB
319    bool useRSDebugContext = false;
320
321    bcinfo::MetadataExtractor bitcodeMetadata((const char *) bitcode, bitcodeSize);
322    if (!bitcodeMetadata.extract()) {
323        ALOGE("Could not extract metadata from bitcode");
324        mCtx->unlockMutex();
325        return false;
326    }
327
328    const char* core_lib = findCoreLib(bitcodeMetadata, (const char*)bitcode, bitcodeSize);
329
330    if (mCtx->getContext()->getContextType() == RS_CONTEXT_TYPE_DEBUG) {
331        useRSDebugContext = true;
332    }
333
334    int optLevel = mCtx->getContext()->getOptLevel();
335
336    std::string bcFileName(cacheDir);
337    bcFileName.append("/");
338    bcFileName.append(resName);
339    bcFileName.append(".bc");
340
341    std::vector<const char*> compileArguments;
342    bool emitGlobalInfo = mCtx->getEmbedGlobalInfo();
343    bool emitGlobalInfoSkipConstant = mCtx->getEmbedGlobalInfoSkipConstant();
344    setCompileArguments(&compileArguments, bcFileName, cacheDir, resName, core_lib,
345                        useRSDebugContext, bccPluginName, emitGlobalInfo,
346                        optLevel, emitGlobalInfoSkipConstant);
347
348    mChecksumNeeded = isChecksumNeeded(cacheDir);
349    if (mChecksumNeeded) {
350        std::vector<const char *> bccFiles = { BCC_EXE_PATH,
351                                               core_lib,
352                                             };
353
354        // The last argument of compileArguments is a nullptr, so remove 1 from
355        // the size.
356        std::unique_ptr<const char> compileCommandLine(
357            rsuJoinStrings(compileArguments.size()-1, compileArguments.data()));
358
359        mBuildChecksum = constructBuildChecksum(bitcode, bitcodeSize,
360                                                compileCommandLine.get(),
361                                                bccFiles.data(), bccFiles.size());
362
363        if (mBuildChecksum == 0) {
364            // cannot compute checksum but verification is enabled
365            mCtx->unlockMutex();
366            return false;
367        }
368    }
369    else {
370        // add a dummy/constant as a checksum if verification is disabled
371        mBuildChecksum = 0xabadcafe;
372    }
373
374    // Append build checksum to commandline
375    // Handle the terminal nullptr in compileArguments
376    compileArguments.pop_back();
377    compileArguments.push_back("-build-checksum");
378    std::stringstream ss;
379    ss << std::hex << mBuildChecksum;
380    compileArguments.push_back(ss.str().c_str());
381    compileArguments.push_back(nullptr);
382
383    if (!is_force_recompile() && !useRSDebugContext) {
384        mScriptSO = SharedLibraryUtils::loadSharedLibrary(cacheDir, resName);
385
386        // Read RS info from the shared object to detect checksum mismatch
387        if (mScriptSO != nullptr && !storeRSInfoFromSO()) {
388            dlclose(mScriptSO);
389            mScriptSO = nullptr;
390        }
391    }
392
393    // If we can't, it's either not there or out of date.  We compile the bit code and try loading
394    // again.
395    if (mScriptSO == nullptr) {
396        if (!compileBitcode(bcFileName, (const char*)bitcode, bitcodeSize,
397                            compileArguments))
398        {
399            ALOGE("bcc: FAILS to compile '%s'", resName);
400            mCtx->unlockMutex();
401            return false;
402        }
403
404        if (!SharedLibraryUtils::createSharedLibrary(mCtx->getContext()->getDriverName(),
405                                                     cacheDir, resName)) {
406            ALOGE("Linker: Failed to link object file '%s'", resName);
407            mCtx->unlockMutex();
408            return false;
409        }
410
411        mScriptSO = SharedLibraryUtils::loadSharedLibrary(cacheDir, resName);
412        if (mScriptSO == nullptr) {
413            ALOGE("Unable to load '%s'", resName);
414            mCtx->unlockMutex();
415            return false;
416        }
417
418        // Read RS symbol information from the .so.
419        if (!storeRSInfoFromSO()) {
420            goto error;
421        }
422    }
423
424    mBitcodeFilePath.setTo(bcFileName.c_str());
425
426#else  // RS_COMPATIBILITY_LIB is defined
427    const char *nativeLibDir = mCtx->getContext()->getNativeLibDir();
428    mScriptSO = SharedLibraryUtils::loadSharedLibrary(cacheDir, resName, nativeLibDir);
429
430    if (!mScriptSO) {
431        goto error;
432    }
433
434    if (!storeRSInfoFromSO()) {
435        goto error;
436    }
437#endif
438    mCtx->unlockMutex();
439    return true;
440
441error:
442
443    mCtx->unlockMutex();
444    if (mScriptSO) {
445        dlclose(mScriptSO);
446        mScriptSO = nullptr;
447    }
448    return false;
449}
450
451#ifndef RS_COMPATIBILITY_LIB
452
453const char* RsdCpuScriptImpl::findCoreLib(const bcinfo::MetadataExtractor& ME, const char* bitcode,
454                                          size_t bitcodeSize) {
455    const char* defaultLib = SYSLIBPATH"/libclcore.bc";
456
457    // If we're debugging, use the debug library.
458    if (mCtx->getContext()->getContextType() == RS_CONTEXT_TYPE_DEBUG) {
459        return SYSLIBPATH"/libclcore_debug.bc";
460    }
461
462    // If a callback has been registered to specify a library, use that.
463    RSSelectRTCallback selectRTCallback = mCtx->getSelectRTCallback();
464    if (selectRTCallback != nullptr) {
465        return selectRTCallback((const char*)bitcode, bitcodeSize);
466    }
467
468    // Check for a platform specific library
469#if defined(ARCH_ARM_HAVE_NEON) && !defined(DISABLE_CLCORE_NEON)
470    enum bcinfo::RSFloatPrecision prec = ME.getRSFloatPrecision();
471    if (prec == bcinfo::RS_FP_Relaxed) {
472        // NEON-capable ARMv7a devices can use an accelerated math library
473        // for all reduced precision scripts.
474        // ARMv8 does not use NEON, as ASIMD can be used with all precision
475        // levels.
476        return SYSLIBPATH"/libclcore_neon.bc";
477    } else {
478        return defaultLib;
479    }
480#elif defined(__i386__) || defined(__x86_64__)
481    // x86 devices will use an optimized library.
482    return SYSLIBPATH"/libclcore_x86.bc";
483#else
484    return defaultLib;
485#endif
486}
487
488#endif
489
490void RsdCpuScriptImpl::populateScript(Script *script) {
491    // Copy info over to runtime
492    script->mHal.info.exportedFunctionCount = mScriptExec->getExportedFunctionCount();
493    script->mHal.info.exportedVariableCount = mScriptExec->getExportedVariableCount();
494    script->mHal.info.exportedPragmaCount = mScriptExec->getPragmaCount();;
495    script->mHal.info.exportedPragmaKeyList = mScriptExec->getPragmaKeys();
496    script->mHal.info.exportedPragmaValueList = mScriptExec->getPragmaValues();
497
498    // Bug, need to stash in metadata
499    if (mRootExpand) {
500        script->mHal.info.root = mRootExpand;
501    } else {
502        script->mHal.info.root = mRoot;
503    }
504}
505
506
507bool RsdCpuScriptImpl::forEachMtlsSetup(const Allocation ** ains,
508                                        uint32_t inLen,
509                                        Allocation * aout,
510                                        const void * usr, uint32_t usrLen,
511                                        const RsScriptCall *sc,
512                                        MTLaunchStruct *mtls) {
513
514    memset(mtls, 0, sizeof(MTLaunchStruct));
515
516    for (int index = inLen; --index >= 0;) {
517        const Allocation* ain = ains[index];
518
519        // possible for this to occur if IO_OUTPUT/IO_INPUT with no bound surface
520        if (ain != nullptr &&
521            (const uint8_t *)ain->mHal.drvState.lod[0].mallocPtr == nullptr) {
522
523            mCtx->getContext()->setError(RS_ERROR_BAD_SCRIPT,
524                                         "rsForEach called with null in allocations");
525            return false;
526        }
527    }
528
529    if (aout &&
530        (const uint8_t *)aout->mHal.drvState.lod[0].mallocPtr == nullptr) {
531
532        mCtx->getContext()->setError(RS_ERROR_BAD_SCRIPT,
533                                     "rsForEach called with null out allocations");
534        return false;
535    }
536
537    if (inLen > 0) {
538        const Allocation *ain0   = ains[0];
539        const Type       *inType = ain0->getType();
540
541        mtls->fep.dim.x = inType->getDimX();
542        mtls->fep.dim.y = inType->getDimY();
543        mtls->fep.dim.z = inType->getDimZ();
544
545        for (int Index = inLen; --Index >= 1;) {
546            if (!ain0->hasSameDims(ains[Index])) {
547                mCtx->getContext()->setError(RS_ERROR_BAD_SCRIPT,
548                  "Failed to launch kernel; dimensions of input and output"
549                  "allocations do not match.");
550
551                return false;
552            }
553        }
554
555    } else if (aout != nullptr) {
556        const Type *outType = aout->getType();
557
558        mtls->fep.dim.x = outType->getDimX();
559        mtls->fep.dim.y = outType->getDimY();
560        mtls->fep.dim.z = outType->getDimZ();
561
562    } else if (sc != nullptr) {
563        mtls->fep.dim.x = sc->xEnd;
564        mtls->fep.dim.y = sc->yEnd;
565        mtls->fep.dim.z = 0;
566    } else {
567        mCtx->getContext()->setError(RS_ERROR_BAD_SCRIPT,
568                                     "rsForEach called with null allocations");
569        return false;
570    }
571
572    if (inLen > 0 && aout != nullptr) {
573        if (!ains[0]->hasSameDims(aout)) {
574            mCtx->getContext()->setError(RS_ERROR_BAD_SCRIPT,
575              "Failed to launch kernel; dimensions of input and output allocations do not match.");
576
577            return false;
578        }
579    }
580
581    if (!sc || (sc->xEnd == 0)) {
582        mtls->end.x = mtls->fep.dim.x;
583    } else {
584        mtls->start.x = rsMin(mtls->fep.dim.x, sc->xStart);
585        mtls->end.x = rsMin(mtls->fep.dim.x, sc->xEnd);
586        if (mtls->start.x >= mtls->end.x) {
587            mCtx->getContext()->setError(RS_ERROR_BAD_SCRIPT,
588              "Failed to launch kernel; Invalid xStart or xEnd.");
589            return false;
590        }
591    }
592
593    if (!sc || (sc->yEnd == 0)) {
594        mtls->end.y = mtls->fep.dim.y;
595    } else {
596        mtls->start.y = rsMin(mtls->fep.dim.y, sc->yStart);
597        mtls->end.y = rsMin(mtls->fep.dim.y, sc->yEnd);
598        if (mtls->start.y >= mtls->end.y) {
599            mCtx->getContext()->setError(RS_ERROR_BAD_SCRIPT,
600              "Failed to launch kernel; Invalid yStart or yEnd.");
601            return false;
602        }
603    }
604
605    if (!sc || (sc->zEnd == 0)) {
606        mtls->end.z = mtls->fep.dim.z;
607    } else {
608        mtls->start.z = rsMin(mtls->fep.dim.z, sc->zStart);
609        mtls->end.z = rsMin(mtls->fep.dim.z, sc->zEnd);
610        if (mtls->start.z >= mtls->end.z) {
611            mCtx->getContext()->setError(RS_ERROR_BAD_SCRIPT,
612              "Failed to launch kernel; Invalid zStart or zEnd.");
613            return false;
614        }
615    }
616
617    if (!sc || (sc->arrayEnd == 0)) {
618        mtls->end.array[0] = mtls->fep.dim.array[0];
619    } else {
620        mtls->start.array[0] = rsMin(mtls->fep.dim.array[0], sc->arrayStart);
621        mtls->end.array[0] = rsMin(mtls->fep.dim.array[0], sc->arrayEnd);
622        if (mtls->start.array[0] >= mtls->end.array[0]) {
623            mCtx->getContext()->setError(RS_ERROR_BAD_SCRIPT,
624              "Failed to launch kernel; Invalid arrayStart or arrayEnd.");
625            return false;
626        }
627    }
628
629    if (!sc || (sc->array2End == 0)) {
630        mtls->end.array[1] = mtls->fep.dim.array[1];
631    } else {
632        mtls->start.array[1] = rsMin(mtls->fep.dim.array[1], sc->array2Start);
633        mtls->end.array[1] = rsMin(mtls->fep.dim.array[1], sc->array2End);
634        if (mtls->start.array[1] >= mtls->end.array[1]) {
635            mCtx->getContext()->setError(RS_ERROR_BAD_SCRIPT,
636              "Failed to launch kernel; Invalid array2Start or array2End.");
637            return false;
638        }
639    }
640
641    if (!sc || (sc->array3End == 0)) {
642        mtls->end.array[2] = mtls->fep.dim.array[2];
643    } else {
644        mtls->start.array[2] = rsMin(mtls->fep.dim.array[2], sc->array3Start);
645        mtls->end.array[2] = rsMin(mtls->fep.dim.array[2], sc->array3End);
646        if (mtls->start.array[2] >= mtls->end.array[2]) {
647            mCtx->getContext()->setError(RS_ERROR_BAD_SCRIPT,
648              "Failed to launch kernel; Invalid array3Start or array3End.");
649            return false;
650        }
651    }
652
653    if (!sc || (sc->array4End == 0)) {
654        mtls->end.array[3] = mtls->fep.dim.array[3];
655    } else {
656        mtls->start.array[3] = rsMin(mtls->fep.dim.array[3], sc->array4Start);
657        mtls->end.array[3] = rsMin(mtls->fep.dim.array[3], sc->array4End);
658        if (mtls->start.array[3] >= mtls->end.array[3]) {
659            mCtx->getContext()->setError(RS_ERROR_BAD_SCRIPT,
660              "Failed to launch kernel; Invalid array4Start or array4End.");
661            return false;
662        }
663    }
664
665
666    // The X & Y walkers always want 0-1 min even if dim is not present
667    mtls->end.x    = rsMax((uint32_t)1, mtls->end.x);
668    mtls->end.y    = rsMax((uint32_t)1, mtls->end.y);
669
670    mtls->rsc        = mCtx;
671    if (ains) {
672        memcpy(mtls->ains, ains, inLen * sizeof(ains[0]));
673    }
674    mtls->aout[0]    = aout;
675    mtls->fep.usr    = usr;
676    mtls->fep.usrLen = usrLen;
677    mtls->mSliceSize = 1;
678    mtls->mSliceNum  = 0;
679
680    mtls->isThreadable  = mIsThreadable;
681
682    if (inLen > 0) {
683        mtls->fep.inLen = inLen;
684        for (int index = inLen; --index >= 0;) {
685            mtls->fep.inPtr[index] = (const uint8_t*)ains[index]->mHal.drvState.lod[0].mallocPtr;
686            mtls->fep.inStride[index] = ains[index]->getType()->getElementSizeBytes();
687        }
688    }
689
690    if (aout != nullptr) {
691        mtls->fep.outPtr[0] = (uint8_t *)aout->mHal.drvState.lod[0].mallocPtr;
692        mtls->fep.outStride[0] = aout->getType()->getElementSizeBytes();
693    }
694
695    // All validation passed, ok to launch threads
696    return true;
697}
698
699
700void RsdCpuScriptImpl::invokeForEach(uint32_t slot,
701                                     const Allocation ** ains,
702                                     uint32_t inLen,
703                                     Allocation * aout,
704                                     const void * usr,
705                                     uint32_t usrLen,
706                                     const RsScriptCall *sc) {
707
708    MTLaunchStruct mtls;
709
710    if (forEachMtlsSetup(ains, inLen, aout, usr, usrLen, sc, &mtls)) {
711        forEachKernelSetup(slot, &mtls);
712
713        RsdCpuScriptImpl * oldTLS = mCtx->setTLS(this);
714        mCtx->launchThreads(ains, inLen, aout, sc, &mtls);
715        mCtx->setTLS(oldTLS);
716    }
717}
718
719void RsdCpuScriptImpl::forEachKernelSetup(uint32_t slot, MTLaunchStruct *mtls) {
720    mtls->script = this;
721    mtls->fep.slot = slot;
722    mtls->kernel = mScriptExec->getForEachFunction(slot);
723    rsAssert(mtls->kernel != nullptr);
724    mtls->sig = mScriptExec->getForEachSignature(slot);
725}
726
727int RsdCpuScriptImpl::invokeRoot() {
728    RsdCpuScriptImpl * oldTLS = mCtx->setTLS(this);
729    int ret = mRoot();
730    mCtx->setTLS(oldTLS);
731    return ret;
732}
733
734void RsdCpuScriptImpl::invokeInit() {
735    if (mInit) {
736        mInit();
737    }
738}
739
740void RsdCpuScriptImpl::invokeFreeChildren() {
741    if (mFreeChildren) {
742        mFreeChildren();
743    }
744}
745
746void RsdCpuScriptImpl::invokeFunction(uint32_t slot, const void *params,
747                                      size_t paramLength) {
748    //ALOGE("invoke %i %p %zu", slot, params, paramLength);
749    void * ap = nullptr;
750
751#if defined(__x86_64__)
752    // The invoked function could have input parameter of vector type for example float4 which
753    // requires void* params to be 16 bytes aligned when using SSE instructions for x86_64 platform.
754    // So try to align void* params before passing them into RS exported function.
755
756    if ((uint8_t)(uint64_t)params & 0x0F) {
757        if ((ap = (void*)memalign(16, paramLength)) != nullptr) {
758            memcpy(ap, params, paramLength);
759        } else {
760            ALOGE("x86_64: invokeFunction memalign error, still use params which"
761                  " is not 16 bytes aligned.");
762        }
763    }
764#endif
765
766    RsdCpuScriptImpl * oldTLS = mCtx->setTLS(this);
767    reinterpret_cast<void (*)(const void *, uint32_t)>(
768        mScriptExec->getInvokeFunction(slot))(ap? (const void *) ap: params, paramLength);
769
770#if defined(__x86_64__)
771    free(ap);
772#endif
773
774    mCtx->setTLS(oldTLS);
775}
776
777void RsdCpuScriptImpl::setGlobalVar(uint32_t slot, const void *data, size_t dataLength) {
778    //rsAssert(!script->mFieldIsObject[slot]);
779    //ALOGE("setGlobalVar %i %p %zu", slot, data, dataLength);
780
781    //if (mIntrinsicID) {
782        //mIntrinsicFuncs.setVar(dc, script, drv->mIntrinsicData, slot, data, dataLength);
783        //return;
784    //}
785
786    int32_t *destPtr = reinterpret_cast<int32_t *>(mScriptExec->getFieldAddress(slot));
787    if (!destPtr) {
788        //ALOGV("Calling setVar on slot = %i which is null", slot);
789        return;
790    }
791
792    memcpy(destPtr, data, dataLength);
793}
794
795void RsdCpuScriptImpl::getGlobalVar(uint32_t slot, void *data, size_t dataLength) {
796    //rsAssert(!script->mFieldIsObject[slot]);
797    //ALOGE("getGlobalVar %i %p %zu", slot, data, dataLength);
798
799    int32_t *srcPtr = reinterpret_cast<int32_t *>(mScriptExec->getFieldAddress(slot));
800    if (!srcPtr) {
801        //ALOGV("Calling setVar on slot = %i which is null", slot);
802        return;
803    }
804    memcpy(data, srcPtr, dataLength);
805}
806
807
808void RsdCpuScriptImpl::setGlobalVarWithElemDims(uint32_t slot, const void *data, size_t dataLength,
809                                                const Element *elem,
810                                                const uint32_t *dims, size_t dimLength) {
811    int32_t *destPtr = reinterpret_cast<int32_t *>(mScriptExec->getFieldAddress(slot));
812    if (!destPtr) {
813        //ALOGV("Calling setVar on slot = %i which is null", slot);
814        return;
815    }
816
817    // We want to look at dimension in terms of integer components,
818    // but dimLength is given in terms of bytes.
819    dimLength /= sizeof(int);
820
821    // Only a single dimension is currently supported.
822    rsAssert(dimLength == 1);
823    if (dimLength == 1) {
824        // First do the increment loop.
825        size_t stride = elem->getSizeBytes();
826        const char *cVal = reinterpret_cast<const char *>(data);
827        for (uint32_t i = 0; i < dims[0]; i++) {
828            elem->incRefs(cVal);
829            cVal += stride;
830        }
831
832        // Decrement loop comes after (to prevent race conditions).
833        char *oldVal = reinterpret_cast<char *>(destPtr);
834        for (uint32_t i = 0; i < dims[0]; i++) {
835            elem->decRefs(oldVal);
836            oldVal += stride;
837        }
838    }
839
840    memcpy(destPtr, data, dataLength);
841}
842
843void RsdCpuScriptImpl::setGlobalBind(uint32_t slot, Allocation *data) {
844
845    //rsAssert(!script->mFieldIsObject[slot]);
846    //ALOGE("setGlobalBind %i %p", slot, data);
847
848    int32_t *destPtr = reinterpret_cast<int32_t *>(mScriptExec->getFieldAddress(slot));
849    if (!destPtr) {
850        //ALOGV("Calling setVar on slot = %i which is null", slot);
851        return;
852    }
853
854    void *ptr = nullptr;
855    mBoundAllocs[slot] = data;
856    if (data) {
857        ptr = data->mHal.drvState.lod[0].mallocPtr;
858    }
859    memcpy(destPtr, &ptr, sizeof(void *));
860}
861
862void RsdCpuScriptImpl::setGlobalObj(uint32_t slot, ObjectBase *data) {
863
864    //rsAssert(script->mFieldIsObject[slot]);
865    //ALOGE("setGlobalObj %i %p", slot, data);
866
867    int32_t *destPtr = reinterpret_cast<int32_t *>(mScriptExec->getFieldAddress(slot));
868    if (!destPtr) {
869        //ALOGV("Calling setVar on slot = %i which is null", slot);
870        return;
871    }
872
873    rsrSetObject(mCtx->getContext(), (rs_object_base *)destPtr, data);
874}
875
876const char* RsdCpuScriptImpl::getFieldName(uint32_t slot) const {
877    return mScriptExec->getFieldName(slot);
878}
879
880RsdCpuScriptImpl::~RsdCpuScriptImpl() {
881    delete mScriptExec;
882    delete[] mBoundAllocs;
883    if (mScriptSO) {
884        dlclose(mScriptSO);
885    }
886}
887
888Allocation * RsdCpuScriptImpl::getAllocationForPointer(const void *ptr) const {
889    if (!ptr) {
890        return nullptr;
891    }
892
893    for (uint32_t ct=0; ct < mScript->mHal.info.exportedVariableCount; ct++) {
894        Allocation *a = mBoundAllocs[ct];
895        if (!a) continue;
896        if (a->mHal.drvState.lod[0].mallocPtr == ptr) {
897            return a;
898        }
899    }
900    ALOGE("rsGetAllocation, failed to find %p", ptr);
901    return nullptr;
902}
903
904int RsdCpuScriptImpl::getGlobalEntries() const {
905    return mScriptExec->getGlobalEntries();
906}
907
908const char * RsdCpuScriptImpl::getGlobalName(int i) const {
909    return mScriptExec->getGlobalName(i);
910}
911
912const void * RsdCpuScriptImpl::getGlobalAddress(int i) const {
913    return mScriptExec->getGlobalAddress(i);
914}
915
916size_t RsdCpuScriptImpl::getGlobalSize(int i) const {
917    return mScriptExec->getGlobalSize(i);
918}
919
920uint32_t RsdCpuScriptImpl::getGlobalProperties(int i) const {
921    return mScriptExec->getGlobalProperties(i);
922}
923
924void RsdCpuScriptImpl::preLaunch(uint32_t slot, const Allocation ** ains,
925                                 uint32_t inLen, Allocation * aout,
926                                 const void * usr, uint32_t usrLen,
927                                 const RsScriptCall *sc) {}
928
929void RsdCpuScriptImpl::postLaunch(uint32_t slot, const Allocation ** ains,
930                                  uint32_t inLen, Allocation * aout,
931                                  const void * usr, uint32_t usrLen,
932                                  const RsScriptCall *sc) {}
933
934
935}
936}
937