rsCpuScript.cpp revision 1efae29f4bbe6c165caf6dfc4b89cf8a5f8c469b 
1/*
2 * Copyright (C) 2011-2012 The Android Open Source Project
3 *
4 * Licensed under the Apache License, Version 2.0 (the "License");
5 * you may not use this file except in compliance with the License.
6 * You may obtain a copy of the License at
7 *
8 *      http://www.apache.org/licenses/LICENSE-2.0
9 *
10 * Unless required by applicable law or agreed to in writing, software
11 * distributed under the License is distributed on an "AS IS" BASIS,
12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 * See the License for the specific language governing permissions and
14 * limitations under the License.
15 */
16
17#include "rsCpuCore.h"
18#include "rsCpuScript.h"
19#include "rsCpuExecutable.h"
20
21#ifdef RS_COMPATIBILITY_LIB
22    #include <stdio.h>
23    #include <sys/stat.h>
24    #include <unistd.h>
25#else
26    #include "rsCppUtils.h"
27
28    #include <bcc/Config/Config.h>
29    #include <bcinfo/MetadataExtractor.h>
30    #include <cutils/properties.h>
31
32    #include <zlib.h>
33    #include <sys/file.h>
34    #include <sys/types.h>
35    #include <unistd.h>
36
37    #include <string>
38    #include <vector>
39#endif
40
41#include <set>
42#include <string>
43#include <dlfcn.h>
44#include <stdlib.h>
45#include <string.h>
46#include <iostream>
47#include <sstream>
48
49namespace {
50
51static const bool kDebugGlobalVariables = false;
52
53#ifndef RS_COMPATIBILITY_LIB
54
55static bool is_force_recompile() {
56#ifdef RS_SERVER
57  return false;
58#else
59  char buf[PROPERTY_VALUE_MAX];
60
61  // Re-compile if floating point precision has been overridden.
62  property_get("debug.rs.precision", buf, "");
63  if (buf[0] != '\0') {
64    return true;
65  }
66
67  // Re-compile if debug.rs.forcerecompile is set.
68  property_get("debug.rs.forcerecompile", buf, "0");
69  if ((::strcmp(buf, "1") == 0) || (::strcmp(buf, "true") == 0)) {
70    return true;
71  } else {
72    return false;
73  }
74#endif  // RS_SERVER
75}
76
77static void setCompileArguments(std::vector<const char*>* args,
78                                const std::string& bcFileName,
79                                const char* cacheDir, const char* resName,
80                                const char* core_lib, bool useRSDebugContext,
81                                const char* bccPluginName, bool emitGlobalInfo,
82                                bool emitGlobalInfoSkipConstant) {
83    rsAssert(cacheDir && resName && core_lib);
84    args->push_back(android::renderscript::RsdCpuScriptImpl::BCC_EXE_PATH);
85    args->push_back("-unroll-runtime");
86    args->push_back("-scalarize-load-store");
87    if (emitGlobalInfo) {
88        args->push_back("-rs-global-info");
89        if (emitGlobalInfoSkipConstant) {
90            args->push_back("-rs-global-info-skip-constant");
91        }
92    }
93    args->push_back("-o");
94    args->push_back(resName);
95    args->push_back("-output_path");
96    args->push_back(cacheDir);
97    args->push_back("-bclib");
98    args->push_back(core_lib);
99    args->push_back("-mtriple");
100    args->push_back(DEFAULT_TARGET_TRIPLE_STRING);
101
102    // Enable workaround for A53 codegen by default.
103#if defined(__aarch64__) && !defined(DISABLE_A53_WORKAROUND)
104    args->push_back("-aarch64-fix-cortex-a53-835769");
105#endif
106
107    // Execute the bcc compiler.
108    if (useRSDebugContext) {
109        args->push_back("-rs-debug-ctx");
110    } else {
111        // Only load additional libraries for compiles that don't use
112        // the debug context.
113        if (bccPluginName && strlen(bccPluginName) > 0) {
114            args->push_back("-load");
115            args->push_back(bccPluginName);
116        }
117    }
118
119    args->push_back("-fPIC");
120    args->push_back("-embedRSInfo");
121
122    args->push_back(bcFileName.c_str());
123    args->push_back(nullptr);
124}
125
126static bool compileBitcode(const std::string &bcFileName,
127                           const char *bitcode,
128                           size_t bitcodeSize,
129                           std::vector<const char *> &compileArguments) {
130    rsAssert(bitcode && bitcodeSize);
131
132    FILE *bcfile = fopen(bcFileName.c_str(), "w");
133    if (!bcfile) {
134        ALOGE("Could not write to %s", bcFileName.c_str());
135        return false;
136    }
137    size_t nwritten = fwrite(bitcode, 1, bitcodeSize, bcfile);
138    fclose(bcfile);
139    if (nwritten != bitcodeSize) {
140        ALOGE("Could not write %zu bytes to %s", bitcodeSize,
141              bcFileName.c_str());
142        return false;
143    }
144
145    return android::renderscript::rsuExecuteCommand(
146                   android::renderscript::RsdCpuScriptImpl::BCC_EXE_PATH,
147                   compileArguments.size()-1, compileArguments.data());
148}
149
150// The checksum is unnecessary under a few conditions, since the primary
151// use-case for it is debugging. If we are loading something from the
152// system partition (read-only), we know that it was precompiled as part of
153// application ahead of time (and thus the checksum is completely
154// unnecessary). The checksum is also unnecessary on release (non-debug)
155// builds, as the only way to get a shared object is to have compiled the
156// script once already. On a release build, there is no way to adjust the
157// other libraries/dependencies, and so the only reason to recompile would
158// be for a source APK change or an OTA. In either case, the APK would be
159// reinstalled, which would already clear the code_cache/ directory.
160bool isChecksumNeeded(const char *cacheDir) {
161    if ((::strcmp(SYSLIBPATH, cacheDir) == 0) ||
162        (::strcmp(SYSLIBPATH_VENDOR, cacheDir) == 0))
163        return false;
164    char buf[PROPERTY_VALUE_MAX];
165    property_get("ro.debuggable", buf, "");
166    return (buf[0] == '1');
167}
168
169bool addFileToChecksum(const char *fileName, uint32_t &checksum) {
170    int FD = open(fileName, O_RDONLY);
171    if (FD == -1) {
172        ALOGE("Cannot open file \'%s\' to compute checksum", fileName);
173        return false;
174    }
175
176    char buf[256];
177    while (true) {
178        ssize_t nread = read(FD, buf, sizeof(buf));
179        if (nread < 0) { // bail out on failed read
180            ALOGE("Error while computing checksum for file \'%s\'", fileName);
181            return false;
182        }
183
184        checksum = adler32(checksum, (const unsigned char *) buf, nread);
185        if (static_cast<size_t>(nread) < sizeof(buf)) // EOF
186            break;
187    }
188
189    if (close(FD) != 0) {
190        ALOGE("Cannot close file \'%s\' after computing checksum", fileName);
191        return false;
192    }
193    return true;
194}
195
196#endif  // !defined(RS_COMPATIBILITY_LIB)
197}  // namespace
198
199namespace android {
200namespace renderscript {
201
202#ifndef RS_COMPATIBILITY_LIB
203
204uint32_t constructBuildChecksum(uint8_t const *bitcode, size_t bitcodeSize,
205                                const char *commandLine,
206                                const char** bccFiles, size_t numFiles) {
207    uint32_t checksum = adler32(0L, Z_NULL, 0);
208
209    // include checksum of bitcode
210    if (bitcode != nullptr && bitcodeSize > 0) {
211        checksum = adler32(checksum, bitcode, bitcodeSize);
212    }
213
214    // include checksum of command line arguments
215    checksum = adler32(checksum, (const unsigned char *) commandLine,
216                       strlen(commandLine));
217
218    // include checksum of bccFiles
219    for (size_t i = 0; i < numFiles; i++) {
220        const char* bccFile = bccFiles[i];
221        if (bccFile[0] != 0 && !addFileToChecksum(bccFile, checksum)) {
222            // return empty checksum instead of something partial/corrupt
223            return 0;
224        }
225    }
226
227    return checksum;
228}
229
230#endif  // !RS_COMPATIBILITY_LIB
231
232RsdCpuScriptImpl::RsdCpuScriptImpl(RsdCpuReferenceImpl *ctx, const Script *s) {
233    mCtx = ctx;
234    mScript = s;
235
236    mScriptSO = nullptr;
237
238    mRoot = nullptr;
239    mRootExpand = nullptr;
240    mInit = nullptr;
241    mFreeChildren = nullptr;
242    mScriptExec = nullptr;
243
244    mBoundAllocs = nullptr;
245    mIntrinsicData = nullptr;
246    mIsThreadable = true;
247
248    mBuildChecksum = 0;
249    mChecksumNeeded = false;
250}
251
252bool RsdCpuScriptImpl::storeRSInfoFromSO() {
253    // The shared object may have an invalid build checksum.
254    // Validate and fail early.
255    mScriptExec = ScriptExecutable::createFromSharedObject(
256            mCtx->getContext(), mScriptSO,
257            mChecksumNeeded ? mBuildChecksum : 0);
258
259    if (mScriptExec == nullptr) {
260        return false;
261    }
262
263    mRoot = (RootFunc_t) dlsym(mScriptSO, "root");
264    if (mRoot) {
265        //ALOGE("Found root(): %p", mRoot);
266    }
267    mRootExpand = (RootFunc_t) dlsym(mScriptSO, "root.expand");
268    if (mRootExpand) {
269        //ALOGE("Found root.expand(): %p", mRootExpand);
270    }
271    mInit = (InvokeFunc_t) dlsym(mScriptSO, "init");
272    if (mInit) {
273        //ALOGE("Found init(): %p", mInit);
274    }
275    mFreeChildren = (InvokeFunc_t) dlsym(mScriptSO, ".rs.dtor");
276    if (mFreeChildren) {
277        //ALOGE("Found .rs.dtor(): %p", mFreeChildren);
278    }
279
280    size_t varCount = mScriptExec->getExportedVariableCount();
281    if (varCount > 0) {
282        mBoundAllocs = new Allocation *[varCount];
283        memset(mBoundAllocs, 0, varCount * sizeof(*mBoundAllocs));
284    }
285
286    mIsThreadable = mScriptExec->getThreadable();
287    //ALOGE("Script isThreadable? %d", mIsThreadable);
288
289    if (kDebugGlobalVariables) {
290        mScriptExec->dumpGlobalInfo();
291    }
292
293    return true;
294}
295
296bool RsdCpuScriptImpl::init(char const *resName, char const *cacheDir,
297                            uint8_t const *bitcode, size_t bitcodeSize,
298                            uint32_t flags, char const *bccPluginName) {
299    //ALOGE("rsdScriptCreate %p %p %p %p %i %i %p", rsc, resName, cacheDir,
300    // bitcode, bitcodeSize, flags, lookupFunc);
301    //ALOGE("rsdScriptInit %p %p", rsc, script);
302
303    mCtx->lockMutex();
304#ifndef RS_COMPATIBILITY_LIB
305    bool useRSDebugContext = false;
306
307    bcinfo::MetadataExtractor bitcodeMetadata((const char *) bitcode, bitcodeSize);
308    if (!bitcodeMetadata.extract()) {
309        ALOGE("Could not extract metadata from bitcode");
310        mCtx->unlockMutex();
311        return false;
312    }
313
314    const char* core_lib = findCoreLib(bitcodeMetadata, (const char*)bitcode, bitcodeSize);
315
316    if (mCtx->getContext()->getContextType() == RS_CONTEXT_TYPE_DEBUG) {
317        useRSDebugContext = true;
318    }
319
320    std::string bcFileName(cacheDir);
321    bcFileName.append("/");
322    bcFileName.append(resName);
323    bcFileName.append(".bc");
324
325    std::vector<const char*> compileArguments;
326    bool emitGlobalInfo = mCtx->getEmbedGlobalInfo();
327    bool emitGlobalInfoSkipConstant = mCtx->getEmbedGlobalInfoSkipConstant();
328    setCompileArguments(&compileArguments, bcFileName, cacheDir, resName, core_lib,
329                        useRSDebugContext, bccPluginName, emitGlobalInfo,
330                        emitGlobalInfoSkipConstant);
331
332    mChecksumNeeded = isChecksumNeeded(cacheDir);
333    if (mChecksumNeeded) {
334        std::vector<const char *> bccFiles = { BCC_EXE_PATH,
335                                               core_lib,
336                                             };
337
338        // The last argument of compileArguments is a nullptr, so remove 1 from
339        // the size.
340        std::unique_ptr<const char> compileCommandLine(
341            rsuJoinStrings(compileArguments.size()-1, compileArguments.data()));
342
343        mBuildChecksum = constructBuildChecksum(bitcode, bitcodeSize,
344                                                compileCommandLine.get(),
345                                                bccFiles.data(), bccFiles.size());
346
347        if (mBuildChecksum == 0) {
348            // cannot compute checksum but verification is enabled
349            mCtx->unlockMutex();
350            return false;
351        }
352    }
353    else {
354        // add a dummy/constant as a checksum if verification is disabled
355        mBuildChecksum = 0xabadcafe;
356    }
357
358    // Append build checksum to commandline
359    // Handle the terminal nullptr in compileArguments
360    compileArguments.pop_back();
361    compileArguments.push_back("-build-checksum");
362    std::stringstream ss;
363    ss << std::hex << mBuildChecksum;
364    compileArguments.push_back(ss.str().c_str());
365    compileArguments.push_back(nullptr);
366
367    if (!is_force_recompile() && !useRSDebugContext) {
368        mScriptSO = SharedLibraryUtils::loadSharedLibrary(cacheDir, resName);
369
370        // Read RS info from the shared object to detect checksum mismatch
371        if (mScriptSO != nullptr && !storeRSInfoFromSO()) {
372            dlclose(mScriptSO);
373            mScriptSO = nullptr;
374        }
375    }
376
377    // If we can't, it's either not there or out of date.  We compile the bit code and try loading
378    // again.
379    if (mScriptSO == nullptr) {
380        if (!compileBitcode(bcFileName, (const char*)bitcode, bitcodeSize,
381                            compileArguments))
382        {
383            ALOGE("bcc: FAILS to compile '%s'", resName);
384            mCtx->unlockMutex();
385            return false;
386        }
387
388        if (!SharedLibraryUtils::createSharedLibrary(mCtx->getContext()->getDriverName(),
389                                                     cacheDir, resName)) {
390            ALOGE("Linker: Failed to link object file '%s'", resName);
391            mCtx->unlockMutex();
392            return false;
393        }
394
395        mScriptSO = SharedLibraryUtils::loadSharedLibrary(cacheDir, resName);
396        if (mScriptSO == nullptr) {
397            ALOGE("Unable to load '%s'", resName);
398            mCtx->unlockMutex();
399            return false;
400        }
401
402        // Read RS symbol information from the .so.
403        if (!storeRSInfoFromSO()) {
404            goto error;
405        }
406    }
407
408    mBitcodeFilePath.setTo(bcFileName.c_str());
409
410#else  // RS_COMPATIBILITY_LIB is defined
411    const char *nativeLibDir = mCtx->getContext()->getNativeLibDir();
412    mScriptSO = SharedLibraryUtils::loadSharedLibrary(cacheDir, resName, nativeLibDir);
413
414    if (!mScriptSO) {
415        goto error;
416    }
417
418    if (!storeRSInfoFromSO()) {
419        goto error;
420    }
421#endif
422    mCtx->unlockMutex();
423    return true;
424
425error:
426
427    mCtx->unlockMutex();
428    if (mScriptSO) {
429        dlclose(mScriptSO);
430        mScriptSO = nullptr;
431    }
432    return false;
433}
434
435#ifndef RS_COMPATIBILITY_LIB
436
437const char* RsdCpuScriptImpl::findCoreLib(const bcinfo::MetadataExtractor& ME, const char* bitcode,
438                                          size_t bitcodeSize) {
439    const char* defaultLib = SYSLIBPATH"/libclcore.bc";
440
441    // If we're debugging, use the debug library.
442    if (mCtx->getContext()->getContextType() == RS_CONTEXT_TYPE_DEBUG) {
443        return SYSLIBPATH"/libclcore_debug.bc";
444    }
445
446    // If a callback has been registered to specify a library, use that.
447    RSSelectRTCallback selectRTCallback = mCtx->getSelectRTCallback();
448    if (selectRTCallback != nullptr) {
449        return selectRTCallback((const char*)bitcode, bitcodeSize);
450    }
451
452    // Check for a platform specific library
453#if defined(ARCH_ARM_HAVE_NEON) && !defined(DISABLE_CLCORE_NEON)
454    enum bcinfo::RSFloatPrecision prec = ME.getRSFloatPrecision();
455    if (prec == bcinfo::RS_FP_Relaxed) {
456        // NEON-capable ARMv7a devices can use an accelerated math library
457        // for all reduced precision scripts.
458        // ARMv8 does not use NEON, as ASIMD can be used with all precision
459        // levels.
460        return SYSLIBPATH"/libclcore_neon.bc";
461    } else {
462        return defaultLib;
463    }
464#elif defined(__i386__) || defined(__x86_64__)
465    // x86 devices will use an optimized library.
466    return SYSLIBPATH"/libclcore_x86.bc";
467#else
468    return defaultLib;
469#endif
470}
471
472#endif
473
474void RsdCpuScriptImpl::populateScript(Script *script) {
475    // Copy info over to runtime
476    script->mHal.info.exportedFunctionCount = mScriptExec->getExportedFunctionCount();
477    script->mHal.info.exportedVariableCount = mScriptExec->getExportedVariableCount();
478    script->mHal.info.exportedPragmaCount = mScriptExec->getPragmaCount();;
479    script->mHal.info.exportedPragmaKeyList = mScriptExec->getPragmaKeys();
480    script->mHal.info.exportedPragmaValueList = mScriptExec->getPragmaValues();
481
482    // Bug, need to stash in metadata
483    if (mRootExpand) {
484        script->mHal.info.root = mRootExpand;
485    } else {
486        script->mHal.info.root = mRoot;
487    }
488}
489
490
491bool RsdCpuScriptImpl::forEachMtlsSetup(const Allocation ** ains,
492                                        uint32_t inLen,
493                                        Allocation * aout,
494                                        const void * usr, uint32_t usrLen,
495                                        const RsScriptCall *sc,
496                                        MTLaunchStruct *mtls) {
497
498    memset(mtls, 0, sizeof(MTLaunchStruct));
499
500    for (int index = inLen; --index >= 0;) {
501        const Allocation* ain = ains[index];
502
503        // possible for this to occur if IO_OUTPUT/IO_INPUT with no bound surface
504        if (ain != nullptr &&
505            (const uint8_t *)ain->mHal.drvState.lod[0].mallocPtr == nullptr) {
506
507            mCtx->getContext()->setError(RS_ERROR_BAD_SCRIPT,
508                                         "rsForEach called with null in allocations");
509            return false;
510        }
511    }
512
513    if (aout &&
514        (const uint8_t *)aout->mHal.drvState.lod[0].mallocPtr == nullptr) {
515
516        mCtx->getContext()->setError(RS_ERROR_BAD_SCRIPT,
517                                     "rsForEach called with null out allocations");
518        return false;
519    }
520
521    if (inLen > 0) {
522        const Allocation *ain0   = ains[0];
523        const Type       *inType = ain0->getType();
524
525        mtls->fep.dim.x = inType->getDimX();
526        mtls->fep.dim.y = inType->getDimY();
527        mtls->fep.dim.z = inType->getDimZ();
528
529        for (int Index = inLen; --Index >= 1;) {
530            if (!ain0->hasSameDims(ains[Index])) {
531                mCtx->getContext()->setError(RS_ERROR_BAD_SCRIPT,
532                  "Failed to launch kernel; dimensions of input and output"
533                  "allocations do not match.");
534
535                return false;
536            }
537        }
538
539    } else if (aout != nullptr) {
540        const Type *outType = aout->getType();
541
542        mtls->fep.dim.x = outType->getDimX();
543        mtls->fep.dim.y = outType->getDimY();
544        mtls->fep.dim.z = outType->getDimZ();
545
546    } else if (sc != nullptr) {
547        mtls->fep.dim.x = sc->xEnd;
548        mtls->fep.dim.y = sc->yEnd;
549        mtls->fep.dim.z = 0;
550    } else {
551        mCtx->getContext()->setError(RS_ERROR_BAD_SCRIPT,
552                                     "rsForEach called with null allocations");
553        return false;
554    }
555
556    if (inLen > 0 && aout != nullptr) {
557        if (!ains[0]->hasSameDims(aout)) {
558            mCtx->getContext()->setError(RS_ERROR_BAD_SCRIPT,
559              "Failed to launch kernel; dimensions of input and output allocations do not match.");
560
561            return false;
562        }
563    }
564
565    if (!sc || (sc->xEnd == 0)) {
566        mtls->end.x = mtls->fep.dim.x;
567    } else {
568        mtls->start.x = rsMin(mtls->fep.dim.x, sc->xStart);
569        mtls->end.x = rsMin(mtls->fep.dim.x, sc->xEnd);
570        if (mtls->start.x >= mtls->end.x) {
571            mCtx->getContext()->setError(RS_ERROR_BAD_SCRIPT,
572              "Failed to launch kernel; Invalid xStart or xEnd.");
573            return false;
574        }
575    }
576
577    if (!sc || (sc->yEnd == 0)) {
578        mtls->end.y = mtls->fep.dim.y;
579    } else {
580        mtls->start.y = rsMin(mtls->fep.dim.y, sc->yStart);
581        mtls->end.y = rsMin(mtls->fep.dim.y, sc->yEnd);
582        if (mtls->start.y >= mtls->end.y) {
583            mCtx->getContext()->setError(RS_ERROR_BAD_SCRIPT,
584              "Failed to launch kernel; Invalid yStart or yEnd.");
585            return false;
586        }
587    }
588
589    if (!sc || (sc->zEnd == 0)) {
590        mtls->end.z = mtls->fep.dim.z;
591    } else {
592        mtls->start.z = rsMin(mtls->fep.dim.z, sc->zStart);
593        mtls->end.z = rsMin(mtls->fep.dim.z, sc->zEnd);
594        if (mtls->start.z >= mtls->end.z) {
595            mCtx->getContext()->setError(RS_ERROR_BAD_SCRIPT,
596              "Failed to launch kernel; Invalid zStart or zEnd.");
597            return false;
598        }
599    }
600
601    if (!sc || (sc->arrayEnd == 0)) {
602        mtls->end.array[0] = mtls->fep.dim.array[0];
603    } else {
604        mtls->start.array[0] = rsMin(mtls->fep.dim.array[0], sc->arrayStart);
605        mtls->end.array[0] = rsMin(mtls->fep.dim.array[0], sc->arrayEnd);
606        if (mtls->start.array[0] >= mtls->end.array[0]) {
607            mCtx->getContext()->setError(RS_ERROR_BAD_SCRIPT,
608              "Failed to launch kernel; Invalid arrayStart or arrayEnd.");
609            return false;
610        }
611    }
612
613    if (!sc || (sc->array2End == 0)) {
614        mtls->end.array[1] = mtls->fep.dim.array[1];
615    } else {
616        mtls->start.array[1] = rsMin(mtls->fep.dim.array[1], sc->array2Start);
617        mtls->end.array[1] = rsMin(mtls->fep.dim.array[1], sc->array2End);
618        if (mtls->start.array[1] >= mtls->end.array[1]) {
619            mCtx->getContext()->setError(RS_ERROR_BAD_SCRIPT,
620              "Failed to launch kernel; Invalid array2Start or array2End.");
621            return false;
622        }
623    }
624
625    if (!sc || (sc->array3End == 0)) {
626        mtls->end.array[2] = mtls->fep.dim.array[2];
627    } else {
628        mtls->start.array[2] = rsMin(mtls->fep.dim.array[2], sc->array3Start);
629        mtls->end.array[2] = rsMin(mtls->fep.dim.array[2], sc->array3End);
630        if (mtls->start.array[2] >= mtls->end.array[2]) {
631            mCtx->getContext()->setError(RS_ERROR_BAD_SCRIPT,
632              "Failed to launch kernel; Invalid array3Start or array3End.");
633            return false;
634        }
635    }
636
637    if (!sc || (sc->array4End == 0)) {
638        mtls->end.array[3] = mtls->fep.dim.array[3];
639    } else {
640        mtls->start.array[3] = rsMin(mtls->fep.dim.array[3], sc->array4Start);
641        mtls->end.array[3] = rsMin(mtls->fep.dim.array[3], sc->array4End);
642        if (mtls->start.array[3] >= mtls->end.array[3]) {
643            mCtx->getContext()->setError(RS_ERROR_BAD_SCRIPT,
644              "Failed to launch kernel; Invalid array4Start or array4End.");
645            return false;
646        }
647    }
648
649
650    // The X & Y walkers always want 0-1 min even if dim is not present
651    mtls->end.x    = rsMax((uint32_t)1, mtls->end.x);
652    mtls->end.y    = rsMax((uint32_t)1, mtls->end.y);
653
654    mtls->rsc        = mCtx;
655    if (ains) {
656        memcpy(mtls->ains, ains, inLen * sizeof(ains[0]));
657    }
658    mtls->aout[0]    = aout;
659    mtls->fep.usr    = usr;
660    mtls->fep.usrLen = usrLen;
661    mtls->mSliceSize = 1;
662    mtls->mSliceNum  = 0;
663
664    mtls->isThreadable  = mIsThreadable;
665
666    if (inLen > 0) {
667        mtls->fep.inLen = inLen;
668        for (int index = inLen; --index >= 0;) {
669            mtls->fep.inPtr[index] = (const uint8_t*)ains[index]->mHal.drvState.lod[0].mallocPtr;
670            mtls->fep.inStride[index] = ains[index]->getType()->getElementSizeBytes();
671        }
672    }
673
674    if (aout != nullptr) {
675        mtls->fep.outPtr[0] = (uint8_t *)aout->mHal.drvState.lod[0].mallocPtr;
676        mtls->fep.outStride[0] = aout->getType()->getElementSizeBytes();
677    }
678
679    // All validation passed, ok to launch threads
680    return true;
681}
682
683
684void RsdCpuScriptImpl::invokeForEach(uint32_t slot,
685                                     const Allocation ** ains,
686                                     uint32_t inLen,
687                                     Allocation * aout,
688                                     const void * usr,
689                                     uint32_t usrLen,
690                                     const RsScriptCall *sc) {
691
692    MTLaunchStruct mtls;
693
694    if (forEachMtlsSetup(ains, inLen, aout, usr, usrLen, sc, &mtls)) {
695        forEachKernelSetup(slot, &mtls);
696
697        RsdCpuScriptImpl * oldTLS = mCtx->setTLS(this);
698        mCtx->launchThreads(ains, inLen, aout, sc, &mtls);
699        mCtx->setTLS(oldTLS);
700    }
701}
702
703void RsdCpuScriptImpl::forEachKernelSetup(uint32_t slot, MTLaunchStruct *mtls) {
704    mtls->script = this;
705    mtls->fep.slot = slot;
706    mtls->kernel = mScriptExec->getForEachFunction(slot);
707    rsAssert(mtls->kernel != nullptr);
708    mtls->sig = mScriptExec->getForEachSignature(slot);
709}
710
711int RsdCpuScriptImpl::invokeRoot() {
712    RsdCpuScriptImpl * oldTLS = mCtx->setTLS(this);
713    int ret = mRoot();
714    mCtx->setTLS(oldTLS);
715    return ret;
716}
717
718void RsdCpuScriptImpl::invokeInit() {
719    if (mInit) {
720        mInit();
721    }
722}
723
724void RsdCpuScriptImpl::invokeFreeChildren() {
725    if (mFreeChildren) {
726        mFreeChildren();
727    }
728}
729
730void RsdCpuScriptImpl::invokeFunction(uint32_t slot, const void *params,
731                                      size_t paramLength) {
732    //ALOGE("invoke %i %p %zu", slot, params, paramLength);
733    void * ap = nullptr;
734
735#if defined(__x86_64__)
736    // The invoked function could have input parameter of vector type for example float4 which
737    // requires void* params to be 16 bytes aligned when using SSE instructions for x86_64 platform.
738    // So try to align void* params before passing them into RS exported function.
739
740    if ((uint8_t)(uint64_t)params & 0x0F) {
741        if ((ap = (void*)memalign(16, paramLength)) != nullptr) {
742            memcpy(ap, params, paramLength);
743        } else {
744            ALOGE("x86_64: invokeFunction memalign error, still use params which"
745                  " is not 16 bytes aligned.");
746        }
747    }
748#endif
749
750    RsdCpuScriptImpl * oldTLS = mCtx->setTLS(this);
751    reinterpret_cast<void (*)(const void *, uint32_t)>(
752        mScriptExec->getInvokeFunction(slot))(ap? (const void *) ap: params, paramLength);
753
754#if defined(__x86_64__)
755    free(ap);
756#endif
757
758    mCtx->setTLS(oldTLS);
759}
760
761void RsdCpuScriptImpl::setGlobalVar(uint32_t slot, const void *data, size_t dataLength) {
762    //rsAssert(!script->mFieldIsObject[slot]);
763    //ALOGE("setGlobalVar %i %p %zu", slot, data, dataLength);
764
765    //if (mIntrinsicID) {
766        //mIntrinsicFuncs.setVar(dc, script, drv->mIntrinsicData, slot, data, dataLength);
767        //return;
768    //}
769
770    int32_t *destPtr = reinterpret_cast<int32_t *>(mScriptExec->getFieldAddress(slot));
771    if (!destPtr) {
772        //ALOGV("Calling setVar on slot = %i which is null", slot);
773        return;
774    }
775
776    memcpy(destPtr, data, dataLength);
777}
778
779void RsdCpuScriptImpl::getGlobalVar(uint32_t slot, void *data, size_t dataLength) {
780    //rsAssert(!script->mFieldIsObject[slot]);
781    //ALOGE("getGlobalVar %i %p %zu", slot, data, dataLength);
782
783    int32_t *srcPtr = reinterpret_cast<int32_t *>(mScriptExec->getFieldAddress(slot));
784    if (!srcPtr) {
785        //ALOGV("Calling setVar on slot = %i which is null", slot);
786        return;
787    }
788    memcpy(data, srcPtr, dataLength);
789}
790
791
792void RsdCpuScriptImpl::setGlobalVarWithElemDims(uint32_t slot, const void *data, size_t dataLength,
793                                                const Element *elem,
794                                                const uint32_t *dims, size_t dimLength) {
795    int32_t *destPtr = reinterpret_cast<int32_t *>(mScriptExec->getFieldAddress(slot));
796    if (!destPtr) {
797        //ALOGV("Calling setVar on slot = %i which is null", slot);
798        return;
799    }
800
801    // We want to look at dimension in terms of integer components,
802    // but dimLength is given in terms of bytes.
803    dimLength /= sizeof(int);
804
805    // Only a single dimension is currently supported.
806    rsAssert(dimLength == 1);
807    if (dimLength == 1) {
808        // First do the increment loop.
809        size_t stride = elem->getSizeBytes();
810        const char *cVal = reinterpret_cast<const char *>(data);
811        for (uint32_t i = 0; i < dims[0]; i++) {
812            elem->incRefs(cVal);
813            cVal += stride;
814        }
815
816        // Decrement loop comes after (to prevent race conditions).
817        char *oldVal = reinterpret_cast<char *>(destPtr);
818        for (uint32_t i = 0; i < dims[0]; i++) {
819            elem->decRefs(oldVal);
820            oldVal += stride;
821        }
822    }
823
824    memcpy(destPtr, data, dataLength);
825}
826
827void RsdCpuScriptImpl::setGlobalBind(uint32_t slot, Allocation *data) {
828
829    //rsAssert(!script->mFieldIsObject[slot]);
830    //ALOGE("setGlobalBind %i %p", slot, data);
831
832    int32_t *destPtr = reinterpret_cast<int32_t *>(mScriptExec->getFieldAddress(slot));
833    if (!destPtr) {
834        //ALOGV("Calling setVar on slot = %i which is null", slot);
835        return;
836    }
837
838    void *ptr = nullptr;
839    mBoundAllocs[slot] = data;
840    if (data) {
841        ptr = data->mHal.drvState.lod[0].mallocPtr;
842    }
843    memcpy(destPtr, &ptr, sizeof(void *));
844}
845
846void RsdCpuScriptImpl::setGlobalObj(uint32_t slot, ObjectBase *data) {
847
848    //rsAssert(script->mFieldIsObject[slot]);
849    //ALOGE("setGlobalObj %i %p", slot, data);
850
851    int32_t *destPtr = reinterpret_cast<int32_t *>(mScriptExec->getFieldAddress(slot));
852    if (!destPtr) {
853        //ALOGV("Calling setVar on slot = %i which is null", slot);
854        return;
855    }
856
857    rsrSetObject(mCtx->getContext(), (rs_object_base *)destPtr, data);
858}
859
860const char* RsdCpuScriptImpl::getFieldName(uint32_t slot) const {
861    return mScriptExec->getFieldName(slot);
862}
863
864RsdCpuScriptImpl::~RsdCpuScriptImpl() {
865    delete mScriptExec;
866    delete[] mBoundAllocs;
867    if (mScriptSO) {
868        dlclose(mScriptSO);
869    }
870}
871
872Allocation * RsdCpuScriptImpl::getAllocationForPointer(const void *ptr) const {
873    if (!ptr) {
874        return nullptr;
875    }
876
877    for (uint32_t ct=0; ct < mScript->mHal.info.exportedVariableCount; ct++) {
878        Allocation *a = mBoundAllocs[ct];
879        if (!a) continue;
880        if (a->mHal.drvState.lod[0].mallocPtr == ptr) {
881            return a;
882        }
883    }
884    ALOGE("rsGetAllocation, failed to find %p", ptr);
885    return nullptr;
886}
887
888int RsdCpuScriptImpl::getGlobalEntries() const {
889    return mScriptExec->getGlobalEntries();
890}
891
892const char * RsdCpuScriptImpl::getGlobalName(int i) const {
893    return mScriptExec->getGlobalName(i);
894}
895
896const void * RsdCpuScriptImpl::getGlobalAddress(int i) const {
897    return mScriptExec->getGlobalAddress(i);
898}
899
900size_t RsdCpuScriptImpl::getGlobalSize(int i) const {
901    return mScriptExec->getGlobalSize(i);
902}
903
904uint32_t RsdCpuScriptImpl::getGlobalProperties(int i) const {
905    return mScriptExec->getGlobalProperties(i);
906}
907
908void RsdCpuScriptImpl::preLaunch(uint32_t slot, const Allocation ** ains,
909                                 uint32_t inLen, Allocation * aout,
910                                 const void * usr, uint32_t usrLen,
911                                 const RsScriptCall *sc) {}
912
913void RsdCpuScriptImpl::postLaunch(uint32_t slot, const Allocation ** ains,
914                                  uint32_t inLen, Allocation * aout,
915                                  const void * usr, uint32_t usrLen,
916                                  const RsScriptCall *sc) {}
917
918
919}
920}
921