1#include "rsCpuScriptGroup2.h"
2
3#include <dlfcn.h>
4#include <stdio.h>
5#include <stdlib.h>
6#include <unistd.h>
7
8#include <set>
9#include <sstream>
10#include <string>
11#include <vector>
12
13#ifndef RS_COMPATIBILITY_LIB
14#include "bcc/Config/Config.h"
15#endif
16
17#include "cpu_ref/rsCpuCore.h"
18#include "rsClosure.h"
19#include "rsContext.h"
20#include "rsCpuCore.h"
21#include "rsCpuExecutable.h"
22#include "rsCpuScript.h"
23#include "rsScript.h"
24#include "rsScriptGroup2.h"
25#include "rsScriptIntrinsic.h"
26
27using std::string;
28using std::vector;
29
30namespace android {
31namespace renderscript {
32
33namespace {
34
35const size_t DefaultKernelArgCount = 2;
36
37void groupRoot(const RsExpandKernelDriverInfo *kinfo, uint32_t xstart,
38               uint32_t xend, uint32_t outstep) {
39    const List<CPUClosure*>& closures = *(List<CPUClosure*>*)kinfo->usr;
40    RsExpandKernelDriverInfo *mutable_kinfo = const_cast<RsExpandKernelDriverInfo *>(kinfo);
41
42    const size_t oldInLen = mutable_kinfo->inLen;
43
44    decltype(mutable_kinfo->inStride) oldInStride;
45    memcpy(&oldInStride, &mutable_kinfo->inStride, sizeof(oldInStride));
46
47    for (CPUClosure* cpuClosure : closures) {
48        const Closure* closure = cpuClosure->mClosure;
49
50        // There had better be enough space in mutable_kinfo
51        rsAssert(closure->mNumArg <= RS_KERNEL_INPUT_LIMIT);
52
53        for (size_t i = 0; i < closure->mNumArg; i++) {
54            const void* arg = closure->mArgs[i];
55            const Allocation* a = (const Allocation*)arg;
56            const uint32_t eStride = a->mHal.state.elementSizeBytes;
57            const uint8_t* ptr = (uint8_t*)(a->mHal.drvState.lod[0].mallocPtr) +
58                    eStride * xstart;
59            if (kinfo->dim.y > 1) {
60                ptr += a->mHal.drvState.lod[0].stride * kinfo->current.y;
61            }
62            mutable_kinfo->inPtr[i] = ptr;
63            mutable_kinfo->inStride[i] = eStride;
64        }
65        mutable_kinfo->inLen = closure->mNumArg;
66
67        const Allocation* out = closure->mReturnValue;
68        const uint32_t ostep = out->mHal.state.elementSizeBytes;
69        const uint8_t* ptr = (uint8_t *)(out->mHal.drvState.lod[0].mallocPtr) +
70                ostep * xstart;
71        if (kinfo->dim.y > 1) {
72            ptr += out->mHal.drvState.lod[0].stride * kinfo->current.y;
73        }
74
75        rsAssert(kinfo->outLen <= 1);
76        mutable_kinfo->outPtr[0] = const_cast<uint8_t*>(ptr);
77
78        // The implementation of an intrinsic relies on kinfo->usr being
79        // the "this" pointer to the intrinsic (an RsdCpuScriptIntrinsic object)
80        mutable_kinfo->usr = cpuClosure->mSi;
81
82        cpuClosure->mFunc(kinfo, xstart, xend, ostep);
83    }
84
85    mutable_kinfo->inLen = oldInLen;
86    mutable_kinfo->usr = &closures;
87    memcpy(&mutable_kinfo->inStride, &oldInStride, sizeof(oldInStride));
88}
89
90}  // namespace
91
92Batch::Batch(CpuScriptGroup2Impl* group, const char* name) :
93    mGroup(group), mFunc(nullptr) {
94    mName = strndup(name, strlen(name));
95}
96
97Batch::~Batch() {
98    for (CPUClosure* c : mClosures) {
99        delete c;
100    }
101    free(mName);
102}
103
104bool Batch::conflict(CPUClosure* cpuClosure) const {
105    if (mClosures.empty()) {
106        return false;
107    }
108
109    const Closure* closure = cpuClosure->mClosure;
110
111    if (!closure->mIsKernel || !mClosures.front()->mClosure->mIsKernel) {
112        // An invoke should be in a batch by itself, so it conflicts with any other
113        // closure.
114        return true;
115    }
116
117    const auto& globalDeps = closure->mGlobalDeps;
118    const auto& argDeps = closure->mArgDeps;
119
120    for (CPUClosure* c : mClosures) {
121        const Closure* batched = c->mClosure;
122        if (globalDeps.find(batched) != globalDeps.end()) {
123            return true;
124        }
125        const auto& it = argDeps.find(batched);
126        if (it != argDeps.end()) {
127            const auto& args = (*it).second;
128            for (const auto &p1 : *args) {
129                if (p1.second.get() != nullptr) {
130                    return true;
131                }
132            }
133        }
134    }
135
136    // The compiler fusion pass in bcc expects that kernels chained up through
137    // (1st) input and output.
138
139    const Closure* lastBatched = mClosures.back()->mClosure;
140    const auto& it = argDeps.find(lastBatched);
141
142    if (it == argDeps.end()) {
143        return true;
144    }
145
146    const auto& args = (*it).second;
147    for (const auto &p1 : *args) {
148        if (p1.first == 0 && p1.second.get() == nullptr) {
149            // The new closure depends on the last batched closure's return
150            // value (fieldId being nullptr) for its first argument (argument 0)
151            return false;
152        }
153    }
154
155    return true;
156}
157
158CpuScriptGroup2Impl::CpuScriptGroup2Impl(RsdCpuReferenceImpl *cpuRefImpl,
159                                         const ScriptGroupBase *sg) :
160    mCpuRefImpl(cpuRefImpl), mGroup((const ScriptGroup2*)(sg)),
161    mExecutable(nullptr), mScriptObj(nullptr) {
162    rsAssert(!mGroup->mClosures.empty());
163
164    mCpuRefImpl->lockMutex();
165    Batch* batch = new Batch(this, "Batch0");
166    int i = 0;
167    for (Closure* closure: mGroup->mClosures) {
168        CPUClosure* cc;
169        const IDBase* funcID = closure->mFunctionID.get();
170        RsdCpuScriptImpl* si =
171                (RsdCpuScriptImpl *)mCpuRefImpl->lookupScript(funcID->mScript);
172        if (closure->mIsKernel) {
173            MTLaunchStructForEach mtls;
174            si->forEachKernelSetup(funcID->mSlot, &mtls);
175            cc = new CPUClosure(closure, si, (ExpandFuncTy)mtls.kernel);
176        } else {
177            cc = new CPUClosure(closure, si);
178        }
179
180        if (batch->conflict(cc)) {
181            mBatches.push_back(batch);
182            std::stringstream ss;
183            ss << "Batch" << ++i;
184            batch = new Batch(this, ss.str().c_str());
185        }
186
187        batch->mClosures.push_back(cc);
188    }
189
190    rsAssert(!batch->mClosures.empty());
191    mBatches.push_back(batch);
192
193#ifndef RS_COMPATIBILITY_LIB
194    compile(mGroup->mCacheDir);
195    if (mScriptObj != nullptr && mExecutable != nullptr) {
196        for (Batch* batch : mBatches) {
197            batch->resolveFuncPtr(mScriptObj);
198        }
199    }
200#endif  // RS_COMPATIBILITY_LIB
201    mCpuRefImpl->unlockMutex();
202}
203
204void Batch::resolveFuncPtr(void* sharedObj) {
205    std::string funcName(mName);
206    if (mClosures.front()->mClosure->mIsKernel) {
207        funcName.append(".expand");
208    }
209    mFunc = dlsym(sharedObj, funcName.c_str());
210    rsAssert (mFunc != nullptr);
211}
212
213CpuScriptGroup2Impl::~CpuScriptGroup2Impl() {
214    for (Batch* batch : mBatches) {
215        delete batch;
216    }
217    delete mExecutable;
218    // TODO: move this dlclose into ~ScriptExecutable().
219    if (mScriptObj != nullptr) {
220        dlclose(mScriptObj);
221    }
222}
223
224namespace {
225
226#ifndef RS_COMPATIBILITY_LIB
227
228string getCoreLibPath(Context* context, string* coreLibRelaxedPath) {
229    *coreLibRelaxedPath = "";
230
231    // If we're debugging, use the debug library.
232    if (context->getContextType() == RS_CONTEXT_TYPE_DEBUG) {
233        return SYSLIBPATH"/libclcore_debug.bc";
234    }
235
236    // Check for a platform specific library
237
238#if defined(ARCH_ARM_HAVE_NEON) && !defined(DISABLE_CLCORE_NEON)
239    // NEON-capable ARMv7a devices can use an accelerated math library
240    // for all reduced precision scripts.
241    // ARMv8 does not use NEON, as ASIMD can be used with all precision
242    // levels.
243    *coreLibRelaxedPath = SYSLIBPATH"/libclcore_neon.bc";
244#endif
245
246#if defined(__i386__) || defined(__x86_64__)
247    // x86 devices will use an optimized library.
248    return SYSLIBPATH"/libclcore_x86.bc";
249#else
250    return SYSLIBPATH"/libclcore.bc";
251#endif
252}
253
254void setupCompileArguments(
255        const vector<const char*>& inputs, const vector<string>& kernelBatches,
256        const vector<string>& invokeBatches,
257        const char* outputDir, const char* outputFileName,
258        const char* coreLibPath, const char* coreLibRelaxedPath,
259        const bool emitGlobalInfo, const bool emitGlobalInfoSkipConstant,
260        int optLevel, vector<const char*>* args) {
261    args->push_back(RsdCpuScriptImpl::BCC_EXE_PATH);
262    args->push_back("-fPIC");
263    args->push_back("-embedRSInfo");
264    if (emitGlobalInfo) {
265        args->push_back("-rs-global-info");
266        if (emitGlobalInfoSkipConstant) {
267            args->push_back("-rs-global-info-skip-constant");
268        }
269    }
270    args->push_back("-mtriple");
271    args->push_back(DEFAULT_TARGET_TRIPLE_STRING);
272    args->push_back("-bclib");
273    args->push_back(coreLibPath);
274    args->push_back("-bclib_relaxed");
275    args->push_back(coreLibRelaxedPath);
276    for (const char* input : inputs) {
277        args->push_back(input);
278    }
279    for (const string& batch : kernelBatches) {
280        args->push_back("-merge");
281        args->push_back(batch.c_str());
282    }
283    for (const string& batch : invokeBatches) {
284        args->push_back("-invoke");
285        args->push_back(batch.c_str());
286    }
287    args->push_back("-output_path");
288    args->push_back(outputDir);
289
290    args->push_back("-O");
291    switch (optLevel) {
292    case 0:
293        args->push_back("0");
294        break;
295    case 3:
296        args->push_back("3");
297        break;
298    default:
299        ALOGW("Expected optimization level of 0 or 3. Received %d", optLevel);
300        args->push_back("3");
301        break;
302    }
303
304    // The output filename has to be the last, in case we need to pop it out and
305    // replace with a different name.
306    args->push_back("-o");
307    args->push_back(outputFileName);
308}
309
310void generateSourceSlot(RsdCpuReferenceImpl* ctxt,
311                        const Closure& closure,
312                        const std::vector<const char*>& inputs,
313                        std::stringstream& ss) {
314    const IDBase* funcID = (const IDBase*)closure.mFunctionID.get();
315    const Script* script = funcID->mScript;
316
317    rsAssert (!script->isIntrinsic());
318
319    const RsdCpuScriptImpl *cpuScript =
320            (const RsdCpuScriptImpl *)ctxt->lookupScript(script);
321    const string& bitcodeFilename = cpuScript->getBitcodeFilePath();
322
323    const int index = find(inputs.begin(), inputs.end(), bitcodeFilename) -
324            inputs.begin();
325
326    ss << index << "," << funcID->mSlot << ".";
327}
328
329#endif  // RS_COMPATIBILTY_LIB
330
331}  // anonymous namespace
332
333void CpuScriptGroup2Impl::compile(const char* cacheDir) {
334#ifndef RS_COMPATIBILITY_LIB
335    if (mGroup->mClosures.size() < 2) {
336        return;
337    }
338
339    auto comparator = [](const char* str1, const char* str2) -> bool {
340        return strcmp(str1, str2) < 0;
341    };
342    std::set<const char*, decltype(comparator)> inputSet(comparator);
343
344    for (Closure* closure : mGroup->mClosures) {
345        const Script* script = closure->mFunctionID.get()->mScript;
346
347        // If any script is an intrinsic, give up trying fusing the kernels.
348        if (script->isIntrinsic()) {
349            return;
350        }
351
352        const RsdCpuScriptImpl *cpuScript =
353            (const RsdCpuScriptImpl *)mCpuRefImpl->lookupScript(script);
354
355        const char* bitcodeFilename = cpuScript->getBitcodeFilePath();
356        inputSet.insert(bitcodeFilename);
357    }
358
359    std::vector<const char*> inputs(inputSet.begin(), inputSet.end());
360
361    std::vector<string> kernelBatches;
362    std::vector<string> invokeBatches;
363
364    int i = 0;
365    for (const auto& batch : mBatches) {
366        rsAssert(batch->size() > 0);
367
368        std::stringstream ss;
369        ss << batch->mName << ":";
370
371        if (!batch->mClosures.front()->mClosure->mIsKernel) {
372            rsAssert(batch->size() == 1);
373            generateSourceSlot(mCpuRefImpl, *batch->mClosures.front()->mClosure, inputs, ss);
374            invokeBatches.push_back(ss.str());
375        } else {
376            for (const auto& cpuClosure : batch->mClosures) {
377                generateSourceSlot(mCpuRefImpl, *cpuClosure->mClosure, inputs, ss);
378            }
379            kernelBatches.push_back(ss.str());
380        }
381    }
382
383    rsAssert(cacheDir != nullptr);
384    string objFilePath(cacheDir);
385    objFilePath.append("/");
386    objFilePath.append(mGroup->mName);
387    objFilePath.append(".o");
388
389    const char* resName = mGroup->mName;
390    string coreLibRelaxedPath;
391    const string& coreLibPath = getCoreLibPath(getCpuRefImpl()->getContext(),
392                                               &coreLibRelaxedPath);
393
394    int optLevel = getCpuRefImpl()->getContext()->getOptLevel();
395
396    vector<const char*> arguments;
397    bool emitGlobalInfo = getCpuRefImpl()->getEmbedGlobalInfo();
398    bool emitGlobalInfoSkipConstant = getCpuRefImpl()->getEmbedGlobalInfoSkipConstant();
399    setupCompileArguments(inputs, kernelBatches, invokeBatches, cacheDir,
400                          resName, coreLibPath.c_str(), coreLibRelaxedPath.c_str(),
401                          emitGlobalInfo, emitGlobalInfoSkipConstant,
402                          optLevel, &arguments);
403
404    std::unique_ptr<const char> cmdLine(rsuJoinStrings(arguments.size() - 1,
405                                                       arguments.data()));
406
407    inputs.push_back(coreLibPath.c_str());
408    inputs.push_back(coreLibRelaxedPath.c_str());
409
410    uint32_t checksum = constructBuildChecksum(nullptr, 0, cmdLine.get(),
411                                               inputs.data(), inputs.size());
412
413    if (checksum == 0) {
414        return;
415    }
416
417    std::stringstream ss;
418    ss << std::hex << checksum;
419    const char* checksumStr = ss.str().c_str();
420
421    //===--------------------------------------------------------------------===//
422    // Try to load a shared lib from code cache matching filename and checksum
423    //===--------------------------------------------------------------------===//
424
425    bool alreadyLoaded = false;
426    std::string cloneName;
427
428    mScriptObj = SharedLibraryUtils::loadSharedLibrary(cacheDir, resName, nullptr,
429                                                       &alreadyLoaded);
430    if (mScriptObj != nullptr) {
431        // A shared library named resName is found in code cache directory
432        // cacheDir, and loaded with the handle stored in mScriptObj.
433
434        mExecutable = ScriptExecutable::createFromSharedObject(
435            mScriptObj, checksum);
436
437        if (mExecutable != nullptr) {
438            // The loaded shared library in mScriptObj has a matching checksum.
439            // An executable object has been created.
440            return;
441        }
442
443        ALOGV("Failed to create an executable object from so file due to "
444              "mismatching checksum");
445
446        if (alreadyLoaded) {
447            // The shared object found in code cache has already been loaded.
448            // A different file name is needed for the new shared library, to
449            // avoid corrupting the currently loaded instance.
450
451            cloneName.append(resName);
452            cloneName.append("#");
453            cloneName.append(SharedLibraryUtils::getRandomString(6).string());
454
455            // The last element in arguments is the output filename.
456            arguments.pop_back();
457            arguments.push_back(cloneName.c_str());
458        }
459
460        dlclose(mScriptObj);
461        mScriptObj = nullptr;
462    }
463
464    //===--------------------------------------------------------------------===//
465    // Fuse the input kernels and generate native code in an object file
466    //===--------------------------------------------------------------------===//
467
468    arguments.push_back("-build-checksum");
469    arguments.push_back(checksumStr);
470    arguments.push_back(nullptr);
471
472    bool compiled = rsuExecuteCommand(RsdCpuScriptImpl::BCC_EXE_PATH,
473                                      arguments.size()-1,
474                                      arguments.data());
475    if (!compiled) {
476        return;
477    }
478
479    //===--------------------------------------------------------------------===//
480    // Create and load the shared lib
481    //===--------------------------------------------------------------------===//
482
483    if (!SharedLibraryUtils::createSharedLibrary(
484            getCpuRefImpl()->getContext()->getDriverName(), cacheDir, resName)) {
485        ALOGE("Failed to link object file '%s'", resName);
486        unlink(objFilePath.c_str());
487        return;
488    }
489
490    unlink(objFilePath.c_str());
491
492    mScriptObj = SharedLibraryUtils::loadSharedLibrary(cacheDir, resName);
493    if (mScriptObj == nullptr) {
494        ALOGE("Unable to load '%s'", resName);
495        return;
496    }
497
498    if (alreadyLoaded) {
499        // Delete the temporary, random-named file that we created to avoid
500        // interfering with an already loaded shared library.
501        string cloneFilePath(cacheDir);
502        cloneFilePath.append("/");
503        cloneFilePath.append(cloneName.c_str());
504        cloneFilePath.append(".so");
505        unlink(cloneFilePath.c_str());
506    }
507
508    mExecutable = ScriptExecutable::createFromSharedObject(mScriptObj);
509
510#endif  // RS_COMPATIBILITY_LIB
511}
512
513void CpuScriptGroup2Impl::execute() {
514    for (auto batch : mBatches) {
515        batch->setGlobalsForBatch();
516        batch->run();
517    }
518}
519
520void Batch::setGlobalsForBatch() {
521    for (CPUClosure* cpuClosure : mClosures) {
522        const Closure* closure = cpuClosure->mClosure;
523        const IDBase* funcID = closure->mFunctionID.get();
524        Script* s = funcID->mScript;;
525        for (const auto& p : closure->mGlobals) {
526            const int64_t value = p.second.first;
527            int size = p.second.second;
528            if (value == 0 && size == 0) {
529                // This indicates the current closure depends on another closure for a
530                // global in their shared module (script). In this case we don't need to
531                // copy the value. For example, an invoke intializes a global variable
532                // which a kernel later reads.
533                continue;
534            }
535            rsAssert(p.first != nullptr);
536            Script* script = p.first->mScript;
537            rsAssert(script == s);
538            RsdCpuReferenceImpl* ctxt = mGroup->getCpuRefImpl();
539            const RsdCpuScriptImpl *cpuScript =
540                    (const RsdCpuScriptImpl *)ctxt->lookupScript(script);
541            int slot = p.first->mSlot;
542            ScriptExecutable* exec = mGroup->getExecutable();
543            if (exec != nullptr) {
544                const char* varName = cpuScript->getFieldName(slot);
545                void* addr = exec->getFieldAddress(varName);
546                if (size < 0) {
547                    rsrSetObject(mGroup->getCpuRefImpl()->getContext(),
548                                 (rs_object_base*)addr, (ObjectBase*)value);
549                } else {
550                    memcpy(addr, (const void*)&value, size);
551                }
552            } else {
553                // We use -1 size to indicate an ObjectBase rather than a primitive type
554                if (size < 0) {
555                    s->setVarObj(slot, (ObjectBase*)value);
556                } else {
557                    s->setVar(slot, (const void*)&value, size);
558                }
559            }
560        }
561    }
562}
563
564void Batch::run() {
565    if (!mClosures.front()->mClosure->mIsKernel) {
566        rsAssert(mClosures.size() == 1);
567
568        // This batch contains a single closure for an invoke function
569        CPUClosure* cc = mClosures.front();
570        const Closure* c = cc->mClosure;
571
572        if (mFunc != nullptr) {
573            // TODO: Need align pointers for x86_64.
574            // See RsdCpuScriptImpl::invokeFunction in rsCpuScript.cpp
575            ((InvokeFuncTy)mFunc)(c->mParams, c->mParamLength);
576        } else {
577            const ScriptInvokeID* invokeID = (const ScriptInvokeID*)c->mFunctionID.get();
578            rsAssert(invokeID != nullptr);
579            cc->mSi->invokeFunction(invokeID->mSlot, c->mParams, c->mParamLength);
580        }
581
582        return;
583    }
584
585    if (mFunc != nullptr) {
586        MTLaunchStructForEach mtls;
587        const CPUClosure* firstCpuClosure = mClosures.front();
588        const CPUClosure* lastCpuClosure = mClosures.back();
589
590        firstCpuClosure->mSi->forEachMtlsSetup(
591                (const Allocation**)firstCpuClosure->mClosure->mArgs,
592                firstCpuClosure->mClosure->mNumArg,
593                lastCpuClosure->mClosure->mReturnValue,
594                nullptr, 0, nullptr, &mtls);
595
596        mtls.script = nullptr;
597        mtls.fep.usr = nullptr;
598        mtls.kernel = (ForEachFunc_t)mFunc;
599
600        mGroup->getCpuRefImpl()->launchForEach(
601                (const Allocation**)firstCpuClosure->mClosure->mArgs,
602                firstCpuClosure->mClosure->mNumArg,
603                lastCpuClosure->mClosure->mReturnValue,
604                nullptr, &mtls);
605
606        return;
607    }
608
609    for (CPUClosure* cpuClosure : mClosures) {
610        const Closure* closure = cpuClosure->mClosure;
611        const ScriptKernelID* kernelID =
612                (const ScriptKernelID*)closure->mFunctionID.get();
613        cpuClosure->mSi->preLaunch(kernelID->mSlot,
614                                   (const Allocation**)closure->mArgs,
615                                   closure->mNumArg, closure->mReturnValue,
616                                   nullptr, 0, nullptr);
617    }
618
619    const CPUClosure* cpuClosure = mClosures.front();
620    const Closure* closure = cpuClosure->mClosure;
621    MTLaunchStructForEach mtls;
622
623    if (cpuClosure->mSi->forEachMtlsSetup((const Allocation**)closure->mArgs,
624                                          closure->mNumArg,
625                                          closure->mReturnValue,
626                                          nullptr, 0, nullptr, &mtls)) {
627
628        mtls.script = nullptr;
629        mtls.kernel = &groupRoot;
630        mtls.fep.usr = &mClosures;
631
632        mGroup->getCpuRefImpl()->launchForEach(nullptr, 0, nullptr, nullptr, &mtls);
633    }
634
635    for (CPUClosure* cpuClosure : mClosures) {
636        const Closure* closure = cpuClosure->mClosure;
637        const ScriptKernelID* kernelID =
638                (const ScriptKernelID*)closure->mFunctionID.get();
639        cpuClosure->mSi->postLaunch(kernelID->mSlot,
640                                    (const Allocation**)closure->mArgs,
641                                    closure->mNumArg, closure->mReturnValue,
642                                    nullptr, 0, nullptr);
643    }
644}
645
646}  // namespace renderscript
647}  // namespace android
648