rsCpuScriptGroup2.cpp revision 1c20667f7a174a7c0a1599d34a40c524fe24c615 
1#include "rsCpuScriptGroup2.h"
2
3#include <dlfcn.h>
4#include <stdio.h>
5#include <stdlib.h>
6#include <unistd.h>
7
8#include <set>
9#include <sstream>
10#include <string>
11#include <vector>
12
13#ifndef RS_COMPATIBILITY_LIB
14#include "bcc/Config/Config.h"
15#endif
16
17#include "cpu_ref/rsCpuCore.h"
18#include "rsClosure.h"
19#include "rsContext.h"
20#include "rsCpuCore.h"
21#include "rsCpuExecutable.h"
22#include "rsCpuScript.h"
23#include "rsScript.h"
24#include "rsScriptGroup2.h"
25#include "rsScriptIntrinsic.h"
26
27using std::string;
28using std::vector;
29
30namespace android {
31namespace renderscript {
32
33namespace {
34
35const size_t DefaultKernelArgCount = 2;
36
37void groupRoot(const RsExpandKernelDriverInfo *kinfo, uint32_t xstart,
38               uint32_t xend, uint32_t outstep) {
39    const List<CPUClosure*>& closures = *(List<CPUClosure*>*)kinfo->usr;
40    RsExpandKernelDriverInfo *mutable_kinfo = const_cast<RsExpandKernelDriverInfo *>(kinfo);
41
42    const size_t oldInLen = mutable_kinfo->inLen;
43
44    decltype(mutable_kinfo->inStride) oldInStride;
45    memcpy(&oldInStride, &mutable_kinfo->inStride, sizeof(oldInStride));
46
47    for (CPUClosure* cpuClosure : closures) {
48        const Closure* closure = cpuClosure->mClosure;
49
50        // There had better be enough space in mutable_kinfo
51        rsAssert(closure->mNumArg <= RS_KERNEL_INPUT_LIMIT);
52
53        for (size_t i = 0; i < closure->mNumArg; i++) {
54            const void* arg = closure->mArgs[i];
55            const Allocation* a = (const Allocation*)arg;
56            const uint32_t eStride = a->mHal.state.elementSizeBytes;
57            const uint8_t* ptr = (uint8_t*)(a->mHal.drvState.lod[0].mallocPtr) +
58                    eStride * xstart;
59            if (kinfo->dim.y > 1) {
60                ptr += a->mHal.drvState.lod[0].stride * kinfo->current.y;
61            }
62            mutable_kinfo->inPtr[i] = ptr;
63            mutable_kinfo->inStride[i] = eStride;
64        }
65        mutable_kinfo->inLen = closure->mNumArg;
66
67        const Allocation* out = closure->mReturnValue;
68        const uint32_t ostep = out->mHal.state.elementSizeBytes;
69        const uint8_t* ptr = (uint8_t *)(out->mHal.drvState.lod[0].mallocPtr) +
70                ostep * xstart;
71        if (kinfo->dim.y > 1) {
72            ptr += out->mHal.drvState.lod[0].stride * kinfo->current.y;
73        }
74
75        rsAssert(kinfo->outLen <= 1);
76        mutable_kinfo->outPtr[0] = const_cast<uint8_t*>(ptr);
77
78        cpuClosure->mFunc(kinfo, xstart, xend, ostep);
79    }
80
81    mutable_kinfo->inLen = oldInLen;
82    memcpy(&mutable_kinfo->inStride, &oldInStride, sizeof(oldInStride));
83}
84
85}  // namespace
86
87Batch::Batch(CpuScriptGroup2Impl* group, const char* name) :
88    mGroup(group), mFunc(nullptr) {
89    mName = strndup(name, strlen(name));
90}
91
92Batch::~Batch() {
93    for (CPUClosure* c : mClosures) {
94        delete c;
95    }
96    free(mName);
97}
98
99bool Batch::conflict(CPUClosure* cpuClosure) const {
100    if (mClosures.empty()) {
101        return false;
102    }
103
104    const Closure* closure = cpuClosure->mClosure;
105
106    if (!closure->mIsKernel || !mClosures.front()->mClosure->mIsKernel) {
107        // An invoke should be in a batch by itself, so it conflicts with any other
108        // closure.
109        return true;
110    }
111
112    const auto& globalDeps = closure->mGlobalDeps;
113    const auto& argDeps = closure->mArgDeps;
114
115    for (CPUClosure* c : mClosures) {
116        const Closure* batched = c->mClosure;
117        if (globalDeps.find(batched) != globalDeps.end()) {
118            return true;
119        }
120        const auto& it = argDeps.find(batched);
121        if (it != argDeps.end()) {
122            const auto& args = (*it).second;
123            for (const auto &p1 : *args) {
124                if (p1.second.get() != nullptr) {
125                    return true;
126                }
127            }
128        }
129    }
130
131    // The compiler fusion pass in bcc expects that kernels chained up through
132    // (1st) input and output.
133
134    const Closure* lastBatched = mClosures.back()->mClosure;
135    const auto& it = argDeps.find(lastBatched);
136
137    if (it == argDeps.end()) {
138        return true;
139    }
140
141    const auto& args = (*it).second;
142    for (const auto &p1 : *args) {
143        if (p1.first == 0 && p1.second.get() == nullptr) {
144            // The new closure depends on the last batched closure's return
145            // value (fieldId being nullptr) for its first argument (argument 0)
146            return false;
147        }
148    }
149
150    return true;
151}
152
153CpuScriptGroup2Impl::CpuScriptGroup2Impl(RsdCpuReferenceImpl *cpuRefImpl,
154                                         const ScriptGroupBase *sg) :
155    mCpuRefImpl(cpuRefImpl), mGroup((const ScriptGroup2*)(sg)),
156    mExecutable(nullptr), mScriptObj(nullptr) {
157    rsAssert(!mGroup->mClosures.empty());
158
159    Batch* batch = new Batch(this, "Batch0");
160    int i = 0;
161    for (Closure* closure: mGroup->mClosures) {
162        CPUClosure* cc;
163        const IDBase* funcID = closure->mFunctionID.get();
164        RsdCpuScriptImpl* si =
165                (RsdCpuScriptImpl *)mCpuRefImpl->lookupScript(funcID->mScript);
166        if (closure->mIsKernel) {
167            MTLaunchStruct mtls;
168            si->forEachKernelSetup(funcID->mSlot, &mtls);
169            cc = new CPUClosure(closure, si, (ExpandFuncTy)mtls.kernel);
170        } else {
171            cc = new CPUClosure(closure, si);
172        }
173
174        if (batch->conflict(cc)) {
175            mBatches.push_back(batch);
176            std::stringstream ss;
177            ss << "Batch" << ++i;
178            batch = new Batch(this, ss.str().c_str());
179        }
180
181        batch->mClosures.push_back(cc);
182    }
183
184    rsAssert(!batch->mClosures.empty());
185    mBatches.push_back(batch);
186
187#ifndef RS_COMPATIBILITY_LIB
188    compile(mGroup->mCacheDir);
189    if (mScriptObj != nullptr && mExecutable != nullptr) {
190        for (Batch* batch : mBatches) {
191            batch->resolveFuncPtr(mScriptObj);
192        }
193    }
194#endif  // RS_COMPATIBILITY_LIB
195}
196
197void Batch::resolveFuncPtr(void* sharedObj) {
198    std::string funcName(mName);
199    if (mClosures.front()->mClosure->mIsKernel) {
200        funcName.append(".expand");
201    }
202    mFunc = dlsym(sharedObj, funcName.c_str());
203    rsAssert (mFunc != nullptr);
204}
205
206CpuScriptGroup2Impl::~CpuScriptGroup2Impl() {
207    for (Batch* batch : mBatches) {
208        delete batch;
209    }
210    delete mExecutable;
211    // TODO: move this dlclose into ~ScriptExecutable().
212    if (mScriptObj != nullptr) {
213        dlclose(mScriptObj);
214    }
215}
216
217namespace {
218
219#ifndef RS_COMPATIBILITY_LIB
220
221string getCoreLibPath(Context* context, string* coreLibRelaxedPath) {
222    *coreLibRelaxedPath = "";
223
224    // If we're debugging, use the debug library.
225    if (context->getContextType() == RS_CONTEXT_TYPE_DEBUG) {
226        return SYSLIBPATH"/libclcore_debug.bc";
227    }
228
229    // Check for a platform specific library
230
231#if defined(ARCH_ARM_HAVE_NEON) && !defined(DISABLE_CLCORE_NEON)
232    // NEON-capable ARMv7a devices can use an accelerated math library
233    // for all reduced precision scripts.
234    // ARMv8 does not use NEON, as ASIMD can be used with all precision
235    // levels.
236    *coreLibRelaxedPath = SYSLIBPATH"/libclcore_neon.bc";
237#endif
238
239#if defined(__i386__) || defined(__x86_64__)
240    // x86 devices will use an optimized library.
241    return SYSLIBPATH"/libclcore_x86.bc";
242#else
243    return SYSLIBPATH"/libclcore.bc";
244#endif
245}
246
247void setupCompileArguments(
248        const vector<const char*>& inputs, const vector<string>& kernelBatches,
249        const vector<string>& invokeBatches,
250        const char* outputDir, const char* outputFileName,
251        const char* coreLibPath, const char* coreLibRelaxedPath,
252        const bool emitGlobalInfo, const bool emitGlobalInfoSkipConstant,
253        vector<const char*>* args) {
254    args->push_back(RsdCpuScriptImpl::BCC_EXE_PATH);
255    args->push_back("-fPIC");
256    args->push_back("-embedRSInfo");
257    if (emitGlobalInfo) {
258        args->push_back("-rs-global-info");
259        if (emitGlobalInfoSkipConstant) {
260            args->push_back("-rs-global-info-skip-constant");
261        }
262    }
263    args->push_back("-mtriple");
264    args->push_back(DEFAULT_TARGET_TRIPLE_STRING);
265    args->push_back("-bclib");
266    args->push_back(coreLibPath);
267    args->push_back("-bclib_relaxed");
268    args->push_back(coreLibRelaxedPath);
269    for (const char* input : inputs) {
270        args->push_back(input);
271    }
272    for (const string& batch : kernelBatches) {
273        args->push_back("-merge");
274        args->push_back(batch.c_str());
275    }
276    for (const string& batch : invokeBatches) {
277        args->push_back("-invoke");
278        args->push_back(batch.c_str());
279    }
280    args->push_back("-output_path");
281    args->push_back(outputDir);
282    args->push_back("-o");
283    args->push_back(outputFileName);
284}
285
286void generateSourceSlot(RsdCpuReferenceImpl* ctxt,
287                        const Closure& closure,
288                        const std::vector<const char*>& inputs,
289                        std::stringstream& ss) {
290    const IDBase* funcID = (const IDBase*)closure.mFunctionID.get();
291    const Script* script = funcID->mScript;
292
293    rsAssert (!script->isIntrinsic());
294
295    const RsdCpuScriptImpl *cpuScript =
296            (const RsdCpuScriptImpl *)ctxt->lookupScript(script);
297    const string& bitcodeFilename = cpuScript->getBitcodeFilePath();
298
299    const int index = find(inputs.begin(), inputs.end(), bitcodeFilename) -
300            inputs.begin();
301
302    ss << index << "," << funcID->mSlot << ".";
303}
304
305#endif  // RS_COMPATIBILTY_LIB
306
307}  // anonymous namespace
308
309void CpuScriptGroup2Impl::compile(const char* cacheDir) {
310#ifndef RS_COMPATIBILITY_LIB
311    if (mGroup->mClosures.size() < 2) {
312        return;
313    }
314
315    auto comparator = [](const char* str1, const char* str2) -> bool {
316        return strcmp(str1, str2) < 0;
317    };
318    std::set<const char*, decltype(comparator)> inputSet(comparator);
319
320    for (Closure* closure : mGroup->mClosures) {
321        const Script* script = closure->mFunctionID.get()->mScript;
322
323        // If any script is an intrinsic, give up trying fusing the kernels.
324        if (script->isIntrinsic()) {
325            return;
326        }
327
328        const RsdCpuScriptImpl *cpuScript =
329            (const RsdCpuScriptImpl *)mCpuRefImpl->lookupScript(script);
330
331        const char* bitcodeFilename = cpuScript->getBitcodeFilePath();
332        inputSet.insert(bitcodeFilename);
333    }
334
335    std::vector<const char*> inputs(inputSet.begin(), inputSet.end());
336
337    std::vector<string> kernelBatches;
338    std::vector<string> invokeBatches;
339
340    int i = 0;
341    for (const auto& batch : mBatches) {
342        rsAssert(batch->size() > 0);
343
344        std::stringstream ss;
345        ss << batch->mName << ":";
346
347        if (!batch->mClosures.front()->mClosure->mIsKernel) {
348            rsAssert(batch->size() == 1);
349            generateSourceSlot(mCpuRefImpl, *batch->mClosures.front()->mClosure, inputs, ss);
350            invokeBatches.push_back(ss.str());
351        } else {
352            for (const auto& cpuClosure : batch->mClosures) {
353                generateSourceSlot(mCpuRefImpl, *cpuClosure->mClosure, inputs, ss);
354            }
355            kernelBatches.push_back(ss.str());
356        }
357    }
358
359    rsAssert(cacheDir != nullptr);
360    string objFilePath(cacheDir);
361    objFilePath.append("/");
362    objFilePath.append(mGroup->mName);
363    objFilePath.append(".o");
364
365    const char* resName = mGroup->mName;
366    string coreLibRelaxedPath;
367    const string& coreLibPath = getCoreLibPath(getCpuRefImpl()->getContext(),
368                                               &coreLibRelaxedPath);
369
370    vector<const char*> arguments;
371    bool emitGlobalInfo = getCpuRefImpl()->getEmbedGlobalInfo();
372    bool emitGlobalInfoSkipConstant = getCpuRefImpl()->getEmbedGlobalInfoSkipConstant();
373    setupCompileArguments(inputs, kernelBatches, invokeBatches, cacheDir,
374                          resName, coreLibPath.c_str(), coreLibRelaxedPath.c_str(),
375                          emitGlobalInfo, emitGlobalInfoSkipConstant,
376                          &arguments);
377
378    std::unique_ptr<const char> cmdLine(rsuJoinStrings(arguments.size() - 1,
379                                                       arguments.data()));
380
381    inputs.push_back(coreLibPath.c_str());
382    inputs.push_back(coreLibRelaxedPath.c_str());
383
384    uint32_t checksum = constructBuildChecksum(nullptr, 0, cmdLine.get(),
385                                               inputs.data(), inputs.size());
386
387    if (checksum == 0) {
388        return;
389    }
390
391    std::stringstream ss;
392    ss << std::hex << checksum;
393    const char* checksumStr = ss.str().c_str();
394
395    //===--------------------------------------------------------------------===//
396    // Try to load a shared lib from code cache matching filename and checksum
397    //===--------------------------------------------------------------------===//
398
399    mScriptObj = SharedLibraryUtils::loadSharedLibrary(cacheDir, resName);
400    if (mScriptObj != nullptr) {
401        mExecutable = ScriptExecutable::createFromSharedObject(
402            getCpuRefImpl()->getContext(), mScriptObj, checksum);
403        if (mExecutable != nullptr) {
404            return;
405        } else {
406            ALOGE("Failed to create an executable object from so file");
407        }
408        dlclose(mScriptObj);
409        mScriptObj = nullptr;
410    }
411
412    //===--------------------------------------------------------------------===//
413    // Fuse the input kernels and generate native code in an object file
414    //===--------------------------------------------------------------------===//
415
416    arguments.push_back("-build-checksum");
417    arguments.push_back(checksumStr);
418    arguments.push_back(nullptr);
419
420    bool compiled = rsuExecuteCommand(RsdCpuScriptImpl::BCC_EXE_PATH,
421                                      arguments.size()-1,
422                                      arguments.data());
423    if (!compiled) {
424        return;
425    }
426
427    //===--------------------------------------------------------------------===//
428    // Create and load the shared lib
429    //===--------------------------------------------------------------------===//
430
431    if (!SharedLibraryUtils::createSharedLibrary(
432            getCpuRefImpl()->getContext()->getDriverName(), cacheDir, resName)) {
433        ALOGE("Failed to link object file '%s'", resName);
434        unlink(objFilePath.c_str());
435        return;
436    }
437
438    unlink(objFilePath.c_str());
439
440    mScriptObj = SharedLibraryUtils::loadSharedLibrary(cacheDir, resName);
441    if (mScriptObj == nullptr) {
442        ALOGE("Unable to load '%s'", resName);
443        return;
444    }
445
446    mExecutable = ScriptExecutable::createFromSharedObject(
447        getCpuRefImpl()->getContext(),
448        mScriptObj);
449
450#endif  // RS_COMPATIBILITY_LIB
451}
452
453void CpuScriptGroup2Impl::execute() {
454    for (auto batch : mBatches) {
455        batch->setGlobalsForBatch();
456        batch->run();
457    }
458}
459
460void Batch::setGlobalsForBatch() {
461    for (CPUClosure* cpuClosure : mClosures) {
462        const Closure* closure = cpuClosure->mClosure;
463        const IDBase* funcID = closure->mFunctionID.get();
464        Script* s = funcID->mScript;;
465        for (const auto& p : closure->mGlobals) {
466            const void* value = p.second.first;
467            int size = p.second.second;
468            if (value == nullptr && size == 0) {
469                // This indicates the current closure depends on another closure for a
470                // global in their shared module (script). In this case we don't need to
471                // copy the value. For example, an invoke intializes a global variable
472                // which a kernel later reads.
473                continue;
474            }
475            rsAssert(p.first != nullptr);
476            Script* script = p.first->mScript;
477            RsdCpuReferenceImpl* ctxt = mGroup->getCpuRefImpl();
478            const RsdCpuScriptImpl *cpuScript =
479                    (const RsdCpuScriptImpl *)ctxt->lookupScript(script);
480            int slot = p.first->mSlot;
481            ScriptExecutable* exec = mGroup->getExecutable();
482            if (exec != nullptr) {
483                const char* varName = cpuScript->getFieldName(slot);
484                void* addr = exec->getFieldAddress(varName);
485                if (size < 0) {
486                    rsrSetObject(mGroup->getCpuRefImpl()->getContext(),
487                                 (rs_object_base*)addr, (ObjectBase*)value);
488                } else {
489                    memcpy(addr, (const void*)&value, size);
490                }
491            } else {
492                // We use -1 size to indicate an ObjectBase rather than a primitive type
493                if (size < 0) {
494                    s->setVarObj(slot, (ObjectBase*)value);
495                } else {
496                    s->setVar(slot, (const void*)&value, size);
497                }
498            }
499        }
500    }
501}
502
503void Batch::run() {
504    if (!mClosures.front()->mClosure->mIsKernel) {
505        rsAssert(mClosures.size() == 1);
506
507        // This batch contains a single closure for an invoke function
508        CPUClosure* cc = mClosures.front();
509        const Closure* c = cc->mClosure;
510
511        if (mFunc != nullptr) {
512            // TODO: Need align pointers for x86_64.
513            // See RsdCpuScriptImpl::invokeFunction in rsCpuScript.cpp
514            ((InvokeFuncTy)mFunc)(c->mParams, c->mParamLength);
515        } else {
516            const ScriptInvokeID* invokeID = (const ScriptInvokeID*)c->mFunctionID.get();
517            rsAssert(invokeID != nullptr);
518            cc->mSi->invokeFunction(invokeID->mSlot, c->mParams, c->mParamLength);
519        }
520
521        return;
522    }
523
524    if (mFunc != nullptr) {
525        MTLaunchStruct mtls;
526        const CPUClosure* firstCpuClosure = mClosures.front();
527        const CPUClosure* lastCpuClosure = mClosures.back();
528
529        firstCpuClosure->mSi->forEachMtlsSetup(
530                (const Allocation**)firstCpuClosure->mClosure->mArgs,
531                firstCpuClosure->mClosure->mNumArg,
532                lastCpuClosure->mClosure->mReturnValue,
533                nullptr, 0, nullptr, &mtls);
534
535        mtls.script = nullptr;
536        mtls.fep.usr = nullptr;
537        mtls.kernel = (ForEachFunc_t)mFunc;
538
539        mGroup->getCpuRefImpl()->launchThreads(
540                (const Allocation**)firstCpuClosure->mClosure->mArgs,
541                firstCpuClosure->mClosure->mNumArg,
542                lastCpuClosure->mClosure->mReturnValue,
543                nullptr, &mtls);
544
545        return;
546    }
547
548    for (CPUClosure* cpuClosure : mClosures) {
549        const Closure* closure = cpuClosure->mClosure;
550        const ScriptKernelID* kernelID =
551                (const ScriptKernelID*)closure->mFunctionID.get();
552        cpuClosure->mSi->preLaunch(kernelID->mSlot,
553                                   (const Allocation**)closure->mArgs,
554                                   closure->mNumArg, closure->mReturnValue,
555                                   nullptr, 0, nullptr);
556    }
557
558    const CPUClosure* cpuClosure = mClosures.front();
559    const Closure* closure = cpuClosure->mClosure;
560    MTLaunchStruct mtls;
561
562    if (cpuClosure->mSi->forEachMtlsSetup((const Allocation**)closure->mArgs,
563                                          closure->mNumArg,
564                                          closure->mReturnValue,
565                                          nullptr, 0, nullptr, &mtls)) {
566
567        mtls.script = nullptr;
568        mtls.kernel = (void (*)())&groupRoot;
569        mtls.fep.usr = &mClosures;
570
571        mGroup->getCpuRefImpl()->launchThreads(nullptr, 0, nullptr, nullptr, &mtls);
572    }
573
574    for (CPUClosure* cpuClosure : mClosures) {
575        const Closure* closure = cpuClosure->mClosure;
576        const ScriptKernelID* kernelID =
577                (const ScriptKernelID*)closure->mFunctionID.get();
578        cpuClosure->mSi->postLaunch(kernelID->mSlot,
579                                    (const Allocation**)closure->mArgs,
580                                    closure->mNumArg, closure->mReturnValue,
581                                    nullptr, 0, nullptr);
582    }
583}
584
585}  // namespace renderscript
586}  // namespace android
587