1709a0978ae141198018ca9769f8d96292a8928e6Jason Sams/*
2709a0978ae141198018ca9769f8d96292a8928e6Jason Sams * Copyright (C) 2012 The Android Open Source Project
3709a0978ae141198018ca9769f8d96292a8928e6Jason Sams *
4709a0978ae141198018ca9769f8d96292a8928e6Jason Sams * Licensed under the Apache License, Version 2.0 (the "License");
5709a0978ae141198018ca9769f8d96292a8928e6Jason Sams * you may not use this file except in compliance with the License.
6709a0978ae141198018ca9769f8d96292a8928e6Jason Sams * You may obtain a copy of the License at
7709a0978ae141198018ca9769f8d96292a8928e6Jason Sams *
8709a0978ae141198018ca9769f8d96292a8928e6Jason Sams *      http://www.apache.org/licenses/LICENSE-2.0
9709a0978ae141198018ca9769f8d96292a8928e6Jason Sams *
10709a0978ae141198018ca9769f8d96292a8928e6Jason Sams * Unless required by applicable law or agreed to in writing, software
11709a0978ae141198018ca9769f8d96292a8928e6Jason Sams * distributed under the License is distributed on an "AS IS" BASIS,
12709a0978ae141198018ca9769f8d96292a8928e6Jason Sams * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13709a0978ae141198018ca9769f8d96292a8928e6Jason Sams * See the License for the specific language governing permissions and
14709a0978ae141198018ca9769f8d96292a8928e6Jason Sams * limitations under the License.
15709a0978ae141198018ca9769f8d96292a8928e6Jason Sams */
16709a0978ae141198018ca9769f8d96292a8928e6Jason Sams
17709a0978ae141198018ca9769f8d96292a8928e6Jason Sams#include "rsCpuCore.h"
18709a0978ae141198018ca9769f8d96292a8928e6Jason Sams#include "rsCpuScript.h"
19709a0978ae141198018ca9769f8d96292a8928e6Jason Sams#include "rsCpuScriptGroup.h"
201ffd86b448d78366190c540f98f8b6d641cdb6cfYang Ni#include "rsCpuScriptGroup2.h"
21709a0978ae141198018ca9769f8d96292a8928e6Jason Sams
22709a0978ae141198018ca9769f8d96292a8928e6Jason Sams#include <malloc.h>
23709a0978ae141198018ca9769f8d96292a8928e6Jason Sams#include "rsContext.h"
24709a0978ae141198018ca9769f8d96292a8928e6Jason Sams
25709a0978ae141198018ca9769f8d96292a8928e6Jason Sams#include <sys/types.h>
26709a0978ae141198018ca9769f8d96292a8928e6Jason Sams#include <sys/resource.h>
27709a0978ae141198018ca9769f8d96292a8928e6Jason Sams#include <sched.h>
28709a0978ae141198018ca9769f8d96292a8928e6Jason Sams#include <sys/syscall.h>
2911fd9ec1ab8dfa7ae45c6edeea48dddc4633efeaMatt Wala#include <stdio.h>
30709a0978ae141198018ca9769f8d96292a8928e6Jason Sams#include <string.h>
31b0934b67b95cc27e2358c2aa4db5f7c1067c8f9bStephen Hines#include <unistd.h>
320b575de8ed0b628d84d256f5846500b0385979bdTim Murray
33b0934b67b95cc27e2358c2aa4db5f7c1067c8f9bStephen Hines#if !defined(RS_SERVER) && !defined(RS_COMPATIBILITY_LIB)
340b575de8ed0b628d84d256f5846500b0385979bdTim Murray#include <cutils/properties.h>
35709a0978ae141198018ca9769f8d96292a8928e6Jason Sams#include "utils/StopWatch.h"
360b575de8ed0b628d84d256f5846500b0385979bdTim Murray#endif
370b575de8ed0b628d84d256f5846500b0385979bdTim Murray
380b575de8ed0b628d84d256f5846500b0385979bdTim Murray#ifdef RS_SERVER
390b575de8ed0b628d84d256f5846500b0385979bdTim Murray// Android exposes gettid(), standard Linux does not
400b575de8ed0b628d84d256f5846500b0385979bdTim Murraystatic pid_t gettid() {
410b575de8ed0b628d84d256f5846500b0385979bdTim Murray    return syscall(SYS_gettid);
420b575de8ed0b628d84d256f5846500b0385979bdTim Murray}
430b575de8ed0b628d84d256f5846500b0385979bdTim Murray#endif
44709a0978ae141198018ca9769f8d96292a8928e6Jason Sams
45709a0978ae141198018ca9769f8d96292a8928e6Jason Samsusing namespace android;
46709a0978ae141198018ca9769f8d96292a8928e6Jason Samsusing namespace android::renderscript;
47709a0978ae141198018ca9769f8d96292a8928e6Jason Sams
48ae2ec3febedfc29376b9104413fb4042028f1265David Gross#define REDUCE_ALOGV(mtls, level, ...) do { if ((mtls)->logReduce >= (level)) ALOGV(__VA_ARGS__); } while(0)
4910adb0c2029f112b5738228617d5645f6ecea0c5David Gross
50709a0978ae141198018ca9769f8d96292a8928e6Jason Samsstatic pthread_key_t gThreadTLSKey = 0;
51709a0978ae141198018ca9769f8d96292a8928e6Jason Samsstatic uint32_t gThreadTLSKeyCount = 0;
52709a0978ae141198018ca9769f8d96292a8928e6Jason Samsstatic pthread_mutex_t gInitMutex = PTHREAD_MUTEX_INITIALIZER;
53709a0978ae141198018ca9769f8d96292a8928e6Jason Sams
54f5ef8df639ba6363aa5d546e57ce872d04144cb6Jason Samsbool android::renderscript::gArchUseSIMD = false;
55f5ef8df639ba6363aa5d546e57ce872d04144cb6Jason Sams
56709a0978ae141198018ca9769f8d96292a8928e6Jason SamsRsdCpuReference::~RsdCpuReference() {
57709a0978ae141198018ca9769f8d96292a8928e6Jason Sams}
58709a0978ae141198018ca9769f8d96292a8928e6Jason Sams
59709a0978ae141198018ca9769f8d96292a8928e6Jason SamsRsdCpuReference * RsdCpuReference::create(Context *rsc, uint32_t version_major,
60cadfac411e6690e39de36c4f9e94deb9b7d2d08eJason Sams        uint32_t version_minor, sym_lookup_t lfn, script_lookup_t slfn
61b043df0676fef226336deb3a00ead2f31e02343fDavid Gross        , RSSelectRTCallback pSelectRTCallback,
62005113297b19ed256b6db9d6bc293ed9266899fcStephen Hines        const char *pBccPluginName
63cadfac411e6690e39de36c4f9e94deb9b7d2d08eJason Sams        ) {
64709a0978ae141198018ca9769f8d96292a8928e6Jason Sams
65709a0978ae141198018ca9769f8d96292a8928e6Jason Sams    RsdCpuReferenceImpl *cpu = new RsdCpuReferenceImpl(rsc);
66709a0978ae141198018ca9769f8d96292a8928e6Jason Sams    if (!cpu) {
6744bef6fba6244292b751387f3d6c31cca96c28adChris Wailes        return nullptr;
68709a0978ae141198018ca9769f8d96292a8928e6Jason Sams    }
69709a0978ae141198018ca9769f8d96292a8928e6Jason Sams    if (!cpu->init(version_major, version_minor, lfn, slfn)) {
70709a0978ae141198018ca9769f8d96292a8928e6Jason Sams        delete cpu;
7144bef6fba6244292b751387f3d6c31cca96c28adChris Wailes        return nullptr;
72709a0978ae141198018ca9769f8d96292a8928e6Jason Sams    }
73f218bf115af4ae4fd79adbb8842608b308a4cf07Stephen Hines
741d476620399d54774e4fd386c1d23cc583d49522Stephen Hines    cpu->setSelectRTCallback(pSelectRTCallback);
75005113297b19ed256b6db9d6bc293ed9266899fcStephen Hines    if (pBccPluginName) {
76005113297b19ed256b6db9d6bc293ed9266899fcStephen Hines        cpu->setBccPluginName(pBccPluginName);
77005113297b19ed256b6db9d6bc293ed9266899fcStephen Hines    }
78f218bf115af4ae4fd79adbb8842608b308a4cf07Stephen Hines
79709a0978ae141198018ca9769f8d96292a8928e6Jason Sams    return cpu;
80709a0978ae141198018ca9769f8d96292a8928e6Jason Sams}
81709a0978ae141198018ca9769f8d96292a8928e6Jason Sams
82709a0978ae141198018ca9769f8d96292a8928e6Jason Sams
83709a0978ae141198018ca9769f8d96292a8928e6Jason SamsContext * RsdCpuReference::getTlsContext() {
84709a0978ae141198018ca9769f8d96292a8928e6Jason Sams    ScriptTLSStruct * tls = (ScriptTLSStruct *)pthread_getspecific(gThreadTLSKey);
85709a0978ae141198018ca9769f8d96292a8928e6Jason Sams    return tls->mContext;
86709a0978ae141198018ca9769f8d96292a8928e6Jason Sams}
87709a0978ae141198018ca9769f8d96292a8928e6Jason Sams
88709a0978ae141198018ca9769f8d96292a8928e6Jason Samsconst Script * RsdCpuReference::getTlsScript() {
89709a0978ae141198018ca9769f8d96292a8928e6Jason Sams    ScriptTLSStruct * tls = (ScriptTLSStruct *)pthread_getspecific(gThreadTLSKey);
90709a0978ae141198018ca9769f8d96292a8928e6Jason Sams    return tls->mScript;
91709a0978ae141198018ca9769f8d96292a8928e6Jason Sams}
92709a0978ae141198018ca9769f8d96292a8928e6Jason Sams
93f218bf115af4ae4fd79adbb8842608b308a4cf07Stephen Hinespthread_key_t RsdCpuReference::getThreadTLSKey(){ return gThreadTLSKey; }
94709a0978ae141198018ca9769f8d96292a8928e6Jason Sams
95709a0978ae141198018ca9769f8d96292a8928e6Jason Sams////////////////////////////////////////////////////////////
96709a0978ae141198018ca9769f8d96292a8928e6Jason Sams///
97709a0978ae141198018ca9769f8d96292a8928e6Jason Sams
98709a0978ae141198018ca9769f8d96292a8928e6Jason SamsRsdCpuReferenceImpl::RsdCpuReferenceImpl(Context *rsc) {
99709a0978ae141198018ca9769f8d96292a8928e6Jason Sams    mRSC = rsc;
100709a0978ae141198018ca9769f8d96292a8928e6Jason Sams
101709a0978ae141198018ca9769f8d96292a8928e6Jason Sams    version_major = 0;
102709a0978ae141198018ca9769f8d96292a8928e6Jason Sams    version_minor = 0;
10310adb0c2029f112b5738228617d5645f6ecea0c5David Gross    mInKernel = false;
104709a0978ae141198018ca9769f8d96292a8928e6Jason Sams    memset(&mWorkers, 0, sizeof(mWorkers));
105709a0978ae141198018ca9769f8d96292a8928e6Jason Sams    memset(&mTlsStruct, 0, sizeof(mTlsStruct));
106709a0978ae141198018ca9769f8d96292a8928e6Jason Sams    mExit = false;
10744bef6fba6244292b751387f3d6c31cca96c28adChris Wailes    mSelectRTCallback = nullptr;
1088409d6414dd4a42aa59779fcfe9fce18648cb135Stephen Hines    mEmbedGlobalInfo = true;
1098409d6414dd4a42aa59779fcfe9fce18648cb135Stephen Hines    mEmbedGlobalInfoSkipConstant = true;
110709a0978ae141198018ca9769f8d96292a8928e6Jason Sams}
111709a0978ae141198018ca9769f8d96292a8928e6Jason Sams
112709a0978ae141198018ca9769f8d96292a8928e6Jason Sams
113709a0978ae141198018ca9769f8d96292a8928e6Jason Samsvoid * RsdCpuReferenceImpl::helperThreadProc(void *vrsc) {
114709a0978ae141198018ca9769f8d96292a8928e6Jason Sams    RsdCpuReferenceImpl *dc = (RsdCpuReferenceImpl *)vrsc;
115709a0978ae141198018ca9769f8d96292a8928e6Jason Sams
1160b575de8ed0b628d84d256f5846500b0385979bdTim Murray    uint32_t idx = __sync_fetch_and_add(&dc->mWorkers.mLaunchCount, 1);
117709a0978ae141198018ca9769f8d96292a8928e6Jason Sams
118709a0978ae141198018ca9769f8d96292a8928e6Jason Sams    //ALOGV("RS helperThread starting %p idx=%i", dc, idx);
119709a0978ae141198018ca9769f8d96292a8928e6Jason Sams
120709a0978ae141198018ca9769f8d96292a8928e6Jason Sams    dc->mWorkers.mLaunchSignals[idx].init();
121709a0978ae141198018ca9769f8d96292a8928e6Jason Sams    dc->mWorkers.mNativeThreadId[idx] = gettid();
122709a0978ae141198018ca9769f8d96292a8928e6Jason Sams
123709a0978ae141198018ca9769f8d96292a8928e6Jason Sams    memset(&dc->mTlsStruct, 0, sizeof(dc->mTlsStruct));
124709a0978ae141198018ca9769f8d96292a8928e6Jason Sams    int status = pthread_setspecific(gThreadTLSKey, &dc->mTlsStruct);
125709a0978ae141198018ca9769f8d96292a8928e6Jason Sams    if (status) {
126709a0978ae141198018ca9769f8d96292a8928e6Jason Sams        ALOGE("pthread_setspecific %i", status);
127709a0978ae141198018ca9769f8d96292a8928e6Jason Sams    }
128709a0978ae141198018ca9769f8d96292a8928e6Jason Sams
129709a0978ae141198018ca9769f8d96292a8928e6Jason Sams#if 0
130709a0978ae141198018ca9769f8d96292a8928e6Jason Sams    typedef struct {uint64_t bits[1024 / 64]; } cpu_set_t;
131709a0978ae141198018ca9769f8d96292a8928e6Jason Sams    cpu_set_t cpuset;
132709a0978ae141198018ca9769f8d96292a8928e6Jason Sams    memset(&cpuset, 0, sizeof(cpuset));
133709a0978ae141198018ca9769f8d96292a8928e6Jason Sams    cpuset.bits[idx / 64] |= 1ULL << (idx % 64);
134709a0978ae141198018ca9769f8d96292a8928e6Jason Sams    int ret = syscall(241, rsc->mWorkers.mNativeThreadId[idx],
135709a0978ae141198018ca9769f8d96292a8928e6Jason Sams              sizeof(cpuset), &cpuset);
136709a0978ae141198018ca9769f8d96292a8928e6Jason Sams    ALOGE("SETAFFINITY ret = %i %s", ret, EGLUtils::strerror(ret));
137709a0978ae141198018ca9769f8d96292a8928e6Jason Sams#endif
138709a0978ae141198018ca9769f8d96292a8928e6Jason Sams
139709a0978ae141198018ca9769f8d96292a8928e6Jason Sams    while (!dc->mExit) {
140709a0978ae141198018ca9769f8d96292a8928e6Jason Sams        dc->mWorkers.mLaunchSignals[idx].wait();
141709a0978ae141198018ca9769f8d96292a8928e6Jason Sams        if (dc->mWorkers.mLaunchCallback) {
142709a0978ae141198018ca9769f8d96292a8928e6Jason Sams           // idx +1 is used because the calling thread is always worker 0.
143709a0978ae141198018ca9769f8d96292a8928e6Jason Sams           dc->mWorkers.mLaunchCallback(dc->mWorkers.mLaunchData, idx+1);
144709a0978ae141198018ca9769f8d96292a8928e6Jason Sams        }
1450b575de8ed0b628d84d256f5846500b0385979bdTim Murray        __sync_fetch_and_sub(&dc->mWorkers.mRunningCount, 1);
146709a0978ae141198018ca9769f8d96292a8928e6Jason Sams        dc->mWorkers.mCompleteSignal.set();
147709a0978ae141198018ca9769f8d96292a8928e6Jason Sams    }
148709a0978ae141198018ca9769f8d96292a8928e6Jason Sams
149709a0978ae141198018ca9769f8d96292a8928e6Jason Sams    //ALOGV("RS helperThread exited %p idx=%i", dc, idx);
15044bef6fba6244292b751387f3d6c31cca96c28adChris Wailes    return nullptr;
151709a0978ae141198018ca9769f8d96292a8928e6Jason Sams}
152709a0978ae141198018ca9769f8d96292a8928e6Jason Sams
15314ce007a633b10e3b9a3fae29d8f53a7e8c9b59fMatt Wala// Launch a kernel.
15414ce007a633b10e3b9a3fae29d8f53a7e8c9b59fMatt Wala// The callback function is called to execute the kernel.
155709a0978ae141198018ca9769f8d96292a8928e6Jason Samsvoid RsdCpuReferenceImpl::launchThreads(WorkerCallback_t cbk, void *data) {
156709a0978ae141198018ca9769f8d96292a8928e6Jason Sams    mWorkers.mLaunchData = data;
157709a0978ae141198018ca9769f8d96292a8928e6Jason Sams    mWorkers.mLaunchCallback = cbk;
1584d252d6e807b89764dad123ac845df298c52ca97Tim Murray
1594d252d6e807b89764dad123ac845df298c52ca97Tim Murray    // fast path for very small launches
16014ce007a633b10e3b9a3fae29d8f53a7e8c9b59fMatt Wala    MTLaunchStructCommon *mtls = (MTLaunchStructCommon *)data;
16114ce007a633b10e3b9a3fae29d8f53a7e8c9b59fMatt Wala    if (mtls && mtls->dimPtr->y <= 1 && mtls->end.x <= mtls->start.x + mtls->mSliceSize) {
1624d252d6e807b89764dad123ac845df298c52ca97Tim Murray        if (mWorkers.mLaunchCallback) {
1634d252d6e807b89764dad123ac845df298c52ca97Tim Murray            mWorkers.mLaunchCallback(mWorkers.mLaunchData, 0);
1644d252d6e807b89764dad123ac845df298c52ca97Tim Murray        }
1654d252d6e807b89764dad123ac845df298c52ca97Tim Murray        return;
1664d252d6e807b89764dad123ac845df298c52ca97Tim Murray    }
1674d252d6e807b89764dad123ac845df298c52ca97Tim Murray
1680b575de8ed0b628d84d256f5846500b0385979bdTim Murray    mWorkers.mRunningCount = mWorkers.mCount;
1690b575de8ed0b628d84d256f5846500b0385979bdTim Murray    __sync_synchronize();
1700b575de8ed0b628d84d256f5846500b0385979bdTim Murray
171709a0978ae141198018ca9769f8d96292a8928e6Jason Sams    for (uint32_t ct = 0; ct < mWorkers.mCount; ct++) {
172709a0978ae141198018ca9769f8d96292a8928e6Jason Sams        mWorkers.mLaunchSignals[ct].set();
173709a0978ae141198018ca9769f8d96292a8928e6Jason Sams    }
174709a0978ae141198018ca9769f8d96292a8928e6Jason Sams
175709a0978ae141198018ca9769f8d96292a8928e6Jason Sams    // We use the calling thread as one of the workers so we can start without
176709a0978ae141198018ca9769f8d96292a8928e6Jason Sams    // the delay of the thread wakeup.
177709a0978ae141198018ca9769f8d96292a8928e6Jason Sams    if (mWorkers.mLaunchCallback) {
1784d252d6e807b89764dad123ac845df298c52ca97Tim Murray        mWorkers.mLaunchCallback(mWorkers.mLaunchData, 0);
179709a0978ae141198018ca9769f8d96292a8928e6Jason Sams    }
180709a0978ae141198018ca9769f8d96292a8928e6Jason Sams
1810b575de8ed0b628d84d256f5846500b0385979bdTim Murray    while (__sync_fetch_and_or(&mWorkers.mRunningCount, 0) != 0) {
182709a0978ae141198018ca9769f8d96292a8928e6Jason Sams        mWorkers.mCompleteSignal.wait();
183709a0978ae141198018ca9769f8d96292a8928e6Jason Sams    }
184709a0978ae141198018ca9769f8d96292a8928e6Jason Sams}
185709a0978ae141198018ca9769f8d96292a8928e6Jason Sams
186709a0978ae141198018ca9769f8d96292a8928e6Jason Sams
187709a0978ae141198018ca9769f8d96292a8928e6Jason Samsvoid RsdCpuReferenceImpl::lockMutex() {
188709a0978ae141198018ca9769f8d96292a8928e6Jason Sams    pthread_mutex_lock(&gInitMutex);
189709a0978ae141198018ca9769f8d96292a8928e6Jason Sams}
190709a0978ae141198018ca9769f8d96292a8928e6Jason Sams
191709a0978ae141198018ca9769f8d96292a8928e6Jason Samsvoid RsdCpuReferenceImpl::unlockMutex() {
192709a0978ae141198018ca9769f8d96292a8928e6Jason Sams    pthread_mutex_unlock(&gInitMutex);
193709a0978ae141198018ca9769f8d96292a8928e6Jason Sams}
194709a0978ae141198018ca9769f8d96292a8928e6Jason Sams
19511fd9ec1ab8dfa7ae45c6edeea48dddc4633efeaMatt Wala// Determine if the CPU we're running on supports SIMD instructions.
196f5ef8df639ba6363aa5d546e57ce872d04144cb6Jason Samsstatic void GetCpuInfo() {
19711fd9ec1ab8dfa7ae45c6edeea48dddc4633efeaMatt Wala    // Read the CPU flags from /proc/cpuinfo.
19811fd9ec1ab8dfa7ae45c6edeea48dddc4633efeaMatt Wala    FILE *cpuinfo = fopen("/proc/cpuinfo", "r");
199f5ef8df639ba6363aa5d546e57ce872d04144cb6Jason Sams
20011fd9ec1ab8dfa7ae45c6edeea48dddc4633efeaMatt Wala    if (!cpuinfo) {
201f5ef8df639ba6363aa5d546e57ce872d04144cb6Jason Sams        return;
202f5ef8df639ba6363aa5d546e57ce872d04144cb6Jason Sams    }
203f5ef8df639ba6363aa5d546e57ce872d04144cb6Jason Sams
20411fd9ec1ab8dfa7ae45c6edeea48dddc4633efeaMatt Wala    char cpuinfostr[4096];
2055d70cb591d78d62d10839a52302ec9087c6f3350Miao Wang    // fgets() ends with newline or EOF, need to check the whole
2065d70cb591d78d62d10839a52302ec9087c6f3350Miao Wang    // "cpuinfo" file to make sure we can use SIMD or not.
2075d70cb591d78d62d10839a52302ec9087c6f3350Miao Wang    while (fgets(cpuinfostr, sizeof(cpuinfostr), cpuinfo)) {
208074424a4ac5b093331df2c92e7a5bcbfff136b71Jason Sams#if defined(ARCH_ARM_HAVE_VFP) || defined(ARCH_ARM_USE_INTRINSICS)
2095d70cb591d78d62d10839a52302ec9087c6f3350Miao Wang        gArchUseSIMD = strstr(cpuinfostr, " neon") || strstr(cpuinfostr, " asimd");
2107b7060c61e4182b29186849c5a857ea5f0898e56Rose, James#elif defined(ARCH_X86_HAVE_SSSE3)
2115d70cb591d78d62d10839a52302ec9087c6f3350Miao Wang        gArchUseSIMD = strstr(cpuinfostr, " ssse3");
2127b7060c61e4182b29186849c5a857ea5f0898e56Rose, James#endif
2135d70cb591d78d62d10839a52302ec9087c6f3350Miao Wang        if (gArchUseSIMD) {
2145d70cb591d78d62d10839a52302ec9087c6f3350Miao Wang            break;
2155d70cb591d78d62d10839a52302ec9087c6f3350Miao Wang        }
2165d70cb591d78d62d10839a52302ec9087c6f3350Miao Wang    }
2175d70cb591d78d62d10839a52302ec9087c6f3350Miao Wang    fclose(cpuinfo);
218f5ef8df639ba6363aa5d546e57ce872d04144cb6Jason Sams}
219f5ef8df639ba6363aa5d546e57ce872d04144cb6Jason Sams
220709a0978ae141198018ca9769f8d96292a8928e6Jason Samsbool RsdCpuReferenceImpl::init(uint32_t version_major, uint32_t version_minor,
221709a0978ae141198018ca9769f8d96292a8928e6Jason Sams                               sym_lookup_t lfn, script_lookup_t slfn) {
222709a0978ae141198018ca9769f8d96292a8928e6Jason Sams    mSymLookupFn = lfn;
223709a0978ae141198018ca9769f8d96292a8928e6Jason Sams    mScriptLookupFn = slfn;
224709a0978ae141198018ca9769f8d96292a8928e6Jason Sams
225709a0978ae141198018ca9769f8d96292a8928e6Jason Sams    lockMutex();
226709a0978ae141198018ca9769f8d96292a8928e6Jason Sams    if (!gThreadTLSKeyCount) {
22744bef6fba6244292b751387f3d6c31cca96c28adChris Wailes        int status = pthread_key_create(&gThreadTLSKey, nullptr);
228709a0978ae141198018ca9769f8d96292a8928e6Jason Sams        if (status) {
229709a0978ae141198018ca9769f8d96292a8928e6Jason Sams            ALOGE("Failed to init thread tls key.");
230709a0978ae141198018ca9769f8d96292a8928e6Jason Sams            unlockMutex();
231709a0978ae141198018ca9769f8d96292a8928e6Jason Sams            return false;
232709a0978ae141198018ca9769f8d96292a8928e6Jason Sams        }
233709a0978ae141198018ca9769f8d96292a8928e6Jason Sams    }
234709a0978ae141198018ca9769f8d96292a8928e6Jason Sams    gThreadTLSKeyCount++;
235709a0978ae141198018ca9769f8d96292a8928e6Jason Sams    unlockMutex();
236709a0978ae141198018ca9769f8d96292a8928e6Jason Sams
237709a0978ae141198018ca9769f8d96292a8928e6Jason Sams    mTlsStruct.mContext = mRSC;
23844bef6fba6244292b751387f3d6c31cca96c28adChris Wailes    mTlsStruct.mScript = nullptr;
239709a0978ae141198018ca9769f8d96292a8928e6Jason Sams    int status = pthread_setspecific(gThreadTLSKey, &mTlsStruct);
240709a0978ae141198018ca9769f8d96292a8928e6Jason Sams    if (status) {
241709a0978ae141198018ca9769f8d96292a8928e6Jason Sams        ALOGE("pthread_setspecific %i", status);
242709a0978ae141198018ca9769f8d96292a8928e6Jason Sams    }
243709a0978ae141198018ca9769f8d96292a8928e6Jason Sams
24410adb0c2029f112b5738228617d5645f6ecea0c5David Gross    mPageSize = sysconf(_SC_PAGE_SIZE);
2456760f7ba7934ddd51938a8d0206fc41c2a7cb419David Gross    // ALOGV("page size = %ld", mPageSize);
24610adb0c2029f112b5738228617d5645f6ecea0c5David Gross
247f5ef8df639ba6363aa5d546e57ce872d04144cb6Jason Sams    GetCpuInfo();
248f5ef8df639ba6363aa5d546e57ce872d04144cb6Jason Sams
24977d57a305f4134e78ebc91869011c4009988104eJason Sams    int cpu = sysconf(_SC_NPROCESSORS_CONF);
250709a0978ae141198018ca9769f8d96292a8928e6Jason Sams    if(mRSC->props.mDebugMaxThreads) {
251709a0978ae141198018ca9769f8d96292a8928e6Jason Sams        cpu = mRSC->props.mDebugMaxThreads;
252709a0978ae141198018ca9769f8d96292a8928e6Jason Sams    }
253709a0978ae141198018ca9769f8d96292a8928e6Jason Sams    if (cpu < 2) {
254709a0978ae141198018ca9769f8d96292a8928e6Jason Sams        mWorkers.mCount = 0;
255709a0978ae141198018ca9769f8d96292a8928e6Jason Sams        return true;
256709a0978ae141198018ca9769f8d96292a8928e6Jason Sams    }
257709a0978ae141198018ca9769f8d96292a8928e6Jason Sams
258709a0978ae141198018ca9769f8d96292a8928e6Jason Sams    // Subtract one from the cpu count because we also use the command thread as a worker.
259709a0978ae141198018ca9769f8d96292a8928e6Jason Sams    mWorkers.mCount = (uint32_t)(cpu - 1);
260709a0978ae141198018ca9769f8d96292a8928e6Jason Sams
261633beab5de5ffe4c5068cf1c7a1c0bd09ee9f195Yang Ni    if (mRSC->props.mLogScripts) {
262633beab5de5ffe4c5068cf1c7a1c0bd09ee9f195Yang Ni      ALOGV("%p Launching thread(s), CPUs %i", mRSC, mWorkers.mCount + 1);
263633beab5de5ffe4c5068cf1c7a1c0bd09ee9f195Yang Ni    }
264709a0978ae141198018ca9769f8d96292a8928e6Jason Sams
265709a0978ae141198018ca9769f8d96292a8928e6Jason Sams    mWorkers.mThreadId = (pthread_t *) calloc(mWorkers.mCount, sizeof(pthread_t));
266709a0978ae141198018ca9769f8d96292a8928e6Jason Sams    mWorkers.mNativeThreadId = (pid_t *) calloc(mWorkers.mCount, sizeof(pid_t));
267709a0978ae141198018ca9769f8d96292a8928e6Jason Sams    mWorkers.mLaunchSignals = new Signal[mWorkers.mCount];
26844bef6fba6244292b751387f3d6c31cca96c28adChris Wailes    mWorkers.mLaunchCallback = nullptr;
269709a0978ae141198018ca9769f8d96292a8928e6Jason Sams
270709a0978ae141198018ca9769f8d96292a8928e6Jason Sams    mWorkers.mCompleteSignal.init();
271709a0978ae141198018ca9769f8d96292a8928e6Jason Sams
2720b575de8ed0b628d84d256f5846500b0385979bdTim Murray    mWorkers.mRunningCount = mWorkers.mCount;
2730b575de8ed0b628d84d256f5846500b0385979bdTim Murray    mWorkers.mLaunchCount = 0;
2740b575de8ed0b628d84d256f5846500b0385979bdTim Murray    __sync_synchronize();
275709a0978ae141198018ca9769f8d96292a8928e6Jason Sams
276709a0978ae141198018ca9769f8d96292a8928e6Jason Sams    pthread_attr_t threadAttr;
277709a0978ae141198018ca9769f8d96292a8928e6Jason Sams    status = pthread_attr_init(&threadAttr);
278709a0978ae141198018ca9769f8d96292a8928e6Jason Sams    if (status) {
279709a0978ae141198018ca9769f8d96292a8928e6Jason Sams        ALOGE("Failed to init thread attribute.");
280709a0978ae141198018ca9769f8d96292a8928e6Jason Sams        return false;
281709a0978ae141198018ca9769f8d96292a8928e6Jason Sams    }
282709a0978ae141198018ca9769f8d96292a8928e6Jason Sams
283709a0978ae141198018ca9769f8d96292a8928e6Jason Sams    for (uint32_t ct=0; ct < mWorkers.mCount; ct++) {
284709a0978ae141198018ca9769f8d96292a8928e6Jason Sams        status = pthread_create(&mWorkers.mThreadId[ct], &threadAttr, helperThreadProc, this);
285709a0978ae141198018ca9769f8d96292a8928e6Jason Sams        if (status) {
286709a0978ae141198018ca9769f8d96292a8928e6Jason Sams            mWorkers.mCount = ct;
287709a0978ae141198018ca9769f8d96292a8928e6Jason Sams            ALOGE("Created fewer than expected number of RS threads.");
288709a0978ae141198018ca9769f8d96292a8928e6Jason Sams            break;
289709a0978ae141198018ca9769f8d96292a8928e6Jason Sams        }
290709a0978ae141198018ca9769f8d96292a8928e6Jason Sams    }
2910b575de8ed0b628d84d256f5846500b0385979bdTim Murray    while (__sync_fetch_and_or(&mWorkers.mRunningCount, 0) != 0) {
292709a0978ae141198018ca9769f8d96292a8928e6Jason Sams        usleep(100);
293709a0978ae141198018ca9769f8d96292a8928e6Jason Sams    }
294709a0978ae141198018ca9769f8d96292a8928e6Jason Sams
295709a0978ae141198018ca9769f8d96292a8928e6Jason Sams    pthread_attr_destroy(&threadAttr);
296709a0978ae141198018ca9769f8d96292a8928e6Jason Sams    return true;
297709a0978ae141198018ca9769f8d96292a8928e6Jason Sams}
298709a0978ae141198018ca9769f8d96292a8928e6Jason Sams
299709a0978ae141198018ca9769f8d96292a8928e6Jason Sams
300709a0978ae141198018ca9769f8d96292a8928e6Jason Samsvoid RsdCpuReferenceImpl::setPriority(int32_t priority) {
301709a0978ae141198018ca9769f8d96292a8928e6Jason Sams    for (uint32_t ct=0; ct < mWorkers.mCount; ct++) {
302709a0978ae141198018ca9769f8d96292a8928e6Jason Sams        setpriority(PRIO_PROCESS, mWorkers.mNativeThreadId[ct], priority);
303709a0978ae141198018ca9769f8d96292a8928e6Jason Sams    }
304709a0978ae141198018ca9769f8d96292a8928e6Jason Sams}
305709a0978ae141198018ca9769f8d96292a8928e6Jason Sams
306709a0978ae141198018ca9769f8d96292a8928e6Jason SamsRsdCpuReferenceImpl::~RsdCpuReferenceImpl() {
307709a0978ae141198018ca9769f8d96292a8928e6Jason Sams    mExit = true;
30844bef6fba6244292b751387f3d6c31cca96c28adChris Wailes    mWorkers.mLaunchData = nullptr;
30944bef6fba6244292b751387f3d6c31cca96c28adChris Wailes    mWorkers.mLaunchCallback = nullptr;
3100b575de8ed0b628d84d256f5846500b0385979bdTim Murray    mWorkers.mRunningCount = mWorkers.mCount;
3110b575de8ed0b628d84d256f5846500b0385979bdTim Murray    __sync_synchronize();
312709a0978ae141198018ca9769f8d96292a8928e6Jason Sams    for (uint32_t ct = 0; ct < mWorkers.mCount; ct++) {
313709a0978ae141198018ca9769f8d96292a8928e6Jason Sams        mWorkers.mLaunchSignals[ct].set();
314709a0978ae141198018ca9769f8d96292a8928e6Jason Sams    }
315709a0978ae141198018ca9769f8d96292a8928e6Jason Sams    void *res;
316709a0978ae141198018ca9769f8d96292a8928e6Jason Sams    for (uint32_t ct = 0; ct < mWorkers.mCount; ct++) {
317709a0978ae141198018ca9769f8d96292a8928e6Jason Sams        pthread_join(mWorkers.mThreadId[ct], &res);
318709a0978ae141198018ca9769f8d96292a8928e6Jason Sams    }
3190b575de8ed0b628d84d256f5846500b0385979bdTim Murray    rsAssert(__sync_fetch_and_or(&mWorkers.mRunningCount, 0) == 0);
32007ef704308b514272ed2f5c3e6a2f4c055550158Jens Gulin    free(mWorkers.mThreadId);
32107ef704308b514272ed2f5c3e6a2f4c055550158Jens Gulin    free(mWorkers.mNativeThreadId);
32207ef704308b514272ed2f5c3e6a2f4c055550158Jens Gulin    delete[] mWorkers.mLaunchSignals;
323709a0978ae141198018ca9769f8d96292a8928e6Jason Sams
324709a0978ae141198018ca9769f8d96292a8928e6Jason Sams    // Global structure cleanup.
325709a0978ae141198018ca9769f8d96292a8928e6Jason Sams    lockMutex();
326709a0978ae141198018ca9769f8d96292a8928e6Jason Sams    --gThreadTLSKeyCount;
327709a0978ae141198018ca9769f8d96292a8928e6Jason Sams    if (!gThreadTLSKeyCount) {
328709a0978ae141198018ca9769f8d96292a8928e6Jason Sams        pthread_key_delete(gThreadTLSKey);
329709a0978ae141198018ca9769f8d96292a8928e6Jason Sams    }
330709a0978ae141198018ca9769f8d96292a8928e6Jason Sams    unlockMutex();
331709a0978ae141198018ca9769f8d96292a8928e6Jason Sams
332709a0978ae141198018ca9769f8d96292a8928e6Jason Sams}
333709a0978ae141198018ca9769f8d96292a8928e6Jason Sams
33414ce007a633b10e3b9a3fae29d8f53a7e8c9b59fMatt Wala// Set up the appropriate input and output pointers to the kernel driver info structure.
33514ce007a633b10e3b9a3fae29d8f53a7e8c9b59fMatt Wala// Inputs:
33614ce007a633b10e3b9a3fae29d8f53a7e8c9b59fMatt Wala//   mtls - The MTLaunchStruct holding information about the kernel launch
33714ce007a633b10e3b9a3fae29d8f53a7e8c9b59fMatt Wala//   fep - The forEach parameters (driver info structure)
33814ce007a633b10e3b9a3fae29d8f53a7e8c9b59fMatt Wala//   x, y, z, lod, face, a1, a2, a3, a4 - The start offsets into each dimension
33914ce007a633b10e3b9a3fae29d8f53a7e8c9b59fMatt Walastatic inline void FepPtrSetup(const MTLaunchStructForEach *mtls, RsExpandKernelDriverInfo *fep,
340c0d68470b978a79ce024fde56f23ea3690603ccdJason Sams                               uint32_t x, uint32_t y,
341c0d68470b978a79ce024fde56f23ea3690603ccdJason Sams                               uint32_t z = 0, uint32_t lod = 0,
342c0d68470b978a79ce024fde56f23ea3690603ccdJason Sams                               RsAllocationCubemapFace face = RS_ALLOCATION_CUBEMAP_FACE_POSITIVE_X,
343c0d68470b978a79ce024fde56f23ea3690603ccdJason Sams                               uint32_t a1 = 0, uint32_t a2 = 0, uint32_t a3 = 0, uint32_t a4 = 0) {
344c0d68470b978a79ce024fde56f23ea3690603ccdJason Sams    for (uint32_t i = 0; i < fep->inLen; i++) {
345c0d68470b978a79ce024fde56f23ea3690603ccdJason Sams        fep->inPtr[i] = (const uint8_t *)mtls->ains[i]->getPointerUnchecked(x, y, z, lod, face, a1, a2, a3, a4);
346709a0978ae141198018ca9769f8d96292a8928e6Jason Sams    }
347c0d68470b978a79ce024fde56f23ea3690603ccdJason Sams    if (mtls->aout[0] != nullptr) {
348c0d68470b978a79ce024fde56f23ea3690603ccdJason Sams        fep->outPtr[0] = (uint8_t *)mtls->aout[0]->getPointerUnchecked(x, y, z, lod, face, a1, a2, a3, a4);
349c0d68470b978a79ce024fde56f23ea3690603ccdJason Sams    }
350709a0978ae141198018ca9769f8d96292a8928e6Jason Sams}
351709a0978ae141198018ca9769f8d96292a8928e6Jason Sams
3526c1876bbef1b2c89975dce91230a168bd2d2ce4cDavid Gross// Set up the appropriate input and output pointers to the kernel driver info structure.
3536c1876bbef1b2c89975dce91230a168bd2d2ce4cDavid Gross// Inputs:
3546c1876bbef1b2c89975dce91230a168bd2d2ce4cDavid Gross//   mtls - The MTLaunchStruct holding information about the kernel launch
3556c1876bbef1b2c89975dce91230a168bd2d2ce4cDavid Gross//   redp - The reduce parameters (driver info structure)
3566c1876bbef1b2c89975dce91230a168bd2d2ce4cDavid Gross//   x, y, z - The start offsets into each dimension
357ae2ec3febedfc29376b9104413fb4042028f1265David Grossstatic inline void RedpPtrSetup(const MTLaunchStructReduce *mtls, RsExpandKernelDriverInfo *redp,
3586c1876bbef1b2c89975dce91230a168bd2d2ce4cDavid Gross                                uint32_t x, uint32_t y, uint32_t z) {
3596c1876bbef1b2c89975dce91230a168bd2d2ce4cDavid Gross    for (uint32_t i = 0; i < redp->inLen; i++) {
3606c1876bbef1b2c89975dce91230a168bd2d2ce4cDavid Gross        redp->inPtr[i] = (const uint8_t *)mtls->ains[i]->getPointerUnchecked(x, y, z);
3616c1876bbef1b2c89975dce91230a168bd2d2ce4cDavid Gross    }
3626c1876bbef1b2c89975dce91230a168bd2d2ce4cDavid Gross}
3636c1876bbef1b2c89975dce91230a168bd2d2ce4cDavid Gross
364bf2111d3b3de310932099514f06924e48fa1d7b2Jason Samsstatic uint32_t sliceInt(uint32_t *p, uint32_t val, uint32_t start, uint32_t end) {
365bf2111d3b3de310932099514f06924e48fa1d7b2Jason Sams    if (start >= end) {
366bf2111d3b3de310932099514f06924e48fa1d7b2Jason Sams        *p = start;
367bf2111d3b3de310932099514f06924e48fa1d7b2Jason Sams        return val;
368bf2111d3b3de310932099514f06924e48fa1d7b2Jason Sams    }
369bf2111d3b3de310932099514f06924e48fa1d7b2Jason Sams
370bf2111d3b3de310932099514f06924e48fa1d7b2Jason Sams    uint32_t div = end - start;
371bf2111d3b3de310932099514f06924e48fa1d7b2Jason Sams
372bf2111d3b3de310932099514f06924e48fa1d7b2Jason Sams    uint32_t n = val / div;
373bf2111d3b3de310932099514f06924e48fa1d7b2Jason Sams    *p = (val - (n * div)) + start;
374bf2111d3b3de310932099514f06924e48fa1d7b2Jason Sams    return n;
375bf2111d3b3de310932099514f06924e48fa1d7b2Jason Sams}
376bf2111d3b3de310932099514f06924e48fa1d7b2Jason Sams
3776c1876bbef1b2c89975dce91230a168bd2d2ce4cDavid Grossstatic bool SelectOuterSlice(const MTLaunchStructCommon *mtls, RsExpandKernelDriverInfo* info, uint32_t sliceNum) {
378bf2111d3b3de310932099514f06924e48fa1d7b2Jason Sams    uint32_t r = sliceNum;
3796c1876bbef1b2c89975dce91230a168bd2d2ce4cDavid Gross    r = sliceInt(&info->current.z, r, mtls->start.z, mtls->end.z);
3806c1876bbef1b2c89975dce91230a168bd2d2ce4cDavid Gross    r = sliceInt(&info->current.lod, r, mtls->start.lod, mtls->end.lod);
3816c1876bbef1b2c89975dce91230a168bd2d2ce4cDavid Gross    r = sliceInt(&info->current.face, r, mtls->start.face, mtls->end.face);
3826c1876bbef1b2c89975dce91230a168bd2d2ce4cDavid Gross    r = sliceInt(&info->current.array[0], r, mtls->start.array[0], mtls->end.array[0]);
3836c1876bbef1b2c89975dce91230a168bd2d2ce4cDavid Gross    r = sliceInt(&info->current.array[1], r, mtls->start.array[1], mtls->end.array[1]);
3846c1876bbef1b2c89975dce91230a168bd2d2ce4cDavid Gross    r = sliceInt(&info->current.array[2], r, mtls->start.array[2], mtls->end.array[2]);
3856c1876bbef1b2c89975dce91230a168bd2d2ce4cDavid Gross    r = sliceInt(&info->current.array[3], r, mtls->start.array[3], mtls->end.array[3]);
386bf2111d3b3de310932099514f06924e48fa1d7b2Jason Sams    return r == 0;
387bf2111d3b3de310932099514f06924e48fa1d7b2Jason Sams}
388bf2111d3b3de310932099514f06924e48fa1d7b2Jason Sams
389415a249300cb9ca3f560814bfde52c6796fed1f7David Grossstatic bool SelectZSlice(const MTLaunchStructCommon *mtls, RsExpandKernelDriverInfo* info, uint32_t sliceNum) {
390415a249300cb9ca3f560814bfde52c6796fed1f7David Gross    return sliceInt(&info->current.z, sliceNum, mtls->start.z, mtls->end.z) == 0;
391415a249300cb9ca3f560814bfde52c6796fed1f7David Gross}
392bf2111d3b3de310932099514f06924e48fa1d7b2Jason Sams
393415a249300cb9ca3f560814bfde52c6796fed1f7David Grossstatic void walk_general_foreach(void *usr, uint32_t idx) {
39414ce007a633b10e3b9a3fae29d8f53a7e8c9b59fMatt Wala    MTLaunchStructForEach *mtls = (MTLaunchStructForEach *)usr;
395bf2111d3b3de310932099514f06924e48fa1d7b2Jason Sams    RsExpandKernelDriverInfo fep = mtls->fep;
396bf2111d3b3de310932099514f06924e48fa1d7b2Jason Sams    fep.lid = idx;
39714ce007a633b10e3b9a3fae29d8f53a7e8c9b59fMatt Wala    ForEachFunc_t fn = mtls->kernel;
398bf2111d3b3de310932099514f06924e48fa1d7b2Jason Sams
399bf2111d3b3de310932099514f06924e48fa1d7b2Jason Sams    while(1) {
400bf2111d3b3de310932099514f06924e48fa1d7b2Jason Sams        uint32_t slice = (uint32_t)__sync_fetch_and_add(&mtls->mSliceNum, 1);
401bf2111d3b3de310932099514f06924e48fa1d7b2Jason Sams
40259b35f29cc576243a322bf88bc16063a9810da55Jason Sams        if (!SelectOuterSlice(mtls, &fep, slice)) {
403bf2111d3b3de310932099514f06924e48fa1d7b2Jason Sams            return;
404bf2111d3b3de310932099514f06924e48fa1d7b2Jason Sams        }
405bf2111d3b3de310932099514f06924e48fa1d7b2Jason Sams
40659b35f29cc576243a322bf88bc16063a9810da55Jason Sams        for (fep.current.y = mtls->start.y; fep.current.y < mtls->end.y;
40759b35f29cc576243a322bf88bc16063a9810da55Jason Sams             fep.current.y++) {
408bf2111d3b3de310932099514f06924e48fa1d7b2Jason Sams
40959b35f29cc576243a322bf88bc16063a9810da55Jason Sams            FepPtrSetup(mtls, &fep, mtls->start.x,
41059b35f29cc576243a322bf88bc16063a9810da55Jason Sams                        fep.current.y, fep.current.z, fep.current.lod,
41159b35f29cc576243a322bf88bc16063a9810da55Jason Sams                        (RsAllocationCubemapFace)fep.current.face,
41259b35f29cc576243a322bf88bc16063a9810da55Jason Sams                        fep.current.array[0], fep.current.array[1],
41359b35f29cc576243a322bf88bc16063a9810da55Jason Sams                        fep.current.array[2], fep.current.array[3]);
414bf2111d3b3de310932099514f06924e48fa1d7b2Jason Sams
41559b35f29cc576243a322bf88bc16063a9810da55Jason Sams            fn(&fep, mtls->start.x, mtls->end.x, mtls->fep.outStride[0]);
416bf2111d3b3de310932099514f06924e48fa1d7b2Jason Sams        }
417bf2111d3b3de310932099514f06924e48fa1d7b2Jason Sams    }
418bf2111d3b3de310932099514f06924e48fa1d7b2Jason Sams}
419c0d68470b978a79ce024fde56f23ea3690603ccdJason Sams
420415a249300cb9ca3f560814bfde52c6796fed1f7David Grossstatic void walk_2d_foreach(void *usr, uint32_t idx) {
42114ce007a633b10e3b9a3fae29d8f53a7e8c9b59fMatt Wala    MTLaunchStructForEach *mtls = (MTLaunchStructForEach *)usr;
422c0d68470b978a79ce024fde56f23ea3690603ccdJason Sams    RsExpandKernelDriverInfo fep = mtls->fep;
423c0d68470b978a79ce024fde56f23ea3690603ccdJason Sams    fep.lid = idx;
42414ce007a633b10e3b9a3fae29d8f53a7e8c9b59fMatt Wala    ForEachFunc_t fn = mtls->kernel;
425709a0978ae141198018ca9769f8d96292a8928e6Jason Sams
426c0d68470b978a79ce024fde56f23ea3690603ccdJason Sams    while (1) {
427c0d68470b978a79ce024fde56f23ea3690603ccdJason Sams        uint32_t slice  = (uint32_t)__sync_fetch_and_add(&mtls->mSliceNum, 1);
428bf2111d3b3de310932099514f06924e48fa1d7b2Jason Sams        uint32_t yStart = mtls->start.y + slice * mtls->mSliceSize;
429c0d68470b978a79ce024fde56f23ea3690603ccdJason Sams        uint32_t yEnd   = yStart + mtls->mSliceSize;
430709a0978ae141198018ca9769f8d96292a8928e6Jason Sams
431bf2111d3b3de310932099514f06924e48fa1d7b2Jason Sams        yEnd = rsMin(yEnd, mtls->end.y);
432709a0978ae141198018ca9769f8d96292a8928e6Jason Sams
433c0d68470b978a79ce024fde56f23ea3690603ccdJason Sams        if (yEnd <= yStart) {
434c0d68470b978a79ce024fde56f23ea3690603ccdJason Sams            return;
435c0d68470b978a79ce024fde56f23ea3690603ccdJason Sams        }
436709a0978ae141198018ca9769f8d96292a8928e6Jason Sams
437c0d68470b978a79ce024fde56f23ea3690603ccdJason Sams        for (fep.current.y = yStart; fep.current.y < yEnd; fep.current.y++) {
438bf2111d3b3de310932099514f06924e48fa1d7b2Jason Sams            FepPtrSetup(mtls, &fep, mtls->start.x, fep.current.y);
439709a0978ae141198018ca9769f8d96292a8928e6Jason Sams
440b0abb140ac51b93d1a85aadaa63fe057f2d29850David Gross            fn(&fep, mtls->start.x, mtls->end.x, fep.outStride[0]);
441709a0978ae141198018ca9769f8d96292a8928e6Jason Sams        }
442c0d68470b978a79ce024fde56f23ea3690603ccdJason Sams    }
443f37121300217d3b39ab66dd9c8881bcbcad932dfChris Wailes}
4444b2bea3dc20865f3a198797702e19912a6a2171cStephen Hines
44510adb0c2029f112b5738228617d5645f6ecea0c5David Grossstatic void walk_1d_foreach(void *usr, uint32_t idx) {
44614ce007a633b10e3b9a3fae29d8f53a7e8c9b59fMatt Wala    MTLaunchStructForEach *mtls = (MTLaunchStructForEach *)usr;
447c0d68470b978a79ce024fde56f23ea3690603ccdJason Sams    RsExpandKernelDriverInfo fep = mtls->fep;
448c0d68470b978a79ce024fde56f23ea3690603ccdJason Sams    fep.lid = idx;
44914ce007a633b10e3b9a3fae29d8f53a7e8c9b59fMatt Wala    ForEachFunc_t fn = mtls->kernel;
45080ef693674f69c0343c41564e30f80e7fb513b60Chris Wailes
451c0d68470b978a79ce024fde56f23ea3690603ccdJason Sams    while (1) {
452c0d68470b978a79ce024fde56f23ea3690603ccdJason Sams        uint32_t slice  = (uint32_t)__sync_fetch_and_add(&mtls->mSliceNum, 1);
453bf2111d3b3de310932099514f06924e48fa1d7b2Jason Sams        uint32_t xStart = mtls->start.x + slice * mtls->mSliceSize;
454c0d68470b978a79ce024fde56f23ea3690603ccdJason Sams        uint32_t xEnd   = xStart + mtls->mSliceSize;
45580ef693674f69c0343c41564e30f80e7fb513b60Chris Wailes
456bf2111d3b3de310932099514f06924e48fa1d7b2Jason Sams        xEnd = rsMin(xEnd, mtls->end.x);
45780ef693674f69c0343c41564e30f80e7fb513b60Chris Wailes
458c0d68470b978a79ce024fde56f23ea3690603ccdJason Sams        if (xEnd <= xStart) {
459c0d68470b978a79ce024fde56f23ea3690603ccdJason Sams            return;
460c0d68470b978a79ce024fde56f23ea3690603ccdJason Sams        }
46180ef693674f69c0343c41564e30f80e7fb513b60Chris Wailes
462bf2111d3b3de310932099514f06924e48fa1d7b2Jason Sams        FepPtrSetup(mtls, &fep, xStart, 0);
46380ef693674f69c0343c41564e30f80e7fb513b60Chris Wailes
464b0abb140ac51b93d1a85aadaa63fe057f2d29850David Gross        fn(&fep, xStart, xEnd, fep.outStride[0]);
465c0d68470b978a79ce024fde56f23ea3690603ccdJason Sams    }
466709a0978ae141198018ca9769f8d96292a8928e6Jason Sams}
467709a0978ae141198018ca9769f8d96292a8928e6Jason Sams
46810adb0c2029f112b5738228617d5645f6ecea0c5David Gross// The function format_bytes() is an auxiliary function to assist in logging.
46910adb0c2029f112b5738228617d5645f6ecea0c5David Gross//
47010adb0c2029f112b5738228617d5645f6ecea0c5David Gross// Bytes are read from an input (inBuf) and written (as pairs of hex digits)
47110adb0c2029f112b5738228617d5645f6ecea0c5David Gross// to an output (outBuf).
47210adb0c2029f112b5738228617d5645f6ecea0c5David Gross//
47310adb0c2029f112b5738228617d5645f6ecea0c5David Gross// Output format:
47410adb0c2029f112b5738228617d5645f6ecea0c5David Gross// - starts with ": "
47510adb0c2029f112b5738228617d5645f6ecea0c5David Gross// - each input byte is translated to a pair of hex digits
47610adb0c2029f112b5738228617d5645f6ecea0c5David Gross// - bytes are separated by "." except that every fourth separator is "|"
47710adb0c2029f112b5738228617d5645f6ecea0c5David Gross// - if the input is sufficiently long, the output is truncated and terminated with "..."
47810adb0c2029f112b5738228617d5645f6ecea0c5David Gross//
47910adb0c2029f112b5738228617d5645f6ecea0c5David Gross// Arguments:
48010adb0c2029f112b5738228617d5645f6ecea0c5David Gross// - outBuf  -- Pointer to buffer of type "FormatBuf" into which output is written
48110adb0c2029f112b5738228617d5645f6ecea0c5David Gross// - inBuf   -- Pointer to bytes which are to be formatted into outBuf
48210adb0c2029f112b5738228617d5645f6ecea0c5David Gross// - inBytes -- Number of bytes in inBuf
48310adb0c2029f112b5738228617d5645f6ecea0c5David Gross//
48410adb0c2029f112b5738228617d5645f6ecea0c5David Gross// Constant:
48510adb0c2029f112b5738228617d5645f6ecea0c5David Gross// - kFormatInBytesMax -- Only min(kFormatInBytesMax, inBytes) bytes will be read
48610adb0c2029f112b5738228617d5645f6ecea0c5David Gross//                        from inBuf
48710adb0c2029f112b5738228617d5645f6ecea0c5David Gross//
48810adb0c2029f112b5738228617d5645f6ecea0c5David Gross// Return value:
48910adb0c2029f112b5738228617d5645f6ecea0c5David Gross// - pointer (const char *) to output (which is part of outBuf)
49010adb0c2029f112b5738228617d5645f6ecea0c5David Gross//
49110adb0c2029f112b5738228617d5645f6ecea0c5David Grossstatic const int kFormatInBytesMax = 16;
49210adb0c2029f112b5738228617d5645f6ecea0c5David Gross// ": " + 2 digits per byte + 1 separator between bytes + "..." + null
49310adb0c2029f112b5738228617d5645f6ecea0c5David Grosstypedef char FormatBuf[2 + kFormatInBytesMax*2 + (kFormatInBytesMax - 1) + 3 + 1];
49410adb0c2029f112b5738228617d5645f6ecea0c5David Grossstatic const char *format_bytes(FormatBuf *outBuf, const uint8_t *inBuf, const int inBytes) {
49510adb0c2029f112b5738228617d5645f6ecea0c5David Gross  strcpy(*outBuf, ": ");
49610adb0c2029f112b5738228617d5645f6ecea0c5David Gross  int pos = 2;
49710adb0c2029f112b5738228617d5645f6ecea0c5David Gross  const int lim = std::min(kFormatInBytesMax, inBytes);
49810adb0c2029f112b5738228617d5645f6ecea0c5David Gross  for (int i = 0; i < lim; ++i) {
49910adb0c2029f112b5738228617d5645f6ecea0c5David Gross    if (i) {
50010adb0c2029f112b5738228617d5645f6ecea0c5David Gross      sprintf(*outBuf + pos, (i % 4 ? "." : "|"));
50110adb0c2029f112b5738228617d5645f6ecea0c5David Gross      ++pos;
50210adb0c2029f112b5738228617d5645f6ecea0c5David Gross    }
50310adb0c2029f112b5738228617d5645f6ecea0c5David Gross    sprintf(*outBuf + pos, "%02x", inBuf[i]);
50410adb0c2029f112b5738228617d5645f6ecea0c5David Gross    pos += 2;
50510adb0c2029f112b5738228617d5645f6ecea0c5David Gross  }
50610adb0c2029f112b5738228617d5645f6ecea0c5David Gross  if (kFormatInBytesMax < inBytes)
50710adb0c2029f112b5738228617d5645f6ecea0c5David Gross    strcpy(*outBuf + pos, "...");
50810adb0c2029f112b5738228617d5645f6ecea0c5David Gross  return *outBuf;
50910adb0c2029f112b5738228617d5645f6ecea0c5David Gross}
51010adb0c2029f112b5738228617d5645f6ecea0c5David Gross
511ae2ec3febedfc29376b9104413fb4042028f1265David Grossstatic void reduce_get_accumulator(uint8_t *&accumPtr, const MTLaunchStructReduce *mtls,
512ae2ec3febedfc29376b9104413fb4042028f1265David Gross                                   const char *walkerName, uint32_t threadIdx) {
513415a249300cb9ca3f560814bfde52c6796fed1f7David Gross  rsAssert(!accumPtr);
514415a249300cb9ca3f560814bfde52c6796fed1f7David Gross
515415a249300cb9ca3f560814bfde52c6796fed1f7David Gross  uint32_t accumIdx = (uint32_t)__sync_fetch_and_add(&mtls->accumCount, 1);
516415a249300cb9ca3f560814bfde52c6796fed1f7David Gross  if (mtls->outFunc) {
517415a249300cb9ca3f560814bfde52c6796fed1f7David Gross    accumPtr = mtls->accumAlloc + mtls->accumStride * accumIdx;
518415a249300cb9ca3f560814bfde52c6796fed1f7David Gross  } else {
519415a249300cb9ca3f560814bfde52c6796fed1f7David Gross    if (accumIdx == 0) {
520415a249300cb9ca3f560814bfde52c6796fed1f7David Gross      accumPtr = mtls->redp.outPtr[0];
521415a249300cb9ca3f560814bfde52c6796fed1f7David Gross    } else {
522415a249300cb9ca3f560814bfde52c6796fed1f7David Gross      accumPtr = mtls->accumAlloc + mtls->accumStride * (accumIdx - 1);
523415a249300cb9ca3f560814bfde52c6796fed1f7David Gross    }
524415a249300cb9ca3f560814bfde52c6796fed1f7David Gross  }
525ae2ec3febedfc29376b9104413fb4042028f1265David Gross  REDUCE_ALOGV(mtls, 2, "%s(%p): idx = %u got accumCount %u and accumPtr %p",
526ae2ec3febedfc29376b9104413fb4042028f1265David Gross               walkerName, mtls->accumFunc, threadIdx, accumIdx, accumPtr);
527415a249300cb9ca3f560814bfde52c6796fed1f7David Gross  // initialize accumulator
528415a249300cb9ca3f560814bfde52c6796fed1f7David Gross  if (mtls->initFunc) {
529415a249300cb9ca3f560814bfde52c6796fed1f7David Gross    mtls->initFunc(accumPtr);
530415a249300cb9ca3f560814bfde52c6796fed1f7David Gross  } else {
531415a249300cb9ca3f560814bfde52c6796fed1f7David Gross    memset(accumPtr, 0, mtls->accumSize);
532415a249300cb9ca3f560814bfde52c6796fed1f7David Gross  }
533415a249300cb9ca3f560814bfde52c6796fed1f7David Gross}
534415a249300cb9ca3f560814bfde52c6796fed1f7David Gross
535ae2ec3febedfc29376b9104413fb4042028f1265David Grossstatic void walk_1d_reduce(void *usr, uint32_t idx) {
536ae2ec3febedfc29376b9104413fb4042028f1265David Gross  const MTLaunchStructReduce *mtls = (const MTLaunchStructReduce *)usr;
53710adb0c2029f112b5738228617d5645f6ecea0c5David Gross  RsExpandKernelDriverInfo redp = mtls->redp;
53810adb0c2029f112b5738228617d5645f6ecea0c5David Gross
53910adb0c2029f112b5738228617d5645f6ecea0c5David Gross  // find accumulator
54010adb0c2029f112b5738228617d5645f6ecea0c5David Gross  uint8_t *&accumPtr = mtls->accumPtr[idx];
54110adb0c2029f112b5738228617d5645f6ecea0c5David Gross  if (!accumPtr) {
542ae2ec3febedfc29376b9104413fb4042028f1265David Gross    reduce_get_accumulator(accumPtr, mtls, __func__, idx);
54310adb0c2029f112b5738228617d5645f6ecea0c5David Gross  }
54410adb0c2029f112b5738228617d5645f6ecea0c5David Gross
54510adb0c2029f112b5738228617d5645f6ecea0c5David Gross  // accumulate
546ae2ec3febedfc29376b9104413fb4042028f1265David Gross  const ReduceAccumulatorFunc_t fn = mtls->accumFunc;
54710adb0c2029f112b5738228617d5645f6ecea0c5David Gross  while (1) {
54810adb0c2029f112b5738228617d5645f6ecea0c5David Gross    uint32_t slice  = (uint32_t)__sync_fetch_and_add(&mtls->mSliceNum, 1);
54910adb0c2029f112b5738228617d5645f6ecea0c5David Gross    uint32_t xStart = mtls->start.x + slice * mtls->mSliceSize;
55010adb0c2029f112b5738228617d5645f6ecea0c5David Gross    uint32_t xEnd   = xStart + mtls->mSliceSize;
55110adb0c2029f112b5738228617d5645f6ecea0c5David Gross
55210adb0c2029f112b5738228617d5645f6ecea0c5David Gross    xEnd = rsMin(xEnd, mtls->end.x);
55310adb0c2029f112b5738228617d5645f6ecea0c5David Gross
55410adb0c2029f112b5738228617d5645f6ecea0c5David Gross    if (xEnd <= xStart) {
55510adb0c2029f112b5738228617d5645f6ecea0c5David Gross      return;
55610adb0c2029f112b5738228617d5645f6ecea0c5David Gross    }
55710adb0c2029f112b5738228617d5645f6ecea0c5David Gross
55810adb0c2029f112b5738228617d5645f6ecea0c5David Gross    RedpPtrSetup(mtls, &redp, xStart, 0, 0);
55910adb0c2029f112b5738228617d5645f6ecea0c5David Gross    fn(&redp, xStart, xEnd, accumPtr);
56010adb0c2029f112b5738228617d5645f6ecea0c5David Gross
561415a249300cb9ca3f560814bfde52c6796fed1f7David Gross    // Emit log line after slice has been run, so that we can include
562415a249300cb9ca3f560814bfde52c6796fed1f7David Gross    // the results of the run on that line.
56310adb0c2029f112b5738228617d5645f6ecea0c5David Gross    FormatBuf fmt;
5646760f7ba7934ddd51938a8d0206fc41c2a7cb419David Gross    if (mtls->logReduce >= 3) {
56510adb0c2029f112b5738228617d5645f6ecea0c5David Gross      format_bytes(&fmt, accumPtr, mtls->accumSize);
56610adb0c2029f112b5738228617d5645f6ecea0c5David Gross    } else {
56710adb0c2029f112b5738228617d5645f6ecea0c5David Gross      fmt[0] = 0;
56810adb0c2029f112b5738228617d5645f6ecea0c5David Gross    }
569ae2ec3febedfc29376b9104413fb4042028f1265David Gross    REDUCE_ALOGV(mtls, 2, "walk_1d_reduce(%p): idx = %u, x in [%u, %u)%s",
570ae2ec3febedfc29376b9104413fb4042028f1265David Gross                 mtls->accumFunc, idx, xStart, xEnd, fmt);
57110adb0c2029f112b5738228617d5645f6ecea0c5David Gross  }
57210adb0c2029f112b5738228617d5645f6ecea0c5David Gross}
57310adb0c2029f112b5738228617d5645f6ecea0c5David Gross
574ae2ec3febedfc29376b9104413fb4042028f1265David Grossstatic void walk_2d_reduce(void *usr, uint32_t idx) {
575ae2ec3febedfc29376b9104413fb4042028f1265David Gross  const MTLaunchStructReduce *mtls = (const MTLaunchStructReduce *)usr;
576415a249300cb9ca3f560814bfde52c6796fed1f7David Gross  RsExpandKernelDriverInfo redp = mtls->redp;
577415a249300cb9ca3f560814bfde52c6796fed1f7David Gross
578415a249300cb9ca3f560814bfde52c6796fed1f7David Gross  // find accumulator
579415a249300cb9ca3f560814bfde52c6796fed1f7David Gross  uint8_t *&accumPtr = mtls->accumPtr[idx];
580415a249300cb9ca3f560814bfde52c6796fed1f7David Gross  if (!accumPtr) {
581ae2ec3febedfc29376b9104413fb4042028f1265David Gross    reduce_get_accumulator(accumPtr, mtls, __func__, idx);
582415a249300cb9ca3f560814bfde52c6796fed1f7David Gross  }
583415a249300cb9ca3f560814bfde52c6796fed1f7David Gross
584415a249300cb9ca3f560814bfde52c6796fed1f7David Gross  // accumulate
585ae2ec3febedfc29376b9104413fb4042028f1265David Gross  const ReduceAccumulatorFunc_t fn = mtls->accumFunc;
586415a249300cb9ca3f560814bfde52c6796fed1f7David Gross  while (1) {
587415a249300cb9ca3f560814bfde52c6796fed1f7David Gross    uint32_t slice  = (uint32_t)__sync_fetch_and_add(&mtls->mSliceNum, 1);
588415a249300cb9ca3f560814bfde52c6796fed1f7David Gross    uint32_t yStart = mtls->start.y + slice * mtls->mSliceSize;
589415a249300cb9ca3f560814bfde52c6796fed1f7David Gross    uint32_t yEnd   = yStart + mtls->mSliceSize;
590415a249300cb9ca3f560814bfde52c6796fed1f7David Gross
591415a249300cb9ca3f560814bfde52c6796fed1f7David Gross    yEnd = rsMin(yEnd, mtls->end.y);
592415a249300cb9ca3f560814bfde52c6796fed1f7David Gross
593415a249300cb9ca3f560814bfde52c6796fed1f7David Gross    if (yEnd <= yStart) {
594415a249300cb9ca3f560814bfde52c6796fed1f7David Gross      return;
595415a249300cb9ca3f560814bfde52c6796fed1f7David Gross    }
596415a249300cb9ca3f560814bfde52c6796fed1f7David Gross
597415a249300cb9ca3f560814bfde52c6796fed1f7David Gross    for (redp.current.y = yStart; redp.current.y < yEnd; redp.current.y++) {
598415a249300cb9ca3f560814bfde52c6796fed1f7David Gross      RedpPtrSetup(mtls, &redp, mtls->start.x, redp.current.y, 0);
599415a249300cb9ca3f560814bfde52c6796fed1f7David Gross      fn(&redp, mtls->start.x, mtls->end.x, accumPtr);
600415a249300cb9ca3f560814bfde52c6796fed1f7David Gross    }
601415a249300cb9ca3f560814bfde52c6796fed1f7David Gross
602415a249300cb9ca3f560814bfde52c6796fed1f7David Gross    FormatBuf fmt;
603415a249300cb9ca3f560814bfde52c6796fed1f7David Gross    if (mtls->logReduce >= 3) {
604415a249300cb9ca3f560814bfde52c6796fed1f7David Gross      format_bytes(&fmt, accumPtr, mtls->accumSize);
605415a249300cb9ca3f560814bfde52c6796fed1f7David Gross    } else {
606415a249300cb9ca3f560814bfde52c6796fed1f7David Gross      fmt[0] = 0;
607415a249300cb9ca3f560814bfde52c6796fed1f7David Gross    }
608ae2ec3febedfc29376b9104413fb4042028f1265David Gross    REDUCE_ALOGV(mtls, 2, "walk_2d_reduce(%p): idx = %u, y in [%u, %u)%s",
609ae2ec3febedfc29376b9104413fb4042028f1265David Gross                 mtls->accumFunc, idx, yStart, yEnd, fmt);
610415a249300cb9ca3f560814bfde52c6796fed1f7David Gross  }
611415a249300cb9ca3f560814bfde52c6796fed1f7David Gross}
612415a249300cb9ca3f560814bfde52c6796fed1f7David Gross
613ae2ec3febedfc29376b9104413fb4042028f1265David Grossstatic void walk_3d_reduce(void *usr, uint32_t idx) {
614ae2ec3febedfc29376b9104413fb4042028f1265David Gross  const MTLaunchStructReduce *mtls = (const MTLaunchStructReduce *)usr;
615415a249300cb9ca3f560814bfde52c6796fed1f7David Gross  RsExpandKernelDriverInfo redp = mtls->redp;
616415a249300cb9ca3f560814bfde52c6796fed1f7David Gross
617415a249300cb9ca3f560814bfde52c6796fed1f7David Gross  // find accumulator
618415a249300cb9ca3f560814bfde52c6796fed1f7David Gross  uint8_t *&accumPtr = mtls->accumPtr[idx];
619415a249300cb9ca3f560814bfde52c6796fed1f7David Gross  if (!accumPtr) {
620ae2ec3febedfc29376b9104413fb4042028f1265David Gross    reduce_get_accumulator(accumPtr, mtls, __func__, idx);
621415a249300cb9ca3f560814bfde52c6796fed1f7David Gross  }
622415a249300cb9ca3f560814bfde52c6796fed1f7David Gross
623415a249300cb9ca3f560814bfde52c6796fed1f7David Gross  // accumulate
624ae2ec3febedfc29376b9104413fb4042028f1265David Gross  const ReduceAccumulatorFunc_t fn = mtls->accumFunc;
625415a249300cb9ca3f560814bfde52c6796fed1f7David Gross  while (1) {
626415a249300cb9ca3f560814bfde52c6796fed1f7David Gross    uint32_t slice  = (uint32_t)__sync_fetch_and_add(&mtls->mSliceNum, 1);
627415a249300cb9ca3f560814bfde52c6796fed1f7David Gross
628415a249300cb9ca3f560814bfde52c6796fed1f7David Gross    if (!SelectZSlice(mtls, &redp, slice)) {
629415a249300cb9ca3f560814bfde52c6796fed1f7David Gross      return;
630415a249300cb9ca3f560814bfde52c6796fed1f7David Gross    }
631415a249300cb9ca3f560814bfde52c6796fed1f7David Gross
632415a249300cb9ca3f560814bfde52c6796fed1f7David Gross    for (redp.current.y = mtls->start.y; redp.current.y < mtls->end.y; redp.current.y++) {
633415a249300cb9ca3f560814bfde52c6796fed1f7David Gross      RedpPtrSetup(mtls, &redp, mtls->start.x, redp.current.y, redp.current.z);
634415a249300cb9ca3f560814bfde52c6796fed1f7David Gross      fn(&redp, mtls->start.x, mtls->end.x, accumPtr);
635415a249300cb9ca3f560814bfde52c6796fed1f7David Gross    }
636415a249300cb9ca3f560814bfde52c6796fed1f7David Gross
637415a249300cb9ca3f560814bfde52c6796fed1f7David Gross    FormatBuf fmt;
638415a249300cb9ca3f560814bfde52c6796fed1f7David Gross    if (mtls->logReduce >= 3) {
639415a249300cb9ca3f560814bfde52c6796fed1f7David Gross      format_bytes(&fmt, accumPtr, mtls->accumSize);
640415a249300cb9ca3f560814bfde52c6796fed1f7David Gross    } else {
641415a249300cb9ca3f560814bfde52c6796fed1f7David Gross      fmt[0] = 0;
642415a249300cb9ca3f560814bfde52c6796fed1f7David Gross    }
643ae2ec3febedfc29376b9104413fb4042028f1265David Gross    REDUCE_ALOGV(mtls, 2, "walk_3d_reduce(%p): idx = %u, z = %u%s",
644ae2ec3febedfc29376b9104413fb4042028f1265David Gross                 mtls->accumFunc, idx, redp.current.z, fmt);
645415a249300cb9ca3f560814bfde52c6796fed1f7David Gross  }
646415a249300cb9ca3f560814bfde52c6796fed1f7David Gross}
647415a249300cb9ca3f560814bfde52c6796fed1f7David Gross
6486c1876bbef1b2c89975dce91230a168bd2d2ce4cDavid Gross// Launch a general reduce-style kernel.
6496c1876bbef1b2c89975dce91230a168bd2d2ce4cDavid Gross// Inputs:
6506c1876bbef1b2c89975dce91230a168bd2d2ce4cDavid Gross//   ains[0..inLen-1]: Array of allocations that contain the inputs
6516c1876bbef1b2c89975dce91230a168bd2d2ce4cDavid Gross//   aout:             The allocation that will hold the output
6526c1876bbef1b2c89975dce91230a168bd2d2ce4cDavid Gross//   mtls:             Holds launch parameters
653ae2ec3febedfc29376b9104413fb4042028f1265David Grossvoid RsdCpuReferenceImpl::launchReduce(const Allocation ** ains,
654ae2ec3febedfc29376b9104413fb4042028f1265David Gross                                       uint32_t inLen,
655ae2ec3febedfc29376b9104413fb4042028f1265David Gross                                       Allocation * aout,
656ae2ec3febedfc29376b9104413fb4042028f1265David Gross                                       MTLaunchStructReduce *mtls) {
6576760f7ba7934ddd51938a8d0206fc41c2a7cb419David Gross  mtls->logReduce = mRSC->props.mLogReduce;
65810adb0c2029f112b5738228617d5645f6ecea0c5David Gross  if ((mWorkers.mCount >= 1) && mtls->isThreadable && !mInKernel) {
659ae2ec3febedfc29376b9104413fb4042028f1265David Gross    launchReduceParallel(ains, inLen, aout, mtls);
66010adb0c2029f112b5738228617d5645f6ecea0c5David Gross  } else {
661ae2ec3febedfc29376b9104413fb4042028f1265David Gross    launchReduceSerial(ains, inLen, aout, mtls);
66210adb0c2029f112b5738228617d5645f6ecea0c5David Gross  }
66310adb0c2029f112b5738228617d5645f6ecea0c5David Gross}
66410adb0c2029f112b5738228617d5645f6ecea0c5David Gross
66510adb0c2029f112b5738228617d5645f6ecea0c5David Gross// Launch a general reduce-style kernel, single-threaded.
66610adb0c2029f112b5738228617d5645f6ecea0c5David Gross// Inputs:
66710adb0c2029f112b5738228617d5645f6ecea0c5David Gross//   ains[0..inLen-1]: Array of allocations that contain the inputs
66810adb0c2029f112b5738228617d5645f6ecea0c5David Gross//   aout:             The allocation that will hold the output
66910adb0c2029f112b5738228617d5645f6ecea0c5David Gross//   mtls:             Holds launch parameters
670ae2ec3febedfc29376b9104413fb4042028f1265David Grossvoid RsdCpuReferenceImpl::launchReduceSerial(const Allocation ** ains,
671ae2ec3febedfc29376b9104413fb4042028f1265David Gross                                             uint32_t inLen,
672ae2ec3febedfc29376b9104413fb4042028f1265David Gross                                             Allocation * aout,
673ae2ec3febedfc29376b9104413fb4042028f1265David Gross                                             MTLaunchStructReduce *mtls) {
674ae2ec3febedfc29376b9104413fb4042028f1265David Gross  REDUCE_ALOGV(mtls, 1, "launchReduceSerial(%p): %u x %u x %u", mtls->accumFunc,
675ae2ec3febedfc29376b9104413fb4042028f1265David Gross               mtls->redp.dim.x, mtls->redp.dim.y, mtls->redp.dim.z);
67610adb0c2029f112b5738228617d5645f6ecea0c5David Gross
6776c1876bbef1b2c89975dce91230a168bd2d2ce4cDavid Gross  // In the presence of outconverter, we allocate temporary memory for
6786c1876bbef1b2c89975dce91230a168bd2d2ce4cDavid Gross  // the accumulator.
6796c1876bbef1b2c89975dce91230a168bd2d2ce4cDavid Gross  //
6806c1876bbef1b2c89975dce91230a168bd2d2ce4cDavid Gross  // In the absence of outconverter, we use the output allocation as the
6816c1876bbef1b2c89975dce91230a168bd2d2ce4cDavid Gross  // accumulator.
6826c1876bbef1b2c89975dce91230a168bd2d2ce4cDavid Gross  uint8_t *const accumPtr = (mtls->outFunc
6836c1876bbef1b2c89975dce91230a168bd2d2ce4cDavid Gross                             ? static_cast<uint8_t *>(malloc(mtls->accumSize))
6846c1876bbef1b2c89975dce91230a168bd2d2ce4cDavid Gross                             : mtls->redp.outPtr[0]);
6856c1876bbef1b2c89975dce91230a168bd2d2ce4cDavid Gross
6866c1876bbef1b2c89975dce91230a168bd2d2ce4cDavid Gross  // initialize
6876c1876bbef1b2c89975dce91230a168bd2d2ce4cDavid Gross  if (mtls->initFunc) {
6886c1876bbef1b2c89975dce91230a168bd2d2ce4cDavid Gross    mtls->initFunc(accumPtr);
6896c1876bbef1b2c89975dce91230a168bd2d2ce4cDavid Gross  } else {
6906c1876bbef1b2c89975dce91230a168bd2d2ce4cDavid Gross    memset(accumPtr, 0, mtls->accumSize);
6916c1876bbef1b2c89975dce91230a168bd2d2ce4cDavid Gross  }
6926c1876bbef1b2c89975dce91230a168bd2d2ce4cDavid Gross
6936c1876bbef1b2c89975dce91230a168bd2d2ce4cDavid Gross  // accumulate
694ae2ec3febedfc29376b9104413fb4042028f1265David Gross  const ReduceAccumulatorFunc_t fn = mtls->accumFunc;
6956c1876bbef1b2c89975dce91230a168bd2d2ce4cDavid Gross  uint32_t slice = 0;
6966c1876bbef1b2c89975dce91230a168bd2d2ce4cDavid Gross  while (SelectOuterSlice(mtls, &mtls->redp, slice++)) {
6976c1876bbef1b2c89975dce91230a168bd2d2ce4cDavid Gross    for (mtls->redp.current.y = mtls->start.y;
6986c1876bbef1b2c89975dce91230a168bd2d2ce4cDavid Gross         mtls->redp.current.y < mtls->end.y;
6996c1876bbef1b2c89975dce91230a168bd2d2ce4cDavid Gross         mtls->redp.current.y++) {
7006c1876bbef1b2c89975dce91230a168bd2d2ce4cDavid Gross      RedpPtrSetup(mtls, &mtls->redp, mtls->start.x, mtls->redp.current.y, mtls->redp.current.z);
701