rsCpuCore.cpp revision 962e720b3d1c27bcfec90374ff393584b99577b3
1/* 2 * Copyright (C) 2012 The Android Open Source Project 3 * 4 * Licensed under the Apache License, Version 2.0 (the "License"); 5 * you may not use this file except in compliance with the License. 6 * You may obtain a copy of the License at 7 * 8 * http://www.apache.org/licenses/LICENSE-2.0 9 * 10 * Unless required by applicable law or agreed to in writing, software 11 * distributed under the License is distributed on an "AS IS" BASIS, 12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 * See the License for the specific language governing permissions and 14 * limitations under the License. 15 */ 16 17#include "rsCpuCore.h" 18#include "rsCpuScript.h" 19#include "rsCpuScriptGroup.h" 20 21#include <malloc.h> 22#include "rsContext.h" 23 24#include <sys/types.h> 25#include <sys/resource.h> 26#include <sched.h> 27#include <sys/syscall.h> 28#include <string.h> 29 30#ifndef RS_SERVER 31#include <cutils/properties.h> 32#include "utils/StopWatch.h" 33#endif 34 35#ifdef RS_SERVER 36// Android exposes gettid(), standard Linux does not 37static pid_t gettid() { 38 return syscall(SYS_gettid); 39} 40#endif 41 42using namespace android; 43using namespace android::renderscript; 44 45typedef void (*outer_foreach_t)( 46 const android::renderscript::RsForEachStubParamStruct *, 47 uint32_t x1, uint32_t x2, 48 uint32_t instep, uint32_t outstep); 49 50 51static pthread_key_t gThreadTLSKey = 0; 52static uint32_t gThreadTLSKeyCount = 0; 53static pthread_mutex_t gInitMutex = PTHREAD_MUTEX_INITIALIZER; 54 55RsdCpuReference::~RsdCpuReference() { 56} 57 58RsdCpuReference * RsdCpuReference::create(Context *rsc, uint32_t version_major, 59 uint32_t version_minor, sym_lookup_t lfn, script_lookup_t slfn 60#ifndef RS_COMPATIBILITY_LIB 61 , bcc::RSLinkRuntimeCallback pLinkRuntimeCallback 62#endif 63 ) { 64 65 RsdCpuReferenceImpl *cpu = new RsdCpuReferenceImpl(rsc); 66 if (!cpu) { 67 return NULL; 68 } 69 if (!cpu->init(version_major, version_minor, lfn, slfn)) { 70 delete cpu; 71 return NULL; 72 } 73 74#ifndef RS_COMPATIBILITY_LIB 75 cpu->setLinkRuntimeCallback(pLinkRuntimeCallback); 76#endif 77 78 return cpu; 79} 80 81 82Context * RsdCpuReference::getTlsContext() { 83 ScriptTLSStruct * tls = (ScriptTLSStruct *)pthread_getspecific(gThreadTLSKey); 84 return tls->mContext; 85} 86 87const Script * RsdCpuReference::getTlsScript() { 88 ScriptTLSStruct * tls = (ScriptTLSStruct *)pthread_getspecific(gThreadTLSKey); 89 return tls->mScript; 90} 91 92pthread_key_t RsdCpuReference::getThreadTLSKey(){ return gThreadTLSKey; } 93 94//////////////////////////////////////////////////////////// 95/// 96 97RsdCpuReferenceImpl::RsdCpuReferenceImpl(Context *rsc) { 98 mRSC = rsc; 99 100 version_major = 0; 101 version_minor = 0; 102 mInForEach = false; 103 memset(&mWorkers, 0, sizeof(mWorkers)); 104 memset(&mTlsStruct, 0, sizeof(mTlsStruct)); 105 mExit = false; 106#ifndef RS_COMPATIBILITY_LIB 107 mLinkRuntimeCallback = NULL; 108#endif 109} 110 111 112void * RsdCpuReferenceImpl::helperThreadProc(void *vrsc) { 113 RsdCpuReferenceImpl *dc = (RsdCpuReferenceImpl *)vrsc; 114 115 uint32_t idx = __sync_fetch_and_add(&dc->mWorkers.mLaunchCount, 1); 116 117 //ALOGV("RS helperThread starting %p idx=%i", dc, idx); 118 119 dc->mWorkers.mLaunchSignals[idx].init(); 120 dc->mWorkers.mNativeThreadId[idx] = gettid(); 121 122 memset(&dc->mTlsStruct, 0, sizeof(dc->mTlsStruct)); 123 int status = pthread_setspecific(gThreadTLSKey, &dc->mTlsStruct); 124 if (status) { 125 ALOGE("pthread_setspecific %i", status); 126 } 127 128#if 0 129 typedef struct {uint64_t bits[1024 / 64]; } cpu_set_t; 130 cpu_set_t cpuset; 131 memset(&cpuset, 0, sizeof(cpuset)); 132 cpuset.bits[idx / 64] |= 1ULL << (idx % 64); 133 int ret = syscall(241, rsc->mWorkers.mNativeThreadId[idx], 134 sizeof(cpuset), &cpuset); 135 ALOGE("SETAFFINITY ret = %i %s", ret, EGLUtils::strerror(ret)); 136#endif 137 138 while (!dc->mExit) { 139 dc->mWorkers.mLaunchSignals[idx].wait(); 140 if (dc->mWorkers.mLaunchCallback) { 141 // idx +1 is used because the calling thread is always worker 0. 142 dc->mWorkers.mLaunchCallback(dc->mWorkers.mLaunchData, idx+1); 143 } 144 __sync_fetch_and_sub(&dc->mWorkers.mRunningCount, 1); 145 dc->mWorkers.mCompleteSignal.set(); 146 } 147 148 //ALOGV("RS helperThread exited %p idx=%i", dc, idx); 149 return NULL; 150} 151 152void RsdCpuReferenceImpl::launchThreads(WorkerCallback_t cbk, void *data) { 153 mWorkers.mLaunchData = data; 154 mWorkers.mLaunchCallback = cbk; 155 156 // fast path for very small launches 157 MTLaunchStruct *mtls = (MTLaunchStruct *)data; 158 if (mtls && mtls->fep.dimY <= 1 && mtls->xEnd <= mtls->xStart + mtls->mSliceSize) { 159 if (mWorkers.mLaunchCallback) { 160 mWorkers.mLaunchCallback(mWorkers.mLaunchData, 0); 161 } 162 return; 163 } 164 165 mWorkers.mRunningCount = mWorkers.mCount; 166 __sync_synchronize(); 167 168 for (uint32_t ct = 0; ct < mWorkers.mCount; ct++) { 169 mWorkers.mLaunchSignals[ct].set(); 170 } 171 172 // We use the calling thread as one of the workers so we can start without 173 // the delay of the thread wakeup. 174 if (mWorkers.mLaunchCallback) { 175 mWorkers.mLaunchCallback(mWorkers.mLaunchData, 0); 176 } 177 178 while (__sync_fetch_and_or(&mWorkers.mRunningCount, 0) != 0) { 179 mWorkers.mCompleteSignal.wait(); 180 } 181} 182 183 184void RsdCpuReferenceImpl::lockMutex() { 185 pthread_mutex_lock(&gInitMutex); 186} 187 188void RsdCpuReferenceImpl::unlockMutex() { 189 pthread_mutex_unlock(&gInitMutex); 190} 191 192bool RsdCpuReferenceImpl::init(uint32_t version_major, uint32_t version_minor, 193 sym_lookup_t lfn, script_lookup_t slfn) { 194 195 mSymLookupFn = lfn; 196 mScriptLookupFn = slfn; 197 198 lockMutex(); 199 if (!gThreadTLSKeyCount) { 200 int status = pthread_key_create(&gThreadTLSKey, NULL); 201 if (status) { 202 ALOGE("Failed to init thread tls key."); 203 unlockMutex(); 204 return false; 205 } 206 } 207 gThreadTLSKeyCount++; 208 unlockMutex(); 209 210 mTlsStruct.mContext = mRSC; 211 mTlsStruct.mScript = NULL; 212 int status = pthread_setspecific(gThreadTLSKey, &mTlsStruct); 213 if (status) { 214 ALOGE("pthread_setspecific %i", status); 215 } 216 217 int cpu = sysconf(_SC_NPROCESSORS_ONLN); 218 if(mRSC->props.mDebugMaxThreads) { 219 cpu = mRSC->props.mDebugMaxThreads; 220 } 221 if (cpu < 2) { 222 mWorkers.mCount = 0; 223 return true; 224 } 225 226 // Subtract one from the cpu count because we also use the command thread as a worker. 227 mWorkers.mCount = (uint32_t)(cpu - 1); 228 229 ALOGV("%p Launching thread(s), CPUs %i", mRSC, mWorkers.mCount + 1); 230 231 mWorkers.mThreadId = (pthread_t *) calloc(mWorkers.mCount, sizeof(pthread_t)); 232 mWorkers.mNativeThreadId = (pid_t *) calloc(mWorkers.mCount, sizeof(pid_t)); 233 mWorkers.mLaunchSignals = new Signal[mWorkers.mCount]; 234 mWorkers.mLaunchCallback = NULL; 235 236 mWorkers.mCompleteSignal.init(); 237 238 mWorkers.mRunningCount = mWorkers.mCount; 239 mWorkers.mLaunchCount = 0; 240 __sync_synchronize(); 241 242 pthread_attr_t threadAttr; 243 status = pthread_attr_init(&threadAttr); 244 if (status) { 245 ALOGE("Failed to init thread attribute."); 246 return false; 247 } 248 249 for (uint32_t ct=0; ct < mWorkers.mCount; ct++) { 250 status = pthread_create(&mWorkers.mThreadId[ct], &threadAttr, helperThreadProc, this); 251 if (status) { 252 mWorkers.mCount = ct; 253 ALOGE("Created fewer than expected number of RS threads."); 254 break; 255 } 256 } 257 while (__sync_fetch_and_or(&mWorkers.mRunningCount, 0) != 0) { 258 usleep(100); 259 } 260 261 pthread_attr_destroy(&threadAttr); 262 return true; 263} 264 265 266void RsdCpuReferenceImpl::setPriority(int32_t priority) { 267 for (uint32_t ct=0; ct < mWorkers.mCount; ct++) { 268 setpriority(PRIO_PROCESS, mWorkers.mNativeThreadId[ct], priority); 269 } 270} 271 272RsdCpuReferenceImpl::~RsdCpuReferenceImpl() { 273 mExit = true; 274 mWorkers.mLaunchData = NULL; 275 mWorkers.mLaunchCallback = NULL; 276 mWorkers.mRunningCount = mWorkers.mCount; 277 __sync_synchronize(); 278 for (uint32_t ct = 0; ct < mWorkers.mCount; ct++) { 279 mWorkers.mLaunchSignals[ct].set(); 280 } 281 void *res; 282 for (uint32_t ct = 0; ct < mWorkers.mCount; ct++) { 283 pthread_join(mWorkers.mThreadId[ct], &res); 284 } 285 rsAssert(__sync_fetch_and_or(&mWorkers.mRunningCount, 0) == 0); 286 287 // Global structure cleanup. 288 lockMutex(); 289 --gThreadTLSKeyCount; 290 if (!gThreadTLSKeyCount) { 291 pthread_key_delete(gThreadTLSKey); 292 } 293 unlockMutex(); 294 295} 296 297typedef void (*rs_t)(const void *, void *, const void *, uint32_t, uint32_t, uint32_t, uint32_t); 298 299static void wc_xy(void *usr, uint32_t idx) { 300 MTLaunchStruct *mtls = (MTLaunchStruct *)usr; 301 RsForEachStubParamStruct p; 302 memcpy(&p, &mtls->fep, sizeof(p)); 303 p.lid = idx; 304 uint32_t sig = mtls->sig; 305 306 outer_foreach_t fn = (outer_foreach_t) mtls->kernel; 307 while (1) { 308 uint32_t slice = (uint32_t)__sync_fetch_and_add(&mtls->mSliceNum, 1); 309 uint32_t yStart = mtls->yStart + slice * mtls->mSliceSize; 310 uint32_t yEnd = yStart + mtls->mSliceSize; 311 yEnd = rsMin(yEnd, mtls->yEnd); 312 if (yEnd <= yStart) { 313 return; 314 } 315 316 //ALOGE("usr idx %i, x %i,%i y %i,%i", idx, mtls->xStart, mtls->xEnd, yStart, yEnd); 317 //ALOGE("usr ptr in %p, out %p", mtls->fep.ptrIn, mtls->fep.ptrOut); 318 319 for (p.y = yStart; p.y < yEnd; p.y++) { 320 p.out = mtls->fep.ptrOut + (mtls->fep.yStrideOut * p.y) + 321 (mtls->fep.eStrideOut * mtls->xStart); 322 p.in = mtls->fep.ptrIn + (mtls->fep.yStrideIn * p.y) + 323 (mtls->fep.eStrideIn * mtls->xStart); 324 fn(&p, mtls->xStart, mtls->xEnd, mtls->fep.eStrideIn, mtls->fep.eStrideOut); 325 } 326 } 327} 328 329static void wc_x(void *usr, uint32_t idx) { 330 MTLaunchStruct *mtls = (MTLaunchStruct *)usr; 331 RsForEachStubParamStruct p; 332 memcpy(&p, &mtls->fep, sizeof(p)); 333 p.lid = idx; 334 uint32_t sig = mtls->sig; 335 336 outer_foreach_t fn = (outer_foreach_t) mtls->kernel; 337 while (1) { 338 uint32_t slice = (uint32_t)__sync_fetch_and_add(&mtls->mSliceNum, 1); 339 uint32_t xStart = mtls->xStart + slice * mtls->mSliceSize; 340 uint32_t xEnd = xStart + mtls->mSliceSize; 341 xEnd = rsMin(xEnd, mtls->xEnd); 342 if (xEnd <= xStart) { 343 return; 344 } 345 346 //ALOGE("usr slice %i idx %i, x %i,%i", slice, idx, xStart, xEnd); 347 //ALOGE("usr ptr in %p, out %p", mtls->fep.ptrIn, mtls->fep.ptrOut); 348 349 p.out = mtls->fep.ptrOut + (mtls->fep.eStrideOut * xStart); 350 p.in = mtls->fep.ptrIn + (mtls->fep.eStrideIn * xStart); 351 fn(&p, xStart, xEnd, mtls->fep.eStrideIn, mtls->fep.eStrideOut); 352 } 353} 354 355void RsdCpuReferenceImpl::launchThreads(const Allocation * ain, Allocation * aout, 356 const RsScriptCall *sc, MTLaunchStruct *mtls) { 357 358 //android::StopWatch kernel_time("kernel time"); 359 360 if ((mWorkers.mCount >= 1) && mtls->isThreadable && !mInForEach) { 361 const size_t targetByteChunk = 16 * 1024; 362 mInForEach = true; 363 if (mtls->fep.dimY > 1) { 364 uint32_t s1 = mtls->fep.dimY / ((mWorkers.mCount + 1) * 4); 365 uint32_t s2 = 0; 366 367 // This chooses our slice size to rate limit atomic ops to 368 // one per 16k bytes of reads/writes. 369 if (mtls->fep.yStrideOut) { 370 s2 = targetByteChunk / mtls->fep.yStrideOut; 371 } else { 372 s2 = targetByteChunk / mtls->fep.yStrideIn; 373 } 374 mtls->mSliceSize = rsMin(s1, s2); 375 376 if(mtls->mSliceSize < 1) { 377 mtls->mSliceSize = 1; 378 } 379 380 // mtls->mSliceSize = 2; 381 launchThreads(wc_xy, mtls); 382 } else { 383 uint32_t s1 = mtls->fep.dimX / ((mWorkers.mCount + 1) * 4); 384 uint32_t s2 = 0; 385 386 // This chooses our slice size to rate limit atomic ops to 387 // one per 16k bytes of reads/writes. 388 if (mtls->fep.eStrideOut) { 389 s2 = targetByteChunk / mtls->fep.eStrideOut; 390 } else { 391 s2 = targetByteChunk / mtls->fep.eStrideIn; 392 } 393 mtls->mSliceSize = rsMin(s1, s2); 394 395 if(mtls->mSliceSize < 1) { 396 mtls->mSliceSize = 1; 397 } 398 399 launchThreads(wc_x, mtls); 400 } 401 mInForEach = false; 402 403 //ALOGE("launch 1"); 404 } else { 405 RsForEachStubParamStruct p; 406 memcpy(&p, &mtls->fep, sizeof(p)); 407 uint32_t sig = mtls->sig; 408 409 //ALOGE("launch 3"); 410 outer_foreach_t fn = (outer_foreach_t) mtls->kernel; 411 for (p.ar[0] = mtls->arrayStart; p.ar[0] < mtls->arrayEnd; p.ar[0]++) { 412 for (p.z = mtls->zStart; p.z < mtls->zEnd; p.z++) { 413 for (p.y = mtls->yStart; p.y < mtls->yEnd; p.y++) { 414 uint32_t offset = mtls->fep.dimY * mtls->fep.dimZ * p.ar[0] + 415 mtls->fep.dimY * p.z + p.y; 416 p.out = mtls->fep.ptrOut + (mtls->fep.yStrideOut * offset) + 417 (mtls->fep.eStrideOut * mtls->xStart); 418 p.in = mtls->fep.ptrIn + (mtls->fep.yStrideIn * offset) + 419 (mtls->fep.eStrideIn * mtls->xStart); 420 fn(&p, mtls->xStart, mtls->xEnd, mtls->fep.eStrideIn, mtls->fep.eStrideOut); 421 } 422 } 423 } 424 } 425} 426 427RsdCpuScriptImpl * RsdCpuReferenceImpl::setTLS(RsdCpuScriptImpl *sc) { 428 //ALOGE("setTls %p", sc); 429 ScriptTLSStruct * tls = (ScriptTLSStruct *)pthread_getspecific(gThreadTLSKey); 430 rsAssert(tls); 431 RsdCpuScriptImpl *old = tls->mImpl; 432 tls->mImpl = sc; 433 tls->mContext = mRSC; 434 if (sc) { 435 tls->mScript = sc->getScript(); 436 } else { 437 tls->mScript = NULL; 438 } 439 return old; 440} 441 442const RsdCpuReference::CpuSymbol * RsdCpuReferenceImpl::symLookup(const char *name) { 443 return mSymLookupFn(mRSC, name); 444} 445 446 447RsdCpuReference::CpuScript * RsdCpuReferenceImpl::createScript(const ScriptC *s, 448 char const *resName, char const *cacheDir, 449 uint8_t const *bitcode, size_t bitcodeSize, 450 uint32_t flags) { 451 452 RsdCpuScriptImpl *i = new RsdCpuScriptImpl(this, s); 453 if (!i->init(resName, cacheDir, bitcode, bitcodeSize, flags)) { 454 delete i; 455 return NULL; 456 } 457 return i; 458} 459 460extern RsdCpuScriptImpl * rsdIntrinsic_3DLUT(RsdCpuReferenceImpl *ctx, 461 const Script *s, const Element *e); 462extern RsdCpuScriptImpl * rsdIntrinsic_Convolve3x3(RsdCpuReferenceImpl *ctx, 463 const Script *s, const Element *e); 464extern RsdCpuScriptImpl * rsdIntrinsic_ColorMatrix(RsdCpuReferenceImpl *ctx, 465 const Script *s, const Element *e); 466extern RsdCpuScriptImpl * rsdIntrinsic_LUT(RsdCpuReferenceImpl *ctx, 467 const Script *s, const Element *e); 468extern RsdCpuScriptImpl * rsdIntrinsic_Convolve5x5(RsdCpuReferenceImpl *ctx, 469 const Script *s, const Element *e); 470extern RsdCpuScriptImpl * rsdIntrinsic_Blur(RsdCpuReferenceImpl *ctx, 471 const Script *s, const Element *e); 472extern RsdCpuScriptImpl * rsdIntrinsic_YuvToRGB(RsdCpuReferenceImpl *ctx, 473 const Script *s, const Element *e); 474extern RsdCpuScriptImpl * rsdIntrinsic_Blend(RsdCpuReferenceImpl *ctx, 475 const Script *s, const Element *e); 476 477RsdCpuReference::CpuScript * RsdCpuReferenceImpl::createIntrinsic(const Script *s, 478 RsScriptIntrinsicID iid, Element *e) { 479 480 RsdCpuScriptImpl *i = NULL; 481 switch (iid) { 482 case RS_SCRIPT_INTRINSIC_ID_3DLUT: 483 i = rsdIntrinsic_3DLUT(this, s, e); 484 break; 485 case RS_SCRIPT_INTRINSIC_ID_CONVOLVE_3x3: 486 i = rsdIntrinsic_Convolve3x3(this, s, e); 487 break; 488 case RS_SCRIPT_INTRINSIC_ID_COLOR_MATRIX: 489 i = rsdIntrinsic_ColorMatrix(this, s, e); 490 break; 491 case RS_SCRIPT_INTRINSIC_ID_LUT: 492 i = rsdIntrinsic_LUT(this, s, e); 493 break; 494 case RS_SCRIPT_INTRINSIC_ID_CONVOLVE_5x5: 495 i = rsdIntrinsic_Convolve5x5(this, s, e); 496 break; 497 case RS_SCRIPT_INTRINSIC_ID_BLUR: 498 i = rsdIntrinsic_Blur(this, s, e); 499 break; 500 case RS_SCRIPT_INTRINSIC_ID_YUV_TO_RGB: 501 i = rsdIntrinsic_YuvToRGB(this, s, e); 502 break; 503 case RS_SCRIPT_INTRINSIC_ID_BLEND: 504 i = rsdIntrinsic_Blend(this, s, e); 505 break; 506 507 default: 508 rsAssert(0); 509 } 510 511 return i; 512} 513 514RsdCpuReference::CpuScriptGroup * RsdCpuReferenceImpl::createScriptGroup(const ScriptGroup *sg) { 515 CpuScriptGroupImpl *sgi = new CpuScriptGroupImpl(this, sg); 516 if (!sgi->init()) { 517 delete sgi; 518 return NULL; 519 } 520 return sgi; 521} 522 523 524