rsCpuCore.cpp revision 140a7acade66ab5d1f3dc55803a3a65a71f3f86c
1/* 2 * Copyright (C) 2012 The Android Open Source Project 3 * 4 * Licensed under the Apache License, Version 2.0 (the "License"); 5 * you may not use this file except in compliance with the License. 6 * You may obtain a copy of the License at 7 * 8 * http://www.apache.org/licenses/LICENSE-2.0 9 * 10 * Unless required by applicable law or agreed to in writing, software 11 * distributed under the License is distributed on an "AS IS" BASIS, 12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 * See the License for the specific language governing permissions and 14 * limitations under the License. 15 */ 16 17#include "rsCpuCore.h" 18#include "rsCpuScript.h" 19#include "rsCpuScriptGroup.h" 20 21#include <malloc.h> 22#include "rsContext.h" 23 24#include <sys/types.h> 25#include <sys/resource.h> 26#include <sched.h> 27#include <sys/syscall.h> 28#include <string.h> 29#include <unistd.h> 30 31#if !defined(RS_SERVER) && !defined(RS_COMPATIBILITY_LIB) 32#include <cutils/properties.h> 33#include "utils/StopWatch.h" 34#endif 35 36#ifdef RS_SERVER 37// Android exposes gettid(), standard Linux does not 38static pid_t gettid() { 39 return syscall(SYS_gettid); 40} 41#endif 42 43using namespace android; 44using namespace android::renderscript; 45 46typedef void (*outer_foreach_t)( 47 const android::renderscript::RsForEachStubParamStruct *, 48 uint32_t x1, uint32_t x2, 49 uint32_t instep, uint32_t outstep); 50 51 52static pthread_key_t gThreadTLSKey = 0; 53static uint32_t gThreadTLSKeyCount = 0; 54static pthread_mutex_t gInitMutex = PTHREAD_MUTEX_INITIALIZER; 55 56RsdCpuReference::~RsdCpuReference() { 57} 58 59RsdCpuReference * RsdCpuReference::create(Context *rsc, uint32_t version_major, 60 uint32_t version_minor, sym_lookup_t lfn, script_lookup_t slfn 61#ifndef RS_COMPATIBILITY_LIB 62 , bcc::RSLinkRuntimeCallback pLinkRuntimeCallback, 63 RSSelectRTCallback pSelectRTCallback 64#endif 65 ) { 66 67 RsdCpuReferenceImpl *cpu = new RsdCpuReferenceImpl(rsc); 68 if (!cpu) { 69 return NULL; 70 } 71 if (!cpu->init(version_major, version_minor, lfn, slfn)) { 72 delete cpu; 73 return NULL; 74 } 75 76#ifndef RS_COMPATIBILITY_LIB 77 cpu->setLinkRuntimeCallback(pLinkRuntimeCallback); 78 cpu->setSelectRTCallback(pSelectRTCallback); 79#endif 80 81 return cpu; 82} 83 84 85Context * RsdCpuReference::getTlsContext() { 86 ScriptTLSStruct * tls = (ScriptTLSStruct *)pthread_getspecific(gThreadTLSKey); 87 return tls->mContext; 88} 89 90const Script * RsdCpuReference::getTlsScript() { 91 ScriptTLSStruct * tls = (ScriptTLSStruct *)pthread_getspecific(gThreadTLSKey); 92 return tls->mScript; 93} 94 95pthread_key_t RsdCpuReference::getThreadTLSKey(){ return gThreadTLSKey; } 96 97//////////////////////////////////////////////////////////// 98/// 99 100RsdCpuReferenceImpl::RsdCpuReferenceImpl(Context *rsc) { 101 mRSC = rsc; 102 103 version_major = 0; 104 version_minor = 0; 105 mInForEach = false; 106 memset(&mWorkers, 0, sizeof(mWorkers)); 107 memset(&mTlsStruct, 0, sizeof(mTlsStruct)); 108 mExit = false; 109#ifndef RS_COMPATIBILITY_LIB 110 mLinkRuntimeCallback = NULL; 111 mSelectRTCallback = NULL; 112 mSetupCompilerCallback = NULL; 113#endif 114} 115 116 117void * RsdCpuReferenceImpl::helperThreadProc(void *vrsc) { 118 RsdCpuReferenceImpl *dc = (RsdCpuReferenceImpl *)vrsc; 119 120 uint32_t idx = __sync_fetch_and_add(&dc->mWorkers.mLaunchCount, 1); 121 122 //ALOGV("RS helperThread starting %p idx=%i", dc, idx); 123 124 dc->mWorkers.mLaunchSignals[idx].init(); 125 dc->mWorkers.mNativeThreadId[idx] = gettid(); 126 127 memset(&dc->mTlsStruct, 0, sizeof(dc->mTlsStruct)); 128 int status = pthread_setspecific(gThreadTLSKey, &dc->mTlsStruct); 129 if (status) { 130 ALOGE("pthread_setspecific %i", status); 131 } 132 133#if 0 134 typedef struct {uint64_t bits[1024 / 64]; } cpu_set_t; 135 cpu_set_t cpuset; 136 memset(&cpuset, 0, sizeof(cpuset)); 137 cpuset.bits[idx / 64] |= 1ULL << (idx % 64); 138 int ret = syscall(241, rsc->mWorkers.mNativeThreadId[idx], 139 sizeof(cpuset), &cpuset); 140 ALOGE("SETAFFINITY ret = %i %s", ret, EGLUtils::strerror(ret)); 141#endif 142 143 while (!dc->mExit) { 144 dc->mWorkers.mLaunchSignals[idx].wait(); 145 if (dc->mWorkers.mLaunchCallback) { 146 // idx +1 is used because the calling thread is always worker 0. 147 dc->mWorkers.mLaunchCallback(dc->mWorkers.mLaunchData, idx+1); 148 } 149 __sync_fetch_and_sub(&dc->mWorkers.mRunningCount, 1); 150 dc->mWorkers.mCompleteSignal.set(); 151 } 152 153 //ALOGV("RS helperThread exited %p idx=%i", dc, idx); 154 return NULL; 155} 156 157void RsdCpuReferenceImpl::launchThreads(WorkerCallback_t cbk, void *data) { 158 mWorkers.mLaunchData = data; 159 mWorkers.mLaunchCallback = cbk; 160 161 // fast path for very small launches 162 MTLaunchStruct *mtls = (MTLaunchStruct *)data; 163 if (mtls && mtls->fep.dimY <= 1 && mtls->xEnd <= mtls->xStart + mtls->mSliceSize) { 164 if (mWorkers.mLaunchCallback) { 165 mWorkers.mLaunchCallback(mWorkers.mLaunchData, 0); 166 } 167 return; 168 } 169 170 mWorkers.mRunningCount = mWorkers.mCount; 171 __sync_synchronize(); 172 173 for (uint32_t ct = 0; ct < mWorkers.mCount; ct++) { 174 mWorkers.mLaunchSignals[ct].set(); 175 } 176 177 // We use the calling thread as one of the workers so we can start without 178 // the delay of the thread wakeup. 179 if (mWorkers.mLaunchCallback) { 180 mWorkers.mLaunchCallback(mWorkers.mLaunchData, 0); 181 } 182 183 while (__sync_fetch_and_or(&mWorkers.mRunningCount, 0) != 0) { 184 mWorkers.mCompleteSignal.wait(); 185 } 186} 187 188 189void RsdCpuReferenceImpl::lockMutex() { 190 pthread_mutex_lock(&gInitMutex); 191} 192 193void RsdCpuReferenceImpl::unlockMutex() { 194 pthread_mutex_unlock(&gInitMutex); 195} 196 197bool RsdCpuReferenceImpl::init(uint32_t version_major, uint32_t version_minor, 198 sym_lookup_t lfn, script_lookup_t slfn) { 199 200 mSymLookupFn = lfn; 201 mScriptLookupFn = slfn; 202 203 lockMutex(); 204 if (!gThreadTLSKeyCount) { 205 int status = pthread_key_create(&gThreadTLSKey, NULL); 206 if (status) { 207 ALOGE("Failed to init thread tls key."); 208 unlockMutex(); 209 return false; 210 } 211 } 212 gThreadTLSKeyCount++; 213 unlockMutex(); 214 215 mTlsStruct.mContext = mRSC; 216 mTlsStruct.mScript = NULL; 217 int status = pthread_setspecific(gThreadTLSKey, &mTlsStruct); 218 if (status) { 219 ALOGE("pthread_setspecific %i", status); 220 } 221 222 int cpu = sysconf(_SC_NPROCESSORS_ONLN); 223 if(mRSC->props.mDebugMaxThreads) { 224 cpu = mRSC->props.mDebugMaxThreads; 225 } 226 if (cpu < 2) { 227 mWorkers.mCount = 0; 228 return true; 229 } 230 231 // Subtract one from the cpu count because we also use the command thread as a worker. 232 mWorkers.mCount = (uint32_t)(cpu - 1); 233 234 ALOGV("%p Launching thread(s), CPUs %i", mRSC, mWorkers.mCount + 1); 235 236 mWorkers.mThreadId = (pthread_t *) calloc(mWorkers.mCount, sizeof(pthread_t)); 237 mWorkers.mNativeThreadId = (pid_t *) calloc(mWorkers.mCount, sizeof(pid_t)); 238 mWorkers.mLaunchSignals = new Signal[mWorkers.mCount]; 239 mWorkers.mLaunchCallback = NULL; 240 241 mWorkers.mCompleteSignal.init(); 242 243 mWorkers.mRunningCount = mWorkers.mCount; 244 mWorkers.mLaunchCount = 0; 245 __sync_synchronize(); 246 247 pthread_attr_t threadAttr; 248 status = pthread_attr_init(&threadAttr); 249 if (status) { 250 ALOGE("Failed to init thread attribute."); 251 return false; 252 } 253 254 for (uint32_t ct=0; ct < mWorkers.mCount; ct++) { 255 status = pthread_create(&mWorkers.mThreadId[ct], &threadAttr, helperThreadProc, this); 256 if (status) { 257 mWorkers.mCount = ct; 258 ALOGE("Created fewer than expected number of RS threads."); 259 break; 260 } 261 } 262 while (__sync_fetch_and_or(&mWorkers.mRunningCount, 0) != 0) { 263 usleep(100); 264 } 265 266 pthread_attr_destroy(&threadAttr); 267 return true; 268} 269 270 271void RsdCpuReferenceImpl::setPriority(int32_t priority) { 272 for (uint32_t ct=0; ct < mWorkers.mCount; ct++) { 273 setpriority(PRIO_PROCESS, mWorkers.mNativeThreadId[ct], priority); 274 } 275} 276 277RsdCpuReferenceImpl::~RsdCpuReferenceImpl() { 278 mExit = true; 279 mWorkers.mLaunchData = NULL; 280 mWorkers.mLaunchCallback = NULL; 281 mWorkers.mRunningCount = mWorkers.mCount; 282 __sync_synchronize(); 283 for (uint32_t ct = 0; ct < mWorkers.mCount; ct++) { 284 mWorkers.mLaunchSignals[ct].set(); 285 } 286 void *res; 287 for (uint32_t ct = 0; ct < mWorkers.mCount; ct++) { 288 pthread_join(mWorkers.mThreadId[ct], &res); 289 } 290 rsAssert(__sync_fetch_and_or(&mWorkers.mRunningCount, 0) == 0); 291 292 // Global structure cleanup. 293 lockMutex(); 294 --gThreadTLSKeyCount; 295 if (!gThreadTLSKeyCount) { 296 pthread_key_delete(gThreadTLSKey); 297 } 298 unlockMutex(); 299 300} 301 302typedef void (*rs_t)(const void *, void *, const void *, uint32_t, uint32_t, uint32_t, uint32_t); 303 304static void wc_xy(void *usr, uint32_t idx) { 305 MTLaunchStruct *mtls = (MTLaunchStruct *)usr; 306 RsForEachStubParamStruct p; 307 memcpy(&p, &mtls->fep, sizeof(p)); 308 p.lid = idx; 309 uint32_t sig = mtls->sig; 310 311 outer_foreach_t fn = (outer_foreach_t) mtls->kernel; 312 while (1) { 313 uint32_t slice = (uint32_t)__sync_fetch_and_add(&mtls->mSliceNum, 1); 314 uint32_t yStart = mtls->yStart + slice * mtls->mSliceSize; 315 uint32_t yEnd = yStart + mtls->mSliceSize; 316 yEnd = rsMin(yEnd, mtls->yEnd); 317 if (yEnd <= yStart) { 318 return; 319 } 320 321 //ALOGE("usr idx %i, x %i,%i y %i,%i", idx, mtls->xStart, mtls->xEnd, yStart, yEnd); 322 //ALOGE("usr ptr in %p, out %p", mtls->fep.ptrIn, mtls->fep.ptrOut); 323 324 for (p.y = yStart; p.y < yEnd; p.y++) { 325 p.out = mtls->fep.ptrOut + (mtls->fep.yStrideOut * p.y) + 326 (mtls->fep.eStrideOut * mtls->xStart); 327 p.in = mtls->fep.ptrIn + (mtls->fep.yStrideIn * p.y) + 328 (mtls->fep.eStrideIn * mtls->xStart); 329 fn(&p, mtls->xStart, mtls->xEnd, mtls->fep.eStrideIn, mtls->fep.eStrideOut); 330 } 331 } 332} 333 334static void wc_x(void *usr, uint32_t idx) { 335 MTLaunchStruct *mtls = (MTLaunchStruct *)usr; 336 RsForEachStubParamStruct p; 337 memcpy(&p, &mtls->fep, sizeof(p)); 338 p.lid = idx; 339 uint32_t sig = mtls->sig; 340 341 outer_foreach_t fn = (outer_foreach_t) mtls->kernel; 342 while (1) { 343 uint32_t slice = (uint32_t)__sync_fetch_and_add(&mtls->mSliceNum, 1); 344 uint32_t xStart = mtls->xStart + slice * mtls->mSliceSize; 345 uint32_t xEnd = xStart + mtls->mSliceSize; 346 xEnd = rsMin(xEnd, mtls->xEnd); 347 if (xEnd <= xStart) { 348 return; 349 } 350 351 //ALOGE("usr slice %i idx %i, x %i,%i", slice, idx, xStart, xEnd); 352 //ALOGE("usr ptr in %p, out %p", mtls->fep.ptrIn, mtls->fep.ptrOut); 353 354 p.out = mtls->fep.ptrOut + (mtls->fep.eStrideOut * xStart); 355 p.in = mtls->fep.ptrIn + (mtls->fep.eStrideIn * xStart); 356 fn(&p, xStart, xEnd, mtls->fep.eStrideIn, mtls->fep.eStrideOut); 357 } 358} 359 360void RsdCpuReferenceImpl::launchThreads(const Allocation * ain, Allocation * aout, 361 const RsScriptCall *sc, MTLaunchStruct *mtls) { 362 363 //android::StopWatch kernel_time("kernel time"); 364 365 if ((mWorkers.mCount >= 1) && mtls->isThreadable && !mInForEach) { 366 const size_t targetByteChunk = 16 * 1024; 367 mInForEach = true; 368 if (mtls->fep.dimY > 1) { 369 uint32_t s1 = mtls->fep.dimY / ((mWorkers.mCount + 1) * 4); 370 uint32_t s2 = 0; 371 372 // This chooses our slice size to rate limit atomic ops to 373 // one per 16k bytes of reads/writes. 374 if (mtls->fep.yStrideOut) { 375 s2 = targetByteChunk / mtls->fep.yStrideOut; 376 } else { 377 s2 = targetByteChunk / mtls->fep.yStrideIn; 378 } 379 mtls->mSliceSize = rsMin(s1, s2); 380 381 if(mtls->mSliceSize < 1) { 382 mtls->mSliceSize = 1; 383 } 384 385 // mtls->mSliceSize = 2; 386 launchThreads(wc_xy, mtls); 387 } else { 388 uint32_t s1 = mtls->fep.dimX / ((mWorkers.mCount + 1) * 4); 389 uint32_t s2 = 0; 390 391 // This chooses our slice size to rate limit atomic ops to 392 // one per 16k bytes of reads/writes. 393 if (mtls->fep.eStrideOut) { 394 s2 = targetByteChunk / mtls->fep.eStrideOut; 395 } else { 396 s2 = targetByteChunk / mtls->fep.eStrideIn; 397 } 398 mtls->mSliceSize = rsMin(s1, s2); 399 400 if(mtls->mSliceSize < 1) { 401 mtls->mSliceSize = 1; 402 } 403 404 launchThreads(wc_x, mtls); 405 } 406 mInForEach = false; 407 408 //ALOGE("launch 1"); 409 } else { 410 RsForEachStubParamStruct p; 411 memcpy(&p, &mtls->fep, sizeof(p)); 412 uint32_t sig = mtls->sig; 413 414 //ALOGE("launch 3"); 415 outer_foreach_t fn = (outer_foreach_t) mtls->kernel; 416 for (p.ar[0] = mtls->arrayStart; p.ar[0] < mtls->arrayEnd; p.ar[0]++) { 417 for (p.z = mtls->zStart; p.z < mtls->zEnd; p.z++) { 418 for (p.y = mtls->yStart; p.y < mtls->yEnd; p.y++) { 419 uint32_t offset = mtls->fep.dimY * mtls->fep.dimZ * p.ar[0] + 420 mtls->fep.dimY * p.z + p.y; 421 p.out = mtls->fep.ptrOut + (mtls->fep.yStrideOut * offset) + 422 (mtls->fep.eStrideOut * mtls->xStart); 423 p.in = mtls->fep.ptrIn + (mtls->fep.yStrideIn * offset) + 424 (mtls->fep.eStrideIn * mtls->xStart); 425 fn(&p, mtls->xStart, mtls->xEnd, mtls->fep.eStrideIn, mtls->fep.eStrideOut); 426 } 427 } 428 } 429 } 430} 431 432RsdCpuScriptImpl * RsdCpuReferenceImpl::setTLS(RsdCpuScriptImpl *sc) { 433 //ALOGE("setTls %p", sc); 434 ScriptTLSStruct * tls = (ScriptTLSStruct *)pthread_getspecific(gThreadTLSKey); 435 rsAssert(tls); 436 RsdCpuScriptImpl *old = tls->mImpl; 437 tls->mImpl = sc; 438 tls->mContext = mRSC; 439 if (sc) { 440 tls->mScript = sc->getScript(); 441 } else { 442 tls->mScript = NULL; 443 } 444 return old; 445} 446 447const RsdCpuReference::CpuSymbol * RsdCpuReferenceImpl::symLookup(const char *name) { 448 return mSymLookupFn(mRSC, name); 449} 450 451 452RsdCpuReference::CpuScript * RsdCpuReferenceImpl::createScript(const ScriptC *s, 453 char const *resName, char const *cacheDir, 454 uint8_t const *bitcode, size_t bitcodeSize, 455 uint32_t flags) { 456 457 RsdCpuScriptImpl *i = new RsdCpuScriptImpl(this, s); 458 if (!i->init(resName, cacheDir, bitcode, bitcodeSize, flags)) { 459 delete i; 460 return NULL; 461 } 462 return i; 463} 464 465extern RsdCpuScriptImpl * rsdIntrinsic_3DLUT(RsdCpuReferenceImpl *ctx, 466 const Script *s, const Element *e); 467extern RsdCpuScriptImpl * rsdIntrinsic_Convolve3x3(RsdCpuReferenceImpl *ctx, 468 const Script *s, const Element *e); 469extern RsdCpuScriptImpl * rsdIntrinsic_ColorMatrix(RsdCpuReferenceImpl *ctx, 470 const Script *s, const Element *e); 471extern RsdCpuScriptImpl * rsdIntrinsic_LUT(RsdCpuReferenceImpl *ctx, 472 const Script *s, const Element *e); 473extern RsdCpuScriptImpl * rsdIntrinsic_Convolve5x5(RsdCpuReferenceImpl *ctx, 474 const Script *s, const Element *e); 475extern RsdCpuScriptImpl * rsdIntrinsic_Blur(RsdCpuReferenceImpl *ctx, 476 const Script *s, const Element *e); 477extern RsdCpuScriptImpl * rsdIntrinsic_YuvToRGB(RsdCpuReferenceImpl *ctx, 478 const Script *s, const Element *e); 479extern RsdCpuScriptImpl * rsdIntrinsic_Blend(RsdCpuReferenceImpl *ctx, 480 const Script *s, const Element *e); 481extern RsdCpuScriptImpl * rsdIntrinsic_Histogram(RsdCpuReferenceImpl *ctx, 482 const Script *s, const Element *e); 483 484RsdCpuReference::CpuScript * RsdCpuReferenceImpl::createIntrinsic(const Script *s, 485 RsScriptIntrinsicID iid, Element *e) { 486 487 RsdCpuScriptImpl *i = NULL; 488 switch (iid) { 489 case RS_SCRIPT_INTRINSIC_ID_3DLUT: 490 i = rsdIntrinsic_3DLUT(this, s, e); 491 break; 492 case RS_SCRIPT_INTRINSIC_ID_CONVOLVE_3x3: 493 i = rsdIntrinsic_Convolve3x3(this, s, e); 494 break; 495 case RS_SCRIPT_INTRINSIC_ID_COLOR_MATRIX: 496 i = rsdIntrinsic_ColorMatrix(this, s, e); 497 break; 498 case RS_SCRIPT_INTRINSIC_ID_LUT: 499 i = rsdIntrinsic_LUT(this, s, e); 500 break; 501 case RS_SCRIPT_INTRINSIC_ID_CONVOLVE_5x5: 502 i = rsdIntrinsic_Convolve5x5(this, s, e); 503 break; 504 case RS_SCRIPT_INTRINSIC_ID_BLUR: 505 i = rsdIntrinsic_Blur(this, s, e); 506 break; 507 case RS_SCRIPT_INTRINSIC_ID_YUV_TO_RGB: 508 i = rsdIntrinsic_YuvToRGB(this, s, e); 509 break; 510 case RS_SCRIPT_INTRINSIC_ID_BLEND: 511 i = rsdIntrinsic_Blend(this, s, e); 512 break; 513 case RS_SCRIPT_INTRINSIC_ID_HISTOGRAM: 514 i = rsdIntrinsic_Histogram(this, s, e); 515 break; 516 517 default: 518 rsAssert(0); 519 } 520 521 return i; 522} 523 524RsdCpuReference::CpuScriptGroup * RsdCpuReferenceImpl::createScriptGroup(const ScriptGroup *sg) { 525 CpuScriptGroupImpl *sgi = new CpuScriptGroupImpl(this, sg); 526 if (!sgi->init()) { 527 delete sgi; 528 return NULL; 529 } 530 return sgi; 531} 532 533 534