rsCpuCore.cpp revision f5ef8df639ba6363aa5d546e57ce872d04144cb6
1/* 2 * Copyright (C) 2012 The Android Open Source Project 3 * 4 * Licensed under the Apache License, Version 2.0 (the "License"); 5 * you may not use this file except in compliance with the License. 6 * You may obtain a copy of the License at 7 * 8 * http://www.apache.org/licenses/LICENSE-2.0 9 * 10 * Unless required by applicable law or agreed to in writing, software 11 * distributed under the License is distributed on an "AS IS" BASIS, 12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 * See the License for the specific language governing permissions and 14 * limitations under the License. 15 */ 16 17#include "rsCpuCore.h" 18#include "rsCpuScript.h" 19#include "rsCpuScriptGroup.h" 20 21#include <malloc.h> 22#include "rsContext.h" 23 24#include <sys/types.h> 25#include <sys/resource.h> 26#include <sched.h> 27#include <sys/syscall.h> 28#include <string.h> 29#include <unistd.h> 30 31#include <stdio.h> 32#include <stdlib.h> 33#include <fcntl.h> 34 35#if !defined(RS_SERVER) && !defined(RS_COMPATIBILITY_LIB) 36#include <cutils/properties.h> 37#include "utils/StopWatch.h" 38#endif 39 40#ifdef RS_SERVER 41// Android exposes gettid(), standard Linux does not 42static pid_t gettid() { 43 return syscall(SYS_gettid); 44} 45#endif 46 47using namespace android; 48using namespace android::renderscript; 49 50typedef void (*outer_foreach_t)( 51 const android::renderscript::RsForEachStubParamStruct *, 52 uint32_t x1, uint32_t x2, 53 uint32_t instep, uint32_t outstep); 54 55 56static pthread_key_t gThreadTLSKey = 0; 57static uint32_t gThreadTLSKeyCount = 0; 58static pthread_mutex_t gInitMutex = PTHREAD_MUTEX_INITIALIZER; 59 60bool android::renderscript::gArchUseSIMD = false; 61 62RsdCpuReference::~RsdCpuReference() { 63} 64 65RsdCpuReference * RsdCpuReference::create(Context *rsc, uint32_t version_major, 66 uint32_t version_minor, sym_lookup_t lfn, script_lookup_t slfn 67#ifndef RS_COMPATIBILITY_LIB 68 , bcc::RSLinkRuntimeCallback pLinkRuntimeCallback, 69 RSSelectRTCallback pSelectRTCallback 70#endif 71 ) { 72 73 RsdCpuReferenceImpl *cpu = new RsdCpuReferenceImpl(rsc); 74 if (!cpu) { 75 return NULL; 76 } 77 if (!cpu->init(version_major, version_minor, lfn, slfn)) { 78 delete cpu; 79 return NULL; 80 } 81 82#ifndef RS_COMPATIBILITY_LIB 83 cpu->setLinkRuntimeCallback(pLinkRuntimeCallback); 84 cpu->setSelectRTCallback(pSelectRTCallback); 85#endif 86 87 return cpu; 88} 89 90 91Context * RsdCpuReference::getTlsContext() { 92 ScriptTLSStruct * tls = (ScriptTLSStruct *)pthread_getspecific(gThreadTLSKey); 93 return tls->mContext; 94} 95 96const Script * RsdCpuReference::getTlsScript() { 97 ScriptTLSStruct * tls = (ScriptTLSStruct *)pthread_getspecific(gThreadTLSKey); 98 return tls->mScript; 99} 100 101pthread_key_t RsdCpuReference::getThreadTLSKey(){ return gThreadTLSKey; } 102 103//////////////////////////////////////////////////////////// 104/// 105 106RsdCpuReferenceImpl::RsdCpuReferenceImpl(Context *rsc) { 107 mRSC = rsc; 108 109 version_major = 0; 110 version_minor = 0; 111 mInForEach = false; 112 memset(&mWorkers, 0, sizeof(mWorkers)); 113 memset(&mTlsStruct, 0, sizeof(mTlsStruct)); 114 mExit = false; 115#ifndef RS_COMPATIBILITY_LIB 116 mLinkRuntimeCallback = NULL; 117 mSelectRTCallback = NULL; 118#endif 119} 120 121 122void * RsdCpuReferenceImpl::helperThreadProc(void *vrsc) { 123 RsdCpuReferenceImpl *dc = (RsdCpuReferenceImpl *)vrsc; 124 125 uint32_t idx = __sync_fetch_and_add(&dc->mWorkers.mLaunchCount, 1); 126 127 //ALOGV("RS helperThread starting %p idx=%i", dc, idx); 128 129 dc->mWorkers.mLaunchSignals[idx].init(); 130 dc->mWorkers.mNativeThreadId[idx] = gettid(); 131 132 memset(&dc->mTlsStruct, 0, sizeof(dc->mTlsStruct)); 133 int status = pthread_setspecific(gThreadTLSKey, &dc->mTlsStruct); 134 if (status) { 135 ALOGE("pthread_setspecific %i", status); 136 } 137 138#if 0 139 typedef struct {uint64_t bits[1024 / 64]; } cpu_set_t; 140 cpu_set_t cpuset; 141 memset(&cpuset, 0, sizeof(cpuset)); 142 cpuset.bits[idx / 64] |= 1ULL << (idx % 64); 143 int ret = syscall(241, rsc->mWorkers.mNativeThreadId[idx], 144 sizeof(cpuset), &cpuset); 145 ALOGE("SETAFFINITY ret = %i %s", ret, EGLUtils::strerror(ret)); 146#endif 147 148 while (!dc->mExit) { 149 dc->mWorkers.mLaunchSignals[idx].wait(); 150 if (dc->mWorkers.mLaunchCallback) { 151 // idx +1 is used because the calling thread is always worker 0. 152 dc->mWorkers.mLaunchCallback(dc->mWorkers.mLaunchData, idx+1); 153 } 154 __sync_fetch_and_sub(&dc->mWorkers.mRunningCount, 1); 155 dc->mWorkers.mCompleteSignal.set(); 156 } 157 158 //ALOGV("RS helperThread exited %p idx=%i", dc, idx); 159 return NULL; 160} 161 162void RsdCpuReferenceImpl::launchThreads(WorkerCallback_t cbk, void *data) { 163 mWorkers.mLaunchData = data; 164 mWorkers.mLaunchCallback = cbk; 165 166 // fast path for very small launches 167 MTLaunchStruct *mtls = (MTLaunchStruct *)data; 168 if (mtls && mtls->fep.dimY <= 1 && mtls->xEnd <= mtls->xStart + mtls->mSliceSize) { 169 if (mWorkers.mLaunchCallback) { 170 mWorkers.mLaunchCallback(mWorkers.mLaunchData, 0); 171 } 172 return; 173 } 174 175 mWorkers.mRunningCount = mWorkers.mCount; 176 __sync_synchronize(); 177 178 for (uint32_t ct = 0; ct < mWorkers.mCount; ct++) { 179 mWorkers.mLaunchSignals[ct].set(); 180 } 181 182 // We use the calling thread as one of the workers so we can start without 183 // the delay of the thread wakeup. 184 if (mWorkers.mLaunchCallback) { 185 mWorkers.mLaunchCallback(mWorkers.mLaunchData, 0); 186 } 187 188 while (__sync_fetch_and_or(&mWorkers.mRunningCount, 0) != 0) { 189 mWorkers.mCompleteSignal.wait(); 190 } 191} 192 193 194void RsdCpuReferenceImpl::lockMutex() { 195 pthread_mutex_lock(&gInitMutex); 196} 197 198void RsdCpuReferenceImpl::unlockMutex() { 199 pthread_mutex_unlock(&gInitMutex); 200} 201 202#if defined(ARCH_ARM_HAVE_VFP) 203static int 204read_file(const char* pathname, char* buffer, size_t buffsize) 205{ 206 int fd, len; 207 208 fd = open(pathname, O_RDONLY); 209 if (fd < 0) 210 return -1; 211 212 do { 213 len = read(fd, buffer, buffsize); 214 } while (len < 0 && errno == EINTR); 215 216 close(fd); 217 218 return len; 219} 220 221static void GetCpuInfo() { 222 char cpuinfo[4096]; 223 int cpuinfo_len; 224 225 cpuinfo_len = read_file("/proc/cpuinfo", cpuinfo, sizeof cpuinfo); 226 if (cpuinfo_len < 0) /* should not happen */ { 227 return; 228 } 229 230 gArchUseSIMD = !!strstr(cpuinfo, " neon"); 231} 232#endif // ARCH_ARM_HAVE_VFP 233 234bool RsdCpuReferenceImpl::init(uint32_t version_major, uint32_t version_minor, 235 sym_lookup_t lfn, script_lookup_t slfn) { 236 237 mSymLookupFn = lfn; 238 mScriptLookupFn = slfn; 239 240 lockMutex(); 241 if (!gThreadTLSKeyCount) { 242 int status = pthread_key_create(&gThreadTLSKey, NULL); 243 if (status) { 244 ALOGE("Failed to init thread tls key."); 245 unlockMutex(); 246 return false; 247 } 248 } 249 gThreadTLSKeyCount++; 250 unlockMutex(); 251 252 mTlsStruct.mContext = mRSC; 253 mTlsStruct.mScript = NULL; 254 int status = pthread_setspecific(gThreadTLSKey, &mTlsStruct); 255 if (status) { 256 ALOGE("pthread_setspecific %i", status); 257 } 258 259#if defined(ARCH_ARM_HAVE_VFP) 260 GetCpuInfo(); 261#endif 262 263 int cpu = sysconf(_SC_NPROCESSORS_ONLN); 264 if(mRSC->props.mDebugMaxThreads) { 265 cpu = mRSC->props.mDebugMaxThreads; 266 } 267 if (cpu < 2) { 268 mWorkers.mCount = 0; 269 return true; 270 } 271 272 // Subtract one from the cpu count because we also use the command thread as a worker. 273 mWorkers.mCount = (uint32_t)(cpu - 1); 274 275 ALOGV("%p Launching thread(s), CPUs %i", mRSC, mWorkers.mCount + 1); 276 277 mWorkers.mThreadId = (pthread_t *) calloc(mWorkers.mCount, sizeof(pthread_t)); 278 mWorkers.mNativeThreadId = (pid_t *) calloc(mWorkers.mCount, sizeof(pid_t)); 279 mWorkers.mLaunchSignals = new Signal[mWorkers.mCount]; 280 mWorkers.mLaunchCallback = NULL; 281 282 mWorkers.mCompleteSignal.init(); 283 284 mWorkers.mRunningCount = mWorkers.mCount; 285 mWorkers.mLaunchCount = 0; 286 __sync_synchronize(); 287 288 pthread_attr_t threadAttr; 289 status = pthread_attr_init(&threadAttr); 290 if (status) { 291 ALOGE("Failed to init thread attribute."); 292 return false; 293 } 294 295 for (uint32_t ct=0; ct < mWorkers.mCount; ct++) { 296 status = pthread_create(&mWorkers.mThreadId[ct], &threadAttr, helperThreadProc, this); 297 if (status) { 298 mWorkers.mCount = ct; 299 ALOGE("Created fewer than expected number of RS threads."); 300 break; 301 } 302 } 303 while (__sync_fetch_and_or(&mWorkers.mRunningCount, 0) != 0) { 304 usleep(100); 305 } 306 307 pthread_attr_destroy(&threadAttr); 308 return true; 309} 310 311 312void RsdCpuReferenceImpl::setPriority(int32_t priority) { 313 for (uint32_t ct=0; ct < mWorkers.mCount; ct++) { 314 setpriority(PRIO_PROCESS, mWorkers.mNativeThreadId[ct], priority); 315 } 316} 317 318RsdCpuReferenceImpl::~RsdCpuReferenceImpl() { 319 mExit = true; 320 mWorkers.mLaunchData = NULL; 321 mWorkers.mLaunchCallback = NULL; 322 mWorkers.mRunningCount = mWorkers.mCount; 323 __sync_synchronize(); 324 for (uint32_t ct = 0; ct < mWorkers.mCount; ct++) { 325 mWorkers.mLaunchSignals[ct].set(); 326 } 327 void *res; 328 for (uint32_t ct = 0; ct < mWorkers.mCount; ct++) { 329 pthread_join(mWorkers.mThreadId[ct], &res); 330 } 331 rsAssert(__sync_fetch_and_or(&mWorkers.mRunningCount, 0) == 0); 332 333 // Global structure cleanup. 334 lockMutex(); 335 --gThreadTLSKeyCount; 336 if (!gThreadTLSKeyCount) { 337 pthread_key_delete(gThreadTLSKey); 338 } 339 unlockMutex(); 340 341} 342 343typedef void (*rs_t)(const void *, void *, const void *, uint32_t, uint32_t, uint32_t, uint32_t); 344 345static void wc_xy(void *usr, uint32_t idx) { 346 MTLaunchStruct *mtls = (MTLaunchStruct *)usr; 347 RsForEachStubParamStruct p; 348 memcpy(&p, &mtls->fep, sizeof(p)); 349 p.lid = idx; 350 uint32_t sig = mtls->sig; 351 352 outer_foreach_t fn = (outer_foreach_t) mtls->kernel; 353 while (1) { 354 uint32_t slice = (uint32_t)__sync_fetch_and_add(&mtls->mSliceNum, 1); 355 uint32_t yStart = mtls->yStart + slice * mtls->mSliceSize; 356 uint32_t yEnd = yStart + mtls->mSliceSize; 357 yEnd = rsMin(yEnd, mtls->yEnd); 358 if (yEnd <= yStart) { 359 return; 360 } 361 362 //ALOGE("usr idx %i, x %i,%i y %i,%i", idx, mtls->xStart, mtls->xEnd, yStart, yEnd); 363 //ALOGE("usr ptr in %p, out %p", mtls->fep.ptrIn, mtls->fep.ptrOut); 364 365 for (p.y = yStart; p.y < yEnd; p.y++) { 366 p.out = mtls->fep.ptrOut + (mtls->fep.yStrideOut * p.y) + 367 (mtls->fep.eStrideOut * mtls->xStart); 368 p.in = mtls->fep.ptrIn + (mtls->fep.yStrideIn * p.y) + 369 (mtls->fep.eStrideIn * mtls->xStart); 370 fn(&p, mtls->xStart, mtls->xEnd, mtls->fep.eStrideIn, mtls->fep.eStrideOut); 371 } 372 } 373} 374 375static void wc_x(void *usr, uint32_t idx) { 376 MTLaunchStruct *mtls = (MTLaunchStruct *)usr; 377 RsForEachStubParamStruct p; 378 memcpy(&p, &mtls->fep, sizeof(p)); 379 p.lid = idx; 380 uint32_t sig = mtls->sig; 381 382 outer_foreach_t fn = (outer_foreach_t) mtls->kernel; 383 while (1) { 384 uint32_t slice = (uint32_t)__sync_fetch_and_add(&mtls->mSliceNum, 1); 385 uint32_t xStart = mtls->xStart + slice * mtls->mSliceSize; 386 uint32_t xEnd = xStart + mtls->mSliceSize; 387 xEnd = rsMin(xEnd, mtls->xEnd); 388 if (xEnd <= xStart) { 389 return; 390 } 391 392 //ALOGE("usr slice %i idx %i, x %i,%i", slice, idx, xStart, xEnd); 393 //ALOGE("usr ptr in %p, out %p", mtls->fep.ptrIn, mtls->fep.ptrOut); 394 395 p.out = mtls->fep.ptrOut + (mtls->fep.eStrideOut * xStart); 396 p.in = mtls->fep.ptrIn + (mtls->fep.eStrideIn * xStart); 397 fn(&p, xStart, xEnd, mtls->fep.eStrideIn, mtls->fep.eStrideOut); 398 } 399} 400 401void RsdCpuReferenceImpl::launchThreads(const Allocation * ain, Allocation * aout, 402 const RsScriptCall *sc, MTLaunchStruct *mtls) { 403 404 //android::StopWatch kernel_time("kernel time"); 405 406 if ((mWorkers.mCount >= 1) && mtls->isThreadable && !mInForEach) { 407 const size_t targetByteChunk = 16 * 1024; 408 mInForEach = true; 409 if (mtls->fep.dimY > 1) { 410 uint32_t s1 = mtls->fep.dimY / ((mWorkers.mCount + 1) * 4); 411 uint32_t s2 = 0; 412 413 // This chooses our slice size to rate limit atomic ops to 414 // one per 16k bytes of reads/writes. 415 if (mtls->fep.yStrideOut) { 416 s2 = targetByteChunk / mtls->fep.yStrideOut; 417 } else { 418 s2 = targetByteChunk / mtls->fep.yStrideIn; 419 } 420 mtls->mSliceSize = rsMin(s1, s2); 421 422 if(mtls->mSliceSize < 1) { 423 mtls->mSliceSize = 1; 424 } 425 426 // mtls->mSliceSize = 2; 427 launchThreads(wc_xy, mtls); 428 } else { 429 uint32_t s1 = mtls->fep.dimX / ((mWorkers.mCount + 1) * 4); 430 uint32_t s2 = 0; 431 432 // This chooses our slice size to rate limit atomic ops to 433 // one per 16k bytes of reads/writes. 434 if (mtls->fep.eStrideOut) { 435 s2 = targetByteChunk / mtls->fep.eStrideOut; 436 } else { 437 s2 = targetByteChunk / mtls->fep.eStrideIn; 438 } 439 mtls->mSliceSize = rsMin(s1, s2); 440 441 if(mtls->mSliceSize < 1) { 442 mtls->mSliceSize = 1; 443 } 444 445 launchThreads(wc_x, mtls); 446 } 447 mInForEach = false; 448 449 //ALOGE("launch 1"); 450 } else { 451 RsForEachStubParamStruct p; 452 memcpy(&p, &mtls->fep, sizeof(p)); 453 uint32_t sig = mtls->sig; 454 455 //ALOGE("launch 3"); 456 outer_foreach_t fn = (outer_foreach_t) mtls->kernel; 457 for (p.ar[0] = mtls->arrayStart; p.ar[0] < mtls->arrayEnd; p.ar[0]++) { 458 for (p.z = mtls->zStart; p.z < mtls->zEnd; p.z++) { 459 for (p.y = mtls->yStart; p.y < mtls->yEnd; p.y++) { 460 uint32_t offset = mtls->fep.dimY * mtls->fep.dimZ * p.ar[0] + 461 mtls->fep.dimY * p.z + p.y; 462 p.out = mtls->fep.ptrOut + (mtls->fep.yStrideOut * offset) + 463 (mtls->fep.eStrideOut * mtls->xStart); 464 p.in = mtls->fep.ptrIn + (mtls->fep.yStrideIn * offset) + 465 (mtls->fep.eStrideIn * mtls->xStart); 466 fn(&p, mtls->xStart, mtls->xEnd, mtls->fep.eStrideIn, mtls->fep.eStrideOut); 467 } 468 } 469 } 470 } 471} 472 473RsdCpuScriptImpl * RsdCpuReferenceImpl::setTLS(RsdCpuScriptImpl *sc) { 474 //ALOGE("setTls %p", sc); 475 ScriptTLSStruct * tls = (ScriptTLSStruct *)pthread_getspecific(gThreadTLSKey); 476 rsAssert(tls); 477 RsdCpuScriptImpl *old = tls->mImpl; 478 tls->mImpl = sc; 479 tls->mContext = mRSC; 480 if (sc) { 481 tls->mScript = sc->getScript(); 482 } else { 483 tls->mScript = NULL; 484 } 485 return old; 486} 487 488const RsdCpuReference::CpuSymbol * RsdCpuReferenceImpl::symLookup(const char *name) { 489 return mSymLookupFn(mRSC, name); 490} 491 492 493RsdCpuReference::CpuScript * RsdCpuReferenceImpl::createScript(const ScriptC *s, 494 char const *resName, char const *cacheDir, 495 uint8_t const *bitcode, size_t bitcodeSize, 496 uint32_t flags) { 497 498 RsdCpuScriptImpl *i = new RsdCpuScriptImpl(this, s); 499 if (!i->init(resName, cacheDir, bitcode, bitcodeSize, flags)) { 500 delete i; 501 return NULL; 502 } 503 return i; 504} 505 506extern RsdCpuScriptImpl * rsdIntrinsic_3DLUT(RsdCpuReferenceImpl *ctx, 507 const Script *s, const Element *e); 508extern RsdCpuScriptImpl * rsdIntrinsic_Convolve3x3(RsdCpuReferenceImpl *ctx, 509 const Script *s, const Element *e); 510extern RsdCpuScriptImpl * rsdIntrinsic_ColorMatrix(RsdCpuReferenceImpl *ctx, 511 const Script *s, const Element *e); 512extern RsdCpuScriptImpl * rsdIntrinsic_LUT(RsdCpuReferenceImpl *ctx, 513 const Script *s, const Element *e); 514extern RsdCpuScriptImpl * rsdIntrinsic_Convolve5x5(RsdCpuReferenceImpl *ctx, 515 const Script *s, const Element *e); 516extern RsdCpuScriptImpl * rsdIntrinsic_Blur(RsdCpuReferenceImpl *ctx, 517 const Script *s, const Element *e); 518extern RsdCpuScriptImpl * rsdIntrinsic_YuvToRGB(RsdCpuReferenceImpl *ctx, 519 const Script *s, const Element *e); 520extern RsdCpuScriptImpl * rsdIntrinsic_Blend(RsdCpuReferenceImpl *ctx, 521 const Script *s, const Element *e); 522 523RsdCpuReference::CpuScript * RsdCpuReferenceImpl::createIntrinsic(const Script *s, 524 RsScriptIntrinsicID iid, Element *e) { 525 526 RsdCpuScriptImpl *i = NULL; 527 switch (iid) { 528 case RS_SCRIPT_INTRINSIC_ID_3DLUT: 529 i = rsdIntrinsic_3DLUT(this, s, e); 530 break; 531 case RS_SCRIPT_INTRINSIC_ID_CONVOLVE_3x3: 532 i = rsdIntrinsic_Convolve3x3(this, s, e); 533 break; 534 case RS_SCRIPT_INTRINSIC_ID_COLOR_MATRIX: 535 i = rsdIntrinsic_ColorMatrix(this, s, e); 536 break; 537 case RS_SCRIPT_INTRINSIC_ID_LUT: 538 i = rsdIntrinsic_LUT(this, s, e); 539 break; 540 case RS_SCRIPT_INTRINSIC_ID_CONVOLVE_5x5: 541 i = rsdIntrinsic_Convolve5x5(this, s, e); 542 break; 543 case RS_SCRIPT_INTRINSIC_ID_BLUR: 544 i = rsdIntrinsic_Blur(this, s, e); 545 break; 546 case RS_SCRIPT_INTRINSIC_ID_YUV_TO_RGB: 547 i = rsdIntrinsic_YuvToRGB(this, s, e); 548 break; 549 case RS_SCRIPT_INTRINSIC_ID_BLEND: 550 i = rsdIntrinsic_Blend(this, s, e); 551 break; 552 553 default: 554 rsAssert(0); 555 } 556 557 return i; 558} 559 560RsdCpuReference::CpuScriptGroup * RsdCpuReferenceImpl::createScriptGroup(const ScriptGroup *sg) { 561 CpuScriptGroupImpl *sgi = new CpuScriptGroupImpl(this, sg); 562 if (!sgi->init()) { 563 delete sgi; 564 return NULL; 565 } 566 return sgi; 567} 568 569 570