1/* NEON optimized code (C) COPYRIGHT 2009 Motorola 2 * 3 * Use of this source code is governed by a BSD-style license that can be 4 * found in the LICENSE file. 5 */ 6 7/* 8 * Modifications done in-house at Motorola 9 * 10 * this is a clone of SkBitmapProcState_matrix.h 11 * and has been tuned to work with the NEON unit. 12 * 13 * Still going back and forth between whether this approach 14 * (clone the entire SkBitmapProcState_matrix.h file or 15 * if I should put just the modified routines in here and 16 * then use a construct like #define DONT_DO_THIS_FUNCTION or 17 * something like that... 18 * 19 * This is for the ClampX_ClampY instance 20 * 21 */ 22 23 24#include <arm_neon.h> 25 26/* 27 * This has been modified on the knowledge that (at the time) 28 * we had the following macro definitions in the parent file 29 * 30 * #define MAKENAME(suffix) ClampX_ClampY ## suffix 31 * #define TILEX_PROCF(fx, max) SkClampMax((fx) >> 16, max) 32 * #define TILEY_PROCF(fy, max) SkClampMax((fy) >> 16, max) 33 * #define TILEX_LOW_BITS(fx, max) (((fx) >> 12) & 0xF) 34 * #define TILEY_LOW_BITS(fy, max) (((fy) >> 12) & 0xF) 35 * #define CHECK_FOR_DECAL 36 */ 37 38/* SkClampMax(val,max) -- bound to 0..max */ 39 40#define SCALE_NOFILTER_NAME MAKENAME(_nofilter_scale) 41#define SCALE_FILTER_NAME MAKENAME(_filter_scale) 42#define AFFINE_NOFILTER_NAME MAKENAME(_nofilter_affine) 43#define AFFINE_FILTER_NAME MAKENAME(_filter_affine) 44#define PERSP_NOFILTER_NAME MAKENAME(_nofilter_persp) 45#define PERSP_FILTER_NAME MAKENAME(_filter_persp) 46 47#define PACK_FILTER_X_NAME MAKENAME(_pack_filter_x) 48#define PACK_FILTER_Y_NAME MAKENAME(_pack_filter_y) 49 50#ifndef PREAMBLE 51 #define PREAMBLE(state) 52 #define PREAMBLE_PARAM_X 53 #define PREAMBLE_PARAM_Y 54 #define PREAMBLE_ARG_X 55 #define PREAMBLE_ARG_Y 56#endif 57 58static void SCALE_NOFILTER_NAME(const SkBitmapProcState& s, 59 uint32_t xy[], int count, int x, int y) { 60 SkASSERT((s.fInvType & ~(SkMatrix::kTranslate_Mask | 61 SkMatrix::kScale_Mask)) == 0); 62 63 PREAMBLE(s); 64 // we store y, x, x, x, x, x 65 66 const unsigned maxX = s.fBitmap->width() - 1; 67 SkFixed fx; 68 { 69 SkPoint pt; 70 s.fInvProc(s.fInvMatrix, SkIntToScalar(x) + SK_ScalarHalf, 71 SkIntToScalar(y) + SK_ScalarHalf, &pt); 72 fx = SkScalarToFixed(pt.fY); 73 const unsigned maxY = s.fBitmap->height() - 1; 74 *xy++ = TILEY_PROCF(fx, maxY); 75 fx = SkScalarToFixed(pt.fX); 76 } 77 78 if (0 == maxX) { 79 // all of the following X values must be 0 80 memset(xy, 0, count * sizeof(uint16_t)); 81 return; 82 } 83 84 const SkFixed dx = s.fInvSx; 85 86#ifdef CHECK_FOR_DECAL 87 // test if we don't need to apply the tile proc 88 if ((unsigned)(fx >> 16) <= maxX && 89 (unsigned)((fx + dx * (count - 1)) >> 16) <= maxX) { 90 decal_nofilter_scale_neon(xy, fx, dx, count); 91 return; 92 } 93#endif 94 95 int i; 96 97 /* very much like done in decal_nofilter, but with 98 * an extra clamping function applied. 99 * TILEX_PROCF(fx,max) SkClampMax((fx)>>16, max) 100 */ 101 if (count >= 8) { 102 /* SkFixed is 16.16 fixed point */ 103 SkFixed dx2 = dx+dx; 104 SkFixed dx4 = dx2+dx2; 105 SkFixed dx8 = dx4+dx4; 106 107 /* now build fx/fx+dx/fx+2dx/fx+3dx */ 108 SkFixed fx1, fx2, fx3; 109 int32x4_t lbase, hbase; 110 int16_t *dst16 = (int16_t *)xy; 111 112 fx1 = fx+dx; 113 fx2 = fx1+dx; 114 fx3 = fx2+dx; 115 116 /* build my template(s) */ 117 /* avoid the 'lbase unitialized' warning */ 118 lbase = vdupq_n_s32(fx); 119 lbase = vsetq_lane_s32(fx1, lbase, 1); 120 lbase = vsetq_lane_s32(fx2, lbase, 2); 121 lbase = vsetq_lane_s32(fx3, lbase, 3); 122 123 hbase = vaddq_s32(lbase, vdupq_n_s32(dx4)); 124 125 /* store & bump */ 126 do { 127 int32x4_t lout; 128 int32x4_t hout; 129 int16x8_t hi16; 130 131 /* get the hi 16s of all those 32s */ 132 lout = lbase; 133 hout = hbase; 134 /* this sets up all lout's then all hout's in hout */ 135 asm ("vuzpq.16 %q0, %q1" : "+w" (lout), "+w" (hout)); 136 hi16 = vreinterpretq_s16_s32(hout); 137 138 /* clamp & output */ 139 hi16 = vmaxq_s16(hi16, vdupq_n_s16(0)); 140 hi16 = vminq_s16(hi16, vdupq_n_s16(maxX)); 141 vst1q_s16(dst16, hi16); 142 143 /* but preserving base & on to the next */ 144 lbase = vaddq_s32 (lbase, vdupq_n_s32(dx8)); 145 hbase = vaddq_s32 (hbase, vdupq_n_s32(dx8)); 146 dst16 += 8; 147 count -= 8; 148 fx += dx8; 149 } while (count >= 8); 150 xy = (uint32_t *) dst16; 151 } 152 153 uint16_t* xx = (uint16_t*)xy; 154 for (i = count; i > 0; --i) { 155 *xx++ = TILEX_PROCF(fx, maxX); fx += dx; 156 } 157} 158 159// note: we could special-case on a matrix which is skewed in X but not Y. 160// this would require a more general setup thatn SCALE does, but could use 161// SCALE's inner loop that only looks at dx 162 163static void AFFINE_NOFILTER_NAME(const SkBitmapProcState& s, 164 uint32_t xy[], int count, int x, int y) { 165 SkASSERT(s.fInvType & SkMatrix::kAffine_Mask); 166 SkASSERT((s.fInvType & ~(SkMatrix::kTranslate_Mask | 167 SkMatrix::kScale_Mask | 168 SkMatrix::kAffine_Mask)) == 0); 169 170 PREAMBLE(s); 171 SkPoint srcPt; 172 s.fInvProc(s.fInvMatrix, 173 SkIntToScalar(x) + SK_ScalarHalf, 174 SkIntToScalar(y) + SK_ScalarHalf, &srcPt); 175 176 SkFixed fx = SkScalarToFixed(srcPt.fX); 177 SkFixed fy = SkScalarToFixed(srcPt.fY); 178 SkFixed dx = s.fInvSx; 179 SkFixed dy = s.fInvKy; 180 int maxX = s.fBitmap->width() - 1; 181 int maxY = s.fBitmap->height() - 1; 182 183 /* NEON lets us do an 8x unrolling */ 184 if (count >= 8) { 185 /* SkFixed is 16.16 fixed point */ 186 SkFixed dx4 = dx * 4; 187 SkFixed dy4 = dy * 4; 188 SkFixed dx8 = dx * 8; 189 SkFixed dy8 = dy * 8; 190 191 int32x4_t xbase, ybase; 192 int32x4_t x2base, y2base; 193 int16_t *dst16 = (int16_t *) xy; 194 195 /* my sets of maxx/maxy for clamping */ 196 int32_t maxpair = (maxX&0xffff) | ((maxY&0xffff)<<16); 197 int16x8_t maxXY = vreinterpretq_s16_s32(vdupq_n_s32(maxpair)); 198 199 /* now build fx/fx+dx/fx+2dx/fx+3dx */ 200 /* avoid the 'xbase unitialized' warning...*/ 201 xbase = vdupq_n_s32(fx); 202 xbase = vsetq_lane_s32(fx+dx, xbase, 1); 203 xbase = vsetq_lane_s32(fx+dx+dx, xbase, 2); 204 xbase = vsetq_lane_s32(fx+dx+dx+dx, xbase, 3); 205 206 /* same for fy */ 207 /* avoid the 'ybase unitialized' warning...*/ 208 ybase = vdupq_n_s32(fy); 209 ybase = vsetq_lane_s32(fy+dy, ybase, 1); 210 ybase = vsetq_lane_s32(fy+dy+dy, ybase, 2); 211 ybase = vsetq_lane_s32(fy+dy+dy+dy, ybase, 3); 212 213 x2base = vaddq_s32(xbase, vdupq_n_s32(dx4)); 214 y2base = vaddq_s32(ybase, vdupq_n_s32(dy4)); 215 216 /* store & bump */ 217 do { 218 int32x4_t xout, yout; 219 int32x4_t x2out, y2out; 220 int16x8_t hi16, hi16_2; 221 222 xout = xbase; 223 yout = ybase; 224 225 /* overlay y's low16 with hi16 from x */ 226 /* so we properly shifted xyxyxyxy */ 227 yout = vsriq_n_s32(yout, xout, 16); 228 hi16 = vreinterpretq_s16_s32 (yout); 229 230 /* do the clamping; both guys get 0's */ 231 hi16 = vmaxq_s16 (hi16, vdupq_n_s16(0)); 232 hi16 = vminq_s16 (hi16, maxXY); 233 234 vst1q_s16 (dst16, hi16); 235 236 /* and for the other 4 pieces of this iteration */ 237 x2out = x2base; 238 y2out = y2base; 239 240 /* overlay y's low16 with hi16 from x */ 241 /* so we properly shifted xyxyxyxy */ 242 y2out = vsriq_n_s32(y2out, x2out, 16); 243 hi16_2 = vreinterpretq_s16_s32 (y2out); 244 245 /* do the clamping; both guys get 0's */ 246 hi16_2 = vmaxq_s16 (hi16_2, vdupq_n_s16(0)); 247 hi16_2 = vminq_s16 (hi16_2, maxXY); 248 249 /* RBE: gcc regenerates dst16+8 all the time instead 250 * of folding it into an addressing mode. *sigh* */ 251 vst1q_s16 (dst16+8, hi16_2); 252 253 /* moving base and on to the next */ 254 xbase = vaddq_s32 (xbase, vdupq_n_s32 (dx8)); 255 ybase = vaddq_s32 (ybase, vdupq_n_s32 (dy8)); 256 x2base = vaddq_s32 (x2base, vdupq_n_s32 (dx8)); 257 y2base = vaddq_s32 (y2base, vdupq_n_s32 (dy8)); 258 259 dst16 += 16; /* 8x32 aka 16x16 */ 260 count -= 8; 261 fx += dx8; 262 fy += dy8; 263 } while (count >= 8); 264 xy = (uint32_t *) dst16; 265 } 266 267 for (int i = count; i > 0; --i) { 268 *xy++ = (TILEY_PROCF(fy, maxY) << 16) | TILEX_PROCF(fx, maxX); 269 fx += dx; fy += dy; 270 } 271} 272 273#undef DEBUG_PERSP_NOFILTER 274 275static void PERSP_NOFILTER_NAME(const SkBitmapProcState& s, 276 uint32_t* SK_RESTRICT xy, 277 int count, int x, int y) { 278 SkASSERT(s.fInvType & SkMatrix::kPerspective_Mask); 279 280 PREAMBLE(s); 281 /* max{X,Y} are int here, but later shown/assumed to fit in 16 bits */ 282 int maxX = s.fBitmap->width() - 1; 283 int maxY = s.fBitmap->height() - 1; 284 285 SkPerspIter iter(s.fInvMatrix, 286 SkIntToScalar(x) + SK_ScalarHalf, 287 SkIntToScalar(y) + SK_ScalarHalf, count); 288 289 while ((count = iter.next()) != 0) { 290 const SkFixed* SK_RESTRICT srcXY = iter.getXY(); 291 292#if defined(DEBUG_PERSP_NOFILTER) 293 /* debugging stuff */ 294 const SkFixed *end_srcXY = srcXY + (count*2); 295 uint32_t *end_xy = xy + (count); 296 const SkFixed *base_srcXY = srcXY; 297 uint32_t *base_xy = xy; 298 int base_count = count; 299#endif 300 301#if 1 302 // 2009/9/30: crashes in ApiDemos - Views - Animation - 3D Transition 303 // 2009/10/9: reworked to avoid illegal (but allowed by gas) insn 304 305 /* srcXY is a batch of 32 bit numbers X0,Y0,X1,Y1... 306 * but we immediately discard the low 16 bits... 307 * so what we're going to do is vld4, which will give us 308 * xlo,xhi,ylo,yhi distribution and we can ignore the 'lo' 309 * parts.... 310 */ 311 if (count >= 8) { 312 int16_t *mysrc = (int16_t *) srcXY; 313 int16_t *mydst = (int16_t *) xy; 314 int16x4_t maxX4 = vdup_n_s16((int16_t)maxX); 315 int16x4_t maxY4 = vdup_n_s16((int16_t)maxY); 316 int16x4_t zero4 = vdup_n_s16(0); 317 318 /* The constructs with local blocks for register assignments 319 * and asm() instructions is to make keep any hard register 320 * assignments to as small a scope as possible. and to avoid 321 * burning call-preserved hard registers on the vld/vst 322 * instructions. 323 */ 324 325 do { 326 int16x4_t xhi, yhi; 327 int16x4_t x2hi, y2hi; 328 329 /* vld4 does the de-interleaving for us */ 330 { 331 register int16x4_t t_xlo asm("d0"); 332 register int16x4_t t_xhi asm("d1"); 333 register int16x4_t t_ylo asm("d2"); 334 register int16x4_t t_yhi asm("d3"); 335 336 asm ("vld4.16 {d0-d3},[%4] /* xlo=%P0 xhi=%P1 ylo=%P2 yhi=%P3 */" 337 : "=w" (t_xlo), "=w" (t_xhi), "=w" (t_ylo), "=w" (t_yhi) 338 : "r" (mysrc) 339 ); 340 xhi = t_xhi; 341 yhi = t_yhi; 342 } 343 344 /* clamp X>>16 (aka xhi) to 0..maxX */ 345 xhi = vmax_s16(xhi, zero4); /* now 0.. */ 346 xhi = vmin_s16(xhi, maxX4); /* now 0..maxX */ 347 348 /* clamp Y>>16 (aka yhi) to 0..maxY */ 349 yhi = vmax_s16(yhi, zero4); /* now 0.. */ 350 yhi = vmin_s16(yhi, maxY4); /* now 0..maxY */ 351 352 /* deal with the second set of numbers */ 353 { 354 register int16x4_t t_xlo asm("d4"); 355 register int16x4_t t_xhi asm("d5"); 356 register int16x4_t t_ylo asm("d6"); 357 register int16x4_t t_yhi asm("d7"); 358 359 /* offset == 256 bits == 32 bytes == 8 longs == 16 shorts */ 360 asm ("vld4.16 {d4-d7},[%4] /* xlo=%P0 xhi=%P1 ylo=%P2 yhi=%P3 */" 361 : "=w" (t_xlo), "=w" (t_xhi), "=w" (t_ylo), "=w" (t_yhi) 362 : "r" (mysrc+16) 363 ); 364 x2hi = t_xhi; 365 y2hi = t_yhi; 366 } 367 368 /* clamp the second 4 here */ 369 370 if (0) { extern void rbe(void); rbe(); } 371 372 /* clamp X>>16 (aka xhi) to 0..maxX */ 373 x2hi = vmax_s16(x2hi, zero4); /* now 0.. */ 374 x2hi = vmin_s16(x2hi, maxX4); /* now 0..maxX */ 375 376 /* clamp Y>>16 (aka yhi) to 0..maxY */ 377 y2hi = vmax_s16(y2hi, zero4); /* now 0.. */ 378 y2hi = vmin_s16(y2hi, maxY4); /* now 0..maxY */ 379 380 /* we're storing as {x,y}s: x is [0], y is [1] */ 381 /* we'll use vst2 to make this happen */ 382 383 { 384 register int16x4_t out_x asm("d16") = xhi; 385 register int16x4_t out_y asm("d17") = yhi; 386 387 asm ("vst2.16 {d16-d17},[%2] /* xlo=%P0 xhi=%P1 */" 388 : 389 : "w" (out_x), "w" (out_y), "r" (mydst) 390 ); 391 } 392 { 393 register int16x4_t out_x asm("d18") = x2hi; 394 register int16x4_t out_y asm("d19") = y2hi; 395 396 asm ("vst2.16 {d18-d19},[%2] /* xlo=%P0 xhi=%P1 */" 397 : 398 : "w" (out_x), "w" (out_y), "r" (mydst+8) 399 ); 400 } 401 402 /* XXX: gcc isn't interleaving these with the NEON ops 403 * but i think that all the scoreboarding works out */ 404 count -= 8; /* 8 iterations */ 405 mysrc += 32; /* 16 longs, aka 32 shorts */ 406 mydst += 16; /* 16 shorts, aka 8 longs */ 407 } while (count >= 8); 408 /* get xy and srcXY fixed up */ 409 srcXY = (const SkFixed *) mysrc; 410 xy = (uint32_t *) mydst; 411 } 412#endif 413 414 while (--count >= 0) { 415 *xy++ = (TILEY_PROCF(srcXY[1], maxY) << 16) | 416 TILEX_PROCF(srcXY[0], maxX); 417 srcXY += 2; 418 } 419 420#if defined(DEBUG_PERSP_NOFILTER) 421 /* for checking our NEON-produced results against vanilla code */ 422 { 423 int bad = (-1); 424 for (int i = 0; i < base_count; i++) { 425 uint32_t val; 426 val = (TILEY_PROCF (base_srcXY[i * 2 + 1], maxY) << 16) | 427 TILEX_PROCF (base_srcXY[i * 2 + 0], maxX); 428 429 if (val != base_xy[i]) { 430 bad = i; 431 break; 432 } 433 } 434 if (bad >= 0) { 435 SkDebugf("clamp-nofilter-persp failed piece %d\n", bad); 436 SkDebugf(" maxX %08x maxY %08x\n", maxX, maxY); 437 bad -= (bad & 0x7); /* align */ 438 for (int i = bad; i < bad + 8; i++) { 439 uint32_t val; 440 val = (TILEY_PROCF (base_srcXY[i * 2 + 1], maxY) << 16) | 441 TILEX_PROCF (base_srcXY[i * 2 + 0], maxX); 442 443 SkDebugf("%d: got %08x want %08x srcXY[0] %08x srcXY[1] %08x\n", 444 i, base_xy[i], val, base_srcXY[i * 2 + 0], 445 base_srcXY[i * 2 + 1]); 446 } 447 SkDebugf ("---\n"); 448 } 449 450 if (end_xy != xy) { 451 SkDebugf("xy ended at %08x, should be %08x\n", xy, end_xy); 452 } 453 if (end_srcXY != srcXY) { 454 SkDebugf("srcXY ended at %08x, should be %08x\n", srcXY, 455 end_srcXY); 456 } 457 } 458#endif 459 } 460} 461 462#undef DEBUG_PERSP_NOFILTER 463 464////////////////////////////////////////////////////////////////////////////// 465 466static inline uint32_t PACK_FILTER_Y_NAME(SkFixed f, unsigned max, 467 SkFixed one PREAMBLE_PARAM_Y) { 468 unsigned i = TILEY_PROCF(f, max); 469 i = (i << 4) | TILEY_LOW_BITS(f, max); 470 return (i << 14) | (TILEY_PROCF((f + one), max)); 471} 472 473static inline uint32_t PACK_FILTER_X_NAME(SkFixed f, unsigned max, 474 SkFixed one PREAMBLE_PARAM_X) { 475 unsigned i = TILEX_PROCF(f, max); 476 i = (i << 4) | TILEX_LOW_BITS(f, max); 477 return (i << 14) | (TILEX_PROCF((f + one), max)); 478} 479 480static void SCALE_FILTER_NAME(const SkBitmapProcState& s, 481 uint32_t xy[], int count, int x, int y) { 482 SkASSERT((s.fInvType & ~(SkMatrix::kTranslate_Mask | 483 SkMatrix::kScale_Mask)) == 0); 484 SkASSERT(s.fInvKy == 0); 485 486 PREAMBLE(s); 487 488 const unsigned maxX = s.fBitmap->width() - 1; 489 const SkFixed one = s.fFilterOneX; 490 const SkFixed dx = s.fInvSx; 491 SkFixed fx; 492 493 { 494 SkPoint pt; 495 s.fInvProc(s.fInvMatrix, SkIntToScalar(x) + SK_ScalarHalf, 496 SkIntToScalar(y) + SK_ScalarHalf, &pt); 497 const SkFixed fy = SkScalarToFixed(pt.fY) - (s.fFilterOneY >> 1); 498 const unsigned maxY = s.fBitmap->height() - 1; 499 // compute our two Y values up front 500 *xy++ = PACK_FILTER_Y_NAME(fy, maxY, s.fFilterOneY PREAMBLE_ARG_Y); 501 // now initialize fx 502 fx = SkScalarToFixed(pt.fX) - (one >> 1); 503 } 504 505#ifdef CHECK_FOR_DECAL 506 // test if we don't need to apply the tile proc 507 if (dx > 0 && 508 (unsigned)(fx >> 16) <= maxX && 509 (unsigned)((fx + dx * (count - 1)) >> 16) < maxX) { 510 decal_filter_scale_neon(xy, fx, dx, count); 511 } else 512#endif 513 514 if (count >= 4) { 515 int32x4_t wide_one, wide_fx, wide_fx1, wide_i, wide_lo; 516 #if 0 517 /* verification hooks -- see below */ 518 SkFixed debug_fx = fx; 519 int count_done = 0; 520 #endif 521 522 wide_fx = vdupq_n_s32(fx); 523 wide_fx = vsetq_lane_s32(fx+dx, wide_fx, 1); 524 wide_fx = vsetq_lane_s32(fx+dx+dx, wide_fx, 2); 525 wide_fx = vsetq_lane_s32(fx+dx+dx+dx, wide_fx, 3); 526 527 wide_one = vdupq_n_s32(one); 528 529 while (count >= 4) { 530 /* original expands to: 531 * unsigned i = SkClampMax((f) >> 16, max); 532 * i = (i << 4) | (((f) >> 12) & 0xF); 533 * return (i << 14) | (SkClampMax(((f + one)) >> 16, max)); 534 */ 535 536 /* i = SkClampMax(f>>16, maxX) */ 537 wide_i = vmaxq_s32(vshrq_n_s32(wide_fx,16), vdupq_n_s32(0)); 538 wide_i = vminq_s32(wide_i, vdupq_n_s32(maxX)); 539 540 /* i<<4 | TILEX_LOW_BITS(fx) */ 541 wide_lo = vshrq_n_s32(wide_fx, 12); 542 wide_i = vsliq_n_s32(wide_lo, wide_i, 4); 543 544 /* i<<14 */ 545 wide_i = vshlq_n_s32(wide_i, 14); 546 547 /* SkClampMax(((f + one)) >> 16, max) */ 548 wide_fx1 = vaddq_s32(wide_fx, wide_one); 549 wide_fx1 = vmaxq_s32(vshrq_n_s32(wide_fx1,16), vdupq_n_s32(0)); 550 wide_fx1 = vminq_s32(wide_fx1, vdupq_n_s32(maxX)); 551 552 /* final combination */ 553 wide_i = vorrq_s32(wide_i, wide_fx1); 554 555 vst1q_u32(xy, vreinterpretq_u32_s32(wide_i)); 556 557 #if 0 558 /* having a verification hook is a good idea */ 559 /* use debug_fx, debug_fx+dx, etc. */ 560 561 for (int i=0;i<4;i++) { 562 uint32_t want = PACK_FILTER_X_NAME(debug_fx, maxX, one PREAMBLE_ARG_X); 563 if (xy[i] != want) 564 { 565 /* print a nastygram */ 566 SkDebugf("clamp-filter-scale fails\n"); 567 SkDebugf("got %08x want %08x\n", xy[i], want); 568 SkDebugf("fx %08x debug_fx %08x dx %08x done %d\n", 569 fx, debug_fx, dx, count_done); 570 SkDebugf(" maxX %08x one %08x\n", maxX, one); 571 572 } 573 debug_fx += dx; 574 count_done++; 575 } 576 #endif 577 wide_fx += vdupq_n_s32(dx+dx+dx+dx); 578 fx += dx+dx+dx+dx; 579 xy += 4; 580 count -= 4; 581 } 582 } 583 584 while (--count >= 0) { 585 *xy++ = PACK_FILTER_X_NAME(fx, maxX, one PREAMBLE_ARG_X); 586 fx += dx; 587 } 588} 589 590static void AFFINE_FILTER_NAME(const SkBitmapProcState& s, 591 uint32_t xy[], int count, int x, int y) { 592 SkASSERT(s.fInvType & SkMatrix::kAffine_Mask); 593 SkASSERT((s.fInvType & ~(SkMatrix::kTranslate_Mask | 594 SkMatrix::kScale_Mask | 595 SkMatrix::kAffine_Mask)) == 0); 596 597 PREAMBLE(s); 598 SkPoint srcPt; 599 s.fInvProc(s.fInvMatrix, 600 SkIntToScalar(x) + SK_ScalarHalf, 601 SkIntToScalar(y) + SK_ScalarHalf, &srcPt); 602 603 SkFixed oneX = s.fFilterOneX; 604 SkFixed oneY = s.fFilterOneY; 605 SkFixed fx = SkScalarToFixed(srcPt.fX) - (oneX >> 1); 606 SkFixed fy = SkScalarToFixed(srcPt.fY) - (oneY >> 1); 607 SkFixed dx = s.fInvSx; 608 SkFixed dy = s.fInvKy; 609 unsigned maxX = s.fBitmap->width() - 1; 610 unsigned maxY = s.fBitmap->height() - 1; 611 612 if (count >= 4) { 613 int32x4_t wide_i, wide_lo; 614 int32x4_t wide_fx, wide_onex, wide_fx1; 615 int32x4_t wide_fy, wide_oney, wide_fy1; 616 617 #undef AFFINE_DEBUG 618 #if defined(AFFINE_DEBUG) 619 SkFixed fyp = fy; 620 SkFixed fxp = fx; 621 uint32_t *xyp = xy; 622 int count_done = 0; 623 #endif 624 625 wide_fx = vdupq_n_s32(fx); 626 wide_fx = vsetq_lane_s32(fx+dx, wide_fx, 1); 627 wide_fx = vsetq_lane_s32(fx+dx+dx, wide_fx, 2); 628 wide_fx = vsetq_lane_s32(fx+dx+dx+dx, wide_fx, 3); 629 630 wide_fy = vdupq_n_s32(fy); 631 wide_fy = vsetq_lane_s32(fy+dy, wide_fy, 1); 632 wide_fy = vsetq_lane_s32(fy+dy+dy, wide_fy, 2); 633 wide_fy = vsetq_lane_s32(fy+dy+dy+dy, wide_fy, 3); 634 635 wide_onex = vdupq_n_s32(oneX); 636 wide_oney = vdupq_n_s32(oneY); 637 638 while (count >= 4) { 639 int32x4_t wide_x; 640 int32x4_t wide_y; 641 642 /* do the X side, then the Y side, then interleave them */ 643 644 /* original expands to: 645 * unsigned i = SkClampMax((f) >> 16, max); 646 * i = (i << 4) | (((f) >> 12) & 0xF); 647 * return (i << 14) | (SkClampMax(((f + one)) >> 16, max)); 648 */ 649 650 /* i = SkClampMax(f>>16, maxX) */ 651 wide_i = vmaxq_s32(vshrq_n_s32(wide_fx,16), vdupq_n_s32(0)); 652 wide_i = vminq_s32(wide_i, vdupq_n_s32(maxX)); 653 654 /* i<<4 | TILEX_LOW_BITS(fx) */ 655 wide_lo = vshrq_n_s32(wide_fx, 12); 656 wide_i = vsliq_n_s32(wide_lo, wide_i, 4); 657 658 /* i<<14 */ 659 wide_i = vshlq_n_s32(wide_i, 14); 660 661 /* SkClampMax(((f + one)) >> 16, max) */ 662 wide_fx1 = vaddq_s32(wide_fx, wide_onex); 663 wide_fx1 = vmaxq_s32(vshrq_n_s32(wide_fx1,16), vdupq_n_s32(0)); 664 wide_fx1 = vminq_s32(wide_fx1, vdupq_n_s32(maxX)); 665 666 /* final combination */ 667 wide_x = vorrq_s32(wide_i, wide_fx1); 668 669 /* And now the Y side */ 670 671 /* i = SkClampMax(f>>16, maxX) */ 672 wide_i = vmaxq_s32(vshrq_n_s32(wide_fy,16), vdupq_n_s32(0)); 673 wide_i = vminq_s32(wide_i, vdupq_n_s32(maxY)); 674 675 /* i<<4 | TILEX_LOW_BITS(fx) */ 676 wide_lo = vshrq_n_s32(wide_fy, 12); 677 wide_i = vsliq_n_s32(wide_lo, wide_i, 4); 678 679 /* i<<14 */ 680 wide_i = vshlq_n_s32(wide_i, 14); 681 682 /* SkClampMax(((f + one)) >> 16, max) */ 683 wide_fy1 = vaddq_s32(wide_fy, wide_oney); 684 wide_fy1 = vmaxq_s32(vshrq_n_s32(wide_fy1,16), vdupq_n_s32(0)); 685 wide_fy1 = vminq_s32(wide_fy1, vdupq_n_s32(maxY)); 686 687 /* final combination */ 688 wide_y = vorrq_s32(wide_i, wide_fy1); 689 690 /* interleave as YXYXYXYX as part of the storing */ 691 { 692 /* vst2.32 needs side-by-side registers */ 693 register int32x4_t t_x asm("q1"); 694 register int32x4_t t_y asm("q0"); 695 696 t_x = wide_x; t_y = wide_y; 697 asm ("vst2.32 {q0-q1},[%2] /* y=%q0 x=%q1 */" 698 : 699 : "w" (t_y), "w" (t_x), "r" (xy) 700 ); 701 } 702 703 #if defined(AFFINE_DEBUG) 704 /* make sure we're good here -- check the 4 we just output */ 705 for (int i = 0; i<4;i++) { 706 uint32_t val; 707 val = PACK_FILTER_Y_NAME(fyp, maxY, oneY PREAMBLE_ARG_Y); 708 if (val != xy[i*2+0]) { 709 /* print a nastygram */ 710 SkDebugf("clamp-filter-affine fails\n"); 711 SkDebugf("[bad-y] got %08x want %08x\n", xy[i*2+0], val); 712 SkDebugf("fy %08x fxp %08x fyp %08x dx %08x dy %08x done %d\n", 713 fy, fxp, fyp, dx, dy, count_done); 714 SkDebugf(" maxY %08x oneY %08x\n", maxY, oneY); 715 } 716 val = PACK_FILTER_X_NAME(fxp, maxX, oneX PREAMBLE_ARG_X); 717 if (val != xy[i*2+1]) { 718 /* print a nastygram */ 719 SkDebugf("clamp-filter-affine fails\n"); 720 SkDebugf("[bad-x] got %08x want %08x\n", xy[i*2+1], val); 721 SkDebugf("fx %08x fxp %08x fyp %08x dx %08x dy %08x done %d\n", 722 fx, fxp, fyp, dx, dy, count_done); 723 SkDebugf(" maxX %08x one %08x\n", maxX, oneX); 724 } 725 fyp += dy; 726 fxp += dx; 727 count_done++; 728 } 729 #endif 730 731 wide_fx += vdupq_n_s32(dx+dx+dx+dx); 732 fx += dx+dx+dx+dx; 733 wide_fy += vdupq_n_s32(dy+dy+dy+dy); 734 fy += dy+dy+dy+dy; 735 xy += 8; /* 4 x's, 4 y's */ 736 count -= 4; 737 } 738 } 739 740 while (--count >= 0) { 741 /* NB: writing Y/X */ 742 *xy++ = PACK_FILTER_Y_NAME(fy, maxY, oneY PREAMBLE_ARG_Y); 743 fy += dy; 744 *xy++ = PACK_FILTER_X_NAME(fx, maxX, oneX PREAMBLE_ARG_X); 745 fx += dx; 746 } 747} 748 749static void PERSP_FILTER_NAME(const SkBitmapProcState& s, 750 uint32_t* SK_RESTRICT xy, int count, 751 int x, int y) { 752 SkASSERT(s.fInvType & SkMatrix::kPerspective_Mask); 753 754 PREAMBLE(s); 755 unsigned maxX = s.fBitmap->width() - 1; 756 unsigned maxY = s.fBitmap->height() - 1; 757 SkFixed oneX = s.fFilterOneX; 758 SkFixed oneY = s.fFilterOneY; 759 760 SkPerspIter iter(s.fInvMatrix, 761 SkIntToScalar(x) + SK_ScalarHalf, 762 SkIntToScalar(y) + SK_ScalarHalf, count); 763 764 while ((count = iter.next()) != 0) { 765 const SkFixed* SK_RESTRICT srcXY = iter.getXY(); 766 767 if (count >= 4) { 768 int32x4_t wide_i, wide_lo; 769 int32x4_t wide_fx1; 770 int32x4_t wide_fy1; 771 int32x4_t wide_x, wide_y; 772 773 while (count >= 4) { 774 /* RBE: it's good, but: 775 * -- we spill a constant that could be easily regnerated 776 * [perhaps tweak gcc's NEON constant costs?] 777 */ 778 779 /* load src: x-y-x-y-x-y-x-y */ 780 { 781 register int32x4_t q0 asm ("q0"); 782 register int32x4_t q1 asm ("q1"); 783 asm ("vld2.32 {q0-q1},[%2] /* x=%q0 y=%q1 */" 784 : "=w" (q0), "=w" (q1) 785 : "r" (srcXY)); 786 wide_x = q0; wide_y = q1; 787 } 788 789 /* do the X side, then the Y side, then interleave them */ 790 791 wide_x = vsubq_s32(wide_x, vdupq_n_s32 (oneX>>1)); 792 793 /* original expands to: 794 * unsigned i = SkClampMax((f) >> 16, max); 795 * i = (i << 4) | (((f) >> 12) & 0xF); 796 * return (i << 14) | (SkClampMax(((f + one)) >> 16, max)); 797 */ 798 799 /* i = SkClampMax(f>>16, maxX) */ 800 wide_i = vmaxq_s32 (vshrq_n_s32 (wide_x, 16), vdupq_n_s32 (0)); 801 wide_i = vminq_s32 (wide_i, vdupq_n_s32 (maxX)); 802 803 /* i<<4 | TILEX_LOW_BITS(fx) */ 804 wide_lo = vshrq_n_s32 (wide_x, 12); 805 wide_i = vsliq_n_s32 (wide_lo, wide_i, 4); 806 807 /* i<<14 */ 808 wide_i = vshlq_n_s32 (wide_i, 14); 809 810 /* SkClampMax(((f + one)) >> 16, max) */ 811 wide_fx1 = vaddq_s32 (wide_x, vdupq_n_s32(oneX)); 812 wide_fx1 = vmaxq_s32 (vshrq_n_s32 (wide_fx1, 16), vdupq_n_s32 (0)); 813 wide_fx1 = vminq_s32 (wide_fx1, vdupq_n_s32 (maxX)); 814 815 /* final combination */ 816 wide_x = vorrq_s32 (wide_i, wide_fx1); 817 818 819 /* And now the Y side */ 820 821 wide_y = vsubq_s32(wide_y, vdupq_n_s32 (oneY>>1)); 822 823 /* i = SkClampMax(f>>16, maxX) */ 824 wide_i = vmaxq_s32 (vshrq_n_s32 (wide_y, 16), vdupq_n_s32 (0)); 825 wide_i = vminq_s32 (wide_i, vdupq_n_s32 (maxY)); 826 827 /* i<<4 | TILEX_LOW_BITS(fx) */ 828 wide_lo = vshrq_n_s32 (wide_y, 12); 829 wide_i = vsliq_n_s32 (wide_lo, wide_i, 4); 830 831 /* i<<14 */ 832 wide_i = vshlq_n_s32 (wide_i, 14); 833 834 /* SkClampMax(((f + one)) >> 16, max) */ 835 836 /* wide_fy1_1 and wide_fy1_2 are just temporary variables to 837 * work-around an ICE in debug */ 838 int32x4_t wide_fy1_1 = vaddq_s32 (wide_y, vdupq_n_s32(oneY)); 839 int32x4_t wide_fy1_2 = vmaxq_s32 (vshrq_n_s32 (wide_fy1_1, 16), 840 vdupq_n_s32 (0)); 841 wide_fy1 = vminq_s32 (wide_fy1_2, vdupq_n_s32 (maxY)); 842 843 /* final combination */ 844 wide_y = vorrq_s32 (wide_i, wide_fy1); 845 846 /* switch them around; have to do it this way to get them 847 * in the proper registers to match our instruction */ 848 849 /* iteration bookkeeping, ahead of the asm() for scheduling */ 850 srcXY += 2*4; 851 count -= 4; 852 853 /* store interleaved as y-x-y-x-y-x-y-x (NB != read order) */ 854 { 855 register int32x4_t q0 asm ("q0") = wide_y; 856 register int32x4_t q1 asm ("q1") = wide_x; 857 858 asm ("vst2.32 {q0-q1},[%2] /* y=%q0 x=%q1 */" 859 : 860 : "w" (q0), "w" (q1), "r" (xy)); 861 } 862 863 /* on to the next iteration */ 864 /* count, srcXY are handled above */ 865 xy += 2*4; 866 } 867 } 868 869 /* was do-while; NEON code invalidates original count>0 assumption */ 870 while (--count >= 0) { 871 /* NB: we read x/y, we write y/x */ 872 *xy++ = PACK_FILTER_Y_NAME(srcXY[1] - (oneY >> 1), maxY, 873 oneY PREAMBLE_ARG_Y); 874 *xy++ = PACK_FILTER_X_NAME(srcXY[0] - (oneX >> 1), maxX, 875 oneX PREAMBLE_ARG_X); 876 srcXY += 2; 877 } 878 } 879} 880 881const SkBitmapProcState::MatrixProc MAKENAME(_Procs)[] = { 882 SCALE_NOFILTER_NAME, 883 SCALE_FILTER_NAME, 884 AFFINE_NOFILTER_NAME, 885 AFFINE_FILTER_NAME, 886 PERSP_NOFILTER_NAME, 887 PERSP_FILTER_NAME 888}; 889 890#undef MAKENAME 891#undef TILEX_PROCF 892#undef TILEY_PROCF 893#ifdef CHECK_FOR_DECAL 894 #undef CHECK_FOR_DECAL 895#endif 896 897#undef SCALE_NOFILTER_NAME 898#undef SCALE_FILTER_NAME 899#undef AFFINE_NOFILTER_NAME 900#undef AFFINE_FILTER_NAME 901#undef PERSP_NOFILTER_NAME 902#undef PERSP_FILTER_NAME 903 904#undef PREAMBLE 905#undef PREAMBLE_PARAM_X 906#undef PREAMBLE_PARAM_Y 907#undef PREAMBLE_ARG_X 908#undef PREAMBLE_ARG_Y 909 910#undef TILEX_LOW_BITS 911#undef TILEY_LOW_BITS 912