1// Copyright 2010 Google Inc. 2// 3// This code is licensed under the same terms as WebM: 4// Software License Agreement: http://www.webmproject.org/license/software/ 5// Additional IP Rights Grant: http://www.webmproject.org/license/additional/ 6// ----------------------------------------------------------------------------- 7// 8// speed-critical functions. 9// 10// Author: Skal (pascal.massimino@gmail.com) 11 12#include "vp8i.h" 13 14#if defined(__cplusplus) || defined(c_plusplus) 15extern "C" { 16#endif 17 18//----------------------------------------------------------------------------- 19// run-time tables (~4k) 20 21static uint8_t abs0[255 + 255 + 1]; // abs(i) 22static uint8_t abs1[255 + 255 + 1]; // abs(i)>>1 23static int8_t sclip1[1020 + 1020 + 1]; // clips [-1020, 1020] to [-128, 127] 24static int8_t sclip2[112 + 112 + 1]; // clips [-112, 112] to [-16, 15] 25static uint8_t clip1[255 + 510 + 1]; // clips [-255,510] to [0,255] 26 27// We declare this variable 'volatile' to prevent instruction reordering 28// and make sure it's set to true _last_ (so as to be thread-safe) 29static volatile int tables_ok = 0; 30 31void VP8DspInitTables(void) { 32 if (!tables_ok) { 33 int i; 34 for (i = -255; i <= 255; ++i) { 35 abs0[255 + i] = (i < 0) ? -i : i; 36 abs1[255 + i] = abs0[255 + i] >> 1; 37 } 38 for (i = -1020; i <= 1020; ++i) { 39 sclip1[1020 + i] = (i < -128) ? -128 : (i > 127) ? 127 : i; 40 } 41 for (i = -112; i <= 112; ++i) { 42 sclip2[112 + i] = (i < -16) ? -16 : (i > 15) ? 15 : i; 43 } 44 for (i = -255; i <= 255 + 255; ++i) { 45 clip1[255 + i] = (i < 0) ? 0 : (i > 255) ? 255 : i; 46 } 47 tables_ok = 1; 48 } 49} 50 51static inline uint8_t clip_8b(int v) { 52 return (!(v & ~0xff)) ? v : (v < 0) ? 0 : 255; 53} 54 55//----------------------------------------------------------------------------- 56// Transforms (Paragraph 14.4) 57 58#define STORE(x, y, v) \ 59 dst[x + y * BPS] = clip_8b(dst[x + y * BPS] + ((v) >> 3)) 60 61static const int kC1 = 20091 + (1 << 16); 62static const int kC2 = 35468; 63#define MUL(a, b) (((a) * (b)) >> 16) 64 65static void TransformOne(const int16_t* in, uint8_t* dst) { 66 int C[4 * 4], *tmp; 67 int i; 68 tmp = C; 69 for (i = 0; i < 4; ++i) { // vertical pass 70 const int a = in[0] + in[8]; // [-4096, 4094] 71 const int b = in[0] - in[8]; // [-4095, 4095] 72 const int c = MUL(in[4], kC2) - MUL(in[12], kC1); // [-3783, 3783] 73 const int d = MUL(in[4], kC1) + MUL(in[12], kC2); // [-3785, 3781] 74 tmp[0] = a + d; // [-7881, 7875] 75 tmp[1] = b + c; // [-7878, 7878] 76 tmp[2] = b - c; // [-7878, 7878] 77 tmp[3] = a - d; // [-7877, 7879] 78 tmp += 4; 79 in++; 80 } 81 // Each pass is expanding the dynamic range by ~3.85 (upper bound). 82 // The exact value is (2. + (kC1 + kC2) / 65536). 83 // After the second pass, maximum interval is [-3794, 3794], assuming 84 // an input in [-2048, 2047] interval. We then need to add a dst value 85 // in the [0, 255] range. 86 // In the worst case scenario, the input to clip_8b() can be as large as 87 // [-60713, 60968]. 88 tmp = C; 89 for (i = 0; i < 4; ++i) { // horizontal pass 90 const int dc = tmp[0] + 4; 91 const int a = dc + tmp[8]; 92 const int b = dc - tmp[8]; 93 const int c = MUL(tmp[4], kC2) - MUL(tmp[12], kC1); 94 const int d = MUL(tmp[4], kC1) + MUL(tmp[12], kC2); 95 STORE(0, 0, a + d); 96 STORE(1, 0, b + c); 97 STORE(2, 0, b - c); 98 STORE(3, 0, a - d); 99 tmp++; 100 dst += BPS; 101 } 102} 103#undef MUL 104 105static void TransformTwo(const int16_t* in, uint8_t* dst, int do_two) { 106 TransformOne(in, dst); 107 if (do_two) { 108 TransformOne(in + 16, dst + 4); 109 } 110} 111 112static void TransformUV(const int16_t* in, uint8_t* dst) { 113 VP8Transform(in + 0 * 16, dst, 1); 114 VP8Transform(in + 2 * 16, dst + 4 * BPS, 1); 115} 116 117static void TransformDC(const int16_t *in, uint8_t* dst) { 118 const int DC = in[0] + 4; 119 int i, j; 120 for (j = 0; j < 4; ++j) { 121 for (i = 0; i < 4; ++i) { 122 STORE(i, j, DC); 123 } 124 } 125} 126 127static void TransformDCUV(const int16_t* in, uint8_t* dst) { 128 if (in[0 * 16]) TransformDC(in + 0 * 16, dst); 129 if (in[1 * 16]) TransformDC(in + 1 * 16, dst + 4); 130 if (in[2 * 16]) TransformDC(in + 2 * 16, dst + 4 * BPS); 131 if (in[3 * 16]) TransformDC(in + 3 * 16, dst + 4 * BPS + 4); 132} 133 134#undef STORE 135 136// default C implementations: 137VP8Idct2 VP8Transform = TransformTwo; 138VP8Idct VP8TransformUV = TransformUV; 139VP8Idct VP8TransformDC = TransformDC; 140VP8Idct VP8TransformDCUV = TransformDCUV; 141 142//----------------------------------------------------------------------------- 143// Paragraph 14.3 144 145static void TransformWHT(const int16_t* in, int16_t* out) { 146 int tmp[16]; 147 int i; 148 for (i = 0; i < 4; ++i) { 149 const int a0 = in[0 + i] + in[12 + i]; 150 const int a1 = in[4 + i] + in[ 8 + i]; 151 const int a2 = in[4 + i] - in[ 8 + i]; 152 const int a3 = in[0 + i] - in[12 + i]; 153 tmp[0 + i] = a0 + a1; 154 tmp[8 + i] = a0 - a1; 155 tmp[4 + i] = a3 + a2; 156 tmp[12 + i] = a3 - a2; 157 } 158 for (i = 0; i < 4; ++i) { 159 const int dc = tmp[0 + i * 4] + 3; // w/ rounder 160 const int a0 = dc + tmp[3 + i * 4]; 161 const int a1 = tmp[1 + i * 4] + tmp[2 + i * 4]; 162 const int a2 = tmp[1 + i * 4] - tmp[2 + i * 4]; 163 const int a3 = dc - tmp[3 + i * 4]; 164 out[ 0] = (a0 + a1) >> 3; 165 out[16] = (a3 + a2) >> 3; 166 out[32] = (a0 - a1) >> 3; 167 out[48] = (a3 - a2) >> 3; 168 out += 64; 169 } 170} 171 172void (*VP8TransformWHT)(const int16_t* in, int16_t* out) = TransformWHT; 173 174//----------------------------------------------------------------------------- 175// Intra predictions 176 177#define OUT(x, y) dst[(x) + (y) * BPS] 178 179static inline void TrueMotion(uint8_t *dst, int size) { 180 const uint8_t* top = dst - BPS; 181 const uint8_t* const clip0 = clip1 + 255 - top[-1]; 182 int y; 183 for (y = 0; y < size; ++y) { 184 const uint8_t* const clip = clip0 + dst[-1]; 185 int x; 186 for (x = 0; x < size; ++x) { 187 dst[x] = clip[top[x]]; 188 } 189 dst += BPS; 190 } 191} 192static void TM4(uint8_t *dst) { TrueMotion(dst, 4); } 193static void TM8uv(uint8_t *dst) { TrueMotion(dst, 8); } 194static void TM16(uint8_t *dst) { TrueMotion(dst, 16); } 195 196//----------------------------------------------------------------------------- 197// 16x16 198 199static void VE16(uint8_t *dst) { // vertical 200 int j; 201 for (j = 0; j < 16; ++j) { 202 memcpy(dst + j * BPS, dst - BPS, 16); 203 } 204} 205 206static void HE16(uint8_t *dst) { // horizontal 207 int j; 208 for (j = 16; j > 0; --j) { 209 memset(dst, dst[-1], 16); 210 dst += BPS; 211 } 212} 213 214static inline void Put16(int v, uint8_t* dst) { 215 int j; 216 for (j = 0; j < 16; ++j) { 217 memset(dst + j * BPS, v, 16); 218 } 219} 220 221static void DC16(uint8_t *dst) { // DC 222 int DC = 16; 223 int j; 224 for (j = 0; j < 16; ++j) { 225 DC += dst[-1 + j * BPS] + dst[j - BPS]; 226 } 227 Put16(DC >> 5, dst); 228} 229 230static void DC16NoTop(uint8_t *dst) { // DC with top samples not available 231 int DC = 8; 232 int j; 233 for (j = 0; j < 16; ++j) { 234 DC += dst[-1 + j * BPS]; 235 } 236 Put16(DC >> 4, dst); 237} 238 239static void DC16NoLeft(uint8_t *dst) { // DC with left samples not available 240 int DC = 8; 241 int i; 242 for (i = 0; i < 16; ++i) { 243 DC += dst[i - BPS]; 244 } 245 Put16(DC >> 4, dst); 246} 247 248static void DC16NoTopLeft(uint8_t *dst) { // DC with no top and left samples 249 Put16(0x80, dst); 250} 251 252//----------------------------------------------------------------------------- 253// 4x4 254 255#define AVG3(a, b, c) (((a) + 2 * (b) + (c) + 2) >> 2) 256#define AVG2(a, b) (((a) + (b) + 1) >> 1) 257 258static void VE4(uint8_t *dst) { // vertical 259 const uint8_t* top = dst - BPS; 260 const uint8_t vals[4] = { 261 AVG3(top[-1], top[0], top[1]), 262 AVG3(top[ 0], top[1], top[2]), 263 AVG3(top[ 1], top[2], top[3]), 264 AVG3(top[ 2], top[3], top[4]) 265 }; 266 int i; 267 for (i = 0; i < 4; ++i) { 268 memcpy(dst + i * BPS, vals, sizeof(vals)); 269 } 270} 271 272static void HE4(uint8_t *dst) { // horizontal 273 const int A = dst[-1 - BPS]; 274 const int B = dst[-1]; 275 const int C = dst[-1 + BPS]; 276 const int D = dst[-1 + 2 * BPS]; 277 const int E = dst[-1 + 3 * BPS]; 278 *(uint32_t*)(dst + 0 * BPS) = 0x01010101U * AVG3(A, B, C); 279 *(uint32_t*)(dst + 1 * BPS) = 0x01010101U * AVG3(B, C, D); 280 *(uint32_t*)(dst + 2 * BPS) = 0x01010101U * AVG3(C, D, E); 281 *(uint32_t*)(dst + 3 * BPS) = 0x01010101U * AVG3(D, E, E); 282} 283 284static void DC4(uint8_t *dst) { // DC 285 uint32_t dc = 4; 286 int i; 287 for (i = 0; i < 4; ++i) dc += dst[i - BPS] + dst[-1 + i * BPS]; 288 dc >>= 3; 289 for (i = 0; i < 4; ++i) memset(dst + i * BPS, dc, 4); 290} 291 292static void RD4(uint8_t *dst) { // Down-right 293 const int I = dst[-1 + 0 * BPS]; 294 const int J = dst[-1 + 1 * BPS]; 295 const int K = dst[-1 + 2 * BPS]; 296 const int L = dst[-1 + 3 * BPS]; 297 const int X = dst[-1 - BPS]; 298 const int A = dst[0 - BPS]; 299 const int B = dst[1 - BPS]; 300 const int C = dst[2 - BPS]; 301 const int D = dst[3 - BPS]; 302 OUT(0, 3) = AVG3(J, K, L); 303 OUT(0, 2) = OUT(1, 3) = AVG3(I, J, K); 304 OUT(0, 1) = OUT(1, 2) = OUT(2, 3) = AVG3(X, I, J); 305 OUT(0, 0) = OUT(1, 1) = OUT(2, 2) = OUT(3, 3) = AVG3(A, X, I); 306 OUT(1, 0) = OUT(2, 1) = OUT(3, 2) = AVG3(B, A, X); 307 OUT(2, 0) = OUT(3, 1) = AVG3(C, B, A); 308 OUT(3, 0) = AVG3(D, C, B); 309} 310 311static void LD4(uint8_t *dst) { // Down-Left 312 const int A = dst[0 - BPS]; 313 const int B = dst[1 - BPS]; 314 const int C = dst[2 - BPS]; 315 const int D = dst[3 - BPS]; 316 const int E = dst[4 - BPS]; 317 const int F = dst[5 - BPS]; 318 const int G = dst[6 - BPS]; 319 const int H = dst[7 - BPS]; 320 OUT(0, 0) = AVG3(A, B, C); 321 OUT(1, 0) = OUT(0, 1) = AVG3(B, C, D); 322 OUT(2, 0) = OUT(1, 1) = OUT(0, 2) = AVG3(C, D, E); 323 OUT(3, 0) = OUT(2, 1) = OUT(1, 2) = OUT(0, 3) = AVG3(D, E, F); 324 OUT(3, 1) = OUT(2, 2) = OUT(1, 3) = AVG3(E, F, G); 325 OUT(3, 2) = OUT(2, 3) = AVG3(F, G, H); 326 OUT(3, 3) = AVG3(G, H, H); 327} 328 329static void VR4(uint8_t *dst) { // Vertical-Right 330 const int I = dst[-1 + 0 * BPS]; 331 const int J = dst[-1 + 1 * BPS]; 332 const int K = dst[-1 + 2 * BPS]; 333 const int X = dst[-1 - BPS]; 334 const int A = dst[0 - BPS]; 335 const int B = dst[1 - BPS]; 336 const int C = dst[2 - BPS]; 337 const int D = dst[3 - BPS]; 338 OUT(0, 0) = OUT(1, 2) = AVG2(X, A); 339 OUT(1, 0) = OUT(2, 2) = AVG2(A, B); 340 OUT(2, 0) = OUT(3, 2) = AVG2(B, C); 341 OUT(3, 0) = AVG2(C, D); 342 343 OUT(0, 3) = AVG3(K, J, I); 344 OUT(0, 2) = AVG3(J, I, X); 345 OUT(0, 1) = OUT(1, 3) = AVG3(I, X, A); 346 OUT(1, 1) = OUT(2, 3) = AVG3(X, A, B); 347 OUT(2, 1) = OUT(3, 3) = AVG3(A, B, C); 348 OUT(3, 1) = AVG3(B, C, D); 349} 350 351static void VL4(uint8_t *dst) { // Vertical-Left 352 const int A = dst[0 - BPS]; 353 const int B = dst[1 - BPS]; 354 const int C = dst[2 - BPS]; 355 const int D = dst[3 - BPS]; 356 const int E = dst[4 - BPS]; 357 const int F = dst[5 - BPS]; 358 const int G = dst[6 - BPS]; 359 const int H = dst[7 - BPS]; 360 OUT(0, 0) = AVG2(A, B); 361 OUT(1, 0) = OUT(0, 2) = AVG2(B, C); 362 OUT(2, 0) = OUT(1, 2) = AVG2(C, D); 363 OUT(3, 0) = OUT(2, 2) = AVG2(D, E); 364 365 OUT(0, 1) = AVG3(A, B, C); 366 OUT(1, 1) = OUT(0, 3) = AVG3(B, C, D); 367 OUT(2, 1) = OUT(1, 3) = AVG3(C, D, E); 368 OUT(3, 1) = OUT(2, 3) = AVG3(D, E, F); 369 OUT(3, 2) = AVG3(E, F, G); 370 OUT(3, 3) = AVG3(F, G, H); 371} 372 373static void HU4(uint8_t *dst) { // Horizontal-Up 374 const int I = dst[-1 + 0 * BPS]; 375 const int J = dst[-1 + 1 * BPS]; 376 const int K = dst[-1 + 2 * BPS]; 377 const int L = dst[-1 + 3 * BPS]; 378 OUT(0, 0) = AVG2(I, J); 379 OUT(2, 0) = OUT(0, 1) = AVG2(J, K); 380 OUT(2, 1) = OUT(0, 2) = AVG2(K, L); 381 OUT(1, 0) = AVG3(I, J, K); 382 OUT(3, 0) = OUT(1, 1) = AVG3(J, K, L); 383 OUT(3, 1) = OUT(1, 2) = AVG3(K, L, L); 384 OUT(3, 2) = OUT(2, 2) = 385 OUT(0, 3) = OUT(1, 3) = OUT(2, 3) = OUT(3, 3) = L; 386} 387 388static void HD4(uint8_t *dst) { // Horizontal-Down 389 const int I = dst[-1 + 0 * BPS]; 390 const int J = dst[-1 + 1 * BPS]; 391 const int K = dst[-1 + 2 * BPS]; 392 const int L = dst[-1 + 3 * BPS]; 393 const int X = dst[-1 - BPS]; 394 const int A = dst[0 - BPS]; 395 const int B = dst[1 - BPS]; 396 const int C = dst[2 - BPS]; 397 398 OUT(0, 0) = OUT(2, 1) = AVG2(I, X); 399 OUT(0, 1) = OUT(2, 2) = AVG2(J, I); 400 OUT(0, 2) = OUT(2, 3) = AVG2(K, J); 401 OUT(0, 3) = AVG2(L, K); 402 403 OUT(3, 0) = AVG3(A, B, C); 404 OUT(2, 0) = AVG3(X, A, B); 405 OUT(1, 0) = OUT(3, 1) = AVG3(I, X, A); 406 OUT(1, 1) = OUT(3, 2) = AVG3(J, I, X); 407 OUT(1, 2) = OUT(3, 3) = AVG3(K, J, I); 408 OUT(1, 3) = AVG3(L, K, J); 409} 410 411#undef AVG3 412#undef AVG2 413 414//----------------------------------------------------------------------------- 415// Chroma 416 417static void VE8uv(uint8_t *dst) { // vertical 418 int j; 419 for (j = 0; j < 8; ++j) { 420 memcpy(dst + j * BPS, dst - BPS, 8); 421 } 422} 423 424static void HE8uv(uint8_t *dst) { // horizontal 425 int j; 426 for (j = 0; j < 8; ++j) { 427 memset(dst, dst[-1], 8); 428 dst += BPS; 429 } 430} 431 432// helper for chroma-DC predictions 433static inline void Put8x8uv(uint64_t v, uint8_t* dst) { 434 int j; 435 for (j = 0; j < 8; ++j) { 436 *(uint64_t*)(dst + j * BPS) = v; 437 } 438} 439 440static void DC8uv(uint8_t *dst) { // DC 441 int dc0 = 8; 442 int i; 443 for (i = 0; i < 8; ++i) { 444 dc0 += dst[i - BPS] + dst[-1 + i * BPS]; 445 } 446 Put8x8uv((uint64_t)((dc0 >> 4) * 0x0101010101010101ULL), dst); 447} 448 449static void DC8uvNoLeft(uint8_t *dst) { // DC with no left samples 450 int dc0 = 4; 451 int i; 452 for (i = 0; i < 8; ++i) { 453 dc0 += dst[i - BPS]; 454 } 455 Put8x8uv((uint64_t)((dc0 >> 3) * 0x0101010101010101ULL), dst); 456} 457 458static void DC8uvNoTop(uint8_t *dst) { // DC with no top samples 459 int dc0 = 4; 460 int i; 461 for (i = 0; i < 8; ++i) { 462 dc0 += dst[-1 + i * BPS]; 463 } 464 Put8x8uv((uint64_t)((dc0 >> 3) * 0x0101010101010101ULL), dst); 465} 466 467static void DC8uvNoTopLeft(uint8_t *dst) { // DC with nothing 468 Put8x8uv(0x8080808080808080ULL, dst); 469} 470 471//----------------------------------------------------------------------------- 472// default C implementations 473 474VP8PredFunc VP8PredLuma4[NUM_BMODES] = { 475 DC4, TM4, VE4, HE4, RD4, VR4, LD4, VL4, HD4, HU4 476}; 477 478VP8PredFunc VP8PredLuma16[NUM_B_DC_MODES] = { 479 DC16, TM16, VE16, HE16, 480 DC16NoTop, DC16NoLeft, DC16NoTopLeft 481}; 482 483VP8PredFunc VP8PredChroma8[NUM_B_DC_MODES] = { 484 DC8uv, TM8uv, VE8uv, HE8uv, 485 DC8uvNoTop, DC8uvNoLeft, DC8uvNoTopLeft 486}; 487 488//----------------------------------------------------------------------------- 489// Edge filtering functions 490 491// 4 pixels in, 2 pixels out 492static inline void do_filter2(uint8_t* p, int step) { 493 const int p1 = p[-2*step], p0 = p[-step], q0 = p[0], q1 = p[step]; 494 const int a = 3 * (q0 - p0) + sclip1[1020 + p1 - q1]; 495 const int a1 = sclip2[112 + ((a + 4) >> 3)]; 496 const int a2 = sclip2[112 + ((a + 3) >> 3)]; 497 p[-step] = clip1[255 + p0 + a2]; 498 p[ 0] = clip1[255 + q0 - a1]; 499} 500 501// 4 pixels in, 4 pixels out 502static inline void do_filter4(uint8_t* p, int step) { 503 const int p1 = p[-2*step], p0 = p[-step], q0 = p[0], q1 = p[step]; 504 const int a = 3 * (q0 - p0); 505 const int a1 = sclip2[112 + ((a + 4) >> 3)]; 506 const int a2 = sclip2[112 + ((a + 3) >> 3)]; 507 const int a3 = (a1 + 1) >> 1; 508 p[-2*step] = clip1[255 + p1 + a3]; 509 p[- step] = clip1[255 + p0 + a2]; 510 p[ 0] = clip1[255 + q0 - a1]; 511 p[ step] = clip1[255 + q1 - a3]; 512} 513 514// 6 pixels in, 6 pixels out 515static inline void do_filter6(uint8_t* p, int step) { 516 const int p2 = p[-3*step], p1 = p[-2*step], p0 = p[-step]; 517 const int q0 = p[0], q1 = p[step], q2 = p[2*step]; 518 const int a = sclip1[1020 + 3 * (q0 - p0) + sclip1[1020 + p1 - q1]]; 519 const int a1 = (27 * a + 63) >> 7; // eq. to ((3 * a + 7) * 9) >> 7 520 const int a2 = (18 * a + 63) >> 7; // eq. to ((2 * a + 7) * 9) >> 7 521 const int a3 = (9 * a + 63) >> 7; // eq. to ((1 * a + 7) * 9) >> 7 522 p[-3*step] = clip1[255 + p2 + a3]; 523 p[-2*step] = clip1[255 + p1 + a2]; 524 p[- step] = clip1[255 + p0 + a1]; 525 p[ 0] = clip1[255 + q0 - a1]; 526 p[ step] = clip1[255 + q1 - a2]; 527 p[ 2*step] = clip1[255 + q2 - a3]; 528} 529 530static inline int hev(const uint8_t* p, int step, int thresh) { 531 const int p1 = p[-2*step], p0 = p[-step], q0 = p[0], q1 = p[step]; 532 return (abs0[255 + p1 - p0] > thresh) || (abs0[255 + q1 - q0] > thresh); 533} 534 535static inline int needs_filter(const uint8_t* p, int step, int thresh) { 536 const int p1 = p[-2*step], p0 = p[-step], q0 = p[0], q1 = p[step]; 537 return (2 * abs0[255 + p0 - q0] + abs1[255 + p1 - q1]) <= thresh; 538} 539 540static inline int needs_filter2(const uint8_t* p, int step, int t, int it) { 541 const int p3 = p[-4*step], p2 = p[-3*step], p1 = p[-2*step], p0 = p[-step]; 542 const int q0 = p[0], q1 = p[step], q2 = p[2*step], q3 = p[3*step]; 543 if ((2 * abs0[255 + p0 - q0] + abs1[255 + p1 - q1]) > t) 544 return 0; 545 return abs0[255 + p3 - p2] <= it && abs0[255 + p2 - p1] <= it && 546 abs0[255 + p1 - p0] <= it && abs0[255 + q3 - q2] <= it && 547 abs0[255 + q2 - q1] <= it && abs0[255 + q1 - q0] <= it; 548} 549 550//----------------------------------------------------------------------------- 551// Simple In-loop filtering (Paragraph 15.2) 552 553static void SimpleVFilter16(uint8_t* p, int stride, int thresh) { 554 int i; 555 for (i = 0; i < 16; ++i) { 556 if (needs_filter(p + i, stride, thresh)) { 557 do_filter2(p + i, stride); 558 } 559 } 560} 561 562static void SimpleHFilter16(uint8_t* p, int stride, int thresh) { 563 int i; 564 for (i = 0; i < 16; ++i) { 565 if (needs_filter(p + i * stride, 1, thresh)) { 566 do_filter2(p + i * stride, 1); 567 } 568 } 569} 570 571static void SimpleVFilter16i(uint8_t* p, int stride, int thresh) { 572 int k; 573 for (k = 3; k > 0; --k) { 574 p += 4 * stride; 575 SimpleVFilter16(p, stride, thresh); 576 } 577} 578 579static void SimpleHFilter16i(uint8_t* p, int stride, int thresh) { 580 int k; 581 for (k = 3; k > 0; --k) { 582 p += 4; 583 SimpleHFilter16(p, stride, thresh); 584 } 585} 586 587//----------------------------------------------------------------------------- 588// Complex In-loop filtering (Paragraph 15.3) 589 590static inline void FilterLoop26(uint8_t* p, int hstride, int vstride, int size, 591 int thresh, int ithresh, int hev_thresh) { 592 while (size-- > 0) { 593 if (needs_filter2(p, hstride, thresh, ithresh)) { 594 if (hev(p, hstride, hev_thresh)) { 595 do_filter2(p, hstride); 596 } else { 597 do_filter6(p, hstride); 598 } 599 } 600 p += vstride; 601 } 602} 603 604static inline void FilterLoop24(uint8_t* p, int hstride, int vstride, int size, 605 int thresh, int ithresh, int hev_thresh) { 606 while (size-- > 0) { 607 if (needs_filter2(p, hstride, thresh, ithresh)) { 608 if (hev(p, hstride, hev_thresh)) { 609 do_filter2(p, hstride); 610 } else { 611 do_filter4(p, hstride); 612 } 613 } 614 p += vstride; 615 } 616} 617 618// on macroblock edges 619static void VFilter16(uint8_t* p, int stride, 620 int thresh, int ithresh, int hev_thresh) { 621 FilterLoop26(p, stride, 1, 16, thresh, ithresh, hev_thresh); 622} 623 624static void HFilter16(uint8_t* p, int stride, 625 int thresh, int ithresh, int hev_thresh) { 626 FilterLoop26(p, 1, stride, 16, thresh, ithresh, hev_thresh); 627} 628 629// on three inner edges 630static void VFilter16i(uint8_t* p, int stride, 631 int thresh, int ithresh, int hev_thresh) { 632 int k; 633 for (k = 3; k > 0; --k) { 634 p += 4 * stride; 635 FilterLoop24(p, stride, 1, 16, thresh, ithresh, hev_thresh); 636 } 637} 638 639static void HFilter16i(uint8_t* p, int stride, 640 int thresh, int ithresh, int hev_thresh) { 641 int k; 642 for (k = 3; k > 0; --k) { 643 p += 4; 644 FilterLoop24(p, 1, stride, 16, thresh, ithresh, hev_thresh); 645 } 646} 647 648// 8-pixels wide variant, for chroma filtering 649static void VFilter8(uint8_t* u, uint8_t* v, int stride, 650 int thresh, int ithresh, int hev_thresh) { 651 FilterLoop26(u, stride, 1, 8, thresh, ithresh, hev_thresh); 652 FilterLoop26(v, stride, 1, 8, thresh, ithresh, hev_thresh); 653} 654 655static void HFilter8(uint8_t* u, uint8_t* v, int stride, 656 int thresh, int ithresh, int hev_thresh) { 657 FilterLoop26(u, 1, stride, 8, thresh, ithresh, hev_thresh); 658 FilterLoop26(v, 1, stride, 8, thresh, ithresh, hev_thresh); 659} 660 661static void VFilter8i(uint8_t* u, uint8_t* v, int stride, 662 int thresh, int ithresh, int hev_thresh) { 663 FilterLoop24(u + 4 * stride, stride, 1, 8, thresh, ithresh, hev_thresh); 664 FilterLoop24(v + 4 * stride, stride, 1, 8, thresh, ithresh, hev_thresh); 665} 666 667static void HFilter8i(uint8_t* u, uint8_t* v, int stride, 668 int thresh, int ithresh, int hev_thresh) { 669 FilterLoop24(u + 4, 1, stride, 8, thresh, ithresh, hev_thresh); 670 FilterLoop24(v + 4, 1, stride, 8, thresh, ithresh, hev_thresh); 671} 672 673//----------------------------------------------------------------------------- 674 675void (*VP8VFilter16)(uint8_t*, int, int, int, int) = VFilter16; 676void (*VP8HFilter16)(uint8_t*, int, int, int, int) = HFilter16; 677void (*VP8VFilter8)(uint8_t*, uint8_t*, int, int, int, int) = VFilter8; 678void (*VP8HFilter8)(uint8_t*, uint8_t*, int, int, int, int) = HFilter8; 679void (*VP8VFilter16i)(uint8_t*, int, int, int, int) = VFilter16i; 680void (*VP8HFilter16i)(uint8_t*, int, int, int, int) = HFilter16i; 681void (*VP8VFilter8i)(uint8_t*, uint8_t*, int, int, int, int) = VFilter8i; 682void (*VP8HFilter8i)(uint8_t*, uint8_t*, int, int, int, int) = HFilter8i; 683 684void (*VP8SimpleVFilter16)(uint8_t*, int, int) = SimpleVFilter16; 685void (*VP8SimpleHFilter16)(uint8_t*, int, int) = SimpleHFilter16; 686void (*VP8SimpleVFilter16i)(uint8_t*, int, int) = SimpleVFilter16i; 687void (*VP8SimpleHFilter16i)(uint8_t*, int, int) = SimpleHFilter16i; 688 689//----------------------------------------------------------------------------- 690// SSE2 detection. 691// 692 693#if defined(__pic__) && defined(__i386__) 694static inline void GetCPUInfo(int cpu_info[4], int info_type) { 695 __asm__ volatile ( 696 "mov %%ebx, %%edi\n" 697 "cpuid\n" 698 "xchg %%edi, %%ebx\n" 699 : "=a"(cpu_info[0]), "=D"(cpu_info[1]), "=c"(cpu_info[2]), "=d"(cpu_info[3]) 700 : "a"(info_type)); 701} 702#elif defined(__i386__) || defined(__x86_64__) 703static inline void GetCPUInfo(int cpu_info[4], int info_type) { 704 __asm__ volatile ( 705 "cpuid\n" 706 : "=a"(cpu_info[0]), "=b"(cpu_info[1]), "=c"(cpu_info[2]), "=d"(cpu_info[3]) 707 : "a"(info_type)); 708} 709#elif defined(_MSC_VER) // Visual C++ 710#define GetCPUInfo __cpuid 711#endif 712 713#if defined(__i386__) || defined(__x86_64__) || defined(_MSC_VER) 714static int x86CPUInfo(CPUFeature feature) { 715 int cpu_info[4]; 716 GetCPUInfo(cpu_info, 1); 717 if (feature == kSSE2) { 718 return 0 != (cpu_info[3] & 0x04000000); 719 } 720 if (feature == kSSE3) { 721 return 0 != (cpu_info[2] & 0x00000001); 722 } 723 return 0; 724} 725VP8CPUInfo VP8DecGetCPUInfo = x86CPUInfo; 726#else 727VP8CPUInfo VP8DecGetCPUInfo = NULL; 728#endif 729 730//----------------------------------------------------------------------------- 731 732extern void VP8DspInitSSE2(void); 733 734void VP8DspInit(void) { 735 // If defined, use CPUInfo() to overwrite some pointers with faster versions. 736 if (VP8DecGetCPUInfo) { 737 if (VP8DecGetCPUInfo(kSSE2)) { 738#if defined(__SSE2__) || defined(_MSC_VER) 739 VP8DspInitSSE2(); 740#endif 741 } 742 if (VP8DecGetCPUInfo(kSSE3)) { 743 // later we'll plug some SSE3 variant here 744 } 745 } 746} 747 748#if defined(__cplusplus) || defined(c_plusplus) 749} // extern "C" 750#endif 751