vp9_loopfilter.c revision da49e34c1fb5e99681f4ad99c21d9cfd83eddb96
1ba164dffc5a6795bce97fae02b51ccf3330e15e4hkuang/*
2ba164dffc5a6795bce97fae02b51ccf3330e15e4hkuang *  Copyright (c) 2010 The WebM project authors. All Rights Reserved.
3ba164dffc5a6795bce97fae02b51ccf3330e15e4hkuang *
4ba164dffc5a6795bce97fae02b51ccf3330e15e4hkuang *  Use of this source code is governed by a BSD-style license
5ba164dffc5a6795bce97fae02b51ccf3330e15e4hkuang *  that can be found in the LICENSE file in the root of the source
6ba164dffc5a6795bce97fae02b51ccf3330e15e4hkuang *  tree. An additional intellectual property rights grant can be found
7ba164dffc5a6795bce97fae02b51ccf3330e15e4hkuang *  in the file PATENTS.  All contributing project authors may
8ba164dffc5a6795bce97fae02b51ccf3330e15e4hkuang *  be found in the AUTHORS file in the root of the source tree.
9ba164dffc5a6795bce97fae02b51ccf3330e15e4hkuang */
10ba164dffc5a6795bce97fae02b51ccf3330e15e4hkuang
115ae7ac49f08a179e4f054d99fcfc9dce78d26e58hkuang#include "./vpx_config.h"
12da49e34c1fb5e99681f4ad99c21d9cfd83eddb96Vignesh Venkatasubramanian#include "./vpx_dsp_rtcd.h"
13ba164dffc5a6795bce97fae02b51ccf3330e15e4hkuang#include "vp9/common/vp9_loopfilter.h"
14ba164dffc5a6795bce97fae02b51ccf3330e15e4hkuang#include "vp9/common/vp9_onyxc_int.h"
15ba164dffc5a6795bce97fae02b51ccf3330e15e4hkuang#include "vp9/common/vp9_reconinter.h"
16ba164dffc5a6795bce97fae02b51ccf3330e15e4hkuang#include "vpx_mem/vpx_mem.h"
17da49e34c1fb5e99681f4ad99c21d9cfd83eddb96Vignesh Venkatasubramanian#include "vpx_ports/mem.h"
18ba164dffc5a6795bce97fae02b51ccf3330e15e4hkuang
19ba164dffc5a6795bce97fae02b51ccf3330e15e4hkuang#include "vp9/common/vp9_seg_common.h"
20ba164dffc5a6795bce97fae02b51ccf3330e15e4hkuang
21ba6c59e9d7d7013b3906b6f4230b663422681848Vignesh Venkatasubramanian// 64 bit masks for left transform size. Each 1 represents a position where
221184aebb761cbeac9124c37189a80a1a58f04b6bhkuang// we should apply a loop filter across the left border of an 8x8 block
231184aebb761cbeac9124c37189a80a1a58f04b6bhkuang// boundary.
241184aebb761cbeac9124c37189a80a1a58f04b6bhkuang//
251184aebb761cbeac9124c37189a80a1a58f04b6bhkuang// In the case of TX_16X16->  ( in low order byte first we end up with
261184aebb761cbeac9124c37189a80a1a58f04b6bhkuang// a mask that looks like this
271184aebb761cbeac9124c37189a80a1a58f04b6bhkuang//
281184aebb761cbeac9124c37189a80a1a58f04b6bhkuang//    10101010
291184aebb761cbeac9124c37189a80a1a58f04b6bhkuang//    10101010
301184aebb761cbeac9124c37189a80a1a58f04b6bhkuang//    10101010
311184aebb761cbeac9124c37189a80a1a58f04b6bhkuang//    10101010
321184aebb761cbeac9124c37189a80a1a58f04b6bhkuang//    10101010
331184aebb761cbeac9124c37189a80a1a58f04b6bhkuang//    10101010
341184aebb761cbeac9124c37189a80a1a58f04b6bhkuang//    10101010
351184aebb761cbeac9124c37189a80a1a58f04b6bhkuang//    10101010
361184aebb761cbeac9124c37189a80a1a58f04b6bhkuang//
371184aebb761cbeac9124c37189a80a1a58f04b6bhkuang// A loopfilter should be applied to every other 8x8 horizontally.
381184aebb761cbeac9124c37189a80a1a58f04b6bhkuangstatic const uint64_t left_64x64_txform_mask[TX_SIZES]= {
39da49e34c1fb5e99681f4ad99c21d9cfd83eddb96Vignesh Venkatasubramanian  0xffffffffffffffffULL,  // TX_4X4
40da49e34c1fb5e99681f4ad99c21d9cfd83eddb96Vignesh Venkatasubramanian  0xffffffffffffffffULL,  // TX_8x8
41da49e34c1fb5e99681f4ad99c21d9cfd83eddb96Vignesh Venkatasubramanian  0x5555555555555555ULL,  // TX_16x16
42da49e34c1fb5e99681f4ad99c21d9cfd83eddb96Vignesh Venkatasubramanian  0x1111111111111111ULL,  // TX_32x32
431184aebb761cbeac9124c37189a80a1a58f04b6bhkuang};
441184aebb761cbeac9124c37189a80a1a58f04b6bhkuang
45ba6c59e9d7d7013b3906b6f4230b663422681848Vignesh Venkatasubramanian// 64 bit masks for above transform size. Each 1 represents a position where
461184aebb761cbeac9124c37189a80a1a58f04b6bhkuang// we should apply a loop filter across the top border of an 8x8 block
471184aebb761cbeac9124c37189a80a1a58f04b6bhkuang// boundary.
481184aebb761cbeac9124c37189a80a1a58f04b6bhkuang//
491184aebb761cbeac9124c37189a80a1a58f04b6bhkuang// In the case of TX_32x32 ->  ( in low order byte first we end up with
501184aebb761cbeac9124c37189a80a1a58f04b6bhkuang// a mask that looks like this
511184aebb761cbeac9124c37189a80a1a58f04b6bhkuang//
521184aebb761cbeac9124c37189a80a1a58f04b6bhkuang//    11111111
531184aebb761cbeac9124c37189a80a1a58f04b6bhkuang//    00000000
541184aebb761cbeac9124c37189a80a1a58f04b6bhkuang//    00000000
551184aebb761cbeac9124c37189a80a1a58f04b6bhkuang//    00000000
561184aebb761cbeac9124c37189a80a1a58f04b6bhkuang//    11111111
571184aebb761cbeac9124c37189a80a1a58f04b6bhkuang//    00000000
581184aebb761cbeac9124c37189a80a1a58f04b6bhkuang//    00000000
591184aebb761cbeac9124c37189a80a1a58f04b6bhkuang//    00000000
601184aebb761cbeac9124c37189a80a1a58f04b6bhkuang//
611184aebb761cbeac9124c37189a80a1a58f04b6bhkuang// A loopfilter should be applied to every other 4 the row vertically.
621184aebb761cbeac9124c37189a80a1a58f04b6bhkuangstatic const uint64_t above_64x64_txform_mask[TX_SIZES]= {
63da49e34c1fb5e99681f4ad99c21d9cfd83eddb96Vignesh Venkatasubramanian  0xffffffffffffffffULL,  // TX_4X4
64da49e34c1fb5e99681f4ad99c21d9cfd83eddb96Vignesh Venkatasubramanian  0xffffffffffffffffULL,  // TX_8x8
65da49e34c1fb5e99681f4ad99c21d9cfd83eddb96Vignesh Venkatasubramanian  0x00ff00ff00ff00ffULL,  // TX_16x16
66da49e34c1fb5e99681f4ad99c21d9cfd83eddb96Vignesh Venkatasubramanian  0x000000ff000000ffULL,  // TX_32x32
671184aebb761cbeac9124c37189a80a1a58f04b6bhkuang};
681184aebb761cbeac9124c37189a80a1a58f04b6bhkuang
69ba6c59e9d7d7013b3906b6f4230b663422681848Vignesh Venkatasubramanian// 64 bit masks for prediction sizes (left). Each 1 represents a position
70ba6c59e9d7d7013b3906b6f4230b663422681848Vignesh Venkatasubramanian// where left border of an 8x8 block. These are aligned to the right most
71ba6c59e9d7d7013b3906b6f4230b663422681848Vignesh Venkatasubramanian// appropriate bit, and then shifted into place.
721184aebb761cbeac9124c37189a80a1a58f04b6bhkuang//
731184aebb761cbeac9124c37189a80a1a58f04b6bhkuang// In the case of TX_16x32 ->  ( low order byte first ) we end up with
741184aebb761cbeac9124c37189a80a1a58f04b6bhkuang// a mask that looks like this :
751184aebb761cbeac9124c37189a80a1a58f04b6bhkuang//
761184aebb761cbeac9124c37189a80a1a58f04b6bhkuang//  10000000
771184aebb761cbeac9124c37189a80a1a58f04b6bhkuang//  10000000
781184aebb761cbeac9124c37189a80a1a58f04b6bhkuang//  10000000
791184aebb761cbeac9124c37189a80a1a58f04b6bhkuang//  10000000
801184aebb761cbeac9124c37189a80a1a58f04b6bhkuang//  00000000
811184aebb761cbeac9124c37189a80a1a58f04b6bhkuang//  00000000
821184aebb761cbeac9124c37189a80a1a58f04b6bhkuang//  00000000
831184aebb761cbeac9124c37189a80a1a58f04b6bhkuang//  00000000
841184aebb761cbeac9124c37189a80a1a58f04b6bhkuangstatic const uint64_t left_prediction_mask[BLOCK_SIZES] = {
85da49e34c1fb5e99681f4ad99c21d9cfd83eddb96Vignesh Venkatasubramanian  0x0000000000000001ULL,  // BLOCK_4X4,
86da49e34c1fb5e99681f4ad99c21d9cfd83eddb96Vignesh Venkatasubramanian  0x0000000000000001ULL,  // BLOCK_4X8,
87da49e34c1fb5e99681f4ad99c21d9cfd83eddb96Vignesh Venkatasubramanian  0x0000000000000001ULL,  // BLOCK_8X4,
88da49e34c1fb5e99681f4ad99c21d9cfd83eddb96Vignesh Venkatasubramanian  0x0000000000000001ULL,  // BLOCK_8X8,
89da49e34c1fb5e99681f4ad99c21d9cfd83eddb96Vignesh Venkatasubramanian  0x0000000000000101ULL,  // BLOCK_8X16,
90da49e34c1fb5e99681f4ad99c21d9cfd83eddb96Vignesh Venkatasubramanian  0x0000000000000001ULL,  // BLOCK_16X8,
91da49e34c1fb5e99681f4ad99c21d9cfd83eddb96Vignesh Venkatasubramanian  0x0000000000000101ULL,  // BLOCK_16X16,
92da49e34c1fb5e99681f4ad99c21d9cfd83eddb96Vignesh Venkatasubramanian  0x0000000001010101ULL,  // BLOCK_16X32,
93da49e34c1fb5e99681f4ad99c21d9cfd83eddb96Vignesh Venkatasubramanian  0x0000000000000101ULL,  // BLOCK_32X16,
94da49e34c1fb5e99681f4ad99c21d9cfd83eddb96Vignesh Venkatasubramanian  0x0000000001010101ULL,  // BLOCK_32X32,
95da49e34c1fb5e99681f4ad99c21d9cfd83eddb96Vignesh Venkatasubramanian  0x0101010101010101ULL,  // BLOCK_32X64,
96da49e34c1fb5e99681f4ad99c21d9cfd83eddb96Vignesh Venkatasubramanian  0x0000000001010101ULL,  // BLOCK_64X32,
97da49e34c1fb5e99681f4ad99c21d9cfd83eddb96Vignesh Venkatasubramanian  0x0101010101010101ULL,  // BLOCK_64X64
981184aebb761cbeac9124c37189a80a1a58f04b6bhkuang};
991184aebb761cbeac9124c37189a80a1a58f04b6bhkuang
1001184aebb761cbeac9124c37189a80a1a58f04b6bhkuang// 64 bit mask to shift and set for each prediction size.
1011184aebb761cbeac9124c37189a80a1a58f04b6bhkuangstatic const uint64_t above_prediction_mask[BLOCK_SIZES] = {
102da49e34c1fb5e99681f4ad99c21d9cfd83eddb96Vignesh Venkatasubramanian  0x0000000000000001ULL,  // BLOCK_4X4
103da49e34c1fb5e99681f4ad99c21d9cfd83eddb96Vignesh Venkatasubramanian  0x0000000000000001ULL,  // BLOCK_4X8
104da49e34c1fb5e99681f4ad99c21d9cfd83eddb96Vignesh Venkatasubramanian  0x0000000000000001ULL,  // BLOCK_8X4
105da49e34c1fb5e99681f4ad99c21d9cfd83eddb96Vignesh Venkatasubramanian  0x0000000000000001ULL,  // BLOCK_8X8
106da49e34c1fb5e99681f4ad99c21d9cfd83eddb96Vignesh Venkatasubramanian  0x0000000000000001ULL,  // BLOCK_8X16,
107da49e34c1fb5e99681f4ad99c21d9cfd83eddb96Vignesh Venkatasubramanian  0x0000000000000003ULL,  // BLOCK_16X8
108da49e34c1fb5e99681f4ad99c21d9cfd83eddb96Vignesh Venkatasubramanian  0x0000000000000003ULL,  // BLOCK_16X16
109da49e34c1fb5e99681f4ad99c21d9cfd83eddb96Vignesh Venkatasubramanian  0x0000000000000003ULL,  // BLOCK_16X32,
110da49e34c1fb5e99681f4ad99c21d9cfd83eddb96Vignesh Venkatasubramanian  0x000000000000000fULL,  // BLOCK_32X16,
111da49e34c1fb5e99681f4ad99c21d9cfd83eddb96Vignesh Venkatasubramanian  0x000000000000000fULL,  // BLOCK_32X32,
112da49e34c1fb5e99681f4ad99c21d9cfd83eddb96Vignesh Venkatasubramanian  0x000000000000000fULL,  // BLOCK_32X64,
113da49e34c1fb5e99681f4ad99c21d9cfd83eddb96Vignesh Venkatasubramanian  0x00000000000000ffULL,  // BLOCK_64X32,
114da49e34c1fb5e99681f4ad99c21d9cfd83eddb96Vignesh Venkatasubramanian  0x00000000000000ffULL,  // BLOCK_64X64
1151184aebb761cbeac9124c37189a80a1a58f04b6bhkuang};
116ba6c59e9d7d7013b3906b6f4230b663422681848Vignesh Venkatasubramanian// 64 bit mask to shift and set for each prediction size. A bit is set for
1171184aebb761cbeac9124c37189a80a1a58f04b6bhkuang// each 8x8 block that would be in the left most block of the given block
1181184aebb761cbeac9124c37189a80a1a58f04b6bhkuang// size in the 64x64 block.
1191184aebb761cbeac9124c37189a80a1a58f04b6bhkuangstatic const uint64_t size_mask[BLOCK_SIZES] = {
120da49e34c1fb5e99681f4ad99c21d9cfd83eddb96Vignesh Venkatasubramanian  0x0000000000000001ULL,  // BLOCK_4X4
121da49e34c1fb5e99681f4ad99c21d9cfd83eddb96Vignesh Venkatasubramanian  0x0000000000000001ULL,  // BLOCK_4X8
122da49e34c1fb5e99681f4ad99c21d9cfd83eddb96Vignesh Venkatasubramanian  0x0000000000000001ULL,  // BLOCK_8X4
123da49e34c1fb5e99681f4ad99c21d9cfd83eddb96Vignesh Venkatasubramanian  0x0000000000000001ULL,  // BLOCK_8X8
124da49e34c1fb5e99681f4ad99c21d9cfd83eddb96Vignesh Venkatasubramanian  0x0000000000000101ULL,  // BLOCK_8X16,
125da49e34c1fb5e99681f4ad99c21d9cfd83eddb96Vignesh Venkatasubramanian  0x0000000000000003ULL,  // BLOCK_16X8
126da49e34c1fb5e99681f4ad99c21d9cfd83eddb96Vignesh Venkatasubramanian  0x0000000000000303ULL,  // BLOCK_16X16
127da49e34c1fb5e99681f4ad99c21d9cfd83eddb96Vignesh Venkatasubramanian  0x0000000003030303ULL,  // BLOCK_16X32,
128da49e34c1fb5e99681f4ad99c21d9cfd83eddb96Vignesh Venkatasubramanian  0x0000000000000f0fULL,  // BLOCK_32X16,
129da49e34c1fb5e99681f4ad99c21d9cfd83eddb96Vignesh Venkatasubramanian  0x000000000f0f0f0fULL,  // BLOCK_32X32,
130da49e34c1fb5e99681f4ad99c21d9cfd83eddb96Vignesh Venkatasubramanian  0x0f0f0f0f0f0f0f0fULL,  // BLOCK_32X64,
131da49e34c1fb5e99681f4ad99c21d9cfd83eddb96Vignesh Venkatasubramanian  0x00000000ffffffffULL,  // BLOCK_64X32,
132da49e34c1fb5e99681f4ad99c21d9cfd83eddb96Vignesh Venkatasubramanian  0xffffffffffffffffULL,  // BLOCK_64X64
1331184aebb761cbeac9124c37189a80a1a58f04b6bhkuang};
1341184aebb761cbeac9124c37189a80a1a58f04b6bhkuang
1351184aebb761cbeac9124c37189a80a1a58f04b6bhkuang// These are used for masking the left and above borders.
136da49e34c1fb5e99681f4ad99c21d9cfd83eddb96Vignesh Venkatasubramanianstatic const uint64_t left_border =  0x1111111111111111ULL;
137da49e34c1fb5e99681f4ad99c21d9cfd83eddb96Vignesh Venkatasubramanianstatic const uint64_t above_border = 0x000000ff000000ffULL;
1381184aebb761cbeac9124c37189a80a1a58f04b6bhkuang
1391184aebb761cbeac9124c37189a80a1a58f04b6bhkuang// 16 bit masks for uv transform sizes.
1401184aebb761cbeac9124c37189a80a1a58f04b6bhkuangstatic const uint16_t left_64x64_txform_mask_uv[TX_SIZES]= {
141ba6c59e9d7d7013b3906b6f4230b663422681848Vignesh Venkatasubramanian  0xffff,  // TX_4X4
142ba6c59e9d7d7013b3906b6f4230b663422681848Vignesh Venkatasubramanian  0xffff,  // TX_8x8
143ba6c59e9d7d7013b3906b6f4230b663422681848Vignesh Venkatasubramanian  0x5555,  // TX_16x16
144ba6c59e9d7d7013b3906b6f4230b663422681848Vignesh Venkatasubramanian  0x1111,  // TX_32x32
1451184aebb761cbeac9124c37189a80a1a58f04b6bhkuang};
1461184aebb761cbeac9124c37189a80a1a58f04b6bhkuang
1471184aebb761cbeac9124c37189a80a1a58f04b6bhkuangstatic const uint16_t above_64x64_txform_mask_uv[TX_SIZES]= {
148ba6c59e9d7d7013b3906b6f4230b663422681848Vignesh Venkatasubramanian  0xffff,  // TX_4X4
149ba6c59e9d7d7013b3906b6f4230b663422681848Vignesh Venkatasubramanian  0xffff,  // TX_8x8
150ba6c59e9d7d7013b3906b6f4230b663422681848Vignesh Venkatasubramanian  0x0f0f,  // TX_16x16
151ba6c59e9d7d7013b3906b6f4230b663422681848Vignesh Venkatasubramanian  0x000f,  // TX_32x32
1521184aebb761cbeac9124c37189a80a1a58f04b6bhkuang};
1531184aebb761cbeac9124c37189a80a1a58f04b6bhkuang
1541184aebb761cbeac9124c37189a80a1a58f04b6bhkuang// 16 bit left mask to shift and set for each uv prediction size.
1551184aebb761cbeac9124c37189a80a1a58f04b6bhkuangstatic const uint16_t left_prediction_mask_uv[BLOCK_SIZES] = {
156ba6c59e9d7d7013b3906b6f4230b663422681848Vignesh Venkatasubramanian  0x0001,  // BLOCK_4X4,
157ba6c59e9d7d7013b3906b6f4230b663422681848Vignesh Venkatasubramanian  0x0001,  // BLOCK_4X8,
158ba6c59e9d7d7013b3906b6f4230b663422681848Vignesh Venkatasubramanian  0x0001,  // BLOCK_8X4,
159ba6c59e9d7d7013b3906b6f4230b663422681848Vignesh Venkatasubramanian  0x0001,  // BLOCK_8X8,
160ba6c59e9d7d7013b3906b6f4230b663422681848Vignesh Venkatasubramanian  0x0001,  // BLOCK_8X16,
161ba6c59e9d7d7013b3906b6f4230b663422681848Vignesh Venkatasubramanian  0x0001,  // BLOCK_16X8,
162ba6c59e9d7d7013b3906b6f4230b663422681848Vignesh Venkatasubramanian  0x0001,  // BLOCK_16X16,
163ba6c59e9d7d7013b3906b6f4230b663422681848Vignesh Venkatasubramanian  0x0011,  // BLOCK_16X32,
164ba6c59e9d7d7013b3906b6f4230b663422681848Vignesh Venkatasubramanian  0x0001,  // BLOCK_32X16,
165ba6c59e9d7d7013b3906b6f4230b663422681848Vignesh Venkatasubramanian  0x0011,  // BLOCK_32X32,
166ba6c59e9d7d7013b3906b6f4230b663422681848Vignesh Venkatasubramanian  0x1111,  // BLOCK_32X64
167ba6c59e9d7d7013b3906b6f4230b663422681848Vignesh Venkatasubramanian  0x0011,  // BLOCK_64X32,
168ba6c59e9d7d7013b3906b6f4230b663422681848Vignesh Venkatasubramanian  0x1111,  // BLOCK_64X64
1691184aebb761cbeac9124c37189a80a1a58f04b6bhkuang};
1701184aebb761cbeac9124c37189a80a1a58f04b6bhkuang// 16 bit above mask to shift and set for uv each prediction size.
1711184aebb761cbeac9124c37189a80a1a58f04b6bhkuangstatic const uint16_t above_prediction_mask_uv[BLOCK_SIZES] = {
172ba6c59e9d7d7013b3906b6f4230b663422681848Vignesh Venkatasubramanian  0x0001,  // BLOCK_4X4
173ba6c59e9d7d7013b3906b6f4230b663422681848Vignesh Venkatasubramanian  0x0001,  // BLOCK_4X8
174ba6c59e9d7d7013b3906b6f4230b663422681848Vignesh Venkatasubramanian  0x0001,  // BLOCK_8X4
175ba6c59e9d7d7013b3906b6f4230b663422681848Vignesh Venkatasubramanian  0x0001,  // BLOCK_8X8
176ba6c59e9d7d7013b3906b6f4230b663422681848Vignesh Venkatasubramanian  0x0001,  // BLOCK_8X16,
177ba6c59e9d7d7013b3906b6f4230b663422681848Vignesh Venkatasubramanian  0x0001,  // BLOCK_16X8
178ba6c59e9d7d7013b3906b6f4230b663422681848Vignesh Venkatasubramanian  0x0001,  // BLOCK_16X16
179ba6c59e9d7d7013b3906b6f4230b663422681848Vignesh Venkatasubramanian  0x0001,  // BLOCK_16X32,
180ba6c59e9d7d7013b3906b6f4230b663422681848Vignesh Venkatasubramanian  0x0003,  // BLOCK_32X16,
181ba6c59e9d7d7013b3906b6f4230b663422681848Vignesh Venkatasubramanian  0x0003,  // BLOCK_32X32,
182ba6c59e9d7d7013b3906b6f4230b663422681848Vignesh Venkatasubramanian  0x0003,  // BLOCK_32X64,
183ba6c59e9d7d7013b3906b6f4230b663422681848Vignesh Venkatasubramanian  0x000f,  // BLOCK_64X32,
184ba6c59e9d7d7013b3906b6f4230b663422681848Vignesh Venkatasubramanian  0x000f,  // BLOCK_64X64
1851184aebb761cbeac9124c37189a80a1a58f04b6bhkuang};
1861184aebb761cbeac9124c37189a80a1a58f04b6bhkuang
1871184aebb761cbeac9124c37189a80a1a58f04b6bhkuang// 64 bit mask to shift and set for each uv prediction size
1881184aebb761cbeac9124c37189a80a1a58f04b6bhkuangstatic const uint16_t size_mask_uv[BLOCK_SIZES] = {
189ba6c59e9d7d7013b3906b6f4230b663422681848Vignesh Venkatasubramanian  0x0001,  // BLOCK_4X4
190ba6c59e9d7d7013b3906b6f4230b663422681848Vignesh Venkatasubramanian  0x0001,  // BLOCK_4X8
191ba6c59e9d7d7013b3906b6f4230b663422681848Vignesh Venkatasubramanian  0x0001,  // BLOCK_8X4
192ba6c59e9d7d7013b3906b6f4230b663422681848Vignesh Venkatasubramanian  0x0001,  // BLOCK_8X8
193ba6c59e9d7d7013b3906b6f4230b663422681848Vignesh Venkatasubramanian  0x0001,  // BLOCK_8X16,
194ba6c59e9d7d7013b3906b6f4230b663422681848Vignesh Venkatasubramanian  0x0001,  // BLOCK_16X8
195ba6c59e9d7d7013b3906b6f4230b663422681848Vignesh Venkatasubramanian  0x0001,  // BLOCK_16X16
196ba6c59e9d7d7013b3906b6f4230b663422681848Vignesh Venkatasubramanian  0x0011,  // BLOCK_16X32,
197ba6c59e9d7d7013b3906b6f4230b663422681848Vignesh Venkatasubramanian  0x0003,  // BLOCK_32X16,
198ba6c59e9d7d7013b3906b6f4230b663422681848Vignesh Venkatasubramanian  0x0033,  // BLOCK_32X32,
199ba6c59e9d7d7013b3906b6f4230b663422681848Vignesh Venkatasubramanian  0x3333,  // BLOCK_32X64,
200ba6c59e9d7d7013b3906b6f4230b663422681848Vignesh Venkatasubramanian  0x00ff,  // BLOCK_64X32,
201ba6c59e9d7d7013b3906b6f4230b663422681848Vignesh Venkatasubramanian  0xffff,  // BLOCK_64X64
2021184aebb761cbeac9124c37189a80a1a58f04b6bhkuang};
2031184aebb761cbeac9124c37189a80a1a58f04b6bhkuangstatic const uint16_t left_border_uv =  0x1111;
2041184aebb761cbeac9124c37189a80a1a58f04b6bhkuangstatic const uint16_t above_border_uv = 0x000f;
2051184aebb761cbeac9124c37189a80a1a58f04b6bhkuang
2062ec72e65689c948e92b826ae1e867bf369e72f13Vignesh Venkatasubramanianstatic const int mode_lf_lut[MB_MODE_COUNT] = {
2072ec72e65689c948e92b826ae1e867bf369e72f13Vignesh Venkatasubramanian  0, 0, 0, 0, 0, 0, 0, 0, 0, 0,  // INTRA_MODES
2082ec72e65689c948e92b826ae1e867bf369e72f13Vignesh Venkatasubramanian  1, 1, 0, 1                     // INTER_MODES (ZEROMV == 0)
2092ec72e65689c948e92b826ae1e867bf369e72f13Vignesh Venkatasubramanian};
210ba164dffc5a6795bce97fae02b51ccf3330e15e4hkuang
2111184aebb761cbeac9124c37189a80a1a58f04b6bhkuangstatic void update_sharpness(loop_filter_info_n *lfi, int sharpness_lvl) {
21291037db265ecdd914a26e056cf69207b4f50924ehkuang  int lvl;
213ba164dffc5a6795bce97fae02b51ccf3330e15e4hkuang
21491037db265ecdd914a26e056cf69207b4f50924ehkuang  // For each possible value for the loop filter fill out limits
21591037db265ecdd914a26e056cf69207b4f50924ehkuang  for (lvl = 0; lvl <= MAX_LOOP_FILTER; lvl++) {
216ba6c59e9d7d7013b3906b6f4230b663422681848Vignesh Venkatasubramanian    // Set loop filter parameters that control sharpness.
21791037db265ecdd914a26e056cf69207b4f50924ehkuang    int block_inside_limit = lvl >> ((sharpness_lvl > 0) + (sharpness_lvl > 4));
218ba164dffc5a6795bce97fae02b51ccf3330e15e4hkuang
219ba164dffc5a6795bce97fae02b51ccf3330e15e4hkuang    if (sharpness_lvl > 0) {
220ba164dffc5a6795bce97fae02b51ccf3330e15e4hkuang      if (block_inside_limit > (9 - sharpness_lvl))
221ba164dffc5a6795bce97fae02b51ccf3330e15e4hkuang        block_inside_limit = (9 - sharpness_lvl);
222ba164dffc5a6795bce97fae02b51ccf3330e15e4hkuang    }
223ba164dffc5a6795bce97fae02b51ccf3330e15e4hkuang
224ba164dffc5a6795bce97fae02b51ccf3330e15e4hkuang    if (block_inside_limit < 1)
225ba164dffc5a6795bce97fae02b51ccf3330e15e4hkuang      block_inside_limit = 1;
226ba164dffc5a6795bce97fae02b51ccf3330e15e4hkuang
227da49e34c1fb5e99681f4ad99c21d9cfd83eddb96Vignesh Venkatasubramanian    memset(lfi->lfthr[lvl].lim, block_inside_limit, SIMD_WIDTH);
228da49e34c1fb5e99681f4ad99c21d9cfd83eddb96Vignesh Venkatasubramanian    memset(lfi->lfthr[lvl].mblim, (2 * (lvl + 2) + block_inside_limit),
229da49e34c1fb5e99681f4ad99c21d9cfd83eddb96Vignesh Venkatasubramanian           SIMD_WIDTH);
230ba164dffc5a6795bce97fae02b51ccf3330e15e4hkuang  }
231ba164dffc5a6795bce97fae02b51ccf3330e15e4hkuang}
232ba164dffc5a6795bce97fae02b51ccf3330e15e4hkuang
2334fb68e5dd4e93c7599dc905d861de11ac39c5585hkuangstatic uint8_t get_filter_level(const loop_filter_info_n *lfi_n,
2344fb68e5dd4e93c7599dc905d861de11ac39c5585hkuang                                const MB_MODE_INFO *mbmi) {
2354fb68e5dd4e93c7599dc905d861de11ac39c5585hkuang  return lfi_n->lvl[mbmi->segment_id][mbmi->ref_frame[0]]
2364fb68e5dd4e93c7599dc905d861de11ac39c5585hkuang                   [mode_lf_lut[mbmi->mode]];
2374fb68e5dd4e93c7599dc905d861de11ac39c5585hkuang}
2384fb68e5dd4e93c7599dc905d861de11ac39c5585hkuang
2391184aebb761cbeac9124c37189a80a1a58f04b6bhkuangvoid vp9_loop_filter_init(VP9_COMMON *cm) {
240ba164dffc5a6795bce97fae02b51ccf3330e15e4hkuang  loop_filter_info_n *lfi = &cm->lf_info;
2411184aebb761cbeac9124c37189a80a1a58f04b6bhkuang  struct loopfilter *lf = &cm->lf;
2425ae7ac49f08a179e4f054d99fcfc9dce78d26e58hkuang  int lvl;
243ba164dffc5a6795bce97fae02b51ccf3330e15e4hkuang
244ba164dffc5a6795bce97fae02b51ccf3330e15e4hkuang  // init limits for given sharpness
24591037db265ecdd914a26e056cf69207b4f50924ehkuang  update_sharpness(lfi, lf->sharpness_level);
24691037db265ecdd914a26e056cf69207b4f50924ehkuang  lf->last_sharpness_level = lf->sharpness_level;
247ba164dffc5a6795bce97fae02b51ccf3330e15e4hkuang
248ba164dffc5a6795bce97fae02b51ccf3330e15e4hkuang  // init hev threshold const vectors
2495ae7ac49f08a179e4f054d99fcfc9dce78d26e58hkuang  for (lvl = 0; lvl <= MAX_LOOP_FILTER; lvl++)
250da49e34c1fb5e99681f4ad99c21d9cfd83eddb96Vignesh Venkatasubramanian    memset(lfi->lfthr[lvl].hev_thr, (lvl >> 4), SIMD_WIDTH);
251ba164dffc5a6795bce97fae02b51ccf3330e15e4hkuang}
252ba164dffc5a6795bce97fae02b51ccf3330e15e4hkuang
2531184aebb761cbeac9124c37189a80a1a58f04b6bhkuangvoid vp9_loop_filter_frame_init(VP9_COMMON *cm, int default_filt_lvl) {
254f3bed9137f66ef693bd406e43b17e9a1114f1e14hkuang  int seg_id;
255ba6c59e9d7d7013b3906b6f4230b663422681848Vignesh Venkatasubramanian  // n_shift is the multiplier for lf_deltas
256ba164dffc5a6795bce97fae02b51ccf3330e15e4hkuang  // the multiplier is 1 for when filter_lvl is between 0 and 31;
257ba164dffc5a6795bce97fae02b51ccf3330e15e4hkuang  // 2 when filter_lvl is between 32 and 63
2582ec72e65689c948e92b826ae1e867bf369e72f13Vignesh Venkatasubramanian  const int scale = 1 << (default_filt_lvl >> 5);
25991037db265ecdd914a26e056cf69207b4f50924ehkuang  loop_filter_info_n *const lfi = &cm->lf_info;
2601184aebb761cbeac9124c37189a80a1a58f04b6bhkuang  struct loopfilter *const lf = &cm->lf;
2612ec72e65689c948e92b826ae1e867bf369e72f13Vignesh Venkatasubramanian  const struct segmentation *const seg = &cm->seg;
26291037db265ecdd914a26e056cf69207b4f50924ehkuang
26391037db265ecdd914a26e056cf69207b4f50924ehkuang  // update limits if sharpness has changed
26491037db265ecdd914a26e056cf69207b4f50924ehkuang  if (lf->last_sharpness_level != lf->sharpness_level) {
26591037db265ecdd914a26e056cf69207b4f50924ehkuang    update_sharpness(lfi, lf->sharpness_level);
26691037db265ecdd914a26e056cf69207b4f50924ehkuang    lf->last_sharpness_level = lf->sharpness_level;
267ba164dffc5a6795bce97fae02b51ccf3330e15e4hkuang  }
268ba164dffc5a6795bce97fae02b51ccf3330e15e4hkuang
269f3bed9137f66ef693bd406e43b17e9a1114f1e14hkuang  for (seg_id = 0; seg_id < MAX_SEGMENTS; seg_id++) {
2702ec72e65689c948e92b826ae1e867bf369e72f13Vignesh Venkatasubramanian    int lvl_seg = default_filt_lvl;
271da49e34c1fb5e99681f4ad99c21d9cfd83eddb96Vignesh Venkatasubramanian    if (segfeature_active(seg, seg_id, SEG_LVL_ALT_LF)) {
272da49e34c1fb5e99681f4ad99c21d9cfd83eddb96Vignesh Venkatasubramanian      const int data = get_segdata(seg, seg_id, SEG_LVL_ALT_LF);
2732ec72e65689c948e92b826ae1e867bf369e72f13Vignesh Venkatasubramanian      lvl_seg = clamp(seg->abs_delta == SEGMENT_ABSDATA ?
2742ec72e65689c948e92b826ae1e867bf369e72f13Vignesh Venkatasubramanian                      data : default_filt_lvl + data,
2752ec72e65689c948e92b826ae1e867bf369e72f13Vignesh Venkatasubramanian                      0, MAX_LOOP_FILTER);
276ba164dffc5a6795bce97fae02b51ccf3330e15e4hkuang    }
277ba164dffc5a6795bce97fae02b51ccf3330e15e4hkuang
27891037db265ecdd914a26e056cf69207b4f50924ehkuang    if (!lf->mode_ref_delta_enabled) {
27991037db265ecdd914a26e056cf69207b4f50924ehkuang      // we could get rid of this if we assume that deltas are set to
28091037db265ecdd914a26e056cf69207b4f50924ehkuang      // zero when not in use; encoder always uses deltas
281da49e34c1fb5e99681f4ad99c21d9cfd83eddb96Vignesh Venkatasubramanian      memset(lfi->lvl[seg_id], lvl_seg, sizeof(lfi->lvl[seg_id]));
2822ec72e65689c948e92b826ae1e867bf369e72f13Vignesh Venkatasubramanian    } else {
2832ec72e65689c948e92b826ae1e867bf369e72f13Vignesh Venkatasubramanian      int ref, mode;
2842ec72e65689c948e92b826ae1e867bf369e72f13Vignesh Venkatasubramanian      const int intra_lvl = lvl_seg + lf->ref_deltas[INTRA_FRAME] * scale;
2852ec72e65689c948e92b826ae1e867bf369e72f13Vignesh Venkatasubramanian      lfi->lvl[seg_id][INTRA_FRAME][0] = clamp(intra_lvl, 0, MAX_LOOP_FILTER);
2862ec72e65689c948e92b826ae1e867bf369e72f13Vignesh Venkatasubramanian
2872ec72e65689c948e92b826ae1e867bf369e72f13Vignesh Venkatasubramanian      for (ref = LAST_FRAME; ref < MAX_REF_FRAMES; ++ref) {
2882ec72e65689c948e92b826ae1e867bf369e72f13Vignesh Venkatasubramanian        for (mode = 0; mode < MAX_MODE_LF_DELTAS; ++mode) {
2892ec72e65689c948e92b826ae1e867bf369e72f13Vignesh Venkatasubramanian          const int inter_lvl = lvl_seg + lf->ref_deltas[ref] * scale
2902ec72e65689c948e92b826ae1e867bf369e72f13Vignesh Venkatasubramanian                                        + lf->mode_deltas[mode] * scale;
2912ec72e65689c948e92b826ae1e867bf369e72f13Vignesh Venkatasubramanian          lfi->lvl[seg_id][ref][mode] = clamp(inter_lvl, 0, MAX_LOOP_FILTER);
2922ec72e65689c948e92b826ae1e867bf369e72f13Vignesh Venkatasubramanian        }
293ba164dffc5a6795bce97fae02b51ccf3330e15e4hkuang      }
2942ec72e65689c948e92b826ae1e867bf369e72f13Vignesh Venkatasubramanian    }
295ba164dffc5a6795bce97fae02b51ccf3330e15e4hkuang  }
296ba164dffc5a6795bce97fae02b51ccf3330e15e4hkuang}
297ba164dffc5a6795bce97fae02b51ccf3330e15e4hkuang
298da49e34c1fb5e99681f4ad99c21d9cfd83eddb96Vignesh Venkatasubramanianstatic void filter_selectively_vert_row2(int subsampling_factor,
2992ec72e65689c948e92b826ae1e867bf369e72f13Vignesh Venkatasubramanian                                         uint8_t *s, int pitch,
3002ec72e65689c948e92b826ae1e867bf369e72f13Vignesh Venkatasubramanian                                         unsigned int mask_16x16_l,
3012ec72e65689c948e92b826ae1e867bf369e72f13Vignesh Venkatasubramanian                                         unsigned int mask_8x8_l,
3022ec72e65689c948e92b826ae1e867bf369e72f13Vignesh Venkatasubramanian                                         unsigned int mask_4x4_l,
3032ec72e65689c948e92b826ae1e867bf369e72f13Vignesh Venkatasubramanian                                         unsigned int mask_4x4_int_l,
3042ec72e65689c948e92b826ae1e867bf369e72f13Vignesh Venkatasubramanian                                         const loop_filter_info_n *lfi_n,
3052ec72e65689c948e92b826ae1e867bf369e72f13Vignesh Venkatasubramanian                                         const uint8_t *lfl) {
306da49e34c1fb5e99681f4ad99c21d9cfd83eddb96Vignesh Venkatasubramanian  const int mask_shift = subsampling_factor ? 4 : 8;
307da49e34c1fb5e99681f4ad99c21d9cfd83eddb96Vignesh Venkatasubramanian  const int mask_cutoff = subsampling_factor ? 0xf : 0xff;
308da49e34c1fb5e99681f4ad99c21d9cfd83eddb96Vignesh Venkatasubramanian  const int lfl_forward = subsampling_factor ? 4 : 8;
3092ec72e65689c948e92b826ae1e867bf369e72f13Vignesh Venkatasubramanian
3102ec72e65689c948e92b826ae1e867bf369e72f13Vignesh Venkatasubramanian  unsigned int mask_16x16_0 = mask_16x16_l & mask_cutoff;
3112ec72e65689c948e92b826ae1e867bf369e72f13Vignesh Venkatasubramanian  unsigned int mask_8x8_0 = mask_8x8_l & mask_cutoff;
3122ec72e65689c948e92b826ae1e867bf369e72f13Vignesh Venkatasubramanian  unsigned int mask_4x4_0 = mask_4x4_l & mask_cutoff;
3132ec72e65689c948e92b826ae1e867bf369e72f13Vignesh Venkatasubramanian  unsigned int mask_4x4_int_0 = mask_4x4_int_l & mask_cutoff;
3142ec72e65689c948e92b826ae1e867bf369e72f13Vignesh Venkatasubramanian  unsigned int mask_16x16_1 = (mask_16x16_l >> mask_shift) & mask_cutoff;
3152ec72e65689c948e92b826ae1e867bf369e72f13Vignesh Venkatasubramanian  unsigned int mask_8x8_1 = (mask_8x8_l >> mask_shift) & mask_cutoff;
3162ec72e65689c948e92b826ae1e867bf369e72f13Vignesh Venkatasubramanian  unsigned int mask_4x4_1 = (mask_4x4_l >> mask_shift) & mask_cutoff;
3172ec72e65689c948e92b826ae1e867bf369e72f13Vignesh Venkatasubramanian  unsigned int mask_4x4_int_1 = (mask_4x4_int_l >> mask_shift) & mask_cutoff;
318ba164dffc5a6795bce97fae02b51ccf3330e15e4hkuang  unsigned int mask;
319ba164dffc5a6795bce97fae02b51ccf3330e15e4hkuang
3202ec72e65689c948e92b826ae1e867bf369e72f13Vignesh Venkatasubramanian  for (mask = mask_16x16_0 | mask_8x8_0 | mask_4x4_0 | mask_4x4_int_0 |
321ba6c59e9d7d7013b3906b6f4230b663422681848Vignesh Venkatasubramanian              mask_16x16_1 | mask_8x8_1 | mask_4x4_1 | mask_4x4_int_1;
322ba6c59e9d7d7013b3906b6f4230b663422681848Vignesh Venkatasubramanian       mask; mask >>= 1) {
3232ec72e65689c948e92b826ae1e867bf369e72f13Vignesh Venkatasubramanian    const loop_filter_thresh *lfi0 = lfi_n->lfthr + *lfl;
3242ec72e65689c948e92b826ae1e867bf369e72f13Vignesh Venkatasubramanian    const loop_filter_thresh *lfi1 = lfi_n->lfthr + *(lfl + lfl_forward);
3255ae7ac49f08a179e4f054d99fcfc9dce78d26e58hkuang
3262ec72e65689c948e92b826ae1e867bf369e72f13Vignesh Venkatasubramanian    // TODO(yunqingwang): count in loopfilter functions should be removed.
327ba164dffc5a6795bce97fae02b51ccf3330e15e4hkuang    if (mask & 1) {
3282ec72e65689c948e92b826ae1e867bf369e72f13Vignesh Venkatasubramanian      if ((mask_16x16_0 | mask_16x16_1) & 1) {
3292ec72e65689c948e92b826ae1e867bf369e72f13Vignesh Venkatasubramanian        if ((mask_16x16_0 & mask_16x16_1) & 1) {
330da49e34c1fb5e99681f4ad99c21d9cfd83eddb96Vignesh Venkatasubramanian          vpx_lpf_vertical_16_dual(s, pitch, lfi0->mblim, lfi0->lim,
3312ec72e65689c948e92b826ae1e867bf369e72f13Vignesh Venkatasubramanian                                   lfi0->hev_thr);
3322ec72e65689c948e92b826ae1e867bf369e72f13Vignesh Venkatasubramanian        } else if (mask_16x16_0 & 1) {
333da49e34c1fb5e99681f4ad99c21d9cfd83eddb96Vignesh Venkatasubramanian          vpx_lpf_vertical_16(s, pitch, lfi0->mblim, lfi0->lim,
3342ec72e65689c948e92b826ae1e867bf369e72f13Vignesh Venkatasubramanian                              lfi0->hev_thr);
3352ec72e65689c948e92b826ae1e867bf369e72f13Vignesh Venkatasubramanian        } else {
336da49e34c1fb5e99681f4ad99c21d9cfd83eddb96Vignesh Venkatasubramanian          vpx_lpf_vertical_16(s + 8 *pitch, pitch, lfi1->mblim,
3372ec72e65689c948e92b826ae1e867bf369e72f13Vignesh Venkatasubramanian                              lfi1->lim, lfi1->hev_thr);
3382ec72e65689c948e92b826ae1e867bf369e72f13Vignesh Venkatasubramanian        }
3392ec72e65689c948e92b826ae1e867bf369e72f13Vignesh Venkatasubramanian      }
3402ec72e65689c948e92b826ae1e867bf369e72f13Vignesh Venkatasubramanian
3412ec72e65689c948e92b826ae1e867bf369e72f13Vignesh Venkatasubramanian      if ((mask_8x8_0 | mask_8x8_1) & 1) {
3422ec72e65689c948e92b826ae1e867bf369e72f13Vignesh Venkatasubramanian        if ((mask_8x8_0 & mask_8x8_1) & 1) {
343da49e34c1fb5e99681f4ad99c21d9cfd83eddb96Vignesh Venkatasubramanian          vpx_lpf_vertical_8_dual(s, pitch, lfi0->mblim, lfi0->lim,
3442ec72e65689c948e92b826ae1e867bf369e72f13Vignesh Venkatasubramanian                                  lfi0->hev_thr, lfi1->mblim, lfi1->lim,
3452ec72e65689c948e92b826ae1e867bf369e72f13Vignesh Venkatasubramanian                                  lfi1->hev_thr);
3462ec72e65689c948e92b826ae1e867bf369e72f13Vignesh Venkatasubramanian        } else if (mask_8x8_0 & 1) {
347da49e34c1fb5e99681f4ad99c21d9cfd83eddb96Vignesh Venkatasubramanian          vpx_lpf_vertical_8(s, pitch, lfi0->mblim, lfi0->lim, lfi0->hev_thr,
3482ec72e65689c948e92b826ae1e867bf369e72f13Vignesh Venkatasubramanian                             1);
3492ec72e65689c948e92b826ae1e867bf369e72f13Vignesh Venkatasubramanian        } else {
350da49e34c1fb5e99681f4ad99c21d9cfd83eddb96Vignesh Venkatasubramanian          vpx_lpf_vertical_8(s + 8 * pitch, pitch, lfi1->mblim, lfi1->lim,
3512ec72e65689c948e92b826ae1e867bf369e72f13Vignesh Venkatasubramanian                             lfi1->hev_thr, 1);
3522ec72e65689c948e92b826ae1e867bf369e72f13Vignesh Venkatasubramanian        }
3532ec72e65689c948e92b826ae1e867bf369e72f13Vignesh Venkatasubramanian      }
3542ec72e65689c948e92b826ae1e867bf369e72f13Vignesh Venkatasubramanian
3552ec72e65689c948e92b826ae1e867bf369e72f13Vignesh Venkatasubramanian      if ((mask_4x4_0 | mask_4x4_1) & 1) {
3562ec72e65689c948e92b826ae1e867bf369e72f13Vignesh Venkatasubramanian        if ((mask_4x4_0 & mask_4x4_1) & 1) {
357da49e34c1fb5e99681f4ad99c21d9cfd83eddb96Vignesh Venkatasubramanian          vpx_lpf_vertical_4_dual(s, pitch, lfi0->mblim, lfi0->lim,
3582ec72e65689c948e92b826ae1e867bf369e72f13Vignesh Venkatasubramanian                                  lfi0->hev_thr, lfi1->mblim, lfi1->lim,
3592ec72e65689c948e92b826ae1e867bf369e72f13Vignesh Venkatasubramanian                                  lfi1->hev_thr);
3602ec72e65689c948e92b826ae1e867bf369e72f13Vignesh Venkatasubramanian        } else if (mask_4x4_0 & 1) {
361da49e34c1fb5e99681f4ad99c21d9cfd83eddb96Vignesh Venkatasubramanian          vpx_lpf_vertical_4(s, pitch, lfi0->mblim, lfi0->lim, lfi0->hev_thr,
3622ec72e65689c948e92b826ae1e867bf369e72f13Vignesh Venkatasubramanian                             1);
3632ec72e65689c948e92b826ae1e867bf369e72f13Vignesh Venkatasubramanian        } else {
364da49e34c1fb5e99681f4ad99c21d9cfd83eddb96Vignesh Venkatasubramanian          vpx_lpf_vertical_4(s + 8 * pitch, pitch, lfi1->mblim, lfi1->lim,
3652ec72e65689c948e92b826ae1e867bf369e72f13Vignesh Venkatasubramanian                             lfi1->hev_thr, 1);
3662ec72e65689c948e92b826ae1e867bf369e72f13Vignesh Venkatasubramanian        }
3672ec72e65689c948e92b826ae1e867bf369e72f13Vignesh Venkatasubramanian      }
3682ec72e65689c948e92b826ae1e867bf369e72f13Vignesh Venkatasubramanian
3692ec72e65689c948e92b826ae1e867bf369e72f13Vignesh Venkatasubramanian      if ((mask_4x4_int_0 | mask_4x4_int_1) & 1) {
3702ec72e65689c948e92b826ae1e867bf369e72f13Vignesh Venkatasubramanian        if ((mask_4x4_int_0 & mask_4x4_int_1) & 1) {
371da49e34c1fb5e99681f4ad99c21d9cfd83eddb96Vignesh Venkatasubramanian          vpx_lpf_vertical_4_dual(s + 4, pitch, lfi0->mblim, lfi0->lim,
3722ec72e65689c948e92b826ae1e867bf369e72f13Vignesh Venkatasubramanian                                  lfi0->hev_thr, lfi1->mblim, lfi1->lim,
3732ec72e65689c948e92b826ae1e867bf369e72f13Vignesh Venkatasubramanian                                  lfi1->hev_thr);
3742ec72e65689c948e92b826ae1e867bf369e72f13Vignesh Venkatasubramanian        } else if (mask_4x4_int_0 & 1) {
375da49e34c1fb5e99681f4ad99c21d9cfd83eddb96Vignesh Venkatasubramanian          vpx_lpf_vertical_4(s + 4, pitch, lfi0->mblim, lfi0->lim,
3762ec72e65689c948e92b826ae1e867bf369e72f13Vignesh Venkatasubramanian                             lfi0->hev_thr, 1);
3772ec72e65689c948e92b826ae1e867bf369e72f13Vignesh Venkatasubramanian        } else {
378da49e34c1fb5e99681f4ad99c21d9cfd83eddb96Vignesh Venkatasubramanian          vpx_lpf_vertical_4(s + 8 * pitch + 4, pitch, lfi1->mblim, lfi1->lim,
3792ec72e65689c948e92b826ae1e867bf369e72f13Vignesh Venkatasubramanian                             lfi1->hev_thr, 1);
3802ec72e65689c948e92b826ae1e867bf369e72f13Vignesh Venkatasubramanian        }
381ba164dffc5a6795bce97fae02b51ccf3330e15e4hkuang      }
382ba164dffc5a6795bce97fae02b51ccf3330e15e4hkuang    }
3832ec72e65689c948e92b826ae1e867bf369e72f13Vignesh Venkatasubramanian
384ba164dffc5a6795bce97fae02b51ccf3330e15e4hkuang    s += 8;
3859b35249446b07f40ac5fcc3205f2c048616efacchkuang    lfl += 1;
3862ec72e65689c948e92b826ae1e867bf369e72f13Vignesh Venkatasubramanian    mask_16x16_0 >>= 1;
3872ec72e65689c948e92b826ae1e867bf369e72f13Vignesh Venkatasubramanian    mask_8x8_0 >>= 1;
3882ec72e65689c948e92b826ae1e867bf369e72f13Vignesh Venkatasubramanian    mask_4x4_0 >>= 1;
3892ec72e65689c948e92b826ae1e867bf369e72f13Vignesh Venkatasubramanian    mask_4x4_int_0 >>= 1;
3902ec72e65689c948e92b826ae1e867bf369e72f13Vignesh Venkatasubramanian    mask_16x16_1 >>= 1;
3912ec72e65689c948e92b826ae1e867bf369e72f13Vignesh Venkatasubramanian    mask_8x8_1 >>= 1;
3922ec72e65689c948e92b826ae1e867bf369e72f13Vignesh Venkatasubramanian    mask_4x4_1 >>= 1;
3932ec72e65689c948e92b826ae1e867bf369e72f13Vignesh Venkatasubramanian    mask_4x4_int_1 >>= 1;
394ba164dffc5a6795bce97fae02b51ccf3330e15e4hkuang  }
395ba164dffc5a6795bce97fae02b51ccf3330e15e4hkuang}
396ba164dffc5a6795bce97fae02b51ccf3330e15e4hkuang
397da49e34c1fb5e99681f4ad99c21d9cfd83eddb96Vignesh Venkatasubramanian#if CONFIG_VP9_HIGHBITDEPTH
398da49e34c1fb5e99681f4ad99c21d9cfd83eddb96Vignesh Venkatasubramanianstatic void highbd_filter_selectively_vert_row2(int subsampling_factor,
399da49e34c1fb5e99681f4ad99c21d9cfd83eddb96Vignesh Venkatasubramanian                                                uint16_t *s, int pitch,
400da49e34c1fb5e99681f4ad99c21d9cfd83eddb96Vignesh Venkatasubramanian                                                unsigned int mask_16x16_l,
401da49e34c1fb5e99681f4ad99c21d9cfd83eddb96Vignesh Venkatasubramanian                                                unsigned int mask_8x8_l,
402da49e34c1fb5e99681f4ad99c21d9cfd83eddb96Vignesh Venkatasubramanian                                                unsigned int mask_4x4_l,
403da49e34c1fb5e99681f4ad99c21d9cfd83eddb96Vignesh Venkatasubramanian                                                unsigned int mask_4x4_int_l,
404da49e34c1fb5e99681f4ad99c21d9cfd83eddb96Vignesh Venkatasubramanian                                                const loop_filter_info_n *lfi_n,
405da49e34c1fb5e99681f4ad99c21d9cfd83eddb96Vignesh Venkatasubramanian                                                const uint8_t *lfl, int bd) {
406da49e34c1fb5e99681f4ad99c21d9cfd83eddb96Vignesh Venkatasubramanian  const int mask_shift = subsampling_factor ? 4 : 8;
407da49e34c1fb5e99681f4ad99c21d9cfd83eddb96Vignesh Venkatasubramanian  const int mask_cutoff = subsampling_factor ? 0xf : 0xff;
408da49e34c1fb5e99681f4ad99c21d9cfd83eddb96Vignesh Venkatasubramanian  const int lfl_forward = subsampling_factor ? 4 : 8;
409da49e34c1fb5e99681f4ad99c21d9cfd83eddb96Vignesh Venkatasubramanian
410da49e34c1fb5e99681f4ad99c21d9cfd83eddb96Vignesh Venkatasubramanian  unsigned int mask_16x16_0 = mask_16x16_l & mask_cutoff;
411da49e34c1fb5e99681f4ad99c21d9cfd83eddb96Vignesh Venkatasubramanian  unsigned int mask_8x8_0 = mask_8x8_l & mask_cutoff;
412da49e34c1fb5e99681f4ad99c21d9cfd83eddb96Vignesh Venkatasubramanian  unsigned int mask_4x4_0 = mask_4x4_l & mask_cutoff;
413da49e34c1fb5e99681f4ad99c21d9cfd83eddb96Vignesh Venkatasubramanian  unsigned int mask_4x4_int_0 = mask_4x4_int_l & mask_cutoff;
414da49e34c1fb5e99681f4ad99c21d9cfd83eddb96Vignesh Venkatasubramanian  unsigned int mask_16x16_1 = (mask_16x16_l >> mask_shift) & mask_cutoff;
415da49e34c1fb5e99681f4ad99c21d9cfd83eddb96Vignesh Venkatasubramanian  unsigned int mask_8x8_1 = (mask_8x8_l >> mask_shift) & mask_cutoff;
416da49e34c1fb5e99681f4ad99c21d9cfd83eddb96Vignesh Venkatasubramanian  unsigned int mask_4x4_1 = (mask_4x4_l >> mask_shift) & mask_cutoff;
417da49e34c1fb5e99681f4ad99c21d9cfd83eddb96Vignesh Venkatasubramanian  unsigned int mask_4x4_int_1 = (mask_4x4_int_l >> mask_shift) & mask_cutoff;
418da49e34c1fb5e99681f4ad99c21d9cfd83eddb96Vignesh Venkatasubramanian  unsigned int mask;
419da49e34c1fb5e99681f4ad99c21d9cfd83eddb96Vignesh Venkatasubramanian
420da49e34c1fb5e99681f4ad99c21d9cfd83eddb96Vignesh Venkatasubramanian  for (mask = mask_16x16_0 | mask_8x8_0 | mask_4x4_0 | mask_4x4_int_0 |
421da49e34c1fb5e99681f4ad99c21d9cfd83eddb96Vignesh Venkatasubramanian       mask_16x16_1 | mask_8x8_1 | mask_4x4_1 | mask_4x4_int_1;
422da49e34c1fb5e99681f4ad99c21d9cfd83eddb96Vignesh Venkatasubramanian       mask; mask >>= 1) {
423da49e34c1fb5e99681f4ad99c21d9cfd83eddb96Vignesh Venkatasubramanian    const loop_filter_thresh *lfi0 = lfi_n->lfthr + *lfl;
424da49e34c1fb5e99681f4ad99c21d9cfd83eddb96Vignesh Venkatasubramanian    const loop_filter_thresh *lfi1 = lfi_n->lfthr + *(lfl + lfl_forward);
425da49e34c1fb5e99681f4ad99c21d9cfd83eddb96Vignesh Venkatasubramanian
426da49e34c1fb5e99681f4ad99c21d9cfd83eddb96Vignesh Venkatasubramanian    // TODO(yunqingwang): count in loopfilter functions should be removed.
427da49e34c1fb5e99681f4ad99c21d9cfd83eddb96Vignesh Venkatasubramanian    if (mask & 1) {
428da49e34c1fb5e99681f4ad99c21d9cfd83eddb96Vignesh Venkatasubramanian      if ((mask_16x16_0 | mask_16x16_1) & 1) {
429da49e34c1fb5e99681f4ad99c21d9cfd83eddb96Vignesh Venkatasubramanian        if ((mask_16x16_0 & mask_16x16_1) & 1) {
430da49e34c1fb5e99681f4ad99c21d9cfd83eddb96Vignesh Venkatasubramanian          vpx_highbd_lpf_vertical_16_dual(s, pitch, lfi0->mblim, lfi0->lim,
431da49e34c1fb5e99681f4ad99c21d9cfd83eddb96Vignesh Venkatasubramanian                                          lfi0->hev_thr, bd);
432da49e34c1fb5e99681f4ad99c21d9cfd83eddb96Vignesh Venkatasubramanian        } else if (mask_16x16_0 & 1) {
433da49e34c1fb5e99681f4ad99c21d9cfd83eddb96Vignesh Venkatasubramanian          vpx_highbd_lpf_vertical_16(s, pitch, lfi0->mblim, lfi0->lim,
434da49e34c1fb5e99681f4ad99c21d9cfd83eddb96Vignesh Venkatasubramanian                                     lfi0->hev_thr, bd);
435da49e34c1fb5e99681f4ad99c21d9cfd83eddb96Vignesh Venkatasubramanian        } else {
436da49e34c1fb5e99681f4ad99c21d9cfd83eddb96Vignesh Venkatasubramanian          vpx_highbd_lpf_vertical_16(s + 8 *pitch, pitch, lfi1->mblim,
437da49e34c1fb5e99681f4ad99c21d9cfd83eddb96Vignesh Venkatasubramanian                                     lfi1->lim, lfi1->hev_thr, bd);
438da49e34c1fb5e99681f4ad99c21d9cfd83eddb96Vignesh Venkatasubramanian        }
439da49e34c1fb5e99681f4ad99c21d9cfd83eddb96Vignesh Venkatasubramanian      }
440da49e34c1fb5e99681f4ad99c21d9cfd83eddb96Vignesh Venkatasubramanian
441da49e34c1fb5e99681f4ad99c21d9cfd83eddb96Vignesh Venkatasubramanian      if ((mask_8x8_0 | mask_8x8_1) & 1) {
442da49e34c1fb5e99681f4ad99c21d9cfd83eddb96Vignesh Venkatasubramanian        if ((mask_8x8_0 & mask_8x8_1) & 1) {
443da49e34c1fb5e99681f4ad99c21d9cfd83eddb96Vignesh Venkatasubramanian          vpx_highbd_lpf_vertical_8_dual(s, pitch, lfi0->mblim, lfi0->lim,
444da49e34c1fb5e99681f4ad99c21d9cfd83eddb96Vignesh Venkatasubramanian                                         lfi0->hev_thr, lfi1->mblim, lfi1->lim,
445da49e34c1fb5e99681f4ad99c21d9cfd83eddb96Vignesh Venkatasubramanian                                         lfi1->hev_thr, bd);
446da49e34c1fb5e99681f4ad99c21d9cfd83eddb96Vignesh Venkatasubramanian        } else if (mask_8x8_0 & 1) {
447da49e34c1fb5e99681f4ad99c21d9cfd83eddb96Vignesh Venkatasubramanian          vpx_highbd_lpf_vertical_8(s, pitch, lfi0->mblim, lfi0->lim,
448da49e34c1fb5e99681f4ad99c21d9cfd83eddb96Vignesh Venkatasubramanian                                    lfi0->hev_thr, 1, bd);
449da49e34c1fb5e99681f4ad99c21d9cfd83eddb96Vignesh Venkatasubramanian        } else {
450da49e34c1fb5e99681f4ad99c21d9cfd83eddb96Vignesh Venkatasubramanian          vpx_highbd_lpf_vertical_8(s + 8 * pitch, pitch, lfi1->mblim,
451da49e34c1fb5e99681f4ad99c21d9cfd83eddb96Vignesh Venkatasubramanian                                    lfi1->lim, lfi1->hev_thr, 1, bd);
452da49e34c1fb5e99681f4ad99c21d9cfd83eddb96Vignesh Venkatasubramanian        }
453da49e34c1fb5e99681f4ad99c21d9cfd83eddb96Vignesh Venkatasubramanian      }
454da49e34c1fb5e99681f4ad99c21d9cfd83eddb96Vignesh Venkatasubramanian
455da49e34c1fb5e99681f4ad99c21d9cfd83eddb96Vignesh Venkatasubramanian      if ((mask_4x4_0 | mask_4x4_1) & 1) {
456da49e34c1fb5e99681f4ad99c21d9cfd83eddb96Vignesh Venkatasubramanian        if ((mask_4x4_0 & mask_4x4_1) & 1) {
457da49e34c1fb5e99681f4ad99c21d9cfd83eddb96Vignesh Venkatasubramanian          vpx_highbd_lpf_vertical_4_dual(s, pitch, lfi0->mblim, lfi0->lim,
458da49e34c1fb5e99681f4ad99c21d9cfd83eddb96Vignesh Venkatasubramanian                                         lfi0->hev_thr, lfi1->mblim, lfi1->lim,
459da49e34c1fb5e99681f4ad99c21d9cfd83eddb96Vignesh Venkatasubramanian                                         lfi1->hev_thr, bd);
460da49e34c1fb5e99681f4ad99c21d9cfd83eddb96Vignesh Venkatasubramanian        } else if (mask_4x4_0 & 1) {
461da49e34c1fb5e99681f4ad99c21d9cfd83eddb96Vignesh Venkatasubramanian          vpx_highbd_lpf_vertical_4(s, pitch, lfi0->mblim, lfi0->lim,
462da49e34c1fb5e99681f4ad99c21d9cfd83eddb96Vignesh Venkatasubramanian                                    lfi0->hev_thr, 1, bd);
463da49e34c1fb5e99681f4ad99c21d9cfd83eddb96Vignesh Venkatasubramanian        } else {
464da49e34c1fb5e99681f4ad99c21d9cfd83eddb96Vignesh Venkatasubramanian          vpx_highbd_lpf_vertical_4(s + 8 * pitch, pitch, lfi1->mblim,
465da49e34c1fb5e99681f4ad99c21d9cfd83eddb96Vignesh Venkatasubramanian                                    lfi1->lim, lfi1->hev_thr, 1, bd);
466da49e34c1fb5e99681f4ad99c21d9cfd83eddb96Vignesh Venkatasubramanian        }
467da49e34c1fb5e99681f4ad99c21d9cfd83eddb96Vignesh Venkatasubramanian      }
468da49e34c1fb5e99681f4ad99c21d9cfd83eddb96Vignesh Venkatasubramanian
469da49e34c1fb5e99681f4ad99c21d9cfd83eddb96Vignesh Venkatasubramanian      if ((mask_4x4_int_0 | mask_4x4_int_1) & 1) {
470da49e34c1fb5e99681f4ad99c21d9cfd83eddb96Vignesh Venkatasubramanian        if ((mask_4x4_int_0 & mask_4x4_int_1) & 1) {
471da49e34c1fb5e99681f4ad99c21d9cfd83eddb96Vignesh Venkatasubramanian          vpx_highbd_lpf_vertical_4_dual(s + 4, pitch, lfi0->mblim, lfi0->lim,
472da49e34c1fb5e99681f4ad99c21d9cfd83eddb96Vignesh Venkatasubramanian                                         lfi0->hev_thr, lfi1->mblim, lfi1->lim,
473da49e34c1fb5e99681f4ad99c21d9cfd83eddb96Vignesh Venkatasubramanian                                         lfi1->hev_thr, bd);
474da49e34c1fb5e99681f4ad99c21d9cfd83eddb96Vignesh Venkatasubramanian        } else if (mask_4x4_int_0 & 1) {
475da49e34c1fb5e99681f4ad99c21d9cfd83eddb96Vignesh Venkatasubramanian          vpx_highbd_lpf_vertical_4(s + 4, pitch, lfi0->mblim, lfi0->lim,
476da49e34c1fb5e99681f4ad99c21d9cfd83eddb96Vignesh Venkatasubramanian                                    lfi0->hev_thr, 1, bd);
477da49e34c1fb5e99681f4ad99c21d9cfd83eddb96Vignesh Venkatasubramanian        } else {
478da49e34c1fb5e99681f4ad99c21d9cfd83eddb96Vignesh Venkatasubramanian          vpx_highbd_lpf_vertical_4(s + 8 * pitch + 4, pitch, lfi1->mblim,
479da49e34c1fb5e99681f4ad99c21d9cfd83eddb96Vignesh Venkatasubramanian                                    lfi1->lim, lfi1->hev_thr, 1, bd);
480da49e34c1fb5e99681f4ad99c21d9cfd83eddb96Vignesh Venkatasubramanian        }
481da49e34c1fb5e99681f4ad99c21d9cfd83eddb96Vignesh Venkatasubramanian      }
482da49e34c1fb5e99681f4ad99c21d9cfd83eddb96Vignesh Venkatasubramanian    }
483da49e34c1fb5e99681f4ad99c21d9cfd83eddb96Vignesh Venkatasubramanian
484da49e34c1fb5e99681f4ad99c21d9cfd83eddb96Vignesh Venkatasubramanian    s += 8;
485da49e34c1fb5e99681f4ad99c21d9cfd83eddb96Vignesh Venkatasubramanian    lfl += 1;
486da49e34c1fb5e99681f4ad99c21d9cfd83eddb96Vignesh Venkatasubramanian    mask_16x16_0 >>= 1;
487da49e34c1fb5e99681f4ad99c21d9cfd83eddb96Vignesh Venkatasubramanian    mask_8x8_0 >>= 1;
488da49e34c1fb5e99681f4ad99c21d9cfd83eddb96Vignesh Venkatasubramanian    mask_4x4_0 >>= 1;
489da49e34c1fb5e99681f4ad99c21d9cfd83eddb96Vignesh Venkatasubramanian    mask_4x4_int_0 >>= 1;
490da49e34c1fb5e99681f4ad99c21d9cfd83eddb96Vignesh Venkatasubramanian    mask_16x16_1 >>= 1;
491da49e34c1fb5e99681f4ad99c21d9cfd83eddb96Vignesh Venkatasubramanian    mask_8x8_1 >>= 1;
492da49e34c1fb5e99681f4ad99c21d9cfd83eddb96Vignesh Venkatasubramanian    mask_4x4_1 >>= 1;
493da49e34c1fb5e99681f4ad99c21d9cfd83eddb96Vignesh Venkatasubramanian    mask_4x4_int_1 >>= 1;
494da49e34c1fb5e99681f4ad99c21d9cfd83eddb96Vignesh Venkatasubramanian  }
495da49e34c1fb5e99681f4ad99c21d9cfd83eddb96Vignesh Venkatasubramanian}
496da49e34c1fb5e99681f4ad99c21d9cfd83eddb96Vignesh Venkatasubramanian#endif  // CONFIG_VP9_HIGHBITDEPTH
497da49e34c1fb5e99681f4ad99c21d9cfd83eddb96Vignesh Venkatasubramanian
498ba164dffc5a6795bce97fae02b51ccf3330e15e4hkuangstatic void filter_selectively_horiz(uint8_t *s, int pitch,
499ba164dffc5a6795bce97fae02b51ccf3330e15e4hkuang                                     unsigned int mask_16x16,
500ba164dffc5a6795bce97fae02b51ccf3330e15e4hkuang                                     unsigned int mask_8x8,
501ba164dffc5a6795bce97fae02b51ccf3330e15e4hkuang                                     unsigned int mask_4x4,
502ba164dffc5a6795bce97fae02b51ccf3330e15e4hkuang                                     unsigned int mask_4x4_int,
5039b35249446b07f40ac5fcc3205f2c048616efacchkuang                                     const loop_filter_info_n *lfi_n,
5049b35249446b07f40ac5fcc3205f2c048616efacchkuang                                     const uint8_t *lfl) {
505ba164dffc5a6795bce97fae02b51ccf3330e15e4hkuang  unsigned int mask;
50691037db265ecdd914a26e056cf69207b4f50924ehkuang  int count;
507ba164dffc5a6795bce97fae02b51ccf3330e15e4hkuang
50891037db265ecdd914a26e056cf69207b4f50924ehkuang  for (mask = mask_16x16 | mask_8x8 | mask_4x4 | mask_4x4_int;
50991037db265ecdd914a26e056cf69207b4f50924ehkuang       mask; mask >>= count) {
5109b35249446b07f40ac5fcc3205f2c048616efacchkuang    const loop_filter_thresh *lfi = lfi_n->lfthr + *lfl;
5115ae7ac49f08a179e4f054d99fcfc9dce78d26e58hkuang
51291037db265ecdd914a26e056cf69207b4f50924ehkuang    count = 1;
513ba164dffc5a6795bce97fae02b51ccf3330e15e4hkuang    if (mask & 1) {
5149b35249446b07f40ac5fcc3205f2c048616efacchkuang      if (mask_16x16 & 1) {
5159b35249446b07f40ac5fcc3205f2c048616efacchkuang        if ((mask_16x16 & 3) == 3) {
516da49e34c1fb5e99681f4ad99c21d9cfd83eddb96Vignesh Venkatasubramanian          vpx_lpf_horizontal_16(s, pitch, lfi->mblim, lfi->lim,
5172ec72e65689c948e92b826ae1e867bf369e72f13Vignesh Venkatasubramanian                                lfi->hev_thr, 2);
5189b35249446b07f40ac5fcc3205f2c048616efacchkuang          count = 2;
5199b35249446b07f40ac5fcc3205f2c048616efacchkuang        } else {
520da49e34c1fb5e99681f4ad99c21d9cfd83eddb96Vignesh Venkatasubramanian          vpx_lpf_horizontal_16(s, pitch, lfi->mblim, lfi->lim,
5212ec72e65689c948e92b826ae1e867bf369e72f13Vignesh Venkatasubramanian                                lfi->hev_thr, 1);
5229b35249446b07f40ac5fcc3205f2c048616efacchkuang        }
5239b35249446b07f40ac5fcc3205f2c048616efacchkuang      } else if (mask_8x8 & 1) {
5249b35249446b07f40ac5fcc3205f2c048616efacchkuang        if ((mask_8x8 & 3) == 3) {
525da49e34c1fb5e99681f4ad99c21d9cfd83eddb96Vignesh Venkatasubramanian          // Next block's thresholds.
5269b35249446b07f40ac5fcc3205f2c048616efacchkuang          const loop_filter_thresh *lfin = lfi_n->lfthr + *(lfl + 1);
5279b35249446b07f40ac5fcc3205f2c048616efacchkuang
528da49e34c1fb5e99681f4ad99c21d9cfd83eddb96Vignesh Venkatasubramanian          vpx_lpf_horizontal_8_dual(s, pitch, lfi->mblim, lfi->lim,
5292ec72e65689c948e92b826ae1e867bf369e72f13Vignesh Venkatasubramanian                                    lfi->hev_thr, lfin->mblim, lfin->lim,
5302ec72e65689c948e92b826ae1e867bf369e72f13Vignesh Venkatasubramanian                                    lfin->hev_thr);
5319b35249446b07f40ac5fcc3205f2c048616efacchkuang
5329b35249446b07f40ac5fcc3205f2c048616efacchkuang          if ((mask_4x4_int & 3) == 3) {
533da49e34c1fb5e99681f4ad99c21d9cfd83eddb96Vignesh Venkatasubramanian            vpx_lpf_horizontal_4_dual(s + 4 * pitch, pitch, lfi->mblim,
5342ec72e65689c948e92b826ae1e867bf369e72f13Vignesh Venkatasubramanian                                      lfi->lim, lfi->hev_thr, lfin->mblim,
5352ec72e65689c948e92b826ae1e867bf369e72f13Vignesh Venkatasubramanian                                      lfin->lim, lfin->hev_thr);
53691037db265ecdd914a26e056cf69207b4f50924ehkuang          } else {
5379b35249446b07f40ac5fcc3205f2c048616efacchkuang            if (mask_4x4_int & 1)
538da49e34c1fb5e99681f4ad99c21d9cfd83eddb96Vignesh Venkatasubramanian              vpx_lpf_horizontal_4(s + 4 * pitch, pitch, lfi->mblim, lfi->lim,
5392ec72e65689c948e92b826ae1e867bf369e72f13Vignesh Venkatasubramanian                                   lfi->hev_thr, 1);
5409b35249446b07f40ac5fcc3205f2c048616efacchkuang            else if (mask_4x4_int & 2)
541da49e34c1fb5e99681f4ad99c21d9cfd83eddb96Vignesh Venkatasubramanian              vpx_lpf_horizontal_4(s + 8 + 4 * pitch, pitch, lfin->mblim,
5422ec72e65689c948e92b826ae1e867bf369e72f13Vignesh Venkatasubramanian                                   lfin->lim, lfin->hev_thr, 1);
54391037db265ecdd914a26e056cf69207b4f50924ehkuang          }
5449b35249446b07f40ac5fcc3205f2c048616efacchkuang          count = 2;
5459b35249446b07f40ac5fcc3205f2c048616efacchkuang        } else {
546da49e34c1fb5e99681f4ad99c21d9cfd83eddb96Vignesh Venkatasubramanian          vpx_lpf_horizontal_8(s, pitch, lfi->mblim, lfi->lim, lfi->hev_thr, 1);
5479b35249446b07f40ac5fcc3205f2c048616efacchkuang
5489b35249446b07f40ac5fcc3205f2c048616efacchkuang          if (mask_4x4_int & 1)
549da49e34c1fb5e99681f4ad99c21d9cfd83eddb96Vignesh Venkatasubramanian            vpx_lpf_horizontal_4(s + 4 * pitch, pitch, lfi->mblim, lfi->lim,
5502ec72e65689c948e92b826ae1e867bf369e72f13Vignesh Venkatasubramanian                                 lfi->hev_thr, 1);
551ba164dffc5a6795bce97fae02b51ccf3330e15e4hkuang        }
5529b35249446b07f40ac5fcc3205f2c048616efacchkuang      } else if (mask_4x4 & 1) {
5539b35249446b07f40ac5fcc3205f2c048616efacchkuang        if ((mask_4x4 & 3) == 3) {
554da49e34c1fb5e99681f4ad99c21d9cfd83eddb96Vignesh Venkatasubramanian          // Next block's thresholds.
5559b35249446b07f40ac5fcc3205f2c048616efacchkuang          const loop_filter_thresh *lfin = lfi_n->lfthr + *(lfl + 1);
556ba164dffc5a6795bce97fae02b51ccf3330e15e4hkuang
557da49e34c1fb5e99681f4ad99c21d9cfd83eddb96Vignesh Venkatasubramanian          vpx_lpf_horizontal_4_dual(s, pitch, lfi->mblim, lfi->lim,
5582ec72e65689c948e92b826ae1e867bf369e72f13Vignesh Venkatasubramanian                                    lfi->hev_thr, lfin->mblim, lfin->lim,
5592ec72e65689c948e92b826ae1e867bf369e72f13Vignesh Venkatasubramanian                                    lfin->hev_thr);
5609b35249446b07f40ac5fcc3205f2c048616efacchkuang          if ((mask_4x4_int & 3) == 3) {
561da49e34c1fb5e99681f4ad99c21d9cfd83eddb96Vignesh Venkatasubramanian            vpx_lpf_horizontal_4_dual(s + 4 * pitch, pitch, lfi->mblim,
5622ec72e65689c948e92b826ae1e867bf369e72f13Vignesh Venkatasubramanian                                      lfi->lim, lfi->hev_thr, lfin->mblim,
5632ec72e65689c948e92b826ae1e867bf369e72f13Vignesh Venkatasubramanian                                      lfin->lim, lfin->hev_thr);
5649b35249446b07f40ac5fcc3205f2c048616efacchkuang          } else {
5659b35249446b07f40ac5fcc3205f2c048616efacchkuang            if (mask_4x4_int & 1)
566da49e34c1fb5e99681f4ad99c21d9cfd83eddb96Vignesh Venkatasubramanian              vpx_lpf_horizontal_4(s + 4 * pitch, pitch, lfi->mblim, lfi->lim,
5672ec72e65689c948e92b826ae1e867bf369e72f13Vignesh Venkatasubramanian                                   lfi->hev_thr, 1);
5689b35249446b07f40ac5fcc3205f2c048616efacchkuang            else if (mask_4x4_int & 2)
569da49e34c1fb5e99681f4ad99c21d9cfd83eddb96Vignesh Venkatasubramanian              vpx_lpf_horizontal_4(s + 8 + 4 * pitch, pitch, lfin->mblim,
5702ec72e65689c948e92b826ae1e867bf369e72f13Vignesh Venkatasubramanian                                   lfin->lim, lfin->hev_thr, 1);
5719b35249446b07f40ac5fcc3205f2c048616efacchkuang          }
5729b35249446b07f40ac5fcc3205f2c048616efacchkuang          count = 2;
5739b35249446b07f40ac5fcc3205f2c048616efacchkuang        } else {
574da49e34c1fb5e99681f4ad99c21d9cfd83eddb96Vignesh Venkatasubramanian          vpx_lpf_horizontal_4(s, pitch, lfi->mblim, lfi->lim, lfi->hev_thr, 1);
5759b35249446b07f40ac5fcc3205f2c048616efacchkuang
5762ec72e65689c948e92b826ae1e867bf369e72f13Vignesh Venkatasubramanian          if (mask_4x4_int & 1)
577da49e34c1fb5e99681f4ad99c21d9cfd83eddb96Vignesh Venkatasubramanian            vpx_lpf_horizontal_4(s + 4 * pitch, pitch, lfi->mblim, lfi->lim,
5782ec72e65689c948e92b826ae1e867bf369e72f13Vignesh Venkatasubramanian                                 lfi->hev_thr, 1);
5799b35249446b07f40ac5fcc3205f2c048616efacchkuang        }
5809b35249446b07f40ac5fcc3205f2c048616efacchkuang      } else if (mask_4x4_int & 1) {
581da49e34c1fb5e99681f4ad99c21d9cfd83eddb96Vignesh Venkatasubramanian        vpx_lpf_horizontal_4(s + 4 * pitch, pitch, lfi->mblim, lfi->lim,
5822ec72e65689c948e92b826ae1e867bf369e72f13Vignesh Venkatasubramanian                             lfi->hev_thr, 1);
5839b35249446b07f40ac5fcc3205f2c048616efacchkuang      }
584ba164dffc5a6795bce97fae02b51ccf3330e15e4hkuang    }
58591037db265ecdd914a26e056cf69207b4f50924ehkuang    s += 8 * count;
5869b35249446b07f40ac5fcc3205f2c048616efacchkuang    lfl += count;
58791037db265ecdd914a26e056cf69207b4f50924ehkuang    mask_16x16 >>= count;
58891037db265ecdd914a26e056cf69207b4f50924ehkuang    mask_8x8 >>= count;
58991037db265ecdd914a26e056cf69207b4f50924ehkuang    mask_4x4 >>= count;
59091037db265ecdd914a26e056cf69207b4f50924ehkuang    mask_4x4_int >>= count;
591ba164dffc5a6795bce97fae02b51ccf3330e15e4hkuang  }
592ba164dffc5a6795bce97fae02b51ccf3330e15e4hkuang}
593ba164dffc5a6795bce97fae02b51ccf3330e15e4hkuang
594da49e34c1fb5e99681f4ad99c21d9cfd83eddb96Vignesh Venkatasubramanian#if CONFIG_VP9_HIGHBITDEPTH
595da49e34c1fb5e99681f4ad99c21d9cfd83eddb96Vignesh Venkatasubramanianstatic void highbd_filter_selectively_horiz(uint16_t *s, int pitch,
596da49e34c1fb5e99681f4ad99c21d9cfd83eddb96Vignesh Venkatasubramanian                                            unsigned int mask_16x16,
597da49e34c1fb5e99681f4ad99c21d9cfd83eddb96Vignesh Venkatasubramanian                                            unsigned int mask_8x8,
598da49e34c1fb5e99681f4ad99c21d9cfd83eddb96Vignesh Venkatasubramanian                                            unsigned int mask_4x4,
599da49e34c1fb5e99681f4ad99c21d9cfd83eddb96Vignesh Venkatasubramanian                                            unsigned int mask_4x4_int,
600da49e34c1fb5e99681f4ad99c21d9cfd83eddb96Vignesh Venkatasubramanian                                            const loop_filter_info_n *lfi_n,
601da49e34c1fb5e99681f4ad99c21d9cfd83eddb96Vignesh Venkatasubramanian                                            const uint8_t *lfl, int bd) {
602da49e34c1fb5e99681f4ad99c21d9cfd83eddb96Vignesh Venkatasubramanian  unsigned int mask;
603da49e34c1fb5e99681f4ad99c21d9cfd83eddb96Vignesh Venkatasubramanian  int count;
604da49e34c1fb5e99681f4ad99c21d9cfd83eddb96Vignesh Venkatasubramanian
605da49e34c1fb5e99681f4ad99c21d9cfd83eddb96Vignesh Venkatasubramanian  for (mask = mask_16x16 | mask_8x8 | mask_4x4 | mask_4x4_int;
606da49e34c1fb5e99681f4ad99c21d9cfd83eddb96Vignesh Venkatasubramanian       mask; mask >>= count) {
607da49e34c1fb5e99681f4ad99c21d9cfd83eddb96Vignesh Venkatasubramanian    const loop_filter_thresh *lfi = lfi_n->lfthr + *lfl;
608da49e34c1fb5e99681f4ad99c21d9cfd83eddb96Vignesh Venkatasubramanian
609da49e34c1fb5e99681f4ad99c21d9cfd83eddb96Vignesh Venkatasubramanian    count = 1;
610da49e34c1fb5e99681f4ad99c21d9cfd83eddb96Vignesh Venkatasubramanian    if (mask & 1) {
611da49e34c1fb5e99681f4ad99c21d9cfd83eddb96Vignesh Venkatasubramanian      if (mask_16x16 & 1) {
612da49e34c1fb5e99681f4ad99c21d9cfd83eddb96Vignesh Venkatasubramanian        if ((mask_16x16 & 3) == 3) {
613da49e34c1fb5e99681f4ad99c21d9cfd83eddb96Vignesh Venkatasubramanian          vpx_highbd_lpf_horizontal_16(s, pitch, lfi->mblim, lfi->lim,
614da49e34c1fb5e99681f4ad99c21d9cfd83eddb96Vignesh Venkatasubramanian                                       lfi->hev_thr, 2, bd);
615da49e34c1fb5e99681f4ad99c21d9cfd83eddb96Vignesh Venkatasubramanian          count = 2;
616da49e34c1fb5e99681f4ad99c21d9cfd83eddb96Vignesh Venkatasubramanian        } else {
617da49e34c1fb5e99681f4ad99c21d9cfd83eddb96Vignesh Venkatasubramanian          vpx_highbd_lpf_horizontal_16(s, pitch, lfi->mblim, lfi->lim,
618da49e34c1fb5e99681f4ad99c21d9cfd83eddb96Vignesh Venkatasubramanian                                       lfi->hev_thr, 1, bd);
619da49e34c1fb5e99681f4ad99c21d9cfd83eddb96Vignesh Venkatasubramanian        }
620da49e34c1fb5e99681f4ad99c21d9cfd83eddb96Vignesh Venkatasubramanian      } else if (mask_8x8 & 1) {
621da49e34c1fb5e99681f4ad99c21d9cfd83eddb96Vignesh Venkatasubramanian        if ((mask_8x8 & 3) == 3) {
622da49e34c1fb5e99681f4ad99c21d9cfd83eddb96Vignesh Venkatasubramanian          // Next block's thresholds.
623da49e34c1fb5e99681f4ad99c21d9cfd83eddb96Vignesh Venkatasubramanian          const loop_filter_thresh *lfin = lfi_n->lfthr + *(lfl + 1);
624da49e34c1fb5e99681f4ad99c21d9cfd83eddb96Vignesh Venkatasubramanian
625da49e34c1fb5e99681f4ad99c21d9cfd83eddb96Vignesh Venkatasubramanian          vpx_highbd_lpf_horizontal_8_dual(s, pitch, lfi->mblim, lfi->lim,
626da49e34c1fb5e99681f4ad99c21d9cfd83eddb96Vignesh Venkatasubramanian                                           lfi->hev_thr, lfin->mblim, lfin->lim,
627da49e34c1fb5e99681f4ad99c21d9cfd83eddb96Vignesh Venkatasubramanian                                           lfin->hev_thr, bd);
628da49e34c1fb5e99681f4ad99c21d9cfd83eddb96Vignesh Venkatasubramanian
629da49e34c1fb5e99681f4ad99c21d9cfd83eddb96Vignesh Venkatasubramanian          if ((mask_4x4_int & 3) == 3) {
630da49e34c1fb5e99681f4ad99c21d9cfd83eddb96Vignesh Venkatasubramanian            vpx_highbd_lpf_horizontal_4_dual(s + 4 * pitch, pitch, lfi->mblim,
631da49e34c1fb5e99681f4ad99c21d9cfd83eddb96Vignesh Venkatasubramanian                                             lfi->lim, lfi->hev_thr,
632da49e34c1fb5e99681f4ad99c21d9cfd83eddb96Vignesh Venkatasubramanian                                             lfin->mblim, lfin->lim,
633da49e34c1fb5e99681f4ad99c21d9cfd83eddb96Vignesh Venkatasubramanian                                             lfin->hev_thr, bd);
634da49e34c1fb5e99681f4ad99c21d9cfd83eddb96Vignesh Venkatasubramanian          } else {
635da49e34c1fb5e99681f4ad99c21d9cfd83eddb96Vignesh Venkatasubramanian            if (mask_4x4_int & 1) {
636da49e34c1fb5e99681f4ad99c21d9cfd83eddb96Vignesh Venkatasubramanian              vpx_highbd_lpf_horizontal_4(s + 4 * pitch, pitch, lfi->mblim,
637da49e34c1fb5e99681f4ad99c21d9cfd83eddb96Vignesh Venkatasubramanian                                          lfi->lim, lfi->hev_thr, 1, bd);
638da49e34c1fb5e99681f4ad99c21d9cfd83eddb96Vignesh Venkatasubramanian            } else if (mask_4x4_int & 2) {
639da49e34c1fb5e99681f4ad99c21d9cfd83eddb96Vignesh Venkatasubramanian              vpx_highbd_lpf_horizontal_4(s + 8 + 4 * pitch, pitch, lfin->mblim,
640da49e34c1fb5e99681f4ad99c21d9cfd83eddb96Vignesh Venkatasubramanian                                          lfin->lim, lfin->hev_thr, 1, bd);
641da49e34c1fb5e99681f4ad99c21d9cfd83eddb96Vignesh Venkatasubramanian            }
642da49e34c1fb5e99681f4ad99c21d9cfd83eddb96Vignesh Venkatasubramanian          }
643da49e34c1fb5e99681f4ad99c21d9cfd83eddb96Vignesh Venkatasubramanian          count = 2;
644da49e34c1fb5e99681f4ad99c21d9cfd83eddb96Vignesh Venkatasubramanian        } else {
645da49e34c1fb5e99681f4ad99c21d9cfd83eddb96Vignesh Venkatasubramanian          vpx_highbd_lpf_horizontal_8(s, pitch, lfi->mblim, lfi->lim,
646da49e34c1fb5e99681f4ad99c21d9cfd83eddb96Vignesh Venkatasubramanian                                      lfi->hev_thr, 1, bd);
647da49e34c1fb5e99681f4ad99c21d9cfd83eddb96Vignesh Venkatasubramanian
648da49e34c1fb5e99681f4ad99c21d9cfd83eddb96Vignesh Venkatasubramanian          if (mask_4x4_int & 1) {
649da49e34c1fb5e99681f4ad99c21d9cfd83eddb96Vignesh Venkatasubramanian            vpx_highbd_lpf_horizontal_4(s + 4 * pitch, pitch, lfi->mblim,
650da49e34c1fb5e99681f4ad99c21d9cfd83eddb96Vignesh Venkatasubramanian                                        lfi->lim, lfi->hev_thr, 1, bd);
651da49e34c1fb5e99681f4ad99c21d9cfd83eddb96Vignesh Venkatasubramanian          }
652da49e34c1fb5e99681f4ad99c21d9cfd83eddb96Vignesh Venkatasubramanian        }
653da49e34c1fb5e99681f4ad99c21d9cfd83eddb96Vignesh Venkatasubramanian      } else if (mask_4x4 & 1) {
654da49e34c1fb5e99681f4ad99c21d9cfd83eddb96Vignesh Venkatasubramanian        if ((mask_4x4 & 3) == 3) {
655da49e34c1fb5e99681f4ad99c21d9cfd83eddb96Vignesh Venkatasubramanian          // Next block's thresholds.
656da49e34c1fb5e99681f4ad99c21d9cfd83eddb96Vignesh Venkatasubramanian          const loop_filter_thresh *lfin = lfi_n->lfthr + *(lfl + 1);
657da49e34c1fb5e99681f4ad99c21d9cfd83eddb96Vignesh Venkatasubramanian
658da49e34c1fb5e99681f4ad99c21d9cfd83eddb96Vignesh Venkatasubramanian          vpx_highbd_lpf_horizontal_4_dual(s, pitch, lfi->mblim, lfi->lim,
659da49e34c1fb5e99681f4ad99c21d9cfd83eddb96Vignesh Venkatasubramanian                                           lfi->hev_thr, lfin->mblim, lfin->lim,
660da49e34c1fb5e99681f4ad99c21d9cfd83eddb96Vignesh Venkatasubramanian                                           lfin->hev_thr, bd);
661da49e34c1fb5e99681f4ad99c21d9cfd83eddb96Vignesh Venkatasubramanian          if ((mask_4x4_int & 3) == 3) {
662da49e34c1fb5e99681f4ad99c21d9cfd83eddb96Vignesh Venkatasubramanian            vpx_highbd_lpf_horizontal_4_dual(s + 4 * pitch, pitch, lfi->mblim,
663da49e34c1fb5e99681f4ad99c21d9cfd83eddb96Vignesh Venkatasubramanian                                             lfi->lim, lfi->hev_thr,
664da49e34c1fb5e99681f4ad99c21d9cfd83eddb96Vignesh Venkatasubramanian                                             lfin->mblim, lfin->lim,
665da49e34c1fb5e99681f4ad99c21d9cfd83eddb96Vignesh Venkatasubramanian                                             lfin->hev_thr, bd);
666da49e34c1fb5e99681f4ad99c21d9cfd83eddb96Vignesh Venkatasubramanian          } else {
667da49e34c1fb5e99681f4ad99c21d9cfd83eddb96Vignesh Venkatasubramanian            if (mask_4x4_int & 1) {
668da49e34c1fb5e99681f4ad99c21d9cfd83eddb96Vignesh Venkatasubramanian              vpx_highbd_lpf_horizontal_4(s + 4 * pitch, pitch, lfi->mblim,
669da49e34c1fb5e99681f4ad99c21d9cfd83eddb96Vignesh Venkatasubramanian                                          lfi->lim, lfi->hev_thr, 1, bd);
670da49e34c1fb5e99681f4ad99c21d9cfd83eddb96Vignesh Venkatasubramanian            } else if (mask_4x4_int & 2) {
671da49e34c1fb5e99681f4ad99c21d9cfd83eddb96Vignesh Venkatasubramanian              vpx_highbd_lpf_horizontal_4(s + 8 + 4 * pitch, pitch, lfin->mblim,
672da49e34c1fb5e99681f4ad99c21d9cfd83eddb96Vignesh Venkatasubramanian                                          lfin->lim, lfin->hev_thr, 1, bd);
673da49e34c1fb5e99681f4ad99c21d9cfd83eddb96Vignesh Venkatasubramanian            }
674da49e34c1fb5e99681f4ad99c21d9cfd83eddb96Vignesh Venkatasubramanian          }
675da49e34c1fb5e99681f4ad99c21d9cfd83eddb96Vignesh Venkatasubramanian          count = 2;
676da49e34c1fb5e99681f4ad99c21d9cfd83eddb96Vignesh Venkatasubramanian        } else {
677da49e34c1fb5e99681f4ad99c21d9cfd83eddb96Vignesh Venkatasubramanian          vpx_highbd_lpf_horizontal_4(s, pitch, lfi->mblim, lfi->lim,
678da49e34c1fb5e99681f4ad99c21d9cfd83eddb96Vignesh Venkatasubramanian                                      lfi->hev_thr, 1, bd);
679da49e34c1fb5e99681f4ad99c21d9cfd83eddb96Vignesh Venkatasubramanian
680da49e34c1fb5e99681f4ad99c21d9cfd83eddb96Vignesh Venkatasubramanian          if (mask_4x4_int & 1) {
681da49e34c1fb5e99681f4ad99c21d9cfd83eddb96Vignesh Venkatasubramanian            vpx_highbd_lpf_horizontal_4(s + 4 * pitch, pitch, lfi->mblim,
682da49e34c1fb5e99681f4ad99c21d9cfd83eddb96Vignesh Venkatasubramanian                                        lfi->lim, lfi->hev_thr, 1, bd);
683da49e34c1fb5e99681f4ad99c21d9cfd83eddb96Vignesh Venkatasubramanian          }
684da49e34c1fb5e99681f4ad99c21d9cfd83eddb96Vignesh Venkatasubramanian        }
685da49e34c1fb5e99681f4ad99c21d9cfd83eddb96Vignesh Venkatasubramanian      } else if (mask_4x4_int & 1) {
686da49e34c1fb5e99681f4ad99c21d9cfd83eddb96Vignesh Venkatasubramanian        vpx_highbd_lpf_horizontal_4(s + 4 * pitch, pitch, lfi->mblim, lfi->lim,
687da49e34c1fb5e99681f4ad99c21d9cfd83eddb96Vignesh Venkatasubramanian                                    lfi->hev_thr, 1, bd);
688da49e34c1fb5e99681f4ad99c21d9cfd83eddb96Vignesh Venkatasubramanian      }
689da49e34c1fb5e99681f4ad99c21d9cfd83eddb96Vignesh Venkatasubramanian    }
690da49e34c1fb5e99681f4ad99c21d9cfd83eddb96Vignesh Venkatasubramanian    s += 8 * count;
691da49e34c1fb5e99681f4ad99c21d9cfd83eddb96Vignesh Venkatasubramanian    lfl += count;
692da49e34c1fb5e99681f4ad99c21d9cfd83eddb96Vignesh Venkatasubramanian    mask_16x16 >>= count;
693da49e34c1fb5e99681f4ad99c21d9cfd83eddb96Vignesh Venkatasubramanian    mask_8x8 >>= count;
694da49e34c1fb5e99681f4ad99c21d9cfd83eddb96Vignesh Venkatasubramanian    mask_4x4 >>= count;
695da49e34c1fb5e99681f4ad99c21d9cfd83eddb96Vignesh Venkatasubramanian    mask_4x4_int >>= count;
696da49e34c1fb5e99681f4ad99c21d9cfd83eddb96Vignesh Venkatasubramanian  }
697da49e34c1fb5e99681f4ad99c21d9cfd83eddb96Vignesh Venkatasubramanian}
698da49e34c1fb5e99681f4ad99c21d9cfd83eddb96Vignesh Venkatasubramanian#endif  // CONFIG_VP9_HIGHBITDEPTH
699da49e34c1fb5e99681f4ad99c21d9cfd83eddb96Vignesh Venkatasubramanian
7001184aebb761cbeac9124c37189a80a1a58f04b6bhkuang// This function ors into the current lfm structure, where to do loop
701ba6c59e9d7d7013b3906b6f4230b663422681848Vignesh Venkatasubramanian// filters for the specific mi we are looking at. It uses information
702ba6c59e9d7d7013b3906b6f4230b663422681848Vignesh Venkatasubramanian// including the block_size_type (32x16, 32x32, etc.), the transform size,
7031184aebb761cbeac9124c37189a80a1a58f04b6bhkuang// whether there were any coefficients encoded, and the loop filter strength
7041184aebb761cbeac9124c37189a80a1a58f04b6bhkuang// block we are currently looking at. Shift is used to position the
7051184aebb761cbeac9124c37189a80a1a58f04b6bhkuang// 1's we produce.
7061184aebb761cbeac9124c37189a80a1a58f04b6bhkuang// TODO(JBB) Need another function for different resolution color..
7071184aebb761cbeac9124c37189a80a1a58f04b6bhkuangstatic void build_masks(const loop_filter_info_n *const lfi_n,
7081184aebb761cbeac9124c37189a80a1a58f04b6bhkuang                        const MODE_INFO *mi, const int shift_y,
7091184aebb761cbeac9124c37189a80a1a58f04b6bhkuang                        const int shift_uv,
7101184aebb761cbeac9124c37189a80a1a58f04b6bhkuang                        LOOP_FILTER_MASK *lfm) {
7114fb68e5dd4e93c7599dc905d861de11ac39c5585hkuang  const MB_MODE_INFO *mbmi = &mi->mbmi;
7124fb68e5dd4e93c7599dc905d861de11ac39c5585hkuang  const BLOCK_SIZE block_size = mbmi->sb_type;
7134fb68e5dd4e93c7599dc905d861de11ac39c5585hkuang  const TX_SIZE tx_size_y = mbmi->tx_size;
714ba6c59e9d7d7013b3906b6f4230b663422681848Vignesh Venkatasubramanian  const TX_SIZE tx_size_uv = get_uv_tx_size_impl(tx_size_y, block_size, 1, 1);
7154fb68e5dd4e93c7599dc905d861de11ac39c5585hkuang  const int filter_level = get_filter_level(lfi_n, mbmi);
7164fb68e5dd4e93c7599dc905d861de11ac39c5585hkuang  uint64_t *const left_y = &lfm->left_y[tx_size_y];
7174fb68e5dd4e93c7599dc905d861de11ac39c5585hkuang  uint64_t *const above_y = &lfm->above_y[tx_size_y];
7184fb68e5dd4e93c7599dc905d861de11ac39c5585hkuang  uint64_t *const int_4x4_y = &lfm->int_4x4_y;
7194fb68e5dd4e93c7599dc905d861de11ac39c5585hkuang  uint16_t *const left_uv = &lfm->left_uv[tx_size_uv];
7204fb68e5dd4e93c7599dc905d861de11ac39c5585hkuang  uint16_t *const above_uv = &lfm->above_uv[tx_size_uv];
7214fb68e5dd4e93c7599dc905d861de11ac39c5585hkuang  uint16_t *const int_4x4_uv = &lfm->int_4x4_uv;
7229b35249446b07f40ac5fcc3205f2c048616efacchkuang  int i;
7231184aebb761cbeac9124c37189a80a1a58f04b6bhkuang
7241184aebb761cbeac9124c37189a80a1a58f04b6bhkuang  // If filter level is 0 we don't loop filter.
7259b35249446b07f40ac5fcc3205f2c048616efacchkuang  if (!filter_level) {
7261184aebb761cbeac9124c37189a80a1a58f04b6bhkuang    return;
7279b35249446b07f40ac5fcc3205f2c048616efacchkuang  } else {
7284fb68e5dd4e93c7599dc905d861de11ac39c5585hkuang    const int w = num_8x8_blocks_wide_lookup[block_size];
7294fb68e5dd4e93c7599dc905d861de11ac39c5585hkuang    const int h = num_8x8_blocks_high_lookup[block_size];
7309b35249446b07f40ac5fcc3205f2c048616efacchkuang    int index = shift_y;
7319b35249446b07f40ac5fcc3205f2c048616efacchkuang    for (i = 0; i < h; i++) {
732da49e34c1fb5e99681f4ad99c21d9cfd83eddb96Vignesh Venkatasubramanian      memset(&lfm->lfl_y[index], filter_level, w);
7339b35249446b07f40ac5fcc3205f2c048616efacchkuang      index += 8;
7349b35249446b07f40ac5fcc3205f2c048616efacchkuang    }
7359b35249446b07f40ac5fcc3205f2c048616efacchkuang  }
7361184aebb761cbeac9124c37189a80a1a58f04b6bhkuang
7371184aebb761cbeac9124c37189a80a1a58f04b6bhkuang  // These set 1 in the current block size for the block size edges.
738ba6c59e9d7d7013b3906b6f4230b663422681848Vignesh Venkatasubramanian  // For instance if the block size is 32x16, we'll set:
7391184aebb761cbeac9124c37189a80a1a58f04b6bhkuang  //    above =   1111
7401184aebb761cbeac9124c37189a80a1a58f04b6bhkuang  //              0000
7411184aebb761cbeac9124c37189a80a1a58f04b6bhkuang  //    and
7421184aebb761cbeac9124c37189a80a1a58f04b6bhkuang  //    left  =   1000
7431184aebb761cbeac9124c37189a80a1a58f04b6bhkuang  //          =   1000
7441184aebb761cbeac9124c37189a80a1a58f04b6bhkuang  // NOTE : In this example the low bit is left most ( 1000 ) is stored as
7451184aebb761cbeac9124c37189a80a1a58f04b6bhkuang  //        1,  not 8...
7461184aebb761cbeac9124c37189a80a1a58f04b6bhkuang  //
747ba6c59e9d7d7013b3906b6f4230b663422681848Vignesh Venkatasubramanian  // U and V set things on a 16 bit scale.
7481184aebb761cbeac9124c37189a80a1a58f04b6bhkuang  //
7491184aebb761cbeac9124c37189a80a1a58f04b6bhkuang  *above_y |= above_prediction_mask[block_size] << shift_y;
7501184aebb761cbeac9124c37189a80a1a58f04b6bhkuang  *above_uv |= above_prediction_mask_uv[block_size] << shift_uv;
7511184aebb761cbeac9124c37189a80a1a58f04b6bhkuang  *left_y |= left_prediction_mask[block_size] << shift_y;
7521184aebb761cbeac9124c37189a80a1a58f04b6bhkuang  *left_uv |= left_prediction_mask_uv[block_size] << shift_uv;
7531184aebb761cbeac9124c37189a80a1a58f04b6bhkuang
7541184aebb761cbeac9124c37189a80a1a58f04b6bhkuang  // If the block has no coefficients and is not intra we skip applying
7551184aebb761cbeac9124c37189a80a1a58f04b6bhkuang  // the loop filter on block edges.
7564fb68e5dd4e93c7599dc905d861de11ac39c5585hkuang  if (mbmi->skip && is_inter_block(mbmi))
7571184aebb761cbeac9124c37189a80a1a58f04b6bhkuang    return;
7581184aebb761cbeac9124c37189a80a1a58f04b6bhkuang
759ba6c59e9d7d7013b3906b6f4230b663422681848Vignesh Venkatasubramanian  // Here we are adding a mask for the transform size. The transform
7601184aebb761cbeac9124c37189a80a1a58f04b6bhkuang  // size mask is set to be correct for a 64x64 prediction block size. We
7611184aebb761cbeac9124c37189a80a1a58f04b6bhkuang  // mask to match the size of the block we are working on and then shift it
7621184aebb761cbeac9124c37189a80a1a58f04b6bhkuang  // into place..
7631184aebb761cbeac9124c37189a80a1a58f04b6bhkuang  *above_y |= (size_mask[block_size] &
7641184aebb761cbeac9124c37189a80a1a58f04b6bhkuang               above_64x64_txform_mask[tx_size_y]) << shift_y;
7651184aebb761cbeac9124c37189a80a1a58f04b6bhkuang  *above_uv |= (size_mask_uv[block_size] &
7661184aebb761cbeac9124c37189a80a1a58f04b6bhkuang                above_64x64_txform_mask_uv[tx_size_uv]) << shift_uv;
7671184aebb761cbeac9124c37189a80a1a58f04b6bhkuang
7681184aebb761cbeac9124c37189a80a1a58f04b6bhkuang  *left_y |= (size_mask[block_size] &
7691184aebb761cbeac9124c37189a80a1a58f04b6bhkuang              left_64x64_txform_mask[tx_size_y]) << shift_y;
7701184aebb761cbeac9124c37189a80a1a58f04b6bhkuang  *left_uv |= (size_mask_uv[block_size] &
7711184aebb761cbeac9124c37189a80a1a58f04b6bhkuang               left_64x64_txform_mask_uv[tx_size_uv]) << shift_uv;
7721184aebb761cbeac9124c37189a80a1a58f04b6bhkuang
7731184aebb761cbeac9124c37189a80a1a58f04b6bhkuang  // Here we are trying to determine what to do with the internal 4x4 block
7741184aebb761cbeac9124c37189a80a1a58f04b6bhkuang  // boundaries.  These differ from the 4x4 boundaries on the outside edge of
7751184aebb761cbeac9124c37189a80a1a58f04b6bhkuang  // an 8x8 in that the internal ones can be skipped and don't depend on
7761184aebb761cbeac9124c37189a80a1a58f04b6bhkuang  // the prediction block size.
7774fb68e5dd4e93c7599dc905d861de11ac39c5585hkuang  if (tx_size_y == TX_4X4)
778da49e34c1fb5e99681f4ad99c21d9cfd83eddb96Vignesh Venkatasubramanian    *int_4x4_y |= (size_mask[block_size] & 0xffffffffffffffffULL) << shift_y;
7794fb68e5dd4e93c7599dc905d861de11ac39c5585hkuang
7804fb68e5dd4e93c7599dc905d861de11ac39c5585hkuang  if (tx_size_uv == TX_4X4)
7811184aebb761cbeac9124c37189a80a1a58f04b6bhkuang    *int_4x4_uv |= (size_mask_uv[block_size] & 0xffff) << shift_uv;
7821184aebb761cbeac9124c37189a80a1a58f04b6bhkuang}
7831184aebb761cbeac9124c37189a80a1a58f04b6bhkuang
7841184aebb761cbeac9124c37189a80a1a58f04b6bhkuang// This function does the same thing as the one above with the exception that
785ba6c59e9d7d7013b3906b6f4230b663422681848Vignesh Venkatasubramanian// it only affects the y masks. It exists because for blocks < 16x16 in size,
7861184aebb761cbeac9124c37189a80a1a58f04b6bhkuang// we only update u and v masks on the first block.
7871184aebb761cbeac9124c37189a80a1a58f04b6bhkuangstatic void build_y_mask(const loop_filter_info_n *const lfi_n,
7881184aebb761cbeac9124c37189a80a1a58f04b6bhkuang                         const MODE_INFO *mi, const int shift_y,
7891184aebb761cbeac9124c37189a80a1a58f04b6bhkuang                         LOOP_FILTER_MASK *lfm) {
7904fb68e5dd4e93c7599dc905d861de11ac39c5585hkuang  const MB_MODE_INFO *mbmi = &mi->mbmi;
7914fb68e5dd4e93c7599dc905d861de11ac39c5585hkuang  const BLOCK_SIZE block_size = mbmi->sb_type;
7924fb68e5dd4e93c7599dc905d861de11ac39c5585hkuang  const TX_SIZE tx_size_y = mbmi->tx_size;
7934fb68e5dd4e93c7599dc905d861de11ac39c5585hkuang  const int filter_level = get_filter_level(lfi_n, mbmi);
7944fb68e5dd4e93c7599dc905d861de11ac39c5585hkuang  uint64_t *const left_y = &lfm->left_y[tx_size_y];
7954fb68e5dd4e93c7599dc905d861de11ac39c5585hkuang  uint64_t *const above_y = &lfm->above_y[tx_size_y];
7964fb68e5dd4e93c7599dc905d861de11ac39c5585hkuang  uint64_t *const int_4x4_y = &lfm->int_4x4_y;
7979b35249446b07f40ac5fcc3205f2c048616efacchkuang  int i;
7981184aebb761cbeac9124c37189a80a1a58f04b6bhkuang
7999b35249446b07f40ac5fcc3205f2c048616efacchkuang  if (!filter_level) {
8001184aebb761cbeac9124c37189a80a1a58f04b6bhkuang    return;
8019b35249446b07f40ac5fcc3205f2c048616efacchkuang  } else {
8024fb68e5dd4e93c7599dc905d861de11ac39c5585hkuang    const int w = num_8x8_blocks_wide_lookup[block_size];
8034fb68e5dd4e93c7599dc905d861de11ac39c5585hkuang    const int h = num_8x8_blocks_high_lookup[block_size];
8049b35249446b07f40ac5fcc3205f2c048616efacchkuang    int index = shift_y;
8059b35249446b07f40ac5fcc3205f2c048616efacchkuang    for (i = 0; i < h; i++) {
806da49e34c1fb5e99681f4ad99c21d9cfd83eddb96Vignesh Venkatasubramanian      memset(&lfm->lfl_y[index], filter_level, w);
8079b35249446b07f40ac5fcc3205f2c048616efacchkuang      index += 8;
8089b35249446b07f40ac5fcc3205f2c048616efacchkuang    }
8099b35249446b07f40ac5fcc3205f2c048616efacchkuang  }
8101184aebb761cbeac9124c37189a80a1a58f04b6bhkuang
8111184aebb761cbeac9124c37189a80a1a58f04b6bhkuang  *above_y |= above_prediction_mask[block_size] << shift_y;
8121184aebb761cbeac9124c37189a80a1a58f04b6bhkuang  *left_y |= left_prediction_mask[block_size] << shift_y;
8131184aebb761cbeac9124c37189a80a1a58f04b6bhkuang
8144fb68e5dd4e93c7599dc905d861de11ac39c5585hkuang  if (mbmi->skip && is_inter_block(mbmi))
8151184aebb761cbeac9124c37189a80a1a58f04b6bhkuang    return;
8161184aebb761cbeac9124c37189a80a1a58f04b6bhkuang
8171184aebb761cbeac9124c37189a80a1a58f04b6bhkuang  *above_y |= (size_mask[block_size] &
8181184aebb761cbeac9124c37189a80a1a58f04b6bhkuang               above_64x64_txform_mask[tx_size_y]) << shift_y;
8191184aebb761cbeac9124c37189a80a1a58f04b6bhkuang
8201184aebb761cbeac9124c37189a80a1a58f04b6bhkuang  *left_y |= (size_mask[block_size] &
8211184aebb761cbeac9124c37189a80a1a58f04b6bhkuang              left_64x64_txform_mask[tx_size_y]) << shift_y;
8221184aebb761cbeac9124c37189a80a1a58f04b6bhkuang
8234fb68e5dd4e93c7599dc905d861de11ac39c5585hkuang  if (tx_size_y == TX_4X4)
824da49e34c1fb5e99681f4ad99c21d9cfd83eddb96Vignesh Venkatasubramanian    *int_4x4_y |= (size_mask[block_size] & 0xffffffffffffffffULL) << shift_y;
8251184aebb761cbeac9124c37189a80a1a58f04b6bhkuang}
8261184aebb761cbeac9124c37189a80a1a58f04b6bhkuang
8271184aebb761cbeac9124c37189a80a1a58f04b6bhkuang// This function sets up the bit masks for the entire 64x64 region represented
8281184aebb761cbeac9124c37189a80a1a58f04b6bhkuang// by mi_row, mi_col.
8291184aebb761cbeac9124c37189a80a1a58f04b6bhkuang// TODO(JBB): This function only works for yv12.
8302ec72e65689c948e92b826ae1e867bf369e72f13Vignesh Venkatasubramanianvoid vp9_setup_mask(VP9_COMMON *const cm, const int mi_row, const int mi_col,
831ba6c59e9d7d7013b3906b6f4230b663422681848Vignesh Venkatasubramanian                    MODE_INFO **mi, const int mode_info_stride,
8322ec72e65689c948e92b826ae1e867bf369e72f13Vignesh Venkatasubramanian                    LOOP_FILTER_MASK *lfm) {
8331184aebb761cbeac9124c37189a80a1a58f04b6bhkuang  int idx_32, idx_16, idx_8;
8341184aebb761cbeac9124c37189a80a1a58f04b6bhkuang  const loop_filter_info_n *const lfi_n = &cm->lf_info;
835ba6c59e9d7d7013b3906b6f4230b663422681848Vignesh Venkatasubramanian  MODE_INFO **mip = mi;
836ba6c59e9d7d7013b3906b6f4230b663422681848Vignesh Venkatasubramanian  MODE_INFO **mip2 = mi;
8371184aebb761cbeac9124c37189a80a1a58f04b6bhkuang
8381184aebb761cbeac9124c37189a80a1a58f04b6bhkuang  // These are offsets to the next mi in the 64x64 block. It is what gets
839ba6c59e9d7d7013b3906b6f4230b663422681848Vignesh Venkatasubramanian  // added to the mi ptr as we go through each loop. It helps us to avoid
840ba6c59e9d7d7013b3906b6f4230b663422681848Vignesh Venkatasubramanian  // setting up special row and column counters for each index. The last step
8411184aebb761cbeac9124c37189a80a1a58f04b6bhkuang  // brings us out back to the starting position.
8421184aebb761cbeac9124c37189a80a1a58f04b6bhkuang  const int offset_32[] = {4, (mode_info_stride << 2) - 4, 4,
8431184aebb761cbeac9124c37189a80a1a58f04b6bhkuang                           -(mode_info_stride << 2) - 4};
8441184aebb761cbeac9124c37189a80a1a58f04b6bhkuang  const int offset_16[] = {2, (mode_info_stride << 1) - 2, 2,
8451184aebb761cbeac9124c37189a80a1a58f04b6bhkuang                           -(mode_info_stride << 1) - 2};
8461184aebb761cbeac9124c37189a80a1a58f04b6bhkuang  const int offset[] = {1, mode_info_stride - 1, 1, -mode_info_stride - 1};
8471184aebb761cbeac9124c37189a80a1a58f04b6bhkuang
8481184aebb761cbeac9124c37189a80a1a58f04b6bhkuang  // Following variables represent shifts to position the current block
849ba6c59e9d7d7013b3906b6f4230b663422681848Vignesh Venkatasubramanian  // mask over the appropriate block. A shift of 36 to the left will move
8501184aebb761cbeac9124c37189a80a1a58f04b6bhkuang  // the bits for the final 32 by 32 block in the 64x64 up 4 rows and left
8511184aebb761cbeac9124c37189a80a1a58f04b6bhkuang  // 4 rows to the appropriate spot.
8521184aebb761cbeac9124c37189a80a1a58f04b6bhkuang  const int shift_32_y[] = {0, 4, 32, 36};
8531184aebb761cbeac9124c37189a80a1a58f04b6bhkuang  const int shift_16_y[] = {0, 2, 16, 18};
8541184aebb761cbeac9124c37189a80a1a58f04b6bhkuang  const int shift_8_y[] = {0, 1, 8, 9};
8551184aebb761cbeac9124c37189a80a1a58f04b6bhkuang  const int shift_32_uv[] = {0, 2, 8, 10};
8561184aebb761cbeac9124c37189a80a1a58f04b6bhkuang  const int shift_16_uv[] = {0, 1, 4, 5};
8571184aebb761cbeac9124c37189a80a1a58f04b6bhkuang  int i;
8581184aebb761cbeac9124c37189a80a1a58f04b6bhkuang  const int max_rows = (mi_row + MI_BLOCK_SIZE > cm->mi_rows ?
8591184aebb761cbeac9124c37189a80a1a58f04b6bhkuang                        cm->mi_rows - mi_row : MI_BLOCK_SIZE);
8601184aebb761cbeac9124c37189a80a1a58f04b6bhkuang  const int max_cols = (mi_col + MI_BLOCK_SIZE > cm->mi_cols ?
8611184aebb761cbeac9124c37189a80a1a58f04b6bhkuang                        cm->mi_cols - mi_col : MI_BLOCK_SIZE);
8621184aebb761cbeac9124c37189a80a1a58f04b6bhkuang
8631184aebb761cbeac9124c37189a80a1a58f04b6bhkuang  vp9_zero(*lfm);
864ba6c59e9d7d7013b3906b6f4230b663422681848Vignesh Venkatasubramanian  assert(mip[0] != NULL);
8651184aebb761cbeac9124c37189a80a1a58f04b6bhkuang
8661184aebb761cbeac9124c37189a80a1a58f04b6bhkuang  // TODO(jimbankoski): Try moving most of the following code into decode
8671184aebb761cbeac9124c37189a80a1a58f04b6bhkuang  // loop and storing lfm in the mbmi structure so that we don't have to go
8681184aebb761cbeac9124c37189a80a1a58f04b6bhkuang  // through the recursive loop structure multiple times.
8691184aebb761cbeac9124c37189a80a1a58f04b6bhkuang  switch (mip[0]->mbmi.sb_type) {
8701184aebb761cbeac9124c37189a80a1a58f04b6bhkuang    case BLOCK_64X64:
8711184aebb761cbeac9124c37189a80a1a58f04b6bhkuang      build_masks(lfi_n, mip[0] , 0, 0, lfm);
8721184aebb761cbeac9124c37189a80a1a58f04b6bhkuang      break;
8731184aebb761cbeac9124c37189a80a1a58f04b6bhkuang    case BLOCK_64X32:
8741184aebb761cbeac9124c37189a80a1a58f04b6bhkuang      build_masks(lfi_n, mip[0], 0, 0, lfm);
8751184aebb761cbeac9124c37189a80a1a58f04b6bhkuang      mip2 = mip + mode_info_stride * 4;
8761184aebb761cbeac9124c37189a80a1a58f04b6bhkuang      if (4 >= max_rows)
8771184aebb761cbeac9124c37189a80a1a58f04b6bhkuang        break;
8781184aebb761cbeac9124c37189a80a1a58f04b6bhkuang      build_masks(lfi_n, mip2[0], 32, 8, lfm);
8791184aebb761cbeac9124c37189a80a1a58f04b6bhkuang      break;
8801184aebb761cbeac9124c37189a80a1a58f04b6bhkuang    case BLOCK_32X64:
8811184aebb761cbeac9124c37189a80a1a58f04b6bhkuang      build_masks(lfi_n, mip[0], 0, 0, lfm);
8821184aebb761cbeac9124c37189a80a1a58f04b6bhkuang      mip2 = mip + 4;
8831184aebb761cbeac9124c37189a80a1a58f04b6bhkuang      if (4 >= max_cols)
8841184aebb761cbeac9124c37189a80a1a58f04b6bhkuang        break;
8851184aebb761cbeac9124c37189a80a1a58f04b6bhkuang      build_masks(lfi_n, mip2[0], 4, 2, lfm);
8861184aebb761cbeac9124c37189a80a1a58f04b6bhkuang      break;
8871184aebb761cbeac9124c37189a80a1a58f04b6bhkuang    default:
8881184aebb761cbeac9124c37189a80a1a58f04b6bhkuang      for (idx_32 = 0; idx_32 < 4; mip += offset_32[idx_32], ++idx_32) {
8891184aebb761cbeac9124c37189a80a1a58f04b6bhkuang        const int shift_y = shift_32_y[idx_32];
8901184aebb761cbeac9124c37189a80a1a58f04b6bhkuang        const int shift_uv = shift_32_uv[idx_32];
8911184aebb761cbeac9124c37189a80a1a58f04b6bhkuang        const int mi_32_col_offset = ((idx_32 & 1) << 2);
8921184aebb761cbeac9124c37189a80a1a58f04b6bhkuang        const int mi_32_row_offset = ((idx_32 >> 1) << 2);
8931184aebb761cbeac9124c37189a80a1a58f04b6bhkuang        if (mi_32_col_offset >= max_cols || mi_32_row_offset >= max_rows)
8941184aebb761cbeac9124c37189a80a1a58f04b6bhkuang          continue;
8951184aebb761cbeac9124c37189a80a1a58f04b6bhkuang        switch (mip[0]->mbmi.sb_type) {
8961184aebb761cbeac9124c37189a80a1a58f04b6bhkuang          case BLOCK_32X32:
8971184aebb761cbeac9124c37189a80a1a58f04b6bhkuang            build_masks(lfi_n, mip[0], shift_y, shift_uv, lfm);
8981184aebb761cbeac9124c37189a80a1a58f04b6bhkuang            break;
8991184aebb761cbeac9124c37189a80a1a58f04b6bhkuang          case BLOCK_32X16:
9001184aebb761cbeac9124c37189a80a1a58f04b6bhkuang            build_masks(lfi_n, mip[0], shift_y, shift_uv, lfm);
9011184aebb761cbeac9124c37189a80a1a58f04b6bhkuang            if (mi_32_row_offset + 2 >= max_rows)
9021184aebb761cbeac9124c37189a80a1a58f04b6bhkuang              continue;
9031184aebb761cbeac9124c37189a80a1a58f04b6bhkuang            mip2 = mip + mode_info_stride * 2;
9041184aebb761cbeac9124c37189a80a1a58f04b6bhkuang            build_masks(lfi_n, mip2[0], shift_y + 16, shift_uv + 4, lfm);
9051184aebb761cbeac9124c37189a80a1a58f04b6bhkuang            break;
9061184aebb761cbeac9124c37189a80a1a58f04b6bhkuang          case BLOCK_16X32:
9071184aebb761cbeac9124c37189a80a1a58f04b6bhkuang            build_masks(lfi_n, mip[0], shift_y, shift_uv, lfm);
9081184aebb761cbeac9124c37189a80a1a58f04b6bhkuang            if (mi_32_col_offset + 2 >= max_cols)
9091184aebb761cbeac9124c37189a80a1a58f04b6bhkuang              continue;
9101184aebb761cbeac9124c37189a80a1a58f04b6bhkuang            mip2 = mip + 2;
9111184aebb761cbeac9124c37189a80a1a58f04b6bhkuang            build_masks(lfi_n, mip2[0], shift_y + 2, shift_uv + 1, lfm);
9121184aebb761cbeac9124c37189a80a1a58f04b6bhkuang            break;
9131184aebb761cbeac9124c37189a80a1a58f04b6bhkuang          default:
9141184aebb761cbeac9124c37189a80a1a58f04b6bhkuang            for (idx_16 = 0; idx_16 < 4; mip += offset_16[idx_16], ++idx_16) {
9151184aebb761cbeac9124c37189a80a1a58f04b6bhkuang              const int shift_y = shift_32_y[idx_32] + shift_16_y[idx_16];
9161184aebb761cbeac9124c37189a80a1a58f04b6bhkuang              const int shift_uv = shift_32_uv[idx_32] + shift_16_uv[idx_16];
9171184aebb761cbeac9124c37189a80a1a58f04b6bhkuang              const int mi_16_col_offset = mi_32_col_offset +
9181184aebb761cbeac9124c37189a80a1a58f04b6bhkuang                  ((idx_16 & 1) << 1);
9191184aebb761cbeac9124c37189a80a1a58f04b6bhkuang              const int mi_16_row_offset = mi_32_row_offset +
9201184aebb761cbeac9124c37189a80a1a58f04b6bhkuang                  ((idx_16 >> 1) << 1);
9211184aebb761cbeac9124c37189a80a1a58f04b6bhkuang
9221184aebb761cbeac9124c37189a80a1a58f04b6bhkuang              if (mi_16_col_offset >= max_cols || mi_16_row_offset >= max_rows)
9231184aebb761cbeac9124c37189a80a1a58f04b6bhkuang                continue;
9241184aebb761cbeac9124c37189a80a1a58f04b6bhkuang
9251184aebb761cbeac9124c37189a80a1a58f04b6bhkuang              switch (mip[0]->mbmi.sb_type) {
9261184aebb761cbeac9124c37189a80a1a58f04b6bhkuang                case BLOCK_16X16:
9271184aebb761cbeac9124c37189a80a1a58f04b6bhkuang                  build_masks(lfi_n, mip[0], shift_y, shift_uv, lfm);
9281184aebb761cbeac9124c37189a80a1a58f04b6bhkuang                  break;
9291184aebb761cbeac9124c37189a80a1a58f04b6bhkuang                case BLOCK_16X8:
9301184aebb761cbeac9124c37189a80a1a58f04b6bhkuang                  build_masks(lfi_n, mip[0], shift_y, shift_uv, lfm);
9311184aebb761cbeac9124c37189a80a1a58f04b6bhkuang                  if (mi_16_row_offset + 1 >= max_rows)
9321184aebb761cbeac9124c37189a80a1a58f04b6bhkuang                    continue;
9331184aebb761cbeac9124c37189a80a1a58f04b6bhkuang                  mip2 = mip + mode_info_stride;
9341184aebb761cbeac9124c37189a80a1a58f04b6bhkuang                  build_y_mask(lfi_n, mip2[0], shift_y+8, lfm);
9351184aebb761cbeac9124c37189a80a1a58f04b6bhkuang                  break;
9361184aebb761cbeac9124c37189a80a1a58f04b6bhkuang                case BLOCK_8X16:
9371184aebb761cbeac9124c37189a80a1a58f04b6bhkuang                  build_masks(lfi_n, mip[0], shift_y, shift_uv, lfm);
9381184aebb761cbeac9124c37189a80a1a58f04b6bhkuang                  if (mi_16_col_offset +1 >= max_cols)
9391184aebb761cbeac9124c37189a80a1a58f04b6bhkuang                    continue;
9401184aebb761cbeac9124c37189a80a1a58f04b6bhkuang                  mip2 = mip + 1;
9411184aebb761cbeac9124c37189a80a1a58f04b6bhkuang                  build_y_mask(lfi_n, mip2[0], shift_y+1, lfm);
9421184aebb761cbeac9124c37189a80a1a58f04b6bhkuang                  break;
9431184aebb761cbeac9124c37189a80a1a58f04b6bhkuang                default: {
9441184aebb761cbeac9124c37189a80a1a58f04b6bhkuang                  const int shift_y = shift_32_y[idx_32] +
9451184aebb761cbeac9124c37189a80a1a58f04b6bhkuang                                      shift_16_y[idx_16] +
9461184aebb761cbeac9124c37189a80a1a58f04b6bhkuang                                      shift_8_y[0];
9471184aebb761cbeac9124c37189a80a1a58f04b6bhkuang                  build_masks(lfi_n, mip[0], shift_y, shift_uv, lfm);
9481184aebb761cbeac9124c37189a80a1a58f04b6bhkuang                  mip += offset[0];
9491184aebb761cbeac9124c37189a80a1a58f04b6bhkuang                  for (idx_8 = 1; idx_8 < 4; mip += offset[idx_8], ++idx_8) {
9501184aebb761cbeac9124c37189a80a1a58f04b6bhkuang                    const int shift_y = shift_32_y[idx_32] +
9511184aebb761cbeac9124c37189a80a1a58f04b6bhkuang                                        shift_16_y[idx_16] +
9521184aebb761cbeac9124c37189a80a1a58f04b6bhkuang                                        shift_8_y[idx_8];
9531184aebb761cbeac9124c37189a80a1a58f04b6bhkuang                    const int mi_8_col_offset = mi_16_col_offset +
9541184aebb761cbeac9124c37189a80a1a58f04b6bhkuang                        ((idx_8 & 1));
9551184aebb761cbeac9124c37189a80a1a58f04b6bhkuang                    const int mi_8_row_offset = mi_16_row_offset +
9561184aebb761cbeac9124c37189a80a1a58f04b6bhkuang                        ((idx_8 >> 1));
9571184aebb761cbeac9124c37189a80a1a58f04b6bhkuang
9581184aebb761cbeac9124c37189a80a1a58f04b6bhkuang                    if (mi_8_col_offset >= max_cols ||
9591184aebb761cbeac9124c37189a80a1a58f04b6bhkuang                        mi_8_row_offset >= max_rows)
9601184aebb761cbeac9124c37189a80a1a58f04b6bhkuang                      continue;
9611184aebb761cbeac9124c37189a80a1a58f04b6bhkuang                    build_y_mask(lfi_n, mip[0], shift_y, lfm);
9621184aebb761cbeac9124c37189a80a1a58f04b6bhkuang                  }
9631184aebb761cbeac9124c37189a80a1a58f04b6bhkuang                  break;
9641184aebb761cbeac9124c37189a80a1a58f04b6bhkuang                }
9651184aebb761cbeac9124c37189a80a1a58f04b6bhkuang              }
9661184aebb761cbeac9124c37189a80a1a58f04b6bhkuang            }
9671184aebb761cbeac9124c37189a80a1a58f04b6bhkuang            break;
9681184aebb761cbeac9124c37189a80a1a58f04b6bhkuang        }
9691184aebb761cbeac9124c37189a80a1a58f04b6bhkuang      }
9701184aebb761cbeac9124c37189a80a1a58f04b6bhkuang      break;
9711184aebb761cbeac9124c37189a80a1a58f04b6bhkuang  }
9721184aebb761cbeac9124c37189a80a1a58f04b6bhkuang  // The largest loopfilter we have is 16x16 so we use the 16x16 mask
973da49e34c1fb5e99681f4ad99c21d9cfd83eddb96Vignesh Venkatasubramanian  // for 32x32 transforms also.
9741184aebb761cbeac9124c37189a80a1a58f04b6bhkuang  lfm->left_y[TX_16X16] |= lfm->left_y[TX_32X32];
9751184aebb761cbeac9124c37189a80a1a58f04b6bhkuang  lfm->above_y[TX_16X16] |= lfm->above_y[TX_32X32];
9761184aebb761cbeac9124c37189a80a1a58f04b6bhkuang  lfm->left_uv[TX_16X16] |= lfm->left_uv[TX_32X32];
9771184aebb761cbeac9124c37189a80a1a58f04b6bhkuang  lfm->above_uv[TX_16X16] |= lfm->above_uv[TX_32X32];
9781184aebb761cbeac9124c37189a80a1a58f04b6bhkuang
9791184aebb761cbeac9124c37189a80a1a58f04b6bhkuang  // We do at least 8 tap filter on every 32x32 even if the transform size
980ba6c59e9d7d7013b3906b6f4230b663422681848Vignesh Venkatasubramanian  // is 4x4. So if the 4x4 is set on a border pixel add it to the 8x8 and
9811184aebb761cbeac9124c37189a80a1a58f04b6bhkuang  // remove it from the 4x4.
9821184aebb761cbeac9124c37189a80a1a58f04b6bhkuang  lfm->left_y[TX_8X8] |= lfm->left_y[TX_4X4] & left_border;
9831184aebb761cbeac9124c37189a80a1a58f04b6bhkuang  lfm->left_y[TX_4X4] &= ~left_border;
9841184aebb761cbeac9124c37189a80a1a58f04b6bhkuang  lfm->above_y[TX_8X8] |= lfm->above_y[TX_4X4] & above_border;
9851184aebb761cbeac9124c37189a80a1a58f04b6bhkuang  lfm->above_y[TX_4X4] &= ~above_border;
9861184aebb761cbeac9124c37189a80a1a58f04b6bhkuang  lfm->left_uv[TX_8X8] |= lfm->left_uv[TX_4X4] & left_border_uv;
9871184aebb761cbeac9124c37189a80a1a58f04b6bhkuang  lfm->left_uv[TX_4X4] &= ~left_border_uv;
9881184aebb761cbeac9124c37189a80a1a58f04b6bhkuang  lfm->above_uv[TX_8X8] |= lfm->above_uv[TX_4X4] & above_border_uv;
9891184aebb761cbeac9124c37189a80a1a58f04b6bhkuang  lfm->above_uv[TX_4X4] &= ~above_border_uv;
9901184aebb761cbeac9124c37189a80a1a58f04b6bhkuang
9911184aebb761cbeac9124c37189a80a1a58f04b6bhkuang  // We do some special edge handling.
9921184aebb761cbeac9124c37189a80a1a58f04b6bhkuang  if (mi_row + MI_BLOCK_SIZE > cm->mi_rows) {
9931184aebb761cbeac9124c37189a80a1a58f04b6bhkuang    const uint64_t rows = cm->mi_rows - mi_row;
9941184aebb761cbeac9124c37189a80a1a58f04b6bhkuang
9951184aebb761cbeac9124c37189a80a1a58f04b6bhkuang    // Each pixel inside the border gets a 1,
9961184aebb761cbeac9124c37189a80a1a58f04b6bhkuang    const uint64_t mask_y = (((uint64_t) 1 << (rows << 3)) - 1);
9971184aebb761cbeac9124c37189a80a1a58f04b6bhkuang    const uint16_t mask_uv = (((uint16_t) 1 << (((rows + 1) >> 1) << 2)) - 1);
9981184aebb761cbeac9124c37189a80a1a58f04b6bhkuang
9991184aebb761cbeac9124c37189a80a1a58f04b6bhkuang    // Remove values completely outside our border.
10001184aebb761cbeac9124c37189a80a1a58f04b6bhkuang    for (i = 0; i < TX_32X32; i++) {
10011184aebb761cbeac9124c37189a80a1a58f04b6bhkuang      lfm->left_y[i] &= mask_y;
10021184aebb761cbeac9124c37189a80a1a58f04b6bhkuang      lfm->above_y[i] &= mask_y;
10031184aebb761cbeac9124c37189a80a1a58f04b6bhkuang      lfm->left_uv[i] &= mask_uv;
10041184aebb761cbeac9124c37189a80a1a58f04b6bhkuang      lfm->above_uv[i] &= mask_uv;
10051184aebb761cbeac9124c37189a80a1a58f04b6bhkuang    }
10061184aebb761cbeac9124c37189a80a1a58f04b6bhkuang    lfm->int_4x4_y &= mask_y;
10071184aebb761cbeac9124c37189a80a1a58f04b6bhkuang    lfm->int_4x4_uv &= mask_uv;
10081184aebb761cbeac9124c37189a80a1a58f04b6bhkuang
1009ba6c59e9d7d7013b3906b6f4230b663422681848Vignesh Venkatasubramanian    // We don't apply a wide loop filter on the last uv block row. If set
10101184aebb761cbeac9124c37189a80a1a58f04b6bhkuang    // apply the shorter one instead.
10111184aebb761cbeac9124c37189a80a1a58f04b6bhkuang    if (rows == 1) {
10121184aebb761cbeac9124c37189a80a1a58f04b6bhkuang      lfm->above_uv[TX_8X8] |= lfm->above_uv[TX_16X16];
10131184aebb761cbeac9124c37189a80a1a58f04b6bhkuang      lfm->above_uv[TX_16X16] = 0;
10141184aebb761cbeac9124c37189a80a1a58f04b6bhkuang    }
10151184aebb761cbeac9124c37189a80a1a58f04b6bhkuang    if (rows == 5) {
10161184aebb761cbeac9124c37189a80a1a58f04b6bhkuang      lfm->above_uv[TX_8X8] |= lfm->above_uv[TX_16X16] & 0xff00;
10171184aebb761cbeac9124c37189a80a1a58f04b6bhkuang      lfm->above_uv[TX_16X16] &= ~(lfm->above_uv[TX_16X16] & 0xff00);
10181184aebb761cbeac9124c37189a80a1a58f04b6bhkuang    }
10191184aebb761cbeac9124c37189a80a1a58f04b6bhkuang  }
10201184aebb761cbeac9124c37189a80a1a58f04b6bhkuang
10211184aebb761cbeac9124c37189a80a1a58f04b6bhkuang  if (mi_col + MI_BLOCK_SIZE > cm->mi_cols) {
10221184aebb761cbeac9124c37189a80a1a58f04b6bhkuang    const uint64_t columns = cm->mi_cols - mi_col;
10231184aebb761cbeac9124c37189a80a1a58f04b6bhkuang
10241184aebb761cbeac9124c37189a80a1a58f04b6bhkuang    // Each pixel inside the border gets a 1, the multiply copies the border
10251184aebb761cbeac9124c37189a80a1a58f04b6bhkuang    // to where we need it.
1026da49e34c1fb5e99681f4ad99c21d9cfd83eddb96Vignesh Venkatasubramanian    const uint64_t mask_y  = (((1 << columns) - 1)) * 0x0101010101010101ULL;
10271184aebb761cbeac9124c37189a80a1a58f04b6bhkuang    const uint16_t mask_uv = ((1 << ((columns + 1) >> 1)) - 1) * 0x1111;
10281184aebb761cbeac9124c37189a80a1a58f04b6bhkuang
10291184aebb761cbeac9124c37189a80a1a58f04b6bhkuang    // Internal edges are not applied on the last column of the image so
10301184aebb761cbeac9124c37189a80a1a58f04b6bhkuang    // we mask 1 more for the internal edges
10311184aebb761cbeac9124c37189a80a1a58f04b6bhkuang    const uint16_t mask_uv_int = ((1 << (columns >> 1)) - 1) * 0x1111;
10321184aebb761cbeac9124c37189a80a1a58f04b6bhkuang
10331184aebb761cbeac9124c37189a80a1a58f04b6bhkuang    // Remove the bits outside the image edge.
10341184aebb761cbeac9124c37189a80a1a58f04b6bhkuang    for (i = 0; i < TX_32X32; i++) {
10351184aebb761cbeac9124c37189a80a1a58f04b6bhkuang      lfm->left_y[i] &= mask_y;
10361184aebb761cbeac9124c37189a80a1a58f04b6bhkuang      lfm->above_y[i] &= mask_y;
10371184aebb761cbeac9124c37189a80a1a58f04b6bhkuang      lfm->left_uv[i] &= mask_uv;
10381184aebb761cbeac9124c37189a80a1a58f04b6bhkuang      lfm->above_uv[i] &= mask_uv;
10391184aebb761cbeac9124c37189a80a1a58f04b6bhkuang    }
10401184aebb761cbeac9124c37189a80a1a58f04b6bhkuang    lfm->int_4x4_y &= mask_y;
10411184aebb761cbeac9124c37189a80a1a58f04b6bhkuang    lfm->int_4x4_uv &= mask_uv_int;
10421184aebb761cbeac9124c37189a80a1a58f04b6bhkuang
1043ba6c59e9d7d7013b3906b6f4230b663422681848Vignesh Venkatasubramanian    // We don't apply a wide loop filter on the last uv column. If set
10441184aebb761cbeac9124c37189a80a1a58f04b6bhkuang    // apply the shorter one instead.
10451184aebb761cbeac9124c37189a80a1a58f04b6bhkuang    if (columns == 1) {
10461184aebb761cbeac9124c37189a80a1a58f04b6bhkuang      lfm->left_uv[TX_8X8] |= lfm->left_uv[TX_16X16];
10471184aebb761cbeac9124c37189a80a1a58f04b6bhkuang      lfm->left_uv[TX_16X16] = 0;
10481184aebb761cbeac9124c37189a80a1a58f04b6bhkuang    }
10491184aebb761cbeac9124c37189a80a1a58f04b6bhkuang    if (columns == 5) {
10501184aebb761cbeac9124c37189a80a1a58f04b6bhkuang      lfm->left_uv[TX_8X8] |= (lfm->left_uv[TX_16X16] & 0xcccc);
10511184aebb761cbeac9124c37189a80a1a58f04b6bhkuang      lfm->left_uv[TX_16X16] &= ~(lfm->left_uv[TX_16X16] & 0xcccc);
10521184aebb761cbeac9124c37189a80a1a58f04b6bhkuang    }
10531184aebb761cbeac9124c37189a80a1a58f04b6bhkuang  }
1054ba6c59e9d7d7013b3906b6f4230b663422681848Vignesh Venkatasubramanian  // We don't apply a loop filter on the first column in the image, mask that
1055ba6c59e9d7d7013b3906b6f4230b663422681848Vignesh Venkatasubramanian  // out.
10561184aebb761cbeac9124c37189a80a1a58f04b6bhkuang  if (mi_col == 0) {
10571184aebb761cbeac9124c37189a80a1a58f04b6bhkuang    for (i = 0; i < TX_32X32; i++) {
1058da49e34c1fb5e99681f4ad99c21d9cfd83eddb96Vignesh Venkatasubramanian      lfm->left_y[i] &= 0xfefefefefefefefeULL;
10591184aebb761cbeac9124c37189a80a1a58f04b6bhkuang      lfm->left_uv[i] &= 0xeeee;
10601184aebb761cbeac9124c37189a80a1a58f04b6bhkuang    }
10611184aebb761cbeac9124c37189a80a1a58f04b6bhkuang  }
10622ec72e65689c948e92b826ae1e867bf369e72f13Vignesh Venkatasubramanian
10632ec72e65689c948e92b826ae1e867bf369e72f13Vignesh Venkatasubramanian  // Assert if we try to apply 2 different loop filters at the same position.
10642ec72e65689c948e92b826ae1e867bf369e72f13Vignesh Venkatasubramanian  assert(!(lfm->left_y[TX_16X16] & lfm->left_y[TX_8X8]));
10652ec72e65689c948e92b826ae1e867bf369e72f13Vignesh Venkatasubramanian  assert(!(lfm->left_y[TX_16X16] & lfm->left_y[TX_4X4]));
10662ec72e65689c948e92b826ae1e867bf369e72f13Vignesh Venkatasubramanian  assert(!(lfm->left_y[TX_8X8] & lfm->left_y[TX_4X4]));
10672ec72e65689c948e92b826ae1e867bf369e72f13Vignesh Venkatasubramanian  assert(!(lfm->int_4x4_y & lfm->left_y[TX_16X16]));
10682ec72e65689c948e92b826ae1e867bf369e72f13Vignesh Venkatasubramanian  assert(!(lfm->left_uv[TX_16X16]&lfm->left_uv[TX_8X8]));
10692ec72e65689c948e92b826ae1e867bf369e72f13Vignesh Venkatasubramanian  assert(!(lfm->left_uv[TX_16X16] & lfm->left_uv[TX_4X4]));
10702ec72e65689c948e92b826ae1e867bf369e72f13Vignesh Venkatasubramanian  assert(!(lfm->left_uv[TX_8X8] & lfm->left_uv[TX_4X4]));
10712ec72e65689c948e92b826ae1e867bf369e72f13Vignesh Venkatasubramanian  assert(!(lfm->int_4x4_uv & lfm->left_uv[TX_16X16]));
10722ec72e65689c948e92b826ae1e867bf369e72f13Vignesh Venkatasubramanian  assert(!(lfm->above_y[TX_16X16] & lfm->above_y[TX_8X8]));
10732ec72e65689c948e92b826ae1e867bf369e72f13Vignesh Venkatasubramanian  assert(!(lfm->above_y[TX_16X16] & lfm->above_y[TX_4X4]));
10742ec72e65689c948e92b826ae1e867bf369e72f13Vignesh Venkatasubramanian  assert(!(lfm->above_y[TX_8X8] & lfm->above_y[TX_4X4]));
10752ec72e65689c948e92b826ae1e867bf369e72f13Vignesh Venkatasubramanian  assert(!(lfm->int_4x4_y & lfm->above_y[TX_16X16]));
10762ec72e65689c948e92b826ae1e867bf369e72f13Vignesh Venkatasubramanian  assert(!(lfm->above_uv[TX_16X16] & lfm->above_uv[TX_8X8]));
10772ec72e65689c948e92b826ae1e867bf369e72f13Vignesh Venkatasubramanian  assert(!(lfm->above_uv[TX_16X16] & lfm->above_uv[TX_4X4]));
10782ec72e65689c948e92b826ae1e867bf369e72f13Vignesh Venkatasubramanian  assert(!(lfm->above_uv[TX_8X8] & lfm->above_uv[TX_4X4]));
10792ec72e65689c948e92b826ae1e867bf369e72f13Vignesh Venkatasubramanian  assert(!(lfm->int_4x4_uv & lfm->above_uv[TX_16X16]));
10802ec72e65689c948e92b826ae1e867bf369e72f13Vignesh Venkatasubramanian}
10812ec72e65689c948e92b826ae1e867bf369e72f13Vignesh Venkatasubramanian
10822ec72e65689c948e92b826ae1e867bf369e72f13Vignesh Venkatasubramanianstatic void filter_selectively_vert(uint8_t *s, int pitch,
10832ec72e65689c948e92b826ae1e867bf369e72f13Vignesh Venkatasubramanian                                    unsigned int mask_16x16,
10842ec72e65689c948e92b826ae1e867bf369e72f13Vignesh Venkatasubramanian                                    unsigned int mask_8x8,
10852ec72e65689c948e92b826ae1e867bf369e72f13Vignesh Venkatasubramanian                                    unsigned int mask_4x4,
10862ec72e65689c948e92b826ae1e867bf369e72f13Vignesh Venkatasubramanian                                    unsigned int mask_4x4_int,
10872ec72e65689c948e92b826ae1e867bf369e72f13Vignesh Venkatasubramanian                                    const loop_filter_info_n *lfi_n,
10882ec72e65689c948e92b826ae1e867bf369e72f13Vignesh Venkatasubramanian                                    const uint8_t *lfl) {
10892ec72e65689c948e92b826ae1e867bf369e72f13Vignesh Venkatasubramanian  unsigned int mask;
10902ec72e65689c948e92b826ae1e867bf369e72f13Vignesh Venkatasubramanian
10912ec72e65689c948e92b826ae1e867bf369e72f13Vignesh Venkatasubramanian  for (mask = mask_16x16 | mask_8x8 | mask_4x4 | mask_4x4_int;
10922ec72e65689c948e92b826ae1e867bf369e72f13Vignesh Venkatasubramanian       mask; mask >>= 1) {
10932ec72e65689c948e92b826ae1e867bf369e72f13Vignesh Venkatasubramanian    const loop_filter_thresh *lfi = lfi_n->lfthr + *lfl;
10942ec72e65689c948e92b826ae1e867bf369e72f13Vignesh Venkatasubramanian
10952ec72e65689c948e92b826ae1e867bf369e72f13Vignesh Venkatasubramanian    if (mask & 1) {
10962ec72e65689c948e92b826ae1e867bf369e72f13Vignesh Venkatasubramanian      if (mask_16x16 & 1) {
1097da49e34c1fb5e99681f4ad99c21d9cfd83eddb96Vignesh Venkatasubramanian        vpx_lpf_vertical_16(s, pitch, lfi->mblim, lfi->lim, lfi->hev_thr);
1098da49e34c1fb5e99681f4ad99c21d9cfd83eddb96Vignesh Venkatasubramanian      } else if (mask_8x8 & 1) {
1099da49e34c1fb5e99681f4ad99c21d9cfd83eddb96Vignesh Venkatasubramanian        vpx_lpf_vertical_8(s, pitch, lfi->mblim, lfi->lim, lfi->hev_thr, 1);
1100da49e34c1fb5e99681f4ad99c21d9cfd83eddb96Vignesh Venkatasubramanian      } else if (mask_4x4 & 1) {
1101da49e34c1fb5e99681f4ad99c21d9cfd83eddb96Vignesh Venkatasubramanian        vpx_lpf_vertical_4(s, pitch, lfi->mblim, lfi->lim, lfi->hev_thr, 1);
1102da49e34c1fb5e99681f4ad99c21d9cfd83eddb96Vignesh Venkatasubramanian      }
1103da49e34c1fb5e99681f4ad99c21d9cfd83eddb96Vignesh Venkatasubramanian    }
1104da49e34c1fb5e99681f4ad99c21d9cfd83eddb96Vignesh Venkatasubramanian    if (mask_4x4_int & 1)
1105da49e34c1fb5e99681f4ad99c21d9cfd83eddb96Vignesh Venkatasubramanian      vpx_lpf_vertical_4(s + 4, pitch, lfi->mblim, lfi->lim, lfi->hev_thr, 1);
1106da49e34c1fb5e99681f4ad99c21d9cfd83eddb96Vignesh Venkatasubramanian    s += 8;
1107da49e34c1fb5e99681f4ad99c21d9cfd83eddb96Vignesh Venkatasubramanian    lfl += 1;
1108da49e34c1fb5e99681f4ad99c21d9cfd83eddb96Vignesh Venkatasubramanian    mask_16x16 >>= 1;
1109da49e34c1fb5e99681f4ad99c21d9cfd83eddb96Vignesh Venkatasubramanian    mask_8x8 >>= 1;
1110da49e34c1fb5e99681f4ad99c21d9cfd83eddb96Vignesh Venkatasubramanian    mask_4x4 >>= 1;
1111da49e34c1fb5e99681f4ad99c21d9cfd83eddb96Vignesh Venkatasubramanian    mask_4x4_int >>= 1;
1112da49e34c1fb5e99681f4ad99c21d9cfd83eddb96Vignesh Venkatasubramanian  }
1113da49e34c1fb5e99681f4ad99c21d9cfd83eddb96Vignesh Venkatasubramanian}
1114da49e34c1fb5e99681f4ad99c21d9cfd83eddb96Vignesh Venkatasubramanian
1115da49e34c1fb5e99681f4ad99c21d9cfd83eddb96Vignesh Venkatasubramanian#if CONFIG_VP9_HIGHBITDEPTH
1116da49e34c1fb5e99681f4ad99c21d9cfd83eddb96Vignesh Venkatasubramanianstatic void highbd_filter_selectively_vert(uint16_t *s, int pitch,
1117da49e34c1fb5e99681f4ad99c21d9cfd83eddb96Vignesh Venkatasubramanian                                           unsigned int mask_16x16,
1118da49e34c1fb5e99681f4ad99c21d9cfd83eddb96Vignesh Venkatasubramanian                                           unsigned int mask_8x8,
1119da49e34c1fb5e99681f4ad99c21d9cfd83eddb96Vignesh Venkatasubramanian                                           unsigned int mask_4x4,
1120da49e34c1fb5e99681f4ad99c21d9cfd83eddb96Vignesh Venkatasubramanian                                           unsigned int mask_4x4_int,
1121da49e34c1fb5e99681f4ad99c21d9cfd83eddb96Vignesh Venkatasubramanian                                           const loop_filter_info_n *lfi_n,
1122da49e34c1fb5e99681f4ad99c21d9cfd83eddb96Vignesh Venkatasubramanian                                           const uint8_t *lfl, int bd) {
1123da49e34c1fb5e99681f4ad99c21d9cfd83eddb96Vignesh Venkatasubramanian  unsigned int mask;
1124da49e34c1fb5e99681f4ad99c21d9cfd83eddb96Vignesh Venkatasubramanian
1125da49e34c1fb5e99681f4ad99c21d9cfd83eddb96Vignesh Venkatasubramanian  for (mask = mask_16x16 | mask_8x8 | mask_4x4 | mask_4x4_int;
1126da49e34c1fb5e99681f4ad99c21d9cfd83eddb96Vignesh Venkatasubramanian       mask; mask >>= 1) {
1127da49e34c1fb5e99681f4ad99c21d9cfd83eddb96Vignesh Venkatasubramanian    const loop_filter_thresh *lfi = lfi_n->lfthr + *lfl;
1128da49e34c1fb5e99681f4ad99c21d9cfd83eddb96Vignesh Venkatasubramanian
1129da49e34c1fb5e99681f4ad99c21d9cfd83eddb96Vignesh Venkatasubramanian    if (mask & 1) {
1130da49e34c1fb5e99681f4ad99c21d9cfd83eddb96Vignesh Venkatasubramanian      if (mask_16x16 & 1) {
1131da49e34c1fb5e99681f4ad99c21d9cfd83eddb96Vignesh Venkatasubramanian        vpx_highbd_lpf_vertical_16(s, pitch, lfi->mblim, lfi->lim,
1132da49e34c1fb5e99681f4ad99c21d9cfd83eddb96Vignesh Venkatasubramanian                                   lfi->hev_thr, bd);
11332ec72e65689c948e92b826ae1e867bf369e72f13Vignesh Venkatasubramanian      } else if (mask_8x8 & 1) {
1134da49e34c1fb5e99681f4ad99c21d9cfd83eddb96Vignesh Venkatasubramanian        vpx_highbd_lpf_vertical_8(s, pitch, lfi->mblim, lfi->lim,
1135da49e34c1fb5e99681f4ad99c21d9cfd83eddb96Vignesh Venkatasubramanian                                  lfi->hev_thr, 1, bd);
11362ec72e65689c948e92b826ae1e867bf369e72f13Vignesh Venkatasubramanian      } else if (mask_4x4 & 1) {
1137da49e34c1fb5e99681f4ad99c21d9cfd83eddb96Vignesh Venkatasubramanian        vpx_highbd_lpf_vertical_4(s, pitch, lfi->mblim, lfi->lim,
1138da49e34c1fb5e99681f4ad99c21d9cfd83eddb96Vignesh Venkatasubramanian                                lfi->hev_thr, 1, bd);
11392ec72e65689c948e92b826ae1e867bf369e72f13Vignesh Venkatasubramanian      }
11402ec72e65689c948e92b826ae1e867bf369e72f13Vignesh Venkatasubramanian    }
11412ec72e65689c948e92b826ae1e867bf369e72f13Vignesh Venkatasubramanian    if (mask_4x4_int & 1)
1142da49e34c1fb5e99681f4ad99c21d9cfd83eddb96Vignesh Venkatasubramanian      vpx_highbd_lpf_vertical_4(s + 4, pitch, lfi->mblim, lfi->lim,
1143da49e34c1fb5e99681f4ad99c21d9cfd83eddb96Vignesh Venkatasubramanian                                lfi->hev_thr, 1, bd);
11442ec72e65689c948e92b826ae1e867bf369e72f13Vignesh Venkatasubramanian    s += 8;
11452ec72e65689c948e92b826ae1e867bf369e72f13Vignesh Venkatasubramanian    lfl += 1;
11462ec72e65689c948e92b826ae1e867bf369e72f13Vignesh Venkatasubramanian    mask_16x16 >>= 1;
11472ec72e65689c948e92b826ae1e867bf369e72f13Vignesh Venkatasubramanian    mask_8x8 >>= 1;
11482ec72e65689c948e92b826ae1e867bf369e72f13Vignesh Venkatasubramanian    mask_4x4 >>= 1;
11492ec72e65689c948e92b826ae1e867bf369e72f13Vignesh Venkatasubramanian    mask_4x4_int >>= 1;
11502ec72e65689c948e92b826ae1e867bf369e72f13Vignesh Venkatasubramanian  }
11511184aebb761cbeac9124c37189a80a1a58f04b6bhkuang}
1152da49e34c1fb5e99681f4ad99c21d9cfd83eddb96Vignesh Venkatasubramanian#endif  // CONFIG_VP9_HIGHBITDEPTH
11539b35249446b07f40ac5fcc3205f2c048616efacchkuang
1154da49e34c1fb5e99681f4ad99c21d9cfd83eddb96Vignesh Venkatasubramanianvoid vp9_filter_block_plane_non420(VP9_COMMON *cm,
1155da49e34c1fb5e99681f4ad99c21d9cfd83eddb96Vignesh Venkatasubramanian                                   struct macroblockd_plane *plane,
1156da49e34c1fb5e99681f4ad99c21d9cfd83eddb96Vignesh Venkatasubramanian                                   MODE_INFO **mi_8x8,
1157da49e34c1fb5e99681f4ad99c21d9cfd83eddb96Vignesh Venkatasubramanian                                   int mi_row, int mi_col) {
115891037db265ecdd914a26e056cf69207b4f50924ehkuang  const int ss_x = plane->subsampling_x;
115991037db265ecdd914a26e056cf69207b4f50924ehkuang  const int ss_y = plane->subsampling_y;
1160da49e34c1fb5e99681f4ad99c21d9cfd83eddb96Vignesh Venkatasubramanian  const int row_step = 1 << ss_y;
1161da49e34c1fb5e99681f4ad99c21d9cfd83eddb96Vignesh Venkatasubramanian  const int col_step = 1 << ss_x;
11624fb68e5dd4e93c7599dc905d861de11ac39c5585hkuang  const int row_step_stride = cm->mi_stride * row_step;
116391037db265ecdd914a26e056cf69207b4f50924ehkuang  struct buf_2d *const dst = &plane->dst;
1164ba164dffc5a6795bce97fae02b51ccf3330e15e4hkuang  uint8_t* const dst0 = dst->buf;
116591037db265ecdd914a26e056cf69207b4f50924ehkuang  unsigned int mask_16x16[MI_BLOCK_SIZE] = {0};
116691037db265ecdd914a26e056cf69207b4f50924ehkuang  unsigned int mask_8x8[MI_BLOCK_SIZE] = {0};
116791037db265ecdd914a26e056cf69207b4f50924ehkuang  unsigned int mask_4x4[MI_BLOCK_SIZE] = {0};
116891037db265ecdd914a26e056cf69207b4f50924ehkuang  unsigned int mask_4x4_int[MI_BLOCK_SIZE] = {0};
11699b35249446b07f40ac5fcc3205f2c048616efacchkuang  uint8_t lfl[MI_BLOCK_SIZE * MI_BLOCK_SIZE];
1170ba164dffc5a6795bce97fae02b51ccf3330e15e4hkuang  int r, c;
1171ba164dffc5a6795bce97fae02b51ccf3330e15e4hkuang
117291037db265ecdd914a26e056cf69207b4f50924ehkuang  for (r = 0; r < MI_BLOCK_SIZE && mi_row + r < cm->mi_rows; r += row_step) {
1173ba164dffc5a6795bce97fae02b51ccf3330e15e4hkuang    unsigned int mask_16x16_c = 0;
1174ba164dffc5a6795bce97fae02b51ccf3330e15e4hkuang    unsigned int mask_8x8_c = 0;
1175ba164dffc5a6795bce97fae02b51ccf3330e15e4hkuang    unsigned int mask_4x4_c = 0;
1176ba164dffc5a6795bce97fae02b51ccf3330e15e4hkuang    unsigned int border_mask;
1177ba164dffc5a6795bce97fae02b51ccf3330e15e4hkuang
1178ba164dffc5a6795bce97fae02b51ccf3330e15e4hkuang    // Determine the vertical edges that need filtering
117991037db265ecdd914a26e056cf69207b4f50924ehkuang    for (c = 0; c < MI_BLOCK_SIZE && mi_col + c < cm->mi_cols; c += col_step) {
11801184aebb761cbeac9124c37189a80a1a58f04b6bhkuang      const MODE_INFO *mi = mi_8x8[c];
11812ec72e65689c948e92b826ae1e867bf369e72f13Vignesh Venkatasubramanian      const BLOCK_SIZE sb_type = mi[0].mbmi.sb_type;
11822ec72e65689c948e92b826ae1e867bf369e72f13Vignesh Venkatasubramanian      const int skip_this = mi[0].mbmi.skip && is_inter_block(&mi[0].mbmi);
1183ba164dffc5a6795bce97fae02b51ccf3330e15e4hkuang      // left edge of current unit is block/partition edge -> no skip
11842ec72e65689c948e92b826ae1e867bf369e72f13Vignesh Venkatasubramanian      const int block_edge_left = (num_4x4_blocks_wide_lookup[sb_type] > 1) ?
11852ec72e65689c948e92b826ae1e867bf369e72f13Vignesh Venkatasubramanian          !(c & (num_8x8_blocks_wide_lookup[sb_type] - 1)) : 1;
1186ba164dffc5a6795bce97fae02b51ccf3330e15e4hkuang      const int skip_this_c = skip_this && !block_edge_left;
1187ba164dffc5a6795bce97fae02b51ccf3330e15e4hkuang      // top edge of current unit is block/partition edge -> no skip
11882ec72e65689c948e92b826ae1e867bf369e72f13Vignesh Venkatasubramanian      const int block_edge_above = (num_4x4_blocks_high_lookup[sb_type] > 1) ?
11892ec72e65689c948e92b826ae1e867bf369e72f13Vignesh Venkatasubramanian          !(r & (num_8x8_blocks_high_lookup[sb_type] - 1)) : 1;
1190ba164dffc5a6795bce97fae02b51ccf3330e15e4hkuang      const int skip_this_r = skip_this && !block_edge_above;
119191037db265ecdd914a26e056cf69207b4f50924ehkuang      const TX_SIZE tx_size = (plane->plane_type == PLANE_TYPE_UV)
1192ba6c59e9d7d7013b3906b6f4230b663422681848Vignesh Venkatasubramanian                            ? get_uv_tx_size(&mi[0].mbmi, plane)
11931184aebb761cbeac9124c37189a80a1a58f04b6bhkuang                            : mi[0].mbmi.tx_size;
1194ba164dffc5a6795bce97fae02b51ccf3330e15e4hkuang      const int skip_border_4x4_c = ss_x && mi_col + c == cm->mi_cols - 1;
1195ba164dffc5a6795bce97fae02b51ccf3330e15e4hkuang      const int skip_border_4x4_r = ss_y && mi_row + r == cm->mi_rows - 1;
1196ba164dffc5a6795bce97fae02b51ccf3330e15e4hkuang
1197ba164dffc5a6795bce97fae02b51ccf3330e15e4hkuang      // Filter level can vary per MI
11989b35249446b07f40ac5fcc3205f2c048616efacchkuang      if (!(lfl[(r << 3) + (c >> ss_x)] =
11994fb68e5dd4e93c7599dc905d861de11ac39c5585hkuang            get_filter_level(&cm->lf_info, &mi[0].mbmi)))
1200ba164dffc5a6795bce97fae02b51ccf3330e15e4hkuang        continue;
1201ba164dffc5a6795bce97fae02b51ccf3330e15e4hkuang
1202ba164dffc5a6795bce97fae02b51ccf3330e15e4hkuang      // Build masks based on the transform size of each block
1203ba164dffc5a6795bce97fae02b51ccf3330e15e4hkuang      if (tx_size == TX_32X32) {
1204ba164dffc5a6795bce97fae02b51ccf3330e15e4hkuang        if (!skip_this_c && ((c >> ss_x) & 3) == 0) {
1205ba164dffc5a6795bce97fae02b51ccf3330e15e4hkuang          if (!skip_border_4x4_c)
1206ba164dffc5a6795bce97fae02b51ccf3330e15e4hkuang            mask_16x16_c |= 1 << (c >> ss_x);
1207ba164dffc5a6795bce97fae02b51ccf3330e15e4hkuang          else
1208ba164dffc5a6795bce97fae02b51ccf3330e15e4hkuang            mask_8x8_c |= 1 << (c >> ss_x);
1209ba164dffc5a6795bce97fae02b51ccf3330e15e4hkuang        }
1210ba164dffc5a6795bce97fae02b51ccf3330e15e4hkuang        if (!skip_this_r && ((r >> ss_y) & 3) == 0) {
1211ba164dffc5a6795bce97fae02b51ccf3330e15e4hkuang          if (!skip_border_4x4_r)
1212ba164dffc5a6795bce97fae02b51ccf3330e15e4hkuang            mask_16x16[r] |= 1 << (c >> ss_x);
1213ba164dffc5a6795bce97fae02b51ccf3330e15e4hkuang          else
1214ba164dffc5a6795bce97fae02b51ccf3330e15e4hkuang            mask_8x8[r] |= 1 << (c >> ss_x);
1215ba164dffc5a6795bce97fae02b51ccf3330e15e4hkuang        }
1216ba164dffc5a6795bce97fae02b51ccf3330e15e4hkuang      } else if (tx_size == TX_16X16) {
1217ba164dffc5a6795bce97fae02b51ccf3330e15e4hkuang        if (!skip_this_c && ((c >> ss_x) & 1) == 0) {
1218ba164dffc5a6795bce97fae02b51ccf3330e15e4hkuang          if (!skip_border_4x4_c)
1219ba164dffc5a6795bce97fae02b51ccf3330e15e4hkuang            mask_16x16_c |= 1 << (c >> ss_x);
1220ba164dffc5a6795bce97fae02b51ccf3330e15e4hkuang          else
1221ba164dffc5a6795bce97fae02b51ccf3330e15e4hkuang            mask_8x8_c |= 1 << (c >> ss_x);
1222ba164dffc5a6795bce97fae02b51ccf3330e15e4hkuang        }
1223ba164dffc5a6795bce97fae02b51ccf3330e15e4hkuang        if (!skip_this_r && ((r >> ss_y) & 1) == 0) {
1224ba164dffc5a6795bce97fae02b51ccf3330e15e4hkuang          if (!skip_border_4x4_r)
1225ba164dffc5a6795bce97fae02b51ccf3330e15e4hkuang            mask_16x16[r] |= 1 << (c >> ss_x);
1226ba164dffc5a6795bce97fae02b51ccf3330e15e4hkuang          else
1227ba164dffc5a6795bce97fae02b51ccf3330e15e4hkuang            mask_8x8[r] |= 1 << (c >> ss_x);
1228ba164dffc5a6795bce97fae02b51ccf3330e15e4hkuang        }
1229ba164dffc5a6795bce97fae02b51ccf3330e15e4hkuang      } else {
1230ba164dffc5a6795bce97fae02b51ccf3330e15e4hkuang        // force 8x8 filtering on 32x32 boundaries
1231ba164dffc5a6795bce97fae02b51ccf3330e15e4hkuang        if (!skip_this_c) {
1232ba164dffc5a6795bce97fae02b51ccf3330e15e4hkuang          if (tx_size == TX_8X8 || ((c >> ss_x) & 3) == 0)
1233ba164dffc5a6795bce97fae02b51ccf3330e15e4hkuang            mask_8x8_c |= 1 << (c >> ss_x);
1234ba164dffc5a6795bce97fae02b51ccf3330e15e4hkuang          else
1235ba164dffc5a6795bce97fae02b51ccf3330e15e4hkuang            mask_4x4_c |= 1 << (c >> ss_x);
1236ba164dffc5a6795bce97fae02b51ccf3330e15e4hkuang        }
1237ba164dffc5a6795bce97fae02b51ccf3330e15e4hkuang
1238ba164dffc5a6795bce97fae02b51ccf3330e15e4hkuang        if (!skip_this_r) {
1239ba164dffc5a6795bce97fae02b51ccf3330e15e4hkuang          if (tx_size == TX_8X8 || ((r >> ss_y) & 3) == 0)
1240ba164dffc5a6795bce97fae02b51ccf3330e15e4hkuang            mask_8x8[r] |= 1 << (c >> ss_x);
1241ba164dffc5a6795bce97fae02b51ccf3330e15e4hkuang          else
1242ba164dffc5a6795bce97fae02b51ccf3330e15e4hkuang            mask_4x4[r] |= 1 << (c >> ss_x);
1243ba164dffc5a6795bce97fae02b51ccf3330e15e4hkuang        }
1244ba164dffc5a6795bce97fae02b51ccf3330e15e4hkuang
1245ba164dffc5a6795bce97fae02b51ccf3330e15e4hkuang        if (!skip_this && tx_size < TX_8X8 && !skip_border_4x4_c)
1246ba164dffc5a6795bce97fae02b51ccf3330e15e4hkuang          mask_4x4_int[r] |= 1 << (c >> ss_x);
1247ba164dffc5a6795bce97fae02b51ccf3330e15e4hkuang      }
1248ba164dffc5a6795bce97fae02b51ccf3330e15e4hkuang    }
1249ba164dffc5a6795bce97fae02b51ccf3330e15e4hkuang
1250ba164dffc5a6795bce97fae02b51ccf3330e15e4hkuang    // Disable filtering on the leftmost column
1251ba164dffc5a6795bce97fae02b51ccf3330e15e4hkuang    border_mask = ~(mi_col == 0);
1252da49e34c1fb5e99681f4ad99c21d9cfd83eddb96Vignesh Venkatasubramanian#if CONFIG_VP9_HIGHBITDEPTH
1253da49e34c1fb5e99681f4ad99c21d9cfd83eddb96Vignesh Venkatasubramanian    if (cm->use_highbitdepth) {
1254da49e34c1fb5e99681f4ad99c21d9cfd83eddb96Vignesh Venkatasubramanian      highbd_filter_selectively_vert(CONVERT_TO_SHORTPTR(dst->buf),
1255da49e34c1fb5e99681f4ad99c21d9cfd83eddb96Vignesh Venkatasubramanian                                     dst->stride,
1256da49e34c1fb5e99681f4ad99c21d9cfd83eddb96Vignesh Venkatasubramanian                                     mask_16x16_c & border_mask,
1257da49e34c1fb5e99681f4ad99c21d9cfd83eddb96Vignesh Venkatasubramanian                                     mask_8x8_c & border_mask,
1258da49e34c1fb5e99681f4ad99c21d9cfd83eddb96Vignesh Venkatasubramanian                                     mask_4x4_c & border_mask,
1259da49e34c1fb5e99681f4ad99c21d9cfd83eddb96Vignesh Venkatasubramanian                                     mask_4x4_int[r],
1260da49e34c1fb5e99681f4ad99c21d9cfd83eddb96Vignesh Venkatasubramanian                                     &cm->lf_info, &lfl[r << 3],
1261da49e34c1fb5e99681f4ad99c21d9cfd83eddb96Vignesh Venkatasubramanian                                     (int)cm->bit_depth);
1262da49e34c1fb5e99681f4ad99c21d9cfd83eddb96Vignesh Venkatasubramanian    } else {
1263da49e34c1fb5e99681f4ad99c21d9cfd83eddb96Vignesh Venkatasubramanian      filter_selectively_vert(dst->buf, dst->stride,
1264da49e34c1fb5e99681f4ad99c21d9cfd83eddb96Vignesh Venkatasubramanian                              mask_16x16_c & border_mask,
1265da49e34c1fb5e99681f4ad99c21d9cfd83eddb96Vignesh Venkatasubramanian                              mask_8x8_c & border_mask,
1266da49e34c1fb5e99681f4ad99c21d9cfd83eddb96Vignesh Venkatasubramanian                              mask_4x4_c & border_mask,
1267da49e34c1fb5e99681f4ad99c21d9cfd83eddb96Vignesh Venkatasubramanian                              mask_4x4_int[r],
1268da49e34c1fb5e99681f4ad99c21d9cfd83eddb96Vignesh Venkatasubramanian                              &cm->lf_info, &lfl[r << 3]);
1269da49e34c1fb5e99681f4ad99c21d9cfd83eddb96Vignesh Venkatasubramanian    }
1270da49e34c1fb5e99681f4ad99c21d9cfd83eddb96Vignesh Venkatasubramanian#else
1271ba164dffc5a6795bce97fae02b51ccf3330e15e4hkuang    filter_selectively_vert(dst->buf, dst->stride,
1272ba164dffc5a6795bce97fae02b51ccf3330e15e4hkuang                            mask_16x16_c & border_mask,
1273ba164dffc5a6795bce97fae02b51ccf3330e15e4hkuang                            mask_8x8_c & border_mask,
1274ba164dffc5a6795bce97fae02b51ccf3330e15e4hkuang                            mask_4x4_c & border_mask,
12759b35249446b07f40ac5fcc3205f2c048616efacchkuang                            mask_4x4_int[r],
12769b35249446b07f40ac5fcc3205f2c048616efacchkuang                            &cm->lf_info, &lfl[r << 3]);
1277da49e34c1fb5e99681f4ad99c21d9cfd83eddb96Vignesh Venkatasubramanian#endif  // CONFIG_VP9_HIGHBITDEPTH
1278ba164dffc5a6795bce97fae02b51ccf3330e15e4hkuang    dst->buf += 8 * dst->stride;
12791184aebb761cbeac9124c37189a80a1a58f04b6bhkuang    mi_8x8 += row_step_stride;
1280ba164dffc5a6795bce97fae02b51ccf3330e15e4hkuang  }
1281ba164dffc5a6795bce97fae02b51ccf3330e15e4hkuang
1282ba164dffc5a6795bce97fae02b51ccf3330e15e4hkuang  // Now do horizontal pass
1283ba164dffc5a6795bce97fae02b51ccf3330e15e4hkuang  dst->buf = dst0;
128491037db265ecdd914a26e056cf69207b4f50924ehkuang  for (r = 0; r < MI_BLOCK_SIZE && mi_row + r < cm->mi_rows; r += row_step) {
1285ba164dffc5a6795bce97fae02b51ccf3330e15e4hkuang    const int skip_border_4x4_r = ss_y && mi_row + r == cm->mi_rows - 1;
1286ba164dffc5a6795bce97fae02b51ccf3330e15e4hkuang    const unsigned int mask_4x4_int_r = skip_border_4x4_r ? 0 : mask_4x4_int[r];
1287ba164dffc5a6795bce97fae02b51ccf3330e15e4hkuang
12889b35249446b07f40ac5fcc3205f2c048616efacchkuang    unsigned int mask_16x16_r;
12899b35249446b07f40ac5fcc3205f2c048616efacchkuang    unsigned int mask_8x8_r;
12909b35249446b07f40ac5fcc3205f2c048616efacchkuang    unsigned int mask_4x4_r;
12919b35249446b07f40ac5fcc3205f2c048616efacchkuang
12929b35249446b07f40ac5fcc3205f2c048616efacchkuang    if (mi_row + r == 0) {
12939b35249446b07f40ac5fcc3205f2c048616efacchkuang      mask_16x16_r = 0;
12949b35249446b07f40ac5fcc3205f2c048616efacchkuang      mask_8x8_r = 0;
12959b35249446b07f40ac5fcc3205f2c048616efacchkuang      mask_4x4_r = 0;
12969b35249446b07f40ac5fcc3205f2c048616efacchkuang    } else {
12979b35249446b07f40ac5fcc3205f2c048616efacchkuang      mask_16x16_r = mask_16x16[r];
12989b35249446b07f40ac5fcc3205f2c048616efacchkuang      mask_8x8_r = mask_8x8[r];
12999b35249446b07f40ac5fcc3205f2c048616efacchkuang      mask_4x4_r = mask_4x4[r];
13009b35249446b07f40ac5fcc3205f2c048616efacchkuang    }
1301da49e34c1fb5e99681f4ad99c21d9cfd83eddb96Vignesh Venkatasubramanian#if CONFIG_VP9_HIGHBITDEPTH
1302da49e34c1fb5e99681f4ad99c21d9cfd83eddb96Vignesh Venkatasubramanian    if (cm->use_highbitdepth) {
1303da49e34c1fb5e99681f4ad99c21d9cfd83eddb96Vignesh Venkatasubramanian      highbd_filter_selectively_horiz(CONVERT_TO_SHORTPTR(dst->buf),
1304da49e34c1fb5e99681f4ad99c21d9cfd83eddb96Vignesh Venkatasubramanian                                      dst->stride,
1305da49e34c1fb5e99681f4ad99c21d9cfd83eddb96Vignesh Venkatasubramanian                                      mask_16x16_r,
1306da49e34c1fb5e99681f4ad99c21d9cfd83eddb96Vignesh Venkatasubramanian                                      mask_8x8_r,
1307da49e34c1fb5e99681f4ad99c21d9cfd83eddb96Vignesh Venkatasubramanian                                      mask_4x4_r,
1308da49e34c1fb5e99681f4ad99c21d9cfd83eddb96Vignesh Venkatasubramanian                                      mask_4x4_int_r,
1309da49e34c1fb5e99681f4ad99c21d9cfd83eddb96Vignesh Venkatasubramanian                                      &cm->lf_info, &lfl[r << 3],
1310da49e34c1fb5e99681f4ad99c21d9cfd83eddb96Vignesh Venkatasubramanian                                      (int)cm->bit_depth);
1311da49e34c1fb5e99681f4ad99c21d9cfd83eddb96Vignesh Venkatasubramanian    } else {
1312da49e34c1fb5e99681f4ad99c21d9cfd83eddb96Vignesh Venkatasubramanian      filter_selectively_horiz(dst->buf, dst->stride,
1313da49e34c1fb5e99681f4ad99c21d9cfd83eddb96Vignesh Venkatasubramanian                               mask_16x16_r,
1314da49e34c1fb5e99681f4ad99c21d9cfd83eddb96Vignesh Venkatasubramanian                               mask_8x8_r,
1315da49e34c1fb5e99681f4ad99c21d9cfd83eddb96Vignesh Venkatasubramanian                               mask_4x4_r,
1316da49e34c1fb5e99681f4ad99c21d9cfd83eddb96Vignesh Venkatasubramanian                               mask_4x4_int_r,
1317da49e34c1fb5e99681f4ad99c21d9cfd83eddb96Vignesh Venkatasubramanian                               &cm->lf_info, &lfl[r << 3]);
1318da49e34c1fb5e99681f4ad99c21d9cfd83eddb96Vignesh Venkatasubramanian    }
1319da49e34c1fb5e99681f4ad99c21d9cfd83eddb96Vignesh Venkatasubramanian#else
1320ba164dffc5a6795bce97fae02b51ccf3330e15e4hkuang    filter_selectively_horiz(dst->buf, dst->stride,
13219b35249446b07f40ac5fcc3205f2c048616efacchkuang                             mask_16x16_r,
13229b35249446b07f40ac5fcc3205f2c048616efacchkuang                             mask_8x8_r,
13239b35249446b07f40ac5fcc3205f2c048616efacchkuang                             mask_4x4_r,
13249b35249446b07f40ac5fcc3205f2c048616efacchkuang                             mask_4x4_int_r,
13259b35249446b07f40ac5fcc3205f2c048616efacchkuang                             &cm->lf_info, &lfl[r << 3]);
1326da49e34c1fb5e99681f4ad99c21d9cfd83eddb96Vignesh Venkatasubramanian#endif  // CONFIG_VP9_HIGHBITDEPTH
1327ba164dffc5a6795bce97fae02b51ccf3330e15e4hkuang    dst->buf += 8 * dst->stride;
1328ba164dffc5a6795bce97fae02b51ccf3330e15e4hkuang  }
1329ba164dffc5a6795bce97fae02b51ccf3330e15e4hkuang}
13301184aebb761cbeac9124c37189a80a1a58f04b6bhkuang
1331da49e34c1fb5e99681f4ad99c21d9cfd83eddb96Vignesh Venkatasubramanianvoid vp9_filter_block_plane_ss00(VP9_COMMON *const cm,
1332da49e34c1fb5e99681f4ad99c21d9cfd83eddb96Vignesh Venkatasubramanian                                 struct macroblockd_plane *const plane,
1333da49e34c1fb5e99681f4ad99c21d9cfd83eddb96Vignesh Venkatasubramanian                                 int mi_row,
1334da49e34c1fb5e99681f4ad99c21d9cfd83eddb96Vignesh Venkatasubramanian                                 LOOP_FILTER_MASK *lfm) {
13351184aebb761cbeac9124c37189a80a1a58f04b6bhkuang  struct buf_2d *const dst = &plane->dst;
1336da49e34c1fb5e99681f4ad99c21d9cfd83eddb96Vignesh Venkatasubramanian  uint8_t *const dst0 = dst->buf;
1337da49e34c1fb5e99681f4ad99c21d9cfd83eddb96Vignesh Venkatasubramanian  int r;
1338da49e34c1fb5e99681f4ad99c21d9cfd83eddb96Vignesh Venkatasubramanian  uint64_t mask_16x16 = lfm->left_y[TX_16X16];
1339da49e34c1fb5e99681f4ad99c21d9cfd83eddb96Vignesh Venkatasubramanian  uint64_t mask_8x8 = lfm->left_y[TX_8X8];
1340da49e34c1fb5e99681f4ad99c21d9cfd83eddb96Vignesh Venkatasubramanian  uint64_t mask_4x4 = lfm->left_y[TX_4X4];
1341da49e34c1fb5e99681f4ad99c21d9cfd83eddb96Vignesh Venkatasubramanian  uint64_t mask_4x4_int = lfm->int_4x4_y;
1342da49e34c1fb5e99681f4ad99c21d9cfd83eddb96Vignesh Venkatasubramanian
1343da49e34c1fb5e99681f4ad99c21d9cfd83eddb96Vignesh Venkatasubramanian  assert(plane->subsampling_x == 0 && plane->subsampling_y == 0);
1344da49e34c1fb5e99681f4ad99c21d9cfd83eddb96Vignesh Venkatasubramanian
1345da49e34c1fb5e99681f4ad99c21d9cfd83eddb96Vignesh Venkatasubramanian  // Vertical pass: do 2 rows at one time
1346da49e34c1fb5e99681f4ad99c21d9cfd83eddb96Vignesh Venkatasubramanian  for (r = 0; r < MI_BLOCK_SIZE && mi_row + r < cm->mi_rows; r += 2) {
1347da49e34c1fb5e99681f4ad99c21d9cfd83eddb96Vignesh Venkatasubramanian    unsigned int mask_16x16_l = mask_16x16 & 0xffff;
1348da49e34c1fb5e99681f4ad99c21d9cfd83eddb96Vignesh Venkatasubramanian    unsigned int mask_8x8_l = mask_8x8 & 0xffff;
1349da49e34c1fb5e99681f4ad99c21d9cfd83eddb96Vignesh Venkatasubramanian    unsigned int mask_4x4_l = mask_4x4 & 0xffff;
1350da49e34c1fb5e99681f4ad99c21d9cfd83eddb96Vignesh Venkatasubramanian    unsigned int mask_4x4_int_l = mask_4x4_int & 0xffff;
1351da49e34c1fb5e99681f4ad99c21d9cfd83eddb96Vignesh Venkatasubramanian
1352da49e34c1fb5e99681f4ad99c21d9cfd83eddb96Vignesh Venkatasubramanian// Disable filtering on the leftmost column.
1353da49e34c1fb5e99681f4ad99c21d9cfd83eddb96Vignesh Venkatasubramanian#if CONFIG_VP9_HIGHBITDEPTH
1354da49e34c1fb5e99681f4ad99c21d9cfd83eddb96Vignesh Venkatasubramanian    if (cm->use_highbitdepth) {
1355da49e34c1fb5e99681f4ad99c21d9cfd83eddb96Vignesh Venkatasubramanian      highbd_filter_selectively_vert_row2(
1356da49e34c1fb5e99681f4ad99c21d9cfd83eddb96Vignesh Venkatasubramanian          plane->subsampling_x, CONVERT_TO_SHORTPTR(dst->buf), dst->stride,
1357da49e34c1fb5e99681f4ad99c21d9cfd83eddb96Vignesh Venkatasubramanian          mask_16x16_l, mask_8x8_l, mask_4x4_l, mask_4x4_int_l, &cm->lf_info,
1358da49e34c1fb5e99681f4ad99c21d9cfd83eddb96Vignesh Venkatasubramanian          &lfm->lfl_y[r << 3], (int)cm->bit_depth);
1359da49e34c1fb5e99681f4ad99c21d9cfd83eddb96Vignesh Venkatasubramanian    } else {
1360da49e34c1fb5e99681f4ad99c21d9cfd83eddb96Vignesh Venkatasubramanian      filter_selectively_vert_row2(
1361da49e34c1fb5e99681f4ad99c21d9cfd83eddb96Vignesh Venkatasubramanian          plane->subsampling_x, dst->buf, dst->stride, mask_16x16_l, mask_8x8_l,
1362da49e34c1fb5e99681f4ad99c21d9cfd83eddb96Vignesh Venkatasubramanian          mask_4x4_l, mask_4x4_int_l, &cm->lf_info, &lfm->lfl_y[r << 3]);
1363da49e34c1fb5e99681f4ad99c21d9cfd83eddb96Vignesh Venkatasubramanian    }
1364da49e34c1fb5e99681f4ad99c21d9cfd83eddb96Vignesh Venkatasubramanian#else
1365da49e34c1fb5e99681f4ad99c21d9cfd83eddb96Vignesh Venkatasubramanian    filter_selectively_vert_row2(
1366da49e34c1fb5e99681f4ad99c21d9cfd83eddb96Vignesh Venkatasubramanian        plane->subsampling_x, dst->buf, dst->stride, mask_16x16_l, mask_8x8_l,
1367da49e34c1fb5e99681f4ad99c21d9cfd83eddb96Vignesh Venkatasubramanian        mask_4x4_l, mask_4x4_int_l, &cm->lf_info, &lfm->lfl_y[r << 3]);
1368da49e34c1fb5e99681f4ad99c21d9cfd83eddb96Vignesh Venkatasubramanian#endif  // CONFIG_VP9_HIGHBITDEPTH
1369da49e34c1fb5e99681f4ad99c21d9cfd83eddb96Vignesh Venkatasubramanian    dst->buf += 16 * dst->stride;
1370da49e34c1fb5e99681f4ad99c21d9cfd83eddb96Vignesh Venkatasubramanian    mask_16x16 >>= 16;
1371da49e34c1fb5e99681f4ad99c21d9cfd83eddb96Vignesh Venkatasubramanian    mask_8x8 >>= 16;
1372da49e34c1fb5e99681f4ad99c21d9cfd83eddb96Vignesh Venkatasubramanian    mask_4x4 >>= 16;
1373da49e34c1fb5e99681f4ad99c21d9cfd83eddb96Vignesh Venkatasubramanian    mask_4x4_int >>= 16;
1374da49e34c1fb5e99681f4ad99c21d9cfd83eddb96Vignesh Venkatasubramanian  }
1375da49e34c1fb5e99681f4ad99c21d9cfd83eddb96Vignesh Venkatasubramanian
1376da49e34c1fb5e99681f4ad99c21d9cfd83eddb96Vignesh Venkatasubramanian  // Horizontal pass
1377da49e34c1fb5e99681f4ad99c21d9cfd83eddb96Vignesh Venkatasubramanian  dst->buf = dst0;
1378da49e34c1fb5e99681f4ad99c21d9cfd83eddb96Vignesh Venkatasubramanian  mask_16x16 = lfm->above_y[TX_16X16];
1379da49e34c1fb5e99681f4ad99c21d9cfd83eddb96Vignesh Venkatasubramanian  mask_8x8 = lfm->above_y[TX_8X8];
1380da49e34c1fb5e99681f4ad99c21d9cfd83eddb96Vignesh Venkatasubramanian  mask_4x4 = lfm->above_y[TX_4X4];
1381da49e34c1fb5e99681f4ad99c21d9cfd83eddb96Vignesh Venkatasubramanian  mask_4x4_int = lfm->int_4x4_y;
1382da49e34c1fb5e99681f4ad99c21d9cfd83eddb96Vignesh Venkatasubramanian
1383da49e34c1fb5e99681f4ad99c21d9cfd83eddb96Vignesh Venkatasubramanian  for (r = 0; r < MI_BLOCK_SIZE && mi_row + r < cm->mi_rows; r++) {
1384da49e34c1fb5e99681f4ad99c21d9cfd83eddb96Vignesh Venkatasubramanian    unsigned int mask_16x16_r;
1385da49e34c1fb5e99681f4ad99c21d9cfd83eddb96Vignesh Venkatasubramanian    unsigned int mask_8x8_r;
1386da49e34c1fb5e99681f4ad99c21d9cfd83eddb96Vignesh Venkatasubramanian    unsigned int mask_4x4_r;
1387da49e34c1fb5e99681f4ad99c21d9cfd83eddb96Vignesh Venkatasubramanian
1388da49e34c1fb5e99681f4ad99c21d9cfd83eddb96Vignesh Venkatasubramanian    if (mi_row + r == 0) {
1389da49e34c1fb5e99681f4ad99c21d9cfd83eddb96Vignesh Venkatasubramanian      mask_16x16_r = 0;
1390da49e34c1fb5e99681f4ad99c21d9cfd83eddb96Vignesh Venkatasubramanian      mask_8x8_r = 0;
1391da49e34c1fb5e99681f4ad99c21d9cfd83eddb96Vignesh Venkatasubramanian      mask_4x4_r = 0;
1392da49e34c1fb5e99681f4ad99c21d9cfd83eddb96Vignesh Venkatasubramanian    } else {
1393da49e34c1fb5e99681f4ad99c21d9cfd83eddb96Vignesh Venkatasubramanian      mask_16x16_r = mask_16x16 & 0xff;
1394da49e34c1fb5e99681f4ad99c21d9cfd83eddb96Vignesh Venkatasubramanian      mask_8x8_r = mask_8x8 & 0xff;
1395da49e34c1fb5e99681f4ad99c21d9cfd83eddb96Vignesh Venkatasubramanian      mask_4x4_r = mask_4x4 & 0xff;
1396da49e34c1fb5e99681f4ad99c21d9cfd83eddb96Vignesh Venkatasubramanian    }
1397da49e34c1fb5e99681f4ad99c21d9cfd83eddb96Vignesh Venkatasubramanian
1398da49e34c1fb5e99681f4ad99c21d9cfd83eddb96Vignesh Venkatasubramanian#if CONFIG_VP9_HIGHBITDEPTH
1399da49e34c1fb5e99681f4ad99c21d9cfd83eddb96Vignesh Venkatasubramanian    if (cm->use_highbitdepth) {
1400da49e34c1fb5e99681f4ad99c21d9cfd83eddb96Vignesh Venkatasubramanian      highbd_filter_selectively_horiz(
1401da49e34c1fb5e99681f4ad99c21d9cfd83eddb96Vignesh Venkatasubramanian          CONVERT_TO_SHORTPTR(dst->buf), dst->stride, mask_16x16_r, mask_8x8_r,
1402da49e34c1fb5e99681f4ad99c21d9cfd83eddb96Vignesh Venkatasubramanian          mask_4x4_r, mask_4x4_int & 0xff, &cm->lf_info, &lfm->lfl_y[r << 3],
1403da49e34c1fb5e99681f4ad99c21d9cfd83eddb96Vignesh Venkatasubramanian          (int)cm->bit_depth);
1404da49e34c1fb5e99681f4ad99c21d9cfd83eddb96Vignesh Venkatasubramanian    } else {
1405da49e34c1fb5e99681f4ad99c21d9cfd83eddb96Vignesh Venkatasubramanian      filter_selectively_horiz(dst->buf, dst->stride, mask_16x16_r, mask_8x8_r,
1406da49e34c1fb5e99681f4ad99c21d9cfd83eddb96Vignesh Venkatasubramanian                               mask_4x4_r, mask_4x4_int & 0xff, &cm->lf_info,
1407da49e34c1fb5e99681f4ad99c21d9cfd83eddb96Vignesh Venkatasubramanian                               &lfm->lfl_y[r << 3]);
1408da49e34c1fb5e99681f4ad99c21d9cfd83eddb96Vignesh Venkatasubramanian    }
1409da49e34c1fb5e99681f4ad99c21d9cfd83eddb96Vignesh Venkatasubramanian#else
1410da49e34c1fb5e99681f4ad99c21d9cfd83eddb96Vignesh Venkatasubramanian    filter_selectively_horiz(dst->buf, dst->stride, mask_16x16_r, mask_8x8_r,
1411da49e34c1fb5e99681f4ad99c21d9cfd83eddb96Vignesh Venkatasubramanian                             mask_4x4_r, mask_4x4_int & 0xff, &cm->lf_info,
1412da49e34c1fb5e99681f4ad99c21d9cfd83eddb96Vignesh Venkatasubramanian                             &lfm->lfl_y[r << 3]);
1413da49e34c1fb5e99681f4ad99c21d9cfd83eddb96Vignesh Venkatasubramanian#endif  // CONFIG_VP9_HIGHBITDEPTH
1414da49e34c1fb5e99681f4ad99c21d9cfd83eddb96Vignesh Venkatasubramanian
1415da49e34c1fb5e99681f4ad99c21d9cfd83eddb96Vignesh Venkatasubramanian    dst->buf += 8 * dst->stride;
1416da49e34c1fb5e99681f4ad99c21d9cfd83eddb96Vignesh Venkatasubramanian    mask_16x16 >>= 8;
1417da49e34c1fb5e99681f4ad99c21d9cfd83eddb96Vignesh Venkatasubramanian    mask_8x8 >>= 8;
1418da49e34c1fb5e99681f4ad99c21d9cfd83eddb96Vignesh Venkatasubramanian    mask_4x4 >>= 8;
1419da49e34c1fb5e99681f4ad99c21d9cfd83eddb96Vignesh Venkatasubramanian    mask_4x4_int >>= 8;
1420da49e34c1fb5e99681f4ad99c21d9cfd83eddb96Vignesh Venkatasubramanian  }
1421da49e34c1fb5e99681f4ad99c21d9cfd83eddb96Vignesh Venkatasubramanian}
1422da49e34c1fb5e99681f4ad99c21d9cfd83eddb96Vignesh Venkatasubramanian
1423da49e34c1fb5e99681f4ad99c21d9cfd83eddb96Vignesh Venkatasubramanianvoid vp9_filter_block_plane_ss11(VP9_COMMON *const cm,
1424da49e34c1fb5e99681f4ad99c21d9cfd83eddb96Vignesh Venkatasubramanian                                 struct macroblockd_plane *const plane,
1425da49e34c1fb5e99681f4ad99c21d9cfd83eddb96Vignesh Venkatasubramanian                                 int mi_row,
1426da49e34c1fb5e99681f4ad99c21d9cfd83eddb96Vignesh Venkatasubramanian                                 LOOP_FILTER_MASK *lfm) {
1427da49e34c1fb5e99681f4ad99c21d9cfd83eddb96Vignesh Venkatasubramanian  struct buf_2d *const dst = &plane->dst;
1428da49e34c1fb5e99681f4ad99c21d9cfd83eddb96Vignesh Venkatasubramanian  uint8_t *const dst0 = dst->buf;
14291184aebb761cbeac9124c37189a80a1a58f04b6bhkuang  int r, c;
14301184aebb761cbeac9124c37189a80a1a58f04b6bhkuang
1431da49e34c1fb5e99681f4ad99c21d9cfd83eddb96Vignesh Venkatasubramanian  uint16_t mask_16x16 = lfm->left_uv[TX_16X16];
1432da49e34c1fb5e99681f4ad99c21d9cfd83eddb96Vignesh Venkatasubramanian  uint16_t mask_8x8 = lfm->left_uv[TX_8X8];
1433da49e34c1fb5e99681f4ad99c21d9cfd83eddb96Vignesh Venkatasubramanian  uint16_t mask_4x4 = lfm->left_uv[TX_4X4];
1434da49e34c1fb5e99681f4ad99c21d9cfd83eddb96Vignesh Venkatasubramanian  uint16_t mask_4x4_int = lfm->int_4x4_uv;
14352ec72e65689c948e92b826ae1e867bf369e72f13Vignesh Venkatasubramanian
1436da49e34c1fb5e99681f4ad99c21d9cfd83eddb96Vignesh Venkatasubramanian  assert(plane->subsampling_x == 1 && plane->subsampling_y == 1);
1437da49e34c1fb5e99681f4ad99c21d9cfd83eddb96Vignesh Venkatasubramanian
1438da49e34c1fb5e99681f4ad99c21d9cfd83eddb96Vignesh Venkatasubramanian  // Vertical pass: do 2 rows at one time
1439da49e34c1fb5e99681f4ad99c21d9cfd83eddb96Vignesh Venkatasubramanian  for (r = 0; r < MI_BLOCK_SIZE && mi_row + r < cm->mi_rows; r += 4) {
1440da49e34c1fb5e99681f4ad99c21d9cfd83eddb96Vignesh Venkatasubramanian    if (plane->plane_type == 1) {
1441da49e34c1fb5e99681f4ad99c21d9cfd83eddb96Vignesh Venkatasubramanian      for (c = 0; c < (MI_BLOCK_SIZE >> 1); c++) {
1442da49e34c1fb5e99681f4ad99c21d9cfd83eddb96Vignesh Venkatasubramanian        lfm->lfl_uv[(r << 1) + c] = lfm->lfl_y[(r << 3) + (c << 1)];
1443da49e34c1fb5e99681f4ad99c21d9cfd83eddb96Vignesh Venkatasubramanian        lfm->lfl_uv[((r + 2) << 1) + c] = lfm->lfl_y[((r + 2) << 3) + (c << 1)];
1444da49e34c1fb5e99681f4ad99c21d9cfd83eddb96Vignesh Venkatasubramanian      }
14459b35249446b07f40ac5fcc3205f2c048616efacchkuang    }
14469b35249446b07f40ac5fcc3205f2c048616efacchkuang
1447da49e34c1fb5e99681f4ad99c21d9cfd83eddb96Vignesh Venkatasubramanian    {
1448da49e34c1fb5e99681f4ad99c21d9cfd83eddb96Vignesh Venkatasubramanian      unsigned int mask_16x16_l = mask_16x16 & 0xff;
1449da49e34c1fb5e99681f4ad99c21d9cfd83eddb96Vignesh Venkatasubramanian      unsigned int mask_8x8_l = mask_8x8 & 0xff;
1450da49e34c1fb5e99681f4ad99c21d9cfd83eddb96Vignesh Venkatasubramanian      unsigned int mask_4x4_l = mask_4x4 & 0xff;
1451da49e34c1fb5e99681f4ad99c21d9cfd83eddb96Vignesh Venkatasubramanian      unsigned int mask_4x4_int_l = mask_4x4_int & 0xff;
1452da49e34c1fb5e99681f4ad99c21d9cfd83eddb96Vignesh Venkatasubramanian
1453da49e34c1fb5e99681f4ad99c21d9cfd83eddb96Vignesh Venkatasubramanian// Disable filtering on the leftmost column.
1454da49e34c1fb5e99681f4ad99c21d9cfd83eddb96Vignesh Venkatasubramanian#if CONFIG_VP9_HIGHBITDEPTH
1455da49e34c1fb5e99681f4ad99c21d9cfd83eddb96Vignesh Venkatasubramanian      if (cm->use_highbitdepth) {
1456da49e34c1fb5e99681f4ad99c21d9cfd83eddb96Vignesh Venkatasubramanian        highbd_filter_selectively_vert_row2(
1457da49e34c1fb5e99681f4ad99c21d9cfd83eddb96Vignesh Venkatasubramanian            plane->subsampling_x, CONVERT_TO_SHORTPTR(dst->buf), dst->stride,
1458da49e34c1fb5e99681f4ad99c21d9cfd83eddb96Vignesh Venkatasubramanian            mask_16x16_l, mask_8x8_l, mask_4x4_l, mask_4x4_int_l, &cm->lf_info,
1459da49e34c1fb5e99681f4ad99c21d9cfd83eddb96Vignesh Venkatasubramanian            &lfm->lfl_uv[r << 1], (int)cm->bit_depth);
14609b35249446b07f40ac5fcc3205f2c048616efacchkuang      } else {
1461da49e34c1fb5e99681f4ad99c21d9cfd83eddb96Vignesh Venkatasubramanian        filter_selectively_vert_row2(
1462da49e34c1fb5e99681f4ad99c21d9cfd83eddb96Vignesh Venkatasubramanian            plane->subsampling_x, dst->buf, dst->stride,
1463da49e34c1fb5e99681f4ad99c21d9cfd83eddb96Vignesh Venkatasubramanian            mask_16x16_l, mask_8x8_l, mask_4x4_l, mask_4x4_int_l, &cm->lf_info,
1464da49e34c1fb5e99681f4ad99c21d9cfd83eddb96Vignesh Venkatasubramanian            &lfm->lfl_uv[r << 1]);
14659b35249446b07f40ac5fcc3205f2c048616efacchkuang      }
1466da49e34c1fb5e99681f4ad99c21d9cfd83eddb96Vignesh Venkatasubramanian#else
1467da49e34c1fb5e99681f4ad99c21d9cfd83eddb96Vignesh Venkatasubramanian      filter_selectively_vert_row2(
1468da49e34c1fb5e99681f4ad99c21d9cfd83eddb96Vignesh Venkatasubramanian          plane->subsampling_x, dst->buf, dst->stride,
1469da49e34c1fb5e99681f4ad99c21d9cfd83eddb96Vignesh Venkatasubramanian          mask_16x16_l, mask_8x8_l, mask_4x4_l, mask_4x4_int_l, &cm->lf_info,
1470da49e34c1fb5e99681f4ad99c21d9cfd83eddb96Vignesh Venkatasubramanian          &lfm->lfl_uv[r << 1]);
1471da49e34c1fb5e99681f4ad99c21d9cfd83eddb96Vignesh Venkatasubramanian#endif  // CONFIG_VP9_HIGHBITDEPTH
14729b35249446b07f40ac5fcc3205f2c048616efacchkuang
1473da49e34c1fb5e99681f4ad99c21d9cfd83eddb96Vignesh Venkatasubramanian      dst->buf += 16 * dst->stride;
14749b35249446b07f40ac5fcc3205f2c048616efacchkuang      mask_16x16 >>= 8;
14759b35249446b07f40ac5fcc3205f2c048616efacchkuang      mask_8x8 >>= 8;
14769b35249446b07f40ac5fcc3205f2c048616efacchkuang      mask_4x4 >>= 8;
14772ec72e65689c948e92b826ae1e867bf369e72f13Vignesh Venkatasubramanian      mask_4x4_int >>= 8;
14789b35249446b07f40ac5fcc3205f2c048616efacchkuang    }
1479da49e34c1fb5e99681f4ad99c21d9cfd83eddb96Vignesh Venkatasubramanian  }
14801184aebb761cbeac9124c37189a80a1a58f04b6bhkuang
1481da49e34c1fb5e99681f4ad99c21d9cfd83eddb96Vignesh Venkatasubramanian  // Horizontal pass
1482da49e34c1fb5e99681f4ad99c21d9cfd83eddb96Vignesh Venkatasubramanian  dst->buf = dst0;
1483da49e34c1fb5e99681f4ad99c21d9cfd83eddb96Vignesh Venkatasubramanian  mask_16x16 = lfm->above_uv[TX_16X16];
1484da49e34c1fb5e99681f4ad99c21d9cfd83eddb96Vignesh Venkatasubramanian  mask_8x8 = lfm->above_uv[TX_8X8];
1485da49e34c1fb5e99681f4ad99c21d9cfd83eddb96Vignesh Venkatasubramanian  mask_4x4 = lfm->above_uv[TX_4X4];
1486da49e34c1fb5e99681f4ad99c21d9cfd83eddb96Vignesh Venkatasubramanian  mask_4x4_int = lfm->int_4x4_uv;
1487da49e34c1fb5e99681f4ad99c21d9cfd83eddb96Vignesh Venkatasubramanian
1488da49e34c1fb5e99681f4ad99c21d9cfd83eddb96Vignesh Venkatasubramanian  for (r = 0; r < MI_BLOCK_SIZE && mi_row + r < cm->mi_rows; r += 2) {
1489da49e34c1fb5e99681f4ad99c21d9cfd83eddb96Vignesh Venkatasubramanian    const int skip_border_4x4_r = mi_row + r == cm->mi_rows - 1;
1490da49e34c1fb5e99681f4ad99c21d9cfd83eddb96Vignesh Venkatasubramanian    const unsigned int mask_4x4_int_r =
1491da49e34c1fb5e99681f4ad99c21d9cfd83eddb96Vignesh Venkatasubramanian        skip_border_4x4_r ? 0 : (mask_4x4_int & 0xf);
1492da49e34c1fb5e99681f4ad99c21d9cfd83eddb96Vignesh Venkatasubramanian    unsigned int mask_16x16_r;
1493da49e34c1fb5e99681f4ad99c21d9cfd83eddb96Vignesh Venkatasubramanian    unsigned int mask_8x8_r;
1494da49e34c1fb5e99681f4ad99c21d9cfd83eddb96Vignesh Venkatasubramanian    unsigned int mask_4x4_r;
14951184aebb761cbeac9124c37189a80a1a58f04b6bhkuang
1496da49e34c1fb5e99681f4ad99c21d9cfd83eddb96Vignesh Venkatasubramanian    if (mi_row + r == 0) {
1497da49e34c1fb5e99681f4ad99c21d9cfd83eddb96Vignesh Venkatasubramanian      mask_16x16_r = 0;
1498da49e34c1fb5e99681f4ad99c21d9cfd83eddb96Vignesh Venkatasubramanian      mask_8x8_r = 0;
1499da49e34c1fb5e99681f4ad99c21d9cfd83eddb96Vignesh Venkatasubramanian      mask_4x4_r = 0;
1500da49e34c1fb5e99681f4ad99c21d9cfd83eddb96Vignesh Venkatasubramanian    } else {
1501da49e34c1fb5e99681f4ad99c21d9cfd83eddb96Vignesh Venkatasubramanian      mask_16x16_r = mask_16x16 & 0xf;
1502da49e34c1fb5e99681f4ad99c21d9cfd83eddb96Vignesh Venkatasubramanian      mask_8x8_r = mask_8x8 & 0xf;
1503da49e34c1fb5e99681f4ad99c21d9cfd83eddb96Vignesh Venkatasubramanian      mask_4x4_r = mask_4x4 & 0xf;
1504da49e34c1fb5e99681f4ad99c21d9cfd83eddb96Vignesh Venkatasubramanian    }
15059b35249446b07f40ac5fcc3205f2c048616efacchkuang
1506da49e34c1fb5e99681f4ad99c21d9cfd83eddb96Vignesh Venkatasubramanian#if CONFIG_VP9_HIGHBITDEPTH
1507da49e34c1fb5e99681f4ad99c21d9cfd83eddb96Vignesh Venkatasubramanian    if (cm->use_highbitdepth) {
1508da49e34c1fb5e99681f4ad99c21d9cfd83eddb96Vignesh Venkatasubramanian      highbd_filter_selectively_horiz(CONVERT_TO_SHORTPTR(dst->buf),
1509da49e34c1fb5e99681f4ad99c21d9cfd83eddb96Vignesh Venkatasubramanian                                      dst->stride, mask_16x16_r, mask_8x8_r,
1510da49e34c1fb5e99681f4ad99c21d9cfd83eddb96Vignesh Venkatasubramanian                                      mask_4x4_r, mask_4x4_int_r, &cm->lf_info,
1511da49e34c1fb5e99681f4ad99c21d9cfd83eddb96Vignesh Venkatasubramanian                                      &lfm->lfl_uv[r << 1], (int)cm->bit_depth);
1512da49e34c1fb5e99681f4ad99c21d9cfd83eddb96Vignesh Venkatasubramanian    } else {
1513da49e34c1fb5e99681f4ad99c21d9cfd83eddb96Vignesh Venkatasubramanian      filter_selectively_horiz(dst->buf, dst->stride, mask_16x16_r, mask_8x8_r,
1514da49e34c1fb5e99681f4ad99c21d9cfd83eddb96Vignesh Venkatasubramanian                               mask_4x4_r, mask_4x4_int_r, &cm->lf_info,
1515da49e34c1fb5e99681f4ad99c21d9cfd83eddb96Vignesh Venkatasubramanian                               &lfm->lfl_uv[r << 1]);
15161184aebb761cbeac9124c37189a80a1a58f04b6bhkuang    }
1517da49e34c1fb5e99681f4ad99c21d9cfd83eddb96Vignesh Venkatasubramanian#else
1518da49e34c1fb5e99681f4ad99c21d9cfd83eddb96Vignesh Venkatasubramanian    filter_selectively_horiz(dst->buf, dst->stride, mask_16x16_r, mask_8x8_r,
1519da49e34c1fb5e99681f4ad99c21d9cfd83eddb96Vignesh Venkatasubramanian                             mask_4x4_r, mask_4x4_int_r, &cm->lf_info,
1520da49e34c1fb5e99681f4ad99c21d9cfd83eddb96Vignesh Venkatasubramanian                             &lfm->lfl_uv[r << 1]);
1521da49e34c1fb5e99681f4ad99c21d9cfd83eddb96Vignesh Venkatasubramanian#endif  // CONFIG_VP9_HIGHBITDEPTH
1522da49e34c1fb5e99681f4ad99c21d9cfd83eddb96Vignesh Venkatasubramanian
1523da49e34c1fb5e99681f4ad99c21d9cfd83eddb96Vignesh Venkatasubramanian    dst->buf += 8 * dst->stride;
1524da49e34c1fb5e99681f4ad99c21d9cfd83eddb96Vignesh Venkatasubramanian    mask_16x16 >>= 4;
1525da49e34c1fb5e99681f4ad99c21d9cfd83eddb96Vignesh Venkatasubramanian    mask_8x8 >>= 4;
1526da49e34c1fb5e99681f4ad99c21d9cfd83eddb96Vignesh Venkatasubramanian    mask_4x4 >>= 4;
1527da49e34c1fb5e99681f4ad99c21d9cfd83eddb96Vignesh Venkatasubramanian    mask_4x4_int >>= 4;
15281184aebb761cbeac9124c37189a80a1a58f04b6bhkuang  }
15291184aebb761cbeac9124c37189a80a1a58f04b6bhkuang}
1530ba164dffc5a6795bce97fae02b51ccf3330e15e4hkuang
1531da49e34c1fb5e99681f4ad99c21d9cfd83eddb96Vignesh Venkatasubramanianvoid vp9_loop_filter_rows(YV12_BUFFER_CONFIG *frame_buffer,
1532ba6c59e9d7d7013b3906b6f4230b663422681848Vignesh Venkatasubramanian                          VP9_COMMON *cm,
1533ba6c59e9d7d7013b3906b6f4230b663422681848Vignesh Venkatasubramanian                          struct macroblockd_plane planes[MAX_MB_PLANE],
153491037db265ecdd914a26e056cf69207b4f50924ehkuang                          int start, int stop, int y_only) {
153591037db265ecdd914a26e056cf69207b4f50924ehkuang  const int num_planes = y_only ? 1 : MAX_MB_PLANE;
1536da49e34c1fb5e99681f4ad99c21d9cfd83eddb96Vignesh Venkatasubramanian  enum lf_path path;
15371184aebb761cbeac9124c37189a80a1a58f04b6bhkuang  LOOP_FILTER_MASK lfm;
1538ba6c59e9d7d7013b3906b6f4230b663422681848Vignesh Venkatasubramanian  int mi_row, mi_col;
1539ba164dffc5a6795bce97fae02b51ccf3330e15e4hkuang
1540da49e34c1fb5e99681f4ad99c21d9cfd83eddb96Vignesh Venkatasubramanian  if (y_only)
1541da49e34c1fb5e99681f4ad99c21d9cfd83eddb96Vignesh Venkatasubramanian    path = LF_PATH_444;
1542da49e34c1fb5e99681f4ad99c21d9cfd83eddb96Vignesh Venkatasubramanian  else if (planes[1].subsampling_y == 1 && planes[1].subsampling_x == 1)
1543da49e34c1fb5e99681f4ad99c21d9cfd83eddb96Vignesh Venkatasubramanian    path = LF_PATH_420;
1544da49e34c1fb5e99681f4ad99c21d9cfd83eddb96Vignesh Venkatasubramanian  else if (planes[1].subsampling_y == 0 && planes[1].subsampling_x == 0)
1545da49e34c1fb5e99681f4ad99c21d9cfd83eddb96Vignesh Venkatasubramanian    path = LF_PATH_444;
1546da49e34c1fb5e99681f4ad99c21d9cfd83eddb96Vignesh Venkatasubramanian  else
1547da49e34c1fb5e99681f4ad99c21d9cfd83eddb96Vignesh Venkatasubramanian    path = LF_PATH_SLOW;
1548da49e34c1fb5e99681f4ad99c21d9cfd83eddb96Vignesh Venkatasubramanian
154991037db265ecdd914a26e056cf69207b4f50924ehkuang  for (mi_row = start; mi_row < stop; mi_row += MI_BLOCK_SIZE) {
1550ba6c59e9d7d7013b3906b6f4230b663422681848Vignesh Venkatasubramanian    MODE_INFO **mi = cm->mi_grid_visible + mi_row * cm->mi_stride;
1551ba164dffc5a6795bce97fae02b51ccf3330e15e4hkuang
155291037db265ecdd914a26e056cf69207b4f50924ehkuang    for (mi_col = 0; mi_col < cm->mi_cols; mi_col += MI_BLOCK_SIZE) {
1553ba164dffc5a6795bce97fae02b51ccf3330e15e4hkuang      int plane;
1554ba164dffc5a6795bce97fae02b51ccf3330e15e4hkuang
1555ba6c59e9d7d7013b3906b6f4230b663422681848Vignesh Venkatasubramanian      vp9_setup_dst_planes(planes, frame_buffer, mi_row, mi_col);
15561184aebb761cbeac9124c37189a80a1a58f04b6bhkuang
15571184aebb761cbeac9124c37189a80a1a58f04b6bhkuang      // TODO(JBB): Make setup_mask work for non 420.
1558da49e34c1fb5e99681f4ad99c21d9cfd83eddb96Vignesh Venkatasubramanian      vp9_setup_mask(cm, mi_row, mi_col, mi + mi_col, cm->mi_stride,
1559da49e34c1fb5e99681f4ad99c21d9cfd83eddb96Vignesh Venkatasubramanian                     &lfm);
1560da49e34c1fb5e99681f4ad99c21d9cfd83eddb96Vignesh Venkatasubramanian
1561da49e34c1fb5e99681f4ad99c21d9cfd83eddb96Vignesh Venkatasubramanian      vp9_filter_block_plane_ss00(cm, &planes[0], mi_row, &lfm);
1562da49e34c1fb5e99681f4ad99c21d9cfd83eddb96Vignesh Venkatasubramanian      for (plane = 1; plane < num_planes; ++plane) {
1563da49e34c1fb5e99681f4ad99c21d9cfd83eddb96Vignesh Venkatasubramanian        switch (path) {
1564da49e34c1fb5e99681f4ad99c21d9cfd83eddb96Vignesh Venkatasubramanian          case LF_PATH_420:
1565da49e34c1fb5e99681f4ad99c21d9cfd83eddb96Vignesh Venkatasubramanian            vp9_filter_block_plane_ss11(cm, &planes[plane], mi_row, &lfm);
1566da49e34c1fb5e99681f4ad99c21d9cfd83eddb96Vignesh Venkatasubramanian            break;
1567da49e34c1fb5e99681f4ad99c21d9cfd83eddb96Vignesh Venkatasubramanian          case LF_PATH_444:
1568da49e34c1fb5e99681f4ad99c21d9cfd83eddb96Vignesh Venkatasubramanian            vp9_filter_block_plane_ss00(cm, &planes[plane], mi_row, &lfm);
1569da49e34c1fb5e99681f4ad99c21d9cfd83eddb96Vignesh Venkatasubramanian            break;
1570da49e34c1fb5e99681f4ad99c21d9cfd83eddb96Vignesh Venkatasubramanian          case LF_PATH_SLOW:
1571da49e34c1fb5e99681f4ad99c21d9cfd83eddb96Vignesh Venkatasubramanian            vp9_filter_block_plane_non420(cm, &planes[plane], mi + mi_col,
1572da49e34c1fb5e99681f4ad99c21d9cfd83eddb96Vignesh Venkatasubramanian                                          mi_row, mi_col);
1573da49e34c1fb5e99681f4ad99c21d9cfd83eddb96Vignesh Venkatasubramanian            break;
1574da49e34c1fb5e99681f4ad99c21d9cfd83eddb96Vignesh Venkatasubramanian        }
1575ba164dffc5a6795bce97fae02b51ccf3330e15e4hkuang      }
1576ba164dffc5a6795bce97fae02b51ccf3330e15e4hkuang    }
1577ba164dffc5a6795bce97fae02b51ccf3330e15e4hkuang  }
1578ba164dffc5a6795bce97fae02b51ccf3330e15e4hkuang}
157991037db265ecdd914a26e056cf69207b4f50924ehkuang
1580ba6c59e9d7d7013b3906b6f4230b663422681848Vignesh Venkatasubramanianvoid vp9_loop_filter_frame(YV12_BUFFER_CONFIG *frame,
1581ba6c59e9d7d7013b3906b6f4230b663422681848Vignesh Venkatasubramanian                           VP9_COMMON *cm, MACROBLOCKD *xd,
15821184aebb761cbeac9124c37189a80a1a58f04b6bhkuang                           int frame_filter_level,
15832ec72e65689c948e92b826ae1e867bf369e72f13Vignesh Venkatasubramanian                           int y_only, int partial_frame) {
15841184aebb761cbeac9124c37189a80a1a58f04b6bhkuang  int start_mi_row, end_mi_row, mi_rows_to_filter;
158591037db265ecdd914a26e056cf69207b4f50924ehkuang  if (!frame_filter_level) return;
15861184aebb761cbeac9124c37189a80a1a58f04b6bhkuang  start_mi_row = 0;
15871184aebb761cbeac9124c37189a80a1a58f04b6bhkuang  mi_rows_to_filter = cm->mi_rows;
15882ec72e65689c948e92b826ae1e867bf369e72f13Vignesh Venkatasubramanian  if (partial_frame && cm->mi_rows > 8) {
15891184aebb761cbeac9124c37189a80a1a58f04b6bhkuang    start_mi_row = cm->mi_rows >> 1;
15901184aebb761cbeac9124c37189a80a1a58f04b6bhkuang    start_mi_row &= 0xfffffff8;
15911184aebb761cbeac9124c37189a80a1a58f04b6bhkuang    mi_rows_to_filter = MAX(cm->mi_rows / 8, 8);
15921184aebb761cbeac9124c37189a80a1a58f04b6bhkuang  }
15931184aebb761cbeac9124c37189a80a1a58f04b6bhkuang  end_mi_row = start_mi_row + mi_rows_to_filter;
15941184aebb761cbeac9124c37189a80a1a58f04b6bhkuang  vp9_loop_filter_frame_init(cm, frame_filter_level);
1595ba6c59e9d7d7013b3906b6f4230b663422681848Vignesh Venkatasubramanian  vp9_loop_filter_rows(frame, cm, xd->plane,
15961184aebb761cbeac9124c37189a80a1a58f04b6bhkuang                       start_mi_row, end_mi_row,
15971184aebb761cbeac9124c37189a80a1a58f04b6bhkuang                       y_only);
159891037db265ecdd914a26e056cf69207b4f50924ehkuang}
1599f3bed9137f66ef693bd406e43b17e9a1114f1e14hkuang
1600da49e34c1fb5e99681f4ad99c21d9cfd83eddb96Vignesh Venkatasubramanianvoid vp9_loop_filter_data_reset(
1601da49e34c1fb5e99681f4ad99c21d9cfd83eddb96Vignesh Venkatasubramanian    LFWorkerData *lf_data, YV12_BUFFER_CONFIG *frame_buffer,
1602da49e34c1fb5e99681f4ad99c21d9cfd83eddb96Vignesh Venkatasubramanian    struct VP9Common *cm, const struct macroblockd_plane planes[MAX_MB_PLANE]) {
1603da49e34c1fb5e99681f4ad99c21d9cfd83eddb96Vignesh Venkatasubramanian  lf_data->frame_buffer = frame_buffer;
1604da49e34c1fb5e99681f4ad99c21d9cfd83eddb96Vignesh Venkatasubramanian  lf_data->cm = cm;
1605da49e34c1fb5e99681f4ad99c21d9cfd83eddb96Vignesh Venkatasubramanian  lf_data->start = 0;
1606da49e34c1fb5e99681f4ad99c21d9cfd83eddb96Vignesh Venkatasubramanian  lf_data->stop = 0;
1607da49e34c1fb5e99681f4ad99c21d9cfd83eddb96Vignesh Venkatasubramanian  lf_data->y_only = 0;
1608da49e34c1fb5e99681f4ad99c21d9cfd83eddb96Vignesh Venkatasubramanian  memcpy(lf_data->planes, planes, sizeof(lf_data->planes));
1609da49e34c1fb5e99681f4ad99c21d9cfd83eddb96Vignesh Venkatasubramanian}
1610da49e34c1fb5e99681f4ad99c21d9cfd83eddb96Vignesh Venkatasubramanian
1611da49e34c1fb5e99681f4ad99c21d9cfd83eddb96Vignesh Venkatasubramanianint vp9_loop_filter_worker(LFWorkerData *const lf_data, void *unused) {
1612da49e34c1fb5e99681f4ad99c21d9cfd83eddb96Vignesh Venkatasubramanian  (void)unused;
1613ba6c59e9d7d7013b3906b6f4230b663422681848Vignesh Venkatasubramanian  vp9_loop_filter_rows(lf_data->frame_buffer, lf_data->cm, lf_data->planes,
1614f3bed9137f66ef693bd406e43b17e9a1114f1e14hkuang                       lf_data->start, lf_data->stop, lf_data->y_only);
1615f3bed9137f66ef693bd406e43b17e9a1114f1e14hkuang  return 1;
1616f3bed9137f66ef693bd406e43b17e9a1114f1e14hkuang}
1617