vp9_rd.h revision ba6c59e9d7d7013b3906b6f4230b663422681848
1ba6c59e9d7d7013b3906b6f4230b663422681848Vignesh Venkatasubramanian/*
2ba6c59e9d7d7013b3906b6f4230b663422681848Vignesh Venkatasubramanian *  Copyright (c) 2010 The WebM project authors. All Rights Reserved.
3ba6c59e9d7d7013b3906b6f4230b663422681848Vignesh Venkatasubramanian *
4ba6c59e9d7d7013b3906b6f4230b663422681848Vignesh Venkatasubramanian *  Use of this source code is governed by a BSD-style license
5ba6c59e9d7d7013b3906b6f4230b663422681848Vignesh Venkatasubramanian *  that can be found in the LICENSE file in the root of the source
6ba6c59e9d7d7013b3906b6f4230b663422681848Vignesh Venkatasubramanian *  tree. An additional intellectual property rights grant can be found
7ba6c59e9d7d7013b3906b6f4230b663422681848Vignesh Venkatasubramanian *  in the file PATENTS.  All contributing project authors may
8ba6c59e9d7d7013b3906b6f4230b663422681848Vignesh Venkatasubramanian *  be found in the AUTHORS file in the root of the source tree.
9ba6c59e9d7d7013b3906b6f4230b663422681848Vignesh Venkatasubramanian */
10ba6c59e9d7d7013b3906b6f4230b663422681848Vignesh Venkatasubramanian
11ba6c59e9d7d7013b3906b6f4230b663422681848Vignesh Venkatasubramanian#ifndef VP9_ENCODER_VP9_RD_H_
12ba6c59e9d7d7013b3906b6f4230b663422681848Vignesh Venkatasubramanian#define VP9_ENCODER_VP9_RD_H_
13ba6c59e9d7d7013b3906b6f4230b663422681848Vignesh Venkatasubramanian
14ba6c59e9d7d7013b3906b6f4230b663422681848Vignesh Venkatasubramanian#include <limits.h>
15ba6c59e9d7d7013b3906b6f4230b663422681848Vignesh Venkatasubramanian
16ba6c59e9d7d7013b3906b6f4230b663422681848Vignesh Venkatasubramanian#include "vp9/common/vp9_blockd.h"
17ba6c59e9d7d7013b3906b6f4230b663422681848Vignesh Venkatasubramanian
18ba6c59e9d7d7013b3906b6f4230b663422681848Vignesh Venkatasubramanian#include "vp9/encoder/vp9_block.h"
19ba6c59e9d7d7013b3906b6f4230b663422681848Vignesh Venkatasubramanian#include "vp9/encoder/vp9_context_tree.h"
20ba6c59e9d7d7013b3906b6f4230b663422681848Vignesh Venkatasubramanian
21ba6c59e9d7d7013b3906b6f4230b663422681848Vignesh Venkatasubramanian#ifdef __cplusplus
22ba6c59e9d7d7013b3906b6f4230b663422681848Vignesh Venkatasubramanianextern "C" {
23ba6c59e9d7d7013b3906b6f4230b663422681848Vignesh Venkatasubramanian#endif
24ba6c59e9d7d7013b3906b6f4230b663422681848Vignesh Venkatasubramanian
25ba6c59e9d7d7013b3906b6f4230b663422681848Vignesh Venkatasubramanian#define RDDIV_BITS          7
26ba6c59e9d7d7013b3906b6f4230b663422681848Vignesh Venkatasubramanian
27ba6c59e9d7d7013b3906b6f4230b663422681848Vignesh Venkatasubramanian#define RDCOST(RM, DM, R, D) \
28ba6c59e9d7d7013b3906b6f4230b663422681848Vignesh Venkatasubramanian  (((128 + ((int64_t)R) * (RM)) >> 8) + (D << DM))
29ba6c59e9d7d7013b3906b6f4230b663422681848Vignesh Venkatasubramanian#define QIDX_SKIP_THRESH     115
30ba6c59e9d7d7013b3906b6f4230b663422681848Vignesh Venkatasubramanian
31ba6c59e9d7d7013b3906b6f4230b663422681848Vignesh Venkatasubramanian#define MV_COST_WEIGHT      108
32ba6c59e9d7d7013b3906b6f4230b663422681848Vignesh Venkatasubramanian#define MV_COST_WEIGHT_SUB  120
33ba6c59e9d7d7013b3906b6f4230b663422681848Vignesh Venkatasubramanian
34ba6c59e9d7d7013b3906b6f4230b663422681848Vignesh Venkatasubramanian#define INVALID_MV 0x80008000
35ba6c59e9d7d7013b3906b6f4230b663422681848Vignesh Venkatasubramanian
36ba6c59e9d7d7013b3906b6f4230b663422681848Vignesh Venkatasubramanian#define MAX_MODES 30
37ba6c59e9d7d7013b3906b6f4230b663422681848Vignesh Venkatasubramanian#define MAX_REFS  6
38ba6c59e9d7d7013b3906b6f4230b663422681848Vignesh Venkatasubramanian
39ba6c59e9d7d7013b3906b6f4230b663422681848Vignesh Venkatasubramanian// This enumerator type needs to be kept aligned with the mode order in
40ba6c59e9d7d7013b3906b6f4230b663422681848Vignesh Venkatasubramanian// const MODE_DEFINITION vp9_mode_order[MAX_MODES] used in the rd code.
41ba6c59e9d7d7013b3906b6f4230b663422681848Vignesh Venkatasubramaniantypedef enum {
42ba6c59e9d7d7013b3906b6f4230b663422681848Vignesh Venkatasubramanian  THR_NEARESTMV,
43ba6c59e9d7d7013b3906b6f4230b663422681848Vignesh Venkatasubramanian  THR_NEARESTA,
44ba6c59e9d7d7013b3906b6f4230b663422681848Vignesh Venkatasubramanian  THR_NEARESTG,
45ba6c59e9d7d7013b3906b6f4230b663422681848Vignesh Venkatasubramanian
46ba6c59e9d7d7013b3906b6f4230b663422681848Vignesh Venkatasubramanian  THR_DC,
47ba6c59e9d7d7013b3906b6f4230b663422681848Vignesh Venkatasubramanian
48ba6c59e9d7d7013b3906b6f4230b663422681848Vignesh Venkatasubramanian  THR_NEWMV,
49ba6c59e9d7d7013b3906b6f4230b663422681848Vignesh Venkatasubramanian  THR_NEWA,
50ba6c59e9d7d7013b3906b6f4230b663422681848Vignesh Venkatasubramanian  THR_NEWG,
51ba6c59e9d7d7013b3906b6f4230b663422681848Vignesh Venkatasubramanian
52ba6c59e9d7d7013b3906b6f4230b663422681848Vignesh Venkatasubramanian  THR_NEARMV,
53ba6c59e9d7d7013b3906b6f4230b663422681848Vignesh Venkatasubramanian  THR_NEARA,
54ba6c59e9d7d7013b3906b6f4230b663422681848Vignesh Venkatasubramanian  THR_COMP_NEARESTLA,
55ba6c59e9d7d7013b3906b6f4230b663422681848Vignesh Venkatasubramanian  THR_COMP_NEARESTGA,
56ba6c59e9d7d7013b3906b6f4230b663422681848Vignesh Venkatasubramanian
57ba6c59e9d7d7013b3906b6f4230b663422681848Vignesh Venkatasubramanian  THR_TM,
58ba6c59e9d7d7013b3906b6f4230b663422681848Vignesh Venkatasubramanian
59ba6c59e9d7d7013b3906b6f4230b663422681848Vignesh Venkatasubramanian  THR_COMP_NEARLA,
60ba6c59e9d7d7013b3906b6f4230b663422681848Vignesh Venkatasubramanian  THR_COMP_NEWLA,
61ba6c59e9d7d7013b3906b6f4230b663422681848Vignesh Venkatasubramanian  THR_NEARG,
62ba6c59e9d7d7013b3906b6f4230b663422681848Vignesh Venkatasubramanian  THR_COMP_NEARGA,
63ba6c59e9d7d7013b3906b6f4230b663422681848Vignesh Venkatasubramanian  THR_COMP_NEWGA,
64ba6c59e9d7d7013b3906b6f4230b663422681848Vignesh Venkatasubramanian
65ba6c59e9d7d7013b3906b6f4230b663422681848Vignesh Venkatasubramanian  THR_ZEROMV,
66ba6c59e9d7d7013b3906b6f4230b663422681848Vignesh Venkatasubramanian  THR_ZEROG,
67ba6c59e9d7d7013b3906b6f4230b663422681848Vignesh Venkatasubramanian  THR_ZEROA,
68ba6c59e9d7d7013b3906b6f4230b663422681848Vignesh Venkatasubramanian  THR_COMP_ZEROLA,
69ba6c59e9d7d7013b3906b6f4230b663422681848Vignesh Venkatasubramanian  THR_COMP_ZEROGA,
70ba6c59e9d7d7013b3906b6f4230b663422681848Vignesh Venkatasubramanian
71ba6c59e9d7d7013b3906b6f4230b663422681848Vignesh Venkatasubramanian  THR_H_PRED,
72ba6c59e9d7d7013b3906b6f4230b663422681848Vignesh Venkatasubramanian  THR_V_PRED,
73ba6c59e9d7d7013b3906b6f4230b663422681848Vignesh Venkatasubramanian  THR_D135_PRED,
74ba6c59e9d7d7013b3906b6f4230b663422681848Vignesh Venkatasubramanian  THR_D207_PRED,
75ba6c59e9d7d7013b3906b6f4230b663422681848Vignesh Venkatasubramanian  THR_D153_PRED,
76ba6c59e9d7d7013b3906b6f4230b663422681848Vignesh Venkatasubramanian  THR_D63_PRED,
77ba6c59e9d7d7013b3906b6f4230b663422681848Vignesh Venkatasubramanian  THR_D117_PRED,
78ba6c59e9d7d7013b3906b6f4230b663422681848Vignesh Venkatasubramanian  THR_D45_PRED,
79ba6c59e9d7d7013b3906b6f4230b663422681848Vignesh Venkatasubramanian} THR_MODES;
80ba6c59e9d7d7013b3906b6f4230b663422681848Vignesh Venkatasubramanian
81ba6c59e9d7d7013b3906b6f4230b663422681848Vignesh Venkatasubramaniantypedef enum {
82ba6c59e9d7d7013b3906b6f4230b663422681848Vignesh Venkatasubramanian  THR_LAST,
83ba6c59e9d7d7013b3906b6f4230b663422681848Vignesh Venkatasubramanian  THR_GOLD,
84ba6c59e9d7d7013b3906b6f4230b663422681848Vignesh Venkatasubramanian  THR_ALTR,
85ba6c59e9d7d7013b3906b6f4230b663422681848Vignesh Venkatasubramanian  THR_COMP_LA,
86ba6c59e9d7d7013b3906b6f4230b663422681848Vignesh Venkatasubramanian  THR_COMP_GA,
87ba6c59e9d7d7013b3906b6f4230b663422681848Vignesh Venkatasubramanian  THR_INTRA,
88ba6c59e9d7d7013b3906b6f4230b663422681848Vignesh Venkatasubramanian} THR_MODES_SUB8X8;
89ba6c59e9d7d7013b3906b6f4230b663422681848Vignesh Venkatasubramanian
90ba6c59e9d7d7013b3906b6f4230b663422681848Vignesh Venkatasubramaniantypedef struct RD_OPT {
91ba6c59e9d7d7013b3906b6f4230b663422681848Vignesh Venkatasubramanian  // Thresh_mult is used to set a threshold for the rd score. A higher value
92ba6c59e9d7d7013b3906b6f4230b663422681848Vignesh Venkatasubramanian  // means that we will accept the best mode so far more often. This number
93ba6c59e9d7d7013b3906b6f4230b663422681848Vignesh Venkatasubramanian  // is used in combination with the current block size, and thresh_freq_fact
94ba6c59e9d7d7013b3906b6f4230b663422681848Vignesh Venkatasubramanian  // to pick a threshold.
95ba6c59e9d7d7013b3906b6f4230b663422681848Vignesh Venkatasubramanian  int thresh_mult[MAX_MODES];
96ba6c59e9d7d7013b3906b6f4230b663422681848Vignesh Venkatasubramanian  int thresh_mult_sub8x8[MAX_REFS];
97ba6c59e9d7d7013b3906b6f4230b663422681848Vignesh Venkatasubramanian
98ba6c59e9d7d7013b3906b6f4230b663422681848Vignesh Venkatasubramanian  int threshes[MAX_SEGMENTS][BLOCK_SIZES][MAX_MODES];
99ba6c59e9d7d7013b3906b6f4230b663422681848Vignesh Venkatasubramanian  int thresh_freq_fact[BLOCK_SIZES][MAX_MODES];
100ba6c59e9d7d7013b3906b6f4230b663422681848Vignesh Venkatasubramanian
101ba6c59e9d7d7013b3906b6f4230b663422681848Vignesh Venkatasubramanian  int64_t comp_pred_diff[REFERENCE_MODES];
102ba6c59e9d7d7013b3906b6f4230b663422681848Vignesh Venkatasubramanian  int64_t prediction_type_threshes[MAX_REF_FRAMES][REFERENCE_MODES];
103ba6c59e9d7d7013b3906b6f4230b663422681848Vignesh Venkatasubramanian  int64_t tx_select_diff[TX_MODES];
104ba6c59e9d7d7013b3906b6f4230b663422681848Vignesh Venkatasubramanian  // TODO(agrange): can this overflow?
105ba6c59e9d7d7013b3906b6f4230b663422681848Vignesh Venkatasubramanian  int tx_select_threshes[MAX_REF_FRAMES][TX_MODES];
106ba6c59e9d7d7013b3906b6f4230b663422681848Vignesh Venkatasubramanian
107ba6c59e9d7d7013b3906b6f4230b663422681848Vignesh Venkatasubramanian  int64_t filter_diff[SWITCHABLE_FILTER_CONTEXTS];
108ba6c59e9d7d7013b3906b6f4230b663422681848Vignesh Venkatasubramanian  int64_t filter_threshes[MAX_REF_FRAMES][SWITCHABLE_FILTER_CONTEXTS];
109ba6c59e9d7d7013b3906b6f4230b663422681848Vignesh Venkatasubramanian  int64_t filter_cache[SWITCHABLE_FILTER_CONTEXTS];
110ba6c59e9d7d7013b3906b6f4230b663422681848Vignesh Venkatasubramanian  int64_t mask_filter;
111ba6c59e9d7d7013b3906b6f4230b663422681848Vignesh Venkatasubramanian
112ba6c59e9d7d7013b3906b6f4230b663422681848Vignesh Venkatasubramanian  int RDMULT;
113ba6c59e9d7d7013b3906b6f4230b663422681848Vignesh Venkatasubramanian  int RDDIV;
114ba6c59e9d7d7013b3906b6f4230b663422681848Vignesh Venkatasubramanian} RD_OPT;
115ba6c59e9d7d7013b3906b6f4230b663422681848Vignesh Venkatasubramanian
116ba6c59e9d7d7013b3906b6f4230b663422681848Vignesh Venkatasubramanianstruct TileInfo;
117ba6c59e9d7d7013b3906b6f4230b663422681848Vignesh Venkatasubramanianstruct VP9_COMP;
118ba6c59e9d7d7013b3906b6f4230b663422681848Vignesh Venkatasubramanianstruct macroblock;
119ba6c59e9d7d7013b3906b6f4230b663422681848Vignesh Venkatasubramanian
120ba6c59e9d7d7013b3906b6f4230b663422681848Vignesh Venkatasubramanianint vp9_compute_rd_mult(const struct VP9_COMP *cpi, int qindex);
121ba6c59e9d7d7013b3906b6f4230b663422681848Vignesh Venkatasubramanian
122ba6c59e9d7d7013b3906b6f4230b663422681848Vignesh Venkatasubramanianvoid vp9_initialize_rd_consts(struct VP9_COMP *cpi);
123ba6c59e9d7d7013b3906b6f4230b663422681848Vignesh Venkatasubramanian
124ba6c59e9d7d7013b3906b6f4230b663422681848Vignesh Venkatasubramanianvoid vp9_initialize_me_consts(struct VP9_COMP *cpi, int qindex);
125ba6c59e9d7d7013b3906b6f4230b663422681848Vignesh Venkatasubramanian
126ba6c59e9d7d7013b3906b6f4230b663422681848Vignesh Venkatasubramanianvoid vp9_model_rd_from_var_lapndz(unsigned int var, unsigned int n,
127ba6c59e9d7d7013b3906b6f4230b663422681848Vignesh Venkatasubramanian                                  unsigned int qstep, int *rate,
128ba6c59e9d7d7013b3906b6f4230b663422681848Vignesh Venkatasubramanian                                  int64_t *dist);
129ba6c59e9d7d7013b3906b6f4230b663422681848Vignesh Venkatasubramanian
130ba6c59e9d7d7013b3906b6f4230b663422681848Vignesh Venkatasubramanianint vp9_get_switchable_rate(const struct VP9_COMP *cpi);
131ba6c59e9d7d7013b3906b6f4230b663422681848Vignesh Venkatasubramanian
132ba6c59e9d7d7013b3906b6f4230b663422681848Vignesh Venkatasubramanianconst YV12_BUFFER_CONFIG *vp9_get_scaled_ref_frame(const struct VP9_COMP *cpi,
133ba6c59e9d7d7013b3906b6f4230b663422681848Vignesh Venkatasubramanian                                                   int ref_frame);
134ba6c59e9d7d7013b3906b6f4230b663422681848Vignesh Venkatasubramanian
135ba6c59e9d7d7013b3906b6f4230b663422681848Vignesh Venkatasubramanianvoid vp9_init_me_luts();
136ba6c59e9d7d7013b3906b6f4230b663422681848Vignesh Venkatasubramanian
137ba6c59e9d7d7013b3906b6f4230b663422681848Vignesh Venkatasubramanianvoid vp9_get_entropy_contexts(BLOCK_SIZE bsize, TX_SIZE tx_size,
138ba6c59e9d7d7013b3906b6f4230b663422681848Vignesh Venkatasubramanian                              const struct macroblockd_plane *pd,
139ba6c59e9d7d7013b3906b6f4230b663422681848Vignesh Venkatasubramanian                              ENTROPY_CONTEXT t_above[16],
140ba6c59e9d7d7013b3906b6f4230b663422681848Vignesh Venkatasubramanian                              ENTROPY_CONTEXT t_left[16]);
141ba6c59e9d7d7013b3906b6f4230b663422681848Vignesh Venkatasubramanian
142ba6c59e9d7d7013b3906b6f4230b663422681848Vignesh Venkatasubramanianvoid vp9_set_rd_speed_thresholds(struct VP9_COMP *cpi);
143ba6c59e9d7d7013b3906b6f4230b663422681848Vignesh Venkatasubramanian
144ba6c59e9d7d7013b3906b6f4230b663422681848Vignesh Venkatasubramanianvoid vp9_set_rd_speed_thresholds_sub8x8(struct VP9_COMP *cpi);
145ba6c59e9d7d7013b3906b6f4230b663422681848Vignesh Venkatasubramanian
146ba6c59e9d7d7013b3906b6f4230b663422681848Vignesh Venkatasubramanianstatic INLINE int rd_less_than_thresh(int64_t best_rd, int thresh,
147ba6c59e9d7d7013b3906b6f4230b663422681848Vignesh Venkatasubramanian                                      int thresh_fact) {
148ba6c59e9d7d7013b3906b6f4230b663422681848Vignesh Venkatasubramanian    return best_rd < ((int64_t)thresh * thresh_fact >> 5) || thresh == INT_MAX;
149ba6c59e9d7d7013b3906b6f4230b663422681848Vignesh Venkatasubramanian}
150ba6c59e9d7d7013b3906b6f4230b663422681848Vignesh Venkatasubramanian
151ba6c59e9d7d7013b3906b6f4230b663422681848Vignesh Venkatasubramanianvoid vp9_mv_pred(struct VP9_COMP *cpi, MACROBLOCK *x,
152ba6c59e9d7d7013b3906b6f4230b663422681848Vignesh Venkatasubramanian                 uint8_t *ref_y_buffer, int ref_y_stride,
153ba6c59e9d7d7013b3906b6f4230b663422681848Vignesh Venkatasubramanian                 int ref_frame, BLOCK_SIZE block_size);
154ba6c59e9d7d7013b3906b6f4230b663422681848Vignesh Venkatasubramanian
155ba6c59e9d7d7013b3906b6f4230b663422681848Vignesh Venkatasubramanianvoid vp9_setup_pred_block(const MACROBLOCKD *xd,
156ba6c59e9d7d7013b3906b6f4230b663422681848Vignesh Venkatasubramanian                          struct buf_2d dst[MAX_MB_PLANE],
157ba6c59e9d7d7013b3906b6f4230b663422681848Vignesh Venkatasubramanian                          const YV12_BUFFER_CONFIG *src,
158ba6c59e9d7d7013b3906b6f4230b663422681848Vignesh Venkatasubramanian                          int mi_row, int mi_col,
159ba6c59e9d7d7013b3906b6f4230b663422681848Vignesh Venkatasubramanian                          const struct scale_factors *scale,
160ba6c59e9d7d7013b3906b6f4230b663422681848Vignesh Venkatasubramanian                          const struct scale_factors *scale_uv);
161ba6c59e9d7d7013b3906b6f4230b663422681848Vignesh Venkatasubramanian#ifdef __cplusplus
162ba6c59e9d7d7013b3906b6f4230b663422681848Vignesh Venkatasubramanian}  // extern "C"
163ba6c59e9d7d7013b3906b6f4230b663422681848Vignesh Venkatasubramanian#endif
164ba6c59e9d7d7013b3906b6f4230b663422681848Vignesh Venkatasubramanian
165ba6c59e9d7d7013b3906b6f4230b663422681848Vignesh Venkatasubramanian#endif  // VP9_ENCODER_VP9_RD_H_
166