Lines Matching refs:bits
42 dequant = 2^(scale_factor+1) * ((raw * 2.0 + 1.0) / ((2^bits) - 1) - 1)
44 2 <= bits <= 16
45 0 <= raw < (2^bits)-1 (the -1 is because quantized values with all 1's are forbidden)
52 dequant_long_scaled table for 16 bits is as accurate as possible, since it has
60 dequant_long_long[bits] = round(2^31 * 1/((2^bits - 1) / 1.38...) for 2 <= bits <= 16
67 dequant_long_scaled[bits] <= 2^31 / ((2^bits - 1)) for 2 <= bits <= 16
73 d = 2 * raw + 1 1 <= d <= 2^bits - 2
75 d' = d * dequant_long[bits]
77 d * dequant_long_scaled[bits] <= (2^bits - 2) * (2^31 / (2^bits - 1))
78 d * dequant_long_scaled[bits] <= 2^31 * (2^bits - 2)/(2^bits - 1) < 2^31
85 d' =~ 2^31 * (raw * 2.0 + 1.0) / (2^bits - 1) / 1.38...
87 result = d' - 2^31/1.38... =~ 2^31 * ((raw * 2.0 + 1.0) / (2^bits - 1) - 1) / 1.38...
115 /** Scales x by y bits to the right, adding a rounding factor.
125 INLINE float dequant_float(OI_UINT32 raw, OI_UINT scale_factor, OI_UINT bits)
127 float result = (1 << (scale_factor+1)) * ((raw * 2.0f + 1.0f) / ((1 << bits) - 1.0f) - 1.0f);
133 * values with all 1-bits set can, theoretically, trigger this assert. This
144 INLINE OI_INT32 OI_SBC_Dequant(OI_UINT32 raw, OI_UINT scale_factor, OI_UINT bits)
150 OI_ASSERT(bits <= 16);
152 if (bits <= 1) {
157 d *= dequant_long_scaled[bits];
165 float_result = dequant_float(raw, scale_factor, bits);
183 INLINE OI_INT32 OI_SBC_Dequant_Unscaled(OI_UINT32 raw, OI_UINT scale_factor, OI_UINT bits)
189 OI_ASSERT(bits <= 16);
192 if (bits <= 1) {
195 if (bits == 16) {
202 d *= dequant_long_unscaled[bits];