1/*
2 * Copyright (C) 2013, The Android Open Source Project
3 *
4 * Licensed under the Apache License, Version 2.0 (the "License");
5 * you may not use this file except in compliance with the License.
6 * You may obtain a copy of the License at
7 *
8 *     http://www.apache.org/licenses/LICENSE-2.0
9 *
10 * Unless required by applicable law or agreed to in writing, software
11 * distributed under the License is distributed on an "AS IS" BASIS,
12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 * See the License for the specific language governing permissions and
14 * limitations under the License.
15 */
16
17#ifndef LATINIME_PROBABILITY_UTILS_H
18#define LATINIME_PROBABILITY_UTILS_H
19
20#include <algorithm>
21#include <cmath>
22
23#include "defines.h"
24
25namespace latinime {
26
27// TODO: Quit using bigram probability to indicate the delta.
28class ProbabilityUtils {
29 public:
30    static AK_FORCE_INLINE int backoff(const int unigramProbability) {
31        return unigramProbability;
32        // For some reason, applying the backoff weight gives bad results in tests. To apply the
33        // backoff weight, we divide the probability by 2, which in our storing format means
34        // decreasing the score by 8.
35        // TODO: figure out what's wrong with this.
36        // return unigramProbability > 8 ?
37        //         unigramProbability - 8 : (0 == unigramProbability ? 0 : 8);
38    }
39
40    static AK_FORCE_INLINE int computeProbabilityForBigram(
41            const int unigramProbability, const int bigramProbability) {
42        // We divide the range [unigramProbability..255] in 16.5 steps - in other words, we want
43        // the unigram probability to be the median value of the 17th step from the top. A value of
44        // 0 for the bigram probability represents the middle of the 16th step from the top,
45        // while a value of 15 represents the middle of the top step.
46        // See makedict.BinaryDictEncoder#makeBigramFlags for details.
47        const float stepSize = static_cast<float>(MAX_PROBABILITY - unigramProbability)
48                / (1.5f + MAX_BIGRAM_ENCODED_PROBABILITY);
49        return unigramProbability
50                + static_cast<int>(static_cast<float>(bigramProbability + 1) * stepSize);
51    }
52
53    // Encode probability using the same way as we are doing for main dictionaries.
54    static AK_FORCE_INLINE int encodeRawProbability(const float rawProbability) {
55        const float probability = static_cast<float>(MAX_PROBABILITY)
56                + log2f(rawProbability) * PROBABILITY_ENCODING_SCALER;
57        if (probability < 0.0f) {
58            return 0;
59        }
60        return std::min(static_cast<int>(probability + 0.5f), MAX_PROBABILITY);
61    }
62
63 private:
64    DISALLOW_IMPLICIT_CONSTRUCTORS(ProbabilityUtils);
65
66    static const float PROBABILITY_ENCODING_SCALER;
67};
68}
69#endif /* LATINIME_PROBABILITY_UTILS_H */
70