1647c00070712067fc5ae415f9106be5ca4e17464Keisuke Kuroynagi/* 2647c00070712067fc5ae415f9106be5ca4e17464Keisuke Kuroynagi * Copyright (C) 2013, The Android Open Source Project 3647c00070712067fc5ae415f9106be5ca4e17464Keisuke Kuroynagi * 4647c00070712067fc5ae415f9106be5ca4e17464Keisuke Kuroynagi * Licensed under the Apache License, Version 2.0 (the "License"); 5647c00070712067fc5ae415f9106be5ca4e17464Keisuke Kuroynagi * you may not use this file except in compliance with the License. 6647c00070712067fc5ae415f9106be5ca4e17464Keisuke Kuroynagi * You may obtain a copy of the License at 7647c00070712067fc5ae415f9106be5ca4e17464Keisuke Kuroynagi * 8647c00070712067fc5ae415f9106be5ca4e17464Keisuke Kuroynagi * http://www.apache.org/licenses/LICENSE-2.0 9647c00070712067fc5ae415f9106be5ca4e17464Keisuke Kuroynagi * 10647c00070712067fc5ae415f9106be5ca4e17464Keisuke Kuroynagi * Unless required by applicable law or agreed to in writing, software 11647c00070712067fc5ae415f9106be5ca4e17464Keisuke Kuroynagi * distributed under the License is distributed on an "AS IS" BASIS, 12647c00070712067fc5ae415f9106be5ca4e17464Keisuke Kuroynagi * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13647c00070712067fc5ae415f9106be5ca4e17464Keisuke Kuroynagi * See the License for the specific language governing permissions and 14647c00070712067fc5ae415f9106be5ca4e17464Keisuke Kuroynagi * limitations under the License. 15647c00070712067fc5ae415f9106be5ca4e17464Keisuke Kuroynagi */ 16647c00070712067fc5ae415f9106be5ca4e17464Keisuke Kuroynagi 17647c00070712067fc5ae415f9106be5ca4e17464Keisuke Kuroynagi#include "suggest/policyimpl/dictionary/patricia_trie_reading_utils.h" 18647c00070712067fc5ae415f9106be5ca4e17464Keisuke Kuroynagi 19647c00070712067fc5ae415f9106be5ca4e17464Keisuke Kuroynagi#include "defines.h" 2090619ae02b1861e644db6a0d72d0e5f9c195ef3eKeisuke Kuroyanagi#include "suggest/policyimpl/dictionary/utils/byte_array_utils.h" 21647c00070712067fc5ae415f9106be5ca4e17464Keisuke Kuroynagi 22647c00070712067fc5ae415f9106be5ca4e17464Keisuke Kuroynaginamespace latinime { 23647c00070712067fc5ae415f9106be5ca4e17464Keisuke Kuroynagi 24647c00070712067fc5ae415f9106be5ca4e17464Keisuke Kuroynagitypedef PatriciaTrieReadingUtils PtReadingUtils; 25647c00070712067fc5ae415f9106be5ca4e17464Keisuke Kuroynagi 2627b12933cd4e6dcb7363f0f33f3da8d7481bf7caKeisuke Kuroyanagiconst PtReadingUtils::NodeFlags PtReadingUtils::MASK_CHILDREN_POSITION_TYPE = 0xC0; 2727b12933cd4e6dcb7363f0f33f3da8d7481bf7caKeisuke Kuroyanagiconst PtReadingUtils::NodeFlags PtReadingUtils::FLAG_CHILDREN_POSITION_TYPE_NOPOSITION = 0x00; 2827b12933cd4e6dcb7363f0f33f3da8d7481bf7caKeisuke Kuroyanagiconst PtReadingUtils::NodeFlags PtReadingUtils::FLAG_CHILDREN_POSITION_TYPE_ONEBYTE = 0x40; 2927b12933cd4e6dcb7363f0f33f3da8d7481bf7caKeisuke Kuroyanagiconst PtReadingUtils::NodeFlags PtReadingUtils::FLAG_CHILDREN_POSITION_TYPE_TWOBYTES = 0x80; 3027b12933cd4e6dcb7363f0f33f3da8d7481bf7caKeisuke Kuroyanagiconst PtReadingUtils::NodeFlags PtReadingUtils::FLAG_CHILDREN_POSITION_TYPE_THREEBYTES = 0xC0; 31647c00070712067fc5ae415f9106be5ca4e17464Keisuke Kuroynagi 32647c00070712067fc5ae415f9106be5ca4e17464Keisuke Kuroynagi// Flag for single/multiple char group 33647c00070712067fc5ae415f9106be5ca4e17464Keisuke Kuroynagiconst PtReadingUtils::NodeFlags PtReadingUtils::FLAG_HAS_MULTIPLE_CHARS = 0x20; 3427b12933cd4e6dcb7363f0f33f3da8d7481bf7caKeisuke Kuroyanagi// Flag for terminal PtNodes 35647c00070712067fc5ae415f9106be5ca4e17464Keisuke Kuroynagiconst PtReadingUtils::NodeFlags PtReadingUtils::FLAG_IS_TERMINAL = 0x10; 36647c00070712067fc5ae415f9106be5ca4e17464Keisuke Kuroynagi// Flag for shortcut targets presence 37647c00070712067fc5ae415f9106be5ca4e17464Keisuke Kuroynagiconst PtReadingUtils::NodeFlags PtReadingUtils::FLAG_HAS_SHORTCUT_TARGETS = 0x08; 38647c00070712067fc5ae415f9106be5ca4e17464Keisuke Kuroynagi// Flag for bigram presence 39647c00070712067fc5ae415f9106be5ca4e17464Keisuke Kuroynagiconst PtReadingUtils::NodeFlags PtReadingUtils::FLAG_HAS_BIGRAMS = 0x04; 40647c00070712067fc5ae415f9106be5ca4e17464Keisuke Kuroynagi// Flag for non-words (typically, shortcut only entries) 41647c00070712067fc5ae415f9106be5ca4e17464Keisuke Kuroynagiconst PtReadingUtils::NodeFlags PtReadingUtils::FLAG_IS_NOT_A_WORD = 0x02; 42647c00070712067fc5ae415f9106be5ca4e17464Keisuke Kuroynagi// Flag for blacklist 43647c00070712067fc5ae415f9106be5ca4e17464Keisuke Kuroynagiconst PtReadingUtils::NodeFlags PtReadingUtils::FLAG_IS_BLACKLISTED = 0x01; 44647c00070712067fc5ae415f9106be5ca4e17464Keisuke Kuroynagi 459ea9c61c99b2fc8ff9a5bbd02c0ad81a828b930cKeisuke Kuroyanagi/* static */ int PtReadingUtils::getPtNodeArraySizeAndAdvancePosition( 469ea9c61c99b2fc8ff9a5bbd02c0ad81a828b930cKeisuke Kuroyanagi const uint8_t *const buffer, int *const pos) { 479ea9c61c99b2fc8ff9a5bbd02c0ad81a828b930cKeisuke Kuroyanagi const uint8_t firstByte = ByteArrayUtils::readUint8AndAdvancePosition(buffer, pos); 489ea9c61c99b2fc8ff9a5bbd02c0ad81a828b930cKeisuke Kuroyanagi if (firstByte < 0x80) { 499ea9c61c99b2fc8ff9a5bbd02c0ad81a828b930cKeisuke Kuroyanagi return firstByte; 509ea9c61c99b2fc8ff9a5bbd02c0ad81a828b930cKeisuke Kuroyanagi } else { 519ea9c61c99b2fc8ff9a5bbd02c0ad81a828b930cKeisuke Kuroyanagi return ((firstByte & 0x7F) << 8) ^ ByteArrayUtils::readUint8AndAdvancePosition( 529ea9c61c99b2fc8ff9a5bbd02c0ad81a828b930cKeisuke Kuroyanagi buffer, pos); 539ea9c61c99b2fc8ff9a5bbd02c0ad81a828b930cKeisuke Kuroyanagi } 549ea9c61c99b2fc8ff9a5bbd02c0ad81a828b930cKeisuke Kuroyanagi} 559ea9c61c99b2fc8ff9a5bbd02c0ad81a828b930cKeisuke Kuroyanagi 569ea9c61c99b2fc8ff9a5bbd02c0ad81a828b930cKeisuke Kuroyanagi/* static */ PtReadingUtils::NodeFlags PtReadingUtils::getFlagsAndAdvancePosition( 579ea9c61c99b2fc8ff9a5bbd02c0ad81a828b930cKeisuke Kuroyanagi const uint8_t *const buffer, int *const pos) { 589ea9c61c99b2fc8ff9a5bbd02c0ad81a828b930cKeisuke Kuroyanagi return ByteArrayUtils::readUint8AndAdvancePosition(buffer, pos); 599ea9c61c99b2fc8ff9a5bbd02c0ad81a828b930cKeisuke Kuroyanagi} 609ea9c61c99b2fc8ff9a5bbd02c0ad81a828b930cKeisuke Kuroyanagi 619ea9c61c99b2fc8ff9a5bbd02c0ad81a828b930cKeisuke Kuroyanagi/* static */ int PtReadingUtils::getCodePointAndAdvancePosition(const uint8_t *const buffer, 629ea9c61c99b2fc8ff9a5bbd02c0ad81a828b930cKeisuke Kuroyanagi int *const pos) { 639ea9c61c99b2fc8ff9a5bbd02c0ad81a828b930cKeisuke Kuroyanagi return ByteArrayUtils::readCodePointAndAdvancePosition(buffer, pos); 649ea9c61c99b2fc8ff9a5bbd02c0ad81a828b930cKeisuke Kuroyanagi} 659ea9c61c99b2fc8ff9a5bbd02c0ad81a828b930cKeisuke Kuroyanagi 669ea9c61c99b2fc8ff9a5bbd02c0ad81a828b930cKeisuke Kuroyanagi// Returns the number of read characters. 679ea9c61c99b2fc8ff9a5bbd02c0ad81a828b930cKeisuke Kuroyanagi/* static */ int PtReadingUtils::getCharsAndAdvancePosition(const uint8_t *const buffer, 689ea9c61c99b2fc8ff9a5bbd02c0ad81a828b930cKeisuke Kuroyanagi const NodeFlags flags, const int maxLength, int *const outBuffer, int *const pos) { 699ea9c61c99b2fc8ff9a5bbd02c0ad81a828b930cKeisuke Kuroyanagi int length = 0; 709ea9c61c99b2fc8ff9a5bbd02c0ad81a828b930cKeisuke Kuroyanagi if (hasMultipleChars(flags)) { 719ea9c61c99b2fc8ff9a5bbd02c0ad81a828b930cKeisuke Kuroyanagi length = ByteArrayUtils::readStringAndAdvancePosition(buffer, maxLength, outBuffer, 729ea9c61c99b2fc8ff9a5bbd02c0ad81a828b930cKeisuke Kuroyanagi pos); 739ea9c61c99b2fc8ff9a5bbd02c0ad81a828b930cKeisuke Kuroyanagi } else { 749155eec0d9a6749879b413a22f30ede2e170ce19Keisuke Kuroyanagi const int codePoint = getCodePointAndAdvancePosition(buffer, pos); 759155eec0d9a6749879b413a22f30ede2e170ce19Keisuke Kuroyanagi if (codePoint == NOT_A_CODE_POINT) { 769155eec0d9a6749879b413a22f30ede2e170ce19Keisuke Kuroyanagi // CAVEAT: codePoint == NOT_A_CODE_POINT means the code point is 779155eec0d9a6749879b413a22f30ede2e170ce19Keisuke Kuroyanagi // CHARACTER_ARRAY_TERMINATOR. The code point must not be CHARACTER_ARRAY_TERMINATOR 789155eec0d9a6749879b413a22f30ede2e170ce19Keisuke Kuroyanagi // when the PtNode has a single code point. 799155eec0d9a6749879b413a22f30ede2e170ce19Keisuke Kuroyanagi length = 0; 809155eec0d9a6749879b413a22f30ede2e170ce19Keisuke Kuroyanagi AKLOGE("codePoint is NOT_A_CODE_POINT. pos: %d, codePoint: 0x%x, buffer[pos - 1]: 0x%x", 819155eec0d9a6749879b413a22f30ede2e170ce19Keisuke Kuroyanagi *pos - 1, codePoint, buffer[*pos - 1]); 829155eec0d9a6749879b413a22f30ede2e170ce19Keisuke Kuroyanagi ASSERT(false); 839155eec0d9a6749879b413a22f30ede2e170ce19Keisuke Kuroyanagi } else if (maxLength > 0) { 849155eec0d9a6749879b413a22f30ede2e170ce19Keisuke Kuroyanagi outBuffer[0] = codePoint; 859ea9c61c99b2fc8ff9a5bbd02c0ad81a828b930cKeisuke Kuroyanagi length = 1; 869ea9c61c99b2fc8ff9a5bbd02c0ad81a828b930cKeisuke Kuroyanagi } 879ea9c61c99b2fc8ff9a5bbd02c0ad81a828b930cKeisuke Kuroyanagi } 889ea9c61c99b2fc8ff9a5bbd02c0ad81a828b930cKeisuke Kuroyanagi return length; 899ea9c61c99b2fc8ff9a5bbd02c0ad81a828b930cKeisuke Kuroyanagi} 909ea9c61c99b2fc8ff9a5bbd02c0ad81a828b930cKeisuke Kuroyanagi 919ea9c61c99b2fc8ff9a5bbd02c0ad81a828b930cKeisuke Kuroyanagi// Returns the number of skipped characters. 929ea9c61c99b2fc8ff9a5bbd02c0ad81a828b930cKeisuke Kuroyanagi/* static */ int PtReadingUtils::skipCharacters(const uint8_t *const buffer, const NodeFlags flags, 939ea9c61c99b2fc8ff9a5bbd02c0ad81a828b930cKeisuke Kuroyanagi const int maxLength, int *const pos) { 949ea9c61c99b2fc8ff9a5bbd02c0ad81a828b930cKeisuke Kuroyanagi if (hasMultipleChars(flags)) { 959ea9c61c99b2fc8ff9a5bbd02c0ad81a828b930cKeisuke Kuroyanagi return ByteArrayUtils::advancePositionToBehindString(buffer, maxLength, pos); 969ea9c61c99b2fc8ff9a5bbd02c0ad81a828b930cKeisuke Kuroyanagi } else { 979ea9c61c99b2fc8ff9a5bbd02c0ad81a828b930cKeisuke Kuroyanagi if (maxLength > 0) { 989ea9c61c99b2fc8ff9a5bbd02c0ad81a828b930cKeisuke Kuroyanagi getCodePointAndAdvancePosition(buffer, pos); 999ea9c61c99b2fc8ff9a5bbd02c0ad81a828b930cKeisuke Kuroyanagi return 1; 1009ea9c61c99b2fc8ff9a5bbd02c0ad81a828b930cKeisuke Kuroyanagi } else { 1019ea9c61c99b2fc8ff9a5bbd02c0ad81a828b930cKeisuke Kuroyanagi return 0; 1029ea9c61c99b2fc8ff9a5bbd02c0ad81a828b930cKeisuke Kuroyanagi } 1039ea9c61c99b2fc8ff9a5bbd02c0ad81a828b930cKeisuke Kuroyanagi } 1049ea9c61c99b2fc8ff9a5bbd02c0ad81a828b930cKeisuke Kuroyanagi} 1059ea9c61c99b2fc8ff9a5bbd02c0ad81a828b930cKeisuke Kuroyanagi 1069ea9c61c99b2fc8ff9a5bbd02c0ad81a828b930cKeisuke Kuroyanagi/* static */ int PtReadingUtils::readProbabilityAndAdvancePosition(const uint8_t *const buffer, 1079ea9c61c99b2fc8ff9a5bbd02c0ad81a828b930cKeisuke Kuroyanagi int *const pos) { 1089ea9c61c99b2fc8ff9a5bbd02c0ad81a828b930cKeisuke Kuroyanagi return ByteArrayUtils::readUint8AndAdvancePosition(buffer, pos); 1099ea9c61c99b2fc8ff9a5bbd02c0ad81a828b930cKeisuke Kuroyanagi} 1109ea9c61c99b2fc8ff9a5bbd02c0ad81a828b930cKeisuke Kuroyanagi 111647c00070712067fc5ae415f9106be5ca4e17464Keisuke Kuroynagi/* static */ int PtReadingUtils::readChildrenPositionAndAdvancePosition( 112647c00070712067fc5ae415f9106be5ca4e17464Keisuke Kuroynagi const uint8_t *const buffer, const NodeFlags flags, int *const pos) { 113647c00070712067fc5ae415f9106be5ca4e17464Keisuke Kuroynagi const int base = *pos; 114647c00070712067fc5ae415f9106be5ca4e17464Keisuke Kuroynagi int offset = 0; 11527b12933cd4e6dcb7363f0f33f3da8d7481bf7caKeisuke Kuroyanagi switch (MASK_CHILDREN_POSITION_TYPE & flags) { 11627b12933cd4e6dcb7363f0f33f3da8d7481bf7caKeisuke Kuroyanagi case FLAG_CHILDREN_POSITION_TYPE_ONEBYTE: 117647c00070712067fc5ae415f9106be5ca4e17464Keisuke Kuroynagi offset = ByteArrayUtils::readUint8AndAdvancePosition(buffer, pos); 118647c00070712067fc5ae415f9106be5ca4e17464Keisuke Kuroynagi break; 11927b12933cd4e6dcb7363f0f33f3da8d7481bf7caKeisuke Kuroyanagi case FLAG_CHILDREN_POSITION_TYPE_TWOBYTES: 120647c00070712067fc5ae415f9106be5ca4e17464Keisuke Kuroynagi offset = ByteArrayUtils::readUint16AndAdvancePosition(buffer, pos); 121647c00070712067fc5ae415f9106be5ca4e17464Keisuke Kuroynagi break; 12227b12933cd4e6dcb7363f0f33f3da8d7481bf7caKeisuke Kuroyanagi case FLAG_CHILDREN_POSITION_TYPE_THREEBYTES: 123647c00070712067fc5ae415f9106be5ca4e17464Keisuke Kuroynagi offset = ByteArrayUtils::readUint24AndAdvancePosition(buffer, pos); 124647c00070712067fc5ae415f9106be5ca4e17464Keisuke Kuroynagi break; 125647c00070712067fc5ae415f9106be5ca4e17464Keisuke Kuroynagi default: 126647c00070712067fc5ae415f9106be5ca4e17464Keisuke Kuroynagi // If we come here, it means we asked for the children of a word with 127647c00070712067fc5ae415f9106be5ca4e17464Keisuke Kuroynagi // no children. 128647c00070712067fc5ae415f9106be5ca4e17464Keisuke Kuroynagi return NOT_A_DICT_POS; 129647c00070712067fc5ae415f9106be5ca4e17464Keisuke Kuroynagi } 130647c00070712067fc5ae415f9106be5ca4e17464Keisuke Kuroynagi return base + offset; 131647c00070712067fc5ae415f9106be5ca4e17464Keisuke Kuroynagi} 132647c00070712067fc5ae415f9106be5ca4e17464Keisuke Kuroynagi 133647c00070712067fc5ae415f9106be5ca4e17464Keisuke Kuroynagi} // namespace latinime 134