1bfbbee8c5757aef4a20879547c16af0a4d1bf4c7Jean Chalard/* 2bfbbee8c5757aef4a20879547c16af0a4d1bf4c7Jean Chalard * Copyright (C) 2011 The Android Open Source Project 3bfbbee8c5757aef4a20879547c16af0a4d1bf4c7Jean Chalard * 4bfbbee8c5757aef4a20879547c16af0a4d1bf4c7Jean Chalard * Licensed under the Apache License, Version 2.0 (the "License"); you may not 5bfbbee8c5757aef4a20879547c16af0a4d1bf4c7Jean Chalard * use this file except in compliance with the License. You may obtain a copy of 6bfbbee8c5757aef4a20879547c16af0a4d1bf4c7Jean Chalard * the License at 7bfbbee8c5757aef4a20879547c16af0a4d1bf4c7Jean Chalard * 8bfbbee8c5757aef4a20879547c16af0a4d1bf4c7Jean Chalard * http://www.apache.org/licenses/LICENSE-2.0 9bfbbee8c5757aef4a20879547c16af0a4d1bf4c7Jean Chalard * 10bfbbee8c5757aef4a20879547c16af0a4d1bf4c7Jean Chalard * Unless required by applicable law or agreed to in writing, software 11bfbbee8c5757aef4a20879547c16af0a4d1bf4c7Jean Chalard * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT 12bfbbee8c5757aef4a20879547c16af0a4d1bf4c7Jean Chalard * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the 13bfbbee8c5757aef4a20879547c16af0a4d1bf4c7Jean Chalard * License for the specific language governing permissions and limitations under 14bfbbee8c5757aef4a20879547c16af0a4d1bf4c7Jean Chalard * the License. 15bfbbee8c5757aef4a20879547c16af0a4d1bf4c7Jean Chalard */ 16bfbbee8c5757aef4a20879547c16af0a4d1bf4c7Jean Chalard 17cdc51fc6afc7fd374c5c9eeb0539cae5cf1de724Tom Ouyangpackage com.android.inputmethod.latin.makedict; 18bfbbee8c5757aef4a20879547c16af0a4d1bf4c7Jean Chalard 191a347723c5ad4a71076df67f3af3b702db205719Yuichiro Hanadaimport com.android.inputmethod.latin.makedict.FormatSpec.FileHeader; 201a347723c5ad4a71076df67f3af3b702db205719Yuichiro Hanadaimport com.android.inputmethod.latin.makedict.FormatSpec.FormatOptions; 21cdc51fc6afc7fd374c5c9eeb0539cae5cf1de724Tom Ouyangimport com.android.inputmethod.latin.makedict.FusionDictionary.CharGroup; 22f420df28233c26e555d203185fb292e83b94b8c3Jean Chalardimport com.android.inputmethod.latin.makedict.FusionDictionary.DictionaryOptions; 23cdc51fc6afc7fd374c5c9eeb0539cae5cf1de724Tom Ouyangimport com.android.inputmethod.latin.makedict.FusionDictionary.Node; 24cdc51fc6afc7fd374c5c9eeb0539cae5cf1de724Tom Ouyangimport com.android.inputmethod.latin.makedict.FusionDictionary.WeightedString; 25bfbbee8c5757aef4a20879547c16af0a4d1bf4c7Jean Chalard 26df7ebbbd616fa5aff569d00b16cd3f85ddf2da6dTom Ouyangimport java.io.ByteArrayOutputStream; 27d4fe7fda303ff937d2e44c15dde9d90cbf59376bYuichiro Hanadaimport java.io.File; 28d4fe7fda303ff937d2e44c15dde9d90cbf59376bYuichiro Hanadaimport java.io.FileInputStream; 29bfbbee8c5757aef4a20879547c16af0a4d1bf4c7Jean Chalardimport java.io.FileNotFoundException; 30bfbbee8c5757aef4a20879547c16af0a4d1bf4c7Jean Chalardimport java.io.IOException; 31bfbbee8c5757aef4a20879547c16af0a4d1bf4c7Jean Chalardimport java.io.OutputStream; 32d4fe7fda303ff937d2e44c15dde9d90cbf59376bYuichiro Hanadaimport java.nio.ByteBuffer; 33d4fe7fda303ff937d2e44c15dde9d90cbf59376bYuichiro Hanadaimport java.nio.channels.FileChannel; 34bfbbee8c5757aef4a20879547c16af0a4d1bf4c7Jean Chalardimport java.util.ArrayList; 35bfbbee8c5757aef4a20879547c16af0a4d1bf4c7Jean Chalardimport java.util.Arrays; 36c734c2aca1830643d169fd292e0c9d4d9306af5aJean Chalardimport java.util.HashMap; 378edd3067181a425ce1383bb950184f9742af7557Jean Chalardimport java.util.Iterator; 38bfbbee8c5757aef4a20879547c16af0a4d1bf4c7Jean Chalardimport java.util.Map; 39bfbbee8c5757aef4a20879547c16af0a4d1bf4c7Jean Chalardimport java.util.TreeMap; 40bfbbee8c5757aef4a20879547c16af0a4d1bf4c7Jean Chalard 41bfbbee8c5757aef4a20879547c16af0a4d1bf4c7Jean Chalard/** 42bfbbee8c5757aef4a20879547c16af0a4d1bf4c7Jean Chalard * Reads and writes XML files for a FusionDictionary. 43bfbbee8c5757aef4a20879547c16af0a4d1bf4c7Jean Chalard * 44bfbbee8c5757aef4a20879547c16af0a4d1bf4c7Jean Chalard * All the methods in this class are static. 45bfbbee8c5757aef4a20879547c16af0a4d1bf4c7Jean Chalard */ 46a28a05e971cc242b338331a3b78276fa95188d19Tadashi G. Takaokapublic final class BinaryDictInputOutput { 47bfbbee8c5757aef4a20879547c16af0a4d1bf4c7Jean Chalard 48e55b644aefbefb4ac79308c9a59116e69a9c53a2Yuichiro Hanada private static final boolean DBG = MakedictLog.DBG; 494df5b43df8f4b29fbfab9180cffe5742f8b5f512Jean Chalard 50a64a1a46e482664dcebdf4fee0745a890d0d70dcJean Chalard // Arbitrary limit to how much passes we consider address size compression should 51a64a1a46e482664dcebdf4fee0745a890d0d70dcJean Chalard // terminate in. At the time of this writing, our largest dictionary completes 52a64a1a46e482664dcebdf4fee0745a890d0d70dcJean Chalard // compression in five passes. 53a64a1a46e482664dcebdf4fee0745a890d0d70dcJean Chalard // If the number of passes exceeds this number, makedict bails with an exception on 54a64a1a46e482664dcebdf4fee0745a890d0d70dcJean Chalard // suspicion that a bug might be causing an infinite loop. 55a64a1a46e482664dcebdf4fee0745a890d0d70dcJean Chalard private static final int MAX_PASSES = 24; 562ee70804e92b17016a2f042c4f6b0e94b5d23e88Yuichiro Hanada private static final int MAX_JUMPS = 12; 57a64a1a46e482664dcebdf4fee0745a890d0d70dcJean Chalard 58666a4338026866df1f18dd6b3f968c3788943e4cYuichiro Hanada public interface FusionDictionaryBufferInterface { 59f5c4ff481782831329593760b000f0543680930aYuichiro Hanada public int readUnsignedByte(); 60f5c4ff481782831329593760b000f0543680930aYuichiro Hanada public int readUnsignedShort(); 61f5c4ff481782831329593760b000f0543680930aYuichiro Hanada public int readUnsignedInt24(); 62f5c4ff481782831329593760b000f0543680930aYuichiro Hanada public int readInt(); 63f5c4ff481782831329593760b000f0543680930aYuichiro Hanada public int position(); 64f5c4ff481782831329593760b000f0543680930aYuichiro Hanada public void position(int newPosition); 658d031a63b4d59d4d8670b4310dd9e18a0e03435aYuichiro Hanada public void put(final byte b); 66a149c53c8ebe1b2acb7ee92eac51ccdc364162e7Yuichiro Hanada public int limit(); 67a161bdac885fc8e5f0063d33b055b0a6ecdefbdbYuichiro Hanada public int capacity(); 68f5c4ff481782831329593760b000f0543680930aYuichiro Hanada } 69f5c4ff481782831329593760b000f0543680930aYuichiro Hanada 70b2a43a2ed4df8c3cacf21168cd742e30fa37e964Yuichiro Hanada public static final class ByteBufferWrapper implements FusionDictionaryBufferInterface { 71b2a43a2ed4df8c3cacf21168cd742e30fa37e964Yuichiro Hanada private ByteBuffer mBuffer; 72b2a43a2ed4df8c3cacf21168cd742e30fa37e964Yuichiro Hanada 73b2a43a2ed4df8c3cacf21168cd742e30fa37e964Yuichiro Hanada public ByteBufferWrapper(final ByteBuffer buffer) { 74b2a43a2ed4df8c3cacf21168cd742e30fa37e964Yuichiro Hanada mBuffer = buffer; 75f5c4ff481782831329593760b000f0543680930aYuichiro Hanada } 76f5c4ff481782831329593760b000f0543680930aYuichiro Hanada 77f5c4ff481782831329593760b000f0543680930aYuichiro Hanada @Override 78f5c4ff481782831329593760b000f0543680930aYuichiro Hanada public int readUnsignedByte() { 79b2a43a2ed4df8c3cacf21168cd742e30fa37e964Yuichiro Hanada return ((int)mBuffer.get()) & 0xFF; 80f5c4ff481782831329593760b000f0543680930aYuichiro Hanada } 81f5c4ff481782831329593760b000f0543680930aYuichiro Hanada 82f5c4ff481782831329593760b000f0543680930aYuichiro Hanada @Override 83f5c4ff481782831329593760b000f0543680930aYuichiro Hanada public int readUnsignedShort() { 84b2a43a2ed4df8c3cacf21168cd742e30fa37e964Yuichiro Hanada return ((int)mBuffer.getShort()) & 0xFFFF; 85f5c4ff481782831329593760b000f0543680930aYuichiro Hanada } 86f5c4ff481782831329593760b000f0543680930aYuichiro Hanada 87f5c4ff481782831329593760b000f0543680930aYuichiro Hanada @Override 88f5c4ff481782831329593760b000f0543680930aYuichiro Hanada public int readUnsignedInt24() { 89f5c4ff481782831329593760b000f0543680930aYuichiro Hanada final int retval = readUnsignedByte(); 90f5c4ff481782831329593760b000f0543680930aYuichiro Hanada return (retval << 16) + readUnsignedShort(); 91f5c4ff481782831329593760b000f0543680930aYuichiro Hanada } 92f5c4ff481782831329593760b000f0543680930aYuichiro Hanada 93f5c4ff481782831329593760b000f0543680930aYuichiro Hanada @Override 94f5c4ff481782831329593760b000f0543680930aYuichiro Hanada public int readInt() { 95b2a43a2ed4df8c3cacf21168cd742e30fa37e964Yuichiro Hanada return mBuffer.getInt(); 96f5c4ff481782831329593760b000f0543680930aYuichiro Hanada } 97f5c4ff481782831329593760b000f0543680930aYuichiro Hanada 98f5c4ff481782831329593760b000f0543680930aYuichiro Hanada @Override 99f5c4ff481782831329593760b000f0543680930aYuichiro Hanada public int position() { 100b2a43a2ed4df8c3cacf21168cd742e30fa37e964Yuichiro Hanada return mBuffer.position(); 101f5c4ff481782831329593760b000f0543680930aYuichiro Hanada } 102f5c4ff481782831329593760b000f0543680930aYuichiro Hanada 103f5c4ff481782831329593760b000f0543680930aYuichiro Hanada @Override 104f5c4ff481782831329593760b000f0543680930aYuichiro Hanada public void position(int newPos) { 105b2a43a2ed4df8c3cacf21168cd742e30fa37e964Yuichiro Hanada mBuffer.position(newPos); 106f5c4ff481782831329593760b000f0543680930aYuichiro Hanada } 1078d031a63b4d59d4d8670b4310dd9e18a0e03435aYuichiro Hanada 1088d031a63b4d59d4d8670b4310dd9e18a0e03435aYuichiro Hanada @Override 1098d031a63b4d59d4d8670b4310dd9e18a0e03435aYuichiro Hanada public void put(final byte b) { 1108d031a63b4d59d4d8670b4310dd9e18a0e03435aYuichiro Hanada mBuffer.put(b); 1118d031a63b4d59d4d8670b4310dd9e18a0e03435aYuichiro Hanada } 112a149c53c8ebe1b2acb7ee92eac51ccdc364162e7Yuichiro Hanada 113a149c53c8ebe1b2acb7ee92eac51ccdc364162e7Yuichiro Hanada @Override 114a149c53c8ebe1b2acb7ee92eac51ccdc364162e7Yuichiro Hanada public int limit() { 115a149c53c8ebe1b2acb7ee92eac51ccdc364162e7Yuichiro Hanada return mBuffer.limit(); 116a149c53c8ebe1b2acb7ee92eac51ccdc364162e7Yuichiro Hanada } 117a161bdac885fc8e5f0063d33b055b0a6ecdefbdbYuichiro Hanada 118a161bdac885fc8e5f0063d33b055b0a6ecdefbdbYuichiro Hanada @Override 119a161bdac885fc8e5f0063d33b055b0a6ecdefbdbYuichiro Hanada public int capacity() { 120a161bdac885fc8e5f0063d33b055b0a6ecdefbdbYuichiro Hanada return mBuffer.capacity(); 121a161bdac885fc8e5f0063d33b055b0a6ecdefbdbYuichiro Hanada } 122f5c4ff481782831329593760b000f0543680930aYuichiro Hanada } 123f5c4ff481782831329593760b000f0543680930aYuichiro Hanada 124bfbbee8c5757aef4a20879547c16af0a4d1bf4c7Jean Chalard /** 125bfbbee8c5757aef4a20879547c16af0a4d1bf4c7Jean Chalard * A class grouping utility function for our specific character encoding. 126bfbbee8c5757aef4a20879547c16af0a4d1bf4c7Jean Chalard */ 127a28a05e971cc242b338331a3b78276fa95188d19Tadashi G. Takaoka private static final class CharEncoding { 128bfbbee8c5757aef4a20879547c16af0a4d1bf4c7Jean Chalard 129bfbbee8c5757aef4a20879547c16af0a4d1bf4c7Jean Chalard private static final int MINIMAL_ONE_BYTE_CHARACTER_VALUE = 0x20; 130bfbbee8c5757aef4a20879547c16af0a4d1bf4c7Jean Chalard private static final int MAXIMAL_ONE_BYTE_CHARACTER_VALUE = 0xFF; 131bfbbee8c5757aef4a20879547c16af0a4d1bf4c7Jean Chalard 132bfbbee8c5757aef4a20879547c16af0a4d1bf4c7Jean Chalard /** 133bfbbee8c5757aef4a20879547c16af0a4d1bf4c7Jean Chalard * Helper method to find out whether this code fits on one byte 134bfbbee8c5757aef4a20879547c16af0a4d1bf4c7Jean Chalard */ 135e55b644aefbefb4ac79308c9a59116e69a9c53a2Yuichiro Hanada private static boolean fitsOnOneByte(final int character) { 136bfbbee8c5757aef4a20879547c16af0a4d1bf4c7Jean Chalard return character >= MINIMAL_ONE_BYTE_CHARACTER_VALUE 137bfbbee8c5757aef4a20879547c16af0a4d1bf4c7Jean Chalard && character <= MAXIMAL_ONE_BYTE_CHARACTER_VALUE; 138bfbbee8c5757aef4a20879547c16af0a4d1bf4c7Jean Chalard } 139bfbbee8c5757aef4a20879547c16af0a4d1bf4c7Jean Chalard 140bfbbee8c5757aef4a20879547c16af0a4d1bf4c7Jean Chalard /** 141bfbbee8c5757aef4a20879547c16af0a4d1bf4c7Jean Chalard * Compute the size of a character given its character code. 142bfbbee8c5757aef4a20879547c16af0a4d1bf4c7Jean Chalard * 143bfbbee8c5757aef4a20879547c16af0a4d1bf4c7Jean Chalard * Char format is: 144bfbbee8c5757aef4a20879547c16af0a4d1bf4c7Jean Chalard * 1 byte = bbbbbbbb match 145bfbbee8c5757aef4a20879547c16af0a4d1bf4c7Jean Chalard * case 000xxxxx: xxxxx << 16 + next byte << 8 + next byte 146bfbbee8c5757aef4a20879547c16af0a4d1bf4c7Jean Chalard * else: if 00011111 (= 0x1F) : this is the terminator. This is a relevant choice because 147bfbbee8c5757aef4a20879547c16af0a4d1bf4c7Jean Chalard * unicode code points range from 0 to 0x10FFFF, so any 3-byte value starting with 148bfbbee8c5757aef4a20879547c16af0a4d1bf4c7Jean Chalard * 00011111 would be outside unicode. 149bfbbee8c5757aef4a20879547c16af0a4d1bf4c7Jean Chalard * else: iso-latin-1 code 150bfbbee8c5757aef4a20879547c16af0a4d1bf4c7Jean Chalard * This allows for the whole unicode range to be encoded, including chars outside of 151bfbbee8c5757aef4a20879547c16af0a4d1bf4c7Jean Chalard * the BMP. Also everything in the iso-latin-1 charset is only 1 byte, except control 152bfbbee8c5757aef4a20879547c16af0a4d1bf4c7Jean Chalard * characters which should never happen anyway (and still work, but take 3 bytes). 153bfbbee8c5757aef4a20879547c16af0a4d1bf4c7Jean Chalard * 154bfbbee8c5757aef4a20879547c16af0a4d1bf4c7Jean Chalard * @param character the character code. 155bfbbee8c5757aef4a20879547c16af0a4d1bf4c7Jean Chalard * @return the size in binary encoded-form, either 1 or 3 bytes. 156bfbbee8c5757aef4a20879547c16af0a4d1bf4c7Jean Chalard */ 157e55b644aefbefb4ac79308c9a59116e69a9c53a2Yuichiro Hanada private static int getCharSize(final int character) { 158bfbbee8c5757aef4a20879547c16af0a4d1bf4c7Jean Chalard // See char encoding in FusionDictionary.java 159bfbbee8c5757aef4a20879547c16af0a4d1bf4c7Jean Chalard if (fitsOnOneByte(character)) return 1; 16081d97eec0e77e72cce606f9c9f96091c0b348190Yuichiro Hanada if (FormatSpec.INVALID_CHARACTER == character) return 1; 161bfbbee8c5757aef4a20879547c16af0a4d1bf4c7Jean Chalard return 3; 162bfbbee8c5757aef4a20879547c16af0a4d1bf4c7Jean Chalard } 163bfbbee8c5757aef4a20879547c16af0a4d1bf4c7Jean Chalard 164bfbbee8c5757aef4a20879547c16af0a4d1bf4c7Jean Chalard /** 165bfbbee8c5757aef4a20879547c16af0a4d1bf4c7Jean Chalard * Compute the byte size of a character array. 166bfbbee8c5757aef4a20879547c16af0a4d1bf4c7Jean Chalard */ 167bfbbee8c5757aef4a20879547c16af0a4d1bf4c7Jean Chalard private static int getCharArraySize(final int[] chars) { 168bfbbee8c5757aef4a20879547c16af0a4d1bf4c7Jean Chalard int size = 0; 169bfbbee8c5757aef4a20879547c16af0a4d1bf4c7Jean Chalard for (int character : chars) size += getCharSize(character); 170bfbbee8c5757aef4a20879547c16af0a4d1bf4c7Jean Chalard return size; 171bfbbee8c5757aef4a20879547c16af0a4d1bf4c7Jean Chalard } 172bfbbee8c5757aef4a20879547c16af0a4d1bf4c7Jean Chalard 173bfbbee8c5757aef4a20879547c16af0a4d1bf4c7Jean Chalard /** 174bfbbee8c5757aef4a20879547c16af0a4d1bf4c7Jean Chalard * Writes a char array to a byte buffer. 175bfbbee8c5757aef4a20879547c16af0a4d1bf4c7Jean Chalard * 1763bbb31f3f00e64cb68bd5877ae69d6dbccfeb519Jean Chalard * @param codePoints the code point array to write. 177bfbbee8c5757aef4a20879547c16af0a4d1bf4c7Jean Chalard * @param buffer the byte buffer to write to. 178bfbbee8c5757aef4a20879547c16af0a4d1bf4c7Jean Chalard * @param index the index in buffer to write the character array to. 179bfbbee8c5757aef4a20879547c16af0a4d1bf4c7Jean Chalard * @return the index after the last character. 180bfbbee8c5757aef4a20879547c16af0a4d1bf4c7Jean Chalard */ 1813bbb31f3f00e64cb68bd5877ae69d6dbccfeb519Jean Chalard private static int writeCharArray(final int[] codePoints, final byte[] buffer, int index) { 1823bbb31f3f00e64cb68bd5877ae69d6dbccfeb519Jean Chalard for (int codePoint : codePoints) { 1833bbb31f3f00e64cb68bd5877ae69d6dbccfeb519Jean Chalard if (1 == getCharSize(codePoint)) { 1843bbb31f3f00e64cb68bd5877ae69d6dbccfeb519Jean Chalard buffer[index++] = (byte)codePoint; 185bfbbee8c5757aef4a20879547c16af0a4d1bf4c7Jean Chalard } else { 1863bbb31f3f00e64cb68bd5877ae69d6dbccfeb519Jean Chalard buffer[index++] = (byte)(0xFF & (codePoint >> 16)); 1873bbb31f3f00e64cb68bd5877ae69d6dbccfeb519Jean Chalard buffer[index++] = (byte)(0xFF & (codePoint >> 8)); 1883bbb31f3f00e64cb68bd5877ae69d6dbccfeb519Jean Chalard buffer[index++] = (byte)(0xFF & codePoint); 189bfbbee8c5757aef4a20879547c16af0a4d1bf4c7Jean Chalard } 190bfbbee8c5757aef4a20879547c16af0a4d1bf4c7Jean Chalard } 191bfbbee8c5757aef4a20879547c16af0a4d1bf4c7Jean Chalard return index; 192bfbbee8c5757aef4a20879547c16af0a4d1bf4c7Jean Chalard } 193bfbbee8c5757aef4a20879547c16af0a4d1bf4c7Jean Chalard 194bfbbee8c5757aef4a20879547c16af0a4d1bf4c7Jean Chalard /** 1953bbb31f3f00e64cb68bd5877ae69d6dbccfeb519Jean Chalard * Writes a string with our character format to a byte buffer. 1963bbb31f3f00e64cb68bd5877ae69d6dbccfeb519Jean Chalard * 1973bbb31f3f00e64cb68bd5877ae69d6dbccfeb519Jean Chalard * This will also write the terminator byte. 1983bbb31f3f00e64cb68bd5877ae69d6dbccfeb519Jean Chalard * 1993bbb31f3f00e64cb68bd5877ae69d6dbccfeb519Jean Chalard * @param buffer the byte buffer to write to. 2003bbb31f3f00e64cb68bd5877ae69d6dbccfeb519Jean Chalard * @param origin the offset to write from. 2013bbb31f3f00e64cb68bd5877ae69d6dbccfeb519Jean Chalard * @param word the string to write. 2023bbb31f3f00e64cb68bd5877ae69d6dbccfeb519Jean Chalard * @return the size written, in bytes. 2033bbb31f3f00e64cb68bd5877ae69d6dbccfeb519Jean Chalard */ 2043bbb31f3f00e64cb68bd5877ae69d6dbccfeb519Jean Chalard private static int writeString(final byte[] buffer, final int origin, 2053bbb31f3f00e64cb68bd5877ae69d6dbccfeb519Jean Chalard final String word) { 2063bbb31f3f00e64cb68bd5877ae69d6dbccfeb519Jean Chalard final int length = word.length(); 2073bbb31f3f00e64cb68bd5877ae69d6dbccfeb519Jean Chalard int index = origin; 2083bbb31f3f00e64cb68bd5877ae69d6dbccfeb519Jean Chalard for (int i = 0; i < length; i = word.offsetByCodePoints(i, 1)) { 2093bbb31f3f00e64cb68bd5877ae69d6dbccfeb519Jean Chalard final int codePoint = word.codePointAt(i); 2103bbb31f3f00e64cb68bd5877ae69d6dbccfeb519Jean Chalard if (1 == getCharSize(codePoint)) { 2113bbb31f3f00e64cb68bd5877ae69d6dbccfeb519Jean Chalard buffer[index++] = (byte)codePoint; 2123bbb31f3f00e64cb68bd5877ae69d6dbccfeb519Jean Chalard } else { 2133bbb31f3f00e64cb68bd5877ae69d6dbccfeb519Jean Chalard buffer[index++] = (byte)(0xFF & (codePoint >> 16)); 2143bbb31f3f00e64cb68bd5877ae69d6dbccfeb519Jean Chalard buffer[index++] = (byte)(0xFF & (codePoint >> 8)); 2153bbb31f3f00e64cb68bd5877ae69d6dbccfeb519Jean Chalard buffer[index++] = (byte)(0xFF & codePoint); 2163bbb31f3f00e64cb68bd5877ae69d6dbccfeb519Jean Chalard } 2173bbb31f3f00e64cb68bd5877ae69d6dbccfeb519Jean Chalard } 21881d97eec0e77e72cce606f9c9f96091c0b348190Yuichiro Hanada buffer[index++] = FormatSpec.GROUP_CHARACTERS_TERMINATOR; 2193bbb31f3f00e64cb68bd5877ae69d6dbccfeb519Jean Chalard return index - origin; 2203bbb31f3f00e64cb68bd5877ae69d6dbccfeb519Jean Chalard } 2213bbb31f3f00e64cb68bd5877ae69d6dbccfeb519Jean Chalard 2223bbb31f3f00e64cb68bd5877ae69d6dbccfeb519Jean Chalard /** 223df7ebbbd616fa5aff569d00b16cd3f85ddf2da6dTom Ouyang * Writes a string with our character format to a ByteArrayOutputStream. 224df7ebbbd616fa5aff569d00b16cd3f85ddf2da6dTom Ouyang * 225df7ebbbd616fa5aff569d00b16cd3f85ddf2da6dTom Ouyang * This will also write the terminator byte. 226df7ebbbd616fa5aff569d00b16cd3f85ddf2da6dTom Ouyang * 227df7ebbbd616fa5aff569d00b16cd3f85ddf2da6dTom Ouyang * @param buffer the ByteArrayOutputStream to write to. 228df7ebbbd616fa5aff569d00b16cd3f85ddf2da6dTom Ouyang * @param word the string to write. 229df7ebbbd616fa5aff569d00b16cd3f85ddf2da6dTom Ouyang */ 230e55b644aefbefb4ac79308c9a59116e69a9c53a2Yuichiro Hanada private static void writeString(final ByteArrayOutputStream buffer, final String word) { 231df7ebbbd616fa5aff569d00b16cd3f85ddf2da6dTom Ouyang final int length = word.length(); 232df7ebbbd616fa5aff569d00b16cd3f85ddf2da6dTom Ouyang for (int i = 0; i < length; i = word.offsetByCodePoints(i, 1)) { 233df7ebbbd616fa5aff569d00b16cd3f85ddf2da6dTom Ouyang final int codePoint = word.codePointAt(i); 234df7ebbbd616fa5aff569d00b16cd3f85ddf2da6dTom Ouyang if (1 == getCharSize(codePoint)) { 235df7ebbbd616fa5aff569d00b16cd3f85ddf2da6dTom Ouyang buffer.write((byte) codePoint); 236df7ebbbd616fa5aff569d00b16cd3f85ddf2da6dTom Ouyang } else { 237df7ebbbd616fa5aff569d00b16cd3f85ddf2da6dTom Ouyang buffer.write((byte) (0xFF & (codePoint >> 16))); 238df7ebbbd616fa5aff569d00b16cd3f85ddf2da6dTom Ouyang buffer.write((byte) (0xFF & (codePoint >> 8))); 239df7ebbbd616fa5aff569d00b16cd3f85ddf2da6dTom Ouyang buffer.write((byte) (0xFF & codePoint)); 240df7ebbbd616fa5aff569d00b16cd3f85ddf2da6dTom Ouyang } 241df7ebbbd616fa5aff569d00b16cd3f85ddf2da6dTom Ouyang } 24281d97eec0e77e72cce606f9c9f96091c0b348190Yuichiro Hanada buffer.write(FormatSpec.GROUP_CHARACTERS_TERMINATOR); 243df7ebbbd616fa5aff569d00b16cd3f85ddf2da6dTom Ouyang } 244df7ebbbd616fa5aff569d00b16cd3f85ddf2da6dTom Ouyang 245df7ebbbd616fa5aff569d00b16cd3f85ddf2da6dTom Ouyang /** 246f5c4ff481782831329593760b000f0543680930aYuichiro Hanada * Reads a string from a buffer. This is the converse of the above method. 247752996540ff3a6dd5b48819849c06355c4270e03Jean Chalard */ 248f5c4ff481782831329593760b000f0543680930aYuichiro Hanada private static String readString(final FusionDictionaryBufferInterface buffer) { 249752996540ff3a6dd5b48819849c06355c4270e03Jean Chalard final StringBuilder s = new StringBuilder(); 250d4fe7fda303ff937d2e44c15dde9d90cbf59376bYuichiro Hanada int character = readChar(buffer); 25181d97eec0e77e72cce606f9c9f96091c0b348190Yuichiro Hanada while (character != FormatSpec.INVALID_CHARACTER) { 252752996540ff3a6dd5b48819849c06355c4270e03Jean Chalard s.appendCodePoint(character); 253d4fe7fda303ff937d2e44c15dde9d90cbf59376bYuichiro Hanada character = readChar(buffer); 254752996540ff3a6dd5b48819849c06355c4270e03Jean Chalard } 255752996540ff3a6dd5b48819849c06355c4270e03Jean Chalard return s.toString(); 256752996540ff3a6dd5b48819849c06355c4270e03Jean Chalard } 257752996540ff3a6dd5b48819849c06355c4270e03Jean Chalard 258752996540ff3a6dd5b48819849c06355c4270e03Jean Chalard /** 259f5c4ff481782831329593760b000f0543680930aYuichiro Hanada * Reads a character from the buffer. 260bfbbee8c5757aef4a20879547c16af0a4d1bf4c7Jean Chalard * 261bfbbee8c5757aef4a20879547c16af0a4d1bf4c7Jean Chalard * This follows the character format documented earlier in this source file. 262bfbbee8c5757aef4a20879547c16af0a4d1bf4c7Jean Chalard * 263d4fe7fda303ff937d2e44c15dde9d90cbf59376bYuichiro Hanada * @param buffer the buffer, positioned over an encoded character. 264bfbbee8c5757aef4a20879547c16af0a4d1bf4c7Jean Chalard * @return the character code. 265bfbbee8c5757aef4a20879547c16af0a4d1bf4c7Jean Chalard */ 266f5c4ff481782831329593760b000f0543680930aYuichiro Hanada private static int readChar(final FusionDictionaryBufferInterface buffer) { 267f5c4ff481782831329593760b000f0543680930aYuichiro Hanada int character = buffer.readUnsignedByte(); 268bfbbee8c5757aef4a20879547c16af0a4d1bf4c7Jean Chalard if (!fitsOnOneByte(character)) { 26981d97eec0e77e72cce606f9c9f96091c0b348190Yuichiro Hanada if (FormatSpec.GROUP_CHARACTERS_TERMINATOR == character) { 27081d97eec0e77e72cce606f9c9f96091c0b348190Yuichiro Hanada return FormatSpec.INVALID_CHARACTER; 27181d97eec0e77e72cce606f9c9f96091c0b348190Yuichiro Hanada } 272bfbbee8c5757aef4a20879547c16af0a4d1bf4c7Jean Chalard character <<= 16; 273f5c4ff481782831329593760b000f0543680930aYuichiro Hanada character += buffer.readUnsignedShort(); 274bfbbee8c5757aef4a20879547c16af0a4d1bf4c7Jean Chalard } 275bfbbee8c5757aef4a20879547c16af0a4d1bf4c7Jean Chalard return character; 276bfbbee8c5757aef4a20879547c16af0a4d1bf4c7Jean Chalard } 277bfbbee8c5757aef4a20879547c16af0a4d1bf4c7Jean Chalard } 278bfbbee8c5757aef4a20879547c16af0a4d1bf4c7Jean Chalard 279bfbbee8c5757aef4a20879547c16af0a4d1bf4c7Jean Chalard /** 280bfbbee8c5757aef4a20879547c16af0a4d1bf4c7Jean Chalard * Compute the binary size of the character array in a group 281bfbbee8c5757aef4a20879547c16af0a4d1bf4c7Jean Chalard * 282bfbbee8c5757aef4a20879547c16af0a4d1bf4c7Jean Chalard * If only one character, this is the size of this character. If many, it's the sum of their 283bfbbee8c5757aef4a20879547c16af0a4d1bf4c7Jean Chalard * sizes + 1 byte for the terminator. 284bfbbee8c5757aef4a20879547c16af0a4d1bf4c7Jean Chalard * 285bfbbee8c5757aef4a20879547c16af0a4d1bf4c7Jean Chalard * @param group the group 286bfbbee8c5757aef4a20879547c16af0a4d1bf4c7Jean Chalard * @return the size of the char array, including the terminator if any 287bfbbee8c5757aef4a20879547c16af0a4d1bf4c7Jean Chalard */ 288e55b644aefbefb4ac79308c9a59116e69a9c53a2Yuichiro Hanada private static int getGroupCharactersSize(final CharGroup group) { 289bfbbee8c5757aef4a20879547c16af0a4d1bf4c7Jean Chalard int size = CharEncoding.getCharArraySize(group.mChars); 29081d97eec0e77e72cce606f9c9f96091c0b348190Yuichiro Hanada if (group.hasSeveralChars()) size += FormatSpec.GROUP_TERMINATOR_SIZE; 291bfbbee8c5757aef4a20879547c16af0a4d1bf4c7Jean Chalard return size; 292bfbbee8c5757aef4a20879547c16af0a4d1bf4c7Jean Chalard } 293bfbbee8c5757aef4a20879547c16af0a4d1bf4c7Jean Chalard 294bfbbee8c5757aef4a20879547c16af0a4d1bf4c7Jean Chalard /** 2959b8b2c757d9a31481aa775809724bf53277721d7Jean Chalard * Compute the binary size of the group count 2969b8b2c757d9a31481aa775809724bf53277721d7Jean Chalard * @param count the group count 2972c0903b1d17a4669fee417c2f0b8f19bd7091ed7Jean Chalard * @return the size of the group count, either 1 or 2 bytes. 2982c0903b1d17a4669fee417c2f0b8f19bd7091ed7Jean Chalard */ 29965feee12e5889601e375d92dfdf5f8e8fbb05092Yuichiro Hanada public static int getGroupCountSize(final int count) { 30081d97eec0e77e72cce606f9c9f96091c0b348190Yuichiro Hanada if (FormatSpec.MAX_CHARGROUPS_FOR_ONE_BYTE_CHARGROUP_COUNT >= count) { 3012c0903b1d17a4669fee417c2f0b8f19bd7091ed7Jean Chalard return 1; 30281d97eec0e77e72cce606f9c9f96091c0b348190Yuichiro Hanada } else if (FormatSpec.MAX_CHARGROUPS_IN_A_NODE >= count) { 3032c0903b1d17a4669fee417c2f0b8f19bd7091ed7Jean Chalard return 2; 3042c0903b1d17a4669fee417c2f0b8f19bd7091ed7Jean Chalard } else { 30581d97eec0e77e72cce606f9c9f96091c0b348190Yuichiro Hanada throw new RuntimeException("Can't have more than " 30681d97eec0e77e72cce606f9c9f96091c0b348190Yuichiro Hanada + FormatSpec.MAX_CHARGROUPS_IN_A_NODE + " groups in a node (found " + count 30781d97eec0e77e72cce606f9c9f96091c0b348190Yuichiro Hanada + ")"); 3082c0903b1d17a4669fee417c2f0b8f19bd7091ed7Jean Chalard } 3092c0903b1d17a4669fee417c2f0b8f19bd7091ed7Jean Chalard } 3102c0903b1d17a4669fee417c2f0b8f19bd7091ed7Jean Chalard 3112c0903b1d17a4669fee417c2f0b8f19bd7091ed7Jean Chalard /** 3129b8b2c757d9a31481aa775809724bf53277721d7Jean Chalard * Compute the binary size of the group count for a node 3139b8b2c757d9a31481aa775809724bf53277721d7Jean Chalard * @param node the node 3149b8b2c757d9a31481aa775809724bf53277721d7Jean Chalard * @return the size of the group count, either 1 or 2 bytes. 3159b8b2c757d9a31481aa775809724bf53277721d7Jean Chalard */ 3169b8b2c757d9a31481aa775809724bf53277721d7Jean Chalard private static int getGroupCountSize(final Node node) { 3179b8b2c757d9a31481aa775809724bf53277721d7Jean Chalard return getGroupCountSize(node.mData.size()); 3189b8b2c757d9a31481aa775809724bf53277721d7Jean Chalard } 3199b8b2c757d9a31481aa775809724bf53277721d7Jean Chalard 3209b8b2c757d9a31481aa775809724bf53277721d7Jean Chalard /** 3213bbb31f3f00e64cb68bd5877ae69d6dbccfeb519Jean Chalard * Compute the size of a shortcut in bytes. 3223bbb31f3f00e64cb68bd5877ae69d6dbccfeb519Jean Chalard */ 3233bbb31f3f00e64cb68bd5877ae69d6dbccfeb519Jean Chalard private static int getShortcutSize(final WeightedString shortcut) { 32481d97eec0e77e72cce606f9c9f96091c0b348190Yuichiro Hanada int size = FormatSpec.GROUP_ATTRIBUTE_FLAGS_SIZE; 3253bbb31f3f00e64cb68bd5877ae69d6dbccfeb519Jean Chalard final String word = shortcut.mWord; 3263bbb31f3f00e64cb68bd5877ae69d6dbccfeb519Jean Chalard final int length = word.length(); 3273bbb31f3f00e64cb68bd5877ae69d6dbccfeb519Jean Chalard for (int i = 0; i < length; i = word.offsetByCodePoints(i, 1)) { 3283bbb31f3f00e64cb68bd5877ae69d6dbccfeb519Jean Chalard final int codePoint = word.codePointAt(i); 3293bbb31f3f00e64cb68bd5877ae69d6dbccfeb519Jean Chalard size += CharEncoding.getCharSize(codePoint); 3303bbb31f3f00e64cb68bd5877ae69d6dbccfeb519Jean Chalard } 33181d97eec0e77e72cce606f9c9f96091c0b348190Yuichiro Hanada size += FormatSpec.GROUP_TERMINATOR_SIZE; 3323bbb31f3f00e64cb68bd5877ae69d6dbccfeb519Jean Chalard return size; 3333bbb31f3f00e64cb68bd5877ae69d6dbccfeb519Jean Chalard } 3343bbb31f3f00e64cb68bd5877ae69d6dbccfeb519Jean Chalard 3353bbb31f3f00e64cb68bd5877ae69d6dbccfeb519Jean Chalard /** 3363bbb31f3f00e64cb68bd5877ae69d6dbccfeb519Jean Chalard * Compute the size of a shortcut list in bytes. 3373bbb31f3f00e64cb68bd5877ae69d6dbccfeb519Jean Chalard * 3383bbb31f3f00e64cb68bd5877ae69d6dbccfeb519Jean Chalard * This is known in advance and does not change according to position in the file 3393bbb31f3f00e64cb68bd5877ae69d6dbccfeb519Jean Chalard * like address lists do. 3403bbb31f3f00e64cb68bd5877ae69d6dbccfeb519Jean Chalard */ 3413bbb31f3f00e64cb68bd5877ae69d6dbccfeb519Jean Chalard private static int getShortcutListSize(final ArrayList<WeightedString> shortcutList) { 3423bbb31f3f00e64cb68bd5877ae69d6dbccfeb519Jean Chalard if (null == shortcutList) return 0; 34381d97eec0e77e72cce606f9c9f96091c0b348190Yuichiro Hanada int size = FormatSpec.GROUP_SHORTCUT_LIST_SIZE_SIZE; 3443bbb31f3f00e64cb68bd5877ae69d6dbccfeb519Jean Chalard for (final WeightedString shortcut : shortcutList) { 3453bbb31f3f00e64cb68bd5877ae69d6dbccfeb519Jean Chalard size += getShortcutSize(shortcut); 3463bbb31f3f00e64cb68bd5877ae69d6dbccfeb519Jean Chalard } 3473bbb31f3f00e64cb68bd5877ae69d6dbccfeb519Jean Chalard return size; 3483bbb31f3f00e64cb68bd5877ae69d6dbccfeb519Jean Chalard } 3493bbb31f3f00e64cb68bd5877ae69d6dbccfeb519Jean Chalard 3503bbb31f3f00e64cb68bd5877ae69d6dbccfeb519Jean Chalard /** 351bfbbee8c5757aef4a20879547c16af0a4d1bf4c7Jean Chalard * Compute the maximum size of a CharGroup, assuming 3-byte addresses for everything. 352bfbbee8c5757aef4a20879547c16af0a4d1bf4c7Jean Chalard * 353bfbbee8c5757aef4a20879547c16af0a4d1bf4c7Jean Chalard * @param group the CharGroup to compute the size of. 354e55b644aefbefb4ac79308c9a59116e69a9c53a2Yuichiro Hanada * @param options file format options. 355bfbbee8c5757aef4a20879547c16af0a4d1bf4c7Jean Chalard * @return the maximum size of the group. 356bfbbee8c5757aef4a20879547c16af0a4d1bf4c7Jean Chalard */ 357e55b644aefbefb4ac79308c9a59116e69a9c53a2Yuichiro Hanada private static int getCharGroupMaximumSize(final CharGroup group, final FormatOptions options) { 358e55b644aefbefb4ac79308c9a59116e69a9c53a2Yuichiro Hanada int size = getGroupHeaderSize(group, options); 359bfbbee8c5757aef4a20879547c16af0a4d1bf4c7Jean Chalard // If terminal, one byte for the frequency 36081d97eec0e77e72cce606f9c9f96091c0b348190Yuichiro Hanada if (group.isTerminal()) size += FormatSpec.GROUP_FREQUENCY_SIZE; 36181d97eec0e77e72cce606f9c9f96091c0b348190Yuichiro Hanada size += FormatSpec.GROUP_MAX_ADDRESS_SIZE; // For children address 3623bbb31f3f00e64cb68bd5877ae69d6dbccfeb519Jean Chalard size += getShortcutListSize(group.mShortcutTargets); 363bfbbee8c5757aef4a20879547c16af0a4d1bf4c7Jean Chalard if (null != group.mBigrams) { 36481d97eec0e77e72cce606f9c9f96091c0b348190Yuichiro Hanada size += (FormatSpec.GROUP_ATTRIBUTE_FLAGS_SIZE 36581d97eec0e77e72cce606f9c9f96091c0b348190Yuichiro Hanada + FormatSpec.GROUP_ATTRIBUTE_MAX_ADDRESS_SIZE) 3668edd3067181a425ce1383bb950184f9742af7557Jean Chalard * group.mBigrams.size(); 367bfbbee8c5757aef4a20879547c16af0a4d1bf4c7Jean Chalard } 368bfbbee8c5757aef4a20879547c16af0a4d1bf4c7Jean Chalard return size; 369bfbbee8c5757aef4a20879547c16af0a4d1bf4c7Jean Chalard } 370bfbbee8c5757aef4a20879547c16af0a4d1bf4c7Jean Chalard 371bfbbee8c5757aef4a20879547c16af0a4d1bf4c7Jean Chalard /** 372bfbbee8c5757aef4a20879547c16af0a4d1bf4c7Jean Chalard * Compute the maximum size of a node, assuming 3-byte addresses for everything, and caches 373bfbbee8c5757aef4a20879547c16af0a4d1bf4c7Jean Chalard * it in the 'actualSize' member of the node. 374bfbbee8c5757aef4a20879547c16af0a4d1bf4c7Jean Chalard * 375bfbbee8c5757aef4a20879547c16af0a4d1bf4c7Jean Chalard * @param node the node to compute the maximum size of. 376e55b644aefbefb4ac79308c9a59116e69a9c53a2Yuichiro Hanada * @param options file format options. 377bfbbee8c5757aef4a20879547c16af0a4d1bf4c7Jean Chalard */ 378e55b644aefbefb4ac79308c9a59116e69a9c53a2Yuichiro Hanada private static void setNodeMaximumSize(final Node node, final FormatOptions options) { 3792c0903b1d17a4669fee417c2f0b8f19bd7091ed7Jean Chalard int size = getGroupCountSize(node); 380bfbbee8c5757aef4a20879547c16af0a4d1bf4c7Jean Chalard for (CharGroup g : node.mData) { 381e55b644aefbefb4ac79308c9a59116e69a9c53a2Yuichiro Hanada final int groupSize = getCharGroupMaximumSize(g, options); 382bfbbee8c5757aef4a20879547c16af0a4d1bf4c7Jean Chalard g.mCachedSize = groupSize; 383bfbbee8c5757aef4a20879547c16af0a4d1bf4c7Jean Chalard size += groupSize; 384bfbbee8c5757aef4a20879547c16af0a4d1bf4c7Jean Chalard } 38582d9deaaf252cd20f8918adbc7a4b9b8f2647c38Yuichiro Hanada if (options.mSupportsDynamicUpdate) { 386bf45dc4860ab28e97c3e7d116a642802fe960239Yuichiro Hanada size += FormatSpec.FORWARD_LINK_ADDRESS_SIZE; 387bf45dc4860ab28e97c3e7d116a642802fe960239Yuichiro Hanada } 388bfbbee8c5757aef4a20879547c16af0a4d1bf4c7Jean Chalard node.mCachedSize = size; 389bfbbee8c5757aef4a20879547c16af0a4d1bf4c7Jean Chalard } 390bfbbee8c5757aef4a20879547c16af0a4d1bf4c7Jean Chalard 391bfbbee8c5757aef4a20879547c16af0a4d1bf4c7Jean Chalard /** 392bfbbee8c5757aef4a20879547c16af0a4d1bf4c7Jean Chalard * Helper method to hide the actual value of the no children address. 393bfbbee8c5757aef4a20879547c16af0a4d1bf4c7Jean Chalard */ 39465feee12e5889601e375d92dfdf5f8e8fbb05092Yuichiro Hanada public static boolean hasChildrenAddress(final int address) { 39581d97eec0e77e72cce606f9c9f96091c0b348190Yuichiro Hanada return FormatSpec.NO_CHILDREN_ADDRESS != address; 396bfbbee8c5757aef4a20879547c16af0a4d1bf4c7Jean Chalard } 397bfbbee8c5757aef4a20879547c16af0a4d1bf4c7Jean Chalard 398bfbbee8c5757aef4a20879547c16af0a4d1bf4c7Jean Chalard /** 3992ee70804e92b17016a2f042c4f6b0e94b5d23e88Yuichiro Hanada * Helper method to check whether the group is moved. 4002ee70804e92b17016a2f042c4f6b0e94b5d23e88Yuichiro Hanada */ 4012ee70804e92b17016a2f042c4f6b0e94b5d23e88Yuichiro Hanada public static boolean isMovedGroup(final int flags, final FormatOptions options) { 4022ee70804e92b17016a2f042c4f6b0e94b5d23e88Yuichiro Hanada return options.mSupportsDynamicUpdate && ((flags & FormatSpec.FLAG_IS_MOVED) == 1); 4032ee70804e92b17016a2f042c4f6b0e94b5d23e88Yuichiro Hanada } 4042ee70804e92b17016a2f042c4f6b0e94b5d23e88Yuichiro Hanada 4052ee70804e92b17016a2f042c4f6b0e94b5d23e88Yuichiro Hanada /** 40682d9deaaf252cd20f8918adbc7a4b9b8f2647c38Yuichiro Hanada * Helper method to check whether the dictionary can be updated dynamically. 407e55b644aefbefb4ac79308c9a59116e69a9c53a2Yuichiro Hanada */ 40882d9deaaf252cd20f8918adbc7a4b9b8f2647c38Yuichiro Hanada public static boolean supportsDynamicUpdate(final FormatOptions options) { 40982d9deaaf252cd20f8918adbc7a4b9b8f2647c38Yuichiro Hanada return options.mVersion >= FormatSpec.FIRST_VERSION_WITH_DYNAMIC_UPDATE 41082d9deaaf252cd20f8918adbc7a4b9b8f2647c38Yuichiro Hanada && options.mSupportsDynamicUpdate; 411e55b644aefbefb4ac79308c9a59116e69a9c53a2Yuichiro Hanada } 412e55b644aefbefb4ac79308c9a59116e69a9c53a2Yuichiro Hanada 413e55b644aefbefb4ac79308c9a59116e69a9c53a2Yuichiro Hanada /** 414e55b644aefbefb4ac79308c9a59116e69a9c53a2Yuichiro Hanada * Compute the size of the header (flag + [parent address] + characters size) of a CharGroup. 415e55b644aefbefb4ac79308c9a59116e69a9c53a2Yuichiro Hanada * 416e55b644aefbefb4ac79308c9a59116e69a9c53a2Yuichiro Hanada * @param group the group of which to compute the size of the header 417e55b644aefbefb4ac79308c9a59116e69a9c53a2Yuichiro Hanada * @param options file format options. 418e55b644aefbefb4ac79308c9a59116e69a9c53a2Yuichiro Hanada */ 419e55b644aefbefb4ac79308c9a59116e69a9c53a2Yuichiro Hanada private static int getGroupHeaderSize(final CharGroup group, final FormatOptions options) { 42082d9deaaf252cd20f8918adbc7a4b9b8f2647c38Yuichiro Hanada if (supportsDynamicUpdate(options)) { 42181d97eec0e77e72cce606f9c9f96091c0b348190Yuichiro Hanada return FormatSpec.GROUP_FLAGS_SIZE + FormatSpec.PARENT_ADDRESS_SIZE 42281d97eec0e77e72cce606f9c9f96091c0b348190Yuichiro Hanada + getGroupCharactersSize(group); 423e55b644aefbefb4ac79308c9a59116e69a9c53a2Yuichiro Hanada } else { 42481d97eec0e77e72cce606f9c9f96091c0b348190Yuichiro Hanada return FormatSpec.GROUP_FLAGS_SIZE + getGroupCharactersSize(group); 425e55b644aefbefb4ac79308c9a59116e69a9c53a2Yuichiro Hanada } 426e55b644aefbefb4ac79308c9a59116e69a9c53a2Yuichiro Hanada } 427e55b644aefbefb4ac79308c9a59116e69a9c53a2Yuichiro Hanada 4288ec0064c49e80945dbe1bb31129eb890478b7e06Yuichiro Hanada private static final int UINT8_MAX = 0xFF; 4298ec0064c49e80945dbe1bb31129eb890478b7e06Yuichiro Hanada private static final int UINT16_MAX = 0xFFFF; 4308ec0064c49e80945dbe1bb31129eb890478b7e06Yuichiro Hanada private static final int UINT24_MAX = 0xFFFFFF; 4318ec0064c49e80945dbe1bb31129eb890478b7e06Yuichiro Hanada 432e55b644aefbefb4ac79308c9a59116e69a9c53a2Yuichiro Hanada /** 433bfbbee8c5757aef4a20879547c16af0a4d1bf4c7Jean Chalard * Compute the size, in bytes, that an address will occupy. 434bfbbee8c5757aef4a20879547c16af0a4d1bf4c7Jean Chalard * 435bfbbee8c5757aef4a20879547c16af0a4d1bf4c7Jean Chalard * This can be used either for children addresses (which are always positive) or for 436bfbbee8c5757aef4a20879547c16af0a4d1bf4c7Jean Chalard * attribute, which may be positive or negative but 437bfbbee8c5757aef4a20879547c16af0a4d1bf4c7Jean Chalard * store their sign bit separately. 438bfbbee8c5757aef4a20879547c16af0a4d1bf4c7Jean Chalard * 439bfbbee8c5757aef4a20879547c16af0a4d1bf4c7Jean Chalard * @param address the address 440bfbbee8c5757aef4a20879547c16af0a4d1bf4c7Jean Chalard * @return the byte size. 441bfbbee8c5757aef4a20879547c16af0a4d1bf4c7Jean Chalard */ 442e55b644aefbefb4ac79308c9a59116e69a9c53a2Yuichiro Hanada private static int getByteSize(final int address) { 4438ec0064c49e80945dbe1bb31129eb890478b7e06Yuichiro Hanada assert(address <= UINT24_MAX); 444bfbbee8c5757aef4a20879547c16af0a4d1bf4c7Jean Chalard if (!hasChildrenAddress(address)) { 445bfbbee8c5757aef4a20879547c16af0a4d1bf4c7Jean Chalard return 0; 4468ec0064c49e80945dbe1bb31129eb890478b7e06Yuichiro Hanada } else if (Math.abs(address) <= UINT8_MAX) { 447bfbbee8c5757aef4a20879547c16af0a4d1bf4c7Jean Chalard return 1; 4488ec0064c49e80945dbe1bb31129eb890478b7e06Yuichiro Hanada } else if (Math.abs(address) <= UINT16_MAX) { 449bfbbee8c5757aef4a20879547c16af0a4d1bf4c7Jean Chalard return 2; 450bfbbee8c5757aef4a20879547c16af0a4d1bf4c7Jean Chalard } else { 451bfbbee8c5757aef4a20879547c16af0a4d1bf4c7Jean Chalard return 3; 452bfbbee8c5757aef4a20879547c16af0a4d1bf4c7Jean Chalard } 453bfbbee8c5757aef4a20879547c16af0a4d1bf4c7Jean Chalard } 4548ec0064c49e80945dbe1bb31129eb890478b7e06Yuichiro Hanada 4558ec0064c49e80945dbe1bb31129eb890478b7e06Yuichiro Hanada private static final int SINT8_MAX = 0x7F; 4568ec0064c49e80945dbe1bb31129eb890478b7e06Yuichiro Hanada private static final int SINT16_MAX = 0x7FFF; 4578ec0064c49e80945dbe1bb31129eb890478b7e06Yuichiro Hanada private static final int SINT24_MAX = 0x7FFFFF; 4588ec0064c49e80945dbe1bb31129eb890478b7e06Yuichiro Hanada private static final int MSB8 = 0x80; 4598ec0064c49e80945dbe1bb31129eb890478b7e06Yuichiro Hanada private static final int MSB16 = 0x8000; 4608ec0064c49e80945dbe1bb31129eb890478b7e06Yuichiro Hanada private static final int MSB24 = 0x800000; 4618ec0064c49e80945dbe1bb31129eb890478b7e06Yuichiro Hanada 462bfbbee8c5757aef4a20879547c16af0a4d1bf4c7Jean Chalard // End utility methods. 463bfbbee8c5757aef4a20879547c16af0a4d1bf4c7Jean Chalard 464bfbbee8c5757aef4a20879547c16af0a4d1bf4c7Jean Chalard // This method is responsible for finding a nice ordering of the nodes that favors run-time 465bfbbee8c5757aef4a20879547c16af0a4d1bf4c7Jean Chalard // cache performance and dictionary size. 466e55b644aefbefb4ac79308c9a59116e69a9c53a2Yuichiro Hanada /* package for tests */ static ArrayList<Node> flattenTree(final Node root) { 467bfbbee8c5757aef4a20879547c16af0a4d1bf4c7Jean Chalard final int treeSize = FusionDictionary.countCharGroups(root); 468bfbbee8c5757aef4a20879547c16af0a4d1bf4c7Jean Chalard MakedictLog.i("Counted nodes : " + treeSize); 469bfbbee8c5757aef4a20879547c16af0a4d1bf4c7Jean Chalard final ArrayList<Node> flatTree = new ArrayList<Node>(treeSize); 470bfbbee8c5757aef4a20879547c16af0a4d1bf4c7Jean Chalard return flattenTreeInner(flatTree, root); 471bfbbee8c5757aef4a20879547c16af0a4d1bf4c7Jean Chalard } 472bfbbee8c5757aef4a20879547c16af0a4d1bf4c7Jean Chalard 473e55b644aefbefb4ac79308c9a59116e69a9c53a2Yuichiro Hanada private static ArrayList<Node> flattenTreeInner(final ArrayList<Node> list, final Node node) { 474bfbbee8c5757aef4a20879547c16af0a4d1bf4c7Jean Chalard // Removing the node is necessary if the tails are merged, because we would then 475bfbbee8c5757aef4a20879547c16af0a4d1bf4c7Jean Chalard // add the same node several times when we only want it once. A number of places in 476bfbbee8c5757aef4a20879547c16af0a4d1bf4c7Jean Chalard // the code also depends on any node being only once in the list. 477bfbbee8c5757aef4a20879547c16af0a4d1bf4c7Jean Chalard // Merging tails can only be done if there are no attributes. Searching for attributes 478bfbbee8c5757aef4a20879547c16af0a4d1bf4c7Jean Chalard // in LatinIME code depends on a total breadth-first ordering, which merging tails 479bfbbee8c5757aef4a20879547c16af0a4d1bf4c7Jean Chalard // breaks. If there are no attributes, it should be fine (and reduce the file size) 4803b1b72ac4d8975d24a3176dd1b5a39b5fead71a8Jean Chalard // to merge tails, and removing the node from the list would be necessary. However, 4813b1b72ac4d8975d24a3176dd1b5a39b5fead71a8Jean Chalard // we don't merge tails because breaking the breadth-first ordering would result in 4823b1b72ac4d8975d24a3176dd1b5a39b5fead71a8Jean Chalard // extreme overhead at bigram lookup time (it would make the search function O(n) instead 4833b1b72ac4d8975d24a3176dd1b5a39b5fead71a8Jean Chalard // of the current O(log(n)), where n=number of nodes in the dictionary which is pretty 4843b1b72ac4d8975d24a3176dd1b5a39b5fead71a8Jean Chalard // high). 4853b1b72ac4d8975d24a3176dd1b5a39b5fead71a8Jean Chalard // If no nodes are ever merged, we can't have the same node twice in the list, hence 4863b1b72ac4d8975d24a3176dd1b5a39b5fead71a8Jean Chalard // searching for duplicates in unnecessary. It is also very performance consuming, 4873b1b72ac4d8975d24a3176dd1b5a39b5fead71a8Jean Chalard // since `list' is an ArrayList so it's an O(n) operation that runs on all nodes, making 4883b1b72ac4d8975d24a3176dd1b5a39b5fead71a8Jean Chalard // this simple list.remove operation O(n*n) overall. On Android this overhead is very 4893b1b72ac4d8975d24a3176dd1b5a39b5fead71a8Jean Chalard // high. 4903b1b72ac4d8975d24a3176dd1b5a39b5fead71a8Jean Chalard // For future reference, the code to remove duplicate is a simple : list.remove(node); 491bfbbee8c5757aef4a20879547c16af0a4d1bf4c7Jean Chalard list.add(node); 492bfbbee8c5757aef4a20879547c16af0a4d1bf4c7Jean Chalard final ArrayList<CharGroup> branches = node.mData; 493bfbbee8c5757aef4a20879547c16af0a4d1bf4c7Jean Chalard final int nodeSize = branches.size(); 494bfbbee8c5757aef4a20879547c16af0a4d1bf4c7Jean Chalard for (CharGroup group : branches) { 495bfbbee8c5757aef4a20879547c16af0a4d1bf4c7Jean Chalard if (null != group.mChildren) flattenTreeInner(list, group.mChildren); 496bfbbee8c5757aef4a20879547c16af0a4d1bf4c7Jean Chalard } 497bfbbee8c5757aef4a20879547c16af0a4d1bf4c7Jean Chalard return list; 498bfbbee8c5757aef4a20879547c16af0a4d1bf4c7Jean Chalard } 499bfbbee8c5757aef4a20879547c16af0a4d1bf4c7Jean Chalard 500bfbbee8c5757aef4a20879547c16af0a4d1bf4c7Jean Chalard /** 501bfbbee8c5757aef4a20879547c16af0a4d1bf4c7Jean Chalard * Finds the absolute address of a word in the dictionary. 502bfbbee8c5757aef4a20879547c16af0a4d1bf4c7Jean Chalard * 503bfbbee8c5757aef4a20879547c16af0a4d1bf4c7Jean Chalard * @param dict the dictionary in which to search. 504bfbbee8c5757aef4a20879547c16af0a4d1bf4c7Jean Chalard * @param word the word we are searching for. 505bfbbee8c5757aef4a20879547c16af0a4d1bf4c7Jean Chalard * @return the word address. If it is not found, an exception is thrown. 506bfbbee8c5757aef4a20879547c16af0a4d1bf4c7Jean Chalard */ 507bfbbee8c5757aef4a20879547c16af0a4d1bf4c7Jean Chalard private static int findAddressOfWord(final FusionDictionary dict, final String word) { 508bfbbee8c5757aef4a20879547c16af0a4d1bf4c7Jean Chalard return FusionDictionary.findWordInTree(dict.mRoot, word).mCachedAddress; 509bfbbee8c5757aef4a20879547c16af0a4d1bf4c7Jean Chalard } 510bfbbee8c5757aef4a20879547c16af0a4d1bf4c7Jean Chalard 511bfbbee8c5757aef4a20879547c16af0a4d1bf4c7Jean Chalard /** 512bfbbee8c5757aef4a20879547c16af0a4d1bf4c7Jean Chalard * Computes the actual node size, based on the cached addresses of the children nodes. 513bfbbee8c5757aef4a20879547c16af0a4d1bf4c7Jean Chalard * 514bfbbee8c5757aef4a20879547c16af0a4d1bf4c7Jean Chalard * Each node stores its tentative address. During dictionary address computing, these 515bfbbee8c5757aef4a20879547c16af0a4d1bf4c7Jean Chalard * are not final, but they can be used to compute the node size (the node size depends 516bfbbee8c5757aef4a20879547c16af0a4d1bf4c7Jean Chalard * on the address of the children because the number of bytes necessary to store an 517a64a1a46e482664dcebdf4fee0745a890d0d70dcJean Chalard * address depends on its numeric value. The return value indicates whether the node 518a64a1a46e482664dcebdf4fee0745a890d0d70dcJean Chalard * contents (as in, any of the addresses stored in the cache fields) have changed with 519a64a1a46e482664dcebdf4fee0745a890d0d70dcJean Chalard * respect to their previous value. 520bfbbee8c5757aef4a20879547c16af0a4d1bf4c7Jean Chalard * 521bfbbee8c5757aef4a20879547c16af0a4d1bf4c7Jean Chalard * @param node the node to compute the size of. 522bfbbee8c5757aef4a20879547c16af0a4d1bf4c7Jean Chalard * @param dict the dictionary in which the word/attributes are to be found. 523e55b644aefbefb4ac79308c9a59116e69a9c53a2Yuichiro Hanada * @param formatOptions file format options. 524a64a1a46e482664dcebdf4fee0745a890d0d70dcJean Chalard * @return false if none of the cached addresses inside the node changed, true otherwise. 525bfbbee8c5757aef4a20879547c16af0a4d1bf4c7Jean Chalard */ 526e55b644aefbefb4ac79308c9a59116e69a9c53a2Yuichiro Hanada private static boolean computeActualNodeSize(final Node node, final FusionDictionary dict, 527e55b644aefbefb4ac79308c9a59116e69a9c53a2Yuichiro Hanada final FormatOptions formatOptions) { 528a64a1a46e482664dcebdf4fee0745a890d0d70dcJean Chalard boolean changed = false; 5292c0903b1d17a4669fee417c2f0b8f19bd7091ed7Jean Chalard int size = getGroupCountSize(node); 530bfbbee8c5757aef4a20879547c16af0a4d1bf4c7Jean Chalard for (CharGroup group : node.mData) { 531a64a1a46e482664dcebdf4fee0745a890d0d70dcJean Chalard if (group.mCachedAddress != node.mCachedAddress + size) { 532a64a1a46e482664dcebdf4fee0745a890d0d70dcJean Chalard changed = true; 533a64a1a46e482664dcebdf4fee0745a890d0d70dcJean Chalard group.mCachedAddress = node.mCachedAddress + size; 534a64a1a46e482664dcebdf4fee0745a890d0d70dcJean Chalard } 535e55b644aefbefb4ac79308c9a59116e69a9c53a2Yuichiro Hanada int groupSize = getGroupHeaderSize(group, formatOptions); 53681d97eec0e77e72cce606f9c9f96091c0b348190Yuichiro Hanada if (group.isTerminal()) groupSize += FormatSpec.GROUP_FREQUENCY_SIZE; 5378ec0064c49e80945dbe1bb31129eb890478b7e06Yuichiro Hanada if (null == group.mChildren && formatOptions.mSupportsDynamicUpdate) { 5388ec0064c49e80945dbe1bb31129eb890478b7e06Yuichiro Hanada groupSize += FormatSpec.SIGNED_CHILDREN_ADDRESS_SIZE; 5398ec0064c49e80945dbe1bb31129eb890478b7e06Yuichiro Hanada } else if (null != group.mChildren) { 540e55b644aefbefb4ac79308c9a59116e69a9c53a2Yuichiro Hanada final int offsetBasePoint = groupSize + node.mCachedAddress + size; 541bfbbee8c5757aef4a20879547c16af0a4d1bf4c7Jean Chalard final int offset = group.mChildren.mCachedAddress - offsetBasePoint; 542e55b644aefbefb4ac79308c9a59116e69a9c53a2Yuichiro Hanada // assign my address to children's parent address 543e55b644aefbefb4ac79308c9a59116e69a9c53a2Yuichiro Hanada group.mChildren.mCachedParentAddress = group.mCachedAddress 544e55b644aefbefb4ac79308c9a59116e69a9c53a2Yuichiro Hanada - group.mChildren.mCachedAddress; 5458ec0064c49e80945dbe1bb31129eb890478b7e06Yuichiro Hanada if (formatOptions.mSupportsDynamicUpdate) { 5468ec0064c49e80945dbe1bb31129eb890478b7e06Yuichiro Hanada groupSize += FormatSpec.SIGNED_CHILDREN_ADDRESS_SIZE; 5478ec0064c49e80945dbe1bb31129eb890478b7e06Yuichiro Hanada } else { 5488ec0064c49e80945dbe1bb31129eb890478b7e06Yuichiro Hanada groupSize += getByteSize(offset); 5498ec0064c49e80945dbe1bb31129eb890478b7e06Yuichiro Hanada } 550bfbbee8c5757aef4a20879547c16af0a4d1bf4c7Jean Chalard } 5513bbb31f3f00e64cb68bd5877ae69d6dbccfeb519Jean Chalard groupSize += getShortcutListSize(group.mShortcutTargets); 552bfbbee8c5757aef4a20879547c16af0a4d1bf4c7Jean Chalard if (null != group.mBigrams) { 553bfbbee8c5757aef4a20879547c16af0a4d1bf4c7Jean Chalard for (WeightedString bigram : group.mBigrams) { 554bfbbee8c5757aef4a20879547c16af0a4d1bf4c7Jean Chalard final int offsetBasePoint = groupSize + node.mCachedAddress + size 55581d97eec0e77e72cce606f9c9f96091c0b348190Yuichiro Hanada + FormatSpec.GROUP_FLAGS_SIZE; 556bfbbee8c5757aef4a20879547c16af0a4d1bf4c7Jean Chalard final int addressOfBigram = findAddressOfWord(dict, bigram.mWord); 557bfbbee8c5757aef4a20879547c16af0a4d1bf4c7Jean Chalard final int offset = addressOfBigram - offsetBasePoint; 55881d97eec0e77e72cce606f9c9f96091c0b348190Yuichiro Hanada groupSize += getByteSize(offset) + FormatSpec.GROUP_FLAGS_SIZE; 559bfbbee8c5757aef4a20879547c16af0a4d1bf4c7Jean Chalard } 560bfbbee8c5757aef4a20879547c16af0a4d1bf4c7Jean Chalard } 561bfbbee8c5757aef4a20879547c16af0a4d1bf4c7Jean Chalard group.mCachedSize = groupSize; 562bfbbee8c5757aef4a20879547c16af0a4d1bf4c7Jean Chalard size += groupSize; 563bfbbee8c5757aef4a20879547c16af0a4d1bf4c7Jean Chalard } 56482d9deaaf252cd20f8918adbc7a4b9b8f2647c38Yuichiro Hanada if (formatOptions.mSupportsDynamicUpdate) { 565bf45dc4860ab28e97c3e7d116a642802fe960239Yuichiro Hanada size += FormatSpec.FORWARD_LINK_ADDRESS_SIZE; 566bf45dc4860ab28e97c3e7d116a642802fe960239Yuichiro Hanada } 567a64a1a46e482664dcebdf4fee0745a890d0d70dcJean Chalard if (node.mCachedSize != size) { 568a64a1a46e482664dcebdf4fee0745a890d0d70dcJean Chalard node.mCachedSize = size; 569a64a1a46e482664dcebdf4fee0745a890d0d70dcJean Chalard changed = true; 570a64a1a46e482664dcebdf4fee0745a890d0d70dcJean Chalard } 571a64a1a46e482664dcebdf4fee0745a890d0d70dcJean Chalard return changed; 572bfbbee8c5757aef4a20879547c16af0a4d1bf4c7Jean Chalard } 573bfbbee8c5757aef4a20879547c16af0a4d1bf4c7Jean Chalard 574bfbbee8c5757aef4a20879547c16af0a4d1bf4c7Jean Chalard /** 575bfbbee8c5757aef4a20879547c16af0a4d1bf4c7Jean Chalard * Computes the byte size of a list of nodes and updates each node cached position. 576bfbbee8c5757aef4a20879547c16af0a4d1bf4c7Jean Chalard * 577bfbbee8c5757aef4a20879547c16af0a4d1bf4c7Jean Chalard * @param flatNodes the array of nodes. 578bf45dc4860ab28e97c3e7d116a642802fe960239Yuichiro Hanada * @param formatOptions file format options. 579bfbbee8c5757aef4a20879547c16af0a4d1bf4c7Jean Chalard * @return the byte size of the entire stack. 580bfbbee8c5757aef4a20879547c16af0a4d1bf4c7Jean Chalard */ 581bf45dc4860ab28e97c3e7d116a642802fe960239Yuichiro Hanada private static int stackNodes(final ArrayList<Node> flatNodes, 582bf45dc4860ab28e97c3e7d116a642802fe960239Yuichiro Hanada final FormatOptions formatOptions) { 583bfbbee8c5757aef4a20879547c16af0a4d1bf4c7Jean Chalard int nodeOffset = 0; 584bfbbee8c5757aef4a20879547c16af0a4d1bf4c7Jean Chalard for (Node n : flatNodes) { 585bfbbee8c5757aef4a20879547c16af0a4d1bf4c7Jean Chalard n.mCachedAddress = nodeOffset; 5862c0903b1d17a4669fee417c2f0b8f19bd7091ed7Jean Chalard int groupCountSize = getGroupCountSize(n); 587bfbbee8c5757aef4a20879547c16af0a4d1bf4c7Jean Chalard int groupOffset = 0; 588bfbbee8c5757aef4a20879547c16af0a4d1bf4c7Jean Chalard for (CharGroup g : n.mData) { 5892c0903b1d17a4669fee417c2f0b8f19bd7091ed7Jean Chalard g.mCachedAddress = groupCountSize + nodeOffset + groupOffset; 590bfbbee8c5757aef4a20879547c16af0a4d1bf4c7Jean Chalard groupOffset += g.mCachedSize; 591bfbbee8c5757aef4a20879547c16af0a4d1bf4c7Jean Chalard } 592bf45dc4860ab28e97c3e7d116a642802fe960239Yuichiro Hanada final int nodeSize = groupCountSize + groupOffset 59382d9deaaf252cd20f8918adbc7a4b9b8f2647c38Yuichiro Hanada + (formatOptions.mSupportsDynamicUpdate 59482d9deaaf252cd20f8918adbc7a4b9b8f2647c38Yuichiro Hanada ? FormatSpec.FORWARD_LINK_ADDRESS_SIZE : 0); 595bf45dc4860ab28e97c3e7d116a642802fe960239Yuichiro Hanada if (nodeSize != n.mCachedSize) { 596bfbbee8c5757aef4a20879547c16af0a4d1bf4c7Jean Chalard throw new RuntimeException("Bug : Stored and computed node size differ"); 597bfbbee8c5757aef4a20879547c16af0a4d1bf4c7Jean Chalard } 598bfbbee8c5757aef4a20879547c16af0a4d1bf4c7Jean Chalard nodeOffset += n.mCachedSize; 599bfbbee8c5757aef4a20879547c16af0a4d1bf4c7Jean Chalard } 600bfbbee8c5757aef4a20879547c16af0a4d1bf4c7Jean Chalard return nodeOffset; 601bfbbee8c5757aef4a20879547c16af0a4d1bf4c7Jean Chalard } 602bfbbee8c5757aef4a20879547c16af0a4d1bf4c7Jean Chalard 603bfbbee8c5757aef4a20879547c16af0a4d1bf4c7Jean Chalard /** 604bfbbee8c5757aef4a20879547c16af0a4d1bf4c7Jean Chalard * Compute the addresses and sizes of an ordered node array. 605bfbbee8c5757aef4a20879547c16af0a4d1bf4c7Jean Chalard * 606bfbbee8c5757aef4a20879547c16af0a4d1bf4c7Jean Chalard * This method takes a node array and will update its cached address and size values 607bfbbee8c5757aef4a20879547c16af0a4d1bf4c7Jean Chalard * so that they can be written into a file. It determines the smallest size each of the 608bfbbee8c5757aef4a20879547c16af0a4d1bf4c7Jean Chalard * nodes can be given the addresses of its children and attributes, and store that into 609bfbbee8c5757aef4a20879547c16af0a4d1bf4c7Jean Chalard * each node. 610bfbbee8c5757aef4a20879547c16af0a4d1bf4c7Jean Chalard * The order of the node is given by the order of the array. This method makes no effort 611bfbbee8c5757aef4a20879547c16af0a4d1bf4c7Jean Chalard * to find a good order; it only mechanically computes the size this order results in. 612bfbbee8c5757aef4a20879547c16af0a4d1bf4c7Jean Chalard * 613bfbbee8c5757aef4a20879547c16af0a4d1bf4c7Jean Chalard * @param dict the dictionary 614bfbbee8c5757aef4a20879547c16af0a4d1bf4c7Jean Chalard * @param flatNodes the ordered array of nodes 615e55b644aefbefb4ac79308c9a59116e69a9c53a2Yuichiro Hanada * @param formatOptions file format options. 616bfbbee8c5757aef4a20879547c16af0a4d1bf4c7Jean Chalard * @return the same array it was passed. The nodes have been updated for address and size. 617bfbbee8c5757aef4a20879547c16af0a4d1bf4c7Jean Chalard */ 618e55b644aefbefb4ac79308c9a59116e69a9c53a2Yuichiro Hanada private static ArrayList<Node> computeAddresses(final FusionDictionary dict, 619e55b644aefbefb4ac79308c9a59116e69a9c53a2Yuichiro Hanada final ArrayList<Node> flatNodes, final FormatOptions formatOptions) { 620bfbbee8c5757aef4a20879547c16af0a4d1bf4c7Jean Chalard // First get the worst sizes and offsets 621e55b644aefbefb4ac79308c9a59116e69a9c53a2Yuichiro Hanada for (Node n : flatNodes) setNodeMaximumSize(n, formatOptions); 622bf45dc4860ab28e97c3e7d116a642802fe960239Yuichiro Hanada final int offset = stackNodes(flatNodes, formatOptions); 623bfbbee8c5757aef4a20879547c16af0a4d1bf4c7Jean Chalard 624bfbbee8c5757aef4a20879547c16af0a4d1bf4c7Jean Chalard MakedictLog.i("Compressing the array addresses. Original size : " + offset); 625bfbbee8c5757aef4a20879547c16af0a4d1bf4c7Jean Chalard MakedictLog.i("(Recursively seen size : " + offset + ")"); 626bfbbee8c5757aef4a20879547c16af0a4d1bf4c7Jean Chalard 627bfbbee8c5757aef4a20879547c16af0a4d1bf4c7Jean Chalard int passes = 0; 628bfbbee8c5757aef4a20879547c16af0a4d1bf4c7Jean Chalard boolean changesDone = false; 629bfbbee8c5757aef4a20879547c16af0a4d1bf4c7Jean Chalard do { 630bfbbee8c5757aef4a20879547c16af0a4d1bf4c7Jean Chalard changesDone = false; 631bfbbee8c5757aef4a20879547c16af0a4d1bf4c7Jean Chalard for (Node n : flatNodes) { 632bfbbee8c5757aef4a20879547c16af0a4d1bf4c7Jean Chalard final int oldNodeSize = n.mCachedSize; 633e55b644aefbefb4ac79308c9a59116e69a9c53a2Yuichiro Hanada final boolean changed = computeActualNodeSize(n, dict, formatOptions); 634bfbbee8c5757aef4a20879547c16af0a4d1bf4c7Jean Chalard final int newNodeSize = n.mCachedSize; 635bfbbee8c5757aef4a20879547c16af0a4d1bf4c7Jean Chalard if (oldNodeSize < newNodeSize) throw new RuntimeException("Increased size ?!"); 636a64a1a46e482664dcebdf4fee0745a890d0d70dcJean Chalard changesDone |= changed; 637bfbbee8c5757aef4a20879547c16af0a4d1bf4c7Jean Chalard } 638bf45dc4860ab28e97c3e7d116a642802fe960239Yuichiro Hanada stackNodes(flatNodes, formatOptions); 639bfbbee8c5757aef4a20879547c16af0a4d1bf4c7Jean Chalard ++passes; 640a64a1a46e482664dcebdf4fee0745a890d0d70dcJean Chalard if (passes > MAX_PASSES) throw new RuntimeException("Too many passes - probably a bug"); 641bfbbee8c5757aef4a20879547c16af0a4d1bf4c7Jean Chalard } while (changesDone); 642bfbbee8c5757aef4a20879547c16af0a4d1bf4c7Jean Chalard 643bfbbee8c5757aef4a20879547c16af0a4d1bf4c7Jean Chalard final Node lastNode = flatNodes.get(flatNodes.size() - 1); 644bfbbee8c5757aef4a20879547c16af0a4d1bf4c7Jean Chalard MakedictLog.i("Compression complete in " + passes + " passes."); 645bfbbee8c5757aef4a20879547c16af0a4d1bf4c7Jean Chalard MakedictLog.i("After address compression : " 646bfbbee8c5757aef4a20879547c16af0a4d1bf4c7Jean Chalard + (lastNode.mCachedAddress + lastNode.mCachedSize)); 647bfbbee8c5757aef4a20879547c16af0a4d1bf4c7Jean Chalard 648bfbbee8c5757aef4a20879547c16af0a4d1bf4c7Jean Chalard return flatNodes; 649bfbbee8c5757aef4a20879547c16af0a4d1bf4c7Jean Chalard } 650bfbbee8c5757aef4a20879547c16af0a4d1bf4c7Jean Chalard 651bfbbee8c5757aef4a20879547c16af0a4d1bf4c7Jean Chalard /** 652bfbbee8c5757aef4a20879547c16af0a4d1bf4c7Jean Chalard * Sanity-checking method. 653bfbbee8c5757aef4a20879547c16af0a4d1bf4c7Jean Chalard * 654bfbbee8c5757aef4a20879547c16af0a4d1bf4c7Jean Chalard * This method checks an array of node for juxtaposition, that is, it will do 655bfbbee8c5757aef4a20879547c16af0a4d1bf4c7Jean Chalard * nothing if each node's cached address is actually the previous node's address 656bfbbee8c5757aef4a20879547c16af0a4d1bf4c7Jean Chalard * plus the previous node's size. 657bfbbee8c5757aef4a20879547c16af0a4d1bf4c7Jean Chalard * If this is not the case, it will throw an exception. 658bfbbee8c5757aef4a20879547c16af0a4d1bf4c7Jean Chalard * 659bfbbee8c5757aef4a20879547c16af0a4d1bf4c7Jean Chalard * @param array the array node to check 660bfbbee8c5757aef4a20879547c16af0a4d1bf4c7Jean Chalard */ 661e55b644aefbefb4ac79308c9a59116e69a9c53a2Yuichiro Hanada private static void checkFlatNodeArray(final ArrayList<Node> array) { 662bfbbee8c5757aef4a20879547c16af0a4d1bf4c7Jean Chalard int offset = 0; 663bfbbee8c5757aef4a20879547c16af0a4d1bf4c7Jean Chalard int index = 0; 664bfbbee8c5757aef4a20879547c16af0a4d1bf4c7Jean Chalard for (Node n : array) { 665bfbbee8c5757aef4a20879547c16af0a4d1bf4c7Jean Chalard if (n.mCachedAddress != offset) { 666bfbbee8c5757aef4a20879547c16af0a4d1bf4c7Jean Chalard throw new RuntimeException("Wrong address for node " + index 667bfbbee8c5757aef4a20879547c16af0a4d1bf4c7Jean Chalard + " : expected " + offset + ", got " + n.mCachedAddress); 668bfbbee8c5757aef4a20879547c16af0a4d1bf4c7Jean Chalard } 669bfbbee8c5757aef4a20879547c16af0a4d1bf4c7Jean Chalard ++index; 670bfbbee8c5757aef4a20879547c16af0a4d1bf4c7Jean Chalard offset += n.mCachedSize; 671bfbbee8c5757aef4a20879547c16af0a4d1bf4c7Jean Chalard } 672bfbbee8c5757aef4a20879547c16af0a4d1bf4c7Jean Chalard } 673bfbbee8c5757aef4a20879547c16af0a4d1bf4c7Jean Chalard 674bfbbee8c5757aef4a20879547c16af0a4d1bf4c7Jean Chalard /** 675bfbbee8c5757aef4a20879547c16af0a4d1bf4c7Jean Chalard * Helper method to write a variable-size address to a file. 676bfbbee8c5757aef4a20879547c16af0a4d1bf4c7Jean Chalard * 677bfbbee8c5757aef4a20879547c16af0a4d1bf4c7Jean Chalard * @param buffer the buffer to write to. 678bfbbee8c5757aef4a20879547c16af0a4d1bf4c7Jean Chalard * @param index the index in the buffer to write the address to. 679bfbbee8c5757aef4a20879547c16af0a4d1bf4c7Jean Chalard * @param address the address to write. 680bfbbee8c5757aef4a20879547c16af0a4d1bf4c7Jean Chalard * @return the size in bytes the address actually took. 681bfbbee8c5757aef4a20879547c16af0a4d1bf4c7Jean Chalard */ 6823bbb31f3f00e64cb68bd5877ae69d6dbccfeb519Jean Chalard private static int writeVariableAddress(final byte[] buffer, int index, final int address) { 683bfbbee8c5757aef4a20879547c16af0a4d1bf4c7Jean Chalard switch (getByteSize(address)) { 684bfbbee8c5757aef4a20879547c16af0a4d1bf4c7Jean Chalard case 1: 685bfbbee8c5757aef4a20879547c16af0a4d1bf4c7Jean Chalard buffer[index++] = (byte)address; 686bfbbee8c5757aef4a20879547c16af0a4d1bf4c7Jean Chalard return 1; 687bfbbee8c5757aef4a20879547c16af0a4d1bf4c7Jean Chalard case 2: 688bfbbee8c5757aef4a20879547c16af0a4d1bf4c7Jean Chalard buffer[index++] = (byte)(0xFF & (address >> 8)); 689bfbbee8c5757aef4a20879547c16af0a4d1bf4c7Jean Chalard buffer[index++] = (byte)(0xFF & address); 690bfbbee8c5757aef4a20879547c16af0a4d1bf4c7Jean Chalard return 2; 691bfbbee8c5757aef4a20879547c16af0a4d1bf4c7Jean Chalard case 3: 692bfbbee8c5757aef4a20879547c16af0a4d1bf4c7Jean Chalard buffer[index++] = (byte)(0xFF & (address >> 16)); 693bfbbee8c5757aef4a20879547c16af0a4d1bf4c7Jean Chalard buffer[index++] = (byte)(0xFF & (address >> 8)); 694bfbbee8c5757aef4a20879547c16af0a4d1bf4c7Jean Chalard buffer[index++] = (byte)(0xFF & address); 695bfbbee8c5757aef4a20879547c16af0a4d1bf4c7Jean Chalard return 3; 696bfbbee8c5757aef4a20879547c16af0a4d1bf4c7Jean Chalard case 0: 697bfbbee8c5757aef4a20879547c16af0a4d1bf4c7Jean Chalard return 0; 698bfbbee8c5757aef4a20879547c16af0a4d1bf4c7Jean Chalard default: 699bfbbee8c5757aef4a20879547c16af0a4d1bf4c7Jean Chalard throw new RuntimeException("Address " + address + " has a strange size"); 700bfbbee8c5757aef4a20879547c16af0a4d1bf4c7Jean Chalard } 701bfbbee8c5757aef4a20879547c16af0a4d1bf4c7Jean Chalard } 702bfbbee8c5757aef4a20879547c16af0a4d1bf4c7Jean Chalard 7038ec0064c49e80945dbe1bb31129eb890478b7e06Yuichiro Hanada /** 7048ec0064c49e80945dbe1bb31129eb890478b7e06Yuichiro Hanada * Helper method to write a variable-size signed address to a file. 7058ec0064c49e80945dbe1bb31129eb890478b7e06Yuichiro Hanada * 7068ec0064c49e80945dbe1bb31129eb890478b7e06Yuichiro Hanada * @param buffer the buffer to write to. 7078ec0064c49e80945dbe1bb31129eb890478b7e06Yuichiro Hanada * @param index the index in the buffer to write the address to. 7088ec0064c49e80945dbe1bb31129eb890478b7e06Yuichiro Hanada * @param address the address to write. 7098ec0064c49e80945dbe1bb31129eb890478b7e06Yuichiro Hanada * @return the size in bytes the address actually took. 7108ec0064c49e80945dbe1bb31129eb890478b7e06Yuichiro Hanada */ 7118ec0064c49e80945dbe1bb31129eb890478b7e06Yuichiro Hanada private static int writeVariableSignedAddress(final byte[] buffer, int index, 7128ec0064c49e80945dbe1bb31129eb890478b7e06Yuichiro Hanada final int address) { 7138ec0064c49e80945dbe1bb31129eb890478b7e06Yuichiro Hanada if (!hasChildrenAddress(address)) { 7148ec0064c49e80945dbe1bb31129eb890478b7e06Yuichiro Hanada buffer[index] = buffer[index + 1] = buffer[index + 2] = 0; 7158ec0064c49e80945dbe1bb31129eb890478b7e06Yuichiro Hanada } else { 7168ec0064c49e80945dbe1bb31129eb890478b7e06Yuichiro Hanada final int absAddress = Math.abs(address); 7178ec0064c49e80945dbe1bb31129eb890478b7e06Yuichiro Hanada buffer[index++] = (byte)((address < 0 ? MSB8 : 0) | (0xFF & (absAddress >> 16))); 7188ec0064c49e80945dbe1bb31129eb890478b7e06Yuichiro Hanada buffer[index++] = (byte)(0xFF & (absAddress >> 8)); 7198ec0064c49e80945dbe1bb31129eb890478b7e06Yuichiro Hanada buffer[index++] = (byte)(0xFF & absAddress); 7208ec0064c49e80945dbe1bb31129eb890478b7e06Yuichiro Hanada } 7218ec0064c49e80945dbe1bb31129eb890478b7e06Yuichiro Hanada return 3; 7228ec0064c49e80945dbe1bb31129eb890478b7e06Yuichiro Hanada } 7238ec0064c49e80945dbe1bb31129eb890478b7e06Yuichiro Hanada 724bfbbee8c5757aef4a20879547c16af0a4d1bf4c7Jean Chalard private static byte makeCharGroupFlags(final CharGroup group, final int groupAddress, 7258ec0064c49e80945dbe1bb31129eb890478b7e06Yuichiro Hanada final int childrenOffset, final FormatOptions formatOptions) { 726bfbbee8c5757aef4a20879547c16af0a4d1bf4c7Jean Chalard byte flags = 0; 72781d97eec0e77e72cce606f9c9f96091c0b348190Yuichiro Hanada if (group.mChars.length > 1) flags |= FormatSpec.FLAG_HAS_MULTIPLE_CHARS; 728bfbbee8c5757aef4a20879547c16af0a4d1bf4c7Jean Chalard if (group.mFrequency >= 0) { 72981d97eec0e77e72cce606f9c9f96091c0b348190Yuichiro Hanada flags |= FormatSpec.FLAG_IS_TERMINAL; 730bfbbee8c5757aef4a20879547c16af0a4d1bf4c7Jean Chalard } 731bfbbee8c5757aef4a20879547c16af0a4d1bf4c7Jean Chalard if (null != group.mChildren) { 7328ec0064c49e80945dbe1bb31129eb890478b7e06Yuichiro Hanada final int byteSize = formatOptions.mSupportsDynamicUpdate 7338ec0064c49e80945dbe1bb31129eb890478b7e06Yuichiro Hanada ? FormatSpec.SIGNED_CHILDREN_ADDRESS_SIZE : getByteSize(childrenOffset); 7348ec0064c49e80945dbe1bb31129eb890478b7e06Yuichiro Hanada switch (byteSize) { 7358ec0064c49e80945dbe1bb31129eb890478b7e06Yuichiro Hanada case 1: 7368ec0064c49e80945dbe1bb31129eb890478b7e06Yuichiro Hanada flags |= FormatSpec.FLAG_GROUP_ADDRESS_TYPE_ONEBYTE; 7378ec0064c49e80945dbe1bb31129eb890478b7e06Yuichiro Hanada break; 7388ec0064c49e80945dbe1bb31129eb890478b7e06Yuichiro Hanada case 2: 7398ec0064c49e80945dbe1bb31129eb890478b7e06Yuichiro Hanada flags |= FormatSpec.FLAG_GROUP_ADDRESS_TYPE_TWOBYTES; 7408ec0064c49e80945dbe1bb31129eb890478b7e06Yuichiro Hanada break; 7418ec0064c49e80945dbe1bb31129eb890478b7e06Yuichiro Hanada case 3: 7428ec0064c49e80945dbe1bb31129eb890478b7e06Yuichiro Hanada flags |= FormatSpec.FLAG_GROUP_ADDRESS_TYPE_THREEBYTES; 7438ec0064c49e80945dbe1bb31129eb890478b7e06Yuichiro Hanada break; 7448ec0064c49e80945dbe1bb31129eb890478b7e06Yuichiro Hanada default: 7458ec0064c49e80945dbe1bb31129eb890478b7e06Yuichiro Hanada throw new RuntimeException("Node with a strange address"); 7468ec0064c49e80945dbe1bb31129eb890478b7e06Yuichiro Hanada } 7478ec0064c49e80945dbe1bb31129eb890478b7e06Yuichiro Hanada } else if (formatOptions.mSupportsDynamicUpdate) { 7488ec0064c49e80945dbe1bb31129eb890478b7e06Yuichiro Hanada flags |= FormatSpec.FLAG_GROUP_ADDRESS_TYPE_THREEBYTES; 749bfbbee8c5757aef4a20879547c16af0a4d1bf4c7Jean Chalard } 7508edd3067181a425ce1383bb950184f9742af7557Jean Chalard if (null != group.mShortcutTargets) { 7514df5b43df8f4b29fbfab9180cffe5742f8b5f512Jean Chalard if (DBG && 0 == group.mShortcutTargets.size()) { 7528edd3067181a425ce1383bb950184f9742af7557Jean Chalard throw new RuntimeException("0-sized shortcut list must be null"); 7538edd3067181a425ce1383bb950184f9742af7557Jean Chalard } 75481d97eec0e77e72cce606f9c9f96091c0b348190Yuichiro Hanada flags |= FormatSpec.FLAG_HAS_SHORTCUT_TARGETS; 7558edd3067181a425ce1383bb950184f9742af7557Jean Chalard } 7568edd3067181a425ce1383bb950184f9742af7557Jean Chalard if (null != group.mBigrams) { 7574df5b43df8f4b29fbfab9180cffe5742f8b5f512Jean Chalard if (DBG && 0 == group.mBigrams.size()) { 7588edd3067181a425ce1383bb950184f9742af7557Jean Chalard throw new RuntimeException("0-sized bigram list must be null"); 7598edd3067181a425ce1383bb950184f9742af7557Jean Chalard } 76081d97eec0e77e72cce606f9c9f96091c0b348190Yuichiro Hanada flags |= FormatSpec.FLAG_HAS_BIGRAMS; 7618edd3067181a425ce1383bb950184f9742af7557Jean Chalard } 76272b1c9394105b6fbc0d8c6ff00f3574ee37a9aaaJean Chalard if (group.mIsNotAWord) { 76381d97eec0e77e72cce606f9c9f96091c0b348190Yuichiro Hanada flags |= FormatSpec.FLAG_IS_NOT_A_WORD; 76472b1c9394105b6fbc0d8c6ff00f3574ee37a9aaaJean Chalard } 76572b1c9394105b6fbc0d8c6ff00f3574ee37a9aaaJean Chalard if (group.mIsBlacklistEntry) { 76681d97eec0e77e72cce606f9c9f96091c0b348190Yuichiro Hanada flags |= FormatSpec.FLAG_IS_BLACKLISTED; 76772b1c9394105b6fbc0d8c6ff00f3574ee37a9aaaJean Chalard } 768bfbbee8c5757aef4a20879547c16af0a4d1bf4c7Jean Chalard return flags; 769bfbbee8c5757aef4a20879547c16af0a4d1bf4c7Jean Chalard } 770bfbbee8c5757aef4a20879547c16af0a4d1bf4c7Jean Chalard 771bfbbee8c5757aef4a20879547c16af0a4d1bf4c7Jean Chalard /** 7724455fe2c894f8aabaf2b3105b72f9193226d4abaJean Chalard * Makes the flag value for a bigram. 773bfbbee8c5757aef4a20879547c16af0a4d1bf4c7Jean Chalard * 7744455fe2c894f8aabaf2b3105b72f9193226d4abaJean Chalard * @param more whether there are more bigrams after this one. 7754455fe2c894f8aabaf2b3105b72f9193226d4abaJean Chalard * @param offset the offset of the bigram. 776f7346de94a902b0d0675a85425e68eba96cece7eJean Chalard * @param bigramFrequency the frequency of the bigram, 0..255. 777f7346de94a902b0d0675a85425e68eba96cece7eJean Chalard * @param unigramFrequency the unigram frequency of the same word, 0..255. 778f7346de94a902b0d0675a85425e68eba96cece7eJean Chalard * @param word the second bigram, for debugging purposes 779bfbbee8c5757aef4a20879547c16af0a4d1bf4c7Jean Chalard * @return the flags 780bfbbee8c5757aef4a20879547c16af0a4d1bf4c7Jean Chalard */ 7814455fe2c894f8aabaf2b3105b72f9193226d4abaJean Chalard private static final int makeBigramFlags(final boolean more, final int offset, 782f7346de94a902b0d0675a85425e68eba96cece7eJean Chalard int bigramFrequency, final int unigramFrequency, final String word) { 78381d97eec0e77e72cce606f9c9f96091c0b348190Yuichiro Hanada int bigramFlags = (more ? FormatSpec.FLAG_ATTRIBUTE_HAS_NEXT : 0) 78481d97eec0e77e72cce606f9c9f96091c0b348190Yuichiro Hanada + (offset < 0 ? FormatSpec.FLAG_ATTRIBUTE_OFFSET_NEGATIVE : 0); 785bfbbee8c5757aef4a20879547c16af0a4d1bf4c7Jean Chalard switch (getByteSize(offset)) { 786bfbbee8c5757aef4a20879547c16af0a4d1bf4c7Jean Chalard case 1: 78781d97eec0e77e72cce606f9c9f96091c0b348190Yuichiro Hanada bigramFlags |= FormatSpec.FLAG_ATTRIBUTE_ADDRESS_TYPE_ONEBYTE; 788bfbbee8c5757aef4a20879547c16af0a4d1bf4c7Jean Chalard break; 789bfbbee8c5757aef4a20879547c16af0a4d1bf4c7Jean Chalard case 2: 79081d97eec0e77e72cce606f9c9f96091c0b348190Yuichiro Hanada bigramFlags |= FormatSpec.FLAG_ATTRIBUTE_ADDRESS_TYPE_TWOBYTES; 791bfbbee8c5757aef4a20879547c16af0a4d1bf4c7Jean Chalard break; 792bfbbee8c5757aef4a20879547c16af0a4d1bf4c7Jean Chalard case 3: 79381d97eec0e77e72cce606f9c9f96091c0b348190Yuichiro Hanada bigramFlags |= FormatSpec.FLAG_ATTRIBUTE_ADDRESS_TYPE_THREEBYTES; 794bfbbee8c5757aef4a20879547c16af0a4d1bf4c7Jean Chalard break; 795bfbbee8c5757aef4a20879547c16af0a4d1bf4c7Jean Chalard default: 796bfbbee8c5757aef4a20879547c16af0a4d1bf4c7Jean Chalard throw new RuntimeException("Strange offset size"); 797bfbbee8c5757aef4a20879547c16af0a4d1bf4c7Jean Chalard } 798f7346de94a902b0d0675a85425e68eba96cece7eJean Chalard if (unigramFrequency > bigramFrequency) { 799f7346de94a902b0d0675a85425e68eba96cece7eJean Chalard MakedictLog.e("Unigram freq is superior to bigram freq for \"" + word 800f7346de94a902b0d0675a85425e68eba96cece7eJean Chalard + "\". Bigram freq is " + bigramFrequency + ", unigram freq for " 801f7346de94a902b0d0675a85425e68eba96cece7eJean Chalard + word + " is " + unigramFrequency); 802f7346de94a902b0d0675a85425e68eba96cece7eJean Chalard bigramFrequency = unigramFrequency; 803f7346de94a902b0d0675a85425e68eba96cece7eJean Chalard } 804f7346de94a902b0d0675a85425e68eba96cece7eJean Chalard // We compute the difference between 255 (which means probability = 1) and the 805418b34379733aa7f3d31729090797c747c8a43a8Jean Chalard // unigram score. We split this into a number of discrete steps. 806418b34379733aa7f3d31729090797c747c8a43a8Jean Chalard // Now, the steps are numbered 0~15; 0 represents an increase of 1 step while 15 807418b34379733aa7f3d31729090797c747c8a43a8Jean Chalard // represents an increase of 16 steps: a value of 15 will be interpreted as the median 808418b34379733aa7f3d31729090797c747c8a43a8Jean Chalard // value of the 16th step. In all justice, if the bigram frequency is low enough to be 809418b34379733aa7f3d31729090797c747c8a43a8Jean Chalard // rounded below the first step (which means it is less than half a step higher than the 810418b34379733aa7f3d31729090797c747c8a43a8Jean Chalard // unigram frequency) then the unigram frequency itself is the best approximation of the 811418b34379733aa7f3d31729090797c747c8a43a8Jean Chalard // bigram freq that we could possibly supply, hence we should *not* include this bigram 812418b34379733aa7f3d31729090797c747c8a43a8Jean Chalard // in the file at all. 813418b34379733aa7f3d31729090797c747c8a43a8Jean Chalard // until this is done, we'll write 0 and slightly overestimate this case. 814418b34379733aa7f3d31729090797c747c8a43a8Jean Chalard // In other words, 0 means "between 0.5 step and 1.5 step", 1 means "between 1.5 step 815418b34379733aa7f3d31729090797c747c8a43a8Jean Chalard // and 2.5 steps", and 15 means "between 15.5 steps and 16.5 steps". So we want to 816418b34379733aa7f3d31729090797c747c8a43a8Jean Chalard // divide our range [unigramFreq..MAX_TERMINAL_FREQUENCY] in 16.5 steps to get the 817418b34379733aa7f3d31729090797c747c8a43a8Jean Chalard // step size. Then we compute the start of the first step (the one where value 0 starts) 818418b34379733aa7f3d31729090797c747c8a43a8Jean Chalard // by adding half-a-step to the unigramFrequency. From there, we compute the integer 819418b34379733aa7f3d31729090797c747c8a43a8Jean Chalard // number of steps to the bigramFrequency. One last thing: we want our steps to include 820418b34379733aa7f3d31729090797c747c8a43a8Jean Chalard // their lower bound and exclude their higher bound so we need to have the first step 821418b34379733aa7f3d31729090797c747c8a43a8Jean Chalard // start at exactly 1 unit higher than floor(unigramFreq + half a step). 822418b34379733aa7f3d31729090797c747c8a43a8Jean Chalard // Note : to reconstruct the score, the dictionary reader will need to divide 823aa27635a8a234b23d6db0957ab398443498eb558Jean Chalard // MAX_TERMINAL_FREQUENCY - unigramFreq by 16.5 likewise to get the value of the step, 824aa27635a8a234b23d6db0957ab398443498eb558Jean Chalard // and add (discretizedFrequency + 0.5 + 0.5) times this value to get the best 825aa27635a8a234b23d6db0957ab398443498eb558Jean Chalard // approximation. (0.5 to get the first step start, and 0.5 to get the middle of the 826aa27635a8a234b23d6db0957ab398443498eb558Jean Chalard // step pointed by the discretized frequency. 827d10c473347c7e21c383c56786c9eb96fd6513a5cJean Chalard final float stepSize = 82881d97eec0e77e72cce606f9c9f96091c0b348190Yuichiro Hanada (FormatSpec.MAX_TERMINAL_FREQUENCY - unigramFrequency) 82981d97eec0e77e72cce606f9c9f96091c0b348190Yuichiro Hanada / (1.5f + FormatSpec.MAX_BIGRAM_FREQUENCY); 830d10c473347c7e21c383c56786c9eb96fd6513a5cJean Chalard final float firstStepStart = 1 + unigramFrequency + (stepSize / 2.0f); 831418b34379733aa7f3d31729090797c747c8a43a8Jean Chalard final int discretizedFrequency = (int)((bigramFrequency - firstStepStart) / stepSize); 832418b34379733aa7f3d31729090797c747c8a43a8Jean Chalard // If the bigram freq is less than half-a-step higher than the unigram freq, we get -1 833418b34379733aa7f3d31729090797c747c8a43a8Jean Chalard // here. The best approximation would be the unigram freq itself, so we should not 834418b34379733aa7f3d31729090797c747c8a43a8Jean Chalard // include this bigram in the dictionary. For now, register as 0, and live with the 835418b34379733aa7f3d31729090797c747c8a43a8Jean Chalard // small over-estimation that we get in this case. TODO: actually remove this bigram 836418b34379733aa7f3d31729090797c747c8a43a8Jean Chalard // if discretizedFrequency < 0. 837418b34379733aa7f3d31729090797c747c8a43a8Jean Chalard final int finalBigramFrequency = discretizedFrequency > 0 ? discretizedFrequency : 0; 83881d97eec0e77e72cce606f9c9f96091c0b348190Yuichiro Hanada bigramFlags += finalBigramFrequency & FormatSpec.FLAG_ATTRIBUTE_FREQUENCY; 839bfbbee8c5757aef4a20879547c16af0a4d1bf4c7Jean Chalard return bigramFlags; 840bfbbee8c5757aef4a20879547c16af0a4d1bf4c7Jean Chalard } 841bfbbee8c5757aef4a20879547c16af0a4d1bf4c7Jean Chalard 842bfbbee8c5757aef4a20879547c16af0a4d1bf4c7Jean Chalard /** 843f420df28233c26e555d203185fb292e83b94b8c3Jean Chalard * Makes the 2-byte value for options flags. 844f420df28233c26e555d203185fb292e83b94b8c3Jean Chalard */ 845e55b644aefbefb4ac79308c9a59116e69a9c53a2Yuichiro Hanada private static final int makeOptionsValue(final FusionDictionary dictionary, 846e55b644aefbefb4ac79308c9a59116e69a9c53a2Yuichiro Hanada final FormatOptions formatOptions) { 84720a6dea1cabfd8822824f7dca828d898e5b91cbcJean Chalard final DictionaryOptions options = dictionary.mOptions; 84820a6dea1cabfd8822824f7dca828d898e5b91cbcJean Chalard final boolean hasBigrams = dictionary.hasBigrams(); 84981d97eec0e77e72cce606f9c9f96091c0b348190Yuichiro Hanada return (options.mFrenchLigatureProcessing ? FormatSpec.FRENCH_LIGATURE_PROCESSING_FLAG : 0) 85081d97eec0e77e72cce606f9c9f96091c0b348190Yuichiro Hanada + (options.mGermanUmlautProcessing ? FormatSpec.GERMAN_UMLAUT_PROCESSING_FLAG : 0) 85181d97eec0e77e72cce606f9c9f96091c0b348190Yuichiro Hanada + (hasBigrams ? FormatSpec.CONTAINS_BIGRAMS_FLAG : 0) 85282d9deaaf252cd20f8918adbc7a4b9b8f2647c38Yuichiro Hanada + (formatOptions.mSupportsDynamicUpdate ? FormatSpec.SUPPORTS_DYNAMIC_UPDATE : 0); 853f420df28233c26e555d203185fb292e83b94b8c3Jean Chalard } 854f420df28233c26e555d203185fb292e83b94b8c3Jean Chalard 855f420df28233c26e555d203185fb292e83b94b8c3Jean Chalard /** 8563bbb31f3f00e64cb68bd5877ae69d6dbccfeb519Jean Chalard * Makes the flag value for a shortcut. 8573bbb31f3f00e64cb68bd5877ae69d6dbccfeb519Jean Chalard * 8583bbb31f3f00e64cb68bd5877ae69d6dbccfeb519Jean Chalard * @param more whether there are more attributes after this one. 8593bbb31f3f00e64cb68bd5877ae69d6dbccfeb519Jean Chalard * @param frequency the frequency of the attribute, 0..15 8603bbb31f3f00e64cb68bd5877ae69d6dbccfeb519Jean Chalard * @return the flags 8613bbb31f3f00e64cb68bd5877ae69d6dbccfeb519Jean Chalard */ 8623bbb31f3f00e64cb68bd5877ae69d6dbccfeb519Jean Chalard private static final int makeShortcutFlags(final boolean more, final int frequency) { 86381d97eec0e77e72cce606f9c9f96091c0b348190Yuichiro Hanada return (more ? FormatSpec.FLAG_ATTRIBUTE_HAS_NEXT : 0) 86481d97eec0e77e72cce606f9c9f96091c0b348190Yuichiro Hanada + (frequency & FormatSpec.FLAG_ATTRIBUTE_FREQUENCY); 8653bbb31f3f00e64cb68bd5877ae69d6dbccfeb519Jean Chalard } 8663bbb31f3f00e64cb68bd5877ae69d6dbccfeb519Jean Chalard 8678ec0064c49e80945dbe1bb31129eb890478b7e06Yuichiro Hanada private static final int writeParentAddress(final byte[] buffer, final int index, 8688ec0064c49e80945dbe1bb31129eb890478b7e06Yuichiro Hanada final int address, final FormatOptions formatOptions) { 8698ec0064c49e80945dbe1bb31129eb890478b7e06Yuichiro Hanada if (supportsDynamicUpdate(formatOptions)) { 8708ec0064c49e80945dbe1bb31129eb890478b7e06Yuichiro Hanada if (address == FormatSpec.NO_PARENT_ADDRESS) { 8718ec0064c49e80945dbe1bb31129eb890478b7e06Yuichiro Hanada buffer[index] = buffer[index + 1] = buffer[index + 2] = 0; 8728ec0064c49e80945dbe1bb31129eb890478b7e06Yuichiro Hanada } else { 8738ec0064c49e80945dbe1bb31129eb890478b7e06Yuichiro Hanada final int absAddress = Math.abs(address); 8748ec0064c49e80945dbe1bb31129eb890478b7e06Yuichiro Hanada assert(absAddress <= SINT24_MAX); 8758ec0064c49e80945dbe1bb31129eb890478b7e06Yuichiro Hanada buffer[index] = (byte)((address < 0 ? MSB8 : 0) 8768ec0064c49e80945dbe1bb31129eb890478b7e06Yuichiro Hanada | ((absAddress >> 16) & 0xFF)); 8778ec0064c49e80945dbe1bb31129eb890478b7e06Yuichiro Hanada buffer[index + 1] = (byte)((absAddress >> 8) & 0xFF); 8788ec0064c49e80945dbe1bb31129eb890478b7e06Yuichiro Hanada buffer[index + 2] = (byte)(absAddress & 0xFF); 8798ec0064c49e80945dbe1bb31129eb890478b7e06Yuichiro Hanada } 8808ec0064c49e80945dbe1bb31129eb890478b7e06Yuichiro Hanada return index + 3; 8818ec0064c49e80945dbe1bb31129eb890478b7e06Yuichiro Hanada } else { 8828ec0064c49e80945dbe1bb31129eb890478b7e06Yuichiro Hanada return index; 8838ec0064c49e80945dbe1bb31129eb890478b7e06Yuichiro Hanada } 8848ec0064c49e80945dbe1bb31129eb890478b7e06Yuichiro Hanada } 8858ec0064c49e80945dbe1bb31129eb890478b7e06Yuichiro Hanada 8863bbb31f3f00e64cb68bd5877ae69d6dbccfeb519Jean Chalard /** 887bfbbee8c5757aef4a20879547c16af0a4d1bf4c7Jean Chalard * Write a node to memory. The node is expected to have its final position cached. 888bfbbee8c5757aef4a20879547c16af0a4d1bf4c7Jean Chalard * 889bfbbee8c5757aef4a20879547c16af0a4d1bf4c7Jean Chalard * This can be an empty map, but the more is inside the faster the lookups will be. It can 890bfbbee8c5757aef4a20879547c16af0a4d1bf4c7Jean Chalard * be carried on as long as nodes do not move. 891bfbbee8c5757aef4a20879547c16af0a4d1bf4c7Jean Chalard * 892bfbbee8c5757aef4a20879547c16af0a4d1bf4c7Jean Chalard * @param dict the dictionary the node is a part of (for relative offsets). 893bfbbee8c5757aef4a20879547c16af0a4d1bf4c7Jean Chalard * @param buffer the memory buffer to write to. 894bfbbee8c5757aef4a20879547c16af0a4d1bf4c7Jean Chalard * @param node the node to write. 895e55b644aefbefb4ac79308c9a59116e69a9c53a2Yuichiro Hanada * @param formatOptions file format options. 896bfbbee8c5757aef4a20879547c16af0a4d1bf4c7Jean Chalard * @return the address of the END of the node. 897bfbbee8c5757aef4a20879547c16af0a4d1bf4c7Jean Chalard */ 898e55b644aefbefb4ac79308c9a59116e69a9c53a2Yuichiro Hanada private static int writePlacedNode(final FusionDictionary dict, byte[] buffer, 899e55b644aefbefb4ac79308c9a59116e69a9c53a2Yuichiro Hanada final Node node, final FormatOptions formatOptions) { 900bfbbee8c5757aef4a20879547c16af0a4d1bf4c7Jean Chalard int index = node.mCachedAddress; 901bfbbee8c5757aef4a20879547c16af0a4d1bf4c7Jean Chalard 9022c0903b1d17a4669fee417c2f0b8f19bd7091ed7Jean Chalard final int groupCount = node.mData.size(); 9032c0903b1d17a4669fee417c2f0b8f19bd7091ed7Jean Chalard final int countSize = getGroupCountSize(node); 904e55b644aefbefb4ac79308c9a59116e69a9c53a2Yuichiro Hanada final int parentAddress = node.mCachedParentAddress; 9052c0903b1d17a4669fee417c2f0b8f19bd7091ed7Jean Chalard if (1 == countSize) { 9062c0903b1d17a4669fee417c2f0b8f19bd7091ed7Jean Chalard buffer[index++] = (byte)groupCount; 9072c0903b1d17a4669fee417c2f0b8f19bd7091ed7Jean Chalard } else if (2 == countSize) { 9082c0903b1d17a4669fee417c2f0b8f19bd7091ed7Jean Chalard // We need to signal 2-byte size by setting the top bit of the MSB to 1, so 9092c0903b1d17a4669fee417c2f0b8f19bd7091ed7Jean Chalard // we | 0x80 to do this. 9102c0903b1d17a4669fee417c2f0b8f19bd7091ed7Jean Chalard buffer[index++] = (byte)((groupCount >> 8) | 0x80); 9112c0903b1d17a4669fee417c2f0b8f19bd7091ed7Jean Chalard buffer[index++] = (byte)(groupCount & 0xFF); 9122c0903b1d17a4669fee417c2f0b8f19bd7091ed7Jean Chalard } else { 9132c0903b1d17a4669fee417c2f0b8f19bd7091ed7Jean Chalard throw new RuntimeException("Strange size from getGroupCountSize : " + countSize); 9142c0903b1d17a4669fee417c2f0b8f19bd7091ed7Jean Chalard } 915bfbbee8c5757aef4a20879547c16af0a4d1bf4c7Jean Chalard int groupAddress = index; 9162c0903b1d17a4669fee417c2f0b8f19bd7091ed7Jean Chalard for (int i = 0; i < groupCount; ++i) { 917bfbbee8c5757aef4a20879547c16af0a4d1bf4c7Jean Chalard CharGroup group = node.mData.get(i); 918bfbbee8c5757aef4a20879547c16af0a4d1bf4c7Jean Chalard if (index != group.mCachedAddress) throw new RuntimeException("Bug: write index is not " 9193bbb31f3f00e64cb68bd5877ae69d6dbccfeb519Jean Chalard + "the same as the cached address of the group : " 9203bbb31f3f00e64cb68bd5877ae69d6dbccfeb519Jean Chalard + index + " <> " + group.mCachedAddress); 921e55b644aefbefb4ac79308c9a59116e69a9c53a2Yuichiro Hanada groupAddress += getGroupHeaderSize(group, formatOptions); 922bfbbee8c5757aef4a20879547c16af0a4d1bf4c7Jean Chalard // Sanity checks. 92381d97eec0e77e72cce606f9c9f96091c0b348190Yuichiro Hanada if (DBG && group.mFrequency > FormatSpec.MAX_TERMINAL_FREQUENCY) { 92481d97eec0e77e72cce606f9c9f96091c0b348190Yuichiro Hanada throw new RuntimeException("A node has a frequency > " 92581d97eec0e77e72cce606f9c9f96091c0b348190Yuichiro Hanada + FormatSpec.MAX_TERMINAL_FREQUENCY 926bfbbee8c5757aef4a20879547c16af0a4d1bf4c7Jean Chalard + " : " + group.mFrequency); 927bfbbee8c5757aef4a20879547c16af0a4d1bf4c7Jean Chalard } 92881d97eec0e77e72cce606f9c9f96091c0b348190Yuichiro Hanada if (group.mFrequency >= 0) groupAddress += FormatSpec.GROUP_FREQUENCY_SIZE; 929bfbbee8c5757aef4a20879547c16af0a4d1bf4c7Jean Chalard final int childrenOffset = null == group.mChildren 93081d97eec0e77e72cce606f9c9f96091c0b348190Yuichiro Hanada ? FormatSpec.NO_CHILDREN_ADDRESS 93181d97eec0e77e72cce606f9c9f96091c0b348190Yuichiro Hanada : group.mChildren.mCachedAddress - groupAddress; 9328ec0064c49e80945dbe1bb31129eb890478b7e06Yuichiro Hanada byte flags = makeCharGroupFlags(group, groupAddress, childrenOffset, formatOptions); 933bfbbee8c5757aef4a20879547c16af0a4d1bf4c7Jean Chalard buffer[index++] = flags; 934e55b644aefbefb4ac79308c9a59116e69a9c53a2Yuichiro Hanada 9358ec0064c49e80945dbe1bb31129eb890478b7e06Yuichiro Hanada if (parentAddress == FormatSpec.NO_PARENT_ADDRESS) { 9368ec0064c49e80945dbe1bb31129eb890478b7e06Yuichiro Hanada index = writeParentAddress(buffer, index, parentAddress, formatOptions); 9378ec0064c49e80945dbe1bb31129eb890478b7e06Yuichiro Hanada } else { 9388ec0064c49e80945dbe1bb31129eb890478b7e06Yuichiro Hanada index = writeParentAddress(buffer, index, 9398ec0064c49e80945dbe1bb31129eb890478b7e06Yuichiro Hanada parentAddress + (node.mCachedAddress - group.mCachedAddress), 9408ec0064c49e80945dbe1bb31129eb890478b7e06Yuichiro Hanada formatOptions); 941e55b644aefbefb4ac79308c9a59116e69a9c53a2Yuichiro Hanada } 942e55b644aefbefb4ac79308c9a59116e69a9c53a2Yuichiro Hanada 943bfbbee8c5757aef4a20879547c16af0a4d1bf4c7Jean Chalard index = CharEncoding.writeCharArray(group.mChars, buffer, index); 944bfbbee8c5757aef4a20879547c16af0a4d1bf4c7Jean Chalard if (group.hasSeveralChars()) { 94581d97eec0e77e72cce606f9c9f96091c0b348190Yuichiro Hanada buffer[index++] = FormatSpec.GROUP_CHARACTERS_TERMINATOR; 946bfbbee8c5757aef4a20879547c16af0a4d1bf4c7Jean Chalard } 947bfbbee8c5757aef4a20879547c16af0a4d1bf4c7Jean Chalard if (group.mFrequency >= 0) { 948bfbbee8c5757aef4a20879547c16af0a4d1bf4c7Jean Chalard buffer[index++] = (byte) group.mFrequency; 949bfbbee8c5757aef4a20879547c16af0a4d1bf4c7Jean Chalard } 9508ec0064c49e80945dbe1bb31129eb890478b7e06Yuichiro Hanada 9518ec0064c49e80945dbe1bb31129eb890478b7e06Yuichiro Hanada final int shift; 9528ec0064c49e80945dbe1bb31129eb890478b7e06Yuichiro Hanada if (formatOptions.mSupportsDynamicUpdate) { 9538ec0064c49e80945dbe1bb31129eb890478b7e06Yuichiro Hanada shift = writeVariableSignedAddress(buffer, index, childrenOffset); 9548ec0064c49e80945dbe1bb31129eb890478b7e06Yuichiro Hanada } else { 9558ec0064c49e80945dbe1bb31129eb890478b7e06Yuichiro Hanada shift = writeVariableAddress(buffer, index, childrenOffset); 9568ec0064c49e80945dbe1bb31129eb890478b7e06Yuichiro Hanada } 957bfbbee8c5757aef4a20879547c16af0a4d1bf4c7Jean Chalard index += shift; 958bfbbee8c5757aef4a20879547c16af0a4d1bf4c7Jean Chalard groupAddress += shift; 959bfbbee8c5757aef4a20879547c16af0a4d1bf4c7Jean Chalard 9608edd3067181a425ce1383bb950184f9742af7557Jean Chalard // Write shortcuts 9618edd3067181a425ce1383bb950184f9742af7557Jean Chalard if (null != group.mShortcutTargets) { 9623bbb31f3f00e64cb68bd5877ae69d6dbccfeb519Jean Chalard final int indexOfShortcutByteSize = index; 96381d97eec0e77e72cce606f9c9f96091c0b348190Yuichiro Hanada index += FormatSpec.GROUP_SHORTCUT_LIST_SIZE_SIZE; 96481d97eec0e77e72cce606f9c9f96091c0b348190Yuichiro Hanada groupAddress += FormatSpec.GROUP_SHORTCUT_LIST_SIZE_SIZE; 96593ebf74bae44728e0d5f7e738ea28376187a876eTadashi G. Takaoka final Iterator<WeightedString> shortcutIterator = group.mShortcutTargets.iterator(); 9668edd3067181a425ce1383bb950184f9742af7557Jean Chalard while (shortcutIterator.hasNext()) { 96793ebf74bae44728e0d5f7e738ea28376187a876eTadashi G. Takaoka final WeightedString target = shortcutIterator.next(); 9688edd3067181a425ce1383bb950184f9742af7557Jean Chalard ++groupAddress; 9693bbb31f3f00e64cb68bd5877ae69d6dbccfeb519Jean Chalard int shortcutFlags = makeShortcutFlags(shortcutIterator.hasNext(), 9708edd3067181a425ce1383bb950184f9742af7557Jean Chalard target.mFrequency); 9718edd3067181a425ce1383bb950184f9742af7557Jean Chalard buffer[index++] = (byte)shortcutFlags; 9723bbb31f3f00e64cb68bd5877ae69d6dbccfeb519Jean Chalard final int shortcutShift = CharEncoding.writeString(buffer, index, target.mWord); 9738edd3067181a425ce1383bb950184f9742af7557Jean Chalard index += shortcutShift; 9748edd3067181a425ce1383bb950184f9742af7557Jean Chalard groupAddress += shortcutShift; 9758edd3067181a425ce1383bb950184f9742af7557Jean Chalard } 9763bbb31f3f00e64cb68bd5877ae69d6dbccfeb519Jean Chalard final int shortcutByteSize = index - indexOfShortcutByteSize; 9773bbb31f3f00e64cb68bd5877ae69d6dbccfeb519Jean Chalard if (shortcutByteSize > 0xFFFF) { 9783bbb31f3f00e64cb68bd5877ae69d6dbccfeb519Jean Chalard throw new RuntimeException("Shortcut list too large"); 9793bbb31f3f00e64cb68bd5877ae69d6dbccfeb519Jean Chalard } 9803bbb31f3f00e64cb68bd5877ae69d6dbccfeb519Jean Chalard buffer[indexOfShortcutByteSize] = (byte)(shortcutByteSize >> 8); 9813bbb31f3f00e64cb68bd5877ae69d6dbccfeb519Jean Chalard buffer[indexOfShortcutByteSize + 1] = (byte)(shortcutByteSize & 0xFF); 9828edd3067181a425ce1383bb950184f9742af7557Jean Chalard } 983bfbbee8c5757aef4a20879547c16af0a4d1bf4c7Jean Chalard // Write bigrams 984bfbbee8c5757aef4a20879547c16af0a4d1bf4c7Jean Chalard if (null != group.mBigrams) { 98593ebf74bae44728e0d5f7e738ea28376187a876eTadashi G. Takaoka final Iterator<WeightedString> bigramIterator = group.mBigrams.iterator(); 9868edd3067181a425ce1383bb950184f9742af7557Jean Chalard while (bigramIterator.hasNext()) { 98793ebf74bae44728e0d5f7e738ea28376187a876eTadashi G. Takaoka final WeightedString bigram = bigramIterator.next(); 9884455fe2c894f8aabaf2b3105b72f9193226d4abaJean Chalard final CharGroup target = 9894455fe2c894f8aabaf2b3105b72f9193226d4abaJean Chalard FusionDictionary.findWordInTree(dict.mRoot, bigram.mWord); 9904455fe2c894f8aabaf2b3105b72f9193226d4abaJean Chalard final int addressOfBigram = target.mCachedAddress; 9914455fe2c894f8aabaf2b3105b72f9193226d4abaJean Chalard final int unigramFrequencyForThisWord = target.mFrequency; 992bfbbee8c5757aef4a20879547c16af0a4d1bf4c7Jean Chalard ++groupAddress; 993bfbbee8c5757aef4a20879547c16af0a4d1bf4c7Jean Chalard final int offset = addressOfBigram - groupAddress; 9944455fe2c894f8aabaf2b3105b72f9193226d4abaJean Chalard int bigramFlags = makeBigramFlags(bigramIterator.hasNext(), offset, 995f7346de94a902b0d0675a85425e68eba96cece7eJean Chalard bigram.mFrequency, unigramFrequencyForThisWord, bigram.mWord); 996bfbbee8c5757aef4a20879547c16af0a4d1bf4c7Jean Chalard buffer[index++] = (byte)bigramFlags; 997bfbbee8c5757aef4a20879547c16af0a4d1bf4c7Jean Chalard final int bigramShift = writeVariableAddress(buffer, index, Math.abs(offset)); 998bfbbee8c5757aef4a20879547c16af0a4d1bf4c7Jean Chalard index += bigramShift; 999bfbbee8c5757aef4a20879547c16af0a4d1bf4c7Jean Chalard groupAddress += bigramShift; 1000bfbbee8c5757aef4a20879547c16af0a4d1bf4c7Jean Chalard } 1001bfbbee8c5757aef4a20879547c16af0a4d1bf4c7Jean Chalard } 1002bfbbee8c5757aef4a20879547c16af0a4d1bf4c7Jean Chalard 1003bfbbee8c5757aef4a20879547c16af0a4d1bf4c7Jean Chalard } 100482d9deaaf252cd20f8918adbc7a4b9b8f2647c38Yuichiro Hanada if (formatOptions.mSupportsDynamicUpdate) { 1005bf45dc4860ab28e97c3e7d116a642802fe960239Yuichiro Hanada buffer[index] = buffer[index + 1] = buffer[index + 2] 1006bf45dc4860ab28e97c3e7d116a642802fe960239Yuichiro Hanada = FormatSpec.NO_FORWARD_LINK_ADDRESS; 1007bf45dc4860ab28e97c3e7d116a642802fe960239Yuichiro Hanada index += FormatSpec.FORWARD_LINK_ADDRESS_SIZE; 1008bf45dc4860ab28e97c3e7d116a642802fe960239Yuichiro Hanada } 1009bfbbee8c5757aef4a20879547c16af0a4d1bf4c7Jean Chalard if (index != node.mCachedAddress + node.mCachedSize) throw new RuntimeException( 1010bfbbee8c5757aef4a20879547c16af0a4d1bf4c7Jean Chalard "Not the same size : written " 1011bfbbee8c5757aef4a20879547c16af0a4d1bf4c7Jean Chalard + (index - node.mCachedAddress) + " bytes out of a node that should have " 1012bfbbee8c5757aef4a20879547c16af0a4d1bf4c7Jean Chalard + node.mCachedSize + " bytes"); 1013bfbbee8c5757aef4a20879547c16af0a4d1bf4c7Jean Chalard return index; 1014bfbbee8c5757aef4a20879547c16af0a4d1bf4c7Jean Chalard } 1015bfbbee8c5757aef4a20879547c16af0a4d1bf4c7Jean Chalard 1016bfbbee8c5757aef4a20879547c16af0a4d1bf4c7Jean Chalard /** 1017bfbbee8c5757aef4a20879547c16af0a4d1bf4c7Jean Chalard * Dumps a collection of useful statistics about a node array. 1018bfbbee8c5757aef4a20879547c16af0a4d1bf4c7Jean Chalard * 1019bfbbee8c5757aef4a20879547c16af0a4d1bf4c7Jean Chalard * This prints purely informative stuff, like the total estimated file size, the 1020bfbbee8c5757aef4a20879547c16af0a4d1bf4c7Jean Chalard * number of nodes, of character groups, the repartition of each address size, etc 1021bfbbee8c5757aef4a20879547c16af0a4d1bf4c7Jean Chalard * 1022bfbbee8c5757aef4a20879547c16af0a4d1bf4c7Jean Chalard * @param nodes the node array. 1023bfbbee8c5757aef4a20879547c16af0a4d1bf4c7Jean Chalard */ 1024bfbbee8c5757aef4a20879547c16af0a4d1bf4c7Jean Chalard private static void showStatistics(ArrayList<Node> nodes) { 1025bfbbee8c5757aef4a20879547c16af0a4d1bf4c7Jean Chalard int firstTerminalAddress = Integer.MAX_VALUE; 1026bfbbee8c5757aef4a20879547c16af0a4d1bf4c7Jean Chalard int lastTerminalAddress = Integer.MIN_VALUE; 1027bfbbee8c5757aef4a20879547c16af0a4d1bf4c7Jean Chalard int size = 0; 1028bfbbee8c5757aef4a20879547c16af0a4d1bf4c7Jean Chalard int charGroups = 0; 1029bfbbee8c5757aef4a20879547c16af0a4d1bf4c7Jean Chalard int maxGroups = 0; 1030bfbbee8c5757aef4a20879547c16af0a4d1bf4c7Jean Chalard int maxRuns = 0; 1031bfbbee8c5757aef4a20879547c16af0a4d1bf4c7Jean Chalard for (Node n : nodes) { 1032bfbbee8c5757aef4a20879547c16af0a4d1bf4c7Jean Chalard if (maxGroups < n.mData.size()) maxGroups = n.mData.size(); 1033bfbbee8c5757aef4a20879547c16af0a4d1bf4c7Jean Chalard for (CharGroup cg : n.mData) { 1034bfbbee8c5757aef4a20879547c16af0a4d1bf4c7Jean Chalard ++charGroups; 1035bfbbee8c5757aef4a20879547c16af0a4d1bf4c7Jean Chalard if (cg.mChars.length > maxRuns) maxRuns = cg.mChars.length; 1036bfbbee8c5757aef4a20879547c16af0a4d1bf4c7Jean Chalard if (cg.mFrequency >= 0) { 1037bfbbee8c5757aef4a20879547c16af0a4d1bf4c7Jean Chalard if (n.mCachedAddress < firstTerminalAddress) 1038bfbbee8c5757aef4a20879547c16af0a4d1bf4c7Jean Chalard firstTerminalAddress = n.mCachedAddress; 1039bfbbee8c5757aef4a20879547c16af0a4d1bf4c7Jean Chalard if (n.mCachedAddress > lastTerminalAddress) 1040bfbbee8c5757aef4a20879547c16af0a4d1bf4c7Jean Chalard lastTerminalAddress = n.mCachedAddress; 1041bfbbee8c5757aef4a20879547c16af0a4d1bf4c7Jean Chalard } 1042bfbbee8c5757aef4a20879547c16af0a4d1bf4c7Jean Chalard } 1043bfbbee8c5757aef4a20879547c16af0a4d1bf4c7Jean Chalard if (n.mCachedAddress + n.mCachedSize > size) size = n.mCachedAddress + n.mCachedSize; 1044bfbbee8c5757aef4a20879547c16af0a4d1bf4c7Jean Chalard } 1045bfbbee8c5757aef4a20879547c16af0a4d1bf4c7Jean Chalard final int[] groupCounts = new int[maxGroups + 1]; 1046bfbbee8c5757aef4a20879547c16af0a4d1bf4c7Jean Chalard final int[] runCounts = new int[maxRuns + 1]; 1047bfbbee8c5757aef4a20879547c16af0a4d1bf4c7Jean Chalard for (Node n : nodes) { 1048bfbbee8c5757aef4a20879547c16af0a4d1bf4c7Jean Chalard ++groupCounts[n.mData.size()]; 1049bfbbee8c5757aef4a20879547c16af0a4d1bf4c7Jean Chalard for (CharGroup cg : n.mData) { 1050bfbbee8c5757aef4a20879547c16af0a4d1bf4c7Jean Chalard ++runCounts[cg.mChars.length]; 1051bfbbee8c5757aef4a20879547c16af0a4d1bf4c7Jean Chalard } 1052bfbbee8c5757aef4a20879547c16af0a4d1bf4c7Jean Chalard } 1053bfbbee8c5757aef4a20879547c16af0a4d1bf4c7Jean Chalard 1054bfbbee8c5757aef4a20879547c16af0a4d1bf4c7Jean Chalard MakedictLog.i("Statistics:\n" 1055bfbbee8c5757aef4a20879547c16af0a4d1bf4c7Jean Chalard + " total file size " + size + "\n" 1056bfbbee8c5757aef4a20879547c16af0a4d1bf4c7Jean Chalard + " " + nodes.size() + " nodes\n" 1057bfbbee8c5757aef4a20879547c16af0a4d1bf4c7Jean Chalard + " " + charGroups + " groups (" + ((float)charGroups / nodes.size()) 1058bfbbee8c5757aef4a20879547c16af0a4d1bf4c7Jean Chalard + " groups per node)\n" 1059bfbbee8c5757aef4a20879547c16af0a4d1bf4c7Jean Chalard + " first terminal at " + firstTerminalAddress + "\n" 1060bfbbee8c5757aef4a20879547c16af0a4d1bf4c7Jean Chalard + " last terminal at " + lastTerminalAddress + "\n" 1061bfbbee8c5757aef4a20879547c16af0a4d1bf4c7Jean Chalard + " Group stats : max = " + maxGroups); 1062bfbbee8c5757aef4a20879547c16af0a4d1bf4c7Jean Chalard for (int i = 0; i < groupCounts.length; ++i) { 1063bfbbee8c5757aef4a20879547c16af0a4d1bf4c7Jean Chalard MakedictLog.i(" " + i + " : " + groupCounts[i]); 1064bfbbee8c5757aef4a20879547c16af0a4d1bf4c7Jean Chalard } 1065bfbbee8c5757aef4a20879547c16af0a4d1bf4c7Jean Chalard MakedictLog.i(" Character run stats : max = " + maxRuns); 1066bfbbee8c5757aef4a20879547c16af0a4d1bf4c7Jean Chalard for (int i = 0; i < runCounts.length; ++i) { 1067bfbbee8c5757aef4a20879547c16af0a4d1bf4c7Jean Chalard MakedictLog.i(" " + i + " : " + runCounts[i]); 1068bfbbee8c5757aef4a20879547c16af0a4d1bf4c7Jean Chalard } 1069bfbbee8c5757aef4a20879547c16af0a4d1bf4c7Jean Chalard } 1070bfbbee8c5757aef4a20879547c16af0a4d1bf4c7Jean Chalard 1071bfbbee8c5757aef4a20879547c16af0a4d1bf4c7Jean Chalard /** 1072bfbbee8c5757aef4a20879547c16af0a4d1bf4c7Jean Chalard * Dumps a FusionDictionary to a file. 1073bfbbee8c5757aef4a20879547c16af0a4d1bf4c7Jean Chalard * 1074bfbbee8c5757aef4a20879547c16af0a4d1bf4c7Jean Chalard * This is the public entry point to write a dictionary to a file. 1075bfbbee8c5757aef4a20879547c16af0a4d1bf4c7Jean Chalard * 1076bfbbee8c5757aef4a20879547c16af0a4d1bf4c7Jean Chalard * @param destination the stream to write the binary data to. 1077bfbbee8c5757aef4a20879547c16af0a4d1bf4c7Jean Chalard * @param dict the dictionary to write. 1078e55b644aefbefb4ac79308c9a59116e69a9c53a2Yuichiro Hanada * @param formatOptions file format options. 1079bfbbee8c5757aef4a20879547c16af0a4d1bf4c7Jean Chalard */ 108089dfb2bae171956dbcc880c9192ff29949388770Jean Chalard public static void writeDictionaryBinary(final OutputStream destination, 108183dfe0fd8c7e2bce2717930dbf8732f5414ee39dYuichiro Hanada final FusionDictionary dict, final FormatOptions formatOptions) 108289dfb2bae171956dbcc880c9192ff29949388770Jean Chalard throws IOException, UnsupportedFormatException { 1083bfbbee8c5757aef4a20879547c16af0a4d1bf4c7Jean Chalard 1084df7ebbbd616fa5aff569d00b16cd3f85ddf2da6dTom Ouyang // Addresses are limited to 3 bytes, but since addresses can be relative to each node, the 1085df7ebbbd616fa5aff569d00b16cd3f85ddf2da6dTom Ouyang // structure itself is not limited to 16MB. However, if it is over 16MB deciding the order 1086df7ebbbd616fa5aff569d00b16cd3f85ddf2da6dTom Ouyang // of the nodes becomes a quite complicated problem, because though the dictionary itself 1087df7ebbbd616fa5aff569d00b16cd3f85ddf2da6dTom Ouyang // does not have a size limit, each node must still be within 16MB of all its children and 1088df7ebbbd616fa5aff569d00b16cd3f85ddf2da6dTom Ouyang // parents. As long as this is ensured, the dictionary file may grow to any size. 1089bfbbee8c5757aef4a20879547c16af0a4d1bf4c7Jean Chalard 109083dfe0fd8c7e2bce2717930dbf8732f5414ee39dYuichiro Hanada final int version = formatOptions.mVersion; 109181d97eec0e77e72cce606f9c9f96091c0b348190Yuichiro Hanada if (version < FormatSpec.MINIMUM_SUPPORTED_VERSION 109281d97eec0e77e72cce606f9c9f96091c0b348190Yuichiro Hanada || version > FormatSpec.MAXIMUM_SUPPORTED_VERSION) { 1093e0f6cc181457613766fd480f499b3e83711cf989Jean Chalard throw new UnsupportedFormatException("Requested file format version " + version 1094e0f6cc181457613766fd480f499b3e83711cf989Jean Chalard + ", but this implementation only supports versions " 109581d97eec0e77e72cce606f9c9f96091c0b348190Yuichiro Hanada + FormatSpec.MINIMUM_SUPPORTED_VERSION + " through " 109681d97eec0e77e72cce606f9c9f96091c0b348190Yuichiro Hanada + FormatSpec.MAXIMUM_SUPPORTED_VERSION); 1097e0f6cc181457613766fd480f499b3e83711cf989Jean Chalard } 1098e0f6cc181457613766fd480f499b3e83711cf989Jean Chalard 1099df7ebbbd616fa5aff569d00b16cd3f85ddf2da6dTom Ouyang ByteArrayOutputStream headerBuffer = new ByteArrayOutputStream(256); 1100df7ebbbd616fa5aff569d00b16cd3f85ddf2da6dTom Ouyang 110189dfb2bae171956dbcc880c9192ff29949388770Jean Chalard // The magic number in big-endian order. 110281d97eec0e77e72cce606f9c9f96091c0b348190Yuichiro Hanada if (version >= FormatSpec.FIRST_VERSION_WITH_HEADER_SIZE) { 110389dfb2bae171956dbcc880c9192ff29949388770Jean Chalard // Magic number for version 2+. 110481d97eec0e77e72cce606f9c9f96091c0b348190Yuichiro Hanada headerBuffer.write((byte) (0xFF & (FormatSpec.VERSION_2_MAGIC_NUMBER >> 24))); 110581d97eec0e77e72cce606f9c9f96091c0b348190Yuichiro Hanada headerBuffer.write((byte) (0xFF & (FormatSpec.VERSION_2_MAGIC_NUMBER >> 16))); 110681d97eec0e77e72cce606f9c9f96091c0b348190Yuichiro Hanada headerBuffer.write((byte) (0xFF & (FormatSpec.VERSION_2_MAGIC_NUMBER >> 8))); 110781d97eec0e77e72cce606f9c9f96091c0b348190Yuichiro Hanada headerBuffer.write((byte) (0xFF & FormatSpec.VERSION_2_MAGIC_NUMBER)); 1108a9a5dd07b573718cc9a6bf99394b2db646e73d1aJean Chalard // Dictionary version. 1109df7ebbbd616fa5aff569d00b16cd3f85ddf2da6dTom Ouyang headerBuffer.write((byte) (0xFF & (version >> 8))); 1110df7ebbbd616fa5aff569d00b16cd3f85ddf2da6dTom Ouyang headerBuffer.write((byte) (0xFF & version)); 111189dfb2bae171956dbcc880c9192ff29949388770Jean Chalard } else { 111289dfb2bae171956dbcc880c9192ff29949388770Jean Chalard // Magic number for version 1. 111381d97eec0e77e72cce606f9c9f96091c0b348190Yuichiro Hanada headerBuffer.write((byte) (0xFF & (FormatSpec.VERSION_1_MAGIC_NUMBER >> 8))); 111481d97eec0e77e72cce606f9c9f96091c0b348190Yuichiro Hanada headerBuffer.write((byte) (0xFF & FormatSpec.VERSION_1_MAGIC_NUMBER)); 1115a9a5dd07b573718cc9a6bf99394b2db646e73d1aJean Chalard // Dictionary version. 1116df7ebbbd616fa5aff569d00b16cd3f85ddf2da6dTom Ouyang headerBuffer.write((byte) (0xFF & version)); 111789dfb2bae171956dbcc880c9192ff29949388770Jean Chalard } 1118bfbbee8c5757aef4a20879547c16af0a4d1bf4c7Jean Chalard // Options flags 1119e55b644aefbefb4ac79308c9a59116e69a9c53a2Yuichiro Hanada final int options = makeOptionsValue(dict, formatOptions); 1120df7ebbbd616fa5aff569d00b16cd3f85ddf2da6dTom Ouyang headerBuffer.write((byte) (0xFF & (options >> 8))); 1121df7ebbbd616fa5aff569d00b16cd3f85ddf2da6dTom Ouyang headerBuffer.write((byte) (0xFF & options)); 112281d97eec0e77e72cce606f9c9f96091c0b348190Yuichiro Hanada if (version >= FormatSpec.FIRST_VERSION_WITH_HEADER_SIZE) { 1123df7ebbbd616fa5aff569d00b16cd3f85ddf2da6dTom Ouyang final int headerSizeOffset = headerBuffer.size(); 1124df7ebbbd616fa5aff569d00b16cd3f85ddf2da6dTom Ouyang // Placeholder to be written later with header size. 1125df7ebbbd616fa5aff569d00b16cd3f85ddf2da6dTom Ouyang for (int i = 0; i < 4; ++i) { 1126df7ebbbd616fa5aff569d00b16cd3f85ddf2da6dTom Ouyang headerBuffer.write(0); 1127df7ebbbd616fa5aff569d00b16cd3f85ddf2da6dTom Ouyang } 1128c734c2aca1830643d169fd292e0c9d4d9306af5aJean Chalard // Write out the options. 1129c734c2aca1830643d169fd292e0c9d4d9306af5aJean Chalard for (final String key : dict.mOptions.mAttributes.keySet()) { 1130c734c2aca1830643d169fd292e0c9d4d9306af5aJean Chalard final String value = dict.mOptions.mAttributes.get(key); 1131df7ebbbd616fa5aff569d00b16cd3f85ddf2da6dTom Ouyang CharEncoding.writeString(headerBuffer, key); 1132df7ebbbd616fa5aff569d00b16cd3f85ddf2da6dTom Ouyang CharEncoding.writeString(headerBuffer, value); 1133c734c2aca1830643d169fd292e0c9d4d9306af5aJean Chalard } 1134df7ebbbd616fa5aff569d00b16cd3f85ddf2da6dTom Ouyang final int size = headerBuffer.size(); 1135df7ebbbd616fa5aff569d00b16cd3f85ddf2da6dTom Ouyang final byte[] bytes = headerBuffer.toByteArray(); 113689dfb2bae171956dbcc880c9192ff29949388770Jean Chalard // Write out the header size. 1137df7ebbbd616fa5aff569d00b16cd3f85ddf2da6dTom Ouyang bytes[headerSizeOffset] = (byte) (0xFF & (size >> 24)); 1138df7ebbbd616fa5aff569d00b16cd3f85ddf2da6dTom Ouyang bytes[headerSizeOffset + 1] = (byte) (0xFF & (size >> 16)); 1139df7ebbbd616fa5aff569d00b16cd3f85ddf2da6dTom Ouyang bytes[headerSizeOffset + 2] = (byte) (0xFF & (size >> 8)); 1140df7ebbbd616fa5aff569d00b16cd3f85ddf2da6dTom Ouyang bytes[headerSizeOffset + 3] = (byte) (0xFF & (size >> 0)); 1141df7ebbbd616fa5aff569d00b16cd3f85ddf2da6dTom Ouyang destination.write(bytes); 1142df7ebbbd616fa5aff569d00b16cd3f85ddf2da6dTom Ouyang } else { 1143df7ebbbd616fa5aff569d00b16cd3f85ddf2da6dTom Ouyang headerBuffer.writeTo(destination); 1144e0f6cc181457613766fd480f499b3e83711cf989Jean Chalard } 1145bfbbee8c5757aef4a20879547c16af0a4d1bf4c7Jean Chalard 1146df7ebbbd616fa5aff569d00b16cd3f85ddf2da6dTom Ouyang headerBuffer.close(); 1147bfbbee8c5757aef4a20879547c16af0a4d1bf4c7Jean Chalard 1148bfbbee8c5757aef4a20879547c16af0a4d1bf4c7Jean Chalard // Leave the choice of the optimal node order to the flattenTree function. 1149bfbbee8c5757aef4a20879547c16af0a4d1bf4c7Jean Chalard MakedictLog.i("Flattening the tree..."); 1150bfbbee8c5757aef4a20879547c16af0a4d1bf4c7Jean Chalard ArrayList<Node> flatNodes = flattenTree(dict.mRoot); 1151bfbbee8c5757aef4a20879547c16af0a4d1bf4c7Jean Chalard 1152bfbbee8c5757aef4a20879547c16af0a4d1bf4c7Jean Chalard MakedictLog.i("Computing addresses..."); 1153e55b644aefbefb4ac79308c9a59116e69a9c53a2Yuichiro Hanada computeAddresses(dict, flatNodes, formatOptions); 1154bfbbee8c5757aef4a20879547c16af0a4d1bf4c7Jean Chalard MakedictLog.i("Checking array..."); 11554df5b43df8f4b29fbfab9180cffe5742f8b5f512Jean Chalard if (DBG) checkFlatNodeArray(flatNodes); 1156bfbbee8c5757aef4a20879547c16af0a4d1bf4c7Jean Chalard 1157df7ebbbd616fa5aff569d00b16cd3f85ddf2da6dTom Ouyang // Create a buffer that matches the final dictionary size. 1158df7ebbbd616fa5aff569d00b16cd3f85ddf2da6dTom Ouyang final Node lastNode = flatNodes.get(flatNodes.size() - 1); 1159e55b644aefbefb4ac79308c9a59116e69a9c53a2Yuichiro Hanada final int bufferSize = lastNode.mCachedAddress + lastNode.mCachedSize; 1160df7ebbbd616fa5aff569d00b16cd3f85ddf2da6dTom Ouyang final byte[] buffer = new byte[bufferSize]; 1161df7ebbbd616fa5aff569d00b16cd3f85ddf2da6dTom Ouyang int index = 0; 1162df7ebbbd616fa5aff569d00b16cd3f85ddf2da6dTom Ouyang 1163bfbbee8c5757aef4a20879547c16af0a4d1bf4c7Jean Chalard MakedictLog.i("Writing file..."); 1164bfbbee8c5757aef4a20879547c16af0a4d1bf4c7Jean Chalard int dataEndOffset = 0; 1165bfbbee8c5757aef4a20879547c16af0a4d1bf4c7Jean Chalard for (Node n : flatNodes) { 1166e55b644aefbefb4ac79308c9a59116e69a9c53a2Yuichiro Hanada dataEndOffset = writePlacedNode(dict, buffer, n, formatOptions); 1167bfbbee8c5757aef4a20879547c16af0a4d1bf4c7Jean Chalard } 1168bfbbee8c5757aef4a20879547c16af0a4d1bf4c7Jean Chalard 116976319c6931becbe2994226a0e52925fc77bd0c92Jean Chalard if (DBG) showStatistics(flatNodes); 1170bfbbee8c5757aef4a20879547c16af0a4d1bf4c7Jean Chalard 1171bfbbee8c5757aef4a20879547c16af0a4d1bf4c7Jean Chalard destination.write(buffer, 0, dataEndOffset); 1172bfbbee8c5757aef4a20879547c16af0a4d1bf4c7Jean Chalard 1173bfbbee8c5757aef4a20879547c16af0a4d1bf4c7Jean Chalard destination.close(); 1174bfbbee8c5757aef4a20879547c16af0a4d1bf4c7Jean Chalard MakedictLog.i("Done"); 1175bfbbee8c5757aef4a20879547c16af0a4d1bf4c7Jean Chalard } 1176bfbbee8c5757aef4a20879547c16af0a4d1bf4c7Jean Chalard 1177bfbbee8c5757aef4a20879547c16af0a4d1bf4c7Jean Chalard 1178bfbbee8c5757aef4a20879547c16af0a4d1bf4c7Jean Chalard // Input methods: Read a binary dictionary to memory. 1179bfbbee8c5757aef4a20879547c16af0a4d1bf4c7Jean Chalard // readDictionaryBinary is the public entry point for them. 1180bfbbee8c5757aef4a20879547c16af0a4d1bf4c7Jean Chalard 11818ec0064c49e80945dbe1bb31129eb890478b7e06Yuichiro Hanada private static int getChildrenAddressSize(final int optionFlags, 11828ec0064c49e80945dbe1bb31129eb890478b7e06Yuichiro Hanada final FormatOptions formatOptions) { 11838ec0064c49e80945dbe1bb31129eb890478b7e06Yuichiro Hanada if (formatOptions.mSupportsDynamicUpdate) return FormatSpec.SIGNED_CHILDREN_ADDRESS_SIZE; 11848ec0064c49e80945dbe1bb31129eb890478b7e06Yuichiro Hanada switch (optionFlags & FormatSpec.MASK_GROUP_ADDRESS_TYPE) { 11858ec0064c49e80945dbe1bb31129eb890478b7e06Yuichiro Hanada case FormatSpec.FLAG_GROUP_ADDRESS_TYPE_ONEBYTE: 11868ec0064c49e80945dbe1bb31129eb890478b7e06Yuichiro Hanada return 1; 11878ec0064c49e80945dbe1bb31129eb890478b7e06Yuichiro Hanada case FormatSpec.FLAG_GROUP_ADDRESS_TYPE_TWOBYTES: 11888ec0064c49e80945dbe1bb31129eb890478b7e06Yuichiro Hanada return 2; 11898ec0064c49e80945dbe1bb31129eb890478b7e06Yuichiro Hanada case FormatSpec.FLAG_GROUP_ADDRESS_TYPE_THREEBYTES: 11908ec0064c49e80945dbe1bb31129eb890478b7e06Yuichiro Hanada return 3; 11918ec0064c49e80945dbe1bb31129eb890478b7e06Yuichiro Hanada case FormatSpec.FLAG_GROUP_ADDRESS_TYPE_NOADDRESS: 11928ec0064c49e80945dbe1bb31129eb890478b7e06Yuichiro Hanada default: 11938ec0064c49e80945dbe1bb31129eb890478b7e06Yuichiro Hanada return 0; 11948ec0064c49e80945dbe1bb31129eb890478b7e06Yuichiro Hanada } 11958ec0064c49e80945dbe1bb31129eb890478b7e06Yuichiro Hanada } 11968ec0064c49e80945dbe1bb31129eb890478b7e06Yuichiro Hanada 11978ec0064c49e80945dbe1bb31129eb890478b7e06Yuichiro Hanada private static int readChildrenAddress(final FusionDictionaryBufferInterface buffer, 11988ec0064c49e80945dbe1bb31129eb890478b7e06Yuichiro Hanada final int optionFlags, final FormatOptions options) { 11998ec0064c49e80945dbe1bb31129eb890478b7e06Yuichiro Hanada if (options.mSupportsDynamicUpdate) { 12008ec0064c49e80945dbe1bb31129eb890478b7e06Yuichiro Hanada final int address = buffer.readUnsignedInt24(); 12018ec0064c49e80945dbe1bb31129eb890478b7e06Yuichiro Hanada if (address == 0) return FormatSpec.NO_CHILDREN_ADDRESS; 12028ec0064c49e80945dbe1bb31129eb890478b7e06Yuichiro Hanada if ((address & MSB24) != 0) { 12038ec0064c49e80945dbe1bb31129eb890478b7e06Yuichiro Hanada return -(address & SINT24_MAX); 12048ec0064c49e80945dbe1bb31129eb890478b7e06Yuichiro Hanada } else { 12058ec0064c49e80945dbe1bb31129eb890478b7e06Yuichiro Hanada return address; 12068ec0064c49e80945dbe1bb31129eb890478b7e06Yuichiro Hanada } 12078ec0064c49e80945dbe1bb31129eb890478b7e06Yuichiro Hanada } 12088ec0064c49e80945dbe1bb31129eb890478b7e06Yuichiro Hanada int address; 12098ec0064c49e80945dbe1bb31129eb890478b7e06Yuichiro Hanada switch (optionFlags & FormatSpec.MASK_GROUP_ADDRESS_TYPE) { 12108ec0064c49e80945dbe1bb31129eb890478b7e06Yuichiro Hanada case FormatSpec.FLAG_GROUP_ADDRESS_TYPE_ONEBYTE: 12118ec0064c49e80945dbe1bb31129eb890478b7e06Yuichiro Hanada return buffer.readUnsignedByte(); 12128ec0064c49e80945dbe1bb31129eb890478b7e06Yuichiro Hanada case FormatSpec.FLAG_GROUP_ADDRESS_TYPE_TWOBYTES: 12138ec0064c49e80945dbe1bb31129eb890478b7e06Yuichiro Hanada return buffer.readUnsignedShort(); 12148ec0064c49e80945dbe1bb31129eb890478b7e06Yuichiro Hanada case FormatSpec.FLAG_GROUP_ADDRESS_TYPE_THREEBYTES: 12158ec0064c49e80945dbe1bb31129eb890478b7e06Yuichiro Hanada return buffer.readUnsignedInt24(); 12168ec0064c49e80945dbe1bb31129eb890478b7e06Yuichiro Hanada case FormatSpec.FLAG_GROUP_ADDRESS_TYPE_NOADDRESS: 12178ec0064c49e80945dbe1bb31129eb890478b7e06Yuichiro Hanada default: 12188ec0064c49e80945dbe1bb31129eb890478b7e06Yuichiro Hanada return FormatSpec.NO_CHILDREN_ADDRESS; 12198ec0064c49e80945dbe1bb31129eb890478b7e06Yuichiro Hanada } 12208ec0064c49e80945dbe1bb31129eb890478b7e06Yuichiro Hanada } 12218ec0064c49e80945dbe1bb31129eb890478b7e06Yuichiro Hanada 12228ec0064c49e80945dbe1bb31129eb890478b7e06Yuichiro Hanada private static int readParentAddress(final FusionDictionaryBufferInterface buffer, 12238ec0064c49e80945dbe1bb31129eb890478b7e06Yuichiro Hanada final FormatOptions formatOptions) { 12248ec0064c49e80945dbe1bb31129eb890478b7e06Yuichiro Hanada if (supportsDynamicUpdate(formatOptions)) { 12258ec0064c49e80945dbe1bb31129eb890478b7e06Yuichiro Hanada final int parentAddress = buffer.readUnsignedInt24(); 12268ec0064c49e80945dbe1bb31129eb890478b7e06Yuichiro Hanada final int sign = ((parentAddress & MSB24) != 0) ? -1 : 1; 12278ec0064c49e80945dbe1bb31129eb890478b7e06Yuichiro Hanada return sign * (parentAddress & SINT24_MAX); 12288ec0064c49e80945dbe1bb31129eb890478b7e06Yuichiro Hanada } else { 12298ec0064c49e80945dbe1bb31129eb890478b7e06Yuichiro Hanada return FormatSpec.NO_PARENT_ADDRESS; 12308ec0064c49e80945dbe1bb31129eb890478b7e06Yuichiro Hanada } 12318ec0064c49e80945dbe1bb31129eb890478b7e06Yuichiro Hanada } 12328ec0064c49e80945dbe1bb31129eb890478b7e06Yuichiro Hanada 123381d97eec0e77e72cce606f9c9f96091c0b348190Yuichiro Hanada private static final int[] CHARACTER_BUFFER = new int[FormatSpec.MAX_WORD_LENGTH]; 123465feee12e5889601e375d92dfdf5f8e8fbb05092Yuichiro Hanada public static CharGroupInfo readCharGroup(final FusionDictionaryBufferInterface buffer, 1235e55b644aefbefb4ac79308c9a59116e69a9c53a2Yuichiro Hanada final int originalGroupAddress, final FormatOptions options) { 1236bfbbee8c5757aef4a20879547c16af0a4d1bf4c7Jean Chalard int addressPointer = originalGroupAddress; 1237f5c4ff481782831329593760b000f0543680930aYuichiro Hanada final int flags = buffer.readUnsignedByte(); 1238bfbbee8c5757aef4a20879547c16af0a4d1bf4c7Jean Chalard ++addressPointer; 1239e55b644aefbefb4ac79308c9a59116e69a9c53a2Yuichiro Hanada 12408ec0064c49e80945dbe1bb31129eb890478b7e06Yuichiro Hanada final int parentAddress = readParentAddress(buffer, options); 124182d9deaaf252cd20f8918adbc7a4b9b8f2647c38Yuichiro Hanada if (supportsDynamicUpdate(options)) { 1242e55b644aefbefb4ac79308c9a59116e69a9c53a2Yuichiro Hanada addressPointer += 3; 1243e55b644aefbefb4ac79308c9a59116e69a9c53a2Yuichiro Hanada } 1244e55b644aefbefb4ac79308c9a59116e69a9c53a2Yuichiro Hanada 1245bfbbee8c5757aef4a20879547c16af0a4d1bf4c7Jean Chalard final int characters[]; 124681d97eec0e77e72cce606f9c9f96091c0b348190Yuichiro Hanada if (0 != (flags & FormatSpec.FLAG_HAS_MULTIPLE_CHARS)) { 1247bfbbee8c5757aef4a20879547c16af0a4d1bf4c7Jean Chalard int index = 0; 1248d4fe7fda303ff937d2e44c15dde9d90cbf59376bYuichiro Hanada int character = CharEncoding.readChar(buffer); 1249bfbbee8c5757aef4a20879547c16af0a4d1bf4c7Jean Chalard addressPointer += CharEncoding.getCharSize(character); 1250bfbbee8c5757aef4a20879547c16af0a4d1bf4c7Jean Chalard while (-1 != character) { 1251e55b644aefbefb4ac79308c9a59116e69a9c53a2Yuichiro Hanada // FusionDictionary is making sure that the length of the word is smaller than 1252e55b644aefbefb4ac79308c9a59116e69a9c53a2Yuichiro Hanada // MAX_WORD_LENGTH. 1253e55b644aefbefb4ac79308c9a59116e69a9c53a2Yuichiro Hanada // So we'll never write past the end of CHARACTER_BUFFER. 1254e55b644aefbefb4ac79308c9a59116e69a9c53a2Yuichiro Hanada CHARACTER_BUFFER[index++] = character; 1255d4fe7fda303ff937d2e44c15dde9d90cbf59376bYuichiro Hanada character = CharEncoding.readChar(buffer); 1256bfbbee8c5757aef4a20879547c16af0a4d1bf4c7Jean Chalard addressPointer += CharEncoding.getCharSize(character); 1257bfbbee8c5757aef4a20879547c16af0a4d1bf4c7Jean Chalard } 1258e55b644aefbefb4ac79308c9a59116e69a9c53a2Yuichiro Hanada characters = Arrays.copyOfRange(CHARACTER_BUFFER, 0, index); 1259bfbbee8c5757aef4a20879547c16af0a4d1bf4c7Jean Chalard } else { 1260d4fe7fda303ff937d2e44c15dde9d90cbf59376bYuichiro Hanada final int character = CharEncoding.readChar(buffer); 1261bfbbee8c5757aef4a20879547c16af0a4d1bf4c7Jean Chalard addressPointer += CharEncoding.getCharSize(character); 1262bfbbee8c5757aef4a20879547c16af0a4d1bf4c7Jean Chalard characters = new int[] { character }; 1263bfbbee8c5757aef4a20879547c16af0a4d1bf4c7Jean Chalard } 1264bfbbee8c5757aef4a20879547c16af0a4d1bf4c7Jean Chalard final int frequency; 126581d97eec0e77e72cce606f9c9f96091c0b348190Yuichiro Hanada if (0 != (FormatSpec.FLAG_IS_TERMINAL & flags)) { 1266bfbbee8c5757aef4a20879547c16af0a4d1bf4c7Jean Chalard ++addressPointer; 1267f5c4ff481782831329593760b000f0543680930aYuichiro Hanada frequency = buffer.readUnsignedByte(); 1268bfbbee8c5757aef4a20879547c16af0a4d1bf4c7Jean Chalard } else { 1269bfbbee8c5757aef4a20879547c16af0a4d1bf4c7Jean Chalard frequency = CharGroup.NOT_A_TERMINAL; 1270bfbbee8c5757aef4a20879547c16af0a4d1bf4c7Jean Chalard } 12718ec0064c49e80945dbe1bb31129eb890478b7e06Yuichiro Hanada int childrenAddress = readChildrenAddress(buffer, flags, options); 12728ec0064c49e80945dbe1bb31129eb890478b7e06Yuichiro Hanada if (childrenAddress != FormatSpec.NO_CHILDREN_ADDRESS) { 12738ec0064c49e80945dbe1bb31129eb890478b7e06Yuichiro Hanada childrenAddress += addressPointer; 1274bfbbee8c5757aef4a20879547c16af0a4d1bf4c7Jean Chalard } 12758ec0064c49e80945dbe1bb31129eb890478b7e06Yuichiro Hanada addressPointer += getChildrenAddressSize(flags, options); 1276752996540ff3a6dd5b48819849c06355c4270e03Jean Chalard ArrayList<WeightedString> shortcutTargets = null; 127781d97eec0e77e72cce606f9c9f96091c0b348190Yuichiro Hanada if (0 != (flags & FormatSpec.FLAG_HAS_SHORTCUT_TARGETS)) { 1278d4fe7fda303ff937d2e44c15dde9d90cbf59376bYuichiro Hanada final int pointerBefore = buffer.position(); 1279752996540ff3a6dd5b48819849c06355c4270e03Jean Chalard shortcutTargets = new ArrayList<WeightedString>(); 1280f5c4ff481782831329593760b000f0543680930aYuichiro Hanada buffer.readUnsignedShort(); // Skip the size 12818edd3067181a425ce1383bb950184f9742af7557Jean Chalard while (true) { 1282f5c4ff481782831329593760b000f0543680930aYuichiro Hanada final int targetFlags = buffer.readUnsignedByte(); 1283d4fe7fda303ff937d2e44c15dde9d90cbf59376bYuichiro Hanada final String word = CharEncoding.readString(buffer); 1284752996540ff3a6dd5b48819849c06355c4270e03Jean Chalard shortcutTargets.add(new WeightedString(word, 128581d97eec0e77e72cce606f9c9f96091c0b348190Yuichiro Hanada targetFlags & FormatSpec.FLAG_ATTRIBUTE_FREQUENCY)); 128681d97eec0e77e72cce606f9c9f96091c0b348190Yuichiro Hanada if (0 == (targetFlags & FormatSpec.FLAG_ATTRIBUTE_HAS_NEXT)) break; 12878edd3067181a425ce1383bb950184f9742af7557Jean Chalard } 1288d4fe7fda303ff937d2e44c15dde9d90cbf59376bYuichiro Hanada addressPointer += buffer.position() - pointerBefore; 12898edd3067181a425ce1383bb950184f9742af7557Jean Chalard } 1290bfbbee8c5757aef4a20879547c16af0a4d1bf4c7Jean Chalard ArrayList<PendingAttribute> bigrams = null; 129181d97eec0e77e72cce606f9c9f96091c0b348190Yuichiro Hanada if (0 != (flags & FormatSpec.FLAG_HAS_BIGRAMS)) { 1292bfbbee8c5757aef4a20879547c16af0a4d1bf4c7Jean Chalard bigrams = new ArrayList<PendingAttribute>(); 12938edd3067181a425ce1383bb950184f9742af7557Jean Chalard while (true) { 1294f5c4ff481782831329593760b000f0543680930aYuichiro Hanada final int bigramFlags = buffer.readUnsignedByte(); 1295bfbbee8c5757aef4a20879547c16af0a4d1bf4c7Jean Chalard ++addressPointer; 129681d97eec0e77e72cce606f9c9f96091c0b348190Yuichiro Hanada final int sign = 0 == (bigramFlags & FormatSpec.FLAG_ATTRIBUTE_OFFSET_NEGATIVE) 129781d97eec0e77e72cce606f9c9f96091c0b348190Yuichiro Hanada ? 1 : -1; 1298bfbbee8c5757aef4a20879547c16af0a4d1bf4c7Jean Chalard int bigramAddress = addressPointer; 129981d97eec0e77e72cce606f9c9f96091c0b348190Yuichiro Hanada switch (bigramFlags & FormatSpec.MASK_ATTRIBUTE_ADDRESS_TYPE) { 130081d97eec0e77e72cce606f9c9f96091c0b348190Yuichiro Hanada case FormatSpec.FLAG_ATTRIBUTE_ADDRESS_TYPE_ONEBYTE: 1301f5c4ff481782831329593760b000f0543680930aYuichiro Hanada bigramAddress += sign * buffer.readUnsignedByte(); 1302bfbbee8c5757aef4a20879547c16af0a4d1bf4c7Jean Chalard addressPointer += 1; 1303bfbbee8c5757aef4a20879547c16af0a4d1bf4c7Jean Chalard break; 130481d97eec0e77e72cce606f9c9f96091c0b348190Yuichiro Hanada case FormatSpec.FLAG_ATTRIBUTE_ADDRESS_TYPE_TWOBYTES: 1305f5c4ff481782831329593760b000f0543680930aYuichiro Hanada bigramAddress += sign * buffer.readUnsignedShort(); 1306bfbbee8c5757aef4a20879547c16af0a4d1bf4c7Jean Chalard addressPointer += 2; 1307bfbbee8c5757aef4a20879547c16af0a4d1bf4c7Jean Chalard break; 130881d97eec0e77e72cce606f9c9f96091c0b348190Yuichiro Hanada case FormatSpec.FLAG_ATTRIBUTE_ADDRESS_TYPE_THREEBYTES: 1309f5c4ff481782831329593760b000f0543680930aYuichiro Hanada final int offset = (buffer.readUnsignedByte() << 16) 1310f5c4ff481782831329593760b000f0543680930aYuichiro Hanada + buffer.readUnsignedShort(); 1311bfbbee8c5757aef4a20879547c16af0a4d1bf4c7Jean Chalard bigramAddress += sign * offset; 1312bfbbee8c5757aef4a20879547c16af0a4d1bf4c7Jean Chalard addressPointer += 3; 1313bfbbee8c5757aef4a20879547c16af0a4d1bf4c7Jean Chalard break; 1314bfbbee8c5757aef4a20879547c16af0a4d1bf4c7Jean Chalard default: 13159b8b2c757d9a31481aa775809724bf53277721d7Jean Chalard throw new RuntimeException("Has bigrams with no address"); 1316bfbbee8c5757aef4a20879547c16af0a4d1bf4c7Jean Chalard } 131781d97eec0e77e72cce606f9c9f96091c0b348190Yuichiro Hanada bigrams.add(new PendingAttribute(bigramFlags & FormatSpec.FLAG_ATTRIBUTE_FREQUENCY, 1318bfbbee8c5757aef4a20879547c16af0a4d1bf4c7Jean Chalard bigramAddress)); 131981d97eec0e77e72cce606f9c9f96091c0b348190Yuichiro Hanada if (0 == (bigramFlags & FormatSpec.FLAG_ATTRIBUTE_HAS_NEXT)) break; 1320bfbbee8c5757aef4a20879547c16af0a4d1bf4c7Jean Chalard } 1321bfbbee8c5757aef4a20879547c16af0a4d1bf4c7Jean Chalard } 1322bfbbee8c5757aef4a20879547c16af0a4d1bf4c7Jean Chalard return new CharGroupInfo(originalGroupAddress, addressPointer, flags, characters, frequency, 1323e55b644aefbefb4ac79308c9a59116e69a9c53a2Yuichiro Hanada parentAddress, childrenAddress, shortcutTargets, bigrams); 1324bfbbee8c5757aef4a20879547c16af0a4d1bf4c7Jean Chalard } 1325bfbbee8c5757aef4a20879547c16af0a4d1bf4c7Jean Chalard 1326bfbbee8c5757aef4a20879547c16af0a4d1bf4c7Jean Chalard /** 1327d4fe7fda303ff937d2e44c15dde9d90cbf59376bYuichiro Hanada * Reads and returns the char group count out of a buffer and forwards the pointer. 13289b8b2c757d9a31481aa775809724bf53277721d7Jean Chalard */ 132965feee12e5889601e375d92dfdf5f8e8fbb05092Yuichiro Hanada public static int readCharGroupCount(final FusionDictionaryBufferInterface buffer) { 1330f5c4ff481782831329593760b000f0543680930aYuichiro Hanada final int msb = buffer.readUnsignedByte(); 133181d97eec0e77e72cce606f9c9f96091c0b348190Yuichiro Hanada if (FormatSpec.MAX_CHARGROUPS_FOR_ONE_BYTE_CHARGROUP_COUNT >= msb) { 13329b8b2c757d9a31481aa775809724bf53277721d7Jean Chalard return msb; 13339b8b2c757d9a31481aa775809724bf53277721d7Jean Chalard } else { 133481d97eec0e77e72cce606f9c9f96091c0b348190Yuichiro Hanada return ((FormatSpec.MAX_CHARGROUPS_FOR_ONE_BYTE_CHARGROUP_COUNT & msb) << 8) 1335f5c4ff481782831329593760b000f0543680930aYuichiro Hanada + buffer.readUnsignedByte(); 13369b8b2c757d9a31481aa775809724bf53277721d7Jean Chalard } 13379b8b2c757d9a31481aa775809724bf53277721d7Jean Chalard } 13389b8b2c757d9a31481aa775809724bf53277721d7Jean Chalard 13391d80a7f395290cd0e7344210bb3960f685059264Jean Chalard // The word cache here is a stopgap bandaid to help the catastrophic performance 13401d80a7f395290cd0e7344210bb3960f685059264Jean Chalard // of this method. Since it performs direct, unbuffered random access to the file and 13411d80a7f395290cd0e7344210bb3960f685059264Jean Chalard // may be called hundreds of thousands of times, the resulting performance is not 13421d80a7f395290cd0e7344210bb3960f685059264Jean Chalard // reasonable without some kind of cache. Thus: 13431d80a7f395290cd0e7344210bb3960f685059264Jean Chalard private static TreeMap<Integer, String> wordCache = new TreeMap<Integer, String>(); 13449b8b2c757d9a31481aa775809724bf53277721d7Jean Chalard /** 1345bfbbee8c5757aef4a20879547c16af0a4d1bf4c7Jean Chalard * Finds, as a string, the word at the address passed as an argument. 1346bfbbee8c5757aef4a20879547c16af0a4d1bf4c7Jean Chalard * 1347d4fe7fda303ff937d2e44c15dde9d90cbf59376bYuichiro Hanada * @param buffer the buffer to read from. 1348bfbbee8c5757aef4a20879547c16af0a4d1bf4c7Jean Chalard * @param headerSize the size of the header. 1349bfbbee8c5757aef4a20879547c16af0a4d1bf4c7Jean Chalard * @param address the address to seek. 1350e55b644aefbefb4ac79308c9a59116e69a9c53a2Yuichiro Hanada * @param formatOptions file format options. 1351bfbbee8c5757aef4a20879547c16af0a4d1bf4c7Jean Chalard * @return the word, as a string. 1352bfbbee8c5757aef4a20879547c16af0a4d1bf4c7Jean Chalard */ 1353d36245fad292ea660ca49f38a3ec36e07727dda5Yuichiro Hanada /* packages for tests */ static String getWordAtAddress( 1354d36245fad292ea660ca49f38a3ec36e07727dda5Yuichiro Hanada final FusionDictionaryBufferInterface buffer, final int headerSize, final int address, 1355d36245fad292ea660ca49f38a3ec36e07727dda5Yuichiro Hanada final FormatOptions formatOptions) { 13561d80a7f395290cd0e7344210bb3960f685059264Jean Chalard final String cachedString = wordCache.get(address); 13571d80a7f395290cd0e7344210bb3960f685059264Jean Chalard if (null != cachedString) return cachedString; 1358e55b644aefbefb4ac79308c9a59116e69a9c53a2Yuichiro Hanada 1359e55b644aefbefb4ac79308c9a59116e69a9c53a2Yuichiro Hanada final String result; 1360d4fe7fda303ff937d2e44c15dde9d90cbf59376bYuichiro Hanada final int originalPointer = buffer.position(); 13618ec0064c49e80945dbe1bb31129eb890478b7e06Yuichiro Hanada buffer.position(address); 1362e55b644aefbefb4ac79308c9a59116e69a9c53a2Yuichiro Hanada 136382d9deaaf252cd20f8918adbc7a4b9b8f2647c38Yuichiro Hanada if (supportsDynamicUpdate(formatOptions)) { 1364e55b644aefbefb4ac79308c9a59116e69a9c53a2Yuichiro Hanada result = getWordAtAddressWithParentAddress(buffer, headerSize, address, formatOptions); 1365e55b644aefbefb4ac79308c9a59116e69a9c53a2Yuichiro Hanada } else { 1366e55b644aefbefb4ac79308c9a59116e69a9c53a2Yuichiro Hanada result = getWordAtAddressWithoutParentAddress(buffer, headerSize, address, 1367e55b644aefbefb4ac79308c9a59116e69a9c53a2Yuichiro Hanada formatOptions); 1368e55b644aefbefb4ac79308c9a59116e69a9c53a2Yuichiro Hanada } 1369e55b644aefbefb4ac79308c9a59116e69a9c53a2Yuichiro Hanada 1370e55b644aefbefb4ac79308c9a59116e69a9c53a2Yuichiro Hanada wordCache.put(address, result); 1371e55b644aefbefb4ac79308c9a59116e69a9c53a2Yuichiro Hanada buffer.position(originalPointer); 1372e55b644aefbefb4ac79308c9a59116e69a9c53a2Yuichiro Hanada return result; 1373e55b644aefbefb4ac79308c9a59116e69a9c53a2Yuichiro Hanada } 1374e55b644aefbefb4ac79308c9a59116e69a9c53a2Yuichiro Hanada 137581d97eec0e77e72cce606f9c9f96091c0b348190Yuichiro Hanada private static int[] sGetWordBuffer = new int[FormatSpec.MAX_WORD_LENGTH]; 1376e55b644aefbefb4ac79308c9a59116e69a9c53a2Yuichiro Hanada private static String getWordAtAddressWithParentAddress( 1377e55b644aefbefb4ac79308c9a59116e69a9c53a2Yuichiro Hanada final FusionDictionaryBufferInterface buffer, final int headerSize, final int address, 1378e55b644aefbefb4ac79308c9a59116e69a9c53a2Yuichiro Hanada final FormatOptions options) { 1379e55b644aefbefb4ac79308c9a59116e69a9c53a2Yuichiro Hanada final StringBuilder builder = new StringBuilder(); 1380e55b644aefbefb4ac79308c9a59116e69a9c53a2Yuichiro Hanada 1381e55b644aefbefb4ac79308c9a59116e69a9c53a2Yuichiro Hanada int currentAddress = address; 138281d97eec0e77e72cce606f9c9f96091c0b348190Yuichiro Hanada int index = FormatSpec.MAX_WORD_LENGTH - 1; 1383e55b644aefbefb4ac79308c9a59116e69a9c53a2Yuichiro Hanada // the length of the path from the root to the leaf is limited by MAX_WORD_LENGTH 138481d97eec0e77e72cce606f9c9f96091c0b348190Yuichiro Hanada for (int count = 0; count < FormatSpec.MAX_WORD_LENGTH; ++count) { 13852ee70804e92b17016a2f042c4f6b0e94b5d23e88Yuichiro Hanada CharGroupInfo currentInfo; 13862ee70804e92b17016a2f042c4f6b0e94b5d23e88Yuichiro Hanada int loopCounter = 0; 13872ee70804e92b17016a2f042c4f6b0e94b5d23e88Yuichiro Hanada do { 13882ee70804e92b17016a2f042c4f6b0e94b5d23e88Yuichiro Hanada buffer.position(currentAddress + headerSize); 13892ee70804e92b17016a2f042c4f6b0e94b5d23e88Yuichiro Hanada currentInfo = readCharGroup(buffer, currentAddress, options); 13902ee70804e92b17016a2f042c4f6b0e94b5d23e88Yuichiro Hanada if (isMovedGroup(currentInfo.mFlags, options)) { 13912ee70804e92b17016a2f042c4f6b0e94b5d23e88Yuichiro Hanada currentAddress = currentInfo.mParentAddress + currentInfo.mOriginalAddress; 13922ee70804e92b17016a2f042c4f6b0e94b5d23e88Yuichiro Hanada } 13932ee70804e92b17016a2f042c4f6b0e94b5d23e88Yuichiro Hanada if (DBG && loopCounter++ > MAX_JUMPS) { 13942ee70804e92b17016a2f042c4f6b0e94b5d23e88Yuichiro Hanada MakedictLog.d("Too many jumps - probably a bug"); 13952ee70804e92b17016a2f042c4f6b0e94b5d23e88Yuichiro Hanada } 13962ee70804e92b17016a2f042c4f6b0e94b5d23e88Yuichiro Hanada } while (isMovedGroup(currentInfo.mFlags, options)); 1397e55b644aefbefb4ac79308c9a59116e69a9c53a2Yuichiro Hanada for (int i = 0; i < currentInfo.mCharacters.length; ++i) { 1398e55b644aefbefb4ac79308c9a59116e69a9c53a2Yuichiro Hanada sGetWordBuffer[index--] = 1399e55b644aefbefb4ac79308c9a59116e69a9c53a2Yuichiro Hanada currentInfo.mCharacters[currentInfo.mCharacters.length - i - 1]; 1400e55b644aefbefb4ac79308c9a59116e69a9c53a2Yuichiro Hanada } 140181d97eec0e77e72cce606f9c9f96091c0b348190Yuichiro Hanada if (currentInfo.mParentAddress == FormatSpec.NO_PARENT_ADDRESS) break; 1402e55b644aefbefb4ac79308c9a59116e69a9c53a2Yuichiro Hanada currentAddress = currentInfo.mParentAddress + currentInfo.mOriginalAddress; 1403e55b644aefbefb4ac79308c9a59116e69a9c53a2Yuichiro Hanada } 1404e55b644aefbefb4ac79308c9a59116e69a9c53a2Yuichiro Hanada 140581d97eec0e77e72cce606f9c9f96091c0b348190Yuichiro Hanada return new String(sGetWordBuffer, index + 1, FormatSpec.MAX_WORD_LENGTH - index - 1); 1406e55b644aefbefb4ac79308c9a59116e69a9c53a2Yuichiro Hanada } 1407e55b644aefbefb4ac79308c9a59116e69a9c53a2Yuichiro Hanada 1408e55b644aefbefb4ac79308c9a59116e69a9c53a2Yuichiro Hanada private static String getWordAtAddressWithoutParentAddress( 1409e55b644aefbefb4ac79308c9a59116e69a9c53a2Yuichiro Hanada final FusionDictionaryBufferInterface buffer, final int headerSize, final int address, 1410e55b644aefbefb4ac79308c9a59116e69a9c53a2Yuichiro Hanada final FormatOptions options) { 1411d4fe7fda303ff937d2e44c15dde9d90cbf59376bYuichiro Hanada buffer.position(headerSize); 1412d4fe7fda303ff937d2e44c15dde9d90cbf59376bYuichiro Hanada final int count = readCharGroupCount(buffer); 14139b8b2c757d9a31481aa775809724bf53277721d7Jean Chalard int groupOffset = getGroupCountSize(count); 1414bfbbee8c5757aef4a20879547c16af0a4d1bf4c7Jean Chalard final StringBuilder builder = new StringBuilder(); 1415bfbbee8c5757aef4a20879547c16af0a4d1bf4c7Jean Chalard String result = null; 1416bfbbee8c5757aef4a20879547c16af0a4d1bf4c7Jean Chalard 1417bfbbee8c5757aef4a20879547c16af0a4d1bf4c7Jean Chalard CharGroupInfo last = null; 1418bfbbee8c5757aef4a20879547c16af0a4d1bf4c7Jean Chalard for (int i = count - 1; i >= 0; --i) { 1419e55b644aefbefb4ac79308c9a59116e69a9c53a2Yuichiro Hanada CharGroupInfo info = readCharGroup(buffer, groupOffset, options); 1420bfbbee8c5757aef4a20879547c16af0a4d1bf4c7Jean Chalard groupOffset = info.mEndAddress; 1421bfbbee8c5757aef4a20879547c16af0a4d1bf4c7Jean Chalard if (info.mOriginalAddress == address) { 1422bfbbee8c5757aef4a20879547c16af0a4d1bf4c7Jean Chalard builder.append(new String(info.mCharacters, 0, info.mCharacters.length)); 1423bfbbee8c5757aef4a20879547c16af0a4d1bf4c7Jean Chalard result = builder.toString(); 1424bfbbee8c5757aef4a20879547c16af0a4d1bf4c7Jean Chalard break; // and return 1425bfbbee8c5757aef4a20879547c16af0a4d1bf4c7Jean Chalard } 1426bfbbee8c5757aef4a20879547c16af0a4d1bf4c7Jean Chalard if (hasChildrenAddress(info.mChildrenAddress)) { 1427bfbbee8c5757aef4a20879547c16af0a4d1bf4c7Jean Chalard if (info.mChildrenAddress > address) { 1428bfbbee8c5757aef4a20879547c16af0a4d1bf4c7Jean Chalard if (null == last) continue; 1429bfbbee8c5757aef4a20879547c16af0a4d1bf4c7Jean Chalard builder.append(new String(last.mCharacters, 0, last.mCharacters.length)); 1430d4fe7fda303ff937d2e44c15dde9d90cbf59376bYuichiro Hanada buffer.position(last.mChildrenAddress + headerSize); 1431bfbbee8c5757aef4a20879547c16af0a4d1bf4c7Jean Chalard groupOffset = last.mChildrenAddress + 1; 1432f5c4ff481782831329593760b000f0543680930aYuichiro Hanada i = buffer.readUnsignedByte(); 1433bfbbee8c5757aef4a20879547c16af0a4d1bf4c7Jean Chalard last = null; 1434bfbbee8c5757aef4a20879547c16af0a4d1bf4c7Jean Chalard continue; 1435bfbbee8c5757aef4a20879547c16af0a4d1bf4c7Jean Chalard } 1436bfbbee8c5757aef4a20879547c16af0a4d1bf4c7Jean Chalard last = info; 1437bfbbee8c5757aef4a20879547c16af0a4d1bf4c7Jean Chalard } 1438bfbbee8c5757aef4a20879547c16af0a4d1bf4c7Jean Chalard if (0 == i && hasChildrenAddress(last.mChildrenAddress)) { 1439bfbbee8c5757aef4a20879547c16af0a4d1bf4c7Jean Chalard builder.append(new String(last.mCharacters, 0, last.mCharacters.length)); 1440d4fe7fda303ff937d2e44c15dde9d90cbf59376bYuichiro Hanada buffer.position(last.mChildrenAddress + headerSize); 1441bfbbee8c5757aef4a20879547c16af0a4d1bf4c7Jean Chalard groupOffset = last.mChildrenAddress + 1; 1442f5c4ff481782831329593760b000f0543680930aYuichiro Hanada i = buffer.readUnsignedByte(); 1443bfbbee8c5757aef4a20879547c16af0a4d1bf4c7Jean Chalard last = null; 1444bfbbee8c5757aef4a20879547c16af0a4d1bf4c7Jean Chalard continue; 1445bfbbee8c5757aef4a20879547c16af0a4d1bf4c7Jean Chalard } 1446bfbbee8c5757aef4a20879547c16af0a4d1bf4c7Jean Chalard } 1447bfbbee8c5757aef4a20879547c16af0a4d1bf4c7Jean Chalard return result; 1448bfbbee8c5757aef4a20879547c16af0a4d1bf4c7Jean Chalard } 1449bfbbee8c5757aef4a20879547c16af0a4d1bf4c7Jean Chalard 1450bfbbee8c5757aef4a20879547c16af0a4d1bf4c7Jean Chalard /** 1451f5c4ff481782831329593760b000f0543680930aYuichiro Hanada * Reads a single node from a buffer. 1452bfbbee8c5757aef4a20879547c16af0a4d1bf4c7Jean Chalard * 1453f5c4ff481782831329593760b000f0543680930aYuichiro Hanada * This methods reads the file at the current position. A node is fully expected to start at 1454f5c4ff481782831329593760b000f0543680930aYuichiro Hanada * the current position. 1455bfbbee8c5757aef4a20879547c16af0a4d1bf4c7Jean Chalard * This will recursively read other nodes into the structure, populating the reverse 1456bfbbee8c5757aef4a20879547c16af0a4d1bf4c7Jean Chalard * maps on the fly and using them to keep track of already read nodes. 1457bfbbee8c5757aef4a20879547c16af0a4d1bf4c7Jean Chalard * 1458d4fe7fda303ff937d2e44c15dde9d90cbf59376bYuichiro Hanada * @param buffer the buffer, correctly positioned at the start of a node. 1459bfbbee8c5757aef4a20879547c16af0a4d1bf4c7Jean Chalard * @param headerSize the size, in bytes, of the file header. 1460bfbbee8c5757aef4a20879547c16af0a4d1bf4c7Jean Chalard * @param reverseNodeMap a mapping from addresses to already read nodes. 1461bfbbee8c5757aef4a20879547c16af0a4d1bf4c7Jean Chalard * @param reverseGroupMap a mapping from addresses to already read character groups. 1462e55b644aefbefb4ac79308c9a59116e69a9c53a2Yuichiro Hanada * @param options file format options. 1463bfbbee8c5757aef4a20879547c16af0a4d1bf4c7Jean Chalard * @return the read node with all his children already read. 1464bfbbee8c5757aef4a20879547c16af0a4d1bf4c7Jean Chalard */ 1465f5c4ff481782831329593760b000f0543680930aYuichiro Hanada private static Node readNode(final FusionDictionaryBufferInterface buffer, final int headerSize, 1466e55b644aefbefb4ac79308c9a59116e69a9c53a2Yuichiro Hanada final Map<Integer, Node> reverseNodeMap, final Map<Integer, CharGroup> reverseGroupMap, 1467e55b644aefbefb4ac79308c9a59116e69a9c53a2Yuichiro Hanada final FormatOptions options) 1468bfbbee8c5757aef4a20879547c16af0a4d1bf4c7Jean Chalard throws IOException { 1469bfbbee8c5757aef4a20879547c16af0a4d1bf4c7Jean Chalard final ArrayList<CharGroup> nodeContents = new ArrayList<CharGroup>(); 1470c2fdf0dfbf1c43f7ed8fdf3d91576bbf71146ef3Yuichiro Hanada final int nodeOrigin = buffer.position() - headerSize; 1471c2fdf0dfbf1c43f7ed8fdf3d91576bbf71146ef3Yuichiro Hanada 1472c2fdf0dfbf1c43f7ed8fdf3d91576bbf71146ef3Yuichiro Hanada do { // Scan the linked-list node. 1473c2fdf0dfbf1c43f7ed8fdf3d91576bbf71146ef3Yuichiro Hanada final int nodeHeadPosition = buffer.position() - headerSize; 1474c2fdf0dfbf1c43f7ed8fdf3d91576bbf71146ef3Yuichiro Hanada final int count = readCharGroupCount(buffer); 1475c2fdf0dfbf1c43f7ed8fdf3d91576bbf71146ef3Yuichiro Hanada int groupOffset = nodeHeadPosition + getGroupCountSize(count); 1476c2fdf0dfbf1c43f7ed8fdf3d91576bbf71146ef3Yuichiro Hanada for (int i = count; i > 0; --i) { // Scan the array of CharGroup. 1477c2fdf0dfbf1c43f7ed8fdf3d91576bbf71146ef3Yuichiro Hanada CharGroupInfo info = readCharGroup(buffer, groupOffset, options); 14782ee70804e92b17016a2f042c4f6b0e94b5d23e88Yuichiro Hanada if (isMovedGroup(info.mFlags, options)) continue; 1479c2fdf0dfbf1c43f7ed8fdf3d91576bbf71146ef3Yuichiro Hanada ArrayList<WeightedString> shortcutTargets = info.mShortcutTargets; 1480c2fdf0dfbf1c43f7ed8fdf3d91576bbf71146ef3Yuichiro Hanada ArrayList<WeightedString> bigrams = null; 1481c2fdf0dfbf1c43f7ed8fdf3d91576bbf71146ef3Yuichiro Hanada if (null != info.mBigrams) { 1482c2fdf0dfbf1c43f7ed8fdf3d91576bbf71146ef3Yuichiro Hanada bigrams = new ArrayList<WeightedString>(); 1483c2fdf0dfbf1c43f7ed8fdf3d91576bbf71146ef3Yuichiro Hanada for (PendingAttribute bigram : info.mBigrams) { 1484c2fdf0dfbf1c43f7ed8fdf3d91576bbf71146ef3Yuichiro Hanada final String word = getWordAtAddress( 1485c2fdf0dfbf1c43f7ed8fdf3d91576bbf71146ef3Yuichiro Hanada buffer, headerSize, bigram.mAddress, options); 1486c2fdf0dfbf1c43f7ed8fdf3d91576bbf71146ef3Yuichiro Hanada bigrams.add(new WeightedString(word, bigram.mFrequency)); 1487c2fdf0dfbf1c43f7ed8fdf3d91576bbf71146ef3Yuichiro Hanada } 1488c2fdf0dfbf1c43f7ed8fdf3d91576bbf71146ef3Yuichiro Hanada } 1489c2fdf0dfbf1c43f7ed8fdf3d91576bbf71146ef3Yuichiro Hanada if (hasChildrenAddress(info.mChildrenAddress)) { 1490c2fdf0dfbf1c43f7ed8fdf3d91576bbf71146ef3Yuichiro Hanada Node children = reverseNodeMap.get(info.mChildrenAddress); 1491c2fdf0dfbf1c43f7ed8fdf3d91576bbf71146ef3Yuichiro Hanada if (null == children) { 1492c2fdf0dfbf1c43f7ed8fdf3d91576bbf71146ef3Yuichiro Hanada final int currentPosition = buffer.position(); 1493c2fdf0dfbf1c43f7ed8fdf3d91576bbf71146ef3Yuichiro Hanada buffer.position(info.mChildrenAddress + headerSize); 1494c2fdf0dfbf1c43f7ed8fdf3d91576bbf71146ef3Yuichiro Hanada children = readNode( 1495c2fdf0dfbf1c43f7ed8fdf3d91576bbf71146ef3Yuichiro Hanada buffer, headerSize, reverseNodeMap, reverseGroupMap, options); 1496c2fdf0dfbf1c43f7ed8fdf3d91576bbf71146ef3Yuichiro Hanada buffer.position(currentPosition); 1497c2fdf0dfbf1c43f7ed8fdf3d91576bbf71146ef3Yuichiro Hanada } 1498c2fdf0dfbf1c43f7ed8fdf3d91576bbf71146ef3Yuichiro Hanada nodeContents.add( 1499c2fdf0dfbf1c43f7ed8fdf3d91576bbf71146ef3Yuichiro Hanada new CharGroup(info.mCharacters, shortcutTargets, bigrams, 1500c2fdf0dfbf1c43f7ed8fdf3d91576bbf71146ef3Yuichiro Hanada info.mFrequency, 1501c2fdf0dfbf1c43f7ed8fdf3d91576bbf71146ef3Yuichiro Hanada 0 != (info.mFlags & FormatSpec.FLAG_IS_NOT_A_WORD), 1502c2fdf0dfbf1c43f7ed8fdf3d91576bbf71146ef3Yuichiro Hanada 0 != (info.mFlags & FormatSpec.FLAG_IS_BLACKLISTED), children)); 1503c2fdf0dfbf1c43f7ed8fdf3d91576bbf71146ef3Yuichiro Hanada } else { 1504c2fdf0dfbf1c43f7ed8fdf3d91576bbf71146ef3Yuichiro Hanada nodeContents.add( 1505c2fdf0dfbf1c43f7ed8fdf3d91576bbf71146ef3Yuichiro Hanada new CharGroup(info.mCharacters, shortcutTargets, bigrams, 1506c2fdf0dfbf1c43f7ed8fdf3d91576bbf71146ef3Yuichiro Hanada info.mFrequency, 1507c2fdf0dfbf1c43f7ed8fdf3d91576bbf71146ef3Yuichiro Hanada 0 != (info.mFlags & FormatSpec.FLAG_IS_NOT_A_WORD), 1508c2fdf0dfbf1c43f7ed8fdf3d91576bbf71146ef3Yuichiro Hanada 0 != (info.mFlags & FormatSpec.FLAG_IS_BLACKLISTED))); 1509bfbbee8c5757aef4a20879547c16af0a4d1bf4c7Jean Chalard } 1510c2fdf0dfbf1c43f7ed8fdf3d91576bbf71146ef3Yuichiro Hanada groupOffset = info.mEndAddress; 1511bfbbee8c5757aef4a20879547c16af0a4d1bf4c7Jean Chalard } 1512c2fdf0dfbf1c43f7ed8fdf3d91576bbf71146ef3Yuichiro Hanada 1513c2fdf0dfbf1c43f7ed8fdf3d91576bbf71146ef3Yuichiro Hanada // reach the end of the array. 151482d9deaaf252cd20f8918adbc7a4b9b8f2647c38Yuichiro Hanada if (options.mSupportsDynamicUpdate) { 1515c2fdf0dfbf1c43f7ed8fdf3d91576bbf71146ef3Yuichiro Hanada final int nextAddress = buffer.readUnsignedInt24(); 1516c2fdf0dfbf1c43f7ed8fdf3d91576bbf71146ef3Yuichiro Hanada if (nextAddress >= 0 && nextAddress < buffer.limit()) { 1517c2fdf0dfbf1c43f7ed8fdf3d91576bbf71146ef3Yuichiro Hanada buffer.position(nextAddress); 1518c2fdf0dfbf1c43f7ed8fdf3d91576bbf71146ef3Yuichiro Hanada } else { 1519c2fdf0dfbf1c43f7ed8fdf3d91576bbf71146ef3Yuichiro Hanada break; 1520bfbbee8c5757aef4a20879547c16af0a4d1bf4c7Jean Chalard } 1521bfbbee8c5757aef4a20879547c16af0a4d1bf4c7Jean Chalard } 152282d9deaaf252cd20f8918adbc7a4b9b8f2647c38Yuichiro Hanada } while (options.mSupportsDynamicUpdate && 1523c2fdf0dfbf1c43f7ed8fdf3d91576bbf71146ef3Yuichiro Hanada buffer.position() != FormatSpec.NO_FORWARD_LINK_ADDRESS); 1524c2fdf0dfbf1c43f7ed8fdf3d91576bbf71146ef3Yuichiro Hanada 1525bfbbee8c5757aef4a20879547c16af0a4d1bf4c7Jean Chalard final Node node = new Node(nodeContents); 1526bfbbee8c5757aef4a20879547c16af0a4d1bf4c7Jean Chalard node.mCachedAddress = nodeOrigin; 1527bfbbee8c5757aef4a20879547c16af0a4d1bf4c7Jean Chalard reverseNodeMap.put(node.mCachedAddress, node); 1528bfbbee8c5757aef4a20879547c16af0a4d1bf4c7Jean Chalard return node; 1529bfbbee8c5757aef4a20879547c16af0a4d1bf4c7Jean Chalard } 1530bfbbee8c5757aef4a20879547c16af0a4d1bf4c7Jean Chalard 1531bfbbee8c5757aef4a20879547c16af0a4d1bf4c7Jean Chalard /** 1532a9a5dd07b573718cc9a6bf99394b2db646e73d1aJean Chalard * Helper function to get the binary format version from the header. 1533d4fe7fda303ff937d2e44c15dde9d90cbf59376bYuichiro Hanada * @throws IOException 153489dfb2bae171956dbcc880c9192ff29949388770Jean Chalard */ 1535f5c4ff481782831329593760b000f0543680930aYuichiro Hanada private static int getFormatVersion(final FusionDictionaryBufferInterface buffer) 1536f5c4ff481782831329593760b000f0543680930aYuichiro Hanada throws IOException { 1537f5c4ff481782831329593760b000f0543680930aYuichiro Hanada final int magic_v1 = buffer.readUnsignedShort(); 153881d97eec0e77e72cce606f9c9f96091c0b348190Yuichiro Hanada if (FormatSpec.VERSION_1_MAGIC_NUMBER == magic_v1) return buffer.readUnsignedByte(); 1539f5c4ff481782831329593760b000f0543680930aYuichiro Hanada final int magic_v2 = (magic_v1 << 16) + buffer.readUnsignedShort(); 154081d97eec0e77e72cce606f9c9f96091c0b348190Yuichiro Hanada if (FormatSpec.VERSION_2_MAGIC_NUMBER == magic_v2) return buffer.readUnsignedShort(); 154181d97eec0e77e72cce606f9c9f96091c0b348190Yuichiro Hanada return FormatSpec.NOT_A_VERSION_NUMBER; 154289dfb2bae171956dbcc880c9192ff29949388770Jean Chalard } 154389dfb2bae171956dbcc880c9192ff29949388770Jean Chalard 154489dfb2bae171956dbcc880c9192ff29949388770Jean Chalard /** 154562ed9011002817cc78f3a1de39a1171cb136176aYuichiro Hanada * Helper function to get and validate the binary format version. 154662ed9011002817cc78f3a1de39a1171cb136176aYuichiro Hanada * @throws UnsupportedFormatException 154762ed9011002817cc78f3a1de39a1171cb136176aYuichiro Hanada * @throws IOException 154862ed9011002817cc78f3a1de39a1171cb136176aYuichiro Hanada */ 154962ed9011002817cc78f3a1de39a1171cb136176aYuichiro Hanada private static int checkFormatVersion(final FusionDictionaryBufferInterface buffer) 155062ed9011002817cc78f3a1de39a1171cb136176aYuichiro Hanada throws IOException, UnsupportedFormatException { 155162ed9011002817cc78f3a1de39a1171cb136176aYuichiro Hanada final int version = getFormatVersion(buffer); 155281d97eec0e77e72cce606f9c9f96091c0b348190Yuichiro Hanada if (version < FormatSpec.MINIMUM_SUPPORTED_VERSION 155381d97eec0e77e72cce606f9c9f96091c0b348190Yuichiro Hanada || version > FormatSpec.MAXIMUM_SUPPORTED_VERSION) { 155462ed9011002817cc78f3a1de39a1171cb136176aYuichiro Hanada throw new UnsupportedFormatException("This file has version " + version 155562ed9011002817cc78f3a1de39a1171cb136176aYuichiro Hanada + ", but this implementation does not support versions above " 155681d97eec0e77e72cce606f9c9f96091c0b348190Yuichiro Hanada + FormatSpec.MAXIMUM_SUPPORTED_VERSION); 155762ed9011002817cc78f3a1de39a1171cb136176aYuichiro Hanada } 155862ed9011002817cc78f3a1de39a1171cb136176aYuichiro Hanada return version; 155962ed9011002817cc78f3a1de39a1171cb136176aYuichiro Hanada } 156062ed9011002817cc78f3a1de39a1171cb136176aYuichiro Hanada 156162ed9011002817cc78f3a1de39a1171cb136176aYuichiro Hanada /** 156262ed9011002817cc78f3a1de39a1171cb136176aYuichiro Hanada * Reads a header from a buffer. 156383dfe0fd8c7e2bce2717930dbf8732f5414ee39dYuichiro Hanada * @param buffer the buffer to read. 156462ed9011002817cc78f3a1de39a1171cb136176aYuichiro Hanada * @throws IOException 156562ed9011002817cc78f3a1de39a1171cb136176aYuichiro Hanada * @throws UnsupportedFormatException 156662ed9011002817cc78f3a1de39a1171cb136176aYuichiro Hanada */ 156765feee12e5889601e375d92dfdf5f8e8fbb05092Yuichiro Hanada public static FileHeader readHeader(final FusionDictionaryBufferInterface buffer) 156862ed9011002817cc78f3a1de39a1171cb136176aYuichiro Hanada throws IOException, UnsupportedFormatException { 156983dfe0fd8c7e2bce2717930dbf8732f5414ee39dYuichiro Hanada final int version = checkFormatVersion(buffer); 157083dfe0fd8c7e2bce2717930dbf8732f5414ee39dYuichiro Hanada final int optionsFlags = buffer.readUnsignedShort(); 157183dfe0fd8c7e2bce2717930dbf8732f5414ee39dYuichiro Hanada 157283dfe0fd8c7e2bce2717930dbf8732f5414ee39dYuichiro Hanada final HashMap<String, String> attributes = new HashMap<String, String>(); 157362ed9011002817cc78f3a1de39a1171cb136176aYuichiro Hanada final int headerSize; 157481d97eec0e77e72cce606f9c9f96091c0b348190Yuichiro Hanada if (version < FormatSpec.FIRST_VERSION_WITH_HEADER_SIZE) { 157562ed9011002817cc78f3a1de39a1171cb136176aYuichiro Hanada headerSize = buffer.position(); 157662ed9011002817cc78f3a1de39a1171cb136176aYuichiro Hanada } else { 157762ed9011002817cc78f3a1de39a1171cb136176aYuichiro Hanada headerSize = buffer.readInt(); 157883dfe0fd8c7e2bce2717930dbf8732f5414ee39dYuichiro Hanada populateOptions(buffer, headerSize, attributes); 157962ed9011002817cc78f3a1de39a1171cb136176aYuichiro Hanada buffer.position(headerSize); 158062ed9011002817cc78f3a1de39a1171cb136176aYuichiro Hanada } 158162ed9011002817cc78f3a1de39a1171cb136176aYuichiro Hanada 158262ed9011002817cc78f3a1de39a1171cb136176aYuichiro Hanada if (headerSize < 0) { 158362ed9011002817cc78f3a1de39a1171cb136176aYuichiro Hanada throw new UnsupportedFormatException("header size can't be negative."); 158462ed9011002817cc78f3a1de39a1171cb136176aYuichiro Hanada } 158583dfe0fd8c7e2bce2717930dbf8732f5414ee39dYuichiro Hanada 158683dfe0fd8c7e2bce2717930dbf8732f5414ee39dYuichiro Hanada final FileHeader header = new FileHeader(headerSize, 158783dfe0fd8c7e2bce2717930dbf8732f5414ee39dYuichiro Hanada new FusionDictionary.DictionaryOptions(attributes, 158881d97eec0e77e72cce606f9c9f96091c0b348190Yuichiro Hanada 0 != (optionsFlags & FormatSpec.GERMAN_UMLAUT_PROCESSING_FLAG), 158981d97eec0e77e72cce606f9c9f96091c0b348190Yuichiro Hanada 0 != (optionsFlags & FormatSpec.FRENCH_LIGATURE_PROCESSING_FLAG)), 1590e55b644aefbefb4ac79308c9a59116e69a9c53a2Yuichiro Hanada new FormatOptions(version, 159182d9deaaf252cd20f8918adbc7a4b9b8f2647c38Yuichiro Hanada 0 != (optionsFlags & FormatSpec.SUPPORTS_DYNAMIC_UPDATE))); 159283dfe0fd8c7e2bce2717930dbf8732f5414ee39dYuichiro Hanada return header; 159362ed9011002817cc78f3a1de39a1171cb136176aYuichiro Hanada } 159462ed9011002817cc78f3a1de39a1171cb136176aYuichiro Hanada 159562ed9011002817cc78f3a1de39a1171cb136176aYuichiro Hanada /** 1596f5c4ff481782831329593760b000f0543680930aYuichiro Hanada * Reads options from a buffer and populate a map with their contents. 159713822d2b056543de5a54b5ed338ca2cc250d8287Jean Chalard * 1598f5c4ff481782831329593760b000f0543680930aYuichiro Hanada * The buffer is read at the current position, so the caller must take care the pointer 159913822d2b056543de5a54b5ed338ca2cc250d8287Jean Chalard * is in the right place before calling this. 160013822d2b056543de5a54b5ed338ca2cc250d8287Jean Chalard */ 1601f5c4ff481782831329593760b000f0543680930aYuichiro Hanada public static void populateOptions(final FusionDictionaryBufferInterface buffer, 1602f5c4ff481782831329593760b000f0543680930aYuichiro Hanada final int headerSize, final HashMap<String, String> options) { 1603d4fe7fda303ff937d2e44c15dde9d90cbf59376bYuichiro Hanada while (buffer.position() < headerSize) { 1604d4fe7fda303ff937d2e44c15dde9d90cbf59376bYuichiro Hanada final String key = CharEncoding.readString(buffer); 1605d4fe7fda303ff937d2e44c15dde9d90cbf59376bYuichiro Hanada final String value = CharEncoding.readString(buffer); 160613822d2b056543de5a54b5ed338ca2cc250d8287Jean Chalard options.put(key, value); 160713822d2b056543de5a54b5ed338ca2cc250d8287Jean Chalard } 160813822d2b056543de5a54b5ed338ca2cc250d8287Jean Chalard } 160913822d2b056543de5a54b5ed338ca2cc250d8287Jean Chalard 161013822d2b056543de5a54b5ed338ca2cc250d8287Jean Chalard /** 1611f5c4ff481782831329593760b000f0543680930aYuichiro Hanada * Reads a buffer and returns the memory representation of the dictionary. 1612bfbbee8c5757aef4a20879547c16af0a4d1bf4c7Jean Chalard * 1613f5c4ff481782831329593760b000f0543680930aYuichiro Hanada * This high-level method takes a buffer and reads its contents, populating a 1614bfbbee8c5757aef4a20879547c16af0a4d1bf4c7Jean Chalard * FusionDictionary structure. The optional dict argument is an existing dictionary to 1615f5c4ff481782831329593760b000f0543680930aYuichiro Hanada * which words from the buffer should be added. If it is null, a new dictionary is created. 1616bfbbee8c5757aef4a20879547c16af0a4d1bf4c7Jean Chalard * 1617d4fe7fda303ff937d2e44c15dde9d90cbf59376bYuichiro Hanada * @param buffer the buffer to read. 1618bfbbee8c5757aef4a20879547c16af0a4d1bf4c7Jean Chalard * @param dict an optional dictionary to add words to, or null. 1619bfbbee8c5757aef4a20879547c16af0a4d1bf4c7Jean Chalard * @return the created (or merged) dictionary. 1620bfbbee8c5757aef4a20879547c16af0a4d1bf4c7Jean Chalard */ 1621f5c4ff481782831329593760b000f0543680930aYuichiro Hanada public static FusionDictionary readDictionaryBinary( 1622f5c4ff481782831329593760b000f0543680930aYuichiro Hanada final FusionDictionaryBufferInterface buffer, final FusionDictionary dict) 1623f5c4ff481782831329593760b000f0543680930aYuichiro Hanada throws IOException, UnsupportedFormatException { 1624d4fe7fda303ff937d2e44c15dde9d90cbf59376bYuichiro Hanada // clear cache 1625d4fe7fda303ff937d2e44c15dde9d90cbf59376bYuichiro Hanada wordCache.clear(); 1626d4fe7fda303ff937d2e44c15dde9d90cbf59376bYuichiro Hanada 162762ed9011002817cc78f3a1de39a1171cb136176aYuichiro Hanada // Read header 162883dfe0fd8c7e2bce2717930dbf8732f5414ee39dYuichiro Hanada final FileHeader header = readHeader(buffer); 1629e0f6cc181457613766fd480f499b3e83711cf989Jean Chalard 1630bfbbee8c5757aef4a20879547c16af0a4d1bf4c7Jean Chalard Map<Integer, Node> reverseNodeMapping = new TreeMap<Integer, Node>(); 1631bfbbee8c5757aef4a20879547c16af0a4d1bf4c7Jean Chalard Map<Integer, CharGroup> reverseGroupMapping = new TreeMap<Integer, CharGroup>(); 163283dfe0fd8c7e2bce2717930dbf8732f5414ee39dYuichiro Hanada final Node root = readNode(buffer, header.mHeaderSize, reverseNodeMapping, 1633e55b644aefbefb4ac79308c9a59116e69a9c53a2Yuichiro Hanada reverseGroupMapping, header.mFormatOptions); 1634bfbbee8c5757aef4a20879547c16af0a4d1bf4c7Jean Chalard 163583dfe0fd8c7e2bce2717930dbf8732f5414ee39dYuichiro Hanada FusionDictionary newDict = new FusionDictionary(root, header.mDictionaryOptions); 1636bfbbee8c5757aef4a20879547c16af0a4d1bf4c7Jean Chalard if (null != dict) { 163744c64f46a143623dd793facd889c8d6eab5e230cJean Chalard for (final Word w : dict) { 163872b1c9394105b6fbc0d8c6ff00f3574ee37a9aaaJean Chalard if (w.mIsBlacklistEntry) { 163972b1c9394105b6fbc0d8c6ff00f3574ee37a9aaaJean Chalard newDict.addBlacklistEntry(w.mWord, w.mShortcutTargets, w.mIsNotAWord); 164072b1c9394105b6fbc0d8c6ff00f3574ee37a9aaaJean Chalard } else { 164172b1c9394105b6fbc0d8c6ff00f3574ee37a9aaaJean Chalard newDict.add(w.mWord, w.mFrequency, w.mShortcutTargets, w.mIsNotAWord); 164272b1c9394105b6fbc0d8c6ff00f3574ee37a9aaaJean Chalard } 164344c64f46a143623dd793facd889c8d6eab5e230cJean Chalard } 164444c64f46a143623dd793facd889c8d6eab5e230cJean Chalard for (final Word w : dict) { 164544c64f46a143623dd793facd889c8d6eab5e230cJean Chalard // By construction a binary dictionary may not have bigrams pointing to 164644c64f46a143623dd793facd889c8d6eab5e230cJean Chalard // words that are not also registered as unigrams so we don't have to avoid 164744c64f46a143623dd793facd889c8d6eab5e230cJean Chalard // them explicitly here. 164844c64f46a143623dd793facd889c8d6eab5e230cJean Chalard for (final WeightedString bigram : w.mBigrams) { 164944c64f46a143623dd793facd889c8d6eab5e230cJean Chalard newDict.setBigram(w.mWord, bigram.mWord, bigram.mFrequency); 165044c64f46a143623dd793facd889c8d6eab5e230cJean Chalard } 1651bfbbee8c5757aef4a20879547c16af0a4d1bf4c7Jean Chalard } 1652bfbbee8c5757aef4a20879547c16af0a4d1bf4c7Jean Chalard } 1653bfbbee8c5757aef4a20879547c16af0a4d1bf4c7Jean Chalard 1654bfbbee8c5757aef4a20879547c16af0a4d1bf4c7Jean Chalard return newDict; 1655bfbbee8c5757aef4a20879547c16af0a4d1bf4c7Jean Chalard } 1656bfbbee8c5757aef4a20879547c16af0a4d1bf4c7Jean Chalard 1657d4fe7fda303ff937d2e44c15dde9d90cbf59376bYuichiro Hanada /** 1658bfbbee8c5757aef4a20879547c16af0a4d1bf4c7Jean Chalard * Basic test to find out whether the file is a binary dictionary or not. 1659bfbbee8c5757aef4a20879547c16af0a4d1bf4c7Jean Chalard * 1660bfbbee8c5757aef4a20879547c16af0a4d1bf4c7Jean Chalard * Concretely this only tests the magic number. 1661bfbbee8c5757aef4a20879547c16af0a4d1bf4c7Jean Chalard * 1662bfbbee8c5757aef4a20879547c16af0a4d1bf4c7Jean Chalard * @param filename The name of the file to test. 1663bfbbee8c5757aef4a20879547c16af0a4d1bf4c7Jean Chalard * @return true if it's a binary dictionary, false otherwise 1664bfbbee8c5757aef4a20879547c16af0a4d1bf4c7Jean Chalard */ 166589dfb2bae171956dbcc880c9192ff29949388770Jean Chalard public static boolean isBinaryDictionary(final String filename) { 1666d4fe7fda303ff937d2e44c15dde9d90cbf59376bYuichiro Hanada FileInputStream inStream = null; 1667bfbbee8c5757aef4a20879547c16af0a4d1bf4c7Jean Chalard try { 1668d4fe7fda303ff937d2e44c15dde9d90cbf59376bYuichiro Hanada final File file = new File(filename); 1669d4fe7fda303ff937d2e44c15dde9d90cbf59376bYuichiro Hanada inStream = new FileInputStream(file); 1670d4fe7fda303ff937d2e44c15dde9d90cbf59376bYuichiro Hanada final ByteBuffer buffer = inStream.getChannel().map( 1671d4fe7fda303ff937d2e44c15dde9d90cbf59376bYuichiro Hanada FileChannel.MapMode.READ_ONLY, 0, file.length()); 1672f5c4ff481782831329593760b000f0543680930aYuichiro Hanada final int version = getFormatVersion(new ByteBufferWrapper(buffer)); 167381d97eec0e77e72cce606f9c9f96091c0b348190Yuichiro Hanada return (version >= FormatSpec.MINIMUM_SUPPORTED_VERSION 167481d97eec0e77e72cce606f9c9f96091c0b348190Yuichiro Hanada && version <= FormatSpec.MAXIMUM_SUPPORTED_VERSION); 1675bfbbee8c5757aef4a20879547c16af0a4d1bf4c7Jean Chalard } catch (FileNotFoundException e) { 1676bfbbee8c5757aef4a20879547c16af0a4d1bf4c7Jean Chalard return false; 1677bfbbee8c5757aef4a20879547c16af0a4d1bf4c7Jean Chalard } catch (IOException e) { 1678bfbbee8c5757aef4a20879547c16af0a4d1bf4c7Jean Chalard return false; 1679d4fe7fda303ff937d2e44c15dde9d90cbf59376bYuichiro Hanada } finally { 1680d4fe7fda303ff937d2e44c15dde9d90cbf59376bYuichiro Hanada if (inStream != null) { 1681d4fe7fda303ff937d2e44c15dde9d90cbf59376bYuichiro Hanada try { 1682d4fe7fda303ff937d2e44c15dde9d90cbf59376bYuichiro Hanada inStream.close(); 1683d4fe7fda303ff937d2e44c15dde9d90cbf59376bYuichiro Hanada } catch (IOException e) { 1684d4fe7fda303ff937d2e44c15dde9d90cbf59376bYuichiro Hanada // do nothing 1685d4fe7fda303ff937d2e44c15dde9d90cbf59376bYuichiro Hanada } 1686d4fe7fda303ff937d2e44c15dde9d90cbf59376bYuichiro Hanada } 1687bfbbee8c5757aef4a20879547c16af0a4d1bf4c7Jean Chalard } 1688bfbbee8c5757aef4a20879547c16af0a4d1bf4c7Jean Chalard } 1689c0a75c8ecbd373c4eaee4f866e4080c0b800470bYuichiro Hanada 1690c0a75c8ecbd373c4eaee4f866e4080c0b800470bYuichiro Hanada /** 1691c0a75c8ecbd373c4eaee4f866e4080c0b800470bYuichiro Hanada * Calculate bigram frequency from compressed value 1692c0a75c8ecbd373c4eaee4f866e4080c0b800470bYuichiro Hanada * 1693c0a75c8ecbd373c4eaee4f866e4080c0b800470bYuichiro Hanada * @see #makeBigramFlags 1694c0a75c8ecbd373c4eaee4f866e4080c0b800470bYuichiro Hanada * 1695c0a75c8ecbd373c4eaee4f866e4080c0b800470bYuichiro Hanada * @param unigramFrequency 1696c0a75c8ecbd373c4eaee4f866e4080c0b800470bYuichiro Hanada * @param bigramFrequency compressed frequency 1697c0a75c8ecbd373c4eaee4f866e4080c0b800470bYuichiro Hanada * @return approximate bigram frequency 1698c0a75c8ecbd373c4eaee4f866e4080c0b800470bYuichiro Hanada */ 1699c0a75c8ecbd373c4eaee4f866e4080c0b800470bYuichiro Hanada public static int reconstructBigramFrequency(final int unigramFrequency, 1700c0a75c8ecbd373c4eaee4f866e4080c0b800470bYuichiro Hanada final int bigramFrequency) { 170181d97eec0e77e72cce606f9c9f96091c0b348190Yuichiro Hanada final float stepSize = (FormatSpec.MAX_TERMINAL_FREQUENCY - unigramFrequency) 170281d97eec0e77e72cce606f9c9f96091c0b348190Yuichiro Hanada / (1.5f + FormatSpec.MAX_BIGRAM_FREQUENCY); 1703c0a75c8ecbd373c4eaee4f866e4080c0b800470bYuichiro Hanada final float resultFreqFloat = (float)unigramFrequency 1704c0a75c8ecbd373c4eaee4f866e4080c0b800470bYuichiro Hanada + stepSize * (bigramFrequency + 1.0f); 1705c0a75c8ecbd373c4eaee4f866e4080c0b800470bYuichiro Hanada return (int)resultFreqFloat; 1706c0a75c8ecbd373c4eaee4f866e4080c0b800470bYuichiro Hanada } 1707bfbbee8c5757aef4a20879547c16af0a4d1bf4c7Jean Chalard} 1708