/* * Copyright (C) 2012 The Android Open Source Project * * Licensed under the Apache License, Version 2.0 (the "License"); you may not * use this file except in compliance with the License. You may obtain a copy of * the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the * License for the specific language governing permissions and limitations under * the License. */ package com.android.inputmethod.latin.makedict; import com.android.inputmethod.latin.Constants; import com.android.inputmethod.latin.makedict.FusionDictionary.DictionaryOptions; /** * Dictionary File Format Specification. */ public final class FormatSpec { /* * Array of Node(FusionDictionary.Node) layout is as follows: * * g | * r | the number of groups, 1 or 2 bytes. * o | 1 byte = bbbbbbbb match * u | case 1xxxxxxx => xxxxxxx << 8 + next byte * p | otherwise => bbbbbbbb * c | * ount * * g | * r | sequence of groups, * o | the layout of each group is described below. * u | * ps * * f | * o | IF HAS_LINKEDLIST_NODE (defined in the file header) * r | forward link address, 3byte * w | the address must be positive. * a | * rdlinkaddress */ /* Node(CharGroup) layout is as follows: * | addressType xx : mask with MASK_GROUP_ADDRESS_TYPE * 2 bits, 00 = no children : FLAG_GROUP_ADDRESS_TYPE_NOADDRESS * f | 01 = 1 byte : FLAG_GROUP_ADDRESS_TYPE_ONEBYTE * l | 10 = 2 bytes : FLAG_GROUP_ADDRESS_TYPE_TWOBYTES * a | 11 = 3 bytes : FLAG_GROUP_ADDRESS_TYPE_THREEBYTES * g | has several chars ? 1 bit, 1 = yes, 0 = no : FLAG_HAS_MULTIPLE_CHARS * s | has a terminal ? 1 bit, 1 = yes, 0 = no : FLAG_IS_TERMINAL * | has shortcut targets ? 1 bit, 1 = yes, 0 = no : FLAG_HAS_SHORTCUT_TARGETS * | has bigrams ? 1 bit, 1 = yes, 0 = no : FLAG_HAS_BIGRAMS * | is not a word ? 1 bit, 1 = yes, 0 = no : FLAG_IS_NOT_A_WORD * | is blacklisted ? 1 bit, 1 = yes, 0 = no : FLAG_IS_BLACKLISTED * * p | * a | IF HAS_PARENT_ADDRESS (defined in the file header) * r | parent address, 3byte * e | the address must be negative, so the absolute value of the address is stored. * n | * taddress * * c | IF FLAG_HAS_MULTIPLE_CHARS * h | char, char, char, char n * (1 or 3 bytes) : use CharGroupInfo for i/o helpers * a | end 1 byte, = 0 * r | ELSE * s | char 1 or 3 bytes * | END * * f | * r | IF FLAG_IS_TERMINAL * e | frequency 1 byte * q | * * c | IF 00 = FLAG_GROUP_ADDRESS_TYPE_NOADDRESS = addressType * h | // nothing * i | ELSIF 01 = FLAG_GROUP_ADDRESS_TYPE_ONEBYTE == addressType * l | children address, 1 byte * d | ELSIF 10 = FLAG_GROUP_ADDRESS_TYPE_TWOBYTES == addressType * r | children address, 2 bytes * e | ELSE // 11 = FLAG_GROUP_ADDRESS_TYPE_THREEBYTES = addressType * n | children address, 3 bytes * A | END * d * dress * * | IF FLAG_IS_TERMINAL && FLAG_HAS_SHORTCUT_TARGETS * | shortcut string list * | IF FLAG_IS_TERMINAL && FLAG_HAS_BIGRAMS * | bigrams address list * * Char format is: * 1 byte = bbbbbbbb match * case 000xxxxx: xxxxx << 16 + next byte << 8 + next byte * else: if 00011111 (= 0x1F) : this is the terminator. This is a relevant choice because * unicode code points range from 0 to 0x10FFFF, so any 3-byte value starting with * 00011111 would be outside unicode. * else: iso-latin-1 code * This allows for the whole unicode range to be encoded, including chars outside of * the BMP. Also everything in the iso-latin-1 charset is only 1 byte, except control * characters which should never happen anyway (and still work, but take 3 bytes). * * bigram address list is: * = | hasNext = 1 bit, 1 = yes, 0 = no : FLAG_ATTRIBUTE_HAS_NEXT * | addressSign = 1 bit, : FLAG_ATTRIBUTE_OFFSET_NEGATIVE * | 1 = must take -address, 0 = must take +address * | xx : mask with MASK_ATTRIBUTE_ADDRESS_TYPE * | addressFormat = 2 bits, 00 = unused : FLAG_ATTRIBUTE_ADDRESS_TYPE_ONEBYTE * | 01 = 1 byte : FLAG_ATTRIBUTE_ADDRESS_TYPE_ONEBYTE * | 10 = 2 bytes : FLAG_ATTRIBUTE_ADDRESS_TYPE_TWOBYTES * | 11 = 3 bytes : FLAG_ATTRIBUTE_ADDRESS_TYPE_THREEBYTES * | 4 bits : frequency : mask with FLAG_ATTRIBUTE_FREQUENCY *
| IF (01 == FLAG_ATTRIBUTE_ADDRESS_TYPE_ONEBYTE == addressFormat) * | read 1 byte, add top 4 bits * | ELSIF (10 == FLAG_ATTRIBUTE_ADDRESS_TYPE_TWOBYTES == addressFormat) * | read 2 bytes, add top 4 bits * | ELSE // 11 == FLAG_ATTRIBUTE_ADDRESS_TYPE_THREEBYTES == addressFormat * | read 3 bytes, add top 4 bits * | END * | if (FLAG_ATTRIBUTE_OFFSET_NEGATIVE) then address = -address * if (FLAG_ATTRIBUTE_HAS_NEXT) goto bigram_and_shortcut_address_list_is * * shortcut string list is: * = GROUP_SHORTCUT_LIST_SIZE_SIZE bytes, big-endian: size of the list, in bytes. * = | hasNext = 1 bit, 1 = yes, 0 = no : FLAG_ATTRIBUTE_HAS_NEXT * | reserved = 3 bits, must be 0 * | 4 bits : frequency : mask with FLAG_ATTRIBUTE_FREQUENCY * = | string of characters at the char format described above, with the terminator * | used to signal the end of the string. * if (FLAG_ATTRIBUTE_HAS_NEXT goto flags */ static final int VERSION_1_MAGIC_NUMBER = 0x78B1; public static final int VERSION_2_MAGIC_NUMBER = 0x9BC13AFE; static final int MINIMUM_SUPPORTED_VERSION = 1; static final int MAXIMUM_SUPPORTED_VERSION = 3; static final int NOT_A_VERSION_NUMBER = -1; static final int FIRST_VERSION_WITH_HEADER_SIZE = 2; static final int FIRST_VERSION_WITH_PARENT_ADDRESS = 3; static final int FIRST_VERSION_WITH_LINKEDLIST_NODE = 3; // These options need to be the same numeric values as the one in the native reading code. static final int GERMAN_UMLAUT_PROCESSING_FLAG = 0x1; static final int HAS_PARENT_ADDRESS = 0x2; static final int FRENCH_LIGATURE_PROCESSING_FLAG = 0x4; static final int CONTAINS_BIGRAMS_FLAG = 0x8; // TODO: Make this value adaptative to content data, store it in the header, and // use it in the reading code. static final int MAX_WORD_LENGTH = Constants.Dictionary.MAX_WORD_LENGTH; static final int PARENT_ADDRESS_SIZE = 3; static final int FORWARD_LINK_ADDRESS_SIZE = 3; static final int MASK_GROUP_ADDRESS_TYPE = 0xC0; static final int FLAG_GROUP_ADDRESS_TYPE_NOADDRESS = 0x00; static final int FLAG_GROUP_ADDRESS_TYPE_ONEBYTE = 0x40; static final int FLAG_GROUP_ADDRESS_TYPE_TWOBYTES = 0x80; static final int FLAG_GROUP_ADDRESS_TYPE_THREEBYTES = 0xC0; static final int FLAG_HAS_MULTIPLE_CHARS = 0x20; static final int FLAG_IS_TERMINAL = 0x10; static final int FLAG_HAS_SHORTCUT_TARGETS = 0x08; static final int FLAG_HAS_BIGRAMS = 0x04; static final int FLAG_IS_NOT_A_WORD = 0x02; static final int FLAG_IS_BLACKLISTED = 0x01; static final int FLAG_ATTRIBUTE_HAS_NEXT = 0x80; static final int FLAG_ATTRIBUTE_OFFSET_NEGATIVE = 0x40; static final int MASK_ATTRIBUTE_ADDRESS_TYPE = 0x30; static final int FLAG_ATTRIBUTE_ADDRESS_TYPE_ONEBYTE = 0x10; static final int FLAG_ATTRIBUTE_ADDRESS_TYPE_TWOBYTES = 0x20; static final int FLAG_ATTRIBUTE_ADDRESS_TYPE_THREEBYTES = 0x30; static final int FLAG_ATTRIBUTE_FREQUENCY = 0x0F; static final int GROUP_CHARACTERS_TERMINATOR = 0x1F; static final int GROUP_TERMINATOR_SIZE = 1; static final int GROUP_FLAGS_SIZE = 1; static final int GROUP_FREQUENCY_SIZE = 1; static final int GROUP_MAX_ADDRESS_SIZE = 3; static final int GROUP_ATTRIBUTE_FLAGS_SIZE = 1; static final int GROUP_ATTRIBUTE_MAX_ADDRESS_SIZE = 3; static final int GROUP_SHORTCUT_LIST_SIZE_SIZE = 2; static final int NO_CHILDREN_ADDRESS = Integer.MIN_VALUE; static final int NO_PARENT_ADDRESS = 0; static final int NO_FORWARD_LINK_ADDRESS = 0; static final int INVALID_CHARACTER = -1; static final int MAX_CHARGROUPS_FOR_ONE_BYTE_CHARGROUP_COUNT = 0x7F; // 127 static final int MAX_CHARGROUPS_IN_A_NODE = 0x7FFF; // 32767 static final int MAX_TERMINAL_FREQUENCY = 255; static final int MAX_BIGRAM_FREQUENCY = 15; /** * Options about file format. */ public static class FormatOptions { public final int mVersion; public final boolean mHasParentAddress; public final boolean mHasLinkedListNode; public FormatOptions(final int version) { this(version, false); } public FormatOptions(final int version, final boolean hasParentAddress) { this(version, hasParentAddress, false); } public FormatOptions(final int version, final boolean hasParentAddress, final boolean hasLinkedListNode) { mVersion = version; if (version < FIRST_VERSION_WITH_PARENT_ADDRESS && hasParentAddress) { throw new RuntimeException("Parent addresses are only supported with versions " + FIRST_VERSION_WITH_PARENT_ADDRESS + " and ulterior."); } mHasParentAddress = hasParentAddress; if (version < FIRST_VERSION_WITH_LINKEDLIST_NODE && hasLinkedListNode) { throw new RuntimeException("Linked list nodes are only supported with versions " + FIRST_VERSION_WITH_LINKEDLIST_NODE + " and ulterior."); } if (!hasParentAddress && hasLinkedListNode) { throw new RuntimeException("Linked list nodes need parent addresses."); } mHasLinkedListNode = hasLinkedListNode; } } /** * Class representing file header. */ static final class FileHeader { public final int mHeaderSize; public final DictionaryOptions mDictionaryOptions; public final FormatOptions mFormatOptions; public FileHeader(final int headerSize, final DictionaryOptions dictionaryOptions, final FormatOptions formatOptions) { mHeaderSize = headerSize; mDictionaryOptions = dictionaryOptions; mFormatOptions = formatOptions; } } private FormatSpec() { // This utility class is not publicly instantiable. } }