1// © 2016 and later: Unicode, Inc. and others. 2// License & terms of use: http://www.unicode.org/copyright.html#License 3/* 4 ******************************************************************************* 5 * Copyright (C) 1996-2004, International Business Machines Corporation and * 6 * others. All Rights Reserved. * 7 ******************************************************************************* 8 */ 9package com.ibm.icu.dev.test.rbbi; 10 11import java.util.ListResourceBundle; 12 13/** 14 * This resource bundle is included for testing and demonstration purposes only. 15 * It applies the dictionary-based algorithm to English text that has had all the 16 * spaces removed. Once we have good test cases for Thai, we will replace this 17 * with good resource data (and a good dictionary file) for Thai 18 */ 19public class BreakIteratorRules_en_US_TEST extends ListResourceBundle { 20 private static final String DATA_NAME = "/com/ibm/icu/dev/data/rbbi/english.dict"; 21 22 // calling code will handle case where dictionary does not exist 23 24 public Object[][] getContents() { 25 return new Object[][] { 26 // names of classes to instantiate for the different kinds of break 27 // iterator. Notice we're now using DictionaryBasedBreakIterator 28 // for word and line breaking. 29 { "BreakIteratorClasses", 30 new String[] { 31 "RuleBasedBreakIterator", 32 // character-break iterator class 33 "DictionaryBasedBreakIterator", 34 // word-break iterator class 35 "DictionaryBasedBreakIterator", 36 // line-break iterator class 37 "RuleBasedBreakIterator" } // sentence-break iterator class 38 }, 39 40 // These are the same word-breaking rules as are specified in the default 41 // resource, except that the Latin letters, apostrophe, and hyphen are 42 // specified as dictionary characters 43 { 44 "WordBreakRules", 45 // ignore non-spacing marks, enclosing marks, and format characters, 46 // all of which should not influence the algorithm 47 "$_ignore_=[[:Mn:][:Me:][:Cf:]];" 48 49 // lower and upper case Roman letters, apostrophy and dash are 50 // in the English dictionary 51 +"$_dictionary_=[a-zA-Z\\'\\-];" 52 53 // Hindi phrase separator, kanji, katakana, hiragana, CJK diacriticals, 54 // other letters, and digits 55 +"$danda=[\u0964\u0965];" 56 + "$kanji=[\u3005\u4e00-\u9fa5\uf900-\ufa2d];" 57 + "$kata=[\u3099-\u309c\u30a1-\u30fe];" 58 + "$hira=[\u3041-\u309e\u30fc];" 59 + "$let=[[[:L:][:Mc:]]-[$kanji$kata$hira]];" 60 + "$dgt=[:N:];" 61 62 // punctuation that can occur in the middle of a word: currently 63 // dashes, apostrophes, and quotation marks 64 +"$mid_word=[[:Pd:]\u00ad\u2027\\\"\\\'];" 65 66 // punctuation that can occur in the middle of a number: currently 67 // apostrophes, qoutation marks, periods, commas, and the Arabic 68 // decimal point 69 +"$mid_num=[\\\"\\\'\\,\u066b\\.];" 70 71 // punctuation that can occur at the beginning of a number: currently 72 // the period, the number sign, and all currency symbols except the cents sign 73 +"$pre_num=[[[:Sc:]-[\u00a2]]\\#\\.];" 74 75 // punctuation that can occur at the end of a number: currently 76 // the percent, per-thousand, per-ten-thousand, and Arabic percent 77 // signs, the cents sign, and the ampersand 78 +"$post_num=[\\%\\&\u00a2\u066a\u2030\u2031];" 79 80 // line separators: currently LF, FF, PS, and LS 81 +"$ls=[\n\u000c\u2028\u2029];" 82 83 // whitespace: all space separators and the tab character 84 +"$ws=[[:Zs:]\t];" 85 86 // a word is a sequence of letters that may contain internal 87 // punctuation, as long as it begins and ends with a letter and 88 // never contains two punctuation marks in a row 89 +"$word=($let+($mid_word$let+)*$danda?);" 90 91 // a number is a sequence of digits that may contain internal 92 // punctuation, as long as it begins and ends with a digit and 93 // never contains two punctuation marks in a row. 94 +"$number=($dgt+($mid_num$dgt+)*);" 95 96 // break after every character, with the following exceptions 97 // (this will cause punctuation marks that aren't considered 98 // part of words or numbers to be treated as words unto themselves) 99 +".;" 100 101 // keep together any sequence of contiguous words and numbers 102 // (including just one of either), plus an optional trailing 103 // number-suffix character 104 +"$word?($number$word)*($number$post_num?)?;" 105 106 // keep together and sequence of contiguous words and numbers 107 // that starts with a number-prefix character and a number, 108 // and may end with a number-suffix character 109 +"$pre_num($number$word)*($number$post_num?)?;" 110 111 // keep together runs of whitespace (optionally with a single trailing 112 // line separator or CRLF sequence) 113 +"$ws*\r?$ls?;" 114 115 // keep together runs of Katakana 116 +"$kata*;" 117 118 // keep together runs of Hiragana 119 +"$hira*;" 120 121 // keep together runs of Kanji 122 +"$kanji*;" }, 123 124 // These are the same line-breaking rules as are specified in the default 125 // resource, except that the Latin letters, apostrophe, and hyphen are 126 // specified as dictionary characters 127 { "LineBreakRules", 128 // ignore non-spacing marks, enclosing marks, and format characters 129 "$_ignore_=[[:Mn:][:Me:][:Cf:]];" 130 131 // lower and upper case Roman letters, apostrophy and dash 132 // are in the English dictionary 133 +"$_dictionary_=[a-zA-Z\\'\\-];" 134 135 // Hindi phrase separators 136 +"$danda=[\u0964\u0965];" 137 138 // characters that always cause a break: ETX, tab, LF, FF, LS, and PS 139 +"$break=[\u0003\t\n\f\u2028\u2029];" 140 141 // characters that always prevent a break: the non-breaking space 142 // and similar characters 143 +"$nbsp=[\u00a0\u2007\u2011\ufeff];" 144 145 // whitespace: space separators and control characters, except for 146 // CR and the other characters mentioned above 147 +"$space=[[[:Zs:][:Cc:]]-[$nbsp$break\r]];" 148 149 // dashes: dash punctuation and the discretionary hyphen, except for 150 // non-breaking hyphens 151 +"$dash=[[[:Pd:]\u00ad]-[$nbsp]];" 152 153 // characters that stick to a word if they precede it: currency symbols 154 // (except the cents sign) and starting punctuation 155 +"$pre_word=[[[:Sc:]-[\u00a2]][:Ps:]\\\"\\\'];" 156 157 // characters that stick to a word if they follow it: ending punctuation, 158 // other punctuation that usually occurs at the end of a sentence, 159 // small Kana characters, some CJK diacritics, etc. 160 +"$post_word=[[:Pe:]\\!\\\"\\\'\\%\\.\\,\\:\\;\\?\u00a2\u00b0\u066a\u2030-\u2034" 161 + "\u2103\u2105\u2109\u3001\u3002\u3005\u3041\u3043\u3045\u3047\u3049\u3063" 162 + "\u3083\u3085\u3087\u308e\u3099-\u309e\u30a1\u30a3\u30a5\u30a7\u30a9" 163 + "\u30c3\u30e3\u30e5\u30e7\u30ee\u30f5\u30f6\u30fc-\u30fe\uff01\uff0c" 164 + "\uff0e\uff1f];" 165 166 // Kanji: actually includes both Kanji and Kana, except for small Kana and 167 // CJK diacritics 168 +"$kanji=[[\u4e00-\u9fa5\uf900-\ufa2d\u3041-\u3094\u30a1-\u30fa]-[$post_word$_ignore_]];" 169 170 // digits 171 +"$digit=[[:Nd:][:No:]];" 172 173 // punctuation that can occur in the middle of a number: periods and commas 174 +"$mid_num=[\\.\\,];" 175 176 // everything not mentioned above, plus the quote marks (which are both 177 // <pre-word>, <post-word>, and <char>) 178 +"$char=[^$break$space$dash$kanji$nbsp$_ignore_$pre_word$post_word$mid_num$danda\r\\\"\\\'];" 179 180 // a "number" is a run of prefix characters and dashes, followed by one or 181 // more digits with isolated number-punctuation characters interspersed 182 +"$number=([$pre_word$dash]*$digit+($mid_num$digit+)*);" 183 184 // the basic core of a word can be either a "number" as defined above, a single 185 // "Kanji" character, or a run of any number of not-explicitly-mentioned 186 // characters (this includes Latin letters) 187 +"$word_core=([$pre_word$char]*|$kanji|$number);" 188 189 // a word may end with an optional suffix that be either a run of one or 190 // more dashes or a run of word-suffix characters, followed by an optional 191 // run of whitespace 192 +"$word_suffix=(($dash+|$post_word*)$space*);" 193 194 // a word, thus, is an optional run of word-prefix characters, followed by 195 // a word core and a word suffix (the syntax of <word-core> and <word-suffix> 196 // actually allows either of them to match the empty string, putting a break 197 // between things like ")(" or "aaa(aaa" 198 +"$word=($pre_word*$word_core$word_suffix);" 199 200 // finally, the rule that does the work: Keep together any run of words that 201 // are joined by runs of one of more non-spacing mark. Also keep a trailing 202 // line-break character or CRLF combination with the word. (line separators 203 // "win" over nbsp's) 204 +"$word($nbsp+$word)*\r?$break?;" }, 205 206 // these two resources specify the pathnames of the dictionary files to 207 // use for word breaking and line breaking. Both currently refer to 208 // a file called english.dict placed in com.ibm.icu.impl.data 209 // somewhere in the class path. It's important to note that 210 // english.dict was created for testing purposes only, and doesn't 211 // come anywhere close to being an exhaustive dictionary of English 212 // words (basically, it contains all the words in the Declaration of 213 // Independence, and the Revised Standard Version of the book of Genesis, 214 // plus a few other words thrown in to show more interesting cases). 215 // { "WordBreakDictionary", "com\\ibm\\text\\resources\\english.dict" }, 216 // { "LineBreakDictionary", "com\\ibm\\text\\resources\\english.dict" } 217 { "WordBreakDictionary", DATA_NAME }, 218 { "LineBreakDictionary", DATA_NAME } 219 }; 220 } 221} 222