1// © 2016 and later: Unicode, Inc. and others.
2// License & terms of use: http://www.unicode.org/copyright.html#License
3/*
4 *******************************************************************************
5 * Copyright (C) 1996-2004, International Business Machines Corporation and    *
6 * others. All Rights Reserved.                                                *
7 *******************************************************************************
8 */
9package com.ibm.icu.dev.test.rbbi;
10
11import java.util.ListResourceBundle;
12
13/**
14 * This resource bundle is included for testing and demonstration purposes only.
15 * It applies the dictionary-based algorithm to English text that has had all the
16 * spaces removed.  Once we have good test cases for Thai, we will replace this
17 * with good resource data (and a good dictionary file) for Thai
18 */
19public class BreakIteratorRules_en_US_TEST extends ListResourceBundle {
20    private static final String DATA_NAME = "/com/ibm/icu/dev/data/rbbi/english.dict";
21
22    // calling code will handle case where dictionary does not exist
23
24    public Object[][] getContents() {
25        return new Object[][] {
26            // names of classes to instantiate for the different kinds of break
27            // iterator.  Notice we're now using DictionaryBasedBreakIterator
28            // for word and line breaking.
29            {   "BreakIteratorClasses",
30                new String[] {
31                    "RuleBasedBreakIterator",
32                    // character-break iterator class
33                    "DictionaryBasedBreakIterator",
34                    // word-break iterator class
35                    "DictionaryBasedBreakIterator",
36                    // line-break iterator class
37                    "RuleBasedBreakIterator" } // sentence-break iterator class
38            },
39
40            // These are the same word-breaking rules as are specified in the default
41            // resource, except that the Latin letters, apostrophe, and hyphen are
42            // specified as dictionary characters
43            {
44                "WordBreakRules",
45                // ignore non-spacing marks, enclosing marks, and format characters,
46                // all of which should not influence the algorithm
47                "$_ignore_=[[:Mn:][:Me:][:Cf:]];"
48
49                // lower and upper case Roman letters, apostrophy and dash are
50                // in the English dictionary
51                +"$_dictionary_=[a-zA-Z\\'\\-];"
52
53                // Hindi phrase separator, kanji, katakana, hiragana, CJK diacriticals,
54                // other letters, and digits
55                +"$danda=[\u0964\u0965];"
56                    + "$kanji=[\u3005\u4e00-\u9fa5\uf900-\ufa2d];"
57                    + "$kata=[\u3099-\u309c\u30a1-\u30fe];"
58                    + "$hira=[\u3041-\u309e\u30fc];"
59                    + "$let=[[[:L:][:Mc:]]-[$kanji$kata$hira]];"
60                    + "$dgt=[:N:];"
61
62                // punctuation that can occur in the middle of a word: currently
63                // dashes, apostrophes, and quotation marks
64                +"$mid_word=[[:Pd:]\u00ad\u2027\\\"\\\'];"
65
66                // punctuation that can occur in the middle of a number: currently
67                // apostrophes, qoutation marks, periods, commas, and the Arabic
68                // decimal point
69                +"$mid_num=[\\\"\\\'\\,\u066b\\.];"
70
71                // punctuation that can occur at the beginning of a number: currently
72                // the period, the number sign, and all currency symbols except the cents sign
73                +"$pre_num=[[[:Sc:]-[\u00a2]]\\#\\.];"
74
75                // punctuation that can occur at the end of a number: currently
76                // the percent, per-thousand, per-ten-thousand, and Arabic percent
77                // signs, the cents sign, and the ampersand
78                +"$post_num=[\\%\\&\u00a2\u066a\u2030\u2031];"
79
80                // line separators: currently LF, FF, PS, and LS
81                +"$ls=[\n\u000c\u2028\u2029];"
82
83                // whitespace: all space separators and the tab character
84                +"$ws=[[:Zs:]\t];"
85
86                // a word is a sequence of letters that may contain internal
87                // punctuation, as long as it begins and ends with a letter and
88                // never contains two punctuation marks in a row
89                +"$word=($let+($mid_word$let+)*$danda?);"
90
91                // a number is a sequence of digits that may contain internal
92                // punctuation, as long as it begins and ends with a digit and
93                // never contains two punctuation marks in a row.
94                +"$number=($dgt+($mid_num$dgt+)*);"
95
96                // break after every character, with the following exceptions
97                // (this will cause punctuation marks that aren't considered
98                // part of words or numbers to be treated as words unto themselves)
99                +".;"
100
101                // keep together any sequence of contiguous words and numbers
102                // (including just one of either), plus an optional trailing
103                // number-suffix character
104                +"$word?($number$word)*($number$post_num?)?;"
105
106                // keep together and sequence of contiguous words and numbers
107                // that starts with a number-prefix character and a number,
108                // and may end with a number-suffix character
109                +"$pre_num($number$word)*($number$post_num?)?;"
110
111                // keep together runs of whitespace (optionally with a single trailing
112                // line separator or CRLF sequence)
113                +"$ws*\r?$ls?;"
114
115                // keep together runs of Katakana
116                +"$kata*;"
117
118                // keep together runs of Hiragana
119                +"$hira*;"
120
121                // keep together runs of Kanji
122                +"$kanji*;" },
123
124            // These are the same line-breaking rules as are specified in the default
125            // resource, except that the Latin letters, apostrophe, and hyphen are
126            // specified as dictionary characters
127            {   "LineBreakRules",
128                // ignore non-spacing marks, enclosing marks, and format characters
129                "$_ignore_=[[:Mn:][:Me:][:Cf:]];"
130
131                // lower and upper case Roman letters, apostrophy and dash
132                // are in the English dictionary
133                +"$_dictionary_=[a-zA-Z\\'\\-];"
134
135                // Hindi phrase separators
136                +"$danda=[\u0964\u0965];"
137
138                // characters that always cause a break: ETX, tab, LF, FF, LS, and PS
139                +"$break=[\u0003\t\n\f\u2028\u2029];"
140
141                // characters that always prevent a break: the non-breaking space
142                // and similar characters
143                +"$nbsp=[\u00a0\u2007\u2011\ufeff];"
144
145                // whitespace: space separators and control characters, except for
146                // CR and the other characters mentioned above
147                +"$space=[[[:Zs:][:Cc:]]-[$nbsp$break\r]];"
148
149                // dashes: dash punctuation and the discretionary hyphen, except for
150                // non-breaking hyphens
151                +"$dash=[[[:Pd:]\u00ad]-[$nbsp]];"
152
153                // characters that stick to a word if they precede it: currency symbols
154                // (except the cents sign) and starting punctuation
155                +"$pre_word=[[[:Sc:]-[\u00a2]][:Ps:]\\\"\\\'];"
156
157                // characters that stick to a word if they follow it: ending punctuation,
158                // other punctuation that usually occurs at the end of a sentence,
159                // small Kana characters, some CJK diacritics, etc.
160                +"$post_word=[[:Pe:]\\!\\\"\\\'\\%\\.\\,\\:\\;\\?\u00a2\u00b0\u066a\u2030-\u2034"
161                    + "\u2103\u2105\u2109\u3001\u3002\u3005\u3041\u3043\u3045\u3047\u3049\u3063"
162                    + "\u3083\u3085\u3087\u308e\u3099-\u309e\u30a1\u30a3\u30a5\u30a7\u30a9"
163                    + "\u30c3\u30e3\u30e5\u30e7\u30ee\u30f5\u30f6\u30fc-\u30fe\uff01\uff0c"
164                    + "\uff0e\uff1f];"
165
166                // Kanji: actually includes both Kanji and Kana, except for small Kana and
167                // CJK diacritics
168                +"$kanji=[[\u4e00-\u9fa5\uf900-\ufa2d\u3041-\u3094\u30a1-\u30fa]-[$post_word$_ignore_]];"
169
170                // digits
171                +"$digit=[[:Nd:][:No:]];"
172
173                // punctuation that can occur in the middle of a number: periods and commas
174                +"$mid_num=[\\.\\,];"
175
176                // everything not mentioned above, plus the quote marks (which are both
177                // <pre-word>, <post-word>, and <char>)
178                +"$char=[^$break$space$dash$kanji$nbsp$_ignore_$pre_word$post_word$mid_num$danda\r\\\"\\\'];"
179
180                // a "number" is a run of prefix characters and dashes, followed by one or
181                // more digits with isolated number-punctuation characters interspersed
182                +"$number=([$pre_word$dash]*$digit+($mid_num$digit+)*);"
183
184                // the basic core of a word can be either a "number" as defined above, a single
185                // "Kanji" character, or a run of any number of not-explicitly-mentioned
186                // characters (this includes Latin letters)
187                +"$word_core=([$pre_word$char]*|$kanji|$number);"
188
189                // a word may end with an optional suffix that be either a run of one or
190                // more dashes or a run of word-suffix characters, followed by an optional
191                // run of whitespace
192                +"$word_suffix=(($dash+|$post_word*)$space*);"
193
194                // a word, thus, is an optional run of word-prefix characters, followed by
195                // a word core and a word suffix (the syntax of <word-core> and <word-suffix>
196                // actually allows either of them to match the empty string, putting a break
197                // between things like ")(" or "aaa(aaa"
198                +"$word=($pre_word*$word_core$word_suffix);"
199
200                // finally, the rule that does the work: Keep together any run of words that
201                // are joined by runs of one of more non-spacing mark.  Also keep a trailing
202                // line-break character or CRLF combination with the word.  (line separators
203                // "win" over nbsp's)
204                +"$word($nbsp+$word)*\r?$break?;" },
205
206            // these two resources specify the pathnames of the dictionary files to
207            // use for word breaking and line breaking.  Both currently refer to
208            // a file called english.dict placed in com.ibm.icu.impl.data
209            // somewhere in the class path.  It's important to note that
210            // english.dict was created for testing purposes only, and doesn't
211            // come anywhere close to being an exhaustive dictionary of English
212            // words (basically, it contains all the words in the Declaration of
213            // Independence, and the Revised Standard Version of the book of Genesis,
214            // plus a few other words thrown in to show more interesting cases).
215            // { "WordBreakDictionary", "com\\ibm\\text\\resources\\english.dict" },
216            // { "LineBreakDictionary", "com\\ibm\\text\\resources\\english.dict" }
217            {   "WordBreakDictionary", DATA_NAME },
218            {   "LineBreakDictionary", DATA_NAME }
219        };
220    }
221}
222