1/*
2*******************************************************************************
3*   Copyright (C) 2001-2013, International Business Machines
4*   Corporation and others.  All Rights Reserved.
5*******************************************************************************
6*/
7
8package com.ibm.icu.dev.test.bidi;
9
10import java.util.Arrays;
11
12import com.ibm.icu.dev.test.TestFmwk;
13import com.ibm.icu.impl.Utility;
14import com.ibm.icu.lang.UCharacter;
15import com.ibm.icu.text.Bidi;
16import com.ibm.icu.text.BidiRun;
17import com.ibm.icu.util.VersionInfo;
18
19/**
20 * A base class for the Bidi test suite.
21 *
22 * @author Lina Kemmel, Matitiahu Allouche
23 */
24
25public class BidiTest extends TestFmwk {
26
27    protected static final char[] charFromDirProp = {
28         /* L      R    EN    ES    ET     AN    CS    B    S    WS    ON */
29         0x61, 0x5d0, 0x30, 0x2f, 0x25, 0x660, 0x2c, 0xa, 0x9, 0x20, 0x26,
30         /* LRE     LRO     AL     RLE     RLO     PDF    NSM      BN */
31         0x202a, 0x202d, 0x627, 0x202b, 0x202e, 0x202c, 0x308, 0x200c,
32         /* FSI     LRI     RLI     PDI */
33         0x2068, 0x2066, 0x2067, 0x2069  /* new in Unicode 6.3/ICU 52 */
34    };
35
36    static {
37        initCharFromDirProps();
38    }
39
40    private static void initCharFromDirProps() {
41        final VersionInfo ucd401 =  VersionInfo.getInstance(4, 0, 1, 0);
42        VersionInfo ucdVersion = VersionInfo.getInstance(0, 0, 0, 0);
43
44        /* lazy initialization */
45        if (ucdVersion.getMajor() > 0) {
46            return;
47
48        }
49        ucdVersion = UCharacter.getUnicodeVersion();
50        if (ucdVersion.compareTo(ucd401) >= 0) {
51            /* Unicode 4.0.1 changes bidi classes for +-/ */
52            /* change ES character from / to + */
53            charFromDirProp[TestData.ES] = 0x2b;
54        }
55    }
56
57    protected boolean assertEquals(String message, String expected, String actual,
58                                   String src, String mode, String option,
59                                   String level) {
60        if (expected == null || actual == null) {
61            return super.assertEquals(message, expected, actual);
62        }
63        if (expected.equals(actual)) {
64            return true;
65        }
66        errln("");
67        errcontln(message);
68        if (src != null) {
69            errcontln("source            : \"" + Utility.escape(src) + "\"");
70        }
71        errcontln("expected          : \"" + Utility.escape(expected) + "\"");
72        errcontln("actual            : \"" + Utility.escape(actual) + "\"");
73        if (mode != null) {
74            errcontln("reordering mode   : " + mode);
75        }
76        if (option != null) {
77            errcontln("reordering option : " + option);
78        }
79        if (level != null) {
80            errcontln("paragraph level   : " + level);
81        }
82        return false;
83    }
84
85    protected static String valueOf(int[] array) {
86        StringBuffer result = new StringBuffer(array.length * 4);
87        for (int i = 0; i < array.length; i++) {
88            result.append(' ');
89            result.append(array[i]);
90        }
91        return result.toString();
92    }
93
94    private static final String[] modeDescriptions = {
95        "REORDER_DEFAULT",
96        "REORDER_NUMBERS_SPECIAL",
97        "REORDER_GROUP_NUMBERS_WITH_R",
98        "REORDER_RUNS_ONLY",
99        "REORDER_INVERSE_NUMBERS_AS_L",
100        "REORDER_INVERSE_LIKE_DIRECT",
101        "REORDER_INVERSE_FOR_NUMBERS_SPECIAL"
102    };
103
104    protected static String modeToString(int mode) {
105        if (mode < Bidi.REORDER_DEFAULT ||
106            mode > Bidi.REORDER_INVERSE_FOR_NUMBERS_SPECIAL) {
107            return "INVALID";
108        }
109        return modeDescriptions[mode];
110    }
111
112    private static final short SETPARA_MASK = Bidi.OPTION_INSERT_MARKS |
113        Bidi.OPTION_REMOVE_CONTROLS | Bidi.OPTION_STREAMING;
114
115    private static final String[] setParaDescriptions = {
116        "OPTION_INSERT_MARKS",
117        "OPTION_REMOVE_CONTROLS",
118        "OPTION_STREAMING"
119    };
120
121    protected static String spOptionsToString(int option) {
122        return optionToString(option, SETPARA_MASK, setParaDescriptions);
123    }
124
125    private static final int MAX_WRITE_REORDERED_OPTION = Bidi.OUTPUT_REVERSE;
126    private static final int REORDER_MASK = (MAX_WRITE_REORDERED_OPTION << 1) - 1;
127
128    private static final String[] writeReorderedDescriptions = {
129        "KEEP_BASE_COMBINING",      //  1
130        "DO_MIRRORING",             //  2
131        "INSERT_LRM_FOR_NUMERIC",   //  4
132        "REMOVE_BIDI_CONTROLS",     //  8
133        "OUTPUT_REVERSE"            // 16
134    };
135
136    public static String wrOptionsToString(int option) {
137        return optionToString(option, REORDER_MASK, writeReorderedDescriptions);
138    }
139    public static String optionToString(int option, int mask,
140                                        String[] descriptions) {
141        StringBuffer desc = new StringBuffer(50);
142
143        if ((option &= mask) == 0) {
144            return "0";
145        }
146        desc.setLength(0);
147
148        for (int i = 0; option > 0; i++, option >>= 1) {
149            if ((option & 1) != 0) {
150                if (desc.length() > 0) {
151                    desc.append(" | ");
152                }
153                desc.append(descriptions[i]);
154            }
155        }
156        return desc.toString();
157    }
158
159    static final String columnString =
160        "0123456789ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz";
161    static final char[] columns = columnString.toCharArray();
162    private static final int TABLE_SIZE = 256;
163    private static boolean tablesInitialized = false;
164    private static char[] pseudoToUChar;
165    private static char[] UCharToPseudo;    /* used for Unicode chars < 0x0100 */
166    private static char[] UCharToPseud2;    /* used for Unicode chars >=0x0100 */
167
168    static void buildPseudoTables()
169    /*
170        The rules for pseudo-Bidi are as follows:
171        - [ == LRE
172        - ] == RLE
173        - { == LRO
174        - } == RLO
175        - ^ == PDF
176        - @ == LRM
177        - & == RLM
178        - A-F == Arabic Letters 0631-0636
179        - G-V == Hebrew letters 05d7-05ea
180        - W-Z == Unassigned RTL 08d0-08d3
181        - 0-5 == western digits 0030-0035
182        - 6-9 == Arabic-Indic digits 0666-0669
183        - ` == Combining Grave Accent 0300 (NSM)
184        - ~ == Delete 007f (BN)
185        - | == Paragraph Separator 2029 (B)
186        - _ == Info Separator 1 001f (S)
187        All other characters represent themselves as Latin-1, with the corresponding
188        Bidi properties.
189    */
190    {
191        int     i;
192        char    uchar;
193        char    c;
194
195        /* initialize all tables to unknown */
196        pseudoToUChar = new char[TABLE_SIZE];
197        UCharToPseudo = new char[TABLE_SIZE];
198        UCharToPseud2 = new char[TABLE_SIZE];
199        for (i = 0; i < TABLE_SIZE; i++) {
200            pseudoToUChar[i] = 0xFFFD;
201            UCharToPseudo[i] = '?';
202            UCharToPseud2[i] = '?';
203        }
204        /* initialize non letters or digits */
205        pseudoToUChar[ 0 ] = 0x0000;    UCharToPseudo[0x00] =  0 ;
206        pseudoToUChar[' '] = 0x0020;    UCharToPseudo[0x20] = ' ';
207        pseudoToUChar['!'] = 0x0021;    UCharToPseudo[0x21] = '!';
208        pseudoToUChar['"'] = 0x0022;    UCharToPseudo[0x22] = '"';
209        pseudoToUChar['#'] = 0x0023;    UCharToPseudo[0x23] = '#';
210        pseudoToUChar['$'] = 0x0024;    UCharToPseudo[0x24] = '$';
211        pseudoToUChar['%'] = 0x0025;    UCharToPseudo[0x25] = '%';
212        pseudoToUChar['\'']= 0x0027;    UCharToPseudo[0x27] = '\'';
213        pseudoToUChar['('] = 0x0028;    UCharToPseudo[0x28] = '(';
214        pseudoToUChar[')'] = 0x0029;    UCharToPseudo[0x29] = ')';
215        pseudoToUChar['*'] = 0x002A;    UCharToPseudo[0x2A] = '*';
216        pseudoToUChar['+'] = 0x002B;    UCharToPseudo[0x2B] = '+';
217        pseudoToUChar[','] = 0x002C;    UCharToPseudo[0x2C] = ',';
218        pseudoToUChar['-'] = 0x002D;    UCharToPseudo[0x2D] = '-';
219        pseudoToUChar['.'] = 0x002E;    UCharToPseudo[0x2E] = '.';
220        pseudoToUChar['/'] = 0x002F;    UCharToPseudo[0x2F] = '/';
221        pseudoToUChar[':'] = 0x003A;    UCharToPseudo[0x3A] = ':';
222        pseudoToUChar[';'] = 0x003B;    UCharToPseudo[0x3B] = ';';
223        pseudoToUChar['<'] = 0x003C;    UCharToPseudo[0x3C] = '<';
224        pseudoToUChar['='] = 0x003D;    UCharToPseudo[0x3D] = '=';
225        pseudoToUChar['>'] = 0x003E;    UCharToPseudo[0x3E] = '>';
226        pseudoToUChar['?'] = 0x003F;    UCharToPseudo[0x3F] = '?';
227        pseudoToUChar['\\']= 0x005C;    UCharToPseudo[0x5C] = '\\';
228        /* initialize specially used characters */
229        pseudoToUChar['`'] = 0x0300;    UCharToPseud2[0x00] = '`';  /* NSM */
230        pseudoToUChar['@'] = 0x200E;    UCharToPseud2[0x0E] = '@';  /* LRM */
231        pseudoToUChar['&'] = 0x200F;    UCharToPseud2[0x0F] = '&';  /* RLM */
232        pseudoToUChar['_'] = 0x001F;    UCharToPseudo[0x1F] = '_';  /* S   */
233        pseudoToUChar['|'] = 0x2029;    UCharToPseud2[0x29] = '|';  /* B   */
234        pseudoToUChar['['] = 0x202A;    UCharToPseud2[0x2A] = '[';  /* LRE */
235        pseudoToUChar[']'] = 0x202B;    UCharToPseud2[0x2B] = ']';  /* RLE */
236        pseudoToUChar['^'] = 0x202C;    UCharToPseud2[0x2C] = '^';  /* PDF */
237        pseudoToUChar['{'] = 0x202D;    UCharToPseud2[0x2D] = '{';  /* LRO */
238        pseudoToUChar['}'] = 0x202E;    UCharToPseud2[0x2E] = '}';  /* RLO */
239        pseudoToUChar['~'] = 0x007F;    UCharToPseudo[0x7F] = '~';  /* BN  */
240        /* initialize western digits */
241        for (i = 0, uchar = 0x0030; i < 6; i++, uchar++) {
242            c = columns[i];
243            pseudoToUChar[c] = uchar;
244            UCharToPseudo[uchar & 0x00ff] = c;
245        }
246        /* initialize Hindi digits */
247        for (i = 6, uchar = 0x0666; i < 10; i++, uchar++) {
248            c = columns[i];
249            pseudoToUChar[c] = uchar;
250            UCharToPseud2[uchar & 0x00ff] = c;
251        }
252        /* initialize Arabic letters */
253        for (i = 10, uchar = 0x0631; i < 16; i++, uchar++) {
254            c = columns[i];
255            pseudoToUChar[c] = uchar;
256            UCharToPseud2[uchar & 0x00ff] = c;
257        }
258        /* initialize Hebrew letters */
259        for (i = 16, uchar = 0x05D7; i < 32; i++, uchar++) {
260            c = columns[i];
261            pseudoToUChar[c] = uchar;
262            UCharToPseud2[uchar & 0x00ff] = c;
263        }
264        /* initialize Unassigned code points */
265        for (i = 32, uchar = 0x08D0; i < 36; i++, uchar++) {
266            c = columns[i];
267            pseudoToUChar[c] = uchar;
268            UCharToPseud2[uchar & 0x00ff] = c;
269        }
270        /* initialize Latin lower case letters */
271        for (i = 36, uchar = 0x0061; i < 62; i++, uchar++) {
272            c = columns[i];
273            pseudoToUChar[c] = uchar;
274            UCharToPseudo[uchar & 0x00ff] = c;
275        }
276        tablesInitialized = true;
277    }
278
279    /*----------------------------------------------------------------------*/
280
281    static String pseudoToU16(String input)
282    /*  This function converts a pseudo-Bidi string into a char string.
283        It returns the char string.
284    */
285    {
286        int len = input.length();
287        char[] output = new char[len];
288        int i;
289        if (!tablesInitialized) {
290            buildPseudoTables();
291        }
292        for (i = 0; i < len; i++)
293            output[i] = pseudoToUChar[input.charAt(i)];
294        return new String(output);
295    }
296
297    /*----------------------------------------------------------------------*/
298
299    static String u16ToPseudo(String input)
300    /*  This function converts a char string into a pseudo-Bidi string.
301        It returns the pseudo-Bidi string.
302    */
303    {
304        int len = input.length();
305        char[] output = new char[len];
306        int i;
307        char uchar;
308        if (!tablesInitialized) {
309            buildPseudoTables();
310        }
311        for (i = 0; i < len; i++)
312        {
313            uchar = input.charAt(i);
314            output[i] = uchar < 0x0100 ? UCharToPseudo[uchar] :
315                                         UCharToPseud2[uchar & 0x00ff];
316        }
317        return new String(output);
318    }
319
320    void errcont(String message) {
321        msg(message, ERR, false, false);
322    }
323
324    void errcontln(String message) {
325        msg(message, ERR, false, true);
326    }
327
328    void printCaseInfo(Bidi bidi, String src, String dst)
329    {
330        int length = bidi.getProcessedLength();
331        byte[] levels = bidi.getLevels();
332        char[] levelChars  = new char[length];
333        byte lev;
334        int runCount = bidi.countRuns();
335        errcontln("========================================");
336        errcontln("Processed length: " + length);
337        for (int i = 0; i < length; i++) {
338            lev = levels[i];
339            if (lev < 0) {
340                levelChars[i] = '-';
341            } else if (lev < columns.length) {
342                levelChars[i] = columns[lev];
343            } else {
344                levelChars[i] = '+';
345            }
346        }
347        errcontln("Levels: " + new String(levelChars));
348        errcontln("Source: " + src);
349        errcontln("Result: " + dst);
350        errcontln("Direction: " + bidi.getDirection());
351        errcontln("paraLevel: " + Byte.toString(bidi.getParaLevel()));
352        errcontln("reorderingMode: " + modeToString(bidi.getReorderingMode()));
353        errcontln("reorderingOptions: " + spOptionsToString(bidi.getReorderingOptions()));
354        errcont("Runs: " + runCount + " => logicalStart.length/level: ");
355        for (int i = 0; i < runCount; i++) {
356            BidiRun run;
357            run = bidi.getVisualRun(i);
358            errcont(" " + run.getStart() + "." + run.getLength() + "/" +
359                    run.getEmbeddingLevel());
360        }
361        errcont("\n");
362    }
363
364    static final String mates1 = "<>()[]{}";
365    static final String mates2 = "><)(][}{";
366    static final char[] mates1Chars = mates1.toCharArray();
367    static final char[] mates2Chars = mates2.toCharArray();
368
369    boolean matchingPair(Bidi bidi, int i, char c1, char c2)
370    {
371        if (c1 == c2) {
372            return true;
373        }
374        /* For REORDER_RUNS_ONLY, it would not be correct to check levels[i],
375           so we use the appropriate run's level, which is good for all cases.
376         */
377        if (bidi.getLogicalRun(i).getDirection() == 0) {
378            return false;
379        }
380        for (int k = 0; k < mates1Chars.length; k++) {
381            if ((c1 == mates1Chars[k]) && (c2 == mates2Chars[k])) {
382                return true;
383            }
384        }
385        return false;
386    }
387
388    boolean checkWhatYouCan(Bidi bidi, String src, String dst)
389    {
390        int i, idx, logLimit, visLimit;
391        boolean testOK, errMap, errDst;
392        char[] srcChars = src.toCharArray();
393        char[] dstChars = dst.toCharArray();
394        int[] visMap = bidi.getVisualMap();
395        int[] logMap = bidi.getLogicalMap();
396
397        testOK = true;
398        errMap = errDst = false;
399        logLimit = bidi.getProcessedLength();
400        visLimit = bidi.getResultLength();
401        if (visLimit > dstChars.length) {
402            visLimit = dstChars.length;
403        }
404        char[] accumSrc = new char[logLimit];
405        char[] accumDst = new char[visLimit];
406        Arrays.fill(accumSrc, '?');
407        Arrays.fill(accumDst, '?');
408
409        if (logMap.length != logLimit) {
410            errMap = true;
411        }
412        for (i = 0; i < logLimit; i++) {
413            idx = bidi.getVisualIndex(i);
414            if (idx != logMap[i]) {
415                errMap = true;
416            }
417            if (idx == Bidi.MAP_NOWHERE) {
418                continue;
419            }
420            if (idx >= visLimit) {
421                continue;
422            }
423            accumDst[idx] = srcChars[i];
424            if (!matchingPair(bidi, i, srcChars[i], dstChars[idx])) {
425                errDst = true;
426            }
427        }
428        if (errMap) {
429            if (testOK) {
430                printCaseInfo(bidi, src, dst);
431                testOK = false;
432            }
433            errln("Mismatch between getLogicalMap() and getVisualIndex()");
434            errcont("Map    :" + valueOf(logMap));
435            errcont("\n");
436            errcont("Indexes:");
437            for (i = 0; i < logLimit; i++) {
438                errcont(" " + bidi.getVisualIndex(i));
439            }
440            errcont("\n");
441        }
442        if (errDst) {
443            if (testOK) {
444                printCaseInfo(bidi, src, dst);
445                testOK = false;
446            }
447            errln("Source does not map to Result");
448            errcontln("We got: " + new String(accumDst));
449        }
450
451        errMap = errDst = false;
452        if (visMap.length != visLimit) {
453            errMap = true;
454        }
455        for (i = 0; i < visLimit; i++) {
456            idx = bidi.getLogicalIndex(i);
457            if (idx != visMap[i]) {
458                errMap = true;
459            }
460            if (idx == Bidi.MAP_NOWHERE) {
461                continue;
462            }
463            if (idx >= logLimit) {
464                continue;
465            }
466            accumSrc[idx] = dstChars[i];
467            if (!matchingPair(bidi, idx, srcChars[idx], dstChars[i])) {
468                errDst = true;
469            }
470        }
471        if (errMap) {
472            if (testOK) {
473                printCaseInfo(bidi, src, dst);
474                testOK = false;
475            }
476            errln("Mismatch between getVisualMap() and getLogicalIndex()");
477            errcont("Map    :" + valueOf(visMap));
478            errcont("\n");
479            errcont("Indexes:");
480            for (i = 0; i < visLimit; i++) {
481                errcont(" " + bidi.getLogicalIndex(i));
482            }
483            errcont("\n");
484        }
485        if (errDst) {
486            if (testOK) {
487                printCaseInfo(bidi, src, dst);
488                testOK = false;
489            }
490            errln("Result does not map to Source");
491            errcontln("We got: " + new String(accumSrc));
492        }
493        return testOK;
494    }
495
496}
497