WordBreakerTests.cpp revision c88ef135fcc2661ec7addc171ebc60787df38aff
157b6dae9894b9362ef04517ff477fd491f9d433bRaph Levien/* 257b6dae9894b9362ef04517ff477fd491f9d433bRaph Levien * Copyright (C) 2015 The Android Open Source Project 357b6dae9894b9362ef04517ff477fd491f9d433bRaph Levien * 457b6dae9894b9362ef04517ff477fd491f9d433bRaph Levien * Licensed under the Apache License, Version 2.0 (the "License"); 557b6dae9894b9362ef04517ff477fd491f9d433bRaph Levien * you may not use this file except in compliance with the License. 657b6dae9894b9362ef04517ff477fd491f9d433bRaph Levien * You may obtain a copy of the License at 757b6dae9894b9362ef04517ff477fd491f9d433bRaph Levien * 857b6dae9894b9362ef04517ff477fd491f9d433bRaph Levien * http://www.apache.org/licenses/LICENSE-2.0 957b6dae9894b9362ef04517ff477fd491f9d433bRaph Levien * 1057b6dae9894b9362ef04517ff477fd491f9d433bRaph Levien * Unless required by applicable law or agreed to in writing, software 1157b6dae9894b9362ef04517ff477fd491f9d433bRaph Levien * distributed under the License is distributed on an "AS IS" BASIS, 1257b6dae9894b9362ef04517ff477fd491f9d433bRaph Levien * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 1357b6dae9894b9362ef04517ff477fd491f9d433bRaph Levien * See the License for the specific language governing permissions and 1457b6dae9894b9362ef04517ff477fd491f9d433bRaph Levien * limitations under the License. 1557b6dae9894b9362ef04517ff477fd491f9d433bRaph Levien */ 1657b6dae9894b9362ef04517ff477fd491f9d433bRaph Levien 1757b6dae9894b9362ef04517ff477fd491f9d433bRaph Levien#include <gtest/gtest.h> 1857b6dae9894b9362ef04517ff477fd491f9d433bRaph Levien#include "ICUTestBase.h" 1957b6dae9894b9362ef04517ff477fd491f9d433bRaph Levien#include "UnicodeUtils.h" 2057b6dae9894b9362ef04517ff477fd491f9d433bRaph Levien#include <minikin/WordBreaker.h> 2157b6dae9894b9362ef04517ff477fd491f9d433bRaph Levien#include <unicode/locid.h> 2257b6dae9894b9362ef04517ff477fd491f9d433bRaph Levien#include <unicode/uclean.h> 2357b6dae9894b9362ef04517ff477fd491f9d433bRaph Levien#include <unicode/udata.h> 2457b6dae9894b9362ef04517ff477fd491f9d433bRaph Levien 2557b6dae9894b9362ef04517ff477fd491f9d433bRaph Levien#define LOG_TAG "Minikin" 2657b6dae9894b9362ef04517ff477fd491f9d433bRaph Levien#include <cutils/log.h> 2757b6dae9894b9362ef04517ff477fd491f9d433bRaph Levien 2857b6dae9894b9362ef04517ff477fd491f9d433bRaph Levien#ifndef NELEM 2957b6dae9894b9362ef04517ff477fd491f9d433bRaph Levien#define NELEM(x) ((sizeof(x) / sizeof((x)[0]))) 3057b6dae9894b9362ef04517ff477fd491f9d433bRaph Levien#endif 3157b6dae9894b9362ef04517ff477fd491f9d433bRaph Levien 3257b6dae9894b9362ef04517ff477fd491f9d433bRaph Levienusing namespace android; 3357b6dae9894b9362ef04517ff477fd491f9d433bRaph Levien 3457b6dae9894b9362ef04517ff477fd491f9d433bRaph Levientypedef ICUTestBase WordBreakerTest; 3557b6dae9894b9362ef04517ff477fd491f9d433bRaph Levien 3657b6dae9894b9362ef04517ff477fd491f9d433bRaph LevienTEST_F(WordBreakerTest, basic) { 3757b6dae9894b9362ef04517ff477fd491f9d433bRaph Levien uint16_t buf[] = {'h', 'e', 'l', 'l' ,'o', ' ', 'w', 'o', 'r', 'l', 'd'}; 3857b6dae9894b9362ef04517ff477fd491f9d433bRaph Levien WordBreaker breaker; 3957b6dae9894b9362ef04517ff477fd491f9d433bRaph Levien breaker.setLocale(icu::Locale::getEnglish()); 4057b6dae9894b9362ef04517ff477fd491f9d433bRaph Levien breaker.setText(buf, NELEM(buf)); 4157b6dae9894b9362ef04517ff477fd491f9d433bRaph Levien EXPECT_EQ(0, breaker.current()); 4257b6dae9894b9362ef04517ff477fd491f9d433bRaph Levien EXPECT_EQ(6, breaker.next()); // after "hello " 4357b6dae9894b9362ef04517ff477fd491f9d433bRaph Levien EXPECT_EQ(0, breaker.wordStart()); // "hello" 4457b6dae9894b9362ef04517ff477fd491f9d433bRaph Levien EXPECT_EQ(5, breaker.wordEnd()); 45c88ef135fcc2661ec7addc171ebc60787df38affRaph Levien EXPECT_EQ(0, breaker.breakBadness()); 4657b6dae9894b9362ef04517ff477fd491f9d433bRaph Levien EXPECT_EQ(6, breaker.current()); 4757b6dae9894b9362ef04517ff477fd491f9d433bRaph Levien EXPECT_EQ((ssize_t)NELEM(buf), breaker.next()); // end 4857b6dae9894b9362ef04517ff477fd491f9d433bRaph Levien EXPECT_EQ(6, breaker.wordStart()); // "world" 4957b6dae9894b9362ef04517ff477fd491f9d433bRaph Levien EXPECT_EQ(11, breaker.wordEnd()); 50c88ef135fcc2661ec7addc171ebc60787df38affRaph Levien EXPECT_EQ(0, breaker.breakBadness()); 5157b6dae9894b9362ef04517ff477fd491f9d433bRaph Levien EXPECT_EQ(11, breaker.current()); 5257b6dae9894b9362ef04517ff477fd491f9d433bRaph Levien} 5357b6dae9894b9362ef04517ff477fd491f9d433bRaph Levien 5457b6dae9894b9362ef04517ff477fd491f9d433bRaph LevienTEST_F(WordBreakerTest, softHyphen) { 5557b6dae9894b9362ef04517ff477fd491f9d433bRaph Levien uint16_t buf[] = {'h', 'e', 'l', 0x00AD, 'l' ,'o', ' ', 'w', 'o', 'r', 'l', 'd'}; 5657b6dae9894b9362ef04517ff477fd491f9d433bRaph Levien WordBreaker breaker; 5757b6dae9894b9362ef04517ff477fd491f9d433bRaph Levien breaker.setLocale(icu::Locale::getEnglish()); 5857b6dae9894b9362ef04517ff477fd491f9d433bRaph Levien breaker.setText(buf, NELEM(buf)); 5957b6dae9894b9362ef04517ff477fd491f9d433bRaph Levien EXPECT_EQ(0, breaker.current()); 6057b6dae9894b9362ef04517ff477fd491f9d433bRaph Levien EXPECT_EQ(7, breaker.next()); // after "hel{SOFT HYPHEN}lo " 6157b6dae9894b9362ef04517ff477fd491f9d433bRaph Levien EXPECT_EQ(0, breaker.wordStart()); // "hel{SOFT HYPHEN}lo" 6257b6dae9894b9362ef04517ff477fd491f9d433bRaph Levien EXPECT_EQ(6, breaker.wordEnd()); 63c88ef135fcc2661ec7addc171ebc60787df38affRaph Levien EXPECT_EQ(0, breaker.breakBadness()); 6457b6dae9894b9362ef04517ff477fd491f9d433bRaph Levien EXPECT_EQ((ssize_t)NELEM(buf), breaker.next()); // end 6557b6dae9894b9362ef04517ff477fd491f9d433bRaph Levien EXPECT_EQ(7, breaker.wordStart()); // "world" 6657b6dae9894b9362ef04517ff477fd491f9d433bRaph Levien EXPECT_EQ(12, breaker.wordEnd()); 67c88ef135fcc2661ec7addc171ebc60787df38affRaph Levien EXPECT_EQ(0, breaker.breakBadness()); 6857b6dae9894b9362ef04517ff477fd491f9d433bRaph Levien} 6957b6dae9894b9362ef04517ff477fd491f9d433bRaph Levien 7057b6dae9894b9362ef04517ff477fd491f9d433bRaph LevienTEST_F(WordBreakerTest, punct) { 7157b6dae9894b9362ef04517ff477fd491f9d433bRaph Levien uint16_t buf[] = {0x00A1, 0x00A1, 'h', 'e', 'l', 'l' ,'o', ',', ' ', 'w', 'o', 'r', 'l', 'd', 7257b6dae9894b9362ef04517ff477fd491f9d433bRaph Levien '!', '!'}; 7357b6dae9894b9362ef04517ff477fd491f9d433bRaph Levien WordBreaker breaker; 7457b6dae9894b9362ef04517ff477fd491f9d433bRaph Levien breaker.setLocale(icu::Locale::getEnglish()); 7557b6dae9894b9362ef04517ff477fd491f9d433bRaph Levien breaker.setText(buf, NELEM(buf)); 7657b6dae9894b9362ef04517ff477fd491f9d433bRaph Levien EXPECT_EQ(0, breaker.current()); 7757b6dae9894b9362ef04517ff477fd491f9d433bRaph Levien EXPECT_EQ(9, breaker.next()); // after "¡¡hello, " 7857b6dae9894b9362ef04517ff477fd491f9d433bRaph Levien EXPECT_EQ(2, breaker.wordStart()); // "hello" 7957b6dae9894b9362ef04517ff477fd491f9d433bRaph Levien EXPECT_EQ(7, breaker.wordEnd()); 80c88ef135fcc2661ec7addc171ebc60787df38affRaph Levien EXPECT_EQ(0, breaker.breakBadness()); 8157b6dae9894b9362ef04517ff477fd491f9d433bRaph Levien EXPECT_EQ((ssize_t)NELEM(buf), breaker.next()); // end 8257b6dae9894b9362ef04517ff477fd491f9d433bRaph Levien EXPECT_EQ(9, breaker.wordStart()); // "world" 8357b6dae9894b9362ef04517ff477fd491f9d433bRaph Levien EXPECT_EQ(14, breaker.wordEnd()); 84c88ef135fcc2661ec7addc171ebc60787df38affRaph Levien EXPECT_EQ(0, breaker.breakBadness()); 8557b6dae9894b9362ef04517ff477fd491f9d433bRaph Levien} 869c4cc648abcae144f3b99d612e58ef01d5e52cceRaph Levien 879c4cc648abcae144f3b99d612e58ef01d5e52cceRaph LevienTEST_F(WordBreakerTest, email) { 889c4cc648abcae144f3b99d612e58ef01d5e52cceRaph Levien uint16_t buf[] = {'f', 'o', 'o', '@', 'e', 'x', 'a', 'm', 'p', 'l', 'e', '.', 'c', 'o', 'm', 899c4cc648abcae144f3b99d612e58ef01d5e52cceRaph Levien ' ', 'x'}; 909c4cc648abcae144f3b99d612e58ef01d5e52cceRaph Levien WordBreaker breaker; 919c4cc648abcae144f3b99d612e58ef01d5e52cceRaph Levien breaker.setLocale(icu::Locale::getEnglish()); 929c4cc648abcae144f3b99d612e58ef01d5e52cceRaph Levien breaker.setText(buf, NELEM(buf)); 939c4cc648abcae144f3b99d612e58ef01d5e52cceRaph Levien EXPECT_EQ(0, breaker.current()); 946d15657e4a3826d4d47d5358f1dde211484527e9Raph Levien EXPECT_EQ(11, breaker.next()); // after "foo@example" 956d15657e4a3826d4d47d5358f1dde211484527e9Raph Levien EXPECT_TRUE(breaker.wordStart() >= breaker.wordEnd()); 96c88ef135fcc2661ec7addc171ebc60787df38affRaph Levien EXPECT_EQ(1, breaker.breakBadness()); 976d15657e4a3826d4d47d5358f1dde211484527e9Raph Levien EXPECT_EQ(16, breaker.next()); // after ".com " 989c4cc648abcae144f3b99d612e58ef01d5e52cceRaph Levien EXPECT_TRUE(breaker.wordStart() >= breaker.wordEnd()); 99c88ef135fcc2661ec7addc171ebc60787df38affRaph Levien EXPECT_EQ(0, breaker.breakBadness()); 1009c4cc648abcae144f3b99d612e58ef01d5e52cceRaph Levien EXPECT_EQ((ssize_t)NELEM(buf), breaker.next()); // end 1019c4cc648abcae144f3b99d612e58ef01d5e52cceRaph Levien EXPECT_EQ(16, breaker.wordStart()); // "x" 1029c4cc648abcae144f3b99d612e58ef01d5e52cceRaph Levien EXPECT_EQ(17, breaker.wordEnd()); 103c88ef135fcc2661ec7addc171ebc60787df38affRaph Levien EXPECT_EQ(0, breaker.breakBadness()); 1049c4cc648abcae144f3b99d612e58ef01d5e52cceRaph Levien} 1059c4cc648abcae144f3b99d612e58ef01d5e52cceRaph Levien 1069c4cc648abcae144f3b99d612e58ef01d5e52cceRaph LevienTEST_F(WordBreakerTest, mailto) { 1079c4cc648abcae144f3b99d612e58ef01d5e52cceRaph Levien uint16_t buf[] = {'m', 'a', 'i', 'l', 't', 'o', ':', 'f', 'o', 'o', '@', 1089c4cc648abcae144f3b99d612e58ef01d5e52cceRaph Levien 'e', 'x', 'a', 'm', 'p', 'l', 'e', '.', 'c', 'o', 'm', ' ', 'x'}; 1099c4cc648abcae144f3b99d612e58ef01d5e52cceRaph Levien WordBreaker breaker; 1109c4cc648abcae144f3b99d612e58ef01d5e52cceRaph Levien breaker.setLocale(icu::Locale::getEnglish()); 1119c4cc648abcae144f3b99d612e58ef01d5e52cceRaph Levien breaker.setText(buf, NELEM(buf)); 1129c4cc648abcae144f3b99d612e58ef01d5e52cceRaph Levien EXPECT_EQ(0, breaker.current()); 1136d15657e4a3826d4d47d5358f1dde211484527e9Raph Levien EXPECT_EQ(7, breaker.next()); // after "mailto:" 1146d15657e4a3826d4d47d5358f1dde211484527e9Raph Levien EXPECT_TRUE(breaker.wordStart() >= breaker.wordEnd()); 115c88ef135fcc2661ec7addc171ebc60787df38affRaph Levien EXPECT_EQ(1, breaker.breakBadness()); 1166d15657e4a3826d4d47d5358f1dde211484527e9Raph Levien EXPECT_EQ(18, breaker.next()); // after "foo@example" 1176d15657e4a3826d4d47d5358f1dde211484527e9Raph Levien EXPECT_TRUE(breaker.wordStart() >= breaker.wordEnd()); 118c88ef135fcc2661ec7addc171ebc60787df38affRaph Levien EXPECT_EQ(1, breaker.breakBadness()); 1196d15657e4a3826d4d47d5358f1dde211484527e9Raph Levien EXPECT_EQ(23, breaker.next()); // after ".com " 1209c4cc648abcae144f3b99d612e58ef01d5e52cceRaph Levien EXPECT_TRUE(breaker.wordStart() >= breaker.wordEnd()); 121c88ef135fcc2661ec7addc171ebc60787df38affRaph Levien EXPECT_EQ(0, breaker.breakBadness()); 1229c4cc648abcae144f3b99d612e58ef01d5e52cceRaph Levien EXPECT_EQ((ssize_t)NELEM(buf), breaker.next()); // end 1239c4cc648abcae144f3b99d612e58ef01d5e52cceRaph Levien EXPECT_EQ(23, breaker.wordStart()); // "x" 1249c4cc648abcae144f3b99d612e58ef01d5e52cceRaph Levien EXPECT_EQ(24, breaker.wordEnd()); 125c88ef135fcc2661ec7addc171ebc60787df38affRaph Levien EXPECT_EQ(0, breaker.breakBadness()); 1269c4cc648abcae144f3b99d612e58ef01d5e52cceRaph Levien} 1279c4cc648abcae144f3b99d612e58ef01d5e52cceRaph Levien 1286d15657e4a3826d4d47d5358f1dde211484527e9Raph Levien// The current logic always places a line break after a detected email address or URL 1296d15657e4a3826d4d47d5358f1dde211484527e9Raph Levien// and an immediately following non-ASCII character. 1309c4cc648abcae144f3b99d612e58ef01d5e52cceRaph LevienTEST_F(WordBreakerTest, emailNonAscii) { 1319c4cc648abcae144f3b99d612e58ef01d5e52cceRaph Levien uint16_t buf[] = {'f', 'o', 'o', '@', 'e', 'x', 'a', 'm', 'p', 'l', 'e', '.', 'c', 'o', 'm', 1329c4cc648abcae144f3b99d612e58ef01d5e52cceRaph Levien 0x4E00}; 1339c4cc648abcae144f3b99d612e58ef01d5e52cceRaph Levien WordBreaker breaker; 1349c4cc648abcae144f3b99d612e58ef01d5e52cceRaph Levien breaker.setLocale(icu::Locale::getEnglish()); 1359c4cc648abcae144f3b99d612e58ef01d5e52cceRaph Levien breaker.setText(buf, NELEM(buf)); 1369c4cc648abcae144f3b99d612e58ef01d5e52cceRaph Levien EXPECT_EQ(0, breaker.current()); 1376d15657e4a3826d4d47d5358f1dde211484527e9Raph Levien EXPECT_EQ(11, breaker.next()); // after "foo@example" 1386d15657e4a3826d4d47d5358f1dde211484527e9Raph Levien EXPECT_TRUE(breaker.wordStart() >= breaker.wordEnd()); 139c88ef135fcc2661ec7addc171ebc60787df38affRaph Levien EXPECT_EQ(1, breaker.breakBadness()); 1406d15657e4a3826d4d47d5358f1dde211484527e9Raph Levien EXPECT_EQ(15, breaker.next()); // after ".com" 1419c4cc648abcae144f3b99d612e58ef01d5e52cceRaph Levien EXPECT_TRUE(breaker.wordStart() >= breaker.wordEnd()); 142c88ef135fcc2661ec7addc171ebc60787df38affRaph Levien EXPECT_EQ(0, breaker.breakBadness()); 1439c4cc648abcae144f3b99d612e58ef01d5e52cceRaph Levien EXPECT_EQ((ssize_t)NELEM(buf), breaker.next()); // end 1449c4cc648abcae144f3b99d612e58ef01d5e52cceRaph Levien EXPECT_EQ(15, breaker.wordStart()); // "一" 1459c4cc648abcae144f3b99d612e58ef01d5e52cceRaph Levien EXPECT_EQ(16, breaker.wordEnd()); 146c88ef135fcc2661ec7addc171ebc60787df38affRaph Levien EXPECT_EQ(0, breaker.breakBadness()); 1479c4cc648abcae144f3b99d612e58ef01d5e52cceRaph Levien} 1489c4cc648abcae144f3b99d612e58ef01d5e52cceRaph Levien 1499c4cc648abcae144f3b99d612e58ef01d5e52cceRaph LevienTEST_F(WordBreakerTest, emailCombining) { 1509c4cc648abcae144f3b99d612e58ef01d5e52cceRaph Levien uint16_t buf[] = {'f', 'o', 'o', '@', 'e', 'x', 'a', 'm', 'p', 'l', 'e', '.', 'c', 'o', 'm', 1519c4cc648abcae144f3b99d612e58ef01d5e52cceRaph Levien 0x0303, ' ', 'x'}; 1529c4cc648abcae144f3b99d612e58ef01d5e52cceRaph Levien WordBreaker breaker; 1539c4cc648abcae144f3b99d612e58ef01d5e52cceRaph Levien breaker.setLocale(icu::Locale::getEnglish()); 1549c4cc648abcae144f3b99d612e58ef01d5e52cceRaph Levien breaker.setText(buf, NELEM(buf)); 1559c4cc648abcae144f3b99d612e58ef01d5e52cceRaph Levien EXPECT_EQ(0, breaker.current()); 1566d15657e4a3826d4d47d5358f1dde211484527e9Raph Levien EXPECT_EQ(11, breaker.next()); // after "foo@example" 1576d15657e4a3826d4d47d5358f1dde211484527e9Raph Levien EXPECT_TRUE(breaker.wordStart() >= breaker.wordEnd()); 158c88ef135fcc2661ec7addc171ebc60787df38affRaph Levien EXPECT_EQ(1, breaker.breakBadness()); 1596d15657e4a3826d4d47d5358f1dde211484527e9Raph Levien EXPECT_EQ(17, breaker.next()); // after ".com̃ " 1609c4cc648abcae144f3b99d612e58ef01d5e52cceRaph Levien EXPECT_TRUE(breaker.wordStart() >= breaker.wordEnd()); 161c88ef135fcc2661ec7addc171ebc60787df38affRaph Levien EXPECT_EQ(0, breaker.breakBadness()); 1629c4cc648abcae144f3b99d612e58ef01d5e52cceRaph Levien EXPECT_EQ((ssize_t)NELEM(buf), breaker.next()); // end 1639c4cc648abcae144f3b99d612e58ef01d5e52cceRaph Levien EXPECT_EQ(17, breaker.wordStart()); // "x" 1649c4cc648abcae144f3b99d612e58ef01d5e52cceRaph Levien EXPECT_EQ(18, breaker.wordEnd()); 165c88ef135fcc2661ec7addc171ebc60787df38affRaph Levien EXPECT_EQ(0, breaker.breakBadness()); 1669c4cc648abcae144f3b99d612e58ef01d5e52cceRaph Levien} 1679c4cc648abcae144f3b99d612e58ef01d5e52cceRaph Levien 1686d15657e4a3826d4d47d5358f1dde211484527e9Raph LevienTEST_F(WordBreakerTest, lonelyAt) { 1696d15657e4a3826d4d47d5358f1dde211484527e9Raph Levien uint16_t buf[] = {'a', ' ', '@', ' ', 'b'}; 1706d15657e4a3826d4d47d5358f1dde211484527e9Raph Levien WordBreaker breaker; 1716d15657e4a3826d4d47d5358f1dde211484527e9Raph Levien breaker.setLocale(icu::Locale::getEnglish()); 1726d15657e4a3826d4d47d5358f1dde211484527e9Raph Levien breaker.setText(buf, NELEM(buf)); 1736d15657e4a3826d4d47d5358f1dde211484527e9Raph Levien EXPECT_EQ(0, breaker.current()); 1746d15657e4a3826d4d47d5358f1dde211484527e9Raph Levien EXPECT_EQ(2, breaker.next()); // after "a " 1756d15657e4a3826d4d47d5358f1dde211484527e9Raph Levien EXPECT_EQ(0, breaker.wordStart()); // "a" 1766d15657e4a3826d4d47d5358f1dde211484527e9Raph Levien EXPECT_EQ(1, breaker.wordEnd()); 177c88ef135fcc2661ec7addc171ebc60787df38affRaph Levien EXPECT_EQ(0, breaker.breakBadness()); 1786d15657e4a3826d4d47d5358f1dde211484527e9Raph Levien EXPECT_EQ(4, breaker.next()); // after "@ " 1796d15657e4a3826d4d47d5358f1dde211484527e9Raph Levien EXPECT_TRUE(breaker.wordStart() >= breaker.wordEnd()); 180c88ef135fcc2661ec7addc171ebc60787df38affRaph Levien EXPECT_EQ(0, breaker.breakBadness()); 1816d15657e4a3826d4d47d5358f1dde211484527e9Raph Levien EXPECT_EQ((ssize_t)NELEM(buf), breaker.next()); // end 1826d15657e4a3826d4d47d5358f1dde211484527e9Raph Levien EXPECT_EQ(4, breaker.wordStart()); // "b" 1836d15657e4a3826d4d47d5358f1dde211484527e9Raph Levien EXPECT_EQ(5, breaker.wordEnd()); 184c88ef135fcc2661ec7addc171ebc60787df38affRaph Levien EXPECT_EQ(0, breaker.breakBadness()); 1856d15657e4a3826d4d47d5358f1dde211484527e9Raph Levien} 1866d15657e4a3826d4d47d5358f1dde211484527e9Raph Levien 1879c4cc648abcae144f3b99d612e58ef01d5e52cceRaph LevienTEST_F(WordBreakerTest, url) { 1889c4cc648abcae144f3b99d612e58ef01d5e52cceRaph Levien uint16_t buf[] = {'h', 't', 't', 'p', ':', '/', '/', 'e', 'x', 'a', 'm', 'p', 'l', 'e', 1899c4cc648abcae144f3b99d612e58ef01d5e52cceRaph Levien '.', 'c', 'o', 'm', ' ', 'x'}; 1909c4cc648abcae144f3b99d612e58ef01d5e52cceRaph Levien WordBreaker breaker; 1919c4cc648abcae144f3b99d612e58ef01d5e52cceRaph Levien breaker.setLocale(icu::Locale::getEnglish()); 1929c4cc648abcae144f3b99d612e58ef01d5e52cceRaph Levien breaker.setText(buf, NELEM(buf)); 1939c4cc648abcae144f3b99d612e58ef01d5e52cceRaph Levien EXPECT_EQ(0, breaker.current()); 1946d15657e4a3826d4d47d5358f1dde211484527e9Raph Levien EXPECT_EQ(5, breaker.next()); // after "http:" 1956d15657e4a3826d4d47d5358f1dde211484527e9Raph Levien EXPECT_TRUE(breaker.wordStart() >= breaker.wordEnd()); 196c88ef135fcc2661ec7addc171ebc60787df38affRaph Levien EXPECT_EQ(1, breaker.breakBadness()); 1976d15657e4a3826d4d47d5358f1dde211484527e9Raph Levien EXPECT_EQ(7, breaker.next()); // after "//" 1986d15657e4a3826d4d47d5358f1dde211484527e9Raph Levien EXPECT_TRUE(breaker.wordStart() >= breaker.wordEnd()); 199c88ef135fcc2661ec7addc171ebc60787df38affRaph Levien EXPECT_EQ(1, breaker.breakBadness()); 2006d15657e4a3826d4d47d5358f1dde211484527e9Raph Levien EXPECT_EQ(14, breaker.next()); // after "example" 2016d15657e4a3826d4d47d5358f1dde211484527e9Raph Levien EXPECT_TRUE(breaker.wordStart() >= breaker.wordEnd()); 202c88ef135fcc2661ec7addc171ebc60787df38affRaph Levien EXPECT_EQ(1, breaker.breakBadness()); 2036d15657e4a3826d4d47d5358f1dde211484527e9Raph Levien EXPECT_EQ(19, breaker.next()); // after ".com " 2049c4cc648abcae144f3b99d612e58ef01d5e52cceRaph Levien EXPECT_TRUE(breaker.wordStart() >= breaker.wordEnd()); 205c88ef135fcc2661ec7addc171ebc60787df38affRaph Levien EXPECT_EQ(0, breaker.breakBadness()); 2069c4cc648abcae144f3b99d612e58ef01d5e52cceRaph Levien EXPECT_EQ((ssize_t)NELEM(buf), breaker.next()); // end 2079c4cc648abcae144f3b99d612e58ef01d5e52cceRaph Levien EXPECT_EQ(19, breaker.wordStart()); // "x" 2089c4cc648abcae144f3b99d612e58ef01d5e52cceRaph Levien EXPECT_EQ(20, breaker.wordEnd()); 209c88ef135fcc2661ec7addc171ebc60787df38affRaph Levien EXPECT_EQ(0, breaker.breakBadness()); 2109c4cc648abcae144f3b99d612e58ef01d5e52cceRaph Levien} 2116d15657e4a3826d4d47d5358f1dde211484527e9Raph Levien 2126d15657e4a3826d4d47d5358f1dde211484527e9Raph Levien// Breaks according to section 14.12 of Chicago Manual of Style, *URLs or DOIs and line breaks* 2136d15657e4a3826d4d47d5358f1dde211484527e9Raph LevienTEST_F(WordBreakerTest, urlBreakChars) { 2146d15657e4a3826d4d47d5358f1dde211484527e9Raph Levien uint16_t buf[] = {'h', 't', 't', 'p', ':', '/', '/', 'a', '.', 'b', '/', '~', 'c', ',', 'd', 2156d15657e4a3826d4d47d5358f1dde211484527e9Raph Levien '-', 'e', '?', 'f', '=', 'g', '&', 'h', '#', 'i', '%', 'j', '_', 'k', '/', 'l'}; 2166d15657e4a3826d4d47d5358f1dde211484527e9Raph Levien WordBreaker breaker; 2176d15657e4a3826d4d47d5358f1dde211484527e9Raph Levien breaker.setLocale(icu::Locale::getEnglish()); 2186d15657e4a3826d4d47d5358f1dde211484527e9Raph Levien breaker.setText(buf, NELEM(buf)); 2196d15657e4a3826d4d47d5358f1dde211484527e9Raph Levien EXPECT_EQ(0, breaker.current()); 2206d15657e4a3826d4d47d5358f1dde211484527e9Raph Levien EXPECT_EQ(5, breaker.next()); // after "http:" 2216d15657e4a3826d4d47d5358f1dde211484527e9Raph Levien EXPECT_TRUE(breaker.wordStart() >= breaker.wordEnd()); 222c88ef135fcc2661ec7addc171ebc60787df38affRaph Levien EXPECT_EQ(1, breaker.breakBadness()); 2236d15657e4a3826d4d47d5358f1dde211484527e9Raph Levien EXPECT_EQ(7, breaker.next()); // after "//" 2246d15657e4a3826d4d47d5358f1dde211484527e9Raph Levien EXPECT_TRUE(breaker.wordStart() >= breaker.wordEnd()); 225c88ef135fcc2661ec7addc171ebc60787df38affRaph Levien EXPECT_EQ(1, breaker.breakBadness()); 2266d15657e4a3826d4d47d5358f1dde211484527e9Raph Levien EXPECT_EQ(8, breaker.next()); // after "a" 2276d15657e4a3826d4d47d5358f1dde211484527e9Raph Levien EXPECT_TRUE(breaker.wordStart() >= breaker.wordEnd()); 228c88ef135fcc2661ec7addc171ebc60787df38affRaph Levien EXPECT_EQ(1, breaker.breakBadness()); 2296d15657e4a3826d4d47d5358f1dde211484527e9Raph Levien EXPECT_EQ(10, breaker.next()); // after ".b" 2306d15657e4a3826d4d47d5358f1dde211484527e9Raph Levien EXPECT_TRUE(breaker.wordStart() >= breaker.wordEnd()); 231c88ef135fcc2661ec7addc171ebc60787df38affRaph Levien EXPECT_EQ(1, breaker.breakBadness()); 2326d15657e4a3826d4d47d5358f1dde211484527e9Raph Levien EXPECT_EQ(11, breaker.next()); // after "/" 2336d15657e4a3826d4d47d5358f1dde211484527e9Raph Levien EXPECT_TRUE(breaker.wordStart() >= breaker.wordEnd()); 234c88ef135fcc2661ec7addc171ebc60787df38affRaph Levien EXPECT_EQ(1, breaker.breakBadness()); 2356d15657e4a3826d4d47d5358f1dde211484527e9Raph Levien EXPECT_EQ(13, breaker.next()); // after "~c" 2366d15657e4a3826d4d47d5358f1dde211484527e9Raph Levien EXPECT_TRUE(breaker.wordStart() >= breaker.wordEnd()); 237c88ef135fcc2661ec7addc171ebc60787df38affRaph Levien EXPECT_EQ(1, breaker.breakBadness()); 2386d15657e4a3826d4d47d5358f1dde211484527e9Raph Levien EXPECT_EQ(15, breaker.next()); // after ",d" 2396d15657e4a3826d4d47d5358f1dde211484527e9Raph Levien EXPECT_TRUE(breaker.wordStart() >= breaker.wordEnd()); 240c88ef135fcc2661ec7addc171ebc60787df38affRaph Levien EXPECT_EQ(1, breaker.breakBadness()); 2416d15657e4a3826d4d47d5358f1dde211484527e9Raph Levien EXPECT_EQ(17, breaker.next()); // after "-e" 2426d15657e4a3826d4d47d5358f1dde211484527e9Raph Levien EXPECT_TRUE(breaker.wordStart() >= breaker.wordEnd()); 243c88ef135fcc2661ec7addc171ebc60787df38affRaph Levien EXPECT_EQ(1, breaker.breakBadness()); 2446d15657e4a3826d4d47d5358f1dde211484527e9Raph Levien EXPECT_EQ(19, breaker.next()); // after "?f" 2456d15657e4a3826d4d47d5358f1dde211484527e9Raph Levien EXPECT_TRUE(breaker.wordStart() >= breaker.wordEnd()); 246c88ef135fcc2661ec7addc171ebc60787df38affRaph Levien EXPECT_EQ(1, breaker.breakBadness()); 2476d15657e4a3826d4d47d5358f1dde211484527e9Raph Levien EXPECT_EQ(20, breaker.next()); // after "=" 2486d15657e4a3826d4d47d5358f1dde211484527e9Raph Levien EXPECT_TRUE(breaker.wordStart() >= breaker.wordEnd()); 249c88ef135fcc2661ec7addc171ebc60787df38affRaph Levien EXPECT_EQ(1, breaker.breakBadness()); 2506d15657e4a3826d4d47d5358f1dde211484527e9Raph Levien EXPECT_EQ(21, breaker.next()); // after "g" 2516d15657e4a3826d4d47d5358f1dde211484527e9Raph Levien EXPECT_TRUE(breaker.wordStart() >= breaker.wordEnd()); 252c88ef135fcc2661ec7addc171ebc60787df38affRaph Levien EXPECT_EQ(1, breaker.breakBadness()); 2536d15657e4a3826d4d47d5358f1dde211484527e9Raph Levien EXPECT_EQ(22, breaker.next()); // after "&" 2546d15657e4a3826d4d47d5358f1dde211484527e9Raph Levien EXPECT_TRUE(breaker.wordStart() >= breaker.wordEnd()); 255c88ef135fcc2661ec7addc171ebc60787df38affRaph Levien EXPECT_EQ(1, breaker.breakBadness()); 2566d15657e4a3826d4d47d5358f1dde211484527e9Raph Levien EXPECT_EQ(23, breaker.next()); // after "h" 2576d15657e4a3826d4d47d5358f1dde211484527e9Raph Levien EXPECT_TRUE(breaker.wordStart() >= breaker.wordEnd()); 258c88ef135fcc2661ec7addc171ebc60787df38affRaph Levien EXPECT_EQ(1, breaker.breakBadness()); 2596d15657e4a3826d4d47d5358f1dde211484527e9Raph Levien EXPECT_EQ(25, breaker.next()); // after "#i" 2606d15657e4a3826d4d47d5358f1dde211484527e9Raph Levien EXPECT_TRUE(breaker.wordStart() >= breaker.wordEnd()); 261c88ef135fcc2661ec7addc171ebc60787df38affRaph Levien EXPECT_EQ(1, breaker.breakBadness()); 2626d15657e4a3826d4d47d5358f1dde211484527e9Raph Levien EXPECT_EQ(27, breaker.next()); // after "%j" 2636d15657e4a3826d4d47d5358f1dde211484527e9Raph Levien EXPECT_TRUE(breaker.wordStart() >= breaker.wordEnd()); 264c88ef135fcc2661ec7addc171ebc60787df38affRaph Levien EXPECT_EQ(1, breaker.breakBadness()); 2656d15657e4a3826d4d47d5358f1dde211484527e9Raph Levien EXPECT_EQ(29, breaker.next()); // after "_k" 2666d15657e4a3826d4d47d5358f1dde211484527e9Raph Levien EXPECT_TRUE(breaker.wordStart() >= breaker.wordEnd()); 267c88ef135fcc2661ec7addc171ebc60787df38affRaph Levien EXPECT_EQ(1, breaker.breakBadness()); 2686d15657e4a3826d4d47d5358f1dde211484527e9Raph Levien EXPECT_EQ((ssize_t)NELEM(buf), breaker.next()); // end 2696d15657e4a3826d4d47d5358f1dde211484527e9Raph Levien EXPECT_TRUE(breaker.wordStart() >= breaker.wordEnd()); 270c88ef135fcc2661ec7addc171ebc60787df38affRaph Levien EXPECT_EQ(0, breaker.breakBadness()); 2716d15657e4a3826d4d47d5358f1dde211484527e9Raph Levien} 2726d15657e4a3826d4d47d5358f1dde211484527e9Raph Levien 2736d15657e4a3826d4d47d5358f1dde211484527e9Raph LevienTEST_F(WordBreakerTest, urlNoHyphenBreak) { 2746d15657e4a3826d4d47d5358f1dde211484527e9Raph Levien uint16_t buf[] = {'h', 't', 't', 'p', ':', '/', '/', 'a', '-', '/', 'b'}; 2756d15657e4a3826d4d47d5358f1dde211484527e9Raph Levien WordBreaker breaker; 2766d15657e4a3826d4d47d5358f1dde211484527e9Raph Levien breaker.setLocale(icu::Locale::getEnglish()); 2776d15657e4a3826d4d47d5358f1dde211484527e9Raph Levien breaker.setText(buf, NELEM(buf)); 2786d15657e4a3826d4d47d5358f1dde211484527e9Raph Levien EXPECT_EQ(0, breaker.current()); 2796d15657e4a3826d4d47d5358f1dde211484527e9Raph Levien EXPECT_EQ(5, breaker.next()); // after "http:" 2806d15657e4a3826d4d47d5358f1dde211484527e9Raph Levien EXPECT_TRUE(breaker.wordStart() >= breaker.wordEnd()); 2816d15657e4a3826d4d47d5358f1dde211484527e9Raph Levien EXPECT_EQ(7, breaker.next()); // after "//" 2826d15657e4a3826d4d47d5358f1dde211484527e9Raph Levien EXPECT_TRUE(breaker.wordStart() >= breaker.wordEnd()); 2836d15657e4a3826d4d47d5358f1dde211484527e9Raph Levien EXPECT_EQ(8, breaker.next()); // after "a" 2846d15657e4a3826d4d47d5358f1dde211484527e9Raph Levien EXPECT_TRUE(breaker.wordStart() >= breaker.wordEnd()); 2856d15657e4a3826d4d47d5358f1dde211484527e9Raph Levien EXPECT_EQ((ssize_t)NELEM(buf), breaker.next()); // end 2866d15657e4a3826d4d47d5358f1dde211484527e9Raph Levien EXPECT_TRUE(breaker.wordStart() >= breaker.wordEnd()); 2876d15657e4a3826d4d47d5358f1dde211484527e9Raph Levien} 2886d15657e4a3826d4d47d5358f1dde211484527e9Raph Levien 2896d15657e4a3826d4d47d5358f1dde211484527e9Raph LevienTEST_F(WordBreakerTest, urlEndsWithSlash) { 2906d15657e4a3826d4d47d5358f1dde211484527e9Raph Levien uint16_t buf[] = {'h', 't', 't', 'p', ':', '/', '/', 'a', '/'}; 2916d15657e4a3826d4d47d5358f1dde211484527e9Raph Levien WordBreaker breaker; 2926d15657e4a3826d4d47d5358f1dde211484527e9Raph Levien breaker.setLocale(icu::Locale::getEnglish()); 2936d15657e4a3826d4d47d5358f1dde211484527e9Raph Levien breaker.setText(buf, NELEM(buf)); 2946d15657e4a3826d4d47d5358f1dde211484527e9Raph Levien EXPECT_EQ(0, breaker.current()); 2956d15657e4a3826d4d47d5358f1dde211484527e9Raph Levien EXPECT_EQ(5, breaker.next()); // after "http:" 2966d15657e4a3826d4d47d5358f1dde211484527e9Raph Levien EXPECT_TRUE(breaker.wordStart() >= breaker.wordEnd()); 2976d15657e4a3826d4d47d5358f1dde211484527e9Raph Levien EXPECT_EQ(7, breaker.next()); // after "//" 2986d15657e4a3826d4d47d5358f1dde211484527e9Raph Levien EXPECT_TRUE(breaker.wordStart() >= breaker.wordEnd()); 2996d15657e4a3826d4d47d5358f1dde211484527e9Raph Levien EXPECT_EQ(8, breaker.next()); // after "a" 3006d15657e4a3826d4d47d5358f1dde211484527e9Raph Levien EXPECT_TRUE(breaker.wordStart() >= breaker.wordEnd()); 3016d15657e4a3826d4d47d5358f1dde211484527e9Raph Levien EXPECT_EQ((ssize_t)NELEM(buf), breaker.next()); // end 3026d15657e4a3826d4d47d5358f1dde211484527e9Raph Levien EXPECT_TRUE(breaker.wordStart() >= breaker.wordEnd()); 3036d15657e4a3826d4d47d5358f1dde211484527e9Raph Levien} 3046d15657e4a3826d4d47d5358f1dde211484527e9Raph Levien 3056d15657e4a3826d4d47d5358f1dde211484527e9Raph LevienTEST_F(WordBreakerTest, emailStartsWithSlash) { 3066d15657e4a3826d4d47d5358f1dde211484527e9Raph Levien uint16_t buf[] = {'/', 'a', '@', 'b'}; 3076d15657e4a3826d4d47d5358f1dde211484527e9Raph Levien WordBreaker breaker; 3086d15657e4a3826d4d47d5358f1dde211484527e9Raph Levien breaker.setLocale(icu::Locale::getEnglish()); 3096d15657e4a3826d4d47d5358f1dde211484527e9Raph Levien breaker.setText(buf, NELEM(buf)); 3106d15657e4a3826d4d47d5358f1dde211484527e9Raph Levien EXPECT_EQ(0, breaker.current()); 3116d15657e4a3826d4d47d5358f1dde211484527e9Raph Levien EXPECT_EQ((ssize_t)NELEM(buf), breaker.next()); // end 3126d15657e4a3826d4d47d5358f1dde211484527e9Raph Levien EXPECT_TRUE(breaker.wordStart() >= breaker.wordEnd()); 3136d15657e4a3826d4d47d5358f1dde211484527e9Raph Levien} 314