1ac04d0bbe12b3ef54518635711412f178cb4d16Jean-Baptiste Queru# 283a171d1a62abf406f7f44ae671823d5ec20db7dCraig Cornelius# Copyright (C) 2002-2011, International Business Machines Corporation and others. 3ac04d0bbe12b3ef54518635711412f178cb4d16Jean-Baptiste Queru# All Rights Reserved. 4ac04d0bbe12b3ef54518635711412f178cb4d16Jean-Baptiste Queru# 5ac04d0bbe12b3ef54518635711412f178cb4d16Jean-Baptiste Queru# file: sent.txt 6ac04d0bbe12b3ef54518635711412f178cb4d16Jean-Baptiste Queru# 7ac04d0bbe12b3ef54518635711412f178cb4d16Jean-Baptiste Queru# ICU Sentence Break Rules 8ac04d0bbe12b3ef54518635711412f178cb4d16Jean-Baptiste Queru# See Unicode Standard Annex #29. 983a171d1a62abf406f7f44ae671823d5ec20db7dCraig Cornelius# These rules are based on UAX #29 Revision 19 for Unicode Version 6.1 10ac04d0bbe12b3ef54518635711412f178cb4d16Jean-Baptiste Queru# 11ac04d0bbe12b3ef54518635711412f178cb4d16Jean-Baptiste Queru 12ac04d0bbe12b3ef54518635711412f178cb4d16Jean-Baptiste Queru 13ac04d0bbe12b3ef54518635711412f178cb4d16Jean-Baptiste Queru# 14ac04d0bbe12b3ef54518635711412f178cb4d16Jean-Baptiste Queru# Character categories as defined in TR 29 15ac04d0bbe12b3ef54518635711412f178cb4d16Jean-Baptiste Queru# 1685bf2e2fbc60a9f938064abc8127d61da7d19882Claire Ho$CR = [\p{Sentence_Break = CR}]; 1785bf2e2fbc60a9f938064abc8127d61da7d19882Claire Ho$LF = [\p{Sentence_Break = LF}]; 1885bf2e2fbc60a9f938064abc8127d61da7d19882Claire Ho$Extend = [\p{Sentence_Break = Extend}]; 19ac04d0bbe12b3ef54518635711412f178cb4d16Jean-Baptiste Queru$Sep = [\p{Sentence_Break = Sep}]; 20ac04d0bbe12b3ef54518635711412f178cb4d16Jean-Baptiste Queru$Format = [\p{Sentence_Break = Format}]; 21ac04d0bbe12b3ef54518635711412f178cb4d16Jean-Baptiste Queru$Sp = [\p{Sentence_Break = Sp}]; 22ac04d0bbe12b3ef54518635711412f178cb4d16Jean-Baptiste Queru$Lower = [\p{Sentence_Break = Lower}]; 23ac04d0bbe12b3ef54518635711412f178cb4d16Jean-Baptiste Queru$Upper = [\p{Sentence_Break = Upper}]; 2485bf2e2fbc60a9f938064abc8127d61da7d19882Claire Ho$OLetter = [\p{Sentence_Break = OLetter}]; 25ac04d0bbe12b3ef54518635711412f178cb4d16Jean-Baptiste Queru$Numeric = [\p{Sentence_Break = Numeric}]; 26ac04d0bbe12b3ef54518635711412f178cb4d16Jean-Baptiste Queru$ATerm = [\p{Sentence_Break = ATerm}]; 2785bf2e2fbc60a9f938064abc8127d61da7d19882Claire Ho$SContinue = [\p{Sentence_Break = SContinue}]; 28ac04d0bbe12b3ef54518635711412f178cb4d16Jean-Baptiste Queru$STerm = [\p{Sentence_Break = STerm}]; 29ac04d0bbe12b3ef54518635711412f178cb4d16Jean-Baptiste Queru$Close = [\p{Sentence_Break = Close}]; 30ac04d0bbe12b3ef54518635711412f178cb4d16Jean-Baptiste Queru 31ac04d0bbe12b3ef54518635711412f178cb4d16Jean-Baptiste Queru# 32ac04d0bbe12b3ef54518635711412f178cb4d16Jean-Baptiste Queru# Define extended forms of the character classes, 3385bf2e2fbc60a9f938064abc8127d61da7d19882Claire Ho# incorporate trailing Extend or Format chars. 34ac04d0bbe12b3ef54518635711412f178cb4d16Jean-Baptiste Queru# Rules 4 and 5. 35ac04d0bbe12b3ef54518635711412f178cb4d16Jean-Baptiste Queru 36ac04d0bbe12b3ef54518635711412f178cb4d16Jean-Baptiste Queru$SpEx = $Sp ($Extend | $Format)*; 37ac04d0bbe12b3ef54518635711412f178cb4d16Jean-Baptiste Queru$LowerEx = $Lower ($Extend | $Format)*; 38ac04d0bbe12b3ef54518635711412f178cb4d16Jean-Baptiste Queru$UpperEx = $Upper ($Extend | $Format)*; 39ac04d0bbe12b3ef54518635711412f178cb4d16Jean-Baptiste Queru$OLetterEx = $OLetter ($Extend | $Format)*; 40ac04d0bbe12b3ef54518635711412f178cb4d16Jean-Baptiste Queru$NumericEx = $Numeric ($Extend | $Format)*; 41ac04d0bbe12b3ef54518635711412f178cb4d16Jean-Baptiste Queru$ATermEx = $ATerm ($Extend | $Format)*; 4285bf2e2fbc60a9f938064abc8127d61da7d19882Claire Ho$SContinueEx= $SContinue ($Extend | $Format)*; 43ac04d0bbe12b3ef54518635711412f178cb4d16Jean-Baptiste Queru$STermEx = $STerm ($Extend | $Format)*; 44ac04d0bbe12b3ef54518635711412f178cb4d16Jean-Baptiste Queru$CloseEx = $Close ($Extend | $Format)*; 45ac04d0bbe12b3ef54518635711412f178cb4d16Jean-Baptiste Queru 46ac04d0bbe12b3ef54518635711412f178cb4d16Jean-Baptiste Queru 47ac04d0bbe12b3ef54518635711412f178cb4d16Jean-Baptiste Queru## ------------------------------------------------- 48ac04d0bbe12b3ef54518635711412f178cb4d16Jean-Baptiste Queru 49ac04d0bbe12b3ef54518635711412f178cb4d16Jean-Baptiste Queru!!chain; 50ac04d0bbe12b3ef54518635711412f178cb4d16Jean-Baptiste Queru!!forward; 51ac04d0bbe12b3ef54518635711412f178cb4d16Jean-Baptiste Queru 52ac04d0bbe12b3ef54518635711412f178cb4d16Jean-Baptiste Queru# Rule 3 - break after separators. Keep CR/LF together. 53ac04d0bbe12b3ef54518635711412f178cb4d16Jean-Baptiste Queru# 54ac04d0bbe12b3ef54518635711412f178cb4d16Jean-Baptiste Queru$CR $LF; 55ac04d0bbe12b3ef54518635711412f178cb4d16Jean-Baptiste Queru 56ac04d0bbe12b3ef54518635711412f178cb4d16Jean-Baptiste Queru 57ac04d0bbe12b3ef54518635711412f178cb4d16Jean-Baptiste Queru# Rule 4 - Break after $Sep. 58ac04d0bbe12b3ef54518635711412f178cb4d16Jean-Baptiste Queru# Rule 5 - Ignore $Format and $Extend 59ac04d0bbe12b3ef54518635711412f178cb4d16Jean-Baptiste Queru# 6085bf2e2fbc60a9f938064abc8127d61da7d19882Claire Ho[^$Sep $CR $LF]? ($Extend | $Format)*; 61ac04d0bbe12b3ef54518635711412f178cb4d16Jean-Baptiste Queru 62ac04d0bbe12b3ef54518635711412f178cb4d16Jean-Baptiste Queru 63ac04d0bbe12b3ef54518635711412f178cb4d16Jean-Baptiste Queru# Rule 6 64ac04d0bbe12b3ef54518635711412f178cb4d16Jean-Baptiste Queru$ATermEx $NumericEx; 65ac04d0bbe12b3ef54518635711412f178cb4d16Jean-Baptiste Queru 66ac04d0bbe12b3ef54518635711412f178cb4d16Jean-Baptiste Queru# Rule 7 67ac04d0bbe12b3ef54518635711412f178cb4d16Jean-Baptiste Queru$UpperEx $ATermEx $UpperEx; 68ac04d0bbe12b3ef54518635711412f178cb4d16Jean-Baptiste Queru 69ac04d0bbe12b3ef54518635711412f178cb4d16Jean-Baptiste Queru#Rule 8 7085bf2e2fbc60a9f938064abc8127d61da7d19882Claire Ho$NotLettersEx = [^$OLetter $Upper $Lower $Sep $CR $LF $ATerm $STerm] ($Extend | $Format)*; 71ac04d0bbe12b3ef54518635711412f178cb4d16Jean-Baptiste Queru$ATermEx $CloseEx* $SpEx* $NotLettersEx* $Lower; 72ac04d0bbe12b3ef54518635711412f178cb4d16Jean-Baptiste Queru 73ac04d0bbe12b3ef54518635711412f178cb4d16Jean-Baptiste Queru# Rule 8a 7485bf2e2fbc60a9f938064abc8127d61da7d19882Claire Ho($STermEx | $ATermEx) $CloseEx* $SpEx* ($SContinueEx | $STermEx | $ATermEx); 75ac04d0bbe12b3ef54518635711412f178cb4d16Jean-Baptiste Queru 76ac04d0bbe12b3ef54518635711412f178cb4d16Jean-Baptiste Queru#Rule 9, 10, 11 7785bf2e2fbc60a9f938064abc8127d61da7d19882Claire Ho($STermEx | $ATermEx) $CloseEx* $SpEx* ($Sep | $CR | $LF)?; 78ac04d0bbe12b3ef54518635711412f178cb4d16Jean-Baptiste Queru 79ac04d0bbe12b3ef54518635711412f178cb4d16Jean-Baptiste Queru#Rule 12 8085bf2e2fbc60a9f938064abc8127d61da7d19882Claire Ho[[^$STerm $ATerm $Close $Sp $Sep $LF $CR $Format $Extend]{bof}] ($Extend | $Format | $Close | $Sp)* .; 8185bf2e2fbc60a9f938064abc8127d61da7d19882Claire Ho[[^$STerm $ATerm $Close $Sp $Sep $LF $CR $Format $Extend]{bof}] ($Extend | $Format | $Close | $Sp)* ([$Sep $LF $CR {eof}] | $CR $LF){100}; 82ac04d0bbe12b3ef54518635711412f178cb4d16Jean-Baptiste Queru 83ac04d0bbe12b3ef54518635711412f178cb4d16Jean-Baptiste Queru## ------------------------------------------------- 84ac04d0bbe12b3ef54518635711412f178cb4d16Jean-Baptiste Queru 85ac04d0bbe12b3ef54518635711412f178cb4d16Jean-Baptiste Queru!!reverse; 86ac04d0bbe12b3ef54518635711412f178cb4d16Jean-Baptiste Queru 87ac04d0bbe12b3ef54518635711412f178cb4d16Jean-Baptiste Queru$SpEx_R = ($Extend | $Format)* $Sp; 88ac04d0bbe12b3ef54518635711412f178cb4d16Jean-Baptiste Queru$ATermEx_R = ($Extend | $Format)* $ATerm; 89ac04d0bbe12b3ef54518635711412f178cb4d16Jean-Baptiste Queru$STermEx_R = ($Extend | $Format)* $STerm; 90ac04d0bbe12b3ef54518635711412f178cb4d16Jean-Baptiste Queru$CloseEx_R = ($Extend | $Format)* $Close; 91ac04d0bbe12b3ef54518635711412f178cb4d16Jean-Baptiste Queru 92ac04d0bbe12b3ef54518635711412f178cb4d16Jean-Baptiste Queru# 93ac04d0bbe12b3ef54518635711412f178cb4d16Jean-Baptiste Queru# Reverse rules. 94ac04d0bbe12b3ef54518635711412f178cb4d16Jean-Baptiste Queru# For now, use the old style inexact reverse rules, which are easier 95ac04d0bbe12b3ef54518635711412f178cb4d16Jean-Baptiste Queru# to write, but less efficient. 96ac04d0bbe12b3ef54518635711412f178cb4d16Jean-Baptiste Queru# TODO: exact reverse rules. It appears that exact reverse rules 97ac04d0bbe12b3ef54518635711412f178cb4d16Jean-Baptiste Queru# may require improving support for look-ahead breaks in the 98ac04d0bbe12b3ef54518635711412f178cb4d16Jean-Baptiste Queru# builder. Needs more investigation. 99ac04d0bbe12b3ef54518635711412f178cb4d16Jean-Baptiste Queru# 100ac04d0bbe12b3ef54518635711412f178cb4d16Jean-Baptiste Queru 10185bf2e2fbc60a9f938064abc8127d61da7d19882Claire Ho[{bof}] (.? | $LF $CR) [^$Sep $CR $LF]* [$Sep $CR $LF {eof}] ($SpEx_R* $CloseEx_R* ($STermEx_R | $ATermEx_R))*; 102ac04d0bbe12b3ef54518635711412f178cb4d16Jean-Baptiste Queru#.*; 103ac04d0bbe12b3ef54518635711412f178cb4d16Jean-Baptiste Queru 104ac04d0bbe12b3ef54518635711412f178cb4d16Jean-Baptiste Queru# Explanation for this rule: 105ac04d0bbe12b3ef54518635711412f178cb4d16Jean-Baptiste Queru# 106ac04d0bbe12b3ef54518635711412f178cb4d16Jean-Baptiste Queru# It needs to back over 107ac04d0bbe12b3ef54518635711412f178cb4d16Jean-Baptiste Queru# The $Sep at which we probably begin 108ac04d0bbe12b3ef54518635711412f178cb4d16Jean-Baptiste Queru# All of the non $Sep chars leading to the preceding $Sep 109ac04d0bbe12b3ef54518635711412f178cb4d16Jean-Baptiste Queru# The preceding $Sep, which will be the second one that the rule matches. 110ac04d0bbe12b3ef54518635711412f178cb4d16Jean-Baptiste Queru# Any immediately preceding STerm or ATerm sequences. We need to see these 111ac04d0bbe12b3ef54518635711412f178cb4d16Jean-Baptiste Queru# to get the correct rule status when moving forwards again. 112ac04d0bbe12b3ef54518635711412f178cb4d16Jean-Baptiste Queru# 113ac04d0bbe12b3ef54518635711412f178cb4d16Jean-Baptiste Queru# [{bof}] inhibit rule chaining. Without this, rule would loop on itself and match 114ac04d0bbe12b3ef54518635711412f178cb4d16Jean-Baptiste Queru# the entire string. 115ac04d0bbe12b3ef54518635711412f178cb4d16Jean-Baptiste Queru# 116ac04d0bbe12b3ef54518635711412f178cb4d16Jean-Baptiste Queru# (.? | $LF $CR) Match one $Sep instance. Use .? rather than $Sep because position might be 117ac04d0bbe12b3ef54518635711412f178cb4d16Jean-Baptiste Queru# at the beginning of the string at this point, and we don't want to fail. 118ac04d0bbe12b3ef54518635711412f178cb4d16Jean-Baptiste Queru# Can only use {eof} once, and it is used later. 119ac04d0bbe12b3ef54518635711412f178cb4d16Jean-Baptiste Queru# 120