1#
2# Copyright (C) 2002-2014, International Business Machines Corporation
3# and others. All Rights Reserved.
4#
5# file:  word_POSIX.txt
6#
7# ICU Word Break Rules, POSIX locale.
8#      See Unicode Standard Annex #29.
9#      These rules are based on UAX #29 Revision 22 for Unicode Version 6.3
10#
11# Note:  Updates to word.txt will usually need to be merged into
12#        word_POSIX.txt also.
13
14##############################################################################
15#
16#  Character class definitions from TR 29
17#
18##############################################################################
19
20!!chain;
21
22
23#
24#  Character Class Definitions.
25#
26
27$CR                 = [\p{Word_Break = CR}];
28$LF                 = [\p{Word_Break = LF}];
29$Newline            = [\p{Word_Break = Newline}];
30$Extend             = [\p{Word_Break = Extend}];
31$Regional_Indicator = [\p{Word_Break = Regional_Indicator}];
32$Format             = [\p{Word_Break = Format}];
33$Katakana           = [\p{Word_Break = Katakana}];
34$Hebrew_Letter      = [\p{Word_Break = Hebrew_Letter}];
35$ALetter            = [\p{Word_Break = ALetter}];
36$Single_Quote       = [\p{Word_Break = Single_Quote}];
37$Double_Quote       = [\p{Word_Break = Double_Quote}];
38$MidNumLet          = [\p{Word_Break = MidNumLet} - [.]];
39$MidLetter          = [\p{Word_Break = MidLetter} - [\:]];
40$MidNum             = [\p{Word_Break = MidNum} [.]];
41$Numeric            = [\p{Word_Break = Numeric}];
42$ExtendNumLet       = [\p{Word_Break = ExtendNumLet}];
43
44$Han                = [:Han:];
45$Hiragana           = [:Hiragana:];
46
47
48#   Dictionary character set, for triggering language-based break engines. Currently
49#   limited to LineBreak=Complex_Context. Note that this set only works in Unicode
50#   5.0 or later as the definition of Complex_Context was corrected to include all
51#   characters requiring dictionary break.
52
53$Control        = [\p{Grapheme_Cluster_Break = Control}];
54$HangulSyllable = [\uac00-\ud7a3];
55$ComplexContext = [:LineBreak = Complex_Context:];
56$KanaKanji      = [$Han $Hiragana $Katakana];
57$dictionaryCJK  = [$KanaKanji $HangulSyllable];
58$dictionary     = [$ComplexContext $dictionaryCJK];
59
60# leave CJK scripts out of ALetterPlus
61$ALetterPlus  = [$ALetter-$dictionaryCJK [$ComplexContext-$Extend-$Control]];
62
63
64#
65#  Rules 4    Ignore Format and Extend characters,
66#             except when they appear at the beginning of a region of text.
67#
68# TODO: check if handling of katakana in dictionary makes rules incorrect/void
69$KatakanaEx           = $Katakana           ($Extend |  $Format)*;
70$Hebrew_LetterEx      = $Hebrew_Letter      ($Extend |  $Format)*;
71$ALetterEx            = $ALetterPlus        ($Extend |  $Format)*;
72$Single_QuoteEx       = $Single_Quote       ($Extend |  $Format)*;
73$Double_QuoteEx       = $Double_Quote       ($Extend |  $Format)*;
74$MidNumLetEx          = $MidNumLet          ($Extend |  $Format)*;
75$MidLetterEx          = $MidLetter          ($Extend |  $Format)*;
76$MidNumEx             = $MidNum             ($Extend |  $Format)*;
77$NumericEx            = $Numeric            ($Extend |  $Format)*;
78$ExtendNumLetEx       = $ExtendNumLet       ($Extend |  $Format)*;
79$Regional_IndicatorEx = $Regional_Indicator ($Extend |  $Format)*;
80
81$Ideographic    = [\p{Ideographic}];
82$HiraganaEx     = $Hiragana     ($Extend |  $Format)*;
83$IdeographicEx  = $Ideographic  ($Extend |  $Format)*;
84
85## -------------------------------------------------
86
87!!forward;
88
89
90# Rule 3 - CR x LF
91#
92$CR $LF;
93
94# Rule 4 - ignore Format and Extend characters, except when they appear at the beginning
95#          of a region of Text.   The rule here comes into play when the start of text
96#          begins with a group of Format chars, or with a "word" consisting of a single
97#          char that is not in any of the listed word break categories followed by
98#          format char(s), or is not a CJK dictionary character.
99[^$CR $LF $Newline]? ($Extend |  $Format)+;
100
101$NumericEx {100};
102$ALetterEx {200};
103$HangulSyllable {200};
104$Hebrew_LetterEx{200};
105$KatakanaEx {400};       # note:  these status values override those from rule 5
106$HiraganaEx {400};       #        by virtue of being numerically larger.
107$IdeographicEx {400};    #
108
109#
110# rule 5
111#    Do not break between most letters.
112#
113($ALetterEx | $Hebrew_LetterEx)  ($ALetterEx | $Hebrew_LetterEx) {200};
114
115# rule 6 and 7
116($ALetterEx | $Hebrew_LetterEx) ($MidLetterEx | $MidNumLetEx | $Single_QuoteEx) ($ALetterEx | $Hebrew_LetterEx) {200};
117
118# rule 7a
119$Hebrew_LetterEx $Single_QuoteEx {200};
120
121# rule 7b and 7c
122$Hebrew_LetterEx $Double_QuoteEx $Hebrew_LetterEx {200};
123
124# rule 8
125
126$NumericEx $NumericEx {100};
127
128# rule 9
129
130($ALetterEx | $Hebrew_LetterEx) $NumericEx {200};
131
132# rule 10
133
134$NumericEx ($ALetterEx | $Hebrew_LetterEx) {200};
135
136# rule 11 and 12
137
138$NumericEx ($MidNumEx | $MidNumLetEx | $Single_QuoteEx) $NumericEx {100};
139
140# rule 13
141# to be consistent with $KanaKanji $KanaKanhi, changed
142# from 300 to 400.
143# See also TestRuleStatus in intltest/rbbiapts.cpp
144$KatakanaEx  $KatakanaEx {400};
145
146# rule 13a/b
147
148$ALetterEx       $ExtendNumLetEx {200};    #  (13a)
149$Hebrew_LetterEx $ExtendNumLetEx {200};    #  (13a)
150$NumericEx       $ExtendNumLetEx {100};    #  (13a)
151$KatakanaEx      $ExtendNumLetEx {400};    #  (13a)
152$ExtendNumLetEx  $ExtendNumLetEx {200};    #  (13a)
153
154$ExtendNumLetEx  $ALetterEx      {200};    #  (13b)
155$ExtendNumLetEx  $Hebrew_Letter  {200};    #  (13b)
156$ExtendNumLetEx  $NumericEx      {100};    #  (13b)
157$ExtendNumLetEx  $KatakanaEx     {400};    #  (13b)
158
159# rule 13c
160
161$Regional_IndicatorEx $Regional_IndicatorEx;
162
163# special handling for CJK characters: chain for later dictionary segmentation
164$HangulSyllable $HangulSyllable {200};
165$KanaKanji $KanaKanji {400}; # different rule status if both kana and kanji found
166
167
168## -------------------------------------------------
169
170!!reverse;
171
172$BackHebrew_LetterEx      = ($Format | $Extend)* $Hebrew_Letter;
173$BackALetterEx            = ($Format | $Extend)* $ALetterPlus;
174$BackSingle_QuoteEx       = ($Format | $Extend)* $Single_Quote;
175$BackDouble_QuoteEx       = ($Format | $Extend)* $Double_Quote;
176$BackMidNumLetEx          = ($Format | $Extend)* $MidNumLet;
177$BackNumericEx            = ($Format | $Extend)* $Numeric;
178$BackMidNumEx             = ($Format | $Extend)* $MidNum;
179$BackMidLetterEx          = ($Format | $Extend)* $MidLetter;
180$BackKatakanaEx           = ($Format | $Extend)* $Katakana;
181$BackHiraganaEx           = ($Format | $Extend)* $Hiragana;
182$BackExtendNumLetEx       = ($Format | $Extend)* $ExtendNumLet;
183$BackRegional_IndicatorEx = ($Format | $Extend)* $Regional_Indicator;
184
185# rule 3
186$LF $CR;
187
188# rule 4
189($Format | $Extend)*  [^$CR $LF $Newline]?;
190
191# rule 5
192
193($BackALetterEx | $BackHebrew_LetterEx) ($BackALetterEx | $BackHebrew_LetterEx);
194
195# rule 6 and 7
196
197($BackALetterEx | $BackHebrew_LetterEx) ($BackMidLetterEx | $BackMidNumLetEx | $BackSingle_QuoteEx) ($BackALetterEx | $BackHebrew_LetterEx);
198
199# rule 7a
200$BackSingle_QuoteEx $BackHebrew_LetterEx;
201
202# Rule 7b and 7c
203$BackHebrew_LetterEx $BackDouble_QuoteEx $BackHebrew_LetterEx;
204
205# rule 8
206
207$BackNumericEx $BackNumericEx;
208
209# rule 9
210
211$BackNumericEx ($BackALetterEx | $BackHebrew_LetterEx);
212
213# rule 10
214
215($BackALetterEx | $BackHebrew_LetterEx) $BackNumericEx;
216
217# rule 11 and 12
218
219$BackNumericEx ($BackMidNumEx | $BackMidNumLetEx | $BackSingle_QuoteEx) $BackNumericEx;
220
221# rule 13
222
223$BackKatakanaEx $BackKatakanaEx;
224
225# rules 13 a/b
226#
227$BackExtendNumLetEx ($BackALetterEx | $BackHebrew_LetterEx | $BackNumericEx | $BackKatakanaEx | $BackExtendNumLetEx);
228($BackALetterEx | $BackHebrew_LetterEx | $BackNumericEx | $BackKatakanaEx) $BackExtendNumLetEx;
229
230# rule 13c
231
232$BackRegional_IndicatorEx $BackRegional_IndicatorEx;
233
234# special handling for CJK characters: chain for later dictionary segmentation
235$HangulSyllable $HangulSyllable;
236$KanaKanji $KanaKanji; #different rule status if both kanji and kana found
237
238## -------------------------------------------------
239
240!!safe_reverse;
241
242# rule 3
243($Extend | $Format)+ .?;
244
245# rule 6
246($MidLetter | $MidNumLet | $Single_Quote) ($BackALetterEx | $BackHebrew_LetterEx);
247
248# rule 7b
249$Double_Quote $BackHebrew_LetterEx;
250
251
252# rule 11
253($MidNum | $MidNumLet | $Single_Quote) $BackNumericEx;
254
255# For dictionary-based break
256$dictionary $dictionary;
257
258## -------------------------------------------------
259
260!!safe_forward;
261
262# rule 4
263($Extend | $Format)+ .?;
264
265# rule 6
266($MidLetterEx | $MidNumLetEx | $Single_QuoteEx) ($ALetterEx | $Hebrew_LetterEx);
267
268# rule 7b
269$Double_QuoteEx $Hebrew_LetterEx;
270
271# rule 11
272($MidNumEx | $MidNumLetEx | $Single_QuoteEx) $NumericEx;
273
274# For dictionary-based break
275$dictionary $dictionary;
276