Home
last modified time | relevance | path

Searched refs:new_token (Results 1 – 6 of 6) sorted by relevance

/external/antlr/runtime/Ruby/test/unit/
Dtest-trees.rb163 r0 = new_node new_token(101)
164 r1 = new_node new_token(102)
166 r1.add_child(new_node new_token(103))
167 r2 = new_node new_token(106)
168 r2.add_child new_node( new_token 107 )
170 r0.add_child new_node( new_token 104 )
171 r0.add_child new_node( new_token 105 )
189 r0 = new_node new_token(101)
190 r1 = new_node new_token(102)
192 r1.add_child new_node( new_token 103 )
[all …]
Dtest-streams.rb264 @source2.tokens << new_token( 10, :channel => ANTLR3::HIDDEN ) << new_token( 11 )
284 @source.tokens << new_token(12)
292 @source.tokens << new_token(12, :channel => ANTLR3::HIDDEN_CHANNEL) <<
293 new_token(13)
300 @source.tokens << new_token(12) <<
301 new_token(13, :channel => ANTLR3::HIDDEN_CHANNEL)
309 @source.tokens << new_token(12) << new_token(13)
318 @source.tokens << new_token(12) << new_token(13)
328 @source.tokens << new_token(12) << new_token(13)
335 @source.tokens << new_token(10) <<
[all …]
/external/libtextclassifier/utils/
Dtokenizer.cc120 Token new_token("", 0, 0); in InternalTokenize() local
133 if (!new_token.value.empty()) { in InternalTokenize()
134 result.push_back(new_token); in InternalTokenize()
136 new_token = Token("", codepoint_index, codepoint_index); in InternalTokenize()
139 new_token.value += std::string( in InternalTokenize()
142 ++new_token.end; in InternalTokenize()
145 if (!new_token.value.empty()) { in InternalTokenize()
146 result.push_back(new_token); in InternalTokenize()
148 new_token = Token("", codepoint_index + 1, codepoint_index + 1); in InternalTokenize()
153 if (!new_token.value.empty()) { in InternalTokenize()
[all …]
/external/python/oauth2client/tests/contrib/
Dtest_multiprocess_file_storage.py66 def _generate_token_response_http(new_token='new_token'): argument
68 'access_token': new_token,
/external/mesa3d/src/compiler/glsl/glcpp/
Dglcpp-parse.y1035 token_t *new_token = linear_alloc_child(parser->linalloc, sizeof(token_t)); in _token_list_copy() local
1036 *new_token = *node->token; in _token_list_copy()
1037 _token_list_append (parser, copy, new_token); in _token_list_copy()
1792 token_t *new_token; in _glcpp_parser_expand_function() local
1794 new_token = _token_create_ival(parser, PLACEHOLDER, in _glcpp_parser_expand_function()
1796 _token_list_append(parser, substituted, new_token); in _glcpp_parser_expand_function()
/external/libtextclassifier/annotator/
Dfeature-processor.cc119 Token new_token(token_word.UTF8Substring(last_start, split_point), in SplitTokensOnSelectionBoundaries() local
124 current_pos = new_token.end; in SplitTokensOnSelectionBoundaries()
126 replacement_tokens.push_back(new_token); in SplitTokensOnSelectionBoundaries()