Lines Matching refs:new_token
120 Token new_token("", 0, 0); in InternalTokenize() local
133 if (!new_token.value.empty()) { in InternalTokenize()
134 result.push_back(new_token); in InternalTokenize()
136 new_token = Token("", codepoint_index, codepoint_index); in InternalTokenize()
139 new_token.value += std::string( in InternalTokenize()
142 ++new_token.end; in InternalTokenize()
145 if (!new_token.value.empty()) { in InternalTokenize()
146 result.push_back(new_token); in InternalTokenize()
148 new_token = Token("", codepoint_index + 1, codepoint_index + 1); in InternalTokenize()
153 if (!new_token.value.empty()) { in InternalTokenize()
154 result.push_back(new_token); in InternalTokenize()