Searched refs:codepoint_config (Results 1 – 2 of 2) sorted by relevance
47 std::vector<const TokenizationCodepointRange*> codepoint_config; in CreateTokenizer() local49 codepoint_config.insert(codepoint_config.end(), in CreateTokenizer()64 options->type(), unilib, codepoint_config, internal_codepoint_config, in CreateTokenizer()
33 std::vector<const TokenizationCodepointRange*> codepoint_config; in BuildTokenizer() local35 codepoint_config.insert(codepoint_config.end(), in BuildTokenizer()49 return Tokenizer(options->tokenization_type(), unilib, codepoint_config, in BuildTokenizer()