Searched refs:CreateTokenizer (Results 1 – 3 of 3) sorted by relevance
45 std::unique_ptr<Tokenizer> CreateTokenizer( in CreateTokenizer() function71 tokenizer_(CreateTokenizer(options->tokenizer_options(), unilib)), in ActionsFeatureProcessor()
33 std::unique_ptr<Tokenizer> CreateTokenizer(
82 owned_tokenizer_ = CreateTokenizer(model->tokenizer_options(), unilib); in NGramModel()