Lines Matching refs:tokenizer
46 Tokenizer tokenizer("a bcd ef"); variable
49 CHECK(tokenizer.next() == "a");
50 CHECK(tokenizer.next() == "bcd");
51 CHECK(tokenizer.next() == "ef");
52 CHECK(tokenizer.next() == "");
60 CHECK(tokenizer.split() == expected);
65 Tokenizer tokenizer(""); variable
68 CHECK(tokenizer.next() == "");
73 CHECK(tokenizer.split().empty());
78 Tokenizer tokenizer("/a/bcd/ef g/h/", "/"); variable
81 CHECK(tokenizer.next() == "a");
82 CHECK(tokenizer.next() == "bcd");
83 CHECK(tokenizer.next() == "ef g");
84 CHECK(tokenizer.next() == "h");
85 CHECK(tokenizer.next() == "");
94 CHECK(tokenizer.split() == expected);
99 Tokenizer tokenizer(" a \n\t bc "); variable
102 CHECK(tokenizer.next() == "a");
103 CHECK(tokenizer.next() == "bc");
104 CHECK(tokenizer.next() == "");
111 CHECK(tokenizer.split() == expected);