1 //===- unittest/Support/YAMLParserTest ------------------------------------===//
2 //
3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4 // See https://llvm.org/LICENSE.txt for license information.
5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6 //
7 //===----------------------------------------------------------------------===//
8 
9 #include "llvm/Support/YAMLParser.h"
10 #include "llvm/ADT/Twine.h"
11 #include "llvm/Support/Casting.h"
12 #include "llvm/Support/MemoryBuffer.h"
13 #include "llvm/Support/SourceMgr.h"
14 #include "gtest/gtest.h"
15 
16 namespace llvm {
17 
SuppressDiagnosticsOutput(const SMDiagnostic &,void *)18 static void SuppressDiagnosticsOutput(const SMDiagnostic &, void *) {
19   // Prevent SourceMgr from writing errors to stderr
20   // to reduce noise in unit test runs.
21 }
22 
23 // Assumes Ctx is an SMDiagnostic where Diag can be stored.
CollectDiagnosticsOutput(const SMDiagnostic & Diag,void * Ctx)24 static void CollectDiagnosticsOutput(const SMDiagnostic &Diag, void *Ctx) {
25   SMDiagnostic* DiagOut = static_cast<SMDiagnostic*>(Ctx);
26   *DiagOut = Diag;
27 }
28 
29 // Checks that the given input gives a parse error. Makes sure that an error
30 // text is available and the parse fails.
ExpectParseError(StringRef Message,StringRef Input)31 static void ExpectParseError(StringRef Message, StringRef Input) {
32   SourceMgr SM;
33   yaml::Stream Stream(Input, SM);
34   SM.setDiagHandler(SuppressDiagnosticsOutput);
35   EXPECT_FALSE(Stream.validate()) << Message << ": " << Input;
36   EXPECT_TRUE(Stream.failed()) << Message << ": " << Input;
37 }
38 
39 // Checks that the given input can be parsed without error.
ExpectParseSuccess(StringRef Message,StringRef Input)40 static void ExpectParseSuccess(StringRef Message, StringRef Input) {
41   SourceMgr SM;
42   yaml::Stream Stream(Input, SM);
43   EXPECT_TRUE(Stream.validate()) << Message << ": " << Input;
44 }
45 
TEST(YAMLParser,ParsesEmptyArray)46 TEST(YAMLParser, ParsesEmptyArray) {
47   ExpectParseSuccess("Empty array", "[]");
48 }
49 
TEST(YAMLParser,FailsIfNotClosingArray)50 TEST(YAMLParser, FailsIfNotClosingArray) {
51   ExpectParseError("Not closing array", "[");
52   ExpectParseError("Not closing array", "  [  ");
53   ExpectParseError("Not closing array", "  [x");
54 }
55 
TEST(YAMLParser,ParsesEmptyArrayWithWhitespace)56 TEST(YAMLParser, ParsesEmptyArrayWithWhitespace) {
57   ExpectParseSuccess("Array with spaces", "  [  ]  ");
58   ExpectParseSuccess("All whitespaces", "\t\r\n[\t\n \t\r ]\t\r \n\n");
59 }
60 
TEST(YAMLParser,ParsesEmptyObject)61 TEST(YAMLParser, ParsesEmptyObject) {
62   ExpectParseSuccess("Empty object", "[{}]");
63 }
64 
TEST(YAMLParser,ParsesObject)65 TEST(YAMLParser, ParsesObject) {
66   ExpectParseSuccess("Object with an entry", "[{\"a\":\"/b\"}]");
67 }
68 
TEST(YAMLParser,ParsesMultipleKeyValuePairsInObject)69 TEST(YAMLParser, ParsesMultipleKeyValuePairsInObject) {
70   ExpectParseSuccess("Multiple key, value pairs",
71                      "[{\"a\":\"/b\",\"c\":\"d\",\"e\":\"f\"}]");
72 }
73 
TEST(YAMLParser,FailsIfNotClosingObject)74 TEST(YAMLParser, FailsIfNotClosingObject) {
75   ExpectParseError("Missing close on empty", "[{]");
76   ExpectParseError("Missing close after pair", "[{\"a\":\"b\"]");
77 }
78 
TEST(YAMLParser,FailsIfMissingColon)79 TEST(YAMLParser, FailsIfMissingColon) {
80   ExpectParseError("Missing colon between key and value", "[{\"a\"\"/b\"}]");
81   ExpectParseError("Missing colon between key and value", "[{\"a\" \"b\"}]");
82 }
83 
TEST(YAMLParser,FailsOnMissingQuote)84 TEST(YAMLParser, FailsOnMissingQuote) {
85   ExpectParseError("Missing open quote", "[{a\":\"b\"}]");
86   ExpectParseError("Missing closing quote", "[{\"a\":\"b}]");
87 }
88 
TEST(YAMLParser,ParsesEscapedQuotes)89 TEST(YAMLParser, ParsesEscapedQuotes) {
90   ExpectParseSuccess("Parses escaped string in key and value",
91                      "[{\"a\":\"\\\"b\\\"  \\\" \\\"\"}]");
92 }
93 
TEST(YAMLParser,ParsesEmptyString)94 TEST(YAMLParser, ParsesEmptyString) {
95   ExpectParseSuccess("Parses empty string in value", "[{\"a\":\"\"}]");
96 }
97 
TEST(YAMLParser,ParsesMultipleObjects)98 TEST(YAMLParser, ParsesMultipleObjects) {
99   ExpectParseSuccess(
100       "Multiple objects in array",
101       "["
102       " { \"a\" : \"b\" },"
103       " { \"a\" : \"b\" },"
104       " { \"a\" : \"b\" }"
105       "]");
106 }
107 
TEST(YAMLParser,FailsOnMissingComma)108 TEST(YAMLParser, FailsOnMissingComma) {
109   ExpectParseError(
110       "Missing comma",
111       "["
112       " { \"a\" : \"b\" }"
113       " { \"a\" : \"b\" }"
114       "]");
115 }
116 
TEST(YAMLParser,ParsesSpacesInBetweenTokens)117 TEST(YAMLParser, ParsesSpacesInBetweenTokens) {
118   ExpectParseSuccess(
119       "Various whitespace between tokens",
120       " \t \n\n \r [ \t \n\n \r"
121       " \t \n\n \r { \t \n\n \r\"a\"\t \n\n \r :"
122       " \t \n\n \r \"b\"\t \n\n \r } \t \n\n \r,\t \n\n \r"
123       " \t \n\n \r { \t \n\n \r\"a\"\t \n\n \r :"
124       " \t \n\n \r \"b\"\t \n\n \r } \t \n\n \r]\t \n\n \r");
125 }
126 
TEST(YAMLParser,ParsesArrayOfArrays)127 TEST(YAMLParser, ParsesArrayOfArrays) {
128   ExpectParseSuccess("Array of arrays", "[[]]");
129 }
130 
TEST(YAMLParser,ParsesBlockLiteralScalars)131 TEST(YAMLParser, ParsesBlockLiteralScalars) {
132   ExpectParseSuccess("Block literal scalar", "test: |\n  Hello\n  World\n");
133   ExpectParseSuccess("Block literal scalar EOF", "test: |\n  Hello\n  World");
134   ExpectParseSuccess("Empty block literal scalar header EOF", "test: | ");
135   ExpectParseSuccess("Empty block literal scalar", "test: |\ntest2: 20");
136   ExpectParseSuccess("Empty block literal scalar 2", "- | \n  \n\n \n- 42");
137   ExpectParseSuccess("Block literal scalar in sequence",
138                      "- |\n  Testing\n  Out\n\n- 22");
139   ExpectParseSuccess("Block literal scalar in document",
140                      "--- |\n  Document\n...");
141   ExpectParseSuccess("Empty non indented lines still count",
142                      "- |\n  First line\n \n\n  Another line\n\n- 2");
143   ExpectParseSuccess("Comment in block literal scalar header",
144                      "test: | # Comment \n  No Comment\ntest 2: | # Void");
145   ExpectParseSuccess("Chomping indicators in block literal scalar header",
146                      "test: |- \n  Hello\n\ntest 2: |+ \n\n  World\n\n\n");
147   ExpectParseSuccess("Indent indicators in block literal scalar header",
148                      "test: |1 \n  \n Hello \n  World\n");
149   ExpectParseSuccess("Chomping and indent indicators in block literals",
150                      "test: |-1\n Hello\ntest 2: |9+\n         World");
151   ExpectParseSuccess("Trailing comments in block literals",
152                      "test: |\n  Content\n # Trailing\n  #Comment\ntest 2: 3");
153   ExpectParseError("Invalid block scalar header", "test: | failure");
154   ExpectParseError("Invalid line indentation", "test: |\n  First line\n Error");
155   ExpectParseError("Long leading space line", "test: |\n   \n  Test\n");
156 }
157 
TEST(YAMLParser,NullTerminatedBlockScalars)158 TEST(YAMLParser, NullTerminatedBlockScalars) {
159   SourceMgr SM;
160   yaml::Stream Stream("test: |\n  Hello\n  World\n", SM);
161   yaml::Document &Doc = *Stream.begin();
162   yaml::MappingNode *Map = cast<yaml::MappingNode>(Doc.getRoot());
163   StringRef Value =
164       cast<yaml::BlockScalarNode>(Map->begin()->getValue())->getValue();
165 
166   EXPECT_EQ(Value, "Hello\nWorld\n");
167   EXPECT_EQ(Value.data()[Value.size()], '\0');
168 }
169 
TEST(YAMLParser,HandlesEndOfFileGracefully)170 TEST(YAMLParser, HandlesEndOfFileGracefully) {
171   ExpectParseError("In string starting with EOF", "[\"");
172   ExpectParseError("In string hitting EOF", "[\"   ");
173   ExpectParseError("In string escaping EOF", "[\"  \\");
174   ExpectParseError("In array starting with EOF", "[");
175   ExpectParseError("In array element starting with EOF", "[[], ");
176   ExpectParseError("In array hitting EOF", "[[] ");
177   ExpectParseError("In array hitting EOF", "[[]");
178   ExpectParseError("In object hitting EOF", "{\"\"");
179 }
180 
TEST(YAMLParser,HandlesNullValuesInKeyValueNodesGracefully)181 TEST(YAMLParser, HandlesNullValuesInKeyValueNodesGracefully) {
182   ExpectParseError("KeyValueNode with null key", "? \"\n:");
183   ExpectParseError("KeyValueNode with null value", "test: '");
184 }
185 
186 // Checks that the given string can be parsed into an identical string inside
187 // of an array.
ExpectCanParseString(StringRef String)188 static void ExpectCanParseString(StringRef String) {
189   std::string StringInArray = (llvm::Twine("[\"") + String + "\"]").str();
190   SourceMgr SM;
191   yaml::Stream Stream(StringInArray, SM);
192   yaml::SequenceNode *ParsedSequence
193     = dyn_cast<yaml::SequenceNode>(Stream.begin()->getRoot());
194   StringRef ParsedString
195     = dyn_cast<yaml::ScalarNode>(
196       static_cast<yaml::Node*>(ParsedSequence->begin()))->getRawValue();
197   ParsedString = ParsedString.substr(1, ParsedString.size() - 2);
198   EXPECT_EQ(String, ParsedString.str());
199 }
200 
201 // Checks that parsing the given string inside an array fails.
ExpectCannotParseString(StringRef String)202 static void ExpectCannotParseString(StringRef String) {
203   std::string StringInArray = (llvm::Twine("[\"") + String + "\"]").str();
204   ExpectParseError((Twine("When parsing string \"") + String + "\"").str(),
205                    StringInArray);
206 }
207 
TEST(YAMLParser,ParsesStrings)208 TEST(YAMLParser, ParsesStrings) {
209   ExpectCanParseString("");
210   ExpectCannotParseString("\\");
211   ExpectCannotParseString("\"");
212   ExpectCanParseString(" ");
213   ExpectCanParseString("\\ ");
214   ExpectCanParseString("\\\"");
215   ExpectCannotParseString("\"\\");
216   ExpectCannotParseString(" \\");
217   ExpectCanParseString("\\\\");
218   ExpectCannotParseString("\\\\\\");
219   ExpectCanParseString("\\\\\\\\");
220   ExpectCanParseString("\\\" ");
221   ExpectCannotParseString("\\\\\" ");
222   ExpectCanParseString("\\\\\\\" ");
223   ExpectCanParseString("    \\\\  \\\"  \\\\\\\"   ");
224 }
225 
TEST(YAMLParser,WorksWithIteratorAlgorithms)226 TEST(YAMLParser, WorksWithIteratorAlgorithms) {
227   SourceMgr SM;
228   yaml::Stream Stream("[\"1\", \"2\", \"3\", \"4\", \"5\", \"6\"]", SM);
229   yaml::SequenceNode *Array
230     = dyn_cast<yaml::SequenceNode>(Stream.begin()->getRoot());
231   EXPECT_EQ(6, std::distance(Array->begin(), Array->end()));
232 }
233 
TEST(YAMLParser,DefaultDiagnosticFilename)234 TEST(YAMLParser, DefaultDiagnosticFilename) {
235   SourceMgr SM;
236 
237   SMDiagnostic GeneratedDiag;
238   SM.setDiagHandler(CollectDiagnosticsOutput, &GeneratedDiag);
239 
240   // When we construct a YAML stream over an unnamed string,
241   // the filename is hard-coded as "YAML".
242   yaml::Stream UnnamedStream("[]", SM);
243   UnnamedStream.printError(UnnamedStream.begin()->getRoot(), "Hello, World!");
244   EXPECT_EQ("YAML", GeneratedDiag.getFilename());
245 }
246 
TEST(YAMLParser,DiagnosticFilenameFromBufferID)247 TEST(YAMLParser, DiagnosticFilenameFromBufferID) {
248   SourceMgr SM;
249 
250   SMDiagnostic GeneratedDiag;
251   SM.setDiagHandler(CollectDiagnosticsOutput, &GeneratedDiag);
252 
253   // When we construct a YAML stream over a named buffer,
254   // we get its ID as filename in diagnostics.
255   std::unique_ptr<MemoryBuffer> Buffer =
256       MemoryBuffer::getMemBuffer("[]", "buffername.yaml");
257   yaml::Stream Stream(Buffer->getMemBufferRef(), SM);
258   Stream.printError(Stream.begin()->getRoot(), "Hello, World!");
259   EXPECT_EQ("buffername.yaml", GeneratedDiag.getFilename());
260 }
261 
TEST(YAMLParser,SameNodeIteratorOperatorNotEquals)262 TEST(YAMLParser, SameNodeIteratorOperatorNotEquals) {
263   SourceMgr SM;
264   yaml::Stream Stream("[\"1\", \"2\"]", SM);
265 
266   yaml::SequenceNode *Node = dyn_cast<yaml::SequenceNode>(
267                                               Stream.begin()->getRoot());
268 
269   auto Begin = Node->begin();
270   auto End = Node->end();
271 
272   EXPECT_TRUE(Begin != End);
273   EXPECT_FALSE(Begin != Begin);
274   EXPECT_FALSE(End != End);
275 }
276 
TEST(YAMLParser,SameNodeIteratorOperatorEquals)277 TEST(YAMLParser, SameNodeIteratorOperatorEquals) {
278   SourceMgr SM;
279   yaml::Stream Stream("[\"1\", \"2\"]", SM);
280 
281   yaml::SequenceNode *Node = dyn_cast<yaml::SequenceNode>(
282                                               Stream.begin()->getRoot());
283 
284   auto Begin = Node->begin();
285   auto End = Node->end();
286 
287   EXPECT_FALSE(Begin == End);
288   EXPECT_TRUE(Begin == Begin);
289   EXPECT_TRUE(End == End);
290 }
291 
TEST(YAMLParser,DifferentNodesIteratorOperatorNotEquals)292 TEST(YAMLParser, DifferentNodesIteratorOperatorNotEquals) {
293   SourceMgr SM;
294   yaml::Stream Stream("[\"1\", \"2\"]", SM);
295   yaml::Stream AnotherStream("[\"1\", \"2\"]", SM);
296 
297   yaml::SequenceNode *Node = dyn_cast<yaml::SequenceNode>(
298                                                   Stream.begin()->getRoot());
299   yaml::SequenceNode *AnotherNode = dyn_cast<yaml::SequenceNode>(
300                                               AnotherStream.begin()->getRoot());
301 
302   auto Begin = Node->begin();
303   auto End = Node->end();
304 
305   auto AnotherBegin = AnotherNode->begin();
306   auto AnotherEnd = AnotherNode->end();
307 
308   EXPECT_TRUE(Begin != AnotherBegin);
309   EXPECT_TRUE(Begin != AnotherEnd);
310   EXPECT_FALSE(End != AnotherEnd);
311 }
312 
TEST(YAMLParser,DifferentNodesIteratorOperatorEquals)313 TEST(YAMLParser, DifferentNodesIteratorOperatorEquals) {
314   SourceMgr SM;
315   yaml::Stream Stream("[\"1\", \"2\"]", SM);
316   yaml::Stream AnotherStream("[\"1\", \"2\"]", SM);
317 
318   yaml::SequenceNode *Node = dyn_cast<yaml::SequenceNode>(
319                                                     Stream.begin()->getRoot());
320   yaml::SequenceNode *AnotherNode = dyn_cast<yaml::SequenceNode>(
321                                              AnotherStream.begin()->getRoot());
322 
323   auto Begin = Node->begin();
324   auto End = Node->end();
325 
326   auto AnotherBegin = AnotherNode->begin();
327   auto AnotherEnd = AnotherNode->end();
328 
329   EXPECT_FALSE(Begin == AnotherBegin);
330   EXPECT_FALSE(Begin == AnotherEnd);
331   EXPECT_TRUE(End == AnotherEnd);
332 }
333 
TEST(YAMLParser,FlowSequenceTokensOutsideFlowSequence)334 TEST(YAMLParser, FlowSequenceTokensOutsideFlowSequence) {
335   auto FlowSequenceStrs = {",", "]", "}"};
336   SourceMgr SM;
337 
338   for (auto &Str : FlowSequenceStrs) {
339     yaml::Stream Stream(Str, SM);
340     yaml::Document &Doc = *Stream.begin();
341     EXPECT_FALSE(Doc.skip());
342   }
343 }
344 
345 } // end namespace llvm
346