1import io
2import shlex
3import string
4import unittest
5
6
7
8# The original test data set was from shellwords, by Hartmut Goebel.
9
10data = r"""x|x|
11foo bar|foo|bar|
12 foo bar|foo|bar|
13 foo bar |foo|bar|
14foo   bar    bla     fasel|foo|bar|bla|fasel|
15x y  z              xxxx|x|y|z|xxxx|
16\x bar|\|x|bar|
17\ x bar|\|x|bar|
18\ bar|\|bar|
19foo \x bar|foo|\|x|bar|
20foo \ x bar|foo|\|x|bar|
21foo \ bar|foo|\|bar|
22foo "bar" bla|foo|"bar"|bla|
23"foo" "bar" "bla"|"foo"|"bar"|"bla"|
24"foo" bar "bla"|"foo"|bar|"bla"|
25"foo" bar bla|"foo"|bar|bla|
26foo 'bar' bla|foo|'bar'|bla|
27'foo' 'bar' 'bla'|'foo'|'bar'|'bla'|
28'foo' bar 'bla'|'foo'|bar|'bla'|
29'foo' bar bla|'foo'|bar|bla|
30blurb foo"bar"bar"fasel" baz|blurb|foo"bar"bar"fasel"|baz|
31blurb foo'bar'bar'fasel' baz|blurb|foo'bar'bar'fasel'|baz|
32""|""|
33''|''|
34foo "" bar|foo|""|bar|
35foo '' bar|foo|''|bar|
36foo "" "" "" bar|foo|""|""|""|bar|
37foo '' '' '' bar|foo|''|''|''|bar|
38\""|\|""|
39"\"|"\"|
40"foo\ bar"|"foo\ bar"|
41"foo\\ bar"|"foo\\ bar"|
42"foo\\ bar\"|"foo\\ bar\"|
43"foo\\" bar\""|"foo\\"|bar|\|""|
44"foo\\ bar\" dfadf"|"foo\\ bar\"|dfadf"|
45"foo\\\ bar\" dfadf"|"foo\\\ bar\"|dfadf"|
46"foo\\\x bar\" dfadf"|"foo\\\x bar\"|dfadf"|
47"foo\x bar\" dfadf"|"foo\x bar\"|dfadf"|
48\''|\|''|
49'foo\ bar'|'foo\ bar'|
50'foo\\ bar'|'foo\\ bar'|
51"foo\\\x bar\" df'a\ 'df'|"foo\\\x bar\"|df'a|\|'df'|
52\"foo"|\|"foo"|
53\"foo"\x|\|"foo"|\|x|
54"foo\x"|"foo\x"|
55"foo\ "|"foo\ "|
56foo\ xx|foo|\|xx|
57foo\ x\x|foo|\|x|\|x|
58foo\ x\x\""|foo|\|x|\|x|\|""|
59"foo\ x\x"|"foo\ x\x"|
60"foo\ x\x\\"|"foo\ x\x\\"|
61"foo\ x\x\\""foobar"|"foo\ x\x\\"|"foobar"|
62"foo\ x\x\\"\''"foobar"|"foo\ x\x\\"|\|''|"foobar"|
63"foo\ x\x\\"\'"fo'obar"|"foo\ x\x\\"|\|'"fo'|obar"|
64"foo\ x\x\\"\'"fo'obar" 'don'\''t'|"foo\ x\x\\"|\|'"fo'|obar"|'don'|\|''|t'|
65'foo\ bar'|'foo\ bar'|
66'foo\\ bar'|'foo\\ bar'|
67foo\ bar|foo|\|bar|
68foo#bar\nbaz|foobaz|
69:-) ;-)|:|-|)|;|-|)|
70áéíóú|á|é|í|ó|ú|
71"""
72
73posix_data = r"""x|x|
74foo bar|foo|bar|
75 foo bar|foo|bar|
76 foo bar |foo|bar|
77foo   bar    bla     fasel|foo|bar|bla|fasel|
78x y  z              xxxx|x|y|z|xxxx|
79\x bar|x|bar|
80\ x bar| x|bar|
81\ bar| bar|
82foo \x bar|foo|x|bar|
83foo \ x bar|foo| x|bar|
84foo \ bar|foo| bar|
85foo "bar" bla|foo|bar|bla|
86"foo" "bar" "bla"|foo|bar|bla|
87"foo" bar "bla"|foo|bar|bla|
88"foo" bar bla|foo|bar|bla|
89foo 'bar' bla|foo|bar|bla|
90'foo' 'bar' 'bla'|foo|bar|bla|
91'foo' bar 'bla'|foo|bar|bla|
92'foo' bar bla|foo|bar|bla|
93blurb foo"bar"bar"fasel" baz|blurb|foobarbarfasel|baz|
94blurb foo'bar'bar'fasel' baz|blurb|foobarbarfasel|baz|
95""||
96''||
97foo "" bar|foo||bar|
98foo '' bar|foo||bar|
99foo "" "" "" bar|foo||||bar|
100foo '' '' '' bar|foo||||bar|
101\"|"|
102"\""|"|
103"foo\ bar"|foo\ bar|
104"foo\\ bar"|foo\ bar|
105"foo\\ bar\""|foo\ bar"|
106"foo\\" bar\"|foo\|bar"|
107"foo\\ bar\" dfadf"|foo\ bar" dfadf|
108"foo\\\ bar\" dfadf"|foo\\ bar" dfadf|
109"foo\\\x bar\" dfadf"|foo\\x bar" dfadf|
110"foo\x bar\" dfadf"|foo\x bar" dfadf|
111\'|'|
112'foo\ bar'|foo\ bar|
113'foo\\ bar'|foo\\ bar|
114"foo\\\x bar\" df'a\ 'df"|foo\\x bar" df'a\ 'df|
115\"foo|"foo|
116\"foo\x|"foox|
117"foo\x"|foo\x|
118"foo\ "|foo\ |
119foo\ xx|foo xx|
120foo\ x\x|foo xx|
121foo\ x\x\"|foo xx"|
122"foo\ x\x"|foo\ x\x|
123"foo\ x\x\\"|foo\ x\x\|
124"foo\ x\x\\""foobar"|foo\ x\x\foobar|
125"foo\ x\x\\"\'"foobar"|foo\ x\x\'foobar|
126"foo\ x\x\\"\'"fo'obar"|foo\ x\x\'fo'obar|
127"foo\ x\x\\"\'"fo'obar" 'don'\''t'|foo\ x\x\'fo'obar|don't|
128"foo\ x\x\\"\'"fo'obar" 'don'\''t' \\|foo\ x\x\'fo'obar|don't|\|
129'foo\ bar'|foo\ bar|
130'foo\\ bar'|foo\\ bar|
131foo\ bar|foo bar|
132foo#bar\nbaz|foo|baz|
133:-) ;-)|:-)|;-)|
134áéíóú|áéíóú|
135"""
136
137class ShlexTest(unittest.TestCase):
138    def setUp(self):
139        self.data = [x.split("|")[:-1]
140                     for x in data.splitlines()]
141        self.posix_data = [x.split("|")[:-1]
142                           for x in posix_data.splitlines()]
143        for item in self.data:
144            item[0] = item[0].replace(r"\n", "\n")
145        for item in self.posix_data:
146            item[0] = item[0].replace(r"\n", "\n")
147
148    def splitTest(self, data, comments):
149        for i in range(len(data)):
150            l = shlex.split(data[i][0], comments=comments)
151            self.assertEqual(l, data[i][1:],
152                             "%s: %s != %s" %
153                             (data[i][0], l, data[i][1:]))
154
155    def oldSplit(self, s):
156        ret = []
157        lex = shlex.shlex(io.StringIO(s))
158        tok = lex.get_token()
159        while tok:
160            ret.append(tok)
161            tok = lex.get_token()
162        return ret
163
164    def testSplitPosix(self):
165        """Test data splitting with posix parser"""
166        self.splitTest(self.posix_data, comments=True)
167
168    def testCompat(self):
169        """Test compatibility interface"""
170        for i in range(len(self.data)):
171            l = self.oldSplit(self.data[i][0])
172            self.assertEqual(l, self.data[i][1:],
173                             "%s: %s != %s" %
174                             (self.data[i][0], l, self.data[i][1:]))
175
176    def testSyntaxSplitAmpersandAndPipe(self):
177        """Test handling of syntax splitting of &, |"""
178        # Could take these forms: &&, &, |&, ;&, ;;&
179        # of course, the same applies to | and ||
180        # these should all parse to the same output
181        for delimiter in ('&&', '&', '|&', ';&', ';;&',
182                          '||', '|', '&|', ';|', ';;|'):
183            src = ['echo hi %s echo bye' % delimiter,
184                   'echo hi%secho bye' % delimiter]
185            ref = ['echo', 'hi', delimiter, 'echo', 'bye']
186            for ss in src:
187                s = shlex.shlex(ss, punctuation_chars=True)
188                result = list(s)
189                self.assertEqual(ref, result, "While splitting '%s'" % ss)
190
191    def testSyntaxSplitSemicolon(self):
192        """Test handling of syntax splitting of ;"""
193        # Could take these forms: ;, ;;, ;&, ;;&
194        # these should all parse to the same output
195        for delimiter in (';', ';;', ';&', ';;&'):
196            src = ['echo hi %s echo bye' % delimiter,
197                   'echo hi%s echo bye' % delimiter,
198                   'echo hi%secho bye' % delimiter]
199            ref = ['echo', 'hi', delimiter, 'echo', 'bye']
200            for ss in src:
201                s = shlex.shlex(ss, punctuation_chars=True)
202                result = list(s)
203                self.assertEqual(ref, result, "While splitting '%s'" % ss)
204
205    def testSyntaxSplitRedirect(self):
206        """Test handling of syntax splitting of >"""
207        # of course, the same applies to <, |
208        # these should all parse to the same output
209        for delimiter in ('<', '|'):
210            src = ['echo hi %s out' % delimiter,
211                   'echo hi%s out' % delimiter,
212                   'echo hi%sout' % delimiter]
213            ref = ['echo', 'hi', delimiter, 'out']
214            for ss in src:
215                s = shlex.shlex(ss, punctuation_chars=True)
216                result = list(s)
217                self.assertEqual(ref, result, "While splitting '%s'" % ss)
218
219    def testSyntaxSplitParen(self):
220        """Test handling of syntax splitting of ()"""
221        # these should all parse to the same output
222        src = ['( echo hi )',
223               '(echo hi)']
224        ref = ['(', 'echo', 'hi', ')']
225        for ss in src:
226            s = shlex.shlex(ss, punctuation_chars=True)
227            result = list(s)
228            self.assertEqual(ref, result, "While splitting '%s'" % ss)
229
230    def testSyntaxSplitCustom(self):
231        """Test handling of syntax splitting with custom chars"""
232        ref = ['~/a', '&', '&', 'b-c', '--color=auto', '||', 'd', '*.py?']
233        ss = "~/a && b-c --color=auto || d *.py?"
234        s = shlex.shlex(ss, punctuation_chars="|")
235        result = list(s)
236        self.assertEqual(ref, result, "While splitting '%s'" % ss)
237
238    def testTokenTypes(self):
239        """Test that tokens are split with types as expected."""
240        for source, expected in (
241                                ('a && b || c',
242                                 [('a', 'a'), ('&&', 'c'), ('b', 'a'),
243                                  ('||', 'c'), ('c', 'a')]),
244                              ):
245            s = shlex.shlex(source, punctuation_chars=True)
246            observed = []
247            while True:
248                t = s.get_token()
249                if t == s.eof:
250                    break
251                if t[0] in s.punctuation_chars:
252                    tt = 'c'
253                else:
254                    tt = 'a'
255                observed.append((t, tt))
256            self.assertEqual(observed, expected)
257
258    def testPunctuationInWordChars(self):
259        """Test that any punctuation chars are removed from wordchars"""
260        s = shlex.shlex('a_b__c', punctuation_chars='_')
261        self.assertNotIn('_', s.wordchars)
262        self.assertEqual(list(s), ['a', '_', 'b', '__', 'c'])
263
264    def testPunctuationWithWhitespaceSplit(self):
265        """Test that with whitespace_split, behaviour is as expected"""
266        s = shlex.shlex('a  && b  ||  c', punctuation_chars='&')
267        # whitespace_split is False, so splitting will be based on
268        # punctuation_chars
269        self.assertEqual(list(s), ['a', '&&', 'b', '|', '|', 'c'])
270        s = shlex.shlex('a  && b  ||  c', punctuation_chars='&')
271        s.whitespace_split = True
272        # whitespace_split is True, so splitting will be based on
273        # white space
274        self.assertEqual(list(s), ['a', '&&', 'b', '||', 'c'])
275
276    def testPunctuationWithPosix(self):
277        """Test that punctuation_chars and posix behave correctly together."""
278        # see Issue #29132
279        s = shlex.shlex('f >"abc"', posix=True, punctuation_chars=True)
280        self.assertEqual(list(s), ['f', '>', 'abc'])
281        s = shlex.shlex('f >\\"abc\\"', posix=True, punctuation_chars=True)
282        self.assertEqual(list(s), ['f', '>', '"abc"'])
283
284    def testEmptyStringHandling(self):
285        """Test that parsing of empty strings is correctly handled."""
286        # see Issue #21999
287        expected = ['', ')', 'abc']
288        for punct in (False, True):
289            s = shlex.shlex("'')abc", posix=True, punctuation_chars=punct)
290            slist = list(s)
291            self.assertEqual(slist, expected)
292        expected = ["''", ')', 'abc']
293        s = shlex.shlex("'')abc", punctuation_chars=True)
294        self.assertEqual(list(s), expected)
295
296    def testQuote(self):
297        safeunquoted = string.ascii_letters + string.digits + '@%_-+=:,./'
298        unicode_sample = '\xe9\xe0\xdf'  # e + acute accent, a + grave, sharp s
299        unsafe = '"`$\\!' + unicode_sample
300
301        self.assertEqual(shlex.quote(''), "''")
302        self.assertEqual(shlex.quote(safeunquoted), safeunquoted)
303        self.assertEqual(shlex.quote('test file name'), "'test file name'")
304        for u in unsafe:
305            self.assertEqual(shlex.quote('test%sname' % u),
306                             "'test%sname'" % u)
307        for u in unsafe:
308            self.assertEqual(shlex.quote("test%s'name'" % u),
309                             "'test%s'\"'\"'name'\"'\"''" % u)
310
311# Allow this test to be used with old shlex.py
312if not getattr(shlex, "split", None):
313    for methname in dir(ShlexTest):
314        if methname.startswith("test") and methname != "testCompat":
315            delattr(ShlexTest, methname)
316
317if __name__ == "__main__":
318    unittest.main()
319