1# (c) 2005 Ian Bicking and contributors; written for Paste (http://pythonpaste.org)
2# Licensed under the MIT license: http://www.opensource.org/licenses/mit-license.php
3#!/usr/bin/env python2.4
4# (c) 2005 Ian Bicking and contributors; written for Paste (http://pythonpaste.org)
5# Licensed under the MIT license: http://www.opensource.org/licenses/mit-license.php
6
7"""
8These are functions for use when doctest-testing a document.
9"""
10
11import subprocess
12import doctest
13import os
14import sys
15import shutil
16import re
17import cgi
18import rfc822
19from cStringIO import StringIO
20from paste.util import PySourceColor
21
22
23here = os.path.abspath(__file__)
24paste_parent = os.path.dirname(
25    os.path.dirname(os.path.dirname(here)))
26
27def run(command):
28    data = run_raw(command)
29    if data:
30        print(data)
31
32def run_raw(command):
33    """
34    Runs the string command, returns any output.
35    """
36    proc = subprocess.Popen(command, shell=True,
37                            stderr=subprocess.STDOUT,
38                            stdout=subprocess.PIPE, env=_make_env())
39    data = proc.stdout.read()
40    proc.wait()
41    while data.endswith('\n') or data.endswith('\r'):
42        data = data[:-1]
43    if data:
44        data = '\n'.join(
45            [l for l in data.splitlines() if l])
46        return data
47    else:
48        return ''
49
50def run_command(command, name, and_print=False):
51    output = run_raw(command)
52    data = '$ %s\n%s' % (command, output)
53    show_file('shell-command', name, description='shell transcript',
54              data=data)
55    if and_print and output:
56        print(output)
57
58def _make_env():
59    env = os.environ.copy()
60    env['PATH'] = (env.get('PATH', '')
61                   + ':'
62                   + os.path.join(paste_parent, 'scripts')
63                   + ':'
64                   + os.path.join(paste_parent, 'paste', '3rd-party',
65                                  'sqlobject-files', 'scripts'))
66    env['PYTHONPATH'] = (env.get('PYTHONPATH', '')
67                         + ':'
68                         + paste_parent)
69    return env
70
71def clear_dir(dir):
72    """
73    Clears (deletes) the given directory
74    """
75    shutil.rmtree(dir, True)
76
77def ls(dir=None, recurse=False, indent=0):
78    """
79    Show a directory listing
80    """
81    dir = dir or os.getcwd()
82    fns = os.listdir(dir)
83    fns.sort()
84    for fn in fns:
85        full = os.path.join(dir, fn)
86        if os.path.isdir(full):
87            fn = fn + '/'
88        print(' '*indent + fn)
89        if os.path.isdir(full) and recurse:
90            ls(dir=full, recurse=True, indent=indent+2)
91
92default_app = None
93default_url = None
94
95def set_default_app(app, url):
96    global default_app
97    global default_url
98    default_app = app
99    default_url = url
100
101def resource_filename(fn):
102    """
103    Returns the filename of the resource -- generally in the directory
104    resources/DocumentName/fn
105    """
106    return os.path.join(
107        os.path.dirname(sys.testing_document_filename),
108        'resources',
109        os.path.splitext(os.path.basename(sys.testing_document_filename))[0],
110        fn)
111
112def show(path_info, example_name):
113    fn = resource_filename(example_name + '.html')
114    out = StringIO()
115    assert default_app is not None, (
116        "No default_app set")
117    url = default_url + path_info
118    out.write('<span class="doctest-url"><a href="%s">%s</a></span><br>\n'
119              % (url, url))
120    out.write('<div class="doctest-example">\n')
121    proc = subprocess.Popen(
122        ['paster', 'serve' '--server=console', '--no-verbose',
123         '--url=' + path_info],
124        stderr=subprocess.PIPE,
125        stdout=subprocess.PIPE,
126        env=_make_env())
127    stdout, errors = proc.communicate()
128    stdout = StringIO(stdout)
129    headers = rfc822.Message(stdout)
130    content = stdout.read()
131    for header, value in headers.items():
132        if header.lower() == 'status' and int(value.split()[0]) == 200:
133            continue
134        if header.lower() in ('content-type', 'content-length'):
135            continue
136        if (header.lower() == 'set-cookie'
137            and value.startswith('_SID_')):
138            continue
139        out.write('<span class="doctest-header">%s: %s</span><br>\n'
140                  % (header, value))
141    lines = [l for l in content.splitlines() if l.strip()]
142    for line in lines:
143        out.write(line + '\n')
144    if errors:
145        out.write('<pre class="doctest-errors">%s</pre>'
146                  % errors)
147    out.write('</div>\n')
148    result = out.getvalue()
149    if not os.path.exists(fn):
150        f = open(fn, 'wb')
151        f.write(result)
152        f.close()
153    else:
154        f = open(fn, 'rb')
155        expected = f.read()
156        f.close()
157        if not html_matches(expected, result):
158            print('Pages did not match.  Expected from %s:' % fn)
159            print('-'*60)
160            print(expected)
161            print('='*60)
162            print('Actual output:')
163            print('-'*60)
164            print(result)
165
166def html_matches(pattern, text):
167    regex = re.escape(pattern)
168    regex = regex.replace(r'\.\.\.', '.*')
169    regex = re.sub(r'0x[0-9a-f]+', '.*', regex)
170    regex = '^%s$' % regex
171    return re.search(regex, text)
172
173def convert_docstring_string(data):
174    if data.startswith('\n'):
175        data = data[1:]
176    lines = data.splitlines()
177    new_lines = []
178    for line in lines:
179        if line.rstrip() == '.':
180            new_lines.append('')
181        else:
182            new_lines.append(line)
183    data = '\n'.join(new_lines) + '\n'
184    return data
185
186def create_file(path, version, data):
187    data = convert_docstring_string(data)
188    write_data(path, data)
189    show_file(path, version)
190
191def append_to_file(path, version, data):
192    data = convert_docstring_string(data)
193    f = open(path, 'a')
194    f.write(data)
195    f.close()
196    # I think these appends can happen so quickly (in less than a second)
197    # that the .pyc file doesn't appear to be expired, even though it
198    # is after we've made this change; so we have to get rid of the .pyc
199    # file:
200    if path.endswith('.py'):
201        pyc_file = path + 'c'
202        if os.path.exists(pyc_file):
203            os.unlink(pyc_file)
204    show_file(path, version, description='added to %s' % path,
205              data=data)
206
207def show_file(path, version, description=None, data=None):
208    ext = os.path.splitext(path)[1]
209    if data is None:
210        f = open(path, 'rb')
211        data = f.read()
212        f.close()
213    if ext == '.py':
214        html = ('<div class="source-code">%s</div>'
215                % PySourceColor.str2html(data, PySourceColor.dark))
216    else:
217        html = '<pre class="source-code">%s</pre>' % cgi.escape(data, 1)
218    html = '<span class="source-filename">%s</span><br>%s' % (
219        description or path, html)
220    write_data(resource_filename('%s.%s.gen.html' % (path, version)),
221               html)
222
223def call_source_highlight(input, format):
224    proc = subprocess.Popen(['source-highlight', '--out-format=html',
225                             '--no-doc', '--css=none',
226                             '--src-lang=%s' % format], shell=False,
227                            stdout=subprocess.PIPE)
228    stdout, stderr = proc.communicate(input)
229    result = stdout
230    proc.wait()
231    return result
232
233
234def write_data(path, data):
235    dir = os.path.dirname(os.path.abspath(path))
236    if not os.path.exists(dir):
237        os.makedirs(dir)
238    f = open(path, 'wb')
239    f.write(data)
240    f.close()
241
242
243def change_file(path, changes):
244    f = open(os.path.abspath(path), 'rb')
245    lines = f.readlines()
246    f.close()
247    for change_type, line, text in changes:
248        if change_type == 'insert':
249            lines[line:line] = [text]
250        elif change_type == 'delete':
251            lines[line:text] = []
252        else:
253            assert 0, (
254                "Unknown change_type: %r" % change_type)
255    f = open(path, 'wb')
256    f.write(''.join(lines))
257    f.close()
258
259class LongFormDocTestParser(doctest.DocTestParser):
260
261    """
262    This parser recognizes some reST comments as commands, without
263    prompts or expected output, like:
264
265    .. run:
266
267        do_this(...
268        ...)
269    """
270
271    _EXAMPLE_RE = re.compile(r"""
272        # Source consists of a PS1 line followed by zero or more PS2 lines.
273        (?: (?P<source>
274                (?:^(?P<indent> [ ]*) >>>    .*)    # PS1 line
275                (?:\n           [ ]*  \.\.\. .*)*)  # PS2 lines
276            \n?
277            # Want consists of any non-blank lines that do not start with PS1.
278            (?P<want> (?:(?![ ]*$)    # Not a blank line
279                         (?![ ]*>>>)  # Not a line starting with PS1
280                         .*$\n?       # But any other line
281                      )*))
282        |
283        (?: # This is for longer commands that are prefixed with a reST
284            # comment like '.. run:' (two colons makes that a directive).
285            # These commands cannot have any output.
286
287            (?:^\.\.[ ]*(?P<run>run):[ ]*\n) # Leading command/command
288            (?:[ ]*\n)?         # Blank line following
289            (?P<runsource>
290                (?:(?P<runindent> [ ]+)[^ ].*$)
291                (?:\n [ ]+ .*)*)
292            )
293        |
294        (?: # This is for shell commands
295
296            (?P<shellsource>
297                (?:^(P<shellindent> [ ]*) [$] .*)   # Shell line
298                (?:\n               [ ]*  [>] .*)*) # Continuation
299            \n?
300            # Want consists of any non-blank lines that do not start with $
301            (?P<shellwant> (?:(?![ ]*$)
302                              (?![ ]*[$]$)
303                              .*$\n?
304                           )*))
305        """, re.MULTILINE | re.VERBOSE)
306
307    def _parse_example(self, m, name, lineno):
308        r"""
309        Given a regular expression match from `_EXAMPLE_RE` (`m`),
310        return a pair `(source, want)`, where `source` is the matched
311        example's source code (with prompts and indentation stripped);
312        and `want` is the example's expected output (with indentation
313        stripped).
314
315        `name` is the string's name, and `lineno` is the line number
316        where the example starts; both are used for error messages.
317
318        >>> def parseit(s):
319        ...     p = LongFormDocTestParser()
320        ...     return p._parse_example(p._EXAMPLE_RE.search(s), '<string>', 1)
321        >>> parseit('>>> 1\n1')
322        ('1', {}, '1', None)
323        >>> parseit('>>> (1\n... +1)\n2')
324        ('(1\n+1)', {}, '2', None)
325        >>> parseit('.. run:\n\n    test1\n    test2\n')
326        ('test1\ntest2', {}, '', None)
327        """
328        # Get the example's indentation level.
329        runner = m.group('run') or ''
330        indent = len(m.group('%sindent' % runner))
331
332        # Divide source into lines; check that they're properly
333        # indented; and then strip their indentation & prompts.
334        source_lines = m.group('%ssource' % runner).split('\n')
335        if runner:
336            self._check_prefix(source_lines[1:], ' '*indent, name, lineno)
337        else:
338            self._check_prompt_blank(source_lines, indent, name, lineno)
339            self._check_prefix(source_lines[2:], ' '*indent + '.', name, lineno)
340        if runner:
341            source = '\n'.join([sl[indent:] for sl in source_lines])
342        else:
343            source = '\n'.join([sl[indent+4:] for sl in source_lines])
344
345        if runner:
346            want = ''
347            exc_msg = None
348        else:
349            # Divide want into lines; check that it's properly indented; and
350            # then strip the indentation.  Spaces before the last newline should
351            # be preserved, so plain rstrip() isn't good enough.
352            want = m.group('want')
353            want_lines = want.split('\n')
354            if len(want_lines) > 1 and re.match(r' *$', want_lines[-1]):
355                del want_lines[-1]  # forget final newline & spaces after it
356            self._check_prefix(want_lines, ' '*indent, name,
357                               lineno + len(source_lines))
358            want = '\n'.join([wl[indent:] for wl in want_lines])
359
360            # If `want` contains a traceback message, then extract it.
361            m = self._EXCEPTION_RE.match(want)
362            if m:
363                exc_msg = m.group('msg')
364            else:
365                exc_msg = None
366
367        # Extract options from the source.
368        options = self._find_options(source, name, lineno)
369
370        return source, options, want, exc_msg
371
372
373    def parse(self, string, name='<string>'):
374        """
375        Divide the given string into examples and intervening text,
376        and return them as a list of alternating Examples and strings.
377        Line numbers for the Examples are 0-based.  The optional
378        argument `name` is a name identifying this string, and is only
379        used for error messages.
380        """
381        string = string.expandtabs()
382        # If all lines begin with the same indentation, then strip it.
383        min_indent = self._min_indent(string)
384        if min_indent > 0:
385            string = '\n'.join([l[min_indent:] for l in string.split('\n')])
386
387        output = []
388        charno, lineno = 0, 0
389        # Find all doctest examples in the string:
390        for m in self._EXAMPLE_RE.finditer(string):
391            # Add the pre-example text to `output`.
392            output.append(string[charno:m.start()])
393            # Update lineno (lines before this example)
394            lineno += string.count('\n', charno, m.start())
395            # Extract info from the regexp match.
396            (source, options, want, exc_msg) = \
397                     self._parse_example(m, name, lineno)
398            # Create an Example, and add it to the list.
399            if not self._IS_BLANK_OR_COMMENT(source):
400                # @@: Erg, this is the only line I need to change...
401                output.append(doctest.Example(
402                    source, want, exc_msg,
403                    lineno=lineno,
404                    indent=min_indent+len(m.group('indent') or m.group('runindent')),
405                    options=options))
406            # Update lineno (lines inside this example)
407            lineno += string.count('\n', m.start(), m.end())
408            # Update charno.
409            charno = m.end()
410        # Add any remaining post-example text to `output`.
411        output.append(string[charno:])
412        return output
413
414
415
416if __name__ == '__main__':
417    if sys.argv[1:] and sys.argv[1] == 'doctest':
418        doctest.testmod()
419        sys.exit()
420    if not paste_parent in sys.path:
421        sys.path.append(paste_parent)
422    for fn in sys.argv[1:]:
423        fn = os.path.abspath(fn)
424        # @@: OK, ick; but this module gets loaded twice
425        sys.testing_document_filename = fn
426        doctest.testfile(
427            fn, module_relative=False,
428            optionflags=doctest.ELLIPSIS|doctest.REPORT_ONLY_FIRST_FAILURE,
429            parser=LongFormDocTestParser())
430        new = os.path.splitext(fn)[0] + '.html'
431        assert new != fn
432        os.system('rst2html.py %s > %s' % (fn, new))
433