1"""
2A small templating language
3
4This implements a small templating language for use internally in
5Paste and Paste Script.  This language implements if/elif/else,
6for/continue/break, expressions, and blocks of Python code.  The
7syntax is::
8
9  {{any expression (function calls etc)}}
10  {{any expression | filter}}
11  {{for x in y}}...{{endfor}}
12  {{if x}}x{{elif y}}y{{else}}z{{endif}}
13  {{py:x=1}}
14  {{py:
15  def foo(bar):
16      return 'baz'
17  }}
18  {{default var = default_value}}
19  {{# comment}}
20
21You use this with the ``Template`` class or the ``sub`` shortcut.
22The ``Template`` class takes the template string and the name of
23the template (for errors) and a default namespace.  Then (like
24``string.Template``) you can call the ``tmpl.substitute(**kw)``
25method to make a substitution (or ``tmpl.substitute(a_dict)``).
26
27``sub(content, **kw)`` substitutes the template immediately.  You
28can use ``__name='tmpl.html'`` to set the name of the template.
29
30If there are syntax errors ``TemplateError`` will be raised.
31"""
32
33import re
34import six
35import sys
36import cgi
37from six.moves.urllib.parse import quote
38from paste.util.looper import looper
39
40__all__ = ['TemplateError', 'Template', 'sub', 'HTMLTemplate',
41           'sub_html', 'html', 'bunch']
42
43token_re = re.compile(r'\{\{|\}\}')
44in_re = re.compile(r'\s+in\s+')
45var_re = re.compile(r'^[a-z_][a-z0-9_]*$', re.I)
46
47class TemplateError(Exception):
48    """Exception raised while parsing a template
49    """
50
51    def __init__(self, message, position, name=None):
52        self.message = message
53        self.position = position
54        self.name = name
55
56    def __str__(self):
57        msg = '%s at line %s column %s' % (
58            self.message, self.position[0], self.position[1])
59        if self.name:
60            msg += ' in %s' % self.name
61        return msg
62
63class _TemplateContinue(Exception):
64    pass
65
66class _TemplateBreak(Exception):
67    pass
68
69class Template(object):
70
71    default_namespace = {
72        'start_braces': '{{',
73        'end_braces': '}}',
74        'looper': looper,
75        }
76
77    default_encoding = 'utf8'
78
79    def __init__(self, content, name=None, namespace=None):
80        self.content = content
81        self._unicode = isinstance(content, six.text_type)
82        self.name = name
83
84        if not self._unicode:
85            content = content.decode(self.default_encoding)
86            self._unicode = True
87
88        self._parsed = parse(content, name=name)
89        if namespace is None:
90            namespace = {}
91        self.namespace = namespace
92
93    def from_filename(cls, filename, namespace=None, encoding=None):
94        f = open(filename, 'rb')
95        c = f.read()
96        f.close()
97        if encoding:
98            c = c.decode(encoding)
99        return cls(content=c, name=filename, namespace=namespace)
100
101    from_filename = classmethod(from_filename)
102
103    def __repr__(self):
104        return '<%s %s name=%r>' % (
105            self.__class__.__name__,
106            hex(id(self))[2:], self.name)
107
108    def substitute(self, *args, **kw):
109        if args:
110            if kw:
111                raise TypeError(
112                    "You can only give positional *or* keyword arguments")
113            if len(args) > 1:
114                raise TypeError(
115                    "You can only give on positional argument")
116            kw = args[0]
117        ns = self.default_namespace.copy()
118        ns.update(self.namespace)
119        ns.update(kw)
120        result = self._interpret(ns)
121        return result
122
123    def _interpret(self, ns):
124        __traceback_hide__ = True
125        parts = []
126        self._interpret_codes(self._parsed, ns, out=parts)
127        return ''.join(parts)
128
129    def _interpret_codes(self, codes, ns, out):
130        __traceback_hide__ = True
131        for item in codes:
132            if isinstance(item, six.string_types):
133                out.append(item)
134            else:
135                self._interpret_code(item, ns, out)
136
137    def _interpret_code(self, code, ns, out):
138        __traceback_hide__ = True
139        name, pos = code[0], code[1]
140        if name == 'py':
141            self._exec(code[2], ns, pos)
142        elif name == 'continue':
143            raise _TemplateContinue()
144        elif name == 'break':
145            raise _TemplateBreak()
146        elif name == 'for':
147            vars, expr, content = code[2], code[3], code[4]
148            expr = self._eval(expr, ns, pos)
149            self._interpret_for(vars, expr, content, ns, out)
150        elif name == 'cond':
151            parts = code[2:]
152            self._interpret_if(parts, ns, out)
153        elif name == 'expr':
154            parts = code[2].split('|')
155            base = self._eval(parts[0], ns, pos)
156            for part in parts[1:]:
157                func = self._eval(part, ns, pos)
158                base = func(base)
159            out.append(self._repr(base, pos))
160        elif name == 'default':
161            var, expr = code[2], code[3]
162            if var not in ns:
163                result = self._eval(expr, ns, pos)
164                ns[var] = result
165        elif name == 'comment':
166            return
167        else:
168            assert 0, "Unknown code: %r" % name
169
170    def _interpret_for(self, vars, expr, content, ns, out):
171        __traceback_hide__ = True
172        for item in expr:
173            if len(vars) == 1:
174                ns[vars[0]] = item
175            else:
176                if len(vars) != len(item):
177                    raise ValueError(
178                        'Need %i items to unpack (got %i items)'
179                        % (len(vars), len(item)))
180                for name, value in zip(vars, item):
181                    ns[name] = value
182            try:
183                self._interpret_codes(content, ns, out)
184            except _TemplateContinue:
185                continue
186            except _TemplateBreak:
187                break
188
189    def _interpret_if(self, parts, ns, out):
190        __traceback_hide__ = True
191        # @@: if/else/else gets through
192        for part in parts:
193            assert not isinstance(part, six.string_types)
194            name, pos = part[0], part[1]
195            if name == 'else':
196                result = True
197            else:
198                result = self._eval(part[2], ns, pos)
199            if result:
200                self._interpret_codes(part[3], ns, out)
201                break
202
203    def _eval(self, code, ns, pos):
204        __traceback_hide__ = True
205        try:
206            value = eval(code, ns)
207            return value
208        except:
209            exc_info = sys.exc_info()
210            e = exc_info[1]
211            if getattr(e, 'args'):
212                arg0 = e.args[0]
213            else:
214                arg0 = str(e)
215            e.args = (self._add_line_info(arg0, pos),)
216            six.reraise(exc_info[0], e, exc_info[2])
217
218    def _exec(self, code, ns, pos):
219        __traceback_hide__ = True
220        try:
221            six.exec_(code, ns)
222        except:
223            exc_info = sys.exc_info()
224            e = exc_info[1]
225            e.args = (self._add_line_info(e.args[0], pos),)
226            six.reraise(exc_info[0], e, exc_info[2])
227
228    def _repr(self, value, pos):
229        __traceback_hide__ = True
230        try:
231            if value is None:
232                return ''
233            if self._unicode:
234                try:
235                    value = six.text_type(value)
236                except UnicodeDecodeError:
237                    value = str(value)
238            else:
239                value = str(value)
240        except:
241            exc_info = sys.exc_info()
242            e = exc_info[1]
243            e.args = (self._add_line_info(e.args[0], pos),)
244            six.reraise(exc_info[0], e, exc_info[2])
245        else:
246            if self._unicode and isinstance(value, six.binary_type):
247                if not self.default_encoding:
248                    raise UnicodeDecodeError(
249                        'Cannot decode str value %r into unicode '
250                        '(no default_encoding provided)' % value)
251                value = value.decode(self.default_encoding)
252            elif not self._unicode and isinstance(value, six.text_type):
253                if not self.default_encoding:
254                    raise UnicodeEncodeError(
255                        'Cannot encode unicode value %r into str '
256                        '(no default_encoding provided)' % value)
257                value = value.encode(self.default_encoding)
258            return value
259
260
261    def _add_line_info(self, msg, pos):
262        msg = "%s at line %s column %s" % (
263            msg, pos[0], pos[1])
264        if self.name:
265            msg += " in file %s" % self.name
266        return msg
267
268def sub(content, **kw):
269    name = kw.get('__name')
270    tmpl = Template(content, name=name)
271    return tmpl.substitute(kw)
272
273def paste_script_template_renderer(content, vars, filename=None):
274    tmpl = Template(content, name=filename)
275    return tmpl.substitute(vars)
276
277class bunch(dict):
278
279    def __init__(self, **kw):
280        for name, value in kw.items():
281            setattr(self, name, value)
282
283    def __setattr__(self, name, value):
284        self[name] = value
285
286    def __getattr__(self, name):
287        try:
288            return self[name]
289        except KeyError:
290            raise AttributeError(name)
291
292    def __getitem__(self, key):
293        if 'default' in self:
294            try:
295                return dict.__getitem__(self, key)
296            except KeyError:
297                return dict.__getitem__(self, 'default')
298        else:
299            return dict.__getitem__(self, key)
300
301    def __repr__(self):
302        items = [
303            (k, v) for k, v in self.items()]
304        items.sort()
305        return '<%s %s>' % (
306            self.__class__.__name__,
307            ' '.join(['%s=%r' % (k, v) for k, v in items]))
308
309############################################################
310## HTML Templating
311############################################################
312
313class html(object):
314    def __init__(self, value):
315        self.value = value
316    def __str__(self):
317        return self.value
318    def __repr__(self):
319        return '<%s %r>' % (
320            self.__class__.__name__, self.value)
321
322def html_quote(value):
323    if value is None:
324        return ''
325    if not isinstance(value, six.string_types):
326        if hasattr(value, '__unicode__'):
327            value = unicode(value)
328        else:
329            value = str(value)
330    value = cgi.escape(value, 1)
331    if isinstance(value, unicode):
332        value = value.encode('ascii', 'xmlcharrefreplace')
333    return value
334
335def url(v):
336    if not isinstance(v, six.string_types):
337        if hasattr(v, '__unicode__'):
338            v = unicode(v)
339        else:
340            v = str(v)
341    if isinstance(v, unicode):
342        v = v.encode('utf8')
343    return quote(v)
344
345def attr(**kw):
346    kw = kw.items()
347    kw.sort()
348    parts = []
349    for name, value in kw:
350        if value is None:
351            continue
352        if name.endswith('_'):
353            name = name[:-1]
354        parts.append('%s="%s"' % (html_quote(name), html_quote(value)))
355    return html(' '.join(parts))
356
357class HTMLTemplate(Template):
358
359    default_namespace = Template.default_namespace.copy()
360    default_namespace.update(dict(
361        html=html,
362        attr=attr,
363        url=url,
364        ))
365
366    def _repr(self, value, pos):
367        plain = Template._repr(self, value, pos)
368        if isinstance(value, html):
369            return plain
370        else:
371            return html_quote(plain)
372
373def sub_html(content, **kw):
374    name = kw.get('__name')
375    tmpl = HTMLTemplate(content, name=name)
376    return tmpl.substitute(kw)
377
378
379############################################################
380## Lexing and Parsing
381############################################################
382
383def lex(s, name=None, trim_whitespace=True):
384    """
385    Lex a string into chunks:
386
387        >>> lex('hey')
388        ['hey']
389        >>> lex('hey {{you}}')
390        ['hey ', ('you', (1, 7))]
391        >>> lex('hey {{')
392        Traceback (most recent call last):
393            ...
394        TemplateError: No }} to finish last expression at line 1 column 7
395        >>> lex('hey }}')
396        Traceback (most recent call last):
397            ...
398        TemplateError: }} outside expression at line 1 column 7
399        >>> lex('hey {{ {{')
400        Traceback (most recent call last):
401            ...
402        TemplateError: {{ inside expression at line 1 column 10
403
404    """
405    in_expr = False
406    chunks = []
407    last = 0
408    last_pos = (1, 1)
409    for match in token_re.finditer(s):
410        expr = match.group(0)
411        pos = find_position(s, match.end())
412        if expr == '{{' and in_expr:
413            raise TemplateError('{{ inside expression', position=pos,
414                                name=name)
415        elif expr == '}}' and not in_expr:
416            raise TemplateError('}} outside expression', position=pos,
417                                name=name)
418        if expr == '{{':
419            part = s[last:match.start()]
420            if part:
421                chunks.append(part)
422            in_expr = True
423        else:
424            chunks.append((s[last:match.start()], last_pos))
425            in_expr = False
426        last = match.end()
427        last_pos = pos
428    if in_expr:
429        raise TemplateError('No }} to finish last expression',
430                            name=name, position=last_pos)
431    part = s[last:]
432    if part:
433        chunks.append(part)
434    if trim_whitespace:
435        chunks = trim_lex(chunks)
436    return chunks
437
438statement_re = re.compile(r'^(?:if |elif |else |for |py:)')
439single_statements = ['endif', 'endfor', 'continue', 'break']
440trail_whitespace_re = re.compile(r'\n[\t ]*$')
441lead_whitespace_re = re.compile(r'^[\t ]*\n')
442
443def trim_lex(tokens):
444    r"""
445    Takes a lexed set of tokens, and removes whitespace when there is
446    a directive on a line by itself:
447
448       >>> tokens = lex('{{if x}}\nx\n{{endif}}\ny', trim_whitespace=False)
449       >>> tokens
450       [('if x', (1, 3)), '\nx\n', ('endif', (3, 3)), '\ny']
451       >>> trim_lex(tokens)
452       [('if x', (1, 3)), 'x\n', ('endif', (3, 3)), 'y']
453    """
454    for i in range(len(tokens)):
455        current = tokens[i]
456        if isinstance(tokens[i], six.string_types):
457            # we don't trim this
458            continue
459        item = current[0]
460        if not statement_re.search(item) and item not in single_statements:
461            continue
462        if not i:
463            prev = ''
464        else:
465            prev = tokens[i-1]
466        if i+1 >= len(tokens):
467            next = ''
468        else:
469            next = tokens[i+1]
470        if (not isinstance(next, six.string_types)
471            or not isinstance(prev, six.string_types)):
472            continue
473        if ((not prev or trail_whitespace_re.search(prev))
474            and (not next or lead_whitespace_re.search(next))):
475            if prev:
476                m = trail_whitespace_re.search(prev)
477                # +1 to leave the leading \n on:
478                prev = prev[:m.start()+1]
479                tokens[i-1] = prev
480            if next:
481                m = lead_whitespace_re.search(next)
482                next = next[m.end():]
483                tokens[i+1] = next
484    return tokens
485
486
487def find_position(string, index):
488    """Given a string and index, return (line, column)"""
489    leading = string[:index].splitlines()
490    return (len(leading), len(leading[-1])+1)
491
492def parse(s, name=None):
493    r"""
494    Parses a string into a kind of AST
495
496        >>> parse('{{x}}')
497        [('expr', (1, 3), 'x')]
498        >>> parse('foo')
499        ['foo']
500        >>> parse('{{if x}}test{{endif}}')
501        [('cond', (1, 3), ('if', (1, 3), 'x', ['test']))]
502        >>> parse('series->{{for x in y}}x={{x}}{{endfor}}')
503        ['series->', ('for', (1, 11), ('x',), 'y', ['x=', ('expr', (1, 27), 'x')])]
504        >>> parse('{{for x, y in z:}}{{continue}}{{endfor}}')
505        [('for', (1, 3), ('x', 'y'), 'z', [('continue', (1, 21))])]
506        >>> parse('{{py:x=1}}')
507        [('py', (1, 3), 'x=1')]
508        >>> parse('{{if x}}a{{elif y}}b{{else}}c{{endif}}')
509        [('cond', (1, 3), ('if', (1, 3), 'x', ['a']), ('elif', (1, 12), 'y', ['b']), ('else', (1, 23), None, ['c']))]
510
511    Some exceptions::
512
513        >>> parse('{{continue}}')
514        Traceback (most recent call last):
515            ...
516        TemplateError: continue outside of for loop at line 1 column 3
517        >>> parse('{{if x}}foo')
518        Traceback (most recent call last):
519            ...
520        TemplateError: No {{endif}} at line 1 column 3
521        >>> parse('{{else}}')
522        Traceback (most recent call last):
523            ...
524        TemplateError: else outside of an if block at line 1 column 3
525        >>> parse('{{if x}}{{for x in y}}{{endif}}{{endfor}}')
526        Traceback (most recent call last):
527            ...
528        TemplateError: Unexpected endif at line 1 column 25
529        >>> parse('{{if}}{{endif}}')
530        Traceback (most recent call last):
531            ...
532        TemplateError: if with no expression at line 1 column 3
533        >>> parse('{{for x y}}{{endfor}}')
534        Traceback (most recent call last):
535            ...
536        TemplateError: Bad for (no "in") in 'x y' at line 1 column 3
537        >>> parse('{{py:x=1\ny=2}}')
538        Traceback (most recent call last):
539            ...
540        TemplateError: Multi-line py blocks must start with a newline at line 1 column 3
541    """
542    tokens = lex(s, name=name)
543    result = []
544    while tokens:
545        next, tokens = parse_expr(tokens, name)
546        result.append(next)
547    return result
548
549def parse_expr(tokens, name, context=()):
550    if isinstance(tokens[0], six.string_types):
551        return tokens[0], tokens[1:]
552    expr, pos = tokens[0]
553    expr = expr.strip()
554    if expr.startswith('py:'):
555        expr = expr[3:].lstrip(' \t')
556        if expr.startswith('\n'):
557            expr = expr[1:]
558        else:
559            if '\n' in expr:
560                raise TemplateError(
561                    'Multi-line py blocks must start with a newline',
562                    position=pos, name=name)
563        return ('py', pos, expr), tokens[1:]
564    elif expr in ('continue', 'break'):
565        if 'for' not in context:
566            raise TemplateError(
567                'continue outside of for loop',
568                position=pos, name=name)
569        return (expr, pos), tokens[1:]
570    elif expr.startswith('if '):
571        return parse_cond(tokens, name, context)
572    elif (expr.startswith('elif ')
573          or expr == 'else'):
574        raise TemplateError(
575            '%s outside of an if block' % expr.split()[0],
576            position=pos, name=name)
577    elif expr in ('if', 'elif', 'for'):
578        raise TemplateError(
579            '%s with no expression' % expr,
580            position=pos, name=name)
581    elif expr in ('endif', 'endfor'):
582        raise TemplateError(
583            'Unexpected %s' % expr,
584            position=pos, name=name)
585    elif expr.startswith('for '):
586        return parse_for(tokens, name, context)
587    elif expr.startswith('default '):
588        return parse_default(tokens, name, context)
589    elif expr.startswith('#'):
590        return ('comment', pos, tokens[0][0]), tokens[1:]
591    return ('expr', pos, tokens[0][0]), tokens[1:]
592
593def parse_cond(tokens, name, context):
594    start = tokens[0][1]
595    pieces = []
596    context = context + ('if',)
597    while 1:
598        if not tokens:
599            raise TemplateError(
600                'Missing {{endif}}',
601                position=start, name=name)
602        if (isinstance(tokens[0], tuple)
603            and tokens[0][0] == 'endif'):
604            return ('cond', start) + tuple(pieces), tokens[1:]
605        next, tokens = parse_one_cond(tokens, name, context)
606        pieces.append(next)
607
608def parse_one_cond(tokens, name, context):
609    (first, pos), tokens = tokens[0], tokens[1:]
610    content = []
611    if first.endswith(':'):
612        first = first[:-1]
613    if first.startswith('if '):
614        part = ('if', pos, first[3:].lstrip(), content)
615    elif first.startswith('elif '):
616        part = ('elif', pos, first[5:].lstrip(), content)
617    elif first == 'else':
618        part = ('else', pos, None, content)
619    else:
620        assert 0, "Unexpected token %r at %s" % (first, pos)
621    while 1:
622        if not tokens:
623            raise TemplateError(
624                'No {{endif}}',
625                position=pos, name=name)
626        if (isinstance(tokens[0], tuple)
627            and (tokens[0][0] == 'endif'
628                 or tokens[0][0].startswith('elif ')
629                 or tokens[0][0] == 'else')):
630            return part, tokens
631        next, tokens = parse_expr(tokens, name, context)
632        content.append(next)
633
634def parse_for(tokens, name, context):
635    first, pos = tokens[0]
636    tokens = tokens[1:]
637    context = ('for',) + context
638    content = []
639    assert first.startswith('for ')
640    if first.endswith(':'):
641        first = first[:-1]
642    first = first[3:].strip()
643    match = in_re.search(first)
644    if not match:
645        raise TemplateError(
646            'Bad for (no "in") in %r' % first,
647            position=pos, name=name)
648    vars = first[:match.start()]
649    if '(' in vars:
650        raise TemplateError(
651            'You cannot have () in the variable section of a for loop (%r)'
652            % vars, position=pos, name=name)
653    vars = tuple([
654        v.strip() for v in first[:match.start()].split(',')
655        if v.strip()])
656    expr = first[match.end():]
657    while 1:
658        if not tokens:
659            raise TemplateError(
660                'No {{endfor}}',
661                position=pos, name=name)
662        if (isinstance(tokens[0], tuple)
663            and tokens[0][0] == 'endfor'):
664            return ('for', pos, vars, expr, content), tokens[1:]
665        next, tokens = parse_expr(tokens, name, context)
666        content.append(next)
667
668def parse_default(tokens, name, context):
669    first, pos = tokens[0]
670    assert first.startswith('default ')
671    first = first.split(None, 1)[1]
672    parts = first.split('=', 1)
673    if len(parts) == 1:
674        raise TemplateError(
675            "Expression must be {{default var=value}}; no = found in %r" % first,
676            position=pos, name=name)
677    var = parts[0].strip()
678    if ',' in var:
679        raise TemplateError(
680            "{{default x, y = ...}} is not supported",
681            position=pos, name=name)
682    if not var_re.search(var):
683        raise TemplateError(
684            "Not a valid variable name for {{default}}: %r"
685            % var, position=pos, name=name)
686    expr = parts[1].strip()
687    return ('default', pos, var, expr), tokens[1:]
688
689_fill_command_usage = """\
690%prog [OPTIONS] TEMPLATE arg=value
691
692Use py:arg=value to set a Python value; otherwise all values are
693strings.
694"""
695
696def fill_command(args=None):
697    import sys, optparse, pkg_resources, os
698    if args is None:
699        args = sys.argv[1:]
700    dist = pkg_resources.get_distribution('Paste')
701    parser = optparse.OptionParser(
702        version=str(dist),
703        usage=_fill_command_usage)
704    parser.add_option(
705        '-o', '--output',
706        dest='output',
707        metavar="FILENAME",
708        help="File to write output to (default stdout)")
709    parser.add_option(
710        '--html',
711        dest='use_html',
712        action='store_true',
713        help="Use HTML style filling (including automatic HTML quoting)")
714    parser.add_option(
715        '--env',
716        dest='use_env',
717        action='store_true',
718        help="Put the environment in as top-level variables")
719    options, args = parser.parse_args(args)
720    if len(args) < 1:
721        print('You must give a template filename')
722        print(dir(parser))
723        assert 0
724    template_name = args[0]
725    args = args[1:]
726    vars = {}
727    if options.use_env:
728        vars.update(os.environ)
729    for value in args:
730        if '=' not in value:
731            print('Bad argument: %r' % value)
732            sys.exit(2)
733        name, value = value.split('=', 1)
734        if name.startswith('py:'):
735            name = name[:3]
736            value = eval(value)
737        vars[name] = value
738    if template_name == '-':
739        template_content = sys.stdin.read()
740        template_name = '<stdin>'
741    else:
742        f = open(template_name, 'rb')
743        template_content = f.read()
744        f.close()
745    if options.use_html:
746        TemplateClass = HTMLTemplate
747    else:
748        TemplateClass = Template
749    template = TemplateClass(template_content, name=template_name)
750    result = template.substitute(vars)
751    if options.output:
752        f = open(options.output, 'wb')
753        f.write(result)
754        f.close()
755    else:
756        sys.stdout.write(result)
757
758if __name__ == '__main__':
759    from paste.util.template import fill_command
760    fill_command()
761
762
763