1"""
2A small templating language
3
4This implements a small templating language for use internally in
5Paste and Paste Script.  This language implements if/elif/else,
6for/continue/break, expressions, and blocks of Python code.  The
7syntax is::
8
9  {{any expression (function calls etc)}}
10  {{any expression | filter}}
11  {{for x in y}}...{{endfor}}
12  {{if x}}x{{elif y}}y{{else}}z{{endif}}
13  {{py:x=1}}
14  {{py:
15  def foo(bar):
16      return 'baz'
17  }}
18  {{default var = default_value}}
19  {{# comment}}
20
21You use this with the ``Template`` class or the ``sub`` shortcut.
22The ``Template`` class takes the template string and the name of
23the template (for errors) and a default namespace.  Then (like
24``string.Template``) you can call the ``tmpl.substitute(**kw)``
25method to make a substitution (or ``tmpl.substitute(a_dict)``).
26
27``sub(content, **kw)`` substitutes the template immediately.  You
28can use ``__name='tmpl.html'`` to set the name of the template.
29
30If there are syntax errors ``TemplateError`` will be raised.
31"""
32
33import re
34import six
35import sys
36import cgi
37from six.moves.urllib.parse import quote
38from paste.util.looper import looper
39
40__all__ = ['TemplateError', 'Template', 'sub', 'HTMLTemplate',
41           'sub_html', 'html', 'bunch']
42
43token_re = re.compile(r'\{\{|\}\}')
44in_re = re.compile(r'\s+in\s+')
45var_re = re.compile(r'^[a-z_][a-z0-9_]*$', re.I)
46
47class TemplateError(Exception):
48    """Exception raised while parsing a template
49    """
50
51    def __init__(self, message, position, name=None):
52        self.message = message
53        self.position = position
54        self.name = name
55
56    def __str__(self):
57        msg = '%s at line %s column %s' % (
58            self.message, self.position[0], self.position[1])
59        if self.name:
60            msg += ' in %s' % self.name
61        return msg
62
63class _TemplateContinue(Exception):
64    pass
65
66class _TemplateBreak(Exception):
67    pass
68
69class Template(object):
70
71    default_namespace = {
72        'start_braces': '{{',
73        'end_braces': '}}',
74        'looper': looper,
75        }
76
77    default_encoding = 'utf8'
78
79    def __init__(self, content, name=None, namespace=None):
80        self.content = content
81        self._unicode = isinstance(content, six.text_type)
82        self.name = name
83        self._parsed = parse(content, name=name)
84        if namespace is None:
85            namespace = {}
86        self.namespace = namespace
87
88    def from_filename(cls, filename, namespace=None, encoding=None):
89        f = open(filename, 'rb')
90        c = f.read()
91        f.close()
92        if encoding:
93            c = c.decode(encoding)
94        return cls(content=c, name=filename, namespace=namespace)
95
96    from_filename = classmethod(from_filename)
97
98    def __repr__(self):
99        return '<%s %s name=%r>' % (
100            self.__class__.__name__,
101            hex(id(self))[2:], self.name)
102
103    def substitute(self, *args, **kw):
104        if args:
105            if kw:
106                raise TypeError(
107                    "You can only give positional *or* keyword arguments")
108            if len(args) > 1:
109                raise TypeError(
110                    "You can only give on positional argument")
111            kw = args[0]
112        ns = self.default_namespace.copy()
113        ns.update(self.namespace)
114        ns.update(kw)
115        result = self._interpret(ns)
116        return result
117
118    def _interpret(self, ns):
119        __traceback_hide__ = True
120        parts = []
121        self._interpret_codes(self._parsed, ns, out=parts)
122        return ''.join(parts)
123
124    def _interpret_codes(self, codes, ns, out):
125        __traceback_hide__ = True
126        for item in codes:
127            if isinstance(item, six.string_types):
128                out.append(item)
129            else:
130                self._interpret_code(item, ns, out)
131
132    def _interpret_code(self, code, ns, out):
133        __traceback_hide__ = True
134        name, pos = code[0], code[1]
135        if name == 'py':
136            self._exec(code[2], ns, pos)
137        elif name == 'continue':
138            raise _TemplateContinue()
139        elif name == 'break':
140            raise _TemplateBreak()
141        elif name == 'for':
142            vars, expr, content = code[2], code[3], code[4]
143            expr = self._eval(expr, ns, pos)
144            self._interpret_for(vars, expr, content, ns, out)
145        elif name == 'cond':
146            parts = code[2:]
147            self._interpret_if(parts, ns, out)
148        elif name == 'expr':
149            parts = code[2].split('|')
150            base = self._eval(parts[0], ns, pos)
151            for part in parts[1:]:
152                func = self._eval(part, ns, pos)
153                base = func(base)
154            out.append(self._repr(base, pos))
155        elif name == 'default':
156            var, expr = code[2], code[3]
157            if var not in ns:
158                result = self._eval(expr, ns, pos)
159                ns[var] = result
160        elif name == 'comment':
161            return
162        else:
163            assert 0, "Unknown code: %r" % name
164
165    def _interpret_for(self, vars, expr, content, ns, out):
166        __traceback_hide__ = True
167        for item in expr:
168            if len(vars) == 1:
169                ns[vars[0]] = item
170            else:
171                if len(vars) != len(item):
172                    raise ValueError(
173                        'Need %i items to unpack (got %i items)'
174                        % (len(vars), len(item)))
175                for name, value in zip(vars, item):
176                    ns[name] = value
177            try:
178                self._interpret_codes(content, ns, out)
179            except _TemplateContinue:
180                continue
181            except _TemplateBreak:
182                break
183
184    def _interpret_if(self, parts, ns, out):
185        __traceback_hide__ = True
186        # @@: if/else/else gets through
187        for part in parts:
188            assert not isinstance(part, six.string_types)
189            name, pos = part[0], part[1]
190            if name == 'else':
191                result = True
192            else:
193                result = self._eval(part[2], ns, pos)
194            if result:
195                self._interpret_codes(part[3], ns, out)
196                break
197
198    def _eval(self, code, ns, pos):
199        __traceback_hide__ = True
200        try:
201            value = eval(code, ns)
202            return value
203        except:
204            exc_info = sys.exc_info()
205            e = exc_info[1]
206            if getattr(e, 'args'):
207                arg0 = e.args[0]
208            else:
209                arg0 = str(e)
210            e.args = (self._add_line_info(arg0, pos),)
211            six.reraise(exc_info[0], e, exc_info[2])
212
213    def _exec(self, code, ns, pos):
214        __traceback_hide__ = True
215        try:
216            six.exec_(code, ns)
217        except:
218            exc_info = sys.exc_info()
219            e = exc_info[1]
220            e.args = (self._add_line_info(e.args[0], pos),)
221            six.reraise(exc_info[0], e, exc_info[2])
222
223    def _repr(self, value, pos):
224        __traceback_hide__ = True
225        try:
226            if value is None:
227                return ''
228            if self._unicode:
229                try:
230                    value = six.text_type(value)
231                except UnicodeDecodeError:
232                    value = str(value)
233            else:
234                value = str(value)
235        except:
236            exc_info = sys.exc_info()
237            e = exc_info[1]
238            e.args = (self._add_line_info(e.args[0], pos),)
239            six.reraise(exc_info[0], e, exc_info[2])
240        else:
241            if self._unicode and isinstance(value, six.binary_type):
242                if not self.decode_encoding:
243                    raise UnicodeDecodeError(
244                        'Cannot decode str value %r into unicode '
245                        '(no default_encoding provided)' % value)
246                value = value.decode(self.default_encoding)
247            elif not self._unicode and isinstance(value, six.text_type):
248                if not self.decode_encoding:
249                    raise UnicodeEncodeError(
250                        'Cannot encode unicode value %r into str '
251                        '(no default_encoding provided)' % value)
252                value = value.encode(self.default_encoding)
253            return value
254
255
256    def _add_line_info(self, msg, pos):
257        msg = "%s at line %s column %s" % (
258            msg, pos[0], pos[1])
259        if self.name:
260            msg += " in file %s" % self.name
261        return msg
262
263def sub(content, **kw):
264    name = kw.get('__name')
265    tmpl = Template(content, name=name)
266    return tmpl.substitute(kw)
267
268def paste_script_template_renderer(content, vars, filename=None):
269    tmpl = Template(content, name=filename)
270    return tmpl.substitute(vars)
271
272class bunch(dict):
273
274    def __init__(self, **kw):
275        for name, value in kw.items():
276            setattr(self, name, value)
277
278    def __setattr__(self, name, value):
279        self[name] = value
280
281    def __getattr__(self, name):
282        try:
283            return self[name]
284        except KeyError:
285            raise AttributeError(name)
286
287    def __getitem__(self, key):
288        if 'default' in self:
289            try:
290                return dict.__getitem__(self, key)
291            except KeyError:
292                return dict.__getitem__(self, 'default')
293        else:
294            return dict.__getitem__(self, key)
295
296    def __repr__(self):
297        items = [
298            (k, v) for k, v in self.items()]
299        items.sort()
300        return '<%s %s>' % (
301            self.__class__.__name__,
302            ' '.join(['%s=%r' % (k, v) for k, v in items]))
303
304############################################################
305## HTML Templating
306############################################################
307
308class html(object):
309    def __init__(self, value):
310        self.value = value
311    def __str__(self):
312        return self.value
313    def __repr__(self):
314        return '<%s %r>' % (
315            self.__class__.__name__, self.value)
316
317def html_quote(value):
318    if value is None:
319        return ''
320    if not isinstance(value, six.string_types):
321        if hasattr(value, '__unicode__'):
322            value = unicode(value)
323        else:
324            value = str(value)
325    value = cgi.escape(value, 1)
326    if isinstance(value, unicode):
327        value = value.encode('ascii', 'xmlcharrefreplace')
328    return value
329
330def url(v):
331    if not isinstance(v, six.string_types):
332        if hasattr(v, '__unicode__'):
333            v = unicode(v)
334        else:
335            v = str(v)
336    if isinstance(v, unicode):
337        v = v.encode('utf8')
338    return quote(v)
339
340def attr(**kw):
341    kw = kw.items()
342    kw.sort()
343    parts = []
344    for name, value in kw:
345        if value is None:
346            continue
347        if name.endswith('_'):
348            name = name[:-1]
349        parts.append('%s="%s"' % (html_quote(name), html_quote(value)))
350    return html(' '.join(parts))
351
352class HTMLTemplate(Template):
353
354    default_namespace = Template.default_namespace.copy()
355    default_namespace.update(dict(
356        html=html,
357        attr=attr,
358        url=url,
359        ))
360
361    def _repr(self, value, pos):
362        plain = Template._repr(self, value, pos)
363        if isinstance(value, html):
364            return plain
365        else:
366            return html_quote(plain)
367
368def sub_html(content, **kw):
369    name = kw.get('__name')
370    tmpl = HTMLTemplate(content, name=name)
371    return tmpl.substitute(kw)
372
373
374############################################################
375## Lexing and Parsing
376############################################################
377
378def lex(s, name=None, trim_whitespace=True):
379    """
380    Lex a string into chunks:
381
382        >>> lex('hey')
383        ['hey']
384        >>> lex('hey {{you}}')
385        ['hey ', ('you', (1, 7))]
386        >>> lex('hey {{')
387        Traceback (most recent call last):
388            ...
389        TemplateError: No }} to finish last expression at line 1 column 7
390        >>> lex('hey }}')
391        Traceback (most recent call last):
392            ...
393        TemplateError: }} outside expression at line 1 column 7
394        >>> lex('hey {{ {{')
395        Traceback (most recent call last):
396            ...
397        TemplateError: {{ inside expression at line 1 column 10
398
399    """
400    in_expr = False
401    chunks = []
402    last = 0
403    last_pos = (1, 1)
404    for match in token_re.finditer(s):
405        expr = match.group(0)
406        pos = find_position(s, match.end())
407        if expr == '{{' and in_expr:
408            raise TemplateError('{{ inside expression', position=pos,
409                                name=name)
410        elif expr == '}}' and not in_expr:
411            raise TemplateError('}} outside expression', position=pos,
412                                name=name)
413        if expr == '{{':
414            part = s[last:match.start()]
415            if part:
416                chunks.append(part)
417            in_expr = True
418        else:
419            chunks.append((s[last:match.start()], last_pos))
420            in_expr = False
421        last = match.end()
422        last_pos = pos
423    if in_expr:
424        raise TemplateError('No }} to finish last expression',
425                            name=name, position=last_pos)
426    part = s[last:]
427    if part:
428        chunks.append(part)
429    if trim_whitespace:
430        chunks = trim_lex(chunks)
431    return chunks
432
433statement_re = re.compile(r'^(?:if |elif |else |for |py:)')
434single_statements = ['endif', 'endfor', 'continue', 'break']
435trail_whitespace_re = re.compile(r'\n[\t ]*$')
436lead_whitespace_re = re.compile(r'^[\t ]*\n')
437
438def trim_lex(tokens):
439    r"""
440    Takes a lexed set of tokens, and removes whitespace when there is
441    a directive on a line by itself:
442
443       >>> tokens = lex('{{if x}}\nx\n{{endif}}\ny', trim_whitespace=False)
444       >>> tokens
445       [('if x', (1, 3)), '\nx\n', ('endif', (3, 3)), '\ny']
446       >>> trim_lex(tokens)
447       [('if x', (1, 3)), 'x\n', ('endif', (3, 3)), 'y']
448    """
449    for i in range(len(tokens)):
450        current = tokens[i]
451        if isinstance(tokens[i], six.string_types):
452            # we don't trim this
453            continue
454        item = current[0]
455        if not statement_re.search(item) and item not in single_statements:
456            continue
457        if not i:
458            prev = ''
459        else:
460            prev = tokens[i-1]
461        if i+1 >= len(tokens):
462            next = ''
463        else:
464            next = tokens[i+1]
465        if (not isinstance(next, six.string_types)
466            or not isinstance(prev, six.string_types)):
467            continue
468        if ((not prev or trail_whitespace_re.search(prev))
469            and (not next or lead_whitespace_re.search(next))):
470            if prev:
471                m = trail_whitespace_re.search(prev)
472                # +1 to leave the leading \n on:
473                prev = prev[:m.start()+1]
474                tokens[i-1] = prev
475            if next:
476                m = lead_whitespace_re.search(next)
477                next = next[m.end():]
478                tokens[i+1] = next
479    return tokens
480
481
482def find_position(string, index):
483    """Given a string and index, return (line, column)"""
484    leading = string[:index].splitlines()
485    return (len(leading), len(leading[-1])+1)
486
487def parse(s, name=None):
488    r"""
489    Parses a string into a kind of AST
490
491        >>> parse('{{x}}')
492        [('expr', (1, 3), 'x')]
493        >>> parse('foo')
494        ['foo']
495        >>> parse('{{if x}}test{{endif}}')
496        [('cond', (1, 3), ('if', (1, 3), 'x', ['test']))]
497        >>> parse('series->{{for x in y}}x={{x}}{{endfor}}')
498        ['series->', ('for', (1, 11), ('x',), 'y', ['x=', ('expr', (1, 27), 'x')])]
499        >>> parse('{{for x, y in z:}}{{continue}}{{endfor}}')
500        [('for', (1, 3), ('x', 'y'), 'z', [('continue', (1, 21))])]
501        >>> parse('{{py:x=1}}')
502        [('py', (1, 3), 'x=1')]
503        >>> parse('{{if x}}a{{elif y}}b{{else}}c{{endif}}')
504        [('cond', (1, 3), ('if', (1, 3), 'x', ['a']), ('elif', (1, 12), 'y', ['b']), ('else', (1, 23), None, ['c']))]
505
506    Some exceptions::
507
508        >>> parse('{{continue}}')
509        Traceback (most recent call last):
510            ...
511        TemplateError: continue outside of for loop at line 1 column 3
512        >>> parse('{{if x}}foo')
513        Traceback (most recent call last):
514            ...
515        TemplateError: No {{endif}} at line 1 column 3
516        >>> parse('{{else}}')
517        Traceback (most recent call last):
518            ...
519        TemplateError: else outside of an if block at line 1 column 3
520        >>> parse('{{if x}}{{for x in y}}{{endif}}{{endfor}}')
521        Traceback (most recent call last):
522            ...
523        TemplateError: Unexpected endif at line 1 column 25
524        >>> parse('{{if}}{{endif}}')
525        Traceback (most recent call last):
526            ...
527        TemplateError: if with no expression at line 1 column 3
528        >>> parse('{{for x y}}{{endfor}}')
529        Traceback (most recent call last):
530            ...
531        TemplateError: Bad for (no "in") in 'x y' at line 1 column 3
532        >>> parse('{{py:x=1\ny=2}}')
533        Traceback (most recent call last):
534            ...
535        TemplateError: Multi-line py blocks must start with a newline at line 1 column 3
536    """
537    tokens = lex(s, name=name)
538    result = []
539    while tokens:
540        next, tokens = parse_expr(tokens, name)
541        result.append(next)
542    return result
543
544def parse_expr(tokens, name, context=()):
545    if isinstance(tokens[0], six.string_types):
546        return tokens[0], tokens[1:]
547    expr, pos = tokens[0]
548    expr = expr.strip()
549    if expr.startswith('py:'):
550        expr = expr[3:].lstrip(' \t')
551        if expr.startswith('\n'):
552            expr = expr[1:]
553        else:
554            if '\n' in expr:
555                raise TemplateError(
556                    'Multi-line py blocks must start with a newline',
557                    position=pos, name=name)
558        return ('py', pos, expr), tokens[1:]
559    elif expr in ('continue', 'break'):
560        if 'for' not in context:
561            raise TemplateError(
562                'continue outside of for loop',
563                position=pos, name=name)
564        return (expr, pos), tokens[1:]
565    elif expr.startswith('if '):
566        return parse_cond(tokens, name, context)
567    elif (expr.startswith('elif ')
568          or expr == 'else'):
569        raise TemplateError(
570            '%s outside of an if block' % expr.split()[0],
571            position=pos, name=name)
572    elif expr in ('if', 'elif', 'for'):
573        raise TemplateError(
574            '%s with no expression' % expr,
575            position=pos, name=name)
576    elif expr in ('endif', 'endfor'):
577        raise TemplateError(
578            'Unexpected %s' % expr,
579            position=pos, name=name)
580    elif expr.startswith('for '):
581        return parse_for(tokens, name, context)
582    elif expr.startswith('default '):
583        return parse_default(tokens, name, context)
584    elif expr.startswith('#'):
585        return ('comment', pos, tokens[0][0]), tokens[1:]
586    return ('expr', pos, tokens[0][0]), tokens[1:]
587
588def parse_cond(tokens, name, context):
589    start = tokens[0][1]
590    pieces = []
591    context = context + ('if',)
592    while 1:
593        if not tokens:
594            raise TemplateError(
595                'Missing {{endif}}',
596                position=start, name=name)
597        if (isinstance(tokens[0], tuple)
598            and tokens[0][0] == 'endif'):
599            return ('cond', start) + tuple(pieces), tokens[1:]
600        next, tokens = parse_one_cond(tokens, name, context)
601        pieces.append(next)
602
603def parse_one_cond(tokens, name, context):
604    (first, pos), tokens = tokens[0], tokens[1:]
605    content = []
606    if first.endswith(':'):
607        first = first[:-1]
608    if first.startswith('if '):
609        part = ('if', pos, first[3:].lstrip(), content)
610    elif first.startswith('elif '):
611        part = ('elif', pos, first[5:].lstrip(), content)
612    elif first == 'else':
613        part = ('else', pos, None, content)
614    else:
615        assert 0, "Unexpected token %r at %s" % (first, pos)
616    while 1:
617        if not tokens:
618            raise TemplateError(
619                'No {{endif}}',
620                position=pos, name=name)
621        if (isinstance(tokens[0], tuple)
622            and (tokens[0][0] == 'endif'
623                 or tokens[0][0].startswith('elif ')
624                 or tokens[0][0] == 'else')):
625            return part, tokens
626        next, tokens = parse_expr(tokens, name, context)
627        content.append(next)
628
629def parse_for(tokens, name, context):
630    first, pos = tokens[0]
631    tokens = tokens[1:]
632    context = ('for',) + context
633    content = []
634    assert first.startswith('for ')
635    if first.endswith(':'):
636        first = first[:-1]
637    first = first[3:].strip()
638    match = in_re.search(first)
639    if not match:
640        raise TemplateError(
641            'Bad for (no "in") in %r' % first,
642            position=pos, name=name)
643    vars = first[:match.start()]
644    if '(' in vars:
645        raise TemplateError(
646            'You cannot have () in the variable section of a for loop (%r)'
647            % vars, position=pos, name=name)
648    vars = tuple([
649        v.strip() for v in first[:match.start()].split(',')
650        if v.strip()])
651    expr = first[match.end():]
652    while 1:
653        if not tokens:
654            raise TemplateError(
655                'No {{endfor}}',
656                position=pos, name=name)
657        if (isinstance(tokens[0], tuple)
658            and tokens[0][0] == 'endfor'):
659            return ('for', pos, vars, expr, content), tokens[1:]
660        next, tokens = parse_expr(tokens, name, context)
661        content.append(next)
662
663def parse_default(tokens, name, context):
664    first, pos = tokens[0]
665    assert first.startswith('default ')
666    first = first.split(None, 1)[1]
667    parts = first.split('=', 1)
668    if len(parts) == 1:
669        raise TemplateError(
670            "Expression must be {{default var=value}}; no = found in %r" % first,
671            position=pos, name=name)
672    var = parts[0].strip()
673    if ',' in var:
674        raise TemplateError(
675            "{{default x, y = ...}} is not supported",
676            position=pos, name=name)
677    if not var_re.search(var):
678        raise TemplateError(
679            "Not a valid variable name for {{default}}: %r"
680            % var, position=pos, name=name)
681    expr = parts[1].strip()
682    return ('default', pos, var, expr), tokens[1:]
683
684_fill_command_usage = """\
685%prog [OPTIONS] TEMPLATE arg=value
686
687Use py:arg=value to set a Python value; otherwise all values are
688strings.
689"""
690
691def fill_command(args=None):
692    import sys, optparse, pkg_resources, os
693    if args is None:
694        args = sys.argv[1:]
695    dist = pkg_resources.get_distribution('Paste')
696    parser = optparse.OptionParser(
697        version=str(dist),
698        usage=_fill_command_usage)
699    parser.add_option(
700        '-o', '--output',
701        dest='output',
702        metavar="FILENAME",
703        help="File to write output to (default stdout)")
704    parser.add_option(
705        '--html',
706        dest='use_html',
707        action='store_true',
708        help="Use HTML style filling (including automatic HTML quoting)")
709    parser.add_option(
710        '--env',
711        dest='use_env',
712        action='store_true',
713        help="Put the environment in as top-level variables")
714    options, args = parser.parse_args(args)
715    if len(args) < 1:
716        print('You must give a template filename')
717        print(dir(parser))
718        assert 0
719    template_name = args[0]
720    args = args[1:]
721    vars = {}
722    if options.use_env:
723        vars.update(os.environ)
724    for value in args:
725        if '=' not in value:
726            print('Bad argument: %r' % value)
727            sys.exit(2)
728        name, value = value.split('=', 1)
729        if name.startswith('py:'):
730            name = name[:3]
731            value = eval(value)
732        vars[name] = value
733    if template_name == '-':
734        template_content = sys.stdin.read()
735        template_name = '<stdin>'
736    else:
737        f = open(template_name, 'rb')
738        template_content = f.read()
739        f.close()
740    if options.use_html:
741        TemplateClass = HTMLTemplate
742    else:
743        TemplateClass = Template
744    template = TemplateClass(template_content, name=template_name)
745    result = template.substitute(vars)
746    if options.output:
747        f = open(options.output, 'wb')
748        f.write(result)
749        f.close()
750    else:
751        sys.stdout.write(result)
752
753if __name__ == '__main__':
754    from paste.util.template import fill_command
755    fill_command()
756
757
758