1import builtins
2import keyword
3import re
4import time
5
6from idlelib.config import idleConf
7from idlelib.delegator import Delegator
8
9DEBUG = False
10
11def any(name, alternates):
12    "Return a named group pattern matching list of alternates."
13    return "(?P<%s>" % name + "|".join(alternates) + ")"
14
15def make_pat():
16    kw = r"\b" + any("KEYWORD", keyword.kwlist) + r"\b"
17    builtinlist = [str(name) for name in dir(builtins)
18                                        if not name.startswith('_') and \
19                                        name not in keyword.kwlist]
20    builtin = r"([^.'\"\\#]\b|^)" + any("BUILTIN", builtinlist) + r"\b"
21    comment = any("COMMENT", [r"#[^\n]*"])
22    stringprefix = r"(?i:r|u|f|fr|rf|b|br|rb)?"
23    sqstring = stringprefix + r"'[^'\\\n]*(\\.[^'\\\n]*)*'?"
24    dqstring = stringprefix + r'"[^"\\\n]*(\\.[^"\\\n]*)*"?'
25    sq3string = stringprefix + r"'''[^'\\]*((\\.|'(?!''))[^'\\]*)*(''')?"
26    dq3string = stringprefix + r'"""[^"\\]*((\\.|"(?!""))[^"\\]*)*(""")?'
27    string = any("STRING", [sq3string, dq3string, sqstring, dqstring])
28    return kw + "|" + builtin + "|" + comment + "|" + string +\
29           "|" + any("SYNC", [r"\n"])
30
31prog = re.compile(make_pat(), re.S)
32idprog = re.compile(r"\s+(\w+)", re.S)
33
34def color_config(text):
35    """Set color options of Text widget.
36
37    If ColorDelegator is used, this should be called first.
38    """
39    # Called from htest, TextFrame, Editor, and Turtledemo.
40    # Not automatic because ColorDelegator does not know 'text'.
41    theme = idleConf.CurrentTheme()
42    normal_colors = idleConf.GetHighlight(theme, 'normal')
43    cursor_color = idleConf.GetHighlight(theme, 'cursor', fgBg='fg')
44    select_colors = idleConf.GetHighlight(theme, 'hilite')
45    text.config(
46        foreground=normal_colors['foreground'],
47        background=normal_colors['background'],
48        insertbackground=cursor_color,
49        selectforeground=select_colors['foreground'],
50        selectbackground=select_colors['background'],
51        inactiveselectbackground=select_colors['background'],  # new in 8.5
52    )
53
54
55class ColorDelegator(Delegator):
56    """Delegator for syntax highlighting (text coloring).
57
58    Instance variables:
59        delegate: Delegator below this one in the stack, meaning the
60                one this one delegates to.
61
62        Used to track state:
63        after_id: Identifier for scheduled after event, which is a
64                timer for colorizing the text.
65        allow_colorizing: Boolean toggle for applying colorizing.
66        colorizing: Boolean flag when colorizing is in process.
67        stop_colorizing: Boolean flag to end an active colorizing
68                process.
69    """
70
71    def __init__(self):
72        Delegator.__init__(self)
73        self.init_state()
74        self.prog = prog
75        self.idprog = idprog
76        self.LoadTagDefs()
77
78    def init_state(self):
79        "Initialize variables that track colorizing state."
80        self.after_id = None
81        self.allow_colorizing = True
82        self.stop_colorizing = False
83        self.colorizing = False
84
85    def setdelegate(self, delegate):
86        """Set the delegate for this instance.
87
88        A delegate is an instance of a Delegator class and each
89        delegate points to the next delegator in the stack.  This
90        allows multiple delegators to be chained together for a
91        widget.  The bottom delegate for a colorizer is a Text
92        widget.
93
94        If there is a delegate, also start the colorizing process.
95        """
96        if self.delegate is not None:
97            self.unbind("<<toggle-auto-coloring>>")
98        Delegator.setdelegate(self, delegate)
99        if delegate is not None:
100            self.config_colors()
101            self.bind("<<toggle-auto-coloring>>", self.toggle_colorize_event)
102            self.notify_range("1.0", "end")
103        else:
104            # No delegate - stop any colorizing.
105            self.stop_colorizing = True
106            self.allow_colorizing = False
107
108    def config_colors(self):
109        "Configure text widget tags with colors from tagdefs."
110        for tag, cnf in self.tagdefs.items():
111            self.tag_configure(tag, **cnf)
112        self.tag_raise('sel')
113
114    def LoadTagDefs(self):
115        "Create dictionary of tag names to text colors."
116        theme = idleConf.CurrentTheme()
117        self.tagdefs = {
118            "COMMENT": idleConf.GetHighlight(theme, "comment"),
119            "KEYWORD": idleConf.GetHighlight(theme, "keyword"),
120            "BUILTIN": idleConf.GetHighlight(theme, "builtin"),
121            "STRING": idleConf.GetHighlight(theme, "string"),
122            "DEFINITION": idleConf.GetHighlight(theme, "definition"),
123            "SYNC": {'background':None,'foreground':None},
124            "TODO": {'background':None,'foreground':None},
125            "ERROR": idleConf.GetHighlight(theme, "error"),
126            # The following is used by ReplaceDialog:
127            "hit": idleConf.GetHighlight(theme, "hit"),
128            }
129
130        if DEBUG: print('tagdefs',self.tagdefs)
131
132    def insert(self, index, chars, tags=None):
133        "Insert chars into widget at index and mark for colorizing."
134        index = self.index(index)
135        self.delegate.insert(index, chars, tags)
136        self.notify_range(index, index + "+%dc" % len(chars))
137
138    def delete(self, index1, index2=None):
139        "Delete chars between indexes and mark for colorizing."
140        index1 = self.index(index1)
141        self.delegate.delete(index1, index2)
142        self.notify_range(index1)
143
144    def notify_range(self, index1, index2=None):
145        "Mark text changes for processing and restart colorizing, if active."
146        self.tag_add("TODO", index1, index2)
147        if self.after_id:
148            if DEBUG: print("colorizing already scheduled")
149            return
150        if self.colorizing:
151            self.stop_colorizing = True
152            if DEBUG: print("stop colorizing")
153        if self.allow_colorizing:
154            if DEBUG: print("schedule colorizing")
155            self.after_id = self.after(1, self.recolorize)
156        return
157
158    def close(self):
159        if self.after_id:
160            after_id = self.after_id
161            self.after_id = None
162            if DEBUG: print("cancel scheduled recolorizer")
163            self.after_cancel(after_id)
164        self.allow_colorizing = False
165        self.stop_colorizing = True
166
167    def toggle_colorize_event(self, event=None):
168        """Toggle colorizing on and off.
169
170        When toggling off, if colorizing is scheduled or is in
171        process, it will be cancelled and/or stopped.
172
173        When toggling on, colorizing will be scheduled.
174        """
175        if self.after_id:
176            after_id = self.after_id
177            self.after_id = None
178            if DEBUG: print("cancel scheduled recolorizer")
179            self.after_cancel(after_id)
180        if self.allow_colorizing and self.colorizing:
181            if DEBUG: print("stop colorizing")
182            self.stop_colorizing = True
183        self.allow_colorizing = not self.allow_colorizing
184        if self.allow_colorizing and not self.colorizing:
185            self.after_id = self.after(1, self.recolorize)
186        if DEBUG:
187            print("auto colorizing turned",\
188                  self.allow_colorizing and "on" or "off")
189        return "break"
190
191    def recolorize(self):
192        """Timer event (every 1ms) to colorize text.
193
194        Colorizing is only attempted when the text widget exists,
195        when colorizing is toggled on, and when the colorizing
196        process is not already running.
197
198        After colorizing is complete, some cleanup is done to
199        make sure that all the text has been colorized.
200        """
201        self.after_id = None
202        if not self.delegate:
203            if DEBUG: print("no delegate")
204            return
205        if not self.allow_colorizing:
206            if DEBUG: print("auto colorizing is off")
207            return
208        if self.colorizing:
209            if DEBUG: print("already colorizing")
210            return
211        try:
212            self.stop_colorizing = False
213            self.colorizing = True
214            if DEBUG: print("colorizing...")
215            t0 = time.perf_counter()
216            self.recolorize_main()
217            t1 = time.perf_counter()
218            if DEBUG: print("%.3f seconds" % (t1-t0))
219        finally:
220            self.colorizing = False
221        if self.allow_colorizing and self.tag_nextrange("TODO", "1.0"):
222            if DEBUG: print("reschedule colorizing")
223            self.after_id = self.after(1, self.recolorize)
224
225    def recolorize_main(self):
226        "Evaluate text and apply colorizing tags."
227        next = "1.0"
228        while True:
229            item = self.tag_nextrange("TODO", next)
230            if not item:
231                break
232            head, tail = item
233            self.tag_remove("SYNC", head, tail)
234            item = self.tag_prevrange("SYNC", head)
235            if item:
236                head = item[1]
237            else:
238                head = "1.0"
239
240            chars = ""
241            next = head
242            lines_to_get = 1
243            ok = False
244            while not ok:
245                mark = next
246                next = self.index(mark + "+%d lines linestart" %
247                                         lines_to_get)
248                lines_to_get = min(lines_to_get * 2, 100)
249                ok = "SYNC" in self.tag_names(next + "-1c")
250                line = self.get(mark, next)
251                ##print head, "get", mark, next, "->", repr(line)
252                if not line:
253                    return
254                for tag in self.tagdefs:
255                    self.tag_remove(tag, mark, next)
256                chars = chars + line
257                m = self.prog.search(chars)
258                while m:
259                    for key, value in m.groupdict().items():
260                        if value:
261                            a, b = m.span(key)
262                            self.tag_add(key,
263                                         head + "+%dc" % a,
264                                         head + "+%dc" % b)
265                            if value in ("def", "class"):
266                                m1 = self.idprog.match(chars, b)
267                                if m1:
268                                    a, b = m1.span(1)
269                                    self.tag_add("DEFINITION",
270                                                 head + "+%dc" % a,
271                                                 head + "+%dc" % b)
272                    m = self.prog.search(chars, m.end())
273                if "SYNC" in self.tag_names(next + "-1c"):
274                    head = next
275                    chars = ""
276                else:
277                    ok = False
278                if not ok:
279                    # We're in an inconsistent state, and the call to
280                    # update may tell us to stop.  It may also change
281                    # the correct value for "next" (since this is a
282                    # line.col string, not a true mark).  So leave a
283                    # crumb telling the next invocation to resume here
284                    # in case update tells us to leave.
285                    self.tag_add("TODO", next)
286                self.update()
287                if self.stop_colorizing:
288                    if DEBUG: print("colorizing stopped")
289                    return
290
291    def removecolors(self):
292        "Remove all colorizing tags."
293        for tag in self.tagdefs:
294            self.tag_remove(tag, "1.0", "end")
295
296
297def _color_delegator(parent):  # htest #
298    from tkinter import Toplevel, Text
299    from idlelib.percolator import Percolator
300
301    top = Toplevel(parent)
302    top.title("Test ColorDelegator")
303    x, y = map(int, parent.geometry().split('+')[1:])
304    top.geometry("700x250+%d+%d" % (x + 20, y + 175))
305    source = (
306        "if True: int ('1') # keyword, builtin, string, comment\n"
307        "elif False: print(0)\n"
308        "else: float(None)\n"
309        "if iF + If + IF: 'keyword matching must respect case'\n"
310        "if'': x or''  # valid string-keyword no-space combinations\n"
311        "async def f(): await g()\n"
312        "# All valid prefixes for unicode and byte strings should be colored.\n"
313        "'x', '''x''', \"x\", \"\"\"x\"\"\"\n"
314        "r'x', u'x', R'x', U'x', f'x', F'x'\n"
315        "fr'x', Fr'x', fR'x', FR'x', rf'x', rF'x', Rf'x', RF'x'\n"
316        "b'x',B'x', br'x',Br'x',bR'x',BR'x', rb'x', rB'x',Rb'x',RB'x'\n"
317        "# Invalid combinations of legal characters should be half colored.\n"
318        "ur'x', ru'x', uf'x', fu'x', UR'x', ufr'x', rfu'x', xf'x', fx'x'\n"
319        )
320    text = Text(top, background="white")
321    text.pack(expand=1, fill="both")
322    text.insert("insert", source)
323    text.focus_set()
324
325    color_config(text)
326    p = Percolator(text)
327    d = ColorDelegator()
328    p.insertfilter(d)
329
330
331if __name__ == "__main__":
332    from unittest import main
333    main('idlelib.idle_test.test_colorizer', verbosity=2, exit=False)
334
335    from idlelib.idle_test.htest import run
336    run(_color_delegator)
337