1"Test colorizer, coverage 99%."
2from idlelib import colorizer
3from test.support import requires
4import unittest
5from unittest import mock
6from idlelib.idle_test.tkinter_testing_utils import run_in_tk_mainloop
7
8from functools import partial
9import textwrap
10from tkinter import Tk, Text
11from idlelib import config
12from idlelib.percolator import Percolator
13
14
15usercfg = colorizer.idleConf.userCfg
16testcfg = {
17    'main': config.IdleUserConfParser(''),
18    'highlight': config.IdleUserConfParser(''),
19    'keys': config.IdleUserConfParser(''),
20    'extensions': config.IdleUserConfParser(''),
21}
22
23source = textwrap.dedent("""\
24    if True: int ('1') # keyword, builtin, string, comment
25    elif False: print(0)  # 'string' in comment
26    else: float(None)  # if in comment
27    if iF + If + IF: 'keyword matching must respect case'
28    if'': x or''  # valid keyword-string no-space combinations
29    async def f(): await g()
30    # Strings should be entirely colored, including quotes.
31    'x', '''x''', "x", \"""x\"""
32    'abc\\
33    def'
34    '''abc\\
35    def'''
36    # All valid prefixes for unicode and byte strings should be colored.
37    r'x', u'x', R'x', U'x', f'x', F'x'
38    fr'x', Fr'x', fR'x', FR'x', rf'x', rF'x', Rf'x', RF'x'
39    b'x',B'x', br'x',Br'x',bR'x',BR'x', rb'x', rB'x',Rb'x',RB'x'
40    # Invalid combinations of legal characters should be half colored.
41    ur'x', ru'x', uf'x', fu'x', UR'x', ufr'x', rfu'x', xf'x', fx'x'
42    match point:
43        case (x, 0) as _:
44            print(f"X={x}")
45        case [_, [_], "_",
46                _]:
47            pass
48        case _ if ("a" if _ else set()): pass
49        case _:
50            raise ValueError("Not a point _")
51    '''
52    case _:'''
53    "match x:"
54    """)
55
56
57def setUpModule():
58    colorizer.idleConf.userCfg = testcfg
59
60
61def tearDownModule():
62    colorizer.idleConf.userCfg = usercfg
63
64
65class FunctionTest(unittest.TestCase):
66
67    def test_any(self):
68        self.assertEqual(colorizer.any('test', ('a', 'b', 'cd')),
69                         '(?P<test>a|b|cd)')
70
71    def test_make_pat(self):
72        # Tested in more detail by testing prog.
73        self.assertTrue(colorizer.make_pat())
74
75    def test_prog(self):
76        prog = colorizer.prog
77        eq = self.assertEqual
78        line = 'def f():\n    print("hello")\n'
79        m = prog.search(line)
80        eq(m.groupdict()['KEYWORD'], 'def')
81        m = prog.search(line, m.end())
82        eq(m.groupdict()['SYNC'], '\n')
83        m = prog.search(line, m.end())
84        eq(m.groupdict()['BUILTIN'], 'print')
85        m = prog.search(line, m.end())
86        eq(m.groupdict()['STRING'], '"hello"')
87        m = prog.search(line, m.end())
88        eq(m.groupdict()['SYNC'], '\n')
89
90    def test_idprog(self):
91        idprog = colorizer.idprog
92        m = idprog.match('nospace')
93        self.assertIsNone(m)
94        m = idprog.match(' space')
95        self.assertEqual(m.group(0), ' space')
96
97
98class ColorConfigTest(unittest.TestCase):
99
100    @classmethod
101    def setUpClass(cls):
102        requires('gui')
103        root = cls.root = Tk()
104        root.withdraw()
105        cls.text = Text(root)
106
107    @classmethod
108    def tearDownClass(cls):
109        del cls.text
110        cls.root.update_idletasks()
111        cls.root.destroy()
112        del cls.root
113
114    def test_color_config(self):
115        text = self.text
116        eq = self.assertEqual
117        colorizer.color_config(text)
118        # Uses IDLE Classic theme as default.
119        eq(text['background'], '#ffffff')
120        eq(text['foreground'], '#000000')
121        eq(text['selectbackground'], 'gray')
122        eq(text['selectforeground'], '#000000')
123        eq(text['insertbackground'], 'black')
124        eq(text['inactiveselectbackground'], 'gray')
125
126
127class ColorDelegatorInstantiationTest(unittest.TestCase):
128
129    @classmethod
130    def setUpClass(cls):
131        requires('gui')
132        root = cls.root = Tk()
133        root.withdraw()
134        cls.text = Text(root)
135
136    @classmethod
137    def tearDownClass(cls):
138        del cls.text
139        cls.root.update_idletasks()
140        cls.root.destroy()
141        del cls.root
142
143    def setUp(self):
144        self.color = colorizer.ColorDelegator()
145
146    def tearDown(self):
147        self.color.close()
148        self.text.delete('1.0', 'end')
149        self.color.resetcache()
150        del self.color
151
152    def test_init(self):
153        color = self.color
154        self.assertIsInstance(color, colorizer.ColorDelegator)
155
156    def test_init_state(self):
157        # init_state() is called during the instantiation of
158        # ColorDelegator in setUp().
159        color = self.color
160        self.assertIsNone(color.after_id)
161        self.assertTrue(color.allow_colorizing)
162        self.assertFalse(color.colorizing)
163        self.assertFalse(color.stop_colorizing)
164
165
166class ColorDelegatorTest(unittest.TestCase):
167
168    @classmethod
169    def setUpClass(cls):
170        requires('gui')
171        root = cls.root = Tk()
172        root.withdraw()
173        text = cls.text = Text(root)
174        cls.percolator = Percolator(text)
175        # Delegator stack = [Delegator(text)]
176
177    @classmethod
178    def tearDownClass(cls):
179        cls.percolator.close()
180        del cls.percolator, cls.text
181        cls.root.update_idletasks()
182        cls.root.destroy()
183        del cls.root
184
185    def setUp(self):
186        self.color = colorizer.ColorDelegator()
187        self.percolator.insertfilter(self.color)
188        # Calls color.setdelegate(Delegator(text)).
189
190    def tearDown(self):
191        self.color.close()
192        self.percolator.removefilter(self.color)
193        self.text.delete('1.0', 'end')
194        self.color.resetcache()
195        del self.color
196
197    def test_setdelegate(self):
198        # Called in setUp when filter is attached to percolator.
199        color = self.color
200        self.assertIsInstance(color.delegate, colorizer.Delegator)
201        # It is too late to mock notify_range, so test side effect.
202        self.assertEqual(self.root.tk.call(
203            'after', 'info', color.after_id)[1], 'timer')
204
205    def test_LoadTagDefs(self):
206        highlight = partial(config.idleConf.GetHighlight, theme='IDLE Classic')
207        for tag, colors in self.color.tagdefs.items():
208            with self.subTest(tag=tag):
209                self.assertIn('background', colors)
210                self.assertIn('foreground', colors)
211                if tag not in ('SYNC', 'TODO'):
212                    self.assertEqual(colors, highlight(element=tag.lower()))
213
214    def test_config_colors(self):
215        text = self.text
216        highlight = partial(config.idleConf.GetHighlight, theme='IDLE Classic')
217        for tag in self.color.tagdefs:
218            for plane in ('background', 'foreground'):
219                with self.subTest(tag=tag, plane=plane):
220                    if tag in ('SYNC', 'TODO'):
221                        self.assertEqual(text.tag_cget(tag, plane), '')
222                    else:
223                        self.assertEqual(text.tag_cget(tag, plane),
224                                         highlight(element=tag.lower())[plane])
225        # 'sel' is marked as the highest priority.
226        self.assertEqual(text.tag_names()[-1], 'sel')
227
228    @mock.patch.object(colorizer.ColorDelegator, 'notify_range')
229    def test_insert(self, mock_notify):
230        text = self.text
231        # Initial text.
232        text.insert('insert', 'foo')
233        self.assertEqual(text.get('1.0', 'end'), 'foo\n')
234        mock_notify.assert_called_with('1.0', '1.0+3c')
235        # Additional text.
236        text.insert('insert', 'barbaz')
237        self.assertEqual(text.get('1.0', 'end'), 'foobarbaz\n')
238        mock_notify.assert_called_with('1.3', '1.3+6c')
239
240    @mock.patch.object(colorizer.ColorDelegator, 'notify_range')
241    def test_delete(self, mock_notify):
242        text = self.text
243        # Initialize text.
244        text.insert('insert', 'abcdefghi')
245        self.assertEqual(text.get('1.0', 'end'), 'abcdefghi\n')
246        # Delete single character.
247        text.delete('1.7')
248        self.assertEqual(text.get('1.0', 'end'), 'abcdefgi\n')
249        mock_notify.assert_called_with('1.7')
250        # Delete multiple characters.
251        text.delete('1.3', '1.6')
252        self.assertEqual(text.get('1.0', 'end'), 'abcgi\n')
253        mock_notify.assert_called_with('1.3')
254
255    def test_notify_range(self):
256        text = self.text
257        color = self.color
258        eq = self.assertEqual
259
260        # Colorizing already scheduled.
261        save_id = color.after_id
262        eq(self.root.tk.call('after', 'info', save_id)[1], 'timer')
263        self.assertFalse(color.colorizing)
264        self.assertFalse(color.stop_colorizing)
265        self.assertTrue(color.allow_colorizing)
266
267        # Coloring scheduled and colorizing in progress.
268        color.colorizing = True
269        color.notify_range('1.0', 'end')
270        self.assertFalse(color.stop_colorizing)
271        eq(color.after_id, save_id)
272
273        # No colorizing scheduled and colorizing in progress.
274        text.after_cancel(save_id)
275        color.after_id = None
276        color.notify_range('1.0', '1.0+3c')
277        self.assertTrue(color.stop_colorizing)
278        self.assertIsNotNone(color.after_id)
279        eq(self.root.tk.call('after', 'info', color.after_id)[1], 'timer')
280        # New event scheduled.
281        self.assertNotEqual(color.after_id, save_id)
282
283        # No colorizing scheduled and colorizing off.
284        text.after_cancel(color.after_id)
285        color.after_id = None
286        color.allow_colorizing = False
287        color.notify_range('1.4', '1.4+10c')
288        # Nothing scheduled when colorizing is off.
289        self.assertIsNone(color.after_id)
290
291    def test_toggle_colorize_event(self):
292        color = self.color
293        eq = self.assertEqual
294
295        # Starts with colorizing allowed and scheduled.
296        self.assertFalse(color.colorizing)
297        self.assertFalse(color.stop_colorizing)
298        self.assertTrue(color.allow_colorizing)
299        eq(self.root.tk.call('after', 'info', color.after_id)[1], 'timer')
300
301        # Toggle colorizing off.
302        color.toggle_colorize_event()
303        self.assertIsNone(color.after_id)
304        self.assertFalse(color.colorizing)
305        self.assertFalse(color.stop_colorizing)
306        self.assertFalse(color.allow_colorizing)
307
308        # Toggle on while colorizing in progress (doesn't add timer).
309        color.colorizing = True
310        color.toggle_colorize_event()
311        self.assertIsNone(color.after_id)
312        self.assertTrue(color.colorizing)
313        self.assertFalse(color.stop_colorizing)
314        self.assertTrue(color.allow_colorizing)
315
316        # Toggle off while colorizing in progress.
317        color.toggle_colorize_event()
318        self.assertIsNone(color.after_id)
319        self.assertTrue(color.colorizing)
320        self.assertTrue(color.stop_colorizing)
321        self.assertFalse(color.allow_colorizing)
322
323        # Toggle on while colorizing not in progress.
324        color.colorizing = False
325        color.toggle_colorize_event()
326        eq(self.root.tk.call('after', 'info', color.after_id)[1], 'timer')
327        self.assertFalse(color.colorizing)
328        self.assertTrue(color.stop_colorizing)
329        self.assertTrue(color.allow_colorizing)
330
331    @mock.patch.object(colorizer.ColorDelegator, 'recolorize_main')
332    def test_recolorize(self, mock_recmain):
333        text = self.text
334        color = self.color
335        eq = self.assertEqual
336        # Call recolorize manually and not scheduled.
337        text.after_cancel(color.after_id)
338
339        # No delegate.
340        save_delegate = color.delegate
341        color.delegate = None
342        color.recolorize()
343        mock_recmain.assert_not_called()
344        color.delegate = save_delegate
345
346        # Toggle off colorizing.
347        color.allow_colorizing = False
348        color.recolorize()
349        mock_recmain.assert_not_called()
350        color.allow_colorizing = True
351
352        # Colorizing in progress.
353        color.colorizing = True
354        color.recolorize()
355        mock_recmain.assert_not_called()
356        color.colorizing = False
357
358        # Colorizing is done, but not completed, so rescheduled.
359        color.recolorize()
360        self.assertFalse(color.stop_colorizing)
361        self.assertFalse(color.colorizing)
362        mock_recmain.assert_called()
363        eq(mock_recmain.call_count, 1)
364        # Rescheduled when TODO tag still exists.
365        eq(self.root.tk.call('after', 'info', color.after_id)[1], 'timer')
366
367        # No changes to text, so no scheduling added.
368        text.tag_remove('TODO', '1.0', 'end')
369        color.recolorize()
370        self.assertFalse(color.stop_colorizing)
371        self.assertFalse(color.colorizing)
372        mock_recmain.assert_called()
373        eq(mock_recmain.call_count, 2)
374        self.assertIsNone(color.after_id)
375
376    @mock.patch.object(colorizer.ColorDelegator, 'notify_range')
377    def test_recolorize_main(self, mock_notify):
378        text = self.text
379        color = self.color
380        eq = self.assertEqual
381
382        text.insert('insert', source)
383        expected = (('1.0', ('KEYWORD',)), ('1.2', ()), ('1.3', ('KEYWORD',)),
384                    ('1.7', ()), ('1.9', ('BUILTIN',)), ('1.14', ('STRING',)),
385                    ('1.19', ('COMMENT',)),
386                    ('2.1', ('KEYWORD',)), ('2.18', ()), ('2.25', ('COMMENT',)),
387                    ('3.6', ('BUILTIN',)), ('3.12', ('KEYWORD',)), ('3.21', ('COMMENT',)),
388                    ('4.0', ('KEYWORD',)), ('4.3', ()), ('4.6', ()),
389                    ('5.2', ('STRING',)), ('5.8', ('KEYWORD',)), ('5.10', ('STRING',)),
390                    ('6.0', ('KEYWORD',)), ('6.10', ('DEFINITION',)), ('6.11', ()),
391                    ('8.0', ('STRING',)), ('8.4', ()), ('8.5', ('STRING',)),
392                    ('8.12', ()), ('8.14', ('STRING',)),
393                    ('19.0', ('KEYWORD',)),
394                    ('20.4', ('KEYWORD',)), ('20.16', ('KEYWORD',)),# ('20.19', ('KEYWORD',)),
395                    #('22.4', ('KEYWORD',)), ('22.10', ('KEYWORD',)), ('22.14', ('KEYWORD',)), ('22.19', ('STRING',)),
396                    #('23.12', ('KEYWORD',)),
397                    ('24.8', ('KEYWORD',)),
398                    ('25.4', ('KEYWORD',)), ('25.9', ('KEYWORD',)),
399                    ('25.11', ('KEYWORD',)), ('25.15', ('STRING',)),
400                    ('25.19', ('KEYWORD',)), ('25.22', ()),
401                    ('25.24', ('KEYWORD',)), ('25.29', ('BUILTIN',)), ('25.37', ('KEYWORD',)),
402                    ('26.4', ('KEYWORD',)), ('26.9', ('KEYWORD',)),# ('26.11', ('KEYWORD',)), ('26.14', (),),
403                    ('27.25', ('STRING',)), ('27.38', ('STRING',)),
404                    ('29.0', ('STRING',)),
405                    ('30.1', ('STRING',)),
406                    # SYNC at the end of every line.
407                    ('1.55', ('SYNC',)), ('2.50', ('SYNC',)), ('3.34', ('SYNC',)),
408                   )
409
410        # Nothing marked to do therefore no tags in text.
411        text.tag_remove('TODO', '1.0', 'end')
412        color.recolorize_main()
413        for tag in text.tag_names():
414            with self.subTest(tag=tag):
415                eq(text.tag_ranges(tag), ())
416
417        # Source marked for processing.
418        text.tag_add('TODO', '1.0', 'end')
419        # Check some indexes.
420        color.recolorize_main()
421        for index, expected_tags in expected:
422            with self.subTest(index=index):
423                eq(text.tag_names(index), expected_tags)
424
425        # Check for some tags for ranges.
426        eq(text.tag_nextrange('TODO', '1.0'), ())
427        eq(text.tag_nextrange('KEYWORD', '1.0'), ('1.0', '1.2'))
428        eq(text.tag_nextrange('COMMENT', '2.0'), ('2.22', '2.43'))
429        eq(text.tag_nextrange('SYNC', '2.0'), ('2.43', '3.0'))
430        eq(text.tag_nextrange('STRING', '2.0'), ('4.17', '4.53'))
431        eq(text.tag_nextrange('STRING', '8.0'), ('8.0', '8.3'))
432        eq(text.tag_nextrange('STRING', '8.3'), ('8.5', '8.12'))
433        eq(text.tag_nextrange('STRING', '8.12'), ('8.14', '8.17'))
434        eq(text.tag_nextrange('STRING', '8.17'), ('8.19', '8.26'))
435        eq(text.tag_nextrange('SYNC', '8.0'), ('8.26', '9.0'))
436        eq(text.tag_nextrange('SYNC', '30.0'), ('30.10', '32.0'))
437
438    def _assert_highlighting(self, source, tag_ranges):
439        """Check highlighting of a given piece of code.
440
441        This inserts just this code into the Text widget. It will then
442        check that the resulting highlighting tag ranges exactly match
443        those described in the given `tag_ranges` dict.
444
445        Note that the irrelevant tags 'sel', 'TODO' and 'SYNC' are
446        ignored.
447        """
448        text = self.text
449
450        with mock.patch.object(colorizer.ColorDelegator, 'notify_range'):
451            text.delete('1.0', 'end-1c')
452            text.insert('insert', source)
453            text.tag_add('TODO', '1.0', 'end-1c')
454            self.color.recolorize_main()
455
456        # Make a dict with highlighting tag ranges in the Text widget.
457        text_tag_ranges = {}
458        for tag in set(text.tag_names()) - {'sel', 'TODO', 'SYNC'}:
459            indexes = [rng.string for rng in text.tag_ranges(tag)]
460            for index_pair in zip(indexes[::2], indexes[1::2]):
461                text_tag_ranges.setdefault(tag, []).append(index_pair)
462
463        self.assertEqual(text_tag_ranges, tag_ranges)
464
465        with mock.patch.object(colorizer.ColorDelegator, 'notify_range'):
466            text.delete('1.0', 'end-1c')
467
468    def test_def_statement(self):
469        # empty def
470        self._assert_highlighting('def', {'KEYWORD': [('1.0', '1.3')]})
471
472        # def followed by identifier
473        self._assert_highlighting('def foo:', {'KEYWORD': [('1.0', '1.3')],
474                                               'DEFINITION': [('1.4', '1.7')]})
475
476        # def followed by partial identifier
477        self._assert_highlighting('def fo', {'KEYWORD': [('1.0', '1.3')],
478                                             'DEFINITION': [('1.4', '1.6')]})
479
480        # def followed by non-keyword
481        self._assert_highlighting('def ++', {'KEYWORD': [('1.0', '1.3')]})
482
483    def test_match_soft_keyword(self):
484        # empty match
485        self._assert_highlighting('match', {'KEYWORD': [('1.0', '1.5')]})
486
487        # match followed by partial identifier
488        self._assert_highlighting('match fo', {'KEYWORD': [('1.0', '1.5')]})
489
490        # match followed by identifier and colon
491        self._assert_highlighting('match foo:', {'KEYWORD': [('1.0', '1.5')]})
492
493        # match followed by keyword
494        self._assert_highlighting('match and', {'KEYWORD': [('1.6', '1.9')]})
495
496        # match followed by builtin with keyword prefix
497        self._assert_highlighting('match int:', {'KEYWORD': [('1.0', '1.5')],
498                                                 'BUILTIN': [('1.6', '1.9')]})
499
500        # match followed by non-text operator
501        self._assert_highlighting('match^', {})
502        self._assert_highlighting('match @', {})
503
504        # match followed by colon
505        self._assert_highlighting('match :', {})
506
507        # match followed by comma
508        self._assert_highlighting('match\t,', {})
509
510        # match followed by a lone underscore
511        self._assert_highlighting('match _:', {'KEYWORD': [('1.0', '1.5')]})
512
513    def test_case_soft_keyword(self):
514        # empty case
515        self._assert_highlighting('case', {'KEYWORD': [('1.0', '1.4')]})
516
517        # case followed by partial identifier
518        self._assert_highlighting('case fo', {'KEYWORD': [('1.0', '1.4')]})
519
520        # case followed by identifier and colon
521        self._assert_highlighting('case foo:', {'KEYWORD': [('1.0', '1.4')]})
522
523        # case followed by keyword
524        self._assert_highlighting('case and', {'KEYWORD': [('1.5', '1.8')]})
525
526        # case followed by builtin with keyword prefix
527        self._assert_highlighting('case int:', {'KEYWORD': [('1.0', '1.4')],
528                                                'BUILTIN': [('1.5', '1.8')]})
529
530        # case followed by non-text operator
531        self._assert_highlighting('case^', {})
532        self._assert_highlighting('case @', {})
533
534        # case followed by colon
535        self._assert_highlighting('case :', {})
536
537        # case followed by comma
538        self._assert_highlighting('case\t,', {})
539
540        # case followed by a lone underscore
541        self._assert_highlighting('case _:', {'KEYWORD': [('1.0', '1.4'),
542                                                          ('1.5', '1.6')]})
543
544    def test_long_multiline_string(self):
545        source = textwrap.dedent('''\
546            """a
547            b
548            c
549            d
550            e"""
551            ''')
552        self._assert_highlighting(source, {'STRING': [('1.0', '5.4')]})
553
554    @run_in_tk_mainloop(delay=50)
555    def test_incremental_editing(self):
556        text = self.text
557        eq = self.assertEqual
558
559        # Simulate typing 'inte'. During this, the highlighting should
560        # change from normal to keyword to builtin to normal.
561        text.insert('insert', 'i')
562        yield
563        eq(text.tag_nextrange('BUILTIN', '1.0'), ())
564        eq(text.tag_nextrange('KEYWORD', '1.0'), ())
565
566        text.insert('insert', 'n')
567        yield
568        eq(text.tag_nextrange('BUILTIN', '1.0'), ())
569        eq(text.tag_nextrange('KEYWORD', '1.0'), ('1.0', '1.2'))
570
571        text.insert('insert', 't')
572        yield
573        eq(text.tag_nextrange('BUILTIN', '1.0'), ('1.0', '1.3'))
574        eq(text.tag_nextrange('KEYWORD', '1.0'), ())
575
576        text.insert('insert', 'e')
577        yield
578        eq(text.tag_nextrange('BUILTIN', '1.0'), ())
579        eq(text.tag_nextrange('KEYWORD', '1.0'), ())
580
581        # Simulate deleting three characters from the end of 'inte'.
582        # During this, the highlighting should change from normal to
583        # builtin to keyword to normal.
584        text.delete('insert-1c', 'insert')
585        yield
586        eq(text.tag_nextrange('BUILTIN', '1.0'), ('1.0', '1.3'))
587        eq(text.tag_nextrange('KEYWORD', '1.0'), ())
588
589        text.delete('insert-1c', 'insert')
590        yield
591        eq(text.tag_nextrange('BUILTIN', '1.0'), ())
592        eq(text.tag_nextrange('KEYWORD', '1.0'), ('1.0', '1.2'))
593
594        text.delete('insert-1c', 'insert')
595        yield
596        eq(text.tag_nextrange('BUILTIN', '1.0'), ())
597        eq(text.tag_nextrange('KEYWORD', '1.0'), ())
598
599    @mock.patch.object(colorizer.ColorDelegator, 'recolorize')
600    @mock.patch.object(colorizer.ColorDelegator, 'notify_range')
601    def test_removecolors(self, mock_notify, mock_recolorize):
602        text = self.text
603        color = self.color
604        text.insert('insert', source)
605
606        color.recolorize_main()
607        # recolorize_main doesn't add these tags.
608        text.tag_add("ERROR", "1.0")
609        text.tag_add("TODO", "1.0")
610        text.tag_add("hit", "1.0")
611        for tag in color.tagdefs:
612            with self.subTest(tag=tag):
613                self.assertNotEqual(text.tag_ranges(tag), ())
614
615        color.removecolors()
616        for tag in color.tagdefs:
617            with self.subTest(tag=tag):
618                self.assertEqual(text.tag_ranges(tag), ())
619
620
621if __name__ == '__main__':
622    unittest.main(verbosity=2)
623