diff --git a/Doc/library/idle.rst b/Doc/library/idle.rst index c7c30e5300c2a4a..6e8ebfbaf6072b6 100644 --- a/Doc/library/idle.rst +++ b/Doc/library/idle.rst @@ -295,6 +295,21 @@ Stack Viewer Auto-open Stack Viewer Toggle automatically opening the stack viewer on an unhandled exception. +Browse menu (Shell and Editor) +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +Token Browser + Open a window listing the Python tokens of the editor content + (or, in the Shell, the current input), + or of the selection if there is one. + Token type names are colored as by ``python -m tokenize``. + Selecting rows highlights the matching regions in the editor + and moves the cursor there; + selecting text or moving the cursor in the editor + selects the matching rows. + Double-click a row, or press :kbd:`Escape`, + to hide the browser and return to the editor at the token. + Options menu (Shell and Editor) ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ diff --git a/Lib/idlelib/News3.txt b/Lib/idlelib/News3.txt index bcbd7fa542a02a4..61103bc3a2f04ac 100644 --- a/Lib/idlelib/News3.txt +++ b/Lib/idlelib/News3.txt @@ -4,6 +4,11 @@ Released on 2026-10-01 ========================= +gh-152941: Add a Token Browser to IDLE, opened from the new Browse menu. +It lists the Python tokens of the editor content, the Shell input, or +the selection, with token type names colored as by `python -m tokenize`. +Patch by Serhiy Storchaka and Claude Code. + gh-152745: When "Run... Customized" with "Restart shell" unchecked while Shell is running code, including waiting for an input('prompt:') response, just report that the shell is executing instead of diff --git a/Lib/idlelib/editor.py b/Lib/idlelib/editor.py index a040d791bdeb528..3157910716bc911 100644 --- a/Lib/idlelib/editor.py +++ b/Lib/idlelib/editor.py @@ -166,6 +166,7 @@ def __init__(self, flist=None, filename=None, key=None, root=None): text.bind("<>", self.flist.close_all_callback) text.bind("<>", self.open_module_browser) text.bind("<>", self.open_path_browser) + text.bind("<>", self.open_token_browser) text.bind("<>", self.open_turtle_demo) self.set_status_bar() @@ -425,6 +426,7 @@ def set_line_and_column(self, event=None): ("edit", "_Edit"), ("format", "F_ormat"), ("run", "_Run"), + ("browse", "_Browse"), ("options", "_Options"), ("window", "_Window"), ("help", "_Help"), @@ -740,6 +742,11 @@ def open_path_browser(self, event=None): pathbrowser.PathBrowser(self.root) return "break" + def open_token_browser(self, event=None): + from idlelib import tokenbrowser + tokenbrowser.open(self) + return "break" + def open_turtle_demo(self, event = None): import subprocess diff --git a/Lib/idlelib/idle_test/htest.py b/Lib/idlelib/idle_test/htest.py index 778e5c3d84e4963..f5c27e817dff45c 100644 --- a/Lib/idlelib/idle_test/htest.py +++ b/Lib/idlelib/idle_test/htest.py @@ -79,6 +79,16 @@ "Verify x.y.z versions and test each button, including Close.\n " } +_token_browser_spec = { + 'file': 'tokenbrowser', + 'kwds': {}, + 'msg': "Select rows in the token table and verify the matching regions\n" + "are highlighted in the sample editor above. Select the whole\n" + "editor text, or part of it, and press Refresh.\n" + "Double-click a row and verify the editor cursor jumps to the\n" + "start of that token and the editor gets focus." + } + # TODO implement ^\; adding '' to function does not work. _calltip_window_spec = { 'file': 'calltip_w', diff --git a/Lib/idlelib/idle_test/test_config.py b/Lib/idlelib/idle_test/test_config.py index 6d75cf7aa67dcce..dd444ac27bd0be5 100644 --- a/Lib/idlelib/idle_test/test_config.py +++ b/Lib/idlelib/idle_test/test_config.py @@ -424,7 +424,8 @@ def test_get_extensions(self): eq(iGE(), ['ZzDummy']) eq(iGE(editor_only=True), ['ZzDummy']) eq(iGE(active_only=False), ['ZzDummy', 'DISABLE']) - eq(iGE(active_only=False, editor_only=True), ['ZzDummy', 'DISABLE']) + eq(iGE(active_only=False, editor_only=True), + ['ZzDummy', 'DISABLE']) userextn.remove_section('ZzDummy') userextn.remove_section('DISABLE') @@ -434,7 +435,8 @@ def test_remove_key_bind_names(self): self.assertCountEqual( conf.RemoveKeyBindNames(conf.GetSectionList('default', 'extensions')), - ['AutoComplete', 'CodeContext', 'FormatParagraph', 'ParenMatch', 'ZzDummy']) + ['AutoComplete', 'CodeContext', 'FormatParagraph', 'ParenMatch', + 'ZzDummy']) def test_get_extn_name_for_event(self): userextn.read_string(''' diff --git a/Lib/idlelib/idle_test/test_mainmenu.py b/Lib/idlelib/idle_test/test_mainmenu.py index 51d2accfe48a1c7..9eefe7b2e66e6e0 100644 --- a/Lib/idlelib/idle_test/test_mainmenu.py +++ b/Lib/idlelib/idle_test/test_mainmenu.py @@ -11,7 +11,7 @@ class MainMenuTest(unittest.TestCase): def test_menudefs(self): actual = [item[0] for item in mainmenu.menudefs] expect = ['file', 'edit', 'format', 'run', 'shell', - 'debug', 'options', 'window', 'help'] + 'debug', 'browse', 'options', 'window', 'help'] self.assertEqual(actual, expect) def test_default_keydefs(self): diff --git a/Lib/idlelib/idle_test/test_tokenbrowser.py b/Lib/idlelib/idle_test/test_tokenbrowser.py new file mode 100644 index 000000000000000..bdded543e0f1226 --- /dev/null +++ b/Lib/idlelib/idle_test/test_tokenbrowser.py @@ -0,0 +1,286 @@ +"Test tokenbrowser, coverage 95%." +from idlelib import tokenbrowser +from test.support import requires + +import unittest +from unittest import mock +from tkinter import Tk, Text +from idlelib.idle_test.mock_idle import Func + +code_sample = "import sys\n\ndef f(x):\n return x + 1\n" + + +class TokenBrowserOpenTest(unittest.TestCase): + "Test the open() entry point (no gui needed)." + + def make_editwin(self): + editwin = Func() # Only .top and .text are used. + editwin.top = 'toplevel' + editwin.text = 'text' + return editwin + + def test_open_creates_window(self): + editwin = self.make_editwin() + with mock.patch.object(tokenbrowser, 'TokenBrowserWindow', + Func(result='window')) as window: + tokenbrowser.open(editwin) + self.assertEqual(window.args, ('toplevel', 'text')) + self.assertEqual(editwin.token_browser, 'window') + + def test_open_reuses_window(self): + editwin = self.make_editwin() + editwin.token_browser = existing = Func() # A live window. + existing.winfo_exists = Func(result=1) + existing.refresh = Func() + with mock.patch.object(tokenbrowser, 'TokenBrowserWindow', + Func()) as new_window: + tokenbrowser.open(editwin) + self.assertTrue(existing.refresh.called) # Refreshed, not recreated. + self.assertFalse(new_window.called) + + +class TokenBrowserWindowTest(unittest.TestCase): + + @classmethod + def setUpClass(cls): + requires('gui') + cls.root = Tk() + cls.root.withdraw() + cls.text = Text(cls.root) + cls.window = tokenbrowser.TokenBrowserWindow( + cls.root, cls.text, _utest=True) + + @classmethod + def tearDownClass(cls): + cls.window.destroy() + cls.root.update_idletasks() + cls.root.destroy() + del cls.window, cls.text, cls.root + + def setUp(self): + self.text.delete("1.0", "end") + self.text.insert("1.0", code_sample) + self.window.populate() + + def find(self, type=None, string=None): + "Return the first tree item matching a token type and/or string." + tree = self.window.tree + for item in tree.get_children(): + typ, s = tree.item(item, "values") + if (type is None or typ == type) and \ + (string is None or s == repr(string)): + return item + self.fail(f"no token {type} {string!r}") + + def test_populate_text(self): + window = self.window + self.assertGreater(len(window.ranges), 0) + self.assertEqual(len(window.ranges), len(window.tree.get_children())) + self.assertIn("in text", window.status.cget("text")) + self.assertEqual(window.base, (1, 0)) + + def test_token_row_values(self): + tree = self.window.tree + item = tree.get_children()[0] + # First token is NAME 'import', shown as two columns, mapped to 1.0-1.6. + self.assertEqual(tree.item(item, "values"), ("NAME", repr("import"))) + self.assertEqual(self.window.ranges[item], ("1.0", "1.6")) + # Operators show their exact type. + self.find(type="PLUS", string="+") + + def test_token_colors(self): + self.text.delete("1.0", "end") + self.text.insert("1.0", "x = 'a' + 1 # c\n") + self.window.populate() + tree = self.window.tree + tags = {tree.item(item, "values")[1]: tree.item(item, "tags") + for item in tree.get_children()} + self.assertIn("string", tags[repr("'a'")]) + self.assertIn("number", tags[repr("1")]) + self.assertIn("comment", tags[repr("# c")]) + self.assertNotIn("string", tags[repr("x")]) # NAME: default color. + + def test_editor_index(self): + window = self.window + window.base = (1, 0) + self.assertEqual(window.editor_index(1, 0), "1.0") + self.assertEqual(window.editor_index(3, 4), "3.4") + + def test_selection_scope(self): + self.text.tag_add("sel", "4.11", "4.16") # 'x + 1' on line 4. + self.window.populate() + window = self.window + self.assertEqual(window.base, (4, 11)) + self.assertIn("in selection", window.status.cget("text")) + # Tokens map back to editor coordinates. + item = self.find(type="NAME", string="x") + self.assertEqual(window.ranges[item], ("4.11", "4.12")) + + def test_focused_highlights_and_moves_cursor(self): + # Browser drives the selection (it has focus): highlight the token + # in the editor and move the cursor to it. + window = self.window + window.focused = True + window.tree.selection_set(self.find(type="NAME", string="sys")) + window.select_tokens() + ranges = [str(i) for i in self.text.tag_ranges(tokenbrowser.TAG)] + self.assertEqual(ranges, ["1.7", "1.10"]) + self.assertEqual(self.text.index("insert"), "1.7") + + def test_not_focused_keeps_editor_clean(self): + # Editor drives the selection (browser not focused): select_tokens + # neither highlights the editor nor moves its cursor. + window = self.window + window.focused = False + self.text.mark_set("insert", "1.0") + window.tree.selection_set(self.find(type="NAME", string="sys")) + window.select_tokens() + self.assertEqual(self.text.tag_ranges(tokenbrowser.TAG), ()) + self.assertEqual(self.text.index("insert"), "1.0") + + def test_select_multiple_highlights(self): + window = self.window + window.focused = True + items = [self.find(type="NAME", string="import"), + self.find(type="NAME", string="sys")] + window.tree.selection_set(items) + window.select_tokens() + ranges = self.text.tag_ranges(tokenbrowser.TAG) + self.assertEqual(len(ranges), 4) # Two (start, end) pairs. + + def test_highlight_follows_focus(self): + window = self.window + window.tree.selection_set(self.find(type="NAME", string="sys")) + window.on_focus_in() # The browser has focus. + self.assertNotEqual(self.text.tag_ranges(tokenbrowser.TAG), ()) + window.on_focus_out() # Focus moves to the editor. + self.assertEqual(self.text.tag_ranges(tokenbrowser.TAG), ()) + window.on_focus_in() # Focus returns to the browser. + self.assertNotEqual(self.text.tag_ranges(tokenbrowser.TAG), ()) + + def test_extend_selection(self): + tree = self.window.tree + rows = tree.get_children() + tree.selection_set(rows[0]) + tree.focus(rows[0]) + self.window.extend_selection(1) + self.assertEqual(set(tree.selection()), {rows[0], rows[1]}) + self.window.extend_selection(1) + self.assertEqual(set(tree.selection()), {rows[0], rows[1], rows[2]}) + + def test_extend_selection_at_edge(self): + tree = self.window.tree + last = tree.get_children()[-1] + tree.selection_set(last) + tree.focus(last) + self.window.extend_selection(1) # No next row to add. + self.assertEqual(tree.selection(), (last,)) + + def test_zero_width_not_highlighted(self): + window = self.window + window.focused = True + item = self.find(type="ENDMARKER") + start, end = window.ranges[item] + self.assertEqual(start, end) + window.tree.selection_set(item) + window.select_tokens() + self.assertEqual(self.text.tag_ranges(tokenbrowser.TAG), ()) + + def test_sync_cursor_row(self): + # With no editor selection, sync selects the single row of the + # token under the cursor, without moving the cursor. + window = self.window + self.text.mark_set("insert", "1.8") # Inside 'sys' (1.7-1.10). + window.sync_from_editor() + selection = window.tree.selection() + self.assertEqual(len(selection), 1) + self.assertEqual(window.tree.item(selection[0], "values"), + ("NAME", repr("sys"))) + self.assertEqual(self.text.index("insert"), "1.8") + + def test_sync_selection_selects_rows(self): + # An editor selection selects every overlapping token's row. + window = self.window + self.text.tag_add("sel", "4.11", "4.16") # 'x + 1' on line 4. + window.sync_from_editor() + values = {window.tree.item(item, "values") + for item in window.tree.selection()} + self.assertEqual(values, {("NAME", repr("x")), + ("PLUS", repr("+")), + ("NUMBER", repr("1"))}) + + def test_refresh(self): + window = self.window + self.text.delete("1.0", "end") + self.text.insert("1.0", "spam = 1\n") + window.refresh() + strings = [window.tree.item(i, "values")[1] + for i in window.tree.get_children()] + self.assertIn(repr("spam"), strings) + + def test_move_cursor(self): + window = self.window + item = self.find(type="NAME", string="return") + window.move_cursor(item) + self.assertEqual(self.text.index("insert"), window.ranges[item][0]) + + def test_move_cursor_no_item(self): + self.window.move_cursor("") # identify_row returns "" off a row. + + def test_hide(self): + text = Text(self.root) + text.insert("1.0", code_sample) + window = tokenbrowser.TokenBrowserWindow(self.root, text, _utest=True) + window.deiconify() + window.focused = True + window.tree.selection_set(window.tree.get_children()[0]) + window.select_tokens() + self.assertNotEqual(text.tag_ranges(tokenbrowser.TAG), ()) + window.hide() # Double-click (or Escape) hides it. + self.assertEqual(window.wm_state(), "withdrawn") # Not destroyed. + self.assertTrue(window.winfo_exists()) + self.assertEqual(text.tag_ranges(tokenbrowser.TAG), ()) + window.destroy() + text.destroy() + + def test_shell_input_scope(self): + # In the Shell (a Text with an "iomark"), browse only the current + # input, which starts after the prompt at the iomark. + text = Text(self.root) + text.insert("1.0", ">>> x = 1\n") + text.mark_set("iomark", "1.4") # After the ">>> " prompt. + window = tokenbrowser.TokenBrowserWindow(self.root, text, _utest=True) + self.assertEqual(window.base, (1, 4)) + self.assertIn("in input", window.status.cget("text")) + # The prompt is not tokenized; the first token is NAME 'x' at 1.4. + first = window.tree.get_children()[0] + self.assertEqual(window.tree.item(first, "values"), ("NAME", repr("x"))) + self.assertEqual(window.ranges[first], ("1.4", "1.5")) + window.destroy() + text.destroy() + + def test_no_selection_empty_index(self): + # The IDLE editor returns '' (not a TclError) for a missing selection + # or mark; that must be treated as "browse the whole text", not crash. + class EditorText(Text): + def index(self, spec): + if spec.startswith("sel.") or spec == "iomark": + return "" + return super().index(spec) + text = EditorText(self.root) + text.insert("1.0", code_sample) + window = tokenbrowser.TokenBrowserWindow(self.root, text, _utest=True) + self.assertEqual(window.base, (1, 0)) + self.assertIn("in text", window.status.cget("text")) + window.destroy() + text.destroy() + + def test_incomplete_source(self): + self.text.delete("1.0", "end") + self.text.insert("1.0", "def f(:\n") # Unbalanced/invalid. + self.window.populate() + self.assertIn("incomplete", self.window.status.cget("text")) + + +if __name__ == '__main__': + unittest.main(verbosity=2) diff --git a/Lib/idlelib/mainmenu.py b/Lib/idlelib/mainmenu.py index 91a32cebb513f91..787d3a4dac9c2f7 100644 --- a/Lib/idlelib/mainmenu.py +++ b/Lib/idlelib/mainmenu.py @@ -97,6 +97,10 @@ ('!_Auto-open Stack Viewer', '<>'), ]), + ('browse', [ + ('_Token Browser', '<>'), + ]), + ('options', [ ('Configure _IDLE', '<>'), None, diff --git a/Lib/idlelib/pyshell.py b/Lib/idlelib/pyshell.py index ef3d014d936ce85..1b59807ff5277f5 100755 --- a/Lib/idlelib/pyshell.py +++ b/Lib/idlelib/pyshell.py @@ -865,6 +865,7 @@ class PyShell(OutputWindow): ("file", "_File"), ("edit", "_Edit"), ("debug", "_Debug"), + ("browse", "_Browse"), ("options", "_Options"), ("window", "_Window"), ("help", "_Help"), diff --git a/Lib/idlelib/tokenbrowser.py b/Lib/idlelib/tokenbrowser.py new file mode 100644 index 000000000000000..a6d41680d680098 --- /dev/null +++ b/Lib/idlelib/tokenbrowser.py @@ -0,0 +1,345 @@ +"""A token browser for IDLE. + +The Browse menu's "Token Browser" command (see open() below) opens a +window listing the Python tokens of the editor content (or, in the Shell, +the current input), or of the selection if there is one. Selecting rows +highlights the matching regions in the editor and moves the editor cursor +there; selecting text (or moving the cursor) in the editor selects the +matching rows. Double-clicking a row hides the browser (as does Escape), +revealing the editor at the token. +""" +import io +import token +import tokenize + +from tkinter import Toplevel, TclError +from tkinter import TOP, BOTTOM, LEFT, RIGHT, X, Y, BOTH, W, END, VERTICAL +from tkinter import ttk + +from idlelib.config import idleConf + +# The editor tag that highlights the tokens of the selected rows. +TAG = "TOKENBROWSER" + +# Row colors per token group, mirroring the "python -m tokenize" CLI +# (see tokenize._get_token_colors and _colorize.Syntax/Tokenize). Token +# groups not listed here (NAME, OP) keep the default foreground. +GROUP_COLORS = { + 'comment': '#cc0000', # RED + 'string': '#008700', # GREEN + 'number': '#a67c00', # YELLOW + 'soft_keyword': '#0000cc', # BOLD_BLUE + 'whitespace': '#808080', # GREY + 'error': '#e40000', # BOLD_RED +} + + +def token_groups(): + "Map token type numbers to a color group name (mirrors the CLI)." + groups = {} + for group, names in ( + ('comment', ['COMMENT']), + ('whitespace', ['DEDENT', 'ENCODING', 'ENDMARKER', 'INDENT', + 'NEWLINE', 'NL']), + ('error', ['ERRORTOKEN']), + ('string', ['STRING', 'FSTRING_START', 'FSTRING_MIDDLE', + 'FSTRING_END', 'TSTRING_START', 'TSTRING_MIDDLE', + 'TSTRING_END']), + ('number', ['NUMBER']), + ('soft_keyword', ['SOFT_KEYWORD'])): + for name in names: + value = getattr(token, name, None) + if value is not None: # Some token types are version-specific. + groups[value] = group + return groups + + +TOKEN_GROUPS = token_groups() + + +def open(editwin): + "Open the token browser for editwin, reusing one already open." + window = getattr(editwin, "token_browser", None) + if window is not None and window.winfo_exists(): + window.refresh() + else: + editwin.token_browser = TokenBrowserWindow(editwin.top, editwin.text) + + +class TokenBrowserWindow(Toplevel): + "List the Python tokens of a Text widget's content or selection." + + def __init__(self, parent, text, *, _htest=False, _utest=False): + """Create the token browser. + + parent - the master widget of this window. + text - the editor Text widget to browse and drive. + _htest - bool; change box location when running htest. + _utest - bool; don't wait for user interaction when unit testing. + """ + super().__init__(parent) + self.text = text + self.base = (1, 0) # Editor index of the tokenized region's start. + self.ranges = {} # Tree item id -> (start index, end index). + self.focused = False # Whether the browser currently has the focus. + self.title("Token Browser") + self.protocol("WM_DELETE_WINDOW", self.hide) + self.bind("", self.hide) + x = parent.winfo_rootx() + 20 + y = parent.winfo_rooty() + (100 if _htest else 20) + self.geometry(f"640x480+{x}+{y}") + self.minsize(400, 300) + + self.create_widgets() + self.configure_tag() + self.populate() + # Follow the editor and select the matching rows. <> + # covers selection changes by keyboard or mouse (a generic + # is shadowed by IDLE's specific key bindings); the release events + # cover plain cursor moves that leave no selection. These bindings + # live as long as the editor Text and are torn down together with it + # (and with this child window), so there is nothing to unbind. + text.bind("<>", self.sync_from_editor, add="+") + text.bind("", self.sync_from_editor, add="+") + text.bind("", self.sync_from_editor, add="+") + if not _utest: + self.deiconify() + + def create_widgets(self): + bar = ttk.Frame(self, padding=(6, 6, 6, 0)) + bar.pack(side=TOP, fill=X) + ttk.Button(bar, text="Refresh", command=self.populate).pack(side=LEFT) + + self.status = ttk.Label(self, anchor=W, relief="sunken", padding=3) + self.status.pack(side=BOTTOM, fill=X) + + frame = ttk.Frame(self, padding=6) + frame.pack(side=TOP, fill=BOTH, expand=True) + self.tree = ttk.Treeview(frame, columns=("type", "string"), + show="headings", selectmode="extended") + for name, title, width, stretch in ( + ("type", "Type", 120, False), + ("string", "String", 260, True)): + self.tree.heading(name, text=title) + self.tree.column(name, width=width, stretch=stretch, anchor=W) + for group, color in GROUP_COLORS.items(): + self.tree.tag_configure(group, foreground=color) + vbar = ttk.Scrollbar(frame, orient=VERTICAL, command=self.tree.yview) + self.tree.configure(yscrollcommand=vbar.set) + vbar.pack(side=RIGHT, fill=Y) + self.tree.pack(side=LEFT, fill=BOTH, expand=True) + self.tree.bind("<>", self.select_tokens) + self.tree.bind("", self.goto_token) + # Shift + Up/Down extends the selection with the keyboard. + self.tree.bind("", lambda e: self.extend_selection(-1)) + self.tree.bind("", lambda e: self.extend_selection(1)) + # The highlight is shown only while the browser has the focus. + self.bind("", self.on_focus_in) + self.bind("", self.on_focus_out) + + def configure_tag(self): + "Give the highlight tag the theme's 'hit' colors." + try: + colors = idleConf.GetHighlight(idleConf.CurrentTheme(), 'hit') + except Exception: + colors = {'foreground': '#000000', 'background': '#ffff80'} + self.text.tag_configure(TAG, **colors) + + def editor_index(self, row, col): + "Map a token (row, col) to an editor index, honoring the selection." + base_row, base_col = self.base + if row == 1: + col += base_col + return f"{base_row + row - 1}.{col}" + + def editor_selection(self): + "Return the editor's (first, last) selection, or ('', '') if none." + try: + # A plain Text raises without a selection; the IDLE editor + # returns an empty string instead. + return self.text.index("sel.first"), self.text.index("sel.last") + except TclError: + return "", "" + + def populate(self, event=None): + "Tokenize the content (or selection) and fill the table." + self.hide_highlight() + self.tree.delete(*self.tree.get_children()) + self.ranges.clear() + text = self.text + first, last = self.editor_selection() + if first and last: + scope = "selection" + else: + last = text.index("end-1c") + # In the Shell, browse just the current input, which starts at the + # "iomark"; a plain editor has no such mark. IDLE's editor returns + # '' for a missing mark, while a plain Text raises TclError. + try: + first = text.index("iomark") + except TclError: + first = "" + if first: + scope = "input" + else: + first, scope = "1.0", "text" + self.base = tuple(int(i) for i in first.split(".")) + source = text.get(first, last) + if not source.endswith("\n"): + source += "\n" + error = None + try: + for tok in tokenize.generate_tokens(io.StringIO(source).readline): + self.add_token(tok) + except (tokenize.TokenError, IndentationError, SyntaxError) as exc: + error = exc.args[0] if exc.args else type(exc).__name__ + status = f"{len(self.ranges)} tokens in {scope}" + if error: + status += f" — incomplete: {error}" + self.status.configure(text=status) + self.sync_from_editor() + + def refresh(self): + "Re-tokenize the current range and bring the browser to the front." + self.populate() + self.deiconify() + self.lift() + self.focus_set() + + def sync_from_editor(self, event=None): + "Select the rows matching the editor's selection, or the cursor's row." + first, last = self.editor_selection() + if first and last: + # Select every token whose range overlaps the editor selection. + text = self.text + self.select_rows( + [item for item, (start, end) in self.ranges.items() + if text.compare(start, "<", last) + and text.compare(end, ">", first)]) + else: + self.select_cursor_row() + + def select_cursor_row(self): + "Select the row of the token that contains the editor's cursor." + insert = self.text.index("insert") + chosen = None + for item, (start, end) in self.ranges.items(): + if self.text.compare(start, "<=", insert): + chosen = item # Last token starting at or before it. + if self.text.compare(insert, "<", end): + break # The cursor is inside this token. + self.select_rows([chosen] if chosen else []) + + def select_rows(self, items): + "Select the given tree rows and reveal the first." + if items: + self.tree.selection_set(items) + self.tree.focus(items[0]) + self.tree.see(items[0]) + + def add_token(self, tok): + name = token.tok_name[tok.exact_type] + start = self.editor_index(*tok.start) + end = self.editor_index(*tok.end) + group = TOKEN_GROUPS.get(tok.type, '') # '' means the default color. + item = self.tree.insert("", END, values=(name, repr(tok.string)), + tags=(group,) if group else ()) + self.ranges[item] = (start, end) + + def select_tokens(self, event=None): + "Highlight the selected rows and, while focused, follow with the cursor." + self.show_highlight(see=True) + # Move the editor cursor only when the browser drives the selection + # (it has the focus). When the editor drives it, the browser is not + # focused, so the cursor is left alone and there is no feedback loop. + if self.focused: + self.move_cursor() + + def show_highlight(self, see=False): + "Highlight the selected rows' tokens while the browser has focus." + if not self.focused: # Keep the editor clean while it is in use. + return + text = self.text + self.hide_highlight() + first = None + for item in self.tree.selection(): + start, end = self.ranges[item] + if start != end: # Skip zero-width tokens (NEWLINE, INDENT, ...). + text.tag_add(TAG, start, end) + if first is None: + first = start + text.tag_raise(TAG) + if see and first is not None: + text.see(first) + + def on_focus_in(self, event=None): + "Restore the highlight when the browser regains focus." + self.focused = True + self.show_highlight() + + def on_focus_out(self, event=None): + "Hide the highlight while the editor (or another window) has focus." + self.focused = False + self.hide_highlight() + + def extend_selection(self, direction): + "Extend the selection to the previous or next row (Shift+Up/Down)." + tree = self.tree + item = tree.next(tree.focus()) if direction > 0 else tree.prev(tree.focus()) + if item: + tree.selection_add(item) + tree.focus(item) + tree.see(item) + return "break" + + def goto_token(self, event=None): + "Move the cursor to the double-clicked token and hide the browser." + self.move_cursor(self.tree.identify_row(event.y)) + self.hide() + return "break" # Suppress the default double-click handling. + + def move_cursor(self, item=None): + "Move the editor cursor to a token (the first selected row by default)." + if item is None: + selection = self.tree.selection() + item = selection[0] if selection else None + if not item: + return + start, end = self.ranges[item] + self.text.mark_set("insert", start) + self.text.see(start) + + def hide(self, event=None): + """Withdraw the browser, revealing the editor and giving it focus. + + Hiding our own window sidesteps the window manager's focus-stealing + prevention, which blocks a background editor window from being raised. + """ + self.hide_highlight() + self.withdraw() + self.text.focus_set() + + def hide_highlight(self, event=None): + try: + self.text.tag_remove(TAG, "1.0", "end") + except TclError: # The editor may already be gone. + pass + + +def _token_browser(parent): # htest # + "Set up a sample editor Text and open a token browser on it." + from tkinter import Text + top = Toplevel(parent) + top.title("Sample editor") + text = Text(top, width=40, height=8) + text.insert("1.0", "import sys\n\ndef f(x):\n return x + 1 # add one\n") + text.pack(fill=BOTH, expand=True) + return TokenBrowserWindow(top, text, _htest=True) + + +if __name__ == "__main__": + from unittest import main + main('idlelib.idle_test.test_tokenbrowser', verbosity=2, exit=False) + + from idlelib.idle_test.htest import run + run(_token_browser) diff --git a/Misc/NEWS.d/next/IDLE/2026-07-03-12-00-00.gh-issue-152941.ToKeNs.rst b/Misc/NEWS.d/next/IDLE/2026-07-03-12-00-00.gh-issue-152941.ToKeNs.rst new file mode 100644 index 000000000000000..0cf33bcdfea92e3 --- /dev/null +++ b/Misc/NEWS.d/next/IDLE/2026-07-03-12-00-00.gh-issue-152941.ToKeNs.rst @@ -0,0 +1,3 @@ +Add a Token Browser to IDLE, opened from the new Browse menu. It lists the +Python tokens of the editor content, the Shell input, or the selection, with +token type names colored as by ``python -m tokenize``.