From c773a79424b4896c6fc48775485af20e392b9aba Mon Sep 17 00:00:00 2001 From: Todd Leonhardt Date: Fri, 27 Feb 2026 13:10:39 -0500 Subject: [PATCH 1/4] Fixed multiline command highlighting in Cmd2Lexer --- cmd2/pt_utils.py | 137 ++++++++++++++++++++++------------------- tests/test_pt_utils.py | 19 +++++- 2 files changed, 90 insertions(+), 66 deletions(-) diff --git a/cmd2/pt_utils.py b/cmd2/pt_utils.py index c99d7c97..a1558c88 100644 --- a/cmd2/pt_utils.py +++ b/cmd2/pt_utils.py @@ -219,77 +219,84 @@ def __init__( def lex_document(self, document: Document) -> Callable[[int], Any]: """Lex the document.""" + # Get redirection tokens and terminators to avoid highlighting them as values + exclude_tokens = set(constants.REDIRECTION_TOKENS) + exclude_tokens.update(self.cmd_app.statement_parser.terminators) + arg_pattern = re.compile(r'(\s+)|(--?[^\s\'"]+)|("[^"]*"?|\'[^\']*\'?)|([^\s\'"]+)') + + def highlight_args(text: str, tokens: list[tuple[str, str]]) -> None: + """Highlight arguments in a string.""" + for m in arg_pattern.finditer(text): + space, flag, quoted, word = m.groups() + match_text = m.group(0) + + if space: + tokens.append(('', match_text)) + elif flag: + tokens.append((self.flag_color, match_text)) + elif (quoted or word) and match_text not in exclude_tokens: + tokens.append((self.argument_color, match_text)) + else: + tokens.append(('', match_text)) def get_line(lineno: int) -> list[tuple[str, str]]: """Return the tokens for the given line number.""" line = document.lines[lineno] tokens: list[tuple[str, str]] = [] - # Use cmd2's command pattern to find the first word (the command) - if ru.ALLOW_STYLE != ru.AllowStyle.NEVER and ( - match := self.cmd_app.statement_parser._command_pattern.search(line) - ): - # Group 1 is the command, Group 2 is the character(s) that terminated the command match - command = match.group(1) - cmd_start = match.start(1) - cmd_end = match.end(1) - - # Add any leading whitespace - if cmd_start > 0: - tokens.append(('', line[:cmd_start])) - - if command: - # Determine the style for the command - shortcut_found = False - for shortcut, _ in self.cmd_app.statement_parser.shortcuts: - if command.startswith(shortcut): - # Add the shortcut with the command style - tokens.append((self.command_color, shortcut)) - - # If there's more in the command word, it's an argument - if len(command) > len(shortcut): - tokens.append((self.argument_color, command[len(shortcut) :])) - - shortcut_found = True - break - - if not shortcut_found: - style = '' - if command in self.cmd_app.get_all_commands(): - style = self.command_color - elif command in self.cmd_app.aliases: - style = self.alias_color - elif command in self.cmd_app.macros: - style = self.macro_color - - # Add the command with the determined style - tokens.append((style, command)) - - # Add the rest of the line - if cmd_end < len(line): - rest = line[cmd_end:] - # Regex to match whitespace, flags, quoted strings, or other words - arg_pattern = re.compile(r'(\s+)|(--?[^\s\'"]+)|("[^"]*"?|\'[^\']*\'?)|([^\s\'"]+)') - - # Get redirection tokens and terminators to avoid highlighting them as values - exclude_tokens = set(constants.REDIRECTION_TOKENS) - exclude_tokens.update(self.cmd_app.statement_parser.terminators) - - for m in arg_pattern.finditer(rest): - space, flag, quoted, word = m.groups() - text = m.group(0) - - if space: - tokens.append(('', text)) - elif flag: - tokens.append((self.flag_color, text)) - elif (quoted or word) and text not in exclude_tokens: - tokens.append((self.argument_color, text)) - else: - tokens.append(('', text)) - elif line: - # No command match found or colors aren't allowed, add the entire line unstyled - tokens.append(('', line)) + # Only attempt to match a command on the first line + if lineno == 0 and ru.ALLOW_STYLE != ru.AllowStyle.NEVER: + # Use cmd2's command pattern to find the first word (the command) + match = self.cmd_app.statement_parser._command_pattern.search(line) + if match: + # Group 1 is the command, Group 2 is the character(s) that terminated the command match + command = match.group(1) + cmd_start = match.start(1) + cmd_end = match.end(1) + + # Add any leading whitespace + if cmd_start > 0: + tokens.append(('', line[:cmd_start])) + + if command: + # Determine the style for the command + shortcut_found = False + for shortcut, _ in self.cmd_app.statement_parser.shortcuts: + if command.startswith(shortcut): + # Add the shortcut with the command style + tokens.append((self.command_color, shortcut)) + + # If there's more in the command word, it's an argument + if len(command) > len(shortcut): + tokens.append((self.argument_color, command[len(shortcut) :])) + + shortcut_found = True + break + + if not shortcut_found: + style = '' + if command in self.cmd_app.get_all_commands(): + style = self.command_color + elif command in self.cmd_app.aliases: + style = self.alias_color + elif command in self.cmd_app.macros: + style = self.macro_color + + # Add the command with the determined style + tokens.append((style, command)) + + # Add the rest of the line as arguments + if cmd_end < len(line): + highlight_args(line[cmd_end:], tokens) + else: + # No command match found on the first line + tokens.append(('', line)) + else: + # All other lines are unstyled or treated as arguments + if ru.ALLOW_STYLE != ru.AllowStyle.NEVER: + highlight_args(line, tokens) + else: + tokens.append(('', line)) return tokens diff --git a/tests/test_pt_utils.py b/tests/test_pt_utils.py index 15d37672..70e379d8 100644 --- a/tests/test_pt_utils.py +++ b/tests/test_pt_utils.py @@ -210,13 +210,30 @@ def test_lex_document_shortcut(self, mock_cmd_app): tokens = get_line(0) assert tokens == [('ansigreen', '!'), ('ansiyellow', 'ls')] - # Case 2: Shortcut with space line = "! ls" document = Document(line) get_line = lexer.lex_document(document) tokens = get_line(0) assert tokens == [('ansigreen', '!'), ('', ' '), ('ansiyellow', 'ls')] + def test_lex_document_multiline(self, mock_cmd_app): + """Test lexing a multiline command.""" + mock_cmd_app.all_commands = ["orate"] + lexer = pt_utils.Cmd2Lexer(cast(Any, mock_cmd_app)) + + # Command on first line, argument on second line that looks like a command + line = "orate\nhelp" + document = Document(line) + get_line = lexer.lex_document(document) + + # First line should have command + tokens0 = get_line(0) + assert tokens0 == [('ansigreen', 'orate')] + + # Second line should have argument (not command) + tokens1 = get_line(1) + assert tokens1 == [('ansiyellow', 'help')] + class TestCmd2Completer: def test_get_completions(self, mock_cmd_app: MockCmd, monkeypatch) -> None: From 10a9d0444db79e1a8581fdcdfd60861d128e9e59 Mon Sep 17 00:00:00 2001 From: Todd Leonhardt Date: Fri, 27 Feb 2026 13:48:57 -0500 Subject: [PATCH 2/4] Simplified logic by adding guard clause for case that allow_style is NEVER --- cmd2/pt_utils.py | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/cmd2/pt_utils.py b/cmd2/pt_utils.py index a1558c88..f13855bb 100644 --- a/cmd2/pt_utils.py +++ b/cmd2/pt_utils.py @@ -244,8 +244,13 @@ def get_line(lineno: int) -> list[tuple[str, str]]: line = document.lines[lineno] tokens: list[tuple[str, str]] = [] + # No syntax highlighting if styles are disallowed + if ru.ALLOW_STYLE == ru.AllowStyle.NEVER: + tokens.append(('', line)) + return tokens + # Only attempt to match a command on the first line - if lineno == 0 and ru.ALLOW_STYLE != ru.AllowStyle.NEVER: + if lineno == 0: # Use cmd2's command pattern to find the first word (the command) match = self.cmd_app.statement_parser._command_pattern.search(line) if match: @@ -292,11 +297,8 @@ def get_line(lineno: int) -> list[tuple[str, str]]: # No command match found on the first line tokens.append(('', line)) else: - # All other lines are unstyled or treated as arguments - if ru.ALLOW_STYLE != ru.AllowStyle.NEVER: - highlight_args(line, tokens) - else: - tokens.append(('', line)) + # All other lines are treated as arguments + highlight_args(line, tokens) return tokens From 91f0ddc8a3b0f15e64caf1cc52613f42707ba1e3 Mon Sep 17 00:00:00 2001 From: Todd Leonhardt Date: Fri, 27 Feb 2026 14:08:06 -0500 Subject: [PATCH 3/4] Add a Cmd2Lexer test to cover the case when allow_style is NEVER --- tests/test_pt_utils.py | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/tests/test_pt_utils.py b/tests/test_pt_utils.py index 70e379d8..42cd1fdc 100644 --- a/tests/test_pt_utils.py +++ b/tests/test_pt_utils.py @@ -92,6 +92,18 @@ def test_pt_filter_style_never() -> None: class TestCmd2Lexer: + @with_ansi_style(ru.AllowStyle.NEVER) + def test_lex_document_no_style(self, mock_cmd_app): + """Test lexing when styles are disallowed.""" + lexer = pt_utils.Cmd2Lexer(cast(Any, mock_cmd_app)) + + line = "help something" + document = Document(line) + get_line = lexer.lex_document(document) + tokens = get_line(0) + + assert tokens == [('', line)] + def test_lex_document_command(self, mock_cmd_app): """Test lexing a command name.""" mock_cmd_app.all_commands = ["help"] From acd1d53bf48bac697fd81781c54ae9a103c88651 Mon Sep 17 00:00:00 2001 From: Todd Leonhardt Date: Fri, 27 Feb 2026 14:26:26 -0500 Subject: [PATCH 4/4] Add test for case when lexer has no command match on first line --- tests/test_pt_utils.py | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/tests/test_pt_utils.py b/tests/test_pt_utils.py index 42cd1fdc..69ef4c10 100644 --- a/tests/test_pt_utils.py +++ b/tests/test_pt_utils.py @@ -174,6 +174,19 @@ def test_lex_document_no_command(self, mock_cmd_app): assert tokens == [('', ' ')] + def test_lex_document_no_match(self, mock_cmd_app): + """Test lexing when command pattern fails to match.""" + # Force the pattern to not match anything + mock_cmd_app.statement_parser._command_pattern = re.compile(r'something_impossible') + lexer = pt_utils.Cmd2Lexer(cast(Any, mock_cmd_app)) + + line = "test command" + document = Document(line) + get_line = lexer.lex_document(document) + tokens = get_line(0) + + assert tokens == [('', line)] + def test_lex_document_arguments(self, mock_cmd_app): """Test lexing a command with flags and values.""" mock_cmd_app.all_commands = ["help"]