Fixed multiline command highlighting in Cmd2Lexer (#1591)

Fixed multiline command highlighting in Cmd2Lexer

Author: tleonhardt

cmd2/pt_utils.py

@@ -219,77 +219,86 @@ def __init__(
 
     def lex_document(self, document: Document) -> Callable[[int], Any]:
         """Lex the document."""
+        # Get redirection tokens and terminators to avoid highlighting them as values
+        exclude_tokens = set(constants.REDIRECTION_TOKENS)
+        exclude_tokens.update(self.cmd_app.statement_parser.terminators)
+        arg_pattern = re.compile(r'(\s+)|(--?[^\s\'"]+)|("[^"]*"?|\'[^\']*\'?)|([^\s\'"]+)')
+
+        def highlight_args(text: str, tokens: list[tuple[str, str]]) -> None:
+            """Highlight arguments in a string."""
+            for m in arg_pattern.finditer(text):
+                space, flag, quoted, word = m.groups()
+                match_text = m.group(0)
+
+                if space:
+                    tokens.append(('', match_text))
+                elif flag:
+                    tokens.append((self.flag_color, match_text))
+                elif (quoted or word) and match_text not in exclude_tokens:
+                    tokens.append((self.argument_color, match_text))
+                else:
+                    tokens.append(('', match_text))
 
         def get_line(lineno: int) -> list[tuple[str, str]]:
             """Return the tokens for the given line number."""
             line = document.lines[lineno]
             tokens: list[tuple[str, str]] = []
 
-            # Use cmd2's command pattern to find the first word (the command)
-            if ru.ALLOW_STYLE != ru.AllowStyle.NEVER and (
-                match := self.cmd_app.statement_parser._command_pattern.search(line)
-            ):
-                # Group 1 is the command, Group 2 is the character(s) that terminated the command match
-                command = match.group(1)
-                cmd_start = match.start(1)
-                cmd_end = match.end(1)
-
-                # Add any leading whitespace
-                if cmd_start > 0:
-                    tokens.append(('', line[:cmd_start]))
-
-                if command:
-                    # Determine the style for the command
-                    shortcut_found = False
-                    for shortcut, _ in self.cmd_app.statement_parser.shortcuts:
-                        if command.startswith(shortcut):
-                            # Add the shortcut with the command style
-                            tokens.append((self.command_color, shortcut))
-
-                            # If there's more in the command word, it's an argument
-                            if len(command) > len(shortcut):
-                                tokens.append((self.argument_color, command[len(shortcut) :]))
-
-                            shortcut_found = True
-                            break
-
-                    if not shortcut_found:
-                        style = ''
-                        if command in self.cmd_app.get_all_commands():
-                            style = self.command_color
-                        elif command in self.cmd_app.aliases:
-                            style = self.alias_color
-                        elif command in self.cmd_app.macros:
-                            style = self.macro_color
-
-                        # Add the command with the determined style
-                        tokens.append((style, command))
-
-                # Add the rest of the line
-                if cmd_end < len(line):
-                    rest = line[cmd_end:]
-                    # Regex to match whitespace, flags, quoted strings, or other words
-                    arg_pattern = re.compile(r'(\s+)|(--?[^\s\'"]+)|("[^"]*"?|\'[^\']*\'?)|([^\s\'"]+)')
-
-                    # Get redirection tokens and terminators to avoid highlighting them as values
-                    exclude_tokens = set(constants.REDIRECTION_TOKENS)
-                    exclude_tokens.update(self.cmd_app.statement_parser.terminators)
-
-                    for m in arg_pattern.finditer(rest):
-                        space, flag, quoted, word = m.groups()
-                        text = m.group(0)
-
-                        if space:
-                            tokens.append(('', text))
-                        elif flag:
-                            tokens.append((self.flag_color, text))
-                        elif (quoted or word) and text not in exclude_tokens:
-                            tokens.append((self.argument_color, text))
-                        else:
-                            tokens.append(('', text))
-            elif line:
-                # No command match found or colors aren't allowed, add the entire line unstyled
+            # No syntax highlighting if styles are disallowed
+            if ru.ALLOW_STYLE == ru.AllowStyle.NEVER:
                 tokens.append(('', line))
+                return tokens
+
+            # Only attempt to match a command on the first line
+            if lineno == 0:
+                # Use cmd2's command pattern to find the first word (the command)
+                match = self.cmd_app.statement_parser._command_pattern.search(line)
+                if match:
+                    # Group 1 is the command, Group 2 is the character(s) that terminated the command match
+                    command = match.group(1)
+                    cmd_start = match.start(1)
+                    cmd_end = match.end(1)
+
+                    # Add any leading whitespace
+                    if cmd_start > 0:
+                        tokens.append(('', line[:cmd_start]))
+
+                    if command:
+                        # Determine the style for the command
+                        shortcut_found = False
+                        for shortcut, _ in self.cmd_app.statement_parser.shortcuts:
+                            if command.startswith(shortcut):
+                                # Add the shortcut with the command style
+                                tokens.append((self.command_color, shortcut))
+
+                                # If there's more in the command word, it's an argument
+                                if len(command) > len(shortcut):
+                                    tokens.append((self.argument_color, command[len(shortcut) :]))
+
+                                shortcut_found = True
+                                break
+
+                        if not shortcut_found:
+                            style = ''
+                            if command in self.cmd_app.get_all_commands():
+                                style = self.command_color
+                            elif command in self.cmd_app.aliases:
+                                style = self.alias_color
+                            elif command in self.cmd_app.macros:
+                                style = self.macro_color
+
+                            # Add the command with the determined style
+                            tokens.append((style, command))
+
+                    # Add the rest of the line as arguments
+                    if cmd_end < len(line):
+                        highlight_args(line[cmd_end:], tokens)
+                else:
+                    # No command match found on the first line
+                    tokens.append(('', line))
+            else:
+                # All other lines are treated as arguments
+                highlight_args(line, tokens)
 
             return tokens
 

tests/test_pt_utils.py

@@ -92,6 +92,18 @@ def test_pt_filter_style_never() -> None:
 
 
 class TestCmd2Lexer:
+    @with_ansi_style(ru.AllowStyle.NEVER)
+    def test_lex_document_no_style(self, mock_cmd_app):
+        """Test lexing when styles are disallowed."""
+        lexer = pt_utils.Cmd2Lexer(cast(Any, mock_cmd_app))
+
+        line = "help something"
+        document = Document(line)
+        get_line = lexer.lex_document(document)
+        tokens = get_line(0)
+
+        assert tokens == [('', line)]
+
     def test_lex_document_command(self, mock_cmd_app):
         """Test lexing a command name."""
         mock_cmd_app.all_commands = ["help"]
@@ -162,6 +174,19 @@ def test_lex_document_no_command(self, mock_cmd_app):
 
         assert tokens == [('', '   ')]
 
+    def test_lex_document_no_match(self, mock_cmd_app):
+        """Test lexing when command pattern fails to match."""
+        # Force the pattern to not match anything
+        mock_cmd_app.statement_parser._command_pattern = re.compile(r'something_impossible')
+        lexer = pt_utils.Cmd2Lexer(cast(Any, mock_cmd_app))
+
+        line = "test command"
+        document = Document(line)
+        get_line = lexer.lex_document(document)
+        tokens = get_line(0)
+
+        assert tokens == [('', line)]
+
     def test_lex_document_arguments(self, mock_cmd_app):
         """Test lexing a command with flags and values."""
         mock_cmd_app.all_commands = ["help"]
@@ -210,13 +235,30 @@ def test_lex_document_shortcut(self, mock_cmd_app):
         tokens = get_line(0)
         assert tokens == [('ansigreen', '!'), ('ansiyellow', 'ls')]
 
-        # Case 2: Shortcut with space
         line = "! ls"
         document = Document(line)
         get_line = lexer.lex_document(document)
         tokens = get_line(0)
         assert tokens == [('ansigreen', '!'), ('', ' '), ('ansiyellow', 'ls')]
 
+    def test_lex_document_multiline(self, mock_cmd_app):
+        """Test lexing a multiline command."""
+        mock_cmd_app.all_commands = ["orate"]
+        lexer = pt_utils.Cmd2Lexer(cast(Any, mock_cmd_app))
+
+        # Command on first line, argument on second line that looks like a command
+        line = "orate\nhelp"
+        document = Document(line)
+        get_line = lexer.lex_document(document)
+
+        # First line should have command
+        tokens0 = get_line(0)
+        assert tokens0 == [('ansigreen', 'orate')]
+
+        # Second line should have argument (not command)
+        tokens1 = get_line(1)
+        assert tokens1 == [('ansiyellow', 'help')]
+
 
 class TestCmd2Completer:
     def test_get_completions(self, mock_cmd_app: MockCmd, monkeypatch) -> None: