diff --git a/lib/syntax_suggest/clean_document.rb b/lib/syntax_suggest/clean_document.rb index ba307af..bf32a17 100644 --- a/lib/syntax_suggest/clean_document.rb +++ b/lib/syntax_suggest/clean_document.rb @@ -182,8 +182,8 @@ def join_heredoc! start_index_stack = [] heredoc_beg_end_index = [] lines.each do |line| - line.lex.each do |lex_value| - case lex_value.type + line.tokens.each do |token| + case token.type when :on_heredoc_beg start_index_stack << line.index when :on_heredoc_end @@ -273,7 +273,7 @@ def join_groups(groups) # Join group into the first line @document[line.index] = CodeLine.new( - lex: lines.map(&:lex).flatten, + tokens: lines.map(&:tokens).flatten, line: lines.join, index: line.index ) @@ -282,7 +282,7 @@ def join_groups(groups) lines[1..].each do |line| # The above lines already have newlines in them, if add more # then there will be double newline, use an empty line instead - @document[line.index] = CodeLine.new(line: "", index: line.index, lex: []) + @document[line.index] = CodeLine.new(line: "", index: line.index, tokens: []) end end self diff --git a/lib/syntax_suggest/code_line.rb b/lib/syntax_suggest/code_line.rb index 76ca892..2a681cb 100644 --- a/lib/syntax_suggest/code_line.rb +++ b/lib/syntax_suggest/code_line.rb @@ -28,19 +28,20 @@ class CodeLine # from the source string def self.from_source(source, lines: nil) lines ||= source.lines - lex_array_for_line = LexAll.new(source: source, source_lines: lines).each_with_object(Hash.new { |h, k| h[k] = [] }) { |lex, hash| hash[lex.line] << lex } + tokens = LexAll.new(source: source, source_lines: lines) + tokens_for_line = tokens.each_with_object(Hash.new { |h, k| h[k] = [] }) { |token, hash| hash[token.line] << token } lines.map.with_index do |line, index| CodeLine.new( line: line, index: index, - lex: lex_array_for_line[index + 1] + tokens: tokens_for_line[index + 1] ) end end - attr_reader :line, :index, :lex, :line_number, :indent - def initialize(line:, index:, lex:) - @lex = lex + attr_reader :line, :index, :tokens, :line_number, :indent + def initialize(line:, index:, tokens:) + @tokens = tokens @line = line @index = index @original = line @@ -181,12 +182,12 @@ def ignore_newline_not_beg? # expect(lines.first.trailing_slash?).to eq(true) # def trailing_slash? - last = @lex.last + last = @tokens.last # Older versions of prism diverged slightly from Ripper in compatibility mode case last&.type when :on_sp - last.token == TRAILING_SLASH + last.value == TRAILING_SLASH when :on_tstring_end true else @@ -210,21 +211,21 @@ def trailing_slash? end_count = 0 @ignore_newline_not_beg = false - @lex.each do |lex| - kw_count += 1 if lex.is_kw? - end_count += 1 if lex.is_end? + @tokens.each do |token| + kw_count += 1 if token.is_kw? + end_count += 1 if token.is_end? - if lex.type == :on_ignored_nl - @ignore_newline_not_beg = !lex.expr_beg? + if token.type == :on_ignored_nl + @ignore_newline_not_beg = !token.expr_beg? end if in_oneliner_def.nil? - in_oneliner_def = :ENDFN if lex.state.allbits?(Ripper::EXPR_ENDFN) - elsif lex.state.allbits?(Ripper::EXPR_ENDFN) + in_oneliner_def = :ENDFN if token.state.allbits?(Ripper::EXPR_ENDFN) + elsif token.state.allbits?(Ripper::EXPR_ENDFN) # Continue - elsif lex.state.allbits?(Ripper::EXPR_BEG) - in_oneliner_def = :BODY if lex.token == "=" - elsif lex.state.allbits?(Ripper::EXPR_END) + elsif token.state.allbits?(Ripper::EXPR_BEG) + in_oneliner_def = :BODY if token.value == "=" + elsif token.state.allbits?(Ripper::EXPR_END) # We found an endless method, count it oneliner_count += 1 if in_oneliner_def == :BODY diff --git a/lib/syntax_suggest/explain_syntax.rb b/lib/syntax_suggest/explain_syntax.rb index 0d80c4d..9638ff4 100644 --- a/lib/syntax_suggest/explain_syntax.rb +++ b/lib/syntax_suggest/explain_syntax.rb @@ -1,6 +1,6 @@ # frozen_string_literal: true -require_relative "left_right_lex_count" +require_relative "left_right_token_count" if !SyntaxSuggest.use_prism_parser? require_relative "ripper_errors" @@ -53,14 +53,14 @@ class ExplainSyntax def initialize(code_lines:) @code_lines = code_lines - @left_right = LeftRightLexCount.new + @left_right = LeftRightTokenCount.new @missing = nil end def call @code_lines.each do |line| - line.lex.each do |lex| - @left_right.count_lex(lex) + line.tokens.each do |token| + @left_right.count_token(token) end end diff --git a/lib/syntax_suggest/left_right_lex_count.rb b/lib/syntax_suggest/left_right_token_count.rb similarity index 85% rename from lib/syntax_suggest/left_right_lex_count.rb rename to lib/syntax_suggest/left_right_token_count.rb index 6fcae74..87a1c39 100644 --- a/lib/syntax_suggest/left_right_lex_count.rb +++ b/lib/syntax_suggest/left_right_token_count.rb @@ -9,19 +9,19 @@ module SyntaxSuggest # # Example: # - # left_right = LeftRightLexCount.new + # left_right = LeftRightTokenCount.new # left_right.count_kw # left_right.missing.first # # => "end" # - # left_right = LeftRightLexCount.new + # left_right = LeftRightTokenCount.new # source = "{ a: b, c: d" # Note missing '}' - # LexAll.new(source: source).each do |lex| - # left_right.count_lex(lex) + # LexAll.new(source: source).each do |token| + # left_right.count_token(token) # end # left_right.missing.first # # => "}" - class LeftRightLexCount + class LeftRightTokenCount def initialize @kw_count = 0 @end_count = 0 @@ -49,14 +49,14 @@ def count_end # # Example: # - # left_right = LeftRightLexCount.new - # left_right.count_lex(LexValue.new(1, :on_lbrace, "{", Ripper::EXPR_BEG)) + # left_right = LeftRightTokenCount.new + # left_right.count_token(Token.new(1, :on_lbrace, "{", Ripper::EXPR_BEG)) # left_right.count_for_char("{") # # => 1 # left_right.count_for_char("}") # # => 0 - def count_lex(lex) - case lex.type + def count_token(token) + case token.type when :on_tstring_content # ^^^ # Means it's a string or a symbol `"{"` rather than being @@ -70,7 +70,7 @@ def count_lex(lex) # The start token will be the full thing `%Q{` but we # need to count it as if it's a `{`. Any token # can be used - char = lex.token[-1] + char = token.value[-1] @count_for_char[char] += 1 if @count_for_char.key?(char) when :on_embexpr_beg # ^^^ @@ -87,14 +87,14 @@ def count_lex(lex) # When we see `#{` count it as a `{` or we will # have a mis-match count. # - case lex.token + case token.value when "\#{" @count_for_char["{"] += 1 end else - @end_count += 1 if lex.is_end? - @kw_count += 1 if lex.is_kw? - @count_for_char[lex.token] += 1 if @count_for_char.key?(lex.token) + @end_count += 1 if token.is_end? + @kw_count += 1 if token.is_kw? + @count_for_char[token.value] += 1 if @count_for_char.key?(token.value) end end diff --git a/lib/syntax_suggest/lex_all.rb b/lib/syntax_suggest/lex_all.rb index c16fbb5..b04309d 100644 --- a/lib/syntax_suggest/lex_all.rb +++ b/lib/syntax_suggest/lex_all.rb @@ -11,32 +11,32 @@ module SyntaxSuggest # # Example usage: # - # lex = LexAll.new(source: source) - # lex.each do |value| - # puts value.line + # tokens = LexAll.new(source: source) + # tokens.each do |token| + # puts token.line # end class LexAll include Enumerable def initialize(source:, source_lines: nil) - @lex = self.class.lex(source, 1) - lineno = @lex.last[0][0] + 1 + @tokens = self.class.lex(source, 1) + lineno = @tokens.last[0][0] + 1 source_lines ||= source.lines last_lineno = source_lines.length until lineno >= last_lineno lines = source_lines[lineno..] - @lex.concat( + @tokens.concat( self.class.lex(lines.join, lineno + 1) ) - lineno = @lex.last[0].first + 1 + lineno = @tokens.last[0].first + 1 end - last_lex = nil - @lex.map! { |elem| - last_lex = LexValue.new(elem[0].first, elem[1], elem[2], elem[3], last_lex) + last_token = nil + @tokens.map! { |elem| + last_token = Token.new(elem[0].first, elem[1], elem[2], elem[3], last_token) } end @@ -51,24 +51,24 @@ def self.lex(source, line_number) end def to_a - @lex + @tokens end def each - return @lex.each unless block_given? - @lex.each do |x| - yield x + return @tokens.each unless block_given? + @tokens.each do |token| + yield token end end def [](index) - @lex[index] + @tokens[index] end def last - @lex.last + @tokens.last end end end -require_relative "lex_value" +require_relative "token" diff --git a/lib/syntax_suggest/lex_value.rb b/lib/syntax_suggest/token.rb similarity index 71% rename from lib/syntax_suggest/lex_value.rb rename to lib/syntax_suggest/token.rb index b46a332..d4577f5 100644 --- a/lib/syntax_suggest/lex_value.rb +++ b/lib/syntax_suggest/token.rb @@ -9,29 +9,29 @@ module SyntaxSuggest # # Would translate into: # - # lex.line # => 1 - # lex.type # => :on_indent - # lex.token # => "describe" - class LexValue - attr_reader :line, :type, :token, :state + # token.line # => 1 + # token.type # => :on_indent + # token.value # => "describe" + class Token + attr_reader :line, :type, :value, :state - def initialize(line, type, token, state, last_lex = nil) + def initialize(line, type, value, state, last_token = nil) @line = line @type = type - @token = token + @value = value @state = state - set_kw_end(last_lex) + set_kw_end(last_token) end - private def set_kw_end(last_lex) + private def set_kw_end(last_token) @is_end = false @is_kw = false return if type != :on_kw - return if last_lex && last_lex.fname? # https://github.com/ruby/ruby/commit/776759e300e4659bb7468e2b97c8c2d4359a2953 + return if last_token && last_token.fname? # https://github.com/ruby/ruby/commit/776759e300e4659bb7468e2b97c8c2d4359a2953 - case token + case value when "if", "unless", "while", "until" # Only count if/unless when it's not a "trailing" if/unless # https://github.com/ruby/ruby/blob/06b44f819eb7b5ede1ff69cecb25682b56a1d60c/lib/irb/ruby-lex.rb#L374-L375 diff --git a/spec/unit/code_block_spec.rb b/spec/unit/code_block_spec.rb index 3ab2751..baf1c63 100644 --- a/spec/unit/code_block_spec.rb +++ b/spec/unit/code_block_spec.rb @@ -33,7 +33,7 @@ def foo array = [block_2, block_1, block_0].sort expect(array.last).to eq(block_2) - block = CodeBlock.new(lines: CodeLine.new(line: " " * 8 + "foo", index: 4, lex: [])) + block = CodeBlock.new(lines: CodeLine.new(line: " " * 8 + "foo", index: 4, tokens: [])) array.prepend(block) expect(array.max).to eq(block) end diff --git a/spec/unit/lex_all_spec.rb b/spec/unit/lex_all_spec.rb index 9621c9e..b88ae65 100644 --- a/spec/unit/lex_all_spec.rb +++ b/spec/unit/lex_all_spec.rb @@ -17,10 +17,10 @@ module SyntaxSuggest end # 9 EOM - lex = LexAll.new(source: source) - expect(lex.map(&:token).to_s).to include("dog") - expect(lex.first.line).to eq(1) - expect(lex.last.line).to eq(9) + tokens = LexAll.new(source: source) + expect(tokens.map(&:value)).to include("dog") + expect(tokens.first.line).to eq(1) + expect(tokens.last.line).to eq(9) end end end