Use native prism

.github/workflows/ci.yml

@@ -24,6 +24,7 @@ jobs:
     uses: ruby/actions/.github/workflows/ruby_versions.yml@master
     with:
       engine: cruby
+      min_version: 3.3
 
   test:
     needs: ruby-versions
@@ -33,7 +34,9 @@ jobs:
       matrix:
         ruby: ${{ fromJson(needs.ruby-versions.outputs.versions) }}
         prism_version:
-          - 1.2.0 # Shipped with Ruby 3.4 as default parser https://www.ruby-lang.org/en/news/2024/12/25/ruby-3-4-0-released/
+          # See https://stdgems.org/prism for which ruby version shipped with which prism version
+          - 0.19.0
+          - 1.2.0
           - 1.8.0
           - head
     env:
@@ -52,22 +55,3 @@ jobs:
       - name: test
         run: bin/rake test
         continue-on-error: ${{ matrix.ruby == 'head' }}
-
-  test-disable-prism:
-    needs: ruby-versions
-    runs-on: ubuntu-latest
-    strategy:
-      fail-fast: false
-      matrix:
-        ruby: ${{ fromJson(needs.ruby-versions.outputs.versions) }}
-    steps:
-      - name: Checkout code
-        uses: actions/checkout@v6
-      - name: Set up Ruby
-        uses: ruby/setup-ruby@v1
-        with:
-          ruby-version: ${{ matrix.ruby }}
-          bundler-cache: true
-      - name: test
-        run: SYNTAX_SUGGEST_DISABLE_PRISM=1 bin/rake test
-        continue-on-error: ${{ matrix.ruby == 'head' }}

.standard.yml

@@ -1 +1 @@
-ruby_version: 3.0.0
+ruby_version: 3.3.0

CHANGELOG.md

@@ -1,8 +1,11 @@
 ## HEAD (unreleased)
 
+- Changed: Changed: Minimum supported Ruby version is now 3.3. (https://github.com/ruby/syntax_suggest/pull/246)
+
 ## 2.0.3
 
 - Fix: Correctly identify trailing slashes when using Prism > 1.8.0. (https://github.com/ruby/syntax_suggest/pull/243)
+- Fix: Correctly handle `%I` delimiters. (https://github.com/ruby/syntax_suggest/pull/249)
 - Internal: Add tests to multiple versions of prism
 
 ## 2.0.2

lib/syntax_suggest/api.rb

@@ -7,25 +7,8 @@
 require "pathname"
 require "timeout"
 
-# We need Ripper loaded for `Prism.lex_compat` even if we're using Prism
-# for lexing and parsing
-require "ripper"
-
 # Prism is the new parser, replacing Ripper
-#
-# We need to "dual boot" both for now because syntax_suggest
-# supports older rubies that do not ship with syntax suggest.
-#
-# We also need the ability to control loading of this library
-# so we can test that both modes work correctly in CI.
-if (value = ENV["SYNTAX_SUGGEST_DISABLE_PRISM"])
-  warn "Skipping loading prism due to SYNTAX_SUGGEST_DISABLE_PRISM=#{value}"
-else
-  begin
-    require "prism"
-  rescue LoadError
-  end
-end
+require "prism"
 
 module SyntaxSuggest
   # Used to indicate a default value that cannot
@@ -35,14 +18,6 @@ module SyntaxSuggest
   class Error < StandardError; end
   TIMEOUT_DEFAULT = ENV.fetch("SYNTAX_SUGGEST_TIMEOUT", 1).to_i
 
-  # SyntaxSuggest.use_prism_parser? [Private]
-  #
-  # Tells us if the prism parser is available for use
-  # or if we should fallback to `Ripper`
-  def self.use_prism_parser?
-    defined?(Prism)
-  end
-
   # SyntaxSuggest.handle_error [Public]
   #
   # Takes a `SyntaxError` exception, uses the
@@ -152,20 +127,11 @@ def self.valid_without?(without_lines:, code_lines:)
   # SyntaxSuggest.invalid? [Private]
   #
   # Opposite of `SyntaxSuggest.valid?`
-  if defined?(Prism)
-    def self.invalid?(source)
-      source = source.join if source.is_a?(Array)
-      source = source.to_s
-
-      Prism.parse(source).failure?
-    end
-  else
-    def self.invalid?(source)
-      source = source.join if source.is_a?(Array)
-      source = source.to_s
+  def self.invalid?(source)
+    source = source.join if source.is_a?(Array)
+    source = source.to_s
 
-      Ripper.new(source).tap(&:parse).error?
-    end
+    Prism.parse(source).failure?
   end
 
   # SyntaxSuggest.valid? [Private]
@@ -219,7 +185,6 @@ def self.valid?(source)
 require_relative "clean_document"
 
 # Helpers
-require_relative "lex_all"
 require_relative "code_line"
 require_relative "code_block"
 require_relative "block_expand"
@@ -231,3 +196,5 @@ def self.valid?(source)
 require_relative "pathname_from_message"
 require_relative "display_invalid_blocks"
 require_relative "parse_blocks_from_indent_line"
+require_relative "visitor"
+require_relative "token"

lib/syntax_suggest/clean_document.rb

@@ -67,26 +67,9 @@ module SyntaxSuggest
   # All of these problems are fixed by joining the whole heredoc into a single
   # line.
   #
-  # ## Comments and whitespace
-  #
-  # Comments can throw off the way the lexer tells us that the line
-  # logically belongs with the next line. This is valid ruby but
-  # results in a different lex output than before:
-  #
-  #     1 User.
-  #     2   where(name: "schneems").
-  #     3   # Comment here
-  #     4   first
-  #
-  # To handle this we can replace comment lines with empty lines
-  # and then re-lex the source. This removal and re-lexing preserves
-  # line index and document size, but generates an easier to work with
-  # document.
-  #
   class CleanDocument
     def initialize(source:)
-      lines = clean_sweep(source: source)
-      @document = CodeLine.from_source(lines.join, lines: lines)
+      @document = CodeLine.from_source(source)
     end
 
     # Call all of the document "cleaners"
@@ -110,62 +93,6 @@ def to_s
       @document.join
     end
 
-    # Remove comments
-    #
-    # replace with empty newlines
-    #
-    #     source = <<~'EOM'
-    #       # Comment 1
-    #       puts "hello"
-    #       # Comment 2
-    #       puts "world"
-    #     EOM
-    #
-    #     lines = CleanDocument.new(source: source).lines
-    #     expect(lines[0].to_s).to eq("\n")
-    #     expect(lines[1].to_s).to eq("puts "hello")
-    #     expect(lines[2].to_s).to eq("\n")
-    #     expect(lines[3].to_s).to eq("puts "world")
-    #
-    # Important: This must be done before lexing.
-    #
-    # After this change is made, we lex the document because
-    # removing comments can change how the doc is parsed.
-    #
-    # For example:
-    #
-    #     values = LexAll.new(source: <<~EOM))
-    #       User.
-    #         # comment
-    #         where(name: 'schneems')
-    #     EOM
-    #     expect(
-    #       values.count {|v| v.type == :on_ignored_nl}
-    #     ).to eq(1)
-    #
-    # After the comment is removed:
-    #
-    #     values = LexAll.new(source: <<~EOM))
-    #       User.
-    #
-    #         where(name: 'schneems')
-    #     EOM
-    #     expect(
-    #      values.count {|v| v.type == :on_ignored_nl}
-    #    ).to eq(2)
-    #
-    def clean_sweep(source:)
-      # Match comments, but not HEREDOC strings with #{variable} interpolation
-      # https://rubular.com/r/HPwtW9OYxKUHXQ
-      source.lines.map do |line|
-        if line.match?(/^\s*#([^{].*|)$/)
-          $/
-        else
-          line
-        end
-      end
-    end
-
     # Smushes all heredoc lines into one line
     #
     #     source = <<~'EOM'
@@ -182,11 +109,11 @@ def join_heredoc!
       start_index_stack = []
       heredoc_beg_end_index = []
       lines.each do |line|
-        line.lex.each do |lex_value|
-          case lex_value.type
-          when :on_heredoc_beg
+        line.tokens.each do |token|
+          case token.type
+          when :HEREDOC_START
             start_index_stack << line.index
-          when :on_heredoc_end
+          when :HEREDOC_END
             start_index = start_index_stack.pop
             end_index = line.index
             heredoc_beg_end_index << [start_index, end_index]
@@ -212,20 +139,10 @@ def join_heredoc!
     #     expect(lines[0].to_s).to eq(source)
     #     expect(lines[1].to_s).to eq("")
     #
-    # The one known case this doesn't handle is:
-    #
-    #     Ripper.lex <<~EOM
-    #       a &&
-    #        b ||
-    #        c
-    #     EOM
-    #
-    # For some reason this introduces `on_ignore_newline` but with BEG type
-    #
     def join_consecutive!
-      consecutive_groups = @document.select(&:ignore_newline_not_beg?).map do |code_line|
+      consecutive_groups = @document.select(&:consecutive?).map do |code_line|
         take_while_including(code_line.index..) do |line|
-          line.ignore_newline_not_beg?
+          line.consecutive?
         end
       end
 
@@ -273,16 +190,17 @@ def join_groups(groups)
 
         # Join group into the first line
         @document[line.index] = CodeLine.new(
-          lex: lines.map(&:lex).flatten,
+          tokens: lines.map(&:tokens).flatten,
           line: lines.join,
-          index: line.index
+          index: line.index,
+          consecutive: false
         )
 
         # Hide the rest of the lines
         lines[1..].each do |line|
           # The above lines already have newlines in them, if add more
           # then there will be double newline, use an empty line instead
-          @document[line.index] = CodeLine.new(line: "", index: line.index, lex: [])
+          @document[line.index] = CodeLine.new(line: "", index: line.index, tokens: [], consecutive: false)
         end
       end
       self