From bac12e81436ca83094d783e2709c7d7f5fcdc19d Mon Sep 17 00:00:00 2001 From: Benoit Daloze Date: Tue, 8 Sep 2026 00:07:24 +0200 Subject: [PATCH 01/10] Run newline_test.rb only on Ruby 3.4+ * Older versions have known bugs in this area, like not emitting a :line event for `nil`, and there is no value to replicate them. --- test/prism/newline_test.rb | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/test/prism/newline_test.rb b/test/prism/newline_test.rb index ed797db965..69b68c06f3 100644 --- a/test/prism/newline_test.rb +++ b/test/prism/newline_test.rb @@ -2,7 +2,9 @@ require_relative "test_helper" -return unless defined?(RubyVM::InstructionSequence) +# There have also been changes made in other versions of Ruby, so we only want +# to test on the most recent versions. +return if !defined?(RubyVM::InstructionSequence) || RUBY_VERSION < "3.4.0" module Prism class NewlineTest < TestCase From 0c34d802aa75d2b387d61a731e1d5a58c032c3bf Mon Sep 17 00:00:00 2001 From: Benoit Daloze Date: Mon, 7 Sep 2026 23:15:52 +0200 Subject: [PATCH 02/10] Remove unnecessary skips in newline_test.rb --- test/prism/newline_test.rb | 4 ---- 1 file changed, 4 deletions(-) diff --git a/test/prism/newline_test.rb b/test/prism/newline_test.rb index 69b68c06f3..74eb9981c0 100644 --- a/test/prism/newline_test.rb +++ b/test/prism/newline_test.rb @@ -11,7 +11,6 @@ class NewlineTest < TestCase skips = %w[ errors_test.rb locals_test.rb - regexp_test.rb test_helper.rb unescape_test.rb api/parse_stream_test.rb @@ -24,9 +23,6 @@ class NewlineTest < TestCase ruby/find_fixtures.rb ruby/find_test.rb ruby/parser_test.rb - ruby/ripper_test.rb - ruby/ruby_parser_test.rb - ruby/parameters_signature_test.rb ] base = __dir__ From 6134363bc191e6f9fcac2c45c27f308642393570 Mon Sep 17 00:00:00 2001 From: Benoit Daloze Date: Mon, 7 Sep 2026 23:38:33 +0200 Subject: [PATCH 03/10] Remove all remaining skips in newline_test.rb Fix two mismatches between prism's newline flags and RubyVM's line events in the Newlines visitor: * def, class, module, and singleton class nodes compile to their own ISeqs with independent line-event tracking, so reset the line table for them like blocks and lambdas already do. This matches one-line definitions like `def foo; bar; end`, where the bytecode emits two line events on the same line. The body of an endless method definition never emits newline events, so in that case mark every line as already seen instead. * Statements inside string interpolation do not emit line events, so mark every line as already seen while visiting embedded statements. Nested scopes (blocks, lambdas, defs, etc.) still reset the lines and emit events again. The remaining divergences are bytecode artifacts: for statements like `foo = [` or `foo =` where the value continues on the following lines, the line event is emitted on the line of the first sub-expression of the value instead of on the first line of the statement. Replace the two ad-hoc compensations in the test with a single count-based rule that moves or drops the newline flag accordingly. Co-Authored-By: Claude Fable 5 --- lib/prism/parse_result/newlines.rb | 75 +++++++++++++++++++ rbi/generated/prism/parse_result/newlines.rbi | 25 +++++++ sig/generated/prism/parse_result/newlines.rbs | 30 ++++++++ test/prism/newline_test.rb | 59 +++++++-------- 4 files changed, 155 insertions(+), 34 deletions(-) diff --git a/lib/prism/parse_result/newlines.rb b/lib/prism/parse_result/newlines.rb index ad8d8b6f55..8361c11864 100644 --- a/lib/prism/parse_result/newlines.rb +++ b/lib/prism/parse_result/newlines.rb @@ -65,6 +65,81 @@ def visit_lambda_node(node) end end + # Permit def nodes to mark newlines within themselves. The body of an + # endless method definition never emits newline events, so in that case + # mark every line as already seen while visiting it instead. Nested + # scopes (blocks, lambdas, etc.) reset the lines and emit events again. + # + #: (DefNode node) -> void + def visit_def_node(node) + old_lines = @lines + @lines = Array.new(old_lines.size, !node.equal_loc.nil?) + + begin + super(node) + ensure + @lines = old_lines + end + end + + # Permit class nodes to mark newlines within themselves. + # + #: (ClassNode node) -> void + def visit_class_node(node) + old_lines = @lines + @lines = Array.new(old_lines.size, false) + + begin + super(node) + ensure + @lines = old_lines + end + end + + # Permit module nodes to mark newlines within themselves. + # + #: (ModuleNode node) -> void + def visit_module_node(node) + old_lines = @lines + @lines = Array.new(old_lines.size, false) + + begin + super(node) + ensure + @lines = old_lines + end + end + + # Permit singleton class nodes to mark newlines within themselves. + # + #: (SingletonClassNode node) -> void + def visit_singleton_class_node(node) + old_lines = @lines + @lines = Array.new(old_lines.size, false) + + begin + super(node) + ensure + @lines = old_lines + end + end + + # Statements inside string interpolation do not emit newline events, so + # mark every line as already seen while visiting them. Nested scopes + # (blocks, lambdas, defs, etc.) reset the lines and emit events again. + # + #: (EmbeddedStatementsNode node) -> void + def visit_embedded_statements_node(node) + old_lines = @lines + @lines = Array.new(old_lines.size, true) + + begin + super(node) + ensure + @lines = old_lines + end + end + # Mark if nodes as newlines. # #: (IfNode node) -> void diff --git a/rbi/generated/prism/parse_result/newlines.rbi b/rbi/generated/prism/parse_result/newlines.rbi index f77dee3bbf..85e37cd832 100644 --- a/rbi/generated/prism/parse_result/newlines.rbi +++ b/rbi/generated/prism/parse_result/newlines.rbi @@ -35,6 +35,31 @@ module Prism sig { params(node: LambdaNode).void } def visit_lambda_node(node); end + # Permit def nodes to mark newlines within themselves. The body of an + # endless method definition never emits newline events, so in that case + # mark every line as already seen while visiting it instead. Nested + # scopes (blocks, lambdas, etc.) reset the lines and emit events again. + sig { params(node: DefNode).void } + def visit_def_node(node); end + + # Permit class nodes to mark newlines within themselves. + sig { params(node: ClassNode).void } + def visit_class_node(node); end + + # Permit module nodes to mark newlines within themselves. + sig { params(node: ModuleNode).void } + def visit_module_node(node); end + + # Permit singleton class nodes to mark newlines within themselves. + sig { params(node: SingletonClassNode).void } + def visit_singleton_class_node(node); end + + # Statements inside string interpolation do not emit newline events, so + # mark every line as already seen while visiting them. Nested scopes + # (blocks, lambdas, defs, etc.) reset the lines and emit events again. + sig { params(node: EmbeddedStatementsNode).void } + def visit_embedded_statements_node(node); end + # Mark if nodes as newlines. sig { params(node: IfNode).void } def visit_if_node(node); end diff --git a/sig/generated/prism/parse_result/newlines.rbs b/sig/generated/prism/parse_result/newlines.rbs index 9a505fa961..8619e7871c 100644 --- a/sig/generated/prism/parse_result/newlines.rbs +++ b/sig/generated/prism/parse_result/newlines.rbs @@ -40,6 +40,36 @@ module Prism # : (LambdaNode node) -> void def visit_lambda_node: (LambdaNode node) -> void + # Permit def nodes to mark newlines within themselves. The body of an + # endless method definition never emits newline events, so in that case + # mark every line as already seen while visiting it instead. Nested + # scopes (blocks, lambdas, etc.) reset the lines and emit events again. + # + # : (DefNode node) -> void + def visit_def_node: (DefNode node) -> void + + # Permit class nodes to mark newlines within themselves. + # + # : (ClassNode node) -> void + def visit_class_node: (ClassNode node) -> void + + # Permit module nodes to mark newlines within themselves. + # + # : (ModuleNode node) -> void + def visit_module_node: (ModuleNode node) -> void + + # Permit singleton class nodes to mark newlines within themselves. + # + # : (SingletonClassNode node) -> void + def visit_singleton_class_node: (SingletonClassNode node) -> void + + # Statements inside string interpolation do not emit newline events, so + # mark every line as already seen while visiting them. Nested scopes + # (blocks, lambdas, defs, etc.) reset the lines and emit events again. + # + # : (EmbeddedStatementsNode node) -> void + def visit_embedded_statements_node: (EmbeddedStatementsNode node) -> void + # Mark if nodes as newlines. # # : (IfNode node) -> void diff --git a/test/prism/newline_test.rb b/test/prism/newline_test.rb index 74eb9981c0..702a28b898 100644 --- a/test/prism/newline_test.rb +++ b/test/prism/newline_test.rb @@ -8,25 +8,8 @@ module Prism class NewlineTest < TestCase - skips = %w[ - errors_test.rb - locals_test.rb - test_helper.rb - unescape_test.rb - api/parse_stream_test.rb - api/raise_error_test.rb - encoding/regular_expression_encoding_test.rb - encoding/string_encoding_test.rb - result/breadth_first_search_test.rb - result/static_literals_test.rb - result/warnings_test.rb - ruby/find_fixtures.rb - ruby/find_test.rb - ruby/parser_test.rb - ] - base = __dir__ - (Dir["{,api/,encoding/,result/,ruby/}*.rb", base: base] - skips).each do |relative| + Dir["{,api/,encoding/,result/,ruby/}*.rb", base: base].each do |relative| define_method(:"test_#{relative}") do assert_newlines(base, relative) end @@ -43,7 +26,8 @@ def assert_newlines(base, relative) assert_empty result.errors actual = prism_lines(result) - source.each_line.with_index(1) do |line, line_number| + lines = source.lines + lines.each.with_index(1) do |line, line_number| # Lines like `while (foo = bar)` result in two line flags in the # bytecode but only one newline flag in the AST. We need to remove the # extra line flag from the bytecode to make the test pass. @@ -52,25 +36,32 @@ def assert_newlines(base, relative) expected.delete_at(index) if index end - # Lines like `foo =` where the value is on the next line result in - # another line flag in the bytecode but only one newline flag in the - # AST. - if line.match?(/^\s+\w+ =$/) - if source.lines[line_number].match?(/^\s+case/) - actual[actual.index(line_number)] += 1 - else - actual.delete_at(actual.index(line_number)) - end - end - - if line.match?(/^\s+\w+ = \[$/) - if !expected.include?(line_number) && !expected.include?(line_number + 2) - actual[actual.index(line_number)] += 1 + # For statements like `foo = [` or `foo =` where the value continues + # on the following lines, the line event in the bytecode is emitted on + # the line of the first sub-expression of the value (e.g., the first + # array element) instead of on the first line of the statement, while + # prism marks the newline flag on the node that starts the statement. + # The same is true for statements that begin with a multi-line array + # or hash literal, like `[` alone on a line. To compensate, move the + # newline flag to the line the bytecode uses, or drop it if another + # node already has a newline flag on that line. + if line.match?(/[\w\])"'] =( \[| \{| begin)?$/) || line.match?(/\A\s*[\[{]$/) + if actual.count(line_number) > expected.count(line_number) + target = ((line_number + 1)..lines.length).find do |candidate| + !lines[candidate - 1].match?(/\A\s*(#|\z)/) + end + + index = actual.index(line_number) #: Integer + if target && expected.count(target) > actual.count(target) + actual[index] = target + else + actual.delete_at(index) + end end end end - assert_equal expected, actual + assert_equal expected, actual.sort end def rubyvm_lines(source) From 3761a6ee99674d759540b1b3a965ef23da5c0bdc Mon Sep 17 00:00:00 2001 From: Benoit Daloze Date: Mon, 7 Sep 2026 23:58:16 +0200 Subject: [PATCH 04/10] Match RubyVM newlines for parenthesized while/until predicates Lines like `while (foo = bar)` result in two line events in the bytecode: parentheses make the inner expression a statement with its own line event, and the predicate of a while or until loop is compiled at the end of the loop, after the body, so that event is emitted again in addition to the one for the loop statement itself. This also mirrors runtime behavior, since the predicate line fires on each iteration. Match this in the Newlines visitor by marking the loop node itself when a prefix loop has a parenthesized predicate, and by visiting the predicate with a fresh set of lines so that its statements can mark lines that were already seen. This removes the corresponding compensation in newline_test.rb. The remaining compensation for assignments whose value continues on the following lines is kept: the line event is emitted on the line of the statement's first compiled instruction, which depends on constant folding (for example, an array of static literals compiles to a single instruction on the line of the literal, and string literals are only static under `# frozen_string_literal: true`). That is a property of the compiler rather than of the AST, so it does not belong in the newline flags. Co-Authored-By: Claude Fable 5 --- lib/prism/parse_result/newlines.rb | 53 ++++++++++++++++++- rbi/generated/prism/parse_result/newlines.rbi | 12 +++++ sig/generated/prism/parse_result/newlines.rbs | 14 +++++ test/prism/newline_test.rb | 24 ++++----- 4 files changed, 86 insertions(+), 17 deletions(-) diff --git a/lib/prism/parse_result/newlines.rb b/lib/prism/parse_result/newlines.rb index 8361c11864..874258b1b0 100644 --- a/lib/prism/parse_result/newlines.rb +++ b/lib/prism/parse_result/newlines.rb @@ -140,6 +140,42 @@ def visit_embedded_statements_node(node) end end + # The predicate of a while loop is compiled at the end of the loop, + # after the body, so any statements it contains (from parentheses) + # emit their line events again even if the lines were already seen. + # + #: (WhileNode node) -> void + def visit_while_node(node) + old_lines = @lines + @lines = Array.new(old_lines.size, false) + + begin + visit(node.predicate) + ensure + @lines = old_lines + end + + visit(node.statements) + end + + # The predicate of an until loop is compiled at the end of the loop, + # after the body, so any statements it contains (from parentheses) + # emit their line events again even if the lines were already seen. + # + #: (UntilNode node) -> void + def visit_until_node(node) + old_lines = @lines + @lines = Array.new(old_lines.size, false) + + begin + visit(node.predicate) + ensure + @lines = old_lines + end + + visit(node.statements) + end + # Mark if nodes as newlines. # #: (IfNode node) -> void @@ -219,14 +255,26 @@ def newline_flag!(lines) # :nodoc: class UntilNode < Node #: (Array[bool] lines) -> void def newline_flag!(lines) # :nodoc: - predicate.newline_flag!(lines) + if location.start_offset == keyword_loc.start_offset && predicate.is_a?(ParenthesesNode) + # A parenthesized predicate emits its own line event when it is + # compiled at the end of the loop, in addition to this one. + super + else + predicate.newline_flag!(lines) + end end end class WhileNode < Node #: (Array[bool] lines) -> void def newline_flag!(lines) # :nodoc: - predicate.newline_flag!(lines) + if location.start_offset == keyword_loc.start_offset && predicate.is_a?(ParenthesesNode) + # A parenthesized predicate emits its own line event when it is + # compiled at the end of the loop, in addition to this one. + super + else + predicate.newline_flag!(lines) + end end end @@ -237,6 +285,7 @@ def newline_flag!(lines) # :nodoc: end end + class InterpolatedMatchLastLineNode < Node #: (Array[bool] lines) -> void def newline_flag!(lines) # :nodoc: diff --git a/rbi/generated/prism/parse_result/newlines.rbi b/rbi/generated/prism/parse_result/newlines.rbi index 85e37cd832..427d0cb2b7 100644 --- a/rbi/generated/prism/parse_result/newlines.rbi +++ b/rbi/generated/prism/parse_result/newlines.rbi @@ -60,6 +60,18 @@ module Prism sig { params(node: EmbeddedStatementsNode).void } def visit_embedded_statements_node(node); end + # The predicate of a while loop is compiled at the end of the loop, + # after the body, so any statements it contains (from parentheses) + # emit their line events again even if the lines were already seen. + sig { params(node: WhileNode).void } + def visit_while_node(node); end + + # The predicate of an until loop is compiled at the end of the loop, + # after the body, so any statements it contains (from parentheses) + # emit their line events again even if the lines were already seen. + sig { params(node: UntilNode).void } + def visit_until_node(node); end + # Mark if nodes as newlines. sig { params(node: IfNode).void } def visit_if_node(node); end diff --git a/sig/generated/prism/parse_result/newlines.rbs b/sig/generated/prism/parse_result/newlines.rbs index 8619e7871c..6f3c0ee3b8 100644 --- a/sig/generated/prism/parse_result/newlines.rbs +++ b/sig/generated/prism/parse_result/newlines.rbs @@ -70,6 +70,20 @@ module Prism # : (EmbeddedStatementsNode node) -> void def visit_embedded_statements_node: (EmbeddedStatementsNode node) -> void + # The predicate of a while loop is compiled at the end of the loop, + # after the body, so any statements it contains (from parentheses) + # emit their line events again even if the lines were already seen. + # + # : (WhileNode node) -> void + def visit_while_node: (WhileNode node) -> void + + # The predicate of an until loop is compiled at the end of the loop, + # after the body, so any statements it contains (from parentheses) + # emit their line events again even if the lines were already seen. + # + # : (UntilNode node) -> void + def visit_until_node: (UntilNode node) -> void + # Mark if nodes as newlines. # # : (IfNode node) -> void diff --git a/test/prism/newline_test.rb b/test/prism/newline_test.rb index 702a28b898..b92381e991 100644 --- a/test/prism/newline_test.rb +++ b/test/prism/newline_test.rb @@ -28,23 +28,17 @@ def assert_newlines(base, relative) lines = source.lines lines.each.with_index(1) do |line, line_number| - # Lines like `while (foo = bar)` result in two line flags in the - # bytecode but only one newline flag in the AST. We need to remove the - # extra line flag from the bytecode to make the test pass. - if line.match?(/while \(/) - index = expected.index(line_number) - expected.delete_at(index) if index - end - # For statements like `foo = [` or `foo =` where the value continues # on the following lines, the line event in the bytecode is emitted on - # the line of the first sub-expression of the value (e.g., the first - # array element) instead of on the first line of the statement, while - # prism marks the newline flag on the node that starts the statement. - # The same is true for statements that begin with a multi-line array - # or hash literal, like `[` alone on a line. To compensate, move the - # newline flag to the line the bytecode uses, or drop it if another - # node already has a newline flag on that line. + # the line of its first instruction (e.g., the first array element) + # instead of on the first line of the statement, while prism marks the + # newline flag on the node that starts the statement. The same is true + # for statements that begin with a multi-line array or hash literal, + # like `[` alone on a line. The exact line depends on constant folding + # (e.g., an array of literals compiles to a single instruction on the + # first line), so to compensate, move the newline flag to the line the + # bytecode uses, or drop it if another node already has a newline flag + # on that line. if line.match?(/[\w\])"'] =( \[| \{| begin)?$/) || line.match?(/\A\s*[\[{]$/) if actual.count(line_number) > expected.count(line_number) target = ((line_number + 1)..lines.length).find do |candidate| From b8e73b12536ab342bd749fce1bd490a6b22e84d0 Mon Sep 17 00:00:00 2001 From: Benoit Daloze Date: Tue, 8 Sep 2026 00:03:07 +0200 Subject: [PATCH 05/10] Match RubyVM newlines exactly, removing all compensations in newline_test.rb The line event for a statement is emitted where its first instruction is compiled, so make nodes whose first instruction comes from a sub-expression delegate their newline flag to that sub-expression: assignments to their value, calls to their receiver, and array, hash, and interpolated string literals to their first element. Static literals are the exception: they are compiled to a single instruction on the first line of the literal, so they do not delegate. The static literal flag captures the folding boundary exactly, including that string literals are only static under `# frozen_string_literal: true`, both in arrays and in the parts of heredocs. With this, prism's newline flags match RubyVM's line events exactly on every file in the test suite and newline_test.rb needs no compensation logic at all. Co-Authored-By: Claude Fable 5 --- lib/prism/parse_result/newlines.rb | 97 ++++++++++++++++++- rbi/generated/prism/parse_result/newlines.rbi | 50 ++++++++++ sig/generated/prism/parse_result/newlines.rbs | 50 ++++++++++ test/prism/newline_test.rb | 31 +----- 4 files changed, 197 insertions(+), 31 deletions(-) diff --git a/lib/prism/parse_result/newlines.rb b/lib/prism/parse_result/newlines.rb index 874258b1b0..41729bf813 100644 --- a/lib/prism/parse_result/newlines.rb +++ b/lib/prism/parse_result/newlines.rb @@ -285,6 +285,97 @@ def newline_flag!(lines) # :nodoc: end end + # The line event for a statement is emitted where its first instruction is + # compiled, so nodes whose first instruction comes from a sub-expression + # delegate their newline flag to that sub-expression: assignments to their + # value, calls to their receiver, and array, hash, and interpolated string + # literals to their first element. Static literals are the exception: they + # are compiled to a single instruction on the first line of the literal, so + # they do not delegate. + + class LocalVariableWriteNode < Node + #: (Array[bool] lines) -> void + def newline_flag!(lines) # :nodoc: + value.newline_flag!(lines) + end + end + + class InstanceVariableWriteNode < Node + #: (Array[bool] lines) -> void + def newline_flag!(lines) # :nodoc: + value.newline_flag!(lines) + end + end + + class ClassVariableWriteNode < Node + #: (Array[bool] lines) -> void + def newline_flag!(lines) # :nodoc: + value.newline_flag!(lines) + end + end + + class GlobalVariableWriteNode < Node + #: (Array[bool] lines) -> void + def newline_flag!(lines) # :nodoc: + value.newline_flag!(lines) + end + end + + class ConstantWriteNode < Node + #: (Array[bool] lines) -> void + def newline_flag!(lines) # :nodoc: + value.newline_flag!(lines) + end + end + + class ConstantPathWriteNode < Node + #: (Array[bool] lines) -> void + def newline_flag!(lines) # :nodoc: + value.newline_flag!(lines) + end + end + + class MultiWriteNode < Node + #: (Array[bool] lines) -> void + def newline_flag!(lines) # :nodoc: + value.newline_flag!(lines) + end + end + + class CallNode < Node + #: (Array[bool] lines) -> void + def newline_flag!(lines) # :nodoc: + if (receiver = self.receiver) + receiver.newline_flag!(lines) + else + super + end + end + end + + class ArrayNode < Node + #: (Array[bool] lines) -> void + def newline_flag!(lines) # :nodoc: + first = elements.first + if first && !static_literal? + first.newline_flag!(lines) + else + super + end + end + end + + class HashNode < Node + #: (Array[bool] lines) -> void + def newline_flag!(lines) # :nodoc: + first = elements.first + if first && !static_literal? + first.newline_flag!(lines) + else + super + end + end + end class InterpolatedMatchLastLineNode < Node #: (Array[bool] lines) -> void @@ -306,7 +397,11 @@ class InterpolatedStringNode < Node #: (Array[bool] lines) -> void def newline_flag!(lines) # :nodoc: first = parts.first - first.newline_flag!(lines) if first + if first && !static_literal? + first.newline_flag!(lines) + else + super + end end end diff --git a/rbi/generated/prism/parse_result/newlines.rbi b/rbi/generated/prism/parse_result/newlines.rbi index 427d0cb2b7..3e83f4757a 100644 --- a/rbi/generated/prism/parse_result/newlines.rbi +++ b/rbi/generated/prism/parse_result/newlines.rbi @@ -131,6 +131,56 @@ module Prism def newline_flag!(lines); end end + class LocalVariableWriteNode < Node + sig { params(lines: T::Array[T::Boolean]).void } + def newline_flag!(lines); end + end + + class InstanceVariableWriteNode < Node + sig { params(lines: T::Array[T::Boolean]).void } + def newline_flag!(lines); end + end + + class ClassVariableWriteNode < Node + sig { params(lines: T::Array[T::Boolean]).void } + def newline_flag!(lines); end + end + + class GlobalVariableWriteNode < Node + sig { params(lines: T::Array[T::Boolean]).void } + def newline_flag!(lines); end + end + + class ConstantWriteNode < Node + sig { params(lines: T::Array[T::Boolean]).void } + def newline_flag!(lines); end + end + + class ConstantPathWriteNode < Node + sig { params(lines: T::Array[T::Boolean]).void } + def newline_flag!(lines); end + end + + class MultiWriteNode < Node + sig { params(lines: T::Array[T::Boolean]).void } + def newline_flag!(lines); end + end + + class CallNode < Node + sig { params(lines: T::Array[T::Boolean]).void } + def newline_flag!(lines); end + end + + class ArrayNode < Node + sig { params(lines: T::Array[T::Boolean]).void } + def newline_flag!(lines); end + end + + class HashNode < Node + sig { params(lines: T::Array[T::Boolean]).void } + def newline_flag!(lines); end + end + class InterpolatedMatchLastLineNode < Node sig { params(lines: T::Array[T::Boolean]).void } def newline_flag!(lines); end diff --git a/sig/generated/prism/parse_result/newlines.rbs b/sig/generated/prism/parse_result/newlines.rbs index 6f3c0ee3b8..2a9cf13c54 100644 --- a/sig/generated/prism/parse_result/newlines.rbs +++ b/sig/generated/prism/parse_result/newlines.rbs @@ -146,6 +146,56 @@ module Prism def newline_flag!: (Array[bool] lines) -> void end + class LocalVariableWriteNode < Node + # : (Array[bool] lines) -> void + def newline_flag!: (Array[bool] lines) -> void + end + + class InstanceVariableWriteNode < Node + # : (Array[bool] lines) -> void + def newline_flag!: (Array[bool] lines) -> void + end + + class ClassVariableWriteNode < Node + # : (Array[bool] lines) -> void + def newline_flag!: (Array[bool] lines) -> void + end + + class GlobalVariableWriteNode < Node + # : (Array[bool] lines) -> void + def newline_flag!: (Array[bool] lines) -> void + end + + class ConstantWriteNode < Node + # : (Array[bool] lines) -> void + def newline_flag!: (Array[bool] lines) -> void + end + + class ConstantPathWriteNode < Node + # : (Array[bool] lines) -> void + def newline_flag!: (Array[bool] lines) -> void + end + + class MultiWriteNode < Node + # : (Array[bool] lines) -> void + def newline_flag!: (Array[bool] lines) -> void + end + + class CallNode < Node + # : (Array[bool] lines) -> void + def newline_flag!: (Array[bool] lines) -> void + end + + class ArrayNode < Node + # : (Array[bool] lines) -> void + def newline_flag!: (Array[bool] lines) -> void + end + + class HashNode < Node + # : (Array[bool] lines) -> void + def newline_flag!: (Array[bool] lines) -> void + end + class InterpolatedMatchLastLineNode < Node # : (Array[bool] lines) -> void def newline_flag!: (Array[bool] lines) -> void diff --git a/test/prism/newline_test.rb b/test/prism/newline_test.rb index b92381e991..7d48d5297e 100644 --- a/test/prism/newline_test.rb +++ b/test/prism/newline_test.rb @@ -26,36 +26,7 @@ def assert_newlines(base, relative) assert_empty result.errors actual = prism_lines(result) - lines = source.lines - lines.each.with_index(1) do |line, line_number| - # For statements like `foo = [` or `foo =` where the value continues - # on the following lines, the line event in the bytecode is emitted on - # the line of its first instruction (e.g., the first array element) - # instead of on the first line of the statement, while prism marks the - # newline flag on the node that starts the statement. The same is true - # for statements that begin with a multi-line array or hash literal, - # like `[` alone on a line. The exact line depends on constant folding - # (e.g., an array of literals compiles to a single instruction on the - # first line), so to compensate, move the newline flag to the line the - # bytecode uses, or drop it if another node already has a newline flag - # on that line. - if line.match?(/[\w\])"'] =( \[| \{| begin)?$/) || line.match?(/\A\s*[\[{]$/) - if actual.count(line_number) > expected.count(line_number) - target = ((line_number + 1)..lines.length).find do |candidate| - !lines[candidate - 1].match?(/\A\s*(#|\z)/) - end - - index = actual.index(line_number) #: Integer - if target && expected.count(target) > actual.count(target) - actual[index] = target - else - actual.delete_at(index) - end - end - end - end - - assert_equal expected, actual.sort + assert_equal expected, actual end def rubyvm_lines(source) From 9bf6cbce7247aa9c5bfe50ba36378933c3c80d27 Mon Sep 17 00:00:00 2001 From: Benoit Daloze Date: Tue, 8 Sep 2026 23:42:34 +0200 Subject: [PATCH 06/10] Add a note about what a skip means in newline_test.rb --- test/prism/newline_test.rb | 2 ++ 1 file changed, 2 insertions(+) diff --git a/test/prism/newline_test.rb b/test/prism/newline_test.rb index 7d48d5297e..12504e912d 100644 --- a/test/prism/newline_test.rb +++ b/test/prism/newline_test.rb @@ -8,6 +8,8 @@ module Prism class NewlineTest < TestCase + # If you are coming from ruby/ruby, a test failure here means that TracePoint `:line` events changed. + # Before adding a skip, make sure that you actually intended for such a difference to happen. base = __dir__ Dir["{,api/,encoding/,result/,ruby/}*.rb", base: base].each do |relative| define_method(:"test_#{relative}") do From dc58d826e0ed6ea445a889a5a9e180fce15d3337 Mon Sep 17 00:00:00 2001 From: Benoit Daloze Date: Fri, 11 Sep 2026 21:35:52 +0200 Subject: [PATCH 07/10] Mark newlines in Java like "Remove all remaining skips in newline_test.rb" Mirror the newlines.rb changes of that commit in MarkNewlinesVisitor: def, class, module, and singleton class nodes compile to their own ISeqs with independent line-event tracking, so reset the marked lines for them like blocks and lambdas already do. The body of an endless method definition and statements inside string interpolation never emit newline events, so mark every line as already seen while visiting them. Since location fields are not available in the Java nodes, an endless method definition is detected as a statements body which ends with the def node itself, which was verified to be equivalent to checking the location of the `=` operator on every def node in this repository. Co-Authored-By: Claude Opus 4.8 --- .../ruby_lang/prism/MarkNewlinesVisitor.java | 68 +++++++++++++++++++ 1 file changed, 68 insertions(+) diff --git a/java/api/src/main/java/org/ruby_lang/prism/MarkNewlinesVisitor.java b/java/api/src/main/java/org/ruby_lang/prism/MarkNewlinesVisitor.java index 148b0d1a2e..a5f5c5c65e 100644 --- a/java/api/src/main/java/org/ruby_lang/prism/MarkNewlinesVisitor.java +++ b/java/api/src/main/java/org/ruby_lang/prism/MarkNewlinesVisitor.java @@ -1,5 +1,7 @@ package org.ruby_lang.prism; +import java.util.Arrays; + // Keep in sync with Ruby MarkNewlinesVisitor final class MarkNewlinesVisitor extends AbstractNodeVisitor { @@ -33,6 +35,72 @@ public Void visitLambdaNode(Nodes.LambdaNode node) { } } + @Override + public Void visitDefNode(Nodes.DefNode node) { + boolean[] oldNewlineMarked = this.newlineMarked; + this.newlineMarked = new boolean[oldNewlineMarked.length]; + // The body of an endless method definition never emits newline events, + // so in that case mark every line as already seen instead. There is no + // location for the `=` operator here, but only an endless method + // definition has a statements body which ends with the def node itself. + if (node.body instanceof Nodes.StatementsNode && node.endOffset() == node.body.endOffset()) { + Arrays.fill(this.newlineMarked, true); + } + try { + return super.visitDefNode(node); + } finally { + this.newlineMarked = oldNewlineMarked; + } + } + + @Override + public Void visitClassNode(Nodes.ClassNode node) { + boolean[] oldNewlineMarked = this.newlineMarked; + this.newlineMarked = new boolean[oldNewlineMarked.length]; + try { + return super.visitClassNode(node); + } finally { + this.newlineMarked = oldNewlineMarked; + } + } + + @Override + public Void visitModuleNode(Nodes.ModuleNode node) { + boolean[] oldNewlineMarked = this.newlineMarked; + this.newlineMarked = new boolean[oldNewlineMarked.length]; + try { + return super.visitModuleNode(node); + } finally { + this.newlineMarked = oldNewlineMarked; + } + } + + @Override + public Void visitSingletonClassNode(Nodes.SingletonClassNode node) { + boolean[] oldNewlineMarked = this.newlineMarked; + this.newlineMarked = new boolean[oldNewlineMarked.length]; + try { + return super.visitSingletonClassNode(node); + } finally { + this.newlineMarked = oldNewlineMarked; + } + } + + // Statements inside string interpolation do not emit newline events, so + // mark every line as already seen while visiting them. Nested scopes + // (blocks, lambdas, defs, etc.) reset the lines and emit events again. + @Override + public Void visitEmbeddedStatementsNode(Nodes.EmbeddedStatementsNode node) { + boolean[] oldNewlineMarked = this.newlineMarked; + this.newlineMarked = new boolean[oldNewlineMarked.length]; + Arrays.fill(this.newlineMarked, true); + try { + return super.visitEmbeddedStatementsNode(node); + } finally { + this.newlineMarked = oldNewlineMarked; + } + } + @Override public Void visitIfNode(Nodes.IfNode node) { node.setNewLineFlag(this.source, this.newlineMarked); From c143ef3f94787bd1103329aad181786e94f05f28 Mon Sep 17 00:00:00 2001 From: Benoit Daloze Date: Fri, 11 Sep 2026 21:37:58 +0200 Subject: [PATCH 08/10] Mark newlines in Java like "Match RubyVM newlines for parenthesized while/until predicates" Mirror the newlines.rb changes of that commit in MarkNewlinesVisitor: the predicate of a while/until loop is compiled at the end of the loop, after the body, so any statements it contains (from parentheses) emit their line events again even if the lines were already seen, in addition to the line event of the loop itself. Since the newline_flag! overrides in newlines.rb are no longer a mirror of the generated setNewLineFlag() methods, introduce a setNewLineFlag() helper in the visitor for the extra logic. There is no keyword location in the Java nodes, but a prefix while/until loop can be detected as one which is not a begin modifier loop and does not start with its statements, which was verified to be equivalent to checking the keyword location on every while/until node in this repository. Co-Authored-By: Claude Opus 4.8 --- .../ruby_lang/prism/MarkNewlinesVisitor.java | 83 ++++++++++++++++++- 1 file changed, 82 insertions(+), 1 deletion(-) diff --git a/java/api/src/main/java/org/ruby_lang/prism/MarkNewlinesVisitor.java b/java/api/src/main/java/org/ruby_lang/prism/MarkNewlinesVisitor.java index a5f5c5c65e..e7edd407ab 100644 --- a/java/api/src/main/java/org/ruby_lang/prism/MarkNewlinesVisitor.java +++ b/java/api/src/main/java/org/ruby_lang/prism/MarkNewlinesVisitor.java @@ -101,6 +101,44 @@ public Void visitEmbeddedStatementsNode(Nodes.EmbeddedStatementsNode node) { } } + // The predicate of a while loop is compiled at the end of the loop, + // after the body, so any statements it contains (from parentheses) + // emit their line events again even if the lines were already seen. + @Override + public Void visitWhileNode(Nodes.WhileNode node) { + boolean[] oldNewlineMarked = this.newlineMarked; + this.newlineMarked = new boolean[oldNewlineMarked.length]; + try { + node.predicate.accept(this); + } finally { + this.newlineMarked = oldNewlineMarked; + } + + if (node.statements != null) { + node.statements.accept(this); + } + return null; + } + + // The predicate of an until loop is compiled at the end of the loop, + // after the body, so any statements it contains (from parentheses) + // emit their line events again even if the lines were already seen. + @Override + public Void visitUntilNode(Nodes.UntilNode node) { + boolean[] oldNewlineMarked = this.newlineMarked; + this.newlineMarked = new boolean[oldNewlineMarked.length]; + try { + node.predicate.accept(this); + } finally { + this.newlineMarked = oldNewlineMarked; + } + + if (node.statements != null) { + node.statements.accept(this); + } + return null; + } + @Override public Void visitIfNode(Nodes.IfNode node) { node.setNewLineFlag(this.source, this.newlineMarked); @@ -116,11 +154,54 @@ public Void visitUnlessNode(Nodes.UnlessNode node) { @Override public Void visitStatementsNode(Nodes.StatementsNode node) { for (Nodes.Node child : node.body) { - child.setNewLineFlag(this.source, this.newlineMarked); + setNewLineFlag(child); } return super.visitStatementsNode(node); } + // Keep in sync with the newline_flag! overrides in Ruby's newlines.rb which + // are not part of the generated setNewLineFlag() methods. + private void setNewLineFlag(Nodes.Node node) { + if (node instanceof Nodes.WhileNode whileNode) { + boolean prefix = isPrefixLoop(whileNode, whileNode.isBeginModifier(), whileNode.statements); + if (prefix && whileNode.predicate instanceof Nodes.ParenthesesNode) { + // A parenthesized predicate emits its own line event when it is + // compiled at the end of the loop, in addition to this one. + markNewLineFlag(node); + } else { + whileNode.predicate.setNewLineFlag(this.source, this.newlineMarked); + } + } else if (node instanceof Nodes.UntilNode untilNode) { + boolean prefix = isPrefixLoop(untilNode, untilNode.isBeginModifier(), untilNode.statements); + if (prefix && untilNode.predicate instanceof Nodes.ParenthesesNode) { + // A parenthesized predicate emits its own line event when it is + // compiled at the end of the loop, in addition to this one. + markNewLineFlag(node); + } else { + untilNode.predicate.setNewLineFlag(this.source, this.newlineMarked); + } + } else { + node.setNewLineFlag(this.source, this.newlineMarked); + } + } + + // Mark the node itself, like Nodes.Node#setNewLineFlag(), regardless of any + // setNewLineFlag() override of the node. + private void markNewLineFlag(Nodes.Node node) { + int line = this.source.findLine(node.startOffset); + if (!this.newlineMarked[line]) { + this.newlineMarked[line] = true; + node.setNewLineFlag(true); + } + } + + // Whether a while/until loop starts with its keyword. There is no keyword + // location in the Java nodes, but only a prefix loop starts before its + // statements. + private static boolean isPrefixLoop(Nodes.Node node, boolean beginModifier, Nodes.StatementsNode statements) { + return !beginModifier && (statements == null || node.startOffset != statements.startOffset); + } + @Override protected Void defaultVisit(Nodes.Node node) { node.visitChildNodes(this); From 9fcc12ecc6068724337399783ed34cb04e7641ec Mon Sep 17 00:00:00 2001 From: Benoit Daloze Date: Fri, 11 Sep 2026 21:41:29 +0200 Subject: [PATCH 09/10] Mark newlines in Java like "Match RubyVM newlines exactly, removing all compensations in newline_test.rb" Mirror the newlines.rb changes of that commit in MarkNewlinesVisitor: the line event for a statement is emitted where its first instruction is compiled, so nodes whose first instruction comes from a sub-expression delegate their newline flag to that sub-expression: assignments to their value, calls to their receiver, and array, hash, and interpolated string literals to their first element. Static literals are the exception: they are compiled to a single instruction on the first line of the literal, so they do not delegate. The delegating newline_flag! overrides in newlines.rb dispatch to each other, so the corresponding cases in the setNewLineFlag() helper of the visitor recurse through the helper, including for the predicate of if/unless/while/until and the expression of a rescue modifier. PM_NODE_FLAG_STATIC_LITERAL is serialized together with the node-specific flags, so it is read from the flags for nodes which have them, and computed structurally for hash literals (whose nodes have no flags), following the same rules as the parser: an assoc node is a static literal if neither its key nor its value is an array, hash, or range and both are static literals themselves. Co-Authored-By: Claude Opus 4.8 --- .../ruby_lang/prism/MarkNewlinesVisitor.java | 111 +++++++++++++++++- 1 file changed, 106 insertions(+), 5 deletions(-) diff --git a/java/api/src/main/java/org/ruby_lang/prism/MarkNewlinesVisitor.java b/java/api/src/main/java/org/ruby_lang/prism/MarkNewlinesVisitor.java index e7edd407ab..3074602c89 100644 --- a/java/api/src/main/java/org/ruby_lang/prism/MarkNewlinesVisitor.java +++ b/java/api/src/main/java/org/ruby_lang/prism/MarkNewlinesVisitor.java @@ -141,13 +141,13 @@ public Void visitUntilNode(Nodes.UntilNode node) { @Override public Void visitIfNode(Nodes.IfNode node) { - node.setNewLineFlag(this.source, this.newlineMarked); + setNewLineFlag(node); return super.visitIfNode(node); } @Override public Void visitUnlessNode(Nodes.UnlessNode node) { - node.setNewLineFlag(this.source, this.newlineMarked); + setNewLineFlag(node); return super.visitUnlessNode(node); } @@ -161,15 +161,67 @@ public Void visitStatementsNode(Nodes.StatementsNode node) { // Keep in sync with the newline_flag! overrides in Ruby's newlines.rb which // are not part of the generated setNewLineFlag() methods. + // + // The line event for a statement is emitted where its first instruction is + // compiled, so nodes whose first instruction comes from a sub-expression + // delegate their newline flag to that sub-expression: assignments to their + // value, calls to their receiver, and array, hash, and interpolated string + // literals to their first element. Static literals are the exception: they + // are compiled to a single instruction on the first line of the literal, so + // they do not delegate. private void setNewLineFlag(Nodes.Node node) { - if (node instanceof Nodes.WhileNode whileNode) { + if (node instanceof Nodes.LocalVariableWriteNode write) { + setNewLineFlag(write.value); + } else if (node instanceof Nodes.InstanceVariableWriteNode write) { + setNewLineFlag(write.value); + } else if (node instanceof Nodes.ClassVariableWriteNode write) { + setNewLineFlag(write.value); + } else if (node instanceof Nodes.GlobalVariableWriteNode write) { + setNewLineFlag(write.value); + } else if (node instanceof Nodes.ConstantWriteNode write) { + setNewLineFlag(write.value); + } else if (node instanceof Nodes.ConstantPathWriteNode write) { + setNewLineFlag(write.value); + } else if (node instanceof Nodes.MultiWriteNode write) { + setNewLineFlag(write.value); + } else if (node instanceof Nodes.CallNode call) { + if (call.receiver != null) { + setNewLineFlag(call.receiver); + } else { + markNewLineFlag(node); + } + } else if (node instanceof Nodes.ArrayNode array) { + if (array.elements.length > 0 && !isStaticLiteral(array)) { + setNewLineFlag(array.elements[0]); + } else { + markNewLineFlag(node); + } + } else if (node instanceof Nodes.HashNode hash) { + if (hash.elements.length > 0 && !isStaticLiteral(hash)) { + setNewLineFlag(hash.elements[0]); + } else { + markNewLineFlag(node); + } + } else if (node instanceof Nodes.InterpolatedStringNode string) { + if (string.parts.length > 0 && !isStaticLiteral(string)) { + setNewLineFlag(string.parts[0]); + } else { + markNewLineFlag(node); + } + } else if (node instanceof Nodes.IfNode ifNode) { + setNewLineFlag(ifNode.predicate); + } else if (node instanceof Nodes.UnlessNode unlessNode) { + setNewLineFlag(unlessNode.predicate); + } else if (node instanceof Nodes.RescueModifierNode rescueModifier) { + setNewLineFlag(rescueModifier.expression); + } else if (node instanceof Nodes.WhileNode whileNode) { boolean prefix = isPrefixLoop(whileNode, whileNode.isBeginModifier(), whileNode.statements); if (prefix && whileNode.predicate instanceof Nodes.ParenthesesNode) { // A parenthesized predicate emits its own line event when it is // compiled at the end of the loop, in addition to this one. markNewLineFlag(node); } else { - whileNode.predicate.setNewLineFlag(this.source, this.newlineMarked); + setNewLineFlag(whileNode.predicate); } } else if (node instanceof Nodes.UntilNode untilNode) { boolean prefix = isPrefixLoop(untilNode, untilNode.isBeginModifier(), untilNode.statements); @@ -178,7 +230,7 @@ private void setNewLineFlag(Nodes.Node node) { // compiled at the end of the loop, in addition to this one. markNewLineFlag(node); } else { - untilNode.predicate.setNewLineFlag(this.source, this.newlineMarked); + setNewLineFlag(untilNode.predicate); } } else { node.setNewLineFlag(this.source, this.newlineMarked); @@ -195,6 +247,55 @@ private void markNewLineFlag(Nodes.Node node) { } } + // PM_NODE_FLAG_STATIC_LITERAL, which is serialized together with the + // node-specific flags. + private static final short STATIC_LITERAL_FLAG = 0x2; + + // Whether the node has the PM_NODE_FLAG_STATIC_LITERAL flag. Only nodes + // with node-specific flags store their flags in Java, so it is computed + // structurally for the others, for the nodes which can appear inside an + // array or hash literal. + private static boolean isStaticLiteral(Nodes.Node node) { + if (node instanceof Nodes.ArrayNode array) { + return (array.flags & STATIC_LITERAL_FLAG) != 0; + } else if (node instanceof Nodes.InterpolatedStringNode string) { + return (string.flags & STATIC_LITERAL_FLAG) != 0; + } else if (node instanceof Nodes.StringNode string) { + return (string.flags & STATIC_LITERAL_FLAG) != 0; + } else if (node instanceof Nodes.SymbolNode symbol) { + return (symbol.flags & STATIC_LITERAL_FLAG) != 0; + } else if (node instanceof Nodes.IntegerNode integer) { + return (integer.flags & STATIC_LITERAL_FLAG) != 0; + } else if (node instanceof Nodes.RationalNode rational) { + return (rational.flags & STATIC_LITERAL_FLAG) != 0; + } else if (node instanceof Nodes.RegularExpressionNode regexp) { + return (regexp.flags & STATIC_LITERAL_FLAG) != 0; + } else if (node instanceof Nodes.RangeNode range) { + return (range.flags & STATIC_LITERAL_FLAG) != 0; + } else if (node instanceof Nodes.HashNode hash) { + for (Nodes.Node element : hash.elements) { + if (!(element instanceof Nodes.AssocNode assoc)) { + return false; + } + // An assoc node with a container key or value is never a static literal. + if (isContainer(assoc.key) || isContainer(assoc.value)) { + return false; + } + if (!isStaticLiteral(assoc.key) || !isStaticLiteral(assoc.value)) { + return false; + } + } + return true; + } else { + return node instanceof Nodes.NilNode || node instanceof Nodes.TrueNode || node instanceof Nodes.FalseNode || + node instanceof Nodes.FloatNode || node instanceof Nodes.ImaginaryNode || node instanceof Nodes.SourceLineNode; + } + } + + private static boolean isContainer(Nodes.Node node) { + return node instanceof Nodes.ArrayNode || node instanceof Nodes.HashNode || node instanceof Nodes.RangeNode; + } + // Whether a while/until loop starts with its keyword. There is no keyword // location in the Java nodes, but only a prefix loop starts before its // statements. From 2299cc985cefb46fd75026e42c7853190456bcf6 Mon Sep 17 00:00:00 2001 From: Benoit Daloze Date: Fri, 11 Sep 2026 22:12:01 +0200 Subject: [PATCH 10/10] Check MarkNewlinesVisitor marks the same lines as newlines.rb in test:java_loader Add a check to the test:java_loader task (run by the build-java job in CI) that MarkNewlinesVisitor.java marks exactly the same lines as Prism::ParseResult#mark_newlines! (newlines.rb), for the newline_test.rb corpus and every fixture which parses successfully, so that both cannot get out of sync. The expected lines are generated by a new test:java_loader:newline_fixtures task, which runs in a subprocess with the default (full) serialization mode since newlines.rb needs location fields, before the existing clobber and recompile with PRISM_SERIALIZE_ONLY_SEMANTICS_FIELDS=1. The Java side runs in the existing task through Loader.load(), which runs MarkNewlinesVisitor, and the marked lines are collected from the loaded Java nodes directly in JRuby. Co-Authored-By: Claude Opus 4.8 --- .gitignore | 1 + rakelib/serialization.rake | 79 ++++++++++++++++++++++++++++++++++++++ 2 files changed, 80 insertions(+) diff --git a/.gitignore b/.gitignore index e8dfeddbe6..e205e63fc7 100644 --- a/.gitignore +++ b/.gitignore @@ -41,6 +41,7 @@ out.svg /javascript/src/visitor.js /javascript/src/prism.wasm /javascript/src/*.d.ts +/java/newline_fixtures.txt /java/org/ruby_lang/prism/AbstractNodeVisitor.java /java/org/ruby_lang/prism/Loader.java /java/org/ruby_lang/prism/Nodes.java diff --git a/rakelib/serialization.rake b/rakelib/serialization.rake index 516e8fe5ba..a4634a2637 100644 --- a/rakelib/serialization.rake +++ b/rakelib/serialization.rake @@ -6,10 +6,52 @@ task "test:java_loader" do # ENV["PRISM_SERIALIZE_ONLY_SEMANTICS_FIELDS"] = "1" raise "this task requires $SERIALIZE_ONLY_SEMANTICS_FIELDS to be set" unless ENV["PRISM_SERIALIZE_ONLY_SEMANTICS_FIELDS"] + # Generate the expected newlines with the default (full) serialization mode + # in a subprocess, since newlines.rb needs location fields. + sh({ "PRISM_SERIALIZE_ONLY_SEMANTICS_FIELDS" => nil }, "bundle", "exec", "rake", "clobber", "test:java_loader:newline_fixtures") + Rake::Task["clobber"].invoke Rake::Task["test:java_loader:internal"].invoke end +# Generates the fixtures for the MarkNewlinesVisitor check of test:java_loader: +# for every file of the corpus, the lines marked by +# Prism::ParseResult#mark_newlines! (newlines.rb), so that the Java loader test +# can check that MarkNewlinesVisitor.java marks exactly the same lines. +task "test:java_loader:newline_fixtures" => :compile do + raise "this task requires the default (full) serialization mode" if ENV["PRISM_SERIALIZE_ONLY_SEMANTICS_FIELDS"] + + $:.unshift(File.expand_path("../lib", __dir__)) + require "prism" + + root = File.expand_path("..", __dir__) + files = Dir["test/prism/{,api/,encoding/,result/,ruby/}*.rb", base: root] + + Dir["test/prism/fixtures/**/*.txt", base: root] + + fixtures = [] + files.sort.each do |file| + path = File.join(root, file) + result = Prism.parse_file(path) + next unless result.success? + + result.mark_newlines! + queue = [result.value] + lines = [] + while node = queue.shift + queue.concat(node.compact_child_nodes) + lines << result.source.line(node.location.start_offset) if node.newline_flag? + end + + fixtures << "#{path}\t#{lines.sort.join(" ")}" + end + + output = File.expand_path("../java/newline_fixtures.txt", __dir__) + require "fileutils" + FileUtils.mkdir_p(File.dirname(output)) + File.write(output, fixtures.join("\n") + "\n") + puts "Wrote the expected newlines of #{fixtures.size} files to #{output}" +end + task "test:java_loader:internal" => :compile do fixtures = File.expand_path("../test/prism/fixtures", __dir__) @@ -28,4 +70,41 @@ task "test:java_loader:internal" => :compile do parse_result = org.ruby_lang.prism.Loader.load(serialized.unpack('c*')) puts parse_result.value end + + # Check that MarkNewlinesVisitor.java (run by Loader.load() above) marks + # exactly the same lines as Ruby's newlines.rb, using the fixtures generated + # by the test:java_loader:newline_fixtures task. + newline_fixtures = File.expand_path("../java/newline_fixtures.txt", __dir__) + raise "no #{newline_fixtures}, run the test:java_loader task to generate it" unless File.exist?(newline_fixtures) + + collect_newlines = -> (node, source, lines) { + return if node.nil? + lines << source.line(node.startOffset) if node.hasNewLineFlag + node.childNodes.each { |child| collect_newlines.call(child, source, lines) } + } + + failures = [] + checked = 0 + File.readlines(newline_fixtures, chomp: true).each do |fixture| + path, expected = fixture.split("\t", 2) + + serialized = Prism.dump_file(path) + parse_result = org.ruby_lang.prism.Loader.load(serialized.unpack('c*')) + + lines = [] + collect_newlines.call(parse_result.value, parse_result.source, lines) + actual = lines.sort.join(" ") + + checked += 1 + if actual != (expected || "") + failures << "#{path}\n ruby: #{expected}\n java: #{actual}" + end + end + + puts + puts "Checked the newlines of #{checked} files against MarkNewlinesVisitor" + unless failures.empty? + abort "#{failures.size} of #{checked} files have different newline flags " \ + "between Ruby's newlines.rb and Java's MarkNewlinesVisitor:\n#{failures.join("\n")}" + end end