diff --git a/.gitignore b/.gitignore index e8dfeddbe6..e205e63fc7 100644 --- a/.gitignore +++ b/.gitignore @@ -41,6 +41,7 @@ out.svg /javascript/src/visitor.js /javascript/src/prism.wasm /javascript/src/*.d.ts +/java/newline_fixtures.txt /java/org/ruby_lang/prism/AbstractNodeVisitor.java /java/org/ruby_lang/prism/Loader.java /java/org/ruby_lang/prism/Nodes.java diff --git a/java/api/src/main/java/org/ruby_lang/prism/MarkNewlinesVisitor.java b/java/api/src/main/java/org/ruby_lang/prism/MarkNewlinesVisitor.java index 148b0d1a2e..3074602c89 100644 --- a/java/api/src/main/java/org/ruby_lang/prism/MarkNewlinesVisitor.java +++ b/java/api/src/main/java/org/ruby_lang/prism/MarkNewlinesVisitor.java @@ -1,5 +1,7 @@ package org.ruby_lang.prism; +import java.util.Arrays; + // Keep in sync with Ruby MarkNewlinesVisitor final class MarkNewlinesVisitor extends AbstractNodeVisitor { @@ -33,26 +35,274 @@ public Void visitLambdaNode(Nodes.LambdaNode node) { } } + @Override + public Void visitDefNode(Nodes.DefNode node) { + boolean[] oldNewlineMarked = this.newlineMarked; + this.newlineMarked = new boolean[oldNewlineMarked.length]; + // The body of an endless method definition never emits newline events, + // so in that case mark every line as already seen instead. There is no + // location for the `=` operator here, but only an endless method + // definition has a statements body which ends with the def node itself. + if (node.body instanceof Nodes.StatementsNode && node.endOffset() == node.body.endOffset()) { + Arrays.fill(this.newlineMarked, true); + } + try { + return super.visitDefNode(node); + } finally { + this.newlineMarked = oldNewlineMarked; + } + } + + @Override + public Void visitClassNode(Nodes.ClassNode node) { + boolean[] oldNewlineMarked = this.newlineMarked; + this.newlineMarked = new boolean[oldNewlineMarked.length]; + try { + return super.visitClassNode(node); + } finally { + this.newlineMarked = oldNewlineMarked; + } + } + + @Override + public Void visitModuleNode(Nodes.ModuleNode node) { + boolean[] oldNewlineMarked = this.newlineMarked; + this.newlineMarked = new boolean[oldNewlineMarked.length]; + try { + return super.visitModuleNode(node); + } finally { + this.newlineMarked = oldNewlineMarked; + } + } + + @Override + public Void visitSingletonClassNode(Nodes.SingletonClassNode node) { + boolean[] oldNewlineMarked = this.newlineMarked; + this.newlineMarked = new boolean[oldNewlineMarked.length]; + try { + return super.visitSingletonClassNode(node); + } finally { + this.newlineMarked = oldNewlineMarked; + } + } + + // Statements inside string interpolation do not emit newline events, so + // mark every line as already seen while visiting them. Nested scopes + // (blocks, lambdas, defs, etc.) reset the lines and emit events again. + @Override + public Void visitEmbeddedStatementsNode(Nodes.EmbeddedStatementsNode node) { + boolean[] oldNewlineMarked = this.newlineMarked; + this.newlineMarked = new boolean[oldNewlineMarked.length]; + Arrays.fill(this.newlineMarked, true); + try { + return super.visitEmbeddedStatementsNode(node); + } finally { + this.newlineMarked = oldNewlineMarked; + } + } + + // The predicate of a while loop is compiled at the end of the loop, + // after the body, so any statements it contains (from parentheses) + // emit their line events again even if the lines were already seen. + @Override + public Void visitWhileNode(Nodes.WhileNode node) { + boolean[] oldNewlineMarked = this.newlineMarked; + this.newlineMarked = new boolean[oldNewlineMarked.length]; + try { + node.predicate.accept(this); + } finally { + this.newlineMarked = oldNewlineMarked; + } + + if (node.statements != null) { + node.statements.accept(this); + } + return null; + } + + // The predicate of an until loop is compiled at the end of the loop, + // after the body, so any statements it contains (from parentheses) + // emit their line events again even if the lines were already seen. + @Override + public Void visitUntilNode(Nodes.UntilNode node) { + boolean[] oldNewlineMarked = this.newlineMarked; + this.newlineMarked = new boolean[oldNewlineMarked.length]; + try { + node.predicate.accept(this); + } finally { + this.newlineMarked = oldNewlineMarked; + } + + if (node.statements != null) { + node.statements.accept(this); + } + return null; + } + @Override public Void visitIfNode(Nodes.IfNode node) { - node.setNewLineFlag(this.source, this.newlineMarked); + setNewLineFlag(node); return super.visitIfNode(node); } @Override public Void visitUnlessNode(Nodes.UnlessNode node) { - node.setNewLineFlag(this.source, this.newlineMarked); + setNewLineFlag(node); return super.visitUnlessNode(node); } @Override public Void visitStatementsNode(Nodes.StatementsNode node) { for (Nodes.Node child : node.body) { - child.setNewLineFlag(this.source, this.newlineMarked); + setNewLineFlag(child); } return super.visitStatementsNode(node); } + // Keep in sync with the newline_flag! overrides in Ruby's newlines.rb which + // are not part of the generated setNewLineFlag() methods. + // + // The line event for a statement is emitted where its first instruction is + // compiled, so nodes whose first instruction comes from a sub-expression + // delegate their newline flag to that sub-expression: assignments to their + // value, calls to their receiver, and array, hash, and interpolated string + // literals to their first element. Static literals are the exception: they + // are compiled to a single instruction on the first line of the literal, so + // they do not delegate. + private void setNewLineFlag(Nodes.Node node) { + if (node instanceof Nodes.LocalVariableWriteNode write) { + setNewLineFlag(write.value); + } else if (node instanceof Nodes.InstanceVariableWriteNode write) { + setNewLineFlag(write.value); + } else if (node instanceof Nodes.ClassVariableWriteNode write) { + setNewLineFlag(write.value); + } else if (node instanceof Nodes.GlobalVariableWriteNode write) { + setNewLineFlag(write.value); + } else if (node instanceof Nodes.ConstantWriteNode write) { + setNewLineFlag(write.value); + } else if (node instanceof Nodes.ConstantPathWriteNode write) { + setNewLineFlag(write.value); + } else if (node instanceof Nodes.MultiWriteNode write) { + setNewLineFlag(write.value); + } else if (node instanceof Nodes.CallNode call) { + if (call.receiver != null) { + setNewLineFlag(call.receiver); + } else { + markNewLineFlag(node); + } + } else if (node instanceof Nodes.ArrayNode array) { + if (array.elements.length > 0 && !isStaticLiteral(array)) { + setNewLineFlag(array.elements[0]); + } else { + markNewLineFlag(node); + } + } else if (node instanceof Nodes.HashNode hash) { + if (hash.elements.length > 0 && !isStaticLiteral(hash)) { + setNewLineFlag(hash.elements[0]); + } else { + markNewLineFlag(node); + } + } else if (node instanceof Nodes.InterpolatedStringNode string) { + if (string.parts.length > 0 && !isStaticLiteral(string)) { + setNewLineFlag(string.parts[0]); + } else { + markNewLineFlag(node); + } + } else if (node instanceof Nodes.IfNode ifNode) { + setNewLineFlag(ifNode.predicate); + } else if (node instanceof Nodes.UnlessNode unlessNode) { + setNewLineFlag(unlessNode.predicate); + } else if (node instanceof Nodes.RescueModifierNode rescueModifier) { + setNewLineFlag(rescueModifier.expression); + } else if (node instanceof Nodes.WhileNode whileNode) { + boolean prefix = isPrefixLoop(whileNode, whileNode.isBeginModifier(), whileNode.statements); + if (prefix && whileNode.predicate instanceof Nodes.ParenthesesNode) { + // A parenthesized predicate emits its own line event when it is + // compiled at the end of the loop, in addition to this one. + markNewLineFlag(node); + } else { + setNewLineFlag(whileNode.predicate); + } + } else if (node instanceof Nodes.UntilNode untilNode) { + boolean prefix = isPrefixLoop(untilNode, untilNode.isBeginModifier(), untilNode.statements); + if (prefix && untilNode.predicate instanceof Nodes.ParenthesesNode) { + // A parenthesized predicate emits its own line event when it is + // compiled at the end of the loop, in addition to this one. + markNewLineFlag(node); + } else { + setNewLineFlag(untilNode.predicate); + } + } else { + node.setNewLineFlag(this.source, this.newlineMarked); + } + } + + // Mark the node itself, like Nodes.Node#setNewLineFlag(), regardless of any + // setNewLineFlag() override of the node. + private void markNewLineFlag(Nodes.Node node) { + int line = this.source.findLine(node.startOffset); + if (!this.newlineMarked[line]) { + this.newlineMarked[line] = true; + node.setNewLineFlag(true); + } + } + + // PM_NODE_FLAG_STATIC_LITERAL, which is serialized together with the + // node-specific flags. + private static final short STATIC_LITERAL_FLAG = 0x2; + + // Whether the node has the PM_NODE_FLAG_STATIC_LITERAL flag. Only nodes + // with node-specific flags store their flags in Java, so it is computed + // structurally for the others, for the nodes which can appear inside an + // array or hash literal. + private static boolean isStaticLiteral(Nodes.Node node) { + if (node instanceof Nodes.ArrayNode array) { + return (array.flags & STATIC_LITERAL_FLAG) != 0; + } else if (node instanceof Nodes.InterpolatedStringNode string) { + return (string.flags & STATIC_LITERAL_FLAG) != 0; + } else if (node instanceof Nodes.StringNode string) { + return (string.flags & STATIC_LITERAL_FLAG) != 0; + } else if (node instanceof Nodes.SymbolNode symbol) { + return (symbol.flags & STATIC_LITERAL_FLAG) != 0; + } else if (node instanceof Nodes.IntegerNode integer) { + return (integer.flags & STATIC_LITERAL_FLAG) != 0; + } else if (node instanceof Nodes.RationalNode rational) { + return (rational.flags & STATIC_LITERAL_FLAG) != 0; + } else if (node instanceof Nodes.RegularExpressionNode regexp) { + return (regexp.flags & STATIC_LITERAL_FLAG) != 0; + } else if (node instanceof Nodes.RangeNode range) { + return (range.flags & STATIC_LITERAL_FLAG) != 0; + } else if (node instanceof Nodes.HashNode hash) { + for (Nodes.Node element : hash.elements) { + if (!(element instanceof Nodes.AssocNode assoc)) { + return false; + } + // An assoc node with a container key or value is never a static literal. + if (isContainer(assoc.key) || isContainer(assoc.value)) { + return false; + } + if (!isStaticLiteral(assoc.key) || !isStaticLiteral(assoc.value)) { + return false; + } + } + return true; + } else { + return node instanceof Nodes.NilNode || node instanceof Nodes.TrueNode || node instanceof Nodes.FalseNode || + node instanceof Nodes.FloatNode || node instanceof Nodes.ImaginaryNode || node instanceof Nodes.SourceLineNode; + } + } + + private static boolean isContainer(Nodes.Node node) { + return node instanceof Nodes.ArrayNode || node instanceof Nodes.HashNode || node instanceof Nodes.RangeNode; + } + + // Whether a while/until loop starts with its keyword. There is no keyword + // location in the Java nodes, but only a prefix loop starts before its + // statements. + private static boolean isPrefixLoop(Nodes.Node node, boolean beginModifier, Nodes.StatementsNode statements) { + return !beginModifier && (statements == null || node.startOffset != statements.startOffset); + } + @Override protected Void defaultVisit(Nodes.Node node) { node.visitChildNodes(this); diff --git a/lib/prism/parse_result/newlines.rb b/lib/prism/parse_result/newlines.rb index ad8d8b6f55..41729bf813 100644 --- a/lib/prism/parse_result/newlines.rb +++ b/lib/prism/parse_result/newlines.rb @@ -65,6 +65,117 @@ def visit_lambda_node(node) end end + # Permit def nodes to mark newlines within themselves. The body of an + # endless method definition never emits newline events, so in that case + # mark every line as already seen while visiting it instead. Nested + # scopes (blocks, lambdas, etc.) reset the lines and emit events again. + # + #: (DefNode node) -> void + def visit_def_node(node) + old_lines = @lines + @lines = Array.new(old_lines.size, !node.equal_loc.nil?) + + begin + super(node) + ensure + @lines = old_lines + end + end + + # Permit class nodes to mark newlines within themselves. + # + #: (ClassNode node) -> void + def visit_class_node(node) + old_lines = @lines + @lines = Array.new(old_lines.size, false) + + begin + super(node) + ensure + @lines = old_lines + end + end + + # Permit module nodes to mark newlines within themselves. + # + #: (ModuleNode node) -> void + def visit_module_node(node) + old_lines = @lines + @lines = Array.new(old_lines.size, false) + + begin + super(node) + ensure + @lines = old_lines + end + end + + # Permit singleton class nodes to mark newlines within themselves. + # + #: (SingletonClassNode node) -> void + def visit_singleton_class_node(node) + old_lines = @lines + @lines = Array.new(old_lines.size, false) + + begin + super(node) + ensure + @lines = old_lines + end + end + + # Statements inside string interpolation do not emit newline events, so + # mark every line as already seen while visiting them. Nested scopes + # (blocks, lambdas, defs, etc.) reset the lines and emit events again. + # + #: (EmbeddedStatementsNode node) -> void + def visit_embedded_statements_node(node) + old_lines = @lines + @lines = Array.new(old_lines.size, true) + + begin + super(node) + ensure + @lines = old_lines + end + end + + # The predicate of a while loop is compiled at the end of the loop, + # after the body, so any statements it contains (from parentheses) + # emit their line events again even if the lines were already seen. + # + #: (WhileNode node) -> void + def visit_while_node(node) + old_lines = @lines + @lines = Array.new(old_lines.size, false) + + begin + visit(node.predicate) + ensure + @lines = old_lines + end + + visit(node.statements) + end + + # The predicate of an until loop is compiled at the end of the loop, + # after the body, so any statements it contains (from parentheses) + # emit their line events again even if the lines were already seen. + # + #: (UntilNode node) -> void + def visit_until_node(node) + old_lines = @lines + @lines = Array.new(old_lines.size, false) + + begin + visit(node.predicate) + ensure + @lines = old_lines + end + + visit(node.statements) + end + # Mark if nodes as newlines. # #: (IfNode node) -> void @@ -144,14 +255,26 @@ def newline_flag!(lines) # :nodoc: class UntilNode < Node #: (Array[bool] lines) -> void def newline_flag!(lines) # :nodoc: - predicate.newline_flag!(lines) + if location.start_offset == keyword_loc.start_offset && predicate.is_a?(ParenthesesNode) + # A parenthesized predicate emits its own line event when it is + # compiled at the end of the loop, in addition to this one. + super + else + predicate.newline_flag!(lines) + end end end class WhileNode < Node #: (Array[bool] lines) -> void def newline_flag!(lines) # :nodoc: - predicate.newline_flag!(lines) + if location.start_offset == keyword_loc.start_offset && predicate.is_a?(ParenthesesNode) + # A parenthesized predicate emits its own line event when it is + # compiled at the end of the loop, in addition to this one. + super + else + predicate.newline_flag!(lines) + end end end @@ -162,6 +285,98 @@ def newline_flag!(lines) # :nodoc: end end + # The line event for a statement is emitted where its first instruction is + # compiled, so nodes whose first instruction comes from a sub-expression + # delegate their newline flag to that sub-expression: assignments to their + # value, calls to their receiver, and array, hash, and interpolated string + # literals to their first element. Static literals are the exception: they + # are compiled to a single instruction on the first line of the literal, so + # they do not delegate. + + class LocalVariableWriteNode < Node + #: (Array[bool] lines) -> void + def newline_flag!(lines) # :nodoc: + value.newline_flag!(lines) + end + end + + class InstanceVariableWriteNode < Node + #: (Array[bool] lines) -> void + def newline_flag!(lines) # :nodoc: + value.newline_flag!(lines) + end + end + + class ClassVariableWriteNode < Node + #: (Array[bool] lines) -> void + def newline_flag!(lines) # :nodoc: + value.newline_flag!(lines) + end + end + + class GlobalVariableWriteNode < Node + #: (Array[bool] lines) -> void + def newline_flag!(lines) # :nodoc: + value.newline_flag!(lines) + end + end + + class ConstantWriteNode < Node + #: (Array[bool] lines) -> void + def newline_flag!(lines) # :nodoc: + value.newline_flag!(lines) + end + end + + class ConstantPathWriteNode < Node + #: (Array[bool] lines) -> void + def newline_flag!(lines) # :nodoc: + value.newline_flag!(lines) + end + end + + class MultiWriteNode < Node + #: (Array[bool] lines) -> void + def newline_flag!(lines) # :nodoc: + value.newline_flag!(lines) + end + end + + class CallNode < Node + #: (Array[bool] lines) -> void + def newline_flag!(lines) # :nodoc: + if (receiver = self.receiver) + receiver.newline_flag!(lines) + else + super + end + end + end + + class ArrayNode < Node + #: (Array[bool] lines) -> void + def newline_flag!(lines) # :nodoc: + first = elements.first + if first && !static_literal? + first.newline_flag!(lines) + else + super + end + end + end + + class HashNode < Node + #: (Array[bool] lines) -> void + def newline_flag!(lines) # :nodoc: + first = elements.first + if first && !static_literal? + first.newline_flag!(lines) + else + super + end + end + end + class InterpolatedMatchLastLineNode < Node #: (Array[bool] lines) -> void def newline_flag!(lines) # :nodoc: @@ -182,7 +397,11 @@ class InterpolatedStringNode < Node #: (Array[bool] lines) -> void def newline_flag!(lines) # :nodoc: first = parts.first - first.newline_flag!(lines) if first + if first && !static_literal? + first.newline_flag!(lines) + else + super + end end end diff --git a/rakelib/serialization.rake b/rakelib/serialization.rake index 516e8fe5ba..a4634a2637 100644 --- a/rakelib/serialization.rake +++ b/rakelib/serialization.rake @@ -6,10 +6,52 @@ task "test:java_loader" do # ENV["PRISM_SERIALIZE_ONLY_SEMANTICS_FIELDS"] = "1" raise "this task requires $SERIALIZE_ONLY_SEMANTICS_FIELDS to be set" unless ENV["PRISM_SERIALIZE_ONLY_SEMANTICS_FIELDS"] + # Generate the expected newlines with the default (full) serialization mode + # in a subprocess, since newlines.rb needs location fields. + sh({ "PRISM_SERIALIZE_ONLY_SEMANTICS_FIELDS" => nil }, "bundle", "exec", "rake", "clobber", "test:java_loader:newline_fixtures") + Rake::Task["clobber"].invoke Rake::Task["test:java_loader:internal"].invoke end +# Generates the fixtures for the MarkNewlinesVisitor check of test:java_loader: +# for every file of the corpus, the lines marked by +# Prism::ParseResult#mark_newlines! (newlines.rb), so that the Java loader test +# can check that MarkNewlinesVisitor.java marks exactly the same lines. +task "test:java_loader:newline_fixtures" => :compile do + raise "this task requires the default (full) serialization mode" if ENV["PRISM_SERIALIZE_ONLY_SEMANTICS_FIELDS"] + + $:.unshift(File.expand_path("../lib", __dir__)) + require "prism" + + root = File.expand_path("..", __dir__) + files = Dir["test/prism/{,api/,encoding/,result/,ruby/}*.rb", base: root] + + Dir["test/prism/fixtures/**/*.txt", base: root] + + fixtures = [] + files.sort.each do |file| + path = File.join(root, file) + result = Prism.parse_file(path) + next unless result.success? + + result.mark_newlines! + queue = [result.value] + lines = [] + while node = queue.shift + queue.concat(node.compact_child_nodes) + lines << result.source.line(node.location.start_offset) if node.newline_flag? + end + + fixtures << "#{path}\t#{lines.sort.join(" ")}" + end + + output = File.expand_path("../java/newline_fixtures.txt", __dir__) + require "fileutils" + FileUtils.mkdir_p(File.dirname(output)) + File.write(output, fixtures.join("\n") + "\n") + puts "Wrote the expected newlines of #{fixtures.size} files to #{output}" +end + task "test:java_loader:internal" => :compile do fixtures = File.expand_path("../test/prism/fixtures", __dir__) @@ -28,4 +70,41 @@ task "test:java_loader:internal" => :compile do parse_result = org.ruby_lang.prism.Loader.load(serialized.unpack('c*')) puts parse_result.value end + + # Check that MarkNewlinesVisitor.java (run by Loader.load() above) marks + # exactly the same lines as Ruby's newlines.rb, using the fixtures generated + # by the test:java_loader:newline_fixtures task. + newline_fixtures = File.expand_path("../java/newline_fixtures.txt", __dir__) + raise "no #{newline_fixtures}, run the test:java_loader task to generate it" unless File.exist?(newline_fixtures) + + collect_newlines = -> (node, source, lines) { + return if node.nil? + lines << source.line(node.startOffset) if node.hasNewLineFlag + node.childNodes.each { |child| collect_newlines.call(child, source, lines) } + } + + failures = [] + checked = 0 + File.readlines(newline_fixtures, chomp: true).each do |fixture| + path, expected = fixture.split("\t", 2) + + serialized = Prism.dump_file(path) + parse_result = org.ruby_lang.prism.Loader.load(serialized.unpack('c*')) + + lines = [] + collect_newlines.call(parse_result.value, parse_result.source, lines) + actual = lines.sort.join(" ") + + checked += 1 + if actual != (expected || "") + failures << "#{path}\n ruby: #{expected}\n java: #{actual}" + end + end + + puts + puts "Checked the newlines of #{checked} files against MarkNewlinesVisitor" + unless failures.empty? + abort "#{failures.size} of #{checked} files have different newline flags " \ + "between Ruby's newlines.rb and Java's MarkNewlinesVisitor:\n#{failures.join("\n")}" + end end diff --git a/rbi/generated/prism/parse_result/newlines.rbi b/rbi/generated/prism/parse_result/newlines.rbi index f77dee3bbf..3e83f4757a 100644 --- a/rbi/generated/prism/parse_result/newlines.rbi +++ b/rbi/generated/prism/parse_result/newlines.rbi @@ -35,6 +35,43 @@ module Prism sig { params(node: LambdaNode).void } def visit_lambda_node(node); end + # Permit def nodes to mark newlines within themselves. The body of an + # endless method definition never emits newline events, so in that case + # mark every line as already seen while visiting it instead. Nested + # scopes (blocks, lambdas, etc.) reset the lines and emit events again. + sig { params(node: DefNode).void } + def visit_def_node(node); end + + # Permit class nodes to mark newlines within themselves. + sig { params(node: ClassNode).void } + def visit_class_node(node); end + + # Permit module nodes to mark newlines within themselves. + sig { params(node: ModuleNode).void } + def visit_module_node(node); end + + # Permit singleton class nodes to mark newlines within themselves. + sig { params(node: SingletonClassNode).void } + def visit_singleton_class_node(node); end + + # Statements inside string interpolation do not emit newline events, so + # mark every line as already seen while visiting them. Nested scopes + # (blocks, lambdas, defs, etc.) reset the lines and emit events again. + sig { params(node: EmbeddedStatementsNode).void } + def visit_embedded_statements_node(node); end + + # The predicate of a while loop is compiled at the end of the loop, + # after the body, so any statements it contains (from parentheses) + # emit their line events again even if the lines were already seen. + sig { params(node: WhileNode).void } + def visit_while_node(node); end + + # The predicate of an until loop is compiled at the end of the loop, + # after the body, so any statements it contains (from parentheses) + # emit their line events again even if the lines were already seen. + sig { params(node: UntilNode).void } + def visit_until_node(node); end + # Mark if nodes as newlines. sig { params(node: IfNode).void } def visit_if_node(node); end @@ -94,6 +131,56 @@ module Prism def newline_flag!(lines); end end + class LocalVariableWriteNode < Node + sig { params(lines: T::Array[T::Boolean]).void } + def newline_flag!(lines); end + end + + class InstanceVariableWriteNode < Node + sig { params(lines: T::Array[T::Boolean]).void } + def newline_flag!(lines); end + end + + class ClassVariableWriteNode < Node + sig { params(lines: T::Array[T::Boolean]).void } + def newline_flag!(lines); end + end + + class GlobalVariableWriteNode < Node + sig { params(lines: T::Array[T::Boolean]).void } + def newline_flag!(lines); end + end + + class ConstantWriteNode < Node + sig { params(lines: T::Array[T::Boolean]).void } + def newline_flag!(lines); end + end + + class ConstantPathWriteNode < Node + sig { params(lines: T::Array[T::Boolean]).void } + def newline_flag!(lines); end + end + + class MultiWriteNode < Node + sig { params(lines: T::Array[T::Boolean]).void } + def newline_flag!(lines); end + end + + class CallNode < Node + sig { params(lines: T::Array[T::Boolean]).void } + def newline_flag!(lines); end + end + + class ArrayNode < Node + sig { params(lines: T::Array[T::Boolean]).void } + def newline_flag!(lines); end + end + + class HashNode < Node + sig { params(lines: T::Array[T::Boolean]).void } + def newline_flag!(lines); end + end + class InterpolatedMatchLastLineNode < Node sig { params(lines: T::Array[T::Boolean]).void } def newline_flag!(lines); end diff --git a/sig/generated/prism/parse_result/newlines.rbs b/sig/generated/prism/parse_result/newlines.rbs index 9a505fa961..2a9cf13c54 100644 --- a/sig/generated/prism/parse_result/newlines.rbs +++ b/sig/generated/prism/parse_result/newlines.rbs @@ -40,6 +40,50 @@ module Prism # : (LambdaNode node) -> void def visit_lambda_node: (LambdaNode node) -> void + # Permit def nodes to mark newlines within themselves. The body of an + # endless method definition never emits newline events, so in that case + # mark every line as already seen while visiting it instead. Nested + # scopes (blocks, lambdas, etc.) reset the lines and emit events again. + # + # : (DefNode node) -> void + def visit_def_node: (DefNode node) -> void + + # Permit class nodes to mark newlines within themselves. + # + # : (ClassNode node) -> void + def visit_class_node: (ClassNode node) -> void + + # Permit module nodes to mark newlines within themselves. + # + # : (ModuleNode node) -> void + def visit_module_node: (ModuleNode node) -> void + + # Permit singleton class nodes to mark newlines within themselves. + # + # : (SingletonClassNode node) -> void + def visit_singleton_class_node: (SingletonClassNode node) -> void + + # Statements inside string interpolation do not emit newline events, so + # mark every line as already seen while visiting them. Nested scopes + # (blocks, lambdas, defs, etc.) reset the lines and emit events again. + # + # : (EmbeddedStatementsNode node) -> void + def visit_embedded_statements_node: (EmbeddedStatementsNode node) -> void + + # The predicate of a while loop is compiled at the end of the loop, + # after the body, so any statements it contains (from parentheses) + # emit their line events again even if the lines were already seen. + # + # : (WhileNode node) -> void + def visit_while_node: (WhileNode node) -> void + + # The predicate of an until loop is compiled at the end of the loop, + # after the body, so any statements it contains (from parentheses) + # emit their line events again even if the lines were already seen. + # + # : (UntilNode node) -> void + def visit_until_node: (UntilNode node) -> void + # Mark if nodes as newlines. # # : (IfNode node) -> void @@ -102,6 +146,56 @@ module Prism def newline_flag!: (Array[bool] lines) -> void end + class LocalVariableWriteNode < Node + # : (Array[bool] lines) -> void + def newline_flag!: (Array[bool] lines) -> void + end + + class InstanceVariableWriteNode < Node + # : (Array[bool] lines) -> void + def newline_flag!: (Array[bool] lines) -> void + end + + class ClassVariableWriteNode < Node + # : (Array[bool] lines) -> void + def newline_flag!: (Array[bool] lines) -> void + end + + class GlobalVariableWriteNode < Node + # : (Array[bool] lines) -> void + def newline_flag!: (Array[bool] lines) -> void + end + + class ConstantWriteNode < Node + # : (Array[bool] lines) -> void + def newline_flag!: (Array[bool] lines) -> void + end + + class ConstantPathWriteNode < Node + # : (Array[bool] lines) -> void + def newline_flag!: (Array[bool] lines) -> void + end + + class MultiWriteNode < Node + # : (Array[bool] lines) -> void + def newline_flag!: (Array[bool] lines) -> void + end + + class CallNode < Node + # : (Array[bool] lines) -> void + def newline_flag!: (Array[bool] lines) -> void + end + + class ArrayNode < Node + # : (Array[bool] lines) -> void + def newline_flag!: (Array[bool] lines) -> void + end + + class HashNode < Node + # : (Array[bool] lines) -> void + def newline_flag!: (Array[bool] lines) -> void + end + class InterpolatedMatchLastLineNode < Node # : (Array[bool] lines) -> void def newline_flag!: (Array[bool] lines) -> void diff --git a/test/prism/newline_test.rb b/test/prism/newline_test.rb index ed797db965..12504e912d 100644 --- a/test/prism/newline_test.rb +++ b/test/prism/newline_test.rb @@ -2,33 +2,16 @@ require_relative "test_helper" -return unless defined?(RubyVM::InstructionSequence) +# There have also been changes made in other versions of Ruby, so we only want +# to test on the most recent versions. +return if !defined?(RubyVM::InstructionSequence) || RUBY_VERSION < "3.4.0" module Prism class NewlineTest < TestCase - skips = %w[ - errors_test.rb - locals_test.rb - regexp_test.rb - test_helper.rb - unescape_test.rb - api/parse_stream_test.rb - api/raise_error_test.rb - encoding/regular_expression_encoding_test.rb - encoding/string_encoding_test.rb - result/breadth_first_search_test.rb - result/static_literals_test.rb - result/warnings_test.rb - ruby/find_fixtures.rb - ruby/find_test.rb - ruby/parser_test.rb - ruby/ripper_test.rb - ruby/ruby_parser_test.rb - ruby/parameters_signature_test.rb - ] - + # If you are coming from ruby/ruby, a test failure here means that TracePoint `:line` events changed. + # Before adding a skip, make sure that you actually intended for such a difference to happen. base = __dir__ - (Dir["{,api/,encoding/,result/,ruby/}*.rb", base: base] - skips).each do |relative| + Dir["{,api/,encoding/,result/,ruby/}*.rb", base: base].each do |relative| define_method(:"test_#{relative}") do assert_newlines(base, relative) end @@ -45,33 +28,6 @@ def assert_newlines(base, relative) assert_empty result.errors actual = prism_lines(result) - source.each_line.with_index(1) do |line, line_number| - # Lines like `while (foo = bar)` result in two line flags in the - # bytecode but only one newline flag in the AST. We need to remove the - # extra line flag from the bytecode to make the test pass. - if line.match?(/while \(/) - index = expected.index(line_number) - expected.delete_at(index) if index - end - - # Lines like `foo =` where the value is on the next line result in - # another line flag in the bytecode but only one newline flag in the - # AST. - if line.match?(/^\s+\w+ =$/) - if source.lines[line_number].match?(/^\s+case/) - actual[actual.index(line_number)] += 1 - else - actual.delete_at(actual.index(line_number)) - end - end - - if line.match?(/^\s+\w+ = \[$/) - if !expected.include?(line_number) && !expected.include?(line_number + 2) - actual[actual.index(line_number)] += 1 - end - end - end - assert_equal expected, actual end