diff --git a/lib/lrama/lexer.rb b/lib/lrama/lexer.rb index 8e039125..bc32377e 100644 --- a/lib/lrama/lexer.rb +++ b/lib/lrama/lexer.rb @@ -86,7 +86,7 @@ def next_token # @rbs () -> Integer def column - @scanner.pos - @head + @scanner.string.byteslice(@head...@scanner.pos).length end # @rbs () -> Location @@ -104,8 +104,7 @@ def lex_token case when @scanner.scan(/\n/) newline - when @scanner.scan(/\s+/) - @scanner.matched.count("\n").times { newline } + when @scanner.scan(/[^\S\n]+/) when @scanner.scan(/\/\*/) lex_comment when @scanner.scan(/\/\/.*(?\n)?/) diff --git a/spec/lrama/lexer_spec.rb b/spec/lrama/lexer_spec.rb index 0b37cf9e..f06fd6cc 100644 --- a/spec/lrama/lexer_spec.rb +++ b/spec/lrama/lexer_spec.rb @@ -409,6 +409,28 @@ end end + it 'reports an unexpected token after indentation at its actual column' do + grammar_file = Lrama::Lexer::GrammarFile.new("indented.y", "%%\nprogram: \n @@@ ;") + lexer = Lrama::Lexer.new(grammar_file) + + expect { loop { lexer.next_token } }.to raise_error(ParseError, <<~MSG) + indented.y:3:4: Unexpected token + 3 | @@@ ; + | ^ + MSG + end + + it 'reports columns as character offsets' do + grammar_file = Lrama::Lexer::GrammarFile.new("multibyte.y", '"あ" @') + lexer = Lrama::Lexer.new(grammar_file) + + expect { loop { lexer.next_token } }.to raise_error(ParseError, <<~MSG) + multibyte.y:1:4: Unexpected token + 1 | "あ" @ + | ^ + MSG + end + context 'unexpected_c_code.y' do it do grammar_file = Lrama::Lexer::GrammarFile.new("invalid.y", "@invalid")