Fix lexer column tracking

lib/lrama/lexer.rb

@@ -86,7 +86,7 @@ def next_token
 
     # @rbs () -> Integer
     def column
-      @scanner.pos - @head
+      @scanner.string.byteslice(@head...@scanner.pos).length
     end
 
     # @rbs () -> Location
@@ -104,8 +104,7 @@ def lex_token
         case
         when @scanner.scan(/\n/)
           newline
-        when @scanner.scan(/\s+/)
-          @scanner.matched.count("\n").times { newline }
+        when @scanner.scan(/[^\S\n]+/)
         when @scanner.scan(/\/\*/)
           lex_comment
         when @scanner.scan(/\/\/.*(?<newline>\n)?/)

spec/lrama/lexer_spec.rb

@@ -409,6 +409,28 @@
     end
   end
 
+  it 'reports an unexpected token after indentation at its actual column' do
+    grammar_file = Lrama::Lexer::GrammarFile.new("indented.y", "%%\nprogram:   \n    @@@ ;")
+    lexer = Lrama::Lexer.new(grammar_file)
+
+    expect { loop { lexer.next_token } }.to raise_error(ParseError, <<~MSG)
+      indented.y:3:4: Unexpected token
+         3 |     @@@ ;
+           |     ^
+    MSG
+  end
+
+  it 'reports columns as character offsets' do
+    grammar_file = Lrama::Lexer::GrammarFile.new("multibyte.y", '"あ" @')
+    lexer = Lrama::Lexer.new(grammar_file)
+
+    expect { loop { lexer.next_token } }.to raise_error(ParseError, <<~MSG)
+      multibyte.y:1:4: Unexpected token
+         1 | "あ" @
+           |     ^
+    MSG
+  end
+
   context 'unexpected_c_code.y' do
     it do
       grammar_file = Lrama::Lexer::GrammarFile.new("invalid.y", "@invalid")