Preserve and validate redeclared token metadata

lib/lrama/context.rb

@@ -26,7 +26,7 @@ def initialize(states)
     # enum yytokentype
     def yytokentype
       @states.terms.reject do |term|
-        term.id.is_a?(Lexer::Token::Char)
+        term.id.is_a?(Lexer::Token::Char) || term.id.s_value.start_with?('"')
       end.map do |term|
         [term.id.s_value, term.token_id, term.display_name]
       end.unshift(["YYEMPTY", -2, nil])

lib/lrama/grammar/symbol.rb

@@ -107,7 +107,7 @@ def enum_name
           res = "YYACCEPT"
         when eof_symbol?
           res = "YYEOF"
-        when term? && id.is_a?(Lrama::Lexer::Token::Char)
+        when term? && (id.is_a?(Lrama::Lexer::Token::Char) || name.start_with?('"'))
           res = number.to_s + display_name
         when term? && id.is_a?(Lrama::Lexer::Token::Ident)
           res = name

lib/lrama/grammar/symbols/resolver.rb

@@ -51,6 +51,41 @@ def sort_by_number!
 
         # @rbs (id: Lexer::Token::Base, ?alias_name: String?, ?tag: Lexer::Token::Tag?, ?token_id: Integer?, ?replace: bool) -> Grammar::Symbol
         def add_term(id:, alias_name: nil, tag: nil, token_id: nil, replace: false)
+          sym = find_symbol_by_id(id)
+          if alias_name&.start_with?('"') && (!sym || !sym.alias_name) && (literal = find_term_by_s_value(alias_name))
+            if sym || (token_id && find_symbol_by_token_id(token_id))
+              @terms.delete(literal)
+              @symbols = nil
+            else
+              sym = literal
+              sym.id = id
+            end
+          end
+
+          if sym
+            if tag && (previous_tag = sym.tag)
+              message = "%type redeclaration for #{id.s_value}"
+              message = id.location&.generate_error_message(message) || message
+              if previous_tag.location
+                message += "\n" + previous_tag.location.generate_error_message("previous declaration")
+              end
+              raise message
+            end
+
+            if token_id && sym.token_id && sym.token_id != token_id
+              message = "redefining code of token #{id.s_value}"
+              raise(id.location&.generate_error_message(message) || message)
+            end
+
+            if alias_name && sym.alias_name && sym.alias_name != alias_name
+              add_term(id: Lexer::Token::Str.new(s_value: alias_name, location: id.location))
+            end
+            sym.alias_name ||= alias_name
+            sym.tag ||= tag
+            sym.token_id = token_id if token_id
+            return sym
+          end
+
           if token_id && (sym = find_symbol_by_token_id(token_id))
             if replace
               sym.id = id
@@ -61,10 +96,6 @@ def add_term(id:, alias_name: nil, tag: nil, token_id: nil, replace: false)
             return sym
           end
 
-          if (sym = find_symbol_by_id(id))
-            return sym
-          end
-
           @symbols = nil
           term = Symbol.new(
             id: id, alias_name: alias_name, number: nil, tag: tag,

lib/lrama/warnings.rb

@@ -7,6 +7,7 @@
 require_relative 'warnings/name_conflicts'
 require_relative 'warnings/redefined_rules'
 require_relative 'warnings/required'
+require_relative 'warnings/token_redeclarations'
 require_relative 'warnings/useless_precedence'
 
 module Lrama
@@ -19,11 +20,13 @@ def initialize(logger, warnings)
       @name_conflicts = NameConflicts.new(logger, warnings)
       @redefined_rules = RedefinedRules.new(logger, warnings)
       @required = Required.new(logger, warnings)
+      @token_redeclarations = TokenRedeclarations.new(logger)
       @useless_precedence = UselessPrecedence.new(logger, warnings)
     end
 
     # @rbs (Lrama::Grammar grammar, Lrama::States states) -> void
     def warn(grammar, states)
+      @token_redeclarations.warn(grammar)
       @conflicts.warn(states)
       @implicit_empty.warn(grammar)
       @mixed_references.warn(grammar)

lib/lrama/warnings/token_redeclarations.rb

@@ -0,0 +1,47 @@
+# rbs_inline: enabled
+# frozen_string_literal: true
+
+module Lrama
+  class Warnings
+    class TokenRedeclarations
+      # @rbs (Lrama::Logger logger) -> void
+      def initialize(logger)
+        @logger = logger
+      end
+
+      # @rbs (Lrama::Grammar grammar) -> void
+      def warn(grammar)
+        declarations = {} #: Hash[String, Grammar::Node::Token]
+        aliases = {} #: Hash[String, String]
+
+        grammar.nodes.each do |node|
+          next unless node.is_a?(Grammar::Node::TokenDecl)
+
+          node.tokens.each do |token|
+            name = token.id.s_value
+            if (previous = declarations[name])
+              @logger.warn(token.location.generate_error_message("symbol #{name} redeclared") + "\n" +
+                           previous.location.generate_error_message("previous declaration"))
+            else
+              declarations[name] = token
+            end
+
+            if (alias_name = token.alias_name)
+              if (previous_alias = aliases[name])
+                message =
+                  if previous_alias == alias_name
+                    "symbol #{alias_name} used more than once as a literal string"
+                  else
+                    "symbol #{name} given more than one literal string"
+                  end
+                @logger.warn(token.location.generate_error_message(message))
+              else
+                aliases[name] = alias_name
+              end
+            end
+          end
+        end
+      end
+    end
+  end
+end

spec/lrama/command_spec.rb

@@ -14,6 +14,16 @@
       end
     end
 
+    it "warns about redeclared aliases without -W and still generates a parser" do
+      y = "%token PLUS \"+\"\n%token PLUS \"plus\"\n%%\nstart: PLUS \"+\" \"plus\";\n"
+      allow(STDIN).to receive(:read).and_return(y)
+      command = Lrama::Command.new(o_option + ["-", "redeclared_token.y"])
+
+      expect { command.run }.to output(/symbol PLUS given more than one literal string/).to_stderr_from_any_process
+      expect(File.read(outfile)).to include("PLUS =")
+      expect(File.read(outfile)).not_to include('"plus" =')
+    end
+
     context "when `--diagram` is specified" do
       it "generates links and reverse references without text diagrams" do
         diagram_file = File.join(Dir.tmpdir, "diagram.html")

spec/lrama/context_spec.rb

@@ -18,6 +18,35 @@
     expect(context.yytranslate[300]).to eq(grammar.find_symbol_by_s_value!(%q('\x41')).number)
   end
 
+  it 'keeps additional aliases as distinct anonymous tokens with valid enums' do
+    grammar = Lrama::Parser.new(<<~'GRAMMAR', "redeclared_alias.y").parse
+      %token <i> PLUS "+"
+      %token PLUS "-"
+      %token PLUS "*"
+      %%
+      start: PLUS "+" "-" "*";
+    GRAMMAR
+    grammar.prepare
+    grammar.validate!
+    states = Lrama::States.new(grammar, Lrama::Tracer.new(Lrama::Logger.new))
+    states.compute
+    context = Lrama::Context.new(states)
+
+    plus, alias_plus, minus, star = grammar.rules.last.rhs
+    expect(plus.alias_name).to eq('"+"')
+    expect(plus.tag.s_value).to eq("<i>")
+    expect(alias_plus).to be(plus)
+    expect(minus).not_to be(plus)
+    expect(minus.tag).to be_nil
+    expect(minus.enum_name).to eq("YYSYMBOL_#{minus.number}_")
+    expect(context.yytranslate[minus.token_id]).to eq(minus.number)
+    expect(star).not_to be(minus)
+    expect(star.tag).to be_nil
+    expect(star.enum_name).to eq("YYSYMBOL_#{star.number}_")
+    expect(context.yytranslate[star.token_id]).to eq(star.number)
+    expect(context.yytokentype.map(&:first)).to eq(["YYEMPTY", "YYEOF", "YYerror", "YYUNDEF", "PLUS"])
+  end
+
   describe "basic" do
     it do
       path = "context/basic.y"

spec/lrama/parser_spec.rb

@@ -799,6 +799,104 @@ def rule_builder_rhs_values(builder)
       ])
     end
 
+    [
+      "%left PLUS\n%token <i> PLUS \"+\"",
+      "%token PLUS \"+\"\n%left <i> PLUS",
+      "%left <i> PLUS\n%token PLUS \"+\"",
+      "%token <i> PLUS \"+\"\n%left PLUS",
+    ].each do |declarations|
+      it "preserves token metadata across declarations: #{declarations.inspect}" do
+        y = <<~GRAMMAR
+          %union { int i; }
+          #{declarations}
+          %type <i> start
+          %%
+          start: PLUS "+" { $$ = $1 + $2; };
+        GRAMMAR
+        grammar = Lrama::Parser.new(y, "redeclared_token.y").parse
+        grammar.prepare
+        grammar.validate!
+
+        token = grammar.find_symbol_by_s_value!("PLUS")
+        rule = grammar.rules.last
+        expect(token.alias_name).to eq('"+"')
+        expect(token.tag).to eq(T::Tag.new(s_value: "<i>"))
+        expect(token.precedence).to eq(Precedence.new(type: :left, precedence: 0))
+        expect(rule.rhs).to eq([token, token])
+        expect(rule.translated_code(grammar)).to eq(" (yyval.i) = (yyvsp[-1].i) + (yyvsp[0].i); ")
+      end
+    end
+
+    [
+      "%token <i> PLUS\n%token <i> PLUS",
+      "%token <i> PLUS\n%token <l> PLUS",
+      "%token <i> PLUS\n%left <i> PLUS",
+      "%left <i> PLUS\n%token <l> PLUS",
+      "%token <i> PLUS 300\n%token <i> PLUS 300",
+    ].each do |declarations|
+      it "rejects a redeclared token type: #{declarations.inspect}" do
+        y = "%union { int i; long l; }\n#{declarations}\n%%\nstart: PLUS;\n"
+
+        expect { Lrama::Parser.new(y, "redeclared_token.y").parse }
+          .to raise_error(/redeclared_token.y:3:\d+: %type redeclaration for PLUS.*previous declaration/m)
+      end
+    end
+
+    ["%token <i> PLUS 300 \"+\"", "%left <i> PLUS\n%token PLUS 300 \"+\""].each do |declaration|
+      it "preserves metadata when repeating a numbered token: #{declaration.inspect}" do
+        y = <<~GRAMMAR
+          %union { int i; }
+          #{declaration}
+          %token PLUS 300
+          %type <i> start
+          %%
+          start: "+" { $$ = $1; };
+        GRAMMAR
+        grammar = Lrama::Parser.new(y, "redeclared_token.y").parse
+        grammar.prepare
+        grammar.validate!
+
+        token = grammar.find_symbol_by_s_value!("PLUS")
+        expect(token.token_id).to eq(300)
+        expect(token.alias_name).to eq('"+"')
+        expect(token.tag).to eq(T::Tag.new(s_value: "<i>"))
+        expect(grammar.rules.last.translated_code(grammar)).to eq(" (yyval.i) = (yyvsp[0].i); ")
+      end
+    end
+
+    it "rejects changing the code of a declared token" do
+      y = "%token PLUS 300\n%token PLUS 301\n%%\nstart: PLUS;\n"
+
+      expect { Lrama::Parser.new(y, "redeclared_token.y").parse }
+        .to raise_error(/redeclared_token.y:2:7: redefining code of token PLUS/)
+    end
+
+    ["", "%token B\n"].each do |declaration|
+      it "attaches an additional literal to a later named token: #{declaration.inspect}" do
+        y = "%token A \"a\"\n%token A \"b\"\n#{declaration}%token B \"b\"\n%%\nstart: B \"b\";\n"
+        grammar = Lrama::Parser.new(y, "redeclared_alias.y").parse
+        grammar.prepare
+        grammar.validate!
+
+        token = grammar.find_symbol_by_s_value!("B")
+        expect(token.alias_name).to eq('"b"')
+        expect(grammar.rules.last.rhs).to eq([token, token])
+        expect(grammar.terms.map(&:name)).to contain_exactly("YYEOF", "YYerror", "YYUNDEF", "A", "B")
+      end
+    end
+
+    it "uses the EOF symbol when an additional literal names token zero" do
+      y = "%token A \"a\"\n%token A \"b\"\n%token END 0 \"b\"\n%%\nstart: A;\n"
+      grammar = Lrama::Parser.new(y, "redeclared_alias.y").parse
+      grammar.prepare
+      grammar.validate!
+
+      expect(grammar.eof_symbol.name).to eq("END")
+      expect(grammar.eof_symbol.alias_name).to eq('"b"')
+      expect(grammar.terms.select { |token| token.token_id == 0 }).to eq([grammar.eof_symbol])
+      expect(grammar.terms.map(&:name)).to contain_exactly("END", "YYerror", "YYUNDEF", "A")
+    end
+
     it "precedence" do
       path = "common/precedence.y"
       y = File.read(fixture_path(path))

spec/lrama/warnings_spec.rb

@@ -2,6 +2,32 @@
 
 RSpec.describe Lrama::Warnings do
   describe "#warn" do
+    [false, true].each do |warnings|
+      {
+        "%token PLUS\n%token PLUS" => ["symbol PLUS redeclared"],
+        "%token PLUS \"+\"\n%token PLUS \"+\"" => ["symbol PLUS redeclared", 'symbol "+" used more than once as a literal string'],
+        "%token PLUS \"+\"\n%token PLUS \"plus\"" => ["symbol PLUS redeclared", "symbol PLUS given more than one literal string"],
+        "%left PLUS\n%token PLUS \"+\"" => [],
+      }.each do |declarations, messages|
+        it "reports token redeclarations with warnings=#{warnings}: #{declarations.inspect}" do
+          grammar = Lrama::Parser.new("#{declarations}\n%%\nstart: PLUS;\n", "redeclared_token.y").parse
+          grammar.prepare
+          grammar.validate!
+          states = Lrama::States.new(grammar, Lrama::Tracer.new(Lrama::Logger.new))
+          states.compute
+          logger = Lrama::Logger.new
+          allow(logger).to receive(:warn)
+
+          Lrama::Warnings.new(logger, warnings).warn(grammar, states)
+
+          messages.each do |message|
+            expect(logger).to have_received(:warn).with(a_string_including("redeclared_token.y:2:", message)).once
+          end
+          expect(logger).not_to have_received(:warn).with(a_string_including("redeclared")) if messages.empty?
+        end
+      end
+    end
+
     context "when rule has conflicts" do
       let(:y) do
         <<~'STR'