diff --git a/lib/lrama/context.rb b/lib/lrama/context.rb index b82ecdc9..eb1781ae 100644 --- a/lib/lrama/context.rb +++ b/lib/lrama/context.rb @@ -26,7 +26,7 @@ def initialize(states) # enum yytokentype def yytokentype @states.terms.reject do |term| - term.id.is_a?(Lexer::Token::Char) + term.id.is_a?(Lexer::Token::Char) || term.id.s_value.start_with?('"') end.map do |term| [term.id.s_value, term.token_id, term.display_name] end.unshift(["YYEMPTY", -2, nil]) diff --git a/lib/lrama/grammar/symbol.rb b/lib/lrama/grammar/symbol.rb index 07aee0c0..3d34b1e6 100644 --- a/lib/lrama/grammar/symbol.rb +++ b/lib/lrama/grammar/symbol.rb @@ -107,7 +107,7 @@ def enum_name res = "YYACCEPT" when eof_symbol? res = "YYEOF" - when term? && id.is_a?(Lrama::Lexer::Token::Char) + when term? && (id.is_a?(Lrama::Lexer::Token::Char) || name.start_with?('"')) res = number.to_s + display_name when term? && id.is_a?(Lrama::Lexer::Token::Ident) res = name diff --git a/lib/lrama/grammar/symbols/resolver.rb b/lib/lrama/grammar/symbols/resolver.rb index 2bcc96d4..61c17ac4 100644 --- a/lib/lrama/grammar/symbols/resolver.rb +++ b/lib/lrama/grammar/symbols/resolver.rb @@ -51,6 +51,41 @@ def sort_by_number! # @rbs (id: Lexer::Token::Base, ?alias_name: String?, ?tag: Lexer::Token::Tag?, ?token_id: Integer?, ?replace: bool) -> Grammar::Symbol def add_term(id:, alias_name: nil, tag: nil, token_id: nil, replace: false) + sym = find_symbol_by_id(id) + if alias_name&.start_with?('"') && (!sym || !sym.alias_name) && (literal = find_term_by_s_value(alias_name)) + if sym || (token_id && find_symbol_by_token_id(token_id)) + @terms.delete(literal) + @symbols = nil + else + sym = literal + sym.id = id + end + end + + if sym + if tag && (previous_tag = sym.tag) + message = "%type redeclaration for #{id.s_value}" + message = id.location&.generate_error_message(message) || message + if previous_tag.location + message += "\n" + previous_tag.location.generate_error_message("previous declaration") + end + raise message + end + + if token_id && sym.token_id && sym.token_id != token_id + message = "redefining code of token #{id.s_value}" + raise(id.location&.generate_error_message(message) || message) + end + + if alias_name && sym.alias_name && sym.alias_name != alias_name + add_term(id: Lexer::Token::Str.new(s_value: alias_name, location: id.location)) + end + sym.alias_name ||= alias_name + sym.tag ||= tag + sym.token_id = token_id if token_id + return sym + end + if token_id && (sym = find_symbol_by_token_id(token_id)) if replace sym.id = id @@ -61,10 +96,6 @@ def add_term(id:, alias_name: nil, tag: nil, token_id: nil, replace: false) return sym end - if (sym = find_symbol_by_id(id)) - return sym - end - @symbols = nil term = Symbol.new( id: id, alias_name: alias_name, number: nil, tag: tag, diff --git a/lib/lrama/warnings.rb b/lib/lrama/warnings.rb index fb7856c8..30085c92 100644 --- a/lib/lrama/warnings.rb +++ b/lib/lrama/warnings.rb @@ -7,6 +7,7 @@ require_relative 'warnings/name_conflicts' require_relative 'warnings/redefined_rules' require_relative 'warnings/required' +require_relative 'warnings/token_redeclarations' require_relative 'warnings/useless_precedence' module Lrama @@ -19,11 +20,13 @@ def initialize(logger, warnings) @name_conflicts = NameConflicts.new(logger, warnings) @redefined_rules = RedefinedRules.new(logger, warnings) @required = Required.new(logger, warnings) + @token_redeclarations = TokenRedeclarations.new(logger) @useless_precedence = UselessPrecedence.new(logger, warnings) end # @rbs (Lrama::Grammar grammar, Lrama::States states) -> void def warn(grammar, states) + @token_redeclarations.warn(grammar) @conflicts.warn(states) @implicit_empty.warn(grammar) @mixed_references.warn(grammar) diff --git a/lib/lrama/warnings/token_redeclarations.rb b/lib/lrama/warnings/token_redeclarations.rb new file mode 100644 index 00000000..4885aa62 --- /dev/null +++ b/lib/lrama/warnings/token_redeclarations.rb @@ -0,0 +1,47 @@ +# rbs_inline: enabled +# frozen_string_literal: true + +module Lrama + class Warnings + class TokenRedeclarations + # @rbs (Lrama::Logger logger) -> void + def initialize(logger) + @logger = logger + end + + # @rbs (Lrama::Grammar grammar) -> void + def warn(grammar) + declarations = {} #: Hash[String, Grammar::Node::Token] + aliases = {} #: Hash[String, String] + + grammar.nodes.each do |node| + next unless node.is_a?(Grammar::Node::TokenDecl) + + node.tokens.each do |token| + name = token.id.s_value + if (previous = declarations[name]) + @logger.warn(token.location.generate_error_message("symbol #{name} redeclared") + "\n" + + previous.location.generate_error_message("previous declaration")) + else + declarations[name] = token + end + + if (alias_name = token.alias_name) + if (previous_alias = aliases[name]) + message = + if previous_alias == alias_name + "symbol #{alias_name} used more than once as a literal string" + else + "symbol #{name} given more than one literal string" + end + @logger.warn(token.location.generate_error_message(message)) + else + aliases[name] = alias_name + end + end + end + end + end + end + end +end diff --git a/sig/generated/lrama/warnings/token_redeclarations.rbs b/sig/generated/lrama/warnings/token_redeclarations.rbs new file mode 100644 index 00000000..f048c361 --- /dev/null +++ b/sig/generated/lrama/warnings/token_redeclarations.rbs @@ -0,0 +1,13 @@ +# Generated from lib/lrama/warnings/token_redeclarations.rb with RBS::Inline + +module Lrama + class Warnings + class TokenRedeclarations + # @rbs (Lrama::Logger logger) -> void + def initialize: (Lrama::Logger logger) -> void + + # @rbs (Lrama::Grammar grammar) -> void + def warn: (Lrama::Grammar grammar) -> void + end + end +end diff --git a/spec/lrama/command_spec.rb b/spec/lrama/command_spec.rb index cacb0163..bca481e5 100644 --- a/spec/lrama/command_spec.rb +++ b/spec/lrama/command_spec.rb @@ -14,6 +14,16 @@ end end + it "warns about redeclared aliases without -W and still generates a parser" do + y = "%token PLUS \"+\"\n%token PLUS \"plus\"\n%%\nstart: PLUS \"+\" \"plus\";\n" + allow(STDIN).to receive(:read).and_return(y) + command = Lrama::Command.new(o_option + ["-", "redeclared_token.y"]) + + expect { command.run }.to output(/symbol PLUS given more than one literal string/).to_stderr_from_any_process + expect(File.read(outfile)).to include("PLUS =") + expect(File.read(outfile)).not_to include('"plus" =') + end + context "when `--diagram` is specified" do it "generates links and reverse references without text diagrams" do diagram_file = File.join(Dir.tmpdir, "diagram.html") diff --git a/spec/lrama/context_spec.rb b/spec/lrama/context_spec.rb index 858a61c3..c8da0472 100644 --- a/spec/lrama/context_spec.rb +++ b/spec/lrama/context_spec.rb @@ -18,6 +18,35 @@ expect(context.yytranslate[300]).to eq(grammar.find_symbol_by_s_value!(%q('\x41')).number) end + it 'keeps additional aliases as distinct anonymous tokens with valid enums' do + grammar = Lrama::Parser.new(<<~'GRAMMAR', "redeclared_alias.y").parse + %token PLUS "+" + %token PLUS "-" + %token PLUS "*" + %% + start: PLUS "+" "-" "*"; + GRAMMAR + grammar.prepare + grammar.validate! + states = Lrama::States.new(grammar, Lrama::Tracer.new(Lrama::Logger.new)) + states.compute + context = Lrama::Context.new(states) + + plus, alias_plus, minus, star = grammar.rules.last.rhs + expect(plus.alias_name).to eq('"+"') + expect(plus.tag.s_value).to eq("") + expect(alias_plus).to be(plus) + expect(minus).not_to be(plus) + expect(minus.tag).to be_nil + expect(minus.enum_name).to eq("YYSYMBOL_#{minus.number}_") + expect(context.yytranslate[minus.token_id]).to eq(minus.number) + expect(star).not_to be(minus) + expect(star.tag).to be_nil + expect(star.enum_name).to eq("YYSYMBOL_#{star.number}_") + expect(context.yytranslate[star.token_id]).to eq(star.number) + expect(context.yytokentype.map(&:first)).to eq(["YYEMPTY", "YYEOF", "YYerror", "YYUNDEF", "PLUS"]) + end + describe "basic" do it do path = "context/basic.y" diff --git a/spec/lrama/parser_spec.rb b/spec/lrama/parser_spec.rb index 48fd69c6..ebd8de80 100644 --- a/spec/lrama/parser_spec.rb +++ b/spec/lrama/parser_spec.rb @@ -799,6 +799,104 @@ def rule_builder_rhs_values(builder) ]) end + [ + "%left PLUS\n%token PLUS \"+\"", + "%token PLUS \"+\"\n%left PLUS", + "%left PLUS\n%token PLUS \"+\"", + "%token PLUS \"+\"\n%left PLUS", + ].each do |declarations| + it "preserves token metadata across declarations: #{declarations.inspect}" do + y = <<~GRAMMAR + %union { int i; } + #{declarations} + %type start + %% + start: PLUS "+" { $$ = $1 + $2; }; + GRAMMAR + grammar = Lrama::Parser.new(y, "redeclared_token.y").parse + grammar.prepare + grammar.validate! + + token = grammar.find_symbol_by_s_value!("PLUS") + rule = grammar.rules.last + expect(token.alias_name).to eq('"+"') + expect(token.tag).to eq(T::Tag.new(s_value: "")) + expect(token.precedence).to eq(Precedence.new(type: :left, precedence: 0)) + expect(rule.rhs).to eq([token, token]) + expect(rule.translated_code(grammar)).to eq(" (yyval.i) = (yyvsp[-1].i) + (yyvsp[0].i); ") + end + end + + [ + "%token PLUS\n%token PLUS", + "%token PLUS\n%token PLUS", + "%token PLUS\n%left PLUS", + "%left PLUS\n%token PLUS", + "%token PLUS 300\n%token PLUS 300", + ].each do |declarations| + it "rejects a redeclared token type: #{declarations.inspect}" do + y = "%union { int i; long l; }\n#{declarations}\n%%\nstart: PLUS;\n" + + expect { Lrama::Parser.new(y, "redeclared_token.y").parse } + .to raise_error(/redeclared_token.y:3:\d+: %type redeclaration for PLUS.*previous declaration/m) + end + end + + ["%token PLUS 300 \"+\"", "%left PLUS\n%token PLUS 300 \"+\""].each do |declaration| + it "preserves metadata when repeating a numbered token: #{declaration.inspect}" do + y = <<~GRAMMAR + %union { int i; } + #{declaration} + %token PLUS 300 + %type start + %% + start: "+" { $$ = $1; }; + GRAMMAR + grammar = Lrama::Parser.new(y, "redeclared_token.y").parse + grammar.prepare + grammar.validate! + + token = grammar.find_symbol_by_s_value!("PLUS") + expect(token.token_id).to eq(300) + expect(token.alias_name).to eq('"+"') + expect(token.tag).to eq(T::Tag.new(s_value: "")) + expect(grammar.rules.last.translated_code(grammar)).to eq(" (yyval.i) = (yyvsp[0].i); ") + end + end + + it "rejects changing the code of a declared token" do + y = "%token PLUS 300\n%token PLUS 301\n%%\nstart: PLUS;\n" + + expect { Lrama::Parser.new(y, "redeclared_token.y").parse } + .to raise_error(/redeclared_token.y:2:7: redefining code of token PLUS/) + end + + ["", "%token B\n"].each do |declaration| + it "attaches an additional literal to a later named token: #{declaration.inspect}" do + y = "%token A \"a\"\n%token A \"b\"\n#{declaration}%token B \"b\"\n%%\nstart: B \"b\";\n" + grammar = Lrama::Parser.new(y, "redeclared_alias.y").parse + grammar.prepare + grammar.validate! + + token = grammar.find_symbol_by_s_value!("B") + expect(token.alias_name).to eq('"b"') + expect(grammar.rules.last.rhs).to eq([token, token]) + expect(grammar.terms.map(&:name)).to contain_exactly("YYEOF", "YYerror", "YYUNDEF", "A", "B") + end + end + + it "uses the EOF symbol when an additional literal names token zero" do + y = "%token A \"a\"\n%token A \"b\"\n%token END 0 \"b\"\n%%\nstart: A;\n" + grammar = Lrama::Parser.new(y, "redeclared_alias.y").parse + grammar.prepare + grammar.validate! + + expect(grammar.eof_symbol.name).to eq("END") + expect(grammar.eof_symbol.alias_name).to eq('"b"') + expect(grammar.terms.select { |token| token.token_id == 0 }).to eq([grammar.eof_symbol]) + expect(grammar.terms.map(&:name)).to contain_exactly("END", "YYerror", "YYUNDEF", "A") + end + it "precedence" do path = "common/precedence.y" y = File.read(fixture_path(path)) diff --git a/spec/lrama/warnings_spec.rb b/spec/lrama/warnings_spec.rb index 8ebe6a6d..91b63854 100644 --- a/spec/lrama/warnings_spec.rb +++ b/spec/lrama/warnings_spec.rb @@ -2,6 +2,32 @@ RSpec.describe Lrama::Warnings do describe "#warn" do + [false, true].each do |warnings| + { + "%token PLUS\n%token PLUS" => ["symbol PLUS redeclared"], + "%token PLUS \"+\"\n%token PLUS \"+\"" => ["symbol PLUS redeclared", 'symbol "+" used more than once as a literal string'], + "%token PLUS \"+\"\n%token PLUS \"plus\"" => ["symbol PLUS redeclared", "symbol PLUS given more than one literal string"], + "%left PLUS\n%token PLUS \"+\"" => [], + }.each do |declarations, messages| + it "reports token redeclarations with warnings=#{warnings}: #{declarations.inspect}" do + grammar = Lrama::Parser.new("#{declarations}\n%%\nstart: PLUS;\n", "redeclared_token.y").parse + grammar.prepare + grammar.validate! + states = Lrama::States.new(grammar, Lrama::Tracer.new(Lrama::Logger.new)) + states.compute + logger = Lrama::Logger.new + allow(logger).to receive(:warn) + + Lrama::Warnings.new(logger, warnings).warn(grammar, states) + + messages.each do |message| + expect(logger).to have_received(:warn).with(a_string_including("redeclared_token.y:2:", message)).once + end + expect(logger).not_to have_received(:warn).with(a_string_including("redeclared")) if messages.empty? + end + end + end + context "when rule has conflicts" do let(:y) do <<~'STR'