From b4259dff3cd8bf7002e772c4e2d3f85f8815b2a1 Mon Sep 17 00:00:00 2001 From: ydah Date: Wed, 8 Jul 2026 21:06:53 +0900 Subject: [PATCH 01/15] pslr: add grammar directives for PSLR(1) lexical specification Introduce the specification language for PSLR(1) pseudo-scanners: * %token-pattern declares a regex pattern for a terminal (with optional tag and alias), registering the terminal on the fly * %lex-prec declares lexical precedence with the seven PSLR operators (<~ <- -~ << -< Array[Grammar::Symbol] # def find_symbol_by_s_value!: (::String s_value) -> Grammar::Symbol # def ielr_defined?: () -> bool + # def pslr_defined?: () -> bool + # def token_patterns: () -> Array[Grammar::TokenPattern] + # def lex_prec: () -> Grammar::LexPrec + # def pslr_max_states: () -> Integer? + # def pslr_max_state_ratio: () -> Float? # end # # include Symbols::Resolver::_DelegatedMethods @@ -68,6 +79,10 @@ class Grammar # @union: Union # @precedences: Array[Precedence] # @start_nterm: Lrama::Lexer::Token::Base? + # @token_patterns: Array[Grammar::TokenPattern] + # @lex_prec: Grammar::LexPrec + # @symbol_sets: Hash[String, Array[Lexer::Token::Base]] + # @lex_tie: Grammar::LexTie extend Forwardable @@ -100,6 +115,18 @@ class Grammar attr_accessor :locations #: bool attr_accessor :define #: Hash[String, String] attr_accessor :required #: bool + attr_reader :token_patterns #: Array[Grammar::TokenPattern] + attr_reader :lex_prec #: Grammar::LexPrec + attr_reader :symbol_sets #: Hash[String, Array[Lexer::Token::Base]] + attr_reader :lex_tie #: Grammar::LexTie + attr_reader :lexer_contexts #: Hash[String, Grammar::LexerContext] + attr_reader :token_actions #: Array[Grammar::TokenAction] + + # Argument symbol names for each parameterized rule expansion. + # @rbs () -> Hash[String, Array[String]] + def parameterized_expansion_args + @parameterized_resolver.expansion_args + end def_delegators "@symbols_resolver", :symbols, :nterms, :terms, :add_nterm, :add_term, :find_term_by_s_value, :find_symbol_by_number!, :find_symbol_by_id!, :token_to_symbol, @@ -133,6 +160,14 @@ def initialize(rule_counter, locations, define = {}) @required = false @precedences = [] @start_nterm = nil + @token_patterns = [] + @lex_prec = Grammar::LexPrec.new + @symbol_sets = {} + @lex_tie = Grammar::LexTie.new + @lexer_contexts = {} + @lexer_context_counter = 0 + @token_pattern_counter = 0 + @token_actions = [] append_special_symbols end @@ -277,6 +312,7 @@ def validate! validate_no_precedence_for_nterm! validate_rule_lhs_is_nterm! validate_duplicated_precedence! + validate_pslr_configuration! end # @rbs (Grammar::Symbol sym) -> Array[Rule] @@ -304,8 +340,360 @@ def ielr_defined? @define.key?('lr.type') && @define['lr.type'] == 'ielr' end + # @rbs () -> bool + def pslr_defined? + @define.key?('lr.type') && @define['lr.type'] == 'pslr' + end + + # @rbs () -> String? + def pslr_state_member + @define['api.pslr.state-member'] + end + + # @rbs () -> Integer? + def pslr_max_states + parse_pslr_positive_integer('pslr.max-states') + end + + # @rbs () -> Float? + def pslr_max_state_ratio + parse_pslr_positive_float('pslr.max-state-ratio') + end + + # @rbs () -> Array[Grammar::TokenPattern] + def layout_token_patterns + @token_patterns.select(&:layout?) + end + + # @rbs () -> Set[String] + def layout_token_names + layout_token_patterns.map(&:name).to_set + end + + # Add a token pattern from %token-pattern directive + # @rbs (id: Lexer::Token::Ident, pattern: Lexer::Token::Regex, ?alias_name: String?, ?tag: Lexer::Token::Tag?, lineno: Integer) -> Grammar::TokenPattern + def add_token_pattern(id:, pattern:, alias_name: nil, tag: nil, lineno:) + token_pattern = Grammar::TokenPattern.new( + id: id, + pattern: pattern, + alias_name: alias_name, + tag: tag, + lineno: lineno, + definition_order: @token_pattern_counter + ) + @token_pattern_counter += 1 + @token_patterns << token_pattern + + # Also register as a terminal symbol + add_term(id: id, alias_name: alias_name, tag: tag) + + token_pattern + end + + # Add a symbol set from %symbol-set directive. + # @rbs (name: String, symbols: Array[Lexer::Token::Base]) -> Array[Lexer::Token::Base] + def add_symbol_set(name:, symbols:) + @symbol_sets[name] = symbols + symbols.each {|id| add_term(id: id) } + end + + # Add lex-prec rules from %lex-prec directive. + # Stores as raw declaration for delayed expansion after implicit literal synthesis. + # @rbs (left_token: Lexer::Token::Base, operator: Symbol, right_token: Lexer::Token::Base, lineno: Integer) -> Grammar::LexPrec::Declaration + def add_lex_prec_rule(left_token:, operator:, right_token:, lineno:) + # Register terminals so they are known to the symbol resolver + add_term(id: left_token) unless left_token.s_value == "yyall" || @symbol_sets.key?(left_token.s_value) + add_term(id: right_token) unless right_token.s_value == "yyall" || @symbol_sets.key?(right_token.s_value) + + @lex_prec.add_declaration( + left_operand: left_token, + operator: operator, + right_operand: right_token, + lineno: lineno + ) + end + + # Finalize lexical declarations after implicit literal synthesis. + # Expands yyall and symbol-set operands using the post-synthesis token universe. + # Validates that identity-component operators are not used as self-pairs. + # @rbs () -> void + def finalize_lexical_declarations! + @lex_prec.declarations.each do |decl| + expand_pslr_operand(decl.left_operand).product(expand_pslr_operand(decl.right_operand)).each do |left, right| + validate_lex_prec_self_pair!(left, right, decl.operator, decl.lineno) + next if left.s_value == right.s_value && Grammar::LexPrec::IDENTITY_OPERATORS.include?(decl.operator) + + @lex_prec.add_rule( + left_token: left, + operator: decl.operator, + right_token: right, + lineno: decl.lineno + ) + end + end + end + + # Add lexical tie relationships from %lex-tie directive. + # @rbs (operands: Array[Lexer::Token::Base]) -> void + def add_lex_tie(operands:) + groups = operands.map {|operand| pslr_operand_group(operand) } + @lex_tie.add_tie_declaration(groups: groups) + end + + # Add no-tie declarations from %lex-no-tie directive. + # @rbs (operands: Array[Lexer::Token::Base]) -> void + def add_lex_no_tie(operands:) + groups = operands.map {|operand| pslr_operand_group(operand) } + @lex_tie.add_no_tie_declaration(groups: groups) + expanded = groups.map {|group| group.names.map {|name| Lrama::Lexer::Token::Ident.new(s_value: name) } } + i = 0 + while i < expanded.size + j = i + 1 + while j < expanded.size + left_group = expanded.fetch(i) + right_group = expanded.fetch(j) + left_group.product(right_group).each do |left, right| + next if left.s_value == right.s_value + + @lex_tie.add_no_tie(left.s_value, right.s_value) + end + j += 1 + end + i += 1 + end + end + + # Add a lexer context from %lexer-context directive + # @rbs (name: String, symbols: Array[Lexer::Token::Ident]) -> Grammar::LexerContext + def add_lexer_context(name:, symbols:) + unless ctx = @lexer_contexts[name] + ctx = Grammar::LexerContext.new(name: name, index: @lexer_context_counter) + @lexer_context_counter += 1 + @lexer_contexts[name] = ctx + end + ctx.add_symbols(symbols) + ctx + end + + # Add a token action from %token-action directive + # @rbs (id: Lexer::Token::Ident, code: Lexer::Token::UserCode, lineno: Integer) -> Grammar::TokenAction + def add_token_action(id:, code:, lineno:) + token_action = Grammar::TokenAction.new( + token_id: id, + code: code, + lineno: lineno + ) + @token_actions << token_action + token_action + end + + # Find a token pattern by its name + # @rbs (String name) -> Grammar::TokenPattern? + def find_token_pattern(name) + @token_patterns.find { |tp| tp.name == name } + end + + # @rbs (Set[String] tokens) -> Set[String] + def expand_lexical_ties(tokens) + tokens.each_with_object(Set.new) do |token, expanded| + @lex_tie.tied_names(token).each {|name| expanded << name } + end + end + + # @rbs (ScannerFSA scanner_fsa) -> void + def finalize_lexical_ties!(scanner_fsa) + @lex_tie.finalize!( + @token_patterns.map(&:name), + scanner_fsa.pairwise_conflict_pairs + ) + end + + REGEX_LITERAL_ESCAPES = ["/", "\\", "*", "+", "?", "(", ")", "[", "]", "{", "}", ".", "|", "^", "$", "-"].freeze #: Array[String] + REGEX_CONTROL_ESCAPES = { + "\n" => "\\n", + "\t" => "\\t", + "\r" => "\\r", + "\f" => "\\f", + "\v" => "\\v" + }.freeze #: Hash[String, String] + + # @rbs () -> void + def synthesize_implicit_literal_token_patterns! + existing_names = @token_patterns.map(&:name).to_set + + terms.each do |term| + pattern = implicit_literal_regex_pattern(term.id) + next unless pattern + next if existing_names.include?(term.id.s_value) + + @token_patterns << Grammar::TokenPattern.new( + id: term.id, + pattern: Lrama::Lexer::Token::Regex.new(s_value: "/#{pattern}/", location: term.id.location), + lineno: token_lineno(term.id), + definition_order: @token_pattern_counter + ) + @token_pattern_counter += 1 + existing_names << term.id.s_value + end + end + private + # Validate that identity-component operators are not applied to self-pairs. + # Self-pair is allowed only for length-only operators (-~ and -s). + # @rbs (Lexer::Token::Base left, Lexer::Token::Base right, Symbol operator, Integer lineno) -> void + def validate_lex_prec_self_pair!(left, right, operator, lineno) + return unless left.s_value == right.s_value + + # Length-only operators are fine for self-pairs (e.g., %lex-prec COM -s COM) + return if operator == Grammar::LexPrec::LONGEST + return if operator == Grammar::LexPrec::SHORTEST + + # Identity-component operators on self-pairs are errors + if Grammar::LexPrec::IDENTITY_OPERATORS.include?(operator) + raise "%lex-prec self-pair identity rule is invalid: #{left.s_value} cannot have an identity conflict with itself (line #{lineno})." + end + + # TOKEN_RIGHT_LENGTH (-<) on self-pair is contradictory + if operator == Grammar::LexPrec::TOKEN_RIGHT_LENGTH + raise "%lex-prec self-pair with -< is contradictory: #{left.s_value} (line #{lineno})." + end + end + + # @rbs (Lexer::Token::Base id) -> String? + def implicit_literal_regex_pattern(id) + case id + when Lrama::Lexer::Token::Char + literal = char_literal_value(id.s_value) + return nil unless literal&.ascii_only? + + escape_regex_literal(literal) + when Lrama::Lexer::Token::Str + literal = str_literal_value(id.s_value) + return nil unless literal + return nil unless literal.ascii_only? + + escape_regex_literal(literal) + end + end + + # Extract the string content from a quoted string literal (e.g., "=>" -> =>) + # @rbs (String s_value) -> String? + def str_literal_value(s_value) + # String literals are stored as "..." with quotes + return nil if s_value.length < 2 + + inner = s_value[1..-2] + return nil if inner.nil? || inner.empty? + + inner + end + + # @rbs (String s_value) -> String? + def char_literal_value(s_value) + inner = s_value[1..-2] + case inner + when "\\b" + "\b" + when "\\f" + "\f" + when "\\n" + "\n" + when "\\r" + "\r" + when "\\t" + "\t" + when "\\v" + "\v" + when "\\\\" + "\\" + when /\A\\(\d+)\z/ + octal = Regexp.last_match(1) + octal ? octal.oct.chr : nil + when /\A.\z/m + inner + end + end + + # @rbs (String literal) -> String + def escape_regex_literal(literal) + escaped = REGEX_CONTROL_ESCAPES[literal] + return escaped if escaped + + literal.each_char.map do |char| + if REGEX_LITERAL_ESCAPES.include?(char) + "\\#{char}" + else + char + end + end.join + end + + # @rbs (Lexer::Token::Base id) -> Integer + def token_lineno(id) + id.location ? id.first_line : 0 + end + + # @rbs () -> void + def validate_pslr_configuration! + return unless pslr_defined? + + member = pslr_state_member + if member && member !~ /\A[a-zA-Z_][a-zA-Z0-9_]*\z/ + raise %(%define api.pslr.state-member must be a valid C identifier, got "#{member}".) + end + + pslr_max_states + pslr_max_state_ratio + end + + # @rbs (Lexer::Token::Base operand) -> Array[Lexer::Token::Base] + def expand_pslr_operand(operand) + return @token_patterns.map(&:id) if operand.s_value == "yyall" + return @symbol_sets.fetch(operand.s_value) if @symbol_sets.key?(operand.s_value) + + add_term(id: operand) + [operand] + end + + # @rbs (Lexer::Token::Base operand) -> Grammar::LexTie::OperandGroup + def pslr_operand_group(operand) + if operand.s_value == "yyall" + Grammar::LexTie::OperandGroup.new(names: @token_patterns.map(&:name), kind: :all) + elsif @symbol_sets.key?(operand.s_value) + Grammar::LexTie::OperandGroup.new(names: @symbol_sets.fetch(operand.s_value).map(&:s_value), kind: :set) + else + add_term(id: operand) + Grammar::LexTie::OperandGroup.new(names: [operand.s_value], kind: :token) + end + end + + # @rbs (String key) -> Integer? + def parse_pslr_positive_integer(key) + value = @define[key] + return nil if value.nil? || value.empty? + + parsed = Integer(value, 10) + raise %(%define #{key} must be greater than 0, got "#{value}".) unless 0 < parsed + + parsed + rescue ArgumentError + raise %(%define #{key} must be an integer, got "#{value}".) + end + + # @rbs (String key) -> Float? + def parse_pslr_positive_float(key) + value = @define[key] + return nil if value.nil? || value.empty? + + parsed = Float(value) + raise %(%define #{key} must be greater than or equal to 1.0, got "#{value}".) unless 1.0 <= parsed + + parsed + rescue ArgumentError + raise %(%define #{key} must be a number, got "#{value}".) + end + # @rbs () -> void def sort_precedence @precedences.sort_by! do |prec| diff --git a/lib/lrama/grammar/lex_prec.rb b/lib/lrama/grammar/lex_prec.rb new file mode 100644 index 000000000..22c787356 --- /dev/null +++ b/lib/lrama/grammar/lex_prec.rb @@ -0,0 +1,200 @@ +# rbs_inline: enabled +# frozen_string_literal: true + +module Lrama + class Grammar + # Represents lexical precedence rules defined by %lex-prec. + # + # Lrama accepts ASCII spellings for the PSLR paper operators: + # <~ identity conflict: right token wins; length conflict: longest wins + # <- identity conflict: right token wins + # -~ length conflict: longest wins + # << identity and length conflicts: right token wins + # -< length conflict: right token wins + # void + def initialize(left_operand:, operator:, right_operand:, lineno:) + @left_operand = left_operand + @operator = operator + @right_operand = right_operand + @lineno = lineno + end + end + + class Rule + attr_reader :left_token #: Lexer::Token::Base + attr_reader :operator #: Symbol + attr_reader :right_token #: Lexer::Token::Base + attr_reader :lineno #: Integer + + # @rbs (left_token: Lexer::Token::Base, operator: Symbol, right_token: Lexer::Token::Base, lineno: Integer) -> void + def initialize(left_token:, operator:, right_token:, lineno:) + @left_token = left_token + @operator = operator + @right_token = right_token + @lineno = lineno + end + + # @rbs () -> String + def left_name + @left_token.s_value + end + + # @rbs () -> String + def right_name + @right_token.s_value + end + end + + attr_reader :rules #: Array[Rule] + attr_reader :declarations #: Array[Declaration] + attr_reader :used_rules #: Set[Integer] + + # @rbs () -> void + def initialize + @rules = [] + @declarations = [] + @used_rules = Set.new + end + + # Mark a rule as used by conflict resolution. + # @rbs (Integer rule_index) -> void + def mark_used(rule_index) + @used_rules << rule_index + end + + # Returns rules that were never used in conflict resolution. + # @rbs () -> Array[Rule] + def useless_rules + @rules.each_with_index.select { |_, i| !@used_rules.include?(i) }.map(&:first) + end + + # Store a raw declaration for delayed expansion. + # @rbs (left_operand: Lexer::Token::Base, operator: Symbol, right_operand: Lexer::Token::Base, lineno: Integer) -> Declaration + def add_declaration(left_operand:, operator:, right_operand:, lineno:) + decl = Declaration.new( + left_operand: left_operand, + operator: operator, + right_operand: right_operand, + lineno: lineno + ) + @declarations << decl + decl + end + + # @rbs (left_token: Lexer::Token::Base, operator: Symbol, right_token: Lexer::Token::Base, lineno: Integer) -> Rule + def add_rule(left_token:, operator:, right_token:, lineno:) + rule = Rule.new( + left_token: left_token, + operator: operator, + right_token: right_token, + lineno: lineno + ) + @rules << rule + rule + end + + # True when winner explicitly wins an identity conflict against loser. + # The relation is intentionally not transitive. + # @rbs (String winner, String loser, ?track: bool) -> bool + def identity_precedes?(winner, loser, track: false) + return true if winner == loser + + @rules.each_with_index.any? do |rule, i| + if IDENTITY_OPERATORS.include?(rule.operator) && + rule.left_name == loser && + rule.right_name == winner + mark_used(i) if track + true + else + false + end + end + end + + # True when rule declares a longest-match length relation for the pair. + # @rbs (String token1, String token2) -> bool + def longest_pair?(token1, token2) + pair_rule?(token1, token2, LONGEST_OPERATORS) + end + + # True when rule declares a shortest-match length relation for the pair. + # @rbs (String token1, String token2) -> bool + def shortest_pair?(token1, token2) + pair_rule?(token1, token2, SHORTEST_OPERATORS) + end + + # Returns the explicit right-token length winner for a pair, if any. + # @rbs (String token1, String token2) -> String? + def right_token_length_winner(token1, token2) + rule = @rules.find do |r| + RIGHT_TOKEN_LENGTH_OPERATORS.include?(r.operator) && + ((r.left_name == token1 && r.right_name == token2) || + (r.left_name == token2 && r.right_name == token1)) + end + + rule&.right_name + end + + private + + # @rbs (String token1, String token2, Array[Symbol] operators) -> bool + def pair_rule?(token1, token2, operators) + @rules.any? do |rule| + operators.include?(rule.operator) && + ((rule.left_name == token1 && rule.right_name == token2) || + (rule.left_name == token2 && rule.right_name == token1)) + end + end + end + end +end diff --git a/lib/lrama/grammar/lex_tie.rb b/lib/lrama/grammar/lex_tie.rb new file mode 100644 index 000000000..b4d89557c --- /dev/null +++ b/lib/lrama/grammar/lex_tie.rb @@ -0,0 +1,290 @@ +# rbs_inline: enabled +# frozen_string_literal: true + +require "set" + +module Lrama + class Grammar + # Stores PSLR lexical ties and explicit no-tie declarations. + # + # Lexical ties expand acc(sp); they never resolve a scanner conflict by + # themselves. Conflict selection is still handled by %lex-prec. + class LexTie + class OperandGroup + attr_reader :names #: Array[String] + attr_reader :kind #: ::Symbol + + # @rbs (names: Array[String], kind: ::Symbol) -> void + def initialize(names:, kind:) + @names = names + @kind = kind + end + end + + class Declaration + attr_reader :kind #: ::Symbol + attr_reader :groups #: Array[OperandGroup] + attr_reader :lineno #: Integer + + # @rbs (kind: ::Symbol, groups: Array[OperandGroup], lineno: Integer) -> void + def initialize(kind:, groups:, lineno:) + @kind = kind + @groups = groups + @lineno = lineno + end + end + + class Decision + attr_reader :kind #: ::Symbol + attr_reader :specificity #: Integer + attr_reader :lineno #: Integer + + # @rbs (kind: ::Symbol, specificity: Integer, lineno: Integer) -> void + def initialize(kind:, specificity:, lineno:) + @kind = kind + @specificity = specificity + @lineno = lineno + end + end + + attr_reader :ties #: Hash[String, Set[String]] + attr_reader :no_ties #: Set[[String, String]] + attr_reader :declarations #: Array[Declaration] + + # @rbs () -> void + def initialize + @ties = Hash.new { |h, k| h[k] = Set.new([k]) } + @no_ties = Set.new + @declarations = [] + end + + # @rbs (String left, String right) -> void + def add_tie(left, right) + left_set = tied_names(left) + right_set = tied_names(right) + merged = left_set | right_set + + merged.each do |name| + @ties[name] = merged.dup + end + end + + # @rbs (String left, String right) -> void + def add_no_tie(left, right) + @no_ties << pair_key(left, right) + end + + # @rbs (groups: Array[OperandGroup], ?lineno: Integer) -> void + def add_tie_declaration(groups:, lineno: 0) + @declarations << Declaration.new(kind: :tie, groups: groups, lineno: lineno) + end + + # @rbs (groups: Array[OperandGroup], ?lineno: Integer) -> void + def add_no_tie_declaration(groups:, lineno: 0) + @declarations << Declaration.new(kind: :no_tie, groups: groups, lineno: lineno) + end + + # @rbs (Array[String] token_names, Set[[String, String]] conflict_pairs) -> void + def finalize!(token_names, conflict_pairs) + decisions = {} #: Hash[[String, String], Decision] + + @declarations.each do |declaration| + declaration_pairs(declaration, token_names, conflict_pairs).each do |pair, specificity| + apply_decision(decisions, pair, Decision.new(kind: declaration.kind, specificity: specificity, lineno: declaration.lineno)) + end + end + + rebuild_relations(token_names, decisions) + end + + # @rbs (String name) -> Set[String] + def tied_names(name) + @ties[name].dup + end + + # @rbs (String left, String right) -> bool + def tied?(left, right) + tied_names(left).include?(right) + end + + # @rbs (String left, String right) -> bool + def no_tie?(left, right) + @no_ties.include?(pair_key(left, right)) + end + + # @rbs () -> Array[[String, String]] + def no_ties_conflicting_with_ties + @no_ties.select do |left, right| + tied?(left, right) + end + end + + private + + # @rbs (Hash[[String, String], Decision] decisions, [String, String] pair, Decision decision) -> void + def apply_decision(decisions, pair, decision) + current = decisions[pair] + if current.nil? || current.specificity < decision.specificity + decisions[pair] = decision + return + end + + if current.specificity == decision.specificity && current.kind != decision.kind + raise "%lex-tie and %lex-no-tie conflict for #{pair.join(' ')}." + end + end + + # @rbs (Declaration declaration, Array[String] token_names, Set[[String, String]] conflict_pairs) -> Array[[[String, String], Integer]] + def declaration_pairs(declaration, token_names, conflict_pairs) + pairs = [] #: Array[[[String, String], Integer]] + + declaration.groups.combination(2) do |left_group, right_group| + next unless left_group && right_group + + specificity = group_specificity(left_group, right_group) + left_names = names_for_group(left_group, token_names) + right_names = names_for_group(right_group, token_names) + + left_names.product(right_names).each do |left, right| + next if left == right + + pair = pair_key(left, right) + if declaration.kind == :tie && specificity < 3 + next unless conflict_pairs.include?(pair) + end + + if declaration.kind == :no_tie && specificity < 3 + next unless conflict_pairs.include?(pair) + end + + pairs << [pair, specificity] + end + end + + pairs + end + + # @rbs (OperandGroup group, Array[String] token_names) -> Array[String] + def names_for_group(group, token_names) + return token_names if group.kind == :all + + group.names + end + + # @rbs (OperandGroup left, OperandGroup right) -> Integer + def group_specificity(left, right) + return 3 if left.kind == :token && right.kind == :token + return 0 if left.kind == :all && right.kind == :all + return 2 if left.kind == :token || right.kind == :token + + 1 + end + + # @rbs (Array[String] token_names, Hash[[String, String], Decision] decisions) -> void + def rebuild_relations(token_names, decisions) + parents = {} #: Hash[String, String] + tie_specificities = {} #: Hash[[String, String], Integer] + token_names.each {|name| parents[name] = name } + + decisions.each do |pair, decision| + next unless decision.kind == :tie + + union(parents, pair[0], pair[1]) + tie_specificities[pair] = decision.specificity + end + + @ties = Hash.new { |h, k| h[k] = Set.new([k]) } + groups = token_names.group_by {|name| root(parents, name) } + groups.each_value do |names| + tied = names.to_set + names.each {|name| @ties[name] = tied.dup } + end + + closure_specificities = {} #: Hash[[String, String], Integer] + groups.each_value do |names| + names.combination(2) do |left, right| + next unless left && right + + pair = pair_key(left, right) + closure_specificities[pair] = tie_specificity_between(left, right, tie_specificities) + end + end + + @no_ties = Set.new + decisions.each do |pair, decision| + next unless decision.kind == :no_tie + + tie_specificity = closure_specificities[pair] + if tie_specificity && decision.specificity >= tie_specificity + raise "%lex-no-tie #{pair[0]} #{pair[1]} conflicts with an existing %lex-tie closure." + end + + @no_ties << pair unless tie_specificity + end + end + + # @rbs (Hash[String, String] parents, String name) -> String + def root(parents, name) + parents[name] ||= name + while parents[name] != name + parents[name] = parents[parents[name]] + name = parents[name] + end + name + end + + # @rbs (Hash[String, String] parents, String left, String right) -> void + def union(parents, left, right) + left_root = root(parents, left) + right_root = root(parents, right) + return if left_root == right_root + + parents[right_root] = left_root + end + + # Compute closure specificity between two tokens via tie graph BFS. + # Path specificity = min(edge specificities on the path). + # Result = max over all paths connecting left and right. + # @rbs (String left, String right, Hash[[String, String], Integer] tie_specificities) -> Integer + def tie_specificity_between(left, right, tie_specificities) + direct = tie_specificities[pair_key(left, right)] + return direct if direct + + # Build adjacency list from tie edges + graph = Hash.new { |h, k| h[k] = [] } #: Hash[String, Array[[String, Integer]]] + tie_specificities.each do |(a, b), specificity| + graph[a] << [b, specificity] + graph[b] << [a, specificity] + end + + return 0 unless graph.key?(left) + + # BFS/Dijkstra-like: find path from left to right maximizing min-edge specificity + # best[node] = best (max) path-min-specificity to reach node from left + best = { left => Float::INFINITY } #: Hash[String, Integer | Float] + queue = [[left, Float::INFINITY]] #: Array[[String, Integer | Float]] + + until queue.empty? + node, path_min = queue.shift + next unless node && path_min + + graph[node].each do |neighbor, edge_spec| + new_min = [path_min, edge_spec].min + if !best.key?(neighbor) || new_min > best[neighbor] + best[neighbor] = new_min + queue << [neighbor, new_min] + end + end + end + + result = best[right] + result && result != Float::INFINITY ? result.to_i : 0 + end + + # @rbs (String left, String right) -> [String, String] + def pair_key(left, right) + left <= right ? [left, right] : [right, left] + end + end + end +end diff --git a/lib/lrama/grammar/lexer_context.rb b/lib/lrama/grammar/lexer_context.rb new file mode 100644 index 000000000..e779899f4 --- /dev/null +++ b/lib/lrama/grammar/lexer_context.rb @@ -0,0 +1,39 @@ +# rbs_inline: enabled +# frozen_string_literal: true + +module Lrama + class Grammar + # Represents a lexer context defined by %lexer-context directive. + # + # Example: + # %lexer-context BEG keyword_if keyword_unless '(' '[' '{' + # + # The bitmask value is automatically assigned by definition order (1 << index). + class LexerContext + attr_reader :name #: String + attr_reader :index #: Integer + attr_reader :symbols #: Array[Lexer::Token::Ident] + + # @rbs (name: String, index: Integer) -> void + def initialize(name:, index:) + @name = name + @index = index + @symbols = [] + end + + # Bitmask value for this context (1 << index). + # @rbs () -> Integer + def bitmask + 1 << @index + end + + # Add symbols that belong to this context. + # @rbs (Array[Lexer::Token::Ident] syms) -> void + def add_symbols(syms) + syms.each do |sym| + @symbols << sym + end + end + end + end +end diff --git a/lib/lrama/grammar/parameterized/resolver.rb b/lib/lrama/grammar/parameterized/resolver.rb index 558f30819..9544a8cbf 100644 --- a/lib/lrama/grammar/parameterized/resolver.rb +++ b/lib/lrama/grammar/parameterized/resolver.rb @@ -7,11 +7,13 @@ class Parameterized class Resolver attr_accessor :rules #: Array[Rule] attr_accessor :created_lhs_list #: Array[Lexer::Token::Base] + attr_reader :expansion_args #: Hash[String, Array[String]] # @rbs () -> void def initialize @rules = [] @created_lhs_list = [] + @expansion_args = {} end # @rbs (Rule rule) -> Array[Rule] @@ -34,6 +36,14 @@ def created_lhs(lhs_s_value) @created_lhs_list.reverse.find { |created_lhs| created_lhs.s_value == lhs_s_value } end + # Register the argument symbol names for a parameterized rule expansion. + # Used by LexerContextClassifier to inherit context from arguments. + # + # @rbs (String lhs_s_value, Array[Lexer::Token::Base] args) -> void + def register_expansion_args(lhs_s_value, args) + @expansion_args[lhs_s_value] = args.map(&:s_value) + end + # @rbs () -> Array[Rule] def redefined_rules @rules.select { |rule| @rules.count { |r| r.name == rule.name && r.required_parameters_count == rule.required_parameters_count } > 1 } diff --git a/lib/lrama/grammar/rule_builder.rb b/lib/lrama/grammar/rule_builder.rb index 34fdca6c8..33bf959d9 100644 --- a/lib/lrama/grammar/rule_builder.rb +++ b/lib/lrama/grammar/rule_builder.rb @@ -157,6 +157,7 @@ def process_rhs lhs_token = Lrama::Lexer::Token::Ident.new(s_value: lhs_s_value, location: token.location) replaced_rhs << lhs_token @parameterized_resolver.created_lhs_list << lhs_token + @parameterized_resolver.register_expansion_args(lhs_s_value, token.args) parameterized_rule.rhs.each do |r| rule_builder = RuleBuilder.new(@rule_counter, @midrule_action_counter, @parameterized_resolver, lhs_tag: token.lhs_tag || parameterized_rule.tag) rule_builder.lhs = lhs_token diff --git a/lib/lrama/grammar/symbols/resolver.rb b/lib/lrama/grammar/symbols/resolver.rb index 085a835d2..72ab17a1c 100644 --- a/lib/lrama/grammar/symbols/resolver.rb +++ b/lib/lrama/grammar/symbols/resolver.rb @@ -52,15 +52,17 @@ def sort_by_number! def add_term(id:, alias_name: nil, tag: nil, token_id: nil, replace: false) if token_id && (sym = find_symbol_by_token_id(token_id)) if replace - sym.id = id - sym.alias_name = alias_name - sym.tag = tag + replace_term_attributes(sym, id: id, alias_name: alias_name, tag: tag, token_id: token_id) end return sym end if (sym = find_symbol_by_id(id)) + if replace + replace_term_attributes(sym, id: id, alias_name: alias_name, tag: tag, token_id: token_id) + end + return sym end @@ -229,6 +231,14 @@ def find_nterm_by_id!(id) end || (raise "Symbol not found. #{id}") end + # @rbs (Grammar::Symbol sym, id: Lexer::Token::Base, ?alias_name: String?, ?tag: Lexer::Token::Tag?, ?token_id: Integer?) -> void + def replace_term_attributes(sym, id:, alias_name: nil, tag: nil, token_id: nil) + sym.id = id + sym.alias_name = alias_name + sym.tag = tag + sym.token_id = token_id if token_id + end + # @rbs () -> void def fill_terms_number # Character literal in grammar file has diff --git a/lib/lrama/grammar/token_action.rb b/lib/lrama/grammar/token_action.rb new file mode 100644 index 000000000..cb3a154f6 --- /dev/null +++ b/lib/lrama/grammar/token_action.rb @@ -0,0 +1,33 @@ +# rbs_inline: enabled +# frozen_string_literal: true + +module Lrama + class Grammar + # Represents a token action defined by %token-action directive. + # + # Token actions are user code blocks associated with token patterns. + # When a token is matched by the pseudo-scanner, the associated code runs. + # Layout tokens are accumulated, and the accumulated text is available + # to the next non-layout token's action. + # + # Example: + # %token-action ID { printf("matched ID: %.*s\n", yyleng, yytext); } + class TokenAction + attr_reader :token_id #: Lexer::Token::Ident + attr_reader :code #: Lexer::Token::UserCode + attr_reader :lineno #: Integer + + # @rbs (token_id: Lexer::Token::Ident, code: Lexer::Token::UserCode, lineno: Integer) -> void + def initialize(token_id:, code:, lineno:) + @token_id = token_id + @code = code + @lineno = lineno + end + + # @rbs () -> String + def token_name + @token_id.s_value + end + end + end +end diff --git a/lib/lrama/grammar/token_pattern.rb b/lib/lrama/grammar/token_pattern.rb new file mode 100644 index 000000000..ff9a834c3 --- /dev/null +++ b/lib/lrama/grammar/token_pattern.rb @@ -0,0 +1,43 @@ +# rbs_inline: enabled +# frozen_string_literal: true + +module Lrama + class Grammar + # Represents a token pattern defined by %token-pattern directive + # Example: %token-pattern RSHIFT />>/ "right shift" + class TokenPattern + attr_reader :id #: Lexer::Token::Base + attr_reader :pattern #: Lexer::Token::Regex + attr_reader :alias_name #: String? + attr_reader :tag #: Lexer::Token::Tag? + attr_reader :lineno #: Integer + attr_reader :definition_order #: Integer + + # @rbs (id: Lexer::Token::Base, pattern: Lexer::Token::Regex, ?alias_name: String?, ?tag: Lexer::Token::Tag?, lineno: Integer, definition_order: Integer) -> void + def initialize(id:, pattern:, alias_name: nil, tag: nil, lineno:, definition_order:) + @id = id + @pattern = pattern + @alias_name = alias_name + @tag = tag + @lineno = lineno + @definition_order = definition_order + end + + # @rbs () -> String + def name + @id.s_value + end + + # Returns the regex pattern string (without slashes) + # @rbs () -> String + def regex_pattern + @pattern.pattern + end + + # @rbs () -> bool + def layout? + name.start_with?("YYLAYOUT") + end + end + end +end diff --git a/lib/lrama/lexer.rb b/lib/lrama/lexer.rb index d9b53810b..8b1c53599 100644 --- a/lib/lrama/lexer.rb +++ b/lib/lrama/lexer.rb @@ -18,7 +18,8 @@ class Lexer # [::Symbol, Token::Char] | # [::Symbol, Token::Str] | # [::Symbol, Token::Int] | - # [::Symbol, Token::Ident] + # [::Symbol, Token::Ident] | + # [::Symbol, Token::Regex] # # type c_token = [:C_DECLARATION, Token::UserCode] @@ -32,6 +33,8 @@ class Lexer PERCENT_TOKENS = %w( %union %token + %token-pattern + %symbol-set %type %nterm %left @@ -43,6 +46,11 @@ class Lexer %printer %destructor %lex-param + %lexer-context + %lex-prec + %lex-tie + %lex-no-tie + %token-action %parse-param %initial-action %precedence @@ -71,6 +79,8 @@ def initialize(grammar_file) @head_line = @line = 1 @status = :initial @end_symbol = nil + @token_pattern_context = false + @expect_token_pattern_regex = false end # @rbs () -> token? @@ -107,7 +117,7 @@ def lex_token @scanner.matched.count("\n").times { newline } when @scanner.scan(/\/\*/) lex_comment - when @scanner.scan(/\/\/.*(?\n)?/) + when !@expect_token_pattern_regex && @scanner.scan(/\/\/.*(?\n)?/) newline if @scanner[:newline] else break @@ -120,8 +130,12 @@ def lex_token when @scanner.eos? return when @scanner.scan(/#{SYMBOLS.join('|')}/) + @token_pattern_context = false if @scanner.matched == "%%" + @expect_token_pattern_regex = false if @scanner.matched == "%%" return [@scanner.matched, Lrama::Lexer::Token::Token.new(s_value: @scanner.matched, location: location)] - when @scanner.scan(/#{PERCENT_TOKENS.join('|')}/) + when @scanner.scan(/#{PERCENT_TOKENS.sort_by { |s| -s.length }.join('|')}/) + @token_pattern_context = @scanner.matched == "%token-pattern" + @expect_token_pattern_regex = false return [@scanner.matched, Lrama::Lexer::Token::Token.new(s_value: @scanner.matched, location: location)] when @scanner.scan(/[\?\+\*]/) return [@scanner.matched, Lrama::Lexer::Token::Token.new(s_value: @scanner.matched, location: location)] @@ -133,10 +147,20 @@ def lex_token return [:CHARACTER, Lrama::Lexer::Token::Char.new(s_value: @scanner.matched, location: location)] when @scanner.scan(/".*?"/) return [:STRING, Lrama::Lexer::Token::Str.new(s_value: %Q(#{@scanner.matched}), location: location)] + when (regex = scan_regex_token) + @expect_token_pattern_regex = false + return [:REGEX, regex] + when @scanner.scan(/<~|<-|-~|<<|-<| Token::Regex? + def scan_regex_token + return nil unless @scanner.peek(1) == "/" + + start_pos = @scanner.pos + @scanner.getch + escaped = false + in_char_class = false + + until @scanner.eos? + char = @scanner.getch + + if escaped + escaped = false + elsif char == "\\" + escaped = true + elsif char == "[" + in_char_class = true + elsif char == "]" + in_char_class = false + elsif char == "/" && !in_char_class + return Lrama::Lexer::Token::Regex.new( + s_value: @scanner.string[start_pos...@scanner.pos], + location: location + ) + elsif char == "\n" + newline + end + end + + raise ParseError, location.generate_error_message("Unclosed regex pattern") # steep:ignore UnknownConstant + end + # @rbs () -> void def reset_first_position @head_line = line diff --git a/lib/lrama/lexer/token.rb b/lib/lrama/lexer/token.rb index 37f77aa06..3932486e0 100644 --- a/lib/lrama/lexer/token.rb +++ b/lib/lrama/lexer/token.rb @@ -7,6 +7,7 @@ require_relative 'token/ident' require_relative 'token/instantiate_rule' require_relative 'token/int' +require_relative 'token/regex' require_relative 'token/str' require_relative 'token/tag' require_relative 'token/token' diff --git a/lib/lrama/lexer/token/regex.rb b/lib/lrama/lexer/token/regex.rb new file mode 100644 index 000000000..c4295f408 --- /dev/null +++ b/lib/lrama/lexer/token/regex.rb @@ -0,0 +1,19 @@ +# rbs_inline: enabled +# frozen_string_literal: true + +module Lrama + class Lexer + module Token + # Token class for regex patterns used in %token-pattern directive + # Example: /[a-zA-Z_][a-zA-Z0-9_]*/ + class Regex < Base + # Returns the regex pattern without the surrounding slashes + # @rbs () -> String + def pattern + # Remove leading and trailing slashes + s_value[1..-2].to_s + end + end + end + end +end diff --git a/lib/lrama/parser.rb b/lib/lrama/parser.rb index 04632cbae..c9ef86185 100644 --- a/lib/lrama/parser.rb +++ b/lib/lrama/parser.rb @@ -655,7 +655,7 @@ def token_to_str(t) module Lrama class Parser < Racc::Parser -module_eval(<<'...end parser.y/module_eval...', 'parser.y', 505) +module_eval(<<'...end parser.y/module_eval...', 'parser.y', 636) include Lrama::Tracer::Duration @@ -745,325 +745,395 @@ def raise_parse_error(error_message, location) ##### State transition tables begin ### racc_action_table = [ - 98, 98, 99, 99, 87, 53, 53, 52, 178, 110, - 110, 97, 53, 53, 184, 178, 110, 110, 53, 181, - 184, 162, 110, 6, 163, 181, 181, 53, 53, 52, - 52, 181, 79, 79, 53, 53, 52, 52, 43, 79, - 79, 53, 4, 52, 5, 110, 88, 94, 182, 125, - 126, 163, 100, 100, 180, 193, 194, 195, 137, 185, - 188, 180, 4, 44, 5, 185, 188, 94, 24, 25, - 26, 27, 28, 29, 30, 31, 32, 46, 33, 34, - 35, 36, 37, 38, 39, 40, 41, 47, 24, 25, - 26, 27, 28, 29, 30, 31, 32, 47, 33, 34, - 35, 36, 37, 38, 39, 40, 41, 12, 13, 50, - 57, 14, 15, 16, 17, 18, 19, 20, 24, 25, - 26, 27, 28, 29, 30, 31, 32, 57, 33, 34, - 35, 36, 37, 38, 39, 40, 41, 12, 13, 57, - 60, 14, 15, 16, 17, 18, 19, 20, 24, 25, - 26, 27, 28, 29, 30, 31, 32, 57, 33, 34, - 35, 36, 37, 38, 39, 40, 41, 53, 53, 52, - 52, 110, 105, 53, 53, 52, 52, 110, 105, 53, - 53, 52, 52, 110, 105, 53, 53, 52, 52, 110, - 105, 53, 53, 52, 52, 110, 110, 53, 53, 52, - 209, 110, 110, 53, 53, 209, 225, 110, 110, 53, - 53, 209, 209, 110, 110, 193, 194, 195, 137, 216, - 222, 232, 217, 217, 217, 235, 57, 53, 217, 52, - 53, 53, 52, 52, 193, 194, 195, 57, 57, 57, - 66, 67, 68, 69, 70, 72, 72, 72, 86, 89, - 47, 57, 57, 113, 117, 117, 79, 123, 124, 131, - 47, 133, 137, 139, 143, 149, 150, 151, 152, 133, - 155, 156, 157, 110, 166, 149, 169, 172, 173, 72, - 175, 176, 183, 189, 166, 196, 137, 200, 202, 137, - 166, 211, 166, 137, 72, 176, 218, 176, 72, 137, - 228, 137, 72, 231, 72 ] + 125, 125, 126, 126, 114, 4, 60, 5, 227, 60, + 96, 233, 124, 96, 60, 6, 227, 60, 96, 233, + 230, 96, 60, 230, 59, 50, 211, 106, 230, 212, + 51, 230, 151, 152, 153, 154, 155, 156, 157, 151, + 152, 153, 154, 155, 156, 157, 60, 60, 59, 59, + 53, 106, 106, 60, 231, 59, 54, 212, 106, 121, + 60, 115, 59, 265, 96, 132, 266, 127, 127, 229, + 271, 234, 237, 266, 60, 54, 59, 229, 96, 234, + 237, 24, 25, 26, 27, 28, 29, 30, 31, 32, + 57, 33, 34, 35, 36, 37, 38, 39, 40, 41, + 42, 43, 44, 45, 46, 47, 121, 60, 4, 59, + 5, 96, 132, 48, 60, 60, 59, 59, 96, 132, + 60, 64, 59, 281, 96, 132, 266, 64, 24, 25, + 26, 27, 28, 29, 30, 31, 32, 64, 33, 34, + 35, 36, 37, 38, 39, 40, 41, 42, 43, 44, + 45, 46, 47, 242, 243, 244, 180, 12, 13, 67, + 48, 14, 15, 16, 17, 18, 19, 20, 24, 25, + 26, 27, 28, 29, 30, 31, 32, 64, 33, 34, + 35, 36, 37, 38, 39, 40, 41, 42, 43, 44, + 45, 46, 47, 242, 243, 244, 180, 12, 13, 64, + 48, 14, 15, 16, 17, 18, 19, 20, 24, 25, + 26, 27, 28, 29, 30, 31, 32, 64, 33, 34, + 35, 36, 37, 38, 39, 40, 41, 42, 43, 44, + 45, 46, 47, 60, 60, 59, 59, 96, 96, 60, + 48, 59, 60, 96, 59, 60, 96, 59, 60, 96, + 59, 60, 96, 59, 60, 96, 59, 60, 96, 59, + 60, 96, 59, 60, 96, 59, 60, 96, 59, 60, + 96, 59, 60, 96, 258, 60, 96, 258, 60, 96, + 274, 60, 96, 258, 60, 96, 258, 284, 96, 60, + 266, 59, 60, 60, 59, 59, 60, 60, 59, 59, + 60, 64, 59, 242, 243, 244, 168, 169, 64, 73, + 74, 75, 76, 77, 79, 79, 86, 88, 90, 79, + 79, 113, 116, 54, 64, 64, 137, 141, 144, 86, + 64, 161, 161, 106, 166, 167, 174, 54, 176, 180, + 182, 186, 144, 144, 190, 197, 198, 199, 200, 176, + 203, 204, 205, 96, 96, 215, 197, 218, 221, 222, + 79, 224, 225, 232, 238, 215, 245, 180, 249, 251, + 180, 215, 260, 215, 180, 79, 225, 267, 225, 79, + 180, 277, 180, 79, 280, 79 ] racc_action_check = [ - 51, 97, 51, 97, 41, 75, 165, 75, 165, 75, - 165, 51, 171, 190, 171, 190, 171, 190, 201, 165, - 201, 148, 201, 1, 148, 171, 190, 36, 37, 36, - 37, 201, 36, 37, 38, 39, 38, 39, 5, 38, - 39, 117, 0, 117, 0, 117, 41, 46, 168, 88, - 88, 168, 51, 97, 165, 177, 177, 177, 177, 171, - 171, 190, 2, 6, 2, 201, 201, 90, 46, 46, - 46, 46, 46, 46, 46, 46, 46, 9, 46, 46, - 46, 46, 46, 46, 46, 46, 46, 10, 90, 90, - 90, 90, 90, 90, 90, 90, 90, 11, 90, 90, - 90, 90, 90, 90, 90, 90, 90, 3, 3, 12, - 14, 3, 3, 3, 3, 3, 3, 3, 3, 3, - 3, 3, 3, 3, 3, 3, 3, 15, 3, 3, - 3, 3, 3, 3, 3, 3, 3, 8, 8, 16, - 17, 8, 8, 8, 8, 8, 8, 8, 8, 8, - 8, 8, 8, 8, 8, 8, 8, 18, 8, 8, - 8, 8, 8, 8, 8, 8, 8, 63, 13, 63, - 13, 63, 63, 64, 73, 64, 73, 64, 64, 65, - 78, 65, 78, 65, 65, 106, 79, 106, 79, 106, - 106, 118, 180, 118, 180, 118, 180, 188, 196, 188, - 196, 188, 196, 202, 217, 202, 217, 202, 217, 218, - 231, 218, 231, 218, 231, 186, 186, 186, 186, 208, - 213, 227, 208, 213, 227, 234, 24, 113, 234, 113, - 114, 123, 114, 123, 210, 210, 210, 25, 26, 27, - 28, 29, 30, 31, 32, 33, 34, 35, 40, 42, - 47, 55, 60, 71, 74, 76, 80, 81, 87, 91, - 92, 93, 94, 102, 116, 124, 125, 126, 127, 133, - 136, 137, 138, 144, 150, 151, 153, 156, 158, 162, - 163, 164, 170, 174, 176, 178, 179, 182, 184, 187, - 189, 199, 200, 204, 205, 207, 209, 212, 214, 216, - 221, 222, 224, 225, 229 ] + 58, 124, 58, 124, 48, 0, 214, 0, 214, 220, + 214, 220, 58, 220, 239, 1, 239, 250, 239, 250, + 214, 250, 43, 220, 43, 5, 196, 43, 239, 196, + 6, 250, 92, 92, 92, 92, 92, 92, 92, 93, + 93, 93, 93, 93, 93, 93, 44, 45, 44, 45, + 9, 44, 45, 46, 217, 46, 10, 217, 46, 53, + 70, 48, 70, 257, 70, 70, 257, 58, 124, 214, + 262, 220, 220, 262, 38, 11, 38, 239, 38, 250, + 250, 53, 53, 53, 53, 53, 53, 53, 53, 53, + 12, 53, 53, 53, 53, 53, 53, 53, 53, 53, + 53, 53, 53, 53, 53, 53, 117, 71, 2, 71, + 2, 71, 71, 53, 72, 13, 72, 13, 72, 72, + 133, 14, 133, 276, 133, 133, 276, 15, 117, 117, + 117, 117, 117, 117, 117, 117, 117, 16, 117, 117, + 117, 117, 117, 117, 117, 117, 117, 117, 117, 117, + 117, 117, 117, 226, 226, 226, 226, 3, 3, 17, + 117, 3, 3, 3, 3, 3, 3, 3, 3, 3, + 3, 3, 3, 3, 3, 3, 3, 18, 3, 3, + 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, + 3, 3, 3, 235, 235, 235, 235, 8, 8, 24, + 3, 8, 8, 8, 8, 8, 8, 8, 8, 8, + 8, 8, 8, 8, 8, 8, 8, 25, 8, 8, + 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, + 8, 8, 8, 39, 40, 39, 40, 39, 40, 88, + 8, 88, 90, 88, 90, 98, 90, 98, 100, 98, + 100, 102, 100, 102, 147, 102, 147, 150, 147, 150, + 158, 150, 158, 161, 158, 161, 229, 161, 229, 237, + 229, 237, 245, 237, 245, 251, 245, 251, 266, 251, + 266, 267, 266, 267, 280, 267, 280, 283, 280, 80, + 283, 80, 105, 106, 105, 106, 137, 138, 137, 138, + 166, 26, 166, 259, 259, 259, 115, 115, 27, 28, + 29, 30, 31, 32, 33, 34, 35, 36, 37, 41, + 42, 47, 49, 54, 62, 67, 78, 81, 82, 84, + 86, 101, 103, 107, 108, 114, 118, 119, 120, 121, + 129, 140, 141, 142, 144, 167, 168, 169, 170, 176, + 179, 180, 181, 187, 190, 198, 199, 201, 204, 206, + 211, 212, 213, 219, 223, 225, 227, 228, 231, 233, + 236, 238, 248, 249, 253, 254, 256, 258, 261, 263, + 265, 270, 271, 273, 274, 278 ] racc_action_pointer = [ - 32, 23, 52, 93, nil, 31, 63, nil, 123, 68, - 74, 84, 103, 165, 94, 111, 123, 135, 141, nil, - nil, nil, nil, nil, 210, 221, 222, 223, 235, 236, - 237, 238, 239, 237, 238, 239, 24, 25, 31, 32, - 243, -1, 247, nil, nil, nil, 43, 237, nil, nil, - nil, -5, nil, nil, nil, 235, nil, nil, nil, nil, - 236, nil, nil, 164, 170, 176, nil, nil, nil, nil, - nil, 245, nil, 171, 246, 2, 247, nil, 177, 183, - 248, 249, nil, nil, nil, nil, nil, 214, 45, nil, - 63, 250, 247, 248, 207, nil, nil, -4, nil, nil, - nil, nil, 261, nil, nil, nil, 182, nil, nil, nil, - nil, nil, nil, 224, 227, nil, 258, 38, 188, nil, - nil, nil, nil, 228, 260, 220, 223, 257, nil, nil, - nil, nil, nil, 256, nil, nil, 224, 266, 255, nil, - nil, nil, nil, nil, 266, nil, nil, nil, -24, nil, - 224, 270, nil, 274, nil, nil, 221, nil, 261, nil, - nil, nil, 271, 275, 232, 3, nil, nil, 3, nil, - 233, 9, nil, nil, 237, nil, 234, 3, 241, 231, - 189, nil, 241, nil, 244, nil, 163, 234, 194, 240, - 10, nil, nil, nil, nil, nil, 195, nil, nil, 289, - 242, 15, 200, nil, 238, 286, nil, 246, 174, 252, - 182, nil, 248, 175, 290, nil, 244, 201, 206, nil, - nil, 283, 246, nil, 294, 259, nil, 176, nil, 296, - nil, 207, nil, nil, 180, nil ] + -6, 15, 97, 142, nil, 18, 30, nil, 182, 40, + 42, 61, 84, 112, 104, 110, 120, 154, 160, nil, + nil, nil, nil, nil, 182, 200, 284, 291, 304, 305, + 306, 307, 308, 306, 307, 311, 312, 313, 71, 230, + 231, 311, 312, 19, 43, 44, 50, 316, -1, 320, + nil, nil, nil, 55, 309, nil, nil, nil, -5, nil, + nil, nil, 307, nil, nil, nil, nil, 308, nil, nil, + 57, 104, 111, nil, nil, nil, nil, nil, 318, nil, + 286, 319, 323, nil, 324, nil, 313, nil, 236, nil, + 239, nil, -19, -12, nil, nil, nil, nil, 242, nil, + 245, 323, 248, 324, nil, 289, 290, 325, 326, nil, + nil, nil, nil, nil, 276, 302, nil, 102, 326, 323, + 324, 269, nil, nil, -4, nil, nil, nil, nil, 338, + nil, nil, nil, 117, nil, nil, nil, 293, 294, nil, + 335, 337, 338, nil, 335, nil, nil, 251, nil, nil, + 254, nil, nil, nil, nil, nil, nil, nil, 257, nil, + nil, 260, nil, nil, nil, nil, 297, 340, 285, 288, + 336, nil, nil, nil, nil, nil, 335, nil, nil, 289, + 346, 334, nil, nil, nil, nil, nil, 346, nil, nil, + 347, nil, nil, nil, nil, nil, -34, nil, 290, 351, + nil, 355, nil, nil, 287, nil, 341, nil, nil, nil, + nil, 352, 356, 298, 3, nil, nil, -6, nil, 299, + 6, nil, nil, 303, nil, 300, 86, 307, 297, 263, + nil, 307, nil, 310, nil, 126, 300, 266, 306, 11, + nil, nil, nil, nil, nil, 269, nil, nil, 370, 308, + 14, 272, nil, 304, 367, nil, 312, 3, 318, 236, + nil, 314, 10, 371, nil, 310, 275, 278, nil, nil, + 363, 312, nil, 375, 325, nil, 63, nil, 377, nil, + 281, nil, nil, 227, nil ] racc_action_default = [ - -1, -137, -1, -3, -10, -137, -137, -2, -3, -137, - -14, -14, -137, -137, -137, -137, -137, -137, -137, -28, - -29, -34, -35, -36, -137, -137, -137, -137, -137, -137, - -137, -137, -137, -54, -54, -54, -137, -137, -137, -137, - -137, -137, -137, -13, 236, -4, -137, -14, -16, -17, - -20, -132, -100, -101, -131, -18, -23, -89, -24, -25, - -137, -27, -37, -137, -137, -137, -41, -42, -43, -44, - -45, -46, -55, -137, -47, -137, -48, -49, -92, -137, - -95, -97, -98, -50, -51, -52, -53, -137, -137, -11, - -5, -7, -14, -137, -72, -15, -21, -132, -133, -134, - -135, -19, -137, -26, -30, -31, -32, -38, -87, -88, - -136, -39, -40, -137, -56, -58, -60, -137, -83, -85, - -93, -94, -96, -137, -137, -137, -137, -137, -6, -8, - -9, -129, -104, -102, -105, -73, -137, -137, -137, -90, - -33, -59, -57, -61, -80, -86, -84, -99, -137, -66, - -70, -137, -12, -137, -103, -109, -137, -22, -137, -62, - -81, -82, -54, -137, -64, -68, -71, -74, -137, -130, - -106, -107, -128, -91, -137, -67, -70, -72, -100, -72, - -137, -125, -137, -109, -100, -110, -72, -72, -137, -70, - -69, -75, -76, -116, -117, -118, -137, -78, -79, -137, - -70, -108, -137, -111, -72, -54, -115, -63, -137, -100, - -119, -126, -65, -137, -54, -114, -72, -137, -137, -120, - -121, -137, -72, -112, -54, -100, -122, -137, -127, -54, - -77, -137, -124, -113, -137, -123 ] + -1, -167, -1, -3, -10, -167, -167, -2, -3, -167, + -14, -14, -167, -167, -167, -167, -167, -167, -167, -28, + -29, -34, -35, -36, -167, -167, -167, -167, -167, -167, + -167, -167, -167, -61, -61, -167, -167, -167, -167, -167, + -167, -61, -61, -167, -167, -167, -167, -167, -167, -167, + -13, 285, -4, -167, -14, -16, -17, -20, -162, -130, + -131, -161, -18, -23, -119, -24, -25, -167, -27, -37, + -167, -167, -167, -41, -42, -43, -44, -45, -46, -62, + -167, -47, -167, -48, -91, -93, -167, -49, -167, -50, + -167, -51, -79, -167, -117, -118, -166, -52, -167, -53, + -167, -54, -167, -55, -56, -122, -167, -125, -127, -128, + -57, -58, -59, -60, -167, -167, -11, -5, -7, -14, + -167, -104, -15, -21, -162, -163, -164, -165, -19, -167, + -26, -30, -31, -32, -38, -39, -40, -167, -63, -65, + -67, -167, -70, -72, -167, -92, -94, -75, -78, -77, + -167, -82, -83, -84, -85, -86, -87, -88, -167, -89, + -90, -167, -115, -123, -124, -126, -167, -167, -167, -167, + -167, -6, -8, -9, -159, -134, -132, -135, -105, -167, + -167, -167, -120, -33, -66, -64, -68, -112, -73, -71, + -112, -76, -81, -80, -116, -129, -167, -98, -102, -167, + -12, -167, -133, -139, -167, -22, -167, -69, -113, -114, + -74, -61, -167, -96, -100, -103, -106, -167, -160, -136, + -137, -158, -121, -167, -99, -102, -104, -130, -104, -167, + -155, -167, -139, -130, -140, -104, -104, -167, -102, -101, + -107, -108, -146, -147, -148, -167, -110, -111, -167, -102, + -138, -167, -141, -104, -61, -145, -95, -167, -130, -149, + -156, -97, -167, -61, -144, -104, -167, -167, -150, -151, + -167, -104, -142, -61, -130, -152, -167, -157, -61, -109, + -167, -154, -143, -167, -153 ] racc_goto_table = [ - 73, 118, 136, 54, 48, 49, 164, 96, 91, 120, - 121, 93, 187, 208, 107, 111, 112, 119, 134, 213, - 56, 58, 59, 171, 61, 1, 78, 78, 78, 78, - 62, 63, 64, 65, 115, 227, 129, 192, 148, 74, - 76, 95, 187, 118, 118, 207, 204, 3, 234, 7, - 130, 201, 128, 138, 147, 93, 212, 140, 154, 145, - 146, 101, 9, 116, 42, 168, 103, 45, 78, 78, - 219, 127, 51, 71, 141, 142, 77, 83, 84, 85, - 159, 144, 190, 160, 161, 191, 132, 197, 102, 158, - 122, 177, 170, 220, 203, 205, 199, 186, 221, 153, - nil, nil, nil, 116, 116, nil, 198, nil, nil, nil, - nil, nil, 214, 78, 206, nil, 177, nil, nil, nil, - nil, nil, 210, nil, 224, nil, nil, 186, 210, 174, - 229, nil, nil, nil, nil, nil, nil, nil, nil, nil, - nil, nil, nil, 226, 210, nil, nil, nil, nil, nil, - nil, nil, nil, nil, nil, nil, nil, 210, nil, nil, + 80, 82, 61, 179, 93, 98, 100, 148, 123, 149, + 55, 56, 118, 120, 163, 164, 68, 159, 85, 160, + 213, 162, 69, 70, 71, 72, 134, 135, 136, 139, + 241, 143, 105, 105, 105, 105, 131, 131, 131, 253, + 177, 196, 9, 1, 63, 65, 66, 52, 236, 3, + 257, 7, 220, 172, 122, 207, 262, 173, 210, 208, + 256, 49, 208, 268, 170, 130, 191, 145, 58, 140, + 78, 261, 276, 217, 181, 195, 171, 120, 236, 81, + 194, 250, 101, 103, 146, 283, 184, 185, 83, 183, + 188, 189, 128, 87, 105, 105, 202, 150, 158, 131, + 104, 110, 111, 112, 89, 91, 97, 99, 240, 187, + 246, 92, 239, 129, 206, 165, 192, 252, 254, 175, + 219, 269, 248, 270, 193, 201, 140, 140, nil, nil, + nil, nil, nil, nil, nil, 263, nil, nil, nil, nil, + nil, nil, nil, nil, nil, nil, nil, 273, nil, nil, + nil, nil, nil, 278, nil, 105, nil, nil, nil, nil, nil, nil, nil, nil, nil, nil, nil, nil, nil, nil, - nil, nil, 215, nil, nil, nil, nil, nil, nil, nil, - nil, 223, nil, nil, nil, nil, nil, nil, nil, nil, - nil, 230, nil, nil, nil, nil, 233 ] + nil, nil, nil, nil, nil, nil, nil, nil, 223, nil, + 226, nil, nil, nil, nil, nil, 235, nil, nil, nil, + nil, nil, nil, nil, nil, 247, nil, nil, nil, nil, + nil, nil, nil, 255, nil, 226, nil, nil, nil, nil, + nil, 259, nil, nil, nil, nil, 235, 259, nil, nil, + nil, 264, nil, nil, nil, nil, nil, nil, nil, nil, + 272, nil, 275, 259, nil, nil, nil, nil, nil, nil, + 279, nil, nil, nil, nil, 282, 259 ] racc_goto_check = [ - 29, 22, 42, 31, 14, 14, 35, 16, 8, 48, - 48, 13, 40, 39, 24, 24, 24, 45, 52, 39, - 18, 18, 18, 54, 17, 1, 31, 31, 31, 31, - 17, 17, 17, 17, 30, 39, 5, 38, 34, 26, - 26, 14, 40, 22, 22, 35, 38, 6, 39, 6, - 9, 54, 8, 16, 48, 13, 35, 24, 52, 45, - 45, 18, 7, 31, 10, 34, 17, 7, 31, 31, - 38, 11, 15, 25, 30, 30, 27, 27, 27, 27, - 32, 33, 36, 43, 44, 42, 14, 42, 46, 47, - 50, 22, 53, 55, 42, 42, 56, 22, 57, 58, - nil, nil, nil, 31, 31, nil, 22, nil, nil, nil, - nil, nil, 42, 31, 22, nil, 22, nil, nil, nil, - nil, nil, 22, nil, 42, nil, nil, 22, 22, 29, - 42, nil, nil, nil, nil, nil, nil, nil, nil, nil, - nil, nil, nil, 22, 22, nil, nil, nil, nil, nil, - nil, nil, nil, nil, nil, nil, nil, 22, nil, nil, + 36, 36, 38, 56, 22, 22, 22, 43, 16, 43, + 14, 14, 8, 13, 61, 61, 17, 43, 47, 43, + 49, 43, 17, 17, 17, 17, 24, 24, 24, 37, + 52, 42, 38, 38, 38, 38, 22, 22, 22, 52, + 65, 48, 7, 1, 18, 18, 18, 7, 54, 6, + 53, 6, 67, 5, 14, 39, 53, 9, 39, 57, + 49, 10, 57, 52, 11, 17, 43, 47, 15, 38, + 25, 49, 53, 48, 16, 61, 8, 13, 54, 26, + 43, 67, 33, 33, 17, 53, 37, 37, 27, 24, + 42, 42, 18, 28, 38, 38, 65, 45, 45, 22, + 34, 34, 34, 34, 29, 30, 31, 32, 56, 40, + 56, 44, 50, 59, 60, 63, 22, 56, 56, 14, + 66, 68, 69, 70, 22, 71, 38, 38, nil, nil, + nil, nil, nil, nil, nil, 56, nil, nil, nil, nil, + nil, nil, nil, nil, nil, nil, nil, 56, nil, nil, + nil, nil, nil, 56, nil, 38, nil, nil, nil, nil, nil, nil, nil, nil, nil, nil, nil, nil, nil, nil, - nil, nil, 29, nil, nil, nil, nil, nil, nil, nil, - nil, 29, nil, nil, nil, nil, nil, nil, nil, nil, - nil, 29, nil, nil, nil, nil, 29 ] + nil, nil, nil, nil, nil, nil, nil, nil, 36, nil, + 22, nil, nil, nil, nil, nil, 22, nil, nil, nil, + nil, nil, nil, nil, nil, 22, nil, nil, nil, nil, + nil, nil, nil, 22, nil, 22, nil, nil, nil, nil, + nil, 22, nil, nil, nil, nil, 22, 22, nil, nil, + nil, 36, nil, nil, nil, nil, nil, nil, nil, nil, + 36, nil, 22, 22, nil, nil, nil, nil, nil, nil, + 36, nil, nil, nil, nil, 36, 22 ] racc_goto_pointer = [ - nil, 25, nil, nil, nil, -55, 47, 59, -38, -41, - 60, -18, nil, -35, -6, 59, -44, 6, 6, nil, - nil, nil, -74, nil, -49, 40, 5, 40, nil, -33, - -39, -10, -64, -35, -86, -144, -94, nil, -140, -183, - -159, nil, -92, -61, -60, -58, 31, -50, -69, nil, - 10, nil, -75, -63, -132, -117, -85, -113, -32 ] + nil, 43, nil, nil, nil, -65, 49, 39, -41, -61, + 57, -52, nil, -40, 0, 55, -50, -2, 30, nil, + nil, nil, -34, nil, -44, 37, 45, 53, 57, 67, + 67, 67, 67, 41, 57, nil, -33, -51, -11, -132, + -31, nil, -51, -81, 73, 5, nil, -17, -126, -178, + -113, nil, -196, -195, -172, nil, -118, -128, nil, 49, + -68, -91, nil, 8, nil, -80, -83, -151, -138, -108, + -137, -49 ] racc_goto_default = [ - nil, nil, 2, 8, 90, nil, nil, nil, nil, nil, - nil, nil, 10, 11, nil, nil, nil, 55, nil, 21, - 22, 23, 104, 106, nil, nil, nil, nil, 114, 75, - nil, 108, nil, nil, nil, nil, 165, 135, nil, nil, - 179, 167, nil, 109, nil, nil, nil, nil, 81, 80, - 82, 92, nil, nil, nil, nil, nil, nil, nil ] + nil, nil, 2, 8, 117, nil, nil, nil, nil, nil, + nil, nil, 10, 11, nil, nil, nil, 62, nil, 21, + 22, 23, 147, 133, nil, nil, nil, nil, nil, nil, + nil, nil, nil, nil, nil, 138, 102, nil, 94, nil, + nil, 142, nil, nil, nil, nil, 84, nil, nil, nil, + 214, 178, nil, nil, 228, 216, nil, 95, 209, nil, + nil, 108, 107, 109, 119, nil, nil, nil, nil, nil, + nil, nil ] racc_reduce_table = [ 0, 0, :racc_error, - 0, 64, :_reduce_1, - 2, 64, :_reduce_2, - 0, 65, :_reduce_3, - 2, 65, :_reduce_4, - 1, 66, :_reduce_5, - 2, 66, :_reduce_6, - 0, 67, :_reduce_none, - 1, 67, :_reduce_none, - 5, 59, :_reduce_none, - 0, 68, :_reduce_10, - 0, 69, :_reduce_11, - 5, 60, :_reduce_12, - 2, 60, :_reduce_13, - 0, 72, :_reduce_14, - 2, 72, :_reduce_15, - 2, 61, :_reduce_none, - 2, 61, :_reduce_none, - 1, 76, :_reduce_18, - 2, 76, :_reduce_19, - 2, 70, :_reduce_20, - 3, 70, :_reduce_21, - 5, 70, :_reduce_22, - 2, 70, :_reduce_none, - 2, 70, :_reduce_24, - 2, 70, :_reduce_25, - 3, 70, :_reduce_26, - 2, 70, :_reduce_27, - 1, 70, :_reduce_28, - 1, 70, :_reduce_29, - 1, 81, :_reduce_30, - 1, 81, :_reduce_31, - 1, 82, :_reduce_32, - 2, 82, :_reduce_33, - 1, 71, :_reduce_none, - 1, 71, :_reduce_none, - 1, 71, :_reduce_none, - 2, 71, :_reduce_37, - 3, 71, :_reduce_38, - 3, 71, :_reduce_39, - 3, 71, :_reduce_40, - 2, 71, :_reduce_41, - 2, 71, :_reduce_42, - 2, 71, :_reduce_43, - 2, 71, :_reduce_44, - 2, 71, :_reduce_45, - 2, 77, :_reduce_none, - 2, 77, :_reduce_47, - 2, 77, :_reduce_48, - 2, 77, :_reduce_49, - 2, 77, :_reduce_50, - 2, 77, :_reduce_51, - 2, 77, :_reduce_52, - 2, 77, :_reduce_53, - 0, 87, :_reduce_none, - 1, 87, :_reduce_none, - 1, 88, :_reduce_56, - 2, 88, :_reduce_57, - 2, 83, :_reduce_58, - 3, 83, :_reduce_59, - 0, 91, :_reduce_none, - 1, 91, :_reduce_none, - 3, 86, :_reduce_62, - 8, 78, :_reduce_63, - 5, 79, :_reduce_64, - 8, 79, :_reduce_65, - 1, 92, :_reduce_66, - 3, 92, :_reduce_67, - 1, 93, :_reduce_68, - 3, 93, :_reduce_69, - 0, 99, :_reduce_none, - 1, 99, :_reduce_none, - 0, 100, :_reduce_none, + 0, 79, :_reduce_1, + 2, 79, :_reduce_2, + 0, 80, :_reduce_3, + 2, 80, :_reduce_4, + 1, 81, :_reduce_5, + 2, 81, :_reduce_6, + 0, 82, :_reduce_none, + 1, 82, :_reduce_none, + 5, 74, :_reduce_none, + 0, 83, :_reduce_10, + 0, 84, :_reduce_11, + 5, 75, :_reduce_12, + 2, 75, :_reduce_13, + 0, 87, :_reduce_14, + 2, 87, :_reduce_15, + 2, 76, :_reduce_none, + 2, 76, :_reduce_none, + 1, 91, :_reduce_18, + 2, 91, :_reduce_19, + 2, 85, :_reduce_20, + 3, 85, :_reduce_21, + 5, 85, :_reduce_22, + 2, 85, :_reduce_none, + 2, 85, :_reduce_24, + 2, 85, :_reduce_25, + 3, 85, :_reduce_26, + 2, 85, :_reduce_27, + 1, 85, :_reduce_28, + 1, 85, :_reduce_29, + 1, 96, :_reduce_30, + 1, 96, :_reduce_31, + 1, 97, :_reduce_32, + 2, 97, :_reduce_33, + 1, 86, :_reduce_none, + 1, 86, :_reduce_none, + 1, 86, :_reduce_none, + 2, 86, :_reduce_37, + 3, 86, :_reduce_38, + 3, 86, :_reduce_39, + 3, 86, :_reduce_40, + 2, 86, :_reduce_41, + 2, 86, :_reduce_42, + 2, 86, :_reduce_43, + 2, 86, :_reduce_44, + 2, 86, :_reduce_45, + 2, 92, :_reduce_none, + 2, 92, :_reduce_none, + 2, 92, :_reduce_none, + 2, 92, :_reduce_none, + 2, 92, :_reduce_none, + 2, 92, :_reduce_none, + 2, 92, :_reduce_none, + 2, 92, :_reduce_none, + 2, 92, :_reduce_54, + 2, 92, :_reduce_55, + 2, 92, :_reduce_56, + 2, 92, :_reduce_57, + 2, 92, :_reduce_58, + 2, 92, :_reduce_59, + 2, 92, :_reduce_60, + 0, 109, :_reduce_none, + 1, 109, :_reduce_none, + 1, 110, :_reduce_63, + 2, 110, :_reduce_64, + 2, 98, :_reduce_65, + 3, 98, :_reduce_66, + 0, 113, :_reduce_none, + 1, 113, :_reduce_none, + 3, 108, :_reduce_69, + 1, 115, :_reduce_70, + 2, 115, :_reduce_71, + 2, 99, :_reduce_72, + 3, 99, :_reduce_73, + 3, 114, :_reduce_74, + 1, 116, :_reduce_75, + 2, 116, :_reduce_76, + 2, 102, :_reduce_77, + 2, 101, :_reduce_78, + 1, 103, :_reduce_79, + 3, 117, :_reduce_80, + 3, 117, :_reduce_81, + 1, 118, :_reduce_82, + 1, 118, :_reduce_83, + 1, 118, :_reduce_84, + 1, 118, :_reduce_85, + 1, 118, :_reduce_86, + 1, 118, :_reduce_87, + 1, 118, :_reduce_88, + 2, 104, :_reduce_89, + 2, 105, :_reduce_90, + 1, 120, :_reduce_91, + 2, 120, :_reduce_92, 1, 100, :_reduce_none, - 1, 94, :_reduce_74, - 3, 94, :_reduce_75, - 3, 94, :_reduce_76, - 7, 94, :_reduce_77, - 3, 94, :_reduce_78, - 3, 94, :_reduce_79, - 0, 102, :_reduce_none, - 1, 102, :_reduce_none, - 1, 90, :_reduce_82, - 1, 103, :_reduce_83, - 2, 103, :_reduce_84, - 2, 84, :_reduce_85, - 3, 84, :_reduce_86, - 1, 80, :_reduce_none, - 1, 80, :_reduce_none, - 0, 104, :_reduce_89, - 0, 105, :_reduce_90, - 5, 75, :_reduce_91, - 1, 106, :_reduce_92, - 2, 106, :_reduce_93, - 2, 107, :_reduce_94, - 1, 108, :_reduce_95, - 2, 108, :_reduce_96, - 1, 85, :_reduce_97, - 1, 85, :_reduce_98, - 3, 85, :_reduce_99, + 2, 119, :_reduce_94, + 8, 93, :_reduce_95, + 5, 94, :_reduce_96, + 8, 94, :_reduce_97, + 1, 121, :_reduce_98, + 3, 121, :_reduce_99, + 1, 122, :_reduce_100, + 3, 122, :_reduce_101, + 0, 128, :_reduce_none, + 1, 128, :_reduce_none, + 0, 129, :_reduce_none, + 1, 129, :_reduce_none, + 1, 123, :_reduce_106, + 3, 123, :_reduce_107, + 3, 123, :_reduce_108, + 7, 123, :_reduce_109, + 3, 123, :_reduce_110, + 3, 123, :_reduce_111, + 0, 131, :_reduce_none, + 1, 131, :_reduce_none, + 1, 112, :_reduce_114, + 2, 106, :_reduce_115, + 3, 106, :_reduce_116, + 1, 95, :_reduce_none, + 1, 95, :_reduce_none, + 0, 132, :_reduce_119, + 0, 133, :_reduce_120, + 5, 90, :_reduce_121, + 1, 134, :_reduce_122, + 2, 134, :_reduce_123, + 2, 135, :_reduce_124, + 1, 136, :_reduce_125, + 2, 136, :_reduce_126, + 1, 107, :_reduce_127, + 1, 107, :_reduce_128, + 3, 107, :_reduce_129, + 1, 111, :_reduce_none, + 1, 111, :_reduce_none, + 1, 138, :_reduce_132, + 2, 138, :_reduce_133, + 2, 77, :_reduce_none, + 2, 77, :_reduce_none, + 4, 137, :_reduce_136, + 1, 139, :_reduce_137, + 3, 139, :_reduce_138, + 0, 140, :_reduce_139, + 2, 140, :_reduce_140, + 3, 140, :_reduce_141, + 5, 140, :_reduce_142, + 7, 140, :_reduce_143, + 4, 140, :_reduce_144, + 3, 140, :_reduce_145, + 1, 125, :_reduce_146, + 1, 125, :_reduce_147, + 1, 125, :_reduce_148, + 0, 141, :_reduce_none, + 1, 141, :_reduce_none, + 2, 126, :_reduce_151, + 3, 126, :_reduce_152, + 6, 126, :_reduce_153, + 4, 126, :_reduce_154, + 0, 142, :_reduce_155, + 0, 143, :_reduce_156, + 5, 127, :_reduce_157, + 3, 124, :_reduce_158, + 0, 144, :_reduce_159, + 3, 78, :_reduce_160, + 1, 88, :_reduce_none, + 0, 89, :_reduce_none, 1, 89, :_reduce_none, 1, 89, :_reduce_none, - 1, 110, :_reduce_102, - 2, 110, :_reduce_103, - 2, 62, :_reduce_none, - 2, 62, :_reduce_none, - 4, 109, :_reduce_106, - 1, 111, :_reduce_107, - 3, 111, :_reduce_108, - 0, 112, :_reduce_109, - 2, 112, :_reduce_110, - 3, 112, :_reduce_111, - 5, 112, :_reduce_112, - 7, 112, :_reduce_113, - 4, 112, :_reduce_114, - 3, 112, :_reduce_115, - 1, 96, :_reduce_116, - 1, 96, :_reduce_117, - 1, 96, :_reduce_118, - 0, 113, :_reduce_none, - 1, 113, :_reduce_none, - 2, 97, :_reduce_121, - 3, 97, :_reduce_122, - 6, 97, :_reduce_123, - 4, 97, :_reduce_124, - 0, 114, :_reduce_125, - 0, 115, :_reduce_126, - 5, 98, :_reduce_127, - 3, 95, :_reduce_128, - 0, 116, :_reduce_129, - 3, 63, :_reduce_130, - 1, 73, :_reduce_none, - 0, 74, :_reduce_none, - 1, 74, :_reduce_none, - 1, 74, :_reduce_none, - 1, 74, :_reduce_none, - 1, 101, :_reduce_136 ] - -racc_reduce_n = 137 - -racc_shift_n = 236 + 1, 89, :_reduce_none, + 1, 130, :_reduce_166 ] + +racc_reduce_n = 167 + +racc_shift_n = 285 racc_token_table = { false => 0, @@ -1075,57 +1145,72 @@ def raise_parse_error(error_message, location) :INTEGER => 6, :STRING => 7, :TAG => 8, - "%%" => 9, - "%{" => 10, - "%}" => 11, - "%require" => 12, - ";" => 13, - "%expect" => 14, - "%define" => 15, - "{" => 16, - "}" => 17, - "%param" => 18, - "%lex-param" => 19, - "%parse-param" => 20, - "%code" => 21, - "%initial-action" => 22, - "%no-stdlib" => 23, - "%locations" => 24, - "%union" => 25, - "%destructor" => 26, - "%printer" => 27, - "%error-token" => 28, - "%after-shift" => 29, - "%before-reduce" => 30, - "%after-reduce" => 31, - "%after-shift-error-token" => 32, - "%after-pop-stack" => 33, - "-temp-group" => 34, - "%token" => 35, - "%type" => 36, - "%nterm" => 37, - "%left" => 38, - "%right" => 39, - "%precedence" => 40, - "%nonassoc" => 41, - "%start" => 42, - "%rule" => 43, - "(" => 44, - ")" => 45, - ":" => 46, - "%inline" => 47, - "," => 48, - "|" => 49, - "%empty" => 50, - "%prec" => 51, - "?" => 52, - "+" => 53, - "*" => 54, - "[" => 55, - "]" => 56, - "{...}" => 57 } - -racc_nt_base = 58 + :REGEX => 9, + "%%" => 10, + "%{" => 11, + "%}" => 12, + "%require" => 13, + ";" => 14, + "%expect" => 15, + "%define" => 16, + "{" => 17, + "}" => 18, + "%param" => 19, + "%lex-param" => 20, + "%parse-param" => 21, + "%code" => 22, + "%initial-action" => 23, + "%no-stdlib" => 24, + "%locations" => 25, + "%union" => 26, + "%destructor" => 27, + "%printer" => 28, + "%error-token" => 29, + "%after-shift" => 30, + "%before-reduce" => 31, + "%after-reduce" => 32, + "%after-shift-error-token" => 33, + "%after-pop-stack" => 34, + "-temp-group" => 35, + "%token" => 36, + "%token-pattern" => 37, + "%token-action" => 38, + "%symbol-set" => 39, + "%lexer-context" => 40, + "%lex-prec" => 41, + "%lex-tie" => 42, + "%lex-no-tie" => 43, + "%type" => 44, + "%nterm" => 45, + "%left" => 46, + "%right" => 47, + "%precedence" => 48, + "%nonassoc" => 49, + "%start" => 50, + "<~" => 51, + "<-" => 52, + "-~" => 53, + "<<" => 54, + "-<" => 55, + " 56, + "-s" => 57, + "%rule" => 58, + "(" => 59, + ")" => 60, + ":" => 61, + "%inline" => 62, + "," => 63, + "|" => 64, + "%empty" => 65, + "%prec" => 66, + "?" => 67, + "+" => 68, + "*" => 69, + "[" => 70, + "]" => 71, + "{...}" => 72 } + +racc_nt_base = 73 racc_use_result_var = true @@ -1156,6 +1241,7 @@ def raise_parse_error(error_message, location) "INTEGER", "STRING", "TAG", + "REGEX", "\"%%\"", "\"%{\"", "\"%}\"", @@ -1183,6 +1269,13 @@ def raise_parse_error(error_message, location) "\"%after-pop-stack\"", "\"-temp-group\"", "\"%token\"", + "\"%token-pattern\"", + "\"%token-action\"", + "\"%symbol-set\"", + "\"%lexer-context\"", + "\"%lex-prec\"", + "\"%lex-tie\"", + "\"%lex-no-tie\"", "\"%type\"", "\"%nterm\"", "\"%left\"", @@ -1190,6 +1283,13 @@ def raise_parse_error(error_message, location) "\"%precedence\"", "\"%nonassoc\"", "\"%start\"", + "\"<~\"", + "\"<-\"", + "\"-~\"", + "\"<<\"", + "\"-<\"", + "\" 1 empties = val[0].rhs.select { |sym| sym.is_a?(Lrama::Lexer::Token::Empty) } empties.each do |empty| @@ -2049,8 +2378,8 @@ def _reduce_107(val, _values, result) end .,., -module_eval(<<'.,.,', 'parser.y', 374) - def _reduce_108(val, _values, result) +module_eval(<<'.,.,', 'parser.y', 505) + def _reduce_138(val, _values, result) builder = val[2] if !builder.line builder.line = @lexer.line - 1 @@ -2061,8 +2390,8 @@ def _reduce_108(val, _values, result) end .,., -module_eval(<<'.,.,', 'parser.y', 384) - def _reduce_109(val, _values, result) +module_eval(<<'.,.,', 'parser.y', 515) + def _reduce_139(val, _values, result) reset_precs result = @grammar.create_rule_builder(@rule_counter, @midrule_action_counter) @@ -2070,8 +2399,8 @@ def _reduce_109(val, _values, result) end .,., -module_eval(<<'.,.,', 'parser.y', 389) - def _reduce_110(val, _values, result) +module_eval(<<'.,.,', 'parser.y', 520) + def _reduce_140(val, _values, result) builder = val[0] builder.add_rhs(Lrama::Lexer::Token::Empty.new(location: @lexer.location)) result = builder @@ -2080,8 +2409,8 @@ def _reduce_110(val, _values, result) end .,., -module_eval(<<'.,.,', 'parser.y', 395) - def _reduce_111(val, _values, result) +module_eval(<<'.,.,', 'parser.y', 526) + def _reduce_141(val, _values, result) on_action_error("intermediate %prec in a rule", val[1]) if @trailing_prec_seen token = val[1] token.alias_name = val[2] @@ -2093,8 +2422,8 @@ def _reduce_111(val, _values, result) end .,., -module_eval(<<'.,.,', 'parser.y', 404) - def _reduce_112(val, _values, result) +module_eval(<<'.,.,', 'parser.y', 535) + def _reduce_142(val, _values, result) on_action_error("intermediate %prec in a rule", val[1]) if @trailing_prec_seen token = Lrama::Lexer::Token::InstantiateRule.new(s_value: val[2], alias_name: val[3], location: @lexer.location, args: [val[1]], lhs_tag: val[4]) builder = val[0] @@ -2106,8 +2435,8 @@ def _reduce_112(val, _values, result) end .,., -module_eval(<<'.,.,', 'parser.y', 413) - def _reduce_113(val, _values, result) +module_eval(<<'.,.,', 'parser.y', 544) + def _reduce_143(val, _values, result) on_action_error("intermediate %prec in a rule", val[1]) if @trailing_prec_seen token = Lrama::Lexer::Token::InstantiateRule.new(s_value: val[1].s_value, alias_name: val[5], location: @lexer.location, args: val[3], lhs_tag: val[6]) builder = val[0] @@ -2119,8 +2448,8 @@ def _reduce_113(val, _values, result) end .,., -module_eval(<<'.,.,', 'parser.y', 422) - def _reduce_114(val, _values, result) +module_eval(<<'.,.,', 'parser.y', 553) + def _reduce_144(val, _values, result) user_code = val[1] user_code.alias_name = val[2] user_code.tag = val[3] @@ -2132,8 +2461,8 @@ def _reduce_114(val, _values, result) end .,., -module_eval(<<'.,.,', 'parser.y', 431) - def _reduce_115(val, _values, result) +module_eval(<<'.,.,', 'parser.y', 562) + def _reduce_145(val, _values, result) on_action_error("multiple %prec in a rule", val[0]) if prec_seen? sym = @grammar.find_symbol_by_id!(val[2]) if val[0].rhs.empty? @@ -2149,33 +2478,33 @@ def _reduce_115(val, _values, result) end .,., -module_eval(<<'.,.,', 'parser.y', 444) - def _reduce_116(val, _values, result) +module_eval(<<'.,.,', 'parser.y', 575) + def _reduce_146(val, _values, result) result = "option" result end .,., -module_eval(<<'.,.,', 'parser.y', 445) - def _reduce_117(val, _values, result) +module_eval(<<'.,.,', 'parser.y', 576) + def _reduce_147(val, _values, result) result = "nonempty_list" result end .,., -module_eval(<<'.,.,', 'parser.y', 446) - def _reduce_118(val, _values, result) +module_eval(<<'.,.,', 'parser.y', 577) + def _reduce_148(val, _values, result) result = "list" result end .,., -# reduce 119 omitted +# reduce 149 omitted -# reduce 120 omitted +# reduce 150 omitted -module_eval(<<'.,.,', 'parser.y', 451) - def _reduce_121(val, _values, result) +module_eval(<<'.,.,', 'parser.y', 582) + def _reduce_151(val, _values, result) result = if val[1] [Lrama::Lexer::Token::InstantiateRule.new(s_value: val[1].s_value, location: @lexer.location, args: val[0])] else @@ -2186,29 +2515,29 @@ def _reduce_121(val, _values, result) end .,., -module_eval(<<'.,.,', 'parser.y', 457) - def _reduce_122(val, _values, result) +module_eval(<<'.,.,', 'parser.y', 588) + def _reduce_152(val, _values, result) result = val[0].append(val[2]) result end .,., -module_eval(<<'.,.,', 'parser.y', 458) - def _reduce_123(val, _values, result) +module_eval(<<'.,.,', 'parser.y', 589) + def _reduce_153(val, _values, result) result = val[0].append(Lrama::Lexer::Token::InstantiateRule.new(s_value: val[2].s_value, location: @lexer.location, args: val[4])) result end .,., -module_eval(<<'.,.,', 'parser.y', 459) - def _reduce_124(val, _values, result) +module_eval(<<'.,.,', 'parser.y', 590) + def _reduce_154(val, _values, result) result = [Lrama::Lexer::Token::InstantiateRule.new(s_value: val[0].s_value, location: @lexer.location, args: val[2])] result end .,., -module_eval(<<'.,.,', 'parser.y', 464) - def _reduce_125(val, _values, result) +module_eval(<<'.,.,', 'parser.y', 595) + def _reduce_155(val, _values, result) if prec_seen? on_action_error("multiple User_code after %prec", val[0]) if @code_after_prec @code_after_prec = true @@ -2219,39 +2548,39 @@ def _reduce_125(val, _values, result) end .,., -module_eval(<<'.,.,', 'parser.y', 472) - def _reduce_126(val, _values, result) +module_eval(<<'.,.,', 'parser.y', 603) + def _reduce_156(val, _values, result) end_c_declaration result end .,., -module_eval(<<'.,.,', 'parser.y', 476) - def _reduce_127(val, _values, result) +module_eval(<<'.,.,', 'parser.y', 607) + def _reduce_157(val, _values, result) result = val[2] result end .,., -module_eval(<<'.,.,', 'parser.y', 479) - def _reduce_128(val, _values, result) +module_eval(<<'.,.,', 'parser.y', 610) + def _reduce_158(val, _values, result) result = val[1].s_value result end .,., -module_eval(<<'.,.,', 'parser.y', 484) - def _reduce_129(val, _values, result) +module_eval(<<'.,.,', 'parser.y', 615) + def _reduce_159(val, _values, result) begin_c_declaration('\Z') result end .,., -module_eval(<<'.,.,', 'parser.y', 488) - def _reduce_130(val, _values, result) +module_eval(<<'.,.,', 'parser.y', 619) + def _reduce_160(val, _values, result) end_c_declaration @grammar.epilogue_first_lineno = val[0].first_line + 1 @grammar.epilogue = val[2].s_value @@ -2260,18 +2589,18 @@ def _reduce_130(val, _values, result) end .,., -# reduce 131 omitted +# reduce 161 omitted -# reduce 132 omitted +# reduce 162 omitted -# reduce 133 omitted +# reduce 163 omitted -# reduce 134 omitted +# reduce 164 omitted -# reduce 135 omitted +# reduce 165 omitted -module_eval(<<'.,.,', 'parser.y', 500) - def _reduce_136(val, _values, result) +module_eval(<<'.,.,', 'parser.y', 631) + def _reduce_166(val, _values, result) result = Lrama::Lexer::Token::Ident.new(s_value: val[0].s_value) result end diff --git a/parser.y b/parser.y index f256d5330..ecf7ec66a 100644 --- a/parser.y +++ b/parser.y @@ -2,7 +2,7 @@ class Lrama::Parser expect 0 error_on_expect_mismatch - token C_DECLARATION CHARACTER IDENT_COLON IDENTIFIER INTEGER STRING TAG + token C_DECLARATION CHARACTER IDENT_COLON IDENTIFIER INTEGER STRING TAG REGEX rule @@ -132,6 +132,13 @@ rule symbol_declaration: "%token" token_declarations + | "%token-pattern" token_pattern_declarations + | "%token-action" token_action_declarations + | "%symbol-set" symbol_set_declaration + | "%lexer-context" lexer_context_declaration + | "%lex-prec" lex_prec_declarations + | "%lex-tie" lex_tie_declaration + | "%lex-no-tie" lex_no_tie_declaration | "%type" symbol_declarations { val[1].each {|hash| @@ -213,6 +220,130 @@ rule token_declaration: id INTEGER? alias { result = val } + token_pattern_declarations: + TAG? token_pattern_declaration+ + { + val[1].each {|decl| + @grammar.add_token_pattern( + id: decl[:id], + pattern: decl[:pattern], + alias_name: decl[:alias], + tag: val[0], + lineno: decl[:id].first_line + ) + } + } + | token_pattern_declarations TAG token_pattern_declaration+ + { + val[2].each {|decl| + @grammar.add_token_pattern( + id: decl[:id], + pattern: decl[:pattern], + alias_name: decl[:alias], + tag: val[1], + lineno: decl[:id].first_line + ) + } + } + + token_pattern_declaration: + IDENTIFIER REGEX alias + { + result = { id: val[0], pattern: val[1], alias: val[2] } + } + + lexer_context_declaration: + IDENTIFIER symbol+ + { + @grammar.add_lexer_context(name: val[0].s_value, symbols: val[1]) + } + + symbol_set_declaration: + IDENTIFIER symbol+ + { + @grammar.add_symbol_set(name: val[0].s_value, symbols: val[1]) + } + + lex_prec_declarations: + lex_prec_chain + { + val[0].each {|rule| + @grammar.add_lex_prec_rule( + left_token: rule[:left], + operator: rule[:op], + right_token: rule[:right], + lineno: rule[:left].first_line + ) + } + } + + lex_prec_chain: + symbol lex_prec_op symbol + { + result = [{ left: val[0], op: val[1], right: val[2] }] + } + | lex_prec_chain lex_prec_op symbol + { + last_right = val[0].last[:right] + result = val[0] + [{ left: last_right, op: val[1], right: val[2] }] + } + + lex_prec_op: + "<~" + { + result = Lrama::Grammar::LexPrec::IDENTITY_RIGHT_LONGEST + } + | "<-" + { + result = Lrama::Grammar::LexPrec::IDENTITY_RIGHT + } + | "-~" + { + result = Lrama::Grammar::LexPrec::LONGEST + } + | "<<" + { + result = Lrama::Grammar::LexPrec::TOKEN_RIGHT + } + | "-<" + { + result = Lrama::Grammar::LexPrec::TOKEN_RIGHT_LENGTH + } + | " Grammar::Symbol def ielr_defined?: () -> bool + + def pslr_defined?: () -> bool + + def token_patterns: () -> Array[Grammar::TokenPattern] + + def lex_prec: () -> Grammar::LexPrec + + def pslr_max_states: () -> Integer? + + def pslr_max_state_ratio: () -> Float? end include Symbols::Resolver::_DelegatedMethods @@ -76,6 +86,14 @@ module Lrama @start_nterm: Lrama::Lexer::Token::Base? + @token_patterns: Array[Grammar::TokenPattern] + + @lex_prec: Grammar::LexPrec + + @symbol_sets: Hash[String, Array[Lexer::Token::Base]] + + @lex_tie: Grammar::LexTie + extend Forwardable attr_reader percent_codes: Array[PercentCode] @@ -136,6 +154,22 @@ module Lrama attr_accessor required: bool + attr_reader token_patterns: Array[Grammar::TokenPattern] + + attr_reader lex_prec: Grammar::LexPrec + + attr_reader symbol_sets: Hash[String, Array[Lexer::Token::Base]] + + attr_reader lex_tie: Grammar::LexTie + + attr_reader lexer_contexts: Hash[String, Grammar::LexerContext] + + attr_reader token_actions: Array[Grammar::TokenAction] + + # Argument symbol names for each parameterized rule expansion. + # @rbs () -> Hash[String, Array[String]] + def parameterized_expansion_args: () -> Hash[String, Array[String]] + # @rbs (Counter rule_counter, bool locations, Hash[String, String] define) -> void def initialize: (Counter rule_counter, bool locations, Hash[String, String] define) -> void @@ -227,8 +261,114 @@ module Lrama # @rbs () -> bool def ielr_defined?: () -> bool + # @rbs () -> bool + def pslr_defined?: () -> bool + + # @rbs () -> String? + def pslr_state_member: () -> String? + + # @rbs () -> Integer? + def pslr_max_states: () -> Integer? + + # @rbs () -> Float? + def pslr_max_state_ratio: () -> Float? + + # @rbs () -> Array[Grammar::TokenPattern] + def layout_token_patterns: () -> Array[Grammar::TokenPattern] + + # @rbs () -> Set[String] + def layout_token_names: () -> Set[String] + + # Add a token pattern from %token-pattern directive + # @rbs (id: Lexer::Token::Ident, pattern: Lexer::Token::Regex, ?alias_name: String?, ?tag: Lexer::Token::Tag?, lineno: Integer) -> Grammar::TokenPattern + def add_token_pattern: (id: Lexer::Token::Ident, pattern: Lexer::Token::Regex, lineno: Integer, ?alias_name: String?, ?tag: Lexer::Token::Tag?) -> Grammar::TokenPattern + + # Add a symbol set from %symbol-set directive. + # @rbs (name: String, symbols: Array[Lexer::Token::Base]) -> Array[Lexer::Token::Base] + def add_symbol_set: (name: String, symbols: Array[Lexer::Token::Base]) -> Array[Lexer::Token::Base] + + # Add lex-prec rules from %lex-prec directive. + # Stores as raw declaration for delayed expansion after implicit literal synthesis. + # @rbs (left_token: Lexer::Token::Base, operator: Symbol, right_token: Lexer::Token::Base, lineno: Integer) -> Grammar::LexPrec::Declaration + def add_lex_prec_rule: (left_token: Lexer::Token::Base, operator: Symbol, right_token: Lexer::Token::Base, lineno: Integer) -> Grammar::LexPrec::Declaration + + # Finalize lexical declarations after implicit literal synthesis. + # Expands yyall and symbol-set operands using the post-synthesis token universe. + # Validates that identity-component operators are not used as self-pairs. + # @rbs () -> void + def finalize_lexical_declarations!: () -> void + + # Add lexical tie relationships from %lex-tie directive. + # @rbs (operands: Array[Lexer::Token::Base]) -> void + def add_lex_tie: (operands: Array[Lexer::Token::Base]) -> void + + # Add no-tie declarations from %lex-no-tie directive. + # @rbs (operands: Array[Lexer::Token::Base]) -> void + def add_lex_no_tie: (operands: Array[Lexer::Token::Base]) -> void + + # Add a lexer context from %lexer-context directive + # @rbs (name: String, symbols: Array[Lexer::Token::Ident]) -> Grammar::LexerContext + def add_lexer_context: (name: String, symbols: Array[Lexer::Token::Ident]) -> Grammar::LexerContext + + # Add a token action from %token-action directive + # @rbs (id: Lexer::Token::Ident, code: Lexer::Token::UserCode, lineno: Integer) -> Grammar::TokenAction + def add_token_action: (id: Lexer::Token::Ident, code: Lexer::Token::UserCode, lineno: Integer) -> Grammar::TokenAction + + # Find a token pattern by its name + # @rbs (String name) -> Grammar::TokenPattern? + def find_token_pattern: (String name) -> Grammar::TokenPattern? + + # @rbs (Set[String] tokens) -> Set[String] + def expand_lexical_ties: (Set[String] tokens) -> Set[String] + + # @rbs (ScannerFSA scanner_fsa) -> void + def finalize_lexical_ties!: (ScannerFSA scanner_fsa) -> void + + REGEX_LITERAL_ESCAPES: Array[String] + + REGEX_CONTROL_ESCAPES: Hash[String, String] + + # @rbs () -> void + def synthesize_implicit_literal_token_patterns!: () -> void + private + # Validate that identity-component operators are not applied to self-pairs. + # Self-pair is allowed only for length-only operators (-~ and -s). + # @rbs (Lexer::Token::Base left, Lexer::Token::Base right, Symbol operator, Integer lineno) -> void + def validate_lex_prec_self_pair!: (Lexer::Token::Base left, Lexer::Token::Base right, Symbol operator, Integer lineno) -> void + + # @rbs (Lexer::Token::Base id) -> String? + def implicit_literal_regex_pattern: (Lexer::Token::Base id) -> String? + + # Extract the string content from a quoted string literal (e.g., "=>" -> =>) + # @rbs (String s_value) -> String? + def str_literal_value: (String s_value) -> String? + + # @rbs (String s_value) -> String? + def char_literal_value: (String s_value) -> String? + + # @rbs (String literal) -> String + def escape_regex_literal: (String literal) -> String + + # @rbs (Lexer::Token::Base id) -> Integer + def token_lineno: (Lexer::Token::Base id) -> Integer + + # @rbs () -> void + def validate_pslr_configuration!: () -> void + + # @rbs (Lexer::Token::Base operand) -> Array[Lexer::Token::Base] + def expand_pslr_operand: (Lexer::Token::Base operand) -> Array[Lexer::Token::Base] + + # @rbs (Lexer::Token::Base operand) -> Grammar::LexTie::OperandGroup + def pslr_operand_group: (Lexer::Token::Base operand) -> Grammar::LexTie::OperandGroup + + # @rbs (String key) -> Integer? + def parse_pslr_positive_integer: (String key) -> Integer? + + # @rbs (String key) -> Float? + def parse_pslr_positive_float: (String key) -> Float? + # @rbs () -> void def sort_precedence: () -> void diff --git a/sig/generated/lrama/grammar/lex_prec.rbs b/sig/generated/lrama/grammar/lex_prec.rbs new file mode 100644 index 000000000..b6e6c5e21 --- /dev/null +++ b/sig/generated/lrama/grammar/lex_prec.rbs @@ -0,0 +1,121 @@ +# Generated from lib/lrama/grammar/lex_prec.rb with RBS::Inline + +module Lrama + class Grammar + # Represents lexical precedence rules defined by %lex-prec. + # + # Lrama accepts ASCII spellings for the PSLR paper operators: + # <~ identity conflict: right token wins; length conflict: longest wins + # <- identity conflict: right token wins + # -~ length conflict: longest wins + # << identity and length conflicts: right token wins + # -< length conflict: right token wins + # void + def initialize: (left_operand: Lexer::Token::Base, operator: Symbol, right_operand: Lexer::Token::Base, lineno: Integer) -> void + end + + class Rule + attr_reader left_token: Lexer::Token::Base + + attr_reader operator: Symbol + + attr_reader right_token: Lexer::Token::Base + + attr_reader lineno: Integer + + # @rbs (left_token: Lexer::Token::Base, operator: Symbol, right_token: Lexer::Token::Base, lineno: Integer) -> void + def initialize: (left_token: Lexer::Token::Base, operator: Symbol, right_token: Lexer::Token::Base, lineno: Integer) -> void + + # @rbs () -> String + def left_name: () -> String + + # @rbs () -> String + def right_name: () -> String + end + + attr_reader rules: Array[Rule] + + attr_reader declarations: Array[Declaration] + + attr_reader used_rules: Set[Integer] + + # @rbs () -> void + def initialize: () -> void + + # Mark a rule as used by conflict resolution. + # @rbs (Integer rule_index) -> void + def mark_used: (Integer rule_index) -> void + + # Returns rules that were never used in conflict resolution. + # @rbs () -> Array[Rule] + def useless_rules: () -> Array[Rule] + + # Store a raw declaration for delayed expansion. + # @rbs (left_operand: Lexer::Token::Base, operator: Symbol, right_operand: Lexer::Token::Base, lineno: Integer) -> Declaration + def add_declaration: (left_operand: Lexer::Token::Base, operator: Symbol, right_operand: Lexer::Token::Base, lineno: Integer) -> Declaration + + # @rbs (left_token: Lexer::Token::Base, operator: Symbol, right_token: Lexer::Token::Base, lineno: Integer) -> Rule + def add_rule: (left_token: Lexer::Token::Base, operator: Symbol, right_token: Lexer::Token::Base, lineno: Integer) -> Rule + + # True when winner explicitly wins an identity conflict against loser. + # The relation is intentionally not transitive. + # @rbs (String winner, String loser, ?track: bool) -> bool + def identity_precedes?: (String winner, String loser, ?track: bool) -> bool + + # True when rule declares a longest-match length relation for the pair. + # @rbs (String token1, String token2) -> bool + def longest_pair?: (String token1, String token2) -> bool + + # True when rule declares a shortest-match length relation for the pair. + # @rbs (String token1, String token2) -> bool + def shortest_pair?: (String token1, String token2) -> bool + + # Returns the explicit right-token length winner for a pair, if any. + # @rbs (String token1, String token2) -> String? + def right_token_length_winner: (String token1, String token2) -> String? + + private + + # @rbs (String token1, String token2, Array[Symbol] operators) -> bool + def pair_rule?: (String token1, String token2, Array[Symbol] operators) -> bool + end + end +end diff --git a/sig/generated/lrama/grammar/lex_tie.rbs b/sig/generated/lrama/grammar/lex_tie.rbs new file mode 100644 index 000000000..7655ca7ba --- /dev/null +++ b/sig/generated/lrama/grammar/lex_tie.rbs @@ -0,0 +1,110 @@ +# Generated from lib/lrama/grammar/lex_tie.rb with RBS::Inline + +module Lrama + class Grammar + # Stores PSLR lexical ties and explicit no-tie declarations. + # + # Lexical ties expand acc(sp); they never resolve a scanner conflict by + # themselves. Conflict selection is still handled by %lex-prec. + class LexTie + class OperandGroup + attr_reader names: Array[String] + + attr_reader kind: ::Symbol + + # @rbs (names: Array[String], kind: ::Symbol) -> void + def initialize: (names: Array[String], kind: ::Symbol) -> void + end + + class Declaration + attr_reader kind: ::Symbol + + attr_reader groups: Array[OperandGroup] + + attr_reader lineno: Integer + + # @rbs (kind: ::Symbol, groups: Array[OperandGroup], lineno: Integer) -> void + def initialize: (kind: ::Symbol, groups: Array[OperandGroup], lineno: Integer) -> void + end + + class Decision + attr_reader kind: ::Symbol + + attr_reader specificity: Integer + + attr_reader lineno: Integer + + # @rbs (kind: ::Symbol, specificity: Integer, lineno: Integer) -> void + def initialize: (kind: ::Symbol, specificity: Integer, lineno: Integer) -> void + end + + attr_reader ties: Hash[String, Set[String]] + + attr_reader no_ties: Set[[ String, String ]] + + attr_reader declarations: Array[Declaration] + + # @rbs () -> void + def initialize: () -> void + + # @rbs (String left, String right) -> void + def add_tie: (String left, String right) -> void + + # @rbs (String left, String right) -> void + def add_no_tie: (String left, String right) -> void + + # @rbs (groups: Array[OperandGroup], ?lineno: Integer) -> void + def add_tie_declaration: (groups: Array[OperandGroup], ?lineno: Integer) -> void + + # @rbs (groups: Array[OperandGroup], ?lineno: Integer) -> void + def add_no_tie_declaration: (groups: Array[OperandGroup], ?lineno: Integer) -> void + + # @rbs (Array[String] token_names, Set[[String, String]] conflict_pairs) -> void + def finalize!: (Array[String] token_names, Set[[ String, String ]] conflict_pairs) -> void + + # @rbs (String name) -> Set[String] + def tied_names: (String name) -> Set[String] + + # @rbs (String left, String right) -> bool + def tied?: (String left, String right) -> bool + + # @rbs (String left, String right) -> bool + def no_tie?: (String left, String right) -> bool + + # @rbs () -> Array[[String, String]] + def no_ties_conflicting_with_ties: () -> Array[[ String, String ]] + + private + + # @rbs (Hash[[String, String], Decision] decisions, [String, String] pair, Decision decision) -> void + def apply_decision: (Hash[[ String, String ], Decision] decisions, [ String, String ] pair, Decision decision) -> void + + # @rbs (Declaration declaration, Array[String] token_names, Set[[String, String]] conflict_pairs) -> Array[[[String, String], Integer]] + def declaration_pairs: (Declaration declaration, Array[String] token_names, Set[[ String, String ]] conflict_pairs) -> Array[[ [ String, String ], Integer ]] + + # @rbs (OperandGroup group, Array[String] token_names) -> Array[String] + def names_for_group: (OperandGroup group, Array[String] token_names) -> Array[String] + + # @rbs (OperandGroup left, OperandGroup right) -> Integer + def group_specificity: (OperandGroup left, OperandGroup right) -> Integer + + # @rbs (Array[String] token_names, Hash[[String, String], Decision] decisions) -> void + def rebuild_relations: (Array[String] token_names, Hash[[ String, String ], Decision] decisions) -> void + + # @rbs (Hash[String, String] parents, String name) -> String + def root: (Hash[String, String] parents, String name) -> String + + # @rbs (Hash[String, String] parents, String left, String right) -> void + def union: (Hash[String, String] parents, String left, String right) -> void + + # Compute closure specificity between two tokens via tie graph BFS. + # Path specificity = min(edge specificities on the path). + # Result = max over all paths connecting left and right. + # @rbs (String left, String right, Hash[[String, String], Integer] tie_specificities) -> Integer + def tie_specificity_between: (String left, String right, Hash[[ String, String ], Integer] tie_specificities) -> Integer + + # @rbs (String left, String right) -> [String, String] + def pair_key: (String left, String right) -> [ String, String ] + end + end +end diff --git a/sig/generated/lrama/grammar/lexer_context.rbs b/sig/generated/lrama/grammar/lexer_context.rbs new file mode 100644 index 000000000..436576d98 --- /dev/null +++ b/sig/generated/lrama/grammar/lexer_context.rbs @@ -0,0 +1,30 @@ +# Generated from lib/lrama/grammar/lexer_context.rb with RBS::Inline + +module Lrama + class Grammar + # Represents a lexer context defined by %lexer-context directive. + # + # Example: + # %lexer-context BEG keyword_if keyword_unless '(' '[' '{' + # + # The bitmask value is automatically assigned by definition order (1 << index). + class LexerContext + attr_reader name: String + + attr_reader index: Integer + + attr_reader symbols: Array[Lexer::Token::Ident] + + # @rbs (name: String, index: Integer) -> void + def initialize: (name: String, index: Integer) -> void + + # Bitmask value for this context (1 << index). + # @rbs () -> Integer + def bitmask: () -> Integer + + # Add symbols that belong to this context. + # @rbs (Array[Lexer::Token::Ident] syms) -> void + def add_symbols: (Array[Lexer::Token::Ident] syms) -> void + end + end +end diff --git a/sig/generated/lrama/grammar/parameterized/resolver.rbs b/sig/generated/lrama/grammar/parameterized/resolver.rbs index d2b07e963..a4d380da7 100644 --- a/sig/generated/lrama/grammar/parameterized/resolver.rbs +++ b/sig/generated/lrama/grammar/parameterized/resolver.rbs @@ -8,6 +8,8 @@ module Lrama attr_accessor created_lhs_list: Array[Lexer::Token::Base] + attr_reader expansion_args: Hash[String, Array[String]] + # @rbs () -> void def initialize: () -> void @@ -23,6 +25,12 @@ module Lrama # @rbs (String lhs_s_value) -> Lexer::Token::Base? def created_lhs: (String lhs_s_value) -> Lexer::Token::Base? + # Register the argument symbol names for a parameterized rule expansion. + # Used by LexerContextClassifier to inherit context from arguments. + # + # @rbs (String lhs_s_value, Array[Lexer::Token::Base] args) -> void + def register_expansion_args: (String lhs_s_value, Array[Lexer::Token::Base] args) -> void + # @rbs () -> Array[Rule] def redefined_rules: () -> Array[Rule] diff --git a/sig/generated/lrama/grammar/symbols/resolver.rbs b/sig/generated/lrama/grammar/symbols/resolver.rbs index 2e5f2ebf7..8c4980b70 100644 --- a/sig/generated/lrama/grammar/symbols/resolver.rbs +++ b/sig/generated/lrama/grammar/symbols/resolver.rbs @@ -108,6 +108,9 @@ module Lrama # @rbs (Lexer::Token::Base id) -> Grammar::Symbol def find_nterm_by_id!: (Lexer::Token::Base id) -> Grammar::Symbol + # @rbs (Grammar::Symbol sym, id: Lexer::Token::Base, ?alias_name: String?, ?tag: Lexer::Token::Tag?, ?token_id: Integer?) -> void + def replace_term_attributes: (Grammar::Symbol sym, id: Lexer::Token::Base, ?alias_name: String?, ?tag: Lexer::Token::Tag?, ?token_id: Integer?) -> void + # @rbs () -> void def fill_terms_number: () -> void diff --git a/sig/generated/lrama/grammar/token_action.rbs b/sig/generated/lrama/grammar/token_action.rbs new file mode 100644 index 000000000..cd208edfd --- /dev/null +++ b/sig/generated/lrama/grammar/token_action.rbs @@ -0,0 +1,28 @@ +# Generated from lib/lrama/grammar/token_action.rb with RBS::Inline + +module Lrama + class Grammar + # Represents a token action defined by %token-action directive. + # + # Token actions are user code blocks associated with token patterns. + # When a token is matched by the pseudo-scanner, the associated code runs. + # Layout tokens are accumulated, and the accumulated text is available + # to the next non-layout token's action. + # + # Example: + # %token-action ID { printf("matched ID: %.*s\n", yyleng, yytext); } + class TokenAction + attr_reader token_id: Lexer::Token::Ident + + attr_reader code: Lexer::Token::UserCode + + attr_reader lineno: Integer + + # @rbs (token_id: Lexer::Token::Ident, code: Lexer::Token::UserCode, lineno: Integer) -> void + def initialize: (token_id: Lexer::Token::Ident, code: Lexer::Token::UserCode, lineno: Integer) -> void + + # @rbs () -> String + def token_name: () -> String + end + end +end diff --git a/sig/generated/lrama/grammar/token_pattern.rbs b/sig/generated/lrama/grammar/token_pattern.rbs new file mode 100644 index 000000000..2e23e14e8 --- /dev/null +++ b/sig/generated/lrama/grammar/token_pattern.rbs @@ -0,0 +1,34 @@ +# Generated from lib/lrama/grammar/token_pattern.rb with RBS::Inline + +module Lrama + class Grammar + # Represents a token pattern defined by %token-pattern directive + # Example: %token-pattern RSHIFT />>/ "right shift" + class TokenPattern + attr_reader id: Lexer::Token::Base + + attr_reader pattern: Lexer::Token::Regex + + attr_reader alias_name: String? + + attr_reader tag: Lexer::Token::Tag? + + attr_reader lineno: Integer + + attr_reader definition_order: Integer + + # @rbs (id: Lexer::Token::Base, pattern: Lexer::Token::Regex, ?alias_name: String?, ?tag: Lexer::Token::Tag?, lineno: Integer, definition_order: Integer) -> void + def initialize: (id: Lexer::Token::Base, pattern: Lexer::Token::Regex, lineno: Integer, definition_order: Integer, ?alias_name: String?, ?tag: Lexer::Token::Tag?) -> void + + # @rbs () -> String + def name: () -> String + + # Returns the regex pattern string (without slashes) + # @rbs () -> String + def regex_pattern: () -> String + + # @rbs () -> bool + def layout?: () -> bool + end + end +end diff --git a/sig/generated/lrama/lexer.rbs b/sig/generated/lrama/lexer.rbs index 232026125..0002648c0 100644 --- a/sig/generated/lrama/lexer.rbs +++ b/sig/generated/lrama/lexer.rbs @@ -4,7 +4,7 @@ module Lrama class Lexer type token = lexer_token | c_token - type lexer_token = [ String, Token::Token ] | [ ::Symbol, Token::Tag ] | [ ::Symbol, Token::Char ] | [ ::Symbol, Token::Str ] | [ ::Symbol, Token::Int ] | [ ::Symbol, Token::Ident ] + type lexer_token = [ String, Token::Token ] | [ ::Symbol, Token::Tag ] | [ ::Symbol, Token::Char ] | [ ::Symbol, Token::Str ] | [ ::Symbol, Token::Int ] | [ ::Symbol, Token::Ident ] | [ ::Symbol, Token::Regex ] type c_token = [ :C_DECLARATION, Token::UserCode ] @@ -45,6 +45,9 @@ module Lrama # @rbs () -> void def lex_comment: () -> void + # @rbs () -> Token::Regex? + def scan_regex_token: () -> Token::Regex? + # @rbs () -> void def reset_first_position: () -> void diff --git a/sig/generated/lrama/lexer/token/regex.rbs b/sig/generated/lrama/lexer/token/regex.rbs new file mode 100644 index 000000000..b832c4be2 --- /dev/null +++ b/sig/generated/lrama/lexer/token/regex.rbs @@ -0,0 +1,15 @@ +# Generated from lib/lrama/lexer/token/regex.rb with RBS::Inline + +module Lrama + class Lexer + module Token + # Token class for regex patterns used in %token-pattern directive + # Example: /[a-zA-Z_][a-zA-Z0-9_]*/ + class Regex < Base + # Returns the regex pattern without the surrounding slashes + # @rbs () -> String + def pattern: () -> String + end + end + end +end diff --git a/spec/lrama/grammar/lex_prec_spec.rb b/spec/lrama/grammar/lex_prec_spec.rb new file mode 100644 index 000000000..8681a3a4f --- /dev/null +++ b/spec/lrama/grammar/lex_prec_spec.rb @@ -0,0 +1,50 @@ +# frozen_string_literal: true + +RSpec.describe Lrama::Grammar::LexPrec do + let(:lex_prec) { Lrama::Grammar::LexPrec.new } + + def ident(name) + Lrama::Lexer::Token::Ident.new(s_value: name) + end + + it "stores identity-right and longest-match rules" do + lex_prec.add_rule( + left_token: ident("ID"), + operator: Lrama::Grammar::LexPrec::IDENTITY_RIGHT_LONGEST, + right_token: ident("IF"), + lineno: 1 + ) + + expect(lex_prec.rules.size).to eq(1) + expect(lex_prec.identity_precedes?("IF", "ID")).to be true + expect(lex_prec.identity_precedes?("ID", "IF")).to be false + expect(lex_prec.longest_pair?("ID", "IF")).to be true + end + + it "does not infer transitive identity precedence" do + lex_prec.add_rule( + left_token: ident("A"), + operator: Lrama::Grammar::LexPrec::IDENTITY_RIGHT, + right_token: ident("B"), + lineno: 1 + ) + lex_prec.add_rule( + left_token: ident("B"), + operator: Lrama::Grammar::LexPrec::IDENTITY_RIGHT, + right_token: ident("C"), + lineno: 1 + ) + + expect(lex_prec.identity_precedes?("B", "A")).to be true + expect(lex_prec.identity_precedes?("C", "B")).to be true + expect(lex_prec.identity_precedes?("C", "A")).to be false + end + + it "separates lexical ties from precedence" do + tie = Lrama::Grammar::LexTie.new + tie.add_tie("ID", "IF") + + expect(tie.tied?("ID", "IF")).to be true + expect(lex_prec.identity_precedes?("IF", "ID")).to be false + end +end diff --git a/spec/lrama/grammar/token_action_spec.rb b/spec/lrama/grammar/token_action_spec.rb new file mode 100644 index 000000000..238d0a2c9 --- /dev/null +++ b/spec/lrama/grammar/token_action_spec.rb @@ -0,0 +1,23 @@ +# frozen_string_literal: true + +RSpec.describe Lrama::Grammar::TokenAction do + def ident(name) + Lrama::Lexer::Token::Ident.new(s_value: name) + end + + def user_code(code) + Lrama::Lexer::Token::UserCode.new(s_value: code) + end + + it "stores token action attributes" do + action = Lrama::Grammar::TokenAction.new( + token_id: ident("ID"), + code: user_code('printf("matched");'), + lineno: 5 + ) + + expect(action.token_name).to eq("ID") + expect(action.code.s_value).to eq('printf("matched");') + expect(action.lineno).to eq(5) + end +end diff --git a/spec/lrama/grammar/token_pattern_spec.rb b/spec/lrama/grammar/token_pattern_spec.rb new file mode 100644 index 000000000..2318865b9 --- /dev/null +++ b/spec/lrama/grammar/token_pattern_spec.rb @@ -0,0 +1,41 @@ +# frozen_string_literal: true + +RSpec.describe Lrama::Grammar::TokenPattern do + it "stores token pattern information" do + id = Lrama::Lexer::Token::Ident.new(s_value: "RSHIFT") + pattern = Lrama::Lexer::Token::Regex.new(s_value: "/>>>/") + + token_pattern = Lrama::Grammar::TokenPattern.new( + id: id, + pattern: pattern, + alias_name: "right shift", + tag: nil, + lineno: 1, + definition_order: 0 + ) + + expect(token_pattern.name).to eq("RSHIFT") + expect(token_pattern.regex_pattern).to eq(">>>") + expect(token_pattern.alias_name).to eq("right shift") + expect(token_pattern.definition_order).to eq(0) + end + + it "recognizes YYLAYOUT-prefixed layout tokens" do + pattern = Lrama::Lexer::Token::Regex.new(s_value: "/[ ]+/") + layout = Lrama::Grammar::TokenPattern.new( + id: Lrama::Lexer::Token::Ident.new(s_value: "YYLAYOUT_COMMENT"), + pattern: pattern, + lineno: 1, + definition_order: 0 + ) + normal = Lrama::Grammar::TokenPattern.new( + id: Lrama::Lexer::Token::Ident.new(s_value: "ID"), + pattern: pattern, + lineno: 1, + definition_order: 1 + ) + + expect(layout).to be_layout + expect(normal).not_to be_layout + end +end diff --git a/spec/lrama/lexer/token/regex_spec.rb b/spec/lrama/lexer/token/regex_spec.rb new file mode 100644 index 000000000..bc4fb38ea --- /dev/null +++ b/spec/lrama/lexer/token/regex_spec.rb @@ -0,0 +1,37 @@ +# frozen_string_literal: true + +RSpec.describe Lrama::Lexer::Token::Regex do + describe "#pattern" do + it "returns the pattern without surrounding slashes" do + regex = Lrama::Lexer::Token::Regex.new(s_value: "/>>>/") + expect(regex.pattern).to eq(">>>") + end + + it "handles character class patterns" do + regex = Lrama::Lexer::Token::Regex.new(s_value: "/[a-zA-Z_][a-zA-Z0-9_]*/") + expect(regex.pattern).to eq("[a-zA-Z_][a-zA-Z0-9_]*") + end + + it "handles escape sequences" do + regex = Lrama::Lexer::Token::Regex.new(s_value: "/\\+/") + expect(regex.pattern).to eq("\\+") + end + + it "handles empty pattern" do + regex = Lrama::Lexer::Token::Regex.new(s_value: "//") + expect(regex.pattern).to eq("") + end + + it "handles single character pattern" do + regex = Lrama::Lexer::Token::Regex.new(s_value: "/>/") + expect(regex.pattern).to eq(">") + end + end + + describe "#s_value" do + it "returns the original value including slashes" do + regex = Lrama::Lexer::Token::Regex.new(s_value: "/>>>/") + expect(regex.s_value).to eq("/>>>/") + end + end +end diff --git a/spec/lrama/parser_spec.rb b/spec/lrama/parser_spec.rb index 6478fef4d..fe102bd42 100644 --- a/spec/lrama/parser_spec.rb +++ b/spec/lrama/parser_spec.rb @@ -3671,7 +3671,7 @@ class : keyword_class { code 1 } tSTRING { code 2 } keyword_end { code 3 } expect(aux.epilogue_first_lineno).to eq(6) expect(aux.epilogue).to eq((<<~CODE).chomp) - + static int yyerror(void) { return 0; } @@ -4622,4 +4622,219 @@ class : keyword_class tSTRING keyword_end end end end + + describe "PSLR directives" do + describe "%token-pattern" do + it "parses a single token pattern" do + y = <<~GRAMMAR + %token-pattern RSHIFT />>/ "right shift" + %% + program: RSHIFT + GRAMMAR + + grammar = Lrama::Parser.new(y, "pslr_test.y").parse + grammar.prepare + grammar.validate! + + expect(grammar.token_patterns.size).to eq(1) + token_pattern = grammar.token_patterns.first + expect(token_pattern.name).to eq("RSHIFT") + expect(token_pattern.regex_pattern).to eq(">>") + expect(token_pattern.alias_name).to eq("\"right shift\"") + end + + it "parses multiple token patterns" do + y = <<~GRAMMAR + %token-pattern RSHIFT />>/ "right shift" + %token-pattern RANGLE />/ "right angle" + %token-pattern LANGLE / RSHIFT />>/ "right shift" + %% + program: RSHIFT + GRAMMAR + + grammar = Lrama::Parser.new(y, "pslr_test.y").parse + grammar.prepare + grammar.validate! + + expect(grammar.token_patterns.size).to eq(1) + token_pattern = grammar.token_patterns.first + expect(token_pattern.tag.s_value).to eq("") + end + + it "parses escaped slashes inside token patterns" do + y = <<~GRAMMAR + %token-pattern SLASH /\\// + %% + program: SLASH + GRAMMAR + + grammar = Lrama::Parser.new(y, "pslr_test.y").parse + grammar.prepare + grammar.validate! + + expect(grammar.token_patterns.first.regex_pattern).to eq("\\/") + end + + it "keeps empty token patterns available for scanner diagnostics" do + y = <<~GRAMMAR + %token-pattern EMPTY // + %% + program: EMPTY + GRAMMAR + + grammar = Lrama::Parser.new(y, "pslr_test.y").parse + grammar.prepare + grammar.validate! + + expect(grammar.token_patterns.first.regex_pattern).to eq("") + end + end + + describe "%lex-prec" do + it "parses shorter priority rule" do + y = <<~GRAMMAR + %token RANGLE RSHIFT + %lex-prec RANGLE -s RSHIFT + %% + program: RANGLE | RSHIFT + GRAMMAR + + grammar = Lrama::Parser.new(y, "pslr_test.y").parse + grammar.prepare + grammar.validate! + + expect(grammar.lex_prec.declarations.size).to eq(1) + grammar.finalize_lexical_declarations! + expect(grammar.lex_prec.shortest_pair?("RANGLE", "RSHIFT")).to be true + end + + it "parses identity-right longest rule" do + y = <<~GRAMMAR + %token IF ID + %lex-prec ID <~ IF + %% + program: IF | ID + GRAMMAR + + grammar = Lrama::Parser.new(y, "pslr_test.y").parse + grammar.prepare + grammar.validate! + + expect(grammar.lex_prec.declarations.size).to eq(1) + grammar.finalize_lexical_declarations! + expect(grammar.lex_prec.identity_precedes?("IF", "ID")).to be true + expect(grammar.lex_prec.longest_pair?("ID", "IF")).to be true + end + + it "parses chained lex-prec rules" do + y = <<~GRAMMAR + %token IF ELSE WHILE ID + %lex-prec ID <- WHILE <- ELSE <- IF + %% + program: IF | ELSE | WHILE | ID + GRAMMAR + + grammar = Lrama::Parser.new(y, "pslr_test.y").parse + grammar.prepare + grammar.validate! + + expect(grammar.lex_prec.declarations.size).to eq(3) + grammar.finalize_lexical_declarations! + expect(grammar.lex_prec.identity_precedes?("WHILE", "ID")).to be true + expect(grammar.lex_prec.identity_precedes?("ELSE", "WHILE")).to be true + expect(grammar.lex_prec.identity_precedes?("IF", "ELSE")).to be true + expect(grammar.lex_prec.identity_precedes?("IF", "ID")).to be false + end + + it "parses symbol sets and lexical ties" do + y = <<~GRAMMAR + %token IF WHILE ID RANGLE RSHIFT + %symbol-set keywords IF WHILE + %lex-tie ID keywords + %lex-no-tie RANGLE RSHIFT + %% + program: IF | WHILE | ID | RANGLE | RSHIFT + GRAMMAR + + grammar = Lrama::Parser.new(y, "pslr_test.y").parse + grammar.prepare + grammar.validate! + + expect(grammar.symbol_sets.fetch("keywords").map(&:s_value)).to eq(["IF", "WHILE"]) + expect(grammar.lex_tie.declarations.map(&:kind)).to eq([:tie, :no_tie]) + expect(grammar.lex_tie.no_tie?("RANGLE", "RSHIFT")).to be true + end + end + + describe "%define lr.type pslr" do + it "recognizes pslr lr.type" do + y = <<~GRAMMAR + %define lr.type pslr + %token ID + %% + program: ID + GRAMMAR + + grammar = Lrama::Parser.new(y, "pslr_test.y").parse + grammar.prepare + grammar.validate! + + expect(grammar.pslr_defined?).to be true + end + end + + describe "%token-action" do + it "parses a single token action" do + y = <<~GRAMMAR + %token ID + %token-action ID { printf("matched ID"); } + %% + program: ID + GRAMMAR + + grammar = Lrama::Parser.new(y, "pslr_test.y").parse + grammar.prepare + grammar.validate! + + expect(grammar.token_actions.size).to eq(1) + action = grammar.token_actions.first + expect(action.token_name).to eq("ID") + expect(action.code.s_value).to include("printf") + end + + it "parses multiple token actions" do + y = <<~GRAMMAR + %token ID NUM + %token-action ID { handle_id(); } NUM { handle_num(); } + %% + program: ID | NUM + GRAMMAR + + grammar = Lrama::Parser.new(y, "pslr_test.y").parse + grammar.prepare + grammar.validate! + + expect(grammar.token_actions.size).to eq(2) + expect(grammar.token_actions.map(&:token_name)).to eq(["ID", "NUM"]) + end + end + + end end From 05b4f5f24afc2f664db0762ca979d2f87d489f9b Mon Sep 17 00:00:00 2001 From: ydah Date: Wed, 8 Jul 2026 21:11:19 +0900 Subject: [PATCH 02/15] pslr: add scanner FSA construction from token patterns Build a single byte-oriented DFA from all %token-pattern regexes via Thompson NFA construction and subset construction (Def 3.2.12/3.2.13). Accepting states keep their full accepting-token sets; identity and length conflicts are intentionally left unresolved inside the DFA so that the parser-state-aware resolver can decide them later (paper section 3.2.6). The regex dialect is a small ASCII subset: literals, escapes, alternation, grouping, quantifiers, character classes with ranges and negation, and dot. Patterns are validated at construction time: nullable patterns, empty patterns/groups/alternatives, dangling escapes, malformed classes and unsupported escapes are rejected (lexemes must be non-empty per Def 2.2.2). ScannerFSA#pairwise_conflict_pairs precomputes the token pairs that can actually be in an identity or length scanner conflict; the set is used to filter %lex-tie expansion and later pair-based analyses. --- lib/lrama.rb | 1 + lib/lrama/scanner_fsa.rb | 692 ++++++++++++++++++++++++++++ sig/generated/lrama/scanner_fsa.rbs | 197 ++++++++ spec/lrama/grammar_spec.rb | 242 ++++++++++ spec/lrama/scanner_fsa_spec.rb | 322 +++++++++++++ 5 files changed, 1454 insertions(+) create mode 100644 lib/lrama/scanner_fsa.rb create mode 100644 sig/generated/lrama/scanner_fsa.rbs create mode 100644 spec/lrama/scanner_fsa_spec.rb diff --git a/lib/lrama.rb b/lib/lrama.rb index 56ba0044d..6cffd7ddc 100644 --- a/lib/lrama.rb +++ b/lib/lrama.rb @@ -15,6 +15,7 @@ require_relative "lrama/output" require_relative "lrama/parser" require_relative "lrama/reporter" +require_relative "lrama/scanner_fsa" require_relative "lrama/state" require_relative "lrama/states" require_relative "lrama/tracer" diff --git a/lib/lrama/scanner_fsa.rb b/lib/lrama/scanner_fsa.rb new file mode 100644 index 000000000..1ed327534 --- /dev/null +++ b/lib/lrama/scanner_fsa.rb @@ -0,0 +1,692 @@ +# rbs_inline: enabled +# frozen_string_literal: true + +require "set" + +module Lrama + # Scanner Finite State Automaton for PSLR(1) + # Built from token patterns defined by %token-pattern directives + # Based on Definitions 3.2.12, 3.2.13 from the PSLR dissertation + class ScannerFSA + # Represents a state in the scanner FSA + class State + attr_reader :id #: Integer + attr_reader :transitions #: Hash[String, Integer] + attr_reader :accepting_tokens #: Array[Grammar::TokenPattern] + + # @rbs (Integer id) -> void + def initialize(id) + @id = id + @transitions = {} + @accepting_tokens = [] + end + + # @rbs () -> bool + def accepting? + !@accepting_tokens.empty? + end + + # @rbs (String char, Integer target_state_id) -> void + def add_transition(char, target_state_id) + @transitions[char] = target_state_id + end + + # @rbs (Grammar::TokenPattern token_pattern) -> void + def add_accepting_token(token_pattern) + @accepting_tokens << token_pattern + end + end + + attr_reader :states #: Array[State] + attr_reader :initial_state #: State + attr_reader :token_patterns #: Array[Grammar::TokenPattern] + + # @rbs (Array[Grammar::TokenPattern] token_patterns) -> void + def initialize(token_patterns) + @token_patterns = token_patterns + @states = [] + @state_counter = 0 + build_fsa + end + + # Returns the accepting state for a given FSA state + # Definition 3.2.13 (state_to_accepting_state) + # @rbs (Integer state_id) -> State? + def state_to_accepting_state(state_id) + state = @states[state_id] + return nil unless state&.accepting? + state + end + + # Returns the set of tokens accepted at FSA state ss + # Definition 3.2.12 acc(ss) + # @rbs (Integer state_id) -> Array[Grammar::TokenPattern] + def acc_ss(state_id) + state = @states[state_id] + return [] unless state + state.accepting_tokens + end + + # Simulate the FSA on input string starting from initial state + # Returns all accepting states reached during the scan + # @rbs (String input) -> Array[{state: State, position: Integer, token: Grammar::TokenPattern}] + def scan(input) + results = [] + current_state_id = 0 + + input.each_char.with_index do |char, index| + current_state = @states[current_state_id] + break unless current_state + + next_state_id = current_state.transitions[char] + break unless next_state_id + + current_state_id = next_state_id + next_state = @states[next_state_id] + + if next_state.accepting? + next_state.accepting_tokens.each do |token_pattern| + results << { state: next_state, position: index + 1, token: token_pattern } + end + end + end + + results + end + + # Returns token pairs that can be in an identity or length scanner conflict. + # Pair keys are sorted token names. + # @rbs () -> Set[[String, String]] + def pairwise_conflict_pairs + pairs = Set.new + + @states.each do |state| + accepting_names = state.accepting_tokens.map(&:name).uniq + accepting_names.combination(2) do |left, right| + pairs << pair_key(left, right) + end + end + + @states.each do |start| + shorter_names = start.accepting_tokens.map(&:name).uniq + next if shorter_names.empty? + + visited = Set.new + stack = start.transitions.values.uniq + + until stack.empty? + state_id = stack.pop + next if visited.include?(state_id) + + visited << state_id + state = @states[state_id] + next unless state + + longer_names = state.accepting_tokens.map(&:name).uniq + shorter_names.product(longer_names).each do |left, right| + pairs << pair_key(left, right) if left != right + end + state.transitions.each_value {|next_id| stack << next_id } + end + end + + pairs + end + + # @rbs (String left, String right) -> bool + def pairwise_conflict?(left, right) + pairwise_conflict_pairs.include?(pair_key(left, right)) + end + + private + + # @rbs (String left, String right) -> [String, String] + def pair_key(left, right) + left <= right ? [left, right] : [right, left] + end + + # Build the FSA from token patterns + # Uses Thompson's construction for NFAs followed by subset construction for DFA + # @rbs () -> void + def build_fsa + return if @token_patterns.empty? + + # Create initial state + @initial_state = create_state + + # Build NFA for each token pattern and convert to DFA + nfa_states = build_nfa + convert_nfa_to_dfa(nfa_states) + end + + # @rbs () -> State + def create_state + state = State.new(@state_counter) + @state_counter += 1 + @states << state + state + end + + # Simple NFA state for regex compilation + class PatternError < StandardError; end + + class NFAState + attr_reader :id #: Integer + attr_accessor :transitions #: Hash[String?, Array[NFAState]] + attr_accessor :accepting_token #: Grammar::TokenPattern? + + # @rbs (Integer id) -> void + def initialize(id) + @id = id + @transitions = Hash.new { |h, k| h[k] = [] } + @accepting_token = nil + end + + # @rbs (String? char, NFAState target) -> void + def add_transition(char, target) + @transitions[char] << target + end + + # @rbs () -> bool + def accepting? + !@accepting_token.nil? + end + end + + class Fragment + attr_reader :start_state #: NFAState + attr_reader :end_state #: NFAState + attr_reader :nullable #: bool + + # @rbs (NFAState start_state, NFAState end_state, bool nullable) -> void + def initialize(start_state, end_state, nullable) + @start_state = start_state + @end_state = end_state + @nullable = nullable + end + + # @rbs () -> [NFAState, NFAState] + def to_ary + [@start_state, @end_state] + end + end + + # Build NFA from all token patterns + # @rbs () -> Array[NFAState] + def build_nfa + nfa_states = [] + nfa_counter = [0] + + # Create NFA start state + nfa_start = create_nfa_state(nfa_counter, nfa_states) + + @token_patterns.each do |token_pattern| + # Build NFA fragment for this pattern + begin + start_state, end_state = compile_regex(token_pattern.regex_pattern, nfa_counter, nfa_states) + rescue PatternError => e + raise PatternError, + "%token-pattern #{token_pattern.name} at line #{token_pattern.lineno} " \ + "has unsupported pattern /#{token_pattern.regex_pattern}/: #{e.message}" + end + + # Connect NFA start to this pattern's start with epsilon + nfa_start.add_transition(nil, start_state) + + # Mark end state as accepting + end_state.accepting_token = token_pattern + end + + nfa_states + end + + # @rbs (Array[Integer] counter, Array[NFAState] states) -> NFAState + def create_nfa_state(counter, states) + state = NFAState.new(counter[0]) + counter[0] += 1 + states << state + state + end + + ASCII_CHARS = (0..127).map(&:chr).freeze #: Array[String] + ANY_CHARS = (ASCII_CHARS - ["\n"]).freeze #: Array[String] + DIGIT_CHARS = ("0".."9").to_a.freeze #: Array[String] + WORD_CHARS = (("a".."z").to_a + ("A".."Z").to_a + DIGIT_CHARS + ["_"]).freeze #: Array[String] + WHITESPACE_CHARS = [" ", "\t", "\n", "\r", "\f", "\v"].freeze #: Array[String] + QUANTIFIERS = ["*", "+", "?"].freeze #: Array[String] + ESCAPED_LITERAL_CHARS = ["/", "\\", "*", "+", "?", "(", ")", "[", "]", "{", "}", ".", "|", "^", "$", "-"].freeze #: Array[String] + + # Compile a regex pattern to NFA fragment. The supported dialect is a small + # ASCII regular-expression subset for PSLR pseudo scanning. + # @rbs (String pattern, Array[Integer] counter, Array[NFAState] states) -> [NFAState, NFAState] + def compile_regex(pattern, counter, states) + raise PatternError, "empty patterns are not allowed" if pattern.empty? + + fragment, pos = compile_expression(pattern, 0, counter, states) + raise PatternError, "unexpected trailing input at offset #{pos}" if pos < pattern.length + raise PatternError, "nullable patterns are not allowed" if fragment.nullable + + [fragment.start_state, fragment.end_state] + end + + # @rbs (String pattern, Integer pos, Array[Integer] counter, Array[NFAState] states, ?String? stop_char) -> [Fragment, Integer] + def compile_expression(pattern, pos, counter, states, stop_char = nil) + fragment, pos = compile_sequence(pattern, pos, counter, states, stop_char) + raise PatternError, empty_sequence_message(stop_char) unless fragment + + alternatives = [fragment] + + while pos < pattern.length && pattern[pos] == "|" + pos += 1 + fragment, pos = compile_sequence(pattern, pos, counter, states, stop_char) + raise PatternError, "empty alternatives are not allowed" unless fragment + + alternatives << fragment + end + + if stop_char + raise PatternError, "unclosed group" unless pos < pattern.length && pattern[pos] == stop_char + + pos += 1 + elsif pos < pattern.length && pattern[pos] == ")" + raise PatternError, "unmatched close group at offset #{pos}" + end + + [alternate_fragments(alternatives, counter, states), pos] + end + + # @rbs (String pattern, Integer pos, Array[Integer] counter, Array[NFAState] states, String? stop_char) -> [Fragment?, Integer] + def compile_sequence(pattern, pos, counter, states, stop_char) + fragments = [] + i = pos + + while i < pattern.length + char = pattern[i] + break if char == "|" || (stop_char && char == stop_char) + + case char + when '\\' + frag, i = compile_escape(pattern, i, counter, states) + fragments << frag + next + when '[' + class_end = find_character_class_end(pattern, i) + raise PatternError, "unclosed character class at offset #{i}" unless class_end + + char_class = pattern[i + 1...class_end] + frag = compile_char_class(char_class, counter, states) + fragments << frag + i = class_end + when '*', '+', '?' + # Quantifier - modify the last fragment + if fragments.empty? + raise PatternError, "quantifier #{char} without preceding element at offset #{i}" + end + fragments << apply_quantifier(fragments.pop, char, counter, states) + when '|' + break + when '(' + frag, i = compile_expression(pattern, i + 1, counter, states, ")") + fragments << frag + next + when ')' + raise PatternError, "unmatched close group at offset #{i}" + when '.' + frag = compile_any_char(counter, states) + fragments << frag + when ']' + raise PatternError, "unmatched character class close at offset #{i}" + else + frag = compile_literal(char, counter, states) + fragments << frag + end + + i += 1 + end + + return [nil, i] if fragments.empty? + + [concatenate_fragments(fragments, counter, states), i] + end + + # @rbs (String? stop_char) -> String + def empty_sequence_message(stop_char) + stop_char ? "empty groups are not allowed" : "empty patterns are not allowed" + end + + # @rbs (String pattern, Integer offset, Array[Integer] counter, Array[NFAState] states) -> [Fragment, Integer] + def compile_escape(pattern, offset, counter, states) + raise PatternError, "dangling escape at offset #{offset}" if offset + 1 >= pattern.length + + escaped = pattern[offset + 1] + fragment = + case escaped + when "d" + compile_chars(DIGIT_CHARS, counter, states) + when "w" + compile_chars(WORD_CHARS, counter, states) + when "s" + compile_chars(WHITESPACE_CHARS, counter, states) + when "n" + compile_literal("\n", counter, states) + when "t" + compile_literal("\t", counter, states) + when "r" + compile_literal("\r", counter, states) + when "f" + compile_literal("\f", counter, states) + when "v" + compile_literal("\v", counter, states) + else + raise PatternError, "unsupported escape \\#{escaped} at offset #{offset}" if escaped.match?(/[[:alnum:]]/) + + unless ESCAPED_LITERAL_CHARS.include?(escaped) + raise PatternError, "unsupported escape \\#{escaped} at offset #{offset}" + end + compile_literal(escaped, counter, states) + end + + [fragment, offset + 2] + end + + # Compile a single literal character + # @rbs (String char, Array[Integer] counter, Array[NFAState] states) -> Fragment + def compile_literal(char, counter, states) + compile_chars([char], counter, states) + end + + # @rbs (Array[String] chars, Array[Integer] counter, Array[NFAState] states) -> Fragment + def compile_chars(chars, counter, states) + raise PatternError, "empty character classes are not allowed" if chars.empty? + + start_state = create_nfa_state(counter, states) + end_state = create_nfa_state(counter, states) + chars.uniq.each do |char| + start_state.add_transition(char, end_state) + end + Fragment.new(start_state, end_state, false) + end + + # Compile a character class [...] + # @rbs (String char_class, Array[Integer] counter, Array[NFAState] states) -> Fragment + def compile_char_class(char_class, counter, states) + compile_chars(expand_char_class(char_class), counter, states) + end + + # Expand character class string to array of characters + # @rbs (String char_class) -> Array[String] + def expand_char_class(char_class) + raise PatternError, "empty character classes are not allowed" if char_class.empty? + + chars = [] + i = 0 + negated = false + + if char_class[0] == '^' + negated = true + i = 1 + raise PatternError, "negated character classes must include at least one character" if i >= char_class.length + end + + while i < char_class.length + element_chars, i = read_char_class_element(char_class, i) + + if i < char_class.length && char_class[i] == "-" && i + 1 < char_class.length + i += 1 + range_end_chars, i = read_char_class_element(char_class, i) + chars.concat(expand_char_range(element_chars, range_end_chars)) + else + chars.concat(element_chars) + end + end + + if negated + chars = ASCII_CHARS - chars + end + + chars.uniq + end + + # @rbs (String pattern, Integer offset) -> Integer? + def find_character_class_end(pattern, offset) + i = offset + 1 + while i < pattern.length + if pattern[i] == "\\" + raise PatternError, "dangling escape in character class at offset #{i}" if i + 1 >= pattern.length + + i += 2 + next + end + + return i if pattern[i] == "]" + + i += 1 + end + + nil + end + + # @rbs (String char_class, Integer offset) -> [Array[String], Integer] + def read_char_class_element(char_class, offset) + raise PatternError, "dangling range operator in character class" if offset >= char_class.length + + char = char_class[offset] + if char == "\\" + raise PatternError, "dangling escape in character class" if offset + 1 >= char_class.length + + escaped = char_class[offset + 1] + return [escaped_char_class_chars(escaped, offset), offset + 2] + end + + [[char], offset + 1] + end + + # @rbs (String char, Integer offset) -> Array[String] + def escaped_char_class_chars(char, offset) + case char + when "d" + DIGIT_CHARS + when "w" + WORD_CHARS + when "s" + WHITESPACE_CHARS + when "t" + ["\t"] + when "n" + ["\n"] + when "r" + ["\r"] + when "f" + ["\f"] + when "v" + ["\v"] + else + raise PatternError, "unsupported escape \\#{char} in character class at offset #{offset}" if char.match?(/[[:alnum:]]/) + + [char] + end + end + + # @rbs (Array[String] start_chars, Array[String] end_chars) -> Array[String] + def expand_char_range(start_chars, end_chars) + if start_chars.size != 1 || end_chars.size != 1 + raise PatternError, "character class ranges must use single-character endpoints" + end + + start_char = start_chars.first + end_char = end_chars.first + if start_char.ord > end_char.ord + raise PatternError, "invalid character class range #{start_char}-#{end_char}" + end + + (start_char.ord..end_char.ord).map(&:chr) + end + + # Compile . (any character) + # @rbs (Array[Integer] counter, Array[NFAState] states) -> Fragment + def compile_any_char(counter, states) + compile_chars(ANY_CHARS, counter, states) + end + + # Apply a quantifier to a fragment + # @rbs (Fragment fragment, String quantifier, Array[Integer] counter, Array[NFAState] states) -> Fragment + def apply_quantifier(fragment, quantifier, counter, states) + frag_start = fragment.start_state + frag_end = fragment.end_state + + case quantifier + when '*' + # Zero or more + new_start = create_nfa_state(counter, states) + new_end = create_nfa_state(counter, states) + + new_start.add_transition(nil, frag_start) + new_start.add_transition(nil, new_end) + frag_end.add_transition(nil, frag_start) + frag_end.add_transition(nil, new_end) + + Fragment.new(new_start, new_end, true) + when '+' + # One or more + new_end = create_nfa_state(counter, states) + + frag_end.add_transition(nil, frag_start) + frag_end.add_transition(nil, new_end) + + Fragment.new(frag_start, new_end, fragment.nullable) + when '?' + # Zero or one + new_start = create_nfa_state(counter, states) + new_end = create_nfa_state(counter, states) + + new_start.add_transition(nil, frag_start) + new_start.add_transition(nil, new_end) + frag_end.add_transition(nil, new_end) + + Fragment.new(new_start, new_end, true) + else + fragment + end + end + + # @rbs (Array[Fragment] fragments, Array[Integer] counter, Array[NFAState] states) -> Fragment + def alternate_fragments(fragments, counter, states) + return fragments.first if fragments.size == 1 + + alt_start = create_nfa_state(counter, states) + alt_end = create_nfa_state(counter, states) + + fragments.each do |fragment| + alt_start.add_transition(nil, fragment.start_state) + fragment.end_state.add_transition(nil, alt_end) + end + + Fragment.new(alt_start, alt_end, fragments.any?(&:nullable)) + end + + # Concatenate multiple NFA fragments into one + # @rbs (Array[Fragment] fragments, Array[Integer] counter, Array[NFAState] states) -> Fragment + def concatenate_fragments(fragments, counter, states) + raise PatternError, "empty sequences are not allowed" if fragments.empty? + return fragments.first if fragments.size == 1 + + result_start = fragments.first.start_state + current_end = fragments.first.end_state + nullable = fragments.all?(&:nullable) + + fragments[1..-1].each do |fragment| + current_end.add_transition(nil, fragment.start_state) + current_end = fragment.end_state + end + + Fragment.new(result_start, current_end, nullable) + end + + # Convert NFA to DFA using subset construction + # @rbs (Array[NFAState] nfa_states) -> void + def convert_nfa_to_dfa(nfa_states) + return if nfa_states.empty? + + # Clear existing DFA states + @states = [] + @state_counter = 0 + + # Compute epsilon closure of start state + nfa_start = nfa_states[0] + start_closure = epsilon_closure([nfa_start]) + + # Map NFA state sets to DFA states + dfa_states = {} + work_list = [start_closure] + dfa_states[start_closure.map(&:id).sort] = create_state + + @initial_state = @states[0] + + # Mark accepting tokens for initial state + start_closure.each do |nfa_state| + if nfa_state.accepting? + @initial_state.add_accepting_token(nfa_state.accepting_token) + end + end + + while !work_list.empty? + current_nfa_set = work_list.shift + current_dfa = dfa_states[current_nfa_set.map(&:id).sort] + + # Find all possible transitions + transitions = {} + current_nfa_set.each do |nfa_state| + nfa_state.transitions.each do |char, targets| + next if char.nil? # Skip epsilon transitions + transitions[char] ||= [] + transitions[char].concat(targets) + end + end + + transitions.each do |char, targets| + target_closure = epsilon_closure(targets.uniq) + target_key = target_closure.map(&:id).sort + + unless dfa_states.key?(target_key) + new_dfa_state = create_state + dfa_states[target_key] = new_dfa_state + + # Mark accepting tokens + target_closure.each do |nfa_state| + if nfa_state.accepting? + new_dfa_state.add_accepting_token(nfa_state.accepting_token) + end + end + + work_list << target_closure + end + + current_dfa.add_transition(char, dfa_states[target_key].id) + end + end + end + + # Compute epsilon closure of a set of NFA states + # @rbs (Array[NFAState] nfa_states) -> Array[NFAState] + def epsilon_closure(nfa_states) + closure = nfa_states.dup + seen = Set.new(nfa_states.map(&:id)) + work_list = nfa_states.dup + + while !work_list.empty? + state = work_list.shift + epsilon_targets = state.transitions[nil] || [] + + epsilon_targets.each do |target| + unless seen.include?(target.id) + seen << target.id + closure << target + work_list << target + end + end + end + + closure + end + end +end diff --git a/sig/generated/lrama/scanner_fsa.rbs b/sig/generated/lrama/scanner_fsa.rbs new file mode 100644 index 000000000..7397a65d3 --- /dev/null +++ b/sig/generated/lrama/scanner_fsa.rbs @@ -0,0 +1,197 @@ +# Generated from lib/lrama/scanner_fsa.rb with RBS::Inline + +module Lrama + # Scanner Finite State Automaton for PSLR(1) + # Built from token patterns defined by %token-pattern directives + # Based on Definitions 3.2.12, 3.2.13 from the PSLR dissertation + class ScannerFSA + # Represents a state in the scanner FSA + class State + attr_reader id: Integer + + attr_reader transitions: Hash[String, Integer] + + attr_reader accepting_tokens: Array[Grammar::TokenPattern] + + # @rbs (Integer id) -> void + def initialize: (Integer id) -> void + + # @rbs () -> bool + def accepting?: () -> bool + + # @rbs (String char, Integer target_state_id) -> void + def add_transition: (String char, Integer target_state_id) -> void + + # @rbs (Grammar::TokenPattern token_pattern) -> void + def add_accepting_token: (Grammar::TokenPattern token_pattern) -> void + end + + attr_reader states: Array[State] + + attr_reader initial_state: State + + attr_reader token_patterns: Array[Grammar::TokenPattern] + + # @rbs (Array[Grammar::TokenPattern] token_patterns) -> void + def initialize: (Array[Grammar::TokenPattern] token_patterns) -> void + + # Returns the accepting state for a given FSA state + # Definition 3.2.13 (state_to_accepting_state) + # @rbs (Integer state_id) -> State? + def state_to_accepting_state: (Integer state_id) -> State? + + # Returns the set of tokens accepted at FSA state ss + # Definition 3.2.12 acc(ss) + # @rbs (Integer state_id) -> Array[Grammar::TokenPattern] + def acc_ss: (Integer state_id) -> Array[Grammar::TokenPattern] + + # Simulate the FSA on input string starting from initial state + # Returns all accepting states reached during the scan + # @rbs (String input) -> Array[{state: State, position: Integer, token: Grammar::TokenPattern}] + def scan: (String input) -> Array[{ state: State, position: Integer, token: Grammar::TokenPattern }] + + # Returns token pairs that can be in an identity or length scanner conflict. + # Pair keys are sorted token names. + # @rbs () -> Set[[String, String]] + def pairwise_conflict_pairs: () -> Set[[ String, String ]] + + # @rbs (String left, String right) -> bool + def pairwise_conflict?: (String left, String right) -> bool + + private + + # @rbs (String left, String right) -> [String, String] + def pair_key: (String left, String right) -> [ String, String ] + + # Build the FSA from token patterns + # Uses Thompson's construction for NFAs followed by subset construction for DFA + # @rbs () -> void + def build_fsa: () -> void + + # @rbs () -> State + def create_state: () -> State + + # Simple NFA state for regex compilation + class PatternError < StandardError + end + + class NFAState + attr_reader id: Integer + + attr_accessor transitions: Hash[String?, Array[NFAState]] + + attr_accessor accepting_token: Grammar::TokenPattern? + + # @rbs (Integer id) -> void + def initialize: (Integer id) -> void + + # @rbs (String? char, NFAState target) -> void + def add_transition: (String? char, NFAState target) -> void + + # @rbs () -> bool + def accepting?: () -> bool + end + + class Fragment + attr_reader start_state: NFAState + + attr_reader end_state: NFAState + + attr_reader nullable: bool + + # @rbs (NFAState start_state, NFAState end_state, bool nullable) -> void + def initialize: (NFAState start_state, NFAState end_state, bool nullable) -> void + + # @rbs () -> [NFAState, NFAState] + def to_ary: () -> [ NFAState, NFAState ] + end + + # Build NFA from all token patterns + # @rbs () -> Array[NFAState] + def build_nfa: () -> Array[NFAState] + + # @rbs (Array[Integer] counter, Array[NFAState] states) -> NFAState + def create_nfa_state: (Array[Integer] counter, Array[NFAState] states) -> NFAState + + ASCII_CHARS: Array[String] + + ANY_CHARS: Array[String] + + DIGIT_CHARS: Array[String] + + WORD_CHARS: Array[String] + + WHITESPACE_CHARS: Array[String] + + QUANTIFIERS: Array[String] + + ESCAPED_LITERAL_CHARS: Array[String] + + # Compile a regex pattern to NFA fragment. The supported dialect is a small + # ASCII regular-expression subset for PSLR pseudo scanning. + # @rbs (String pattern, Array[Integer] counter, Array[NFAState] states) -> [NFAState, NFAState] + def compile_regex: (String pattern, Array[Integer] counter, Array[NFAState] states) -> [ NFAState, NFAState ] + + # @rbs (String pattern, Integer pos, Array[Integer] counter, Array[NFAState] states, ?String? stop_char) -> [Fragment, Integer] + def compile_expression: (String pattern, Integer pos, Array[Integer] counter, Array[NFAState] states, ?String? stop_char) -> [ Fragment, Integer ] + + # @rbs (String pattern, Integer pos, Array[Integer] counter, Array[NFAState] states, String? stop_char) -> [Fragment?, Integer] + def compile_sequence: (String pattern, Integer pos, Array[Integer] counter, Array[NFAState] states, String? stop_char) -> [ Fragment?, Integer ] + + # @rbs (String? stop_char) -> String + def empty_sequence_message: (String? stop_char) -> String + + # @rbs (String pattern, Integer offset, Array[Integer] counter, Array[NFAState] states) -> [Fragment, Integer] + def compile_escape: (String pattern, Integer offset, Array[Integer] counter, Array[NFAState] states) -> [ Fragment, Integer ] + + # Compile a single literal character + # @rbs (String char, Array[Integer] counter, Array[NFAState] states) -> Fragment + def compile_literal: (String char, Array[Integer] counter, Array[NFAState] states) -> Fragment + + # @rbs (Array[String] chars, Array[Integer] counter, Array[NFAState] states) -> Fragment + def compile_chars: (Array[String] chars, Array[Integer] counter, Array[NFAState] states) -> Fragment + + # Compile a character class [...] + # @rbs (String char_class, Array[Integer] counter, Array[NFAState] states) -> Fragment + def compile_char_class: (String char_class, Array[Integer] counter, Array[NFAState] states) -> Fragment + + # Expand character class string to array of characters + # @rbs (String char_class) -> Array[String] + def expand_char_class: (String char_class) -> Array[String] + + # @rbs (String pattern, Integer offset) -> Integer? + def find_character_class_end: (String pattern, Integer offset) -> Integer? + + # @rbs (String char_class, Integer offset) -> [Array[String], Integer] + def read_char_class_element: (String char_class, Integer offset) -> [ Array[String], Integer ] + + # @rbs (String char, Integer offset) -> Array[String] + def escaped_char_class_chars: (String char, Integer offset) -> Array[String] + + # @rbs (Array[String] start_chars, Array[String] end_chars) -> Array[String] + def expand_char_range: (Array[String] start_chars, Array[String] end_chars) -> Array[String] + + # Compile . (any character) + # @rbs (Array[Integer] counter, Array[NFAState] states) -> Fragment + def compile_any_char: (Array[Integer] counter, Array[NFAState] states) -> Fragment + + # Apply a quantifier to a fragment + # @rbs (Fragment fragment, String quantifier, Array[Integer] counter, Array[NFAState] states) -> Fragment + def apply_quantifier: (Fragment fragment, String quantifier, Array[Integer] counter, Array[NFAState] states) -> Fragment + + # @rbs (Array[Fragment] fragments, Array[Integer] counter, Array[NFAState] states) -> Fragment + def alternate_fragments: (Array[Fragment] fragments, Array[Integer] counter, Array[NFAState] states) -> Fragment + + # Concatenate multiple NFA fragments into one + # @rbs (Array[Fragment] fragments, Array[Integer] counter, Array[NFAState] states) -> Fragment + def concatenate_fragments: (Array[Fragment] fragments, Array[Integer] counter, Array[NFAState] states) -> Fragment + + # Convert NFA to DFA using subset construction + # @rbs (Array[NFAState] nfa_states) -> void + def convert_nfa_to_dfa: (Array[NFAState] nfa_states) -> void + + # Compute epsilon closure of a set of NFA states + # @rbs (Array[NFAState] nfa_states) -> Array[NFAState] + def epsilon_closure: (Array[NFAState] nfa_states) -> Array[NFAState] + end +end diff --git a/spec/lrama/grammar_spec.rb b/spec/lrama/grammar_spec.rb index 3be8eab4f..4a663401d 100644 --- a/spec/lrama/grammar_spec.rb +++ b/spec/lrama/grammar_spec.rb @@ -242,5 +242,247 @@ end end end + + context 'when PSLR state member is not a valid C identifier' do + before do + grammar.define = { + 'lr.type' => 'pslr', + 'api.pslr.state-member' => 'current-state' + } + end + + it 'raises an error with the invalid member name' do + expect { grammar.validate! } + .to raise_error(RuntimeError, '%define api.pslr.state-member must be a valid C identifier, got "current-state".') + end + end + + context 'when PSLR max states is not an integer' do + before do + grammar.define = { + 'lr.type' => 'pslr', + 'pslr.max-states' => 'many' + } + end + + it 'raises an error with the invalid value' do + expect { grammar.validate! } + .to raise_error(RuntimeError, '%define pslr.max-states must be an integer, got "many".') + end + end + + context 'when PSLR max state ratio is smaller than one' do + before do + grammar.define = { + 'lr.type' => 'pslr', + 'pslr.max-state-ratio' => '0.5' + } + end + + it 'raises an error with the invalid ratio' do + expect { grammar.validate! } + .to raise_error(RuntimeError, '%define pslr.max-state-ratio must be greater than or equal to 1.0, got "0.5".') + end + end + end + + describe "#finalize_lexical_ties!" do + def build_pslr_grammar(source) + grammar = Lrama::Parser.new(source, "lex_tie.y").parse + grammar.prepare + grammar.validate! + grammar + end + + it "keeps token-token ties even without a scanner conflict" do + grammar = build_pslr_grammar(<<~GRAMMAR) + %define lr.type pslr + %token-pattern A /a/ + %token-pattern B /b/ + %lex-tie A B + %% + start: A | B ; + GRAMMAR + + grammar.finalize_lexical_ties!(Lrama::ScannerFSA.new(grammar.token_patterns)) + + expect(grammar.lex_tie.tied?("A", "B")).to be true + end + + it "limits set-set ties to scanner-conflicting pairs" do + grammar = build_pslr_grammar(<<~GRAMMAR) + %define lr.type pslr + %token-pattern RANGLE />/ + %token-pattern RSHIFT />>/ + %token-pattern DOT /\\./ + %token-pattern COMMA /,/ + %symbol-set punct RANGLE RSHIFT DOT COMMA + %lex-tie punct punct + %% + start: RANGLE | RSHIFT | DOT | COMMA ; + GRAMMAR + + grammar.finalize_lexical_ties!(Lrama::ScannerFSA.new(grammar.token_patterns)) + + expect(grammar.lex_tie.tied?("RANGLE", "RSHIFT")).to be true + expect(grammar.lex_tie.tied?("DOT", "COMMA")).to be false + end + + it "limits set-token ties to scanner-conflicting pairs" do + grammar = build_pslr_grammar(<<~GRAMMAR) + %define lr.type pslr + %token-pattern ID /[a-z]+/ + %token-pattern KW_IF /if/ + %token-pattern KW_WHILE /while/ + %token-pattern PLUS /\\+/ + %symbol-set keywords KW_IF KW_WHILE + %lex-tie ID keywords + %lex-tie PLUS keywords + %% + start: ID | KW_IF | KW_WHILE | PLUS ; + GRAMMAR + + grammar.finalize_lexical_ties!(Lrama::ScannerFSA.new(grammar.token_patterns)) + + expect(grammar.lex_tie.tied?("ID", "KW_IF")).to be true + expect(grammar.lex_tie.tied?("ID", "KW_WHILE")).to be true + expect(grammar.lex_tie.tied?("PLUS", "KW_IF")).to be false + expect(grammar.lex_tie.tied?("PLUS", "KW_WHILE")).to be false + end + + it "limits yyall ties to scanner-conflicting pairs" do + grammar = build_pslr_grammar(<<~GRAMMAR) + %define lr.type pslr + %token-pattern PLUS /\\+/ + %token-pattern PLUSPLUS /\\+\\+/ + %token-pattern DOT /\\./ + %token-pattern SLASH /\\// + %lex-tie yyall yyall + %% + start: PLUS | PLUSPLUS | DOT | SLASH ; + GRAMMAR + + grammar.finalize_lexical_ties!(Lrama::ScannerFSA.new(grammar.token_patterns)) + + expect(grammar.lex_tie.tied?("PLUS", "PLUSPLUS")).to be true + expect(grammar.lex_tie.tied?("DOT", "SLASH")).to be false + expect(grammar.lex_tie.tied?("PLUS", "DOT")).to be false + expect(grammar.lex_tie.tied?("SLASH", "PLUSPLUS")).to be false + end + + it "lets a specific tie override generic yyall no-tie" do + grammar = build_pslr_grammar(<<~GRAMMAR) + %define lr.type pslr + %token-pattern IF /if/ + %token-pattern ID /[a-z]+/ + %symbol-set keywords IF + %lex-no-tie yyall yyall + %lex-tie ID keywords + %% + start: IF | ID ; + GRAMMAR + + grammar.finalize_lexical_ties!(Lrama::ScannerFSA.new(grammar.token_patterns)) + + expect(grammar.lex_tie.tied?("ID", "IF")).to be true + expect(grammar.lex_tie.no_tie?("ID", "IF")).to be false + end + + it "rejects a direct no-tie that conflicts with transitive ties" do + grammar = build_pslr_grammar(<<~GRAMMAR) + %define lr.type pslr + %token-pattern A /a/ + %token-pattern B /a/ + %token-pattern C /a/ + %lex-tie A B + %lex-tie B C + %lex-no-tie A C + %% + start: A | B | C ; + GRAMMAR + + expect do + grammar.finalize_lexical_ties!(Lrama::ScannerFSA.new(grammar.token_patterns)) + end.to raise_error(RuntimeError, /%lex-no-tie A C conflicts/) + end + end + + describe "#synthesize_implicit_literal_token_patterns!" do + it "adds exact-match token patterns for character literal terminals" do + grammar = Lrama::Parser.new(<<~GRAMMAR, "implicit_literal.y").parse + %define lr.type pslr + %token-pattern ID /[a-z]+/ + %% + start: ID ';' ; + GRAMMAR + grammar.prepare + grammar.validate! + + grammar.synthesize_implicit_literal_token_patterns! + + literal_pattern = grammar.token_patterns.find {|pattern| pattern.name == "';'" } + expect(literal_pattern).not_to be_nil + expect(Lrama::ScannerFSA.new(grammar.token_patterns).scan(";").map {|result| result[:token].name }).to include("';'") + end + + it "escapes regex metacharacters in character literal patterns" do + grammar = Lrama::Parser.new(<<~GRAMMAR, "implicit_escape.y").parse + %define lr.type pslr + %token-pattern ID /[a-z]+/ + %% + start + : ID '/' + | ID '[' + | ID ']' + | ID '+' + ; + GRAMMAR + grammar.prepare + grammar.validate! + + grammar.synthesize_implicit_literal_token_patterns! + + slash = grammar.token_patterns.find {|p| p.name == "'/'" } + lbracket = grammar.token_patterns.find {|p| p.name == "'['" } + rbracket = grammar.token_patterns.find {|p| p.name == "']'" } + plus = grammar.token_patterns.find {|p| p.name == "'+'" } + + expect(slash).not_to be_nil + expect(slash.regex_pattern).to eq("\\/") + expect(lbracket).not_to be_nil + expect(lbracket.regex_pattern).to eq("\\[") + expect(rbracket).not_to be_nil + expect(rbracket.regex_pattern).to eq("\\]") + expect(plus).not_to be_nil + expect(plus.regex_pattern).to eq("\\+") + end + + it "handles backslash and control character literals" do + grammar = Lrama::Parser.new(<<~GRAMMAR, "implicit_ctrl.y").parse + %define lr.type pslr + %token-pattern ID /[a-z]+/ + %% + start + : ID '\\\\' + | ID '\\n' + | ID '\\t' + ; + GRAMMAR + grammar.prepare + grammar.validate! + + grammar.synthesize_implicit_literal_token_patterns! + + backslash = grammar.token_patterns.find {|p| p.name == "'\\\\'" } + newline = grammar.token_patterns.find {|p| p.name == "'\\n'" } + tab = grammar.token_patterns.find {|p| p.name == "'\\t'" } + + expect(backslash).not_to be_nil + expect(backslash.regex_pattern).to eq("\\\\") + expect(newline).not_to be_nil + expect(newline.regex_pattern).to eq("\\n") + expect(tab).not_to be_nil + expect(tab.regex_pattern).to eq("\\t") + end end end diff --git a/spec/lrama/scanner_fsa_spec.rb b/spec/lrama/scanner_fsa_spec.rb new file mode 100644 index 000000000..9912945ee --- /dev/null +++ b/spec/lrama/scanner_fsa_spec.rb @@ -0,0 +1,322 @@ +# frozen_string_literal: true + +RSpec.describe Lrama::ScannerFSA do + def token_pattern(name, regex, lineno: 1, order: 0) + Lrama::Grammar::TokenPattern.new( + id: Lrama::Lexer::Token::Ident.new(s_value: name), + pattern: Lrama::Lexer::Token::Regex.new(s_value: "/#{regex}/"), + lineno: lineno, + definition_order: order + ) + end + + describe "initialization" do + it "creates an empty FSA for no patterns" do + fsa = Lrama::ScannerFSA.new([]) + expect(fsa.states).to be_empty + end + + it "creates FSA for single literal pattern" do + id = Lrama::Lexer::Token::Ident.new(s_value: "PLUS") + regex = Lrama::Lexer::Token::Regex.new(s_value: "/\\+/") + token_pattern = Lrama::Grammar::TokenPattern.new( + id: id, + pattern: regex, + lineno: 1, + definition_order: 0 + ) + fsa = Lrama::ScannerFSA.new([token_pattern]) + + expect(fsa.states).not_to be_empty + expect(fsa.initial_state).not_to be_nil + end + end + + describe "#scan" do + it "matches a single character pattern" do + id = Lrama::Lexer::Token::Ident.new(s_value: "RANGLE") + regex = Lrama::Lexer::Token::Regex.new(s_value: "/>/") + token_pattern = Lrama::Grammar::TokenPattern.new( + id: id, + pattern: regex, + lineno: 1, + definition_order: 0 + ) + fsa = Lrama::ScannerFSA.new([token_pattern]) + + results = fsa.scan(">") + expect(results.size).to eq(1) + expect(results[0][:token].name).to eq("RANGLE") + expect(results[0][:position]).to eq(1) + end + + it "matches a multi-character pattern" do + id = Lrama::Lexer::Token::Ident.new(s_value: "RSHIFT") + regex = Lrama::Lexer::Token::Regex.new(s_value: "/>>/") + token_pattern = Lrama::Grammar::TokenPattern.new( + id: id, + pattern: regex, + lineno: 1, + definition_order: 0 + ) + fsa = Lrama::ScannerFSA.new([token_pattern]) + + results = fsa.scan(">>") + expect(results.size).to eq(1) + expect(results[0][:token].name).to eq("RSHIFT") + expect(results[0][:position]).to eq(2) + end + + it "returns multiple matches for overlapping patterns" do + rangle_id = Lrama::Lexer::Token::Ident.new(s_value: "RANGLE") + rangle_regex = Lrama::Lexer::Token::Regex.new(s_value: "/>/") + rangle = Lrama::Grammar::TokenPattern.new( + id: rangle_id, + pattern: rangle_regex, + lineno: 1, + definition_order: 0 + ) + + rshift_id = Lrama::Lexer::Token::Ident.new(s_value: "RSHIFT") + rshift_regex = Lrama::Lexer::Token::Regex.new(s_value: "/>>/") + rshift = Lrama::Grammar::TokenPattern.new( + id: rshift_id, + pattern: rshift_regex, + lineno: 1, + definition_order: 1 + ) + + fsa = Lrama::ScannerFSA.new([rangle, rshift]) + + results = fsa.scan(">>") + + # Should match both RANGLE at position 1 and RSHIFT at position 2 + expect(results.size).to eq(2) + positions = results.map { |r| [r[:token].name, r[:position]] } + expect(positions).to include(["RANGLE", 1]) + expect(positions).to include(["RSHIFT", 2]) + end + + it "matches character class patterns" do + id = Lrama::Lexer::Token::Ident.new(s_value: "ID") + regex = Lrama::Lexer::Token::Regex.new(s_value: "/[a-zA-Z_][a-zA-Z0-9_]*/") + id_pattern = Lrama::Grammar::TokenPattern.new( + id: id, + pattern: regex, + lineno: 1, + definition_order: 0 + ) + fsa = Lrama::ScannerFSA.new([id_pattern]) + + results = fsa.scan("hello_world123") + expect(results).not_to be_empty + # Should have matches at each position as the identifier grows + end + + it "matches digit patterns" do + id = Lrama::Lexer::Token::Ident.new(s_value: "INT") + regex = Lrama::Lexer::Token::Regex.new(s_value: "/[0-9]+/") + int_pattern = Lrama::Grammar::TokenPattern.new( + id: id, + pattern: regex, + lineno: 1, + definition_order: 0 + ) + fsa = Lrama::ScannerFSA.new([int_pattern]) + + results = fsa.scan("12345") + expect(results).not_to be_empty + end + + it "matches escaped whitespace inside character classes" do + id = Lrama::Lexer::Token::Ident.new(s_value: "YYLAYOUT") + regex = Lrama::Lexer::Token::Regex.new(s_value: "/[ \\t\\r\\n]+/") + token_pattern = Lrama::Grammar::TokenPattern.new( + id: id, + pattern: regex, + lineno: 1, + definition_order: 0 + ) + fsa = Lrama::ScannerFSA.new([token_pattern]) + + expect(fsa.scan("\t").map { |result| result[:token].name }).to include("YYLAYOUT") + expect(fsa.scan("\n").map { |result| result[:token].name }).to include("YYLAYOUT") + end + + it "matches escaped literals in and outside character classes" do + slash = token_pattern("SLASH", "\\/", order: 0) + rbrack = token_pattern("RBRACK", "[\\]]", order: 1) + backslash = token_pattern("BACKSLASH", "[\\\\]", order: 2) + + expect(Lrama::ScannerFSA.new([slash]).scan("/").map { |result| result[:token].name }).to include("SLASH") + expect(Lrama::ScannerFSA.new([rbrack]).scan("]").map { |result| result[:token].name }).to include("RBRACK") + expect(Lrama::ScannerFSA.new([backslash]).scan("\\").map { |result| result[:token].name }).to include("BACKSLASH") + end + + it "matches ranges and negated character classes over ASCII" do + not_star = token_pattern("NOT_STAR", "[^*]+") + fsa = Lrama::ScannerFSA.new([not_star]) + + expect(fsa.scan("abc/]").map { |result| result[:token].name }).to include("NOT_STAR") + expect(fsa.scan("a\nb").map { |result| result[:token].name }).to include("NOT_STAR") + expect(fsa.scan("*")).to be_empty + end + end + + describe "pattern validation" do + it "rejects empty token patterns" do + expect { Lrama::ScannerFSA.new([token_pattern("EMPTY", "", lineno: 42)]) } + .to raise_error(Lrama::ScannerFSA::PatternError, /EMPTY at line 42.*empty patterns/m) + end + + it "rejects dangling escapes" do + expect { Lrama::ScannerFSA.new([token_pattern("BAD_ESCAPE", "abc\\", lineno: 7)]) } + .to raise_error(Lrama::ScannerFSA::PatternError, /BAD_ESCAPE at line 7.*dangling escape/m) + end + + it "rejects unclosed character classes" do + expect { Lrama::ScannerFSA.new([token_pattern("BAD_CLASS", "[abc", lineno: 6)]) } + .to raise_error(Lrama::ScannerFSA::PatternError, /BAD_CLASS at line 6.*unclosed character class/m) + end + + it "rejects unsupported alphabetic escapes" do + expect { Lrama::ScannerFSA.new([token_pattern("BAD_ESCAPE", "\\q", lineno: 8)]) } + .to raise_error(Lrama::ScannerFSA::PatternError, /BAD_ESCAPE at line 8.*unsupported escape \\q/m) + end + + it "rejects malformed character class ranges" do + expect { Lrama::ScannerFSA.new([token_pattern("BAD_RANGE", "[z-a]", lineno: 9)]) } + .to raise_error(Lrama::ScannerFSA::PatternError, /BAD_RANGE at line 9.*invalid character class range z-a/m) + end + + it "rejects nullable token patterns" do + expect { Lrama::ScannerFSA.new([token_pattern("NULLABLE", "a*", lineno: 10)]) } + .to raise_error(Lrama::ScannerFSA::PatternError, /NULLABLE at line 10.*nullable patterns/m) + expect { Lrama::ScannerFSA.new([token_pattern("NULLABLE", "a?", lineno: 10)]) } + .to raise_error(Lrama::ScannerFSA::PatternError, /NULLABLE at line 10.*nullable patterns/m) + expect { Lrama::ScannerFSA.new([token_pattern("NULLABLE", "()", lineno: 10)]) } + .to raise_error(Lrama::ScannerFSA::PatternError, /NULLABLE at line 10.*empty groups/m) + end + + it "rejects empty alternatives" do + expect { Lrama::ScannerFSA.new([token_pattern("EMPTY_ALT", "a|", lineno: 11)]) } + .to raise_error(Lrama::ScannerFSA::PatternError, /EMPTY_ALT at line 11.*empty alternatives/m) + end + end + + describe "#pairwise_conflict_pairs" do + it "detects identity and length conflicts" do + rangle = Lrama::Grammar::TokenPattern.new( + id: Lrama::Lexer::Token::Ident.new(s_value: "RANGLE"), + pattern: Lrama::Lexer::Token::Regex.new(s_value: "/>/"), + lineno: 1, + definition_order: 0 + ) + rshift = Lrama::Grammar::TokenPattern.new( + id: Lrama::Lexer::Token::Ident.new(s_value: "RSHIFT"), + pattern: Lrama::Lexer::Token::Regex.new(s_value: "/>>/"), + lineno: 1, + definition_order: 1 + ) + keyword = Lrama::Grammar::TokenPattern.new( + id: Lrama::Lexer::Token::Ident.new(s_value: "IF"), + pattern: Lrama::Lexer::Token::Regex.new(s_value: "/if/"), + lineno: 1, + definition_order: 2 + ) + identifier = Lrama::Grammar::TokenPattern.new( + id: Lrama::Lexer::Token::Ident.new(s_value: "ID"), + pattern: Lrama::Lexer::Token::Regex.new(s_value: "/[a-z]+/"), + lineno: 1, + definition_order: 3 + ) + + pairs = Lrama::ScannerFSA.new([rangle, rshift, keyword, identifier]).pairwise_conflict_pairs + + expect(pairs).to include(["RANGLE", "RSHIFT"]) + expect(pairs).to include(["ID", "IF"]) + end + + it "checks pairwise conflicts for sorted token pairs" do + rangle = token_pattern("RANGLE", ">", order: 0) + rshift = token_pattern("RSHIFT", ">>", order: 1) + dot = token_pattern("DOT", "\\.", order: 2) + comma = token_pattern("COMMA", ",", order: 3) + fsa = Lrama::ScannerFSA.new([rangle, rshift, dot, comma]) + + expect(fsa.pairwise_conflict?("RSHIFT", "RANGLE")).to be true + expect(fsa.pairwise_conflict?("DOT", "COMMA")).to be false + end + end + + describe "#acc_ss" do + it "returns empty array for non-accepting state" do + id = Lrama::Lexer::Token::Ident.new(s_value: "AB") + regex = Lrama::Lexer::Token::Regex.new(s_value: "/ab/") + token_pattern = Lrama::Grammar::TokenPattern.new( + id: id, + pattern: regex, + lineno: 1, + definition_order: 0 + ) + fsa = Lrama::ScannerFSA.new([token_pattern]) + + # Initial state shouldn't be accepting for non-empty pattern + tokens = fsa.acc_ss(0) + expect(tokens).to be_empty + end + + it "returns accepting tokens for accepting state" do + id = Lrama::Lexer::Token::Ident.new(s_value: "A") + regex = Lrama::Lexer::Token::Regex.new(s_value: "/a/") + token_pattern = Lrama::Grammar::TokenPattern.new( + id: id, + pattern: regex, + lineno: 1, + definition_order: 0 + ) + fsa = Lrama::ScannerFSA.new([token_pattern]) + + # Scan to reach accepting state + results = fsa.scan("a") + expect(results).not_to be_empty + + accepting_state = results[0][:state] + tokens = fsa.acc_ss(accepting_state.id) + expect(tokens.map(&:name)).to include("A") + end + end + + describe "#state_to_accepting_state" do + it "returns nil for non-accepting state" do + id = Lrama::Lexer::Token::Ident.new(s_value: "AB") + regex = Lrama::Lexer::Token::Regex.new(s_value: "/ab/") + token_pattern = Lrama::Grammar::TokenPattern.new( + id: id, + pattern: regex, + lineno: 1, + definition_order: 0 + ) + fsa = Lrama::ScannerFSA.new([token_pattern]) + + expect(fsa.state_to_accepting_state(0)).to be_nil + end + + it "returns the state itself for accepting state" do + id = Lrama::Lexer::Token::Ident.new(s_value: "A") + regex = Lrama::Lexer::Token::Regex.new(s_value: "/a/") + token_pattern = Lrama::Grammar::TokenPattern.new( + id: id, + pattern: regex, + lineno: 1, + definition_order: 0 + ) + fsa = Lrama::ScannerFSA.new([token_pattern]) + + results = fsa.scan("a") + accepting_state = results[0][:state] + + expect(fsa.state_to_accepting_state(accepting_state.id)).to eq(accepting_state) + end + end +end From f1e5413a43272d77272516e06fc3d8695ac586b4 Mon Sep 17 00:00:00 2001 From: ydah Date: Wed, 8 Jul 2026 21:12:38 +0900 Subject: [PATCH 03/15] pslr: add length precedence resolution tables LengthPrecedences captures the length component of %lex-prec rules as a pairwise resolution table (Def 3.2.15): longest-match operators (<~ -~) prefer the longer match in both directions, shortest-match operators ( void + def initialize(operator, lineno) + @operator = operator + @lineno = lineno + end + end + + attr_reader :table #: Hash[[String, String], bool] + attr_reader :resolution_table #: Hash[[String, String], Symbol] + + # @rbs (Grammar::LexPrec lex_prec) -> void + def initialize(lex_prec) + @lex_prec = lex_prec + @rule_index_table = {} #: Hash[[String, String], Integer] + @resolution_table = build_resolution_table(lex_prec) + @table = @resolution_table.transform_values {|value| value == PREFER_NEW } + end + + # @rbs (String old_token, String new_token) -> bool + def normal_precedes?(old_token, new_token) + resolution(old_token, new_token) == PREFER_NEW + end + + # @rbs (String old_token, String new_token) -> bool + def precedes?(old_token, new_token) + normal_precedes?(old_token, new_token) + end + + # @rbs (String old_token, String new_token) -> bool + def fallback_precedes?(old_token, new_token) + resolution(old_token, new_token, fallback: true) == PREFER_NEW + end + + # Backward-compatible query used by existing specs. + # @rbs (String old_token, String new_token) -> bool + def prefer_shorter?(old_token, new_token) + resolution(old_token, new_token) == PREFER_OLD + end + + # @rbs (String old_token, String new_token, ?fallback: bool, ?track: bool) -> Symbol + def resolution(old_token, new_token, fallback: false, track: false) + result = @resolution_table.fetch([old_token, new_token]) do + return PREFER_NEW if old_token == new_token + return PREFER_NEW if fallback + + return UNRESOLVED + end + + if track && (rule_idx = @rule_index_table[[old_token, new_token]]) + @lex_prec.mark_used(rule_idx) + end + + result + end + + # @rbs (String old_token, String new_token) -> Symbol + def precedence(old_token, new_token) + case resolution(old_token, new_token) + when PREFER_NEW + RIGHT + when PREFER_OLD + LEFT + else + UNDEFINED + end + end + + # @rbs (Symbol operator) -> String + def self.operator_label(operator) + case operator + when Grammar::LexPrec::IDENTITY_RIGHT_LONGEST then "<~" + when Grammar::LexPrec::IDENTITY_RIGHT then "<-" + when Grammar::LexPrec::LONGEST then "-~" + when Grammar::LexPrec::TOKEN_RIGHT then "<<" + when Grammar::LexPrec::TOKEN_RIGHT_LENGTH then "-<" + when Grammar::LexPrec::IDENTITY_RIGHT_SHORTEST then " Hash[[String, String], Symbol] + def build_resolution_table(lex_prec) + table = {} + sources = {} + + lex_prec.rules.each_with_index do |rule, idx| + left = rule.left_name + right = rule.right_name + + case rule.operator + when Grammar::LexPrec::IDENTITY_RIGHT_LONGEST, Grammar::LexPrec::LONGEST + set_resolution!(table, sources, [left, right], PREFER_NEW, rule, idx) + set_resolution!(table, sources, [right, left], PREFER_NEW, rule, idx) + when Grammar::LexPrec::IDENTITY_RIGHT_SHORTEST, Grammar::LexPrec::SHORTEST + set_resolution!(table, sources, [left, right], PREFER_OLD, rule, idx) + set_resolution!(table, sources, [right, left], PREFER_OLD, rule, idx) + when Grammar::LexPrec::TOKEN_RIGHT, Grammar::LexPrec::TOKEN_RIGHT_LENGTH + set_resolution!(table, sources, [left, right], PREFER_NEW, rule, idx) + set_resolution!(table, sources, [right, left], PREFER_OLD, rule, idx) + end + end + + table + end + + # @rbs (Hash[[String, String], Symbol] table, Hash[[String, String], RuleSource] sources, [String, String] key, Symbol value, Grammar::LexPrec::Rule rule, Integer rule_index) -> void + def set_resolution!(table, sources, key, value, rule, rule_index) + existing = table[key] + if existing.nil? + table[key] = value + sources[key] = RuleSource.new(rule.operator, rule.lineno) + @rule_index_table[key] = rule_index + return + end + + return if existing == value + + source = sources.fetch(key) + old_token, new_token = key + raise LexicalPrecedenceConflictError, + "conflicting %lex-prec length rules for #{old_token} -> #{new_token}: " \ + "#{operator_label(source.operator)} at line #{source.lineno} resolves #{resolution_label(existing)}, " \ + "but #{operator_label(rule.operator)} at line #{rule.lineno} resolves #{resolution_label(value)}" + end + + # @rbs (Symbol operator) -> String + def operator_label(operator) + case operator + when Grammar::LexPrec::IDENTITY_RIGHT_LONGEST + "<~" + when Grammar::LexPrec::IDENTITY_RIGHT + "<-" + when Grammar::LexPrec::LONGEST + "-~" + when Grammar::LexPrec::TOKEN_RIGHT + "<<" + when Grammar::LexPrec::TOKEN_RIGHT_LENGTH + "-<" + when Grammar::LexPrec::IDENTITY_RIGHT_SHORTEST + " String + def resolution_label(value) + case value + when PREFER_NEW + "prefer-new" + when PREFER_OLD + "prefer-old" + else + value.to_s + end + end + end +end diff --git a/sig/generated/lrama/length_precedences.rbs b/sig/generated/lrama/length_precedences.rbs new file mode 100644 index 000000000..ca4c737d5 --- /dev/null +++ b/sig/generated/lrama/length_precedences.rbs @@ -0,0 +1,76 @@ +# Generated from lib/lrama/length_precedences.rb with RBS::Inline + +module Lrama + # Runtime length precedence matrix for PSLR pseudo-scanning. + # + # When a longer match for new_token is reached after an earlier match for + # old_token, #precedes? answers whether the longer match should replace it. + class LengthPrecedences + LEFT: Symbol + + RIGHT: Symbol + + UNDEFINED: Symbol + + PREFER_NEW: Symbol + + PREFER_OLD: Symbol + + UNRESOLVED: Symbol + + class LexicalPrecedenceConflictError < StandardError + end + + class RuleSource + attr_reader operator: Symbol + + attr_reader lineno: Integer + + # @rbs (Symbol operator, Integer lineno) -> void + def initialize: (Symbol operator, Integer lineno) -> void + end + + attr_reader table: Hash[[ String, String ], bool] + + attr_reader resolution_table: Hash[[ String, String ], Symbol] + + # @rbs (Grammar::LexPrec lex_prec) -> void + def initialize: (Grammar::LexPrec lex_prec) -> void + + # @rbs (String old_token, String new_token) -> bool + def normal_precedes?: (String old_token, String new_token) -> bool + + # @rbs (String old_token, String new_token) -> bool + def precedes?: (String old_token, String new_token) -> bool + + # @rbs (String old_token, String new_token) -> bool + def fallback_precedes?: (String old_token, String new_token) -> bool + + # Backward-compatible query used by existing specs. + # @rbs (String old_token, String new_token) -> bool + def prefer_shorter?: (String old_token, String new_token) -> bool + + # @rbs (String old_token, String new_token, ?fallback: bool, ?track: bool) -> Symbol + def resolution: (String old_token, String new_token, ?fallback: bool, ?track: bool) -> Symbol + + # @rbs (String old_token, String new_token) -> Symbol + def precedence: (String old_token, String new_token) -> Symbol + + # @rbs (Symbol operator) -> String + def self.operator_label: (Symbol operator) -> String + + private + + # @rbs (Grammar::LexPrec lex_prec) -> Hash[[String, String], Symbol] + def build_resolution_table: (Grammar::LexPrec lex_prec) -> Hash[[ String, String ], Symbol] + + # @rbs (Hash[[String, String], Symbol] table, Hash[[String, String], RuleSource] sources, [String, String] key, Symbol value, Grammar::LexPrec::Rule rule, Integer rule_index) -> void + def set_resolution!: (Hash[[ String, String ], Symbol] table, Hash[[ String, String ], RuleSource] sources, [ String, String ] key, Symbol value, Grammar::LexPrec::Rule rule, Integer rule_index) -> void + + # @rbs (Symbol operator) -> String + def operator_label: (Symbol operator) -> String + + # @rbs (Symbol value) -> String + def resolution_label: (Symbol value) -> String + end +end diff --git a/spec/lrama/length_precedences_spec.rb b/spec/lrama/length_precedences_spec.rb new file mode 100644 index 000000000..c8eef01d0 --- /dev/null +++ b/spec/lrama/length_precedences_spec.rb @@ -0,0 +1,129 @@ +# frozen_string_literal: true + +RSpec.describe Lrama::LengthPrecedences do + let(:lex_prec) { Lrama::Grammar::LexPrec.new } + + def ident(name) + Lrama::Lexer::Token::Ident.new(s_value: name) + end + + def add_rule(left, operator, right, lineno) + lex_prec.add_rule( + left_token: ident(left), + operator: operator, + right_token: ident(right), + lineno: lineno + ) + end + + describe "#resolution" do + it "defaults same-token autolength conflicts to longest match" do + length_prec = Lrama::LengthPrecedences.new(lex_prec) + + expect(length_prec.resolution("ID", "ID")).to eq(Lrama::LengthPrecedences::PREFER_NEW) + expect(length_prec.precedes?("ID", "ID")).to be true + end + + it "leaves different-token length conflicts unresolved without a rule" do + length_prec = Lrama::LengthPrecedences.new(lex_prec) + + expect(length_prec.resolution("A", "B")).to eq(Lrama::LengthPrecedences::UNRESOLVED) + expect(length_prec.precedence("A", "B")).to eq(:undefined) + end + + it "supports explicit shortest-match precedence" do + lex_prec.add_rule( + left_token: ident("COM"), + operator: Lrama::Grammar::LexPrec::SHORTEST, + right_token: ident("COM"), + lineno: 1 + ) + length_prec = Lrama::LengthPrecedences.new(lex_prec) + + expect(length_prec.resolution("COM", "COM")).to eq(Lrama::LengthPrecedences::PREFER_OLD) + expect(length_prec.prefer_shorter?("COM", "COM")).to be true + end + + it "supports explicit longest-match precedence" do + lex_prec.add_rule( + left_token: ident("ID"), + operator: Lrama::Grammar::LexPrec::LONGEST, + right_token: ident("IF"), + lineno: 1 + ) + length_prec = Lrama::LengthPrecedences.new(lex_prec) + + expect(length_prec.resolution("ID", "IF")).to eq(Lrama::LengthPrecedences::PREFER_NEW) + expect(length_prec.resolution("IF", "ID")).to eq(Lrama::LengthPrecedences::PREFER_NEW) + end + + it "supports right-token length precedence" do + lex_prec.add_rule( + left_token: ident("WORD"), + operator: Lrama::Grammar::LexPrec::TOKEN_RIGHT_LENGTH, + right_token: ident("NON"), + lineno: 1 + ) + length_prec = Lrama::LengthPrecedences.new(lex_prec) + + expect(length_prec.resolution("WORD", "NON")).to eq(Lrama::LengthPrecedences::PREFER_NEW) + expect(length_prec.resolution("NON", "WORD")).to eq(Lrama::LengthPrecedences::PREFER_OLD) + end + end + + describe "#fallback_precedes?" do + it "uses traditional longest match for unspecified fallback length conflicts" do + length_prec = Lrama::LengthPrecedences.new(lex_prec) + + expect(length_prec.normal_precedes?("A", "B")).to be false + expect(length_prec.fallback_precedes?("A", "B")).to be true + end + + it "respects explicit same-token shortest-match precedence" do + add_rule("COM", Lrama::Grammar::LexPrec::SHORTEST, "COM", 1) + length_prec = Lrama::LengthPrecedences.new(lex_prec) + + expect(length_prec.normal_precedes?("COM", "COM")).to be false + expect(length_prec.fallback_precedes?("COM", "COM")).to be false + end + + it "respects explicit right-token length precedence" do + add_rule("WORD", Lrama::Grammar::LexPrec::TOKEN_RIGHT_LENGTH, "NON", 1) + length_prec = Lrama::LengthPrecedences.new(lex_prec) + + expect(length_prec.fallback_precedes?("WORD", "NON")).to be true + expect(length_prec.fallback_precedes?("NON", "WORD")).to be false + end + end + + describe "#initialize" do + it "rejects contradictory shortest and longest rules for the same scan direction" do + add_rule("RANGLE", Lrama::Grammar::LexPrec::SHORTEST, "RSHIFT", 10) + add_rule("RANGLE", Lrama::Grammar::LexPrec::LONGEST, "RSHIFT", 12) + + expect { Lrama::LengthPrecedences.new(lex_prec) } + .to raise_error( + Lrama::LengthPrecedences::LexicalPrecedenceConflictError, + /RANGLE -> RSHIFT.*-s at line 10.*-~ at line 12/m + ) + end + + it "rejects contradictory right-token length winners in reverse declarations" do + add_rule("RANGLE", Lrama::Grammar::LexPrec::TOKEN_RIGHT_LENGTH, "RSHIFT", 20) + add_rule("RSHIFT", Lrama::Grammar::LexPrec::TOKEN_RIGHT_LENGTH, "RANGLE", 21) + + expect { Lrama::LengthPrecedences.new(lex_prec) } + .to raise_error( + Lrama::LengthPrecedences::LexicalPrecedenceConflictError, + /RSHIFT -> RANGLE.*-< at line 20.*-< at line 21/m + ) + end + + it "allows repeated declarations with the same length resolution" do + add_rule("RANGLE", Lrama::Grammar::LexPrec::LONGEST, "RSHIFT", 30) + add_rule("RSHIFT", Lrama::Grammar::LexPrec::LONGEST, "RANGLE", 31) + + expect { Lrama::LengthPrecedences.new(lex_prec) }.not_to raise_error + end + end +end From dc88596476581c72d9b965c2fe39a4852bad2be6 Mon Sep 17 00:00:00 2001 From: ydah Date: Wed, 8 Jul 2026 21:39:04 +0900 Subject: [PATCH 04/15] pslr: integrate pseudo-scanner into state computation Add States#compute_pslr, an IELR(1) pipeline extended with scanner awareness (paper section 3.4): * Phase 2 builds the scanner FSA and length precedence tables, and the lexical tie closure is finalized against the FSA conflict pairs * Phase 3 state splitting additionally requires PSLR compatibility: states merge only when their acceptable-token sets produce the same pseudo-scanner decisions, checked with unfiltered kernel lookaheads so lookahead filtering cannot mask scanner-visible differences * after conflict resolution, the scanner_accepts table is built with profile-based complete-conflict resolution (Def 3.2.17-3.2.20): each parser state row is computed by walking the DFA with (Ts, ts, Tl) conflict profiles; unresolved complete conflicts are recorded instead of falling back to declaration order * a fallback row over the whole token universe is computed for syntax-error handling, completing residual conflicts with traditional longest-match/declaration-order semantics * acc(sp) uses true reduce lookaheads (not default reductions), expands through lexical ties, and always includes layout tokens States#validate! now fails generation on unresolved scanner conflicts and on pslr.max-states / pslr.max-state-ratio guard violations, and warns about useless %lex-prec declarations. %lexer-context classification and context-based splitting are kept isolated as a Lrama/Ruby extension executed after the PSLR core. --- lib/lrama/command.rb | 8 +- lib/lrama/lexer_context_classifier.rb | 161 +++++ lib/lrama/state.rb | 52 +- lib/lrama/state/pslr_inadequacy.rb | 79 +++ lib/lrama/state/scanner_accepts.rb | 489 ++++++++++++++ lib/lrama/states.rb | 611 +++++++++++++++++- .../lrama/lexer_context_classifier.rbs | 78 +++ sig/generated/lrama/state.rbs | 16 + sig/generated/lrama/state/pslr_inadequacy.rbs | 52 ++ sig/generated/lrama/state/scanner_accepts.rbs | 192 ++++++ sig/generated/lrama/states.rbs | 143 ++++ spec/lrama/lexer_context_classifier_spec.rb | 457 +++++++++++++ spec/lrama/state/pslr_inadequacy_spec.rb | 234 +++++++ spec/lrama/state/scanner_accepts_spec.rb | 394 +++++++++++ spec/lrama/states_spec.rb | 497 ++++++++++++++ spec/spec_helper.rb | 3 + spec/support/pslr_family_helper.rb | 159 +++++ 17 files changed, 3611 insertions(+), 14 deletions(-) create mode 100644 lib/lrama/lexer_context_classifier.rb create mode 100644 lib/lrama/state/pslr_inadequacy.rb create mode 100644 lib/lrama/state/scanner_accepts.rb create mode 100644 sig/generated/lrama/lexer_context_classifier.rbs create mode 100644 sig/generated/lrama/state/pslr_inadequacy.rbs create mode 100644 sig/generated/lrama/state/scanner_accepts.rbs create mode 100644 spec/lrama/lexer_context_classifier_spec.rb create mode 100644 spec/lrama/state/pslr_inadequacy_spec.rb create mode 100644 spec/lrama/state/scanner_accepts_spec.rb create mode 100644 spec/support/pslr_family_helper.rb diff --git a/lib/lrama/command.rb b/lib/lrama/command.rb index 17aad1a1c..236dca054 100644 --- a/lib/lrama/command.rb +++ b/lib/lrama/command.rb @@ -30,11 +30,11 @@ def execute_command_workflow text = read_input grammar = build_grammar(text) states, context = compute_status(grammar) + states.validate!(@logger) render_reports(states) if @options.report_file @tracer.trace(grammar) render_diagram(grammar) render_output(context, grammar) - states.validate!(@logger) @warnings.warn(grammar, states) end @@ -84,7 +84,11 @@ def prepare_grammar(grammar) def compute_status(grammar) states = Lrama::States.new(grammar, @tracer) states.compute - states.compute_ielr if grammar.ielr_defined? + if grammar.pslr_defined? + states.compute_pslr + elsif grammar.ielr_defined? + states.compute_ielr + end [states, Lrama::Context.new(states)] end diff --git a/lib/lrama/lexer_context_classifier.rb b/lib/lrama/lexer_context_classifier.rb new file mode 100644 index 000000000..e61c990a1 --- /dev/null +++ b/lib/lrama/lexer_context_classifier.rb @@ -0,0 +1,161 @@ +# rbs_inline: enabled +# frozen_string_literal: true + +module Lrama + # Classifies parser states into lexer context categories. + # + # When LALR states are merged, states from different grammatical contexts + # (e.g., BEG vs CMDARG) share the same state number, making them + # indistinguishable to the lexer. This classifier analyzes kernel items + # to determine the lexer context of each state, enabling context-aware + # state splitting. + # + # Context definitions come from %lexer-context directives in the grammar file. + # Each directive maps a context name to a set of symbols: + # + # %lexer-context BEG keyword_if keyword_unless '(' '[' '{' + # %lexer-context CMDARG tIDENTIFIER tFID tCONSTANT + # + class LexerContextClassifier + # @rbs (Hash[String, Grammar::LexerContext] lexer_contexts, ?Hash[String, Array[String]] expansion_args) -> void + def initialize(lexer_contexts, expansion_args = {}) + @lexer_contexts = lexer_contexts + @expansion_args = expansion_args + @symbol_to_context = build_symbol_to_context_map + @context_names = build_context_names + end + + # Classify a state's kernel items into context groups. + # + # @rbs (State state) -> Hash[Integer, Array[State::Item]] + def classify(state) + groups = {} + + state.kernels.each do |item| + ctx = infer_item_context(item) + groups[ctx] ||= [] + groups[ctx] << item + end + + groups + end + + # Infer the lexer context for a single kernel item. + # + # @rbs (State::Item item) -> Integer + def infer_item_context(item) + # Position 0 means we're at the start of a rule (just entered via GOTO) + return default_beg_context if item.position == 0 + + prev_sym = item.rhs[item.position - 1] + classify_symbol_context(prev_sym) + end + + # Classify context based on the symbol before the dot. + # + # @rbs (Grammar::Symbol sym) -> Integer + def classify_symbol_context(sym) + name = sym.id.s_value + # Also try without surrounding quotes for single-char tokens + bare = name.gsub(/\A["']|["']\z/, "") + + # Direct match + ctx = @symbol_to_context[name] || @symbol_to_context[bare] + return ctx if ctx + + # Fallback: inherit context from parameterized rule expansion arguments + if (arg_names = @expansion_args[name]) + arg_names.each do |arg_name| + ctx = @symbol_to_context[arg_name] + return ctx if ctx + end + end + + 0 + end + + # For backward compatibility with states.rb split logic + # @rbs (Grammar::Symbol sym) -> Integer + def classify_terminal_context(sym) + classify_symbol_context(sym) + end + + # For backward compatibility with states.rb split logic + # @rbs (Grammar::Symbol sym) -> Integer + def classify_nonterminal_context(sym) + classify_symbol_context(sym) + end + + # Return a human-readable name for a context value. + # + # @rbs (Integer ctx) -> String + def context_name(ctx) + return "UNKNOWN" if ctx == 0 + + names = @context_names.select { |flag, _| (ctx & flag) != 0 }.values + names.empty? ? "UNKNOWN" : names.join("|") + end + + # Class-level context_name for use without an instance (e.g., output.rb). + # Requires lexer_contexts to build the name map. + # + # @rbs (Integer ctx, Hash[String, Grammar::LexerContext] lexer_contexts) -> String + def self.context_name(ctx, lexer_contexts) + return "UNKNOWN" if ctx == 0 + + names = [] + lexer_contexts.each_value do |lc| + names << lc.name if (ctx & lc.bitmask) != 0 + end + names.empty? ? "UNKNOWN" : names.join("|") + end + + # All context bitmasks OR'd together (for "is context known?" checks). + # @rbs () -> Integer + def all_contexts_mask + mask = 0 + @lexer_contexts.each_value { |lc| mask |= lc.bitmask } + mask + end + + # Return the ordered list of context definitions. + # @rbs () -> Array[Grammar::LexerContext] + def contexts + @lexer_contexts.values.sort_by(&:index) + end + + private + + # Build a map from symbol name → context bitmask. + # @rbs () -> Hash[String, Integer] + def build_symbol_to_context_map + map = {} + @lexer_contexts.each_value do |lc| + lc.symbols.each do |sym| + name = sym.s_value + # OR the bitmask in case a symbol appears in multiple contexts + map[name] = (map[name] || 0) | lc.bitmask + end + end + map + end + + # Build a map from bitmask value → context name. + # @rbs () -> Hash[Integer, String] + def build_context_names + names = {} + @lexer_contexts.each_value do |lc| + names[lc.bitmask] = lc.name + end + names + end + + # Return the bitmask for the first defined context (used as default for position-0 items). + # Returns 0 if no contexts are defined. + # @rbs () -> Integer + def default_beg_context + first = @lexer_contexts.values.min_by(&:index) + first ? first.bitmask : 0 + end + end +end diff --git a/lib/lrama/state.rb b/lib/lrama/state.rb index 50912e094..826d40b1a 100644 --- a/lib/lrama/state.rb +++ b/lib/lrama/state.rb @@ -4,8 +4,10 @@ require_relative "state/action" require_relative "state/inadequacy_annotation" require_relative "state/item" +require_relative "state/pslr_inadequacy" require_relative "state/reduce_reduce_conflict" require_relative "state/resolved_conflict" +require_relative "state/scanner_accepts" require_relative "state/shift_reduce_conflict" module Lrama @@ -55,6 +57,8 @@ class State attr_accessor :follow_kernel_items #: Hash[Action::Goto, Hash[Item, bool]] attr_accessor :always_follows #: Hash[Action::Goto, Array[Grammar::Symbol]] attr_accessor :goto_follows #: Hash[Action::Goto, Array[Grammar::Symbol]] + attr_accessor :pslr_item_lookahead_set #: lookahead_set? + attr_accessor :lexer_context #: Integer? # @rbs (Integer id, Grammar::Symbol accessing_symbol, Array[Item] kernels) -> void def initialize(id, accessing_symbol, kernels) @@ -78,6 +82,8 @@ def initialize(id, accessing_symbol, kernels) @follow_kernel_items = {} @always_follows = {} @goto_follows = {} + @pslr_item_lookahead_set = nil + @lexer_context = nil @lhs_contributions = {} @lane_items = {} end @@ -147,6 +153,18 @@ def set_look_ahead(rule, look_ahead) reduce.look_ahead = look_ahead end + # @rbs (Action::Reduce reduce) -> Array[Grammar::Symbol] + def acceptable_reduce_lookahead(reduce) + reduce.look_ahead || item_lookahead_set[reduce.item] || [] + end + + # @rbs (Action::Reduce reduce) -> Array[Grammar::Symbol] + def acceptable_pslr_reduce_lookahead(reduce) + return acceptable_reduce_lookahead(reduce) unless @pslr_item_lookahead_set + + @pslr_item_lookahead_set[reduce.item] || acceptable_reduce_lookahead(reduce) + end + # @rbs (Grammar::Rule rule, Hash[Grammar::Symbol, Array[Action::Goto]] sources) -> void def set_look_ahead_sources(rule, sources) reduce = reduces.find do |r| @@ -288,6 +306,16 @@ def split_state? # # @rbs (State next_state) -> lookahead_set def propagate_lookaheads(next_state) + propagate_lookaheads_with_filter(next_state, true) + end + + # @rbs (State next_state) -> lookahead_set + def propagate_lookaheads_without_filter(next_state) + propagate_lookaheads_with_filter(next_state, false) + end + + # @rbs (State next_state, bool apply_filter) -> lookahead_set + def propagate_lookaheads_with_filter(next_state, apply_filter) next_state.kernels.map {|next_kernel| lookahead_sets = if next_kernel.position > 1 @@ -297,7 +325,14 @@ def propagate_lookaheads(next_state) goto_follow_set(next_kernel.lhs) end - [next_kernel, lookahead_sets & next_state.lookahead_set_filters[next_kernel]] + lookahead_sets = + if apply_filter + lookahead_sets & next_state.lookahead_set_filters[next_kernel] + else + lookahead_sets + end + + [next_kernel, lookahead_sets] }.to_h end @@ -441,11 +476,14 @@ def item_lookahead_set [] elsif kernel.position > 1 prev_items = predecessors_with_item(kernel) - prev_items.map {|st, i| st.item_lookahead_set[i] }.reduce([]) {|acc, syms| acc |= syms } + prev_items + .map {|st, i| st.item_lookahead_set[i] } + .compact + .reduce([]) {|acc, syms| acc | syms } elsif kernel.position == 1 prev_state = @predecessors.find {|p| p.transitions.any? {|transition| transition.next_sym == kernel.lhs } } - goto = prev_state.nterm_transitions.find {|goto| goto.next_sym == kernel.lhs } - prev_state.goto_follows[goto] + goto = prev_state&.nterm_transitions&.find {|goto| goto.next_sym == kernel.lhs } + prev_state&.goto_follows&.fetch(goto, []) || [] end [kernel, value] }.to_h @@ -479,11 +517,15 @@ def append_predecessor(prev_state) def goto_follow_set(nterm_token) return [] if nterm_token.accept_symbol? goto = @lalr_isocore.nterm_transitions.find {|g| g.next_sym == nterm_token } + return [] unless goto + + base_terms = Array(@lalr_isocore.always_follows[goto]) @kernels .select {|kernel| @lalr_isocore.follow_kernel_items[goto][kernel] } .map {|kernel| item_lookahead_set[kernel] } - .reduce(@lalr_isocore.always_follows[goto]) {|result, terms| result |= terms } + .compact + .reduce(base_terms) {|result, terms| result | terms } end # Definition 3.8 (Goto Follows Internal Relation) diff --git a/lib/lrama/state/pslr_inadequacy.rb b/lib/lrama/state/pslr_inadequacy.rb new file mode 100644 index 000000000..451458cf5 --- /dev/null +++ b/lib/lrama/state/pslr_inadequacy.rb @@ -0,0 +1,79 @@ +# rbs_inline: enabled +# frozen_string_literal: true + +module Lrama + class State + # PSLR Inadequacy detection + # Based on Section 3.4.3 from the PSLR dissertation + # + # PSLR inadequacy occurs when state merging causes different + # pseudo-scanner behavior + class PslrInadequacy + # Inadequacy types + LR_RELATIVE = :lr_relative #: Symbol + PSLR_RELATIVE = :pslr_relative #: Symbol + + attr_reader :type #: Symbol + attr_reader :state #: State + attr_reader :conflicting_states #: Array[State] + attr_reader :details #: Hash[Symbol, untyped] + + # @rbs (type: Symbol, state: State, conflicting_states: Array[State], details: Hash[Symbol, untyped]) -> void + def initialize(type:, state:, conflicting_states:, details:) + @type = type + @state = state + @conflicting_states = conflicting_states + @details = details + end + + # @rbs () -> String + def to_s + message = "PSLR Inadequacy (#{type}): state #{state.id} conflicts with states #{conflicting_states.map(&:id).join(', ')}" + return message if details[:profiles].nil? + + profiles = details[:profiles].map do |profile, state_ids| + "#{state_ids.join(', ')} => #{profile.inspect}" + end + + "#{message} (profiles: #{profiles.join(' | ')})" + end + end + + # PSLR Compatibility checker + # Based on Definition 3.4.1 from the dissertation + class PslrCompatibilityChecker + # @rbs (ScannerAccepts scanner_accepts, LengthPrecedences length_prec) -> void + def initialize(scanner_accepts, length_prec) + @scanner_accepts = scanner_accepts + @length_prec = length_prec + end + + # Build a stable scanner profile for a parser state + # @rbs (State state, ScannerFSA scanner_fsa) -> Array[[Integer, String?]] + def profile(state, scanner_fsa) + scanner_fsa.states.each_with_object([]) do |fsa_state, result| + next unless fsa_state.accepting? + + token = @scanner_accepts[state.id, fsa_state.id] + result << [fsa_state.id, token&.name] + end + end + + # Partition states by scanner profile + # @rbs (Array[State] states, ScannerFSA scanner_fsa) -> Hash[Array[[Integer, String?]], Array[State]] + def group_by_profile(states, scanner_fsa) + states.group_by do |state| + profile(state, scanner_fsa) + end + end + + # Check if two states are PSLR-compatible + # Definition 3.4.1: States are compatible if for any input, + # the pseudo-scanner selects the same token + # @rbs (State s1, State s2, ScannerFSA scanner_fsa) -> bool + def compatible?(s1, s2, scanner_fsa) + profile(s1, scanner_fsa) == profile(s2, scanner_fsa) + end + end + end +end diff --git a/lib/lrama/state/scanner_accepts.rb b/lib/lrama/state/scanner_accepts.rb new file mode 100644 index 000000000..bb764043c --- /dev/null +++ b/lib/lrama/state/scanner_accepts.rb @@ -0,0 +1,489 @@ +# rbs_inline: enabled +# frozen_string_literal: true + +require "set" + +module Lrama + class State + # scanner_accepts[parser_state, accepting_scanner_state] for PSLR(1). + # + # Construction follows complete pseudo-scanner conflict profiles. A profile + # contains the shorter token set Ts, the selected shorter token ts, and the + # current/longest token set Tl. + # + # Normal parser-state rows are strict: unresolved pseudo-scanner + # conflicts are reported and are not resolved by token declaration order. + # + # The fallback row is used only for syntax-error handling. It first + # applies explicit PSLR lexical precedence declarations. For scanner + # conflicts that remain unresolved only in the fallback universe, it + # completes the decision with traditional scanner behavior: fallback + # length defaults are handled by LengthPrecedences#fallback_precedes?, + # and otherwise-unresolved identity conflicts are resolved by token + # declaration order. + # + # When %lex-scope declarations are present, each parser state may use + # a merged set of global + scope-active lexical precedence rules. + class ScannerAccepts + class Conflict + attr_reader :parser_state_id #: Integer? + attr_reader :scanner_state_id #: Integer + attr_reader :shorter_tokens #: Array[String] + attr_reader :selected_shorter_token #: String? + attr_reader :current_tokens #: Array[String] + + # @rbs (parser_state_id: Integer?, scanner_state_id: Integer, shorter_tokens: Array[String], selected_shorter_token: String?, current_tokens: Array[String]) -> void + def initialize(parser_state_id:, scanner_state_id:, shorter_tokens:, selected_shorter_token:, current_tokens:) + @parser_state_id = parser_state_id + @scanner_state_id = scanner_state_id + @shorter_tokens = shorter_tokens + @selected_shorter_token = selected_shorter_token + @current_tokens = current_tokens + end + end + + class ProfileOutcome + EMPTY = :empty #: Symbol + RESOLVED = :resolved #: Symbol + UNRESOLVED = :unresolved #: Symbol + + attr_reader :kind #: Symbol + attr_reader :token_name #: String? + attr_reader :conflict #: Conflict? + + # @rbs (kind: Symbol, ?token_name: String?, ?conflict: Conflict?) -> void + def initialize(kind:, token_name: nil, conflict: nil) + @kind = kind + @token_name = token_name + @conflict = conflict + end + + # @rbs () -> bool + def empty? + @kind == EMPTY + end + + # @rbs () -> bool + def resolved? + @kind == RESOLVED + end + + # @rbs () -> bool + def unresolved? + @kind == UNRESOLVED + end + end + + class ProfileResolver + # @rbs (Grammar::LexPrec lex_prec, LengthPrecedences length_prec, ?fallback: bool, ?token_order: Hash[String, Integer]) -> void + def initialize(lex_prec, length_prec, fallback: false, token_order: {}) + @lex_prec = lex_prec + @length_prec = length_prec + @fallback = fallback + @token_order = token_order + end + + # @rbs (Set[String] shorter_tokens, String? selected_shorter_token, Set[String] current_tokens) -> ProfileOutcome + def resolve(shorter_tokens, selected_shorter_token, current_tokens) + if @fallback + resolve_fallback(shorter_tokens, selected_shorter_token, current_tokens) + else + resolve_normal(shorter_tokens, selected_shorter_token, current_tokens) + end + end + + private + + # @rbs (Set[String] shorter_tokens, String? selected_shorter_token, Set[String] current_tokens) -> ProfileOutcome + def resolve_normal(shorter_tokens, selected_shorter_token, current_tokens) + if current_tokens.empty? + return ProfileOutcome.new(kind: ProfileOutcome::RESOLVED, token_name: selected_shorter_token) if selected_shorter_token + + return ProfileOutcome.new(kind: ProfileOutcome::EMPTY) + end + + if selected_shorter_token && current_tokens.all? {|token| length_prefers_old?(selected_shorter_token, token) } + return ProfileOutcome.new(kind: ProfileOutcome::RESOLVED, token_name: selected_shorter_token) + end + + winners = current_tokens.select do |candidate| + identity_winner?(candidate, current_tokens) && + shorter_tokens.all? {|shorter| @length_prec.resolution(shorter, candidate) == LengthPrecedences::PREFER_NEW } + end + + return ProfileOutcome.new(kind: ProfileOutcome::RESOLVED, token_name: winners.first) if winners.size == 1 + + ProfileOutcome.new(kind: ProfileOutcome::UNRESOLVED) + end + + # @rbs (Set[String] shorter_tokens, String? selected_shorter_token, Set[String] current_tokens) -> ProfileOutcome + def resolve_fallback(shorter_tokens, selected_shorter_token, current_tokens) + if current_tokens.empty? + return ProfileOutcome.new(kind: ProfileOutcome::RESOLVED, token_name: selected_shorter_token) if selected_shorter_token + + return ProfileOutcome.new(kind: ProfileOutcome::EMPTY) + end + + if selected_shorter_token && current_tokens.all? {|token| fallback_length_prefers_old?(selected_shorter_token, token) } + return ProfileOutcome.new(kind: ProfileOutcome::RESOLVED, token_name: selected_shorter_token) + end + + explicit_winners = current_tokens.select do |candidate| + fallback_identity_winner?(candidate, current_tokens) && + shorter_tokens.all? {|shorter| @length_prec.fallback_precedes?(shorter, candidate) } + end + + return ProfileOutcome.new(kind: ProfileOutcome::RESOLVED, token_name: explicit_winners.first) if explicit_winners.size == 1 + + declaration_order_winner = current_tokens.sort_by {|token| token_order_key(token) }.find do |candidate| + shorter_tokens.all? {|shorter| @length_prec.fallback_precedes?(shorter, candidate) } + end + return ProfileOutcome.new(kind: ProfileOutcome::RESOLVED, token_name: declaration_order_winner) if declaration_order_winner + + return ProfileOutcome.new(kind: ProfileOutcome::RESOLVED, token_name: selected_shorter_token) if selected_shorter_token + + ProfileOutcome.new(kind: ProfileOutcome::UNRESOLVED) + end + + # @rbs (String old_token, String new_token) -> bool + def length_prefers_old?(old_token, new_token) + @length_prec.resolution(old_token, new_token) == LengthPrecedences::PREFER_OLD + end + + # @rbs (String old_token, String new_token) -> bool + def fallback_length_prefers_old?(old_token, new_token) + !@length_prec.fallback_precedes?(old_token, new_token) + end + + # @rbs (String candidate, Set[String] current_tokens) -> bool + def identity_winner?(candidate, current_tokens) + current_tokens.all? do |other| + candidate == other || @lex_prec.identity_precedes?(candidate, other) + end + end + + # @rbs (String candidate, Set[String] current_tokens) -> bool + def fallback_identity_winner?(candidate, current_tokens) + current_tokens.all? do |other| + next true if candidate == other + + candidate_wins = @lex_prec.identity_precedes?(candidate, other) + other_wins = @lex_prec.identity_precedes?(other, candidate) + if candidate_wins && !other_wins + true + elsif other_wins + false + else + token_order_precedes?(candidate, other) + end + end + end + + # @rbs (String candidate, String other) -> bool + def token_order_precedes?(candidate, other) + (token_order_key(candidate) <=> token_order_key(other)) == -1 + end + + # @rbs (String token) -> [Integer, String] + def token_order_key(token) + [@token_order.fetch(token, @token_order.size), token] + end + end + + class CompleteProfileComputer + attr_reader :table #: Hash[Integer, Grammar::TokenPattern] + attr_reader :conflicts #: Array[Conflict] + + # @rbs (ScannerFSA scanner_fsa, Grammar::LexPrec lex_prec, LengthPrecedences length_prec, Set[String] acceptable_tokens, ?Integer? parser_state_id, ?fallback: bool, ?token_order: Hash[String, Integer]) -> void + def initialize(scanner_fsa, lex_prec, length_prec, acceptable_tokens, parser_state_id = nil, fallback: false, token_order: {}) + @scanner_fsa = scanner_fsa + @resolver = ProfileResolver.new(lex_prec, length_prec, fallback: fallback, token_order: token_order) + @acceptable_tokens = acceptable_tokens + @parser_state_id = parser_state_id + @table = {} + @conflicts = [] + token_patterns_by_name = {} #: Hash[String, Grammar::TokenPattern] + @token_patterns_by_name = scanner_fsa.token_patterns.each_with_object(token_patterns_by_name) do |token_pattern, hash| + hash[token_pattern.name] ||= token_pattern + end + end + + # @rbs () -> void + def compute + visited = Set.new + visit_transitions(0, Set.new, nil, visited) + end + + private + + # @rbs (Integer fsa_state_id, Set[String] shorter_tokens, String? selected_shorter_token, Set[untyped] visited) -> void + def visit_transitions(fsa_state_id, shorter_tokens, selected_shorter_token, visited) + fsa_state = @scanner_fsa.states[fsa_state_id] + return unless fsa_state + + fsa_state.transitions.each_value do |next_state_id| + visit_state(next_state_id, shorter_tokens, selected_shorter_token, visited) + end + end + + # @rbs (Integer fsa_state_id, Set[String] shorter_tokens, String? selected_shorter_token, Set[untyped] visited) -> void + def visit_state(fsa_state_id, shorter_tokens, selected_shorter_token, visited) + fsa_state = @scanner_fsa.states[fsa_state_id] + return unless fsa_state + + current_tokens = current_acceptable_tokens(fsa_state) + key = profile_key(fsa_state_id, shorter_tokens, selected_shorter_token, current_tokens) + return if visited.include?(key) + + visited << key + result = @resolver.resolve(shorter_tokens, selected_shorter_token, current_tokens) + + if result.resolved? + if result.token_name && current_tokens.include?(result.token_name) + token_pattern = token_pattern_for(result.token_name) + existing = @table[fsa_state_id] + if existing && existing.name != token_pattern.name + @conflicts << Conflict.new( + parser_state_id: @parser_state_id, + scanner_state_id: fsa_state_id, + shorter_tokens: shorter_tokens.to_a.sort, + selected_shorter_token: existing.name, + current_tokens: current_tokens.to_a.sort + ) + else + @table[fsa_state_id] = token_pattern + end + end + elsif result.unresolved? + @conflicts << Conflict.new( + parser_state_id: @parser_state_id, + scanner_state_id: fsa_state_id, + shorter_tokens: shorter_tokens.to_a.sort, + selected_shorter_token: selected_shorter_token, + current_tokens: current_tokens.to_a.sort + ) + end + + next_shorter_tokens = shorter_tokens | current_tokens + next_selected = result.resolved? ? result.token_name : nil + visit_transitions(fsa_state_id, next_shorter_tokens, next_selected, visited) + end + + # @rbs (ScannerFSA::State fsa_state) -> Set[String] + def current_acceptable_tokens(fsa_state) + fsa_state.accepting_tokens.each_with_object(Set.new) do |token_pattern, tokens| + tokens << token_pattern.name if @acceptable_tokens.include?(token_pattern.name) + end + end + + # @rbs (String token_name) -> Grammar::TokenPattern + def token_pattern_for(token_name) + @token_patterns_by_name.fetch(token_name) + end + + # @rbs (Integer fsa_state_id, Set[String] shorter_tokens, String? selected_shorter_token, Set[String] current_tokens) -> [Integer, Array[String], String?, Array[String]] + def profile_key(fsa_state_id, shorter_tokens, selected_shorter_token, current_tokens) + [fsa_state_id, shorter_tokens.to_a.sort, selected_shorter_token, current_tokens.to_a.sort] + end + end + + class CompatibilityChecker + # @rbs (ScannerFSA scanner_fsa, Grammar::LexPrec lex_prec, LengthPrecedences length_prec) -> void + def initialize(scanner_fsa, lex_prec, length_prec) + @scanner_fsa = scanner_fsa + @resolver = ProfileResolver.new(lex_prec, length_prec) + end + + # @rbs (Set[String] left_tokens, Set[String] right_tokens) -> bool + def compatible?(left_tokens, right_tokens) + visit_transitions(0, Set.new, nil, Set.new, nil, left_tokens, right_tokens, Set.new) + end + + private + + # @rbs (Integer fsa_state_id, Set[String] left_shorter, String? left_selected, Set[String] right_shorter, String? right_selected, Set[String] left_acc, Set[String] right_acc, Set[untyped] visited) -> bool + def visit_transitions(fsa_state_id, left_shorter, left_selected, right_shorter, right_selected, left_acc, right_acc, visited) + fsa_state = @scanner_fsa.states[fsa_state_id] + return true unless fsa_state + + fsa_state.transitions.each_value do |next_state_id| + return false unless visit_state(next_state_id, left_shorter, left_selected, right_shorter, right_selected, left_acc, right_acc, visited) + end + + true + end + + # @rbs (Integer fsa_state_id, Set[String] left_shorter, String? left_selected, Set[String] right_shorter, String? right_selected, Set[String] left_acc, Set[String] right_acc, Set[untyped] visited) -> bool + def visit_state(fsa_state_id, left_shorter, left_selected, right_shorter, right_selected, left_acc, right_acc, visited) + fsa_state = @scanner_fsa.states[fsa_state_id] + return true unless fsa_state + + left_current = tokens_accepted_by(fsa_state, left_acc) + right_current = tokens_accepted_by(fsa_state, right_acc) + left_outcome = @resolver.resolve(left_shorter, left_selected, left_current) + right_outcome = @resolver.resolve(right_shorter, right_selected, right_current) + + return false unless outcomes_compatible?(left_outcome, right_outcome) + + key = [ + fsa_state_id, + left_shorter.to_a.sort, + left_outcome.kind, + left_outcome.token_name, + right_shorter.to_a.sort, + right_outcome.kind, + right_outcome.token_name + ] + return true if visited.include?(key) + + visited << key + + visit_transitions( + fsa_state_id, + left_shorter | left_current, + left_outcome.resolved? ? left_outcome.token_name : nil, + right_shorter | right_current, + right_outcome.resolved? ? right_outcome.token_name : nil, + left_acc, + right_acc, + visited + ) + end + + # @rbs (ScannerFSA::State fsa_state, Set[String] accepted_names) -> Set[String] + def tokens_accepted_by(fsa_state, accepted_names) + fsa_state.accepting_tokens.each_with_object(Set.new) do |token_pattern, tokens| + tokens << token_pattern.name if accepted_names.include?(token_pattern.name) + end + end + + # @rbs (ProfileOutcome left, ProfileOutcome right) -> bool + def outcomes_compatible?(left, right) + return true if left.empty? || right.empty? + return true if left.unresolved? && right.unresolved? + return false if left.unresolved? || right.unresolved? + + left.token_name == right.token_name + end + end + + FALLBACK_ROW_ID = -1 #: Integer + + attr_reader :table #: Hash[[Integer, Integer], Grammar::TokenPattern?] + attr_reader :fallback_table #: Hash[Integer, Grammar::TokenPattern] + attr_reader :conflicts #: Array[Conflict] + + # @rbs (Array[State] parser_states, ScannerFSA scanner_fsa, Grammar::LexPrec lex_prec, LengthPrecedences length_prec, ?Grammar::LexTie? lex_tie, ?layout_token_names: Set[String]) -> void + def initialize(parser_states, scanner_fsa, lex_prec, length_prec, lex_tie = nil, layout_token_names: Set.new) + @parser_states = parser_states + @scanner_fsa = scanner_fsa + @lex_prec = lex_prec + @length_prec = length_prec + @lex_tie = lex_tie + @layout_token_names = layout_token_names.to_set + @table = {} + @fallback_table = {} + @conflicts = [] + end + + # @rbs (ScannerFSA scanner_fsa, Grammar::LexPrec lex_prec, LengthPrecedences length_prec, Set[String] acceptable_tokens) -> [Hash[Integer, Grammar::TokenPattern], Array[Conflict]] + def self.compute_for_acceptable_tokens(scanner_fsa, lex_prec, length_prec, acceptable_tokens) + computer = CompleteProfileComputer.new(scanner_fsa, lex_prec, length_prec, acceptable_tokens) + computer.compute + [computer.table, computer.conflicts] + end + + # @rbs () -> void + def build + @parser_states.each do |parser_state| + compute_for_parser_state(parser_state) + end + compute_fallback_row + end + + # @rbs (Integer parser_state_id, Integer accepting_state_id) -> Grammar::TokenPattern? + def [](parser_state_id, accepting_state_id) + @table[[parser_state_id, accepting_state_id]] + end + + # @rbs () -> bool + def unresolved_conflicts? + !@conflicts.empty? + end + + private + + # @rbs (State parser_state) -> void + def compute_for_parser_state(parser_state) + computer = CompleteProfileComputer.new( + @scanner_fsa, + @lex_prec, + @length_prec, + compute_acc_sp(parser_state), + parser_state.id + ) + computer.compute + + computer.table.each do |scanner_state_id, token_pattern| + @table[[parser_state.id, scanner_state_id]] = token_pattern + end + @conflicts.concat(computer.conflicts) + end + + # @rbs () -> void + def compute_fallback_row + all_tokens = @scanner_fsa.token_patterns.map(&:name).to_set + computer = CompleteProfileComputer.new( + @scanner_fsa, + @lex_prec, + @length_prec, + all_tokens, + FALLBACK_ROW_ID, + fallback: true, + token_order: token_order + ) + computer.compute + + computer.table.each do |scanner_state_id, token_pattern| + @fallback_table[scanner_state_id] = token_pattern + @table[[FALLBACK_ROW_ID, scanner_state_id]] = token_pattern + end + end + + # @rbs (State parser_state) -> Set[String] + def compute_acc_sp(parser_state) + tokens = Set.new + + parser_state.term_transitions.each do |shift| + next_sym = shift.next_sym + tokens << next_sym.id.s_value if next_sym.term? + end + + parser_state.reduces.each do |reduce| + parser_state.acceptable_pslr_reduce_lookahead(reduce).each do |la| + tokens << la.id.s_value + end + end + + expand_lexical_ties(tokens) | @layout_token_names + end + + # @rbs (Set[String] tokens) -> Set[String] + def expand_lexical_ties(tokens) + return tokens unless @lex_tie + + tokens.each_with_object(Set.new) do |token, expanded| + @lex_tie.tied_names(token).each {|name| expanded << name } + end + end + + # @rbs () -> Hash[String, Integer] + def token_order + initial = {} #: Hash[String, Integer] + @scanner_fsa.token_patterns.each_with_object(initial) do |token_pattern, order| + order[token_pattern.name] ||= token_pattern.definition_order + end + end + end + end +end diff --git a/lib/lrama/states.rb b/lib/lrama/states.rb index ddce627df..af34aefa4 100644 --- a/lib/lrama/states.rb +++ b/lib/lrama/states.rb @@ -2,6 +2,8 @@ # frozen_string_literal: true require "forwardable" +require "set" +require_relative "lexer_context_classifier" require_relative "tracer/duration" require_relative "state/item" @@ -36,12 +38,20 @@ class States include Lrama::Tracer::Duration def_delegators "@grammar", :symbols, :terms, :nterms, :rules, :precedences, - :accept_symbol, :eof_symbol, :undef_symbol, :find_symbol_by_s_value!, :ielr_defined? + :accept_symbol, :eof_symbol, :undef_symbol, :find_symbol_by_s_value!, :ielr_defined?, :pslr_defined?, + :token_patterns, :lex_prec, :lex_tie, :pslr_max_states, :pslr_max_state_ratio attr_reader :states #: Array[State] attr_reader :reads_relation #: Hash[State::Action::Goto, Array[State::Action::Goto]] attr_reader :includes_relation #: Hash[State::Action::Goto, Array[State::Action::Goto]] attr_reader :lookback_relation #: Hash[state_id, Hash[rule_id, Array[State::Action::Goto]]] + attr_reader :scanner_fsa #: ScannerFSA? + attr_reader :length_precedences #: LengthPrecedences? + attr_reader :scanner_accepts_table #: State::ScannerAccepts? + attr_reader :pslr_inadequacies #: Array[State::PslrInadequacy] + attr_reader :pslr_metrics #: Hash[Symbol, Integer | Float | nil] + attr_reader :lexer_context_classifier #: LexerContextClassifier? + attr_reader :lexical_tie_candidates #: Array[[String, String]] # @rbs (Grammar grammar, Tracer tracer) -> void def initialize(grammar, tracer) @@ -105,6 +115,18 @@ def initialize(grammar, tracer) # second key is rule_id, # value is bitmap of term. @la = {} + @pslr_inadequacies = [] + @lexical_tie_candidates = [] + @pslr_metrics = { + base_states_count: nil, + total_states_count: nil, + split_state_count: 0, + growth_count: 0, + growth_ratio: nil, + token_pattern_count: 0, + scanner_fsa_state_count: 0, + inadequacies_count: 0 + } end # @rbs () -> void @@ -141,6 +163,49 @@ def compute_ielr report_duration(:compute_default_reduction) { compute_default_reduction } end + # Compute PSLR(1) states + # Based on Section 3.4 of the PSLR dissertation + # @rbs () -> void + def compute_pslr + capture_pslr_metrics_before_split + # Preparation + report_duration(:clear_conflicts) { clear_conflicts } + # Phase 1 + report_duration(:compute_predecessors) { compute_predecessors } + report_duration(:compute_follow_kernel_items) { compute_follow_kernel_items } + report_duration(:compute_always_follows) { compute_always_follows } + report_duration(:compute_goto_follows) { compute_goto_follows } + # Phase 2 + report_duration(:build_scanner_fsa) { build_scanner_fsa } + report_duration(:build_length_precedences) { build_length_precedences } + report_duration(:compute_inadequacy_annotations) { compute_inadequacy_annotations } + # Phase 3a: PSLR split (Scanner FSA-based) + @pslr_split_enabled = true + report_duration(:split_states) { split_states } + @pslr_split_enabled = false + # Phase 3b: Lexer context classification + context-based split. + # Lrama/Ruby extension, not part of the PSLR dissertation core. + # No-op unless %lexer-context directives are present. + if @grammar.lexer_contexts.any? + report_duration(:classify_lexer_contexts) { classify_lexer_contexts } + report_duration(:split_states_by_context) { split_states_by_context } + end + # Phase 4 + report_duration(:clear_look_ahead_sets) { clear_look_ahead_sets } + report_duration(:compute_look_ahead_sets) { compute_look_ahead_sets } + # Phase 5 + report_duration(:compute_conflicts) { compute_conflicts(:ielr) } + report_duration(:compute_default_reduction) { compute_default_reduction } + report_duration(:build_scanner_accepts) { build_scanner_accepts } + report_duration(:handle_pslr_inadequacies) { handle_pslr_inadequacies } + # Phase 6: Re-classify after all splits. + # Lrama/Ruby extension, not part of the PSLR dissertation core. + if @grammar.lexer_contexts.any? + report_duration(:classify_lexer_contexts) { classify_lexer_contexts } + end + finalize_pslr_metrics + end + # @rbs () -> Integer def states_count @states.count @@ -189,11 +254,60 @@ def rr_conflicts_count # @rbs (Logger logger) -> void def validate!(logger) validate_conflicts_within_threshold!(logger) + validate_pslr_state_growth!(logger) + validate_pslr_scanner_conflicts!(logger) + validate_pslr_inadequacies!(logger) + validate_pslr_useless_lex_prec!(logger) + validate_pslr_lexical_tie_candidates!(logger) + end + + # Classify each state's lexer context based on kernel items. + # + # For each state, analyzes the kernel items to determine what lexer + # context (BEG, CMDARG, ARG, END, ENDFN, MID, DOT) the state belongs to. + # When a state has kernel items from multiple contexts, the context is + # set to the bitwise OR of all contexts (mixed context). + # + # @rbs () -> void + def classify_lexer_contexts + return if @grammar.lexer_contexts.empty? + + @lexer_context_classifier = LexerContextClassifier.new( + @grammar.lexer_contexts, + @grammar.parameterized_expansion_args + ) + + @states.each do |state| + groups = @lexer_context_classifier.classify(state) + + # Combine all contexts into a single bitmask + combined = 0 + groups.each_key do |ctx| + combined |= ctx if ctx > 0 + end + + state.lexer_context = combined + end + end + + # Return the lexer context table as an array of context values, + # one per parser state (indexed by state id). + # + # @rbs () -> Array[Integer] + def lexer_context_table + @states.map { |state| state.lexer_context || 0 } + end + + # Check if lexer context classification has been performed. + # + # @rbs () -> bool + def lexer_context_enabled? + pslr_defined? && @lexer_context_classifier != nil end def compute_la_sources_for_conflicted_states reflexive = {} - @states.each do |state| + reachable_parser_states.each do |state| state.nterm_transitions.each do |goto| reflexive[goto] = [goto] end @@ -755,12 +869,139 @@ def compute_always_follows_bitmaps # @rbs () -> void def split_states @states.each do |state| - state.transitions.each do |transition| + state.transitions.dup.each do |transition| compute_state(state, transition, transition.to_state) end end end + # Split states where different predecessor paths lead to different + # lexer contexts. This resolves LALR state merging that makes + # BEG vs CMDARG (and other context pairs) indistinguishable. + # + # Algorithm: + # 1. For each state, group incoming transitions by the lexer context + # that the predecessor would imply + # 2. If a state has predecessors from multiple different contexts, + # split the state so each split has a unique context + # + # @rbs () -> void + def split_states_by_context + return unless @lexer_context_classifier + + # Iterate over a snapshot of states (new states may be added) + states_snapshot = @states.dup + + states_snapshot.each do |state| + # Skip start state and states with no context + next if state.kernels.any?(&:start_item?) + + # Group predecessor transitions by the context they imply + context_groups = compute_predecessor_context_groups(state) + + # Only split if there are multiple distinct non-zero contexts + meaningful_groups = context_groups.reject { |ctx, _| ctx == 0 } + next if meaningful_groups.size <= 1 + + # The largest group keeps the original state + primary_ctx, = meaningful_groups.max_by { |_, transitions| transitions.size } + + meaningful_groups.each do |ctx, transitions| + next if ctx == primary_ctx + + # Create a new split state for this context group + split = create_context_split_state(state) + split.lexer_context = ctx + + # Update predecessor transitions to point to the new split state + transitions.each do |pred_state, transition| + pred_state.update_transition(transition, split) + end + end + + # Update the original state's context to the primary + state.lexer_context = primary_ctx + end + end + + # For a given state, group its incoming transitions by the lexer context + # that the predecessor state implies for this state. + # + # The implied context is determined by what symbol was used to reach + # this state (the accessing symbol's context). + # + # @rbs (State state) -> Hash[Integer, Array[[State, State::Action::Shift | State::Action::Goto]]] + def compute_predecessor_context_groups(state) + groups = Hash.new { |h, k| h[k] = [] } + + state.predecessors.each do |pred| + pred.transitions.each do |transition| + next unless transition.to_state == state + + # The context is determined by the predecessor's context + # combined with what we're transitioning on + ctx = infer_transition_context(pred, transition) + groups[ctx] << [pred, transition] + end + end + + groups + end + + # Infer the lexer context that a transition implies for the target state. + # + # @rbs (State pred, State::Action::Shift | State::Action::Goto transition) -> Integer + def infer_transition_context(pred, transition) + sym = transition.next_sym + if sym.term? + @lexer_context_classifier.classify_terminal_context(sym) + else + @lexer_context_classifier.classify_nonterminal_context(sym) + end + end + + # Create a new split state that is an isocore copy of the given state. + # + # @rbs (State original) -> State + def create_context_split_state(original) + base = original.lalr_isocore || original + new_state = State.new(@states.count, base.accessing_symbol, base.kernels) + new_state.closure = base.closure + new_state.compute_transitions_and_reduces + + # Copy transition targets from original + original.transitions.each do |transition| + new_state.set_items_to_state(transition.to_items, transition.to_state) + end + + @states << new_state + new_state.lalr_isocore = base + base.ielr_isocores << new_state + base.ielr_isocores.each do |st| + st.ielr_isocores = base.ielr_isocores + end + + new_state.lookaheads_recomputed = true + new_state.item_lookahead_set = original.item_lookahead_set + new_state.pslr_item_lookahead_set = original.pslr_item_lookahead_set + + new_state + end + + # @rbs () -> void + def capture_pslr_metrics_before_split + @pslr_metrics = { + base_states_count: @states.count, + total_states_count: @states.count, + split_state_count: 0, + growth_count: 0, + growth_ratio: 1.0, + token_pattern_count: token_patterns.size, + scanner_fsa_state_count: 0, + inadequacies_count: 0 + } + end + # @rbs () -> void def compute_inadequacy_annotations @states.each do |state| @@ -782,17 +1023,32 @@ def compute_inadequacy_annotations def merge_lookaheads(state, filtered_lookaheads) return if state.kernels.all? {|item| (filtered_lookaheads[item] - state.item_lookahead_set[item]).empty? } - state.item_lookahead_set = state.item_lookahead_set.merge {|_, v1, v2| v1 | v2 } + state.item_lookahead_set = state.item_lookahead_set.merge(filtered_lookaheads) {|_, v1, v2| v1 | v2 } state.transitions.each do |transition| next if transition.to_state.lookaheads_recomputed compute_state(state, transition, transition.to_state) end end + # @rbs (State state, State::lookahead_set pslr_lookaheads) -> void + def merge_pslr_lookaheads(state, pslr_lookaheads) + state.pslr_item_lookahead_set ||= state.kernels.map {|kernel| [kernel, []] }.to_h + return if state.kernels.all? {|item| (pslr_lookaheads[item] - state.pslr_item_lookahead_set[item]).empty? } + + state.pslr_item_lookahead_set = state.pslr_item_lookahead_set.merge(pslr_lookaheads) {|_, v1, v2| v1 | v2 } + end + # @rbs (State state, State::Action::Shift | State::Action::Goto transition, State next_state) -> void def compute_state(state, transition, next_state) propagating_lookaheads = state.propagate_lookaheads(next_state) - s = next_state.ielr_isocores.find {|st| st.is_compatible?(propagating_lookaheads) } + pslr_lookaheads = + if @pslr_split_enabled + state.propagate_lookaheads_without_filter(next_state) + else + propagating_lookaheads + end + + s = next_state.ielr_isocores.find {|st| compatible_split_state?(st, propagating_lookaheads, pslr_lookaheads) } if s.nil? s = next_state.lalr_isocore @@ -809,17 +1065,109 @@ def compute_state(state, transition, next_state) st.ielr_isocores = s.ielr_isocores end new_state.lookaheads_recomputed = true - new_state.item_lookahead_set = propagating_lookaheads + new_state.item_lookahead_set = pslr_lookaheads + new_state.pslr_item_lookahead_set = pslr_lookaheads state.update_transition(transition, new_state) elsif(!s.lookaheads_recomputed) s.lookaheads_recomputed = true - s.item_lookahead_set = propagating_lookaheads + s.item_lookahead_set = pslr_lookaheads + s.pslr_item_lookahead_set = pslr_lookaheads else + merge_pslr_lookaheads(s, pslr_lookaheads) if @pslr_split_enabled merge_lookaheads(s, propagating_lookaheads) state.update_transition(transition, s) if state.items_to_state[transition.to_items].id != s.id end end + # @rbs (State state, State::lookahead_set filtered_lookaheads, ?State::lookahead_set pslr_lookaheads) -> bool + def compatible_split_state?(state, filtered_lookaheads, pslr_lookaheads = nil) + return false unless state.is_compatible?(filtered_lookaheads) + return true unless @pslr_split_enabled && @scanner_fsa + + pslr_lookaheads ||= filtered_lookaheads + + existing_acc = acceptable_tokens_for_pslr(state) + candidate_acc = acceptable_tokens_for_pslr(state, pslr_lookaheads) + pslr_compatible_accept_sets?(existing_acc, candidate_acc) + end + + # @rbs (State state, ?State::lookahead_set filtered_lookaheads) -> Array[[Integer, String?]] + def pslr_state_signature(state, filtered_lookaheads = nil) + return [] unless @scanner_fsa + + acc_sp = acceptable_tokens_for_pslr(state, filtered_lookaheads) + table, conflicts = State::ScannerAccepts.compute_for_acceptable_tokens( + @scanner_fsa, + lex_prec, + @length_precedences || LengthPrecedences.new(lex_prec), + acc_sp + ) + + signature = @scanner_fsa.states.each_with_object([]) do |fsa_state, result| + next unless fsa_state.accepting? + + result << [fsa_state.id, table[fsa_state.id]&.name] + end + + conflicts.each do |conflict| + signature << [ + :unresolved, + conflict.scanner_state_id, + conflict.shorter_tokens, + conflict.selected_shorter_token, + conflict.current_tokens + ] + end + + signature + end + + # @rbs (Set[String] left_acc, Set[String] right_acc) -> bool + def pslr_compatible_accept_sets?(left_acc, right_acc) + return true unless @scanner_fsa + + @pslr_compatibility_checker ||= State::ScannerAccepts::CompatibilityChecker.new( + @scanner_fsa, + lex_prec, + @length_precedences || LengthPrecedences.new(lex_prec) + ) + @pslr_compatibility_checker.compatible?(left_acc, right_acc) + end + + # @rbs (State state, ?State::lookahead_set filtered_lookaheads, ?expand_ties: bool, ?include_layout: bool) -> Set[String] + def acceptable_tokens_for_pslr(state, filtered_lookaheads = nil, expand_ties: true, include_layout: true) + tokens = Set.new + kernel_reduce_items = state.kernels.select(&:end_of_rule?).to_set + + state.term_transitions.each do |shift| + next_sym = shift.next_sym + tokens << next_sym.id.s_value if next_sym.term? + end + + state.reduces.each do |reduce| + look_ahead = + if filtered_lookaheads && kernel_reduce_items.include?(reduce.item) + filtered_lookaheads[reduce.item] || [] + else + state.acceptable_pslr_reduce_lookahead(reduce) + end + + look_ahead.each do |la| + tokens << la.id.s_value + end + end + + tokens = @grammar.expand_lexical_ties(tokens) if expand_ties + tokens | layout_token_names_for_pslr(include_layout: include_layout) + end + + # @rbs (?include_layout: bool) -> Set[String] + def layout_token_names_for_pslr(include_layout: true) + return Set.new unless include_layout + + @grammar.layout_token_names + end + # @rbs (Logger logger) -> void def validate_conflicts_within_threshold!(logger) exit false unless conflicts_within_threshold?(logger) @@ -863,5 +1211,254 @@ def clear_look_ahead_sets @_follow_sets = nil @_la = nil end + + # Build Scanner FSA from token patterns + # @rbs () -> void + def build_scanner_fsa + @grammar.synthesize_implicit_literal_token_patterns! + @grammar.finalize_lexical_declarations! + return if token_patterns.empty? + + @scanner_fsa = ScannerFSA.new(token_patterns) + @grammar.finalize_lexical_ties!(@scanner_fsa) + @pslr_compatibility_checker = nil + end + + # Build length precedences table + # @rbs () -> void + def build_length_precedences + @length_precedences = LengthPrecedences.new(lex_prec) + end + + # Build scanner_accepts table + # @rbs () -> void + def build_scanner_accepts + return unless @scanner_fsa + + collect_lexical_tie_candidates + + @scanner_accepts_table = State::ScannerAccepts.new( + reachable_parser_states, + @scanner_fsa, + lex_prec, + @length_precedences, + lex_tie, + layout_token_names: @grammar.layout_token_names + ) + @scanner_accepts_table.build + end + + # Handle PSLR inadequacies + # Detects and splits states where pseudo-scanner behavior differs + # @rbs () -> void + def handle_pslr_inadequacies + return unless @scanner_fsa && @scanner_accepts_table + + @pslr_inadequacies = detect_pslr_inadequacies + return if @pslr_inadequacies.empty? + + @tracer.warn("Detected #{@pslr_inadequacies.size} unresolved PSLR inadequacies") if @tracer.respond_to?(:warn) + end + + # @rbs () -> void + def finalize_pslr_metrics + return unless pslr_defined? + + base_states_count = @pslr_metrics[:base_states_count] || @states.count + total_states_count = @states.count + + @pslr_metrics = { + base_states_count: base_states_count, + total_states_count: total_states_count, + split_state_count: @states.count(&:split_state?), + growth_count: total_states_count - base_states_count, + growth_ratio: base_states_count.zero? ? nil : total_states_count.to_f / base_states_count, + token_pattern_count: token_patterns.size, + scanner_fsa_state_count: @scanner_fsa ? @scanner_fsa.states.size : 0, + inadequacies_count: @pslr_inadequacies.size + } + end + + # Detect PSLR inadequacies in isocore groups + # @rbs () -> Array[State::PslrInadequacy] + def detect_pslr_inadequacies + inadequacies = [] + + @states.each do |state| + state.transitions.each do |transition| + next_state = transition.to_state + next unless next_state + + propagating_lookaheads = state.propagate_lookaheads_without_filter(next_state.lalr_isocore) + expected_acc = acceptable_tokens_for_pslr(next_state, propagating_lookaheads) + actual_acc = acceptable_tokens_for_pslr(next_state) + + next if pslr_compatible_accept_sets?(expected_acc, actual_acc) + + matching_state = next_state.ielr_isocores.find do |candidate| + pslr_compatible_accept_sets?(acceptable_tokens_for_pslr(candidate), expected_acc) + end + + inadequacies << State::PslrInadequacy.new( + type: State::PslrInadequacy::PSLR_RELATIVE, + state: next_state, + conflicting_states: [matching_state, next_state].compact.uniq, + details: { + reason: "Transition reaches a state with an incompatible PSLR scanner profile", + from_state_id: state.id, + transition_symbol: transition.next_sym.id.s_value, + expected_profile: pslr_state_signature(next_state, propagating_lookaheads), + actual_profile: pslr_state_signature(next_state), + matching_state_id: matching_state&.id + } + ) + end + end + + inadequacies + end + + # @rbs () -> void + def collect_lexical_tie_candidates + @lexical_tie_candidates = [] + return unless @scanner_fsa + + pairs = @scanner_fsa.pairwise_conflict_pairs + return if pairs.empty? + + candidates = Set.new + reachable_parser_states.each do |state| + pre_tie_tokens = acceptable_tokens_for_pslr(state, nil, expand_ties: false, include_layout: false) + pairs.each do |left, right| + next unless pre_tie_tokens.include?(left) ^ pre_tie_tokens.include?(right) + next if lex_tie.tied?(left, right) + next if lex_tie.no_tie?(left, right) + + candidates << [left, right] + end + end + + @lexical_tie_candidates = candidates.to_a.sort + end + + # @rbs () -> Array[State] + def reachable_parser_states + return [] if @states.empty? + + visited = Set.new + stack = [@states.first] + reachable = [] + + until stack.empty? + state = stack.pop + next if visited.include?(state.id) + + visited << state.id + reachable << state + state.transitions.each do |transition| + stack << transition.to_state if transition.to_state + end + end + + reachable.sort_by(&:id) + end + + # @rbs (Logger logger) -> void + def validate_pslr_inadequacies!(logger) + return unless pslr_defined? + return if @pslr_inadequacies.empty? + + @pslr_inadequacies.each do |inadequacy| + logger.error(inadequacy.to_s) + end + + exit false + end + + # @rbs (Logger logger) -> void + def validate_pslr_scanner_conflicts!(logger) + return unless pslr_defined? + return unless @scanner_accepts_table + return unless @scanner_accepts_table.unresolved_conflicts? + + @scanner_accepts_table.conflicts.each do |conflict| + logger.error(pslr_scanner_conflict_message(conflict)) + end + + exit false + end + + # @rbs (State::ScannerAccepts::Conflict conflict) -> String + def pslr_scanner_conflict_message(conflict) + state = conflict.parser_state_id || "unknown" + shorter = conflict.shorter_tokens.empty? ? "(none)" : conflict.shorter_tokens.join(", ") + selected = conflict.selected_shorter_token || "(none)" + current = conflict.current_tokens.empty? ? "(none)" : conflict.current_tokens.join(", ") + + "unresolved PSLR scanner conflict in state #{state}, scanner state #{conflict.scanner_state_id}: " \ + "shorter matches: #{shorter}; selected shorter token: #{selected}; " \ + "current matches: #{current}; add an explicit %lex-prec rule or adjust lexical ties" + end + + # @rbs (Logger logger) -> void + def validate_pslr_state_growth!(logger) + return unless pslr_defined? + + errors = [] + base_states_count = @pslr_metrics[:base_states_count] || @states.count + total_states_count = @pslr_metrics[:total_states_count] || @states.count + split_state_count = @pslr_metrics[:split_state_count] || @states.count(&:split_state?) + growth_ratio = @pslr_metrics[:growth_ratio] || 1.0 + + if (limit = pslr_max_states) && limit < total_states_count + errors << "PSLR state growth exceeded pslr.max-states=#{limit} (total=#{total_states_count}, base=#{base_states_count}, split=#{split_state_count})" + end + + if (limit = pslr_max_state_ratio) && limit < growth_ratio + errors << "PSLR state growth exceeded pslr.max-state-ratio=#{limit} (ratio=#{format('%.2f', growth_ratio)}x, total=#{total_states_count}, base=#{base_states_count})" + end + + return if errors.empty? + + errors.each do |message| + logger.error(message) + end + + exit false + end + + # Report %lex-prec rules that were never used in scanner conflict resolution. + # @rbs (Logger logger) -> void + def validate_pslr_useless_lex_prec!(logger) + return unless pslr_defined? + return unless @scanner_accepts_table + + useless = lex_prec.useless_rules + return if useless.empty? + + useless.each do |rule| + operator_label = LengthPrecedences.operator_label(rule.operator) + logger.warn( + "useless %lex-prec rule at line #{rule.lineno}: " \ + "#{rule.left_name} #{operator_label} #{rule.right_name} " \ + "does not resolve any PSLR scanner conflict" + ) + end + end + + # Report lexical tie candidates that are not covered by %lex-tie or %lex-no-tie. + # @rbs (Logger logger) -> void + def validate_pslr_lexical_tie_candidates!(logger) + return unless pslr_defined? + return if @lexical_tie_candidates.empty? + + @lexical_tie_candidates.each do |left, right| + logger.warn( + "PSLR lexical tie candidate: #{left} and #{right} have scanner conflicts " \ + "but are not always accepted together. " \ + "Add %lex-tie #{left} #{right} or %lex-no-tie #{left} #{right}." + ) + end + end end end diff --git a/sig/generated/lrama/lexer_context_classifier.rbs b/sig/generated/lrama/lexer_context_classifier.rbs new file mode 100644 index 000000000..5a969237d --- /dev/null +++ b/sig/generated/lrama/lexer_context_classifier.rbs @@ -0,0 +1,78 @@ +# Generated from lib/lrama/lexer_context_classifier.rb with RBS::Inline + +module Lrama + # Classifies parser states into lexer context categories. + # + # When LALR states are merged, states from different grammatical contexts + # (e.g., BEG vs CMDARG) share the same state number, making them + # indistinguishable to the lexer. This classifier analyzes kernel items + # to determine the lexer context of each state, enabling context-aware + # state splitting. + # + # Context definitions come from %lexer-context directives in the grammar file. + # Each directive maps a context name to a set of symbols: + # + # %lexer-context BEG keyword_if keyword_unless '(' '[' '{' + # %lexer-context CMDARG tIDENTIFIER tFID tCONSTANT + class LexerContextClassifier + # @rbs (Hash[String, Grammar::LexerContext] lexer_contexts, ?Hash[String, Array[String]] expansion_args) -> void + def initialize: (Hash[String, Grammar::LexerContext] lexer_contexts, ?Hash[String, Array[String]] expansion_args) -> void + + # Classify a state's kernel items into context groups. + # + # @rbs (State state) -> Hash[Integer, Array[State::Item]] + def classify: (State state) -> Hash[Integer, Array[State::Item]] + + # Infer the lexer context for a single kernel item. + # + # @rbs (State::Item item) -> Integer + def infer_item_context: (State::Item item) -> Integer + + # Classify context based on the symbol before the dot. + # + # @rbs (Grammar::Symbol sym) -> Integer + def classify_symbol_context: (Grammar::Symbol sym) -> Integer + + # For backward compatibility with states.rb split logic + # @rbs (Grammar::Symbol sym) -> Integer + def classify_terminal_context: (Grammar::Symbol sym) -> Integer + + # For backward compatibility with states.rb split logic + # @rbs (Grammar::Symbol sym) -> Integer + def classify_nonterminal_context: (Grammar::Symbol sym) -> Integer + + # Return a human-readable name for a context value. + # + # @rbs (Integer ctx) -> String + def context_name: (Integer ctx) -> String + + # Class-level context_name for use without an instance (e.g., output.rb). + # Requires lexer_contexts to build the name map. + # + # @rbs (Integer ctx, Hash[String, Grammar::LexerContext] lexer_contexts) -> String + def self.context_name: (Integer ctx, Hash[String, Grammar::LexerContext] lexer_contexts) -> String + + # All context bitmasks OR'd together (for "is context known?" checks). + # @rbs () -> Integer + def all_contexts_mask: () -> Integer + + # Return the ordered list of context definitions. + # @rbs () -> Array[Grammar::LexerContext] + def contexts: () -> Array[Grammar::LexerContext] + + private + + # Build a map from symbol name → context bitmask. + # @rbs () -> Hash[String, Integer] + def build_symbol_to_context_map: () -> Hash[String, Integer] + + # Build a map from bitmask value → context name. + # @rbs () -> Hash[Integer, String] + def build_context_names: () -> Hash[Integer, String] + + # Return the bitmask for the first defined context (used as default for position-0 items). + # Returns 0 if no contexts are defined. + # @rbs () -> Integer + def default_beg_context: () -> Integer + end +end diff --git a/sig/generated/lrama/state.rbs b/sig/generated/lrama/state.rbs index 8f585c332..42c4030de 100644 --- a/sig/generated/lrama/state.rbs +++ b/sig/generated/lrama/state.rbs @@ -76,6 +76,10 @@ module Lrama attr_accessor goto_follows: Hash[Action::Goto, Array[Grammar::Symbol]] + attr_accessor pslr_item_lookahead_set: lookahead_set? + + attr_accessor lexer_context: Integer? + # @rbs (Integer id, Grammar::Symbol accessing_symbol, Array[Item] kernels) -> void def initialize: (Integer id, Grammar::Symbol accessing_symbol, Array[Item] kernels) -> void @@ -100,6 +104,12 @@ module Lrama # @rbs (Grammar::Rule rule, Array[Grammar::Symbol] look_ahead) -> void def set_look_ahead: (Grammar::Rule rule, Array[Grammar::Symbol] look_ahead) -> void + # @rbs (Action::Reduce reduce) -> Array[Grammar::Symbol] + def acceptable_reduce_lookahead: (Action::Reduce reduce) -> Array[Grammar::Symbol] + + # @rbs (Action::Reduce reduce) -> Array[Grammar::Symbol] + def acceptable_pslr_reduce_lookahead: (Action::Reduce reduce) -> Array[Grammar::Symbol] + # @rbs (Grammar::Rule rule, Hash[Grammar::Symbol, Array[Action::Goto]] sources) -> void def set_look_ahead_sources: (Grammar::Rule rule, Hash[Grammar::Symbol, Array[Action::Goto]] sources) -> void @@ -158,6 +168,12 @@ module Lrama # @rbs (State next_state) -> lookahead_set def propagate_lookaheads: (State next_state) -> lookahead_set + # @rbs (State next_state) -> lookahead_set + def propagate_lookaheads_without_filter: (State next_state) -> lookahead_set + + # @rbs (State next_state, bool apply_filter) -> lookahead_set + def propagate_lookaheads_with_filter: (State next_state, bool apply_filter) -> lookahead_set + # Definition 3.43 (is_compatible) # # @rbs (lookahead_set filtered_lookahead) -> bool diff --git a/sig/generated/lrama/state/pslr_inadequacy.rbs b/sig/generated/lrama/state/pslr_inadequacy.rbs new file mode 100644 index 000000000..54c25cff6 --- /dev/null +++ b/sig/generated/lrama/state/pslr_inadequacy.rbs @@ -0,0 +1,52 @@ +# Generated from lib/lrama/state/pslr_inadequacy.rb with RBS::Inline + +module Lrama + class State + # PSLR Inadequacy detection + # Based on Section 3.4.3 from the PSLR dissertation + # + # PSLR inadequacy occurs when state merging causes different + # pseudo-scanner behavior + class PslrInadequacy + # Inadequacy types + LR_RELATIVE: Symbol + + PSLR_RELATIVE: Symbol + + attr_reader type: Symbol + + attr_reader state: State + + attr_reader conflicting_states: Array[State] + + attr_reader details: Hash[Symbol, untyped] + + # @rbs (type: Symbol, state: State, conflicting_states: Array[State], details: Hash[Symbol, untyped]) -> void + def initialize: (type: Symbol, state: State, conflicting_states: Array[State], details: Hash[Symbol, untyped]) -> void + + # @rbs () -> String + def to_s: () -> String + end + + # PSLR Compatibility checker + # Based on Definition 3.4.1 from the dissertation + class PslrCompatibilityChecker + # @rbs (ScannerAccepts scanner_accepts, LengthPrecedences length_prec) -> void + def initialize: (ScannerAccepts scanner_accepts, LengthPrecedences length_prec) -> void + + # Build a stable scanner profile for a parser state + # @rbs (State state, ScannerFSA scanner_fsa) -> Array[[Integer, String?]] + def profile: (State state, ScannerFSA scanner_fsa) -> Array[[ Integer, String? ]] + + # Partition states by scanner profile + # @rbs (Array[State] states, ScannerFSA scanner_fsa) -> Hash[Array[[Integer, String?]], Array[State]] + def group_by_profile: (Array[State] states, ScannerFSA scanner_fsa) -> Hash[Array[[ Integer, String? ]], Array[State]] + + # Check if two states are PSLR-compatible + # Definition 3.4.1: States are compatible if for any input, + # the pseudo-scanner selects the same token + # @rbs (State s1, State s2, ScannerFSA scanner_fsa) -> bool + def compatible?: (State s1, State s2, ScannerFSA scanner_fsa) -> bool + end + end +end diff --git a/sig/generated/lrama/state/scanner_accepts.rbs b/sig/generated/lrama/state/scanner_accepts.rbs new file mode 100644 index 000000000..93e0260c9 --- /dev/null +++ b/sig/generated/lrama/state/scanner_accepts.rbs @@ -0,0 +1,192 @@ +# Generated from lib/lrama/state/scanner_accepts.rb with RBS::Inline + +module Lrama + class State + # scanner_accepts[parser_state, accepting_scanner_state] for PSLR(1). + # + # Construction follows complete pseudo-scanner conflict profiles. A profile + # contains the shorter token set Ts, the selected shorter token ts, and the + # current/longest token set Tl. + # + # Normal parser-state rows are strict: unresolved pseudo-scanner + # conflicts are reported and are not resolved by token declaration order. + # + # The fallback row is used only for syntax-error handling. It first + # applies explicit PSLR lexical precedence declarations. For scanner + # conflicts that remain unresolved only in the fallback universe, it + # completes the decision with traditional scanner behavior: fallback + # length defaults are handled by LengthPrecedences#fallback_precedes?, + # and otherwise-unresolved identity conflicts are resolved by token + # declaration order. + # + # When %lex-scope declarations are present, each parser state may use + # a merged set of global + scope-active lexical precedence rules. + class ScannerAccepts + class Conflict + attr_reader parser_state_id: Integer? + + attr_reader scanner_state_id: Integer + + attr_reader shorter_tokens: Array[String] + + attr_reader selected_shorter_token: String? + + attr_reader current_tokens: Array[String] + + # @rbs (parser_state_id: Integer?, scanner_state_id: Integer, shorter_tokens: Array[String], selected_shorter_token: String?, current_tokens: Array[String]) -> void + def initialize: (parser_state_id: Integer?, scanner_state_id: Integer, shorter_tokens: Array[String], selected_shorter_token: String?, current_tokens: Array[String]) -> void + end + + class ProfileOutcome + EMPTY: Symbol + + RESOLVED: Symbol + + UNRESOLVED: Symbol + + attr_reader kind: Symbol + + attr_reader token_name: String? + + attr_reader conflict: Conflict? + + # @rbs (kind: Symbol, ?token_name: String?, ?conflict: Conflict?) -> void + def initialize: (kind: Symbol, ?token_name: String?, ?conflict: Conflict?) -> void + + # @rbs () -> bool + def empty?: () -> bool + + # @rbs () -> bool + def resolved?: () -> bool + + # @rbs () -> bool + def unresolved?: () -> bool + end + + class ProfileResolver + # @rbs (Grammar::LexPrec lex_prec, LengthPrecedences length_prec, ?fallback: bool, ?token_order: Hash[String, Integer]) -> void + def initialize: (Grammar::LexPrec lex_prec, LengthPrecedences length_prec, ?fallback: bool, ?token_order: Hash[String, Integer]) -> void + + # @rbs (Set[String] shorter_tokens, String? selected_shorter_token, Set[String] current_tokens) -> ProfileOutcome + def resolve: (Set[String] shorter_tokens, String? selected_shorter_token, Set[String] current_tokens) -> ProfileOutcome + + private + + # @rbs (Set[String] shorter_tokens, String? selected_shorter_token, Set[String] current_tokens) -> ProfileOutcome + def resolve_normal: (Set[String] shorter_tokens, String? selected_shorter_token, Set[String] current_tokens) -> ProfileOutcome + + # @rbs (Set[String] shorter_tokens, String? selected_shorter_token, Set[String] current_tokens) -> ProfileOutcome + def resolve_fallback: (Set[String] shorter_tokens, String? selected_shorter_token, Set[String] current_tokens) -> ProfileOutcome + + # @rbs (String old_token, String new_token) -> bool + def length_prefers_old?: (String old_token, String new_token) -> bool + + # @rbs (String old_token, String new_token) -> bool + def fallback_length_prefers_old?: (String old_token, String new_token) -> bool + + # @rbs (String candidate, Set[String] current_tokens) -> bool + def identity_winner?: (String candidate, Set[String] current_tokens) -> bool + + # @rbs (String candidate, Set[String] current_tokens) -> bool + def fallback_identity_winner?: (String candidate, Set[String] current_tokens) -> bool + + # @rbs (String candidate, String other) -> bool + def token_order_precedes?: (String candidate, String other) -> bool + + # @rbs (String token) -> [Integer, String] + def token_order_key: (String token) -> [ Integer, String ] + end + + class CompleteProfileComputer + attr_reader table: Hash[Integer, Grammar::TokenPattern] + + attr_reader conflicts: Array[Conflict] + + # @rbs (ScannerFSA scanner_fsa, Grammar::LexPrec lex_prec, LengthPrecedences length_prec, Set[String] acceptable_tokens, ?Integer? parser_state_id, ?fallback: bool, ?token_order: Hash[String, Integer]) -> void + def initialize: (ScannerFSA scanner_fsa, Grammar::LexPrec lex_prec, LengthPrecedences length_prec, Set[String] acceptable_tokens, ?Integer? parser_state_id, ?fallback: bool, ?token_order: Hash[String, Integer]) -> void + + # @rbs () -> void + def compute: () -> void + + private + + # @rbs (Integer fsa_state_id, Set[String] shorter_tokens, String? selected_shorter_token, Set[untyped] visited) -> void + def visit_transitions: (Integer fsa_state_id, Set[String] shorter_tokens, String? selected_shorter_token, Set[untyped] visited) -> void + + # @rbs (Integer fsa_state_id, Set[String] shorter_tokens, String? selected_shorter_token, Set[untyped] visited) -> void + def visit_state: (Integer fsa_state_id, Set[String] shorter_tokens, String? selected_shorter_token, Set[untyped] visited) -> void + + # @rbs (ScannerFSA::State fsa_state) -> Set[String] + def current_acceptable_tokens: (ScannerFSA::State fsa_state) -> Set[String] + + # @rbs (String token_name) -> Grammar::TokenPattern + def token_pattern_for: (String token_name) -> Grammar::TokenPattern + + # @rbs (Integer fsa_state_id, Set[String] shorter_tokens, String? selected_shorter_token, Set[String] current_tokens) -> [Integer, Array[String], String?, Array[String]] + def profile_key: (Integer fsa_state_id, Set[String] shorter_tokens, String? selected_shorter_token, Set[String] current_tokens) -> [ Integer, Array[String], String?, Array[String] ] + end + + class CompatibilityChecker + # @rbs (ScannerFSA scanner_fsa, Grammar::LexPrec lex_prec, LengthPrecedences length_prec) -> void + def initialize: (ScannerFSA scanner_fsa, Grammar::LexPrec lex_prec, LengthPrecedences length_prec) -> void + + # @rbs (Set[String] left_tokens, Set[String] right_tokens) -> bool + def compatible?: (Set[String] left_tokens, Set[String] right_tokens) -> bool + + private + + # @rbs (Integer fsa_state_id, Set[String] left_shorter, String? left_selected, Set[String] right_shorter, String? right_selected, Set[String] left_acc, Set[String] right_acc, Set[untyped] visited) -> bool + def visit_transitions: (Integer fsa_state_id, Set[String] left_shorter, String? left_selected, Set[String] right_shorter, String? right_selected, Set[String] left_acc, Set[String] right_acc, Set[untyped] visited) -> bool + + # @rbs (Integer fsa_state_id, Set[String] left_shorter, String? left_selected, Set[String] right_shorter, String? right_selected, Set[String] left_acc, Set[String] right_acc, Set[untyped] visited) -> bool + def visit_state: (Integer fsa_state_id, Set[String] left_shorter, String? left_selected, Set[String] right_shorter, String? right_selected, Set[String] left_acc, Set[String] right_acc, Set[untyped] visited) -> bool + + # @rbs (ScannerFSA::State fsa_state, Set[String] accepted_names) -> Set[String] + def tokens_accepted_by: (ScannerFSA::State fsa_state, Set[String] accepted_names) -> Set[String] + + # @rbs (ProfileOutcome left, ProfileOutcome right) -> bool + def outcomes_compatible?: (ProfileOutcome left, ProfileOutcome right) -> bool + end + + FALLBACK_ROW_ID: Integer + + attr_reader table: Hash[[ Integer, Integer ], Grammar::TokenPattern?] + + attr_reader fallback_table: Hash[Integer, Grammar::TokenPattern] + + attr_reader conflicts: Array[Conflict] + + # @rbs (Array[State] parser_states, ScannerFSA scanner_fsa, Grammar::LexPrec lex_prec, LengthPrecedences length_prec, ?Grammar::LexTie? lex_tie, ?layout_token_names: Set[String]) -> void + def initialize: (Array[State] parser_states, ScannerFSA scanner_fsa, Grammar::LexPrec lex_prec, LengthPrecedences length_prec, ?Grammar::LexTie? lex_tie, ?layout_token_names: Set[String]) -> void + + # @rbs (ScannerFSA scanner_fsa, Grammar::LexPrec lex_prec, LengthPrecedences length_prec, Set[String] acceptable_tokens) -> [Hash[Integer, Grammar::TokenPattern], Array[Conflict]] + def self.compute_for_acceptable_tokens: (ScannerFSA scanner_fsa, Grammar::LexPrec lex_prec, LengthPrecedences length_prec, Set[String] acceptable_tokens) -> [ Hash[Integer, Grammar::TokenPattern], Array[Conflict] ] + + # @rbs () -> void + def build: () -> void + + # @rbs (Integer parser_state_id, Integer accepting_state_id) -> Grammar::TokenPattern? + def []: (Integer parser_state_id, Integer accepting_state_id) -> Grammar::TokenPattern? + + # @rbs () -> bool + def unresolved_conflicts?: () -> bool + + private + + # @rbs (State parser_state) -> void + def compute_for_parser_state: (State parser_state) -> void + + # @rbs () -> void + def compute_fallback_row: () -> void + + # @rbs (State parser_state) -> Set[String] + def compute_acc_sp: (State parser_state) -> Set[String] + + # @rbs (Set[String] tokens) -> Set[String] + def expand_lexical_ties: (Set[String] tokens) -> Set[String] + + # @rbs () -> Hash[String, Integer] + def token_order: () -> Hash[String, Integer] + end + end +end diff --git a/sig/generated/lrama/states.rbs b/sig/generated/lrama/states.rbs index 8e4b296e9..919b6a00c 100644 --- a/sig/generated/lrama/states.rbs +++ b/sig/generated/lrama/states.rbs @@ -44,6 +44,20 @@ module Lrama attr_reader lookback_relation: Hash[state_id, Hash[rule_id, Array[State::Action::Goto]]] + attr_reader scanner_fsa: ScannerFSA? + + attr_reader length_precedences: LengthPrecedences? + + attr_reader scanner_accepts_table: State::ScannerAccepts? + + attr_reader pslr_inadequacies: Array[State::PslrInadequacy] + + attr_reader pslr_metrics: Hash[Symbol, Integer | Float | nil] + + attr_reader lexer_context_classifier: LexerContextClassifier? + + attr_reader lexical_tie_candidates: Array[[ String, String ]] + # @rbs (Grammar grammar, Tracer tracer) -> void def initialize: (Grammar grammar, Tracer tracer) -> void @@ -53,6 +67,11 @@ module Lrama # @rbs () -> void def compute_ielr: () -> void + # Compute PSLR(1) states + # Based on Section 3.4 of the PSLR dissertation + # @rbs () -> void + def compute_pslr: () -> void + # @rbs () -> Integer def states_count: () -> Integer @@ -77,6 +96,27 @@ module Lrama # @rbs (Logger logger) -> void def validate!: (Logger logger) -> void + # Classify each state's lexer context based on kernel items. + # + # For each state, analyzes the kernel items to determine what lexer + # context (BEG, CMDARG, ARG, END, ENDFN, MID, DOT) the state belongs to. + # When a state has kernel items from multiple contexts, the context is + # set to the bitwise OR of all contexts (mixed context). + # + # @rbs () -> void + def classify_lexer_contexts: () -> void + + # Return the lexer context table as an array of context values, + # one per parser state (indexed by state id). + # + # @rbs () -> Array[Integer] + def lexer_context_table: () -> Array[Integer] + + # Check if lexer context classification has been performed. + # + # @rbs () -> bool + def lexer_context_enabled?: () -> bool + def compute_la_sources_for_conflicted_states: () -> untyped private @@ -188,15 +228,68 @@ module Lrama # @rbs () -> void def split_states: () -> void + # Split states where different predecessor paths lead to different + # lexer contexts. This resolves LALR state merging that makes + # BEG vs CMDARG (and other context pairs) indistinguishable. + # + # Algorithm: + # 1. For each state, group incoming transitions by the lexer context + # that the predecessor would imply + # 2. If a state has predecessors from multiple different contexts, + # split the state so each split has a unique context + # + # @rbs () -> void + def split_states_by_context: () -> void + + # For a given state, group its incoming transitions by the lexer context + # that the predecessor state implies for this state. + # + # The implied context is determined by what symbol was used to reach + # this state (the accessing symbol's context). + # + # @rbs (State state) -> Hash[Integer, Array[[State, State::Action::Shift | State::Action::Goto]]] + def compute_predecessor_context_groups: (State state) -> Hash[Integer, Array[[ State, State::Action::Shift | State::Action::Goto ]]] + + # Infer the lexer context that a transition implies for the target state. + # + # @rbs (State pred, State::Action::Shift | State::Action::Goto transition) -> Integer + def infer_transition_context: (State pred, State::Action::Shift | State::Action::Goto transition) -> Integer + + # Create a new split state that is an isocore copy of the given state. + # + # @rbs (State original) -> State + def create_context_split_state: (State original) -> State + + # @rbs () -> void + def capture_pslr_metrics_before_split: () -> void + # @rbs () -> void def compute_inadequacy_annotations: () -> void # @rbs (State state, State::lookahead_set filtered_lookaheads) -> void def merge_lookaheads: (State state, State::lookahead_set filtered_lookaheads) -> void + # @rbs (State state, State::lookahead_set pslr_lookaheads) -> void + def merge_pslr_lookaheads: (State state, State::lookahead_set pslr_lookaheads) -> void + # @rbs (State state, State::Action::Shift | State::Action::Goto transition, State next_state) -> void def compute_state: (State state, State::Action::Shift | State::Action::Goto transition, State next_state) -> void + # @rbs (State state, State::lookahead_set filtered_lookaheads, ?State::lookahead_set pslr_lookaheads) -> bool + def compatible_split_state?: (State state, State::lookahead_set filtered_lookaheads, ?State::lookahead_set pslr_lookaheads) -> bool + + # @rbs (State state, ?State::lookahead_set filtered_lookaheads) -> Array[[Integer, String?]] + def pslr_state_signature: (State state, ?State::lookahead_set filtered_lookaheads) -> Array[[ Integer, String? ]] + + # @rbs (Set[String] left_acc, Set[String] right_acc) -> bool + def pslr_compatible_accept_sets?: (Set[String] left_acc, Set[String] right_acc) -> bool + + # @rbs (State state, ?State::lookahead_set filtered_lookaheads, ?expand_ties: bool, ?include_layout: bool) -> Set[String] + def acceptable_tokens_for_pslr: (State state, ?State::lookahead_set filtered_lookaheads, ?expand_ties: bool, ?include_layout: bool) -> Set[String] + + # @rbs (?include_layout: bool) -> Set[String] + def layout_token_names_for_pslr: (?include_layout: bool) -> Set[String] + # @rbs (Logger logger) -> void def validate_conflicts_within_threshold!: (Logger logger) -> void @@ -211,5 +304,55 @@ module Lrama # @rbs () -> void def clear_look_ahead_sets: () -> void + + # Build Scanner FSA from token patterns + # @rbs () -> void + def build_scanner_fsa: () -> void + + # Build length precedences table + # @rbs () -> void + def build_length_precedences: () -> void + + # Build scanner_accepts table + # @rbs () -> void + def build_scanner_accepts: () -> void + + # Handle PSLR inadequacies + # Detects and splits states where pseudo-scanner behavior differs + # @rbs () -> void + def handle_pslr_inadequacies: () -> void + + # @rbs () -> void + def finalize_pslr_metrics: () -> void + + # Detect PSLR inadequacies in isocore groups + # @rbs () -> Array[State::PslrInadequacy] + def detect_pslr_inadequacies: () -> Array[State::PslrInadequacy] + + # @rbs () -> void + def collect_lexical_tie_candidates: () -> void + + # @rbs () -> Array[State] + def reachable_parser_states: () -> Array[State] + + # @rbs (Logger logger) -> void + def validate_pslr_inadequacies!: (Logger logger) -> void + + # @rbs (Logger logger) -> void + def validate_pslr_scanner_conflicts!: (Logger logger) -> void + + # @rbs (State::ScannerAccepts::Conflict conflict) -> String + def pslr_scanner_conflict_message: (State::ScannerAccepts::Conflict conflict) -> String + + # @rbs (Logger logger) -> void + def validate_pslr_state_growth!: (Logger logger) -> void + + # Report %lex-prec rules that were never used in scanner conflict resolution. + # @rbs (Logger logger) -> void + def validate_pslr_useless_lex_prec!: (Logger logger) -> void + + # Report lexical tie candidates that are not covered by %lex-tie or %lex-no-tie. + # @rbs (Logger logger) -> void + def validate_pslr_lexical_tie_candidates!: (Logger logger) -> void end end diff --git a/spec/lrama/lexer_context_classifier_spec.rb b/spec/lrama/lexer_context_classifier_spec.rb new file mode 100644 index 000000000..0ad703ca1 --- /dev/null +++ b/spec/lrama/lexer_context_classifier_spec.rb @@ -0,0 +1,457 @@ +# frozen_string_literal: true + +RSpec.describe Lrama::LexerContextClassifier do + include PslrFamilyHelper + + # Helper to build a classifier with standard CRuby-like contexts + def build_classifier_with_contexts + lexer_contexts = {} + [ + ["BEG", %w[keyword_if keyword_unless keyword_while keyword_do tPLUS tMINUS tLPAREN tLBRACK tLBRACE]], + ["CMDARG", %w[tIDENTIFIER tFID tCONSTANT]], + ["END", %w[tINTEGER tFLOAT tSTRING_END keyword_end tRPAREN tRBRACK tRBRACE]], + ["ENDFN", %w[keyword_def]], + ["DOT", %w[tDOT tCOLON2 tANDDOT]], + ].each_with_index do |(name, syms), idx| + lc = Lrama::Grammar::LexerContext.new(name: name, index: idx) + syms.each do |s| + lc.add_symbols([double("token", s_value: s)]) + end + lexer_contexts[name] = lc + end + described_class.new(lexer_contexts) + end + + describe "context bitmask assignment" do + it "assigns non-overlapping bitmask flags by definition order" do + classifier = build_classifier_with_contexts + bitmasks = classifier.contexts.map(&:bitmask) + + # All bitmasks should be powers of 2 + bitmasks.each do |bm| + expect(bm).to be > 0 + expect(bm & (bm - 1)).to eq(0), "#{bm} is not a power of 2" + end + + # No two should overlap + bitmasks.combination(2).each do |a, b| + expect(a & b).to eq(0), "Bitmasks #{a} and #{b} overlap" + end + end + end + + describe ".context_name" do + let(:lexer_contexts) do + lcs = {} + lc = Lrama::Grammar::LexerContext.new(name: "BEG", index: 0) + lcs["BEG"] = lc + lc2 = Lrama::Grammar::LexerContext.new(name: "CMDARG", index: 1) + lcs["CMDARG"] = lc2 + lcs + end + + it "returns UNKNOWN for 0" do + expect(described_class.context_name(0, lexer_contexts)).to eq("UNKNOWN") + end + + it "returns single context name for single flag" do + expect(described_class.context_name(0x01, lexer_contexts)).to eq("BEG") + expect(described_class.context_name(0x02, lexer_contexts)).to eq("CMDARG") + end + + it "returns combined name for multiple flags" do + name = described_class.context_name(0x01 | 0x02, lexer_contexts) + expect(name).to include("BEG") + expect(name).to include("CMDARG") + end + end + + describe "#classify_symbol_context" do + let(:classifier) { build_classifier_with_contexts } + + it "classifies operator-like terminals" do + %w[tPLUS tMINUS].each do |name| + sym = double("symbol", id: double("id", s_value: name), term?: true) + ctx = classifier.classify_symbol_context(sym) + # BEG = 1 << 0 = 0x01 + expect(ctx).to eq(0x01), "Expected #{name} to be BEG" + end + end + + it "classifies identifier terminals as CMDARG" do + %w[tIDENTIFIER tFID tCONSTANT].each do |name| + sym = double("symbol", id: double("id", s_value: name), term?: true) + ctx = classifier.classify_symbol_context(sym) + # CMDARG = 1 << 1 = 0x02 + expect(ctx).to eq(0x02), "Expected #{name} to be CMDARG" + end + end + + it "classifies literal terminals as END" do + %w[tINTEGER tFLOAT tSTRING_END].each do |name| + sym = double("symbol", id: double("id", s_value: name), term?: true) + ctx = classifier.classify_symbol_context(sym) + # END = 1 << 2 = 0x04 + expect(ctx).to eq(0x04), "Expected #{name} to be END" + end + end + + it "classifies keyword_def as ENDFN" do + sym = double("symbol", id: double("id", s_value: "keyword_def"), term?: true) + ctx = classifier.classify_symbol_context(sym) + # ENDFN = 1 << 3 = 0x08 + expect(ctx).to eq(0x08) + end + + it "classifies dot tokens as DOT" do + %w[tDOT tCOLON2 tANDDOT].each do |name| + sym = double("symbol", id: double("id", s_value: name), term?: true) + ctx = classifier.classify_symbol_context(sym) + # DOT = 1 << 4 = 0x10 + expect(ctx).to eq(0x10), "Expected #{name} to be DOT" + end + end + + it "classifies open brackets as BEG" do + %w[tLPAREN tLBRACK tLBRACE].each do |name| + sym = double("symbol", id: double("id", s_value: name), term?: true) + ctx = classifier.classify_symbol_context(sym) + expect(ctx).to eq(0x01), "Expected #{name} to be BEG" + end + end + + it "classifies close brackets as END" do + %w[tRPAREN tRBRACK tRBRACE].each do |name| + sym = double("symbol", id: double("id", s_value: name), term?: true) + ctx = classifier.classify_symbol_context(sym) + expect(ctx).to eq(0x04), "Expected #{name} to be END" + end + end + + it "returns 0 for unknown symbols" do + sym = double("symbol", id: double("id", s_value: "unknown_token"), term?: true) + ctx = classifier.classify_symbol_context(sym) + expect(ctx).to eq(0) + end + end + + describe "#classify with grammar-defined contexts" do + context "with %lexer-context directives" do + let(:grammar) do + build_grammar(<<~GRAMMAR, "lexer_context/basic.y") + %define lr.type pslr + %token-pattern IF /if/ + %token-pattern ID /[a-z]+/ + %lex-prec ID <~ IF + + %lexer-context BEG IF + %lexer-context CMDARG ID + + %% + + program + : expr + ; + + expr + : ID + | expr '+' expr + ; + GRAMMAR + end + + it "classifies states without errors" do + states = Lrama::States.new(grammar, Lrama::Tracer.new(Lrama::Logger.new)) + states.compute + states.compute_pslr + + states.states.each do |state| + expect(state.lexer_context).not_to be_nil + end + end + end + + context "with operator-heavy grammar" do + let(:grammar) do + build_grammar(<<~GRAMMAR, "lexer_context/operators.y") + %define lr.type pslr + %token-pattern PLUS /\\+/ + %token-pattern STAR /\\*/ + %token-pattern ID /[a-z]+/ + %token-pattern NUM /[0-9]+/ + + %lexer-context BEG PLUS STAR + %lexer-context CMDARG ID + %lexer-context END NUM + + %% + + program + : expr + ; + + expr + : NUM + | ID + | expr PLUS expr + | expr STAR expr + ; + GRAMMAR + end + + it "classifies all states" do + states = Lrama::States.new(grammar, Lrama::Tracer.new(Lrama::Logger.new)) + states.compute + states.compute_pslr + + states.states.each do |state| + expect(state.lexer_context).not_to be_nil + end + end + end + end + + describe "integration with States" do + context "lexer_context_enabled?" do + it "returns false when no %lexer-context directives" do + grammar = build_grammar(<<~GRAMMAR, "lexer_context/no_ctx.y") + %define lr.type pslr + %token-pattern ID /[a-z]+/ + + %% + + program : ID ; + GRAMMAR + + states = Lrama::States.new(grammar, Lrama::Tracer.new(Lrama::Logger.new)) + states.compute + states.compute_pslr + + expect(states.lexer_context_enabled?).to eq(false) + end + + it "returns true when %lexer-context directives are present" do + grammar = build_grammar(<<~GRAMMAR, "lexer_context/with_ctx.y") + %define lr.type pslr + %token-pattern ID /[a-z]+/ + + %lexer-context BEG ID + + %% + + program : ID ; + GRAMMAR + + states = Lrama::States.new(grammar, Lrama::Tracer.new(Lrama::Logger.new)) + states.compute + states.compute_pslr + + expect(states.lexer_context_enabled?).to eq(true) + end + end + + context "lexer_context_table" do + it "returns an array with one entry per state" do + grammar = build_grammar(<<~GRAMMAR, "lexer_context/table.y") + %define lr.type pslr + %token-pattern ID /[a-z]+/ + %token-pattern NUM /[0-9]+/ + + %lexer-context BEG ID + %lexer-context END NUM + + %% + + program + : expr + ; + + expr + : NUM + | ID + | expr '+' expr + ; + GRAMMAR + + states = Lrama::States.new(grammar, Lrama::Tracer.new(Lrama::Logger.new)) + states.compute + states.compute_pslr + + table = states.lexer_context_table + expect(table.size).to eq(states.states_count) + expect(table.all? { |v| v.is_a?(Integer) }).to eq(true) + end + end + end + + describe "context-based state splitting" do + context "with operator vs identifier predecessor contexts" do + let(:grammar) do + build_grammar(<<~GRAMMAR, "lexer_context/split_expr.y") + %define lr.type pslr + %token-pattern tPLUS /\\+/ + %token-pattern tSTAR /\\*/ + %token-pattern tIDENTIFIER /[a-z]+/ + %token-pattern tINTEGER /[0-9]+/ + + %lexer-context BEG tPLUS tSTAR + %lexer-context CMDARG tIDENTIFIER + %lexer-context END tINTEGER + + %% + + program + : expr + ; + + expr + : tINTEGER + | tIDENTIFIER + | expr tPLUS expr + | expr tSTAR expr + ; + GRAMMAR + end + + it "classifies all states with non-nil context" do + states = Lrama::States.new(grammar, Lrama::Tracer.new(Lrama::Logger.new)) + states.compute + states.compute_pslr + + states.states.each do |state| + expect(state.lexer_context).not_to be_nil, + "State #{state.id} has nil lexer_context" + end + end + + it "has BEG context after operators" do + states = Lrama::States.new(grammar, Lrama::Tracer.new(Lrama::Logger.new)) + states.compute + states.compute_pslr + + # Find states after tPLUS or tSTAR + operator_target_states = [] + states.states.each do |state| + state.term_transitions.each do |shift| + name = shift.next_sym.id.s_value + if name == "tPLUS" || name == "tSTAR" + operator_target_states << shift.to_state + end + end + end + + lexer_contexts = grammar.lexer_contexts + beg_mask = lexer_contexts["BEG"].bitmask + + operator_target_states.each do |target| + ctx = target.lexer_context || 0 + ctx_name = described_class.context_name(ctx, lexer_contexts) + expect(ctx & beg_mask).not_to eq(0), + "State #{target.id} after operator should have BEG context, got #{ctx_name}" + end + end + end + + context "with def keyword creating ENDFN context" do + let(:grammar) do + build_grammar(<<~GRAMMAR, "lexer_context/endfn.y") + %define lr.type pslr + %token-pattern keyword_def /def/ + %token-pattern keyword_end /end/ + %token-pattern tIDENTIFIER /[a-z]+/ + %token-pattern tINTEGER /[0-9]+/ + + %lexer-context ENDFN keyword_def + %lexer-context END keyword_end tINTEGER + %lexer-context CMDARG tIDENTIFIER + + %% + + program + : defn + ; + + defn + : keyword_def tIDENTIFIER keyword_end + ; + GRAMMAR + end + + it "marks state after keyword_def as ENDFN" do + states = Lrama::States.new(grammar, Lrama::Tracer.new(Lrama::Logger.new)) + states.compute + states.compute_pslr + + lexer_contexts = grammar.lexer_contexts + endfn_mask = lexer_contexts["ENDFN"].bitmask + + # Find state reached after keyword_def + def_target = nil + states.states.each do |state| + state.term_transitions.each do |shift| + if shift.next_sym.id.s_value == "keyword_def" + def_target = shift.to_state + end + end + end + + expect(def_target).not_to be_nil + ctx = def_target.lexer_context || 0 + ctx_name = described_class.context_name(ctx, lexer_contexts) + expect(ctx & endfn_mask).not_to eq(0), + "State after keyword_def should have ENDFN context, got #{ctx_name}" + end + end + end + + describe "existing PSLR tests still pass" do + context "pure reduce profile" do + let(:grammar) do + build_grammar(<<~GRAMMAR, "states/pslr_pure_reduce.y") + %define lr.type pslr + %token-pattern RSHIFT />>/ + %token-pattern RANGLE />/ + %token-pattern ID /[a-z]+/ + %lex-prec RANGLE -~ RSHIFT + + %% + + program + : templ + | rshift_expr + ; + + templ + : a RANGLE + ; + + rshift_expr + : a RSHIFT ID + ; + + a + : ID + ; + GRAMMAR + end + + it "does not break PSLR" do + _, pslr_states = compute_ielr_and_pslr(grammar) + expect(pslr_states.pslr_inadequacies).to be_empty + end + end + + context "chained keyword split" do + let(:grammar) do + build_grammar(keyword_context_source(depth: 2), "states/pslr_keyword_ctx.y") + end + + it "does not break PSLR split" do + ielr_states, pslr_states = compute_ielr_and_pslr(grammar) + + expect(pslr_states.states_count).to be > ielr_states.states_count + expect(pslr_states.pslr_inadequacies).to be_empty + end + end + end +end diff --git a/spec/lrama/state/pslr_inadequacy_spec.rb b/spec/lrama/state/pslr_inadequacy_spec.rb new file mode 100644 index 000000000..cef7a5d62 --- /dev/null +++ b/spec/lrama/state/pslr_inadequacy_spec.rb @@ -0,0 +1,234 @@ +# frozen_string_literal: true + +RSpec.describe Lrama::State::PslrInadequacy do + let(:mock_state) do + instance_double(Lrama::State, id: 0) + end + + let(:mock_conflicting_states) do + [ + instance_double(Lrama::State, id: 1), + instance_double(Lrama::State, id: 2) + ] + end + + describe "#initialize" do + it "creates an LR-relative inadequacy" do + inadequacy = Lrama::State::PslrInadequacy.new( + type: Lrama::State::PslrInadequacy::LR_RELATIVE, + state: mock_state, + conflicting_states: mock_conflicting_states, + details: { reason: "test" } + ) + + expect(inadequacy.type).to eq(:lr_relative) + expect(inadequacy.state).to eq(mock_state) + expect(inadequacy.conflicting_states).to eq(mock_conflicting_states) + expect(inadequacy.details[:reason]).to eq("test") + end + + it "creates a PSLR-relative inadequacy" do + inadequacy = Lrama::State::PslrInadequacy.new( + type: Lrama::State::PslrInadequacy::PSLR_RELATIVE, + state: mock_state, + conflicting_states: mock_conflicting_states, + details: {} + ) + + expect(inadequacy.type).to eq(:pslr_relative) + end + end + + describe "#to_s" do + it "returns a human-readable description" do + inadequacy = Lrama::State::PslrInadequacy.new( + type: Lrama::State::PslrInadequacy::PSLR_RELATIVE, + state: mock_state, + conflicting_states: mock_conflicting_states, + details: {} + ) + + expect(inadequacy.to_s).to include("PSLR Inadequacy") + expect(inadequacy.to_s).to include("pslr_relative") + expect(inadequacy.to_s).to include("state 0") + expect(inadequacy.to_s).to include("1, 2") + end + end + + describe "constants" do + it "defines LR_RELATIVE constant" do + expect(Lrama::State::PslrInadequacy::LR_RELATIVE).to eq(:lr_relative) + end + + it "defines PSLR_RELATIVE constant" do + expect(Lrama::State::PslrInadequacy::PSLR_RELATIVE).to eq(:pslr_relative) + end + end +end + +RSpec.describe Lrama::State::PslrCompatibilityChecker do + let(:rangle) do + id = Lrama::Lexer::Token::Ident.new(s_value: "RANGLE") + regex = Lrama::Lexer::Token::Regex.new(s_value: "/>/") + Lrama::Grammar::TokenPattern.new( + id: id, + pattern: regex, + lineno: 1, + definition_order: 0 + ) + end + + let(:rshift) do + id = Lrama::Lexer::Token::Ident.new(s_value: "RSHIFT") + regex = Lrama::Lexer::Token::Regex.new(s_value: "/>>/") + Lrama::Grammar::TokenPattern.new( + id: id, + pattern: regex, + lineno: 1, + definition_order: 1 + ) + end + + let(:scanner_fsa) { Lrama::ScannerFSA.new([rangle, rshift]) } + let(:lex_prec) { Lrama::Grammar::LexPrec.new } + let(:length_prec) { Lrama::LengthPrecedences.new(lex_prec) } + let(:accepting_state_ids) { scanner_fsa.states.select(&:accepting?).map(&:id) } + let(:short_state_id) { accepting_state_ids.min } + let(:long_state_id) { accepting_state_ids.max } + + describe "#initialize" do + it "creates a compatibility checker" do + scanner_accepts = instance_double(Lrama::State::ScannerAccepts) + checker = Lrama::State::PslrCompatibilityChecker.new( + scanner_accepts, + length_prec + ) + + expect(checker).to be_a(Lrama::State::PslrCompatibilityChecker) + end + end + + describe "#compatible?" do + context "when both states select same tokens" do + it "returns true" do + scanner_accepts = instance_double(Lrama::State::ScannerAccepts) + allow(scanner_accepts).to receive(:[]).and_return(rangle) + + checker = Lrama::State::PslrCompatibilityChecker.new( + scanner_accepts, + length_prec + ) + + state1 = instance_double(Lrama::State, id: 0) + state2 = instance_double(Lrama::State, id: 1) + + expect(checker.compatible?(state1, state2, scanner_fsa)).to be true + end + end + + context "when both states have no tokens (nil)" do + it "returns true" do + scanner_accepts = instance_double(Lrama::State::ScannerAccepts) + allow(scanner_accepts).to receive(:[]).and_return(nil) + + checker = Lrama::State::PslrCompatibilityChecker.new( + scanner_accepts, + length_prec + ) + + state1 = instance_double(Lrama::State, id: 0) + state2 = instance_double(Lrama::State, id: 1) + + expect(checker.compatible?(state1, state2, scanner_fsa)).to be true + end + end + + context "when states select different tokens" do + it "returns false" do + scanner_accepts = instance_double(Lrama::State::ScannerAccepts) + + # State 0 selects RANGLE, State 1 selects RSHIFT + allow(scanner_accepts).to receive(:[]) do |state_id, _fsa_state_id| + if state_id == 0 + rangle + else + rshift + end + end + + checker = Lrama::State::PslrCompatibilityChecker.new( + scanner_accepts, + length_prec + ) + + state1 = instance_double(Lrama::State, id: 0) + state2 = instance_double(Lrama::State, id: 1) + + expect(checker.compatible?(state1, state2, scanner_fsa)).to be false + end + end + end + + describe "#profile" do + it "returns a stable accepting-state profile" do + scanner_accepts = instance_double(Lrama::State::ScannerAccepts) + allow(scanner_accepts).to receive(:[]) do |state_id, fsa_state_id| + if state_id == 0 + fsa_state_id == short_state_id ? rangle : rshift + else + fsa_state_id == short_state_id ? rangle : nil + end + end + + checker = Lrama::State::PslrCompatibilityChecker.new( + scanner_accepts, + length_prec + ) + + state = instance_double(Lrama::State, id: 0) + + expect(checker.profile(state, scanner_fsa)).to eq([ + [short_state_id, "RANGLE"], + [long_state_id, "RSHIFT"], + ]) + end + end + + describe "#group_by_profile" do + it "partitions states by scanner behavior" do + scanner_accepts = instance_double(Lrama::State::ScannerAccepts) + allow(scanner_accepts).to receive(:[]) do |state_id, fsa_state_id| + case [state_id, fsa_state_id] + when [0, short_state_id], [1, short_state_id] + rangle + when [0, long_state_id] + rshift + when [1, long_state_id] + nil + when [2, short_state_id] + rshift + when [2, long_state_id] + rshift + end + end + + checker = Lrama::State::PslrCompatibilityChecker.new( + scanner_accepts, + length_prec + ) + + state1 = instance_double(Lrama::State, id: 0) + state2 = instance_double(Lrama::State, id: 1) + state3 = instance_double(Lrama::State, id: 2) + + grouped = checker.group_by_profile([state1, state2, state3], scanner_fsa) + + expect(grouped.values.map(&:size)).to contain_exactly(1, 1, 1) + expect(grouped.keys).to include( + [[short_state_id, "RANGLE"], [long_state_id, "RSHIFT"]], + [[short_state_id, "RANGLE"], [long_state_id, nil]], + [[short_state_id, "RSHIFT"], [long_state_id, "RSHIFT"]], + ) + end + end +end diff --git a/spec/lrama/state/scanner_accepts_spec.rb b/spec/lrama/state/scanner_accepts_spec.rb new file mode 100644 index 000000000..56a0bbcf8 --- /dev/null +++ b/spec/lrama/state/scanner_accepts_spec.rb @@ -0,0 +1,394 @@ +# frozen_string_literal: true + +RSpec.describe Lrama::State::ScannerAccepts do + def ident(name) + Lrama::Lexer::Token::Ident.new(s_value: name) + end + + def token_pattern(name, regex, order) + Lrama::Grammar::TokenPattern.new( + id: ident(name), + pattern: Lrama::Lexer::Token::Regex.new(s_value: "/#{regex}/"), + lineno: 1, + definition_order: order + ) + end + + def shift_for(name) + symbol = instance_double( + Lrama::Grammar::Symbol, + term?: true, + id: ident(name) + ) + instance_double(Lrama::State::Action::Shift, next_sym: symbol) + end + + def parser_state(id, token_names) + instance_double( + Lrama::State, + id: id, + term_transitions: token_names.map {|name| shift_for(name) }, + reduces: [] + ) + end + + describe "#build and #[]" do + let(:rangle) { token_pattern("RANGLE", ">", 0) } + let(:rshift) { token_pattern("RSHIFT", ">>", 1) } + let(:scanner_fsa) { Lrama::ScannerFSA.new([rangle, rshift]) } + let(:lex_prec) { Lrama::Grammar::LexPrec.new } + let(:length_prec) { Lrama::LengthPrecedences.new(lex_prec) } + + it "builds scanner_accepts from parser acceptable tokens" do + state = parser_state(0, ["RANGLE"]) + scanner_accepts = Lrama::State::ScannerAccepts.new( + [state], + scanner_fsa, + lex_prec, + length_prec + ) + + scanner_accepts.build + + accepting = scanner_fsa.states.find {|s| s.accepting_tokens.map(&:name).include?("RANGLE") } + expect(scanner_accepts[0, accepting.id].name).to eq("RANGLE") + end + + it "includes layout tokens in every parser-state accept set" do + div = token_pattern("DIV", "/", 0) + layout = token_pattern("YYLAYOUT_WS", "[ \\t]+", 1) + scanner_fsa = Lrama::ScannerFSA.new([div, layout]) + state = parser_state(0, ["DIV"]) + scanner_accepts = Lrama::State::ScannerAccepts.new( + [state], + scanner_fsa, + lex_prec, + Lrama::LengthPrecedences.new(lex_prec), + layout_token_names: Set["YYLAYOUT_WS"] + ) + + scanner_accepts.build + + accepting = scanner_fsa.states.find {|s| s.accepting_tokens.map(&:name).include?("YYLAYOUT_WS") } + expect(scanner_accepts[0, accepting.id].name).to eq("YYLAYOUT_WS") + end + end + + describe "complete conflict resolution" do + it "does not use declaration order for unresolved identity conflicts" do + tokens = [ + token_pattern("A", "a", 0), + token_pattern("B", "a", 1), + token_pattern("C", "a", 2) + ] + scanner_fsa = Lrama::ScannerFSA.new(tokens) + lex_prec = Lrama::Grammar::LexPrec.new + scanner_accepts = Lrama::State::ScannerAccepts.new( + [parser_state(0, ["A", "B", "C"])], + scanner_fsa, + lex_prec, + Lrama::LengthPrecedences.new(lex_prec) + ) + + scanner_accepts.build + + expect(scanner_accepts.unresolved_conflicts?).to be true + parser_rows = scanner_accepts.table.reject do |(parser_state_id, _scanner_state_id), _token| + parser_state_id == Lrama::State::ScannerAccepts::FALLBACK_ROW_ID + end + expect(parser_rows).to be_empty + expect(scanner_accepts.fallback_table.values.map(&:name)).to contain_exactly("A") + end + + it "selects a unique explicitly declared identity winner" do + tokens = [ + token_pattern("A", "a", 0), + token_pattern("B", "a", 1), + token_pattern("C", "a", 2) + ] + scanner_fsa = Lrama::ScannerFSA.new(tokens) + lex_prec = Lrama::Grammar::LexPrec.new + lex_prec.add_rule(left_token: ident("A"), operator: Lrama::Grammar::LexPrec::IDENTITY_RIGHT, right_token: ident("C"), lineno: 1) + lex_prec.add_rule(left_token: ident("B"), operator: Lrama::Grammar::LexPrec::IDENTITY_RIGHT, right_token: ident("C"), lineno: 1) + scanner_accepts = Lrama::State::ScannerAccepts.new( + [parser_state(0, ["A", "B", "C"])], + scanner_fsa, + lex_prec, + Lrama::LengthPrecedences.new(lex_prec) + ) + + scanner_accepts.build + + accepting = scanner_fsa.states.find(&:accepting?) + expect(scanner_accepts[0, accepting.id].name).to eq("C") + expect(scanner_accepts.unresolved_conflicts?).to be false + end + + it "keeps conflicts finite for looped scanner states" do + id = token_pattern("ID", "[a-z]+", 0) + kw = token_pattern("IF", "if", 1) + scanner_fsa = Lrama::ScannerFSA.new([id, kw]) + lex_prec = Lrama::Grammar::LexPrec.new + scanner_accepts = Lrama::State::ScannerAccepts.new( + [parser_state(0, ["ID", "IF"])], + scanner_fsa, + lex_prec, + Lrama::LengthPrecedences.new(lex_prec) + ) + + scanner_accepts.build + + expect(scanner_accepts.conflicts.size).to be < 10 + end + + it "keeps an empty outcome distinct from an unresolved conflict" do + resolver = Lrama::State::ScannerAccepts::ProfileResolver.new( + Lrama::Grammar::LexPrec.new, + Lrama::LengthPrecedences.new(Lrama::Grammar::LexPrec.new) + ) + + outcome = resolver.resolve(Set.new, nil, Set.new) + + expect(outcome).to be_empty + expect(outcome).not_to be_unresolved + end + + it "uses same-token autolength without fallback mode" do + resolver = Lrama::State::ScannerAccepts::ProfileResolver.new( + Lrama::Grammar::LexPrec.new, + Lrama::LengthPrecedences.new(Lrama::Grammar::LexPrec.new) + ) + + outcome = resolver.resolve(Set["ID"], "ID", Set["ID"]) + + expect(outcome).to be_resolved + expect(outcome.token_name).to eq("ID") + end + + it "uses declaration order only in fallback mode" do + lex_prec = Lrama::Grammar::LexPrec.new + length_prec = Lrama::LengthPrecedences.new(lex_prec) + normal = Lrama::State::ScannerAccepts::ProfileResolver.new( + lex_prec, + length_prec, + token_order: { "A" => 1, "B" => 0 } + ) + fallback = Lrama::State::ScannerAccepts::ProfileResolver.new( + lex_prec, + length_prec, + fallback: true, + token_order: { "A" => 1, "B" => 0 } + ) + + expect(normal.resolve(Set.new, nil, Set["A", "B"])).to be_unresolved + expect(fallback.resolve(Set.new, nil, Set["A", "B"]).token_name).to eq("B") + end + + it "uses explicit identity precedence before fallback declaration order" do + lex_prec = Lrama::Grammar::LexPrec.new + lex_prec.add_rule(left_token: ident("A"), operator: Lrama::Grammar::LexPrec::IDENTITY_RIGHT, right_token: ident("B"), lineno: 1) + fallback = Lrama::State::ScannerAccepts::ProfileResolver.new( + lex_prec, + Lrama::LengthPrecedences.new(lex_prec), + fallback: true, + token_order: { "A" => 0, "B" => 1 } + ) + + expect(fallback.resolve(Set.new, nil, Set["A", "B"]).token_name).to eq("B") + end + + it "preserves explicit identity precedence when fallback length precedence is needed" do + tokens = [ + token_pattern("X", "x", 0), + token_pattern("SHORT", "a", 1), + token_pattern("A", "ab", 2), + token_pattern("B", "ab", 3) + ] + scanner_fsa = Lrama::ScannerFSA.new(tokens) + lex_prec = Lrama::Grammar::LexPrec.new + lex_prec.add_rule(left_token: ident("A"), operator: Lrama::Grammar::LexPrec::IDENTITY_RIGHT, right_token: ident("B"), lineno: 1) + scanner_accepts = Lrama::State::ScannerAccepts.new( + [parser_state(0, ["X"])], + scanner_fsa, + lex_prec, + Lrama::LengthPrecedences.new(lex_prec) + ) + + scanner_accepts.build + + accepting_ab = scanner_fsa.states.find do |state| + state.accepting_tokens.map(&:name).sort == ["A", "B"] + end + expect(scanner_accepts.fallback_table.fetch(accepting_ab.id).name).to eq("B") + end + + it "uses declaration order for fallback-only explicit identity cycles" do + tokens = [ + token_pattern("X", "x", 0), + token_pattern("A", "a", 1), + token_pattern("B", "a", 2), + token_pattern("C", "a", 3) + ] + scanner_fsa = Lrama::ScannerFSA.new(tokens) + lex_prec = Lrama::Grammar::LexPrec.new + lex_prec.add_rule(left_token: ident("A"), operator: Lrama::Grammar::LexPrec::IDENTITY_RIGHT, right_token: ident("B"), lineno: 1) + lex_prec.add_rule(left_token: ident("B"), operator: Lrama::Grammar::LexPrec::IDENTITY_RIGHT, right_token: ident("C"), lineno: 2) + lex_prec.add_rule(left_token: ident("C"), operator: Lrama::Grammar::LexPrec::IDENTITY_RIGHT, right_token: ident("A"), lineno: 3) + scanner_accepts = Lrama::State::ScannerAccepts.new( + [parser_state(0, ["X"])], + scanner_fsa, + lex_prec, + Lrama::LengthPrecedences.new(lex_prec) + ) + + scanner_accepts.build + + accepting = scanner_fsa.states.find {|state| state.accepting_tokens.map(&:name).sort == ["A", "B", "C"] } + expect(scanner_accepts.fallback_table.fetch(accepting.id).name).to eq("A") + end + + it "keeps normal parser-state rows strict for explicit identity cycles" do + tokens = [ + token_pattern("A", "a", 0), + token_pattern("B", "a", 1), + token_pattern("C", "a", 2) + ] + scanner_fsa = Lrama::ScannerFSA.new(tokens) + lex_prec = Lrama::Grammar::LexPrec.new + lex_prec.add_rule(left_token: ident("A"), operator: Lrama::Grammar::LexPrec::IDENTITY_RIGHT, right_token: ident("B"), lineno: 1) + lex_prec.add_rule(left_token: ident("B"), operator: Lrama::Grammar::LexPrec::IDENTITY_RIGHT, right_token: ident("C"), lineno: 2) + lex_prec.add_rule(left_token: ident("C"), operator: Lrama::Grammar::LexPrec::IDENTITY_RIGHT, right_token: ident("A"), lineno: 3) + scanner_accepts = Lrama::State::ScannerAccepts.new( + [parser_state(0, ["A", "B", "C"])], + scanner_fsa, + lex_prec, + Lrama::LengthPrecedences.new(lex_prec) + ) + + scanner_accepts.build + + accepting = scanner_fsa.states.find {|state| state.accepting_tokens.map(&:name).sort == ["A", "B", "C"] } + expect(scanner_accepts[0, accepting.id]).to be_nil + expect(scanner_accepts.unresolved_conflicts?).to be true + end + end + + describe Lrama::State::ScannerAccepts::CompatibilityChecker do + let(:rangle) { token_pattern("RANGLE", ">", 0) } + let(:rshift) { token_pattern("RSHIFT", ">>", 1) } + let(:scanner_fsa) { Lrama::ScannerFSA.new([rangle, rshift]) } + let(:lex_prec) { Lrama::Grammar::LexPrec.new } + let(:checker) do + described_class.new(scanner_fsa, lex_prec, Lrama::LengthPrecedences.new(lex_prec)) + end + + it "treats a missing match on one side as irrelevant" do + a = token_pattern("A", "a", 0) + b = token_pattern("B", "b", 1) + fsa = Lrama::ScannerFSA.new([a, b]) + checker = described_class.new(fsa, lex_prec, Lrama::LengthPrecedences.new(lex_prec)) + + expect(checker.compatible?(Set["A"], Set["B"])).to be true + end + + it "rejects different resolved outcomes when both sides match" do + expect(checker.compatible?(Set["RANGLE"], Set["RSHIFT"])).to be false + end + + it "rejects resolved versus unresolved outcomes" do + a = token_pattern("A", "a", 0) + b = token_pattern("B", "a", 1) + fsa = Lrama::ScannerFSA.new([a, b]) + checker = described_class.new(fsa, lex_prec, Lrama::LengthPrecedences.new(lex_prec)) + + expect(checker.compatible?(Set["A"], Set["A", "B"])).to be false + end + + it "accepts unresolved outcomes on both sides" do + a = token_pattern("A", "a", 0) + b = token_pattern("B", "a", 1) + fsa = Lrama::ScannerFSA.new([a, b]) + checker = described_class.new(fsa, lex_prec, Lrama::LengthPrecedences.new(lex_prec)) + + expect(checker.compatible?(Set["A", "B"], Set["A", "B"])).to be true + end + end + + describe "lexical ties" do + it "expands acc(sp) through tie closure" do + id = token_pattern("ID", "[a-z]+", 0) + kw = token_pattern("IF", "if", 1) + scanner_fsa = Lrama::ScannerFSA.new([id, kw]) + lex_prec = Lrama::Grammar::LexPrec.new + lex_prec.add_rule(left_token: ident("ID"), operator: Lrama::Grammar::LexPrec::IDENTITY_RIGHT_LONGEST, right_token: ident("IF"), lineno: 1) + lex_tie = Lrama::Grammar::LexTie.new + lex_tie.add_tie("ID", "IF") + scanner_accepts = Lrama::State::ScannerAccepts.new( + [parser_state(0, ["ID"])], + scanner_fsa, + lex_prec, + Lrama::LengthPrecedences.new(lex_prec), + lex_tie + ) + + scanner_accepts.build + + accepting = scanner_fsa.states.find {|state| state.accepting_tokens.map(&:name).include?("IF") } + expect(scanner_accepts[0, accepting.id].name).to eq("IF") + end + + it "does not expand tokens tied only through layout injection" do + div = token_pattern("DIV", "/", 0) + layout = token_pattern("YYLAYOUT_WS", "[ \\t]+", 1) + layout_alias = token_pattern("LAYOUT_ALIAS", "[ \\t]+", 2) + scanner_fsa = Lrama::ScannerFSA.new([div, layout, layout_alias]) + lex_prec = Lrama::Grammar::LexPrec.new + lex_tie = Lrama::Grammar::LexTie.new + lex_tie.add_tie("YYLAYOUT_WS", "LAYOUT_ALIAS") + scanner_accepts = Lrama::State::ScannerAccepts.new( + [parser_state(0, ["DIV"])], + scanner_fsa, + lex_prec, + Lrama::LengthPrecedences.new(lex_prec), + lex_tie, + layout_token_names: Set["YYLAYOUT_WS"] + ) + + acc_sp = scanner_accepts.send(:compute_acc_sp, parser_state(0, ["DIV"])) + + expect(acc_sp).to contain_exactly("DIV", "YYLAYOUT_WS") + end + end + + describe "pure reduce states" do + let(:rangle) { token_pattern("RANGLE", ">", 0) } + let(:rshift) { token_pattern("RSHIFT", ">>", 1) } + let(:scanner_fsa) { Lrama::ScannerFSA.new([rangle, rshift]) } + let(:lex_prec) { Lrama::Grammar::LexPrec.new } + let(:length_prec) { Lrama::LengthPrecedences.new(lex_prec) } + let(:reduce) { instance_double(Lrama::State::Action::Reduce) } + let(:parser_state) do + instance_double( + Lrama::State, + term_transitions: [], + reduces: [reduce], + ) + end + + it "uses propagated item lookaheads when explicit reduce lookahead is absent" do + allow(parser_state).to receive(:acceptable_pslr_reduce_lookahead).with(reduce).and_return([ + instance_double(Lrama::Grammar::Symbol, id: ident("RANGLE")), + instance_double(Lrama::Grammar::Symbol, id: ident("RSHIFT")), + ]) + + scanner_accepts = Lrama::State::ScannerAccepts.new( + [parser_state], + scanner_fsa, + lex_prec, + length_prec + ) + + expect(scanner_accepts.send(:compute_acc_sp, parser_state).to_a).to contain_exactly("RANGLE", "RSHIFT") + end + end +end diff --git a/spec/lrama/states_spec.rb b/spec/lrama/states_spec.rb index 28c217e2a..d864b0323 100644 --- a/spec/lrama/states_spec.rb +++ b/spec/lrama/states_spec.rb @@ -3158,5 +3158,502 @@ class : keyword_class tSTRING keyword_end %prec tPLUS expect(logger).not_to have_received(:error) end end + + context "when unresolved PSLR inadequacies remain" do + let(:header) do + <<~STR + %define lr.type pslr + %token-pattern RSHIFT />>/ + %token-pattern RANGLE />/ + %lex-no-tie RANGLE RSHIFT + + %% + + program: RSHIFT | RANGLE + STR + end + + it "fails fast instead of silently generating a parser" do + grammar = Lrama::Parser.new(header, "states/pslr_inadequacy.y").parse + grammar.prepare + grammar.validate! + states = Lrama::States.new(grammar, Lrama::Tracer.new(Lrama::Logger.new)) + states.compute + states.instance_variable_set( + :@pslr_inadequacies, + [ + Lrama::State::PslrInadequacy.new( + type: Lrama::State::PslrInadequacy::PSLR_RELATIVE, + state: instance_double(Lrama::State, id: 3), + conflicting_states: [instance_double(Lrama::State, id: 3), instance_double(Lrama::State, id: 4)], + details: { reason: "Scanner behavior differs between isocore states" } + ) + ] + ) + logger = Lrama::Logger.new + allow(logger).to receive(:error) + + expect { states.validate!(logger) }.to raise_error(SystemExit) + expect(logger).to have_received(:error).with(include("PSLR Inadequacy")) + end + end + end + + describe "PSLR split helpers" do + let(:y) do + <<~GRAMMAR + %define lr.type pslr + %token-pattern RSHIFT />>/ + %token-pattern RANGLE />/ + %lex-no-tie RANGLE RSHIFT + + %% + + program: RSHIFT | RANGLE + GRAMMAR + end + + let(:grammar) do + g = Lrama::Parser.new(y, "states/pslr_split.y").parse + g.prepare + g.validate! + g + end + + let(:states) { Lrama::States.new(grammar, Lrama::Tracer.new(Lrama::Logger.new)) } + let(:kernel_item) { instance_double(Lrama::State::Item, end_of_rule?: true) } + let(:reduce) { instance_double(Lrama::State::Action::Reduce, item: kernel_item, look_ahead: [grammar.find_symbol_by_s_value!("RSHIFT")]) } + let(:mock_state) do + instance_double( + Lrama::State, + is_compatible?: true, + kernels: [kernel_item], + term_transitions: [], + reduces: [reduce], + acceptable_reduce_lookahead: [grammar.find_symbol_by_s_value!("RSHIFT")], + acceptable_pslr_reduce_lookahead: [grammar.find_symbol_by_s_value!("RSHIFT")], + ) + end + + before do + states.instance_variable_set(:@scanner_fsa, Lrama::ScannerFSA.new(grammar.token_patterns)) + states.instance_variable_set(:@pslr_split_enabled, true) + end + + it "derives different PSLR signatures from different propagated lookaheads" do + current = states.send(:pslr_state_signature, mock_state) + filtered = states.send( + :pslr_state_signature, + mock_state, + { kernel_item => [grammar.find_symbol_by_s_value!("RANGLE")] }, + ) + + expect(current.map(&:last)).to include("RSHIFT") + expect(current.map(&:last)).not_to include("RANGLE") + expect(filtered.map(&:last)).to include("RANGLE") + expect(filtered.map(&:last)).not_to include("RSHIFT") + end + + it "treats states with different PSLR signatures as incompatible during splitting" do + filtered_lookaheads = { kernel_item => [grammar.find_symbol_by_s_value!("RANGLE")] } + expect(states.send(:compatible_split_state?, mock_state, filtered_lookaheads)).to be false + end + + it "detects unresolved PSLR inadequacies per transition" do + propagated = { kernel_item => [grammar.find_symbol_by_s_value!("RANGLE")] } + matching_state = instance_double(Lrama::State, id: 8) + next_state = instance_double(Lrama::State, id: 4) + transition_symbol = instance_double( + Lrama::Grammar::Symbol, + id: instance_double(Lrama::Lexer::Token::Ident, s_value: "RSHIFT"), + ) + transition = instance_double(Lrama::State::Action::Shift, to_state: next_state, next_sym: transition_symbol) + from_state = instance_double( + Lrama::State, + id: 1, + transitions: [transition], + propagate_lookaheads_without_filter: propagated, + ) + + allow(next_state).to receive(:lalr_isocore).and_return(next_state) + allow(next_state).to receive(:ielr_isocores).and_return([next_state, matching_state]) + allow(states).to receive(:pslr_state_signature).with(next_state, propagated).and_return([[1, "RANGLE"]]) + allow(states).to receive(:pslr_state_signature).with(next_state).and_return([[1, "RSHIFT"]]) + allow(states).to receive(:pslr_state_signature).with(matching_state).and_return([[1, "RANGLE"]]) + allow(states).to receive(:acceptable_tokens_for_pslr).with(next_state, propagated).and_return(Set["RANGLE"]) + allow(states).to receive(:acceptable_tokens_for_pslr).with(next_state).and_return(Set["RSHIFT"]) + allow(states).to receive(:acceptable_tokens_for_pslr).with(matching_state).and_return(Set["RANGLE"]) + states.instance_variable_set(:@states, [from_state]) + + inadequacies = states.send(:detect_pslr_inadequacies) + + expect(inadequacies.size).to eq(1) + expect(inadequacies.first.details[:matching_state_id]).to eq(8) + expect(inadequacies.first.details[:transition_symbol]).to eq("RSHIFT") + end + + it "merges propagated lookaheads into an existing split state" do + current_lookaheads = { kernel_item => [grammar.find_symbol_by_s_value!("RSHIFT")] } + incoming_lookaheads = { kernel_item => [grammar.find_symbol_by_s_value!("RANGLE")] } + target_state = instance_double(Lrama::State, lookaheads_recomputed: true) + transition = instance_double(Lrama::State::Action::Shift, to_state: target_state) + split_state = instance_double( + Lrama::State, + kernels: [kernel_item], + item_lookahead_set: current_lookaheads, + transitions: [transition], + ) + + allow(split_state).to receive(:item_lookahead_set=) + + states.send(:merge_lookaheads, split_state, incoming_lookaheads) + + expect(split_state).to have_received(:item_lookahead_set=).with( + kernel_item => [grammar.find_symbol_by_s_value!("RSHIFT"), grammar.find_symbol_by_s_value!("RANGLE")], + ) + end + end + + describe "PSLR pure-reduce profile regression" do + let(:y) do + <<~GRAMMAR + %define lr.type pslr + %token-pattern RSHIFT />>/ + %token-pattern RANGLE />/ + %token-pattern ID /[a-z]+/ + %lex-prec RANGLE -~ RSHIFT + + %% + + program + : templ + | rshift_expr + ; + + templ + : a RANGLE + ; + + rshift_expr + : a RSHIFT ID + ; + + a + : ID + ; + GRAMMAR + end + + let(:grammar) do + g = Lrama::Parser.new(y, "states/pslr_pure_reduce.y").parse + g.prepare + g.validate! + g + end + + it "keeps pure reduce states scanner-compatible without forcing a split" do + ielr_states = Lrama::States.new(grammar, Lrama::Tracer.new(Lrama::Logger.new)) + ielr_states.compute + ielr_states.compute_ielr + + pslr_states = Lrama::States.new(grammar, Lrama::Tracer.new(Lrama::Logger.new)) + pslr_states.compute + pslr_states.compute_pslr + + reduce_state = pslr_states.states.find do |state| + state.reduces.any? { |reduce| reduce.rule.display_name == "a -> ID" } + end + + expect(pslr_states.states_count).to eq(ielr_states.states_count) + expect(pslr_states.pslr_inadequacies).to be_empty + expect(pslr_states.send(:acceptable_tokens_for_pslr, reduce_state).to_a).to contain_exactly("RANGLE", "RSHIFT") + end + end + + describe "PSLR chained keyword split regression" do + let(:y) do + <<~GRAMMAR + %define lr.type pslr + %token-pattern P /p/ + %token-pattern Q /q/ + %token-pattern X /x/ + %token-pattern IF /if/ + %token-pattern ID /[a-z]+/ + %lex-prec ID <~ IF + + %% + + program + : kw_context + | id_context + ; + + kw_context + : P shared IF + ; + + id_context + : Q shared ID + ; + + shared + : n1 + ; + + n1 + : n2 + ; + + n2 + : X + ; + GRAMMAR + end + + let(:grammar) do + g = Lrama::Parser.new(y, "states/pslr_keyword_context.y").parse + g.prepare + g.validate! + g + end + + it "splits every chained reduce state by scanner profile" do + ielr_states = Lrama::States.new(grammar, Lrama::Tracer.new(Lrama::Logger.new)) + ielr_states.compute + ielr_states.compute_ielr + + pslr_states = Lrama::States.new(grammar, Lrama::Tracer.new(Lrama::Logger.new)) + pslr_states.compute + pslr_states.compute_pslr + + reduce_states = pslr_states.states + .select { |state| state.reduces.any? } + .group_by { |state| state.reduces.first.rule.display_name } + + expect(pslr_states.states_count).to be > ielr_states.states_count + expect(pslr_states.pslr_inadequacies).to be_empty + + ["shared -> n1", "n1 -> n2", "n2 -> X"].each do |rule_name| + states_for_rule = reduce_states.fetch(rule_name) + token_sets = states_for_rule.map { |state| pslr_states.send(:acceptable_tokens_for_pslr, state) } + + expect(states_for_rule.size).to eq(2) + expect(states_for_rule.count(&:split_state?)).to eq(1) + expect(token_sets.any? { |set| set.include?("IF") && !set.include?("ID") }).to be(true) + expect(token_sets.any? { |set| set.include?("ID") && !set.include?("IF") }).to be(true) + end + end + end + + describe "PSLR chained shift/angle split regression" do + let(:y) do + <<~GRAMMAR + %define lr.type pslr + %token-pattern LT />/ + %token-pattern RANGLE />/ + %token-pattern ID /[a-z]+/ + %lex-no-tie RANGLE RSHIFT + + %% + + program + : template_expr + | shift_expr + ; + + template_expr + : LT shared RANGLE + ; + + shift_expr + : START shared RSHIFT ID + ; + + shared + : n1 + ; + + n1 + : n2 + ; + + n2 + : MARK + ; + GRAMMAR + end + + let(:grammar) do + g = Lrama::Parser.new(y, "states/pslr_shift_chain.y").parse + g.prepare + g.validate! + g + end + + it "splits every chained reduce state by shift/angle scanner profile" do + ielr_states = Lrama::States.new(grammar, Lrama::Tracer.new(Lrama::Logger.new)) + ielr_states.compute + ielr_states.compute_ielr + + pslr_states = Lrama::States.new(grammar, Lrama::Tracer.new(Lrama::Logger.new)) + pslr_states.compute + pslr_states.compute_pslr + + reduce_states = pslr_states.states + .select { |state| state.reduces.any? } + .group_by { |state| state.reduces.first.rule.display_name } + + expect(pslr_states.states_count).to be > ielr_states.states_count + expect(pslr_states.pslr_inadequacies).to be_empty + + ["shared -> n1", "n1 -> n2", "n2 -> MARK"].each do |rule_name| + states_for_rule = reduce_states.fetch(rule_name) + token_sets = states_for_rule.map { |state| pslr_states.send(:acceptable_tokens_for_pslr, state) } + + expect(states_for_rule.size).to eq(2) + expect(states_for_rule.count(&:split_state?)).to eq(1) + expect(token_sets.any? { |set| set.include?("RANGLE") && !set.include?("RSHIFT") }).to be(true) + expect(token_sets.any? { |set| set.include?("RSHIFT") && !set.include?("RANGLE") }).to be(true) + end + end + end + + describe "PSLR mixed family regressions" do + { + "empty shared wrapper" => { + path: "states/pslr_mixed_empty.y", + grows: true, + grammar: <<~GRAMMAR, + %define lr.type pslr + %token-pattern LT />/ + %token-pattern RANGLE />/ + %lex-prec ID <~ IF + %lex-no-tie RANGLE RSHIFT + + %% + + program + : kw + | ident + | templ + | shift_expr + ; + + kw + : P shared IF + ; + + ident + : Q shared ID + ; + + templ + : LT shared RANGLE + ; + + shift_expr + : START shared RSHIFT ID + ; + + shared + : opt n1 + ; + + opt + : + ; + + n1 + : MARK + ; + GRAMMAR + }, + "chain2 shared wrapper" => { + path: "states/pslr_mixed_chain2.y", + grows: true, + grammar: <<~GRAMMAR, + %define lr.type pslr + %token-pattern LT />/ + %token-pattern RANGLE />/ + %lex-prec ID <~ IF + %lex-no-tie RANGLE RSHIFT + + %% + + program + : kw + | ident + | templ + | shift_expr + ; + + kw + : P shared IF + ; + + ident + : Q shared ID + ; + + templ + : LT shared RANGLE + ; + + shift_expr + : START shared RSHIFT ID + ; + + shared + : n1 + ; + + n1 + : n2 + ; + + n2 + : MARK + ; + GRAMMAR + } + }.each do |label, attrs| + it "keeps #{label} scanner-compatible" do + grammar = Lrama::Parser.new(attrs[:grammar], attrs[:path]).parse + grammar.prepare + grammar.validate! + + ielr_states = Lrama::States.new(grammar, Lrama::Tracer.new(Lrama::Logger.new)) + ielr_states.compute + ielr_states.compute_ielr + + pslr_states = Lrama::States.new(grammar, Lrama::Tracer.new(Lrama::Logger.new)) + pslr_states.compute + pslr_states.compute_pslr + + if attrs[:grows] + expect(pslr_states.states_count).to be > ielr_states.states_count + else + expect(pslr_states.states_count).to eq(ielr_states.states_count) + end + expect(pslr_states.pslr_inadequacies).to be_empty + end + end end end diff --git a/spec/spec_helper.rb b/spec/spec_helper.rb index c832a1cea..338404c19 100644 --- a/spec/spec_helper.rb +++ b/spec/spec_helper.rb @@ -26,6 +26,9 @@ end require "lrama" +Dir[File.expand_path("support/**/*.rb", __dir__)].sort.each do |file| + require file +end module RSpecHelper def fixture_path(file_name) diff --git a/spec/support/pslr_family_helper.rb b/spec/support/pslr_family_helper.rb new file mode 100644 index 000000000..77934b9b8 --- /dev/null +++ b/spec/support/pslr_family_helper.rb @@ -0,0 +1,159 @@ +# frozen_string_literal: true + +module PslrFamilyHelper + def build_grammar(source, path) + grammar = Lrama::Parser.new(source, path).parse + grammar.prepare + grammar.validate! + grammar + end + + def compute_ielr_and_pslr(grammar) + ielr_states = Lrama::States.new(grammar, Lrama::Tracer.new(Lrama::Logger.new)) + ielr_states.compute + ielr_states.compute_ielr + + pslr_states = Lrama::States.new(grammar, Lrama::Tracer.new(Lrama::Logger.new)) + pslr_states.compute + pslr_states.compute_pslr + + [ielr_states, pslr_states] + end + + def acceptable_tokens(states, state) + states.send(:acceptable_tokens_for_pslr, state).to_a + end + + def shared_chain_rules(name:, terminal:, depth:, prefix: "n") + return <<~RULES if depth.zero? + #{name} + : #{terminal} + ; + RULES + + rules = [<<~RULE] + #{name} + : #{prefix}1 + ; + RULE + + 1.upto(depth - 1) do |index| + rules << <<~RULE + #{prefix}#{index} + : #{prefix}#{index + 1} + ; + RULE + end + + rules << <<~RULE + #{prefix}#{depth} + : #{terminal} + ; + RULE + + rules.join("\n") + end + + def keyword_context_source(depth:) + <<~GRAMMAR + %define lr.type pslr + %token-pattern P /p/ + %token-pattern Q /q/ + %token-pattern X /x/ + %token-pattern IF /if/ + %token-pattern ID /[a-z]+/ + %lex-prec ID <~ IF + + %% + + program + : kw_context + | id_context + ; + + kw_context + : P shared IF + ; + + id_context + : Q shared ID + ; + + #{shared_chain_rules(name: "shared", terminal: "X", depth: depth)} + GRAMMAR + end + + def shift_angle_source(depth:) + <<~GRAMMAR + %define lr.type pslr + %token-pattern LT />/ + %token-pattern RANGLE />/ + %token-pattern ID /[a-z]+/ + %lex-no-tie RANGLE RSHIFT + + %% + + program + : template_expr + | shift_expr + ; + + template_expr + : LT shared RANGLE + ; + + shift_expr + : START shared RSHIFT ID + ; + + #{shared_chain_rules(name: "shared", terminal: "MARK", depth: depth)} + GRAMMAR + end + + def mixed_context_source(depth:) + <<~GRAMMAR + %define lr.type pslr + %token-pattern LT />/ + %token-pattern RANGLE />/ + %lex-prec ID <~ IF + %lex-no-tie RANGLE RSHIFT + + %% + + program + : kw_context + | id_context + | template_expr + | shift_expr + ; + + kw_context + : P shared IF + ; + + id_context + : Q shared ID + ; + + template_expr + : LT shared RANGLE + ; + + shift_expr + : START shared RSHIFT ID + ; + + #{shared_chain_rules(name: "shared", terminal: "MARK", depth: depth)} + GRAMMAR + end +end From 4276e96a921e755974cb9666dfe19745ed24d510 Mon Sep 17 00:00:00 2001 From: ydah Date: Wed, 8 Jul 2026 21:40:35 +0900 Subject: [PATCH 05/15] pslr: emit pseudo-scanner tables and runtime in generated parsers Generate the PSLR(1) runtime as part of the yacc.c template output: * yy_scanner_transition / yy_state_to_accepting (Def 3.2.13), yy_scanner_accepts rows per parser state plus the fallback row (Def 3.2.14, section 3.5.1), length precedence tables for the normal and fallback faces, token-pattern-to-token-id mapping and layout flags * yy_pseudo_scan / yy_pseudo_scan_full implement Def 3.2.16: walk the DFA, prefer matches per the length precedence tables, fall back to the syntax-error row when no normal-row match exists, and flag layout tokens for skipping * %token-action bodies are emitted with yytext/yyleng and accumulated layout text (yypslr_layout_text/leng) available * yy_state_accepts_token and yy_state_eventually_accepts_token allow a hand-written lexer to query acceptability of a token in a state; the eventual variant follows default-reduction chains (Lrama extension) * a LAC (lookahead correction) checker performs exploratory parses so scanner decisions taken from merged states cannot commit erroneous semantic actions (section 3.5.2) * bridge-mode macros (YYSETSTATE_CONTEXT, YYPSLR_PSEUDO_SCAN*) let an existing yylex consult the pseudo-scanner through %define api.pslr.state-member Integration fixtures cover template argument lists, keyword contexts, layout comments, implicit literals and fallback precedence. --- lib/lrama/output.rb | 731 ++++++++++++++++++ spec/fixtures/integration/pslr_context.l | 50 ++ spec/fixtures/integration/pslr_context.y | 75 ++ .../integration/pslr_fallback_precedence.l | 73 ++ .../integration/pslr_fallback_precedence.y | 93 +++ .../integration/pslr_implicit_literal.l | 30 + .../integration/pslr_implicit_literal.y | 66 ++ .../integration/pslr_keyword_context.l | 41 + .../integration/pslr_keyword_context.y | 87 +++ .../integration/pslr_layout_comment.l | 36 + .../integration/pslr_layout_comment.y | 70 ++ .../fixtures/integration/pslr_mixed_context.l | 69 ++ .../fixtures/integration/pslr_mixed_context.y | 102 +++ spec/fixtures/integration/pslr_shift_chain.l | 58 ++ spec/fixtures/integration/pslr_shift_chain.y | 88 +++ .../pslr_template_argument_lists.l | 36 + .../pslr_template_argument_lists.y | 91 +++ spec/fixtures/integration/pslr_token_action.y | 69 ++ spec/lrama/integration_spec.rb | 382 +++++++++ spec/lrama/output_spec.rb | 312 ++++++++ template/bison/_yacc.h | 4 + template/bison/yacc.c | 271 ++++++- 22 files changed, 2832 insertions(+), 2 deletions(-) create mode 100644 spec/fixtures/integration/pslr_context.l create mode 100644 spec/fixtures/integration/pslr_context.y create mode 100644 spec/fixtures/integration/pslr_fallback_precedence.l create mode 100644 spec/fixtures/integration/pslr_fallback_precedence.y create mode 100644 spec/fixtures/integration/pslr_implicit_literal.l create mode 100644 spec/fixtures/integration/pslr_implicit_literal.y create mode 100644 spec/fixtures/integration/pslr_keyword_context.l create mode 100644 spec/fixtures/integration/pslr_keyword_context.y create mode 100644 spec/fixtures/integration/pslr_layout_comment.l create mode 100644 spec/fixtures/integration/pslr_layout_comment.y create mode 100644 spec/fixtures/integration/pslr_mixed_context.l create mode 100644 spec/fixtures/integration/pslr_mixed_context.y create mode 100644 spec/fixtures/integration/pslr_shift_chain.l create mode 100644 spec/fixtures/integration/pslr_shift_chain.y create mode 100644 spec/fixtures/integration/pslr_template_argument_lists.l create mode 100644 spec/fixtures/integration/pslr_template_argument_lists.y create mode 100644 spec/fixtures/integration/pslr_token_action.y diff --git a/lib/lrama/output.rb b/lib/lrama/output.rb index 24cf725c7..68b5d1139 100644 --- a/lib/lrama/output.rb +++ b/lib/lrama/output.rb @@ -401,6 +401,737 @@ def percent_code(name) end.join end + # PSLR Output Helper Methods + # Based on PSLR::OutputHelper - generates PSLR-specific C code + + # Check if the grammar requested PSLR output. + def pslr_enabled? + @grammar.pslr_defined? + end + + # Check if PSLR scanner tables are available. + def pslr_scanner_enabled? + scanner_fsa = @context.states.scanner_fsa + !scanner_fsa.nil? && !scanner_fsa.states.empty? + end + + def pslr_function_declarations + return "" unless pslr_enabled? + + declarations = [<<~C_CODE] + int yy_state_accepts_token (int yystate, int yychar); + C_CODE + + if pslr_scanner_enabled? + declarations << <<~C_CODE + #ifndef YYPSLR_SCAN_RESULT_DEFINED + # define YYPSLR_SCAN_RESULT_DEFINED + typedef struct yypslr_scan_result { + int token; + int length; + int is_layout; + int is_character_token; + } yypslr_scan_result; + #endif + + int yy_pseudo_scan_result (int parser_state, const char *input, yypslr_scan_result *result); + int yy_pseudo_scan (int parser_state, const char *input, int *match_length); + int yy_pslr_token_is_layout (int token); + C_CODE + + declarations << <<~C_CODE + #define YYPSLR_ENABLED 1 + #define YYPSLR_NO_MATCH YYEMPTY + + #ifndef YYPSLR_PSEUDO_SCAN_STATE + # define YYPSLR_PSEUDO_SCAN_STATE(ParserState, Input, MatchLength) \\ + yy_pseudo_scan ((ParserState), (Input), (MatchLength)) + #endif + + #ifndef YYPSLR_PSEUDO_SCAN_RESULT_STATE + # define YYPSLR_PSEUDO_SCAN_RESULT_STATE(ParserState, Input, Result) \\ + yy_pseudo_scan_result ((ParserState), (Input), (Result)) + #endif + + #ifndef YYPSLR_TOKEN_IS_LAYOUT + # define YYPSLR_TOKEN_IS_LAYOUT(Token) yy_pslr_token_is_layout ((Token)) + #endif + + int yypslr_scan_with_layout (int parser_state, const char **input, + yypslr_scan_result *result); + + #ifndef YYPSLR_SCAN_WITH_LAYOUT + # define YYPSLR_SCAN_WITH_LAYOUT(ParserState, InputPtr, Result) \\ + yypslr_scan_with_layout ((ParserState), (InputPtr), (Result)) + #endif + C_CODE + end + + if (member = pslr_state_member) + declarations << <<~C_CODE + #ifndef YYGETSTATE_CONTEXT + # define YYGETSTATE_CONTEXT(Context) ((Context)->#{member}) + #endif + C_CODE + + if pslr_scanner_enabled? + declarations << <<~C_CODE + #ifndef YYPSLR_PSEUDO_SCAN + # define YYPSLR_PSEUDO_SCAN(Context, Input, MatchLength) \\ + ((Context) != 0 \\ + ? YYPSLR_PSEUDO_SCAN_STATE (YYGETSTATE_CONTEXT (Context), (Input), (MatchLength)) \\ + : YYEMPTY) + #endif + + #ifndef YYPSLR_PSEUDO_SCAN_RESULT + # define YYPSLR_PSEUDO_SCAN_RESULT(Context, Input, Result) \\ + ((Context) != 0 \\ + ? YYPSLR_PSEUDO_SCAN_RESULT_STATE (YYGETSTATE_CONTEXT (Context), (Input), (Result)) \\ + : YYEMPTY) + #endif + C_CODE + end + + if !parse_param_name.empty? + declarations << <<~C_CODE + #ifndef YYSETSTATE_CONTEXT + # define YYSETSTATE_CONTEXT(CurrentState) \\ + do { \\ + if (#{parse_param_name} != 0) { \\ + YYGETSTATE_CONTEXT (#{parse_param_name}) = (CurrentState); \\ + } \\ + } while (0) + #endif + C_CODE + end + end + + declarations.join("\n") + end + + def pslr_state_member + member = @grammar.pslr_state_member + member&.strip + end + + def pslr_accepting_states + return [] unless pslr_scanner_enabled? + + @context.states.scanner_fsa.states.select(&:accepting?) + end + + def pslr_token_pattern_count + @context.states.token_patterns.size + end + + def pslr_token_id(token_pattern) + @context.states.find_symbol_by_s_value!(token_pattern.name).token_id + end + + # Generate Scanner FSA transition table as C code + def scanner_transition_table + return "" unless pslr_scanner_enabled? + scanner_fsa = @context.states.scanner_fsa + + lines = [] + lines << "/* Scanner FSA transition table */" + lines << "#define YY_SCANNER_NUM_STATES #{scanner_fsa.states.size}" + lines << "#define YY_SCANNER_INVALID_STATE (-1)" + lines << "" + lines << "static const int yy_scanner_transition[YY_SCANNER_NUM_STATES][256] = {" + + scanner_fsa.states.each_with_index do |state, idx| + transitions = Array.new(256, -1) + state.transitions.each do |char, target_id| + transitions[char.ord] = target_id + end + lines << " /* state #{idx} */ {#{transitions.join(', ')}}#{idx < scanner_fsa.states.size - 1 ? ',' : ''}" + end + + lines << "};" + lines.join("\n") + end + + # Generate state_to_accepting table as C code + def state_to_accepting_table + return "" unless pslr_scanner_enabled? + scanner_fsa = @context.states.scanner_fsa + accepting_indices = Array.new(scanner_fsa.states.size, -1) + + pslr_accepting_states.each_with_index do |state, index| + accepting_indices[state.id] = index + end + + lines = [] + lines << "" + lines << "/* FSA state -> accepting state index mapping */" + lines << "#define YY_ACCEPTING_NONE (-1)" + lines << "" + lines << "static const int yy_state_to_accepting[YY_SCANNER_NUM_STATES] = {" + lines << " #{accepting_indices.join(', ')}" + lines << "};" + lines.join("\n") + end + + def token_pattern_token_ids_table + return "" unless pslr_scanner_enabled? + + lines = [] + lines << "" + lines << "/* token pattern index -> parser token id */" + lines << "#define YY_PSLR_EMPTY_PATTERN (-1)" + lines << "#define YY_NUM_TOKEN_PATTERNS #{pslr_token_pattern_count}" + lines << "" + lines << "static const int yy_token_pattern_to_token_id[YY_NUM_TOKEN_PATTERNS] = {" + lines << " #{@context.states.token_patterns.map {|token_pattern| pslr_token_id(token_pattern) }.join(', ')}" + lines << "};" + lines << "" + lines << "static const int yy_token_pattern_is_layout[YY_NUM_TOKEN_PATTERNS] = {" + lines << " #{@context.states.token_patterns.map {|token_pattern| token_pattern.layout? ? 1 : 0 }.join(', ')}" + lines << "};" + lines << "" + lines << "int" + lines << "yy_pslr_token_is_layout (int token)" + lines << "{" + lines << " int i;" + lines << " for (i = 0; i < YY_NUM_TOKEN_PATTERNS; i++)" + lines << " if (yy_token_pattern_to_token_id[i] == token)" + lines << " return yy_token_pattern_is_layout[i];" + lines << " return 0;" + lines << "}" + lines.join("\n") + end + + # Generate token IDs for accepting states as C code + def accepting_tokens_table + return "" unless pslr_scanner_enabled? + scanner_fsa = @context.states.scanner_fsa + + lines = [] + lines << "" + lines << "/* Accepting state token IDs */" + lines << "" + + # Collect all unique tokens + all_tokens = @context.states.token_patterns.map(&:name) + lines << "/* Token pattern names: #{all_tokens.join(', ')} */" + lines << "" + + # Generate accepting tokens for each FSA state + scanner_fsa.states.each do |state| + next unless state.accepting? + + token_names = state.accepting_tokens.map(&:name) + lines << "/* State #{state.id} accepts: #{token_names.join(', ')} */" + end + + lines.join("\n") + end + + # Generate scanner_accepts table as C code + def scanner_accepts_table_code + return "" unless pslr_scanner_enabled? + scanner_fsa = @context.states.scanner_fsa + scanner_accepts = @context.states.scanner_accepts_table + return "" unless scanner_accepts + + lines = [] + lines << "" + lines << "/* scanner_accepts[parser_state][accepting_state] -> token pattern index */" + lines << "/* YY_PSLR_EMPTY_PATTERN means no token accepted */" + lines << "" + + num_parser_states = @context.states.states.size + num_accepting_states = pslr_accepting_states.size + + lines << "#define YY_NUM_PARSER_STATES #{num_parser_states}" + lines << "#define YY_NUM_ACCEPTING_STATES #{num_accepting_states}" + lines << "" + + if num_accepting_states > 0 + lines << "static const int yy_scanner_accepts[YY_NUM_PARSER_STATES][YY_NUM_ACCEPTING_STATES] = {" + token_pattern_indexes = @context.states.token_patterns.each_with_index.to_h + + @context.states.states.each_with_index do |parser_state, ps_idx| + row = [] + pslr_accepting_states.each do |fsa_state| + token = scanner_accepts[parser_state.id, fsa_state.id] + if token + row << token_pattern_indexes.fetch(token) + else + row << -1 + end + end + + lines << " /* parser state #{ps_idx} */ {#{row.join(', ')}}#{ps_idx < num_parser_states - 1 ? ',' : ''}" + end + + lines << "};" + lines << "" + lines << "static const int yy_scanner_fallback_accepts[YY_NUM_ACCEPTING_STATES] = {" + fallback_row = pslr_accepting_states.map do |fsa_state| + token = scanner_accepts.fallback_table[fsa_state.id] + token ? token_pattern_indexes.fetch(token) : -1 + end + lines << " #{fallback_row.join(', ')}" + lines << "};" + end + + lines.join("\n") + end + + # Generate length_precedences table as C code + def length_precedences_table_code + return "" unless pslr_scanner_enabled? + length_precedences = @context.states.length_precedences + return "" unless length_precedences + + lines = [] + lines << "" + lines << "/* yy_pslr_length_precedes[old_token][new_token] is true when a longer" + lines << " match for new_token replaces an earlier match for old_token. */" + lines << "" + + num_tokens = pslr_token_pattern_count + if num_tokens > 0 + lines << length_precedence_matrix_code( + "yy_pslr_length_precedes", + num_tokens, + :normal_precedes? + ) + + lines << "" + lines << "/* yy_pslr_fallback_length_precedes keeps explicit PSLR length" + lines << " precedence and otherwise uses traditional longest-match fallback. */" + lines << "" + + lines << length_precedence_matrix_code( + "yy_pslr_fallback_length_precedes", + num_tokens, + :fallback_precedes? + ) + end + + lines.join("\n") + end + + def length_precedence_matrix_code(table_name, num_tokens, query_method) + length_precedences = @context.states.length_precedences + lines = [] + + lines << "static const int #{table_name}[#{num_tokens}][#{num_tokens}] = {" + @context.states.token_patterns.each_with_index do |t1, i| + row = @context.states.token_patterns.map do |t2| + length_precedences.public_send(query_method, t1.name, t2.name) ? 1 : 0 + end + lines << " /* #{t1.name} */ {#{row.join(', ')}}#{i < num_tokens - 1 ? ',' : ''}" + end + lines << "};" + + lines.join("\n") + end + + def pslr_lac_function + return "" unless pslr_enabled? + + <<~C_CODE + + static int + yy_lac_check_ (yy_state_t *yyss, yy_state_t *yyssp, yysymbol_kind_t yytoken) + { + YYPTRDIFF_T yylac_len = yyssp - yyss + 1; + yy_state_t *yylac_base = YY_CAST (yy_state_t *, + YYMALLOC (YY_CAST (YYSIZE_T, YYMAXDEPTH * YYSIZEOF (yy_state_t)))); + yy_state_t *yylac_top; + + if (!yylac_base) + return 1; + + if (YYMAXDEPTH < yylac_len) + { + YYFREE (yylac_base); + return 1; + } + + YYCOPY (yylac_base, yyss, yylac_len); + yylac_top = yylac_base + yylac_len - 1; + + for (;;) + { + int yystate = *yylac_top; + int yyn = yypact[yystate]; + + if (!yypact_value_is_default (yyn)) + { + yyn += yytoken; + if (0 <= yyn && yyn <= YYLAST && yycheck[yyn] == yytoken) + { + yyn = yytable[yyn]; + if (0 < yyn) + { + YYFREE (yylac_base); + return 1; + } + if (yytable_value_is_error (yyn)) + { + YYFREE (yylac_base); + return 0; + } + yyn = -yyn; + } + else + { + yyn = yydefact[yystate]; + if (yyn == 0) + { + YYFREE (yylac_base); + return 0; + } + } + } + else + { + yyn = yydefact[yystate]; + if (yyn == 0) + { + YYFREE (yylac_base); + return 0; + } + } + + yylac_top -= yyr2[yyn]; + { + const int yylhs = yyr1[yyn] - YYNTOKENS; + const int yyi = yypgoto[yylhs] + *yylac_top; + yystate = (0 <= yyi && yyi <= YYLAST && yycheck[yyi] == *yylac_top + ? yytable[yyi] + : yydefgoto[yylhs]); + } + + if (yylac_base + YYMAXDEPTH - 1 <= yylac_top) + { + YYFREE (yylac_base); + return 1; + } + + *++yylac_top = YY_CAST (yy_state_t, yystate); + } + } + C_CODE + end + + # Generate pseudo_scan function as C code + def pseudo_scan_function + return "" unless pslr_scanner_enabled? + + <<~C_CODE + + /* + * pseudo_scan: PSLR(1) scanning function + * Based on Definition 3.2.16 from the PSLR dissertation + * + * Input: + * parser_state: Current parser state + * input: Input buffer pointer + * match_length: Output parameter for matched length + * + * Returns: Selected parser token ID. If neither the parser-state row + * nor fallback row matches, returns YYUNDEF and consumes one byte. + */ + int + yy_pseudo_scan_result (int parser_state, const char *input, yypslr_scan_result *result) + { + int ss = 0; /* FSA initial state */ + int ibest = 0; + int pbest = YY_PSLR_EMPTY_PATTERN; + int fallback_ibest = 0; + int fallback_pbest = YY_PSLR_EMPTY_PATTERN; + int i = 0; + + if (result == NULL) { + return YYEMPTY; + } + + result->token = YYEMPTY; + result->length = 0; + result->is_layout = 0; + result->is_character_token = 0; + + if (parser_state < 0 || parser_state >= YY_NUM_PARSER_STATES || input == NULL) { + return YYEMPTY; + } + + if (input[0] == '\\0') { + result->token = YYEOF; + return result->token; + } + + while (input[i] != '\\0') { + int c = (unsigned char)input[i]; + int next_ss = yy_scanner_transition[ss][c]; + + if (next_ss == YY_SCANNER_INVALID_STATE) { + break; + } + + ss = next_ss; + i++; + + /* Check if this is an accepting state */ + int sa = yy_state_to_accepting[ss]; + if (sa != YY_ACCEPTING_NONE) { + int pattern_index = yy_scanner_accepts[parser_state][sa]; + if (pattern_index != YY_PSLR_EMPTY_PATTERN) { + if (pbest == YY_PSLR_EMPTY_PATTERN || + yy_pslr_length_precedes[pbest][pattern_index]) { + pbest = pattern_index; + ibest = i; + } + } + + pattern_index = yy_scanner_fallback_accepts[sa]; + if (pattern_index != YY_PSLR_EMPTY_PATTERN) { + if (fallback_pbest == YY_PSLR_EMPTY_PATTERN || + yy_pslr_fallback_length_precedes[fallback_pbest][pattern_index]) { + fallback_pbest = pattern_index; + fallback_ibest = i; + } + } + } + } + + if (pbest != YY_PSLR_EMPTY_PATTERN) { + result->token = yy_token_pattern_to_token_id[pbest]; + result->length = ibest; + result->is_layout = yy_token_pattern_is_layout[pbest]; + return result->token; + } + + if (fallback_pbest != YY_PSLR_EMPTY_PATTERN) { + result->token = yy_token_pattern_to_token_id[fallback_pbest]; + result->length = fallback_ibest; + result->is_layout = yy_token_pattern_is_layout[fallback_pbest]; + return result->token; + } + + result->token = YYUNDEF; + result->length = 1; + result->is_character_token = 1; + return result->token; + } + + int + yy_pseudo_scan(int parser_state, const char *input, int *match_length) + { + yypslr_scan_result result; + int token = yy_pseudo_scan_result (parser_state, input, &result); + + if (match_length != NULL) { + *match_length = result.length; + } + + return token; + } + C_CODE + end + + # Check if lexer context table is available. + def lexer_context_enabled? + @context.states.lexer_context_enabled? + end + + # Generate #define constants for lexer contexts, emitted early in the output + # so that user code in %{ ... %} can reference them. + def lexer_context_defines_code + return "" unless lexer_context_enabled? + + classifier = @context.states.lexer_context_classifier + lines = [] + lines << "/* Lexer context constants — generated from %lexer-context directives */" + classifier.contexts.each do |lc| + lines << "#define YY_CTX_%-8s 0x%02x" % [lc.name, lc.bitmask] + end + lines.join("\n") + end + + # Generate the lexer context table as C code. + def lexer_context_table_code + return "" unless lexer_context_enabled? + + table = @context.states.lexer_context_table + lexer_contexts = @grammar.lexer_contexts + lines = [] + + lines << "/* Lexer Context Classification Table */" + lines << "/* Maps parser state -> lexer context flags */" + lines << "" + lines << "static const unsigned char yy_lexer_context[] = {" + + table.each_with_index do |ctx, idx| + ctx_name = LexerContextClassifier.context_name(ctx, lexer_contexts) + comma = idx < table.size - 1 ? "," : "" + lines << " /* state #{idx} */ #{ctx}#{comma} /* #{ctx_name} */" + end + + lines << "};" + lines << "" + lines << "int" + lines << "yy_lexer_context_is(int yystate, int ctx_mask) {" + lines << " if (yystate < 0 || yystate >= #{table.size}) return 0;" + lines << " return yy_lexer_context[yystate] & ctx_mask;" + lines << "}" + + lines.join("\n") + end + + # Check if token actions are defined + def pslr_token_actions_enabled? + pslr_enabled? && !@grammar.token_actions.empty? + end + + # Generate layout accumulation declarations + def pslr_layout_declarations + return "" unless pslr_scanner_enabled? + + <<~C_CODE + + #ifndef YYPSLR_LAYOUT_DEFINED + # define YYPSLR_LAYOUT_DEFINED + + typedef struct yypslr_layout { + const char *text; + int length; + } yypslr_layout; + + # ifndef YYPSLR_LAYOUT_BUFFER_SIZE + # define YYPSLR_LAYOUT_BUFFER_SIZE 4096 + # endif + + static char yypslr_layout_buffer[YYPSLR_LAYOUT_BUFFER_SIZE]; + static int yypslr_layout_length = 0; + + static void + yypslr_layout_clear (void) + { + yypslr_layout_length = 0; + } + + static void + yypslr_layout_append (const char *text, int length) + { + if (yypslr_layout_length + length < YYPSLR_LAYOUT_BUFFER_SIZE) + { + memcpy (yypslr_layout_buffer + yypslr_layout_length, text, length); + yypslr_layout_length += length; + yypslr_layout_buffer[yypslr_layout_length] = '\\0'; + } + } + + # define YYPSLR_LAYOUT_TEXT yypslr_layout_buffer + # define YYPSLR_LAYOUT_LENGTH yypslr_layout_length + # define YYPSLR_LAYOUT_CLEAR() yypslr_layout_clear () + # define YYPSLR_LAYOUT_APPEND(Text, Len) yypslr_layout_append ((Text), (Len)) + + #endif /* YYPSLR_LAYOUT_DEFINED */ + C_CODE + end + + # Generate token action dispatch function + def pslr_token_action_function + return "" unless pslr_token_actions_enabled? + + lines = [] + lines << "" + lines << "/* PSLR token action dispatch */" + lines << "/* Generated from %token-action declarations */" + lines << "" + lines << "static void" + lines << "yypslr_token_action (int token, const char *yytext, int yyleng)" + lines << "{" + + @grammar.token_actions.each_with_index do |action, idx| + token_id = @context.states.find_symbol_by_s_value!(action.token_name).token_id + keyword = idx == 0 ? "if" : "else if" + lines << " #{keyword} (token == #{token_id})" + lines << " {" + lines << "#line #{action.lineno} \"#{@grammar_file_path}\"" + lines << " {#{action.code.s_value}}" + lines << "#line [@oline@] [@ofile@]" + lines << " }" + end + + lines << "}" + lines << "" + lines << "#define YYPSLR_TOKEN_ACTION(Token, Text, Len) \\" + lines << " yypslr_token_action ((Token), (Text), (Len))" + lines.join("\n") + end + + # Generate enhanced pseudo_scan_result with layout accumulation + def pslr_layout_scan_function + return "" unless pslr_scanner_enabled? + + <<~C_CODE + + /* + * yypslr_scan_with_layout: scan with layout accumulation. + * + * Repeatedly invokes yy_pseudo_scan_result to consume input. + * Layout tokens are accumulated in the layout buffer; + * non-layout tokens are returned to the caller with the + * accumulated layout available via YYPSLR_LAYOUT_TEXT. + */ + int + yypslr_scan_with_layout (int parser_state, const char **input, + yypslr_scan_result *result) + { + YYPSLR_LAYOUT_CLEAR (); + + for (;;) + { + int token = yy_pseudo_scan_result (parser_state, *input, result); + + if (token == YYEMPTY || token == YYEOF) + return token; + + if (result->is_layout) + { + YYPSLR_LAYOUT_APPEND (*input, result->length); + #{pslr_token_actions_enabled? ? " YYPSLR_TOKEN_ACTION (token, *input, result->length);" : ""} + *input += result->length; + continue; + } + + #{pslr_token_actions_enabled? ? " YYPSLR_TOKEN_ACTION (token, *input, result->length);" : ""} + return token; + } + } + C_CODE + end + + # Generate all PSLR C code + def pslr_tables_and_functions + return "" unless pslr_scanner_enabled? + + parts = [ + "/* PSLR(1) Scanner Tables and Functions */", + "/* Generated by Lrama PSLR implementation */", + "", + scanner_transition_table, + state_to_accepting_table, + token_pattern_token_ids_table, + accepting_tokens_table, + scanner_accepts_table_code, + length_precedences_table_code, + pslr_layout_declarations, + pseudo_scan_function, + pslr_layout_scan_function, + pslr_token_action_function, + ] + + parts.join("\n") + end + private def eval_template(file, path) diff --git a/spec/fixtures/integration/pslr_context.l b/spec/fixtures/integration/pslr_context.l new file mode 100644 index 000000000..06cbd121d --- /dev/null +++ b/spec/fixtures/integration/pslr_context.l @@ -0,0 +1,50 @@ +%option noinput nounput noyywrap never-interactive + +%{ +#include +#include "pslr_context.h" + +#define YY_DECL int yylex(YYSTYPE *yylval, struct parse_params *p) +%} + +ID_PATTERN [a-zA-Z_][a-zA-Z0-9_]* + +%% + +{ID_PATTERN} { + (void)yylval; + return ID; +} + +"<" { + return LANGLE; +} + +[>]{1,2} { + int match_length = 0; + int token = YYPSLR_PSEUDO_SCAN(p, yytext, &match_length); + + if (token == YYEMPTY) { + token = (yyleng == 2) ? RSHIFT : RANGLE; + match_length = yyleng; + } + + if (0 < match_length && match_length < yyleng) { + yyless(match_length); + } + + return token; +} + +[[:space:]]+ {} + +<> { + return YYEOF; +} + +. { + fprintf(stderr, "Illegal character '%s'\n", yytext); + return YYUNDEF; +} + +%% diff --git a/spec/fixtures/integration/pslr_context.y b/spec/fixtures/integration/pslr_context.y new file mode 100644 index 000000000..74de59abd --- /dev/null +++ b/spec/fixtures/integration/pslr_context.y @@ -0,0 +1,75 @@ +%{ +#include + +#define YY_DECL int yylex(YYSTYPE *lval, struct parse_params *p) + +#include "pslr_context.h" +#include "pslr_context-lexer.h" + +extern int yylex(YYSTYPE *lval, struct parse_params *p); +static int yyerror(YYLTYPE *loc, struct parse_params *p, const char *str); +%} + +%code requires { + struct parse_params { + int current_state; + }; +} + +%define api.pure +%define lr.type pslr +%define api.pslr.state-member current_state + +%lex-param {struct parse_params *p} +%parse-param {struct parse_params *p} + +%token-pattern RSHIFT />>/ +%token-pattern RANGLE />/ +%token-pattern LANGLE / +#include "pslr_fallback_precedence.h" + +#define YY_DECL int yylex(YYSTYPE *yylval, struct parse_params *p) + +static const char * +token_name(int token) +{ + switch (token) { + case X: + return "X"; + case COM: + return "COM"; + case A: + return "A"; + case B: + return "B"; + case SHORT: + return "SHORT"; + case IA: + return "IA"; + case IB: + return "IB"; + case CYC_A: + return "CYC_A"; + case CYC_B: + return "CYC_B"; + case CYC_C: + return "CYC_C"; + case ZA: + return "ZA"; + case ZB: + return "ZB"; + case PA: + return "PA"; + case PB: + return "PB"; + case NON: + return "NON"; + case WORD: + return "WORD"; + default: + return "UNKNOWN"; + } +} +%} + +%% + +(.|\n)+ { + yypslr_scan_result result; + int token; + + (void)yylval; + token = YYPSLR_PSEUDO_SCAN_RESULT(p, yytext, &result); + if (result.length <= 0) { + result.length = 1; + } + printf("%s %d\n", token_name(token), result.length); + if (result.length < yyleng) { + yyless(result.length); + } + return token; +} + +<> { + return YYEOF; +} + +%% diff --git a/spec/fixtures/integration/pslr_fallback_precedence.y b/spec/fixtures/integration/pslr_fallback_precedence.y new file mode 100644 index 000000000..86f708a5e --- /dev/null +++ b/spec/fixtures/integration/pslr_fallback_precedence.y @@ -0,0 +1,93 @@ +%{ +#include +#include + +#define YY_DECL int yylex(YYSTYPE *lval, struct parse_params *p) + +#include "pslr_fallback_precedence.h" +#include "pslr_fallback_precedence-lexer.h" + +extern int yylex(YYSTYPE *lval, struct parse_params *p); +static int yyerror(YYLTYPE *loc, struct parse_params *p, const char *str); +%} + +%code requires { + struct parse_params { + int current_state; + }; +} + +%define api.pure +%define lr.type pslr +%define api.pslr.state-member current_state + +%lex-param {struct parse_params *p} +%parse-param {struct parse_params *p} + +%token-pattern X /x/ +%token-pattern COM /\/\*(.|\n)*\*\// +%token-pattern A /a/ +%token-pattern B /ab/ +%token-pattern SHORT /c/ +%token-pattern IA /cd/ +%token-pattern IB /cd/ +%token-pattern CYC_A /~/ +%token-pattern CYC_B /~/ +%token-pattern CYC_C /~/ +%token-pattern ZA /z/ +%token-pattern ZB /z/ +%token-pattern PA /@/ +%token-pattern PB /@/ +%token-pattern NON /non-/ +%token-pattern WORD /[a-z-]+/ + +%lex-prec COM -s COM +%lex-prec WORD -< NON +%lex-prec PA <- PB +%lex-prec IA <- IB +%lex-prec CYC_A <- CYC_B +%lex-prec CYC_B <- CYC_C +%lex-prec CYC_C <- CYC_A + +%% + +start + : X { printf("ok\n"); } + ; + +%% + +static int +yyerror(YYLTYPE *loc, struct parse_params *p, const char *str) +{ + (void)loc; + (void)p; + (void)str; + return 0; +} + +int +main(int argc, char *argv[]) +{ + struct parse_params params = { 0 }; + + if (argc == 2 && strcmp(argv[1], "__empty__") == 0) { + yypslr_scan_result result; + int match_length = -1; + int token = YYPSLR_PSEUDO_SCAN_RESULT(¶ms, "", &result); + int wrapper_token = YYPSLR_PSEUDO_SCAN(¶ms, "", &match_length); + int ok = token == YYEOF && result.token == YYEOF && wrapper_token == YYEOF && + result.length == 0 && result.is_layout == 0 && result.is_character_token == 0 && + match_length == 0; + + printf("%s %d %d %d %d\n", ok ? "EOF" : "BAD", result.length, result.is_layout, + result.is_character_token, match_length); + return ok ? 0 : 1; + } + + if (argc == 2) { + yy_scan_string(argv[1]); + } + + return yyparse(¶ms); +} diff --git a/spec/fixtures/integration/pslr_implicit_literal.l b/spec/fixtures/integration/pslr_implicit_literal.l new file mode 100644 index 000000000..31121d203 --- /dev/null +++ b/spec/fixtures/integration/pslr_implicit_literal.l @@ -0,0 +1,30 @@ +%option noinput nounput noyywrap never-interactive + +%{ +#include "pslr_implicit_literal.h" + +#define YY_DECL int yylex(YYSTYPE *yylval, struct parse_params *p) +%} + +%% + +(.|\n)+ { + yypslr_scan_result result; + int token; + + (void)yylval; + token = YYPSLR_PSEUDO_SCAN_RESULT(p, yytext, &result); + if (result.length <= 0) { + result.length = 1; + } + if (result.length < yyleng) { + yyless(result.length); + } + return token; +} + +<> { + return YYEOF; +} + +%% diff --git a/spec/fixtures/integration/pslr_implicit_literal.y b/spec/fixtures/integration/pslr_implicit_literal.y new file mode 100644 index 000000000..1f9526429 --- /dev/null +++ b/spec/fixtures/integration/pslr_implicit_literal.y @@ -0,0 +1,66 @@ +%{ +#include +#include + +#define YY_DECL int yylex(YYSTYPE *lval, struct parse_params *p) + +#include "pslr_implicit_literal.h" +#include "pslr_implicit_literal-lexer.h" + +extern int yylex(YYSTYPE *lval, struct parse_params *p); +static int yyerror(YYLTYPE *loc, struct parse_params *p, const char *str); +%} + +%code requires { + struct parse_params { + int current_state; + }; +} + +%define api.pure +%define lr.type pslr +%define api.pslr.state-member current_state + +%lex-param {struct parse_params *p} +%parse-param {struct parse_params *p} + +%token-pattern ID /[a-z]+/ + +%% + +start + : ID ';' { printf("ok\n"); } + ; + +%% + +static int +yyerror(YYLTYPE *loc, struct parse_params *p, const char *str) +{ + (void)loc; + (void)p; + (void)str; + return 0; +} + +int +main(int argc, char *argv[]) +{ + struct parse_params params = { 0 }; + + if (argc == 2 && strcmp(argv[1], "__fallback_semi__") == 0) { + yypslr_scan_result result; + int token = YYPSLR_PSEUDO_SCAN_RESULT(¶ms, ";", &result); + int ok = token == ';' && result.token == ';' && result.length == 1 && + result.is_character_token == 0; + + printf("%s %d %d\n", ok ? "SEMI" : "BAD", result.length, result.is_character_token); + return ok ? 0 : 1; + } + + if (argc == 2) { + yy_scan_string(argv[1]); + } + + return yyparse(¶ms); +} diff --git a/spec/fixtures/integration/pslr_keyword_context.l b/spec/fixtures/integration/pslr_keyword_context.l new file mode 100644 index 000000000..a54322064 --- /dev/null +++ b/spec/fixtures/integration/pslr_keyword_context.l @@ -0,0 +1,41 @@ +%option noinput nounput noyywrap never-interactive + +%{ +#include +#include "pslr_keyword_context.h" + +#define YY_DECL int yylex(YYSTYPE *yylval, struct parse_params *p) +%} + +%% + +[a-z]+ { + (void)yylval; + + int match_length = 0; + int token = YYPSLR_PSEUDO_SCAN(p, yytext, &match_length); + + if (token == YYEMPTY) { + token = ID; + match_length = yyleng; + } + + if (0 < match_length && match_length < yyleng) { + yyless(match_length); + } + + return token; +} + +[[:space:]]+ {} + +<> { + return YYEOF; +} + +. { + fprintf(stderr, "Illegal character '%s'\n", yytext); + return YYUNDEF; +} + +%% diff --git a/spec/fixtures/integration/pslr_keyword_context.y b/spec/fixtures/integration/pslr_keyword_context.y new file mode 100644 index 000000000..6f5f6b509 --- /dev/null +++ b/spec/fixtures/integration/pslr_keyword_context.y @@ -0,0 +1,87 @@ +%{ +#include + +#define YY_DECL int yylex(YYSTYPE *lval, struct parse_params *p) + +#include "pslr_keyword_context.h" +#include "pslr_keyword_context-lexer.h" + +extern int yylex(YYSTYPE *lval, struct parse_params *p); +static int yyerror(YYLTYPE *loc, struct parse_params *p, const char *str); +%} + +%code requires { + struct parse_params { + int current_state; + }; +} + +%define api.pure +%define lr.type pslr +%define api.pslr.state-member current_state + +%lex-param {struct parse_params *p} +%parse-param {struct parse_params *p} + +%token-pattern P /p/ +%token-pattern Q /q/ +%token-pattern X /x/ +%token-pattern IF /if/ +%token-pattern ID /[a-z]+/ + +%lex-prec ID <~ IF + +%% + +program + : kw_context { printf("kw\n"); } + | id_context { printf("id\n"); } + ; + +kw_context + : P shared IF + ; + +id_context + : Q shared ID + ; + +shared + : n1 + ; + +n1 + : n2 + ; + +n2 + : X + ; + +%% + +static int +yyerror(YYLTYPE *loc, struct parse_params *p, const char *str) +{ + (void)loc; + (void)p; + fprintf(stderr, "parse error: %s\n", str); + return 0; +} + +int +main(int argc, char *argv[]) +{ + struct parse_params params = { 0 }; + + if (argc == 2) { + yy_scan_string(argv[1]); + } + + if (yyparse(¶ms)) { + fprintf(stderr, "syntax error\n"); + return 1; + } + + return 0; +} diff --git a/spec/fixtures/integration/pslr_layout_comment.l b/spec/fixtures/integration/pslr_layout_comment.l new file mode 100644 index 000000000..82fc8fa77 --- /dev/null +++ b/spec/fixtures/integration/pslr_layout_comment.l @@ -0,0 +1,36 @@ +%option noinput nounput noyywrap never-interactive + +%{ +#include +#include "pslr_layout_comment.h" + +#define YY_DECL int yylex(YYSTYPE *yylval, struct parse_params *p) +%} + +%% + +(.|\n)+ { + yypslr_scan_result result; + int token; + + (void)yylval; + token = YYPSLR_PSEUDO_SCAN_RESULT(p, yytext, &result); + if (result.length <= 0) { + result.length = 1; + } + if (result.length < yyleng) { + yyless(result.length); + } + /* Layout tokens are scanner-only. Consume them and keep the same parser + state while scanning the remaining input. */ + if (result.is_layout) { + return yylex(yylval, p); + } + return token; +} + +<> { + return YYEOF; +} + +%% diff --git a/spec/fixtures/integration/pslr_layout_comment.y b/spec/fixtures/integration/pslr_layout_comment.y new file mode 100644 index 000000000..185fb927a --- /dev/null +++ b/spec/fixtures/integration/pslr_layout_comment.y @@ -0,0 +1,70 @@ +%{ +#include + +#define YY_DECL int yylex(YYSTYPE *lval, struct parse_params *p) + +#include "pslr_layout_comment.h" +#include "pslr_layout_comment-lexer.h" + +extern int yylex(YYSTYPE *lval, struct parse_params *p); +static int yyerror(YYLTYPE *loc, struct parse_params *p, const char *str); +%} + +%code requires { + struct parse_params { + int current_state; + }; +} + +%define api.pure +%define lr.type pslr +%define api.pslr.state-member current_state + +%lex-param {struct parse_params *p} +%parse-param {struct parse_params *p} + +%token ID +%token DIV "/" +%token SEMI ";" + +%token-pattern ID /[a-zA-Z][a-zA-Z0-9_]*/ +%token-pattern DIV /\// +%token-pattern SEMI /;/ +%token-pattern YYLAYOUT_COMMENT /\/\*([^*]|\*+[^*/])*\*+\// +%token-pattern YYLAYOUT_WS /[ \t\r\n]+/ + +%lex-prec DIV -~ YYLAYOUT_COMMENT + +%% + +start + : ID DIV ID SEMI { printf("ok\n"); } + ; + +%% + +static int +yyerror(YYLTYPE *loc, struct parse_params *p, const char *str) +{ + (void)loc; + (void)p; + fprintf(stderr, "parse error: %s\n", str); + return 0; +} + +int +main(int argc, char *argv[]) +{ + struct parse_params params = { 0 }; + + if (argc == 2) { + yy_scan_string(argv[1]); + } + + if (yyparse(¶ms)) { + fprintf(stderr, "syntax error\n"); + return 1; + } + + return 0; +} diff --git a/spec/fixtures/integration/pslr_mixed_context.l b/spec/fixtures/integration/pslr_mixed_context.l new file mode 100644 index 000000000..5c75c7a31 --- /dev/null +++ b/spec/fixtures/integration/pslr_mixed_context.l @@ -0,0 +1,69 @@ +%option noinput nounput noyywrap never-interactive + +%{ +#include +#include "pslr_mixed_context.h" + +#define YY_DECL int yylex(YYSTYPE *yylval, struct parse_params *p) +%} + +%% + +[a-z]+ { + (void)yylval; + + int match_length = 0; + int token = YYPSLR_PSEUDO_SCAN(p, yytext, &match_length); + + if (token == YYEMPTY) { + token = ID; + match_length = yyleng; + } + + if (0 < match_length && match_length < yyleng) { + yyless(match_length); + } + + return token; +} + +"<" { + return LT; +} + +"@" { + return START; +} + +"#" { + return MARK; +} + +[>]{1,2} { + int match_length = 0; + int token = YYPSLR_PSEUDO_SCAN(p, yytext, &match_length); + + if (token == YYEMPTY) { + token = (yyleng == 2) ? RSHIFT : RANGLE; + match_length = yyleng; + } + + if (0 < match_length && match_length < yyleng) { + yyless(match_length); + } + + return token; +} + +[[:space:]]+ {} + +<> { + return YYEOF; +} + +. { + fprintf(stderr, "Illegal character '%s'\n", yytext); + return YYUNDEF; +} + +%% diff --git a/spec/fixtures/integration/pslr_mixed_context.y b/spec/fixtures/integration/pslr_mixed_context.y new file mode 100644 index 000000000..38eba8c90 --- /dev/null +++ b/spec/fixtures/integration/pslr_mixed_context.y @@ -0,0 +1,102 @@ +%{ +#include + +#define YY_DECL int yylex(YYSTYPE *lval, struct parse_params *p) + +#include "pslr_mixed_context.h" +#include "pslr_mixed_context-lexer.h" + +extern int yylex(YYSTYPE *lval, struct parse_params *p); +static int yyerror(YYLTYPE *loc, struct parse_params *p, const char *str); +%} + +%code requires { + struct parse_params { + int current_state; + }; +} + +%define api.pure +%define lr.type pslr +%define api.pslr.state-member current_state + +%lex-param {struct parse_params *p} +%parse-param {struct parse_params *p} + +%token-pattern LT />/ +%token-pattern RANGLE />/ + +%lex-prec ID <~ IF +%lex-no-tie RANGLE RSHIFT + +%% + +program + : kw_context { printf("kw\n"); } + | id_context { printf("id\n"); } + | template_expr { printf("template\n"); } + | shift_expr { printf("shift\n"); } + ; + +kw_context + : P shared IF + ; + +id_context + : Q shared ID + ; + +template_expr + : LT shared RANGLE + ; + +shift_expr + : START shared RSHIFT ID + ; + +shared + : n1 + ; + +n1 + : n2 + ; + +n2 + : MARK + ; + +%% + +static int +yyerror(YYLTYPE *loc, struct parse_params *p, const char *str) +{ + (void)loc; + (void)p; + fprintf(stderr, "parse error: %s\n", str); + return 0; +} + +int +main(int argc, char *argv[]) +{ + struct parse_params params = { 0 }; + + if (argc == 2) { + yy_scan_string(argv[1]); + } + + if (yyparse(¶ms)) { + fprintf(stderr, "syntax error\n"); + return 1; + } + + return 0; +} diff --git a/spec/fixtures/integration/pslr_shift_chain.l b/spec/fixtures/integration/pslr_shift_chain.l new file mode 100644 index 000000000..885ba62e8 --- /dev/null +++ b/spec/fixtures/integration/pslr_shift_chain.l @@ -0,0 +1,58 @@ +%option noinput nounput noyywrap never-interactive + +%{ +#include +#include "pslr_shift_chain.h" + +#define YY_DECL int yylex(YYSTYPE *yylval, struct parse_params *p) +%} + +ID_PATTERN [a-z]+ + +%% + +{ID_PATTERN} { + (void)yylval; + return ID; +} + +"<" { + return LT; +} + +"@" { + return START; +} + +"#" { + return MARK; +} + +[>]{1,2} { + int match_length = 0; + int token = YYPSLR_PSEUDO_SCAN(p, yytext, &match_length); + + if (token == YYEMPTY) { + token = (yyleng == 2) ? RSHIFT : RANGLE; + match_length = yyleng; + } + + if (0 < match_length && match_length < yyleng) { + yyless(match_length); + } + + return token; +} + +[[:space:]]+ {} + +<> { + return YYEOF; +} + +. { + fprintf(stderr, "Illegal character '%s'\n", yytext); + return YYUNDEF; +} + +%% diff --git a/spec/fixtures/integration/pslr_shift_chain.y b/spec/fixtures/integration/pslr_shift_chain.y new file mode 100644 index 000000000..d7b63d759 --- /dev/null +++ b/spec/fixtures/integration/pslr_shift_chain.y @@ -0,0 +1,88 @@ +%{ +#include + +#define YY_DECL int yylex(YYSTYPE *lval, struct parse_params *p) + +#include "pslr_shift_chain.h" +#include "pslr_shift_chain-lexer.h" + +extern int yylex(YYSTYPE *lval, struct parse_params *p); +static int yyerror(YYLTYPE *loc, struct parse_params *p, const char *str); +%} + +%code requires { + struct parse_params { + int current_state; + }; +} + +%define api.pure +%define lr.type pslr +%define api.pslr.state-member current_state + +%lex-param {struct parse_params *p} +%parse-param {struct parse_params *p} + +%token-pattern LT />/ +%token-pattern RANGLE />/ +%token-pattern ID /[a-z]+/ + +%lex-no-tie RANGLE RSHIFT + +%% + +program + : template_expr { printf("template\n"); } + | shift_expr { printf("shift\n"); } + ; + +template_expr + : LT shared RANGLE + ; + +shift_expr + : START shared RSHIFT ID + ; + +shared + : n1 + ; + +n1 + : n2 + ; + +n2 + : MARK + ; + +%% + +static int +yyerror(YYLTYPE *loc, struct parse_params *p, const char *str) +{ + (void)loc; + (void)p; + fprintf(stderr, "parse error: %s\n", str); + return 0; +} + +int +main(int argc, char *argv[]) +{ + struct parse_params params = { 0 }; + + if (argc == 2) { + yy_scan_string(argv[1]); + } + + if (yyparse(¶ms)) { + fprintf(stderr, "syntax error\n"); + return 1; + } + + return 0; +} diff --git a/spec/fixtures/integration/pslr_template_argument_lists.l b/spec/fixtures/integration/pslr_template_argument_lists.l new file mode 100644 index 000000000..aedab7edf --- /dev/null +++ b/spec/fixtures/integration/pslr_template_argument_lists.l @@ -0,0 +1,36 @@ +%option noinput nounput noyywrap never-interactive + +%{ +#include +#include "pslr_template_argument_lists.h" + +#define YY_DECL int yylex(YYSTYPE *yylval, struct parse_params *p) +%} + +%% + +(.|\n)+ { + yypslr_scan_result result; + int token; + + (void)yylval; + token = YYPSLR_PSEUDO_SCAN_RESULT(p, yytext, &result); + if (result.length <= 0) { + result.length = 1; + } + if (result.length < yyleng) { + yyless(result.length); + } + /* Layout tokens are scanner-only. Consume them and keep the same parser + state while scanning the remaining input. */ + if (result.is_layout) { + return yylex(yylval, p); + } + return token; +} + +<> { + return YYEOF; +} + +%% diff --git a/spec/fixtures/integration/pslr_template_argument_lists.y b/spec/fixtures/integration/pslr_template_argument_lists.y new file mode 100644 index 000000000..baf265b57 --- /dev/null +++ b/spec/fixtures/integration/pslr_template_argument_lists.y @@ -0,0 +1,91 @@ +%{ +#include + +#define YY_DECL int yylex(YYSTYPE *lval, struct parse_params *p) + +#include "pslr_template_argument_lists.h" +#include "pslr_template_argument_lists-lexer.h" + +extern int yylex(YYSTYPE *lval, struct parse_params *p); +static int yyerror(YYLTYPE *loc, struct parse_params *p, const char *str); +%} + +%code requires { + struct parse_params { + int current_state; + }; +} + +%define api.pure +%define lr.type pslr +%define api.pslr.state-member current_state + +%lex-param {struct parse_params *p} +%parse-param {struct parse_params *p} + +%token ID +%token LT "<" +%token RANGLE ">" +%token RSHIFT ">>" +%token SEMI ";" + +%token-pattern ID /[a-zA-Z][a-zA-Z0-9_]*/ +%token-pattern LT // +%token-pattern RSHIFT />>/ +%token-pattern SEMI /;/ +%token-pattern YYLAYOUT /[ \t\r\n]+/ + +%lex-no-tie RANGLE RSHIFT + +%% + +start + : decl SEMI { printf("decl\n"); } + | expr SEMI { printf("expr\n"); } + ; + +decl + : type id + ; + +type + : id LT type RANGLE + | ID + ; + +expr + : id RSHIFT id + ; + +id + : ID + ; + +%% + +static int +yyerror(YYLTYPE *loc, struct parse_params *p, const char *str) +{ + (void)loc; + (void)p; + fprintf(stderr, "parse error: %s\n", str); + return 0; +} + +int +main(int argc, char *argv[]) +{ + struct parse_params params = { 0 }; + + if (argc == 2) { + yy_scan_string(argv[1]); + } + + if (yyparse(¶ms)) { + fprintf(stderr, "syntax error\n"); + return 1; + } + + return 0; +} diff --git a/spec/fixtures/integration/pslr_token_action.y b/spec/fixtures/integration/pslr_token_action.y new file mode 100644 index 000000000..8c36b986f --- /dev/null +++ b/spec/fixtures/integration/pslr_token_action.y @@ -0,0 +1,69 @@ +%{ +#include + +#define YY_DECL int yylex(YYSTYPE *lval, struct parse_params *p) + +#include "pslr_token_action.h" +#include "pslr_token_action-lexer.h" + +extern int yylex(YYSTYPE *lval, struct parse_params *p); +static int yyerror(YYLTYPE *loc, struct parse_params *p, const char *str); +%} + +%code requires { + struct parse_params { + int current_state; + }; +} + +%define api.pure +%define lr.type pslr +%define api.pslr.state-member current_state + +%lex-param {struct parse_params *p} +%parse-param {struct parse_params *p} + +%token ID NUM SEMI + +%token-pattern ID /[a-zA-Z][a-zA-Z0-9_]*/ +%token-pattern NUM /[0-9]+/ +%token-pattern YYLAYOUT_WS /[ \t\r\n]+/ + +%token-action ID { /* token action for ID */ } +%token-action NUM { /* token action for NUM */ } + +%lex-prec ID -~ NUM + +%% + +start + : ID NUM ';' { printf("ok\n"); } + ; + +%% + +static int +yyerror(YYLTYPE *loc, struct parse_params *p, const char *str) +{ + (void)loc; + (void)p; + fprintf(stderr, "parse error: %s\n", str); + return 0; +} + +int +main(int argc, char *argv[]) +{ + struct parse_params params = { 0 }; + + if (argc == 2) { + yy_scan_string(argv[1]); + } + + if (yyparse(¶ms)) { + fprintf(stderr, "syntax error\n"); + return 1; + } + + return 0; +} diff --git a/spec/lrama/integration_spec.rb b/spec/lrama/integration_spec.rb index c0bba17f4..66ed9f35a 100644 --- a/spec/lrama/integration_spec.rb +++ b/spec/lrama/integration_spec.rb @@ -130,6 +130,160 @@ def generate_object(grammar_file_path, c_path, obj_path, command_args: []) end end + describe "PSLR parser and lexer integration" do + it "selects the longer token in shift contexts" do + test_parser("pslr_context", "foo>>bar", "shift\n") + end + + it "can prefer the shorter match in template contexts" do + test_parser("pslr_context", "foo>", "template\n") + end + + it "splits keyword and identifier contexts with the same lexeme" do + test_parser("pslr_keyword_context", "p x if", "kw\n") + test_parser("pslr_keyword_context", "q x if", "id\n") + end + + it "keeps chained shift and template contexts distinct" do + test_parser("pslr_shift_chain", "< # >", "template\n") + test_parser("pslr_shift_chain", "@ # >> foo", "shift\n") + end + + it "handles mixed keyword, identifier, template, and shift contexts in one grammar" do + test_parser("pslr_mixed_context", "p # if", "kw\n") + test_parser("pslr_mixed_context", "q # if", "id\n") + test_parser("pslr_mixed_context", "< # >", "template\n") + test_parser("pslr_mixed_context", "@ # >> foo", "shift\n") + end + + it "handles template argument lists without a short-token lex-prec override" do + test_parser("pslr_template_argument_lists", "vector> v;", "decl\n") + test_parser("pslr_template_argument_lists", "a >> b;", "expr\n") + end + + it "discards layout comments that share a prefix with normal tokens" do + test_parser("pslr_layout_comment", "a/* comment */ / b;", "ok\n") + end + + it "reports unresolved layout scanner conflicts without an explicit length rule" do + grammar_text = File.read(fixture_path("integration/pslr_layout_comment.y")) + .sub("%lex-prec DIV -~ YYLAYOUT_COMMENT\n", "") + grammar = Lrama::Parser.new(grammar_text, "integration/pslr_layout_comment_no_prec.y").parse + grammar.prepare + grammar.validate! + states = Lrama::States.new(grammar, Lrama::Tracer.new(Lrama::Logger.new)) + states.compute + states.compute_pslr + logger = Lrama::Logger.new + allow(logger).to receive(:error) + + expect { states.validate!(logger) }.to raise_error(SystemExit) + expect(logger).to have_received(:error).with(a_string_including("DIV", "YYLAYOUT_COMMENT")) + end + + it "honors fallback shortest-match length precedence" do + test_parser("pslr_fallback_precedence", "/*a*/ b /*c*/", "COM 5\n", expect_success: false) + end + + it "uses traditional longest match for unspecified fallback length conflicts" do + test_parser("pslr_fallback_precedence", "ab", "B 2\n", expect_success: false) + end + + it "honors fallback right-token length precedence" do + test_parser("pslr_fallback_precedence", "non-euclidean", "NON 4\n", expect_success: false) + end + + it "uses declaration order for unspecified fallback identity conflicts" do + test_parser("pslr_fallback_precedence", "z", "ZA 1\n", expect_success: false) + end + + it "uses explicit identity precedence before fallback declaration order" do + test_parser("pslr_fallback_precedence", "@", "PB 1\n", expect_success: false) + end + + it "preserves explicit identity precedence when fallback length precedence is needed" do + test_parser("pslr_fallback_precedence", "cd", "IB 2\n", expect_success: false) + end + + it "uses declaration order for fallback-only explicit identity cycles" do + test_parser("pslr_fallback_precedence", "~", "CYC_A 1\n", expect_success: false) + end + + it "returns YYEOF for empty input in PSLR pseudo-scan helpers" do + test_parser("pslr_fallback_precedence", "__empty__", "EOF 0 0 0 0\n") + end + + it "synthesizes implicit literal token patterns for parser-state rows" do + test_parser("pslr_implicit_literal", "x;", "ok\n") + end + + it "uses implicit literal token patterns in the fallback row" do + test_parser("pslr_implicit_literal", "__fallback_semi__", "SEMI 1 0\n") + end + + it "does not use fallback longest rules for normal parser-state rows" do + grammar_text = <<~GRAMMAR + %define lr.type pslr + %token-pattern A /a/ + %token-pattern B /ab/ + + %% + + start + : A + | B + ; + GRAMMAR + grammar = Lrama::Parser.new(grammar_text, "integration/pslr_normal_conflict.y").parse + grammar.prepare + grammar.validate! + states = Lrama::States.new(grammar, Lrama::Tracer.new(Lrama::Logger.new)) + states.compute + states.compute_pslr + logger = Lrama::Logger.new + allow(logger).to receive(:error) + + expect { states.validate!(logger) }.to raise_error(SystemExit) + expect(logger).to have_received(:error).with(a_string_including("unresolved PSLR scanner conflict", "A", "B")) + end + + it "does not use fallback declaration order for normal parser-state identity cycles" do + grammar_text = <<~GRAMMAR + %define lr.type pslr + %token-pattern A /a/ + %token-pattern B /a/ + %token-pattern C /a/ + + %lex-prec A <- B + %lex-prec B <- C + %lex-prec C <- A + + %% + + start + : A + | B + | C + ; + GRAMMAR + grammar = Lrama::Parser.new(grammar_text, "integration/pslr_normal_identity_cycle.y").parse + grammar.prepare + grammar.validate! + states = Lrama::States.new(grammar, Lrama::Tracer.new(Lrama::Logger.new)) + states.compute + states.compute_pslr + logger = Lrama::Logger.new + allow(logger).to receive(:error) + + expect { states.validate!(logger) }.to raise_error(SystemExit) + expect(logger).to have_received(:error).with(a_string_including("unresolved PSLR scanner conflict", "A", "B", "C")) + end + + it "consumes an unmatched PSLR character token before reporting an error" do + test_parser("pslr_template_argument_lists", "$", "", expect_success: false) + end + end + describe "user defined parameterized rules" do it "prints messages corresponding to rules" do expected = <<~STR @@ -307,4 +461,232 @@ def generate_object(grammar_file_path, c_path, obj_path, command_args: []) end end end + + describe "PSLR context-dependent lexing" do + describe "Scanner FSA with overlapping patterns" do + let(:rangle) do + id = Lrama::Lexer::Token::Ident.new(s_value: "RANGLE") + regex = Lrama::Lexer::Token::Regex.new(s_value: "/>/") + Lrama::Grammar::TokenPattern.new( + id: id, + pattern: regex, + lineno: 1, + definition_order: 0 + ) + end + + let(:rshift) do + id = Lrama::Lexer::Token::Ident.new(s_value: "RSHIFT") + regex = Lrama::Lexer::Token::Regex.new(s_value: "/>>/") + Lrama::Grammar::TokenPattern.new( + id: id, + pattern: regex, + lineno: 1, + definition_order: 1 + ) + end + + let(:scanner_fsa) { Lrama::ScannerFSA.new([rangle, rshift]) } + + it "recognizes both RANGLE and RSHIFT as possible matches for '>>'" do + results = scanner_fsa.scan(">>") + + token_names = results.map { |r| r[:token].name } + expect(token_names).to include("RANGLE") + expect(token_names).to include("RSHIFT") + end + + it "RANGLE matches at position 1, RSHIFT matches at position 2" do + results = scanner_fsa.scan(">>") + + rangle_match = results.find { |r| r[:token].name == "RANGLE" } + rshift_match = results.find { |r| r[:token].name == "RSHIFT" } + + expect(rangle_match[:position]).to eq(1) + expect(rshift_match[:position]).to eq(2) + end + end + + describe "Length precedence resolution" do + let(:lex_prec) { Lrama::Grammar::LexPrec.new } + + before do + left = Lrama::Lexer::Token::Ident.new(s_value: "RANGLE") + right = Lrama::Lexer::Token::Ident.new(s_value: "RSHIFT") + lex_prec.add_rule( + left_token: left, + operator: Lrama::Grammar::LexPrec::SHORTEST, + right_token: right, + lineno: 1 + ) + end + + let(:length_prec) { Lrama::LengthPrecedences.new(lex_prec) } + + it "indicates RANGLE (shorter) should be preferred over RSHIFT (longer)" do + expect(length_prec.prefer_shorter?("RANGLE", "RSHIFT")).to be true + end + + it "returns :left precedence for RANGLE vs RSHIFT" do + expect(length_prec.precedence("RANGLE", "RSHIFT")).to eq(:left) + end + end + + describe "Keyword vs identifier precedence" do + let(:lex_prec) { Lrama::Grammar::LexPrec.new } + + before do + left = Lrama::Lexer::Token::Ident.new(s_value: "ID") + right = Lrama::Lexer::Token::Ident.new(s_value: "IF") + lex_prec.add_rule( + left_token: left, + operator: Lrama::Grammar::LexPrec::IDENTITY_RIGHT_LONGEST, + right_token: right, + lineno: 1 + ) + end + + it "indicates IF has higher priority than ID" do + expect(lex_prec.identity_precedes?("IF", "ID")).to be true + end + + it "indicates ID does not have higher priority than IF" do + expect(lex_prec.identity_precedes?("ID", "IF")).to be false + end + end + + describe "Full PSLR grammar compilation" do + let(:grammar_text) do + <<~GRAMMAR + %define lr.type pslr + %token-pattern RSHIFT />>/ "right shift" + %token-pattern RANGLE />/ "right angle" + %token-pattern LANGLE />") + token_names = results.map { |r| r[:token].name } + + expect(token_names).to include("RANGLE") + expect(token_names).to include("RSHIFT") + end + + describe "context-dependent token selection" do + it "scanner_accepts table is built" do + expect(states.scanner_accepts_table).not_to be_nil + end + + it "different parser states may accept different tokens for same FSA state" do + scanner_accepts = states.scanner_accepts_table + scanner_fsa = states.scanner_fsa + + results = scanner_fsa.scan(">>") + rshift_result = results.find { |r| r[:token].name == "RSHIFT" } + rangle_result = results.find { |r| r[:token].name == "RANGLE" } + + expect(rshift_result).not_to be_nil + expect(rangle_result).not_to be_nil + expect(scanner_accepts.table).to be_a(Hash) + end + end + + describe "generated C code output" do + let(:out) { StringIO.new } + let(:context) { Lrama::Context.new(states) } + let(:output) do + Lrama::Output.new( + out: out, + output_file_path: "pslr_test.c", + template_name: "bison/yacc.c", + grammar_file_path: "pslr_test.y", + context: context, + grammar: grammar + ) + end + + before do + output.render + out.rewind + end + + let(:rendered) { out.read } + + it "includes yy_scanner_transition table" do + expect(rendered).to include("yy_scanner_transition") + expect(rendered).to include("YY_SCANNER_NUM_STATES") + end + + it "includes yy_state_to_accepting mapping" do + expect(rendered).to include("yy_state_to_accepting") + expect(rendered).to include("YY_ACCEPTING_NONE") + end + + it "includes yy_pslr_length_precedes table" do + expect(rendered).to include("yy_pslr_length_precedes") + end + + it "includes yy_pseudo_scan function" do + expect(rendered).to include("yy_pseudo_scan") + expect(rendered).to include("parser_state") + expect(rendered).to include("match_length") + end + + it "pseudo_scan function uses length precedences for token selection" do + expect(rendered).to include("yy_pslr_length_precedes[pbest][pattern_index]") + end + end + end + end end diff --git a/spec/lrama/output_spec.rb b/spec/lrama/output_spec.rb index 5fa1d04f0..a88c93fa8 100644 --- a/spec/lrama/output_spec.rb +++ b/spec/lrama/output_spec.rb @@ -203,4 +203,316 @@ end end end + + describe "PSLR methods" do + let(:token_pattern) do + id = Lrama::Lexer::Token::Ident.new(s_value: "ID") + regex = Lrama::Lexer::Token::Regex.new(s_value: "/[a-z]+/") + Lrama::Grammar::TokenPattern.new( + id: id, + pattern: regex, + lineno: 1, + definition_order: 0 + ) + end + + let(:scanner_fsa) { Lrama::ScannerFSA.new([token_pattern]) } + + let(:mock_states) do + instance_double( + Lrama::States, + scanner_fsa: scanner_fsa, + scanner_accepts_table: nil, + length_precedences: nil, + token_patterns: [token_pattern], + states: [], + find_symbol_by_s_value!: instance_double(Lrama::Grammar::Symbol, token_id: 301), + lexer_context_enabled?: false + ) + end + + let(:mock_context) do + instance_double(Lrama::Context, states: mock_states) + end + + let(:mock_grammar) do + instance_double( + Lrama::Grammar, + eof_symbol: nil, + error_symbol: nil, + undef_symbol: nil, + accept_symbol: nil, + locations: false, + parse_param: "struct parse_params *p", + lex_param: "struct parse_params *p", + pslr_defined?: true, + pslr_state_member: "current_state", + token_actions: [] + ) + end + + let(:pslr_output) do + out = StringIO.new + Lrama::Output.new( + out: out, + output_file_path: "test.c", + template_name: "bison/yacc.c", + grammar_file_path: "test.y", + context: mock_context, + grammar: mock_grammar + ) + end + + describe "#pslr_enabled?" do + it "returns true when grammar requested PSLR output" do + expect(pslr_output.pslr_enabled?).to be true + end + + it "returns false when grammar did not request PSLR output" do + allow(mock_grammar).to receive(:pslr_defined?).and_return(false) + expect(pslr_output.pslr_enabled?).to be false + end + end + + describe "#pslr_scanner_enabled?" do + it "returns true when scanner FSA is built with states" do + expect(pslr_output.pslr_scanner_enabled?).to be true + end + + it "returns false when scanner FSA is nil" do + allow(mock_states).to receive(:scanner_fsa).and_return(nil) + expect(pslr_output.pslr_scanner_enabled?).to be false + end + + it "returns false when scanner FSA has no states" do + empty_fsa = Lrama::ScannerFSA.new([]) + allow(mock_states).to receive(:scanner_fsa).and_return(empty_fsa) + expect(pslr_output.pslr_scanner_enabled?).to be false + end + end + + describe "#scanner_transition_table" do + it "generates C code for scanner transitions" do + result = pslr_output.scanner_transition_table + expect(result).to include("YY_SCANNER_NUM_STATES") + expect(result).to include("yy_scanner_transition") + end + end + + describe "#pseudo_scan_function" do + it "generates the pseudo_scan C function" do + result = pslr_output.pseudo_scan_function + expect(result).to include("yy_pseudo_scan") + expect(result).to include("parser_state") + expect(result).to include("match_length") + expect(result).to include("yy_token_pattern_to_token_id") + expect(result).to include("yy_pslr_fallback_length_precedes") + expect(result).to include("result->token = YYEOF") + end + end + + describe "#pslr_tables_and_functions" do + it "generates all PSLR C code" do + result = pslr_output.pslr_tables_and_functions + expect(result).to include("PSLR(1) Scanner Tables and Functions") + expect(result).to include("YY_SCANNER_NUM_STATES") + expect(result).to include("yy_scanner_transition") + expect(result).to include("yy_pseudo_scan") + expect(result).to include("yy_token_pattern_to_token_id") + end + + it "generates layout accumulation code" do + result = pslr_output.pslr_tables_and_functions + expect(result).to include("yypslr_layout_buffer") + expect(result).to include("YYPSLR_LAYOUT_TEXT") + expect(result).to include("YYPSLR_LAYOUT_LENGTH") + expect(result).to include("yypslr_scan_with_layout") + end + end + + describe "#pslr_lac_function" do + it "generates a LAC checker for PSLR parsers" do + result = pslr_output.pslr_lac_function + + expect(result).to include("yy_lac_check_") + expect(result).to include("yydefact") + end + end + + describe "#state_to_accepting_table" do + it "generates state to accepting mapping" do + result = pslr_output.state_to_accepting_table + expect(result).to include("yy_state_to_accepting") + expect(result).to include("YY_ACCEPTING_NONE") + end + end + + describe "#lexer_context_table_code" do + context "when lexer context is not enabled" do + it "returns empty string" do + result = pslr_output.lexer_context_table_code + expect(result).to eq("") + end + end + + context "when lexer context is enabled" do + let(:lexer_contexts) do + lcs = {} + lcs["BEG"] = Lrama::Grammar::LexerContext.new(name: "BEG", index: 0) + lcs["END"] = Lrama::Grammar::LexerContext.new(name: "END", index: 1) + lcs + end + + let(:classifier) { Lrama::LexerContextClassifier.new(lexer_contexts) } + + before do + allow(mock_states).to receive(:lexer_context_enabled?).and_return(true) + allow(mock_states).to receive(:lexer_context_table).and_return([0x01, 0x02, 0x01, 0x00]) + allow(mock_states).to receive(:lexer_context_classifier).and_return(classifier) + allow(mock_grammar).to receive(:lexer_contexts).and_return(lexer_contexts) + end + + it "generates the C context table" do + result = pslr_output.lexer_context_table_code + expect(result).to include("yy_lexer_context") + expect(result).to include("yy_lexer_context_is") + expect(result).to include("/* state 0 */") + expect(result).to include("/* state 3 */") + end + + it "generates defines via lexer_context_defines_code" do + result = pslr_output.lexer_context_defines_code + expect(result).to include("YY_CTX_BEG") + expect(result).to include("YY_CTX_END") + end + end + end + + describe "#pslr_function_declarations" do + it "declares the PSLR helper entry points" do + result = pslr_output.pslr_function_declarations + expect(result).to include("int yy_state_accepts_token") + expect(result).to include("int yy_pseudo_scan") + expect(result).to include("YYPSLR_PSEUDO_SCAN_STATE") + expect(result).to include("YYPSLR_PSEUDO_SCAN") + expect(result).to include("YYSETSTATE_CONTEXT(CurrentState)") + end + end + + describe "#length_precedences_table_code" do + let(:mock_length_prec) { Lrama::LengthPrecedences.new(Lrama::Grammar::LexPrec.new) } + + before do + allow(mock_states).to receive(:length_precedences).and_return(mock_length_prec) + end + + it "generates length precedences table" do + result = pslr_output.length_precedences_table_code + expect(result).to include("yy_pslr_length_precedes") + expect(result).to include("yy_pslr_fallback_length_precedes") + expect(result).to include("old_token") + end + + it "keeps traditional longest defaults out of the normal row table" do + a = Lrama::Grammar::TokenPattern.new( + id: Lrama::Lexer::Token::Ident.new(s_value: "A"), + pattern: Lrama::Lexer::Token::Regex.new(s_value: "/a/"), + lineno: 1, + definition_order: 0 + ) + b = Lrama::Grammar::TokenPattern.new( + id: Lrama::Lexer::Token::Ident.new(s_value: "B"), + pattern: Lrama::Lexer::Token::Regex.new(s_value: "/ab/"), + lineno: 1, + definition_order: 1 + ) + allow(mock_states).to receive(:token_patterns).and_return([a, b]) + + result = pslr_output.length_precedences_table_code + + expect(result).to include("static const int yy_pslr_length_precedes[2][2] = {\n /* A */ {1, 0},") + expect(result).to include("static const int yy_pslr_fallback_length_precedes[2][2] = {\n /* A */ {1, 1},") + end + end + + describe "#accepting_tokens_table" do + it "generates accepting tokens information" do + result = pslr_output.accepting_tokens_table + expect(result).to include("Accepting state token IDs") + end + end + end + + describe "PSLR integration in render" do + let(:pslr_grammar_text) do + <<~GRAMMAR + %code requires { + struct parse_params { + int current_state; + }; + } + %define lr.type pslr + %define api.pslr.state-member current_state + %parse-param {struct parse_params *p} + %lex-param {struct parse_params *p} + %token-pattern RSHIFT />>/ "right shift" + %token-pattern RANGLE />/ "right angle" + %lex-prec RANGLE -s RSHIFT + %% + program: RSHIFT | RANGLE + GRAMMAR + end + + let(:pslr_grammar) do + grammar = Lrama::Parser.new(pslr_grammar_text, "pslr_test.y").parse + grammar.prepare + grammar.validate! + grammar + end + + let(:pslr_states) do + s = Lrama::States.new(pslr_grammar, Lrama::Tracer.new(Lrama::Logger.new)) + s.compute + s.compute_pslr + s + end + + let(:pslr_context) { Lrama::Context.new(pslr_states) } + let(:pslr_out) { StringIO.new } + + let(:pslr_full_output) do + Lrama::Output.new( + out: pslr_out, + output_file_path: "pslr_test.c", + template_name: "bison/yacc.c", + grammar_file_path: "pslr_test.y", + header_out: header_out, + header_file_path: "pslr_test.h", + context: pslr_context, + grammar: pslr_grammar + ) + end + + it "includes PSLR tables in rendered output" do + pslr_full_output.render + pslr_out.rewind + header_out.rewind + rendered = pslr_out.read + rendered_header = header_out.read + + expect(rendered).to include("PSLR(1) Scanner Tables and Functions") + expect(rendered).to include("YY_SCANNER_NUM_STATES") + expect(rendered).to include("yy_scanner_transition") + expect(rendered).to include("yy_pseudo_scan") + expect(rendered).to include("yy_token_pattern_to_token_id") + expect(rendered).to include("yy_scanner_fallback_accepts") + expect(rendered).to include("yy_token_pattern_is_layout") + expect(rendered).to include("yy_lac_check_") + expect(rendered_header).to include("int yy_state_accepts_token") + expect(rendered_header).to include("int yy_pseudo_scan") + expect(rendered_header).to include("yypslr_scan_result") + expect(rendered_header).to include("YYSETSTATE_CONTEXT(CurrentState)") + expect(rendered_header).to include("YYPSLR_PSEUDO_SCAN(Context, Input, MatchLength)") + end + end end diff --git a/template/bison/_yacc.h b/template/bison/_yacc.h index 3e270c917..7cf4b6b21 100644 --- a/template/bison/_yacc.h +++ b/template/bison/_yacc.h @@ -71,6 +71,10 @@ struct YYLTYPE <%-# b4_declare_yyparse -%> int yyparse (<%= output.parse_param %>); +<%- if output.pslr_enabled? -%> +<%= output.pslr_function_declarations %> +<%- end -%> + <%= output.percent_code("provides") %> <%-# b4_cpp_guard_close([b4_spec_mapped_header_file]) -%> diff --git a/template/bison/yacc.c b/template/bison/yacc.c index 6edd59a0d..a5c195382 100644 --- a/template/bison/yacc.c +++ b/template/bison/yacc.c @@ -68,6 +68,10 @@ #define YYPULL 1 +<%- if output.lexer_context_enabled? -%> +<%= output.lexer_context_defines_code %> +<%- end -%> + <%# b4_user_pre_prologue -%> <%- if output.aux.prologue -%> /* First part of user prologue. */ @@ -582,6 +586,206 @@ static const <%= output.int_type_for(output.context.yyr2) %> yyr2[] = <%= output.int_array_to_string(output.context.yyr2) %> }; +<%- if output.pslr_enabled? -%> +<%= output.pslr_function_declarations %> +<%- end -%> + +<%- if output.pslr_enabled? -%> +<%= output.pslr_tables_and_functions %> +<%- end -%> + +<%- if output.lexer_context_enabled? -%> +<%= output.lexer_context_table_code %> +<%- end -%> + +<%- if output.pslr_enabled? -%> +#ifndef YYSETSTATE_CONTEXT +# define YYSETSTATE_CONTEXT(CurrentState) ((void) 0) +#endif +<%- end -%> + +int +yy_state_accepts_token (int yystate, int yychar) +{ + yysymbol_kind_t yytoken = YYTRANSLATE (yychar); + int yyn = yypact[yystate]; + + if (yypact_value_is_default (yyn)) + return 0; + + yyn += yytoken; + if (yyn < 0 || YYLAST < yyn || yycheck[yyn] != yytoken) + return 0; + + yyn = yytable[yyn]; + if (yyn <= 0) + return !yytable_value_is_error (yyn); + + return 1; +} + +/* + * Like yy_state_accepts_token, but also follows chains of default reductions + * where the rule has zero symbols on the right-hand side (yyr2 == 0). + * This allows the lexer to see tokens that become visible only after + * empty productions are reduced (e.g., opt_terms -> epsilon). + * + * Returns 1 if the token would be accepted in the current state or in a + * state reachable via a chain of empty default reductions; 0 otherwise. + */ +int +yy_state_eventually_accepts_token (int yystate, int yychar) +{ + yysymbol_kind_t yytoken = YYTRANSLATE (yychar); + /* Limit iteration to prevent infinite loops from cyclic empty reductions. */ + int visited[64]; + int visited_count = 0; + + for (;;) + { + int yyn; + + /* 1. Check the current state's action table for the token. */ + yyn = yypact[yystate]; + if (!yypact_value_is_default (yyn)) + { + yyn += yytoken; + if (0 <= yyn && yyn <= YYLAST && yycheck[yyn] == yytoken) + { + yyn = yytable[yyn]; + if (yyn > 0 || !yytable_value_is_error (yyn)) + return 1; + } + } + + /* 2. Try to follow the default reduction if it's an empty rule. */ + { + int rule = yydefact[yystate]; + int lhs, goto_state; + int i; + + if (rule == 0 || yyr2[rule] != 0) + return 0; /* No default or non-empty rule: can't proceed. */ + + /* Cycle detection. */ + for (i = 0; i < visited_count; i++) + if (visited[i] == yystate) + return 0; + if (visited_count < 64) + visited[visited_count++] = yystate; + else + return 0; + + /* Compute GOTO state after reducing by the empty rule. */ + lhs = yyr1[rule] - YYNTOKENS; + goto_state = yypgoto[lhs] + yystate; + if (0 <= goto_state && goto_state <= YYLAST + && yycheck[goto_state] == yystate) + yystate = yytable[goto_state]; + else + yystate = yydefgoto[lhs]; + } + } +} + +/* + * Like yy_state_eventually_accepts_token, but also follows non-empty + * default reductions by using the actual parser stack to determine + * GOTO states. This allows the lexer to see tokens that become visible + * after reductions like stmt -> expr (yyr2 > 0). + * + * stack_base and stack_top point to the parser's state stack (yy_state_t). + * Returns 1 if the token is reachable; 0 otherwise. + */ +int +yy_state_deep_accepts_token (int yystate, int yychar, + const void *stack_base_v, const void *stack_top_v) +{ + typedef short yy_state_t_compat; + yysymbol_kind_t yytoken = YYTRANSLATE (yychar); + const yy_state_t_compat *stack_base = (const yy_state_t_compat *)stack_base_v; + const yy_state_t_compat *stack_top = (const yy_state_t_compat *)stack_top_v; + int visited[64]; + int visited_count = 0; + int stack_consumed = 0; /* how many stack items we've "popped" */ + + if (!stack_base || !stack_top) + return 0; + + for (;;) + { + int yyn; + + /* 1. Check the current state's action table for the token. */ + yyn = yypact[yystate]; + if (!yypact_value_is_default (yyn)) + { + yyn += yytoken; + if (0 <= yyn && yyn <= YYLAST && yycheck[yyn] == yytoken) + { + yyn = yytable[yyn]; + if (yyn > 0 || !yytable_value_is_error (yyn)) + return 1; + } + } + + /* 2. Try to follow the default reduction. */ + { + int rule = yydefact[yystate]; + int rhs_len, lhs, goto_state, uncovered_state; + int i; + + if (rule == 0) + return 0; /* No default action. */ + + /* Cycle detection. */ + for (i = 0; i < visited_count; i++) + if (visited[i] == yystate) + return 0; + if (visited_count < 64) + visited[visited_count++] = yystate; + else + return 0; + + rhs_len = yyr2[rule]; + + if (rhs_len == 0) + { + /* Empty reduction: use current state for GOTO (same as eventually_accepts). */ + lhs = yyr1[rule] - YYNTOKENS; + goto_state = yypgoto[lhs] + yystate; + if (0 <= goto_state && goto_state <= YYLAST + && yycheck[goto_state] == yystate) + yystate = yytable[goto_state]; + else + yystate = yydefgoto[lhs]; + } + else + { + /* Non-empty reduction: need to look at the stack. */ + int total_depth = stack_consumed + rhs_len; + const yy_state_t_compat *target = stack_top - total_depth; + + if (target < stack_base) + return 0; /* Stack too shallow. */ + + uncovered_state = (int)*target; + lhs = yyr1[rule] - YYNTOKENS; + goto_state = yypgoto[lhs] + uncovered_state; + if (0 <= goto_state && goto_state <= YYLAST + && yycheck[goto_state] == uncovered_state) + yystate = yytable[goto_state]; + else + yystate = yydefgoto[lhs]; + + /* After a non-empty reduction, the stack effectively shrank. + * But since we consumed rhs_len items and pushed 1 (the GOTO state), + * net consumption is rhs_len - 1. */ + stack_consumed += rhs_len - 1; + } + } + } +} enum { YYENOMEM = -2 }; @@ -858,10 +1062,14 @@ int yydebug; # define YYMAXDEPTH 10000 #endif +<%- if output.pslr_enabled? -%> +<%= output.pslr_lac_function %> +<%- end -%> /* Context of a parse error. */ typedef struct { + yy_state_t *yyss; yy_state_t *yyssp; yysymbol_kind_t yytoken; YYLTYPE *yylloc; @@ -879,6 +1087,23 @@ yypcontext_expected_tokens (const yypcontext_t *yyctx, { /* Actual size of YYARG. */ int yycount = 0; +<%- if output.pslr_enabled? -%> + int yyx; + for (yyx = 0; yyx < YYNTOKENS; ++yyx) + if (yyx != YYSYMBOL_YYerror + && yy_lac_check_ (yyctx->yyss, yyctx->yyssp, YY_CAST (yysymbol_kind_t, yyx))) + { + if (!yyarg) + ++yycount; + else if (yycount == yyargn) + return 0; + else + yyarg[yycount++] = YY_CAST (yysymbol_kind_t, yyx); + } + if (yyarg && yycount == 0 && 0 < yyargn) + yyarg[0] = YYSYMBOL_YYEMPTY; + return yycount; +<%- else -%> int yyn = yypact[+*yyctx->yyssp]; if (!yypact_value_is_default (yyn)) { @@ -905,6 +1130,7 @@ yypcontext_expected_tokens (const yypcontext_t *yyctx, if (yyarg && yycount == 0 && 0 < yyargn) yyarg[0] = YYSYMBOL_YYEMPTY; return yycount; +<%- end -%> } @@ -1582,6 +1808,9 @@ YYLTYPE yylloc = yyloc_default; YY_IGNORE_USELESS_CAST_BEGIN *yyssp = YY_CAST (yy_state_t, yystate); YY_IGNORE_USELESS_CAST_END +<%- if output.pslr_enabled? -%> + YYSETSTATE_CONTEXT (yystate); +<%- end -%> YY_STACK_PRINT (yyss, yyssp<%= output.user_args %>); if (yyss + yystacksize - 1 <= yyssp) @@ -1668,6 +1897,40 @@ YYLTYPE yylloc = yyloc_default; /* First try to decide what to do without reference to lookahead token. */ yyn = yypact[yystate]; +<%- if output.pslr_enabled? -%> + if (yypact_value_is_default (yyn) && yydefact[yystate] != 0) + { + if (yychar == YYEMPTY) + { + YYDPRINTF ((stderr, "Reading a token\n")); + yychar = yylex <%= output.yylex_formals %>; + } + + if (yychar <= <%= output.eof_symbol.id.s_value %>) + { + yychar = <%= output.eof_symbol.id.s_value %>; + yytoken = <%= output.eof_symbol.enum_name %>; + YYDPRINTF ((stderr, "Now at end of input.\n")); + } + else if (yychar == <%= output.error_symbol.id.s_value %>) + { + yychar = <%= output.undef_symbol.id.s_value %>; + yytoken = <%= output.error_symbol.enum_name %>; + yyerror_range[1] = yylloc; + goto yyerrlab1; + } + else + { + yytoken = YYTRANSLATE (yychar); + YY_SYMBOL_PRINT ("Next token is", yytoken, &yylval, &yylloc<%= output.user_args %>); + } + + if (!yy_lac_check_ (yyss, yyssp, yytoken)) + goto yyerrlab; + + goto yydefault; + } +<%- end -%> if (yypact_value_is_default (yyn)) goto yydefault; @@ -1736,6 +1999,11 @@ YYLTYPE yylloc = yyloc_default; YY_SYMBOL_PRINT ("Next token is", yytoken, &yylval, &yylloc<%= output.user_args %>); } +<%- if output.pslr_enabled? -%> + if (!yy_lac_check_ (yyss, yyssp, yytoken)) + goto yyerrlab; +<%- end -%> + /* If the proper action on seeing token YYTOKEN is to reduce or to detect an error, take that action. */ yyn += yytoken; @@ -1854,7 +2122,7 @@ YYLTYPE yylloc = yyloc_default; ++yynerrs; { yypcontext_t yyctx - = {yyssp, yytoken, &yylloc}; + = {yyss, yyssp, yytoken, &yylloc}; char const *yymsgp = YY_("syntax error"); int yysyntax_error_status; yysyntax_error_status = yysyntax_error (&yymsg_alloc, &yymsg, &yyctx<%= output.user_args %>); @@ -2065,4 +2333,3 @@ YYLTYPE yylloc = yyloc_default; #line <%= output.aux.epilogue_first_lineno - 1 %> "<%= output.grammar_file_path %>" <%= output.aux.epilogue -%> <%- end -%> - From 897d36638abdd8124907c549326788f91bff3306 Mon Sep 17 00:00:00 2001 From: ydah Date: Wed, 8 Jul 2026 21:43:24 +0900 Subject: [PATCH 06/15] pslr: add PSLR reporting, tie-candidate warnings and CLI wiring * --report=pslr renders scanner metrics: state growth before/after splitting, token pattern and FSA state counts, per-state scanner_accepts rows and unresolved-conflict details * warn about lexical tie candidates (Def 3.3.3): token pairs that have a scanner conflict but appear one-sided in some parser state, unless an explicit %lex-tie/%lex-no-tie already covers the pair * Command runs compute_pslr for %define lr.type pslr grammars and validates states before writing any parser output, so unresolved scanner conflicts and state-growth guard violations abort generation * document the PSLR(1) directives in NEWS --- NEWS.md | 136 ++++++ lib/lrama/option_parser.rb | 3 +- lib/lrama/reporter.rb | 3 + lib/lrama/reporter/pslr.rb | 42 ++ lib/lrama/warnings.rb | 3 + lib/lrama/warnings/lexical_tie_candidates.rb | 27 ++ sig/generated/lrama/reporter/pslr.rbs | 18 + .../lrama/warnings/lexical_tie_candidates.rbs | 13 + spec/fixtures/command/pslr_growth_limit.y | 36 ++ spec/fixtures/command/pslr_pure_reduce.y | 26 ++ spec/lrama/command_spec.rb | 100 +++++ spec/lrama/option_parser_spec.rb | 10 +- spec/lrama/pslr_family_regressions_spec.rb | 394 ++++++++++++++++++ spec/lrama/pslr_generated_families_spec.rb | 49 +++ 14 files changed, 858 insertions(+), 2 deletions(-) create mode 100644 lib/lrama/reporter/pslr.rb create mode 100644 lib/lrama/warnings/lexical_tie_candidates.rb create mode 100644 sig/generated/lrama/reporter/pslr.rbs create mode 100644 sig/generated/lrama/warnings/lexical_tie_candidates.rbs create mode 100644 spec/fixtures/command/pslr_growth_limit.y create mode 100644 spec/fixtures/command/pslr_pure_reduce.y create mode 100644 spec/lrama/pslr_family_regressions_spec.rb create mode 100644 spec/lrama/pslr_generated_families_spec.rb diff --git a/NEWS.md b/NEWS.md index 693b46f01..8a1a52ae6 100644 --- a/NEWS.md +++ b/NEWS.md @@ -40,6 +40,142 @@ program: args_list(f_opt(number), opt_tail(string), number) https://github.com/ruby/lrama/pull/779 +### [EXPERIMENTAL] Support core PSLR(1) parser generation + +Added experimental support for generating a PSLR(1)-style parser based on this dissertation. +https://open.clemson.edu/all_dissertations/519/ + +This adds the following PSLR-related grammar directives and integration points: + +- `%define lr.type pslr` enables PSLR parser generation +- `%token-pattern` declares token candidates and their regular expressions for PSLR-aware lexical disambiguation +- `%lex-prec` declares explicit lexical precedence for overlapping token patterns +- `%symbol-set` declares reusable sets of terminal tokens for PSLR lexical declarations +- `%lex-tie` expands parser-state acceptable-token sets for tied terminals +- `%lex-no-tie` records an explicit no-tie decision for terminals with overlapping token patterns +- `YYLAYOUT*` token patterns are recognized in every parser state and discarded by PSLR-aware lexers +- `%define pslr.max-states` and `%define pslr.max-state-ratio` are Lrama-specific safety guards for state growth +- `%define api.pslr.state-member` names the parser-state field to be shared with the lexer when using the generated helper macros + +Typical usage looks like this: + +```yacc +%define api.pure +%define lr.type pslr +%define api.pslr.state-member current_state + +%parse-param {struct parse_params *p} +%lex-param {struct parse_params *p} + +%token-pattern RSHIFT />>/ "right shift" +%token-pattern RANGLE />/ "right angle" +%token-pattern ID /[a-z]+/ + +%lex-no-tie RANGLE RSHIFT +``` + +In this setup, `%token-pattern` lists the tokens that the generated pseudo-scanner FSA should consider, and +`%lex-no-tie` records that `RANGLE` and `RSHIFT` should not be treated as tied tokens. In a template-closing +parser state only `RANGLE` is syntactically acceptable, while a shift-expression state accepts `RSHIFT`; PSLR +state splitting and scanner profiles choose between the two without a global shortest-match rule. Use `%lex-prec` +for real lexical precedence relations, such as comment patterns that need a shortest-match or longest-match rule +in every parser context. + +For normal parser-state scanner rows, unresolved pseudo-scanner conflicts are not resolved by token declaration +order. They are reported as errors so the grammar can add an explicit `%lex-prec`, `%lex-tie`, or `%lex-no-tie` +declaration. For syntax-error handling, Lrama also emits a fallback scanner row. The fallback row first applies +explicit PSLR lexical precedence declarations. If a scanner conflict remains unresolved only because it is not a +pseudo-scanner conflict for any parser state, the fallback row completes the decision with scanner-default rules: +length conflicts use longest match and identity conflicts use token declaration order only for token pairs without +an explicit identity precedence relation. These defaults are composed with the explicit graph, so explicit identity +precedence is still honored when fallback length precedence is needed. For normal parser-state rows, explicit +precedence cycles remain unresolved and are reported as PSLR scanner conflicts. For the syntax-error fallback row +only, if explicit PSLR precedence plus fallback length defaults still do not determine a unique identity winner, +Lrama completes the fallback decision with traditional token declaration order so that a token-pattern match is +returned whenever `M(input, T0)` is non-empty. If no token pattern matches at all, the PSLR helper consumes one +byte and returns `YYUNDEF` as a character-token fallback, so error paths do not loop forever. + +`%lex-prec` uses ASCII spellings for the PSLR lexical precedence operators: + +| Lrama | Meaning | +|---|---| +| `<~` | identity conflict: right token wins; length conflict: longest match wins | +| `<-` | identity conflict: right token wins | +| `-~` | length conflict: longest match wins | +| `<<` | identity and length conflicts: right token wins | +| `-<` | length conflict: right token wins | +| `` versus `>>`; those cases should be handled by PSLR state +splitting plus `%lex-no-tie` when the tokens are intentionally not tied. + +Contradictory length precedence declarations are rejected instead of being silently overwritten. For example, +declaring both `%lex-prec A -~ B` and `%lex-prec A -s B` reports the token pair, the two operators and lines, and +the scan direction where the contradiction occurs. + +Lexical ties are separate from precedence. For example: + +```yacc +%token-pattern IF /if/ +%token-pattern ID /[a-z]+/ +%symbol-set keywords IF +%lex-tie ID keywords +%lex-prec ID <~ keywords +``` + +Here, `IF` can be considered when the parser state accepts `ID`, but `%lex-tie` does not choose a winner. +The `%lex-prec ID <~ keywords` declaration resolves the `if` identity conflict in favor of `IF` while keeping +longer identifiers such as `ifx` as `ID`. + +When both operands of `%lex-tie` are explicit tokens, Lrama ties them even if their token patterns do not conflict. +When either operand is a `%symbol-set` or `yyall`, Lrama follows the PSLR paper and ties only token pairs that have +a pairwise scanner conflict. This avoids unnecessary transitive lexical ties and the pseudo-scanner conflicts they +would create. + +`%lex-no-tie` suppresses lexical tie candidate warnings; it does not break a final transitive tie closure. Generic +declarations such as `%lex-no-tie yyall yyall` can suppress broad candidate reports, and a more specific `%lex-tie` +can still tie the relevant token pair. + +Token patterns named `YYLAYOUT` or starting with `YYLAYOUT` are layout tokens. They are included in every +parser-state scanner row and should be consumed and skipped by the PSLR-aware lexer instead of being returned to +the parser. The generated helpers include `YYPSLR_TOKEN_IS_LAYOUT(Token)` and the structured +`YYPSLR_PSEUDO_SCAN_RESULT(...)` API for this purpose. + +`yy_pseudo_scan_result` is a low-level scanner helper and may report a layout token with `result->is_layout = 1`. +A PSLR-aware `yylex` must consume that text, keep the same parser state, and scan the remaining input instead of +passing the layout token to the parser. + +The generated PSLR scanner FSA considers terminals declared with `%token-pattern` and character literal terminals +for which Lrama can synthesize an exact-match implicit token pattern. Grammar terminals without either a token +pattern or a supported literal spelling remain outside the generated pseudo-scanner helper. Thus, the paper's `T0` +fallback token set corresponds here to the set of terminals known to the generated scanner FSA. Parser-state rows +use the subset accepted by the current state plus tied tokens and layout tokens. The fallback row uses the whole +generated scanner universe so error handling can still identify and consume a token when the current parser state +has no normal scanner decision. + +`%token-pattern` currently uses an ASCII byte-oriented regular-expression subset for PSLR pseudo scanning. +Supported constructs are literals, escaped literals such as `\/`, `\*`, `\+`, `\?`, `\(`, `\)`, `\[`, `\]`, +and `\\`, grouping with `(...)`, alternation with `|`, repetition operators `*`, `+`, `?`, character classes with +escapes such as `[\]]`, `[\\]`, and `[\n\t\r]`, ranges such as `[a-z]`, negated classes such as `[^*]`, common +escapes such as `\n`, `\t`, and `\r`, and `.`. The `.` operator matches ASCII bytes except newline; negated +character classes range over ASCII bytes. Unicode properties and full Ruby/Onigmo regexp syntax are not supported. +Unsupported or malformed constructs are rejected during generation rather than silently reinterpreted. Nullable +token patterns such as `//`, `/()/`, `/a*/`, `/a?/`, and `/a|/` are generation errors because PSLR token lexemes +must be non-empty. + +When the parser and lexer share a context through `%parse-param` / `%lex-param`, the generated header also +provides helpers such as `YYPSLR_PSEUDO_SCAN(...)`, so the lexer can choose a token based on the current parser +state. The paper-compatible scanning path needs the lexer to pass the unconsumed input prefix, not only an +already-decided token fragment, so legacy external lexer bridges may still be limited by the text they provide. + +PSLR parsers enable a lightweight LAC check in the generated parser so syntax errors caused by LR state merging, +default reductions, or `%nonassoc` error actions are detected before user semantic actions are run for the bad +lookahead. PSLR support is still experimental. Scoped lexical declarations, lexical nonterminals, and `%lex` +are not implemented yet. If you find any bugs, please report them. + ## Lrama 0.7.1 (2025-12-24) ### Optimize IELR diff --git a/lib/lrama/option_parser.rb b/lib/lrama/option_parser.rb index 5a15d59c7..29ae759c4 100644 --- a/lib/lrama/option_parser.rb +++ b/lib/lrama/option_parser.rb @@ -97,6 +97,7 @@ def parse_by_option_parser(argv) o.on_tail ' lookaheads explicitly associate lookahead tokens to items' o.on_tail ' solved describe shift/reduce conflicts solving' o.on_tail ' counterexamples, cex generate conflict counterexamples' + o.on_tail ' pslr report PSLR split and scanner metrics' o.on_tail ' rules list unused rules' o.on_tail ' terms list unused terminals' o.on_tail ' verbose report detailed internal state and analysis results' @@ -141,7 +142,7 @@ def parse_by_option_parser(argv) end ALIASED_REPORTS = { cex: :counterexamples }.freeze #: Hash[Symbol, Symbol] - VALID_REPORTS = %i[states itemsets lookaheads solved counterexamples rules terms verbose].freeze #: Array[Symbol] + VALID_REPORTS = %i[states itemsets lookaheads solved counterexamples pslr rules terms verbose].freeze #: Array[Symbol] # @rbs (Array[String]) -> Hash[Symbol, bool] def validate_report(report) diff --git a/lib/lrama/reporter.rb b/lib/lrama/reporter.rb index ed25cc7f8..a98203855 100644 --- a/lib/lrama/reporter.rb +++ b/lib/lrama/reporter.rb @@ -5,6 +5,7 @@ require_relative 'reporter/grammar' require_relative 'reporter/precedences' require_relative 'reporter/profile' +require_relative 'reporter/pslr' require_relative 'reporter/rules' require_relative 'reporter/states' require_relative 'reporter/terms' @@ -20,6 +21,7 @@ def initialize(**options) @terms = Terms.new(**options) @conflicts = Conflicts.new @precedences = Precedences.new + @pslr = Pslr.new(**options) @grammar = Grammar.new(**options) @states = States.new(**options) end @@ -31,6 +33,7 @@ def report(io, states) report_duration(:report_terms) { @terms.report(io, states) } report_duration(:report_conflicts) { @conflicts.report(io, states) } report_duration(:report_precedences) { @precedences.report(io, states) } + report_duration(:report_pslr) { @pslr.report(io, states) } report_duration(:report_grammar) { @grammar.report(io, states) } report_duration(:report_states) { @states.report(io, states, ielr: states.ielr_defined?) } end diff --git a/lib/lrama/reporter/pslr.rb b/lib/lrama/reporter/pslr.rb new file mode 100644 index 000000000..766c5855b --- /dev/null +++ b/lib/lrama/reporter/pslr.rb @@ -0,0 +1,42 @@ +# rbs_inline: enabled +# frozen_string_literal: true + +module Lrama + class Reporter + class Pslr + # @rbs (?pslr: bool, **bool _) -> void + def initialize(pslr: false, **_) + @pslr = pslr + end + + # @rbs (IO io, Lrama::States states) -> void + def report(io, states) + return unless @pslr + return unless states.pslr_defined? + + metrics = states.pslr_metrics + + io << "PSLR Summary\n\n" + io << " Base states: #{metrics[:base_states_count]}\n" + io << " Total states: #{metrics[:total_states_count]}\n" + io << " Split states: #{metrics[:split_state_count]}\n" + io << " State growth: +#{metrics[:growth_count]} (#{format_ratio(metrics[:growth_ratio])})\n" + io << " Token patterns: #{metrics[:token_pattern_count]}\n" + io << " Scanner states: #{metrics[:scanner_fsa_state_count]}\n" + io << " Inadequacies: #{metrics[:inadequacies_count]}\n" + io << " Max states: #{states.pslr_max_states || 'unbounded'}\n" + io << " Max ratio: #{states.pslr_max_state_ratio || 'unbounded'}\n" + io << "\n" + end + + private + + # @rbs (Numeric?) -> String + def format_ratio(value) + return "n/a" if value.nil? + + "#{format('%.2f', value)}x" + end + end + end +end diff --git a/lib/lrama/warnings.rb b/lib/lrama/warnings.rb index 52f09144e..b014f85f7 100644 --- a/lib/lrama/warnings.rb +++ b/lib/lrama/warnings.rb @@ -7,6 +7,7 @@ require_relative 'warnings/redefined_rules' require_relative 'warnings/required' require_relative 'warnings/useless_precedence' +require_relative 'warnings/lexical_tie_candidates' module Lrama class Warnings @@ -18,6 +19,7 @@ def initialize(logger, warnings) @redefined_rules = RedefinedRules.new(logger, warnings) @required = Required.new(logger, warnings) @useless_precedence = UselessPrecedence.new(logger, warnings) + @lexical_tie_candidates = LexicalTieCandidates.new(logger, warnings) end # @rbs (Lrama::Grammar grammar, Lrama::States states) -> void @@ -28,6 +30,7 @@ def warn(grammar, states) @redefined_rules.warn(grammar) @required.warn(grammar) @useless_precedence.warn(grammar, states) + @lexical_tie_candidates.warn(states) end end end diff --git a/lib/lrama/warnings/lexical_tie_candidates.rb b/lib/lrama/warnings/lexical_tie_candidates.rb new file mode 100644 index 000000000..e45cccb2e --- /dev/null +++ b/lib/lrama/warnings/lexical_tie_candidates.rb @@ -0,0 +1,27 @@ +# rbs_inline: enabled +# frozen_string_literal: true + +module Lrama + class Warnings + class LexicalTieCandidates + # @rbs (Logger logger, bool warnings) -> void + def initialize(logger, warnings) + @logger = logger + @warnings = warnings + end + + # @rbs (Lrama::States states) -> void + def warn(states) + return unless @warnings + return unless states.respond_to?(:lexical_tie_candidates) + + states.lexical_tie_candidates.each do |left, right| + @logger.warn( + "lexical tie candidate: #{left} and #{right} conflict lexically but are not tied; " \ + "add %lex-tie #{left} #{right} or %lex-no-tie #{left} #{right}" + ) + end + end + end + end +end diff --git a/sig/generated/lrama/reporter/pslr.rbs b/sig/generated/lrama/reporter/pslr.rbs new file mode 100644 index 000000000..c9de08058 --- /dev/null +++ b/sig/generated/lrama/reporter/pslr.rbs @@ -0,0 +1,18 @@ +# Generated from lib/lrama/reporter/pslr.rb with RBS::Inline + +module Lrama + class Reporter + class Pslr + # @rbs (?pslr: bool, **bool _) -> void + def initialize: (?pslr: bool, **bool _) -> void + + # @rbs (IO io, Lrama::States states) -> void + def report: (IO io, Lrama::States states) -> void + + private + + # @rbs (Numeric?) -> String + def format_ratio: (Numeric?) -> String + end + end +end diff --git a/sig/generated/lrama/warnings/lexical_tie_candidates.rbs b/sig/generated/lrama/warnings/lexical_tie_candidates.rbs new file mode 100644 index 000000000..43e569a41 --- /dev/null +++ b/sig/generated/lrama/warnings/lexical_tie_candidates.rbs @@ -0,0 +1,13 @@ +# Generated from lib/lrama/warnings/lexical_tie_candidates.rb with RBS::Inline + +module Lrama + class Warnings + class LexicalTieCandidates + # @rbs (Logger logger, bool warnings) -> void + def initialize: (Logger logger, bool warnings) -> void + + # @rbs (Lrama::States states) -> void + def warn: (Lrama::States states) -> void + end + end +end diff --git a/spec/fixtures/command/pslr_growth_limit.y b/spec/fixtures/command/pslr_growth_limit.y new file mode 100644 index 000000000..095ed6ae8 --- /dev/null +++ b/spec/fixtures/command/pslr_growth_limit.y @@ -0,0 +1,36 @@ +%define lr.type pslr + +%token-pattern P /p/ +%token-pattern Q /q/ +%token-pattern X /x/ +%token-pattern IF /if/ +%token-pattern ID /[a-z]+/ + +%lex-prec ID <~ IF + +%% + +program + : kw_context + | id_context + ; + +kw_context + : P shared IF + ; + +id_context + : Q shared ID + ; + +shared + : n1 + ; + +n1 + : n2 + ; + +n2 + : X + ; diff --git a/spec/fixtures/command/pslr_pure_reduce.y b/spec/fixtures/command/pslr_pure_reduce.y new file mode 100644 index 000000000..99e8ec6ef --- /dev/null +++ b/spec/fixtures/command/pslr_pure_reduce.y @@ -0,0 +1,26 @@ +%define lr.type pslr + +%token-pattern RSHIFT />>/ +%token-pattern RANGLE />/ +%token-pattern ID /[a-z]+/ + +%lex-prec RANGLE -~ RSHIFT + +%% + +program + : templ + | rshift_expr + ; + +templ + : a RANGLE + ; + +rshift_expr + : a RSHIFT ID + ; + +a + : ID + ; diff --git a/spec/lrama/command_spec.rb b/spec/lrama/command_spec.rb index 58069e4a7..ce30eccf0 100644 --- a/spec/lrama/command_spec.rb +++ b/spec/lrama/command_spec.rb @@ -81,5 +81,105 @@ File.delete("report.output") end end + + context "when a PSLR grammar needs pure-reduce lookahead to choose tokens" do + let(:outfile) { File.join(Dir.tmpdir, "pslr-pure-reduce.c") } + + before do + File.delete(outfile) if File.exist?(outfile) + end + + after do + File.delete(outfile) if File.exist?(outfile) + end + + it "emits parser output successfully" do + command = Lrama::Command.new(["-o", outfile, fixture_path("command/pslr_pure_reduce.y")]) + + expect(command.run).to be_nil + expect(File).to exist(outfile) + end + end + + context "when validation aborts" do + let(:outfile) { File.join(Dir.tmpdir, "validate-abort.c") } + + before do + File.delete(outfile) if File.exist?(outfile) + end + + after do + File.delete(outfile) if File.exist?(outfile) + end + + it "fails before writing parser output" do + allow_any_instance_of(Lrama::States).to receive(:validate!).and_raise(SystemExit) + + command = Lrama::Command.new(["-o", outfile, fixture_path("command/basic.y")]) + + expect { command.run }.to raise_error(SystemExit) + expect(File).not_to exist(outfile) + end + end + + context "when a PSLR grammar exceeds the configured state limit" do + let(:outfile) { File.join(Dir.tmpdir, "pslr-growth-limit.c") } + + before do + File.delete(outfile) if File.exist?(outfile) + end + + after do + File.delete(outfile) if File.exist?(outfile) + end + + it "fails before writing parser output" do + command = Lrama::Command.new([ + "-Dpslr.max-states=5", + "-o", outfile, + fixture_path("command/pslr_growth_limit.y") + ]) + + expect do + begin + command.run + rescue SystemExit + nil + end + end.to output(/error: PSLR state growth exceeded pslr.max-states=5/).to_stderr_from_any_process + + expect(File).not_to exist(outfile) + end + end + + context "when PSLR report output is requested" do + let(:outfile) { File.join(Dir.tmpdir, "pslr-report.c") } + let(:report_file) { File.join(Dir.tmpdir, "pslr-report.output") } + + before do + File.delete(outfile) if File.exist?(outfile) + File.delete(report_file) if File.exist?(report_file) + end + + after do + File.delete(outfile) if File.exist?(outfile) + File.delete(report_file) if File.exist?(report_file) + end + + it "writes PSLR metrics into the report file" do + command = Lrama::Command.new([ + "--report=pslr", + "--report-file=#{report_file}", + "-o", outfile, + fixture_path("command/pslr_growth_limit.y") + ]) + + expect(command.run).to be_nil + report = File.read(report_file) + expect(report).to include("PSLR Summary") + expect(report).to include("Base states:") + expect(report).to include("Total states:") + end + end end end diff --git a/spec/lrama/option_parser_spec.rb b/spec/lrama/option_parser_spec.rb index 7675c4daa..0efaa9008 100644 --- a/spec/lrama/option_parser_spec.rb +++ b/spec/lrama/option_parser_spec.rb @@ -80,6 +80,7 @@ lookaheads explicitly associate lookahead tokens to items solved describe shift/reduce conflicts solving counterexamples, cex generate conflict counterexamples + pslr report PSLR split and scanner metrics rules list unused rules terms list unused terminals verbose report detailed internal state and analysis results @@ -128,12 +129,19 @@ end end + context "when pslr is passed" do + it "returns option hash pslr flag enabled" do + opts = option_parser.send(:validate_report, ["pslr"]) + expect(opts).to eq({grammar: true, pslr: true}) + end + end + context "when all is passed" do it "returns option hash all flags enabled" do opts = option_parser.send(:validate_report, ["all"]) expect(opts).to eq({ grammar: true, states: true, itemsets: true, - lookaheads: true, solved: true, counterexamples: true, + lookaheads: true, solved: true, counterexamples: true, pslr: true, rules: true, terms: true, verbose: true }) end diff --git a/spec/lrama/pslr_family_regressions_spec.rb b/spec/lrama/pslr_family_regressions_spec.rb new file mode 100644 index 000000000..0001e911e --- /dev/null +++ b/spec/lrama/pslr_family_regressions_spec.rb @@ -0,0 +1,394 @@ +# frozen_string_literal: true + +RSpec.describe "PSLR family regressions" do + include PslrFamilyHelper + + describe "pure-reduce profile" do + let(:grammar) do + build_grammar(<<~GRAMMAR, "states/pslr_pure_reduce.y") + %define lr.type pslr + %token-pattern RSHIFT />>/ + %token-pattern RANGLE />/ + %token-pattern ID /[a-z]+/ + %lex-prec RANGLE -~ RSHIFT + + %% + + program + : templ + | rshift_expr + ; + + templ + : a RANGLE + ; + + rshift_expr + : a RSHIFT ID + ; + + a + : ID + ; + GRAMMAR + end + + it "keeps pure reduce states scanner-compatible without forcing a split" do + ielr_states, pslr_states = compute_ielr_and_pslr(grammar) + + reduce_state = pslr_states.states.find do |state| + state.reduces.any? { |reduce| reduce.rule.display_name == "a -> ID" } + end + + expect(pslr_states.states_count).to eq(ielr_states.states_count) + expect(pslr_states.pslr_inadequacies).to be_empty + expect(acceptable_tokens(pslr_states, reduce_state)).to contain_exactly("RANGLE", "RSHIFT") + end + end + + describe "chained keyword split" do + let(:grammar) do + build_grammar(<<~GRAMMAR, "states/pslr_keyword_context.y") + %define lr.type pslr + %token-pattern P /p/ + %token-pattern Q /q/ + %token-pattern X /x/ + %token-pattern IF /if/ + %token-pattern ID /[a-z]+/ + %lex-prec ID <~ IF + + %% + + program + : kw_context + | id_context + ; + + kw_context + : P shared IF + ; + + id_context + : Q shared ID + ; + + shared + : n1 + ; + + n1 + : n2 + ; + + n2 + : X + ; + GRAMMAR + end + + it "splits every chained reduce state by scanner profile" do + ielr_states, pslr_states = compute_ielr_and_pslr(grammar) + + reduce_states = pslr_states.states + .select { |state| state.reduces.any? } + .group_by { |state| state.reduces.first.rule.display_name } + + expect(pslr_states.states_count).to be > ielr_states.states_count + expect(pslr_states.pslr_inadequacies).to be_empty + + ["shared -> n1", "n1 -> n2", "n2 -> X"].each do |rule_name| + states_for_rule = reduce_states.fetch(rule_name) + token_sets = states_for_rule.map { |state| acceptable_tokens(pslr_states, state) } + + expect(states_for_rule.size).to eq(2) + expect(states_for_rule.count(&:split_state?)).to eq(1) + expect(token_sets.any? { |set| set.include?("IF") && !set.include?("ID") }).to be(true) + expect(token_sets.any? { |set| set.include?("ID") && !set.include?("IF") }).to be(true) + end + end + end + + describe "chained shift/angle split" do + let(:grammar) do + build_grammar(<<~GRAMMAR, "states/pslr_shift_chain.y") + %define lr.type pslr + %token-pattern LT />/ + %token-pattern RANGLE />/ + %token-pattern ID /[a-z]+/ + %lex-no-tie RANGLE RSHIFT + + %% + + program + : template_expr + | shift_expr + ; + + template_expr + : LT shared RANGLE + ; + + shift_expr + : START shared RSHIFT ID + ; + + shared + : n1 + ; + + n1 + : n2 + ; + + n2 + : MARK + ; + GRAMMAR + end + + it "splits every chained reduce state by shift/angle scanner profile" do + ielr_states, pslr_states = compute_ielr_and_pslr(grammar) + + reduce_states = pslr_states.states + .select { |state| state.reduces.any? } + .group_by { |state| state.reduces.first.rule.display_name } + + expect(pslr_states.states_count).to be > ielr_states.states_count + expect(pslr_states.pslr_inadequacies).to be_empty + + ["shared -> n1", "n1 -> n2", "n2 -> MARK"].each do |rule_name| + states_for_rule = reduce_states.fetch(rule_name) + token_sets = states_for_rule.map { |state| acceptable_tokens(pslr_states, state) } + + expect(states_for_rule.size).to eq(2) + expect(states_for_rule.count(&:split_state?)).to eq(1) + expect(token_sets.any? { |set| set.include?("RANGLE") && !set.include?("RSHIFT") }).to be(true) + expect(token_sets.any? { |set| set.include?("RSHIFT") && !set.include?("RANGLE") }).to be(true) + end + end + end + + describe "mixed families" do + { + "empty shared wrapper" => { + path: "states/pslr_mixed_empty.y", + grows: true, + grammar: <<~GRAMMAR, + %define lr.type pslr + %token-pattern LT />/ + %token-pattern RANGLE />/ + %lex-prec ID <~ IF + %lex-no-tie RANGLE RSHIFT + + %% + + program + : kw + | ident + | templ + | shift_expr + ; + + kw + : P shared IF + ; + + ident + : Q shared ID + ; + + templ + : LT shared RANGLE + ; + + shift_expr + : START shared RSHIFT ID + ; + + shared + : opt n1 + ; + + opt + : + ; + + n1 + : MARK + ; + GRAMMAR + }, + "chain2 shared wrapper" => { + path: "states/pslr_mixed_chain2.y", + grows: true, + grammar: <<~GRAMMAR, + %define lr.type pslr + %token-pattern LT />/ + %token-pattern RANGLE />/ + %lex-prec ID <~ IF + %lex-no-tie RANGLE RSHIFT + + %% + + program + : kw + | ident + | templ + | shift_expr + ; + + kw + : P shared IF + ; + + ident + : Q shared ID + ; + + templ + : LT shared RANGLE + ; + + shift_expr + : START shared RSHIFT ID + ; + + shared + : n1 + ; + + n1 + : n2 + ; + + n2 + : MARK + ; + GRAMMAR + } + }.each do |label, attrs| + it "keeps #{label} scanner-compatible" do + grammar = build_grammar(attrs[:grammar], attrs[:path]) + ielr_states, pslr_states = compute_ielr_and_pslr(grammar) + + if attrs[:grows] + expect(pslr_states.states_count).to be > ielr_states.states_count + else + expect(pslr_states.states_count).to eq(ielr_states.states_count) + end + expect(pslr_states.pslr_inadequacies).to be_empty + end + end + end + + describe "lexical tie candidates" do + it "reports scanner-conflicting tokens that appear one-sided in parser states" do + grammar = build_grammar(<<~GRAMMAR, "states/pslr_lexical_tie_candidate.y") + %define lr.type pslr + %token-pattern P /p/ + %token-pattern Q /q/ + %token-pattern MARK /#/ + %token-pattern IF /if/ + %token-pattern ID /[a-z]+/ + %lex-prec ID <~ IF + + %% + + start + : P MARK IF + | Q MARK ID + ; + GRAMMAR + + _ielr_states, pslr_states = compute_ielr_and_pslr(grammar) + + expect(pslr_states.lexical_tie_candidates).to include(["ID", "IF"]) + end + + it "suppresses candidates with yyall no-tie" do + grammar = build_grammar(<<~GRAMMAR, "states/pslr_lexical_tie_no_tie.y") + %define lr.type pslr + %token-pattern P /p/ + %token-pattern Q /q/ + %token-pattern MARK /#/ + %token-pattern IF /if/ + %token-pattern ID /[a-z]+/ + %lex-prec ID <~ IF + %lex-no-tie yyall yyall + + %% + + start + : P MARK IF + | Q MARK ID + ; + GRAMMAR + + _ielr_states, pslr_states = compute_ielr_and_pslr(grammar) + + expect(pslr_states.lexical_tie_candidates).to be_empty + end + end + + describe "template argument lists" do + it "splits RANGLE and RSHIFT contexts without short-token precedence" do + grammar = build_grammar(<<~GRAMMAR, "states/pslr_template_argument_lists.y") + %define lr.type pslr + %token-pattern ID /[a-zA-Z][a-zA-Z0-9_]*/ + %token-pattern LT // + %token-pattern RSHIFT />>/ + %token-pattern SEMI /;/ + %token-pattern YYLAYOUT /[ \\t\\r\\n]+/ + %lex-no-tie RANGLE RSHIFT + + %% + + start + : decl SEMI + | expr SEMI + ; + + decl + : type id + ; + + type + : id LT type RANGLE + | ID + ; + + expr + : id RSHIFT id + ; + + id + : ID + ; + GRAMMAR + + ielr_states, pslr_states = compute_ielr_and_pslr(grammar) + + expect(pslr_states.states_count).to be >= ielr_states.states_count + expect(pslr_states.pslr_inadequacies).to be_empty + expect(pslr_states.lexical_tie_candidates).not_to include(["RANGLE", "RSHIFT"]) + end + end +end diff --git a/spec/lrama/pslr_generated_families_spec.rb b/spec/lrama/pslr_generated_families_spec.rb new file mode 100644 index 000000000..795d25d69 --- /dev/null +++ b/spec/lrama/pslr_generated_families_spec.rb @@ -0,0 +1,49 @@ +# frozen_string_literal: true + +RSpec.describe "PSLR generated family coverage" do + include PslrFamilyHelper + + families = [ + { + label: "keyword/id", + builder: :keyword_context_source, + path_prefix: "generated/pslr_keyword_depth", + split_expected: true + }, + { + label: "shift/angle", + builder: :shift_angle_source, + path_prefix: "generated/pslr_shift_depth", + split_expected: true + }, + { + label: "mixed", + builder: :mixed_context_source, + path_prefix: "generated/pslr_mixed_depth", + split_expected: true + } + ].freeze + + families.each do |family| + (0..3).each do |depth| + it "#{family[:label]} depth=#{depth} keeps PSLR inadequacies resolved" do + grammar = build_grammar( + public_send(family[:builder], depth: depth), + "#{family[:path_prefix]}_#{depth}.y" + ) + ielr_states, pslr_states = compute_ielr_and_pslr(grammar) + + aggregate_failures do + expect(pslr_states.pslr_inadequacies).to be_empty + expect(pslr_states.states_count).to be >= ielr_states.states_count + expect(pslr_states.pslr_metrics[:growth_count]).to eq(pslr_states.states_count - pslr_states.pslr_metrics[:base_states_count]) + + next unless family[:split_expected] + next unless 1 <= depth + + expect(pslr_states.states_count).to be > ielr_states.states_count + end + end + end + end +end From ceee1e65491aec94158a5b761c80a023a826beca Mon Sep 17 00:00:00 2001 From: ydah Date: Wed, 8 Jul 2026 21:53:14 +0900 Subject: [PATCH 07/15] pslr: switch pseudo-scan runtime to length-delimited input Ruby sources may contain NUL bytes, so a NUL-terminated scan API cannot represent every valid input. yy_pseudo_scan_result and yy_pseudo_scan now take (input, input_len) and treat input_len == 0 as end of input; the YYPSLR_PSEUDO_SCAN* bridge macros and yypslr_scan_with_layout gain the corresponding length argument, and the layout loop keeps the remaining length in sync as it consumes layout tokens. The scan result reports from_fallback when the token came from the fallback row or from character-token handling, letting callers detect a guaranteed upcoming syntax error before handing the token to the parser. The YYEOF token action now runs before returning YYEOF so a grammar can recover trailing layout (paper section 3.6). --- lib/lrama/output.rb | 78 ++++++++++++------- spec/fixtures/integration/pslr_context.l | 2 +- .../integration/pslr_fallback_precedence.l | 2 +- .../integration/pslr_fallback_precedence.y | 4 +- .../integration/pslr_implicit_literal.l | 2 +- .../integration/pslr_implicit_literal.y | 2 +- .../integration/pslr_keyword_context.l | 2 +- .../integration/pslr_layout_comment.l | 2 +- .../fixtures/integration/pslr_mixed_context.l | 4 +- spec/fixtures/integration/pslr_shift_chain.l | 2 +- .../pslr_template_argument_lists.l | 2 +- spec/lrama/output_spec.rb | 14 +++- 12 files changed, 74 insertions(+), 42 deletions(-) diff --git a/lib/lrama/output.rb b/lib/lrama/output.rb index 68b5d1139..52aaf4e17 100644 --- a/lib/lrama/output.rb +++ b/lib/lrama/output.rb @@ -424,6 +424,8 @@ def pslr_function_declarations if pslr_scanner_enabled? declarations << <<~C_CODE + #include + #ifndef YYPSLR_SCAN_RESULT_DEFINED # define YYPSLR_SCAN_RESULT_DEFINED typedef struct yypslr_scan_result { @@ -431,11 +433,12 @@ def pslr_function_declarations int length; int is_layout; int is_character_token; + int from_fallback; } yypslr_scan_result; #endif - int yy_pseudo_scan_result (int parser_state, const char *input, yypslr_scan_result *result); - int yy_pseudo_scan (int parser_state, const char *input, int *match_length); + int yy_pseudo_scan_result (int parser_state, const char *input, size_t input_len, yypslr_scan_result *result); + int yy_pseudo_scan (int parser_state, const char *input, size_t input_len, int *match_length); int yy_pslr_token_is_layout (int token); C_CODE @@ -444,13 +447,13 @@ def pslr_function_declarations #define YYPSLR_NO_MATCH YYEMPTY #ifndef YYPSLR_PSEUDO_SCAN_STATE - # define YYPSLR_PSEUDO_SCAN_STATE(ParserState, Input, MatchLength) \\ - yy_pseudo_scan ((ParserState), (Input), (MatchLength)) + # define YYPSLR_PSEUDO_SCAN_STATE(ParserState, Input, InputLen, MatchLength) \\ + yy_pseudo_scan ((ParserState), (Input), (InputLen), (MatchLength)) #endif #ifndef YYPSLR_PSEUDO_SCAN_RESULT_STATE - # define YYPSLR_PSEUDO_SCAN_RESULT_STATE(ParserState, Input, Result) \\ - yy_pseudo_scan_result ((ParserState), (Input), (Result)) + # define YYPSLR_PSEUDO_SCAN_RESULT_STATE(ParserState, Input, InputLen, Result) \\ + yy_pseudo_scan_result ((ParserState), (Input), (InputLen), (Result)) #endif #ifndef YYPSLR_TOKEN_IS_LAYOUT @@ -458,11 +461,11 @@ def pslr_function_declarations #endif int yypslr_scan_with_layout (int parser_state, const char **input, - yypslr_scan_result *result); + size_t *input_len, yypslr_scan_result *result); #ifndef YYPSLR_SCAN_WITH_LAYOUT - # define YYPSLR_SCAN_WITH_LAYOUT(ParserState, InputPtr, Result) \\ - yypslr_scan_with_layout ((ParserState), (InputPtr), (Result)) + # define YYPSLR_SCAN_WITH_LAYOUT(ParserState, InputPtr, InputLenPtr, Result) \\ + yypslr_scan_with_layout ((ParserState), (InputPtr), (InputLenPtr), (Result)) #endif C_CODE end @@ -477,16 +480,16 @@ def pslr_function_declarations if pslr_scanner_enabled? declarations << <<~C_CODE #ifndef YYPSLR_PSEUDO_SCAN - # define YYPSLR_PSEUDO_SCAN(Context, Input, MatchLength) \\ + # define YYPSLR_PSEUDO_SCAN(Context, Input, InputLen, MatchLength) \\ ((Context) != 0 \\ - ? YYPSLR_PSEUDO_SCAN_STATE (YYGETSTATE_CONTEXT (Context), (Input), (MatchLength)) \\ + ? YYPSLR_PSEUDO_SCAN_STATE (YYGETSTATE_CONTEXT (Context), (Input), (InputLen), (MatchLength)) \\ : YYEMPTY) #endif #ifndef YYPSLR_PSEUDO_SCAN_RESULT - # define YYPSLR_PSEUDO_SCAN_RESULT(Context, Input, Result) \\ + # define YYPSLR_PSEUDO_SCAN_RESULT(Context, Input, InputLen, Result) \\ ((Context) != 0 \\ - ? YYPSLR_PSEUDO_SCAN_RESULT_STATE (YYGETSTATE_CONTEXT (Context), (Input), (Result)) \\ + ? YYPSLR_PSEUDO_SCAN_RESULT_STATE (YYGETSTATE_CONTEXT (Context), (Input), (InputLen), (Result)) \\ : YYEMPTY) #endif C_CODE @@ -828,25 +831,30 @@ def pseudo_scan_function /* * pseudo_scan: PSLR(1) scanning function - * Based on Definition 3.2.16 from the PSLR dissertation + * Based on Definition 3.2.16 from the PSLR dissertation, + * extended with the fallback row of section 3.5.1. * * Input: * parser_state: Current parser state - * input: Input buffer pointer - * match_length: Output parameter for matched length + * input: Unconsumed input pointer (may contain NUL bytes) + * input_len: Number of remaining input bytes + * result: Output parameter for the match * - * Returns: Selected parser token ID. If neither the parser-state row - * nor fallback row matches, returns YYUNDEF and consumes one byte. + * Returns: Selected parser token ID. YYEOF when input_len is zero. + * If neither the parser-state row nor the fallback row matches, + * returns YYUNDEF and consumes one byte (character-token handling); + * from_fallback is set for both cases because a syntax error is + * then guaranteed. */ int - yy_pseudo_scan_result (int parser_state, const char *input, yypslr_scan_result *result) + yy_pseudo_scan_result (int parser_state, const char *input, size_t input_len, yypslr_scan_result *result) { int ss = 0; /* FSA initial state */ int ibest = 0; int pbest = YY_PSLR_EMPTY_PATTERN; int fallback_ibest = 0; int fallback_pbest = YY_PSLR_EMPTY_PATTERN; - int i = 0; + size_t i = 0; if (result == NULL) { return YYEMPTY; @@ -856,17 +864,18 @@ def pseudo_scan_function result->length = 0; result->is_layout = 0; result->is_character_token = 0; + result->from_fallback = 0; if (parser_state < 0 || parser_state >= YY_NUM_PARSER_STATES || input == NULL) { return YYEMPTY; } - if (input[0] == '\\0') { + if (input_len == 0) { result->token = YYEOF; return result->token; } - while (input[i] != '\\0') { + while (i < input_len) { int c = (unsigned char)input[i]; int next_ss = yy_scanner_transition[ss][c]; @@ -885,7 +894,7 @@ def pseudo_scan_function if (pbest == YY_PSLR_EMPTY_PATTERN || yy_pslr_length_precedes[pbest][pattern_index]) { pbest = pattern_index; - ibest = i; + ibest = (int)i; } } @@ -894,7 +903,7 @@ def pseudo_scan_function if (fallback_pbest == YY_PSLR_EMPTY_PATTERN || yy_pslr_fallback_length_precedes[fallback_pbest][pattern_index]) { fallback_pbest = pattern_index; - fallback_ibest = i; + fallback_ibest = (int)i; } } } @@ -911,20 +920,22 @@ def pseudo_scan_function result->token = yy_token_pattern_to_token_id[fallback_pbest]; result->length = fallback_ibest; result->is_layout = yy_token_pattern_is_layout[fallback_pbest]; + result->from_fallback = 1; return result->token; } result->token = YYUNDEF; result->length = 1; result->is_character_token = 1; + result->from_fallback = 1; return result->token; } int - yy_pseudo_scan(int parser_state, const char *input, int *match_length) + yy_pseudo_scan(int parser_state, const char *input, size_t input_len, int *match_length) { yypslr_scan_result result; - int token = yy_pseudo_scan_result (parser_state, input, &result); + int token = yy_pseudo_scan_result (parser_state, input, input_len, &result); if (match_length != NULL) { *match_length = result.length; @@ -1080,25 +1091,34 @@ def pslr_layout_scan_function * Layout tokens are accumulated in the layout buffer; * non-layout tokens are returned to the caller with the * accumulated layout available via YYPSLR_LAYOUT_TEXT. + * The YYEOF token action (if any) runs before returning YYEOF so + * trailing layout can be recovered (paper section 3.6). */ int yypslr_scan_with_layout (int parser_state, const char **input, - yypslr_scan_result *result) + size_t *input_len, yypslr_scan_result *result) { YYPSLR_LAYOUT_CLEAR (); for (;;) { - int token = yy_pseudo_scan_result (parser_state, *input, result); + int token = yy_pseudo_scan_result (parser_state, *input, *input_len, result); - if (token == YYEMPTY || token == YYEOF) + if (token == YYEMPTY) return token; + if (token == YYEOF) + { + #{pslr_token_actions_enabled? ? " YYPSLR_TOKEN_ACTION (token, *input, 0);" : ""} + return token; + } + if (result->is_layout) { YYPSLR_LAYOUT_APPEND (*input, result->length); #{pslr_token_actions_enabled? ? " YYPSLR_TOKEN_ACTION (token, *input, result->length);" : ""} *input += result->length; + *input_len -= (size_t)result->length; continue; } diff --git a/spec/fixtures/integration/pslr_context.l b/spec/fixtures/integration/pslr_context.l index 06cbd121d..89f639bed 100644 --- a/spec/fixtures/integration/pslr_context.l +++ b/spec/fixtures/integration/pslr_context.l @@ -22,7 +22,7 @@ ID_PATTERN [a-zA-Z_][a-zA-Z0-9_]* [>]{1,2} { int match_length = 0; - int token = YYPSLR_PSEUDO_SCAN(p, yytext, &match_length); + int token = YYPSLR_PSEUDO_SCAN(p, yytext, strlen(yytext), &match_length); if (token == YYEMPTY) { token = (yyleng == 2) ? RSHIFT : RANGLE; diff --git a/spec/fixtures/integration/pslr_fallback_precedence.l b/spec/fixtures/integration/pslr_fallback_precedence.l index e5f7031b0..0f8c6639a 100644 --- a/spec/fixtures/integration/pslr_fallback_precedence.l +++ b/spec/fixtures/integration/pslr_fallback_precedence.l @@ -55,7 +55,7 @@ token_name(int token) int token; (void)yylval; - token = YYPSLR_PSEUDO_SCAN_RESULT(p, yytext, &result); + token = YYPSLR_PSEUDO_SCAN_RESULT(p, yytext, strlen(yytext), &result); if (result.length <= 0) { result.length = 1; } diff --git a/spec/fixtures/integration/pslr_fallback_precedence.y b/spec/fixtures/integration/pslr_fallback_precedence.y index 86f708a5e..8984ac839 100644 --- a/spec/fixtures/integration/pslr_fallback_precedence.y +++ b/spec/fixtures/integration/pslr_fallback_precedence.y @@ -74,8 +74,8 @@ main(int argc, char *argv[]) if (argc == 2 && strcmp(argv[1], "__empty__") == 0) { yypslr_scan_result result; int match_length = -1; - int token = YYPSLR_PSEUDO_SCAN_RESULT(¶ms, "", &result); - int wrapper_token = YYPSLR_PSEUDO_SCAN(¶ms, "", &match_length); + int token = YYPSLR_PSEUDO_SCAN_RESULT(¶ms, "", 0, &result); + int wrapper_token = YYPSLR_PSEUDO_SCAN(¶ms, "", 0, &match_length); int ok = token == YYEOF && result.token == YYEOF && wrapper_token == YYEOF && result.length == 0 && result.is_layout == 0 && result.is_character_token == 0 && match_length == 0; diff --git a/spec/fixtures/integration/pslr_implicit_literal.l b/spec/fixtures/integration/pslr_implicit_literal.l index 31121d203..4e5b4cdf4 100644 --- a/spec/fixtures/integration/pslr_implicit_literal.l +++ b/spec/fixtures/integration/pslr_implicit_literal.l @@ -13,7 +13,7 @@ int token; (void)yylval; - token = YYPSLR_PSEUDO_SCAN_RESULT(p, yytext, &result); + token = YYPSLR_PSEUDO_SCAN_RESULT(p, yytext, strlen(yytext), &result); if (result.length <= 0) { result.length = 1; } diff --git a/spec/fixtures/integration/pslr_implicit_literal.y b/spec/fixtures/integration/pslr_implicit_literal.y index 1f9526429..903a05ad4 100644 --- a/spec/fixtures/integration/pslr_implicit_literal.y +++ b/spec/fixtures/integration/pslr_implicit_literal.y @@ -50,7 +50,7 @@ main(int argc, char *argv[]) if (argc == 2 && strcmp(argv[1], "__fallback_semi__") == 0) { yypslr_scan_result result; - int token = YYPSLR_PSEUDO_SCAN_RESULT(¶ms, ";", &result); + int token = YYPSLR_PSEUDO_SCAN_RESULT(¶ms, ";", 1, &result); int ok = token == ';' && result.token == ';' && result.length == 1 && result.is_character_token == 0; diff --git a/spec/fixtures/integration/pslr_keyword_context.l b/spec/fixtures/integration/pslr_keyword_context.l index a54322064..4214a96f7 100644 --- a/spec/fixtures/integration/pslr_keyword_context.l +++ b/spec/fixtures/integration/pslr_keyword_context.l @@ -13,7 +13,7 @@ (void)yylval; int match_length = 0; - int token = YYPSLR_PSEUDO_SCAN(p, yytext, &match_length); + int token = YYPSLR_PSEUDO_SCAN(p, yytext, strlen(yytext), &match_length); if (token == YYEMPTY) { token = ID; diff --git a/spec/fixtures/integration/pslr_layout_comment.l b/spec/fixtures/integration/pslr_layout_comment.l index 82fc8fa77..68039d05f 100644 --- a/spec/fixtures/integration/pslr_layout_comment.l +++ b/spec/fixtures/integration/pslr_layout_comment.l @@ -14,7 +14,7 @@ int token; (void)yylval; - token = YYPSLR_PSEUDO_SCAN_RESULT(p, yytext, &result); + token = YYPSLR_PSEUDO_SCAN_RESULT(p, yytext, strlen(yytext), &result); if (result.length <= 0) { result.length = 1; } diff --git a/spec/fixtures/integration/pslr_mixed_context.l b/spec/fixtures/integration/pslr_mixed_context.l index 5c75c7a31..e3ab95ede 100644 --- a/spec/fixtures/integration/pslr_mixed_context.l +++ b/spec/fixtures/integration/pslr_mixed_context.l @@ -13,7 +13,7 @@ (void)yylval; int match_length = 0; - int token = YYPSLR_PSEUDO_SCAN(p, yytext, &match_length); + int token = YYPSLR_PSEUDO_SCAN(p, yytext, strlen(yytext), &match_length); if (token == YYEMPTY) { token = ID; @@ -41,7 +41,7 @@ [>]{1,2} { int match_length = 0; - int token = YYPSLR_PSEUDO_SCAN(p, yytext, &match_length); + int token = YYPSLR_PSEUDO_SCAN(p, yytext, strlen(yytext), &match_length); if (token == YYEMPTY) { token = (yyleng == 2) ? RSHIFT : RANGLE; diff --git a/spec/fixtures/integration/pslr_shift_chain.l b/spec/fixtures/integration/pslr_shift_chain.l index 885ba62e8..0b8b32122 100644 --- a/spec/fixtures/integration/pslr_shift_chain.l +++ b/spec/fixtures/integration/pslr_shift_chain.l @@ -30,7 +30,7 @@ ID_PATTERN [a-z]+ [>]{1,2} { int match_length = 0; - int token = YYPSLR_PSEUDO_SCAN(p, yytext, &match_length); + int token = YYPSLR_PSEUDO_SCAN(p, yytext, strlen(yytext), &match_length); if (token == YYEMPTY) { token = (yyleng == 2) ? RSHIFT : RANGLE; diff --git a/spec/fixtures/integration/pslr_template_argument_lists.l b/spec/fixtures/integration/pslr_template_argument_lists.l index aedab7edf..3d0a93526 100644 --- a/spec/fixtures/integration/pslr_template_argument_lists.l +++ b/spec/fixtures/integration/pslr_template_argument_lists.l @@ -14,7 +14,7 @@ int token; (void)yylval; - token = YYPSLR_PSEUDO_SCAN_RESULT(p, yytext, &result); + token = YYPSLR_PSEUDO_SCAN_RESULT(p, yytext, strlen(yytext), &result); if (result.length <= 0) { result.length = 1; } diff --git a/spec/lrama/output_spec.rb b/spec/lrama/output_spec.rb index a88c93fa8..db1bc363f 100644 --- a/spec/lrama/output_spec.rb +++ b/spec/lrama/output_spec.rb @@ -309,6 +309,18 @@ expect(result).to include("yy_pslr_fallback_length_precedes") expect(result).to include("result->token = YYEOF") end + + it "takes a length-delimited input instead of a NUL-terminated string" do + result = pslr_output.pseudo_scan_function + expect(result).to include("size_t input_len") + expect(result).to include("while (i < input_len)") + expect(result).not_to include("input[i] != '\\0'") + end + + it "marks fallback-row and character-token matches with from_fallback" do + result = pslr_output.pseudo_scan_function + expect(result.scan("result->from_fallback = 1;").size).to eq(2) + end end describe "#pslr_tables_and_functions" do @@ -512,7 +524,7 @@ expect(rendered_header).to include("int yy_pseudo_scan") expect(rendered_header).to include("yypslr_scan_result") expect(rendered_header).to include("YYSETSTATE_CONTEXT(CurrentState)") - expect(rendered_header).to include("YYPSLR_PSEUDO_SCAN(Context, Input, MatchLength)") + expect(rendered_header).to include("YYPSLR_PSEUDO_SCAN(Context, Input, InputLen, MatchLength)") end end end From aaca9fa48af2791460ee292e27a1740312360000 Mon Sep 17 00:00:00 2001 From: ydah Date: Wed, 8 Jul 2026 22:00:00 +0900 Subject: [PATCH 08/15] pslr: add conflict witnesses and real %lex-prec usage tracking Unresolved-scanner-conflict reports now carry a witness: an example input that drives the DFA to the conflicting profile. The witness is collected during the profile DFS and included in the error message so users can reproduce the conflict without reverse-engineering scanner state ids. Useless-%lex-prec detection previously relied on usage marks that nothing recorded, so it could not distinguish referenced rules from dead ones. The profile resolver now marks the identity and length rules that actually decide a normal-row resolution (winner selection in both the shorter-token-wins and longer-token-wins cases). The fallback row intentionally does not mark: it spans the whole token universe and would make every declaration look used. Compatibility probes during state splitting do not mark either. --- lib/lrama/state/scanner_accepts.rb | 74 ++++++++++++++++++------ lib/lrama/states.rb | 3 +- spec/lrama/state/scanner_accepts_spec.rb | 48 +++++++++++++++ 3 files changed, 106 insertions(+), 19 deletions(-) diff --git a/lib/lrama/state/scanner_accepts.rb b/lib/lrama/state/scanner_accepts.rb index bb764043c..9d7f54671 100644 --- a/lib/lrama/state/scanner_accepts.rb +++ b/lib/lrama/state/scanner_accepts.rb @@ -31,14 +31,16 @@ class Conflict attr_reader :shorter_tokens #: Array[String] attr_reader :selected_shorter_token #: String? attr_reader :current_tokens #: Array[String] + attr_reader :witness #: String? - # @rbs (parser_state_id: Integer?, scanner_state_id: Integer, shorter_tokens: Array[String], selected_shorter_token: String?, current_tokens: Array[String]) -> void - def initialize(parser_state_id:, scanner_state_id:, shorter_tokens:, selected_shorter_token:, current_tokens:) + # @rbs (parser_state_id: Integer?, scanner_state_id: Integer, shorter_tokens: Array[String], selected_shorter_token: String?, current_tokens: Array[String], ?witness: String?) -> void + def initialize(parser_state_id:, scanner_state_id:, shorter_tokens:, selected_shorter_token:, current_tokens:, witness: nil) @parser_state_id = parser_state_id @scanner_state_id = scanner_state_id @shorter_tokens = shorter_tokens @selected_shorter_token = selected_shorter_token @current_tokens = current_tokens + @witness = witness end end @@ -75,12 +77,13 @@ def unresolved? end class ProfileResolver - # @rbs (Grammar::LexPrec lex_prec, LengthPrecedences length_prec, ?fallback: bool, ?token_order: Hash[String, Integer]) -> void - def initialize(lex_prec, length_prec, fallback: false, token_order: {}) + # @rbs (Grammar::LexPrec lex_prec, LengthPrecedences length_prec, ?fallback: bool, ?token_order: Hash[String, Integer], ?track_rule_usage: bool) -> void + def initialize(lex_prec, length_prec, fallback: false, token_order: {}, track_rule_usage: false) @lex_prec = lex_prec @length_prec = length_prec @fallback = fallback @token_order = token_order + @track_rule_usage = track_rule_usage end # @rbs (Set[String] shorter_tokens, String? selected_shorter_token, Set[String] current_tokens) -> ProfileOutcome @@ -103,6 +106,7 @@ def resolve_normal(shorter_tokens, selected_shorter_token, current_tokens) end if selected_shorter_token && current_tokens.all? {|token| length_prefers_old?(selected_shorter_token, token) } + mark_shorter_win_rules_used(selected_shorter_token, current_tokens) return ProfileOutcome.new(kind: ProfileOutcome::RESOLVED, token_name: selected_shorter_token) end @@ -111,7 +115,11 @@ def resolve_normal(shorter_tokens, selected_shorter_token, current_tokens) shorter_tokens.all? {|shorter| @length_prec.resolution(shorter, candidate) == LengthPrecedences::PREFER_NEW } end - return ProfileOutcome.new(kind: ProfileOutcome::RESOLVED, token_name: winners.first) if winners.size == 1 + if winners.size == 1 + winner = winners.first #: String + mark_longer_win_rules_used(winner, shorter_tokens, current_tokens) + return ProfileOutcome.new(kind: ProfileOutcome::RESOLVED, token_name: winner) + end ProfileOutcome.new(kind: ProfileOutcome::UNRESOLVED) end @@ -179,6 +187,33 @@ def fallback_identity_winner?(candidate, current_tokens) end end + # Mark the %lex-prec rules that made the profile decision as used. + # Only normal-row scanner_accepts construction tracks usage: + # the fallback row spans the whole token universe and exploratory + # resolutions (state compatibility probes) must not mark rules. + # @rbs (String selected_shorter_token, Set[String] current_tokens) -> void + def mark_shorter_win_rules_used(selected_shorter_token, current_tokens) + return unless @track_rule_usage + + current_tokens.each do |token| + @length_prec.resolution(selected_shorter_token, token, track: true) + end + end + + # @rbs (String winner, Set[String] shorter_tokens, Set[String] current_tokens) -> void + def mark_longer_win_rules_used(winner, shorter_tokens, current_tokens) + return unless @track_rule_usage + + current_tokens.each do |other| + next if other == winner + + @lex_prec.identity_precedes?(winner, other, track: true) + end + shorter_tokens.each do |shorter| + @length_prec.resolution(shorter, winner, track: true) + end + end + # @rbs (String candidate, String other) -> bool def token_order_precedes?(candidate, other) (token_order_key(candidate) <=> token_order_key(other)) == -1 @@ -195,9 +230,9 @@ class CompleteProfileComputer attr_reader :conflicts #: Array[Conflict] # @rbs (ScannerFSA scanner_fsa, Grammar::LexPrec lex_prec, LengthPrecedences length_prec, Set[String] acceptable_tokens, ?Integer? parser_state_id, ?fallback: bool, ?token_order: Hash[String, Integer]) -> void - def initialize(scanner_fsa, lex_prec, length_prec, acceptable_tokens, parser_state_id = nil, fallback: false, token_order: {}) + def initialize(scanner_fsa, lex_prec, length_prec, acceptable_tokens, parser_state_id = nil, fallback: false, token_order: {}, track_rule_usage: false) @scanner_fsa = scanner_fsa - @resolver = ProfileResolver.new(lex_prec, length_prec, fallback: fallback, token_order: token_order) + @resolver = ProfileResolver.new(lex_prec, length_prec, fallback: fallback, token_order: token_order, track_rule_usage: track_rule_usage) @acceptable_tokens = acceptable_tokens @parser_state_id = parser_state_id @table = {} @@ -211,23 +246,23 @@ def initialize(scanner_fsa, lex_prec, length_prec, acceptable_tokens, parser_sta # @rbs () -> void def compute visited = Set.new - visit_transitions(0, Set.new, nil, visited) + visit_transitions(0, Set.new, nil, visited, "") end private - # @rbs (Integer fsa_state_id, Set[String] shorter_tokens, String? selected_shorter_token, Set[untyped] visited) -> void - def visit_transitions(fsa_state_id, shorter_tokens, selected_shorter_token, visited) + # @rbs (Integer fsa_state_id, Set[String] shorter_tokens, String? selected_shorter_token, Set[untyped] visited, String path) -> void + def visit_transitions(fsa_state_id, shorter_tokens, selected_shorter_token, visited, path) fsa_state = @scanner_fsa.states[fsa_state_id] return unless fsa_state - fsa_state.transitions.each_value do |next_state_id| - visit_state(next_state_id, shorter_tokens, selected_shorter_token, visited) + fsa_state.transitions.each do |char, next_state_id| + visit_state(next_state_id, shorter_tokens, selected_shorter_token, visited, path + char) end end - # @rbs (Integer fsa_state_id, Set[String] shorter_tokens, String? selected_shorter_token, Set[untyped] visited) -> void - def visit_state(fsa_state_id, shorter_tokens, selected_shorter_token, visited) + # @rbs (Integer fsa_state_id, Set[String] shorter_tokens, String? selected_shorter_token, Set[untyped] visited, String path) -> void + def visit_state(fsa_state_id, shorter_tokens, selected_shorter_token, visited, path) fsa_state = @scanner_fsa.states[fsa_state_id] return unless fsa_state @@ -248,7 +283,8 @@ def visit_state(fsa_state_id, shorter_tokens, selected_shorter_token, visited) scanner_state_id: fsa_state_id, shorter_tokens: shorter_tokens.to_a.sort, selected_shorter_token: existing.name, - current_tokens: current_tokens.to_a.sort + current_tokens: current_tokens.to_a.sort, + witness: path ) else @table[fsa_state_id] = token_pattern @@ -260,13 +296,14 @@ def visit_state(fsa_state_id, shorter_tokens, selected_shorter_token, visited) scanner_state_id: fsa_state_id, shorter_tokens: shorter_tokens.to_a.sort, selected_shorter_token: selected_shorter_token, - current_tokens: current_tokens.to_a.sort + current_tokens: current_tokens.to_a.sort, + witness: path ) end next_shorter_tokens = shorter_tokens | current_tokens next_selected = result.resolved? ? result.token_name : nil - visit_transitions(fsa_state_id, next_shorter_tokens, next_selected, visited) + visit_transitions(fsa_state_id, next_shorter_tokens, next_selected, visited, path) end # @rbs (ScannerFSA::State fsa_state) -> Set[String] @@ -420,7 +457,8 @@ def compute_for_parser_state(parser_state) @lex_prec, @length_prec, compute_acc_sp(parser_state), - parser_state.id + parser_state.id, + track_rule_usage: true ) computer.compute diff --git a/lib/lrama/states.rb b/lib/lrama/states.rb index af34aefa4..2d09bd5a8 100644 --- a/lib/lrama/states.rb +++ b/lib/lrama/states.rb @@ -1394,8 +1394,9 @@ def pslr_scanner_conflict_message(conflict) shorter = conflict.shorter_tokens.empty? ? "(none)" : conflict.shorter_tokens.join(", ") selected = conflict.selected_shorter_token || "(none)" current = conflict.current_tokens.empty? ? "(none)" : conflict.current_tokens.join(", ") + witness = conflict.witness ? " witness input: #{conflict.witness.inspect};" : "" - "unresolved PSLR scanner conflict in state #{state}, scanner state #{conflict.scanner_state_id}: " \ + "unresolved PSLR scanner conflict in state #{state}, scanner state #{conflict.scanner_state_id}:#{witness} " \ "shorter matches: #{shorter}; selected shorter token: #{selected}; " \ "current matches: #{current}; add an explicit %lex-prec rule or adjust lexical ties" end diff --git a/spec/lrama/state/scanner_accepts_spec.rb b/spec/lrama/state/scanner_accepts_spec.rb index 56a0bbcf8..2caa28aba 100644 --- a/spec/lrama/state/scanner_accepts_spec.rb +++ b/spec/lrama/state/scanner_accepts_spec.rb @@ -273,6 +273,54 @@ def parser_state(id, token_names) end end + describe "conflict witnesses" do + it "attaches an example input to unresolved conflicts" do + tokens = [ + token_pattern("A", "ab", 0), + token_pattern("B", "ab", 1) + ] + scanner_fsa = Lrama::ScannerFSA.new(tokens) + lex_prec = Lrama::Grammar::LexPrec.new + scanner_accepts = Lrama::State::ScannerAccepts.new( + [parser_state(0, ["A", "B"])], + scanner_fsa, + lex_prec, + Lrama::LengthPrecedences.new(lex_prec) + ) + + scanner_accepts.build + + witnesses = scanner_accepts.conflicts.map(&:witness) + expect(witnesses).to include("ab") + end + end + + describe "%lex-prec usage tracking" do + it "marks rules referenced by normal-row resolution and keeps unreferenced ones useless" do + tokens = [ + token_pattern("A", "a", 0), + token_pattern("B", "a", 1), + token_pattern("X", "x", 2), + token_pattern("Y", "y", 3) + ] + scanner_fsa = Lrama::ScannerFSA.new(tokens) + lex_prec = Lrama::Grammar::LexPrec.new + lex_prec.add_rule(left_token: ident("A"), operator: Lrama::Grammar::LexPrec::IDENTITY_RIGHT, right_token: ident("B"), lineno: 1) + lex_prec.add_rule(left_token: ident("X"), operator: Lrama::Grammar::LexPrec::IDENTITY_RIGHT, right_token: ident("Y"), lineno: 2) + scanner_accepts = Lrama::State::ScannerAccepts.new( + [parser_state(0, ["A", "B", "X", "Y"])], + scanner_fsa, + lex_prec, + Lrama::LengthPrecedences.new(lex_prec) + ) + + scanner_accepts.build + + useless = lex_prec.useless_rules + expect(useless.map {|rule| [rule.left_name, rule.right_name] }).to eq([["X", "Y"]]) + end + end + describe Lrama::State::ScannerAccepts::CompatibilityChecker do let(:rangle) { token_pattern("RANGLE", ">", 0) } let(:rshift) { token_pattern("RSHIFT", ">>", 1) } From 701479bbac952efc2d3cf71e0552f20cd7130478 Mon Sep 17 00:00:00 2001 From: ydah Date: Wed, 8 Jul 2026 22:06:36 +0900 Subject: [PATCH 09/15] lac: make lookahead correction configurable via %define parse.lac LAC is orthogonal to PSLR (paper section 3.5.2), so expose it as %define parse.lac with the Bison-compatible values full and none: * PSLR parsers default to full; exploratory parses are what keep pseudo-scanning on merged states equivalent to canonical LR(1) * LALR/IELR parsers can now opt in with parse.lac full to get immediate error detection and accurate expected-token lists * parse.lac none on a PSLR parser is honored but warned about, because token selections from merged states may then run semantic actions before the syntax error is reported * invalid values are rejected at grammar validation time All LAC template blocks (checker function, lookahead verification, eager lookahead fetch at default reductions, expected-token enumeration) are now gated on the LAC setting instead of lr.type. --- lib/lrama/grammar.rb | 33 +++++++++++++ lib/lrama/output.rb | 9 +++- lib/lrama/warnings.rb | 3 ++ lib/lrama/warnings/parse_lac.rb | 26 ++++++++++ spec/lrama/output_spec.rb | 88 +++++++++++++++++++++++++++++++++ spec/lrama/warnings_spec.rb | 27 ++++++++++ template/bison/yacc.c | 8 +-- 7 files changed, 189 insertions(+), 5 deletions(-) create mode 100644 lib/lrama/warnings/parse_lac.rb diff --git a/lib/lrama/grammar.rb b/lib/lrama/grammar.rb index 246c99e04..2764c41b3 100644 --- a/lib/lrama/grammar.rb +++ b/lib/lrama/grammar.rb @@ -360,6 +360,28 @@ def pslr_max_state_ratio parse_pslr_positive_float('pslr.max-state-ratio') end + # LAC (lookahead correction) setting from %define parse.lac. + # PSLR parsers default to full because exploratory parses are what + # keep merged-state pseudo-scanning canonical-equivalent; other + # parser types default to none for Bison compatibility. + # @rbs () -> String + def parse_lac + value = @define['parse.lac'] + return (pslr_defined? ? 'full' : 'none') if value.nil? || value.empty? + + value + end + + # @rbs () -> bool + def parse_lac_full? + parse_lac == 'full' + end + + # @rbs () -> bool + def parse_lac_explicit_none? + @define['parse.lac'] == 'none' + end + # @rbs () -> Array[Grammar::TokenPattern] def layout_token_patterns @token_patterns.select(&:layout?) @@ -636,6 +658,8 @@ def token_lineno(id) # @rbs () -> void def validate_pslr_configuration! + validate_parse_lac! + return unless pslr_defined? member = pslr_state_member @@ -647,6 +671,15 @@ def validate_pslr_configuration! pslr_max_state_ratio end + # @rbs () -> void + def validate_parse_lac! + value = @define['parse.lac'] + return if value.nil? || value.empty? + return if %w[full none].include?(value) + + raise %(%define parse.lac must be "full" or "none", got "#{value}".) + end + # @rbs (Lexer::Token::Base operand) -> Array[Lexer::Token::Base] def expand_pslr_operand(operand) return @token_patterns.map(&:id) if operand.s_value == "yyall" diff --git a/lib/lrama/output.rb b/lib/lrama/output.rb index 52aaf4e17..f43c3f469 100644 --- a/lib/lrama/output.rb +++ b/lib/lrama/output.rb @@ -409,6 +409,13 @@ def pslr_enabled? @grammar.pslr_defined? end + # Check if LAC (lookahead correction) is enabled. + # Defaults to full for PSLR parsers; any parser type can opt in + # with %define parse.lac full (section 3.5.2 is orthogonal to PSLR). + def lac_enabled? + @grammar.parse_lac_full? + end + # Check if PSLR scanner tables are available. def pslr_scanner_enabled? scanner_fsa = @context.states.scanner_fsa @@ -735,7 +742,7 @@ def length_precedence_matrix_code(table_name, num_tokens, query_method) end def pslr_lac_function - return "" unless pslr_enabled? + return "" unless lac_enabled? <<~C_CODE diff --git a/lib/lrama/warnings.rb b/lib/lrama/warnings.rb index b014f85f7..d26cdff9c 100644 --- a/lib/lrama/warnings.rb +++ b/lib/lrama/warnings.rb @@ -8,6 +8,7 @@ require_relative 'warnings/required' require_relative 'warnings/useless_precedence' require_relative 'warnings/lexical_tie_candidates' +require_relative 'warnings/parse_lac' module Lrama class Warnings @@ -20,6 +21,7 @@ def initialize(logger, warnings) @required = Required.new(logger, warnings) @useless_precedence = UselessPrecedence.new(logger, warnings) @lexical_tie_candidates = LexicalTieCandidates.new(logger, warnings) + @parse_lac = ParseLac.new(logger, warnings) end # @rbs (Lrama::Grammar grammar, Lrama::States states) -> void @@ -31,6 +33,7 @@ def warn(grammar, states) @required.warn(grammar) @useless_precedence.warn(grammar, states) @lexical_tie_candidates.warn(states) + @parse_lac.warn(grammar) end end end diff --git a/lib/lrama/warnings/parse_lac.rb b/lib/lrama/warnings/parse_lac.rb new file mode 100644 index 000000000..aec57ac88 --- /dev/null +++ b/lib/lrama/warnings/parse_lac.rb @@ -0,0 +1,26 @@ +# rbs_inline: enabled +# frozen_string_literal: true + +module Lrama + class Warnings + class ParseLac + # @rbs (Logger logger, bool warnings) -> void + def initialize(logger, warnings) + @logger = logger + @warnings = warnings + end + + # @rbs (Lrama::Grammar grammar) -> void + def warn(grammar) + return unless @warnings + return unless grammar.pslr_defined? + return unless grammar.parse_lac_explicit_none? + + @logger.warn( + "parse.lac is disabled for a PSLR parser; " \ + "tokens selected in merged states may trigger semantic actions before a syntax error is detected" + ) + end + end + end +end diff --git a/spec/lrama/output_spec.rb b/spec/lrama/output_spec.rb index db1bc363f..0bbd52c0e 100644 --- a/spec/lrama/output_spec.rb +++ b/spec/lrama/output_spec.rb @@ -247,6 +247,7 @@ lex_param: "struct parse_params *p", pslr_defined?: true, pslr_state_member: "current_state", + parse_lac_full?: true, token_actions: [] ) end @@ -527,4 +528,91 @@ expect(rendered_header).to include("YYPSLR_PSEUDO_SCAN(Context, Input, InputLen, MatchLength)") end end + + describe "LAC (%define parse.lac)" do + def render_grammar(text) + grammar = Lrama::Parser.new(text, "lac_test.y").parse + grammar.prepare + grammar.validate! + states = Lrama::States.new(grammar, Lrama::Tracer.new(Lrama::Logger.new)) + states.compute + if grammar.pslr_defined? + states.compute_pslr + elsif grammar.ielr_defined? + states.compute_ielr + end + out = StringIO.new + Lrama::Output.new( + out: out, + output_file_path: "lac_test.c", + template_name: "bison/yacc.c", + grammar_file_path: "lac_test.y", + context: Lrama::Context.new(states), + grammar: grammar + ).render + out.rewind + out.read + end + + it "is enabled by default for PSLR parsers" do + rendered = render_grammar(<<~GRAMMAR) + %define lr.type pslr + %token-pattern RSHIFT />>/ + %token-pattern RANGLE />/ + %lex-prec RANGLE -s RSHIFT + %% + program: RSHIFT | RANGLE + GRAMMAR + + expect(rendered).to include("yy_lac_check_") + end + + it "can be disabled with parse.lac none" do + rendered = render_grammar(<<~GRAMMAR) + %define lr.type pslr + %define parse.lac none + %token-pattern RSHIFT />>/ + %token-pattern RANGLE />/ + %lex-prec RANGLE -s RSHIFT + %% + program: RSHIFT | RANGLE + GRAMMAR + + expect(rendered).not_to include("yy_lac_check_") + end + + it "can be enabled independently of PSLR with parse.lac full" do + rendered = render_grammar(<<~GRAMMAR) + %define lr.type ielr + %define parse.lac full + %token NUM + %% + program: NUM + GRAMMAR + + expect(rendered).to include("yy_lac_check_") + end + + it "is disabled by default for non-PSLR parsers" do + rendered = render_grammar(<<~GRAMMAR) + %token NUM + %% + program: NUM + GRAMMAR + + expect(rendered).not_to include("yy_lac_check_") + end + + it "rejects invalid parse.lac values" do + grammar = Lrama::Parser.new(<<~GRAMMAR, "lac_test.y").parse + %define parse.lac sometimes + %token NUM + %% + program: NUM + GRAMMAR + grammar.prepare + + expect { grammar.validate! }.to raise_error(/parse\.lac must be "full" or "none"/) + end + end end diff --git a/spec/lrama/warnings_spec.rb b/spec/lrama/warnings_spec.rb index 3a4492a3b..e434a7c90 100644 --- a/spec/lrama/warnings_spec.rb +++ b/spec/lrama/warnings_spec.rb @@ -116,5 +116,32 @@ class : keyword_class tSTRING keyword_end %prec tPLUS expect(logger).to have_received(:warn).with("parameterized rule redefined: foo(Y)") end end + + context "when a PSLR grammar disables LAC explicitly" do + let(:y) do + <<~STR + %define lr.type pslr + %define parse.lac none + %token-pattern RSHIFT />>/ + %token-pattern RANGLE />/ + %lex-prec RANGLE -s RSHIFT + %% + program: RSHIFT | RANGLE + STR + end + + it "warns that error detection may be delayed" do + grammar = Lrama::Parser.new(y, "states/pslr_lac_none.y").parse + grammar.prepare + grammar.validate! + states = Lrama::States.new(grammar, Lrama::Tracer.new(Lrama::Logger.new)) + states.compute + states.compute_pslr + logger = Lrama::Logger.new + allow(logger).to receive(:warn) + Lrama::Warnings.new(logger, true).warn(grammar, states) + expect(logger).to have_received(:warn).with(a_string_including("parse.lac is disabled for a PSLR parser")) + end + end end end diff --git a/template/bison/yacc.c b/template/bison/yacc.c index a5c195382..03aad98da 100644 --- a/template/bison/yacc.c +++ b/template/bison/yacc.c @@ -1062,7 +1062,7 @@ int yydebug; # define YYMAXDEPTH 10000 #endif -<%- if output.pslr_enabled? -%> +<%- if output.lac_enabled? -%> <%= output.pslr_lac_function %> <%- end -%> @@ -1087,7 +1087,7 @@ yypcontext_expected_tokens (const yypcontext_t *yyctx, { /* Actual size of YYARG. */ int yycount = 0; -<%- if output.pslr_enabled? -%> +<%- if output.lac_enabled? -%> int yyx; for (yyx = 0; yyx < YYNTOKENS; ++yyx) if (yyx != YYSYMBOL_YYerror @@ -1897,7 +1897,7 @@ YYLTYPE yylloc = yyloc_default; /* First try to decide what to do without reference to lookahead token. */ yyn = yypact[yystate]; -<%- if output.pslr_enabled? -%> +<%- if output.lac_enabled? -%> if (yypact_value_is_default (yyn) && yydefact[yystate] != 0) { if (yychar == YYEMPTY) @@ -1999,7 +1999,7 @@ YYLTYPE yylloc = yyloc_default; YY_SYMBOL_PRINT ("Next token is", yytoken, &yylval, &yylloc<%= output.user_args %>); } -<%- if output.pslr_enabled? -%> +<%- if output.lac_enabled? -%> if (!yy_lac_check_ (yyss, yyssp, yytoken)) goto yyerrlab; <%- end -%> From 25402b038680e150a6fca68cba205fef44763c72 Mon Sep 17 00:00:00 2001 From: ydah Date: Wed, 8 Jul 2026 22:09:22 +0900 Subject: [PATCH 10/15] pslr: use pair-based state compatibility for splitting Phase 3 previously re-ran the dual-profile resolver walk over the whole scanner FSA for every candidate merge, which is the cost the paper's section 3.4.3 warns about. Precompute the FSA's conflict pairs once and reduce the compatibility test to a presence check per pair: two accept sets are compatible when every conflict pair is present identically in both, or absent from one side entirely (unresolved/unresolved pairs merge because their presence patterns match). Only pairs touching the symmetric difference of the two accept sets are examined, so merges that differ in conflict-free tokens are accepted in O(diff). Layout tokens live in every accept set and thus never distinguish states, which realizes the split-stable layout optimization of section 3.6 without extra bookkeeping. --- lib/lrama/pslr/pairwise_resolution.rb | 75 +++++++++++++++++++++ lib/lrama/states.rb | 11 ++- spec/lrama/pslr/pairwise_resolution_spec.rb | 49 ++++++++++++++ 3 files changed, 129 insertions(+), 6 deletions(-) create mode 100644 lib/lrama/pslr/pairwise_resolution.rb create mode 100644 spec/lrama/pslr/pairwise_resolution_spec.rb diff --git a/lib/lrama/pslr/pairwise_resolution.rb b/lib/lrama/pslr/pairwise_resolution.rb new file mode 100644 index 000000000..c7102ff96 --- /dev/null +++ b/lib/lrama/pslr/pairwise_resolution.rb @@ -0,0 +1,75 @@ +# rbs_inline: enabled +# frozen_string_literal: true + +require "set" + +module Lrama + module Pslr + # Pair-based PSLR state compatibility (paper section 3.4.3). + # + # The scanner FSA is scanned once to find every token pair that can be + # in an identity or length scanner conflict. Two acceptable-token sets + # are then compatible when no conflict pair distinguishes them: + # + # * both sets contain the same subset of the pair (the pseudo-scanner + # resolves the conflict identically, or leaves it identically + # unresolved -- unresolved/unresolved states merge), or + # * one set contains neither token of the pair (its match set for + # inputs manifesting the conflict is empty for this pair). + # + # Reducing the check to pairs is the precomputation the paper + # recommends for phase 3: the per-pair verdict no longer depends on + # the rest of the accept set, so state splitting does not have to + # re-run the profile resolver for every candidate merge. Layout + # tokens are members of every accept set, so pairs won by layout can + # never distinguish two states (the split-stable layout optimization + # of section 3.6 falls out of the presence test). + class PairwiseResolution + attr_reader :conflict_pairs #: Set[[String, String]] + + # @rbs (ScannerFSA scanner_fsa) -> void + def initialize(scanner_fsa) + @conflict_pairs = scanner_fsa.pairwise_conflict_pairs + pairs_by_token = Hash.new {|h, k| h[k] = [] } #: Hash[String, Array[[String, String]]] + @pairs_by_token = @conflict_pairs.each_with_object(pairs_by_token) do |pair, hash| + hash[pair[0]] << pair + hash[pair[1]] << pair + end + end + + # @rbs (Set[String] left_acc, Set[String] right_acc) -> bool + def compatible_accept_sets?(left_acc, right_acc) + return true if left_acc == right_acc + + # Only conflict pairs touching the symmetric difference can + # distinguish the two accept sets. + diff = (left_acc - right_acc) | (right_acc - left_acc) + return true if diff.empty? + + checked = Set.new #: Set[[String, String]] + diff.all? do |token| + @pairs_by_token[token].all? do |pair| + next true unless checked.add?(pair) + + pair_compatible?(pair, left_acc, right_acc) + end + end + end + + private + + # @rbs ([String, String] pair, Set[String] left_acc, Set[String] right_acc) -> bool + def pair_compatible?(pair, left_acc, right_acc) + left_presence = presence(pair, left_acc) + right_presence = presence(pair, right_acc) + + left_presence == right_presence || left_presence == 0 || right_presence == 0 + end + + # @rbs ([String, String] pair, Set[String] acc) -> Integer + def presence(pair, acc) + (acc.include?(pair[0]) ? 1 : 0) | (acc.include?(pair[1]) ? 2 : 0) + end + end + end +end diff --git a/lib/lrama/states.rb b/lib/lrama/states.rb index 2d09bd5a8..4f10c7522 100644 --- a/lib/lrama/states.rb +++ b/lib/lrama/states.rb @@ -4,6 +4,7 @@ require "forwardable" require "set" require_relative "lexer_context_classifier" +require_relative "pslr/pairwise_resolution" require_relative "tracer/duration" require_relative "state/item" @@ -1122,16 +1123,14 @@ def pslr_state_signature(state, filtered_lookaheads = nil) signature end + # Pair-based PSLR compatibility (Def 3.4.3, reduced to token pairs as + # recommended by section 3.4.3 for splitting performance). # @rbs (Set[String] left_acc, Set[String] right_acc) -> bool def pslr_compatible_accept_sets?(left_acc, right_acc) return true unless @scanner_fsa - @pslr_compatibility_checker ||= State::ScannerAccepts::CompatibilityChecker.new( - @scanner_fsa, - lex_prec, - @length_precedences || LengthPrecedences.new(lex_prec) - ) - @pslr_compatibility_checker.compatible?(left_acc, right_acc) + @pslr_pairwise_resolution ||= Pslr::PairwiseResolution.new(@scanner_fsa) + @pslr_pairwise_resolution.compatible_accept_sets?(left_acc, right_acc) end # @rbs (State state, ?State::lookahead_set filtered_lookaheads, ?expand_ties: bool, ?include_layout: bool) -> Set[String] diff --git a/spec/lrama/pslr/pairwise_resolution_spec.rb b/spec/lrama/pslr/pairwise_resolution_spec.rb new file mode 100644 index 000000000..7ce6f3c8a --- /dev/null +++ b/spec/lrama/pslr/pairwise_resolution_spec.rb @@ -0,0 +1,49 @@ +# frozen_string_literal: true + +RSpec.describe Lrama::Pslr::PairwiseResolution do + def token_pattern(name, regex, order) + Lrama::Grammar::TokenPattern.new( + id: Lrama::Lexer::Token::Ident.new(s_value: name), + pattern: Lrama::Lexer::Token::Regex.new(s_value: "/#{regex}/"), + lineno: 1, + definition_order: order + ) + end + + let(:rangle) { token_pattern("RANGLE", ">", 0) } + let(:rshift) { token_pattern("RSHIFT", ">>", 1) } + let(:identifier) { token_pattern("ID", "[a-z]+", 2) } + let(:keyword) { token_pattern("IF", "if", 3) } + let(:layout) { token_pattern("YYLAYOUT_WS", "[ ]+", 4) } + let(:scanner_fsa) { Lrama::ScannerFSA.new([rangle, rshift, identifier, keyword, layout]) } + let(:pairwise) { described_class.new(scanner_fsa) } + + it "treats identical accept sets as compatible" do + acc = Set["RANGLE", "RSHIFT"] + expect(pairwise.compatible_accept_sets?(acc, acc.dup)).to be true + end + + it "splits states that pick different tokens of a conflict pair" do + expect(pairwise.compatible_accept_sets?(Set["RANGLE"], Set["RSHIFT"])).to be false + end + + it "splits a resolved state from a state seeing the full conflict" do + expect(pairwise.compatible_accept_sets?(Set["ID"], Set["ID", "IF"])).to be false + end + + it "treats states as compatible when one side has no match for the pair" do + expect(pairwise.compatible_accept_sets?(Set["RANGLE"], Set["ID"])).to be true + end + + it "ignores tokens without scanner conflicts" do + expect(pairwise.compatible_accept_sets?(Set["RANGLE", "ID"], Set["RANGLE"])).to be true + end + + it "never splits on layout tokens because they are in every accept set" do + left = Set["RANGLE", "YYLAYOUT_WS"] + right = Set["RANGLE", "RSHIFT", "YYLAYOUT_WS"] + + expect(pairwise.compatible_accept_sets?(left, right)).to be false + expect(pairwise.compatible_accept_sets?(Set["ID", "YYLAYOUT_WS"], Set["YYLAYOUT_WS", "ID"])).to be true + end +end From a5158b30e9774361adc7523151f1833e7f4744cf Mon Sep 17 00:00:00 2001 From: ydah Date: Wed, 8 Jul 2026 22:17:41 +0900 Subject: [PATCH 11/15] pslr: add pure mode where the generated parser owns lexing %define api.pslr.lexer generated emits a complete lexer: yylex is generated, drives yypslr_scan_with_layout with the parser state kept current by yysetstate, skips layout tokens, runs %token-action bodies and advances an internal input cursor. Callers hand the input to the parser with yypslr_set_input(input, len) before yyparse; no hand-written lexer is involved, matching the paper's model. %token-action bodies now receive the semantic value slot: yylval inside an action refers to the YYSTYPE the caller passed, both in pure mode and through the bridge's YYPSLR_SCAN_WITH_LAYOUT macro (whose signature gains the value pointer). The dispatch is also emitted before the scan loop that calls it, so token actions compile. Pure mode requires full coverage: generation aborts when a terminal has no %token-pattern. Bridge mode instead warns with the list of uncovered terminals the user lexer must keep producing. --- lib/lrama/grammar.rb | 37 +++++++++ lib/lrama/output.rb | 104 +++++++++++++++++++++++--- lib/lrama/states.rb | 18 +++++ lib/lrama/warnings.rb | 3 + lib/lrama/warnings/pslr_coverage.rb | 34 +++++++++ spec/fixtures/integration/pslr_pure.y | 63 ++++++++++++++++ spec/lrama/integration_spec.rb | 60 +++++++++++++++ spec/lrama/output_spec.rb | 1 + template/bison/yacc.c | 4 + 9 files changed, 312 insertions(+), 12 deletions(-) create mode 100644 lib/lrama/warnings/pslr_coverage.rb create mode 100644 spec/fixtures/integration/pslr_pure.y diff --git a/lib/lrama/grammar.rb b/lib/lrama/grammar.rb index 2764c41b3..18d94a1f6 100644 --- a/lib/lrama/grammar.rb +++ b/lib/lrama/grammar.rb @@ -360,6 +360,28 @@ def pslr_max_state_ratio parse_pslr_positive_float('pslr.max-state-ratio') end + # True when the generated parser owns lexical analysis completely + # (%define api.pslr.lexer generated, the paper's own model). + # @rbs () -> bool + def pslr_lexer_generated? + @define['api.pslr.lexer'] == 'generated' + end + + # Terminals that the pseudo-scanner cannot produce: real terminals + # used by the grammar that have no %token-pattern (explicit or + # synthesized). Pure mode requires this list to be empty; bridge + # mode reports it so users know which tokens their yylex must keep + # producing. + # @rbs () -> Array[String] + def uncovered_pslr_terminals + covered = @token_patterns.map(&:name).to_set + + terms.reject do |term| + term == eof_symbol || term == error_symbol || term == undef_symbol || + covered.include?(term.id.s_value) + end.map {|term| term.id.s_value } + end + # LAC (lookahead correction) setting from %define parse.lac. # PSLR parsers default to full because exploratory parses are what # keep merged-state pseudo-scanning canonical-equivalent; other @@ -659,6 +681,7 @@ def token_lineno(id) # @rbs () -> void def validate_pslr_configuration! validate_parse_lac! + validate_pslr_lexer! return unless pslr_defined? @@ -680,6 +703,20 @@ def validate_parse_lac! raise %(%define parse.lac must be "full" or "none", got "#{value}".) end + # @rbs () -> void + def validate_pslr_lexer! + value = @define['api.pslr.lexer'] + return if value.nil? || value.empty? + + unless value == 'generated' + raise %(%define api.pslr.lexer must be "generated", got "#{value}".) + end + + unless pslr_defined? + raise "%define api.pslr.lexer generated requires %define lr.type pslr." + end + end + # @rbs (Lexer::Token::Base operand) -> Array[Lexer::Token::Base] def expand_pslr_operand(operand) return @token_patterns.map(&:id) if operand.s_value == "yyall" diff --git a/lib/lrama/output.rb b/lib/lrama/output.rb index f43c3f469..4dd8b4a9a 100644 --- a/lib/lrama/output.rb +++ b/lib/lrama/output.rb @@ -416,6 +416,12 @@ def lac_enabled? @grammar.parse_lac_full? end + # Pure mode: the generated parser owns lexical analysis + # (%define api.pslr.lexer generated). + def pslr_pure_mode? + pslr_enabled? && @grammar.pslr_lexer_generated? && pslr_scanner_enabled? + end + # Check if PSLR scanner tables are available. def pslr_scanner_enabled? scanner_fsa = @context.states.scanner_fsa @@ -468,15 +474,25 @@ def pslr_function_declarations #endif int yypslr_scan_with_layout (int parser_state, const char **input, - size_t *input_len, yypslr_scan_result *result); + size_t *input_len, YYSTYPE *yylvalp, + yypslr_scan_result *result); #ifndef YYPSLR_SCAN_WITH_LAYOUT - # define YYPSLR_SCAN_WITH_LAYOUT(ParserState, InputPtr, InputLenPtr, Result) \\ - yypslr_scan_with_layout ((ParserState), (InputPtr), (InputLenPtr), (Result)) + # define YYPSLR_SCAN_WITH_LAYOUT(ParserState, InputPtr, InputLenPtr, LvalPtr, Result) \\ + yypslr_scan_with_layout ((ParserState), (InputPtr), (InputLenPtr), (LvalPtr), (Result)) #endif C_CODE end + if pslr_pure_mode? + declarations << <<~C_CODE + /* Pure mode (%define api.pslr.lexer generated): the generated + parser owns lexical analysis. Give it the input before + calling yyparse. */ + void yypslr_set_input (const char *input, size_t input_len); + C_CODE + end + if (member = pslr_state_member) declarations << <<~C_CODE #ifndef YYGETSTATE_CONTEXT @@ -1054,7 +1070,10 @@ def pslr_layout_declarations C_CODE end - # Generate token action dispatch function + # Generate token action dispatch function. + # yylval inside a %token-action refers to the semantic value slot the + # caller passes in, so token actions work with both the generated + # (pure-mode) lexer and a user yylex driving the bridge macros. def pslr_token_action_function return "" unless pslr_token_actions_enabled? @@ -1063,9 +1082,13 @@ def pslr_token_action_function lines << "/* PSLR token action dispatch */" lines << "/* Generated from %token-action declarations */" lines << "" + lines << "#define yylval (*yylvalp)" lines << "static void" - lines << "yypslr_token_action (int token, const char *yytext, int yyleng)" + lines << "yypslr_token_action (int token, const char *yytext, int yyleng, YYSTYPE *yylvalp)" lines << "{" + lines << " YY_USE (yytext);" + lines << " YY_USE (yyleng);" + lines << " YY_USE (yylvalp);" @grammar.token_actions.each_with_index do |action, idx| token_id = @context.states.find_symbol_by_s_value!(action.token_name).token_id @@ -1079,9 +1102,10 @@ def pslr_token_action_function end lines << "}" + lines << "#undef yylval" lines << "" - lines << "#define YYPSLR_TOKEN_ACTION(Token, Text, Len) \\" - lines << " yypslr_token_action ((Token), (Text), (Len))" + lines << "#define YYPSLR_TOKEN_ACTION(Token, Text, Len, LvalPtr) \\" + lines << " yypslr_token_action ((Token), (Text), (Len), (LvalPtr))" lines.join("\n") end @@ -1103,9 +1127,11 @@ def pslr_layout_scan_function */ int yypslr_scan_with_layout (int parser_state, const char **input, - size_t *input_len, yypslr_scan_result *result) + size_t *input_len, YYSTYPE *yylvalp, + yypslr_scan_result *result) { YYPSLR_LAYOUT_CLEAR (); + YY_USE (yylvalp); for (;;) { @@ -1116,26 +1142,79 @@ def pslr_layout_scan_function if (token == YYEOF) { - #{pslr_token_actions_enabled? ? " YYPSLR_TOKEN_ACTION (token, *input, 0);" : ""} + #{pslr_token_actions_enabled? ? " YYPSLR_TOKEN_ACTION (token, *input, 0, yylvalp);" : ""} return token; } if (result->is_layout) { YYPSLR_LAYOUT_APPEND (*input, result->length); - #{pslr_token_actions_enabled? ? " YYPSLR_TOKEN_ACTION (token, *input, result->length);" : ""} + #{pslr_token_actions_enabled? ? " YYPSLR_TOKEN_ACTION (token, *input, result->length, yylvalp);" : ""} *input += result->length; *input_len -= (size_t)result->length; continue; } - #{pslr_token_actions_enabled? ? " YYPSLR_TOKEN_ACTION (token, *input, result->length);" : ""} + #{pslr_token_actions_enabled? ? " YYPSLR_TOKEN_ACTION (token, *input, result->length, yylvalp);" : ""} return token; } } C_CODE end + # Generate the pure-mode lexer: yylex is emitted by the generator and + # drives the pseudo-scanner with the current parser state, so no user + # lexer is involved (%define api.pslr.lexer generated, section 3.1). + def pslr_generated_lexer_function + return "" unless pslr_pure_mode? + + params = ["YYSTYPE *yylvalp"] + params << "YYLTYPE *yyllocp" if @grammar.locations + params << lex_param unless lex_param.empty? + unused = [] + unused << " YY_USE (yyllocp);" if @grammar.locations + unused << " YY_USE (#{lex_param_name});" unless lex_param.empty? + + <<~C_CODE + + /* Pure-mode PSLR lexer: generated, owns the input cursor. */ + static const char *yypslr_input_cursor = 0; + static size_t yypslr_input_remain = 0; + static int yypslr_parser_state = 0; + + void + yypslr_set_input (const char *input, size_t input_len) + { + yypslr_input_cursor = input; + yypslr_input_remain = input_len; + yypslr_parser_state = 0; + } + + #define YYPSLR_SET_PARSER_STATE(State) (yypslr_parser_state = (State)) + + static int + yylex (#{params.join(", ")}) + { + yypslr_scan_result yypslr_result; + int yypslr_token; + #{unused.join("\n")} + + yypslr_token = yypslr_scan_with_layout (yypslr_parser_state, + &yypslr_input_cursor, + &yypslr_input_remain, + yylvalp, + &yypslr_result); + + if (yypslr_token == YYEMPTY || yypslr_token == YYEOF) + return yypslr_token; + + yypslr_input_cursor += yypslr_result.length; + yypslr_input_remain -= (size_t) yypslr_result.length; + return yypslr_token; + } + C_CODE + end + # Generate all PSLR C code def pslr_tables_and_functions return "" unless pslr_scanner_enabled? @@ -1152,8 +1231,9 @@ def pslr_tables_and_functions length_precedences_table_code, pslr_layout_declarations, pseudo_scan_function, - pslr_layout_scan_function, pslr_token_action_function, + pslr_layout_scan_function, + pslr_generated_lexer_function, ] parts.join("\n") diff --git a/lib/lrama/states.rb b/lib/lrama/states.rb index 4f10c7522..f964a4f3f 100644 --- a/lib/lrama/states.rb +++ b/lib/lrama/states.rb @@ -258,6 +258,7 @@ def validate!(logger) validate_pslr_state_growth!(logger) validate_pslr_scanner_conflicts!(logger) validate_pslr_inadequacies!(logger) + validate_pslr_pure_coverage!(logger) validate_pslr_useless_lex_prec!(logger) validate_pslr_lexical_tie_candidates!(logger) end @@ -1387,6 +1388,23 @@ def validate_pslr_scanner_conflicts!(logger) exit false end + # Pure mode owns lexical analysis completely, so every terminal must + # be producible by the pseudo-scanner. + # @rbs (Logger logger) -> void + def validate_pslr_pure_coverage!(logger) + return unless pslr_defined? + return unless @grammar.pslr_lexer_generated? + + uncovered = @grammar.uncovered_pslr_terminals + return if uncovered.empty? + + logger.error( + "api.pslr.lexer generated requires a %token-pattern for every terminal; " \ + "missing: #{uncovered.join(', ')}" + ) + exit false + end + # @rbs (State::ScannerAccepts::Conflict conflict) -> String def pslr_scanner_conflict_message(conflict) state = conflict.parser_state_id || "unknown" diff --git a/lib/lrama/warnings.rb b/lib/lrama/warnings.rb index d26cdff9c..554b82f48 100644 --- a/lib/lrama/warnings.rb +++ b/lib/lrama/warnings.rb @@ -9,6 +9,7 @@ require_relative 'warnings/useless_precedence' require_relative 'warnings/lexical_tie_candidates' require_relative 'warnings/parse_lac' +require_relative 'warnings/pslr_coverage' module Lrama class Warnings @@ -22,6 +23,7 @@ def initialize(logger, warnings) @useless_precedence = UselessPrecedence.new(logger, warnings) @lexical_tie_candidates = LexicalTieCandidates.new(logger, warnings) @parse_lac = ParseLac.new(logger, warnings) + @pslr_coverage = PslrCoverage.new(logger, warnings) end # @rbs (Lrama::Grammar grammar, Lrama::States states) -> void @@ -34,6 +36,7 @@ def warn(grammar, states) @useless_precedence.warn(grammar, states) @lexical_tie_candidates.warn(states) @parse_lac.warn(grammar) + @pslr_coverage.warn(grammar) end end end diff --git a/lib/lrama/warnings/pslr_coverage.rb b/lib/lrama/warnings/pslr_coverage.rb new file mode 100644 index 000000000..2a614c0d2 --- /dev/null +++ b/lib/lrama/warnings/pslr_coverage.rb @@ -0,0 +1,34 @@ +# rbs_inline: enabled +# frozen_string_literal: true + +module Lrama + class Warnings + class PslrCoverage + # @rbs (Logger logger, bool warnings) -> void + def initialize(logger, warnings) + @logger = logger + @warnings = warnings + end + + # Bridge mode allows partial coverage: terminals without a + # %token-pattern must keep coming from the user lexer. List them so + # the boundary between generated and hand-written scanning is + # explicit. + # @rbs (Lrama::Grammar grammar) -> void + def warn(grammar) + return unless @warnings + return unless grammar.pslr_defined? + return if grammar.pslr_lexer_generated? + return if grammar.token_patterns.empty? + + uncovered = grammar.uncovered_pslr_terminals + return if uncovered.empty? + + @logger.warn( + "PSLR pseudo-scanner does not cover these terminals; " \ + "the user lexer must produce them: #{uncovered.join(', ')}" + ) + end + end + end +end diff --git a/spec/fixtures/integration/pslr_pure.y b/spec/fixtures/integration/pslr_pure.y new file mode 100644 index 000000000..fdecd1c24 --- /dev/null +++ b/spec/fixtures/integration/pslr_pure.y @@ -0,0 +1,63 @@ +/* + * Integration test for PSLR pure mode (%define api.pslr.lexer generated). + * The generated parser owns lexical analysis: no user yylex, layout is + * skipped by the generated scan loop, and %token-action builds semantic + * values from the matched lexeme. + */ + +%{ +#include +#include +#include "pslr_pure.h" + +static int yyerror(YYLTYPE *loc, const char *str); +%} + +%define lr.type pslr +%define api.pslr.lexer generated + +%token NUM PLUS + +%token-pattern NUM /[0-9]+/ +%token-pattern PLUS /\+/ +%token-pattern YYLAYOUT_WS /[ \t\r\n]+/ + +%token-action NUM { + int yypslr_value = 0; + int yypslr_i; + for (yypslr_i = 0; yypslr_i < yyleng; yypslr_i++) { + yypslr_value = yypslr_value * 10 + (yytext[yypslr_i] - '0'); + } + yylval = yypslr_value; +} + +%% + +program: expr { printf("=> %d\n", $1); } + ; + +expr: NUM + | expr PLUS NUM { $$ = $1 + $3; } + ; + +%% + +static int yyerror(YYLTYPE *loc, const char *str) +{ + (void)loc; + fprintf(stderr, "parse error: %s\n", str); + return 0; +} + +int main(int argc, char *argv[]) +{ + const char *input = argc == 2 ? argv[1] : ""; + + yypslr_set_input(input, strlen(input)); + + if (yyparse()) { + fprintf(stderr, "syntax error\n"); + return 1; + } + return 0; +} diff --git a/spec/lrama/integration_spec.rb b/spec/lrama/integration_spec.rb index 66ed9f35a..54cb36988 100644 --- a/spec/lrama/integration_spec.rb +++ b/spec/lrama/integration_spec.rb @@ -284,6 +284,66 @@ def generate_object(grammar_file_path, c_path, obj_path, command_args: []) end end + describe "PSLR pure mode (api.pslr.lexer generated)" do + def compile_pure_parser + tmpdir = Dir.tmpdir + grammar_file_path = fixture_path("integration/pslr_pure.y") + c_path = tmpdir + "/pslr_pure#{file_extension}" + h_path = tmpdir + "/pslr_pure.h" + obj_path = tmpdir + "/pslr_pure" + + unless IntegrationHelper.compiled_parsers["pslr_pure"] && File.exist?(obj_path) + Lrama::Command.new(%W[-H#{h_path} -o#{c_path} #{grammar_file_path}]).run + exec_command("#{compiler} -Wall -O0 -g -I#{tmpdir} #{c_path} -o #{obj_path}") + IntegrationHelper.compiled_parsers["pslr_pure"] = true + end + + obj_path + end + + it "parses input with the generated lexer, skipping layout and running token actions" do + obj_path = compile_pure_parser + + out, _err, status = Open3.capture3(obj_path, "1 + 23 +\t4") + expect(status.success?).to be true + expect(out).to eq("=> 28\n") + end + + it "reports a syntax error for invalid input" do + obj_path = compile_pure_parser + + _out, _err, status = Open3.capture3(obj_path, "1 + + 2") + expect(status.success?).to be false + end + + it "rejects pure mode when a terminal has no token pattern" do + grammar_text = <<~GRAMMAR + %define lr.type pslr + %define api.pslr.lexer generated + %token NUM UNCOVERED + %token-pattern NUM /[0-9]+/ + + %% + + start + : NUM + | UNCOVERED + ; + GRAMMAR + grammar = Lrama::Parser.new(grammar_text, "integration/pslr_pure_uncovered.y").parse + grammar.prepare + grammar.validate! + states = Lrama::States.new(grammar, Lrama::Tracer.new(Lrama::Logger.new)) + states.compute + states.compute_pslr + logger = Lrama::Logger.new + allow(logger).to receive(:error) + + expect { states.validate!(logger) }.to raise_error(SystemExit) + expect(logger).to have_received(:error).with(a_string_including("UNCOVERED")) + end + end + describe "user defined parameterized rules" do it "prints messages corresponding to rules" do expected = <<~STR diff --git a/spec/lrama/output_spec.rb b/spec/lrama/output_spec.rb index 0bbd52c0e..9bd9e4520 100644 --- a/spec/lrama/output_spec.rb +++ b/spec/lrama/output_spec.rb @@ -248,6 +248,7 @@ pslr_defined?: true, pslr_state_member: "current_state", parse_lac_full?: true, + pslr_lexer_generated?: false, token_actions: [] ) end diff --git a/template/bison/yacc.c b/template/bison/yacc.c index 03aad98da..db15f70d8 100644 --- a/template/bison/yacc.c +++ b/template/bison/yacc.c @@ -602,6 +602,9 @@ static const <%= output.int_type_for(output.context.yyr2) %> yyr2[] = #ifndef YYSETSTATE_CONTEXT # define YYSETSTATE_CONTEXT(CurrentState) ((void) 0) #endif +#ifndef YYPSLR_SET_PARSER_STATE +# define YYPSLR_SET_PARSER_STATE(State) ((void) 0) +#endif <%- end -%> int @@ -1810,6 +1813,7 @@ YYLTYPE yylloc = yyloc_default; YY_IGNORE_USELESS_CAST_END <%- if output.pslr_enabled? -%> YYSETSTATE_CONTEXT (yystate); + YYPSLR_SET_PARSER_STATE (yystate); <%- end -%> YY_STACK_PRINT (yyss, yyssp<%= output.user_args %>); From b00f868492e63f120e7fb3654542ef0975edf1fe Mon Sep 17 00:00:00 2001 From: ydah Date: Wed, 8 Jul 2026 22:28:03 +0900 Subject: [PATCH 12/15] pslr: add {NAME} references, canonical-lr tables and richer reports * %token-pattern bodies can reference earlier patterns with {NAME}, the idiom the paper uses for shortest-match comment scanning (Fig 3.2c/3.2d). Undefined, forward, self and malformed references are rejected; a literal brace is written \{. * The scanner alphabet is now bytes (0-255) rather than 7-bit ASCII: negated character classes and dot pass multi-byte UTF-8 sequences through unmodified, and Ruby-side scanning walks bytes to match the generated C runtime. * Layout tokens are rejected in grammar rules at validation time; the parser never sees them, so such rules could never match. * %define pslr.tables canonical-lr switches state splitting to canonical LR(1) compatibility (kernel lookahead-set equality) for debugging conflict reports whose contexts IELR merging would blend. * --report=pslr now renders per-state acceptable-token sets, scanner_accepts rows including the fallback row, unresolved-conflict details with witnesses, useless %lex-prec rules and lexical tie candidates in addition to the split metrics summary. --- lib/lrama/grammar.rb | 42 ++++++++++++++ lib/lrama/reporter/pslr.rb | 94 +++++++++++++++++++++++++++++++- lib/lrama/scanner_fsa.rb | 50 +++++++++++++++-- lib/lrama/states.rb | 31 ++++++++++- spec/lrama/grammar_spec.rb | 31 +++++++++++ spec/lrama/integration_spec.rb | 46 ++++++++++++++++ spec/lrama/reporter/pslr_spec.rb | 55 +++++++++++++++++++ spec/lrama/scanner_fsa_spec.rb | 42 ++++++++++++++ 8 files changed, 381 insertions(+), 10 deletions(-) create mode 100644 spec/lrama/reporter/pslr_spec.rb diff --git a/lib/lrama/grammar.rb b/lib/lrama/grammar.rb index 18d94a1f6..4292dab05 100644 --- a/lib/lrama/grammar.rb +++ b/lib/lrama/grammar.rb @@ -367,6 +367,22 @@ def pslr_lexer_generated? @define['api.pslr.lexer'] == 'generated' end + # Base table construction for PSLR: extended IELR (default) or + # canonical LR(1) for debugging unresolved-conflict reports whose + # contexts would otherwise be merged (section 3.4.3). + # @rbs () -> String + def pslr_tables + value = @define['pslr.tables'] + return 'ielr' if value.nil? || value.empty? + + value + end + + # @rbs () -> bool + def pslr_canonical_tables? + pslr_defined? && pslr_tables == 'canonical-lr' + end + # Terminals that the pseudo-scanner cannot produce: real terminals # used by the grammar that have no %token-pattern (explicit or # synthesized). Pure mode requires this list to be empty; bridge @@ -682,9 +698,12 @@ def token_lineno(id) def validate_pslr_configuration! validate_parse_lac! validate_pslr_lexer! + validate_pslr_tables! return unless pslr_defined? + validate_no_layout_tokens_in_rules! + member = pslr_state_member if member && member !~ /\A[a-zA-Z_][a-zA-Z0-9_]*\z/ raise %(%define api.pslr.state-member must be a valid C identifier, got "#{member}".) @@ -703,6 +722,20 @@ def validate_parse_lac! raise %(%define parse.lac must be "full" or "none", got "#{value}".) end + # Layout tokens are consumed by the pseudo-scanner and never reach + # the parser, so they must not appear in grammar rules (section 3.6). + # @rbs () -> void + def validate_no_layout_tokens_in_rules! + @rules.each do |rule| + rule.rhs.each do |sym| + next unless sym.term? + next unless sym.id.s_value.start_with?("YYLAYOUT") + + raise "layout token #{sym.id.s_value} must not appear in a grammar rule (line #{rule.lineno})." + end + end + end + # @rbs () -> void def validate_pslr_lexer! value = @define['api.pslr.lexer'] @@ -717,6 +750,15 @@ def validate_pslr_lexer! end end + # @rbs () -> void + def validate_pslr_tables! + value = @define['pslr.tables'] + return if value.nil? || value.empty? + return if %w[ielr canonical-lr].include?(value) + + raise %(%define pslr.tables must be "ielr" or "canonical-lr", got "#{value}".) + end + # @rbs (Lexer::Token::Base operand) -> Array[Lexer::Token::Base] def expand_pslr_operand(operand) return @token_patterns.map(&:id) if operand.s_value == "yyall" diff --git a/lib/lrama/reporter/pslr.rb b/lib/lrama/reporter/pslr.rb index 766c5855b..ccbf3015a 100644 --- a/lib/lrama/reporter/pslr.rb +++ b/lib/lrama/reporter/pslr.rb @@ -14,6 +14,18 @@ def report(io, states) return unless @pslr return unless states.pslr_defined? + report_summary(io, states) + report_acceptable_tokens(io, states) + report_scanner_accepts(io, states) + report_unresolved_conflicts(io, states) + report_useless_lex_prec(io, states) + report_tie_candidates(io, states) + end + + private + + # @rbs (IO io, Lrama::States states) -> void + def report_summary(io, states) metrics = states.pslr_metrics io << "PSLR Summary\n\n" @@ -29,7 +41,87 @@ def report(io, states) io << "\n" end - private + # @rbs (IO io, Lrama::States states) -> void + def report_acceptable_tokens(io, states) + return unless states.scanner_fsa + + io << "PSLR Acceptable Tokens (acc with ties and layout)\n\n" + states.states.each do |state| + tokens = states.pslr_acceptable_tokens(state).to_a.sort + io << " State #{state.id}: #{tokens.join(' ')}\n" + end + io << "\n" + end + + # @rbs (IO io, Lrama::States states) -> void + def report_scanner_accepts(io, states) + scanner_accepts = states.scanner_accepts_table + scanner_fsa = states.scanner_fsa + return unless scanner_accepts && scanner_fsa + + accepting_states = scanner_fsa.states.select(&:accepting?) + + io << "PSLR Scanner Accepts\n\n" + states.states.each do |state| + cells = accepting_states.filter_map do |fsa_state| + token = scanner_accepts[state.id, fsa_state.id] + "ss#{fsa_state.id}=>#{token.name}" if token + end + next if cells.empty? + + io << " State #{state.id}: #{cells.join(' ')}\n" + end + + fallback_cells = accepting_states.filter_map do |fsa_state| + token = scanner_accepts.fallback_table[fsa_state.id] + "ss#{fsa_state.id}=>#{token.name}" if token + end + io << " Fallback: #{fallback_cells.join(' ')}\n" unless fallback_cells.empty? + io << "\n" + end + + # @rbs (IO io, Lrama::States states) -> void + def report_unresolved_conflicts(io, states) + scanner_accepts = states.scanner_accepts_table + return unless scanner_accepts + return unless scanner_accepts.unresolved_conflicts? + + io << "PSLR Unresolved Scanner Conflicts\n\n" + scanner_accepts.conflicts.each do |conflict| + io << " State #{conflict.parser_state_id || 'fallback'}, scanner state #{conflict.scanner_state_id}\n" + io << " witness: #{conflict.witness.inspect}\n" if conflict.witness + io << " shorter matches: #{conflict.shorter_tokens.join(', ')}\n" unless conflict.shorter_tokens.empty? + io << " current matches: #{conflict.current_tokens.join(', ')}\n" + end + io << "\n" + end + + # @rbs (IO io, Lrama::States states) -> void + def report_useless_lex_prec(io, states) + return unless states.scanner_accepts_table + + useless = states.lex_prec.useless_rules + return if useless.empty? + + io << "PSLR Useless %lex-prec Rules\n\n" + useless.each do |rule| + operator = LengthPrecedences.operator_label(rule.operator) + io << " #{rule.left_name} #{operator} #{rule.right_name} (line #{rule.lineno})\n" + end + io << "\n" + end + + # @rbs (IO io, Lrama::States states) -> void + def report_tie_candidates(io, states) + candidates = states.lexical_tie_candidates + return if candidates.nil? || candidates.empty? + + io << "PSLR Lexical Tie Candidates\n\n" + candidates.each do |left, right| + io << " #{left} #{right}\n" + end + io << "\n" + end # @rbs (Numeric?) -> String def format_ratio(value) diff --git a/lib/lrama/scanner_fsa.rb b/lib/lrama/scanner_fsa.rb index 1ed327534..32e0559a5 100644 --- a/lib/lrama/scanner_fsa.rb +++ b/lib/lrama/scanner_fsa.rb @@ -68,17 +68,18 @@ def acc_ss(state_id) end # Simulate the FSA on input string starting from initial state - # Returns all accepting states reached during the scan + # Returns all accepting states reached during the scan. + # The walk is byte-oriented to match the generated C runtime. # @rbs (String input) -> Array[{state: State, position: Integer, token: Grammar::TokenPattern}] def scan(input) results = [] current_state_id = 0 - input.each_char.with_index do |char, index| + input.each_byte.with_index do |byte, index| current_state = @states[current_state_id] break unless current_state - next_state_id = current_state.transitions[char] + next_state_id = current_state.transitions[byte.chr] break unless next_state_id current_state_id = next_state_id @@ -216,6 +217,7 @@ def to_ary def build_nfa nfa_states = [] nfa_counter = [0] + @pattern_references = {} # Create NFA start state nfa_start = create_nfa_state(nfa_counter, nfa_states) @@ -235,6 +237,10 @@ def build_nfa # Mark end state as accepting end_state.accepting_token = token_pattern + + # {NAME} may only reference patterns that are already defined, so + # register after a successful compile (no self/forward references). + @pattern_references[token_pattern.name] = token_pattern.regex_pattern end nfa_states @@ -248,8 +254,11 @@ def create_nfa_state(counter, states) state end - ASCII_CHARS = (0..127).map(&:chr).freeze #: Array[String] - ANY_CHARS = (ASCII_CHARS - ["\n"]).freeze #: Array[String] + # The input alphabet is bytes (0-255): keywords and operators are + # ASCII, and byte-oriented classes let multi-byte UTF-8 sequences + # pass through negated classes and dot unmodified. + BYTE_CHARS = (0..255).map(&:chr).freeze #: Array[String] + ANY_CHARS = (BYTE_CHARS - ["\n"]).freeze #: Array[String] DIGIT_CHARS = ("0".."9").to_a.freeze #: Array[String] WORD_CHARS = (("a".."z").to_a + ("A".."Z").to_a + DIGIT_CHARS + ["_"]).freeze #: Array[String] WHITESPACE_CHARS = [" ", "\t", "\n", "\r", "\f", "\v"].freeze #: Array[String] @@ -329,6 +338,10 @@ def compile_sequence(pattern, pos, counter, states, stop_char) frag, i = compile_expression(pattern, i + 1, counter, states, ")") fragments << frag next + when '{' + frag, i = compile_pattern_reference(pattern, i, counter, states) + fragments << frag + next when ')' raise PatternError, "unmatched close group at offset #{i}" when '.' @@ -354,6 +367,31 @@ def empty_sequence_message(stop_char) stop_char ? "empty groups are not allowed" : "empty patterns are not allowed" end + # Compile a {NAME} reference by inlining the body of an already + # defined %token-pattern (paper Fig 3.2c/3.2d idiom). Self and + # forward references are rejected because @pattern_references only + # contains patterns compiled before the current one. + # @rbs (String pattern, Integer offset, Array[Integer] counter, Array[NFAState] states) -> [Fragment, Integer] + def compile_pattern_reference(pattern, offset, counter, states) + close = pattern.index("}", offset) + raise PatternError, "unclosed pattern reference at offset #{offset}" unless close + + name = pattern[offset + 1...close] + unless name&.match?(/\A[a-zA-Z_][a-zA-Z0-9_]*\z/) + raise PatternError, "invalid pattern reference {#{name}} at offset #{offset} (escape a literal brace as \\{)" + end + + referenced = (@pattern_references || {})[name] + unless referenced + raise PatternError, "undefined or forward pattern reference {#{name}} (references must name an earlier %token-pattern)" + end + + fragment, consumed = compile_expression(referenced, 0, counter, states) + raise PatternError, "unexpected trailing input in referenced pattern {#{name}}" if consumed < referenced.length + + [fragment, close + 1] + end + # @rbs (String pattern, Integer offset, Array[Integer] counter, Array[NFAState] states) -> [Fragment, Integer] def compile_escape(pattern, offset, counter, states) raise PatternError, "dangling escape at offset #{offset}" if offset + 1 >= pattern.length @@ -441,7 +479,7 @@ def expand_char_class(char_class) end if negated - chars = ASCII_CHARS - chars + chars = BYTE_CHARS - chars end chars.uniq diff --git a/lib/lrama/states.rb b/lib/lrama/states.rb index f964a4f3f..4dd8bcc19 100644 --- a/lib/lrama/states.rb +++ b/lib/lrama/states.rb @@ -348,6 +348,13 @@ def compute_la_sources_for_conflicted_states end end + # Public accessor for reporting: acc(sp) with tie expansion and + # layout injection, as used by the scanner_accepts construction. + # @rbs (State state) -> Set[String] + def pslr_acceptable_tokens(state) + acceptable_tokens_for_pslr(state) + end + private # @rbs (Grammar::Symbol accessing_symbol, Array[State::Item] kernels, Hash[Array[State::Item], State] states_created) -> [State, bool] @@ -1083,16 +1090,34 @@ def compute_state(state, transition, next_state) # @rbs (State state, State::lookahead_set filtered_lookaheads, ?State::lookahead_set pslr_lookaheads) -> bool def compatible_split_state?(state, filtered_lookaheads, pslr_lookaheads = nil) - return false unless state.is_compatible?(filtered_lookaheads) - return true unless @pslr_split_enabled && @scanner_fsa - pslr_lookaheads ||= filtered_lookaheads + if @pslr_split_enabled && @grammar.pslr_canonical_tables? + return false unless canonical_lookaheads_match?(state, pslr_lookaheads) + else + return false unless state.is_compatible?(filtered_lookaheads) + end + return true unless @pslr_split_enabled && @scanner_fsa + existing_acc = acceptable_tokens_for_pslr(state) candidate_acc = acceptable_tokens_for_pslr(state, pslr_lookaheads) pslr_compatible_accept_sets?(existing_acc, candidate_acc) end + # Canonical LR(1) compatibility for %define pslr.tables canonical-lr: + # states merge only when their kernel lookahead sets are identical, + # so unresolved-conflict reports are not merged across contexts. + # @rbs (State state, State::lookahead_set filtered_lookaheads) -> bool + def canonical_lookaheads_match?(state, filtered_lookaheads) + return true unless state.lookaheads_recomputed + + state.kernels.all? do |kernel| + existing = state.item_lookahead_set[kernel] || [] + candidate = filtered_lookaheads[kernel] || [] + existing.to_set == candidate.to_set + end + end + # @rbs (State state, ?State::lookahead_set filtered_lookaheads) -> Array[[Integer, String?]] def pslr_state_signature(state, filtered_lookaheads = nil) return [] unless @scanner_fsa diff --git a/spec/lrama/grammar_spec.rb b/spec/lrama/grammar_spec.rb index 4a663401d..6e28e24dd 100644 --- a/spec/lrama/grammar_spec.rb +++ b/spec/lrama/grammar_spec.rb @@ -284,6 +284,37 @@ .to raise_error(RuntimeError, '%define pslr.max-state-ratio must be greater than or equal to 1.0, got "0.5".') end end + + context 'when pslr.tables has an unknown value' do + before do + grammar.define = { + 'lr.type' => 'pslr', + 'pslr.tables' => 'lalr' + } + end + + it 'raises an error' do + expect { grammar.validate! } + .to raise_error(RuntimeError, '%define pslr.tables must be "ielr" or "canonical-lr", got "lalr".') + end + end + + context 'when a layout token appears in a grammar rule' do + it 'raises an error' do + y = <<~GRAMMAR + %define lr.type pslr + %token-pattern ID /[a-z]+/ + %token-pattern YYLAYOUT_WS /[ ]+/ + %% + program: ID YYLAYOUT_WS ID + GRAMMAR + grammar = Lrama::Parser.new(y, "layout_in_rule.y").parse + grammar.prepare + + expect { grammar.validate! } + .to raise_error(RuntimeError, /layout token YYLAYOUT_WS must not appear in a grammar rule/) + end + end end describe "#finalize_lexical_ties!" do diff --git a/spec/lrama/integration_spec.rb b/spec/lrama/integration_spec.rb index 54cb36988..c615132a2 100644 --- a/spec/lrama/integration_spec.rb +++ b/spec/lrama/integration_spec.rb @@ -316,6 +316,52 @@ def compile_pure_parser expect(status.success?).to be false end + it "computes canonical LR(1) tables when pslr.tables is canonical-lr" do + base_source = <<~GRAMMAR + %token-pattern LT />/ + %token-pattern RANGLE />/ + %token-pattern ID /[a-z]+/ + %lex-no-tie RANGLE RSHIFT + + %% + + program + : template_expr + | shift_expr + ; + + template_expr + : LT shared RANGLE + ; + + shift_expr + : START shared RSHIFT ID + ; + + shared + : MARK + ; + GRAMMAR + + states_by_tables = ["ielr", "canonical-lr"].map do |tables| + source = "%define lr.type pslr\n%define pslr.tables #{tables}\n" + base_source + grammar = Lrama::Parser.new(source, "integration/pslr_tables_#{tables}.y").parse + grammar.prepare + grammar.validate! + states = Lrama::States.new(grammar, Lrama::Tracer.new(Lrama::Logger.new)) + states.compute + states.compute_pslr + states + end + + ielr_states, canonical_states = states_by_tables + expect(canonical_states.states_count).to be >= ielr_states.states_count + expect(canonical_states.scanner_accepts_table).not_to be_nil + end + it "rejects pure mode when a terminal has no token pattern" do grammar_text = <<~GRAMMAR %define lr.type pslr diff --git a/spec/lrama/reporter/pslr_spec.rb b/spec/lrama/reporter/pslr_spec.rb new file mode 100644 index 000000000..0736481e1 --- /dev/null +++ b/spec/lrama/reporter/pslr_spec.rb @@ -0,0 +1,55 @@ +# frozen_string_literal: true + +require "stringio" + +RSpec.describe Lrama::Reporter::Pslr do + def build_states(source) + grammar = Lrama::Parser.new(source, "reporter_pslr.y").parse + grammar.prepare + grammar.validate! + states = Lrama::States.new(grammar, Lrama::Tracer.new(Lrama::Logger.new)) + states.compute + states.compute_pslr + states + end + + let(:source) do + <<~GRAMMAR + %define lr.type pslr + %token-pattern RSHIFT />>/ + %token-pattern RANGLE />/ + %token-pattern USELESS_A /a/ + %token-pattern USELESS_B /b/ + %lex-prec RANGLE -s RSHIFT + %lex-prec USELESS_A -~ USELESS_B + %% + program: RSHIFT | RANGLE + GRAMMAR + end + + it "reports summary, accept sets, scanner accepts rows and useless rules" do + states = build_states(source) + io = StringIO.new + described_class.new(pslr: true).report(io, states) + output = io.string + + expect(output).to include("PSLR Summary") + expect(output).to include("PSLR Acceptable Tokens") + expect(output).to include("PSLR Scanner Accepts") + expect(output).to include("Fallback:") + expect(output).to include("PSLR Useless %lex-prec Rules") + expect(output).to include("USELESS_A -~ USELESS_B") + end + + it "reports nothing for non-PSLR grammars" do + grammar = Lrama::Parser.new("%token NUM\n%%\nprogram: NUM\n", "plain.y").parse + grammar.prepare + grammar.validate! + states = Lrama::States.new(grammar, Lrama::Tracer.new(Lrama::Logger.new)) + states.compute + io = StringIO.new + described_class.new(pslr: true).report(io, states) + + expect(io.string).to eq("") + end +end diff --git a/spec/lrama/scanner_fsa_spec.rb b/spec/lrama/scanner_fsa_spec.rb index 9912945ee..fb45aa7df 100644 --- a/spec/lrama/scanner_fsa_spec.rb +++ b/spec/lrama/scanner_fsa_spec.rb @@ -204,6 +204,48 @@ def token_pattern(name, regex, lineno: 1, order: 0) end end + describe "{NAME} pattern references" do + it "inlines the body of an earlier pattern" do + com_start = token_pattern("COM_START", "\\/\\*[^*]*", order: 0) + com = token_pattern("COM", "{COM_START}\\*\\/", order: 1) + fsa = Lrama::ScannerFSA.new([com_start, com]) + + names = fsa.scan("/*abc*/").map { |result| result[:token].name } + expect(names).to include("COM") + end + + it "rejects undefined and forward references" do + expect { Lrama::ScannerFSA.new([token_pattern("BAD", "{MISSING}x", lineno: 3)]) } + .to raise_error(Lrama::ScannerFSA::PatternError, /BAD at line 3.*undefined or forward pattern reference \{MISSING\}/m) + end + + it "rejects self references" do + expect { Lrama::ScannerFSA.new([token_pattern("SELF", "{SELF}x", lineno: 4)]) } + .to raise_error(Lrama::ScannerFSA::PatternError, /SELF at line 4.*undefined or forward pattern reference \{SELF\}/m) + end + + it "still matches escaped literal braces" do + brace = token_pattern("LBRACE", "\\{") + expect(Lrama::ScannerFSA.new([brace]).scan("{").map { |result| result[:token].name }).to include("LBRACE") + end + + it "rejects malformed references" do + expect { Lrama::ScannerFSA.new([token_pattern("BAD", "{1x}", lineno: 5)]) } + .to raise_error(Lrama::ScannerFSA::PatternError, /invalid pattern reference/) + end + end + + describe "byte-oriented classes" do + it "matches multi-byte UTF-8 sequences through negated classes" do + not_star = token_pattern("NOT_STAR", "[^*]+") + fsa = Lrama::ScannerFSA.new([not_star]) + + results = fsa.scan("あ") + expect(results.map { |result| result[:token].name }).to include("NOT_STAR") + expect(results.map { |result| result[:position] }).to include("あ".bytesize) + end + end + describe "#pairwise_conflict_pairs" do it "detects identity and length conflicts" do rangle = Lrama::Grammar::TokenPattern.new( From 1b5cb84bb569d8aa3e96d965cfcedb1a0c67f3ff Mon Sep 17 00:00:00 2001 From: ydah Date: Wed, 8 Jul 2026 22:40:10 +0900 Subject: [PATCH 13/15] pslr: regenerate RBS signatures and fix operator annotations Annotate %lex-prec operator constants and attributes as ::Symbol so they do not resolve to Lrama::Grammar::Symbol inside the Grammar namespace, and regenerate the rbs-inline signatures for the new pairwise resolution, LAC, pure-mode and reporting code. steep check is clean. --- lib/lrama/grammar.rb | 4 +- lib/lrama/grammar/lex_prec.rb | 38 ++++++------- lib/lrama/state/scanner_accepts.rb | 2 +- rbs_collection.lock.yaml | 16 ++++++ sig/generated/lrama/grammar.rbs | 57 +++++++++++++++++-- sig/generated/lrama/grammar/lex_prec.rbs | 48 ++++++++-------- .../lrama/pslr/pairwise_resolution.rbs | 42 ++++++++++++++ sig/generated/lrama/reporter/pslr.rbs | 18 ++++++ sig/generated/lrama/scanner_fsa.rbs | 15 ++++- sig/generated/lrama/state/scanner_accepts.rbs | 32 +++++++---- sig/generated/lrama/states.rbs | 18 ++++++ sig/generated/lrama/warnings/parse_lac.rbs | 13 +++++ .../lrama/warnings/pslr_coverage.rbs | 17 ++++++ 13 files changed, 258 insertions(+), 62 deletions(-) create mode 100644 sig/generated/lrama/pslr/pairwise_resolution.rbs create mode 100644 sig/generated/lrama/warnings/parse_lac.rbs create mode 100644 sig/generated/lrama/warnings/pslr_coverage.rbs diff --git a/lib/lrama/grammar.rb b/lib/lrama/grammar.rb index 4292dab05..841a42927 100644 --- a/lib/lrama/grammar.rb +++ b/lib/lrama/grammar.rb @@ -459,7 +459,7 @@ def add_symbol_set(name:, symbols:) # Add lex-prec rules from %lex-prec directive. # Stores as raw declaration for delayed expansion after implicit literal synthesis. - # @rbs (left_token: Lexer::Token::Base, operator: Symbol, right_token: Lexer::Token::Base, lineno: Integer) -> Grammar::LexPrec::Declaration + # @rbs (left_token: Lexer::Token::Base, operator: ::Symbol, right_token: Lexer::Token::Base, lineno: Integer) -> Grammar::LexPrec::Declaration def add_lex_prec_rule(left_token:, operator:, right_token:, lineno:) # Register terminals so they are known to the symbol resolver add_term(id: left_token) unless left_token.s_value == "yyall" || @symbol_sets.key?(left_token.s_value) @@ -601,7 +601,7 @@ def synthesize_implicit_literal_token_patterns! # Validate that identity-component operators are not applied to self-pairs. # Self-pair is allowed only for length-only operators (-~ and -s). - # @rbs (Lexer::Token::Base left, Lexer::Token::Base right, Symbol operator, Integer lineno) -> void + # @rbs (Lexer::Token::Base left, Lexer::Token::Base right, ::Symbol operator, Integer lineno) -> void def validate_lex_prec_self_pair!(left, right, operator, lineno) return unless left.s_value == right.s_value diff --git a/lib/lrama/grammar/lex_prec.rb b/lib/lrama/grammar/lex_prec.rb index 22c787356..1d0e8c872 100644 --- a/lib/lrama/grammar/lex_prec.rb +++ b/lib/lrama/grammar/lex_prec.rb @@ -14,20 +14,20 @@ class Grammar # void + # @rbs (left_operand: Lexer::Token::Base, operator: ::Symbol, right_operand: Lexer::Token::Base, lineno: Integer) -> void def initialize(left_operand:, operator:, right_operand:, lineno:) @left_operand = left_operand @operator = operator @@ -72,11 +72,11 @@ def initialize(left_operand:, operator:, right_operand:, lineno:) class Rule attr_reader :left_token #: Lexer::Token::Base - attr_reader :operator #: Symbol + attr_reader :operator #: ::Symbol attr_reader :right_token #: Lexer::Token::Base attr_reader :lineno #: Integer - # @rbs (left_token: Lexer::Token::Base, operator: Symbol, right_token: Lexer::Token::Base, lineno: Integer) -> void + # @rbs (left_token: Lexer::Token::Base, operator: ::Symbol, right_token: Lexer::Token::Base, lineno: Integer) -> void def initialize(left_token:, operator:, right_token:, lineno:) @left_token = left_token @operator = operator @@ -119,7 +119,7 @@ def useless_rules end # Store a raw declaration for delayed expansion. - # @rbs (left_operand: Lexer::Token::Base, operator: Symbol, right_operand: Lexer::Token::Base, lineno: Integer) -> Declaration + # @rbs (left_operand: Lexer::Token::Base, operator: ::Symbol, right_operand: Lexer::Token::Base, lineno: Integer) -> Declaration def add_declaration(left_operand:, operator:, right_operand:, lineno:) decl = Declaration.new( left_operand: left_operand, @@ -131,7 +131,7 @@ def add_declaration(left_operand:, operator:, right_operand:, lineno:) decl end - # @rbs (left_token: Lexer::Token::Base, operator: Symbol, right_token: Lexer::Token::Base, lineno: Integer) -> Rule + # @rbs (left_token: Lexer::Token::Base, operator: ::Symbol, right_token: Lexer::Token::Base, lineno: Integer) -> Rule def add_rule(left_token:, operator:, right_token:, lineno:) rule = Rule.new( left_token: left_token, @@ -187,7 +187,7 @@ def right_token_length_winner(token1, token2) private - # @rbs (String token1, String token2, Array[Symbol] operators) -> bool + # @rbs (String token1, String token2, Array[::Symbol] operators) -> bool def pair_rule?(token1, token2, operators) @rules.any? do |rule| operators.include?(rule.operator) && diff --git a/lib/lrama/state/scanner_accepts.rb b/lib/lrama/state/scanner_accepts.rb index 9d7f54671..04b3bcff1 100644 --- a/lib/lrama/state/scanner_accepts.rb +++ b/lib/lrama/state/scanner_accepts.rb @@ -229,7 +229,7 @@ class CompleteProfileComputer attr_reader :table #: Hash[Integer, Grammar::TokenPattern] attr_reader :conflicts #: Array[Conflict] - # @rbs (ScannerFSA scanner_fsa, Grammar::LexPrec lex_prec, LengthPrecedences length_prec, Set[String] acceptable_tokens, ?Integer? parser_state_id, ?fallback: bool, ?token_order: Hash[String, Integer]) -> void + # @rbs (ScannerFSA scanner_fsa, Grammar::LexPrec lex_prec, LengthPrecedences length_prec, Set[String] acceptable_tokens, ?Integer? parser_state_id, ?fallback: bool, ?token_order: Hash[String, Integer], ?track_rule_usage: bool) -> void def initialize(scanner_fsa, lex_prec, length_prec, acceptable_tokens, parser_state_id = nil, fallback: false, token_order: {}, track_rule_usage: false) @scanner_fsa = scanner_fsa @resolver = ProfileResolver.new(lex_prec, length_prec, fallback: fallback, token_order: token_order, track_rule_usage: track_rule_usage) diff --git a/rbs_collection.lock.yaml b/rbs_collection.lock.yaml index 1bef45faf..11f78f455 100644 --- a/rbs_collection.lock.yaml +++ b/rbs_collection.lock.yaml @@ -1,6 +1,10 @@ --- path: ".gem_rbs_collection" gems: +- name: date + version: '0' + source: + type: stdlib - name: dbm version: '0' source: @@ -25,6 +29,10 @@ gems: version: '0' source: type: stdlib +- name: io-console + version: '0' + source: + type: stdlib - name: optparse version: '0' source: @@ -57,6 +65,10 @@ gems: revision: 01f77ce44d0ac13c8ebfc3cfdd7104320021f991 remote: https://github.com/ruby/gem_rbs_collection.git repo_dir: gems +- name: stringio + version: '0' + source: + type: stdlib - name: strscan version: '0' source: @@ -65,4 +77,8 @@ gems: version: '0' source: type: stdlib +- name: tsort + version: '0' + source: + type: stdlib gemfile_lock_path: Gemfile.lock diff --git a/sig/generated/lrama/grammar.rbs b/sig/generated/lrama/grammar.rbs index e55936e88..bd485b2d8 100644 --- a/sig/generated/lrama/grammar.rbs +++ b/sig/generated/lrama/grammar.rbs @@ -273,6 +273,41 @@ module Lrama # @rbs () -> Float? def pslr_max_state_ratio: () -> Float? + # True when the generated parser owns lexical analysis completely + # (%define api.pslr.lexer generated, the paper's own model). + # @rbs () -> bool + def pslr_lexer_generated?: () -> bool + + # Base table construction for PSLR: extended IELR (default) or + # canonical LR(1) for debugging unresolved-conflict reports whose + # contexts would otherwise be merged (section 3.4.3). + # @rbs () -> String + def pslr_tables: () -> String + + # @rbs () -> bool + def pslr_canonical_tables?: () -> bool + + # Terminals that the pseudo-scanner cannot produce: real terminals + # used by the grammar that have no %token-pattern (explicit or + # synthesized). Pure mode requires this list to be empty; bridge + # mode reports it so users know which tokens their yylex must keep + # producing. + # @rbs () -> Array[String] + def uncovered_pslr_terminals: () -> Array[String] + + # LAC (lookahead correction) setting from %define parse.lac. + # PSLR parsers default to full because exploratory parses are what + # keep merged-state pseudo-scanning canonical-equivalent; other + # parser types default to none for Bison compatibility. + # @rbs () -> String + def parse_lac: () -> String + + # @rbs () -> bool + def parse_lac_full?: () -> bool + + # @rbs () -> bool + def parse_lac_explicit_none?: () -> bool + # @rbs () -> Array[Grammar::TokenPattern] def layout_token_patterns: () -> Array[Grammar::TokenPattern] @@ -289,8 +324,8 @@ module Lrama # Add lex-prec rules from %lex-prec directive. # Stores as raw declaration for delayed expansion after implicit literal synthesis. - # @rbs (left_token: Lexer::Token::Base, operator: Symbol, right_token: Lexer::Token::Base, lineno: Integer) -> Grammar::LexPrec::Declaration - def add_lex_prec_rule: (left_token: Lexer::Token::Base, operator: Symbol, right_token: Lexer::Token::Base, lineno: Integer) -> Grammar::LexPrec::Declaration + # @rbs (left_token: Lexer::Token::Base, operator: ::Symbol, right_token: Lexer::Token::Base, lineno: Integer) -> Grammar::LexPrec::Declaration + def add_lex_prec_rule: (left_token: Lexer::Token::Base, operator: ::Symbol, right_token: Lexer::Token::Base, lineno: Integer) -> Grammar::LexPrec::Declaration # Finalize lexical declarations after implicit literal synthesis. # Expands yyall and symbol-set operands using the post-synthesis token universe. @@ -335,8 +370,8 @@ module Lrama # Validate that identity-component operators are not applied to self-pairs. # Self-pair is allowed only for length-only operators (-~ and -s). - # @rbs (Lexer::Token::Base left, Lexer::Token::Base right, Symbol operator, Integer lineno) -> void - def validate_lex_prec_self_pair!: (Lexer::Token::Base left, Lexer::Token::Base right, Symbol operator, Integer lineno) -> void + # @rbs (Lexer::Token::Base left, Lexer::Token::Base right, ::Symbol operator, Integer lineno) -> void + def validate_lex_prec_self_pair!: (Lexer::Token::Base left, Lexer::Token::Base right, ::Symbol operator, Integer lineno) -> void # @rbs (Lexer::Token::Base id) -> String? def implicit_literal_regex_pattern: (Lexer::Token::Base id) -> String? @@ -357,6 +392,20 @@ module Lrama # @rbs () -> void def validate_pslr_configuration!: () -> void + # @rbs () -> void + def validate_parse_lac!: () -> void + + # Layout tokens are consumed by the pseudo-scanner and never reach + # the parser, so they must not appear in grammar rules (section 3.6). + # @rbs () -> void + def validate_no_layout_tokens_in_rules!: () -> void + + # @rbs () -> void + def validate_pslr_lexer!: () -> void + + # @rbs () -> void + def validate_pslr_tables!: () -> void + # @rbs (Lexer::Token::Base operand) -> Array[Lexer::Token::Base] def expand_pslr_operand: (Lexer::Token::Base operand) -> Array[Lexer::Token::Base] diff --git a/sig/generated/lrama/grammar/lex_prec.rbs b/sig/generated/lrama/grammar/lex_prec.rbs index b6e6c5e21..c88e30410 100644 --- a/sig/generated/lrama/grammar/lex_prec.rbs +++ b/sig/generated/lrama/grammar/lex_prec.rbs @@ -13,56 +13,56 @@ module Lrama # void - def initialize: (left_operand: Lexer::Token::Base, operator: Symbol, right_operand: Lexer::Token::Base, lineno: Integer) -> void + # @rbs (left_operand: Lexer::Token::Base, operator: ::Symbol, right_operand: Lexer::Token::Base, lineno: Integer) -> void + def initialize: (left_operand: Lexer::Token::Base, operator: ::Symbol, right_operand: Lexer::Token::Base, lineno: Integer) -> void end class Rule attr_reader left_token: Lexer::Token::Base - attr_reader operator: Symbol + attr_reader operator: ::Symbol attr_reader right_token: Lexer::Token::Base attr_reader lineno: Integer - # @rbs (left_token: Lexer::Token::Base, operator: Symbol, right_token: Lexer::Token::Base, lineno: Integer) -> void - def initialize: (left_token: Lexer::Token::Base, operator: Symbol, right_token: Lexer::Token::Base, lineno: Integer) -> void + # @rbs (left_token: Lexer::Token::Base, operator: ::Symbol, right_token: Lexer::Token::Base, lineno: Integer) -> void + def initialize: (left_token: Lexer::Token::Base, operator: ::Symbol, right_token: Lexer::Token::Base, lineno: Integer) -> void # @rbs () -> String def left_name: () -> String @@ -89,11 +89,11 @@ module Lrama def useless_rules: () -> Array[Rule] # Store a raw declaration for delayed expansion. - # @rbs (left_operand: Lexer::Token::Base, operator: Symbol, right_operand: Lexer::Token::Base, lineno: Integer) -> Declaration - def add_declaration: (left_operand: Lexer::Token::Base, operator: Symbol, right_operand: Lexer::Token::Base, lineno: Integer) -> Declaration + # @rbs (left_operand: Lexer::Token::Base, operator: ::Symbol, right_operand: Lexer::Token::Base, lineno: Integer) -> Declaration + def add_declaration: (left_operand: Lexer::Token::Base, operator: ::Symbol, right_operand: Lexer::Token::Base, lineno: Integer) -> Declaration - # @rbs (left_token: Lexer::Token::Base, operator: Symbol, right_token: Lexer::Token::Base, lineno: Integer) -> Rule - def add_rule: (left_token: Lexer::Token::Base, operator: Symbol, right_token: Lexer::Token::Base, lineno: Integer) -> Rule + # @rbs (left_token: Lexer::Token::Base, operator: ::Symbol, right_token: Lexer::Token::Base, lineno: Integer) -> Rule + def add_rule: (left_token: Lexer::Token::Base, operator: ::Symbol, right_token: Lexer::Token::Base, lineno: Integer) -> Rule # True when winner explicitly wins an identity conflict against loser. # The relation is intentionally not transitive. @@ -114,8 +114,8 @@ module Lrama private - # @rbs (String token1, String token2, Array[Symbol] operators) -> bool - def pair_rule?: (String token1, String token2, Array[Symbol] operators) -> bool + # @rbs (String token1, String token2, Array[::Symbol] operators) -> bool + def pair_rule?: (String token1, String token2, Array[::Symbol] operators) -> bool end end end diff --git a/sig/generated/lrama/pslr/pairwise_resolution.rbs b/sig/generated/lrama/pslr/pairwise_resolution.rbs new file mode 100644 index 000000000..ccdaba302 --- /dev/null +++ b/sig/generated/lrama/pslr/pairwise_resolution.rbs @@ -0,0 +1,42 @@ +# Generated from lib/lrama/pslr/pairwise_resolution.rb with RBS::Inline + +module Lrama + module Pslr + # Pair-based PSLR state compatibility (paper section 3.4.3). + # + # The scanner FSA is scanned once to find every token pair that can be + # in an identity or length scanner conflict. Two acceptable-token sets + # are then compatible when no conflict pair distinguishes them: + # + # * both sets contain the same subset of the pair (the pseudo-scanner + # resolves the conflict identically, or leaves it identically + # unresolved -- unresolved/unresolved states merge), or + # * one set contains neither token of the pair (its match set for + # inputs manifesting the conflict is empty for this pair). + # + # Reducing the check to pairs is the precomputation the paper + # recommends for phase 3: the per-pair verdict no longer depends on + # the rest of the accept set, so state splitting does not have to + # re-run the profile resolver for every candidate merge. Layout + # tokens are members of every accept set, so pairs won by layout can + # never distinguish two states (the split-stable layout optimization + # of section 3.6 falls out of the presence test). + class PairwiseResolution + attr_reader conflict_pairs: Set[[ String, String ]] + + # @rbs (ScannerFSA scanner_fsa) -> void + def initialize: (ScannerFSA scanner_fsa) -> void + + # @rbs (Set[String] left_acc, Set[String] right_acc) -> bool + def compatible_accept_sets?: (Set[String] left_acc, Set[String] right_acc) -> bool + + private + + # @rbs ([String, String] pair, Set[String] left_acc, Set[String] right_acc) -> bool + def pair_compatible?: ([ String, String ] pair, Set[String] left_acc, Set[String] right_acc) -> bool + + # @rbs ([String, String] pair, Set[String] acc) -> Integer + def presence: ([ String, String ] pair, Set[String] acc) -> Integer + end + end +end diff --git a/sig/generated/lrama/reporter/pslr.rbs b/sig/generated/lrama/reporter/pslr.rbs index c9de08058..874f2e2ca 100644 --- a/sig/generated/lrama/reporter/pslr.rbs +++ b/sig/generated/lrama/reporter/pslr.rbs @@ -11,6 +11,24 @@ module Lrama private + # @rbs (IO io, Lrama::States states) -> void + def report_summary: (IO io, Lrama::States states) -> void + + # @rbs (IO io, Lrama::States states) -> void + def report_acceptable_tokens: (IO io, Lrama::States states) -> void + + # @rbs (IO io, Lrama::States states) -> void + def report_scanner_accepts: (IO io, Lrama::States states) -> void + + # @rbs (IO io, Lrama::States states) -> void + def report_unresolved_conflicts: (IO io, Lrama::States states) -> void + + # @rbs (IO io, Lrama::States states) -> void + def report_useless_lex_prec: (IO io, Lrama::States states) -> void + + # @rbs (IO io, Lrama::States states) -> void + def report_tie_candidates: (IO io, Lrama::States states) -> void + # @rbs (Numeric?) -> String def format_ratio: (Numeric?) -> String end diff --git a/sig/generated/lrama/scanner_fsa.rbs b/sig/generated/lrama/scanner_fsa.rbs index 7397a65d3..fd4e23c2f 100644 --- a/sig/generated/lrama/scanner_fsa.rbs +++ b/sig/generated/lrama/scanner_fsa.rbs @@ -46,7 +46,8 @@ module Lrama def acc_ss: (Integer state_id) -> Array[Grammar::TokenPattern] # Simulate the FSA on input string starting from initial state - # Returns all accepting states reached during the scan + # Returns all accepting states reached during the scan. + # The walk is byte-oriented to match the generated C runtime. # @rbs (String input) -> Array[{state: State, position: Integer, token: Grammar::TokenPattern}] def scan: (String input) -> Array[{ state: State, position: Integer, token: Grammar::TokenPattern }] @@ -113,7 +114,10 @@ module Lrama # @rbs (Array[Integer] counter, Array[NFAState] states) -> NFAState def create_nfa_state: (Array[Integer] counter, Array[NFAState] states) -> NFAState - ASCII_CHARS: Array[String] + # The input alphabet is bytes (0-255): keywords and operators are + # ASCII, and byte-oriented classes let multi-byte UTF-8 sequences + # pass through negated classes and dot unmodified. + BYTE_CHARS: Array[String] ANY_CHARS: Array[String] @@ -141,6 +145,13 @@ module Lrama # @rbs (String? stop_char) -> String def empty_sequence_message: (String? stop_char) -> String + # Compile a {NAME} reference by inlining the body of an already + # defined %token-pattern (paper Fig 3.2c/3.2d idiom). Self and + # forward references are rejected because @pattern_references only + # contains patterns compiled before the current one. + # @rbs (String pattern, Integer offset, Array[Integer] counter, Array[NFAState] states) -> [Fragment, Integer] + def compile_pattern_reference: (String pattern, Integer offset, Array[Integer] counter, Array[NFAState] states) -> [ Fragment, Integer ] + # @rbs (String pattern, Integer offset, Array[Integer] counter, Array[NFAState] states) -> [Fragment, Integer] def compile_escape: (String pattern, Integer offset, Array[Integer] counter, Array[NFAState] states) -> [ Fragment, Integer ] diff --git a/sig/generated/lrama/state/scanner_accepts.rbs b/sig/generated/lrama/state/scanner_accepts.rbs index 93e0260c9..0dee0e954 100644 --- a/sig/generated/lrama/state/scanner_accepts.rbs +++ b/sig/generated/lrama/state/scanner_accepts.rbs @@ -33,8 +33,10 @@ module Lrama attr_reader current_tokens: Array[String] - # @rbs (parser_state_id: Integer?, scanner_state_id: Integer, shorter_tokens: Array[String], selected_shorter_token: String?, current_tokens: Array[String]) -> void - def initialize: (parser_state_id: Integer?, scanner_state_id: Integer, shorter_tokens: Array[String], selected_shorter_token: String?, current_tokens: Array[String]) -> void + attr_reader witness: String? + + # @rbs (parser_state_id: Integer?, scanner_state_id: Integer, shorter_tokens: Array[String], selected_shorter_token: String?, current_tokens: Array[String], ?witness: String?) -> void + def initialize: (parser_state_id: Integer?, scanner_state_id: Integer, shorter_tokens: Array[String], selected_shorter_token: String?, current_tokens: Array[String], ?witness: String?) -> void end class ProfileOutcome @@ -64,8 +66,8 @@ module Lrama end class ProfileResolver - # @rbs (Grammar::LexPrec lex_prec, LengthPrecedences length_prec, ?fallback: bool, ?token_order: Hash[String, Integer]) -> void - def initialize: (Grammar::LexPrec lex_prec, LengthPrecedences length_prec, ?fallback: bool, ?token_order: Hash[String, Integer]) -> void + # @rbs (Grammar::LexPrec lex_prec, LengthPrecedences length_prec, ?fallback: bool, ?token_order: Hash[String, Integer], ?track_rule_usage: bool) -> void + def initialize: (Grammar::LexPrec lex_prec, LengthPrecedences length_prec, ?fallback: bool, ?token_order: Hash[String, Integer], ?track_rule_usage: bool) -> void # @rbs (Set[String] shorter_tokens, String? selected_shorter_token, Set[String] current_tokens) -> ProfileOutcome def resolve: (Set[String] shorter_tokens, String? selected_shorter_token, Set[String] current_tokens) -> ProfileOutcome @@ -90,6 +92,16 @@ module Lrama # @rbs (String candidate, Set[String] current_tokens) -> bool def fallback_identity_winner?: (String candidate, Set[String] current_tokens) -> bool + # Mark the %lex-prec rules that made the profile decision as used. + # Only normal-row scanner_accepts construction tracks usage: + # the fallback row spans the whole token universe and exploratory + # resolutions (state compatibility probes) must not mark rules. + # @rbs (String selected_shorter_token, Set[String] current_tokens) -> void + def mark_shorter_win_rules_used: (String selected_shorter_token, Set[String] current_tokens) -> void + + # @rbs (String winner, Set[String] shorter_tokens, Set[String] current_tokens) -> void + def mark_longer_win_rules_used: (String winner, Set[String] shorter_tokens, Set[String] current_tokens) -> void + # @rbs (String candidate, String other) -> bool def token_order_precedes?: (String candidate, String other) -> bool @@ -102,19 +114,19 @@ module Lrama attr_reader conflicts: Array[Conflict] - # @rbs (ScannerFSA scanner_fsa, Grammar::LexPrec lex_prec, LengthPrecedences length_prec, Set[String] acceptable_tokens, ?Integer? parser_state_id, ?fallback: bool, ?token_order: Hash[String, Integer]) -> void - def initialize: (ScannerFSA scanner_fsa, Grammar::LexPrec lex_prec, LengthPrecedences length_prec, Set[String] acceptable_tokens, ?Integer? parser_state_id, ?fallback: bool, ?token_order: Hash[String, Integer]) -> void + # @rbs (ScannerFSA scanner_fsa, Grammar::LexPrec lex_prec, LengthPrecedences length_prec, Set[String] acceptable_tokens, ?Integer? parser_state_id, ?fallback: bool, ?token_order: Hash[String, Integer], ?track_rule_usage: bool) -> void + def initialize: (ScannerFSA scanner_fsa, Grammar::LexPrec lex_prec, LengthPrecedences length_prec, Set[String] acceptable_tokens, ?Integer? parser_state_id, ?fallback: bool, ?token_order: Hash[String, Integer], ?track_rule_usage: bool) -> void # @rbs () -> void def compute: () -> void private - # @rbs (Integer fsa_state_id, Set[String] shorter_tokens, String? selected_shorter_token, Set[untyped] visited) -> void - def visit_transitions: (Integer fsa_state_id, Set[String] shorter_tokens, String? selected_shorter_token, Set[untyped] visited) -> void + # @rbs (Integer fsa_state_id, Set[String] shorter_tokens, String? selected_shorter_token, Set[untyped] visited, String path) -> void + def visit_transitions: (Integer fsa_state_id, Set[String] shorter_tokens, String? selected_shorter_token, Set[untyped] visited, String path) -> void - # @rbs (Integer fsa_state_id, Set[String] shorter_tokens, String? selected_shorter_token, Set[untyped] visited) -> void - def visit_state: (Integer fsa_state_id, Set[String] shorter_tokens, String? selected_shorter_token, Set[untyped] visited) -> void + # @rbs (Integer fsa_state_id, Set[String] shorter_tokens, String? selected_shorter_token, Set[untyped] visited, String path) -> void + def visit_state: (Integer fsa_state_id, Set[String] shorter_tokens, String? selected_shorter_token, Set[untyped] visited, String path) -> void # @rbs (ScannerFSA::State fsa_state) -> Set[String] def current_acceptable_tokens: (ScannerFSA::State fsa_state) -> Set[String] diff --git a/sig/generated/lrama/states.rbs b/sig/generated/lrama/states.rbs index 919b6a00c..df1e66f90 100644 --- a/sig/generated/lrama/states.rbs +++ b/sig/generated/lrama/states.rbs @@ -119,6 +119,11 @@ module Lrama def compute_la_sources_for_conflicted_states: () -> untyped + # Public accessor for reporting: acc(sp) with tie expansion and + # layout injection, as used by the scanner_accepts construction. + # @rbs (State state) -> Set[String] + def pslr_acceptable_tokens: (State state) -> Set[String] + private # @rbs (Grammar::Symbol accessing_symbol, Array[State::Item] kernels, Hash[Array[State::Item], State] states_created) -> [State, bool] @@ -278,9 +283,17 @@ module Lrama # @rbs (State state, State::lookahead_set filtered_lookaheads, ?State::lookahead_set pslr_lookaheads) -> bool def compatible_split_state?: (State state, State::lookahead_set filtered_lookaheads, ?State::lookahead_set pslr_lookaheads) -> bool + # Canonical LR(1) compatibility for %define pslr.tables canonical-lr: + # states merge only when their kernel lookahead sets are identical, + # so unresolved-conflict reports are not merged across contexts. + # @rbs (State state, State::lookahead_set filtered_lookaheads) -> bool + def canonical_lookaheads_match?: (State state, State::lookahead_set filtered_lookaheads) -> bool + # @rbs (State state, ?State::lookahead_set filtered_lookaheads) -> Array[[Integer, String?]] def pslr_state_signature: (State state, ?State::lookahead_set filtered_lookaheads) -> Array[[ Integer, String? ]] + # Pair-based PSLR compatibility (Def 3.4.3, reduced to token pairs as + # recommended by section 3.4.3 for splitting performance). # @rbs (Set[String] left_acc, Set[String] right_acc) -> bool def pslr_compatible_accept_sets?: (Set[String] left_acc, Set[String] right_acc) -> bool @@ -341,6 +354,11 @@ module Lrama # @rbs (Logger logger) -> void def validate_pslr_scanner_conflicts!: (Logger logger) -> void + # Pure mode owns lexical analysis completely, so every terminal must + # be producible by the pseudo-scanner. + # @rbs (Logger logger) -> void + def validate_pslr_pure_coverage!: (Logger logger) -> void + # @rbs (State::ScannerAccepts::Conflict conflict) -> String def pslr_scanner_conflict_message: (State::ScannerAccepts::Conflict conflict) -> String diff --git a/sig/generated/lrama/warnings/parse_lac.rbs b/sig/generated/lrama/warnings/parse_lac.rbs new file mode 100644 index 000000000..35a8785a4 --- /dev/null +++ b/sig/generated/lrama/warnings/parse_lac.rbs @@ -0,0 +1,13 @@ +# Generated from lib/lrama/warnings/parse_lac.rb with RBS::Inline + +module Lrama + class Warnings + class ParseLac + # @rbs (Logger logger, bool warnings) -> void + def initialize: (Logger logger, bool warnings) -> void + + # @rbs (Lrama::Grammar grammar) -> void + def warn: (Lrama::Grammar grammar) -> void + end + end +end diff --git a/sig/generated/lrama/warnings/pslr_coverage.rbs b/sig/generated/lrama/warnings/pslr_coverage.rbs new file mode 100644 index 000000000..c487f6b08 --- /dev/null +++ b/sig/generated/lrama/warnings/pslr_coverage.rbs @@ -0,0 +1,17 @@ +# Generated from lib/lrama/warnings/pslr_coverage.rb with RBS::Inline + +module Lrama + class Warnings + class PslrCoverage + # @rbs (Logger logger, bool warnings) -> void + def initialize: (Logger logger, bool warnings) -> void + + # Bridge mode allows partial coverage: terminals without a + # %token-pattern must keep coming from the user lexer. List them so + # the boundary between generated and hand-written scanning is + # explicit. + # @rbs (Lrama::Grammar grammar) -> void + def warn: (Lrama::Grammar grammar) -> void + end + end +end From 590bee331e092bfd67c27727daa3e962d8ff7d3e Mon Sep 17 00:00:00 2001 From: ydah Date: Wed, 8 Jul 2026 22:41:45 +0900 Subject: [PATCH 14/15] docs: update NEWS for the completed PSLR(1) feature set Document token actions, pure and bridge operating modes, the length-delimited scan API, {NAME} pattern references, byte-oriented classes, configurable LAC, canonical-lr table debugging, conflict witnesses, useless-rule reporting and the extended --report=pslr output. --- NEWS.md | 54 +++++++++++++++++++++++++++++++++++++----------------- 1 file changed, 37 insertions(+), 17 deletions(-) diff --git a/NEWS.md b/NEWS.md index 8a1a52ae6..caeca88ed 100644 --- a/NEWS.md +++ b/NEWS.md @@ -49,13 +49,20 @@ This adds the following PSLR-related grammar directives and integration points: - `%define lr.type pslr` enables PSLR parser generation - `%token-pattern` declares token candidates and their regular expressions for PSLR-aware lexical disambiguation +- `%token-action` attaches user code to matched token patterns; `yytext`, `yyleng`, `yylval`, and the accumulated + layout text are available in the action body - `%lex-prec` declares explicit lexical precedence for overlapping token patterns - `%symbol-set` declares reusable sets of terminal tokens for PSLR lexical declarations - `%lex-tie` expands parser-state acceptable-token sets for tied terminals - `%lex-no-tie` records an explicit no-tie decision for terminals with overlapping token patterns - `YYLAYOUT*` token patterns are recognized in every parser state and discarded by PSLR-aware lexers +- `%define api.pslr.lexer generated` enables pure mode: the generated parser owns lexical analysis completely +- `%define api.pslr.state-member` names the parser-state field to be shared with the lexer when using the generated helper macros (bridge mode) +- `%define parse.lac` controls lookahead correction (`full` by default for PSLR parsers, available to LALR/IELR parsers too) +- `%define pslr.tables canonical-lr` switches state splitting to canonical LR(1) compatibility for debugging merged conflict reports - `%define pslr.max-states` and `%define pslr.max-state-ratio` are Lrama-specific safety guards for state growth -- `%define api.pslr.state-member` names the parser-state field to be shared with the lexer when using the generated helper macros +- `--report=pslr` renders split metrics, per-state acceptable-token sets, scanner-accepts rows, unresolved-conflict + witnesses, useless `%lex-prec` rules, and lexical tie candidates Typical usage looks like this: @@ -156,25 +163,38 @@ use the subset accepted by the current state plus tied tokens and layout tokens. generated scanner universe so error handling can still identify and consume a token when the current parser state has no normal scanner decision. -`%token-pattern` currently uses an ASCII byte-oriented regular-expression subset for PSLR pseudo scanning. +`%token-pattern` currently uses a byte-oriented regular-expression subset for PSLR pseudo scanning. Supported constructs are literals, escaped literals such as `\/`, `\*`, `\+`, `\?`, `\(`, `\)`, `\[`, `\]`, and `\\`, grouping with `(...)`, alternation with `|`, repetition operators `*`, `+`, `?`, character classes with escapes such as `[\]]`, `[\\]`, and `[\n\t\r]`, ranges such as `[a-z]`, negated classes such as `[^*]`, common -escapes such as `\n`, `\t`, and `\r`, and `.`. The `.` operator matches ASCII bytes except newline; negated -character classes range over ASCII bytes. Unicode properties and full Ruby/Onigmo regexp syntax are not supported. -Unsupported or malformed constructs are rejected during generation rather than silently reinterpreted. Nullable -token patterns such as `//`, `/()/`, `/a*/`, `/a?/`, and `/a|/` are generation errors because PSLR token lexemes -must be non-empty. - -When the parser and lexer share a context through `%parse-param` / `%lex-param`, the generated header also -provides helpers such as `YYPSLR_PSEUDO_SCAN(...)`, so the lexer can choose a token based on the current parser -state. The paper-compatible scanning path needs the lexer to pass the unconsumed input prefix, not only an -already-decided token fragment, so legacy external lexer bridges may still be limited by the text they provide. - -PSLR parsers enable a lightweight LAC check in the generated parser so syntax errors caused by LR state merging, -default reductions, or `%nonassoc` error actions are detected before user semantic actions are run for the bad -lookahead. PSLR support is still experimental. Scoped lexical declarations, lexical nonterminals, and `%lex` -are not implemented yet. If you find any bugs, please report them. +escapes such as `\n`, `\t`, and `\r`, `.`, and `{NAME}` references that inline the body of an earlier +`%token-pattern` (self and forward references are rejected; write a literal brace as `\{`). The `.` operator +matches any byte except newline; negated character classes range over all bytes (0-255), so multi-byte UTF-8 +sequences pass through negated classes and `.` unmodified. Unicode properties and full Ruby/Onigmo regexp syntax +are not supported. Unsupported or malformed constructs are rejected during generation rather than silently +reinterpreted. Nullable token patterns such as `//`, `/()/`, `/a*/`, `/a?/`, and `/a|/` are generation errors +because PSLR token lexemes must be non-empty. + +Two operating modes are supported. In pure mode (`%define api.pslr.lexer generated`), the generated parser owns +lexical analysis: `yylex` is generated, layout tokens are skipped with their text accumulated, `%token-action` +bodies build semantic values from `yytext`/`yyleng`, and the caller hands the input to the parser with +`yypslr_set_input(input, len)` before `yyparse`. Pure mode requires a `%token-pattern` for every terminal. +In bridge mode (`%define api.pslr.state-member`), an existing hand-written lexer consults the pseudo-scanner +through helper macros such as `YYPSLR_PSEUDO_SCAN(Context, Input, InputLen, MatchLength)`; terminals without a +token pattern stay owned by the user lexer and are reported as a warning. The scan API is length-delimited +(`input` plus `input_len`) so inputs containing NUL bytes are handled correctly, and the scan result reports +`from_fallback` when the token came from the syntax-error fallback row. The paper-compatible scanning path needs +the lexer to pass the unconsumed input prefix, not only an already-decided token fragment, so legacy external +lexer bridges may still be limited by the text they provide. + +Unresolved pseudo-scanner conflict reports include a witness: an example input that reaches the conflicting +scanner profile. `%lex-prec` rules that never participate in a normal-row resolution are reported as useless. + +PSLR parsers enable LAC (lookahead correction, `%define parse.lac full`) by default so syntax errors caused by +LR state merging, default reductions, or `%nonassoc` error actions are detected before user semantic actions are +run for the bad lookahead, and expected-token lists in error messages are computed by exploratory parses. LAC can +also be enabled independently for LALR/IELR parsers. PSLR support is still experimental. Scoped lexical +declarations, lexical nonterminals, and `%lex` are not implemented yet. If you find any bugs, please report them. ## Lrama 0.7.1 (2025-12-24) From 225fa5cec6526e05cdcf17c4901938bd1b1f82c7 Mon Sep 17 00:00:00 2001 From: ydah Date: Wed, 8 Jul 2026 22:57:18 +0900 Subject: [PATCH 15/15] ci: fix Ruby 2.5 compatibility and vendored-gem ASCII check Array#filter_map is Ruby 2.7+, so the PSLR report renderer now uses map + compact; the test suite must keep passing on Ruby 2.5 because lrama is executed by BASERUBY when building CRuby. The US-ASCII source check scanned vendor/bundle, where rbs 4.0.3 ships a C file containing a UTF-8 comment, failing the job for every pull request. Exclude vendored gems: they are not sources of this repository. --- .github/workflows/test.yaml | 3 ++- lib/lrama/reporter/pslr.rb | 8 ++++---- 2 files changed, 6 insertions(+), 5 deletions(-) diff --git a/.github/workflows/test.yaml b/.github/workflows/test.yaml index 40bebbdd7..ac6ee5182 100644 --- a/.github/workflows/test.yaml +++ b/.github/workflows/test.yaml @@ -135,9 +135,10 @@ jobs: - run: bundle install # Copy from https://github.com/ruby/ruby/blob/cb9a47f2acd6e373ef868b890a9d07da6f565dd4/.github/workflows/check_misc.yml#L31 + # Exclude vendored gems: they are not part of this repository's sources. - name: Check if C-sources are US-ASCII run: | - grep -r -n --include='*.[chyS]' --include='*.asm' $'[^\t-~]' -- . && exit 1 || : + grep -r -n --exclude-dir=vendor --include='*.[chyS]' --include='*.asm' $'[^\t-~]' -- . && exit 1 || : # Copy from https://github.com/ruby/ruby/blob/089227e94823542acfdafa68541d330eee42ffea/.github/workflows/check_misc.yml#L27 - name: Check for trailing spaces diff --git a/lib/lrama/reporter/pslr.rb b/lib/lrama/reporter/pslr.rb index ccbf3015a..289b5307c 100644 --- a/lib/lrama/reporter/pslr.rb +++ b/lib/lrama/reporter/pslr.rb @@ -63,19 +63,19 @@ def report_scanner_accepts(io, states) io << "PSLR Scanner Accepts\n\n" states.states.each do |state| - cells = accepting_states.filter_map do |fsa_state| + cells = accepting_states.map {|fsa_state| token = scanner_accepts[state.id, fsa_state.id] "ss#{fsa_state.id}=>#{token.name}" if token - end + }.compact next if cells.empty? io << " State #{state.id}: #{cells.join(' ')}\n" end - fallback_cells = accepting_states.filter_map do |fsa_state| + fallback_cells = accepting_states.map {|fsa_state| token = scanner_accepts.fallback_table[fsa_state.id] "ss#{fsa_state.id}=>#{token.name}" if token - end + }.compact io << " Fallback: #{fallback_cells.join(' ')}\n" unless fallback_cells.empty? io << "\n" end