diff --git a/SYNTAX.md b/SYNTAX.md index 3065b0e6..154e7459 100644 --- a/SYNTAX.md +++ b/SYNTAX.md @@ -697,6 +697,7 @@ When a diagnostic is emitted, RustyLR will suggest the exact `%allow (` to inspect parser states. - Read conflict diagnostics before switching to GLR. +- Remember that RustyLR may expand safe, unobserved `P?` occurrences in GLR grammars when the generated optional helper would otherwise create a zero-consuming reduce cycle; unexpandable nullable cycles are reported as generation errors with rewrite guidance. - Check `context.expected_token()` and `context.can_feed(&token)` for interactive or editor-facing parsers. - Use `Debug` on a context for parser stack state and user data; in GLR mode, inspect the branch-grouped output. Use `to_tree_list()`/`to_tree_lists()` when syntax-tree inspection is needed. - Confirm the generated parser compiles and creates contexts with a compatible `rusty_lr` runtime version. Major/minor generator/runtime mismatches panic during context creation. diff --git a/llms.txt b/llms.txt index 8db6baa5..77350187 100644 --- a/llms.txt +++ b/llms.txt @@ -29,6 +29,8 @@ Generated contexts implement `Clone` when their runtime storage and user data sa Context `Debug` output reports parser stack state and user data; GLR output is grouped by active branch. Use the explicit tree APIs for syntax-tree inspection when the `tree` feature is enabled. +In GLR grammars, RustyLR may expand safe, unobserved auto-generated `P?` occurrences when their empty branch would otherwise form a zero-consuming reduce cycle. Unexpandable nullable cycles are reported as generation errors with rewrite guidance. + Important files: - `README.md`: Main quick start and examples. - `USING_RUSTYLR_WITH_AI.md`: Short implementation guide for AI coding agents using RustyLR in another Rust project. diff --git a/rusty_lr_buildscript/src/lib.rs b/rusty_lr_buildscript/src/lib.rs index 88bd424c..92323c64 100644 --- a/rusty_lr_buildscript/src/lib.rs +++ b/rusty_lr_buildscript/src/lib.rs @@ -869,7 +869,7 @@ impl Builder { .with_labels(vec![Label::primary(file_id, range) .with_message("unknown diagnostic name")]) .with_notes(vec![ - "valid diagnostic names: nonterm_unreachable, unused_nonterm_data, nonterm_unproductive, unused_terminals, terminals_merged, redundant_rule_removed, unit_production_eliminated, reduce_reduce_conflict_resolved, shift_reduce_conflict_resolved, shift_reduce_conflict_glr, reduce_reduce_conflict_glr".to_string(), + "valid diagnostic names: nonterm_unreachable, unused_nonterm_data, nonterm_unproductive, unused_terminals, terminals_merged, redundant_rule_removed, unit_production_eliminated, glr_optional_expanded, reduce_reduce_conflict_resolved, shift_reduce_conflict_resolved, shift_reduce_conflict_glr, reduce_reduce_conflict_glr".to_string(), ]) } @@ -889,6 +889,27 @@ impl Builder { "refer to https://github.com/ehwan/RustyLR/blob/main/SYNTAX.md#substitution-errors".to_string(), ]) } + ParseError::GlrNullableReduceCycle { + location, + nullable_rule, + state, + reason, + help, + } => { + let range = span_manager.get_byterange(&location).unwrap_or(0..0); + Diagnostic::error() + .with_message("GLR nullable reduce cycle cannot be expanded safely") + .with_labels(vec![Label::primary(file_id, range).with_message( + "this nullable production participates in a zero-consuming GLR cycle", + )]) + .with_notes(vec![ + format!( + "reducing `{nullable_rule}` in state {state} returns to the same state without consuming input" + ), + format!("why RustyLR cannot rewrite it automatically: {reason}"), + format!("how to fix it: {help}"), + ]) + } }; let writer = self.stream(); @@ -905,7 +926,49 @@ impl Builder { grammar.optimize(25); } - grammar.builder = grammar.create_builder(); + grammar.builder = match grammar.create_builder() { + Ok(builder) => builder, + Err(err) => { + let diag = match err { + ParseError::GlrNullableReduceCycle { + location, + nullable_rule, + state, + reason, + help, + } => { + let range = span_manager.get_byterange(&location).unwrap_or(0..0); + Diagnostic::error() + .with_message("GLR nullable reduce cycle cannot be expanded safely") + .with_labels(vec![Label::primary(file_id, range).with_message( + "this nullable production participates in a zero-consuming GLR cycle", + )]) + .with_notes(vec![ + format!( + "reducing `{nullable_rule}` in state {state} returns to the same state without consuming input" + ), + format!("why RustyLR cannot rewrite it automatically: {reason}"), + format!("how to fix it: {help}"), + ]) + } + err => { + let range = err + .locations() + .first() + .and_then(|loc| span_manager.get_byterange(loc)) + .unwrap_or(0..0); + Diagnostic::error() + .with_message(err.short_message()) + .with_labels(vec![Label::primary(file_id, range)]) + } + }; + let writer = self.stream(); + let config = codespan_reporting::term::Config::default(); + term::emit_to_write_style(&mut writer.lock(), &config, &files, &diag) + .expect("Failed to write to stderr"); + return Err(diag.message); + } + }; let diags_collector = grammar.build_grammar(); let mut conflict_errors = Vec::new(); @@ -1153,6 +1216,31 @@ impl Builder { .to_string(), ]) } + rusty_lr_parser::error::Info::GlrOptionalExpanded { + nonterm_name, + rule_location, + before, + after, + } => { + let nonterm_range = span_manager + .get_byterange(&nonterm_name.location()) + .unwrap_or(0..0); + let rule_range = span_manager.get_byterange(rule_location).unwrap_or(0..0); + let mut notes = vec![ + "This GLR production was expanded because an auto-generated optional empty branch formed a zero-consuming reduce cycle.".to_string(), + format!("before: {before}"), + ]; + notes.extend(after.iter().map(|rule| format!("after: {rule}"))); + Diagnostic::note() + .with_message("Production rule expanded") + .with_labels(vec![ + Label::primary(file_id, rule_range) + .with_message("production expanded here"), + Label::secondary(file_id, nonterm_range) + .with_message("left-hand side defined here"), + ]) + .with_notes(notes) + } rusty_lr_parser::error::Info::ReduceReduceConflictResolved { max_priority, reduce_rules, @@ -1458,7 +1546,8 @@ impl Builder { let should_print = match info_variant { rusty_lr_parser::error::Info::TerminalsMerged { .. } | rusty_lr_parser::error::Info::RedundantRuleRemoved { .. } - | rusty_lr_parser::error::Info::UnitProductionEliminated { .. } => true, + | rusty_lr_parser::error::Info::UnitProductionEliminated { .. } + | rusty_lr_parser::error::Info::GlrOptionalExpanded { .. } => true, _ => false, }; if should_print { diff --git a/rusty_lr_derive/src/lib.rs b/rusty_lr_derive/src/lib.rs index b42ff4c7..3de23921 100644 --- a/rusty_lr_derive/src/lib.rs +++ b/rusty_lr_derive/src/lib.rs @@ -46,7 +46,10 @@ pub fn lr1(input: TokenStream) -> TokenStream { if grammar.optimize { grammar.optimize(15); } - grammar.builder = grammar.create_builder(); + grammar.builder = match grammar.create_builder() { + Ok(builder) => builder, + Err(e) => return e.to_compile_error(&span_manager).into(), + }; let diags = grammar.build_grammar(); if !grammar.glr { if let Some(((term, shift_rules, _), reduce_rules)) = diff --git a/rusty_lr_executable/src/lsp/diagnostics.rs b/rusty_lr_executable/src/lsp/diagnostics.rs index aa9f0dd6..f40aff93 100644 --- a/rusty_lr_executable/src/lsp/diagnostics.rs +++ b/rusty_lr_executable/src/lsp/diagnostics.rs @@ -1,6 +1,7 @@ use lsp_types::{Diagnostic, DiagnosticSeverity, Range}; use proc_macro2::{Spacing, TokenStream, TokenTree}; use rusty_lr_core::{TerminalSymbol, production::LR0ItemRef}; +use rusty_lr_parser::error::ParseError; use rusty_lr_parser::grammar::Grammar; use serde_json::json; use std::collections::{BTreeMap, BTreeSet}; @@ -233,7 +234,53 @@ pub fn compile_and_get_diagnostics(content: &str) -> Vec { if grammar.optimize { grammar.optimize(25); } - grammar.builder = grammar.create_builder(); + grammar.builder = match grammar.create_builder() { + Ok(builder) => builder, + Err(err) => { + match err { + ParseError::GlrNullableReduceCycle { + location, + nullable_rule, + state, + reason, + help, + } => { + let range = span_manager.get_byterange(&location).unwrap_or(0..0); + diagnostics.push(Diagnostic { + range: range_to_lsp_range(content, range), + severity: Some(DiagnosticSeverity::ERROR), + code: None, + code_description: None, + source: Some("rusty_lr".to_string()), + message: format!( + "GLR nullable reduce cycle cannot be expanded safely: reducing `{nullable_rule}` in state {state} returns to the same state without consuming input. Reason: {reason}. Fix: {help}" + ), + related_information: None, + tags: None, + data: None, + }); + } + err => { + let msg = err.short_message(); + for loc in err.locations() { + let range = span_manager.get_byterange(&loc).unwrap_or(0..0); + diagnostics.push(Diagnostic { + range: range_to_lsp_range(content, range), + severity: Some(DiagnosticSeverity::ERROR), + code: None, + code_description: None, + source: Some("rusty_lr".to_string()), + message: msg.clone(), + related_information: None, + tags: None, + data: None, + }); + } + } + } + return diagnostics; + } + }; // 7. Verify Shift/Reduce and Reduce/Reduce conflicts in non-GLR mode let diags_collector = grammar.build_grammar(); diff --git a/rusty_lr_parser/src/error.rs b/rusty_lr_parser/src/error.rs index c63b9330..e760299b 100644 --- a/rusty_lr_parser/src/error.rs +++ b/rusty_lr_parser/src/error.rs @@ -138,6 +138,15 @@ pub enum ParseError { /// unknown diagnostic name allowed InvalidAllowDiagnostic { location: Location, name: String }, + + /// a nullable production can be reduced forever in GLR without consuming input + GlrNullableReduceCycle { + location: Location, + nullable_rule: String, + state: usize, + reason: String, + help: String, + }, } impl ArgError { pub fn to_compile_error( @@ -282,6 +291,7 @@ impl ParseError { ParseError::CircularDependency { location, .. } => vec![*location], ParseError::MaxSubstitutionDepthExceeded { location, .. } => vec![*location], ParseError::InvalidAllowDiagnostic { location, .. } => vec![*location], + ParseError::GlrNullableReduceCycle { location, .. } => vec![*location], } } @@ -358,6 +368,16 @@ impl ParseError { ParseError::InvalidAllowDiagnostic { name, .. } => { format!("unknown diagnostic name: `{}`", name) } + ParseError::GlrNullableReduceCycle { + nullable_rule, + reason, + .. + } => { + format!( + "GLR nullable reduce cycle cannot be expanded safely: `{}` ({})", + nullable_rule, reason + ) + } } } } @@ -543,6 +563,12 @@ pub enum Info { nonterm_name: Located, rule_location: Location, }, + GlrOptionalExpanded { + nonterm_name: Located, + rule_location: Location, + before: String, + after: Vec, + }, ReduceReduceConflictResolved { max_priority: usize, reduce_rules: Vec, @@ -578,6 +604,7 @@ impl Info { Info::TerminalsMerged { .. } => "terminals_merged", Info::RedundantRuleRemoved { .. } => "redundant_rule_removed", Info::UnitProductionEliminated { .. } => "unit_production_eliminated", + Info::GlrOptionalExpanded { .. } => "glr_optional_expanded", Info::ReduceReduceConflictResolved { .. } => "reduce_reduce_conflict_resolved", Info::ShiftReduceConflictResolvedShift { .. } => "shift_reduce_conflict_resolved", Info::ShiftReduceConflictResolvedReduce { .. } => "shift_reduce_conflict_resolved", @@ -592,6 +619,10 @@ impl Info { let name = nonterm_name.value(); format!("%allow {}({});", self.name(), name) } + Info::GlrOptionalExpanded { nonterm_name, .. } => { + let name = nonterm_name.value(); + format!("%allow {}({});", self.name(), name) + } Info::RedundantRuleRemoved { rule_location } => { let mut name = String::new(); for nonterm in &grammar.nonterminals { diff --git a/rusty_lr_parser/src/grammar.rs b/rusty_lr_parser/src/grammar.rs index 1b432768..ee7ff6aa 100644 --- a/rusty_lr_parser/src/grammar.rs +++ b/rusty_lr_parser/src/grammar.rs @@ -201,6 +201,514 @@ impl Grammar { } } + fn rule_id_to_indices(&self, mut rule_idx: usize) -> Option<(usize, usize)> { + // Rule ids are assigned by walking non-terminals in storage order; keep the inverse + // mapping local so table analyses can talk back to the editable grammar. + for (nonterm_idx, nonterm) in self.nonterminals.iter().enumerate() { + if rule_idx < nonterm.rules.len() { + return Some((nonterm_idx, rule_idx)); + } + rule_idx -= nonterm.rules.len(); + } + None + } + + fn create_builder_from_current_rules( + &self, + ) -> rusty_lr_core::builder::Grammar, usize> { + let mut grammar: rusty_lr_core::builder::Grammar, usize> = + rusty_lr_core::builder::Grammar::new(); + + for (term_idx, term_info) in self.terminals.iter().enumerate() { + if let Some(level) = &term_info.precedence { + let class = self.terminal_class_id[term_idx]; + if !grammar.add_precedence(TerminalSymbol::Terminal(class), *level.value()) { + unreachable!("set_reduce_type error"); + } + } + } + if let Some(level) = self.error_precedence { + if !grammar.add_precedence(TerminalSymbol::Error, level) { + unreachable!("set_reduce_type error"); + } + } + grammar.set_precedence_types(self.precedence_types.iter().map(|op| *op.value()).collect()); + + for (nonterm_id, nonterm) in self.nonterminals.iter().enumerate() { + for rule in nonterm.rules.iter() { + let tokens = rule + .tokens + .iter() + .map(|token_mapped| token_mapped.symbol) + .collect(); + + grammar.add_rule( + nonterm_id, + tokens, + rule.prec + .map(Located::into_value) + .unwrap_or(Precedence::None), + rule.dprec.map_or(0, Located::into_value), + ); + } + } + + grammar + } + + fn is_optional_epsilon_self_loop_candidate( + &self, + nonterm_idx: usize, + local_rule_idx: usize, + ) -> bool { + let nonterm = &self.nonterminals[nonterm_idx]; + let rule = &nonterm.rules[local_rule_idx]; + + // Only RustyLR-created optional symbols have known branch semantics; user-written + // nullable productions may contain intentional actions and must not be rewritten. + if nonterm.nonterm_type + != Some(rusty_lr_core::parser::nonterminal::NonTerminalType::Optional) + || !nonterm.is_auto_generated() + || nonterm.is_protected() + { + return false; + } + + // The dangerous closure edge must come from the empty branch itself. Non-empty + // branches consume input and therefore cannot be the zero-progress cycle we are fixing. + rule.tokens.is_empty() + } + + fn rule_uses_positional_bison_variables(rule: &Rule) -> bool { + // Expanding an optional occurrence changes token positions. Named bindings remain stable, + // but already-rewritten `$n` or `@n` identifiers would point at the wrong symbol. + (0..rule.tokens.len()).any(|idx| { + rule.reduce_action_contains_ident(&format!("__rustylr_data_{idx}")) + || rule.reduce_action_contains_ident(&format!("__rustylr_location_{idx}")) + }) + } + + fn can_expand_optional_occurrence(rule: &Rule, token_idx: usize) -> bool { + let token = &rule.tokens[token_idx]; + + // Existing reduce-action chains mean another optimization has semantic work attached to + // this exact symbol occurrence, so leave it intact instead of guessing how to distribute it. + if !token.reduce_action_chains.is_empty() { + return false; + } + + // Identity actions are positional by definition; rewriting their RHS shape would require + // remapping the identity index, so keep this conservative pass focused on custom/empty actions. + if matches!(rule.reduce_action, Some(ReduceAction::Identity(_))) { + return false; + } + + if Self::rule_uses_positional_bison_variables(rule) { + return false; + } + + if let Some(mapto) = &token.mapto { + // If the optional's value or location is observed, replacing it with a present/absent + // structural branch would change the user's reduce action contract. + if rule.reduce_action_contains_ident(mapto.value().as_str()) + || rule.reduce_action_contains_ident(&utils::location_variable_name( + mapto.value().as_str(), + )) + { + return false; + } + } + + true + } + + fn mapped_symbol_pretty_name(&self, token: &MappedSymbol) -> String { + match token.symbol { + Symbol::Terminal(term) => self.class_pretty_name_list(term, 5), + Symbol::NonTerminal(nonterm) => self.nonterm_pretty_name(nonterm), + } + } + + fn production_pretty_name(&self, nonterm_idx: usize, tokens: &[MappedSymbol]) -> String { + // Keep this close to the generated grammar comment format so the diagnostic explains the + // exact table-level rewrite without exposing internal rule ids. + let lhs = self.nonterm_pretty_name(nonterm_idx); + if tokens.is_empty() { + format!("{lhs} ->") + } else { + let rhs = tokens + .iter() + .map(|token| self.mapped_symbol_pretty_name(token)) + .collect::>() + .join(" "); + format!("{lhs} -> {rhs}") + } + } + + fn expand_optional_occurrences(&mut self, optional_idx: usize) -> bool { + let Some(present_token) = self.nonterminals[optional_idx] + .rules + .iter() + .find(|rule| rule.tokens.len() == 1) + .map(|rule| rule.tokens[0].clone()) + else { + return false; + }; + + let mut changed = false; + for nonterm_idx in 0..self.nonterminals.len() { + if nonterm_idx == optional_idx { + continue; + } + + let old_rules = std::mem::take(&mut self.nonterminals[nonterm_idx].rules); + let mut new_rules = Vec::with_capacity(old_rules.len()); + + for rule in old_rules { + if !rule + .tokens + .iter() + .any(|token| token.symbol == Symbol::NonTerminal(optional_idx)) + { + new_rules.push(rule); + continue; + } + + let mut variants = vec![Rule { + tokens: Vec::with_capacity(rule.tokens.len()), + ..rule.clone() + }]; + let mut expanded_this_rule = false; + let mut can_keep_expanding = true; + + for (token_idx, token) in rule.tokens.iter().enumerate() { + if token.symbol != Symbol::NonTerminal(optional_idx) { + for variant in &mut variants { + variant.tokens.push(token.clone()); + } + continue; + } + + if !Self::can_expand_optional_occurrence(&rule, token_idx) { + for variant in &mut variants { + variant.tokens.push(token.clone()); + } + can_keep_expanding = false; + continue; + } + + // Split the occurrence into its absent branch and its present branch. The + // parent action does not observe the optional value, so the present branch + // carries only the underlying grammar symbol. + let mut present = present_token.clone(); + present.mapto = None; + present.location = token.location; + + let mut present_variants = variants.clone(); + for variant in &mut present_variants { + variant.tokens.push(present.clone()); + } + variants.extend(present_variants); + expanded_this_rule = true; + } + + if expanded_this_rule && can_keep_expanding { + changed = true; + self.infos.push(Info::GlrOptionalExpanded { + nonterm_name: self.nonterminals[nonterm_idx].name.clone(), + rule_location: rule.location(), + before: self.production_pretty_name(nonterm_idx, &rule.tokens), + after: variants + .iter() + .map(|variant| { + self.production_pretty_name(nonterm_idx, &variant.tokens) + }) + .collect(), + }); + new_rules.extend(variants); + } else { + new_rules.push(rule); + } + } + + self.nonterminals[nonterm_idx].rules = new_rules; + } + + let still_referenced = self.nonterminals.iter().any(|nonterm| { + nonterm.rules.iter().any(|rule| { + rule.tokens + .iter() + .any(|token| token.symbol == Symbol::NonTerminal(optional_idx)) + }) + }); + if changed && !still_referenced { + // Once every occurrence has been structurally expanded, the generated helper has no + // remaining parser role and can be left empty for the normal cleanup path. + self.nonterminals[optional_idx].rules.clear(); + } + + changed + } + + fn optional_expansion_error_context( + &self, + optional_idx: usize, + ) -> Option<(Location, String, String)> { + for nonterm in &self.nonterminals { + for rule in &nonterm.rules { + for (token_idx, token) in rule.tokens.iter().enumerate() { + if token.symbol != Symbol::NonTerminal(optional_idx) { + continue; + } + + let production = self.production_pretty_name( + *self.nonterminals_index.get(nonterm.name.value()).unwrap(), + &rule.tokens, + ); + let help = format!( + "Rewrite `{production}` explicitly into one production without the optional branch and one production with the optional branch present." + ); + + if !token.reduce_action_chains.is_empty() { + return Some(( + rule.location(), + format!( + "`{production}` has reduce-action chains attached to the optional occurrence" + ), + help, + )); + } + if matches!(rule.reduce_action, Some(ReduceAction::Identity(_))) { + return Some(( + rule.location(), + format!( + "`{production}` uses an identity reduce action whose positional result would change" + ), + help, + )); + } + if Self::rule_uses_positional_bison_variables(rule) { + return Some(( + rule.location(), + format!( + "`{production}` uses positional `$n` or `@n` references, so expansion would change token numbering" + ), + help, + )); + } + if let Some(mapto) = &token.mapto { + if rule.reduce_action_contains_ident(mapto.value().as_str()) { + return Some(( + rule.location(), + format!( + "`{production}` reads the optional value `{}` in its reduce action", + mapto.value() + ), + help, + )); + } + if rule.reduce_action_contains_ident(&utils::location_variable_name( + mapto.value().as_str(), + )) { + return Some(( + rule.location(), + format!( + "`{production}` reads the optional location `@{}` in its reduce action", + mapto.value() + ), + help, + )); + } + } + + if !Self::can_expand_optional_occurrence(rule, token_idx) { + return Some(( + rule.location(), + format!("`{production}` cannot be expanded without changing semantics"), + help, + )); + } + } + } + } + None + } + + fn glr_nullable_cycle_error( + &self, + state_idx: usize, + nonterm_idx: usize, + local_rule_idx: usize, + ) -> ParseError { + let nonterm = &self.nonterminals[nonterm_idx]; + let rule = &nonterm.rules[local_rule_idx]; + let nullable_rule = self.production_pretty_name(nonterm_idx, &rule.tokens); + + let default_location = nonterm.root_location.unwrap_or_else(|| rule.location()); + let (location, reason, help) = match nonterm.nonterm_type { + Some(rusty_lr_core::parser::nonterminal::NonTerminalType::Optional) => { + self.optional_expansion_error_context(nonterm_idx).unwrap_or_else(|| { + ( + default_location, + format!( + "`{}` is an optional helper, but no safe occurrence remained for automatic expansion", + nonterm.pretty_name + ), + "Rewrite the optional occurrence explicitly as separate absent and present productions.".to_string(), + ) + }) + } + Some(rusty_lr_core::parser::nonterminal::NonTerminalType::Star) => ( + default_location, + format!( + "`{}` is a zero-or-more repetition helper; `*` denotes an unbounded family of productions and cannot be expanded into a finite replacement here", + nonterm.pretty_name + ), + "Rewrite the grammar with an explicit non-empty list helper (`P+`) or factor the nullable repetition out of the left-recursive GLR position.".to_string(), + ), + _ if nonterm.is_auto_generated() => ( + default_location, + format!( + "`{}` is an auto-generated nullable helper that RustyLR does not know how to expand safely", + nonterm.pretty_name + ), + "Rewrite the surrounding grammar manually so the nullable helper is not reduced back to the same GLR state without consuming input.".to_string(), + ), + _ => ( + rule.location(), + format!( + "`{}` is a user-defined nullable production; RustyLR cannot remove or duplicate its semantic behavior automatically", + nullable_rule + ), + "Rewrite the grammar manually so this nullable production is not in a GLR reduce cycle, or make the recursive path consume a terminal before returning to the same state.".to_string(), + ), + }; + + ParseError::GlrNullableReduceCycle { + location, + nullable_rule, + state: state_idx, + reason, + help, + } + } + + fn find_glr_nullable_reduce_cycle_error( + &self, + states: &[rusty_lr_core::builder::State, usize>], + ) -> Option { + for (state_idx, state) in states.iter().enumerate() { + for reduce_rules in state.reduce_map.values() { + for &rule_idx in reduce_rules { + let Some((nonterm_idx, local_rule_idx)) = self.rule_id_to_indices(rule_idx) + else { + continue; + }; + if !self.nonterminals[nonterm_idx].rules[local_rule_idx] + .tokens + .is_empty() + { + continue; + } + if state + .shift_goto_map_nonterm + .get(&nonterm_idx) + .is_some_and(|target| *target == state_idx) + { + return Some(self.glr_nullable_cycle_error( + state_idx, + nonterm_idx, + local_rule_idx, + )); + } + } + } + } + None + } + + fn expand_glr_epsilon_self_loop_optionals(&mut self) -> Result<(), ParseError> { + if !self.glr { + return Ok(()); + } + + // A rewrite can reveal another nullable self-loop through a different optional helper, so + // iterate to a fixed point while bounding the pass against accidental expansion churn. + for _ in 0..self.nonterminals.len() { + let mut builder = self.create_builder_from_current_rules(); + let augmented_idx = *self.nonterminals_index.get(utils::AUGMENTED_NAME).unwrap(); + let mut collector = rusty_lr_core::builder::DiagnosticCollector::new(true); + let states = if self.lalr { + builder.build_lalr(augmented_idx, &mut collector) + } else { + builder.build(augmented_idx, &mut collector) + } + .unwrap_or_else(|err| { + unreachable!( + "Error building grammar during GLR nullable expansion: {:?}", + err + ) + }) + .states; + + let mut rewrite_target = None; + 'states: for (state_idx, state) in states.iter().enumerate() { + for reduce_rules in state.reduce_map.values() { + for &rule_idx in reduce_rules { + let Some((nonterm_idx, local_rule_idx)) = self.rule_id_to_indices(rule_idx) + else { + continue; + }; + if !self + .is_optional_epsilon_self_loop_candidate(nonterm_idx, local_rule_idx) + { + continue; + } + if state + .shift_goto_map_nonterm + .iter() + .any(|(nonterm, target)| { + *nonterm == nonterm_idx && *target == state_idx + }) + { + rewrite_target = Some(nonterm_idx); + break 'states; + } + } + } + } + + let Some(optional_idx) = rewrite_target else { + break; + }; + if !self.expand_optional_occurrences(optional_idx) { + break; + } + } + + let mut builder = self.create_builder_from_current_rules(); + let augmented_idx = *self.nonterminals_index.get(utils::AUGMENTED_NAME).unwrap(); + let mut collector = rusty_lr_core::builder::DiagnosticCollector::new(true); + let states = if self.lalr { + builder.build_lalr(augmented_idx, &mut collector) + } else { + builder.build(augmented_idx, &mut collector) + } + .unwrap_or_else(|err| { + unreachable!( + "Error building grammar during GLR nullable cycle validation: {:?}", + err + ) + }) + .states; + + if let Some(err) = self.find_glr_nullable_reduce_cycle_error(&states) { + return Err(err); + } + + Ok(()) + } + /// Resolved Rust type for `%tokentype`, after substitutions such as `$tokentype` /// and storage modifiers such as `box` have been stripped. pub fn token_type(&self) -> &TokenStream { @@ -376,6 +884,15 @@ impl Grammar { } } } + Info::GlrOptionalExpanded { nonterm_name, .. } => { + for opt_target in scopes { + if let Some(ResolvedAllowTarget::Name(name)) = opt_target { + if name == nonterm_name.value() { + return true; + } + } + } + } Info::RedundantRuleRemoved { rule_location } => { for nonterm in &self.nonterminals { for rule in &nonterm.rules { @@ -2091,6 +2608,7 @@ impl Grammar { "terminals_merged", "redundant_rule_removed", "unit_production_eliminated", + "glr_optional_expanded", "reduce_reduce_conflict_resolved", "shift_reduce_conflict_resolved", "shift_reduce_conflict_glr", @@ -2879,54 +3397,12 @@ impl Grammar { /// create the rusty_lr_core::Grammar from the parsed CFGs pub fn create_builder( &mut self, - ) -> rusty_lr_core::builder::Grammar, usize> { - let mut grammar: rusty_lr_core::builder::Grammar, usize> = - rusty_lr_core::builder::Grammar::new(); - - let mut rules = Vec::new(); - for (nonterm_idx, nonterminal) in self.nonterminals.iter().enumerate() { - rules.reserve(nonterminal.rules.len()); - for rule in 0..nonterminal.rules.len() { - rules.push((nonterm_idx, rule)); - } - } - - for (term_idx, term_info) in self.terminals.iter().enumerate() { - if let Some(level) = &term_info.precedence { - let class = self.terminal_class_id[term_idx]; - if !grammar.add_precedence(TerminalSymbol::Terminal(class), *level.value()) { - unreachable!("set_reduce_type error"); - } - } - } - if let Some(level) = self.error_precedence { - if !grammar.add_precedence(TerminalSymbol::Error, level) { - unreachable!("set_reduce_type error"); - } - } - grammar.set_precedence_types(self.precedence_types.iter().map(|op| *op.value()).collect()); - - // add rules - for (nonterm_id, nonterm) in self.nonterminals.iter().enumerate() { - for rule in nonterm.rules.iter() { - let tokens = rule - .tokens - .iter() - .map(|token_mapped| token_mapped.symbol) - .collect(); - - grammar.add_rule( - nonterm_id, - tokens, - rule.prec - .map(Located::into_value) - .unwrap_or(Precedence::None), - rule.dprec.map_or(0, Located::into_value), - ); - } - } - - grammar + ) -> Result, usize>, ParseError> + { + // This normalization is not an optimization knob: it prevents GLR reduce closure from + // repeatedly applying an auto-generated optional empty branch without consuming input. + self.expand_glr_epsilon_self_loop_optionals()?; + Ok(self.create_builder_from_current_rules()) } pub fn build_grammar( @@ -3924,7 +4400,9 @@ mod tests { let grammar_args = Grammar::parse_args(input).expect("Failed to parse grammar args"); let mut grammar = Grammar::from_grammar_args(grammar_args).expect("Failed to build grammar"); - grammar.builder = grammar.create_builder(); + grammar.builder = grammar + .create_builder() + .expect("Failed to create parser builder"); let _ = grammar.build_grammar(); let code = grammar.emit_compiletime(); @@ -3965,7 +4443,9 @@ mod tests { let grammar_args = Grammar::parse_args(input).expect("Failed to parse grammar args"); let mut grammar = Grammar::from_grammar_args(grammar_args).expect("Failed to build grammar"); - grammar.builder = grammar.create_builder(); + grammar.builder = grammar + .create_builder() + .expect("Failed to create parser builder"); let _ = grammar.build_grammar(); let code = grammar.emit_compiletime(); @@ -3981,6 +4461,171 @@ mod tests { ); } + #[test] + fn test_glr_optional_epsilon_self_loop_is_expanded_even_with_nooptim() { + let input = quote! { + %nooptim; + %glr; + %tokentype char; + %start Expr; + + Expr(Ast) + : 't' { Ast::True } + | '!'? left=Expr '&' right=Expr { + Ast::And(Box::new(left), Box::new(right)) + } + ; + }; + + let grammar_args = Grammar::parse_args(input).expect("Failed to parse grammar args"); + let mut grammar = + Grammar::from_grammar_args(grammar_args).expect("Failed to build grammar"); + assert!( + !grammar.optimize, + "the regression must be covered even when ordinary optimization is disabled" + ); + + grammar.builder = grammar + .create_builder() + .expect("Failed to create parser builder"); + let expansion_info = grammar + .infos + .iter() + .find_map(|info| match info { + Info::GlrOptionalExpanded { before, after, .. } => Some((before, after)), + _ => None, + }) + .expect("expected GLR optional expansion info"); + assert_eq!( + expansion_info.0, "Expr -> '!'? Expr '&' Expr", + "the info must report the original production" + ); + assert_eq!( + expansion_info.1, + &vec![ + "Expr -> Expr '&' Expr".to_string(), + "Expr -> '!' Expr '&' Expr".to_string(), + ], + "the info must report the generated replacement productions" + ); + + let _ = grammar.build_grammar(); + + for (state_idx, state) in grammar.states.iter().enumerate() { + for (_, reduce_rules) in &state.reduce_map { + for &rule_idx in reduce_rules { + let Some((nonterm_idx, local_rule_idx)) = grammar.rule_id_to_indices(rule_idx) + else { + continue; + }; + let nonterm = &grammar.nonterminals[nonterm_idx]; + let rule = &nonterm.rules[local_rule_idx]; + let is_optional_empty_rule = nonterm.nonterm_type + == Some(rusty_lr_core::parser::nonterminal::NonTerminalType::Optional) + && rule.tokens.is_empty(); + let has_self_goto = + state + .shift_goto_map_nonterm + .iter() + .any(|(nonterm, target)| { + *nonterm == nonterm_idx && target.state == state_idx + }); + + assert!( + !(is_optional_empty_rule && has_self_goto), + "GLR tables must not keep an optional epsilon self-loop" + ); + } + } + } + } + + #[test] + fn test_glr_optional_epsilon_self_loop_reports_error_when_value_is_used() { + let input = quote! { + %glr; + %tokentype char; + %start Expr; + + Expr(Ast) + : 't' { Ast::True } + | opt='!'? left=Expr '&' right=Expr { + let _ = opt; + Ast::And(Box::new(left), Box::new(right)) + } + ; + }; + + let grammar_args = Grammar::parse_args(input).expect("Failed to parse grammar args"); + let mut grammar = + Grammar::from_grammar_args(grammar_args).expect("Failed to build grammar"); + if grammar.optimize { + grammar.optimize(25); + } + + let err = grammar + .create_builder() + .expect_err("optional value use must block automatic GLR expansion"); + let ParseError::GlrNullableReduceCycle { + nullable_rule, + reason, + help, + .. + } = err + else { + panic!("expected GLR nullable reduce cycle error"); + }; + + assert_eq!(nullable_rule, "'!'? ->"); + assert!( + reason.contains("reads the optional value `opt`"), + "reason should explain why automatic expansion is unsafe: {reason}" + ); + assert!( + help.contains("Rewrite"), + "help should tell the user how to fix the grammar: {help}" + ); + } + + #[test] + fn test_glr_star_epsilon_self_loop_reports_error() { + let input = quote! { + %glr; + %tokentype char; + %start Expr; + + Expr(Ast) + : 't' { Ast::True } + | '!'* left=Expr '&' right=Expr { + Ast::And(Box::new(left), Box::new(right)) + } + ; + }; + + let grammar_args = Grammar::parse_args(input).expect("Failed to parse grammar args"); + let mut grammar = + Grammar::from_grammar_args(grammar_args).expect("Failed to build grammar"); + if grammar.optimize { + grammar.optimize(25); + } + + let err = grammar + .create_builder() + .expect_err("zero-or-more GLR cycle must be rejected"); + let ParseError::GlrNullableReduceCycle { reason, help, .. } = err else { + panic!("expected GLR nullable reduce cycle error"); + }; + + assert!( + reason.contains("zero-or-more repetition helper"), + "reason should explain that star cannot be finitely expanded: {reason}" + ); + assert!( + help.contains("non-empty list helper"), + "help should suggest a useful manual rewrite: {help}" + ); + } + #[test] fn test_variable_substitution_success() { let input = quote! { @@ -4216,7 +4861,9 @@ mod tests { grammar.warnings ); - grammar.builder = grammar.create_builder(); + grammar.builder = grammar + .create_builder() + .expect("Failed to create parser builder"); let _ = grammar.build_grammar(); let has_resolved_info = grammar.infos.iter().any(|info| { diff --git a/rusty_lr_parser/src/nonterminal_info.rs b/rusty_lr_parser/src/nonterminal_info.rs index 052ac4b3..0d8b476e 100644 --- a/rusty_lr_parser/src/nonterminal_info.rs +++ b/rusty_lr_parser/src/nonterminal_info.rs @@ -35,6 +35,7 @@ impl CustomReduceAction { } } +#[derive(Clone)] pub enum ReduceAction { /// reduce action that is function-like TokenStream Custom(CustomReduceAction), @@ -54,6 +55,7 @@ impl ReduceAction { } } +#[derive(Clone)] pub struct Rule { pub tokens: Vec, /// reduce action called when this rule is reduced