diff --git a/crates/codegen/language/definition/src/model/item.rs b/crates/codegen/language/definition/src/model/item.rs index a915d01be6..0cab577dd4 100644 --- a/crates/codegen/language/definition/src/model/item.rs +++ b/crates/codegen/language/definition/src/model/item.rs @@ -39,4 +39,24 @@ impl Item { Item::Fragment { item } => &item.name, } } + + /// Whether the language item corresponds to a dedicated terminal kind. + pub fn is_terminal(&self) -> bool { + // NOTE: `Item::Fragment` is inlined. + matches!( + self, + Item::Trivia { .. } | Item::Keyword { .. } | Item::Token { .. } + ) + } + + pub fn is_nonterminal(&self) -> bool { + matches!( + self, + Item::Struct { .. } + | Item::Enum { .. } + | Item::Repeated { .. } + | Item::Separated { .. } + | Item::Precedence { .. } + ) + } } diff --git a/crates/codegen/language/definition/src/model/utils/version_specifier.rs b/crates/codegen/language/definition/src/model/utils/version_specifier.rs index 5ee03e64f0..55a646b91d 100644 --- a/crates/codegen/language/definition/src/model/utils/version_specifier.rs +++ b/crates/codegen/language/definition/src/model/utils/version_specifier.rs @@ -20,4 +20,16 @@ impl VersionSpecifier { VersionSpecifier::Range { from, till } => from <= version && version < till, } } + + /// Returns an iterator over the versions specified as the upper and lower bound. + pub fn versions(&self) -> impl Iterator { + match self { + VersionSpecifier::Never => [None, None], + VersionSpecifier::From { from } => [Some(from), None], + VersionSpecifier::Till { till } => [None, Some(till)], + VersionSpecifier::Range { from, till } => [Some(from), Some(till)], + } + .into_iter() + .flatten() + } } diff --git a/crates/codegen/runtime/cargo/src/runtime/kinds.rs.jinja2 b/crates/codegen/runtime/cargo/src/runtime/kinds.rs.jinja2 index 30e59186fb..850d2499ce 100644 --- a/crates/codegen/runtime/cargo/src/runtime/kinds.rs.jinja2 +++ b/crates/codegen/runtime/cargo/src/runtime/kinds.rs.jinja2 @@ -22,7 +22,7 @@ pub enum NonterminalKind { Stub2, Stub3, {%- else -%} - {%- for variant in model.parser.nonterminal_kinds -%} + {%- for variant in model.kinds.nonterminal_kinds -%} {# variant.documentation | indent(prefix = "/// ", first = true, blank = true) #} {{ variant }}, {%- endfor -%} @@ -65,7 +65,7 @@ pub enum EdgeLabel { Stub2, Stub3, {%- else -%} - {% for variant in model.parser.labels -%} + {% for variant in model.kinds.labels -%} {{ variant | pascal_case }}, {%- endfor -%} {%- endif -%} @@ -98,7 +98,7 @@ pub enum TerminalKind { Stub2, Stub3, {%- else -%} - {%- for variant in model.parser.terminal_kinds -%} + {%- for variant in model.kinds.terminal_kinds -%} {# variant.documentation | indent(prefix = "/// ", first = true, blank = true) #} {{ variant }}, {%- endfor -%} @@ -112,7 +112,7 @@ impl metaslang_cst::TerminalKind for TerminalKind { {%- else -%} matches!( self, - {%- for variant in model.parser.trivia_scanner_names -%} + {%- for variant in model.kinds.trivia_scanner_names -%} | Self::{{ variant }} {%- endfor -%} ) @@ -128,7 +128,7 @@ pub(crate) enum LexicalContext { Stub2, Stub3, {%- else -%} - {%- for context_name, _ in model.parser.scanner_contexts %} + {%- for context_name in model.kinds.lexical_contexts %} {{ context_name }}, {%- endfor %} {%- endif -%} @@ -143,7 +143,7 @@ pub(crate) trait IsLexicalContext { #[allow(non_snake_case)] pub(crate) mod LexicalContextType { {%- if not rendering_in_stubs -%} - {%- for context_name, _ in model.parser.scanner_contexts %} + {%- for context_name in model.kinds.lexical_contexts %} pub struct {{ context_name }}; impl super::IsLexicalContext for {{ context_name }} { diff --git a/crates/codegen/runtime/cargo/src/runtime/language.rs.jinja2 b/crates/codegen/runtime/cargo/src/runtime/language.rs.jinja2 index 4f5c38fd5b..488add51f2 100644 --- a/crates/codegen/runtime/cargo/src/runtime/language.rs.jinja2 +++ b/crates/codegen/runtime/cargo/src/runtime/language.rs.jinja2 @@ -228,7 +228,7 @@ impl Lexer for Language { if kw_scan == KeywordScan::Absent { input.set_position(save); - // TODO(#638): Don't allocate a string here + // TODO(#1001): Don't allocate a string here let ident_value = input.content(save.utf8..furthest_position.utf8); for keyword_compound_scanner in [ diff --git a/crates/codegen/runtime/generator/src/kinds.rs b/crates/codegen/runtime/generator/src/kinds.rs new file mode 100644 index 0000000000..3d157ba90f --- /dev/null +++ b/crates/codegen/runtime/generator/src/kinds.rs @@ -0,0 +1,98 @@ +use std::collections::BTreeSet; + +use codegen_language_definition::model::{self, Identifier, Item}; +use serde::Serialize; + +#[derive(Default, Serialize)] +pub struct KindsModel { + /// Defines the `NonterminalKind` enum variants. + nonterminal_kinds: BTreeSet, + /// Defines the `TerminalKind` enum variants. + terminal_kinds: BTreeSet, + /// Defines `TerminalKind::is_trivia` method. + trivia_scanner_names: BTreeSet, + /// Defines `EdgeLabel` enum variants. + labels: BTreeSet, + // Defines the `LexicalContext(Type)` enum and type-level variants. + lexical_contexts: BTreeSet, +} + +impl KindsModel { + pub fn create(language: &model::Language) -> Self { + let terminal_kinds = language + .items() + .filter(|item| item.is_terminal() && !matches!(item, Item::Fragment { .. })) + .map(|item| item.name().clone()) + .collect(); + + let mut nonterminal_kinds = BTreeSet::default(); + for item in language.items() { + match item { + Item::Struct { item } => { + nonterminal_kinds.insert(item.name.clone()); + } + Item::Enum { item } => { + nonterminal_kinds.insert(item.name.clone()); + } + Item::Repeated { item } => { + nonterminal_kinds.insert(item.name.clone()); + } + Item::Separated { item } => { + nonterminal_kinds.insert(item.name.clone()); + } + Item::Precedence { item } => { + nonterminal_kinds.insert(item.name.clone()); + for op in &item.precedence_expressions { + nonterminal_kinds.insert(op.name.clone()); + } + } + // Terminals + _ => {} + } + } + + let trivia_scanner_names = language + .items() + .filter_map(|item| match item { + Item::Trivia { item } => Some(item.name.clone()), + _ => None, + }) + .collect(); + + let mut labels = BTreeSet::default(); + for item in language.items() { + match item { + Item::Struct { item } => { + for field_name in item.fields.keys() { + labels.insert(field_name.clone()); + } + } + Item::Precedence { item } => { + for item in &item.precedence_expressions { + for item in &item.operators { + for field_name in item.fields.keys() { + labels.insert(field_name.clone()); + } + } + } + } + _ => {} + } + } + + let lexical_contexts: BTreeSet<_> = language + .topics() + .filter_map(|t| t.lexical_context.as_ref()) + .cloned() + .chain(std::iter::once(Identifier::from("Default"))) + .collect(); + + KindsModel { + nonterminal_kinds, + terminal_kinds, + trivia_scanner_names, + labels, + lexical_contexts, + } + } +} diff --git a/crates/codegen/runtime/generator/src/lib.rs b/crates/codegen/runtime/generator/src/lib.rs index 24d3e5e046..b74b1c8d9f 100644 --- a/crates/codegen/runtime/generator/src/lib.rs +++ b/crates/codegen/runtime/generator/src/lib.rs @@ -10,6 +10,7 @@ use serde::Serialize; use crate::model::RuntimeModel; mod ast; +mod kinds; mod model; mod parser; diff --git a/crates/codegen/runtime/generator/src/model.rs b/crates/codegen/runtime/generator/src/model.rs index a241ff48a7..308d8e8845 100644 --- a/crates/codegen/runtime/generator/src/model.rs +++ b/crates/codegen/runtime/generator/src/model.rs @@ -6,6 +6,7 @@ use semver::Version; use serde::Serialize; use crate::ast::AstModel; +use crate::kinds::KindsModel; use crate::parser::ParserModel; #[derive(Default, Serialize)] @@ -14,6 +15,7 @@ pub struct RuntimeModel { all_versions: BTreeSet, parser: ParserModel, ast: AstModel, + kinds: KindsModel, } impl RuntimeModel { @@ -22,6 +24,7 @@ impl RuntimeModel { all_versions: language.versions.iter().cloned().collect(), ast: AstModel::create(language), parser: ParserModel::from_language(language), + kinds: KindsModel::create(language), } } } diff --git a/crates/codegen/runtime/generator/src/parser.rs b/crates/codegen/runtime/generator/src/parser.rs index e420c13645..8a2c42e847 100644 --- a/crates/codegen/runtime/generator/src/parser.rs +++ b/crates/codegen/runtime/generator/src/parser.rs @@ -4,28 +4,21 @@ use std::collections::{BTreeMap, BTreeSet}; use std::rc::Rc; use codegen_language_definition::model::{Identifier, Language, VersionSpecifier}; -use quote::{format_ident, quote}; use semver::Version; use serde::Serialize; +mod codegen; mod grammar; -mod keyword_scanner_definition; -mod parser_definition; -mod precedence_parser_definition; -mod scanner_definition; -mod trie; -mod versioned; +use codegen::{ + KeywordScannerDefinitionCodegen as _, ParserDefinitionCodegen as _, + PrecedenceParserDefinitionCodegen as _, ScannerDefinitionCodegen as _, Trie, +}; use grammar::{ Grammar, GrammarVisitor, KeywordScannerAtomic, KeywordScannerDefinitionRef, ParserDefinitionNode, ParserDefinitionRef, PrecedenceParserDefinitionRef, ScannerDefinitionNode, ScannerDefinitionRef, TriviaParserDefinitionRef, }; -use keyword_scanner_definition::KeywordScannerDefinitionExtensions as _; -use parser_definition::ParserDefinitionExtensions as _; -use precedence_parser_definition::PrecedenceParserDefinitionExtensions as _; -use scanner_definition::ScannerDefinitionExtensions as _; -use trie::Trie; /// Newtype for the already generated Rust code, not to be confused with regular strings. #[derive(Serialize, Default, Clone)] @@ -36,18 +29,9 @@ pub struct ParserModel { /// Constructs inner `Language` the state to evaluate the version-dependent branches. referenced_versions: BTreeSet, - /// Defines the `NonterminalKind` enum variants. - nonterminal_kinds: BTreeSet, - /// Defines the `TerminalKind` enum variants. - terminal_kinds: BTreeSet, - /// Defines `TerminalKind::is_trivia` method. - trivia_scanner_names: BTreeSet, - /// Defines `EdgeLabel` enum variants. - labels: BTreeSet, - /// Defines the top-level scanner functions in `Language`. scanner_functions: BTreeMap, // (name of scanner, code) - // Defines the `LexicalContext(Type)` enum and type-level variants. + // Defines the `Lexer::next_terminal` method. scanner_contexts: BTreeMap, /// Defines the top-level compound scanners used when lexing in `Language`. keyword_compound_scanners: BTreeMap, // (name of the KW scanner, code) @@ -80,16 +64,7 @@ struct ParserAccumulatorState { /// Constructs inner `Language` the state to evaluate the version-dependent branches. referenced_versions: BTreeSet, - /// Defines the `NonterminalKind` enum variants. - nonterminal_kinds: BTreeSet, - /// Defines the `TerminalKind` enum variants. - terminal_kinds: BTreeSet, - /// Defines `TerminalKind::is_trivia` method. - trivia_scanner_names: BTreeSet, - /// Defines `EdgeLabel` enum variants. - labels: BTreeSet, - - // Defines the `LexicalContext(Type)` enum and type-level variants. + // Defines the `Lexer::next_terminal` method. scanner_contexts: BTreeMap, /// Defines the top-level parser functions in `Language`. @@ -137,7 +112,7 @@ impl ParserAccumulatorState { .expect("context must be set with `set_current_context`") } - fn into_model(mut self) -> ParserModel { + fn into_model(self) -> ParserModel { let contexts = self .scanner_contexts .into_iter() @@ -216,25 +191,8 @@ impl ParserAccumulatorState { }) .collect(); - // Make sure empty strings are not there - self.labels.remove(""); - // These are built-in and already pre-defined - // _SLANG_INTERNAL_RESERVED_NODE_LABELS_ (keep in sync) - self.labels.remove("item"); - self.labels.remove("variant"); - self.labels.remove("separator"); - self.labels.remove("operand"); - self.labels.remove("left_operand"); - self.labels.remove("right_operand"); - self.labels.remove("leading_trivia"); - self.labels.remove("trailing_trivia"); - ParserModel { referenced_versions: self.referenced_versions, - nonterminal_kinds: self.nonterminal_kinds, - terminal_kinds: self.terminal_kinds, - trivia_scanner_names: self.trivia_scanner_names, - labels: self.labels, parser_functions: self.parser_functions, trivia_parser_functions: self.trivia_parser_functions, // These are derived from the accumulated state @@ -253,46 +211,15 @@ impl GrammarVisitor for ParserAccumulatorState { fn keyword_scanner_definition_enter(&mut self, scanner: &KeywordScannerDefinitionRef) { for def in scanner.definitions() { - let versions = def.enabled.iter().chain(def.reserved.iter()); + let specifiers = def.enabled.iter().chain(def.reserved.iter()); - for version in versions { - match version { - VersionSpecifier::Never => {} - VersionSpecifier::From { from } => { - self.referenced_versions.insert(from.clone()); - } - VersionSpecifier::Till { till } => { - self.referenced_versions.insert(till.clone()); - } - VersionSpecifier::Range { from, till } => { - self.referenced_versions.insert(from.clone()); - self.referenced_versions.insert(till.clone()); - } - } - } + self.referenced_versions + .extend(specifiers.flat_map(VersionSpecifier::versions).cloned()); } } fn trivia_parser_definition_enter(&mut self, parser: &TriviaParserDefinitionRef) { self.set_current_context(parser.context().clone()); - let trivia_scanners = { - use crate::parser::grammar::visitor::Visitable; - - #[derive(Default)] - struct CollectTriviaScanners { - scanner_names: BTreeSet, - } - impl crate::parser::grammar::visitor::GrammarVisitor for CollectTriviaScanners { - fn scanner_definition_enter(&mut self, node: &ScannerDefinitionRef) { - self.scanner_names.insert(node.name().clone()); - } - } - - let mut visitor = CollectTriviaScanners::default(); - parser.node().accept_visitor(&mut visitor); - visitor.scanner_names - }; - self.trivia_scanner_names.extend(trivia_scanners); self.trivia_parser_functions.insert( parser.name().clone(), @@ -304,27 +231,15 @@ impl GrammarVisitor for ParserAccumulatorState { // Have to set this regardless so that we can collect referenced scanners self.set_current_context(parser.context().clone()); if !parser.is_inline() { - self.nonterminal_kinds.insert(parser.name().clone()); - let code = parser.to_parser_code(); self.parser_functions.insert( parser.name().clone(), - RustCode( - { - let nonterminal_kind = format_ident!("{}", parser.name()); - quote! { #code.with_kind(NonterminalKind::#nonterminal_kind) } - } - .to_string(), - ), + RustCode(parser.to_parser_code().to_string()), ); } } fn precedence_parser_definition_enter(&mut self, parser: &PrecedenceParserDefinitionRef) { self.set_current_context(parser.context().clone()); - self.nonterminal_kinds.insert(parser.name().clone()); - for (_, name, _) in &parser.node().operators { - self.nonterminal_kinds.insert(name.clone()); - } // While it's not common to parse a precedence expression as a standalone nonterminal, // we generate a function for completeness. @@ -335,88 +250,38 @@ impl GrammarVisitor for ParserAccumulatorState { self.parser_functions.insert( parser.name().clone(), - RustCode( - { - let code = parser.to_parser_code(); - let nonterminal_kind = format_ident!("{}", parser.name()); - quote! { #code.with_kind(NonterminalKind::#nonterminal_kind) } - } - .to_string(), - ), + RustCode(parser.to_parser_code().to_string()), ); } fn scanner_definition_node_enter(&mut self, node: &ScannerDefinitionNode) { if let ScannerDefinitionNode::Versioned(_, version_specifier) = node { - match version_specifier { - VersionSpecifier::Never => {} - VersionSpecifier::From { from } => { - self.referenced_versions.insert(from.clone()); - } - VersionSpecifier::Till { till } => { - self.referenced_versions.insert(till.clone()); - } - VersionSpecifier::Range { from, till } => { - self.referenced_versions.insert(from.clone()); - self.referenced_versions.insert(till.clone()); - } - } + self.referenced_versions + .extend(version_specifier.versions().cloned()); } } fn parser_definition_node_enter(&mut self, node: &ParserDefinitionNode) { match node { - ParserDefinitionNode::Versioned(_, version_specifier) => match version_specifier { - VersionSpecifier::Never => {} - VersionSpecifier::From { from } => { - self.referenced_versions.insert(from.clone()); - } - VersionSpecifier::Till { till } => { - self.referenced_versions.insert(till.clone()); - } - VersionSpecifier::Range { from, till } => { - self.referenced_versions.insert(from.clone()); - self.referenced_versions.insert(till.clone()); - } - }, + ParserDefinitionNode::Versioned(_, version_specifier) => { + self.referenced_versions + .extend(version_specifier.versions().cloned()); + } ParserDefinitionNode::ScannerDefinition(scanner) => { self.top_level_scanner_names.insert(scanner.name().clone()); - self.terminal_kinds.insert(scanner.name().clone()); self.current_context() .scanner_definitions .insert(scanner.name().clone()); } ParserDefinitionNode::KeywordScannerDefinition(scanner) => { - self.terminal_kinds.insert(scanner.name().clone()); - self.current_context() .keyword_scanner_defs .insert(scanner.name().clone(), Rc::clone(scanner)); } - // Collect labels: - ParserDefinitionNode::Choice(choice) => { - self.labels.insert(choice.label.clone()); - } - ParserDefinitionNode::Sequence(sequence) => { - for node in sequence { - self.labels.insert(node.label.clone()); - } - } - ParserDefinitionNode::SeparatedBy(item, separator) => { - self.labels.insert(item.label.clone()); - self.labels.insert(separator.label.clone()); - } - ParserDefinitionNode::TerminatedBy(_, terminator) => { - self.labels.insert(terminator.label.clone()); - } - // Collect delimiters for each context ParserDefinitionNode::DelimitedBy(open, _, close, ..) => { - self.labels.insert(open.label.clone()); - self.labels.insert(close.label.clone()); - let (open, close) = match (open.as_ref(), close.as_ref()) { ( ParserDefinitionNode::ScannerDefinition(open, ..), diff --git a/crates/codegen/runtime/generator/src/parser/codegen.rs b/crates/codegen/runtime/generator/src/parser/codegen.rs new file mode 100644 index 0000000000..5f4fdfcd34 --- /dev/null +++ b/crates/codegen/runtime/generator/src/parser/codegen.rs @@ -0,0 +1,12 @@ +mod keyword_scanner_definition; +mod parser_definition; +mod precedence_parser_definition; +mod scanner_definition; +mod trie; +mod versioned; + +pub use keyword_scanner_definition::KeywordScannerDefinitionCodegen; +pub use parser_definition::ParserDefinitionCodegen; +pub use precedence_parser_definition::PrecedenceParserDefinitionCodegen; +pub use scanner_definition::ScannerDefinitionCodegen; +pub use trie::Trie; diff --git a/crates/codegen/runtime/generator/src/parser/keyword_scanner_definition.rs b/crates/codegen/runtime/generator/src/parser/codegen/keyword_scanner_definition.rs similarity index 90% rename from crates/codegen/runtime/generator/src/parser/keyword_scanner_definition.rs rename to crates/codegen/runtime/generator/src/parser/codegen/keyword_scanner_definition.rs index 3129fdb13f..124a43f186 100644 --- a/crates/codegen/runtime/generator/src/parser/keyword_scanner_definition.rs +++ b/crates/codegen/runtime/generator/src/parser/codegen/keyword_scanner_definition.rs @@ -2,15 +2,15 @@ use codegen_language_definition::model; use proc_macro2::TokenStream; use quote::{format_ident, quote}; +use crate::parser::codegen::scanner_definition::ScannerDefinitionNodeCodegen as _; +use crate::parser::codegen::versioned::VersionedQuote; use crate::parser::grammar::{KeywordScannerDefinitionRef, ScannerDefinitionNode}; -use crate::parser::scanner_definition::ScannerDefinitionNodeExtensions; -use crate::parser::versioned::VersionedQuote; -pub trait KeywordScannerDefinitionExtensions { +pub trait KeywordScannerDefinitionCodegen { fn to_scanner_code(&self) -> TokenStream; } -impl KeywordScannerDefinitionExtensions for KeywordScannerDefinitionRef { +impl KeywordScannerDefinitionCodegen for KeywordScannerDefinitionRef { fn to_scanner_code(&self) -> TokenStream { let name_ident = format_ident!("{}", self.name()); let terminal_kind = quote! { TerminalKind::#name_ident }; @@ -79,7 +79,7 @@ impl KeywordScannerDefinitionExtensions for KeywordScannerDefinitionRef { } } -impl KeywordScannerDefinitionExtensions for model::KeywordValue { +impl KeywordScannerDefinitionCodegen for model::KeywordValue { fn to_scanner_code(&self) -> TokenStream { // This is a subset; let's reuse that ScannerDefinitionNode::from(self.clone()).to_scanner_code() diff --git a/crates/codegen/runtime/generator/src/parser/parser_definition.rs b/crates/codegen/runtime/generator/src/parser/codegen/parser_definition.rs similarity index 95% rename from crates/codegen/runtime/generator/src/parser/parser_definition.rs rename to crates/codegen/runtime/generator/src/parser/codegen/parser_definition.rs index 9bfc928139..438680fc6c 100644 --- a/crates/codegen/runtime/generator/src/parser/parser_definition.rs +++ b/crates/codegen/runtime/generator/src/parser/codegen/parser_definition.rs @@ -3,35 +3,38 @@ use inflector::Inflector; use proc_macro2::TokenStream; use quote::{format_ident, quote}; +use crate::parser::codegen::versioned::{Versioned as _, VersionedQuote as _}; use crate::parser::grammar::{ Labeled, ParserDefinitionNode, ParserDefinitionRef, TriviaParserDefinitionRef, }; -use crate::parser::versioned::{Versioned as _, VersionedQuote as _}; -pub trait ParserDefinitionExtensions { +pub trait ParserDefinitionCodegen { fn to_parser_code(&self) -> TokenStream; } -impl ParserDefinitionExtensions for ParserDefinitionRef { +impl ParserDefinitionCodegen for ParserDefinitionRef { fn to_parser_code(&self) -> TokenStream { - self.node().version_specifier().to_conditional_code( + let code = self.node().version_specifier().to_conditional_code( self.node().to_parser_code(self.context(), false), Some(quote! { ParserResult::disabled() }), - ) + ); + + let nonterminal_kind = format_ident!("{}", self.name()); + quote! { #code.with_kind(NonterminalKind::#nonterminal_kind) } } } -impl ParserDefinitionExtensions for TriviaParserDefinitionRef { +impl ParserDefinitionCodegen for TriviaParserDefinitionRef { fn to_parser_code(&self) -> TokenStream { self.node().to_parser_code(self.context(), true) } } -pub trait ParserDefinitionNodeExtensions { +pub(super) trait ParserDefinitionNodeCodegen { fn to_parser_code(&self, context_name: &Identifier, is_trivia: bool) -> TokenStream; } -impl ParserDefinitionNodeExtensions for ParserDefinitionNode { +impl ParserDefinitionNodeCodegen for ParserDefinitionNode { #[allow(clippy::too_many_lines)] // giant switch over parser definition node types fn to_parser_code(&self, context_name: &Identifier, is_trivia: bool) -> TokenStream { let context = format_ident!("{context_name}"); diff --git a/crates/codegen/runtime/generator/src/parser/precedence_parser_definition.rs b/crates/codegen/runtime/generator/src/parser/codegen/precedence_parser_definition.rs similarity index 94% rename from crates/codegen/runtime/generator/src/parser/precedence_parser_definition.rs rename to crates/codegen/runtime/generator/src/parser/codegen/precedence_parser_definition.rs index 1c27fda53e..30c11f90a3 100644 --- a/crates/codegen/runtime/generator/src/parser/precedence_parser_definition.rs +++ b/crates/codegen/runtime/generator/src/parser/codegen/precedence_parser_definition.rs @@ -3,14 +3,14 @@ use inflector::Inflector; use proc_macro2::{Ident, TokenStream}; use quote::{format_ident, quote}; +use crate::parser::codegen::parser_definition::{ + make_choice, make_sequence, ParserDefinitionNodeCodegen, +}; use crate::parser::grammar::{ PrecedenceOperatorModel, PrecedenceParserDefinitionNode, PrecedenceParserDefinitionRef, }; -use crate::parser::parser_definition::{ - make_choice, make_sequence, ParserDefinitionNodeExtensions, -}; -pub trait PrecedenceParserDefinitionExtensions { +pub trait PrecedenceParserDefinitionCodegen { fn to_parser_code(&self) -> TokenStream; /// Emit a helper parser function for each precedence expression that ensures the main parser /// identifies a single node of the expected type, with a child node being the expected @@ -18,20 +18,22 @@ pub trait PrecedenceParserDefinitionExtensions { fn to_precedence_expression_parser_code(&self) -> Vec<(Identifier, TokenStream)>; } -impl PrecedenceParserDefinitionExtensions for PrecedenceParserDefinitionRef { +impl PrecedenceParserDefinitionCodegen for PrecedenceParserDefinitionRef { fn to_parser_code(&self) -> TokenStream { - self.node().to_parser_code( + let code = self.node().to_parser_code( self.context(), format_ident!("{name}", name = self.name().to_pascal_case()), - ) + ); + + let nonterminal_kind = format_ident!("{}", self.name()); + quote! { #code.with_kind(NonterminalKind::#nonterminal_kind) } } fn to_precedence_expression_parser_code(&self) -> Vec<(Identifier, TokenStream)> { - let mut res = vec![]; let parser_name = format_ident!("{}", self.name().to_snake_case()); let nonterminal_name = format_ident!("{}", self.name().to_pascal_case()); - for name in &self.node().precedence_expression_names { + self.node().precedence_expression_names.iter().map(|name| { let op_nonterminal_name = format_ident!("{}", name.to_pascal_case()); // Ensure that the parser correctly identifies a single node of the expected type, @@ -52,17 +54,18 @@ impl PrecedenceParserDefinitionExtensions for PrecedenceParserDefinitionRef { _ => ParserResult::no_match(vec![]), } }; - res.push((name.clone(), code)); - } - res + + + (name.clone(), code) + }).collect() } } -pub trait PrecedenceParserDefinitionNodeExtensions { +pub(super) trait PrecedenceParserDefinitionNodeCodegen { fn to_parser_code(&self, context_name: &Identifier, expression_kind: Ident) -> TokenStream; } -impl PrecedenceParserDefinitionNodeExtensions for PrecedenceParserDefinitionNode { +impl PrecedenceParserDefinitionNodeCodegen for PrecedenceParserDefinitionNode { // A Pratt parser can be implemented as two simple passes, // only the first of which can generate parse errors. // diff --git a/crates/codegen/runtime/generator/src/parser/scanner_definition.rs b/crates/codegen/runtime/generator/src/parser/codegen/scanner_definition.rs similarity index 94% rename from crates/codegen/runtime/generator/src/parser/scanner_definition.rs rename to crates/codegen/runtime/generator/src/parser/codegen/scanner_definition.rs index efcb4327dd..d34fafb89a 100644 --- a/crates/codegen/runtime/generator/src/parser/scanner_definition.rs +++ b/crates/codegen/runtime/generator/src/parser/codegen/scanner_definition.rs @@ -4,15 +4,15 @@ use inflector::Inflector; use proc_macro2::TokenStream; use quote::{format_ident, quote}; +use crate::parser::codegen::versioned::VersionedQuote; use crate::parser::grammar::{ScannerDefinitionNode, ScannerDefinitionRef}; -use crate::parser::versioned::VersionedQuote; -pub trait ScannerDefinitionExtensions { +pub trait ScannerDefinitionCodegen { fn to_scanner_code(&self) -> TokenStream; fn literals(&self) -> Vec; } -impl ScannerDefinitionExtensions for ScannerDefinitionRef { +impl ScannerDefinitionCodegen for ScannerDefinitionRef { fn to_scanner_code(&self) -> TokenStream { self.node().to_scanner_code() } @@ -26,12 +26,12 @@ impl ScannerDefinitionExtensions for ScannerDefinitionRef { } } -pub trait ScannerDefinitionNodeExtensions { +pub(super) trait ScannerDefinitionNodeCodegen { fn to_scanner_code(&self) -> TokenStream; fn literals(&self, accum: &mut BTreeSet) -> bool; } -impl ScannerDefinitionNodeExtensions for ScannerDefinitionNode { +impl ScannerDefinitionNodeCodegen for ScannerDefinitionNode { // Returns true if this is nothing but a set of literals fn literals(&self, accum: &mut BTreeSet) -> bool { match self { diff --git a/crates/codegen/runtime/generator/src/parser/trie.rs b/crates/codegen/runtime/generator/src/parser/codegen/trie.rs similarity index 98% rename from crates/codegen/runtime/generator/src/parser/trie.rs rename to crates/codegen/runtime/generator/src/parser/codegen/trie.rs index e6d92136b6..a568299832 100644 --- a/crates/codegen/runtime/generator/src/parser/trie.rs +++ b/crates/codegen/runtime/generator/src/parser/codegen/trie.rs @@ -5,8 +5,8 @@ use codegen_language_definition::model::KeywordDefinition; use proc_macro2::TokenStream; use quote::{format_ident, quote}; +use crate::parser::codegen::versioned::{Versioned as _, VersionedQuote as _}; use crate::parser::grammar::{KeywordScannerAtomic, ScannerDefinitionRef}; -use crate::parser::versioned::{Versioned as _, VersionedQuote as _}; #[derive(Clone, Debug, Default)] pub struct Trie { diff --git a/crates/codegen/runtime/generator/src/parser/versioned.rs b/crates/codegen/runtime/generator/src/parser/codegen/versioned.rs similarity index 98% rename from crates/codegen/runtime/generator/src/parser/versioned.rs rename to crates/codegen/runtime/generator/src/parser/codegen/versioned.rs index eb2f3b52ed..4d7cc1a93f 100644 --- a/crates/codegen/runtime/generator/src/parser/versioned.rs +++ b/crates/codegen/runtime/generator/src/parser/codegen/versioned.rs @@ -5,7 +5,7 @@ use semver::Version; use crate::parser::grammar::{Labeled, ParserDefinitionNode, ScannerDefinitionNode}; -pub trait Versioned { +pub(super) trait Versioned { fn version_specifier(&self) -> Option<&VersionSpecifier>; } @@ -39,7 +39,7 @@ impl Versioned for ScannerDefinitionNode { } } -pub trait VersionedQuote { +pub(super) trait VersionedQuote { /// Depending on the `as_bool_expr` result, wraps the given code in an `if` block and optionally includes an `else` block fn to_conditional_code( &self, diff --git a/crates/codegen/runtime/generator/src/parser/grammar/constructor.rs b/crates/codegen/runtime/generator/src/parser/grammar/constructor.rs index 000fd320cf..7b22ac1d6c 100644 --- a/crates/codegen/runtime/generator/src/parser/grammar/constructor.rs +++ b/crates/codegen/runtime/generator/src/parser/grammar/constructor.rs @@ -242,7 +242,7 @@ struct ResolveCtx<'a> { resolved: &'a mut HashMap, } -#[allow(clippy::too_many_lines)] // FIXME: Simplify me when we simplify the v2-to-v1 interface +#[allow(clippy::too_many_lines)] // FIXME(#638): Simplify me when we simplify the v2-to-v1 interface fn resolve_grammar_element(ident: &Identifier, ctx: &mut ResolveCtx<'_>) -> GrammarElement { let (lex_ctx, elem) = ctx.items.get(ident).expect("Missing item"); diff --git a/crates/solidity/outputs/cargo/slang_solidity/src/generated/generated/language.rs b/crates/solidity/outputs/cargo/slang_solidity/src/generated/generated/language.rs index b81cd28140..db1829532e 100644 --- a/crates/solidity/outputs/cargo/slang_solidity/src/generated/generated/language.rs +++ b/crates/solidity/outputs/cargo/slang_solidity/src/generated/generated/language.rs @@ -10906,7 +10906,7 @@ impl Lexer for Language { if kw_scan == KeywordScan::Absent { input.set_position(save); - // TODO(#638): Don't allocate a string here + // TODO(#1001): Don't allocate a string here let ident_value = input.content(save.utf8..furthest_position.utf8); for keyword_compound_scanner in [ @@ -13434,7 +13434,7 @@ impl Lexer for Language { if kw_scan == KeywordScan::Absent { input.set_position(save); - // TODO(#638): Don't allocate a string here + // TODO(#1001): Don't allocate a string here let ident_value = input.content(save.utf8..furthest_position.utf8); for keyword_compound_scanner in [