refactor: Use DSL v2 types in place of ScannerDefinitionNode (#1003)

Based on #1002
Part of #638

The DSL v2's `model::Scanner` is almost 1:1 usable with our current
parser codegen, with the exception of versioned scanners. In short we:
- introduce a helper type `VersionedScanner` that is used in place of
the old `ScannerDefinitionNode::Versioned`
- introduce a `ScannerExt` trait implemented for both `VersionedScanner`
and the `model::Scanner`, which is responsible for generating the main
scanning logic
- Repurposes `ScannerDefinition` slightly to surface more of the
scanner-related logic for trivia/fragment/token
- implements this directly for
`model::{TriviaItem,FragmentItem,TokenItem}` and stores it in the v1
Grammar struct
- similarly, uses `model::KeywordItem` directly in place of v1's
`KeywordScannerDefinition` as they shared the same functionality

crates/codegen/language/definition/src/model/terminals/keyword.rs

@@ -2,7 +2,7 @@ use codegen_language_internal_macros::{derive_spanned_type, ParseInputTokens, Wr
 use itertools::Itertools;
 use serde::{Deserialize, Serialize};
 
-use crate::model::{Identifier, VersionSpecifier};
+use crate::model::{Identifier, Scanner, VersionSpecifier};
 
 #[derive(Clone, Debug, Deserialize, Eq, PartialEq, Serialize)]
 #[derive_spanned_type(Clone, Debug, ParseInputTokens, WriteOutputTokens)]
@@ -32,6 +32,23 @@ pub enum KeywordValue {
  Atom { atom: String },
 }
 
+impl From<KeywordValue> for Scanner {
+ fn from(value: KeywordValue) -> Scanner {
+ match value {
+ KeywordValue::Optional { value } => Scanner::Optional {
+ scanner: Box::new((*value).into()),
+ },
+ KeywordValue::Sequence { values } => Scanner::Sequence {
+ scanners: values.into_iter().map(Into::into).collect(),
+ },
+ KeywordValue::Atom { atom } => Scanner::Atom { atom },
+ KeywordValue::Choice { values } => Scanner::Choice {
+ scanners: values.into_iter().map(Into::into).collect(),
+ },
+ }
+ }
+}
+
 impl KeywordValue {
  /// Collects all possible variations generated by this value.
  pub fn collect_variations(&self) -> Vec<String> {

crates/codegen/runtime/generator/src/parser/codegen.rs

@@ -5,8 +5,7 @@ mod scanner_definition;
 mod trie;
 mod versioned;
 
-pub use keyword_scanner_definition::KeywordScannerDefinitionCodegen;
+pub use keyword_scanner_definition::{KeywordItemAtom, KeywordScannerDefinitionCodegen};
 pub use parser_definition::ParserDefinitionCodegen;
 pub use precedence_parser_definition::PrecedenceParserDefinitionCodegen;
-pub use scanner_definition::ScannerDefinitionCodegen;
 pub use trie::Trie;

crates/codegen/runtime/generator/src/parser/codegen/keyword_scanner_definition.rs

@@ -1,22 +1,23 @@
+use std::rc::Rc;
+
 use codegen_language_definition::model;
 use proc_macro2::TokenStream;
 use quote::{format_ident, quote};
 
-use crate::parser::codegen::scanner_definition::ScannerDefinitionNodeCodegen as _;
+use crate::parser::codegen::scanner_definition::ScannerCodegen as _;
 use crate::parser::codegen::versioned::VersionedQuote;
-use crate::parser::grammar::{KeywordScannerDefinitionRef, ScannerDefinitionNode};
 
 pub trait KeywordScannerDefinitionCodegen {
  fn to_scanner_code(&self) -> TokenStream;
 }
 
-impl KeywordScannerDefinitionCodegen for KeywordScannerDefinitionRef {
+impl KeywordScannerDefinitionCodegen for model::KeywordItem {
  fn to_scanner_code(&self) -> TokenStream {
- let name_ident = format_ident!("{}", self.name());
+ let name_ident = format_ident!("{}", self.name);
  let terminal_kind = quote! { TerminalKind::#name_ident };
 
  let kw_scanners: Vec<_> = self
- .definitions()
+ .definitions
  .iter()
  .map(|versioned_kw| {
  let scanner = versioned_kw.value.to_scanner_code();
@@ -82,6 +83,55 @@ impl KeywordScannerDefinitionCodegen for KeywordScannerDefinitionRef {
 impl KeywordScannerDefinitionCodegen for model::KeywordValue {
  fn to_scanner_code(&self) -> TokenStream {
  // This is a subset; let's reuse that
- ScannerDefinitionNode::from(self.clone()).to_scanner_code()
+ model::Scanner::from(self.clone()).to_scanner_code()
+ }
+}
+
+/// A newtype wrapper around [`model::KeywordItem`] that only has a single atom value.
+///
+/// The main usage for this type is to construct a keyword trie, as trie will
+/// only work with single atom values and keyword promotion needs to additionally account for
+/// keyword reservation, rather than just literal presence.
+#[derive(Clone)]
+pub struct KeywordItemAtom(Rc<model::KeywordItem>);
+
+impl KeywordItemAtom {
+ /// Wraps the keyword scanner definition if it is a single atom value.
+ pub fn try_from_def(def: &Rc<model::KeywordItem>) -> Option<Self> {
+ match def.definitions[..] {
+ [model::KeywordDefinition {
+ value: model::KeywordValue::Atom { .. },
+ ..
+ }] => Some(Self(Rc::clone(def))),
+ _ => None,
+ }
+ }
+}
+
+impl std::ops::Deref for KeywordItemAtom {
+ type Target = Rc<model::KeywordItem>;
+
+ fn deref(&self) -> &Self::Target {
+ &self.0
+ }
+}
+
+impl KeywordItemAtom {
+ pub fn definition(&self) -> &model::KeywordDefinition {
+ self.0
+ .definitions
+ .first()
+ .expect("KeywordItemAtom should have exactly one definition")
+ }
+
+ /// The single atom value that this keyword item matches.
+ pub fn value(&self) -> &str {
+ match self.definition() {
+ model::KeywordDefinition {
+ value: model::KeywordValue::Atom { atom },
+ ..
+ } => atom,
+ _ => unreachable!("KeywordItemAtom should have a single atom value"),
+ }
  }
 }

crates/codegen/runtime/generator/src/parser/codegen/parser_definition.rs

@@ -144,7 +144,7 @@ impl ParserDefinitionNodeCodegen for ParserDefinitionNode {
 
  // Keyword scanner uses the promotion inside the parse_terminal
  Self::KeywordScannerDefinition(scanner_definition) => {
- let kind = format_ident!("{name}", name = scanner_definition.name());
+ let kind = format_ident!("{name}", name = scanner_definition.name);
 
  let parse_terminal = if is_trivia {
  format_ident!("parse_terminal")