Make Lexeme a trait, not a struct.

It is quite probable that in some parsing situations the way that
`Lexeme`s are represented in memory will be performance critical.
Previously we tried to satisfy everyone by having a fairly compact
`Lexeme` representation, but that meant that we couldn't represent some
reasonable size/lengths. I had hoped we could do something clever like
expand the `Lexeme` struct depending on `StorageT`s size, but that's not
possible in any sensible way.

This commit solves the problem in, I think, a more general way: it lets
users provide their own lexeme struct provided it conforms to the
(simple) `Lexeme` trait. This does mean lots of additional type
parameters, but (after three attempts!) these don't leak out to the end
user *too* much, though they do a bit.

As a half-way house, the "original" lexeme struct is now called
`StandardLexeme`. The plan is to move that out of lrpar and into lrlex,
but that can be done in a subsequent commit.

Interestingly, this commit paves the way for splitting lrlex and lrpar
apart in a more satisfactory way: the `Lexeme` trait makes clear that
any lexer can be used with lrpar. In other words, while lrlex only makes
sense with lrpar, the `Lexeme` trait will help make it clearer that
lrpar can be used without lrlex.

Author: ltratt

lrlex/src/lib/lexer.rs

@@ -1,5 +1,6 @@
 use std::{
     collections::{HashMap, HashSet},
+    fmt,
     hash::Hash,
     marker::PhantomData,
     slice::Iter,
@@ -9,7 +10,7 @@ use num_traits::{PrimInt, Unsigned};
 use regex::{self, Regex, RegexBuilder};
 use try_from::TryFrom;
 
-use lrpar::{LexError, Lexeme, Lexer, NonStreamingLexer, Span};
+use lrpar::{LexError, Lexeme, Lexer, NonStreamingLexer, Span, StandardLexeme};
 
 use crate::{parser::LexParser, LexBuildResult};
 
@@ -192,7 +193,7 @@ impl<StorageT: Copy + Eq + Hash + PrimInt + TryFrom<usize> + Unsigned> LexerDef<
     }
 }
 
-impl<StorageT: Copy + Eq + Hash + PrimInt + TryFrom<usize> + Unsigned>
+impl<StorageT: Copy + Eq + fmt::Debug + Hash + PrimInt + TryFrom<usize> + Unsigned>
     LRNonStreamingLexerDef<StorageT>
 {
     /// Return an [LRNonStreamingLexer] for the `String` `s` that will lex relative to this
@@ -208,17 +209,20 @@ impl<StorageT: Copy + Eq + Hash + PrimInt + TryFrom<usize> + Unsigned>
 /// An `LRNonStreamingLexer` holds a reference to a string and can lex it into [lrpar::Lexeme]s.
 /// Although the struct is tied to a single string, no guarantees are made about whether the
 /// lexemes are cached or not.
-pub struct LRNonStreamingLexer<'lexer, 'input: 'lexer, StorageT> {
+pub struct LRNonStreamingLexer<'lexer, 'input: 'lexer, StorageT: fmt::Debug> {
     s: &'input str,
-    lexemes: Vec<Result<Lexeme<StorageT>, LexError>>,
+    lexemes: Vec<Result<StandardLexeme<StorageT>, LexError>>,
     /// A sorted list of the byte index of the start of the following line. i.e. for the input
     /// string `" a\nb\n  c d"` this will contain `[3, 5]`.
     newlines: Vec<usize>,
     phantom: PhantomData<&'lexer ()>,
 }
 
-impl<'lexer, 'input: 'lexer, StorageT: Copy + Eq + Hash + PrimInt + TryFrom<usize> + Unsigned>
-    LRNonStreamingLexer<'lexer, 'input, StorageT>
+impl<
+        'lexer,
+        'input: 'lexer,
+        StorageT: Copy + Eq + fmt::Debug + Hash + PrimInt + TryFrom<usize> + Unsigned,
+    > LRNonStreamingLexer<'lexer, 'input, StorageT>
 {
     fn new(
         lexerdef: &'lexer LRNonStreamingLexerDef<StorageT>,
@@ -278,16 +282,19 @@ impl<'lexer, 'input: 'lexer, StorageT: Copy + Eq + Hash + PrimInt + TryFrom<usiz
     }
 }
 
-impl<'lexer, 'input: 'lexer, StorageT: Copy + Eq + Hash + PrimInt + Unsigned> Lexer<StorageT>
-    for LRNonStreamingLexer<'lexer, 'input, StorageT>
+impl<'lexer, 'input: 'lexer, StorageT: Copy + fmt::Debug + Eq + Hash + PrimInt + Unsigned>
+    Lexer<StandardLexeme<StorageT>, StorageT> for LRNonStreamingLexer<'lexer, 'input, StorageT>
 {
-    fn iter<'a>(&'a self) -> Box<dyn Iterator<Item = Result<Lexeme<StorageT>, LexError>> + 'a> {
+    fn iter<'a>(
+        &'a self,
+    ) -> Box<dyn Iterator<Item = Result<StandardLexeme<StorageT>, LexError>> + 'a> {
         Box::new(self.lexemes.iter().cloned())
     }
 }
 
-impl<'lexer, 'input: 'lexer, StorageT: Copy + Eq + Hash + PrimInt + Unsigned>
-    NonStreamingLexer<'input, StorageT> for LRNonStreamingLexer<'lexer, 'input, StorageT>
+impl<'lexer, 'input: 'lexer, StorageT: Copy + Eq + fmt::Debug + Hash + PrimInt + Unsigned>
+    NonStreamingLexer<'input, StandardLexeme<StorageT>, StorageT>
+    for LRNonStreamingLexer<'lexer, 'input, StorageT>
 {
     fn span_str(&self, span: Span) -> &'input str {
         if span.end() > self.s.len() {
@@ -334,15 +341,18 @@ impl<'lexer, 'input: 'lexer, StorageT: Copy + Eq + Hash + PrimInt + Unsigned>
         }
 
         /// Returns `(line byte offset, line index)`.
-        fn lc_byte<StorageT>(lexer: &LRNonStreamingLexer<StorageT>, i: usize) -> (usize, usize) {
+        fn lc_byte<StorageT: fmt::Debug>(
+            lexer: &LRNonStreamingLexer<StorageT>,
+            i: usize,
+        ) -> (usize, usize) {
             match lexer.newlines.binary_search(&i) {
                 Ok(j) => (lexer.newlines[j], j + 2),
                 Err(0) => (0, 1),
                 Err(j) => (lexer.newlines[j - 1], j + 1),
             }
         }
 
-        fn lc_char<StorageT: Copy + Eq + Hash + PrimInt + Unsigned>(
+        fn lc_char<StorageT: Copy + Eq + fmt::Debug + Hash + PrimInt + Unsigned>(
             lexer: &LRNonStreamingLexer<StorageT>,
             i: usize,
             s: &str,

lrlex/src/main.rs

@@ -8,7 +8,7 @@ use std::{
 };
 
 use lrlex::{LRNonStreamingLexerDef, LexerDef};
-use lrpar::Lexer;
+use lrpar::{Lexeme, Lexer};
 
 fn usage(prog: &str, msg: &str) {
     let path = Path::new(prog);

lrpar/cttests/src/lib.rs

@@ -3,7 +3,7 @@ use lrpar::lrpar_mod;
 #[cfg(test)]
 use lrpar::Span;
 #[cfg(test)]
-use lrpar::{Lexer, NonStreamingLexer};
+use lrpar::{Lexeme, Lexer, NonStreamingLexer};
 
 lrlex_mod!("calc_multitypes.l");
 lrpar_mod!("calc_multitypes.y");

lrpar/examples/calc_ast/src/main.rs

@@ -3,7 +3,7 @@
 use std::io::{self, BufRead, Write};
 
 use lrlex::lrlex_mod;
-use lrpar::{lrpar_mod, NonStreamingLexer, Span};
+use lrpar::{lrpar_mod, NonStreamingLexer, Span, StandardLexeme};
 
 // Using `lrlex_mod!` brings the lexer for `calc.l` into scope. By default the module name will be
 // `calc_l` (i.e. the file name, minus any extensions, with a suffix of `_l`).
@@ -54,7 +54,10 @@ fn main() {
     }
 }
 
-fn eval(lexer: &dyn NonStreamingLexer<u32>, e: Expr) -> Result<u64, (Span, &'static str)> {
+fn eval(
+    lexer: &dyn NonStreamingLexer<StandardLexeme<u32>, u32>,
+    e: Expr,
+) -> Result<u64, (Span, &'static str)> {
     match e {
         Expr::Add { span, lhs, rhs } => eval(lexer, *lhs)?
             .checked_add(eval(lexer, *rhs)?)

lrpar/examples/calc_parsetree/build.rs

@@ -1,6 +1,6 @@
 use cfgrammar::yacc::{YaccKind, YaccOriginalActionKind};
 use lrlex::CTLexerBuilder;
-use lrpar::CTParserBuilder;
+use lrpar::{CTParserBuilder, StandardLexeme};
 
 fn main() -> Result<(), Box<dyn std::error::Error>> {
     // First we create the parser, which returns a HashMap of all the tokens used, then we pass
@@ -9,7 +9,7 @@ fn main() -> Result<(), Box<dyn std::error::Error>> {
     // Note that we specify the integer type (u8) we'll use for token IDs (this type *must* be big
     // enough to fit all IDs in) as well as the input file (which must end in ".y" for lrpar, and
     // ".l" for lrlex).
-    let cp = CTParserBuilder::<u8>::new_with_storaget()
+    let cp = CTParserBuilder::<StandardLexeme<u8>, _>::new_with_storaget()
         .yacckind(YaccKind::Original(YaccOriginalActionKind::GenericParseTree))
         .grammar_in_src_dir("calc.y")?
         .build()?;

lrpar/examples/calc_parsetree/src/main.rs

@@ -2,7 +2,7 @@ use std::io::{self, BufRead, Write};
 
 use cfgrammar::RIdx;
 use lrlex::lrlex_mod;
-use lrpar::{lrpar_mod, Node};
+use lrpar::{lrpar_mod, Lexeme, Node, StandardLexeme};
 
 // Using `lrlex_mod!` brings the lexer for `calc.l` into scope. By default the module name will be
 // `calc_l` (i.e. the file name, minus any extensions, with a suffix of `_l`).
@@ -49,7 +49,7 @@ impl<'a> Eval<'a> {
         Eval { s }
     }
 
-    fn eval(&self, n: &Node<u8>) -> i64 {
+    fn eval(&self, n: &Node<StandardLexeme<u8>, u8>) -> i64 {
         match *n {
             Node::Nonterm {
                 ridx: RIdx(ridx),

lrpar/src/lib/cpctplus.rs

@@ -100,13 +100,25 @@ impl<StorageT: PrimInt + Unsigned> PartialEq for PathFNode<StorageT> {
 
 impl<StorageT: PrimInt + Unsigned> Eq for PathFNode<StorageT> {}
 
-struct CPCTPlus<'a, 'b: 'a, 'input: 'b, StorageT: 'static + Eq + Hash, ActionT: 'a> {
-    parser: &'a Parser<'a, 'b, 'input, StorageT, ActionT>,
+struct CPCTPlus<
+    'a,
+    'b: 'a,
+    'input: 'b,
+    LexemeT: Lexeme<StorageT>,
+    StorageT: 'static + Eq + Hash,
+    ActionT: 'a,
+> {
+    parser: &'a Parser<'a, 'b, 'input, LexemeT, StorageT, ActionT>,
 }
 
-pub(super) fn recoverer<'a, StorageT: 'static + Debug + Hash + PrimInt + Unsigned, ActionT: 'a>(
-    parser: &'a Parser<StorageT, ActionT>,
-) -> Box<dyn Recoverer<StorageT, ActionT> + 'a>
+pub(super) fn recoverer<
+    'a,
+    LexemeT: Lexeme<StorageT>,
+    StorageT: 'static + Debug + Hash + PrimInt + Unsigned,
+    ActionT: 'a,
+>(
+    parser: &'a Parser<LexemeT, StorageT, ActionT>,
+) -> Box<dyn Recoverer<LexemeT, StorageT, ActionT> + 'a>
 where
     usize: AsPrimitive<StorageT>,
 {
@@ -117,21 +129,22 @@ impl<
         'a,
         'b: 'a,
         'input: 'b,
+        LexemeT: Lexeme<StorageT>,
         StorageT: 'static + Debug + Hash + PrimInt + Unsigned,
         ActionT: 'a,
-    > Recoverer<StorageT, ActionT> for CPCTPlus<'a, 'b, 'input, StorageT, ActionT>
+    > Recoverer<LexemeT, StorageT, ActionT> for CPCTPlus<'a, 'b, 'input, LexemeT, StorageT, ActionT>
 where
     usize: AsPrimitive<StorageT>,
 {
     fn recover(
         &self,
         finish_by: Instant,
-        parser: &Parser<StorageT, ActionT>,
+        parser: &Parser<LexemeT, StorageT, ActionT>,
         in_laidx: usize,
         mut in_pstack: &mut Vec<StIdx>,
-        mut astack: &mut Vec<AStackType<ActionT, StorageT>>,
+        mut astack: &mut Vec<AStackType<LexemeT, ActionT>>,
         mut spans: &mut Vec<Span>,
-    ) -> (usize, Vec<Vec<ParseRepair<StorageT>>>) {
+    ) -> (usize, Vec<Vec<ParseRepair<LexemeT, StorageT>>>) {
         // This function implements a minor variant of the algorithm from "Repairing syntax errors
         // in LR parsers" by Rafael Corchuelo, Jose A. Perez, Antonio Ruiz, and Miguel Toro.
         //
@@ -249,9 +262,10 @@ impl<
         'a,
         'b: 'a,
         'input: 'b,
+        LexemeT: Lexeme<StorageT>,
         StorageT: 'static + Debug + Hash + PrimInt + Unsigned,
         ActionT: 'a,
-    > CPCTPlus<'a, 'b, 'input, StorageT, ActionT>
+    > CPCTPlus<'a, 'b, 'input, LexemeT, StorageT, ActionT>
 where
     usize: AsPrimitive<StorageT>,
 {
@@ -357,7 +371,7 @@ where
         &self,
         in_laidx: usize,
         cnds: Vec<PathFNode<StorageT>>,
-    ) -> Vec<Vec<Vec<ParseRepair<StorageT>>>> {
+    ) -> Vec<Vec<Vec<ParseRepair<LexemeT, StorageT>>>> {
         fn traverse<StorageT: PrimInt>(
             rm: &Cactus<RepairMerge<StorageT>>,
         ) -> Vec<Vec<Repair<StorageT>>> {
@@ -411,7 +425,7 @@ where
         &self,
         mut laidx: usize,
         from: &[Repair<StorageT>],
-    ) -> Vec<ParseRepair<StorageT>> {
+    ) -> Vec<ParseRepair<LexemeT, StorageT>> {
         from.iter()
             .map(|y| match *y {
                 Repair::InsertTerm(token_idx) => ParseRepair::Insert(token_idx),
@@ -432,13 +446,18 @@ where
 
 /// Apply the `repairs` to `pstack` starting at position `laidx`: return the resulting parse
 /// distance and a new pstack.
-fn apply_repairs<'a, StorageT: 'static + Debug + Hash + PrimInt + Unsigned, ActionT: 'a>(
-    parser: &Parser<StorageT, ActionT>,
+fn apply_repairs<
+    'a,
+    LexemeT: Lexeme<StorageT>,
+    StorageT: 'static + Debug + Hash + PrimInt + Unsigned,
+    ActionT: 'a,
+>(
+    parser: &Parser<LexemeT, StorageT, ActionT>,
     mut laidx: usize,
     mut pstack: &mut Vec<StIdx>,
-    mut astack: &mut Option<&mut Vec<AStackType<ActionT, StorageT>>>,
+    mut astack: &mut Option<&mut Vec<AStackType<LexemeT, ActionT>>>,
     mut spans: &mut Option<&mut Vec<Span>>,
-    repairs: &[ParseRepair<StorageT>],
+    repairs: &[ParseRepair<LexemeT, StorageT>],
 ) -> usize
 where
     usize: AsPrimitive<StorageT>,
@@ -474,9 +493,13 @@ where
 }
 
 /// Simplifies repair sequences, removes duplicates, and sorts them into order.
-fn simplify_repairs<StorageT: 'static + Hash + PrimInt + Unsigned, ActionT>(
-    parser: &Parser<StorageT, ActionT>,
-    all_rprs: &mut Vec<Vec<ParseRepair<StorageT>>>,
+fn simplify_repairs<
+    LexemeT: Lexeme<StorageT>,
+    StorageT: 'static + Hash + PrimInt + Unsigned,
+    ActionT,
+>(
+    parser: &Parser<LexemeT, StorageT, ActionT>,
+    all_rprs: &mut Vec<Vec<ParseRepair<LexemeT, StorageT>>>,
 ) where
     usize: AsPrimitive<StorageT>,
 {
@@ -494,13 +517,13 @@ fn simplify_repairs<StorageT: 'static + Hash + PrimInt + Unsigned, ActionT>(
     // Use a HashSet as a quick way of deduplicating repair sequences: occasionally we can end up
     // with hundreds of thousands (!), and we don't have a sensible ordering on ParseRepair to make
     // it plausible to do a sort and dedup.
-    let mut hs: HashSet<Vec<ParseRepair<StorageT>>> = all_rprs.drain(..).collect();
+    let mut hs: HashSet<Vec<ParseRepair<LexemeT, StorageT>>> = all_rprs.drain(..).collect();
     all_rprs.extend(hs.drain());
 
     // Sort repair sequences:
     //   1) by whether they contain Inserts that are %insert_avoid
     //   2) by the number of repairs they contain
-    let contains_avoid_insert = |rprs: &Vec<ParseRepair<StorageT>>| -> bool {
+    let contains_avoid_insert = |rprs: &Vec<ParseRepair<LexemeT, StorageT>>| -> bool {
         for r in rprs.iter() {
             if let ParseRepair::Insert(tidx) = r {
                 if parser.grm.avoid_insert(*tidx) {
@@ -528,13 +551,18 @@ fn simplify_repairs<StorageT: 'static + Hash + PrimInt + Unsigned, ActionT>(
 /// `ParseRepair`s allow the same distance of parsing, then the `ParseRepair` which requires
 /// repairs over the shortest distance is preferred. Amongst `ParseRepair`s of the same rank, the
 /// ordering is non-deterministic.
-fn rank_cnds<'a, StorageT: 'static + Debug + Hash + PrimInt + Unsigned, ActionT: 'a>(
-    parser: &Parser<StorageT, ActionT>,
+fn rank_cnds<
+    'a,
+    LexemeT: Lexeme<StorageT>,
+    StorageT: 'static + Debug + Hash + PrimInt + Unsigned,
+    ActionT: 'a,
+>(
+    parser: &Parser<LexemeT, StorageT, ActionT>,
     finish_by: Instant,
     in_laidx: usize,
     in_pstack: &[StIdx],
-    in_cnds: Vec<Vec<Vec<ParseRepair<StorageT>>>>,
-) -> Vec<Vec<ParseRepair<StorageT>>>
+    in_cnds: Vec<Vec<Vec<ParseRepair<LexemeT, StorageT>>>>,
+) -> Vec<Vec<ParseRepair<LexemeT, StorageT>>>
 where
     usize: AsPrimitive<StorageT>,
 {
@@ -603,9 +631,9 @@ mod test {
         parser::{test::do_parse, LexParseError, ParseRepair, RecoveryKind},
     };
 
-    fn pp_repairs<StorageT: 'static + Hash + PrimInt + Unsigned>(
+    fn pp_repairs<LexemeT: Lexeme<StorageT>, StorageT: 'static + Hash + PrimInt + Unsigned>(
         grm: &YaccGrammar<StorageT>,
-        repairs: &[ParseRepair<StorageT>],
+        repairs: &[ParseRepair<LexemeT, StorageT>],
     ) -> String
     where
         usize: AsPrimitive<StorageT>,
@@ -623,9 +651,12 @@ mod test {
         out.join(", ")
     }
 
-    fn check_all_repairs<StorageT: 'static + Debug + Hash + PrimInt + Unsigned>(
+    fn check_all_repairs<
+        LexemeT: Lexeme<StorageT>,
+        StorageT: 'static + Debug + Hash + PrimInt + Unsigned,
+    >(
         grm: &YaccGrammar<StorageT>,
-        err: &LexParseError<StorageT>,
+        err: &LexParseError<LexemeT, StorageT>,
         expected: &[&str],
     ) where
         usize: AsPrimitive<StorageT>,