refactor(tazjin/rlox): Prepare scanner for shared use

In the book, the clox interpreter has its own scanner which uses a
pull-based model for a single pass compiler.

I can't be bothered to write another scanner, or amend this one into
pull-mode to work with the treewalk interpreter, so instead I will
just reuse it and pull from a vector of tokens.

The tokens are shared between both interpreters and the scanner is not
what I'm interested in here.

Change-Id: Ib07e89127fce2b047f9b3e1ff7e9908d798b3b2b
Reviewed-on: https://cl.tvl.fyi/c/depot/+/2420
Reviewed-by: tazjin <mail@tazj.in>
Tested-by: BuildkiteCI

users/tazjin/rlox/src/main.rs

@@ -5,6 +5,7 @@ use std::io::Write;
 use std::process;
 
 mod bytecode;
+mod scanner;
 mod treewalk;
 
 /// Trait for making the different interpreters callable in the same

users/tazjin/rlox/src/scanner.rs

@@ -1,5 +1,3 @@
-use crate::treewalk::errors::{Error, ErrorKind};
-
 #[derive(Clone, Debug, PartialEq)]
 pub enum TokenKind {
     // Single-character tokens.
@@ -59,10 +57,15 @@ pub struct Token {
     pub line: usize,
 }
 
+pub enum ScannerError {
+    UnexpectedChar { line: usize, unexpected: char },
+    UnterminatedString { line: usize },
+}
+
 struct Scanner<'a> {
     source: &'a [char],
     tokens: Vec<Token>,
-    errors: Vec<Error>,
+    errors: Vec<ScannerError>,
     start: usize,   // offset of first character in current lexeme
     current: usize, // current offset into source
     line: usize,    // current line in source
@@ -131,9 +134,9 @@ impl<'a> Scanner<'a> {
 
             chr if chr.is_alphabetic() || chr == '_' => self.scan_identifier(),
 
-            unexpected => self.errors.push(Error {
+            unexpected => self.errors.push(ScannerError::UnexpectedChar {
                 line: self.line,
-                kind: ErrorKind::UnexpectedChar(unexpected),
+                unexpected,
             }),
         };
     }
@@ -181,10 +184,8 @@ impl<'a> Scanner<'a> {
         }
 
         if self.is_at_end() {
-            self.errors.push(Error {
+            self.errors
-                line: self.line,
+                .push(ScannerError::UnterminatedString { line: self.line });
-                kind: ErrorKind::UnterminatedString,
-            });
             return;
         }
 
@@ -263,7 +264,7 @@ impl<'a> Scanner<'a> {
     }
 }
 
-pub fn scan<'a>(input: &'a [char]) -> Result<Vec<Token>, Vec<Error>> {
+pub fn scan<'a>(input: &'a [char]) -> Result<Vec<Token>, Vec<ScannerError>> {
     let mut scanner = Scanner {
         source: &input,
         tokens: vec![],

users/tazjin/rlox/src/treewalk/errors.rs

@@ -1,4 +1,6 @@
+use crate::scanner::ScannerError;
 use crate::treewalk::interpreter::Value;
+
 use std::fmt;
 
 #[derive(Debug)]
@@ -39,3 +41,19 @@ impl fmt::Display for Error {
         write!(f, "[line {}] Error: {:?}", self.line, self.kind)
     }
 }
+
+impl From<ScannerError> for Error {
+    fn from(err: ScannerError) -> Self {
+        match err {
+            ScannerError::UnexpectedChar { line, unexpected } => Error {
+                line,
+                kind: ErrorKind::UnexpectedChar(unexpected),
+            },
+
+            ScannerError::UnterminatedString { line } => Error {
+                line,
+                kind: ErrorKind::UnterminatedString,
+            },
+        }
+    }
+}

users/tazjin/rlox/src/treewalk/interpreter.rs

@@ -200,7 +200,9 @@ impl Lox for Interpreter {
     fn interpret(&mut self, code: String) -> Result<Value, Vec<Error>> {
         let chars: Vec<char> = code.chars().collect();
 
-        let mut program = scanner::scan(&chars).and_then(|tokens| parser::parse(tokens))?;
+        let mut program = scanner::scan(&chars)
+            .map_err(|errors| errors.into_iter().map(Into::into).collect())
+            .and_then(|tokens| parser::parse(tokens))?;
 
         let globals = self
             .env

users/tazjin/rlox/src/treewalk/mod.rs

@@ -1,5 +1,6 @@
+use crate::scanner;
+
 mod errors;
 pub mod interpreter;
 mod parser;
 mod resolver;
-mod scanner;