refactor(tazjin/rlox): Let scanner tokens own their lexeme

This removes the runtime dependency on a borrow into the program
source code.

It's not yet ideal because there are a lot of tokens where we really
don't care about the lexeme, but this is what the book does and I
am not going to change that.

Change-Id: I888e18f98597766d6f725cbf9241e8eb2bd839e2
Reviewed-on: https://cl.tvl.fyi/c/depot/+/2394
Reviewed-by: tazjin <mail@tazj.in>
Tested-by: BuildkiteCI

users/tazjin/rlox/src/scanner.rs

@@ -53,15 +53,15 @@ pub enum TokenKind {
 }
 
 #[derive(Clone, Debug)]
-pub struct Token<'a> {
+pub struct Token {
     pub kind: TokenKind,
-    pub lexeme: &'a [char],
+    pub lexeme: String,
     pub line: usize,
 }
 
 struct Scanner<'a> {
     source: &'a [char],
-    tokens: Vec<Token<'a>>,
+    tokens: Vec<Token>,
     errors: Vec<Error>,
     start: usize,   // offset of first character in current lexeme
     current: usize, // current offset into source
@@ -82,7 +82,7 @@ impl<'a> Scanner<'a> {
         let lexeme = &self.source[self.start..self.current];
         self.tokens.push(Token {
             kind,
-            lexeme,
+            lexeme: lexeme.into_iter().collect(),
             line: self.line,
         })
     }
@@ -263,7 +263,7 @@ impl<'a> Scanner<'a> {
     }
 }
 
-pub fn scan<'a>(input: &'a [char]) -> Result<Vec<Token<'a>>, Vec<Error>> {
+pub fn scan<'a>(input: &'a [char]) -> Result<Vec<Token>, Vec<Error>> {
     let mut scanner = Scanner {
         source: &input,
         tokens: vec![],