refactor(tazjin/rlox): Use &[char] instead of &str in scanner
This makes it easier to work with the Unicode issue. The original string representation can be discarded. Change-Id: I740be4cb9654679ea7950f3899c5c709b1e7a739 Reviewed-on: https://cl.tvl.fyi/c/depot/+/2160 Reviewed-by: tazjin <mail@tazj.in> Tested-by: BuildkiteCI
This commit is contained in:
		
							parent
							
								
									b595553f63
								
							
						
					
					
						commit
						6363efbebf
					
				
					 1 changed files with 3 additions and 7 deletions
				
			
		| 
						 | 
				
			
			@ -55,13 +55,13 @@ pub enum TokenKind {
 | 
			
		|||
#[derive(Debug)]
 | 
			
		||||
pub struct Token<'a> {
 | 
			
		||||
    kind: TokenKind,
 | 
			
		||||
    lexeme: &'a str,
 | 
			
		||||
    lexeme: &'a [char],
 | 
			
		||||
    // literal: Object, // TODO(tazjin): Uhh?
 | 
			
		||||
    line: usize,
 | 
			
		||||
}
 | 
			
		||||
 | 
			
		||||
struct Scanner<'a> {
 | 
			
		||||
    source: &'a str,
 | 
			
		||||
    source: &'a [char],
 | 
			
		||||
    tokens: Vec<Token<'a>>,
 | 
			
		||||
    errors: Vec<Error>,
 | 
			
		||||
    start: usize,   // offset of first character in current lexeme
 | 
			
		||||
| 
						 | 
				
			
			@ -76,11 +76,7 @@ impl<'a> Scanner<'a> {
 | 
			
		|||
 | 
			
		||||
    fn advance(&mut self) -> char {
 | 
			
		||||
        self.current += 1;
 | 
			
		||||
 | 
			
		||||
        // TODO(tazjin): Due to utf8-safety, this is a bit annoying.
 | 
			
		||||
        // Since string iteration is not the point here I'm just
 | 
			
		||||
        // leaving this as is for now.
 | 
			
		||||
        self.source.chars().nth(self.current - 1).unwrap()
 | 
			
		||||
        self.source[self.current-1]
 | 
			
		||||
    }
 | 
			
		||||
 | 
			
		||||
    fn add_token(&mut self, kind: TokenKind) {
 | 
			
		||||
| 
						 | 
				
			
			
 | 
			
		|||
		Loading…
	
	Add table
		Add a link
		
	
		Reference in a new issue