feat(tazjin/rlox): Scan identifiers and keywords
Change-Id: Ifec627605c23c25f199d47eaa91e441ed9590208 Reviewed-on: https://cl.tvl.fyi/c/depot/+/2192 Reviewed-by: tazjin <mail@tazj.in> Tested-by: BuildkiteCI
This commit is contained in:
		
							parent
							
								
									97505eb1e1
								
							
						
					
					
						commit
						af793325c0
					
				
					 1 changed files with 41 additions and 6 deletions
				
			
		| 
						 | 
				
			
			@ -26,7 +26,7 @@ pub enum TokenKind {
 | 
			
		|||
    LessEqual,
 | 
			
		||||
 | 
			
		||||
    // Literals.
 | 
			
		||||
    Identifier,
 | 
			
		||||
    Identifier(String),
 | 
			
		||||
    String(String),
 | 
			
		||||
    Number(f64),
 | 
			
		||||
 | 
			
		||||
| 
						 | 
				
			
			@ -120,15 +120,18 @@ impl<'a> Scanner<'a> {
 | 
			
		|||
            }
 | 
			
		||||
 | 
			
		||||
            // ignore whitespace
 | 
			
		||||
            ' ' => {}
 | 
			
		||||
            '\r' => {}
 | 
			
		||||
            '\t' => {}
 | 
			
		||||
            '\n' => self.line += 1,
 | 
			
		||||
            ws if ws.is_whitespace() => {
 | 
			
		||||
                if ws == '\n' {
 | 
			
		||||
                    self.line += 1
 | 
			
		||||
                }
 | 
			
		||||
            }
 | 
			
		||||
 | 
			
		||||
            '"' => self.scan_string(),
 | 
			
		||||
 | 
			
		||||
            digit if digit.is_digit(10) => self.scan_number(),
 | 
			
		||||
 | 
			
		||||
            chr if chr.is_alphabetic() || chr == '_' => self.scan_identifier(),
 | 
			
		||||
 | 
			
		||||
            unexpected => self.errors.push(Error {
 | 
			
		||||
                line: self.line,
 | 
			
		||||
                kind: ErrorKind::UnexpectedChar(unexpected),
 | 
			
		||||
| 
						 | 
				
			
			@ -162,7 +165,7 @@ impl<'a> Scanner<'a> {
 | 
			
		|||
    }
 | 
			
		||||
 | 
			
		||||
    fn peek_next(&self) -> char {
 | 
			
		||||
        if (self.current + 1 >= self.source.len()) {
 | 
			
		||||
        if self.current + 1 >= self.source.len() {
 | 
			
		||||
            return '\0';
 | 
			
		||||
        } else {
 | 
			
		||||
            return self.source[self.current + 1];
 | 
			
		||||
| 
						 | 
				
			
			@ -220,12 +223,44 @@ impl<'a> Scanner<'a> {
 | 
			
		|||
        self.add_token(TokenKind::Number(num));
 | 
			
		||||
    }
 | 
			
		||||
 | 
			
		||||
    fn scan_identifier(&mut self) {
 | 
			
		||||
        while self.peek().is_alphanumeric() || self.peek() == '_' {
 | 
			
		||||
            self.advance();
 | 
			
		||||
        }
 | 
			
		||||
 | 
			
		||||
        let ident: String = self.source[self.start..self.current].iter().collect();
 | 
			
		||||
 | 
			
		||||
        // Determine whether this is an identifier, or a keyword:
 | 
			
		||||
        let token_kind = match ident.as_str() {
 | 
			
		||||
            "and" => TokenKind::And,
 | 
			
		||||
            "class" => TokenKind::Class,
 | 
			
		||||
            "else" => TokenKind::Else,
 | 
			
		||||
            "false" => TokenKind::False,
 | 
			
		||||
            "for" => TokenKind::For,
 | 
			
		||||
            "fun" => TokenKind::Fun,
 | 
			
		||||
            "if" => TokenKind::If,
 | 
			
		||||
            "nil" => TokenKind::Nil,
 | 
			
		||||
            "or" => TokenKind::Or,
 | 
			
		||||
            "print" => TokenKind::Print,
 | 
			
		||||
            "return" => TokenKind::Return,
 | 
			
		||||
            "super" => TokenKind::Super,
 | 
			
		||||
            "this" => TokenKind::This,
 | 
			
		||||
            "true" => TokenKind::True,
 | 
			
		||||
            "var" => TokenKind::Var,
 | 
			
		||||
            "while" => TokenKind::While,
 | 
			
		||||
            _ => TokenKind::Identifier(ident),
 | 
			
		||||
        };
 | 
			
		||||
 | 
			
		||||
        self.add_token(token_kind);
 | 
			
		||||
    }
 | 
			
		||||
 | 
			
		||||
    fn scan_tokens(mut self) -> Vec<Token<'a>> {
 | 
			
		||||
        while !self.is_at_end() {
 | 
			
		||||
            self.start = self.current;
 | 
			
		||||
            self.scan_token();
 | 
			
		||||
        }
 | 
			
		||||
 | 
			
		||||
        self.add_token(TokenKind::Eof);
 | 
			
		||||
        return self.tokens;
 | 
			
		||||
    }
 | 
			
		||||
}
 | 
			
		||||
| 
						 | 
				
			
			
 | 
			
		|||
		Loading…
	
	Add table
		Add a link
		
	
		Reference in a new issue