Small refactor for lexer

src/lexer.rs

@@ -107,10 +107,29 @@ impl<'a> Lexer<'a> {
                 '}' => tokens.push(Token::RBraces),
                 '!' => tokens.push(Token::LNot),
 
-                // Lex numbers
-                ch @ '0'..='9' => {
+                // Special tokens with variable length
+
+                // Lex multiple characters together as numbers
+                ch @ '0'..='9' => tokens.push(self.lex_number(ch)?),
+
+                // Lex multiple characters together as a string
+                '"' => tokens.push(self.lex_str()?),
+
+                // Lex multiple characters together as identifier
+                ch @ ('a'..='z' | 'A'..='Z' | '_') => tokens.push(self.lex_identifier(ch)?),
+
+                ch => Err(LexErr::UnexpectedChar(ch))?,
+            }
+        }
+
+        Ok(tokens)
+    }
+
+    /// Lex multiple characters as a number until encountering a non numeric digit. This includes
+    /// the first character 
+    fn lex_number(&mut self, first_char: char) -> Result<Token, LexErr> {
         // String representation of the integer value
-                    let mut sval = String::from(ch);
+        let mut sval = String::from(first_char);
 
         // Do as long as a next char exists and it is a numeric char
         loop {
@@ -130,11 +149,11 @@ impl<'a> Lexer<'a> {
 
         // Try to convert the string representation of the value to i64
         let i64val = sval.parse().map_err(|_| LexErr::NumericParse(sval))?;
-                    tokens.push(Token::I64(i64val));
+        Ok(Token::I64(i64val))
     }
 
-                // Lex a string
-                '"' => {
+    /// Lex characters as a string until encountering an unescaped closing doublequoute char '"'
+    fn lex_str(&mut self) -> Result<Token, LexErr> {
         // Opening " was consumed in match
 
         let mut text = String::new();
@@ -164,12 +183,12 @@ impl<'a> Lexer<'a> {
         // Consume closing "
         self.next();
 
-                    tokens.push(Token::String(text))
+        Ok(Token::String(text))
     }
 
-                // Lex characters as identifier
-                ch @ ('a'..='z' | 'A'..='Z' | '_') => {
-                    let mut ident = String::from(ch);
+    /// Lex characters from the text as an identifier. This includes the first character passed in
+    fn lex_identifier(&mut self, first_char: char) -> Result<Token, LexErr> {
+        let mut ident = String::from(first_char);
 
         // Do as long as a next char exists and it is a valid char for an identifier
         loop {
@@ -194,14 +213,7 @@ impl<'a> Lexer<'a> {
             _ => Token::Ident(ident),
         };
 
-                    tokens.push(token);
-                }
-
-                ch => Err(LexErr::UnexpectedChar(ch))?,
-            }
-        }
-
-        Ok(tokens)
+        Ok(token)
     }
 
     /// Advance to next character and return the removed char