Move token into separate file

src/lexer.rs

@@ -1,102 +1,6 @@
 use std::{iter::Peekable, str::Chars};
 
-use crate::parser::BinOpType;
+use crate::token::Token;
-
-#[derive(Debug, PartialEq, Eq)]
-pub enum Token {
-    /// Integer literal (64-bit)
-    I64(i64),
-
-    /// Identifier (name for variables, functions, ...)
-    Ident(String),
-
-    /// Loop keyword (loop)
-    Loop,
-
-    /// Print keyword (print)
-    Print,
-
-    /// If keyword (if)
-    If,
-
-    /// Else keyword (else)
-    Else,
-
-    /// Left Parenthesis ('(')
-    LParen,
-
-    /// Right Parenthesis (')')
-    RParen,
-
-    /// Left curly braces ({)
-    LBraces,
-
-    /// Right curly braces (})
-    RBraces,
-
-    /// Plus (+)
-    Add,
-
-    /// Minus (-)
-    Sub,
-
-    /// Asterisk (*)
-    Mul,
-
-    /// Slash (/)
-    Div,
-
-    /// Percent (%)
-    Mod,
-
-    /// Equal Equal (==)
-    EquEqu,
-
-    /// Exclamationmark Equal (!=)
-    NotEqu,
-
-    /// Pipe (|)
-    BOr,
-
-    /// Ampersand (&)
-    BAnd,
-
-    /// Circumflex (^)
-    BXor,
-
-    /// Shift Left (<<)
-    Shl,
-
-    /// Shift Right (>>)
-    Shr,
-
-    /// Tilde (~)
-    Tilde,
-
-    /// Left angle bracket (<)
-    LAngle,
-
-    /// Right angle bracket (>)
-    RAngle,
-
-    /// Left angle bracket Equal (<=)
-    LAngleEqu,
-
-    /// Left angle bracket Equal (>=)
-    RAngleEqu,
-
-    /// Left arrow (<-)
-    LArrow,
-
-    /// Equal Sign (=)
-    Equ,
-
-    /// Semicolon (;)
-    Semicolon,
-
-    /// End of file
-    EoF,
-}
 
 struct Lexer<'a> {
     code: Peekable<Chars<'a>>,
@@ -245,40 +149,6 @@ pub fn lex(code: &str) -> Vec<Token> {
     lexer.lex()
 }
 
-impl Token {
-    pub fn try_to_binop(&self) -> Option<BinOpType> {
-        Some(match self {
-            Token::Add => BinOpType::Add,
-            Token::Sub => BinOpType::Sub,
-
-            Token::Mul => BinOpType::Mul,
-            Token::Div => BinOpType::Div,
-            Token::Mod => BinOpType::Mod,
-
-            Token::BAnd => BinOpType::BAnd,
-            Token::BOr => BinOpType::BOr,
-            Token::BXor => BinOpType::BXor,
-
-            Token::Shl => BinOpType::Shl,
-            Token::Shr => BinOpType::Shr,
-
-            Token::EquEqu => BinOpType::EquEqu,
-            Token::NotEqu => BinOpType::NotEqu,
-
-            Token::LAngle => BinOpType::Less,
-            Token::LAngleEqu => BinOpType::LessEqu,
-
-            Token::RAngle => BinOpType::Greater,
-            Token::RAngleEqu => BinOpType::GreaterEqu,
-
-            Token::LArrow => BinOpType::Declare,
-            Token::Equ => BinOpType::Assign,
-
-            _ => return None,
-        })
-    }
-}
-
 #[cfg(test)]
 mod tests {
     use super::{lex, Token};

src/lib.rs

@@ -1,3 +1,4 @@
 pub mod lexer;
+pub mod token;
 pub mod parser;
 pub mod interpreter;

src/parser.rs

@@ -1,6 +1,6 @@
 use std::iter::Peekable;
 
-use crate::lexer::Token;
+use crate::token::Token;
 
 /// Types for binary operators
 #[derive(Debug, PartialEq, Eq, Clone)]
@@ -365,7 +365,7 @@ impl BinOpType {
 #[cfg(test)]
 mod tests {
     use super::{parse, Expression, BinOpType};
-    use crate::{lexer::Token, parser::{Statement, Ast}};
+    use crate::{token::Token, parser::{Statement, Ast}};
 
     #[test]
     fn test_parser() {

src/token.rs

@@ -0,0 +1,131 @@
+use crate::parser::BinOpType;
+
+#[derive(Debug, PartialEq, Eq)]
+pub enum Token {
+    /// Integer literal (64-bit)
+    I64(i64),
+
+    /// Identifier (name for variables, functions, ...)
+    Ident(String),
+
+    /// Loop keyword (loop)
+    Loop,
+
+    /// Print keyword (print)
+    Print,
+
+    /// If keyword (if)
+    If,
+
+    /// Else keyword (else)
+    Else,
+
+    /// Left Parenthesis ('(')
+    LParen,
+
+    /// Right Parenthesis (')')
+    RParen,
+
+    /// Left curly braces ({)
+    LBraces,
+
+    /// Right curly braces (})
+    RBraces,
+
+    /// Plus (+)
+    Add,
+
+    /// Minus (-)
+    Sub,
+
+    /// Asterisk (*)
+    Mul,
+
+    /// Slash (/)
+    Div,
+
+    /// Percent (%)
+    Mod,
+
+    /// Equal Equal (==)
+    EquEqu,
+
+    /// Exclamationmark Equal (!=)
+    NotEqu,
+
+    /// Pipe (|)
+    BOr,
+
+    /// Ampersand (&)
+    BAnd,
+
+    /// Circumflex (^)
+    BXor,
+
+    /// Shift Left (<<)
+    Shl,
+
+    /// Shift Right (>>)
+    Shr,
+
+    /// Tilde (~)
+    Tilde,
+
+    /// Left angle bracket (<)
+    LAngle,
+
+    /// Right angle bracket (>)
+    RAngle,
+
+    /// Left angle bracket Equal (<=)
+    LAngleEqu,
+
+    /// Left angle bracket Equal (>=)
+    RAngleEqu,
+
+    /// Left arrow (<-)
+    LArrow,
+
+    /// Equal Sign (=)
+    Equ,
+
+    /// Semicolon (;)
+    Semicolon,
+
+    /// End of file
+    EoF,
+}
+
+impl Token {
+    pub fn try_to_binop(&self) -> Option<BinOpType> {
+        Some(match self {
+            Token::Add => BinOpType::Add,
+            Token::Sub => BinOpType::Sub,
+
+            Token::Mul => BinOpType::Mul,
+            Token::Div => BinOpType::Div,
+            Token::Mod => BinOpType::Mod,
+
+            Token::BAnd => BinOpType::BAnd,
+            Token::BOr => BinOpType::BOr,
+            Token::BXor => BinOpType::BXor,
+
+            Token::Shl => BinOpType::Shl,
+            Token::Shr => BinOpType::Shr,
+
+            Token::EquEqu => BinOpType::EquEqu,
+            Token::NotEqu => BinOpType::NotEqu,
+
+            Token::LAngle => BinOpType::Less,
+            Token::LAngleEqu => BinOpType::LessEqu,
+
+            Token::RAngle => BinOpType::Greater,
+            Token::RAngleEqu => BinOpType::GreaterEqu,
+
+            Token::LArrow => BinOpType::Declare,
+            Token::Equ => BinOpType::Assign,
+
+            _ => return None,
+        })
+    }
+}