From 7e2ef494815769ce3e9c3c5266b449c1861f5409 Mon Sep 17 00:00:00 2001 From: Daniel M Date: Wed, 2 Feb 2022 16:40:05 +0100 Subject: [PATCH] Move token into separate file --- src/lexer.rs | 132 +------------------------------------------------- src/lib.rs | 1 + src/parser.rs | 4 +- src/token.rs | 131 +++++++++++++++++++++++++++++++++++++++++++++++++ 4 files changed, 135 insertions(+), 133 deletions(-) create mode 100644 src/token.rs diff --git a/src/lexer.rs b/src/lexer.rs index a746dfb..447defc 100644 --- a/src/lexer.rs +++ b/src/lexer.rs @@ -1,102 +1,6 @@ use std::{iter::Peekable, str::Chars}; -use crate::parser::BinOpType; - -#[derive(Debug, PartialEq, Eq)] -pub enum Token { - /// Integer literal (64-bit) - I64(i64), - - /// Identifier (name for variables, functions, ...) - Ident(String), - - /// Loop keyword (loop) - Loop, - - /// Print keyword (print) - Print, - - /// If keyword (if) - If, - - /// Else keyword (else) - Else, - - /// Left Parenthesis ('(') - LParen, - - /// Right Parenthesis (')') - RParen, - - /// Left curly braces ({) - LBraces, - - /// Right curly braces (}) - RBraces, - - /// Plus (+) - Add, - - /// Minus (-) - Sub, - - /// Asterisk (*) - Mul, - - /// Slash (/) - Div, - - /// Percent (%) - Mod, - - /// Equal Equal (==) - EquEqu, - - /// Exclamationmark Equal (!=) - NotEqu, - - /// Pipe (|) - BOr, - - /// Ampersand (&) - BAnd, - - /// Circumflex (^) - BXor, - - /// Shift Left (<<) - Shl, - - /// Shift Right (>>) - Shr, - - /// Tilde (~) - Tilde, - - /// Left angle bracket (<) - LAngle, - - /// Right angle bracket (>) - RAngle, - - /// Left angle bracket Equal (<=) - LAngleEqu, - - /// Left angle bracket Equal (>=) - RAngleEqu, - - /// Left arrow (<-) - LArrow, - - /// Equal Sign (=) - Equ, - - /// Semicolon (;) - Semicolon, - - /// End of file - EoF, -} +use crate::token::Token; struct Lexer<'a> { code: Peekable>, @@ -245,40 +149,6 @@ pub fn lex(code: &str) -> Vec { lexer.lex() } -impl Token { - pub fn try_to_binop(&self) -> Option { - Some(match self { - Token::Add => BinOpType::Add, - Token::Sub => BinOpType::Sub, - - Token::Mul => BinOpType::Mul, - Token::Div => BinOpType::Div, - Token::Mod => BinOpType::Mod, - - Token::BAnd => BinOpType::BAnd, - Token::BOr => BinOpType::BOr, - Token::BXor => BinOpType::BXor, - - Token::Shl => BinOpType::Shl, - Token::Shr => BinOpType::Shr, - - Token::EquEqu => BinOpType::EquEqu, - Token::NotEqu => BinOpType::NotEqu, - - Token::LAngle => BinOpType::Less, - Token::LAngleEqu => BinOpType::LessEqu, - - Token::RAngle => BinOpType::Greater, - Token::RAngleEqu => BinOpType::GreaterEqu, - - Token::LArrow => BinOpType::Declare, - Token::Equ => BinOpType::Assign, - - _ => return None, - }) - } -} - #[cfg(test)] mod tests { use super::{lex, Token}; diff --git a/src/lib.rs b/src/lib.rs index 74c1228..c188eff 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -1,3 +1,4 @@ pub mod lexer; +pub mod token; pub mod parser; pub mod interpreter; diff --git a/src/parser.rs b/src/parser.rs index dbb42a6..e9c459f 100644 --- a/src/parser.rs +++ b/src/parser.rs @@ -1,6 +1,6 @@ use std::iter::Peekable; -use crate::lexer::Token; +use crate::token::Token; /// Types for binary operators #[derive(Debug, PartialEq, Eq, Clone)] @@ -365,7 +365,7 @@ impl BinOpType { #[cfg(test)] mod tests { use super::{parse, Expression, BinOpType}; - use crate::{lexer::Token, parser::{Statement, Ast}}; + use crate::{token::Token, parser::{Statement, Ast}}; #[test] fn test_parser() { diff --git a/src/token.rs b/src/token.rs new file mode 100644 index 0000000..7ed275d --- /dev/null +++ b/src/token.rs @@ -0,0 +1,131 @@ +use crate::parser::BinOpType; + +#[derive(Debug, PartialEq, Eq)] +pub enum Token { + /// Integer literal (64-bit) + I64(i64), + + /// Identifier (name for variables, functions, ...) + Ident(String), + + /// Loop keyword (loop) + Loop, + + /// Print keyword (print) + Print, + + /// If keyword (if) + If, + + /// Else keyword (else) + Else, + + /// Left Parenthesis ('(') + LParen, + + /// Right Parenthesis (')') + RParen, + + /// Left curly braces ({) + LBraces, + + /// Right curly braces (}) + RBraces, + + /// Plus (+) + Add, + + /// Minus (-) + Sub, + + /// Asterisk (*) + Mul, + + /// Slash (/) + Div, + + /// Percent (%) + Mod, + + /// Equal Equal (==) + EquEqu, + + /// Exclamationmark Equal (!=) + NotEqu, + + /// Pipe (|) + BOr, + + /// Ampersand (&) + BAnd, + + /// Circumflex (^) + BXor, + + /// Shift Left (<<) + Shl, + + /// Shift Right (>>) + Shr, + + /// Tilde (~) + Tilde, + + /// Left angle bracket (<) + LAngle, + + /// Right angle bracket (>) + RAngle, + + /// Left angle bracket Equal (<=) + LAngleEqu, + + /// Left angle bracket Equal (>=) + RAngleEqu, + + /// Left arrow (<-) + LArrow, + + /// Equal Sign (=) + Equ, + + /// Semicolon (;) + Semicolon, + + /// End of file + EoF, +} + +impl Token { + pub fn try_to_binop(&self) -> Option { + Some(match self { + Token::Add => BinOpType::Add, + Token::Sub => BinOpType::Sub, + + Token::Mul => BinOpType::Mul, + Token::Div => BinOpType::Div, + Token::Mod => BinOpType::Mod, + + Token::BAnd => BinOpType::BAnd, + Token::BOr => BinOpType::BOr, + Token::BXor => BinOpType::BXor, + + Token::Shl => BinOpType::Shl, + Token::Shr => BinOpType::Shr, + + Token::EquEqu => BinOpType::EquEqu, + Token::NotEqu => BinOpType::NotEqu, + + Token::LAngle => BinOpType::Less, + Token::LAngleEqu => BinOpType::LessEqu, + + Token::RAngle => BinOpType::Greater, + Token::RAngleEqu => BinOpType::GreaterEqu, + + Token::LArrow => BinOpType::Declare, + Token::Equ => BinOpType::Assign, + + _ => return None, + }) + } +}