feat: Add token definitions and lexer logic.

This commit adds the `Token` and `TokenKind` definitions in `src/token.rs`, in `src/lexer.rs` I've added the `Lexer` logic.
2026-03-11 23:18:05 +01:00
parent 9e4d4085ba
commit a1ca2f2467
7 changed files with 420 additions and 2 deletions
--- a/Cargo.toml
+++ b/Cargo.toml
@@ -4,4 +4,5 @@ version = "0.1.0"
 edition = "2024"

 [dependencies]
+unicode-xid = "0.2.6"
 yansi = "1.0.1"