feat: Add token definitions and lexer logic.

This commit adds the `Token` and `TokenKind` definitions in `src/token.rs`, in `src/lexer.rs` I've added the `Lexer` logic.
2026-03-11 23:18:05 +01:00
parent 9e4d4085ba
commit a1ca2f2467
7 changed files with 420 additions and 2 deletions
--- a/Cargo.lock
+++ b/Cargo.lock
@@ -6,9 +6,16 @@ version = 4
 name = "buckyc"
 version = "0.1.0"
 dependencies = [
+ "unicode-xid",
 "yansi",
 ]

+[[package]]
+name = "unicode-xid"
+version = "0.2.6"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "ebc1c04c71510c7f702b52b7c350734c9ff1295c464a03335b00bb84fc54f853"
+
 [[package]]
 name = "yansi"
 version = "1.0.1"