init: Add Token definition and Tokenizer logic.

2026-01-12 16:06:55 +01:00
commit 0599a5fb98
6 changed files with 378 additions and 0 deletions
--- a/Cargo.lock
+++ b/Cargo.lock
@@ -0,0 +1,23 @@
+# This file is automatically @generated by Cargo.
+# It is not intended for manual editing.
+version = 4
+
+[[package]]
+name = "bucky"
+version = "0.1.0"
+dependencies = [
+ "multipeek",
+ "unicode-xid",
+]
+
+[[package]]
+name = "multipeek"
+version = "0.1.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "1d6b1cf1c2ae7c8c3898cbf8354ee836bc7037e35592d3739a9901d53c97b6a2"
+
+[[package]]
+name = "unicode-xid"
+version = "0.2.6"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "ebc1c04c71510c7f702b52b7c350734c9ff1295c464a03335b00bb84fc54f853"