init: Add Token definition and Tokenizer logic.

src/main.rs

@@ -0,0 +1,11 @@
+use crate::token::Tokenizer;
+
+pub mod token;
+
+fn main() {
+    let input = include_str!("../example/main.bky");
+
+    for token in Tokenizer::new(input) {
+        println!("{token:?}");
+    }
+}