feat: Add token definitions and lexer logic.

This commit adds the `Token` and `TokenKind` definitions in `src/token.rs`,
in `src/lexer.rs` I've added the `Lexer` logic.

src/main.rs

@@ -1,9 +1,32 @@
-use crate::cli::parse_args;
+use std::fs;
+
+use crate::{
+    cli::{fatal, parse_args},
+    lexer::Lexer,
+};
 
 mod cli;
+mod lexer;
+mod token;
 
 fn main() {
     let opts = parse_args();
 
-    println!("{opts:#?}");
+    for file in &opts.files {
+        let content = match fs::read_to_string(file) {
+            Ok(content) => content,
+            Err(error) => {
+                fatal(format!(
+                    "failed to read {}: {:?}",
+                    file.display(),
+                    error.kind()
+                ));
+            }
+        };
+
+        println!("-- {} --", file.display());
+        for token in Lexer::new(&content) {
+            println!("{}", token);
+        }
+    }
 }