#2 add function keys support to lexer

1 files changed, 9 insertions, 1 deletions

diff --git a/src/tokenize.c b/src/tokenize.c
index 53f294a..333ee67 100644
--- a/src/tokenize.c
+++ b/src/tokenize.c
@@ -58,10 +58,18 @@ eat_hex(struct tokenizer *tokenizer)
     }
 }
 
+// NOTE(koekeishiya): production rules:
+// identifier = <char><char>   | <char><number>
+// number     = <digit><digit> | <digit>
+// digit      = 0..9
 internal void
 eat_identifier(struct tokenizer *tokenizer)
 {
-    while(*tokenizer->at && isalpha(*tokenizer->at)) {
+    while((*tokenizer->at) && isalpha(*tokenizer->at)) {
+        advance(tokenizer);
+    }
+
+    while((*tokenizer->at) && isdigit(*tokenizer->at)) {
         advance(tokenizer);
     }
 }