|
|
@@ -39,12 +39,15 @@
|
|
|
|
|
|
(defun tokenize (input)
|
|
|
(remove :whitespace
|
|
|
- (parse-string*
|
|
|
- (many? (choices (<token>)
|
|
|
- (<punctuation>)
|
|
|
- (chook? :whitespace
|
|
|
- (choices (whitespace?) "---" (<sl-comment>) (<ml-comment>)))))
|
|
|
- input)))
|
|
|
+ (substitute #\_ "_"
|
|
|
+ (parse-string*
|
|
|
+ (many? (choices (<token>)
|
|
|
+ (<punctuation>)
|
|
|
+ (chook? :whitespace
|
|
|
+ (choices (whitespace?) "---"
|
|
|
+ (<sl-comment>) (<ml-comment>)))))
|
|
|
+ input)
|
|
|
+ :test #'equal)))
|
|
|
|
|
|
;; Actual parsers
|
|
|
(defun <hex-digit> ()
|
