From 198a92db125655d1632d1f264b70c68fbdfc5eb8 Mon Sep 17 00:00:00 2001
From: Sergei Parshukov <codedby@bugman.me>
Date: Tue, 28 Jan 2025 06:06:06 +0400
Subject: [PATCH] Fix comment lexer failing on EOF. Fix typos

---
 src/nibble.gleam           |  2 +-
 src/nibble/lexer.gleam     |  7 ++++---
 test/unit/lexer_test.gleam | 36 ++++++++++++++++++++++++++++++++++++
 3 files changed, 41 insertions(+), 4 deletions(-)

diff --git a/src/nibble.gleam b/src/nibble.gleam
index 720c964..1dc04eb 100644
--- a/src/nibble.gleam
+++ b/src/nibble.gleam
@@ -156,7 +156,7 @@ pub fn return(value: a) -> Parser(a, tok, ctx) {
 ///
 /// 💡 [`succeed`](#succeed) and [`return`](#return) are names for the same thing.
 /// We suggest using `succeed` in a pipeline with `nibble.then`, and `return`
-/// unqalified when using `do` with Gleam's `use` syntax.
+/// unqualified when using `do` with Gleam's `use` syntax.
 ///
 pub fn succeed(value: a) -> Parser(a, tok, ctx) {
   return(value)
diff --git a/src/nibble/lexer.gleam b/src/nibble/lexer.gleam
index 0886dde..f5b8235 100644
--- a/src/nibble/lexer.gleam
+++ b/src/nibble/lexer.gleam
@@ -65,7 +65,7 @@
 //// ```
 ////
 //// Here we have some whitespace that would be matched by `lexer.whitespace`, by
-//// we passed that matcher to the [`lexer.ignore`](#ignore) comabinator. The matcher
+//// we passed that matcher to the [`lexer.ignore`](#ignore) combinator. The matcher
 //// will still consume the input, but this time it will not produce a new token
 //// value:
 ////
@@ -249,7 +249,7 @@ pub fn map(matcher: Matcher(a, mode), f: fn(a) -> b) -> Matcher(b, mode) {
 }
 
 /// Take an existing matcher and transform it by applying a function to the value
-/// it producs. The function you provide can return a different [`Match`](#Match)
+/// it produces. The function you provide can return a different [`Match`](#Match)
 /// so you can, for example, take a matcher that `Keep`s a value and turn it into
 /// a matcher that `Drop`s the value instead. This is out [`ignore`](#ignore) works!
 ///
@@ -564,13 +564,14 @@ pub fn whitespace(token: a) -> Matcher(a, mode) {
   }
 }
 
+/// Match a line comment starting with the a given string. Keeps the leading whitespace.
 ///
 pub fn comment(start: String, to_value: fn(String) -> a) -> Matcher(a, mode) {
   let drop_length = string.length(start)
   use mode, lexeme, lookahead <- Matcher
 
   case string.starts_with(lexeme, start), lookahead {
-    True, "\n" ->
+    True, "\n" | True, "" ->
       lexeme
       |> string.drop_start(drop_length)
       |> to_value
diff --git a/test/unit/lexer_test.gleam b/test/unit/lexer_test.gleam
index 0f18552..52e60f7 100644
--- a/test/unit/lexer_test.gleam
+++ b/test/unit/lexer_test.gleam
@@ -14,6 +14,7 @@ type TokenT {
   Num(Float)
   Kwd(String)
   Var(String)
+  Comment(String)
 }
 
 // INTEGER TESTS ---------------------------------------------------------------
@@ -210,6 +211,41 @@ pub fn variable_containing_keyword_test() {
   ])
 }
 
+// COMMENT TESTS --------------------------------------------------------------
+
+pub fn linebreak_terminated_comment_test() {
+  use run <- should("lex a linebreak-terminated comment")
+
+  let input = "# this is a comment\n"
+  let expected = [
+    Token(
+      Span(1, 1, 1, 20),
+      "# this is a comment",
+      Comment(" this is a comment"),
+    ),
+  ]
+
+  run(input, expected, [
+    lexer.comment("#", Comment),
+    lexer.whitespace(Nil) |> lexer.ignore,
+  ])
+}
+
+pub fn bare_comment_test() {
+  use run <- should("lex a bare comment")
+
+  let input = "// this is a comment"
+  let expected = [
+    Token(
+      Span(1, 1, 1, 21),
+      "// this is a comment",
+      Comment(" this is a comment"),
+    ),
+  ]
+
+  run(input, expected, [lexer.comment("//", Comment)])
+}
+
 // UTILS -----------------------------------------------------------------------
 
 fn should(