From 43b9ac85aece51c3bf920ed724cc419251f0f522 Mon Sep 17 00:00:00 2001 From: tufusa Date: Thu, 6 Aug 2026 21:54:18 +0900 Subject: [PATCH 1/4] Add lexer test on CRLF --- test/rbs/parser_test.rb | 30 ++++++++++++++++++++++++++++++ 1 file changed, 30 insertions(+) diff --git a/test/rbs/parser_test.rb b/test/rbs/parser_test.rb index f0e61e7e8..256e1e29f 100644 --- a/test/rbs/parser_test.rb +++ b/test/rbs/parser_test.rb @@ -1030,6 +1030,36 @@ class Foo[T < Integer] < Bar # Comment assert_equal [:pEOF, '', 57...57], tokens.shift.then { |t| [t[0], t[1].source, t[1].range] } end + def test__lex_crlf + content = "# LineComment\r\nclass Foo[T < Integer] < Bar # Comment\r\nend\r\n" + tokens = RBS::Parser._lex(buffer(content), content.length) + assert_equal [:tLINECOMMENT, '# LineComment', 0...13], tokens.shift.then { |t| [t[0], t[1].source, t[1].range] } + assert_equal [:tTRIVIA, "\r", 13...14], tokens.shift.then { |t| [t[0], t[1].source, t[1].range] } + assert_equal [:tTRIVIA, "\n", 14...15], tokens.shift.then { |t| [t[0], t[1].source, t[1].range] } + assert_equal [:kCLASS, 'class', 15...20], tokens.shift.then { |t| [t[0], t[1].source, t[1].range] } + assert_equal [:tTRIVIA, " ", 20...21], tokens.shift.then { |t| [t[0], t[1].source, t[1].range] } + assert_equal [:tUIDENT, 'Foo', 21...24], tokens.shift.then { |t| [t[0], t[1].source, t[1].range] } + assert_equal [:pLBRACKET, '[', 24...25], tokens.shift.then { |t| [t[0], t[1].source, t[1].range] } + assert_equal [:tUIDENT, 'T', 25...26], tokens.shift.then { |t| [t[0], t[1].source, t[1].range] } + assert_equal [:tTRIVIA, " ", 26...27], tokens.shift.then { |t| [t[0], t[1].source, t[1].range] } + assert_equal [:pLT, '<', 27...28], tokens.shift.then { |t| [t[0], t[1].source, t[1].range] } + assert_equal [:tTRIVIA, " ", 28...29], tokens.shift.then { |t| [t[0], t[1].source, t[1].range] } + assert_equal [:tUIDENT, 'Integer', 29...36], tokens.shift.then { |t| [t[0], t[1].source, t[1].range] } + assert_equal [:pRBRACKET, ']', 36...37], tokens.shift.then { |t| [t[0], t[1].source, t[1].range] } + assert_equal [:tTRIVIA, " ", 37...38], tokens.shift.then { |t| [t[0], t[1].source, t[1].range] } + assert_equal [:pLT, '<', 38...39], tokens.shift.then { |t| [t[0], t[1].source, t[1].range] } + assert_equal [:tTRIVIA, " ", 39...40], tokens.shift.then { |t| [t[0], t[1].source, t[1].range] } + assert_equal [:tUIDENT, 'Bar', 40...43], tokens.shift.then { |t| [t[0], t[1].source, t[1].range] } + assert_equal [:tTRIVIA, " ", 43...44], tokens.shift.then { |t| [t[0], t[1].source, t[1].range] } + assert_equal [:tCOMMENT, '# Comment', 44...53], tokens.shift.then { |t| [t[0], t[1].source, t[1].range] } + assert_equal [:tTRIVIA, "\r", 53...54], tokens.shift.then { |t| [t[0], t[1].source, t[1].range] } + assert_equal [:tTRIVIA, "\n", 54...55], tokens.shift.then { |t| [t[0], t[1].source, t[1].range] } + assert_equal [:kEND, 'end', 55...58], tokens.shift.then { |t| [t[0], t[1].source, t[1].range] } + assert_equal [:tTRIVIA, "\r", 58...59], tokens.shift.then { |t| [t[0], t[1].source, t[1].range] } + assert_equal [:tTRIVIA, "\n", 59...60], tokens.shift.then { |t| [t[0], t[1].source, t[1].range] } + assert_equal [:pEOF, '', 60...60], tokens.shift.then { |t| [t[0], t[1].source, t[1].range] } + end + def test_invalid_position_range_raises # Regression: start_pos > end_pos used to cause an infinite loop in the lexer. assert_raises(ArgumentError) do From e0a66c05e5f98bcddcff2e86c55a4fcc6221e2fe Mon Sep 17 00:00:00 2001 From: tufusa Date: Thu, 6 Aug 2026 21:54:22 +0900 Subject: [PATCH 2/4] Exclude CR from comment token pattern --- src/lexer.re | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/lexer.re b/src/lexer.re index 18980eacd..af0f50a2a 100644 --- a/src/lexer.re +++ b/src/lexer.re @@ -70,7 +70,7 @@ rbs_token_t rbs_lexer_next_token(rbs_lexer_t *lexer) { "%a|" [^|\x00]* "|" { return rbs_next_token(lexer, tANNOTATION); } "%a<" [^>\x00]* ">" { return rbs_next_token(lexer, tANNOTATION); } - "#" (. \ [\x00\uFFFD])* { + "#" (. \ [\x00\r\uFFFD])* { return rbs_next_token( lexer, lexer->first_token_of_line ? tLINECOMMENT : tCOMMENT From 7a435898894acfe0d5ef9c6138c6d27a5d9263c4 Mon Sep 17 00:00:00 2001 From: tufusa Date: Thu, 6 Aug 2026 21:54:30 +0900 Subject: [PATCH 3/4] Update lexer.c --- src/lexer.c | 183 ++++++++++++++++++++++++++-------------------------- 1 file changed, 92 insertions(+), 91 deletions(-) diff --git a/src/lexer.c b/src/lexer.c index 144cc071d..54ff6d3f5 100644 --- a/src/lexer.c +++ b/src/lexer.c @@ -177,10 +177,11 @@ rbs_token_t rbs_lexer_next_token(rbs_lexer_t *lexer) { yy10: rbs_skip(lexer); yych = rbs_peek(lexer); - if (yych <= '\n') { + if (yych <= '\f') { if (yych <= 0x00000000) goto yy11; - if (yych <= '\t') goto yy10; + if (yych != '\n') goto yy10; } else { + if (yych <= '\r') goto yy11; if (yych != 0x0000FFFD) goto yy10; } yy11: @@ -191,7 +192,7 @@ rbs_token_t rbs_lexer_next_token(rbs_lexer_t *lexer) { lexer->first_token_of_line ? tLINECOMMENT : tCOMMENT ); } -#line 188 "src/lexer.c" +#line 189 "src/lexer.c" yy12: rbs_skip(lexer); yych = rbs_peek(lexer); @@ -251,7 +252,7 @@ rbs_token_t rbs_lexer_next_token(rbs_lexer_t *lexer) { { return rbs_next_token(lexer, pAMP); } -#line 246 "src/lexer.c" +#line 247 "src/lexer.c" yy15: yyaccept = 0; rbs_skip(lexer); @@ -265,14 +266,14 @@ rbs_token_t rbs_lexer_next_token(rbs_lexer_t *lexer) { { return rbs_next_token(lexer, pLPAREN); } -#line 258 "src/lexer.c" +#line 259 "src/lexer.c" yy17: rbs_skip(lexer); #line 37 "src/lexer.re" { return rbs_next_token(lexer, pRPAREN); } -#line 263 "src/lexer.c" +#line 264 "src/lexer.c" yy18: rbs_skip(lexer); yych = rbs_peek(lexer); @@ -281,7 +282,7 @@ rbs_token_t rbs_lexer_next_token(rbs_lexer_t *lexer) { { return rbs_next_token(lexer, pSTAR); } -#line 270 "src/lexer.c" +#line 271 "src/lexer.c" yy19: rbs_skip(lexer); yych = rbs_peek(lexer); @@ -295,7 +296,7 @@ rbs_token_t rbs_lexer_next_token(rbs_lexer_t *lexer) { { return rbs_next_token(lexer, pCOMMA); } -#line 282 "src/lexer.c" +#line 283 "src/lexer.c" yy21: rbs_skip(lexer); yych = rbs_peek(lexer); @@ -331,7 +332,7 @@ rbs_token_t rbs_lexer_next_token(rbs_lexer_t *lexer) { { return rbs_next_token(lexer, pDOT); } -#line 311 "src/lexer.c" +#line 312 "src/lexer.c" yy24: rbs_skip(lexer); goto yy8; @@ -346,7 +347,7 @@ rbs_token_t rbs_lexer_next_token(rbs_lexer_t *lexer) { { return rbs_next_token(lexer, tINTEGER); } -#line 324 "src/lexer.c" +#line 325 "src/lexer.c" yy27: yyaccept = 3; rbs_skip(lexer); @@ -418,7 +419,7 @@ rbs_token_t rbs_lexer_next_token(rbs_lexer_t *lexer) { { return rbs_next_token(lexer, pCOLON); } -#line 394 "src/lexer.c" +#line 395 "src/lexer.c" yy29: rbs_skip(lexer); yych = rbs_peek(lexer); @@ -430,7 +431,7 @@ rbs_token_t rbs_lexer_next_token(rbs_lexer_t *lexer) { { return rbs_next_token(lexer, pLT); } -#line 404 "src/lexer.c" +#line 405 "src/lexer.c" yy31: rbs_skip(lexer); yych = rbs_peek(lexer); @@ -446,7 +447,7 @@ rbs_token_t rbs_lexer_next_token(rbs_lexer_t *lexer) { { return rbs_next_token(lexer, pEQ); } -#line 418 "src/lexer.c" +#line 419 "src/lexer.c" yy33: rbs_skip(lexer); yych = rbs_peek(lexer); @@ -457,14 +458,14 @@ rbs_token_t rbs_lexer_next_token(rbs_lexer_t *lexer) { { return rbs_next_token(lexer, pGT); } -#line 427 "src/lexer.c" +#line 428 "src/lexer.c" yy35: rbs_skip(lexer); #line 46 "src/lexer.re" { return rbs_next_token(lexer, pQUESTION); } -#line 432 "src/lexer.c" +#line 433 "src/lexer.c" yy36: yyaccept = 0; rbs_skip(lexer); @@ -514,7 +515,7 @@ rbs_token_t rbs_lexer_next_token(rbs_lexer_t *lexer) { { return rbs_next_token(lexer, tUIDENT); } -#line 480 "src/lexer.c" +#line 481 "src/lexer.c" yy39: rbs_skip(lexer); yych = rbs_peek(lexer); @@ -523,21 +524,21 @@ rbs_token_t rbs_lexer_next_token(rbs_lexer_t *lexer) { { return rbs_next_token(lexer, pLBRACKET); } -#line 487 "src/lexer.c" +#line 488 "src/lexer.c" yy40: rbs_skip(lexer); #line 39 "src/lexer.re" { return rbs_next_token(lexer, pRBRACKET); } -#line 492 "src/lexer.c" +#line 493 "src/lexer.c" yy41: rbs_skip(lexer); #line 44 "src/lexer.re" { return rbs_next_token(lexer, pHAT); } -#line 497 "src/lexer.c" +#line 498 "src/lexer.c" yy42: rbs_skip(lexer); yych = rbs_peek(lexer); @@ -563,7 +564,7 @@ rbs_token_t rbs_lexer_next_token(rbs_lexer_t *lexer) { { return rbs_next_token(lexer, tULLIDENT); } -#line 521 "src/lexer.c" +#line 522 "src/lexer.c" yy44: yyaccept = 4; rbs_skip(lexer); @@ -580,7 +581,7 @@ rbs_token_t rbs_lexer_next_token(rbs_lexer_t *lexer) { { return rbs_next_token(lexer, tOPERATOR); } -#line 536 "src/lexer.c" +#line 537 "src/lexer.c" yy46: rbs_skip(lexer); yych = rbs_peek(lexer); @@ -597,7 +598,7 @@ rbs_token_t rbs_lexer_next_token(rbs_lexer_t *lexer) { { return rbs_next_token(lexer, tLIDENT); } -#line 551 "src/lexer.c" +#line 552 "src/lexer.c" yy48: rbs_skip(lexer); yych = rbs_peek(lexer); @@ -719,21 +720,21 @@ rbs_token_t rbs_lexer_next_token(rbs_lexer_t *lexer) { { return rbs_next_token(lexer, pLBRACE); } -#line 671 "src/lexer.c" +#line 672 "src/lexer.c" yy66: rbs_skip(lexer); #line 43 "src/lexer.re" { return rbs_next_token(lexer, pBAR); } -#line 676 "src/lexer.c" +#line 677 "src/lexer.c" yy67: rbs_skip(lexer); #line 41 "src/lexer.re" { return rbs_next_token(lexer, pRBRACE); } -#line 681 "src/lexer.c" +#line 682 "src/lexer.c" yy68: rbs_skip(lexer); yych = rbs_peek(lexer); @@ -759,7 +760,7 @@ rbs_token_t rbs_lexer_next_token(rbs_lexer_t *lexer) { { return rbs_next_token(lexer, tNONASCIIIDENT); } -#line 705 "src/lexer.c" +#line 706 "src/lexer.c" yy70: rbs_skip(lexer); yych = rbs_peek(lexer); @@ -805,7 +806,7 @@ rbs_token_t rbs_lexer_next_token(rbs_lexer_t *lexer) { { return rbs_next_token(lexer, tDQSTRING); } -#line 745 "src/lexer.c" +#line 746 "src/lexer.c" yy74: rbs_skip(lexer); yych = rbs_peek(lexer); @@ -846,7 +847,7 @@ rbs_token_t rbs_lexer_next_token(rbs_lexer_t *lexer) { { return rbs_next_token(lexer, tGIDENT); } -#line 784 "src/lexer.c" +#line 785 "src/lexer.c" yy77: rbs_skip(lexer); goto yy76; @@ -889,7 +890,7 @@ rbs_token_t rbs_lexer_next_token(rbs_lexer_t *lexer) { { return rbs_next_token(lexer, tSQSTRING); } -#line 825 "src/lexer.c" +#line 826 "src/lexer.c" yy83: rbs_skip(lexer); yych = rbs_peek(lexer); @@ -907,7 +908,7 @@ rbs_token_t rbs_lexer_next_token(rbs_lexer_t *lexer) { { return rbs_next_token(lexer, pSTAR2); } -#line 841 "src/lexer.c" +#line 842 "src/lexer.c" yy85: rbs_skip(lexer); yych = rbs_peek(lexer); @@ -916,14 +917,14 @@ rbs_token_t rbs_lexer_next_token(rbs_lexer_t *lexer) { { return rbs_next_token(lexer, tINLINECOMMENT); } -#line 848 "src/lexer.c" +#line 849 "src/lexer.c" yy86: rbs_skip(lexer); #line 53 "src/lexer.re" { return rbs_next_token(lexer, pARROW); } -#line 853 "src/lexer.c" +#line 854 "src/lexer.c" yy87: rbs_skip(lexer); yych = rbs_peek(lexer); @@ -939,7 +940,7 @@ rbs_token_t rbs_lexer_next_token(rbs_lexer_t *lexer) { { return rbs_next_token(lexer, tSYMBOL); } -#line 867 "src/lexer.c" +#line 868 "src/lexer.c" yy90: rbs_skip(lexer); yych = rbs_peek(lexer); @@ -1027,7 +1028,7 @@ rbs_token_t rbs_lexer_next_token(rbs_lexer_t *lexer) { { return rbs_next_token(lexer, pCOLON2); } -#line 953 "src/lexer.c" +#line 954 "src/lexer.c" yy97: rbs_skip(lexer); yych = rbs_peek(lexer); @@ -1097,7 +1098,7 @@ rbs_token_t rbs_lexer_next_token(rbs_lexer_t *lexer) { { return rbs_next_token(lexer, tSYMBOL); } -#line 1021 "src/lexer.c" +#line 1022 "src/lexer.c" yy103: rbs_skip(lexer); yych = rbs_peek(lexer); @@ -1119,7 +1120,7 @@ rbs_token_t rbs_lexer_next_token(rbs_lexer_t *lexer) { { return rbs_next_token(lexer, pFATARROW); } -#line 1041 "src/lexer.c" +#line 1042 "src/lexer.c" yy107: rbs_skip(lexer); yych = rbs_peek(lexer); @@ -1165,7 +1166,7 @@ rbs_token_t rbs_lexer_next_token(rbs_lexer_t *lexer) { { return rbs_next_token(lexer, tAIDENT); } -#line 1085 "src/lexer.c" +#line 1086 "src/lexer.c" yy111: rbs_skip(lexer); yych = rbs_peek(lexer); @@ -1177,14 +1178,14 @@ rbs_token_t rbs_lexer_next_token(rbs_lexer_t *lexer) { { return rbs_next_token(lexer, tBANGIDENT); } -#line 1095 "src/lexer.c" +#line 1096 "src/lexer.c" yy113: rbs_skip(lexer); #line 165 "src/lexer.re" { return rbs_next_token(lexer, tEQIDENT); } -#line 1100 "src/lexer.c" +#line 1101 "src/lexer.c" yy114: rbs_skip(lexer); yych = rbs_peek(lexer); @@ -1193,7 +1194,7 @@ rbs_token_t rbs_lexer_next_token(rbs_lexer_t *lexer) { { return rbs_next_token(lexer, pAREF_OPR); } -#line 1107 "src/lexer.c" +#line 1108 "src/lexer.c" yy115: rbs_skip(lexer); yych = rbs_peek(lexer); @@ -1220,7 +1221,7 @@ rbs_token_t rbs_lexer_next_token(rbs_lexer_t *lexer) { { return rbs_next_token(lexer, tULLIDENT); } -#line 1132 "src/lexer.c" +#line 1133 "src/lexer.c" yy118: rbs_skip(lexer); yych = rbs_peek(lexer); @@ -1246,7 +1247,7 @@ rbs_token_t rbs_lexer_next_token(rbs_lexer_t *lexer) { { return rbs_next_token(lexer, tULIDENT); } -#line 1156 "src/lexer.c" +#line 1157 "src/lexer.c" yy120: rbs_skip(lexer); yych = rbs_peek(lexer); @@ -1288,7 +1289,7 @@ rbs_token_t rbs_lexer_next_token(rbs_lexer_t *lexer) { { return rbs_next_token(lexer, kAS); } -#line 1196 "src/lexer.c" +#line 1197 "src/lexer.c" yy125: rbs_skip(lexer); yych = rbs_peek(lexer); @@ -1367,7 +1368,7 @@ rbs_token_t rbs_lexer_next_token(rbs_lexer_t *lexer) { { return rbs_next_token(lexer, kIN); } -#line 1273 "src/lexer.c" +#line 1274 "src/lexer.c" yy134: rbs_skip(lexer); yych = rbs_peek(lexer); @@ -1515,14 +1516,14 @@ rbs_token_t rbs_lexer_next_token(rbs_lexer_t *lexer) { { return rbs_next_token(lexer, pDOT3); } -#line 1419 "src/lexer.c" +#line 1420 "src/lexer.c" yy158: rbs_skip(lexer); #line 128 "src/lexer.re" { return rbs_next_token(lexer, tDQSYMBOL); } -#line 1424 "src/lexer.c" +#line 1425 "src/lexer.c" yy159: rbs_skip(lexer); yych = rbs_peek(lexer); @@ -1563,7 +1564,7 @@ rbs_token_t rbs_lexer_next_token(rbs_lexer_t *lexer) { { return rbs_next_token(lexer, tSYMBOL); } -#line 1463 "src/lexer.c" +#line 1464 "src/lexer.c" yy162: rbs_skip(lexer); goto yy161; @@ -1574,7 +1575,7 @@ rbs_token_t rbs_lexer_next_token(rbs_lexer_t *lexer) { { return rbs_next_token(lexer, tSQSYMBOL); } -#line 1472 "src/lexer.c" +#line 1473 "src/lexer.c" yy165: rbs_skip(lexer); yych = rbs_peek(lexer); @@ -1643,7 +1644,7 @@ rbs_token_t rbs_lexer_next_token(rbs_lexer_t *lexer) { { return rbs_next_token(lexer, tA2IDENT); } -#line 1539 "src/lexer.c" +#line 1540 "src/lexer.c" yy172: rbs_skip(lexer); yych = rbs_peek(lexer); @@ -1660,7 +1661,7 @@ rbs_token_t rbs_lexer_next_token(rbs_lexer_t *lexer) { { return rbs_next_token(lexer, tQIDENT); } -#line 1554 "src/lexer.c" +#line 1555 "src/lexer.c" yy175: rbs_skip(lexer); yych = rbs_peek(lexer); @@ -1701,7 +1702,7 @@ rbs_token_t rbs_lexer_next_token(rbs_lexer_t *lexer) { { return rbs_next_token(lexer, kBOT); } -#line 1593 "src/lexer.c" +#line 1594 "src/lexer.c" yy180: rbs_skip(lexer); yych = rbs_peek(lexer); @@ -1732,7 +1733,7 @@ rbs_token_t rbs_lexer_next_token(rbs_lexer_t *lexer) { { return rbs_next_token(lexer, kDEF); } -#line 1622 "src/lexer.c" +#line 1623 "src/lexer.c" yy183: rbs_skip(lexer); yych = rbs_peek(lexer); @@ -1758,7 +1759,7 @@ rbs_token_t rbs_lexer_next_token(rbs_lexer_t *lexer) { { return rbs_next_token(lexer, kEND); } -#line 1646 "src/lexer.c" +#line 1647 "src/lexer.c" yy185: rbs_skip(lexer); yych = rbs_peek(lexer); @@ -1814,7 +1815,7 @@ rbs_token_t rbs_lexer_next_token(rbs_lexer_t *lexer) { { return rbs_next_token(lexer, kNIL); } -#line 1700 "src/lexer.c" +#line 1701 "src/lexer.c" yy193: rbs_skip(lexer); yych = rbs_peek(lexer); @@ -1840,7 +1841,7 @@ rbs_token_t rbs_lexer_next_token(rbs_lexer_t *lexer) { { return rbs_next_token(lexer, kOUT); } -#line 1724 "src/lexer.c" +#line 1725 "src/lexer.c" yy195: rbs_skip(lexer); yych = rbs_peek(lexer); @@ -1901,7 +1902,7 @@ rbs_token_t rbs_lexer_next_token(rbs_lexer_t *lexer) { { return rbs_next_token(lexer, kTOP); } -#line 1783 "src/lexer.c" +#line 1784 "src/lexer.c" yy204: rbs_skip(lexer); yych = rbs_peek(lexer); @@ -1947,7 +1948,7 @@ rbs_token_t rbs_lexer_next_token(rbs_lexer_t *lexer) { { return rbs_next_token(lexer, kUSE); } -#line 1827 "src/lexer.c" +#line 1828 "src/lexer.c" yy210: rbs_skip(lexer); yych = rbs_peek(lexer); @@ -1972,35 +1973,35 @@ rbs_token_t rbs_lexer_next_token(rbs_lexer_t *lexer) { { return rbs_next_token(lexer, tANNOTATION); } -#line 1850 "src/lexer.c" +#line 1851 "src/lexer.c" yy213: rbs_skip(lexer); #line 71 "src/lexer.re" { return rbs_next_token(lexer, tANNOTATION); } -#line 1855 "src/lexer.c" +#line 1856 "src/lexer.c" yy214: rbs_skip(lexer); #line 69 "src/lexer.re" { return rbs_next_token(lexer, tANNOTATION); } -#line 1860 "src/lexer.c" +#line 1861 "src/lexer.c" yy215: rbs_skip(lexer); #line 67 "src/lexer.re" { return rbs_next_token(lexer, tANNOTATION); } -#line 1865 "src/lexer.c" +#line 1866 "src/lexer.c" yy216: rbs_skip(lexer); #line 70 "src/lexer.re" { return rbs_next_token(lexer, tANNOTATION); } -#line 1870 "src/lexer.c" +#line 1871 "src/lexer.c" yy217: rbs_skip(lexer); yych = rbs_peek(lexer); @@ -2059,7 +2060,7 @@ rbs_token_t rbs_lexer_next_token(rbs_lexer_t *lexer) { { return rbs_next_token(lexer, kATRBS); } -#line 1927 "src/lexer.c" +#line 1928 "src/lexer.c" yy222: rbs_skip(lexer); yych = rbs_peek(lexer); @@ -2100,7 +2101,7 @@ rbs_token_t rbs_lexer_next_token(rbs_lexer_t *lexer) { { return rbs_next_token(lexer, kBOOL); } -#line 1966 "src/lexer.c" +#line 1967 "src/lexer.c" yy227: rbs_skip(lexer); yych = rbs_peek(lexer); @@ -2181,7 +2182,7 @@ rbs_token_t rbs_lexer_next_token(rbs_lexer_t *lexer) { { return rbs_next_token(lexer, kSELF); } -#line 2045 "src/lexer.c" +#line 2046 "src/lexer.c" yy240: rbs_skip(lexer); yych = rbs_peek(lexer); @@ -2212,7 +2213,7 @@ rbs_token_t rbs_lexer_next_token(rbs_lexer_t *lexer) { { return rbs_next_token(lexer, kSKIP); } -#line 2074 "src/lexer.c" +#line 2075 "src/lexer.c" yy243: rbs_skip(lexer); yych = rbs_peek(lexer); @@ -2238,7 +2239,7 @@ rbs_token_t rbs_lexer_next_token(rbs_lexer_t *lexer) { { return rbs_next_token(lexer, kTRUE); } -#line 2098 "src/lexer.c" +#line 2099 "src/lexer.c" yy245: rbs_skip(lexer); yych = rbs_peek(lexer); @@ -2264,7 +2265,7 @@ rbs_token_t rbs_lexer_next_token(rbs_lexer_t *lexer) { { return rbs_next_token(lexer, kTYPE); } -#line 2122 "src/lexer.c" +#line 2123 "src/lexer.c" yy247: rbs_skip(lexer); yych = rbs_peek(lexer); @@ -2300,7 +2301,7 @@ rbs_token_t rbs_lexer_next_token(rbs_lexer_t *lexer) { { return rbs_next_token(lexer, kVOID); } -#line 2156 "src/lexer.c" +#line 2157 "src/lexer.c" yy251: rbs_skip(lexer); yych = rbs_peek(lexer); @@ -2357,7 +2358,7 @@ rbs_token_t rbs_lexer_next_token(rbs_lexer_t *lexer) { { return rbs_next_token(lexer, kALIAS); } -#line 2211 "src/lexer.c" +#line 2212 "src/lexer.c" yy256: rbs_skip(lexer); yych = rbs_peek(lexer); @@ -2402,7 +2403,7 @@ rbs_token_t rbs_lexer_next_token(rbs_lexer_t *lexer) { { return rbs_next_token(lexer, kCLASS); } -#line 2254 "src/lexer.c" +#line 2255 "src/lexer.c" yy259: rbs_skip(lexer); yych = rbs_peek(lexer); @@ -2433,7 +2434,7 @@ rbs_token_t rbs_lexer_next_token(rbs_lexer_t *lexer) { { return rbs_next_token(lexer, kFALSE); } -#line 2283 "src/lexer.c" +#line 2284 "src/lexer.c" yy262: rbs_skip(lexer); yych = rbs_peek(lexer); @@ -2565,7 +2566,7 @@ rbs_token_t rbs_lexer_next_token(rbs_lexer_t *lexer) { { return rbs_next_token(lexer, kEXTEND); } -#line 2413 "src/lexer.c" +#line 2414 "src/lexer.c" yy282: rbs_skip(lexer); yych = rbs_peek(lexer); @@ -2614,7 +2615,7 @@ rbs_token_t rbs_lexer_next_token(rbs_lexer_t *lexer) { { return rbs_next_token(lexer, kMODULE); } -#line 2460 "src/lexer.c" +#line 2461 "src/lexer.c" yy287: rbs_skip(lexer); yych = rbs_peek(lexer); @@ -2650,7 +2651,7 @@ rbs_token_t rbs_lexer_next_token(rbs_lexer_t *lexer) { { return rbs_next_token(lexer, kPUBLIC); } -#line 2494 "src/lexer.c" +#line 2495 "src/lexer.c" yy291: rbs_skip(lexer); yych = rbs_peek(lexer); @@ -2676,7 +2677,7 @@ rbs_token_t rbs_lexer_next_token(rbs_lexer_t *lexer) { { return rbs_next_token(lexer, kRETURN); } -#line 2518 "src/lexer.c" +#line 2519 "src/lexer.c" yy293: rbs_skip(lexer); yych = rbs_peek(lexer); @@ -2755,7 +2756,7 @@ rbs_token_t rbs_lexer_next_token(rbs_lexer_t *lexer) { { return rbs_next_token(lexer, kINCLUDE); } -#line 2595 "src/lexer.c" +#line 2596 "src/lexer.c" yy304: rbs_skip(lexer); yych = rbs_peek(lexer); @@ -2797,7 +2798,7 @@ rbs_token_t rbs_lexer_next_token(rbs_lexer_t *lexer) { { return rbs_next_token(lexer, kPREPEND); } -#line 2635 "src/lexer.c" +#line 2636 "src/lexer.c" yy309: rbs_skip(lexer); yych = rbs_peek(lexer); @@ -2823,7 +2824,7 @@ rbs_token_t rbs_lexer_next_token(rbs_lexer_t *lexer) { { return rbs_next_token(lexer, kPRIVATE); } -#line 2659 "src/lexer.c" +#line 2660 "src/lexer.c" yy311: rbs_skip(lexer); yych = rbs_peek(lexer); @@ -2859,7 +2860,7 @@ rbs_token_t rbs_lexer_next_token(rbs_lexer_t *lexer) { { return rbs_next_token(lexer, kUNTYPED); } -#line 2693 "src/lexer.c" +#line 2694 "src/lexer.c" yy315: rbs_skip(lexer); yych = rbs_peek(lexer); @@ -2885,7 +2886,7 @@ rbs_token_t rbs_lexer_next_token(rbs_lexer_t *lexer) { { return rbs_next_token(lexer, k__TODO__); } -#line 2717 "src/lexer.c" +#line 2718 "src/lexer.c" yy317: rbs_skip(lexer); yych = rbs_peek(lexer); @@ -2931,7 +2932,7 @@ rbs_token_t rbs_lexer_next_token(rbs_lexer_t *lexer) { { return rbs_next_token(lexer, kINSTANCE); } -#line 2761 "src/lexer.c" +#line 2762 "src/lexer.c" yy323: rbs_skip(lexer); yych = rbs_peek(lexer); @@ -3002,7 +3003,7 @@ rbs_token_t rbs_lexer_next_token(rbs_lexer_t *lexer) { { return rbs_next_token(lexer, kINTERFACE); } -#line 2830 "src/lexer.c" +#line 2831 "src/lexer.c" yy334: rbs_skip(lexer); yych = rbs_peek(lexer); @@ -3038,7 +3039,7 @@ rbs_token_t rbs_lexer_next_token(rbs_lexer_t *lexer) { { return rbs_next_token(lexer, kSINGLETON); } -#line 2864 "src/lexer.c" +#line 2865 "src/lexer.c" yy338: rbs_skip(lexer); yych = rbs_peek(lexer); @@ -3064,7 +3065,7 @@ rbs_token_t rbs_lexer_next_token(rbs_lexer_t *lexer) { { return rbs_next_token(lexer, kUNCHECKED); } -#line 2888 "src/lexer.c" +#line 2889 "src/lexer.c" yy340: rbs_skip(lexer); yych = rbs_peek(lexer); @@ -3125,7 +3126,7 @@ rbs_token_t rbs_lexer_next_token(rbs_lexer_t *lexer) { { return rbs_next_token(lexer, kATTRREADER); } -#line 2947 "src/lexer.c" +#line 2948 "src/lexer.c" yy349: rbs_skip(lexer); yych = rbs_peek(lexer); @@ -3151,14 +3152,14 @@ rbs_token_t rbs_lexer_next_token(rbs_lexer_t *lexer) { { return rbs_next_token(lexer, kATTRWRITER); } -#line 2971 "src/lexer.c" +#line 2972 "src/lexer.c" yy351: rbs_skip(lexer); #line 87 "src/lexer.re" { return rbs_next_token(lexer, kCLASSALIAS); } -#line 2976 "src/lexer.c" +#line 2977 "src/lexer.c" yy352: rbs_skip(lexer); yych = rbs_peek(lexer); @@ -3170,7 +3171,7 @@ rbs_token_t rbs_lexer_next_token(rbs_lexer_t *lexer) { { return rbs_next_token(lexer, kMODULESELF); } -#line 2986 "src/lexer.c" +#line 2987 "src/lexer.c" yy354: rbs_skip(lexer); yych = rbs_peek(lexer); @@ -3182,7 +3183,7 @@ rbs_token_t rbs_lexer_next_token(rbs_lexer_t *lexer) { { return rbs_next_token(lexer, kMODULEALIAS); } -#line 2996 "src/lexer.c" +#line 2997 "src/lexer.c" yy356: rbs_skip(lexer); yych = rbs_peek(lexer); @@ -3208,7 +3209,7 @@ rbs_token_t rbs_lexer_next_token(rbs_lexer_t *lexer) { { return rbs_next_token(lexer, kATTRACCESSOR); } -#line 3020 "src/lexer.c" +#line 3021 "src/lexer.c" } #line 177 "src/lexer.re" } From ea65b1cbf74a72b13cf2b867a117b2efb6e16e6b Mon Sep 17 00:00:00 2001 From: Soutaro Matsumoto Date: Thu, 13 Aug 2026 16:54:08 +0900 Subject: [PATCH 4/4] Handle CRLF comments without changing bare CR behavior --- src/lexer.c | 312 ++++++++++++++++++++-------------------- src/lexer.re | 11 +- test/rbs/parser_test.rb | 12 ++ 3 files changed, 182 insertions(+), 153 deletions(-) diff --git a/src/lexer.c b/src/lexer.c index 54ff6d3f5..886449a4d 100644 --- a/src/lexer.c +++ b/src/lexer.c @@ -129,7 +129,7 @@ rbs_token_t rbs_lexer_next_token(rbs_lexer_t *lexer) { } yy1: rbs_skip(lexer); -#line 175 "src/lexer.re" +#line 184 "src/lexer.re" { return rbs_next_eof_token(lexer); } @@ -137,7 +137,7 @@ rbs_token_t rbs_lexer_next_token(rbs_lexer_t *lexer) { yy2: rbs_skip(lexer); yy3: -#line 176 "src/lexer.re" +#line 185 "src/lexer.re" { return rbs_next_token(lexer, ErrorToken); } @@ -148,7 +148,7 @@ rbs_token_t rbs_lexer_next_token(rbs_lexer_t *lexer) { if (yych == '\t') goto yy4; if (yych == ' ') goto yy4; yy5: -#line 174 "src/lexer.re" +#line 183 "src/lexer.re" { return rbs_next_token(lexer, tTRIVIA); } @@ -177,22 +177,30 @@ rbs_token_t rbs_lexer_next_token(rbs_lexer_t *lexer) { yy10: rbs_skip(lexer); yych = rbs_peek(lexer); - if (yych <= '\f') { + if (yych <= '\n') { if (yych <= 0x00000000) goto yy11; - if (yych != '\n') goto yy10; + if (yych <= '\t') goto yy10; } else { - if (yych <= '\r') goto yy11; if (yych != 0x0000FFFD) goto yy10; } yy11: #line 73 "src/lexer.re" { + // Keep a bare CR in the comment, but leave the CR in CRLF for a trivia token. + if (rbs_peek(lexer) == '\n' && lexer->string.start[lexer->current.byte_pos - 1] == '\r') { + lexer->current.byte_pos -= 1; + lexer->current.char_pos -= 1; + lexer->current.column -= 1; + lexer->current_code_point = '\r'; + lexer->current_character_bytes = 1; + } + return rbs_next_token( lexer, lexer->first_token_of_line ? tLINECOMMENT : tCOMMENT ); } -#line 189 "src/lexer.c" +#line 197 "src/lexer.c" yy12: rbs_skip(lexer); yych = rbs_peek(lexer); @@ -252,7 +260,7 @@ rbs_token_t rbs_lexer_next_token(rbs_lexer_t *lexer) { { return rbs_next_token(lexer, pAMP); } -#line 247 "src/lexer.c" +#line 255 "src/lexer.c" yy15: yyaccept = 0; rbs_skip(lexer); @@ -266,14 +274,14 @@ rbs_token_t rbs_lexer_next_token(rbs_lexer_t *lexer) { { return rbs_next_token(lexer, pLPAREN); } -#line 259 "src/lexer.c" +#line 267 "src/lexer.c" yy17: rbs_skip(lexer); #line 37 "src/lexer.re" { return rbs_next_token(lexer, pRPAREN); } -#line 264 "src/lexer.c" +#line 272 "src/lexer.c" yy18: rbs_skip(lexer); yych = rbs_peek(lexer); @@ -282,7 +290,7 @@ rbs_token_t rbs_lexer_next_token(rbs_lexer_t *lexer) { { return rbs_next_token(lexer, pSTAR); } -#line 271 "src/lexer.c" +#line 279 "src/lexer.c" yy19: rbs_skip(lexer); yych = rbs_peek(lexer); @@ -296,7 +304,7 @@ rbs_token_t rbs_lexer_next_token(rbs_lexer_t *lexer) { { return rbs_next_token(lexer, pCOMMA); } -#line 283 "src/lexer.c" +#line 291 "src/lexer.c" yy21: rbs_skip(lexer); yych = rbs_peek(lexer); @@ -332,7 +340,7 @@ rbs_token_t rbs_lexer_next_token(rbs_lexer_t *lexer) { { return rbs_next_token(lexer, pDOT); } -#line 312 "src/lexer.c" +#line 320 "src/lexer.c" yy24: rbs_skip(lexer); goto yy8; @@ -347,7 +355,7 @@ rbs_token_t rbs_lexer_next_token(rbs_lexer_t *lexer) { { return rbs_next_token(lexer, tINTEGER); } -#line 325 "src/lexer.c" +#line 333 "src/lexer.c" yy27: yyaccept = 3; rbs_skip(lexer); @@ -419,7 +427,7 @@ rbs_token_t rbs_lexer_next_token(rbs_lexer_t *lexer) { { return rbs_next_token(lexer, pCOLON); } -#line 395 "src/lexer.c" +#line 403 "src/lexer.c" yy29: rbs_skip(lexer); yych = rbs_peek(lexer); @@ -431,7 +439,7 @@ rbs_token_t rbs_lexer_next_token(rbs_lexer_t *lexer) { { return rbs_next_token(lexer, pLT); } -#line 405 "src/lexer.c" +#line 413 "src/lexer.c" yy31: rbs_skip(lexer); yych = rbs_peek(lexer); @@ -447,7 +455,7 @@ rbs_token_t rbs_lexer_next_token(rbs_lexer_t *lexer) { { return rbs_next_token(lexer, pEQ); } -#line 419 "src/lexer.c" +#line 427 "src/lexer.c" yy33: rbs_skip(lexer); yych = rbs_peek(lexer); @@ -458,14 +466,14 @@ rbs_token_t rbs_lexer_next_token(rbs_lexer_t *lexer) { { return rbs_next_token(lexer, pGT); } -#line 428 "src/lexer.c" +#line 436 "src/lexer.c" yy35: rbs_skip(lexer); #line 46 "src/lexer.re" { return rbs_next_token(lexer, pQUESTION); } -#line 433 "src/lexer.c" +#line 441 "src/lexer.c" yy36: yyaccept = 0; rbs_skip(lexer); @@ -511,11 +519,11 @@ rbs_token_t rbs_lexer_next_token(rbs_lexer_t *lexer) { } } yy38: -#line 146 "src/lexer.re" +#line 155 "src/lexer.re" { return rbs_next_token(lexer, tUIDENT); } -#line 481 "src/lexer.c" +#line 489 "src/lexer.c" yy39: rbs_skip(lexer); yych = rbs_peek(lexer); @@ -524,21 +532,21 @@ rbs_token_t rbs_lexer_next_token(rbs_lexer_t *lexer) { { return rbs_next_token(lexer, pLBRACKET); } -#line 488 "src/lexer.c" +#line 496 "src/lexer.c" yy40: rbs_skip(lexer); #line 39 "src/lexer.re" { return rbs_next_token(lexer, pRBRACKET); } -#line 493 "src/lexer.c" +#line 501 "src/lexer.c" yy41: rbs_skip(lexer); #line 44 "src/lexer.re" { return rbs_next_token(lexer, pHAT); } -#line 498 "src/lexer.c" +#line 506 "src/lexer.c" yy42: rbs_skip(lexer); yych = rbs_peek(lexer); @@ -560,11 +568,11 @@ rbs_token_t rbs_lexer_next_token(rbs_lexer_t *lexer) { } } yy43: -#line 149 "src/lexer.re" +#line 158 "src/lexer.re" { return rbs_next_token(lexer, tULLIDENT); } -#line 522 "src/lexer.c" +#line 530 "src/lexer.c" yy44: yyaccept = 4; rbs_skip(lexer); @@ -581,7 +589,7 @@ rbs_token_t rbs_lexer_next_token(rbs_lexer_t *lexer) { { return rbs_next_token(lexer, tOPERATOR); } -#line 537 "src/lexer.c" +#line 545 "src/lexer.c" yy46: rbs_skip(lexer); yych = rbs_peek(lexer); @@ -594,11 +602,11 @@ rbs_token_t rbs_lexer_next_token(rbs_lexer_t *lexer) { goto yy54; } yy47: -#line 145 "src/lexer.re" +#line 154 "src/lexer.re" { return rbs_next_token(lexer, tLIDENT); } -#line 552 "src/lexer.c" +#line 560 "src/lexer.c" yy48: rbs_skip(lexer); yych = rbs_peek(lexer); @@ -720,21 +728,21 @@ rbs_token_t rbs_lexer_next_token(rbs_lexer_t *lexer) { { return rbs_next_token(lexer, pLBRACE); } -#line 672 "src/lexer.c" +#line 680 "src/lexer.c" yy66: rbs_skip(lexer); #line 43 "src/lexer.re" { return rbs_next_token(lexer, pBAR); } -#line 677 "src/lexer.c" +#line 685 "src/lexer.c" yy67: rbs_skip(lexer); #line 41 "src/lexer.re" { return rbs_next_token(lexer, pRBRACE); } -#line 682 "src/lexer.c" +#line 690 "src/lexer.c" yy68: rbs_skip(lexer); yych = rbs_peek(lexer); @@ -756,11 +764,11 @@ rbs_token_t rbs_lexer_next_token(rbs_lexer_t *lexer) { } } yy69: -#line 160 "src/lexer.re" +#line 169 "src/lexer.re" { return rbs_next_token(lexer, tNONASCIIIDENT); } -#line 706 "src/lexer.c" +#line 714 "src/lexer.c" yy70: rbs_skip(lexer); yych = rbs_peek(lexer); @@ -802,11 +810,11 @@ rbs_token_t rbs_lexer_next_token(rbs_lexer_t *lexer) { } yy73: rbs_skip(lexer); -#line 126 "src/lexer.re" +#line 135 "src/lexer.re" { return rbs_next_token(lexer, tDQSTRING); } -#line 746 "src/lexer.c" +#line 754 "src/lexer.c" yy74: rbs_skip(lexer); yych = rbs_peek(lexer); @@ -843,11 +851,11 @@ rbs_token_t rbs_lexer_next_token(rbs_lexer_t *lexer) { } } yy76: -#line 170 "src/lexer.re" +#line 179 "src/lexer.re" { return rbs_next_token(lexer, tGIDENT); } -#line 785 "src/lexer.c" +#line 793 "src/lexer.c" yy77: rbs_skip(lexer); goto yy76; @@ -886,11 +894,11 @@ rbs_token_t rbs_lexer_next_token(rbs_lexer_t *lexer) { yy81: rbs_skip(lexer); yy82: -#line 127 "src/lexer.re" +#line 136 "src/lexer.re" { return rbs_next_token(lexer, tSQSTRING); } -#line 826 "src/lexer.c" +#line 834 "src/lexer.c" yy83: rbs_skip(lexer); yych = rbs_peek(lexer); @@ -908,7 +916,7 @@ rbs_token_t rbs_lexer_next_token(rbs_lexer_t *lexer) { { return rbs_next_token(lexer, pSTAR2); } -#line 842 "src/lexer.c" +#line 850 "src/lexer.c" yy85: rbs_skip(lexer); yych = rbs_peek(lexer); @@ -917,14 +925,14 @@ rbs_token_t rbs_lexer_next_token(rbs_lexer_t *lexer) { { return rbs_next_token(lexer, tINLINECOMMENT); } -#line 849 "src/lexer.c" +#line 857 "src/lexer.c" yy86: rbs_skip(lexer); #line 53 "src/lexer.re" { return rbs_next_token(lexer, pARROW); } -#line 854 "src/lexer.c" +#line 862 "src/lexer.c" yy87: rbs_skip(lexer); yych = rbs_peek(lexer); @@ -936,11 +944,11 @@ rbs_token_t rbs_lexer_next_token(rbs_lexer_t *lexer) { if (yych == '=') goto yy92; if (yych == '~') goto yy92; yy89: -#line 143 "src/lexer.re" +#line 152 "src/lexer.re" { return rbs_next_token(lexer, tSYMBOL); } -#line 868 "src/lexer.c" +#line 876 "src/lexer.c" yy90: rbs_skip(lexer); yych = rbs_peek(lexer); @@ -1028,7 +1036,7 @@ rbs_token_t rbs_lexer_next_token(rbs_lexer_t *lexer) { { return rbs_next_token(lexer, pCOLON2); } -#line 954 "src/lexer.c" +#line 962 "src/lexer.c" yy97: rbs_skip(lexer); yych = rbs_peek(lexer); @@ -1094,11 +1102,11 @@ rbs_token_t rbs_lexer_next_token(rbs_lexer_t *lexer) { } } yy102: -#line 141 "src/lexer.re" +#line 150 "src/lexer.re" { return rbs_next_token(lexer, tSYMBOL); } -#line 1022 "src/lexer.c" +#line 1030 "src/lexer.c" yy103: rbs_skip(lexer); yych = rbs_peek(lexer); @@ -1120,7 +1128,7 @@ rbs_token_t rbs_lexer_next_token(rbs_lexer_t *lexer) { { return rbs_next_token(lexer, pFATARROW); } -#line 1042 "src/lexer.c" +#line 1050 "src/lexer.c" yy107: rbs_skip(lexer); yych = rbs_peek(lexer); @@ -1162,11 +1170,11 @@ rbs_token_t rbs_lexer_next_token(rbs_lexer_t *lexer) { } } yy110: -#line 167 "src/lexer.re" +#line 176 "src/lexer.re" { return rbs_next_token(lexer, tAIDENT); } -#line 1086 "src/lexer.c" +#line 1094 "src/lexer.c" yy111: rbs_skip(lexer); yych = rbs_peek(lexer); @@ -1174,18 +1182,18 @@ rbs_token_t rbs_lexer_next_token(rbs_lexer_t *lexer) { goto yy109; yy112: rbs_skip(lexer); -#line 164 "src/lexer.re" +#line 173 "src/lexer.re" { return rbs_next_token(lexer, tBANGIDENT); } -#line 1096 "src/lexer.c" +#line 1104 "src/lexer.c" yy113: rbs_skip(lexer); -#line 165 "src/lexer.re" +#line 174 "src/lexer.re" { return rbs_next_token(lexer, tEQIDENT); } -#line 1101 "src/lexer.c" +#line 1109 "src/lexer.c" yy114: rbs_skip(lexer); yych = rbs_peek(lexer); @@ -1194,7 +1202,7 @@ rbs_token_t rbs_lexer_next_token(rbs_lexer_t *lexer) { { return rbs_next_token(lexer, pAREF_OPR); } -#line 1108 "src/lexer.c" +#line 1116 "src/lexer.c" yy115: rbs_skip(lexer); yych = rbs_peek(lexer); @@ -1217,11 +1225,11 @@ rbs_token_t rbs_lexer_next_token(rbs_lexer_t *lexer) { } } yy117: -#line 147 "src/lexer.re" +#line 156 "src/lexer.re" { return rbs_next_token(lexer, tULLIDENT); } -#line 1133 "src/lexer.c" +#line 1141 "src/lexer.c" yy118: rbs_skip(lexer); yych = rbs_peek(lexer); @@ -1243,11 +1251,11 @@ rbs_token_t rbs_lexer_next_token(rbs_lexer_t *lexer) { } } yy119: -#line 148 "src/lexer.re" +#line 157 "src/lexer.re" { return rbs_next_token(lexer, tULIDENT); } -#line 1157 "src/lexer.c" +#line 1165 "src/lexer.c" yy120: rbs_skip(lexer); yych = rbs_peek(lexer); @@ -1285,11 +1293,11 @@ rbs_token_t rbs_lexer_next_token(rbs_lexer_t *lexer) { } } yy124: -#line 113 "src/lexer.re" +#line 122 "src/lexer.re" { return rbs_next_token(lexer, kAS); } -#line 1197 "src/lexer.c" +#line 1205 "src/lexer.c" yy125: rbs_skip(lexer); yych = rbs_peek(lexer); @@ -1364,11 +1372,11 @@ rbs_token_t rbs_lexer_next_token(rbs_lexer_t *lexer) { } } yy133: -#line 92 "src/lexer.re" +#line 101 "src/lexer.re" { return rbs_next_token(lexer, kIN); } -#line 1274 "src/lexer.c" +#line 1282 "src/lexer.c" yy134: rbs_skip(lexer); yych = rbs_peek(lexer); @@ -1516,14 +1524,14 @@ rbs_token_t rbs_lexer_next_token(rbs_lexer_t *lexer) { { return rbs_next_token(lexer, pDOT3); } -#line 1420 "src/lexer.c" +#line 1428 "src/lexer.c" yy158: rbs_skip(lexer); -#line 128 "src/lexer.re" +#line 137 "src/lexer.re" { return rbs_next_token(lexer, tDQSYMBOL); } -#line 1425 "src/lexer.c" +#line 1433 "src/lexer.c" yy159: rbs_skip(lexer); yych = rbs_peek(lexer); @@ -1560,22 +1568,22 @@ rbs_token_t rbs_lexer_next_token(rbs_lexer_t *lexer) { } } yy161: -#line 142 "src/lexer.re" +#line 151 "src/lexer.re" { return rbs_next_token(lexer, tSYMBOL); } -#line 1464 "src/lexer.c" +#line 1472 "src/lexer.c" yy162: rbs_skip(lexer); goto yy161; yy163: rbs_skip(lexer); yy164: -#line 129 "src/lexer.re" +#line 138 "src/lexer.re" { return rbs_next_token(lexer, tSQSYMBOL); } -#line 1473 "src/lexer.c" +#line 1481 "src/lexer.c" yy165: rbs_skip(lexer); yych = rbs_peek(lexer); @@ -1640,11 +1648,11 @@ rbs_token_t rbs_lexer_next_token(rbs_lexer_t *lexer) { } } yy171: -#line 168 "src/lexer.re" +#line 177 "src/lexer.re" { return rbs_next_token(lexer, tA2IDENT); } -#line 1540 "src/lexer.c" +#line 1548 "src/lexer.c" yy172: rbs_skip(lexer); yych = rbs_peek(lexer); @@ -1661,7 +1669,7 @@ rbs_token_t rbs_lexer_next_token(rbs_lexer_t *lexer) { { return rbs_next_token(lexer, tQIDENT); } -#line 1555 "src/lexer.c" +#line 1563 "src/lexer.c" yy175: rbs_skip(lexer); yych = rbs_peek(lexer); @@ -1698,11 +1706,11 @@ rbs_token_t rbs_lexer_next_token(rbs_lexer_t *lexer) { } } yy179: -#line 85 "src/lexer.re" +#line 94 "src/lexer.re" { return rbs_next_token(lexer, kBOT); } -#line 1594 "src/lexer.c" +#line 1602 "src/lexer.c" yy180: rbs_skip(lexer); yych = rbs_peek(lexer); @@ -1729,11 +1737,11 @@ rbs_token_t rbs_lexer_next_token(rbs_lexer_t *lexer) { } } yy182: -#line 88 "src/lexer.re" +#line 97 "src/lexer.re" { return rbs_next_token(lexer, kDEF); } -#line 1623 "src/lexer.c" +#line 1631 "src/lexer.c" yy183: rbs_skip(lexer); yych = rbs_peek(lexer); @@ -1755,11 +1763,11 @@ rbs_token_t rbs_lexer_next_token(rbs_lexer_t *lexer) { } } yy184: -#line 89 "src/lexer.re" +#line 98 "src/lexer.re" { return rbs_next_token(lexer, kEND); } -#line 1647 "src/lexer.c" +#line 1655 "src/lexer.c" yy185: rbs_skip(lexer); yych = rbs_peek(lexer); @@ -1811,11 +1819,11 @@ rbs_token_t rbs_lexer_next_token(rbs_lexer_t *lexer) { } } yy192: -#line 99 "src/lexer.re" +#line 108 "src/lexer.re" { return rbs_next_token(lexer, kNIL); } -#line 1701 "src/lexer.c" +#line 1709 "src/lexer.c" yy193: rbs_skip(lexer); yych = rbs_peek(lexer); @@ -1837,11 +1845,11 @@ rbs_token_t rbs_lexer_next_token(rbs_lexer_t *lexer) { } } yy194: -#line 100 "src/lexer.re" +#line 109 "src/lexer.re" { return rbs_next_token(lexer, kOUT); } -#line 1725 "src/lexer.c" +#line 1733 "src/lexer.c" yy195: rbs_skip(lexer); yych = rbs_peek(lexer); @@ -1898,11 +1906,11 @@ rbs_token_t rbs_lexer_next_token(rbs_lexer_t *lexer) { } } yy203: -#line 106 "src/lexer.re" +#line 115 "src/lexer.re" { return rbs_next_token(lexer, kTOP); } -#line 1784 "src/lexer.c" +#line 1792 "src/lexer.c" yy204: rbs_skip(lexer); yych = rbs_peek(lexer); @@ -1944,11 +1952,11 @@ rbs_token_t rbs_lexer_next_token(rbs_lexer_t *lexer) { } } yy209: -#line 112 "src/lexer.re" +#line 121 "src/lexer.re" { return rbs_next_token(lexer, kUSE); } -#line 1828 "src/lexer.c" +#line 1836 "src/lexer.c" yy210: rbs_skip(lexer); yych = rbs_peek(lexer); @@ -1973,35 +1981,35 @@ rbs_token_t rbs_lexer_next_token(rbs_lexer_t *lexer) { { return rbs_next_token(lexer, tANNOTATION); } -#line 1851 "src/lexer.c" +#line 1859 "src/lexer.c" yy213: rbs_skip(lexer); #line 71 "src/lexer.re" { return rbs_next_token(lexer, tANNOTATION); } -#line 1856 "src/lexer.c" +#line 1864 "src/lexer.c" yy214: rbs_skip(lexer); #line 69 "src/lexer.re" { return rbs_next_token(lexer, tANNOTATION); } -#line 1861 "src/lexer.c" +#line 1869 "src/lexer.c" yy215: rbs_skip(lexer); #line 67 "src/lexer.re" { return rbs_next_token(lexer, tANNOTATION); } -#line 1866 "src/lexer.c" +#line 1874 "src/lexer.c" yy216: rbs_skip(lexer); #line 70 "src/lexer.re" { return rbs_next_token(lexer, tANNOTATION); } -#line 1871 "src/lexer.c" +#line 1879 "src/lexer.c" yy217: rbs_skip(lexer); yych = rbs_peek(lexer); @@ -2056,11 +2064,11 @@ rbs_token_t rbs_lexer_next_token(rbs_lexer_t *lexer) { } } yy221: -#line 115 "src/lexer.re" +#line 124 "src/lexer.re" { return rbs_next_token(lexer, kATRBS); } -#line 1928 "src/lexer.c" +#line 1936 "src/lexer.c" yy222: rbs_skip(lexer); yych = rbs_peek(lexer); @@ -2097,11 +2105,11 @@ rbs_token_t rbs_lexer_next_token(rbs_lexer_t *lexer) { } } yy226: -#line 84 "src/lexer.re" +#line 93 "src/lexer.re" { return rbs_next_token(lexer, kBOOL); } -#line 1967 "src/lexer.c" +#line 1975 "src/lexer.c" yy227: rbs_skip(lexer); yych = rbs_peek(lexer); @@ -2178,11 +2186,11 @@ rbs_token_t rbs_lexer_next_token(rbs_lexer_t *lexer) { } } yy239: -#line 104 "src/lexer.re" +#line 113 "src/lexer.re" { return rbs_next_token(lexer, kSELF); } -#line 2046 "src/lexer.c" +#line 2054 "src/lexer.c" yy240: rbs_skip(lexer); yych = rbs_peek(lexer); @@ -2209,11 +2217,11 @@ rbs_token_t rbs_lexer_next_token(rbs_lexer_t *lexer) { } } yy242: -#line 116 "src/lexer.re" +#line 125 "src/lexer.re" { return rbs_next_token(lexer, kSKIP); } -#line 2075 "src/lexer.c" +#line 2083 "src/lexer.c" yy243: rbs_skip(lexer); yych = rbs_peek(lexer); @@ -2235,11 +2243,11 @@ rbs_token_t rbs_lexer_next_token(rbs_lexer_t *lexer) { } } yy244: -#line 107 "src/lexer.re" +#line 116 "src/lexer.re" { return rbs_next_token(lexer, kTRUE); } -#line 2099 "src/lexer.c" +#line 2107 "src/lexer.c" yy245: rbs_skip(lexer); yych = rbs_peek(lexer); @@ -2261,11 +2269,11 @@ rbs_token_t rbs_lexer_next_token(rbs_lexer_t *lexer) { } } yy246: -#line 108 "src/lexer.re" +#line 117 "src/lexer.re" { return rbs_next_token(lexer, kTYPE); } -#line 2123 "src/lexer.c" +#line 2131 "src/lexer.c" yy247: rbs_skip(lexer); yych = rbs_peek(lexer); @@ -2297,11 +2305,11 @@ rbs_token_t rbs_lexer_next_token(rbs_lexer_t *lexer) { } } yy250: -#line 111 "src/lexer.re" +#line 120 "src/lexer.re" { return rbs_next_token(lexer, kVOID); } -#line 2157 "src/lexer.c" +#line 2165 "src/lexer.c" yy251: rbs_skip(lexer); yych = rbs_peek(lexer); @@ -2354,11 +2362,11 @@ rbs_token_t rbs_lexer_next_token(rbs_lexer_t *lexer) { } } yy255: -#line 80 "src/lexer.re" +#line 89 "src/lexer.re" { return rbs_next_token(lexer, kALIAS); } -#line 2212 "src/lexer.c" +#line 2220 "src/lexer.c" yy256: rbs_skip(lexer); yych = rbs_peek(lexer); @@ -2399,11 +2407,11 @@ rbs_token_t rbs_lexer_next_token(rbs_lexer_t *lexer) { } } yy258: -#line 86 "src/lexer.re" +#line 95 "src/lexer.re" { return rbs_next_token(lexer, kCLASS); } -#line 2255 "src/lexer.c" +#line 2263 "src/lexer.c" yy259: rbs_skip(lexer); yych = rbs_peek(lexer); @@ -2430,11 +2438,11 @@ rbs_token_t rbs_lexer_next_token(rbs_lexer_t *lexer) { } } yy261: -#line 91 "src/lexer.re" +#line 100 "src/lexer.re" { return rbs_next_token(lexer, kFALSE); } -#line 2284 "src/lexer.c" +#line 2292 "src/lexer.c" yy262: rbs_skip(lexer); yych = rbs_peek(lexer); @@ -2562,11 +2570,11 @@ rbs_token_t rbs_lexer_next_token(rbs_lexer_t *lexer) { } } yy281: -#line 90 "src/lexer.re" +#line 99 "src/lexer.re" { return rbs_next_token(lexer, kEXTEND); } -#line 2414 "src/lexer.c" +#line 2422 "src/lexer.c" yy282: rbs_skip(lexer); yych = rbs_peek(lexer); @@ -2611,11 +2619,11 @@ rbs_token_t rbs_lexer_next_token(rbs_lexer_t *lexer) { } } yy286: -#line 96 "src/lexer.re" +#line 105 "src/lexer.re" { return rbs_next_token(lexer, kMODULE); } -#line 2461 "src/lexer.c" +#line 2469 "src/lexer.c" yy287: rbs_skip(lexer); yych = rbs_peek(lexer); @@ -2647,11 +2655,11 @@ rbs_token_t rbs_lexer_next_token(rbs_lexer_t *lexer) { } } yy290: -#line 103 "src/lexer.re" +#line 112 "src/lexer.re" { return rbs_next_token(lexer, kPUBLIC); } -#line 2495 "src/lexer.c" +#line 2503 "src/lexer.c" yy291: rbs_skip(lexer); yych = rbs_peek(lexer); @@ -2673,11 +2681,11 @@ rbs_token_t rbs_lexer_next_token(rbs_lexer_t *lexer) { } } yy292: -#line 117 "src/lexer.re" +#line 126 "src/lexer.re" { return rbs_next_token(lexer, kRETURN); } -#line 2519 "src/lexer.c" +#line 2527 "src/lexer.c" yy293: rbs_skip(lexer); yych = rbs_peek(lexer); @@ -2752,11 +2760,11 @@ rbs_token_t rbs_lexer_next_token(rbs_lexer_t *lexer) { } } yy303: -#line 93 "src/lexer.re" +#line 102 "src/lexer.re" { return rbs_next_token(lexer, kINCLUDE); } -#line 2596 "src/lexer.c" +#line 2604 "src/lexer.c" yy304: rbs_skip(lexer); yych = rbs_peek(lexer); @@ -2794,11 +2802,11 @@ rbs_token_t rbs_lexer_next_token(rbs_lexer_t *lexer) { } } yy308: -#line 101 "src/lexer.re" +#line 110 "src/lexer.re" { return rbs_next_token(lexer, kPREPEND); } -#line 2636 "src/lexer.c" +#line 2644 "src/lexer.c" yy309: rbs_skip(lexer); yych = rbs_peek(lexer); @@ -2820,11 +2828,11 @@ rbs_token_t rbs_lexer_next_token(rbs_lexer_t *lexer) { } } yy310: -#line 102 "src/lexer.re" +#line 111 "src/lexer.re" { return rbs_next_token(lexer, kPRIVATE); } -#line 2660 "src/lexer.c" +#line 2668 "src/lexer.c" yy311: rbs_skip(lexer); yych = rbs_peek(lexer); @@ -2856,11 +2864,11 @@ rbs_token_t rbs_lexer_next_token(rbs_lexer_t *lexer) { } } yy314: -#line 110 "src/lexer.re" +#line 119 "src/lexer.re" { return rbs_next_token(lexer, kUNTYPED); } -#line 2694 "src/lexer.c" +#line 2702 "src/lexer.c" yy315: rbs_skip(lexer); yych = rbs_peek(lexer); @@ -2882,11 +2890,11 @@ rbs_token_t rbs_lexer_next_token(rbs_lexer_t *lexer) { } } yy316: -#line 114 "src/lexer.re" +#line 123 "src/lexer.re" { return rbs_next_token(lexer, k__TODO__); } -#line 2718 "src/lexer.c" +#line 2726 "src/lexer.c" yy317: rbs_skip(lexer); yych = rbs_peek(lexer); @@ -2928,11 +2936,11 @@ rbs_token_t rbs_lexer_next_token(rbs_lexer_t *lexer) { } } yy322: -#line 94 "src/lexer.re" +#line 103 "src/lexer.re" { return rbs_next_token(lexer, kINSTANCE); } -#line 2762 "src/lexer.c" +#line 2770 "src/lexer.c" yy323: rbs_skip(lexer); yych = rbs_peek(lexer); @@ -2999,11 +3007,11 @@ rbs_token_t rbs_lexer_next_token(rbs_lexer_t *lexer) { } } yy333: -#line 95 "src/lexer.re" +#line 104 "src/lexer.re" { return rbs_next_token(lexer, kINTERFACE); } -#line 2831 "src/lexer.c" +#line 2839 "src/lexer.c" yy334: rbs_skip(lexer); yych = rbs_peek(lexer); @@ -3035,11 +3043,11 @@ rbs_token_t rbs_lexer_next_token(rbs_lexer_t *lexer) { } } yy337: -#line 105 "src/lexer.re" +#line 114 "src/lexer.re" { return rbs_next_token(lexer, kSINGLETON); } -#line 2865 "src/lexer.c" +#line 2873 "src/lexer.c" yy338: rbs_skip(lexer); yych = rbs_peek(lexer); @@ -3061,11 +3069,11 @@ rbs_token_t rbs_lexer_next_token(rbs_lexer_t *lexer) { } } yy339: -#line 109 "src/lexer.re" +#line 118 "src/lexer.re" { return rbs_next_token(lexer, kUNCHECKED); } -#line 2889 "src/lexer.c" +#line 2897 "src/lexer.c" yy340: rbs_skip(lexer); yych = rbs_peek(lexer); @@ -3122,11 +3130,11 @@ rbs_token_t rbs_lexer_next_token(rbs_lexer_t *lexer) { } } yy348: -#line 82 "src/lexer.re" +#line 91 "src/lexer.re" { return rbs_next_token(lexer, kATTRREADER); } -#line 2948 "src/lexer.c" +#line 2956 "src/lexer.c" yy349: rbs_skip(lexer); yych = rbs_peek(lexer); @@ -3148,18 +3156,18 @@ rbs_token_t rbs_lexer_next_token(rbs_lexer_t *lexer) { } } yy350: -#line 83 "src/lexer.re" +#line 92 "src/lexer.re" { return rbs_next_token(lexer, kATTRWRITER); } -#line 2972 "src/lexer.c" +#line 2980 "src/lexer.c" yy351: rbs_skip(lexer); -#line 87 "src/lexer.re" +#line 96 "src/lexer.re" { return rbs_next_token(lexer, kCLASSALIAS); } -#line 2977 "src/lexer.c" +#line 2985 "src/lexer.c" yy352: rbs_skip(lexer); yych = rbs_peek(lexer); @@ -3167,11 +3175,11 @@ rbs_token_t rbs_lexer_next_token(rbs_lexer_t *lexer) { goto yy72; yy353: rbs_skip(lexer); -#line 98 "src/lexer.re" +#line 107 "src/lexer.re" { return rbs_next_token(lexer, kMODULESELF); } -#line 2987 "src/lexer.c" +#line 2995 "src/lexer.c" yy354: rbs_skip(lexer); yych = rbs_peek(lexer); @@ -3179,11 +3187,11 @@ rbs_token_t rbs_lexer_next_token(rbs_lexer_t *lexer) { goto yy54; yy355: rbs_skip(lexer); -#line 97 "src/lexer.re" +#line 106 "src/lexer.re" { return rbs_next_token(lexer, kMODULEALIAS); } -#line 2997 "src/lexer.c" +#line 3005 "src/lexer.c" yy356: rbs_skip(lexer); yych = rbs_peek(lexer); @@ -3205,11 +3213,11 @@ rbs_token_t rbs_lexer_next_token(rbs_lexer_t *lexer) { } } yy357: -#line 81 "src/lexer.re" +#line 90 "src/lexer.re" { return rbs_next_token(lexer, kATTRACCESSOR); } -#line 3021 "src/lexer.c" +#line 3029 "src/lexer.c" } -#line 177 "src/lexer.re" +#line 186 "src/lexer.re" } diff --git a/src/lexer.re b/src/lexer.re index af0f50a2a..2cbb154de 100644 --- a/src/lexer.re +++ b/src/lexer.re @@ -70,7 +70,16 @@ rbs_token_t rbs_lexer_next_token(rbs_lexer_t *lexer) { "%a|" [^|\x00]* "|" { return rbs_next_token(lexer, tANNOTATION); } "%a<" [^>\x00]* ">" { return rbs_next_token(lexer, tANNOTATION); } - "#" (. \ [\x00\r\uFFFD])* { + "#" (. \ [\x00\uFFFD])* { + // Keep a bare CR in the comment, but leave the CR in CRLF for a trivia token. + if (rbs_peek(lexer) == '\n' && lexer->string.start[lexer->current.byte_pos - 1] == '\r') { + lexer->current.byte_pos -= 1; + lexer->current.char_pos -= 1; + lexer->current.column -= 1; + lexer->current_code_point = '\r'; + lexer->current_character_bytes = 1; + } + return rbs_next_token( lexer, lexer->first_token_of_line ? tLINECOMMENT : tCOMMENT diff --git a/test/rbs/parser_test.rb b/test/rbs/parser_test.rb index 256e1e29f..09b7054e2 100644 --- a/test/rbs/parser_test.rb +++ b/test/rbs/parser_test.rb @@ -1032,6 +1032,10 @@ class Foo[T < Integer] < Bar # Comment def test__lex_crlf content = "# LineComment\r\nclass Foo[T < Integer] < Bar # Comment\r\nend\r\n" + + comments = RBS::Parser.lex(content).value.select { %i[tLINECOMMENT tCOMMENT].include?(_1.type) } + assert_equal [["# LineComment", 1], ["# Comment", 2]], comments.map { [_1.location.source, _1.location.end_line] } + tokens = RBS::Parser._lex(buffer(content), content.length) assert_equal [:tLINECOMMENT, '# LineComment', 0...13], tokens.shift.then { |t| [t[0], t[1].source, t[1].range] } assert_equal [:tTRIVIA, "\r", 13...14], tokens.shift.then { |t| [t[0], t[1].source, t[1].range] } @@ -1060,6 +1064,14 @@ def test__lex_crlf assert_equal [:pEOF, '', 60...60], tokens.shift.then { |t| [t[0], t[1].source, t[1].range] } end + def test__lex_comment_with_bare_cr + content = "# Comment\rclass Foo" + tokens = RBS::Parser.lex(content).value + + assert_equal [:tLINECOMMENT, :pEOF], tokens.map(&:type) + assert_equal content, tokens.first.location.source + end + def test_invalid_position_range_raises # Regression: start_pos > end_pos used to cause an infinite loop in the lexer. assert_raises(ArgumentError) do