diff --git a/src/lexer.c b/src/lexer.c index 144cc071d..886449a4d 100644 --- a/src/lexer.c +++ b/src/lexer.c @@ -129,7 +129,7 @@ rbs_token_t rbs_lexer_next_token(rbs_lexer_t *lexer) { } yy1: rbs_skip(lexer); -#line 175 "src/lexer.re" +#line 184 "src/lexer.re" { return rbs_next_eof_token(lexer); } @@ -137,7 +137,7 @@ rbs_token_t rbs_lexer_next_token(rbs_lexer_t *lexer) { yy2: rbs_skip(lexer); yy3: -#line 176 "src/lexer.re" +#line 185 "src/lexer.re" { return rbs_next_token(lexer, ErrorToken); } @@ -148,7 +148,7 @@ rbs_token_t rbs_lexer_next_token(rbs_lexer_t *lexer) { if (yych == '\t') goto yy4; if (yych == ' ') goto yy4; yy5: -#line 174 "src/lexer.re" +#line 183 "src/lexer.re" { return rbs_next_token(lexer, tTRIVIA); } @@ -186,12 +186,21 @@ rbs_token_t rbs_lexer_next_token(rbs_lexer_t *lexer) { yy11: #line 73 "src/lexer.re" { + // Keep a bare CR in the comment, but leave the CR in CRLF for a trivia token. + if (rbs_peek(lexer) == '\n' && lexer->string.start[lexer->current.byte_pos - 1] == '\r') { + lexer->current.byte_pos -= 1; + lexer->current.char_pos -= 1; + lexer->current.column -= 1; + lexer->current_code_point = '\r'; + lexer->current_character_bytes = 1; + } + return rbs_next_token( lexer, lexer->first_token_of_line ? tLINECOMMENT : tCOMMENT ); } -#line 188 "src/lexer.c" +#line 197 "src/lexer.c" yy12: rbs_skip(lexer); yych = rbs_peek(lexer); @@ -251,7 +260,7 @@ rbs_token_t rbs_lexer_next_token(rbs_lexer_t *lexer) { { return rbs_next_token(lexer, pAMP); } -#line 246 "src/lexer.c" +#line 255 "src/lexer.c" yy15: yyaccept = 0; rbs_skip(lexer); @@ -265,14 +274,14 @@ rbs_token_t rbs_lexer_next_token(rbs_lexer_t *lexer) { { return rbs_next_token(lexer, pLPAREN); } -#line 258 "src/lexer.c" +#line 267 "src/lexer.c" yy17: rbs_skip(lexer); #line 37 "src/lexer.re" { return rbs_next_token(lexer, pRPAREN); } -#line 263 "src/lexer.c" +#line 272 "src/lexer.c" yy18: rbs_skip(lexer); yych = rbs_peek(lexer); @@ -281,7 +290,7 @@ rbs_token_t rbs_lexer_next_token(rbs_lexer_t *lexer) { { return rbs_next_token(lexer, pSTAR); } -#line 270 "src/lexer.c" +#line 279 "src/lexer.c" yy19: rbs_skip(lexer); yych = rbs_peek(lexer); @@ -295,7 +304,7 @@ rbs_token_t rbs_lexer_next_token(rbs_lexer_t *lexer) { { return rbs_next_token(lexer, pCOMMA); } -#line 282 "src/lexer.c" +#line 291 "src/lexer.c" yy21: rbs_skip(lexer); yych = rbs_peek(lexer); @@ -331,7 +340,7 @@ rbs_token_t rbs_lexer_next_token(rbs_lexer_t *lexer) { { return rbs_next_token(lexer, pDOT); } -#line 311 "src/lexer.c" +#line 320 "src/lexer.c" yy24: rbs_skip(lexer); goto yy8; @@ -346,7 +355,7 @@ rbs_token_t rbs_lexer_next_token(rbs_lexer_t *lexer) { { return rbs_next_token(lexer, tINTEGER); } -#line 324 "src/lexer.c" +#line 333 "src/lexer.c" yy27: yyaccept = 3; rbs_skip(lexer); @@ -418,7 +427,7 @@ rbs_token_t rbs_lexer_next_token(rbs_lexer_t *lexer) { { return rbs_next_token(lexer, pCOLON); } -#line 394 "src/lexer.c" +#line 403 "src/lexer.c" yy29: rbs_skip(lexer); yych = rbs_peek(lexer); @@ -430,7 +439,7 @@ rbs_token_t rbs_lexer_next_token(rbs_lexer_t *lexer) { { return rbs_next_token(lexer, pLT); } -#line 404 "src/lexer.c" +#line 413 "src/lexer.c" yy31: rbs_skip(lexer); yych = rbs_peek(lexer); @@ -446,7 +455,7 @@ rbs_token_t rbs_lexer_next_token(rbs_lexer_t *lexer) { { return rbs_next_token(lexer, pEQ); } -#line 418 "src/lexer.c" +#line 427 "src/lexer.c" yy33: rbs_skip(lexer); yych = rbs_peek(lexer); @@ -457,14 +466,14 @@ rbs_token_t rbs_lexer_next_token(rbs_lexer_t *lexer) { { return rbs_next_token(lexer, pGT); } -#line 427 "src/lexer.c" +#line 436 "src/lexer.c" yy35: rbs_skip(lexer); #line 46 "src/lexer.re" { return rbs_next_token(lexer, pQUESTION); } -#line 432 "src/lexer.c" +#line 441 "src/lexer.c" yy36: yyaccept = 0; rbs_skip(lexer); @@ -510,11 +519,11 @@ rbs_token_t rbs_lexer_next_token(rbs_lexer_t *lexer) { } } yy38: -#line 146 "src/lexer.re" +#line 155 "src/lexer.re" { return rbs_next_token(lexer, tUIDENT); } -#line 480 "src/lexer.c" +#line 489 "src/lexer.c" yy39: rbs_skip(lexer); yych = rbs_peek(lexer); @@ -523,21 +532,21 @@ rbs_token_t rbs_lexer_next_token(rbs_lexer_t *lexer) { { return rbs_next_token(lexer, pLBRACKET); } -#line 487 "src/lexer.c" +#line 496 "src/lexer.c" yy40: rbs_skip(lexer); #line 39 "src/lexer.re" { return rbs_next_token(lexer, pRBRACKET); } -#line 492 "src/lexer.c" +#line 501 "src/lexer.c" yy41: rbs_skip(lexer); #line 44 "src/lexer.re" { return rbs_next_token(lexer, pHAT); } -#line 497 "src/lexer.c" +#line 506 "src/lexer.c" yy42: rbs_skip(lexer); yych = rbs_peek(lexer); @@ -559,11 +568,11 @@ rbs_token_t rbs_lexer_next_token(rbs_lexer_t *lexer) { } } yy43: -#line 149 "src/lexer.re" +#line 158 "src/lexer.re" { return rbs_next_token(lexer, tULLIDENT); } -#line 521 "src/lexer.c" +#line 530 "src/lexer.c" yy44: yyaccept = 4; rbs_skip(lexer); @@ -580,7 +589,7 @@ rbs_token_t rbs_lexer_next_token(rbs_lexer_t *lexer) { { return rbs_next_token(lexer, tOPERATOR); } -#line 536 "src/lexer.c" +#line 545 "src/lexer.c" yy46: rbs_skip(lexer); yych = rbs_peek(lexer); @@ -593,11 +602,11 @@ rbs_token_t rbs_lexer_next_token(rbs_lexer_t *lexer) { goto yy54; } yy47: -#line 145 "src/lexer.re" +#line 154 "src/lexer.re" { return rbs_next_token(lexer, tLIDENT); } -#line 551 "src/lexer.c" +#line 560 "src/lexer.c" yy48: rbs_skip(lexer); yych = rbs_peek(lexer); @@ -719,21 +728,21 @@ rbs_token_t rbs_lexer_next_token(rbs_lexer_t *lexer) { { return rbs_next_token(lexer, pLBRACE); } -#line 671 "src/lexer.c" +#line 680 "src/lexer.c" yy66: rbs_skip(lexer); #line 43 "src/lexer.re" { return rbs_next_token(lexer, pBAR); } -#line 676 "src/lexer.c" +#line 685 "src/lexer.c" yy67: rbs_skip(lexer); #line 41 "src/lexer.re" { return rbs_next_token(lexer, pRBRACE); } -#line 681 "src/lexer.c" +#line 690 "src/lexer.c" yy68: rbs_skip(lexer); yych = rbs_peek(lexer); @@ -755,11 +764,11 @@ rbs_token_t rbs_lexer_next_token(rbs_lexer_t *lexer) { } } yy69: -#line 160 "src/lexer.re" +#line 169 "src/lexer.re" { return rbs_next_token(lexer, tNONASCIIIDENT); } -#line 705 "src/lexer.c" +#line 714 "src/lexer.c" yy70: rbs_skip(lexer); yych = rbs_peek(lexer); @@ -801,11 +810,11 @@ rbs_token_t rbs_lexer_next_token(rbs_lexer_t *lexer) { } yy73: rbs_skip(lexer); -#line 126 "src/lexer.re" +#line 135 "src/lexer.re" { return rbs_next_token(lexer, tDQSTRING); } -#line 745 "src/lexer.c" +#line 754 "src/lexer.c" yy74: rbs_skip(lexer); yych = rbs_peek(lexer); @@ -842,11 +851,11 @@ rbs_token_t rbs_lexer_next_token(rbs_lexer_t *lexer) { } } yy76: -#line 170 "src/lexer.re" +#line 179 "src/lexer.re" { return rbs_next_token(lexer, tGIDENT); } -#line 784 "src/lexer.c" +#line 793 "src/lexer.c" yy77: rbs_skip(lexer); goto yy76; @@ -885,11 +894,11 @@ rbs_token_t rbs_lexer_next_token(rbs_lexer_t *lexer) { yy81: rbs_skip(lexer); yy82: -#line 127 "src/lexer.re" +#line 136 "src/lexer.re" { return rbs_next_token(lexer, tSQSTRING); } -#line 825 "src/lexer.c" +#line 834 "src/lexer.c" yy83: rbs_skip(lexer); yych = rbs_peek(lexer); @@ -907,7 +916,7 @@ rbs_token_t rbs_lexer_next_token(rbs_lexer_t *lexer) { { return rbs_next_token(lexer, pSTAR2); } -#line 841 "src/lexer.c" +#line 850 "src/lexer.c" yy85: rbs_skip(lexer); yych = rbs_peek(lexer); @@ -916,14 +925,14 @@ rbs_token_t rbs_lexer_next_token(rbs_lexer_t *lexer) { { return rbs_next_token(lexer, tINLINECOMMENT); } -#line 848 "src/lexer.c" +#line 857 "src/lexer.c" yy86: rbs_skip(lexer); #line 53 "src/lexer.re" { return rbs_next_token(lexer, pARROW); } -#line 853 "src/lexer.c" +#line 862 "src/lexer.c" yy87: rbs_skip(lexer); yych = rbs_peek(lexer); @@ -935,11 +944,11 @@ rbs_token_t rbs_lexer_next_token(rbs_lexer_t *lexer) { if (yych == '=') goto yy92; if (yych == '~') goto yy92; yy89: -#line 143 "src/lexer.re" +#line 152 "src/lexer.re" { return rbs_next_token(lexer, tSYMBOL); } -#line 867 "src/lexer.c" +#line 876 "src/lexer.c" yy90: rbs_skip(lexer); yych = rbs_peek(lexer); @@ -1027,7 +1036,7 @@ rbs_token_t rbs_lexer_next_token(rbs_lexer_t *lexer) { { return rbs_next_token(lexer, pCOLON2); } -#line 953 "src/lexer.c" +#line 962 "src/lexer.c" yy97: rbs_skip(lexer); yych = rbs_peek(lexer); @@ -1093,11 +1102,11 @@ rbs_token_t rbs_lexer_next_token(rbs_lexer_t *lexer) { } } yy102: -#line 141 "src/lexer.re" +#line 150 "src/lexer.re" { return rbs_next_token(lexer, tSYMBOL); } -#line 1021 "src/lexer.c" +#line 1030 "src/lexer.c" yy103: rbs_skip(lexer); yych = rbs_peek(lexer); @@ -1119,7 +1128,7 @@ rbs_token_t rbs_lexer_next_token(rbs_lexer_t *lexer) { { return rbs_next_token(lexer, pFATARROW); } -#line 1041 "src/lexer.c" +#line 1050 "src/lexer.c" yy107: rbs_skip(lexer); yych = rbs_peek(lexer); @@ -1161,11 +1170,11 @@ rbs_token_t rbs_lexer_next_token(rbs_lexer_t *lexer) { } } yy110: -#line 167 "src/lexer.re" +#line 176 "src/lexer.re" { return rbs_next_token(lexer, tAIDENT); } -#line 1085 "src/lexer.c" +#line 1094 "src/lexer.c" yy111: rbs_skip(lexer); yych = rbs_peek(lexer); @@ -1173,18 +1182,18 @@ rbs_token_t rbs_lexer_next_token(rbs_lexer_t *lexer) { goto yy109; yy112: rbs_skip(lexer); -#line 164 "src/lexer.re" +#line 173 "src/lexer.re" { return rbs_next_token(lexer, tBANGIDENT); } -#line 1095 "src/lexer.c" +#line 1104 "src/lexer.c" yy113: rbs_skip(lexer); -#line 165 "src/lexer.re" +#line 174 "src/lexer.re" { return rbs_next_token(lexer, tEQIDENT); } -#line 1100 "src/lexer.c" +#line 1109 "src/lexer.c" yy114: rbs_skip(lexer); yych = rbs_peek(lexer); @@ -1193,7 +1202,7 @@ rbs_token_t rbs_lexer_next_token(rbs_lexer_t *lexer) { { return rbs_next_token(lexer, pAREF_OPR); } -#line 1107 "src/lexer.c" +#line 1116 "src/lexer.c" yy115: rbs_skip(lexer); yych = rbs_peek(lexer); @@ -1216,11 +1225,11 @@ rbs_token_t rbs_lexer_next_token(rbs_lexer_t *lexer) { } } yy117: -#line 147 "src/lexer.re" +#line 156 "src/lexer.re" { return rbs_next_token(lexer, tULLIDENT); } -#line 1132 "src/lexer.c" +#line 1141 "src/lexer.c" yy118: rbs_skip(lexer); yych = rbs_peek(lexer); @@ -1242,11 +1251,11 @@ rbs_token_t rbs_lexer_next_token(rbs_lexer_t *lexer) { } } yy119: -#line 148 "src/lexer.re" +#line 157 "src/lexer.re" { return rbs_next_token(lexer, tULIDENT); } -#line 1156 "src/lexer.c" +#line 1165 "src/lexer.c" yy120: rbs_skip(lexer); yych = rbs_peek(lexer); @@ -1284,11 +1293,11 @@ rbs_token_t rbs_lexer_next_token(rbs_lexer_t *lexer) { } } yy124: -#line 113 "src/lexer.re" +#line 122 "src/lexer.re" { return rbs_next_token(lexer, kAS); } -#line 1196 "src/lexer.c" +#line 1205 "src/lexer.c" yy125: rbs_skip(lexer); yych = rbs_peek(lexer); @@ -1363,11 +1372,11 @@ rbs_token_t rbs_lexer_next_token(rbs_lexer_t *lexer) { } } yy133: -#line 92 "src/lexer.re" +#line 101 "src/lexer.re" { return rbs_next_token(lexer, kIN); } -#line 1273 "src/lexer.c" +#line 1282 "src/lexer.c" yy134: rbs_skip(lexer); yych = rbs_peek(lexer); @@ -1515,14 +1524,14 @@ rbs_token_t rbs_lexer_next_token(rbs_lexer_t *lexer) { { return rbs_next_token(lexer, pDOT3); } -#line 1419 "src/lexer.c" +#line 1428 "src/lexer.c" yy158: rbs_skip(lexer); -#line 128 "src/lexer.re" +#line 137 "src/lexer.re" { return rbs_next_token(lexer, tDQSYMBOL); } -#line 1424 "src/lexer.c" +#line 1433 "src/lexer.c" yy159: rbs_skip(lexer); yych = rbs_peek(lexer); @@ -1559,22 +1568,22 @@ rbs_token_t rbs_lexer_next_token(rbs_lexer_t *lexer) { } } yy161: -#line 142 "src/lexer.re" +#line 151 "src/lexer.re" { return rbs_next_token(lexer, tSYMBOL); } -#line 1463 "src/lexer.c" +#line 1472 "src/lexer.c" yy162: rbs_skip(lexer); goto yy161; yy163: rbs_skip(lexer); yy164: -#line 129 "src/lexer.re" +#line 138 "src/lexer.re" { return rbs_next_token(lexer, tSQSYMBOL); } -#line 1472 "src/lexer.c" +#line 1481 "src/lexer.c" yy165: rbs_skip(lexer); yych = rbs_peek(lexer); @@ -1639,11 +1648,11 @@ rbs_token_t rbs_lexer_next_token(rbs_lexer_t *lexer) { } } yy171: -#line 168 "src/lexer.re" +#line 177 "src/lexer.re" { return rbs_next_token(lexer, tA2IDENT); } -#line 1539 "src/lexer.c" +#line 1548 "src/lexer.c" yy172: rbs_skip(lexer); yych = rbs_peek(lexer); @@ -1660,7 +1669,7 @@ rbs_token_t rbs_lexer_next_token(rbs_lexer_t *lexer) { { return rbs_next_token(lexer, tQIDENT); } -#line 1554 "src/lexer.c" +#line 1563 "src/lexer.c" yy175: rbs_skip(lexer); yych = rbs_peek(lexer); @@ -1697,11 +1706,11 @@ rbs_token_t rbs_lexer_next_token(rbs_lexer_t *lexer) { } } yy179: -#line 85 "src/lexer.re" +#line 94 "src/lexer.re" { return rbs_next_token(lexer, kBOT); } -#line 1593 "src/lexer.c" +#line 1602 "src/lexer.c" yy180: rbs_skip(lexer); yych = rbs_peek(lexer); @@ -1728,11 +1737,11 @@ rbs_token_t rbs_lexer_next_token(rbs_lexer_t *lexer) { } } yy182: -#line 88 "src/lexer.re" +#line 97 "src/lexer.re" { return rbs_next_token(lexer, kDEF); } -#line 1622 "src/lexer.c" +#line 1631 "src/lexer.c" yy183: rbs_skip(lexer); yych = rbs_peek(lexer); @@ -1754,11 +1763,11 @@ rbs_token_t rbs_lexer_next_token(rbs_lexer_t *lexer) { } } yy184: -#line 89 "src/lexer.re" +#line 98 "src/lexer.re" { return rbs_next_token(lexer, kEND); } -#line 1646 "src/lexer.c" +#line 1655 "src/lexer.c" yy185: rbs_skip(lexer); yych = rbs_peek(lexer); @@ -1810,11 +1819,11 @@ rbs_token_t rbs_lexer_next_token(rbs_lexer_t *lexer) { } } yy192: -#line 99 "src/lexer.re" +#line 108 "src/lexer.re" { return rbs_next_token(lexer, kNIL); } -#line 1700 "src/lexer.c" +#line 1709 "src/lexer.c" yy193: rbs_skip(lexer); yych = rbs_peek(lexer); @@ -1836,11 +1845,11 @@ rbs_token_t rbs_lexer_next_token(rbs_lexer_t *lexer) { } } yy194: -#line 100 "src/lexer.re" +#line 109 "src/lexer.re" { return rbs_next_token(lexer, kOUT); } -#line 1724 "src/lexer.c" +#line 1733 "src/lexer.c" yy195: rbs_skip(lexer); yych = rbs_peek(lexer); @@ -1897,11 +1906,11 @@ rbs_token_t rbs_lexer_next_token(rbs_lexer_t *lexer) { } } yy203: -#line 106 "src/lexer.re" +#line 115 "src/lexer.re" { return rbs_next_token(lexer, kTOP); } -#line 1783 "src/lexer.c" +#line 1792 "src/lexer.c" yy204: rbs_skip(lexer); yych = rbs_peek(lexer); @@ -1943,11 +1952,11 @@ rbs_token_t rbs_lexer_next_token(rbs_lexer_t *lexer) { } } yy209: -#line 112 "src/lexer.re" +#line 121 "src/lexer.re" { return rbs_next_token(lexer, kUSE); } -#line 1827 "src/lexer.c" +#line 1836 "src/lexer.c" yy210: rbs_skip(lexer); yych = rbs_peek(lexer); @@ -1972,35 +1981,35 @@ rbs_token_t rbs_lexer_next_token(rbs_lexer_t *lexer) { { return rbs_next_token(lexer, tANNOTATION); } -#line 1850 "src/lexer.c" +#line 1859 "src/lexer.c" yy213: rbs_skip(lexer); #line 71 "src/lexer.re" { return rbs_next_token(lexer, tANNOTATION); } -#line 1855 "src/lexer.c" +#line 1864 "src/lexer.c" yy214: rbs_skip(lexer); #line 69 "src/lexer.re" { return rbs_next_token(lexer, tANNOTATION); } -#line 1860 "src/lexer.c" +#line 1869 "src/lexer.c" yy215: rbs_skip(lexer); #line 67 "src/lexer.re" { return rbs_next_token(lexer, tANNOTATION); } -#line 1865 "src/lexer.c" +#line 1874 "src/lexer.c" yy216: rbs_skip(lexer); #line 70 "src/lexer.re" { return rbs_next_token(lexer, tANNOTATION); } -#line 1870 "src/lexer.c" +#line 1879 "src/lexer.c" yy217: rbs_skip(lexer); yych = rbs_peek(lexer); @@ -2055,11 +2064,11 @@ rbs_token_t rbs_lexer_next_token(rbs_lexer_t *lexer) { } } yy221: -#line 115 "src/lexer.re" +#line 124 "src/lexer.re" { return rbs_next_token(lexer, kATRBS); } -#line 1927 "src/lexer.c" +#line 1936 "src/lexer.c" yy222: rbs_skip(lexer); yych = rbs_peek(lexer); @@ -2096,11 +2105,11 @@ rbs_token_t rbs_lexer_next_token(rbs_lexer_t *lexer) { } } yy226: -#line 84 "src/lexer.re" +#line 93 "src/lexer.re" { return rbs_next_token(lexer, kBOOL); } -#line 1966 "src/lexer.c" +#line 1975 "src/lexer.c" yy227: rbs_skip(lexer); yych = rbs_peek(lexer); @@ -2177,11 +2186,11 @@ rbs_token_t rbs_lexer_next_token(rbs_lexer_t *lexer) { } } yy239: -#line 104 "src/lexer.re" +#line 113 "src/lexer.re" { return rbs_next_token(lexer, kSELF); } -#line 2045 "src/lexer.c" +#line 2054 "src/lexer.c" yy240: rbs_skip(lexer); yych = rbs_peek(lexer); @@ -2208,11 +2217,11 @@ rbs_token_t rbs_lexer_next_token(rbs_lexer_t *lexer) { } } yy242: -#line 116 "src/lexer.re" +#line 125 "src/lexer.re" { return rbs_next_token(lexer, kSKIP); } -#line 2074 "src/lexer.c" +#line 2083 "src/lexer.c" yy243: rbs_skip(lexer); yych = rbs_peek(lexer); @@ -2234,11 +2243,11 @@ rbs_token_t rbs_lexer_next_token(rbs_lexer_t *lexer) { } } yy244: -#line 107 "src/lexer.re" +#line 116 "src/lexer.re" { return rbs_next_token(lexer, kTRUE); } -#line 2098 "src/lexer.c" +#line 2107 "src/lexer.c" yy245: rbs_skip(lexer); yych = rbs_peek(lexer); @@ -2260,11 +2269,11 @@ rbs_token_t rbs_lexer_next_token(rbs_lexer_t *lexer) { } } yy246: -#line 108 "src/lexer.re" +#line 117 "src/lexer.re" { return rbs_next_token(lexer, kTYPE); } -#line 2122 "src/lexer.c" +#line 2131 "src/lexer.c" yy247: rbs_skip(lexer); yych = rbs_peek(lexer); @@ -2296,11 +2305,11 @@ rbs_token_t rbs_lexer_next_token(rbs_lexer_t *lexer) { } } yy250: -#line 111 "src/lexer.re" +#line 120 "src/lexer.re" { return rbs_next_token(lexer, kVOID); } -#line 2156 "src/lexer.c" +#line 2165 "src/lexer.c" yy251: rbs_skip(lexer); yych = rbs_peek(lexer); @@ -2353,11 +2362,11 @@ rbs_token_t rbs_lexer_next_token(rbs_lexer_t *lexer) { } } yy255: -#line 80 "src/lexer.re" +#line 89 "src/lexer.re" { return rbs_next_token(lexer, kALIAS); } -#line 2211 "src/lexer.c" +#line 2220 "src/lexer.c" yy256: rbs_skip(lexer); yych = rbs_peek(lexer); @@ -2398,11 +2407,11 @@ rbs_token_t rbs_lexer_next_token(rbs_lexer_t *lexer) { } } yy258: -#line 86 "src/lexer.re" +#line 95 "src/lexer.re" { return rbs_next_token(lexer, kCLASS); } -#line 2254 "src/lexer.c" +#line 2263 "src/lexer.c" yy259: rbs_skip(lexer); yych = rbs_peek(lexer); @@ -2429,11 +2438,11 @@ rbs_token_t rbs_lexer_next_token(rbs_lexer_t *lexer) { } } yy261: -#line 91 "src/lexer.re" +#line 100 "src/lexer.re" { return rbs_next_token(lexer, kFALSE); } -#line 2283 "src/lexer.c" +#line 2292 "src/lexer.c" yy262: rbs_skip(lexer); yych = rbs_peek(lexer); @@ -2561,11 +2570,11 @@ rbs_token_t rbs_lexer_next_token(rbs_lexer_t *lexer) { } } yy281: -#line 90 "src/lexer.re" +#line 99 "src/lexer.re" { return rbs_next_token(lexer, kEXTEND); } -#line 2413 "src/lexer.c" +#line 2422 "src/lexer.c" yy282: rbs_skip(lexer); yych = rbs_peek(lexer); @@ -2610,11 +2619,11 @@ rbs_token_t rbs_lexer_next_token(rbs_lexer_t *lexer) { } } yy286: -#line 96 "src/lexer.re" +#line 105 "src/lexer.re" { return rbs_next_token(lexer, kMODULE); } -#line 2460 "src/lexer.c" +#line 2469 "src/lexer.c" yy287: rbs_skip(lexer); yych = rbs_peek(lexer); @@ -2646,11 +2655,11 @@ rbs_token_t rbs_lexer_next_token(rbs_lexer_t *lexer) { } } yy290: -#line 103 "src/lexer.re" +#line 112 "src/lexer.re" { return rbs_next_token(lexer, kPUBLIC); } -#line 2494 "src/lexer.c" +#line 2503 "src/lexer.c" yy291: rbs_skip(lexer); yych = rbs_peek(lexer); @@ -2672,11 +2681,11 @@ rbs_token_t rbs_lexer_next_token(rbs_lexer_t *lexer) { } } yy292: -#line 117 "src/lexer.re" +#line 126 "src/lexer.re" { return rbs_next_token(lexer, kRETURN); } -#line 2518 "src/lexer.c" +#line 2527 "src/lexer.c" yy293: rbs_skip(lexer); yych = rbs_peek(lexer); @@ -2751,11 +2760,11 @@ rbs_token_t rbs_lexer_next_token(rbs_lexer_t *lexer) { } } yy303: -#line 93 "src/lexer.re" +#line 102 "src/lexer.re" { return rbs_next_token(lexer, kINCLUDE); } -#line 2595 "src/lexer.c" +#line 2604 "src/lexer.c" yy304: rbs_skip(lexer); yych = rbs_peek(lexer); @@ -2793,11 +2802,11 @@ rbs_token_t rbs_lexer_next_token(rbs_lexer_t *lexer) { } } yy308: -#line 101 "src/lexer.re" +#line 110 "src/lexer.re" { return rbs_next_token(lexer, kPREPEND); } -#line 2635 "src/lexer.c" +#line 2644 "src/lexer.c" yy309: rbs_skip(lexer); yych = rbs_peek(lexer); @@ -2819,11 +2828,11 @@ rbs_token_t rbs_lexer_next_token(rbs_lexer_t *lexer) { } } yy310: -#line 102 "src/lexer.re" +#line 111 "src/lexer.re" { return rbs_next_token(lexer, kPRIVATE); } -#line 2659 "src/lexer.c" +#line 2668 "src/lexer.c" yy311: rbs_skip(lexer); yych = rbs_peek(lexer); @@ -2855,11 +2864,11 @@ rbs_token_t rbs_lexer_next_token(rbs_lexer_t *lexer) { } } yy314: -#line 110 "src/lexer.re" +#line 119 "src/lexer.re" { return rbs_next_token(lexer, kUNTYPED); } -#line 2693 "src/lexer.c" +#line 2702 "src/lexer.c" yy315: rbs_skip(lexer); yych = rbs_peek(lexer); @@ -2881,11 +2890,11 @@ rbs_token_t rbs_lexer_next_token(rbs_lexer_t *lexer) { } } yy316: -#line 114 "src/lexer.re" +#line 123 "src/lexer.re" { return rbs_next_token(lexer, k__TODO__); } -#line 2717 "src/lexer.c" +#line 2726 "src/lexer.c" yy317: rbs_skip(lexer); yych = rbs_peek(lexer); @@ -2927,11 +2936,11 @@ rbs_token_t rbs_lexer_next_token(rbs_lexer_t *lexer) { } } yy322: -#line 94 "src/lexer.re" +#line 103 "src/lexer.re" { return rbs_next_token(lexer, kINSTANCE); } -#line 2761 "src/lexer.c" +#line 2770 "src/lexer.c" yy323: rbs_skip(lexer); yych = rbs_peek(lexer); @@ -2998,11 +3007,11 @@ rbs_token_t rbs_lexer_next_token(rbs_lexer_t *lexer) { } } yy333: -#line 95 "src/lexer.re" +#line 104 "src/lexer.re" { return rbs_next_token(lexer, kINTERFACE); } -#line 2830 "src/lexer.c" +#line 2839 "src/lexer.c" yy334: rbs_skip(lexer); yych = rbs_peek(lexer); @@ -3034,11 +3043,11 @@ rbs_token_t rbs_lexer_next_token(rbs_lexer_t *lexer) { } } yy337: -#line 105 "src/lexer.re" +#line 114 "src/lexer.re" { return rbs_next_token(lexer, kSINGLETON); } -#line 2864 "src/lexer.c" +#line 2873 "src/lexer.c" yy338: rbs_skip(lexer); yych = rbs_peek(lexer); @@ -3060,11 +3069,11 @@ rbs_token_t rbs_lexer_next_token(rbs_lexer_t *lexer) { } } yy339: -#line 109 "src/lexer.re" +#line 118 "src/lexer.re" { return rbs_next_token(lexer, kUNCHECKED); } -#line 2888 "src/lexer.c" +#line 2897 "src/lexer.c" yy340: rbs_skip(lexer); yych = rbs_peek(lexer); @@ -3121,11 +3130,11 @@ rbs_token_t rbs_lexer_next_token(rbs_lexer_t *lexer) { } } yy348: -#line 82 "src/lexer.re" +#line 91 "src/lexer.re" { return rbs_next_token(lexer, kATTRREADER); } -#line 2947 "src/lexer.c" +#line 2956 "src/lexer.c" yy349: rbs_skip(lexer); yych = rbs_peek(lexer); @@ -3147,18 +3156,18 @@ rbs_token_t rbs_lexer_next_token(rbs_lexer_t *lexer) { } } yy350: -#line 83 "src/lexer.re" +#line 92 "src/lexer.re" { return rbs_next_token(lexer, kATTRWRITER); } -#line 2971 "src/lexer.c" +#line 2980 "src/lexer.c" yy351: rbs_skip(lexer); -#line 87 "src/lexer.re" +#line 96 "src/lexer.re" { return rbs_next_token(lexer, kCLASSALIAS); } -#line 2976 "src/lexer.c" +#line 2985 "src/lexer.c" yy352: rbs_skip(lexer); yych = rbs_peek(lexer); @@ -3166,11 +3175,11 @@ rbs_token_t rbs_lexer_next_token(rbs_lexer_t *lexer) { goto yy72; yy353: rbs_skip(lexer); -#line 98 "src/lexer.re" +#line 107 "src/lexer.re" { return rbs_next_token(lexer, kMODULESELF); } -#line 2986 "src/lexer.c" +#line 2995 "src/lexer.c" yy354: rbs_skip(lexer); yych = rbs_peek(lexer); @@ -3178,11 +3187,11 @@ rbs_token_t rbs_lexer_next_token(rbs_lexer_t *lexer) { goto yy54; yy355: rbs_skip(lexer); -#line 97 "src/lexer.re" +#line 106 "src/lexer.re" { return rbs_next_token(lexer, kMODULEALIAS); } -#line 2996 "src/lexer.c" +#line 3005 "src/lexer.c" yy356: rbs_skip(lexer); yych = rbs_peek(lexer); @@ -3204,11 +3213,11 @@ rbs_token_t rbs_lexer_next_token(rbs_lexer_t *lexer) { } } yy357: -#line 81 "src/lexer.re" +#line 90 "src/lexer.re" { return rbs_next_token(lexer, kATTRACCESSOR); } -#line 3020 "src/lexer.c" +#line 3029 "src/lexer.c" } -#line 177 "src/lexer.re" +#line 186 "src/lexer.re" } diff --git a/src/lexer.re b/src/lexer.re index 18980eacd..2cbb154de 100644 --- a/src/lexer.re +++ b/src/lexer.re @@ -71,6 +71,15 @@ rbs_token_t rbs_lexer_next_token(rbs_lexer_t *lexer) { "%a<" [^>\x00]* ">" { return rbs_next_token(lexer, tANNOTATION); } "#" (. \ [\x00\uFFFD])* { + // Keep a bare CR in the comment, but leave the CR in CRLF for a trivia token. + if (rbs_peek(lexer) == '\n' && lexer->string.start[lexer->current.byte_pos - 1] == '\r') { + lexer->current.byte_pos -= 1; + lexer->current.char_pos -= 1; + lexer->current.column -= 1; + lexer->current_code_point = '\r'; + lexer->current_character_bytes = 1; + } + return rbs_next_token( lexer, lexer->first_token_of_line ? tLINECOMMENT : tCOMMENT diff --git a/test/rbs/parser_test.rb b/test/rbs/parser_test.rb index f0e61e7e8..09b7054e2 100644 --- a/test/rbs/parser_test.rb +++ b/test/rbs/parser_test.rb @@ -1030,6 +1030,48 @@ class Foo[T < Integer] < Bar # Comment assert_equal [:pEOF, '', 57...57], tokens.shift.then { |t| [t[0], t[1].source, t[1].range] } end + def test__lex_crlf + content = "# LineComment\r\nclass Foo[T < Integer] < Bar # Comment\r\nend\r\n" + + comments = RBS::Parser.lex(content).value.select { %i[tLINECOMMENT tCOMMENT].include?(_1.type) } + assert_equal [["# LineComment", 1], ["# Comment", 2]], comments.map { [_1.location.source, _1.location.end_line] } + + tokens = RBS::Parser._lex(buffer(content), content.length) + assert_equal [:tLINECOMMENT, '# LineComment', 0...13], tokens.shift.then { |t| [t[0], t[1].source, t[1].range] } + assert_equal [:tTRIVIA, "\r", 13...14], tokens.shift.then { |t| [t[0], t[1].source, t[1].range] } + assert_equal [:tTRIVIA, "\n", 14...15], tokens.shift.then { |t| [t[0], t[1].source, t[1].range] } + assert_equal [:kCLASS, 'class', 15...20], tokens.shift.then { |t| [t[0], t[1].source, t[1].range] } + assert_equal [:tTRIVIA, " ", 20...21], tokens.shift.then { |t| [t[0], t[1].source, t[1].range] } + assert_equal [:tUIDENT, 'Foo', 21...24], tokens.shift.then { |t| [t[0], t[1].source, t[1].range] } + assert_equal [:pLBRACKET, '[', 24...25], tokens.shift.then { |t| [t[0], t[1].source, t[1].range] } + assert_equal [:tUIDENT, 'T', 25...26], tokens.shift.then { |t| [t[0], t[1].source, t[1].range] } + assert_equal [:tTRIVIA, " ", 26...27], tokens.shift.then { |t| [t[0], t[1].source, t[1].range] } + assert_equal [:pLT, '<', 27...28], tokens.shift.then { |t| [t[0], t[1].source, t[1].range] } + assert_equal [:tTRIVIA, " ", 28...29], tokens.shift.then { |t| [t[0], t[1].source, t[1].range] } + assert_equal [:tUIDENT, 'Integer', 29...36], tokens.shift.then { |t| [t[0], t[1].source, t[1].range] } + assert_equal [:pRBRACKET, ']', 36...37], tokens.shift.then { |t| [t[0], t[1].source, t[1].range] } + assert_equal [:tTRIVIA, " ", 37...38], tokens.shift.then { |t| [t[0], t[1].source, t[1].range] } + assert_equal [:pLT, '<', 38...39], tokens.shift.then { |t| [t[0], t[1].source, t[1].range] } + assert_equal [:tTRIVIA, " ", 39...40], tokens.shift.then { |t| [t[0], t[1].source, t[1].range] } + assert_equal [:tUIDENT, 'Bar', 40...43], tokens.shift.then { |t| [t[0], t[1].source, t[1].range] } + assert_equal [:tTRIVIA, " ", 43...44], tokens.shift.then { |t| [t[0], t[1].source, t[1].range] } + assert_equal [:tCOMMENT, '# Comment', 44...53], tokens.shift.then { |t| [t[0], t[1].source, t[1].range] } + assert_equal [:tTRIVIA, "\r", 53...54], tokens.shift.then { |t| [t[0], t[1].source, t[1].range] } + assert_equal [:tTRIVIA, "\n", 54...55], tokens.shift.then { |t| [t[0], t[1].source, t[1].range] } + assert_equal [:kEND, 'end', 55...58], tokens.shift.then { |t| [t[0], t[1].source, t[1].range] } + assert_equal [:tTRIVIA, "\r", 58...59], tokens.shift.then { |t| [t[0], t[1].source, t[1].range] } + assert_equal [:tTRIVIA, "\n", 59...60], tokens.shift.then { |t| [t[0], t[1].source, t[1].range] } + assert_equal [:pEOF, '', 60...60], tokens.shift.then { |t| [t[0], t[1].source, t[1].range] } + end + + def test__lex_comment_with_bare_cr + content = "# Comment\rclass Foo" + tokens = RBS::Parser.lex(content).value + + assert_equal [:tLINECOMMENT, :pEOF], tokens.map(&:type) + assert_equal content, tokens.first.location.source + end + def test_invalid_position_range_raises # Regression: start_pos > end_pos used to cause an infinite loop in the lexer. assert_raises(ArgumentError) do