Codebase list golang-github-tdewolff-parse / ff140b1
JS: support parsing of numeric literal separators Taco de Wolff 2 years ago
2 changed file(s) with 19 addition(s) and 6 deletion(s). Raw diff Collapse all Expand all
+18
-6
js/lex.go less more
514514 return true
515515 }
516516
517 func (l *Lexer) consumeNumericSeparator(f func() bool) bool {
518 if l.r.Peek(0) != '_' {
519 return false
520 }
521 l.r.Move(1)
522 if !f() {
523 l.r.Move(-1)
524 return false
525 }
526 return true
527 }
528
517529 func (l *Lexer) consumeNumericToken() TokenType {
518530 // assume to be on 0 1 2 3 4 5 6 7 8 9 .
519531 first := l.r.Peek(0)
522534 if l.r.Peek(0) == 'x' || l.r.Peek(0) == 'X' {
523535 l.r.Move(1)
524536 if l.consumeHexDigit() {
525 for l.consumeHexDigit() {
537 for l.consumeHexDigit() || l.consumeNumericSeparator(l.consumeHexDigit) {
526538 }
527539 return HexadecimalToken
528540 }
531543 } else if l.r.Peek(0) == 'b' || l.r.Peek(0) == 'B' {
532544 l.r.Move(1)
533545 if l.consumeBinaryDigit() {
534 for l.consumeBinaryDigit() {
546 for l.consumeBinaryDigit() || l.consumeNumericSeparator(l.consumeBinaryDigit) {
535547 }
536548 return BinaryToken
537549 }
540552 } else if l.r.Peek(0) == 'o' || l.r.Peek(0) == 'O' {
541553 l.r.Move(1)
542554 if l.consumeOctalDigit() {
543 for l.consumeOctalDigit() {
555 for l.consumeOctalDigit() || l.consumeNumericSeparator(l.consumeOctalDigit) {
544556 }
545557 return OctalToken
546558 }
554566 return ErrorToken
555567 }
556568 } else if first != '.' {
557 for l.consumeDigit() {
569 for l.consumeDigit() || l.consumeNumericSeparator(l.consumeDigit) {
558570 }
559571 }
560572 // we have parsed a 0 or an integer number
562574 if c == '.' {
563575 l.r.Move(1)
564576 if l.consumeDigit() {
565 for l.consumeDigit() {
577 for l.consumeDigit() || l.consumeNumericSeparator(l.consumeDigit) {
566578 }
567579 c = l.r.Peek(0)
568580 } else if first == '.' {
586598 l.err = parse.NewErrorLexer(l.r, "invalid number")
587599 return ErrorToken
588600 }
589 for l.consumeDigit() {
601 for l.consumeDigit() || l.consumeNumericSeparator(l.consumeDigit) {
590602 }
591603 }
592604 return DecimalToken
+1
-0
js/lex_test.go less more
1818 {" \t\v\f\u00A0\uFEFF\u2000", TTs{}}, // WhitespaceToken
1919 {"\n\r\r\n\u2028\u2029", TTs{LineTerminatorToken}},
2020 {"5.2 .04 1. 2.e3 0x0F 5e99", TTs{DecimalToken, DecimalToken, DecimalToken, DecimalToken, HexadecimalToken, DecimalToken}},
21 {"2_3 5_4.1_2 1_1n 0o2_3 0b1_1 0xF_F", TTs{DecimalToken, DecimalToken, BigIntToken, OctalToken, BinaryToken, HexadecimalToken}},
2122 {"0o22 0b11", TTs{OctalToken, BinaryToken}},
2223 {"0n 2345n 435.333n", TTs{BigIntToken, BigIntToken, DecimalToken, ErrorToken}},
2324 {"a = 'string'", TTs{IdentifierToken, EqToken, StringToken}},