cmd/compile/internal/syntax: various cleanups following CL 221603

1) Introduced setLit method to uniformly set the scanner state for
   literals instead of directly manipulating the scanner fields.

2) Use a local variable 'ok' to track validity of literals instead
   of relying on the side-effect of error reporters setting s.bad.
   More code but clearer because it is local and explicit.

3) s/litname/baseName/ and use this function uniformly, also for
   escapes. Consequently we now report always "hexadecimal" and
   not "hex" (in the case of invalid escapes).

4) Added TestDirectives verifying that we get the correct directive
   string (even if that string contains '%').

Verified that lines/s parsing performance is unchanged by comparing

go test -run StdLib -fast -skip "syntax/(scanner|scanner_test)\.go"

before and after (no relevant difference).

Change-Id: I143e4648fdaa31d1c365fb794a1cae4bc1c3f5ba
Reviewed-on: https://go-review.googlesource.com/c/go/+/222258
Run-TryBot: Robert Griesemer <[email protected]>
TryBot-Result: Gobot Gobot <[email protected]>
Reviewed-by: Matthew Dempsky <[email protected]>

Author: griesemer

src/cmd/compile/internal/syntax/scanner.go

@@ -50,16 +50,23 @@ func (s *scanner) init(src io.Reader, errh func(line, col uint, msg string), mod
 
 // errorf reports an error at the most recently read character position.
 func (s *scanner) errorf(format string, args ...interface{}) {
-	s.bad = true
 	s.error(fmt.Sprintf(format, args...))
 }
 
 // errorAtf reports an error at a byte column offset relative to the current token start.
 func (s *scanner) errorAtf(offset int, format string, args ...interface{}) {
-	s.bad = true
 	s.errh(s.line, s.col+uint(offset), fmt.Sprintf(format, args...))
 }
 
+// setLit sets the scanner state for a recognized _Literal token.
+func (s *scanner) setLit(kind LitKind, ok bool) {
+	s.nlsemi = true
+	s.tok = _Literal
+	s.lit = string(s.segment())
+	s.bad = !ok
+	s.kind = kind
+}
+
 // next advances the scanner by reading the next token.
 //
 // If a read, source encoding, or lexical error occurs, next calls
@@ -461,16 +468,15 @@ func (s *scanner) digits(base int, invalid *int) (digsep int) {
 }
 
 func (s *scanner) number(seenPoint bool) {
-	s.bad = false
-
+	ok := true
+	kind := IntLit
 	base := 10        // number base
 	prefix := rune(0) // one of 0 (decimal), '0' (0-octal), 'x', 'o', or 'b'
 	digsep := 0       // bit 0: digit present, bit 1: '_' present
 	invalid := -1     // index of invalid digit in literal, or < 0
 
 	// integer part
 	if !seenPoint {
-		s.kind = IntLit
 		if s.ch == '0' {
 			s.nextch()
 			switch lower(s.ch) {
@@ -491,7 +497,8 @@ func (s *scanner) number(seenPoint bool) {
 		digsep |= s.digits(base, &invalid)
 		if s.ch == '.' {
 			if prefix == 'o' || prefix == 'b' {
-				s.errorf("invalid radix point in %s", litname(prefix))
+				s.errorf("invalid radix point in %s literal", baseName(base))
+				ok = false
 			}
 			s.nextch()
 			seenPoint = true
@@ -500,68 +507,77 @@ func (s *scanner) number(seenPoint bool) {
 
 	// fractional part
 	if seenPoint {
-		s.kind = FloatLit
+		kind = FloatLit
 		digsep |= s.digits(base, &invalid)
 	}
 
-	if digsep&1 == 0 && !s.bad {
-		s.errorf("%s has no digits", litname(prefix))
+	if digsep&1 == 0 && ok {
+		s.errorf("%s literal has no digits", baseName(base))
+		ok = false
 	}
 
 	// exponent
 	if e := lower(s.ch); e == 'e' || e == 'p' {
-		if !s.bad {
+		if ok {
 			switch {
 			case e == 'e' && prefix != 0 && prefix != '0':
 				s.errorf("%q exponent requires decimal mantissa", s.ch)
+				ok = false
 			case e == 'p' && prefix != 'x':
 				s.errorf("%q exponent requires hexadecimal mantissa", s.ch)
+				ok = false
 			}
 		}
 		s.nextch()
-		s.kind = FloatLit
+		kind = FloatLit
 		if s.ch == '+' || s.ch == '-' {
 			s.nextch()
 		}
 		digsep = s.digits(10, nil) | digsep&2 // don't lose sep bit
-		if digsep&1 == 0 && !s.bad {
+		if digsep&1 == 0 && ok {
 			s.errorf("exponent has no digits")
+			ok = false
 		}
-	} else if prefix == 'x' && s.kind == FloatLit && !s.bad {
+	} else if prefix == 'x' && kind == FloatLit && ok {
 		s.errorf("hexadecimal mantissa requires a 'p' exponent")
+		ok = false
 	}
 
 	// suffix 'i'
 	if s.ch == 'i' {
-		s.kind = ImagLit
+		kind = ImagLit
 		s.nextch()
 	}
 
-	s.nlsemi = true
-	s.lit = string(s.segment())
-	s.tok = _Literal
+	s.setLit(kind, ok) // do this now so we can use s.lit below
 
-	if s.kind == IntLit && invalid >= 0 && !s.bad {
-		s.errorAtf(invalid, "invalid digit %q in %s", s.lit[invalid], litname(prefix))
+	if kind == IntLit && invalid >= 0 && ok {
+		s.errorAtf(invalid, "invalid digit %q in %s literal", s.lit[invalid], baseName(base))
+		ok = false
 	}
 
-	if digsep&2 != 0 && !s.bad {
+	if digsep&2 != 0 && ok {
 		if i := invalidSep(s.lit); i >= 0 {
 			s.errorAtf(i, "'_' must separate successive digits")
+			ok = false
 		}
 	}
+
+	s.bad = !ok // correct s.bad
 }
 
-func litname(prefix rune) string {
-	switch prefix {
-	case 'x':
-		return "hexadecimal literal"
-	case 'o', '0':
-		return "octal literal"
-	case 'b':
-		return "binary literal"
-	}
-	return "decimal literal"
+func baseName(base int) string {
+	switch base {
+	case 2:
+		return "binary"
+	case 8:
+		return "octal"
+	case 10:
+		return "decimal"
+	case 16:
+		return "hexadecimal"
+	}
+	panic("invalid base")
 }
 
 // invalidSep returns the index of the first invalid separator in x, or -1.
@@ -605,50 +621,53 @@ func invalidSep(x string) int {
 }
 
 func (s *scanner) rune() {
-	s.bad = false
+	ok := true
 	s.nextch()
 
 	n := 0
 	for ; ; n++ {
 		if s.ch == '\'' {
-			if !s.bad {
+			if ok {
 				if n == 0 {
 					s.errorf("empty rune literal or unescaped '")
+					ok = false
 				} else if n != 1 {
 					s.errorAtf(0, "more than one character in rune literal")
+					ok = false
 				}
 			}
 			s.nextch()
 			break
 		}
 		if s.ch == '\\' {
 			s.nextch()
-			s.escape('\'')
+			if !s.escape('\'') {
+				ok = false
+			}
 			continue
 		}
 		if s.ch == '\n' {
-			if !s.bad {
+			if ok {
 				s.errorf("newline in rune literal")
+				ok = false
 			}
 			break
 		}
 		if s.ch < 0 {
-			if !s.bad {
+			if ok {
 				s.errorAtf(0, "rune literal not terminated")
+				ok = false
 			}
 			break
 		}
 		s.nextch()
 	}
 
-	s.nlsemi = true
-	s.lit = string(s.segment())
-	s.kind = RuneLit
-	s.tok = _Literal
+	s.setLit(RuneLit, ok)
 }
 
 func (s *scanner) stdString() {
-	s.bad = false
+	ok := true
 	s.nextch()
 
 	for {
@@ -658,28 +677,29 @@ func (s *scanner) stdString() {
 		}
 		if s.ch == '\\' {
 			s.nextch()
-			s.escape('"')
+			if !s.escape('"') {
+				ok = false
+			}
 			continue
 		}
 		if s.ch == '\n' {
 			s.errorf("newline in string")
+			ok = false
 			break
 		}
 		if s.ch < 0 {
 			s.errorAtf(0, "string not terminated")
+			ok = false
 			break
 		}
 		s.nextch()
 	}
 
-	s.nlsemi = true
-	s.lit = string(s.segment())
-	s.kind = StringLit
-	s.tok = _Literal
+	s.setLit(StringLit, ok)
 }
 
 func (s *scanner) rawString() {
-	s.bad = false
+	ok := true
 	s.nextch()
 
 	for {
@@ -689,6 +709,7 @@ func (s *scanner) rawString() {
 		}
 		if s.ch < 0 {
 			s.errorAtf(0, "string not terminated")
+			ok = false
 			break
 		}
 		s.nextch()
@@ -697,10 +718,7 @@ func (s *scanner) rawString() {
 	// literal (even though they are not part of the literal
 	// value).
 
-	s.nlsemi = true
-	s.lit = string(s.segment())
-	s.kind = StringLit
-	s.tok = _Literal
+	s.setLit(StringLit, ok)
 }
 
 func (s *scanner) comment(text string) {
@@ -797,14 +815,14 @@ func (s *scanner) fullComment() {
 	}
 }
 
-func (s *scanner) escape(quote rune) {
+func (s *scanner) escape(quote rune) bool {
 	var n int
 	var base, max uint32
 
 	switch s.ch {
 	case quote, 'a', 'b', 'f', 'n', 'r', 't', 'v', '\\':
 		s.nextch()
-		return
+		return true
 	case '0', '1', '2', '3', '4', '5', '6', '7':
 		n, base, max = 3, 8, 255
 	case 'x':
@@ -818,16 +836,16 @@ func (s *scanner) escape(quote rune) {
 		n, base, max = 8, 16, unicode.MaxRune
 	default:
 		if s.ch < 0 {
-			return // complain in caller about EOF
+			return true // complain in caller about EOF
 		}
 		s.errorf("unknown escape")
-		return
+		return false
 	}
 
 	var x uint32
 	for i := n; i > 0; i-- {
 		if s.ch < 0 {
-			return // complain in caller about EOF
+			return true // complain in caller about EOF
 		}
 		d := base
 		if isDecimal(s.ch) {
@@ -836,12 +854,8 @@ func (s *scanner) escape(quote rune) {
 			d = uint32(lower(s.ch)) - 'a' + 10
 		}
 		if d >= base {
-			kind := "hex"
-			if base == 8 {
-				kind = "octal"
-			}
-			s.errorf("invalid character %q in %s escape", s.ch, kind)
-			return
+			s.errorf("invalid character %q in %s escape", s.ch, baseName(int(base)))
+			return false
 		}
 		// d < base
 		x = x*base + d
@@ -850,10 +864,13 @@ func (s *scanner) escape(quote rune) {
 
 	if x > max && base == 8 {
 		s.errorf("octal escape value %d > 255", x)
-		return
+		return false
 	}
 
 	if x > max || 0xD800 <= x && x < 0xE000 /* surrogate range */ {
 		s.errorf("escape is invalid Unicode code point %#U", x)
+		return false
 	}
+
+	return true
 }