Merge pull request #240 from octo/scanner-next

scanner: Update prevPos even when returning utf8.RuneError.

Author: mitchellh

hcl/scanner/scanner.go

@@ -74,21 +74,18 @@ func (s *Scanner) next() rune {
 		return eof
 	}
 
-	if ch == utf8.RuneError && size == 1 {
-		s.srcPos.Column++
-		s.srcPos.Offset += size
-		s.lastCharLen = size
-		s.err("illegal UTF-8 encoding")
-		return ch
-	}
-
 	// remember last position
 	s.prevPos = s.srcPos
 
 	s.srcPos.Column++
 	s.lastCharLen = size
 	s.srcPos.Offset += size
 
+	if ch == utf8.RuneError && size == 1 {
+		s.err("illegal UTF-8 encoding")
+		return ch
+	}
+
 	if ch == '\n' {
 		s.srcPos.Line++
 		s.lastLineLen = s.srcPos.Column

hcl/scanner/scanner_test.go

@@ -592,3 +592,22 @@ func countNewlines(s string) int {
 	}
 	return n
 }
+
+func TestScanHeredocRegexpCompile(t *testing.T) {
+	cases := []string{
+		"0\xe1\n<<ȸ\nhello\nworld\nȸ",
+	}
+
+	for _, c := range cases {
+		s := New([]byte(c))
+		fmt.Printf("START %q\n", c)
+
+		for {
+			tok := s.Scan()
+			if tok.Type == token.EOF {
+				break
+			}
+			t.Logf("s.Scan() = %s", tok)
+		}
+	}
+}