cmd/cue: import non-UTF8 Go string constants as CUE bytes

Otherwise literal.String.Quote replaces invalid UTF-8 sequences
with the replacement character \uFFFD, as can be seen by the test
failure when the fix is reverted:

    > cmp decls/p_go_gen.cue decls/p_go_gen.cue.golden
    diff decls/p_go_gen.cue decls/p_go_gen.cue.golden
    --- decls/p_go_gen.cue
    +++ decls/p_go_gen.cue.golden
    @@ -21,4 +21,4 @@

     #UTF8: "012" // "\x30\x31\x32"

    -#NonUTF8: "a�b��c�" // "a\xffb\xC0\xAFc\xff"
    +#NonUTF8: 'a\xffb\xc0\xafc\xff' // "a\xffb\xC0\xAFc\xff"

Signed-off-by: Daniel Martí <[email protected]>
Change-Id: I36217582d6e47053d25d0a4eea7adea55b432027
Reviewed-on: https://review.gerrithub.io/c/cue-lang/cue/+/1211576
Unity-Result: CUE porcuepine <[email protected]>
Reviewed-by: Roger Peppe <[email protected]>
TryBot-Result: CUEcueckoo <[email protected]>

Author: mvdan

Diff for: cmd/cue/cmd/get_go.go

@@ -30,6 +30,7 @@ import (
 	"strconv"
 	"strings"
 	"unicode"
+	"unicode/utf8"
 
 	"github.com/spf13/cobra"
 	"golang.org/x/tools/go/packages"
@@ -792,10 +793,17 @@ func (e *extractor) reportDecl(x *ast.GenDecl) (a []cueast.Decl) {
 				var cv cueast.Expr
 				switch v.Kind() {
 				case constant.String:
-					cv = &cueast.BasicLit{
-						Kind:  cuetoken.STRING,
-						Value: literal.String.Quote(constant.StringVal(v)),
+					s := constant.StringVal(v)
+					bl := &cueast.BasicLit{Kind: cuetoken.STRING}
+					// Go strings may contain any bytes, even invalid UTF-8.
+					// CUE strings may only contain valid UTF-8, because it has
+					// bytes values for anything that may not be valid UTF-8.
+					if utf8.ValidString(s) {
+						bl.Value = literal.String.Quote(s)
+					} else {
+						bl.Value = literal.Bytes.Quote(s)
 					}
+					cv = bl
 
 				default:
 					// TODO(mvdan): replace this with switch cases for Bool/Int/Float

Diff for: cmd/cue/cmd/testdata/script/get_go_json_compat.txtar

@@ -31,13 +31,17 @@ language: version: "v0.9.0"
 -- decls/export.cue --
 package decls
 
+import "strings"
+
 #Root & {
 	Duration: #ValueSecond
 
 	Strings: [
 		#ErrorPrefix,
 		#ResetColor,
+		#UTF8,
 	]
+	NonUTF8ThirdByte: strings.ByteAt(#NonUTF8, 3)
 }
 -- decls/p.go --
 package decls
@@ -48,12 +52,20 @@ type Root struct {
 	Duration time.Duration
 
 	Strings []string
+
+	// We can't roundtrip NonUTF8 directly because of https://cuelang.org/issue/3823.
+	// Roundtrip its third byte for now, which does work.
+	NonUTF8ThirdByte int
 }
 
 const ValueSecond = 3 * time.Second
 
 const ErrorPrefix = "\033[31mError:\033[0m"
 const ResetColor = "\u001b[0m"
+
+const UTF8 = "\x30\x31\x32" // "012"
+// The third byte here should not be replaced by \uFFFD, the replacement character.
+const NonUTF8 = "a\xffb\xC0\xAFc\xff"
 -- decls/p_go_gen.cue.golden --
 // Code generated by cue get go. DO NOT EDIT.
 
@@ -64,13 +76,21 @@ package decls
 #Root: {
 	Duration: int @go(,time.Duration)
 	Strings: [...string] @go(,[]string)
+
+	// We can't roundtrip NonUTF8 directly because of https://cuelang.org/issue/3823.
+	// Roundtrip its third byte for now, which does work.
+	NonUTF8ThirdByte: int
 }
 
 #ValueSecond: int & 3000000000
 
 #ErrorPrefix: "\u001b[31mError:\u001b[0m" // "\033[31mError:\033[0m"
 
 #ResetColor: "\u001b[0m"
+
+#UTF8: "012" // "\x30\x31\x32"
+
+#NonUTF8: 'a\xffb\xc0\xafc\xff' // "a\xffb\xC0\xAFc\xff"
 -- marshal/main.go --
 package main
 
@@ -88,7 +108,9 @@ func main() {
 		Strings: []string{
 			decls.ErrorPrefix,
 			decls.ResetColor,
+			decls.UTF8,
 		},
+		NonUTF8ThirdByte: int(decls.NonUTF8[3]),
 	}
 	// Mimic the formatting of `cue export` to be able to compare bytes.
 	data, err := json.MarshalIndent(r, "", "    ")
@@ -102,6 +124,8 @@ func main() {
     "Duration": 3000000000,
     "Strings": [
         "\u001b[31mError:\u001b[0m",
-        "\u001b[0m"
-    ]
+        "\u001b[0m",
+        "012"
+    ],
+    "NonUTF8ThirdByte": 192
 }